message_bus 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0b1d52c269db2dcec90068d0ceb828b969666a64
+  data.tar.gz: e2e46e33612cde225c94c17367599b63f3c824f4
+SHA512:
+  metadata.gz: 523d379b630f2c3d23db73835bde99b5fda6003f448ce7d9f3d66c169bbd18775d251aad51dc269bfe987e62279d49475712f5bab32db1baaf2c404fa6110a5c
+  data.tar.gz: 42aa9744353b2a70fb7dfe6422a30eaaa721deb161695f0ea084cec809acde201af451cc253f801d3623257ad7f700fb8921d482d2d61fe21909542ef8e4074a

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Sam Saffron
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# MessageBus
+
+A reliable, robust messaging bus for Ruby processes and web clients built on Redis.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'message_bus'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install message_bus
+
+## Usage
+
+Server to Server messaging
+
+```ruby
+message_id = MessageBus.publish "/channel", "message"
+
+# in another process / spot
+
+MessageBus.subscribe "/channel" do |msg|
+  # block called in a backgroud thread when message is recieved
+end
+
+MessageBus.backlog "/channel", id
+# returns all messages after the id
+
+
+# messages can be targetted at particular users or groups
+MessageBus.publish "/channel", user_ids: [1,2,3], group_ids: [4,5,6]
+
+# message bus determines the user ids and groups based on env
+
+MessageBus.user_id_lookup do |env|
+  # return the user id here
+end
+
+MessageBus.group_ids_lookup do |env|
+  # return the group ids the user belongs to
+  # can be nil or []
+end
+
+```
+
+
+## Similar projects
+
+Faye - http://faye.jcoglan.com/
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/lib/message_bus/client.rb

@@ -0,0 +1,81 @@
+class MessageBus::Client
+  attr_accessor :client_id, :user_id, :group_ids, :connect_time, :subscribed_sets, :site_id, :cleanup_timer, :async_response
+  def initialize(opts)
+    self.client_id = opts[:client_id]
+    self.user_id = opts[:user_id]
+    self.group_ids = opts[:group_ids] || []
+    self.site_id = opts[:site_id]
+    self.connect_time = Time.now
+    @subscriptions = {}
+  end
+
+  def close
+    return unless @async_response
+    write_and_close "[]"
+  end
+
+  def closed
+    !@async_response
+  end
+
+  def subscribe(channel, last_seen_id)
+    last_seen_id ||= MessageBus.last_id(channel)
+    @subscriptions[channel] = last_seen_id
+  end
+
+  def subscriptions
+    @subscriptions
+  end
+
+  def <<(msg)
+    write_and_close messages_to_json([msg])
+  end
+
+  def subscriptions
+    @subscriptions
+  end
+
+  def allowed?(msg)
+    allowed = !msg.user_ids || msg.user_ids.include?(self.user_id)
+    allowed && (
+      msg.group_ids.nil? ||
+      msg.group_ids.length == 0 ||
+      (
+        msg.group_ids - self.group_ids
+      ).length < msg.group_ids.length
+    )
+  end
+
+  def backlog
+    r = []
+    @subscriptions.each do |k,v|
+      next if v.to_i < 0
+      messages = MessageBus.backlog(k,v)
+      messages.each do |msg|
+        r << msg if allowed?(msg)
+      end
+    end
+    # stats message for all newly subscribed
+    status_message = nil
+    @subscriptions.each do |k,v|
+      if v.to_i == -1
+        status_message ||= {}
+        status_message[k] = MessageBus.last_id(k)
+      end
+    end
+    r << MessageBus::Message.new(-1, -1, '/__status', status_message) if status_message
+    r
+  end
+
+  protected
+
+  def write_and_close(data)
+    @async_response << data
+    @async_response.done
+    @async_response = nil
+  end
+
+  def messages_to_json(msgs)
+    MessageBus::Rack::Middleware.backlog_to_json(msgs)
+  end
+end

data/lib/message_bus/connection_manager.rb

@@ -0,0 +1,66 @@
+require 'json' unless defined? ::JSON
+
+class MessageBus::ConnectionManager
+
+  def initialize
+    @clients = {}
+    @subscriptions = {}
+  end
+
+  def notify_clients(msg)
+    begin
+      site_subs = @subscriptions[msg.site_id]
+      subscription = site_subs[msg.channel] if site_subs
+
+      return unless subscription
+
+      subscription.each do |client_id|
+        client = @clients[client_id]
+        if client && client.allowed?(msg)
+          client << msg
+          # turns out you can delete from a set while itereating
+          remove_client(client)
+        end
+      end
+    rescue => e
+      MessageBus.logger.error "notify clients crash #{e} : #{e.backtrace}"
+    end
+  end
+
+  def add_client(client)
+    @clients[client.client_id] = client
+    @subscriptions[client.site_id] ||= {}
+    client.subscriptions.each do |k,v|
+      subscribe_client(client, k)
+    end
+  end
+
+  def remove_client(c)
+    @clients.delete c.client_id
+    @subscriptions[c.site_id].each do |k, set|
+      set.delete c.client_id
+    end
+    c.cleanup_timer.cancel
+  end
+
+  def lookup_client(client_id)
+    @clients[client_id]
+  end
+
+  def subscribe_client(client,channel)
+    set = @subscriptions[client.site_id][channel]
+    unless set
+      set = Set.new
+      @subscriptions[client.site_id][channel] = set
+    end
+    set << client.client_id
+  end
+
+  def stats
+    {
+      client_count: @clients.length,
+      subscriptions: @subscriptions
+    }
+  end
+
+end

data/lib/message_bus/diagnostics.rb

@@ -0,0 +1,53 @@
+class MessageBus::Diagnostics
+  def self.full_process_path
+    begin
+      system = `uname`.strip
+      if system == "Darwin"
+        `ps -o "comm=" -p #{Process.pid}`
+      elsif system == "FreeBSD"
+        `ps -o command -p #{Process.pid}`.split("\n",2)[1].strip()
+      else
+        info = `ps -eo "%p|$|%a" | grep '^\\s*#{Process.pid}'`
+        info.strip.split('|$|')[1]
+      end
+    rescue
+      # skip it ... not linux or something weird
+    end
+  end
+
+  def self.hostname
+    begin
+      `hostname`.strip
+    rescue
+      # skip it
+    end
+  end
+
+  def self.enable
+    full_path = full_process_path
+    start_time = Time.now.to_f
+    hostname = self.hostname
+
+    # it may make sense to add a channel per machine/host to streamline
+    #  process to process comms
+    MessageBus.subscribe('/_diagnostics/hup') do |msg|
+      if Process.pid == msg.data["pid"] && hostname == msg.data["hostname"]
+        $shutdown = true
+        sleep 4
+        Process.kill("HUP", $$)
+      end
+    end
+
+    MessageBus.subscribe('/_diagnostics/discover') do |msg|
+      MessageBus.on_connect.call msg.site_id if MessageBus.on_connect
+      MessageBus.publish '/_diagnostics/process-discovery', {
+        pid: Process.pid,
+        process_name: $0,
+        full_path: full_path,
+        uptime: (Time.now.to_f - start_time).to_i,
+        hostname: hostname
+      }, user_ids: [msg.data["user_id"]]
+      MessageBus.on_disconnect.call msg.site_id if MessageBus.on_disconnect
+    end
+  end
+end

data/lib/message_bus/message.rb

@@ -0,0 +1,17 @@
+class MessageBus::Message < Struct.new(:global_id, :message_id, :channel , :data)
+
+  attr_accessor :site_id, :user_ids, :group_ids
+
+  def self.decode(encoded)
+    s1 = encoded.index("|")
+    s2 = encoded.index("|", s1+1)
+    s3 = encoded.index("|", s2+1)
+
+    MessageBus::Message.new encoded[0..s1].to_i, encoded[s1+1..s2].to_i, encoded[s2+1..s3-1].gsub("$$123$$", "|"), encoded[s3+1..-1]
+  end
+
+  # only tricky thing to encode is pipes in a channel name ... do a straight replace
+  def encode
+    global_id.to_s << "|" << message_id.to_s << "|" << channel.gsub("|","$$123$$") << "|" << data
+  end
+end

data/lib/message_bus/message_handler.rb

@@ -0,0 +1,26 @@
+class MessageBus::MessageHandler
+  def self.load_handlers(path)
+    Dir.glob("#{path}/*.rb").each do |f|
+      load "#{f}"
+    end
+  end
+
+  def self.handle(name,&blk)
+    raise ArgumentError.new("expecting block") unless block_given?
+    raise ArgumentError.new("name") unless name
+
+    @@handlers ||= {}
+    @@handlers[name] = blk
+  end
+
+  def self.call(site_id, name, data, current_user_id)
+    begin
+      MessageBus.on_connect.call(site_id) if MessageBus.on_connect
+      @@handlers[name].call(data,current_user_id)
+    ensure
+      MessageBus.on_disconnect.call(site_id) if MessageBus.on_disconnect
+    end
+  end
+
+
+end

data/lib/message_bus/rack/diagnostics.rb

@@ -0,0 +1,98 @@
+module MessageBus::Rack; end
+
+class MessageBus::Rack::Diagnostics
+  def initialize(app, config = {})
+    @app = app
+  end
+
+  def js_asset(name)
+    return generate_script_tag(name) unless MessageBus.cache_assets
+    @@asset_cache ||= {}
+    @@asset_cache[name] ||= generate_script_tag(name)
+    @@asset_cache[name]
+  end
+
+  def  generate_script_tag(name)
+    "<script src='/message-bus/_diagnostics/assets/#{name}?#{file_hash(name)}' type='text/javascript'></script>"
+  end
+
+  def file_hash(asset)
+    require 'digest/sha1'
+    Digest::SHA1.hexdigest(asset_contents(asset))
+  end
+
+  def asset_contents(asset)
+    File.open(asset_path(asset)).read
+  end
+
+  def asset_path(asset)
+    File.expand_path("../../../../assets/#{asset}", __FILE__)
+  end
+
+  def index
+    html = <<HTML
+<!DOCTYPE html>
+<html>
+  <head>
+  </head>
+  <body>
+    <div id="app"></div>
+    #{js_asset "jquery-1.8.2.js"}
+    #{js_asset "handlebars.js"}
+    #{js_asset "ember.js"}
+    #{js_asset "message-bus.js"}
+    #{js_asset "application.handlebars"}
+    #{js_asset "index.handlebars"}
+    #{js_asset "application.js"}
+  </body>
+</html>
+HTML
+    return [200, {"content-type" => "text/html;"}, [html]]
+  end
+
+  def translate_handlebars(name, content)
+    "Ember.TEMPLATES['#{name}'] = Ember.Handlebars.compile(#{indent(content).inspect});"
+  end
+
+  # from ember-rails
+  def indent(string)
+    string.gsub(/$(.)/m, "\\1  ").strip
+  end
+
+  def call(env)
+
+    return @app.call(env) unless env['PATH_INFO'].start_with? '/message-bus/_diagnostics'
+
+    route = env['PATH_INFO'].split('/message-bus/_diagnostics')[1]
+
+    if MessageBus.is_admin_lookup.nil? || !MessageBus.is_admin_lookup.call(env)
+      return [403, {}, ['not allowed']]
+    end
+
+    return index unless route
+
+    if route == '/discover'
+      user_id =  MessageBus.user_id_lookup.call(env)
+      MessageBus.publish('/_diagnostics/discover', user_id: user_id)
+      return [200, {}, ['ok']]
+    end
+
+    if route =~ /^\/hup\//
+      hostname, pid = route.split('/hup/')[1].split('/')
+      MessageBus.publish('/_diagnostics/hup', {hostname: hostname, pid: pid.to_i})
+      return [200, {}, ['ok']]
+    end
+
+    asset = route.split('/assets/')[1]
+    if asset && !asset !~ /\//
+      content = asset_contents(asset)
+      split = asset.split('.')
+      if split[1] == 'handlebars'
+        content = translate_handlebars(split[0],content)
+      end
+      return [200, {'content-type' => 'text/javascript;'}, [content]]
+    end
+
+    return [404, {}, ['not found']]
+  end
+end

data/lib/message_bus/rack/middleware.rb

@@ -0,0 +1,171 @@
+# our little message bus, accepts long polling and web sockets
+require 'thin'
+require 'eventmachine'
+
+module MessageBus::Rack; end
+
+class MessageBus::Rack::Middleware
+
+  def self.start_listener
+    unless @started_listener
+      MessageBus.subscribe do |msg|
+        if EM.reactor_running?
+          EM.next_tick do
+            @@connection_manager.notify_clients(msg) if @@connection_manager
+          end
+        end
+      end
+      @started_listener = true
+    end
+  end
+
+  def initialize(app, config = {})
+    @app = app
+    @@connection_manager = MessageBus::ConnectionManager.new
+    self.class.start_listener
+  end
+
+  def self.backlog_to_json(backlog)
+    m = backlog.map do |msg|
+      {
+        :global_id => msg.global_id,
+        :message_id => msg.message_id,
+        :channel => msg.channel,
+        :data => msg.data
+      }
+    end.to_a
+    JSON.dump(m)
+  end
+
+  def call(env)
+
+    return @app.call(env) unless env['PATH_INFO'] =~ /^\/message-bus/
+
+    # special debug/test route
+    if ::MessageBus.allow_broadcast? && env['PATH_INFO'] == '/message-bus/broadcast'
+        parsed = Rack::Request.new(env)
+        ::MessageBus.publish parsed["channel"], parsed["data"]
+        return [200,{"Content-Type" => "text/html"},["sent"]]
+    end
+
+    if env['PATH_INFO'].start_with? '/message-bus/_diagnostics'
+      diags = MessageBus::Rack::Diagnostics.new(@app)
+      return diags.call(env)
+    end
+
+    client_id = env['PATH_INFO'].split("/")[2]
+    return [404, {}, ["not found"]] unless client_id
+
+    user_id = MessageBus.user_id_lookup.call(env) if MessageBus.user_id_lookup
+    group_ids = MessageBus.group_ids_lookup.call(env) if MessageBus.group_ids_lookup
+    site_id = MessageBus.site_id_lookup.call(env) if MessageBus.site_id_lookup
+
+    client = MessageBus::Client.new(client_id: client_id, user_id: user_id, site_id: site_id, group_ids: group_ids)
+
+    connection = env['em.connection']
+
+    request = Rack::Request.new(env)
+    request.POST.each do |k,v|
+      client.subscribe(k, v)
+    end
+
+    backlog = client.backlog
+    headers = {}
+    headers["Cache-Control"] = "must-revalidate, private, max-age=0"
+    headers["Content-Type"] ="application/json; charset=utf-8"
+
+    if backlog.length > 0
+      [200, headers, [self.class.backlog_to_json(backlog)] ]
+    elsif MessageBus.long_polling_enabled? && env['QUERY_STRING'] !~ /dlp=t/ && EM.reactor_running?
+      response = Thin::AsyncResponse.new(env)
+      response.headers["Cache-Control"] = "must-revalidate, private, max-age=0"
+      response.headers["Content-Type"] ="application/json; charset=utf-8"
+      response.status = 200
+      client.async_response = response
+
+      @@connection_manager.add_client(client)
+
+      client.cleanup_timer = ::EM::Timer.new(MessageBus.long_polling_interval.to_f / 1000) {
+        client.close
+        @@connection_manager.remove_client(client)
+      }
+
+      throw :async
+    else
+      [200, headers, ["[]"]]
+    end
+
+  end
+end
+
+# there is also another in cramp this is from https://github.com/macournoyer/thin_async/blob/master/lib/thin/async.rb
+module Thin
+  unless defined?(DeferrableBody)
+    # Based on version from James Tucker <raggi@rubyforge.org>
+    class DeferrableBody
+      include ::EM::Deferrable
+
+      def initialize
+        @queue = []
+      end
+
+      def call(body)
+        @queue << body
+        schedule_dequeue
+      end
+
+      def each(&blk)
+        @body_callback = blk
+        schedule_dequeue
+      end
+
+      private
+        def schedule_dequeue
+          return unless @body_callback
+          ::EM.next_tick do
+            next unless body = @queue.shift
+            body.each do |chunk|
+              @body_callback.call(chunk)
+            end
+            schedule_dequeue unless @queue.empty?
+          end
+        end
+    end
+  end
+
+  # Response whos body is sent asynchronously.
+  class AsyncResponse
+    include Rack::Response::Helpers
+
+    attr_reader :headers, :callback, :closed
+    attr_accessor :status
+
+    def initialize(env, status=200, headers={})
+      @callback = env['async.callback']
+      @body = DeferrableBody.new
+      @status = status
+      @headers = headers
+      @headers_sent = false
+    end
+
+    def send_headers
+      return if @headers_sent
+      @callback.call [@status, @headers, @body]
+      @headers_sent = true
+    end
+
+    def write(body)
+      send_headers
+      @body.call(body.respond_to?(:each) ? body : [body])
+    end
+    alias :<< :write
+
+    # Tell Thin the response is complete and the connection can be closed.
+    def done
+      @closed = true
+      send_headers
+      ::EM.next_tick { @body.succeed }
+    end
+
+  end
+end

data/lib/message_bus/rails/railtie.rb

@@ -0,0 +1,9 @@
+module MessageBus; module Rails; end; end
+
+class MessageBus::Rails::Railtie < ::Rails::Railtie
+  initializer "message_bus.configure_init" do |app|
+    MessageBus::MessageHandler.load_handlers("#{Rails.root}/app/message_handlers")
+    app.middleware.use MessageBus::Rack::Middleware
+    MessageBus.logger = Rails.logger
+  end
+end

data/lib/message_bus/reliable_pub_sub.rb

@@ -0,0 +1,262 @@
+require 'redis'
+# the heart of the message bus, it acts as 2 things
+#
+# 1. A channel multiplexer
+# 2. Backlog storage per-multiplexed channel.
+#
+# ids are all sequencially increasing numbers starting at 0
+#
+
+
+class MessageBus::ReliablePubSub
+
+  class NoMoreRetries < StandardError; end
+  class BackLogOutOfOrder < StandardError
+    attr_accessor :highest_id
+
+    def initialize(highest_id)
+      @highest_id = highest_id
+    end
+  end
+
+  def max_publish_retries=(val)
+    @max_publish_retries = val
+  end
+
+  def max_publish_retries
+    @max_publish_retries ||= 10
+  end
+
+  def max_publish_wait=(ms)
+    @max_publish_wait = ms
+  end
+
+  def max_publish_wait
+    @max_publish_wait ||= 500
+  end
+
+  # max_backlog_size is per multiplexed channel
+  def initialize(redis_config = {}, max_backlog_size = 1000)
+    @redis_config = redis_config
+    @max_backlog_size = 1000
+    # we can store a ton here ...
+    @max_global_backlog_size = 100000
+  end
+
+  # amount of global backlog we can spin through
+  def max_global_backlog_size=(val)
+    @max_global_backlog_size = val
+  end
+
+  # per channel backlog size
+  def max_backlog_size=(val)
+    @max_backlog_size = val
+  end
+
+  def new_redis_connection
+    ::Redis.new(@redis_config)
+  end
+
+  def redis_channel_name
+    db = @redis_config[:db] || 0
+    "discourse_#{db}"
+  end
+
+  # redis connection used for publishing messages
+  def pub_redis
+    @pub_redis ||= new_redis_connection
+  end
+
+  def backlog_key(channel)
+    "__mb_backlog_n_#{channel}"
+  end
+
+  def backlog_id_key(channel)
+    "__mb_backlog_id_n_#{channel}"
+  end
+
+  def global_id_key
+    "__mb_global_id_n"
+  end
+
+  def global_backlog_key
+    "__mb_global_backlog_n"
+  end
+
+  # use with extreme care, will nuke all of the data
+  def reset!
+    pub_redis.keys("__mb_*").each do |k|
+      pub_redis.del k
+    end
+  end
+
+  def publish(channel, data)
+    redis = pub_redis
+    backlog_id_key = backlog_id_key(channel)
+    backlog_key = backlog_key(channel)
+
+    global_id = nil
+    backlog_id = nil
+
+    redis.multi do |m|
+      global_id = m.incr(global_id_key)
+      backlog_id = m.incr(backlog_id_key)
+    end
+
+    global_id = global_id.value
+    backlog_id = backlog_id.value
+
+    msg = MessageBus::Message.new global_id, backlog_id, channel, data
+    payload = msg.encode
+
+    redis.zadd backlog_key, backlog_id, payload
+    redis.zadd global_backlog_key, global_id, backlog_id.to_s << "|" << channel
+
+    redis.publish redis_channel_name, payload
+
+    if backlog_id > @max_backlog_size
+      redis.zremrangebyscore backlog_key, 1, backlog_id - @max_backlog_size
+    end
+
+    if global_id > @max_global_backlog_size
+      redis.zremrangebyscore global_backlog_key, 1, backlog_id - @max_backlog_size
+    end
+
+    backlog_id
+  end
+
+  def last_id(channel)
+    redis = pub_redis
+    backlog_id_key = backlog_id_key(channel)
+    redis.get(backlog_id_key).to_i
+  end
+
+  def backlog(channel, last_id = nil)
+    redis = pub_redis
+    backlog_key = backlog_key(channel)
+    items = redis.zrangebyscore backlog_key, last_id.to_i + 1, "+inf"
+
+    items.map do |i|
+      MessageBus::Message.decode(i)
+    end
+  end
+
+  def global_backlog(last_id = nil)
+    last_id = last_id.to_i
+    redis = pub_redis
+
+    items = redis.zrangebyscore global_backlog_key, last_id.to_i + 1, "+inf"
+
+    items.map! do |i|
+      pipe = i.index "|"
+      message_id = i[0..pipe].to_i
+      channel = i[pipe+1..-1]
+      m = get_message(channel, message_id)
+      m
+    end
+
+    items.compact!
+    items
+  end
+
+  def get_message(channel, message_id)
+    redis = pub_redis
+    backlog_key = backlog_key(channel)
+
+    items = redis.zrangebyscore backlog_key, message_id, message_id
+    if items && items[0]
+      MessageBus::Message.decode(items[0])
+    else
+      nil
+    end
+  end
+
+  def subscribe(channel, last_id = nil)
+    # trivial implementation for now,
+    #   can cut down on connections if we only have one global subscriber
+    raise ArgumentError unless block_given?
+
+    if last_id
+      # we need to translate this to a global id, at least give it a shot
+      #   we are subscribing on global and global is always going to be bigger than local
+      #   so worst case is a replay of a few messages
+      message = get_message(channel, last_id)
+      if message
+        last_id = message.global_id
+      end
+    end
+    global_subscribe(last_id) do |m|
+      yield m if m.channel == channel
+    end
+  end
+
+  def process_global_backlog(highest_id, raise_error, &blk)
+    global_backlog(highest_id).each do |old|
+      if highest_id + 1 == old.global_id
+        yield old
+        highest_id = old.global_id
+      else
+        raise BackLogOutOfOrder.new(highest_id) if raise_error
+        if old.global_id > highest_id
+          yield old
+          highest_id = old.global_id
+        end
+      end
+    end
+    highest_id
+  end
+
+  def global_subscribe(last_id=nil, &blk)
+    raise ArgumentError unless block_given?
+    highest_id = last_id
+
+    clear_backlog = lambda do
+      retries = 4
+      begin
+        highest_id = process_global_backlog(highest_id, retries > 0, &blk)
+      rescue BackLogOutOfOrder => e
+        highest_id = e.highest_id
+        retries -= 1
+        sleep(rand(50) / 1000.0)
+        retry
+      end
+    end
+
+
+    begin
+      redis = new_redis_connection
+
+      if highest_id
+        clear_backlog.call(&blk)
+      end
+
+      redis.subscribe(redis_channel_name) do |on|
+        on.subscribe do
+          if highest_id
+            clear_backlog.call(&blk)
+          end
+        end
+        on.message do |c,m|
+          m = MessageBus::Message.decode m
+
+          # we have 2 options
+          #
+          # 1. message came in the correct order GREAT, just deal with it
+          # 2. message came in the incorrect order COMPLICATED, wait a tiny bit and clear backlog
+
+          if highest_id.nil? || m.global_id == highest_id + 1
+            highest_id = m.global_id
+            yield m
+          else
+            clear_backlog.call(&blk)
+          end
+        end
+      end
+    rescue => error
+      MessageBus.logger.warn "#{error} subscribe failed, reconnecting in 1 second. Call stack #{error.backtrace}"
+      sleep 1
+      retry
+    end
+  end
+
+end

data/lib/message_bus/version.rb

@@ -0,0 +1,3 @@
+module MessageBus
+  VERSION = "0.0.1"
+end

data/lib/message_bus.rb

@@ -0,0 +1,272 @@
+# require 'thin'
+# require 'eventmachine'
+# require 'rack'
+# require 'redis'
+
+require "message_bus/version"
+require "message_bus/message"
+require "message_bus/reliable_pub_sub"
+require "message_bus/client"
+require "message_bus/connection_manager"
+require "message_bus/message_handler"
+require "message_bus/diagnostics"
+require "message_bus/rack/middleware"
+require "message_bus/rack/diagnostics"
+
+# we still need to take care of the logger
+if defined?(::Rails)
+  require 'message_bus/rails/railtie'
+end
+
+module MessageBus; end
+module MessageBus::Implementation
+
+  def cache_assets=(val)
+    @cache_assets = val
+  end
+
+  def cache_assets
+    if defined? @cache_assets
+      @cache_assets
+    else
+      true
+    end
+  end
+
+  def logger=(logger)
+    @logger = logger
+  end
+
+  def logger
+    return @logger if @logger
+    require 'logger'
+    @logger = Logger.new(STDOUT)
+  end
+
+  def long_polling_enabled?
+    @long_polling_enabled == false ? false : true
+  end
+
+  def long_polling_enabled=(val)
+    @long_polling_enabled = val
+  end
+
+  def long_polling_interval=(millisecs)
+    @long_polling_interval = millisecs
+  end
+
+  def long_polling_interval
+    @long_polling_interval || 30 * 1000
+  end
+
+  def off
+    @off = true
+  end
+
+  def on
+    @off = false
+  end
+
+  # Allow us to inject a redis db
+  def redis_config=(config)
+    @redis_config = config
+  end
+
+  def redis_config
+    @redis_config ||= {}
+  end
+
+  def site_id_lookup(&blk)
+    @site_id_lookup = blk if blk
+    @site_id_lookup
+  end
+
+  def user_id_lookup(&blk)
+    @user_id_lookup = blk if blk
+    @user_id_lookup
+  end
+
+  def group_ids_lookup(&blk)
+    @group_ids_lookup = blk if blk
+    @group_ids_lookup
+  end
+
+  def is_admin_lookup(&blk)
+    @is_admin_lookup = blk if blk
+    @is_admin_lookup
+  end
+
+  def on_connect(&blk)
+    @on_connect = blk if blk
+    @on_connect
+  end
+
+  def on_disconnect(&blk)
+    @on_disconnect = blk if blk
+    @on_disconnect
+  end
+
+  def allow_broadcast=(val)
+    @allow_broadcast = val
+  end
+
+  def allow_broadcast?
+    @allow_broadcast ||=
+      if defined? ::Rails
+        ::Rails.env.test? || ::Rails.env.development?
+      else
+        false
+      end
+  end
+
+  def reliable_pub_sub
+    @reliable_pub_sub ||= MessageBus::ReliablePubSub.new redis_config
+  end
+
+  def enable_diagnostics
+    MessageBus::Diagnostics.enable
+  end
+
+  def publish(channel, data, opts = nil)
+    return if @off
+
+    user_ids = nil
+    group_ids = nil
+    if opts
+      user_ids = opts[:user_ids]
+      group_ids = opts[:group_ids]
+    end
+
+    encoded_data = JSON.dump({
+      data: data,
+      user_ids: user_ids,
+      group_ids: group_ids
+    })
+
+    reliable_pub_sub.publish(encode_channel_name(channel), encoded_data)
+  end
+
+  def blocking_subscribe(channel=nil, &blk)
+    if channel
+      reliable_pub_sub.subscribe(encode_channel_name(channel), &blk)
+    else
+      reliable_pub_sub.global_subscribe(&blk)
+    end
+  end
+
+  ENCODE_SITE_TOKEN = "$|$"
+
+  # encode channel name to include site
+  def encode_channel_name(channel)
+    if MessageBus.site_id_lookup
+      raise ArgumentError.new channel if channel.include? ENCODE_SITE_TOKEN
+      "#{channel}#{ENCODE_SITE_TOKEN}#{MessageBus.site_id_lookup.call}"
+    else
+      channel
+    end
+  end
+
+  def decode_channel_name(channel)
+    channel.split(ENCODE_SITE_TOKEN)
+  end
+
+  def subscribe(channel=nil, &blk)
+    subscribe_impl(channel, nil, &blk)
+  end
+
+  # subscribe only on current site
+  def local_subscribe(channel=nil, &blk)
+    site_id = MessageBus.site_id_lookup.call if MessageBus.site_id_lookup
+    subscribe_impl(channel, site_id, &blk)
+  end
+
+  def backlog(channel=nil, last_id)
+    old =
+      if channel
+        reliable_pub_sub.backlog(encode_channel_name(channel), last_id)
+      else
+        reliable_pub_sub.global_backlog(encode_channel_name(channel), last_id)
+      end
+
+    old.each{ |m|
+      decode_message!(m)
+    }
+    old
+  end
+
+
+  def last_id(channel)
+    reliable_pub_sub.last_id(encode_channel_name(channel))
+  end
+
+  protected
+
+  def decode_message!(msg)
+    channel, site_id = decode_channel_name(msg.channel)
+    msg.channel = channel
+    msg.site_id = site_id
+    parsed = JSON.parse(msg.data)
+    msg.data = parsed["data"]
+    msg.user_ids = parsed["user_ids"]
+    msg.group_ids = parsed["group_ids"]
+  end
+
+  def subscribe_impl(channel, site_id, &blk)
+    @subscriptions ||= {}
+    @subscriptions[site_id] ||= {}
+    @subscriptions[site_id][channel] ||=  []
+    @subscriptions[site_id][channel] << blk
+    ensure_subscriber_thread
+  end
+
+  def ensure_subscriber_thread
+    @mutex ||= Mutex.new
+    @mutex.synchronize do
+      return if @subscriber_thread
+      @subscriber_thread = Thread.new do
+        reliable_pub_sub.global_subscribe do |msg|
+          begin
+            decode_message!(msg)
+
+            globals = @subscriptions[nil]
+            locals = @subscriptions[msg.site_id] if msg.site_id
+
+            global_globals = globals[nil] if globals
+            local_globals = locals[nil] if locals
+
+            globals = globals[msg.channel] if globals
+            locals = locals[msg.channel] if locals
+
+            multi_each(globals,locals, global_globals, local_globals) do |c|
+              begin
+                c.call msg
+              rescue => e
+                MessageBus.logger.warn "failed to deliver message, skipping #{msg.inspect}\n ex: #{e} backtrace: #{e.backtrace}"
+              end
+            end
+
+          rescue => e
+            MessageBus.logger.warn "failed to process message #{msg.inspect}\n ex: #{e} backtrace: #{e.backtrace}"
+          end
+
+        end
+      end
+    end
+  end
+
+  def multi_each(*args,&block)
+    args.each do |a|
+      a.each(&block) if a
+    end
+  end
+
+end
+
+module MessageBus
+  extend MessageBus::Implementation
+end
+
+# allows for multiple buses per app
+class MessageBus::Instance
+  include MessageBus::Implementation
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: message_bus
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Sam Saffron
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.3
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A message bus built on websockets
+email:
+- sam.saffron@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- LICENSE
+- lib/message_bus.rb
+- lib/message_bus/rack/diagnostics.rb
+- lib/message_bus/rack/middleware.rb
+- lib/message_bus/message.rb
+- lib/message_bus/connection_manager.rb
+- lib/message_bus/message_handler.rb
+- lib/message_bus/rails/railtie.rb
+- lib/message_bus/version.rb
+- lib/message_bus/diagnostics.rb
+- lib/message_bus/client.rb
+- lib/message_bus/reliable_pub_sub.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.2
+signing_key: 
+specification_version: 4
+summary: ''
+test_files: []