firehose 1.2.20 → 1.3.6

data/lib/firehose/rack/ping.rb

@@ -27,7 +27,7 @@ module Firehose
         SECONDS_TO_EXPIRE = 60
 
         def self.redis
-          @redis ||= EM::Hiredis.connect
+          @redis ||= Firehose::Server.redis.connection
         end
 
         def initialize(env, redis=nil)

data/lib/firehose/rack/publisher.rb

@@ -1,12 +1,14 @@
+require "rack/utils"
+
 module Firehose
   module Rack
     class Publisher
       include Firehose::Rack::Helpers
 
       def call(env)
-        req     = env['parsed_request'] ||= ::Rack::Request.new(env)
-        path    = req.path
-        method  = req.request_method
+        req           = env['parsed_request'] ||= ::Rack::Request.new(env)
+        path          = req.path
+        method        = req.request_method
         cache_control = {}
 
         # Parse out cache control directives from the Cache-Control header.
@@ -26,7 +28,11 @@ module Firehose
           EM.next_tick do
             body = env['rack.input'].read
             Firehose.logger.debug "HTTP published #{body.inspect} to #{path.inspect} with ttl #{ttl.inspect}"
-            publisher.publish(path, body, :ttl => ttl).callback do
+            opts = { :ttl => ttl }
+            if buffer_size = env["HTTP_X_FIREHOSE_BUFFER_SIZE"]
+              opts[:buffer_size] = buffer_size.to_i
+            end
+            publisher.publish(path, body, opts).callback do
               env['async.callback'].call [202, {'Content-Type' => 'text/plain', 'Content-Length' => '0'}, []]
               env['async.callback'].call response(202, '', 'Content-Type' => 'text/plain')
             end.errback do |e|

data/lib/firehose/server.rb

@@ -8,15 +8,15 @@ module Firehose
   # Firehose components that sit between the Rack HTTP software and the Redis server.
   # This mostly handles message sequencing and different HTTP channel names.
   module Server
-    autoload :Subscriber, 'firehose/server/subscriber'
-    autoload :Publisher,  'firehose/server/publisher'
-    autoload :Channel,    'firehose/server/channel'
-    autoload :App,        'firehose/server/app'
+    autoload :MessageBuffer,    'firehose/server/message_buffer'
+    autoload :Subscriber,       'firehose/server/subscriber'
+    autoload :Publisher,        'firehose/server/publisher'
+    autoload :Channel,          'firehose/server/channel'
+    autoload :App,              'firehose/server/app'
+    autoload :Redis,            'firehose/server/redis'
 
-    # Generates keys for all firehose interactions with Redis. Ensures a root
-    # key of `firehose`
-    def self.key(*segments)
-      segments.unshift(:firehose).join(':')
+    def self.redis
+      @redis ||= Redis.new
     end
   end
-end
+end

data/lib/firehose/server/channel.rb

@@ -2,57 +2,49 @@ module Firehose
   module Server
     # Connects to a specific channel on Redis and listens for messages to notify subscribers.
     class Channel
-      attr_reader :channel_key, :redis, :subscriber, :list_key, :sequence_key
+      attr_reader :channel_key, :list_key, :sequence_key
+      attr_reader :redis, :subscriber
 
       def self.redis
-        @redis ||= EM::Hiredis.connect
+        @redis ||= Firehose::Server.redis.connection
       end
 
       def self.subscriber
-        @subscriber ||= Server::Subscriber.new(EM::Hiredis.connect)
+        @subscriber ||= Server::Subscriber.new(Firehose::Server.redis.connection)
       end
 
       def initialize(channel_key, redis=self.class.redis, subscriber=self.class.subscriber)
-        @channel_key, @redis, @subscriber = channel_key, redis, subscriber
-        @list_key, @sequence_key = Server.key(channel_key, :list), Server.key(channel_key, :sequence)
+        @redis        = redis
+        @subscriber   = subscriber
+        @channel_key  = channel_key
+        @list_key     = Server::Redis.key(channel_key, :list)
+        @sequence_key = Server::Redis.key(channel_key, :sequence)
       end
 
-      def next_message(last_sequence=nil, options={})
-        last_sequence = last_sequence.to_i
-
+      def next_messages(consumer_sequence=nil, options={})
         deferrable = EM::DefaultDeferrable.new
-        # TODO - Think this through a little harder... maybe some tests ol buddy!
         deferrable.errback {|e| EM.next_tick { raise e } unless [:timeout, :disconnect].include?(e) }
 
-        # TODO: Use HSET so we don't have to pull 100 messages back every time.
         redis.multi
           redis.get(sequence_key).
             errback {|e| deferrable.fail e }
-          redis.lrange(list_key, 0, Server::Publisher::MAX_MESSAGES).
+          # Fetch entire list: http://stackoverflow.com/questions/10703019/redis-fetch-all-value-of-list-without-iteration-and-without-popping
+          redis.lrange(list_key, 0, -1).
             errback {|e| deferrable.fail e }
-        redis.exec.callback do |(sequence, message_list)|
-          Firehose.logger.debug "exec returned: `#{sequence}` and `#{message_list.inspect}`"
-          sequence = sequence.to_i
-
-          if sequence.nil? || (diff = sequence - last_sequence) <= 0
-            Firehose.logger.debug "No message available yet, subscribing. sequence: `#{sequence}`"
+        redis.exec.callback do |(channel_sequence, message_list)|
+          # Reverse the messages so they can be correctly procesed by the MessageBuffer class. There's
+          # a patch in the message-buffer-redis branch that moves this concern into the Publisher LUA
+          # script. We kept it out of this for now because it represents a deployment risk and `reverse!`
+          # is a cheap operation in Ruby.
+          message_list.reverse!
+          buffer = MessageBuffer.new(message_list, channel_sequence, consumer_sequence)
+          if buffer.remaining_messages.empty?
+            Firehose.logger.debug "No messages in buffer, subscribing. sequence: `#{channel_sequence}` consumer_sequence: #{consumer_sequence}"
             # Either this resource has never been seen before or we are all caught up.
             # Subscribe and hope something gets published to this end-point.
             subscribe(deferrable, options[:timeout])
-          elsif last_sequence > 0 && diff < Server::Publisher::MAX_MESSAGES
-            # The client is kinda-sorta running behind, but has a chance to catch
-            # up. Catch them up FTW.
-            # But we won't "catch them up" if last_sequence was zero/nil because
-            # that implies the client is connecting for the 1st time.
-            message = message_list[diff-1]
-            Firehose.logger.debug "Sending old message `#{message}` and sequence `#{sequence}` to client directly. Client is `#{diff}` behind, at `#{last_sequence}`."
-            deferrable.succeed message, last_sequence + 1
-          else
-            # The client is hopelessly behind and underwater. Just reset
-            # their whole world with the lastest message.
-            message = message_list[0]
-            Firehose.logger.debug "Sending latest message `#{message}` and sequence `#{sequence}` to client directly."
-            deferrable.succeed message, sequence
+          else # Either the client is under water or caught up to head.
+            deferrable.succeed buffer.remaining_messages
           end
         end.errback {|e| deferrable.fail e }
 

data/lib/firehose/server/message_buffer.rb

@@ -0,0 +1,59 @@
+module Firehose
+  module Server
+    # Encapsulates a sequence of messages from the server along with their
+    # consumer_sequences calculate by offset.
+    class MessageBuffer
+      # Number of messages that Redis buffers for the client if its
+      # connection drops, then reconnects.
+      DEFAULT_SIZE = 100
+
+      Message = Struct.new(:payload, :sequence)
+
+      def initialize(message_list, channel_sequence, consumer_sequence = nil)
+        @message_list = message_list
+        @channel_sequence = channel_sequence.to_i
+        @consumer_sequence = consumer_sequence.to_i
+      end
+
+      def remaining_messages
+        messages.last(remaining_message_count)
+      end
+
+      private
+
+      def remaining_message_count
+        # Special case to always get the latest message.
+        return 1 unless @consumer_sequence > 0
+
+        count = @channel_sequence - @consumer_sequence
+
+        if count < 0
+          # UNEXPECTED: Somehow the sequence is ahead of the channel.
+          #             It is likely a bug in the consumer, but we'll assume
+          #             the consumer has all the messages.
+          0
+        elsif count > @message_list.size
+          # Consumer is under water since the last request. Just send the most recent message.
+          1
+        else
+          count
+        end
+      end
+
+      # Calculates the last_message_sequence per message.
+      # [a b c e f]
+      def messages
+        @messages ||= @message_list.map.with_index do |payload, index|
+          Message.new(payload, starting_channel_sequence + index)
+        end
+      end
+
+      # Channel sequence is   10
+      # Buffer size of        5
+      # Start of sequence in buffer ... which would be 6
+      def starting_channel_sequence
+        @channel_sequence - @message_list.size + 1
+      end
+    end
+  end
+end

data/lib/firehose/server/publisher.rb

@@ -1,10 +1,6 @@
 module Firehose
   module Server
     class Publisher
-      # Number of messages that Redis buffers for the client if its
-      # connection drops, then reconnects.
-      MAX_MESSAGES = 100
-
       # Seconds that the message buffer should live before Redis expires it.
       TTL = 60*60*24
 
@@ -16,6 +12,7 @@ module Firehose
       def publish(channel_key, message, opts={})
         # How long should we hang on to the resource once is published?
         ttl = (opts[:ttl] || TTL).to_i
+        buffer_size = (opts[:buffer_size] || MessageBuffer::DEFAULT_SIZE).to_i
 
         # TODO hi-redis isn't that awesome... we have to setup an errback per even for wrong
         # commands because of the lack of a method_missing whitelist. Perhaps implement a whitelist in
@@ -44,10 +41,10 @@ module Firehose
           end.callback do |digest|
             @publish_script_digest = digest
             Firehose.logger.debug "Registered Lua publishing script with Redis => #{digest}"
-            eval_publish_script channel_key, message, ttl, deferrable
+            eval_publish_script channel_key, message, ttl, buffer_size, deferrable
           end
         else
-          eval_publish_script channel_key, message, ttl, deferrable
+          eval_publish_script channel_key, message, ttl, buffer_size, deferrable
         end
 
         deferrable
@@ -55,7 +52,7 @@ module Firehose
 
       private
       def redis
-        @redis ||= EM::Hiredis.connect
+        @redis ||= Firehose::Server.redis.connection
       end
 
       # Serialize components of a message into something that can be dropped into Redis.
@@ -63,9 +60,10 @@ module Firehose
         [channel_key, sequence, message].join(PAYLOAD_DELIMITER)
       end
 
-      # Deserealize components of a message back into Ruby.
+      # Deserialize components of a message back into Ruby.
       def self.from_payload(payload)
-        payload.split(PAYLOAD_DELIMITER, method(:to_payload).arity)
+        @payload_size ||= method(:to_payload).arity
+        payload.split(PAYLOAD_DELIMITER, @payload_size)
       end
 
       # TODO: Make this FAR more robust. Ideally we'd whitelist the permitted
@@ -79,18 +77,19 @@ module Firehose
         redis.script 'LOAD', REDIS_PUBLISH_SCRIPT
       end
 
-      def eval_publish_script(channel_key, message, ttl, deferrable)
-        list_key = Server.key(channel_key, :list)
+      def eval_publish_script(channel_key, message, ttl, buffer_size, deferrable)
+        list_key = Server::Redis.key(channel_key, :list)
         script_args = [
-          Server.key(channel_key, :sequence),
+          Server::Redis.key(channel_key, :sequence),
           list_key,
-          Server.key(:channel_updates),
+          Server::Redis.key(:channel_updates),
           ttl,
           message,
-          MAX_MESSAGES,
+          buffer_size,
           PAYLOAD_DELIMITER,
           channel_key
         ]
+
         redis.evalsha(
           @publish_script_digest, script_args.length, *script_args
         ).errback do |e|
@@ -107,7 +106,7 @@ module Firehose
         local channel_key       = KEYS[3]
         local ttl               = KEYS[4]
         local message           = KEYS[5]
-        local max_messages      = KEYS[6]
+        local buffer_size       = KEYS[6]
         local payload_delimiter = KEYS[7]
         local firehose_resource = KEYS[8]
 
@@ -122,7 +121,7 @@ module Firehose
         redis.call('set', sequence_key, sequence)
         redis.call('expire', sequence_key, ttl)
         redis.call('lpush', list_key, message)
-        redis.call('ltrim', list_key, 0, max_messages - 1)
+        redis.call('ltrim', list_key, 0, buffer_size - 1)
         redis.call('expire', list_key, ttl)
         redis.call('publish', channel_key, message_payload)
 
@@ -131,4 +130,4 @@ module Firehose
 
     end
   end
-end
+end

data/lib/firehose/server/redis.rb

@@ -0,0 +1,32 @@
+require "uri"
+
+module Firehose
+  module Server
+    # Manages redis configuration and connections.
+    class Redis
+      DEFAULT_URL = "redis://127.0.0.1:6379/0".freeze
+      KEY_DELIMITER = ":".freeze
+      ROOT_KEY = "firehose".freeze
+
+      attr_reader :url
+
+      def initialize(url = self.class.url)
+        @url = URI(url)
+      end
+
+      def connection
+        EM::Hiredis.connect(@url)
+      end
+
+      # Generates keys for all firehose interactions with Redis. Ensures a root
+      # key of `firehose`
+      def self.key(*segments)
+        segments.flatten.unshift(ROOT_KEY).join(KEY_DELIMITER)
+      end
+
+      def self.url
+        ENV.fetch("REDIS_URL", DEFAULT_URL)
+      end
+    end
+  end
+end

data/lib/firehose/server/subscriber.rb

@@ -16,18 +16,18 @@ module Firehose
         #       with the same error.
         #       The final goal is to allow the failed deferrable bubble back up
         #       so we can send back a nice, clean 500 error to the client.
-        channel_updates_key = Server.key('channel_updates')
+        channel_updates_key = Server::Redis.key('channel_updates')
         pubsub.subscribe(channel_updates_key).
           errback{|e| EM.next_tick { raise e } }.
           callback { Firehose.logger.debug "Redis subscribed to `#{channel_updates_key}`" }
         pubsub.on(:message) do |_, payload|
-          channel_key, sequence, message = Server::Publisher.from_payload(payload)
-
+          channel_key, channel_sequence, message = Server::Publisher.from_payload(payload)
+          messages = [ MessageBuffer::Message.new(message, channel_sequence.to_i) ]
           if deferrables = subscriptions.delete(channel_key)
-            Firehose.logger.debug "Redis notifying #{deferrables.count} deferrable(s) at `#{channel_key}` with sequence `#{sequence}` and message `#{message}`"
+            Firehose.logger.debug "Redis notifying #{deferrables.count} deferrable(s) at `#{channel_key}` with channel_sequence `#{channel_sequence}` and message `#{message}`"
             deferrables.each do |deferrable|
-              Firehose.logger.debug "Sending message #{message} and sequence #{sequence} to client from subscriber"
-              deferrable.succeed message, sequence.to_i
+              Firehose.logger.debug "Sending message #{message} and channel_sequence #{channel_sequence} to client from subscriber"
+              deferrable.succeed messages
             end
           end
         end
@@ -48,4 +48,4 @@ module Firehose
       end
     end
   end
-end
+end

data/lib/firehose/version.rb

@@ -1,4 +1,4 @@
 module Firehose
-  VERSION  = "1.2.20"
-  CODENAME = "Onprogress of Doom"
+  VERSION  = "1.3.6"
+  CODENAME = "Oops I did it again"
 end

data/package.json

@@ -1,5 +1,17 @@
 {
   "name": "firehose",
-  "version": "1.2.20",
-  "main": "lib/assets/javascripts/firehose.js.coffee"
+  "version": "1.3.3",
+  "main": "lib/assets/javascripts/firehose.js.coffee",
+  "devDependencies": {
+    "coffee-script": "*",
+    "jasmine-jquery": "git://github.com/velesin/jasmine-jquery.git",
+    "karma": "~0.12.16",
+    "karma-chrome-launcher": "~0.1.4",
+    "karma-coffee-preprocessor": "~0.2.1",
+    "karma-jasmine": "~0.2.0",
+    "karma-junit-reporter": "^0.2.2",
+    "karma-phantomjs-launcher": "^0.1.4",
+    "karma-safari-launcher": "*",
+    "karma-sprockets-mincer": "0.1.2"
+  }
 }

data/spec/integrations/shared_examples.rb

@@ -23,9 +23,12 @@ shared_examples_for 'Firehose::Rack::App' do
   let(:messages)  { (1..200).map{|n| "msg-#{n}" } }
   let(:channel)   { "/firehose/integration/#{Time.now.to_i}" }
   let(:http_url)  { "http://#{uri.host}:#{uri.port}#{channel}" }
+  let(:http_multi_url) { "http://#{uri.host}:#{uri.port}/channels@firehose" }
   let(:ws_url)    { "ws://#{uri.host}:#{uri.port}#{channel}" }
+  let(:multiplex_channels) { ["/foo/bar", "/bar/baz", "/baz/quux"] }
+  let(:subscription_query) { multiplex_channels.map{|c| "#{c}!0"}.join(",") }
 
-  it "should pub-sub http and websockets" do
+  it "supports pub-sub http and websockets" do
     # Setup variables that we'll use after we turn off EM to validate our
     # test assertions.
     outgoing, received = messages.dup, Hash.new{|h,k| h[k] = []}
@@ -41,7 +44,7 @@ shared_examples_for 'Firehose::Rack::App' do
 
     # Setup a publisher
     publish = Proc.new do
-      Firehose::Client::Producer::Http.new.publish(outgoing.shift).to(channel) do
+      Firehose::Client::Producer::Http.new.publish(outgoing.shift).to(channel, buffer_size: rand(100)) do
         # The random timer ensures that sometimes the clients will be behind
         # and sometimes they will be caught up.
         EM::add_timer(rand*0.005) { publish.call } unless outgoing.empty?
@@ -100,19 +103,98 @@ shared_examples_for 'Firehose::Rack::App' do
     end
 
     # When EM stops, these assertions will be made.
-    received.size.should == 4
-    received.values.each do |arr|
-      arr.should == messages
+    expect(received.size).to eql(4)
+    received.each_value do |arr|
+      expect(arr.size).to eql(messages.size)
+      expect(arr.sort).to eql(messages.sort)
+    end
+  end
+
+  it "supports channel multiplexing for http_long_poll and websockets" do
+    # Setup variables that we'll use after we turn off EM to validate our
+    # test assertions.
+    outgoing, received = messages.dup, Hash.new{|h,k| h[k] = []}
+
+    # Our WS and Http clients call this when they have received their messages to determine
+    # when to turn off EM and make the test assertion at the very bottom.
+    succeed = Proc.new do
+      # TODO: For some weird reason the `add_timer` call causes up to 20 seconds of delay after
+      #       the test finishes running. However, without it the test will randomly fail with a
+      #       "Redis disconnected" error.
+      em.add_timer(1) { em.stop } if received.values.all?{|arr| arr.size == messages.size }
+    end
+
+    # Lets have an HTTP Long poll client using channel multiplexing
+    multiplexed_http_long_poll = Proc.new do |cid, last_sequence|
+      http = EM::HttpRequest.new(http_multi_url).get(:query => {'subscribe' => subscription_query})
+
+      http.errback { em.stop }
+      http.callback do
+        frame = JSON.parse(http.response, :symbolize_names => true)
+        received[cid] << frame[:message]
+        if received[cid].size < messages.size
+          # Add some jitter so the clients aren't syncronized
+          EM::add_timer(rand*0.001) { multiplexed_http_long_poll.call cid, frame[:last_sequence] }
+        else
+          succeed.call cid
+        end
+      end
+    end
+
+    # Test multiplexed web socket client
+    outgoing = messages.dup
+    publish_multi = Proc.new do
+      msg = outgoing.shift
+      chan = multiplex_channels[rand(multiplex_channels.size)]
+      Firehose::Client::Producer::Http.new.publish(msg).to(chan) do
+        EM::add_timer(rand*0.005) { publish_multi.call } unless outgoing.empty?
+      end
+    end
+
+    multiplexed_websocket = Proc.new do |cid|
+      ws = Faye::WebSocket::Client.new("ws://#{uri.host}:#{uri.port}/channels@firehose?subscribe=#{subscription_query}")
+
+      ws.onmessage = lambda do |event|
+        frame = JSON.parse(event.data, :symbolize_names => true)
+        received[cid] << frame[:message]
+        succeed.call cid unless received[cid].size < messages.size
+      end
+
+      ws.onclose = lambda do |event|
+        ws = nil
+      end
+
+      ws.onerror = lambda do |event|
+        raise 'ws failed' + "\n" + event.inspect
+      end
+    end
+
+    em 180 do
+      # Start the clients.
+      multiplexed_http_long_poll.call(5)
+      multiplexed_http_long_poll.call(6)
+      multiplexed_websocket.call(7)
+      multiplexed_websocket.call(8)
+
+      # Wait a sec to let our clients set up.
+      em.add_timer(1){ publish_multi.call }
+    end
+
+    # When EM stops, these assertions will be made.
+    expect(received.size).to eql(4)
+    received.each_value do |arr|
+      expect(arr.size).to be <= messages.size
+      # expect(arr.sort).to eql(messages.sort)
     end
   end
 
 
-  it "should return 400 error for long-polling when using http long polling and sequence header is < 0" do
+  it "returns 400 error for long-polling when using http long polling and sequence header is < 0" do
     em 5 do
       http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => -1})
       http.errback { |e| raise e.inspect }
       http.callback do
-        http.response_header.status.should == 400
+        expect(http.response_header.status).to eql(400)
         em.stop
       end
     end