firehose 1.1.1 → 1.2.0

data/lib/{rainbows_em_swf_policy.rb → firehose/patches/rainbows.rb}

@@ -2,7 +2,7 @@
 # Enable this with something like this in your config/rainbows.rb file:
 #
 #     after_fork do |server, worker|
-#       require 'rainbows_em_swf_policy'
+#       require 'firehose/patches/rainbows'
 #     end if ENV['RACK_ENV'] == 'development'
 #
 # You should only use this in development. It has not been well tested in a
@@ -15,12 +15,14 @@
 # http://www.adobe.com/devnet/flashplayer/articles/socket_policy_files.html
 # http://blog.vokle.com/index.php/2009/06/10/dealing-with-adobe-and-serving-socket-policy-servers-via-nginx-and-10-lines-of-code/
 
+require 'firehose/patches/swf_policy_request'
 require 'rainbows'
+
 # Ensure the class already exists so we are overwriting it.
 Rainbows::EventMachine::Client
 
 class Rainbows::EventMachine::Client
-  include Firehose::SwfPolicyRequest
+  include Firehose::Patches::SwfPolicyRequest
   alias_method :receive_data_without_swf_policy, :receive_data
   # Borrowed from: https://github.com/igrigorik/em-websocket/blob/3e7f7d7760cc23b9d1d34fc1c17bab4423b5d11a/lib/em-websocket/connection.rb#L104
   def receive_data(data)

data/lib/firehose/patches/swf_policy_request.rb

@@ -0,0 +1,26 @@
+module Firehose
+  module Patches
+    # Helpers for making Firehose work with Macromedia Flash Sockets. Since this doesn't use "normal" HTTP, we
+    # have to monkey patch both Rainbows and Thin to recognize when a request is for a SWF policy.
+    module SwfPolicyRequest
+      # Borrowed from: https://github.com/igrigorik/em-websocket/blob/3e7f7d7760cc23b9d1d34fc1c17bab4423b5d11a/lib/em-websocket/connection.rb#L104
+      def handle_swf_policy_request(data)
+        if data =~ /\A<policy-file-request\s*\/>/
+          Firehose.logger.debug "Received SWF Policy request: #{data.inspect}"
+          send_data policy
+          close_connection_after_writing
+          true
+        end
+      end
+
+      def policy
+        <<-EOS
+  <?xml version="1.0"?>
+  <cross-domain-policy>
+    <allow-access-from domain="*" to-ports="*"/>
+  </cross-domain-policy>
+        EOS
+      end
+    end
+  end
+end

data/lib/{thin_em_swf_policy.rb → firehose/patches/thin.rb}

@@ -9,12 +9,13 @@
 # http://www.adobe.com/devnet/flashplayer/articles/socket_policy_files.html
 # http://blog.vokle.com/index.php/2009/06/10/dealing-with-adobe-and-serving-socket-policy-servers-via-nginx-and-10-lines-of-code/
 
+require 'firehose/patches/swf_policy_request'
 require 'thin'
 # Ensure the class already exists so we are overwriting it.
 Thin::Connection
 
 class Thin::Connection
-  include Firehose::SwfPolicyRequest
+  include Firehose::Patches::SwfPolicyRequest
   alias_method :receive_data_without_swf_policy, :receive_data
   def receive_data(data)
     if handle_swf_policy_request(data)

data/lib/firehose/rack.rb

@@ -1,55 +1,28 @@
 module Firehose
   module Rack
-    autoload :ConsumerApp,    'firehose/rack/consumer_app'
-    autoload :PublisherApp,   'firehose/rack/publisher_app'
-    autoload :PingApp,        'firehose/rack/ping_app'
+    autoload :Consumer,   'firehose/rack/consumer'
+    autoload :Publisher,  'firehose/rack/publisher'
+    autoload :Ping,       'firehose/rack/ping'
+    autoload :App,        'firehose/rack/app'
 
-    # Evented web servers recognize this as a response deferral.
+    # Evented web servers recognize the -1 HTTP code as a response deferral, which
+    # is needed to stream responses via WebSockets or HTTP long polling.
     ASYNC_RESPONSE = [-1, {}, []].freeze
 
     # Normally we'd want to use a custom header to reduce the likelihood of some
     # HTTP middleware clobbering the value. But Safari seems to ignore our CORS
     # header instructions, so we are using 'pragma' because it is always allowed.
     LAST_MESSAGE_SEQUENCE_HEADER = 'Pragma'
-    RACK_LAST_MESSAGE_SEQUENCE_HEADER = "HTTP_#{LAST_MESSAGE_SEQUENCE_HEADER.upcase.gsub('-', '_')}"
-    # Don't cache in development mode
-    CORS_OPTIONS_MAX_AGE = ENV['RACK_ENV'] == 'development' ? '1' : '1728000'
 
-    # Allows the publisher and consumer to be mounted on the same port.
-    class App
-      def call(env)
-        # Cache the parsed request so we don't need to re-parse it when we pass
-        # control onto another app.
-        req     = env['parsed_request'] ||= ::Rack::Request.new(env)
-        method  = req.request_method
-
-        case method
-        when 'PUT'
-          publisher.call(env)
-        when 'HEAD'
-          ping.call(env)
-        else
-          consumer.call(env)
-        end
-      end
-
-
-      private
-      def publisher
-        @publisher ||= PublisherApp.new
-      end
-
-      def consumer
-        @consumer ||= ConsumerApp.new
-      end
+    # Rack wants the header to start with HTTP, so we deal with that here.
+    RACK_LAST_MESSAGE_SEQUENCE_HEADER = "HTTP_#{LAST_MESSAGE_SEQUENCE_HEADER.upcase.gsub('-', '_')}"
 
-      def ping
-        @ping ||= PingApp.new
-      end
-    end
+    # Disable CORS preflight caches for requests in development mode.
+    CORS_OPTIONS_MAX_AGE = ENV['RACK_ENV'] == 'development' ? '1' : '1728000'
 
     module Helpers
-      # Calculates the content length for you
+      # Calculates the content of a message body for the response so that HTTP Keep-Alive
+      # connections work.
       def response(status, body='', headers={})
         headers = {'Content-Length' => body.size.to_s}.merge(headers)
         [status, headers, [body]]

data/lib/firehose/rack/app.rb

@@ -0,0 +1,42 @@
+module Firehose
+  module Rack
+    # Acts as the glue between the HTTP/WebSocket world and the Firehose::Server class,
+    # which talks directly to the Redis server. Also dispatches between HTTP and WebSocket
+    # transport handlers depending on the clients' request.
+    class App
+      def call(env)
+        # Cache the parsed request so we don't need to re-parse it when we pass
+        # control onto another app.
+        req     = env['parsed_request'] ||= ::Rack::Request.new(env)
+        method  = req.request_method
+
+        case method
+        when 'PUT'
+          # Firehose::Client::Publisher PUT's payloads to the server.
+          publisher.call(env)
+        when 'HEAD' 
+          # HEAD requests are used to prevent sockets from timing out
+          # from inactivity
+          ping.call(env)
+        else
+          # TODO - 'harden' this up with a GET request and throw a "Bad Request" 
+          # HTTP error code. I'd do it now but I'm in a plane and can't think of it.
+          consumer.call(env)
+        end
+      end
+
+      private
+      def publisher
+        @publisher ||= Publisher.new
+      end
+
+      def consumer
+        @consumer ||= Consumer.new
+      end
+
+      def ping
+        @ping ||= Ping.new
+      end
+    end
+  end
+end

data/lib/firehose/rack/{consumer_app.rb → consumer.rb}

@@ -3,7 +3,10 @@ require 'json'
 
 module Firehose
   module Rack
-    class ConsumerApp
+    # Handles a subscription request over HTTP or WebSockets depeding on its abilities and
+    # binds that to the Firehose::Server::Subscription class, which is bound to a channel that
+    # gets published to.
+    class Consumer
       def call(env)
         websocket_request?(env) ? websocket.call(env) : http_long_poll.call(env)
       end
@@ -47,7 +50,7 @@ module Firehose
               if last_sequence < 0
                 env['async.callback'].call response(400, "The last_message_sequence parameter may not be less than zero", response_headers(env))
               else
-                Channel.new(path).next_message(last_sequence, :timeout => TIMEOUT).callback do |message, sequence|
+                Server::Channel.new(path).next_message(last_sequence, :timeout => TIMEOUT).callback do |message, sequence|
                   env['async.callback'].call response(200, wrap_frame(message, sequence), response_headers(env))
                 end.errback do |e|
                   if e == :timeout
@@ -107,11 +110,10 @@ module Firehose
           return @ws.rack_response
         end
 
-      private
-
+        private
         def subscribe(last_sequence)
           @subscribed = true
-          @channel    = Channel.new @path
+          @channel    = Server::Channel.new @path
           @deferrable = @channel.next_message last_sequence
           @deferrable.callback do |message, sequence|
             Firehose.logger.debug "WS sent `#{message}` to `#{@path}` with sequence `#{sequence}`"

data/lib/firehose/rack/{ping_app.rb → ping.rb}

@@ -1,6 +1,9 @@
 module Firehose
   module Rack
-    class PingApp
+    # Allows the Firehose client to periodically "ping" the server
+    # so that the connection isn't timed out by browsers or proxies from
+    # inactivity.
+    class Ping
       attr_reader :redis
 
       def initialize(redis=nil)
@@ -12,7 +15,6 @@ module Firehose
         ASYNC_RESPONSE
       end
 
-
       # Encapsulate this in a class so we aren't passing a bunch of variables around
       class PingCheck
         include Firehose::Rack::Helpers

data/lib/firehose/rack/{publisher_app.rb → publisher.rb}

@@ -1,6 +1,6 @@
 module Firehose
   module Rack
-    class PublisherApp
+    class Publisher
       include Firehose::Rack::Helpers
 
       def call(env)
@@ -44,9 +44,9 @@ module Firehose
         end
       end
 
-    private
+      private
       def publisher
-        @publisher ||= Firehose::Publisher.new
+        @publisher ||= Firehose::Server::Publisher.new
       end
     end
   end

data/lib/firehose/server.rb

@@ -1,48 +1,22 @@
 require 'faye/websocket'
+require 'em-hiredis'
 
-module Firehose
-  class Server
-    def initialize(opts={})
-      @port   = opts[:port]   || Firehose::Default::URI.port
-      @host   = opts[:host]   || Firehose::Default::URI.host
-      @server = opts[:server] || :rainbows
-
-      Firehose.logger.info "Starting #{Firehose::VERSION} '#{Firehose::CODENAME}', in #{ENV['RACK_ENV']}"
-    end
-
-    def start
-      self.send("start_#{@server}")
-    end
-
-  private
-    def start_rainbows
-      require 'rainbows'
-      Faye::WebSocket.load_adapter('rainbows')
-
-      rackup = Unicorn::Configurator::RACKUP
-      rackup[:port] = @port if @port
-      rackup[:host] = @host if @host
-      rackup[:set_listener] = true
-      opts = rackup[:options]
-      opts[:config_file] = File.expand_path('../../../config/rainbows.rb', __FILE__)
+# Set the EM::Hiredis logger to be the same as the Firehose logger.
+EM::Hiredis.logger = Firehose.logger
 
-      server = Rainbows::HttpServer.new(Firehose::Rack::App.new, opts)
-      server.start.join
-    end
-
-    def start_thin
-      require 'thin'
-      require 'thin_em_swf_policy' if ENV['RACK_ENV'] == 'development'
-
-      Faye::WebSocket.load_adapter('thin')
-
-      # TODO: See if we can just set Thin to use Firehose.logger instead of
-      #       printing out messages by itself.
-      Thin::Logging.silent = true if Firehose.logger.level == Logger::ERROR
-
-      server = Thin::Server.new(@host, @port) do
-        run Firehose::Rack::App.new
-      end.start
+module Firehose
+  # Firehose components that sit between the Rack HTTP software and the Redis server.
+  # This mostly handles message sequencing and different HTTP channel names.
+  module Server
+    autoload :Subscriber, 'firehose/server/subscriber'
+    autoload :Publisher,  'firehose/server/publisher'
+    autoload :Channel,    'firehose/server/channel'
+    autoload :App,        'firehose/server/app'
+
+    # Generates keys for all firehose interactions with Redis. Ensures a root
+    # key of `firehose`
+    def self.key(*segments)
+      segments.unshift(:firehose).join(':')
     end
   end
 end

data/lib/firehose/server/app.rb

@@ -0,0 +1,53 @@
+module Firehose
+  module Server
+    # Configure servers that are booted with-out going through Rack. This is mostly used by
+    # the `firehose server` CLI command or for testing. Production configurations are likely
+    # to boot with custom rack configurations.
+    class App
+      def initialize(opts={})
+        @port   = opts[:port]   || Firehose::URI.port
+        @host   = opts[:host]   || Firehose::URI.host
+        @server = opts[:server] || :rainbows
+
+        Firehose.logger.info "Starting #{Firehose::VERSION} '#{Firehose::CODENAME}', in #{ENV['RACK_ENV']}"
+      end
+
+      def start
+        self.send("start_#{@server}")
+      end
+
+      private
+      # Boot the Firehose server with the Rainbows app server.
+      def start_rainbows
+        require 'rainbows'
+        Faye::WebSocket.load_adapter('rainbows')
+
+        rackup = Unicorn::Configurator::RACKUP
+        rackup[:port] = @port if @port
+        rackup[:host] = @host if @host
+        rackup[:set_listener] = true
+        opts = rackup[:options]
+        opts[:config_file] = File.expand_path('../../../../config/rainbows.rb', __FILE__)
+
+        server = Rainbows::HttpServer.new(Firehose::Rack::App.new, opts)
+        server.start.join
+      end
+
+      # Boot the Firehose server with the Thin app server.
+      def start_thin
+        require 'thin'
+        require 'firehose/patches/thin' if ENV['RACK_ENV'] == 'development'
+
+        Faye::WebSocket.load_adapter('thin')
+
+        # TODO: See if we can just set Thin to use Firehose.logger instead of
+        #       printing out messages by itself.
+        Thin::Logging.silent = true if Firehose.logger.level == Logger::ERROR
+
+        server = Thin::Server.new(@host, @port) do
+          run Firehose::Rack::App.new
+        end.start
+      end
+    end
+  end
+end

data/lib/firehose/server/channel.rb

@@ -0,0 +1,80 @@
+module Firehose
+  module Server
+    # Connects to a specific channel on Redis and listens for messages to notify subscribers.
+    class Channel
+      attr_reader :channel_key, :redis, :subscriber, :list_key, :sequence_key
+
+      def self.redis
+        @redis ||= EM::Hiredis.connect
+      end
+
+      def self.subscriber
+        @subscriber ||= Server::Subscriber.new(EM::Hiredis.connect)
+      end
+
+      def initialize(channel_key, redis=self.class.redis, subscriber=self.class.subscriber)
+        @channel_key, @redis, @subscriber = channel_key, redis, subscriber
+        @list_key, @sequence_key = Server.key(channel_key, :list), Server.key(channel_key, :sequence)
+      end
+
+      def next_message(last_sequence=nil, options={})
+        last_sequence = last_sequence.to_i
+
+        deferrable = EM::DefaultDeferrable.new
+        # TODO - Think this through a little harder... maybe some tests ol buddy!
+        deferrable.errback {|e| EM.next_tick { raise e } unless [:timeout, :disconnect].include?(e) }
+
+        # TODO: Use HSET so we don't have to pull 100 messages back every time.
+        redis.multi
+          redis.get(sequence_key).
+            errback {|e| deferrable.fail e }
+          redis.lrange(list_key, 0, Server::Publisher::MAX_MESSAGES).
+            errback {|e| deferrable.fail e }
+        redis.exec.callback do |(sequence, message_list)|
+          Firehose.logger.debug "exec returened: `#{sequence}` and `#{message_list.inspect}`"
+          sequence = sequence.to_i
+
+          if sequence.nil? || (diff = sequence - last_sequence) <= 0
+            Firehose.logger.debug "No message available yet, subscribing. sequence: `#{sequence}`"
+            # Either this resource has never been seen before or we are all caught up.
+            # Subscribe and hope something gets published to this end-point.
+            subscribe(deferrable, options[:timeout])
+          elsif last_sequence > 0 && diff < Server::Publisher::MAX_MESSAGES
+            # The client is kinda-sorta running behind, but has a chance to catch
+            # up. Catch them up FTW.
+            # But we won't "catch them up" if last_sequence was zero/nil because
+            # that implies the client is connecting for the 1st time.
+            message = message_list[diff-1]
+            Firehose.logger.debug "Sending old message `#{message}` and sequence `#{sequence}` to client directly. Client is `#{diff}` behind, at `#{last_sequence}`."
+            deferrable.succeed message, last_sequence + 1
+          else
+            # The client is hopelessly behind and underwater. Just reset
+            # their whole world with the lastest message.
+            message = message_list[0]
+            Firehose.logger.debug "Sending latest message `#{message}` and sequence `#{sequence}` to client directly."
+            deferrable.succeed message, sequence
+          end
+        end.errback {|e| deferrable.fail e }
+
+        deferrable
+      end
+
+      def unsubscribe(deferrable)
+        subscriber.unsubscribe channel_key, deferrable
+      end
+
+      private
+      def subscribe(deferrable, timeout=nil)
+        subscriber.subscribe(channel_key, deferrable)
+        if timeout
+          timer = EventMachine::Timer.new(timeout) do
+            deferrable.fail :timeout
+            unsubscribe deferrable
+          end
+          # Cancel the timer if when the deferrable succeeds
+          deferrable.callback { timer.cancel }
+        end
+      end
+    end
+  end
+end