firehose 0.1.1 → 0.2.alpha.2

data/lib/firehose/default.rb

@@ -3,6 +3,6 @@ require 'uri'
 module Firehose
   module Default
     # Default URI for the Firehose server. Consider the port "well-known" and bindable from other apps.
-    URI = URI.parse("//127.0.0.1:7474").freeze
+    URI = URI.parse("//0.0.0.0:7474").freeze
   end
-end
+end

data/lib/firehose/logging.rb

@@ -0,0 +1,35 @@
+# Sets up logging
+
+require 'logger'
+
+module Firehose
+  def self.logger
+    @logger ||= Logger.new($stdout)
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  self.logger.level = if ENV['LOG_LEVEL']
+    Logger.const_get(ENV['LOG_LEVEL'].upcase)
+  else
+    case ENV['RACK_ENV']
+      when 'test'         then Logger::ERROR
+      when 'development'  then Logger::DEBUG
+      else Logger::INFO
+    end
+  end
+
+  # TODO: Provide some way to allow this to be configured via an ENV variable.
+  self.logger.formatter = lambda do |severity, time, name, msg|
+    out_time = time.utc.strftime "%Y-%m-%d %H:%M:%S.%L"
+    "[#{out_time} ##$$] #{severity} : #{msg}\n"
+  end
+end
+
+EM::Hiredis.logger = Firehose.logger
+
+# stdout gets "lost" in Foreman if this isn't here
+# https://github.com/ddollar/foreman/wiki/Missing-Output
+$stdout.sync = true if ENV['RACK_ENV'] == 'development' || ENV['SYNC_LOGGING']

data/lib/firehose/producer.rb

@@ -1,3 +1,4 @@
+# TODO Move this into the Firehose::Client:Producer namespace and rename the class to Http (its an HTTP publisher dumby!)
 require "faraday"
 require "uri"
 

data/lib/firehose/publisher.rb

@@ -1,13 +1,65 @@
 module Firehose
   class Publisher
-    def publish(channel, message)
-      Firehose.logger.debug "Redis publishing `#{message}` to `#{channel}`"
-      redis.publish(channel, message).errback { |msg| raise "Error publishing: #{msg}" }
+    MAX_MESSAGES = 100
+    TTL = 60*60*24  # 1 day of time, yay!
+    PAYLOAD_DELIMITER = "\n"
+
+    def publish(channel_key, message)
+      # TODO hi-redis isn't that awesome... we have to setup an errback per even for wrong
+      # commands because of the lack of a method_missing whitelist. Perhaps implement a whitelist in
+      # em-hiredis or us a diff lib?
+      deferrable = EM::DefaultDeferrable.new
+      deferrable.errback {|e| EM.next_tick { raise e } }
+
+      # DRY up keys a little bit for the epic publish command to come.
+      list_key = key(channel_key, :list)
+      sequence_key = key(channel_key, :sequence)
+
+      redis.eval(%(local current_sequence = redis.call('get', KEYS[1])
+                   if (current_sequence == nil) or (current_sequence == false)
+                   then
+                     current_sequence = 0
+                   end
+                   local sequence = current_sequence + 1
+                   redis.call('set', KEYS[1], sequence)
+                   redis.call('expire', KEYS[1], #{TTL})
+                   redis.call('lpush', KEYS[2], "#{lua_escape(message)}")
+                   redis.call('ltrim', KEYS[2], 0, #{MAX_MESSAGES - 1})
+                   redis.call('expire', KEYS[2], #{TTL})
+                   redis.call('publish', KEYS[3], "#{lua_escape(channel_key + PAYLOAD_DELIMITER)}" .. sequence .. "#{lua_escape(PAYLOAD_DELIMITER + message)}")
+                   return sequence
+                  ), 3, sequence_key, list_key, key(:channel_updates)).
+        errback{|e| deferrable.fail e }.
+        callback do |sequence|
+          Firehose.logger.debug "Redis stored/published `#{message}` to list `#{list_key}` with sequence `#{sequence}`"
+          deferrable.succeed
+        end
+
+      deferrable
     end
 
   private
+    def key(*segments)
+      segments.unshift(:firehose).join(':')
+    end
+
     def redis
       @redis ||= EM::Hiredis.connect
     end
+
+    def self.to_payload(channel_key, sequence, message)
+      [channel_key, sequence, message].join(PAYLOAD_DELIMITER)
+    end
+
+    def self.from_payload(payload)
+      payload.split(PAYLOAD_DELIMITER, method(:to_payload).arity)
+    end
+
+    # TODO: Make this FAR more robust. Ideally we'd whitelist the permitted
+    #       characters and then escape or remove everything else.
+    #       See: http://en.wikibooks.org/wiki/Lua_Programming/How_to_Lua/escape_sequence
+    def lua_escape(str)
+      str.gsub(/\\/,'\\\\\\').gsub(/"/,'\"').gsub(/\n/,'\n').gsub(/\r/,'\r')
+    end
   end
-end
+end

data/lib/firehose/rack.rb

@@ -1,142 +1,59 @@
-require 'rack/websocket'
-
 module Firehose
   module Rack
-    AsyncResponse = [-1, {}, []]
-
-    class HttpLongPoll
+    autoload :ConsumerApp,    'firehose/rack/consumer_app'
+    autoload :PublisherApp,   'firehose/rack/publisher_app'
+    autoload :PingApp,        'firehose/rack/ping_app'
+
+    # Evented web servers recognize this as a response deferral.
+    ASYNC_RESPONSE = [-1, {}, []].freeze
+
+    # Normally we'd want to use a custom header to reduce the likelihood of some
+    # HTTP middleware clobbering the value. But Safari seems to ignore our CORS
+    # header instructions, so we are using 'pragma' because it is always allowed.
+    LAST_MESSAGE_SEQUENCE_HEADER = 'pragma'
+    RACK_LAST_MESSAGE_SEQUENCE_HEADER = "HTTP_#{LAST_MESSAGE_SEQUENCE_HEADER.upcase.gsub('-', '_')}"
+    # Don't cache in development mode
+    CORS_OPTIONS_MAX_AGE = ENV['RACK_ENV'] == 'development' ? '1' : '1728000'
+
+    # Allows the publisher and consumer to be mounted on the same port.
+    class App
       def call(env)
-        req     = ::Rack::Request.new(env)
-        cid     = req.params['cid']
-        path    = req.path
+        # Cache the parsed request so we don't need to re-parse it when we pass
+        # control onto another app.
+        req     = env['parsed_request'] ||= ::Rack::Request.new(env)
         method  = req.request_method
-        timeout = 30
-        queue_name = "#{cid}@#{path}"
-
-        # TODO seperate out CORS logic as an async middleware with a Goliath web server.
-        cors_origin = env['HTTP_ORIGIN']
-        cors_headers  = {
-          'Access-Control-Allow-Origin'     => cors_origin,
-          'Access-Control-Allow-Methods'    => 'GET',
-          'Access-Control-Max-Age'          => '1728000',
-          'Access-Control-Allow-Headers'    => 'Content-Type, User-Agent, If-Modified-Since, Cache-Control'
-        }
 
         case method
-        # GET is how clients subscribe to the queue. When a messages comes in, we flush out a response,
-        # close down the requeust, and the client then reconnects.
-        when 'GET'
-          EM.next_tick do
-            # If the request is a CORS request, return those headers, otherwise don't worry 'bout it
-            response_headers = cors_origin ? cors_headers : {}
-
-            # Setup a subscription with a client id. We haven't subscribed yet here.
-            if queue = queues[queue_name]
-              queue.live
-            else
-              queue = queues[queue_name] = Firehose::Subscription::Queue.new(cid, path)
-            end
-
-            # Setup a timeout timer to tell clients that time out that everything is OK
-            # and they should come back for more
-            long_poll_timer = EM::Timer.new(timeout) do
-              # We send a 204 OK to tell the client to reconnect.
-              env['async.callback'].call [204, response_headers, []]
-              Firehose.logger.debug "HTTP wait `#{cid}@#{path}` timed out"
-            end
-
-            # Ok, now subscribe to the subscription.
-            queue.pop do |message, subscription|
-              long_poll_timer.cancel # Turn off the heart beat so we don't execute any of that business.
-              env['async.callback'].call [200, response_headers, [message]]
-              Firehose.logger.debug "HTTP sent `#{message}` to `#{cid}@#{path}`"
-            end
-            Firehose.logger.debug "HTTP subscribed to `#{cid}@#{path}`"
-
-            # Unsubscribe from the subscription if its still open and something bad happened
-            # or the heart beat triggered before we could finish.
-            env['async.close'].callback do
-              # Kill queue if we don't hear back in 30s
-              queue.kill timeout do
-                Firehose.logger.debug "Deleting queue to `#{queue_name}`"
-                queues.delete queue_name
-              end
-              Firehose.logger.debug "HTTP connection `#{cid}@#{path}` closing"
-            end
-          end
-
-          # Tell the web server that this will be an async response.
-          Firehose::Rack::AsyncResponse
-
-        # PUT is how we throw messages on to the fan-out queue.
         when 'PUT'
-          body = env['rack.input'].read
-          Firehose.logger.debug "HTTP published `#{body}` to `#{path}`"
-          publisher.publish(path, body)
-
-          [202, {}, []]
+          publisher.call(env)
+        when 'HEAD'
+          ping.call(env)
         else
-          Firehose.logger.debug "HTTP #{method} not supported"
-          [501, {'Content-Type' => 'text/plain'}, ["#{method} not supported."]]
+          consumer.call(env)
         end
       end
 
-    private 
-      def publisher
-        @publisher ||= Firehose::Publisher.new
-      end
 
-      def queues
-        @queues ||= {}
-      end
-    end
-
-    class WebSocket < ::Rack::WebSocket::Application
-      attr_reader :cid, :path, :subscription
-      
-      # Subscribe to a path and make some magic happen, mmkmay?
-      def on_open(env)
-        req   = ::Rack::Request.new(env)
-        @cid   = req.params['cid']
-        @path  = req.path
-        @subscription = Firehose::Subscription.new(cid, path)
-
-        subscription.subscribe do |message, subscription|
-          Firehose.logger.debug "WS sent `#{message}` to `#{cid}@#{path}`"
-          send_data message
-        end
-        Firehose.logger.debug "WS subscribed to `#{cid}@#{path}`"
+      private
+      def publisher
+        @publisher ||= PublisherApp.new
       end
 
-      # Delete the subscription if the thing even happened.
-      def on_close(env)
-        subscription.unsubscribe
-        Firehose.logger.debug "WS connection `#{cid}@#{path}` closing"
+      def consumer
+        @consumer ||= ConsumerApp.new
       end
 
-      # Log websocket level errors
-      def on_error(env, error)
-        Firehose.logger.error "WS connection `#{cid}@#{path}` error `#{error}`: #{error.backtrace}"
+      def ping
+        @ping ||= PingApp.new
       end
     end
 
-    class App
-      def call(env)
-        websocket_request?(env) ? websocket.call(env) : http_long_poll.call(env)
-      end
-
-    private
-      def websocket
-        WebSocket.new
-      end
-
-      def http_long_poll
-        @http_long_poll ||= HttpLongPoll.new
-      end
-
-      def websocket_request?(env)
-        env['HTTP_UPGRADE'] =~ /websocket/i
+    module Helpers
+      # Calculates the content length for you
+      def response(status, body='', headers={})
+        headers = {'Content-Length' => body.size.to_s}.merge(headers)
+        [status, headers, [body]]
       end
     end
   end
-end
+end

data/lib/firehose/rack/consumer_app.rb

@@ -0,0 +1,143 @@
+require 'faye/websocket'
+
+module Firehose
+  module Rack
+    class ConsumerApp
+      def call(env)
+        websocket_request?(env) ? websocket.call(env) : http_long_poll.call(env)
+      end
+
+      private
+      def websocket
+        WebSocket.new
+      end
+
+      def http_long_poll
+        @http_long_poll ||= HttpLongPoll.new
+      end
+
+      def websocket_request?(env)
+        Faye::WebSocket.websocket?(env)
+      end
+
+      class HttpLongPoll
+        include Firehose::Rack::Helpers
+
+        # How long should we wait before closing out the consuming clients web connection
+        # for long polling? Most browsers timeout after a connection has been idle for 30s.
+        TIMEOUT = 20
+
+        def call(env)
+          req     = env['parsed_request'] ||= ::Rack::Request.new(env)
+          path    = req.path
+          method  = req.request_method
+          # Get the Last Message Sequence from the query string.
+          # Ideally we'd use an HTTP header, but android devices don't let us
+          # set any HTTP headers for CORS requests.
+          last_sequence = req.params['last_message_sequence'].to_i
+
+          case method
+          # GET is how clients subscribe to the queue. When a messages comes in, we flush out a response,
+          # close down the requeust, and the client then reconnects.
+          when 'GET'
+            Firehose.logger.debug "HTTP GET with last_sequence #{last_sequence} for path #{path} with query #{env["QUERY_STRING"].inspect} and params #{req.params.inspect}"
+            EM.next_tick do
+
+              if last_sequence < 0
+                env['async.callback'].call response(400, "Header '#{LAST_MESSAGE_SEQUENCE_HEADER}' may not be less than zero", response_headers(env))
+              else
+                Channel.new(path).next_message(last_sequence, :timeout => TIMEOUT).callback do |message, sequence|
+                  env['async.callback'].call response(200, message, response_headers(env).merge(LAST_MESSAGE_SEQUENCE_HEADER => sequence.to_s))
+                end.errback do |e|
+                  if e == :timeout
+                    env['async.callback'].call response(204, '', response_headers(env))
+                  else
+                    Firehose.logger.error "Unexpected error when trying to GET last_sequence #{last_sequence} for path #{path}: #{e.inspect}"
+                    env['async.callback'].call response(500, 'Unexpected error', response_headers(env))
+                  end
+                end
+              end
+
+            end
+
+            # Tell the web server that this will be an async response.
+            ASYNC_RESPONSE
+
+          else
+            Firehose.logger.debug "HTTP #{method} not supported"
+            response(501, "#{method} not supported.")
+          end
+        end
+
+
+        private
+
+        # If the request is a CORS request, return those headers, otherwise don't worry 'bout it
+        def response_headers(env)
+          cors_origin(env) ? cors_headers(env) : {}
+        end
+
+        def cors_origin(env)
+          env['HTTP_ORIGIN']
+        end
+
+        def cors_headers(env)
+          # TODO seperate out CORS logic as an async middleware with a Goliath web server.
+          {
+            'Access-Control-Allow-Origin'     => cors_origin(env),
+            'Access-Control-Expose-Headers'   => LAST_MESSAGE_SEQUENCE_HEADER
+          }
+        end
+      end
+
+
+      # It _may_ be more memory efficient if we used the same instance of this
+      # class (or even if we just used a proc/lambda) for every
+      # request/connection. However, we couldn't use instance variables, and
+      # so I'd need to confirm that local variables would be accessible from
+      # the callback blocks.
+      class WebSocket
+        def call(env)
+          req   = ::Rack::Request.new(env)
+          @path  = req.path
+          ws = Faye::WebSocket.new(env)
+
+          ws.onopen = lambda do |event|
+            Firehose.logger.debug "WS subscribed to `#{@path}`"
+
+            subscribe = Proc.new do |last_sequence|
+              @channel = Channel.new(@path)
+              @deferrable = @channel.next_message(last_sequence).callback do |message, sequence|
+                Firehose.logger.debug "WS sent `#{message}` to `#{@path}` with sequence `#{sequence}`"
+                ws.send message
+                subscribe.call(sequence)
+              end.errback { |e| EM.next_tick { raise e.inspect } unless e == :disconnect }
+            end
+
+            subscribe.call nil
+          end
+
+          #ws.onmessage = lambda do |event|
+          #  event.data
+          #end
+
+          ws.onclose = lambda do |event|
+            if @deferrable
+              @deferrable.fail :disconnect
+              @channel.unsubscribe(@deferrable) if @channel
+            end
+            Firehose.logger.debug "WS connection `#{@path}` closing. Code: #{event.code.inspect}; Reason #{event.reason.inspect}"
+          end
+
+          ws.onerror = lambda do |event|
+            Firehose.logger.error "WS connection `#{@path}` error `#{error}`: #{error.backtrace}"
+          end
+
+
+          # Return async Rack response
+          ws.rack_response
+        end
+      end
+    end
+  end
+end

data/lib/firehose/rack/ping_app.rb

@@ -0,0 +1,84 @@
+module Firehose
+  module Rack
+    class PingApp
+      attr_reader :redis
+
+      def initialize(redis=nil)
+        @redis = redis
+      end
+
+      def call(env)
+        PingCheck.new(env, redis).call
+        ASYNC_RESPONSE
+      end
+
+
+      # Encapsulate this in a class so we aren't passing a bunch of variables around
+      class PingCheck
+        include Firehose::Rack::Helpers
+
+        attr_reader :req, :env, :key, :redis
+
+        TEST_VALUE = 'Firehose Healthcheck Test Value'
+        SECONDS_TO_EXPIRE = 60
+
+        def self.redis
+          @redis ||= EM::Hiredis.connect
+        end
+
+        def initialize(env, redis=nil)
+          @redis = redis || self.class.redis
+          @env   = env
+          @req   = env['parsed_request'] ||= ::Rack::Request.new(env)
+          @key   = "/firehose/ping/#{Time.now.to_i}/#{rand}"
+        end
+
+        def call
+          log req, 'started'
+          test_redis
+        end
+
+
+        private
+
+        def log(req, msg)
+          Firehose.logger.debug "HTTP PING request for path '#{req.path}': #{msg}"
+        end
+
+        def test_redis
+          redis.set(key, TEST_VALUE).
+            callback { expire_key }.
+            callback { read_and_respond }.
+            errback do |e|
+              log req, "failed with write value to redis: #{e.inspect}"
+              env['async.callback'].call response(500)
+            end
+        end
+
+        def expire_key
+          redis.expire(key, SECONDS_TO_EXPIRE).
+            errback do
+              log req, "failed to expire key #{key.inspect}. If this key is not manually deleted, it may cause a memory leak."
+            end
+        end
+
+        def read_and_respond
+          redis.get(key).
+            callback do |val|
+              if val == TEST_VALUE
+                log req, 'succeeded'
+                env['async.callback'].call response(200)
+              else
+                log req, "failed with unexpected value retrieved from redis: #{val.inspect}"
+                env['async.callback'].call response(500)
+              end
+            end.
+            errback do |e|
+              log req, "failed with read value from redis: #{e.inspect}"
+              env['async.callback'].call response(500)
+            end
+        end
+      end
+    end
+  end
+end