firehose 1.2.20 → 1.3.6

data/lib/firehose/client/producer.rb

@@ -10,6 +10,7 @@ module Firehose
         PublishError = Class.new(RuntimeError)
         TimeoutError = Class.new(Faraday::Error::TimeoutError)
         DEFAULT_TIMEOUT = 1 # How many seconds should we wait for a publish to take?
+        DEFAULT_ERROR_HANDLER = ->(e) { raise e }
 
         # A DSL for publishing requests. This doesn't so much, but lets us call
         # Firehose::Client::Producer::Http#publish('message').to('channel'). Slick eh? If you don't like it,
@@ -41,8 +42,9 @@ module Firehose
 
         # Publish the message via HTTP.
         def put(message, channel, opts, &block)
-          ttl = opts[:ttl]
-          timeout = opts[:timeout] || @timeout || DEFAULT_TIMEOUT
+          ttl         = opts[:ttl]
+          timeout     = opts[:timeout] || @timeout || DEFAULT_TIMEOUT
+          buffer_size = opts[:buffer_size]
 
           response = conn.put do |req|
             req.options[:timeout] = timeout
@@ -59,13 +61,17 @@ module Firehose
             end
             req.body = message
             req.headers['Cache-Control'] = "max-age=#{ttl.to_i}" if ttl
+            req.headers["X-Firehose-Buffer-Size"] = buffer_size.to_s if buffer_size
           end
           response.on_complete do
             case response.status
             when 202 # Fire off the callback if everything worked out OK.
               block.call(response) if block
             else
-              error_handler.call PublishError.new("Could not publish #{message.inspect} to '#{uri.to_s}/#{channel}': #{response.inspect}")
+              # don't pass along basic auth header, if present
+              response_data = response.inspect.gsub(/"Authorization"=>"Basic \S+"/, '"Authorization" => "Basic [HIDDEN]"')
+              endpoint = "#{uri}/#{channel}".gsub(/:\/\/\S+@/, "://")
+              error_handler.call PublishError.new("Could not publish #{message.inspect} to '#{endpoint}': #{response_data}")
             end
           end
 
@@ -81,7 +87,7 @@ module Firehose
 
         # Raise an exception if an error occurs when connecting to the Firehose.
         def error_handler
-          @error_handler || Proc.new{ |e| raise e }
+          @error_handler || DEFAULT_ERROR_HANDLER
         end
 
         # What adapter should Firehose use to PUT the message? List of adapters is

data/lib/firehose/rack/consumer.rb

@@ -10,6 +10,45 @@ module Firehose
       autoload :HttpLongPoll, 'firehose/rack/consumer/http_long_poll'
       autoload :WebSocket,    'firehose/rack/consumer/web_socket'
 
+      MULTIPLEX_CHANNEL = "channels@firehose"
+
+      def self.multiplexing_request?(env)
+        env["PATH_INFO"].include? MULTIPLEX_CHANNEL
+      end
+
+      def self.multiplex_subscriptions(request)
+        if request.get?
+          query_string_subscriptions(request.env)
+        elsif request.post?
+          post_subscriptions(request)
+        end
+      end
+
+      def self.query_string_subscriptions(env)
+        query_params = ::Rack::Utils.parse_query(env["QUERY_STRING"])
+
+        query_params["subscribe"].to_s.split(",").map do |sub|
+          chan, last_sequence = sub.split("!")
+          last_sequence = last_sequence.to_i
+          last_sequence = 0 if last_sequence < 0
+          {
+            channel: chan,
+            message_sequence: last_sequence
+          }
+        end
+      end
+
+      def self.post_subscriptions(request)
+        body = request.body.read
+        subs = JSON.parse(body).map do |chan, last_sequence|
+          last_sequence = 0 if last_sequence < 0
+          {
+            channel: chan,
+            message_sequence: last_sequence
+          }
+        end
+      end
+
       # Let the client configure the consumer on initialization.
       def initialize
         yield self if block_given?

data/lib/firehose/rack/consumer/http_long_poll.rb

@@ -4,13 +4,10 @@ module Firehose
   module Rack
     class Consumer
       class HttpLongPoll
-        include Firehose::Rack::Helpers
-
         # How long should we wait before closing out the consuming clients web connection
         # for long polling? Most browsers timeout after a connection has been idle for 30s.
         TIMEOUT = 20
 
-        # Configures the timeout for the 
         attr_accessor :timeout
 
         def initialize(timeout=TIMEOUT)
@@ -19,68 +16,144 @@ module Firehose
         end
 
         def call(env)
-          req     = env['parsed_request'] ||= ::Rack::Request.new(env)
-          path    = req.path
-          method  = req.request_method
-          # Get the Last Message Sequence from the query string.
-          # Ideally we'd use an HTTP header, but android devices don't let us
-          # set any HTTP headers for CORS requests.
-          last_sequence = req.params['last_message_sequence'].to_i
-
-          case method
-          # GET is how clients subscribe to the queue. When a messages comes in, we flush out a response,
-          # close down the requeust, and the client then reconnects.
-          when 'GET'
-            Firehose.logger.debug "HTTP GET with last_sequence #{last_sequence} for path #{path} with query #{env["QUERY_STRING"].inspect} and params #{req.params.inspect}"
-            EM.next_tick do
+          if Consumer.multiplexing_request?(env)
+            MultiplexingHandler.new(@timeout).call(env)
+          else
+            DefaultHandler.new(@timeout).call(env)
+          end
+        end
+
+        class Handler
+          include Firehose::Rack::Helpers
+
+          def initialize(timeout=TIMEOUT)
+            @timeout = timeout
+            yield self if block_given?
+          end
+
+          def call(env)
+            request = request(env)
+            method  = request.request_method
+
+            case method
+            # GET is how clients subscribe to the queue. When a messages comes in, we flush out a response,
+            # close down the requeust, and the client then reconnects.
+            when "GET"
+              handle_request(request, env)
+              return ASYNC_RESPONSE
+            # we use post messages for http long poll multiplexing
+            when "POST"
+              if Consumer.multiplexing_request?(env)
+                handle_request(request, env)
+                return ASYNC_RESPONSE
+              end
+            end
+
+            Firehose.logger.debug "HTTP #{method} not supported"
+            response(405, "#{method} not supported.", "Allow" => "GET")
+          end
+
+          private
+
+          # If the request is a CORS request, return those headers, otherwise don't worry 'bout it
+          def response_headers(env)
+            cors_origin(env) ? cors_headers(env) : {}
+          end
+
+          def cors_origin(env)
+            env['HTTP_ORIGIN']
+          end
+
+          def cors_headers(env)
+            # TODO seperate out CORS logic as an async middleware with a Goliath web server.
+            {'Access-Control-Allow-Origin' => cors_origin(env)}
+          end
+
+          def request(env)
+            env['parsed_request'] ||= ::Rack::Request.new(env)
+          end
 
+          def async_callback(env, code, message = "", headers = nil)
+            resp_headers = response_headers(env)
+
+            if headers
+              resp_headers.merge!(headers)
+            end
+
+            if cb = env["async.callback"]
+              cb.call response(code, message, resp_headers)
+            else
+              Firehose.logger.error "async.callback not set for response: #{message.inspect}"
+            end
+          end
+
+          def respond_async(channel, last_sequence, env)
+            EM.next_tick do
               if last_sequence < 0
-                env['async.callback'].call response(400, "The last_message_sequence parameter may not be less than zero", response_headers(env))
+                async_callback env, 400, "The last_message_sequence parameter may not be less than zero"
               else
-                Server::Channel.new(path).next_message(last_sequence, :timeout => timeout).callback do |message, sequence|
-                  env['async.callback'].call response(200, wrap_frame(message, sequence), response_headers(env))
+                Server::Channel.new(channel).next_messages(last_sequence, :timeout => @timeout).callback do |messages|
+                  # TODO: Can we send all of these messages down in one request? Sending one message per
+                  # request is slow and inefficient. If we change the protocol (3.0?) we could batch the
+                  # messages and send them all down the pipe, then close the conneciton.
+                  message = messages.first
+                  async_callback env, 200, wrap_frame(channel, message)
                 end.errback do |e|
                   if e == :timeout
-                    env['async.callback'].call response(204, '', response_headers(env))
+                    async_callback env, 204
                   else
-                    Firehose.logger.error "Unexpected error when trying to GET last_sequence #{last_sequence} for path #{path}: #{e.inspect}"
-                    env['async.callback'].call response(500, 'Unexpected error', response_headers(env))
+                    Firehose.logger.error "Unexpected error when trying to GET last_sequence #{last_sequence} for path #{channel}: #{e.inspect}"
+                    async_callback env, 500, "Unexpected error"
                   end
                 end
               end
-
             end
-
-            # Tell the web server that this will be an async response.
-            ASYNC_RESPONSE
-
-          else
-            Firehose.logger.debug "HTTP #{method} not supported"
-            response(501, "#{method} not supported.")
           end
         end
 
+        class DefaultHandler < Handler
+          def wrap_frame(channel, message)
+            JSON.generate :message => message.payload, :last_sequence => message.sequence
+          end
 
-        private
+          def log_request(path, last_sequence, env)
+            Firehose.logger.debug "HTTP GET with last_sequence #{last_sequence} for path #{path} with query #{env["QUERY_STRING"].inspect}"
+          end
 
-        def wrap_frame(message, last_sequence)
-          JSON.generate :message => message, :last_sequence => last_sequence
-        end
+          def handle_request(request, env)
+            # Get the Last Message Sequence from the query string.
+            # Ideally we'd use an HTTP header, but android devices don't let us
+            # set any HTTP headers for CORS requests.
+            last_sequence = request.params['last_message_sequence'].to_i
+            channel       = request.path
 
-        # If the request is a CORS request, return those headers, otherwise don't worry 'bout it
-        def response_headers(env)
-          cors_origin(env) ? cors_headers(env) : {}
+            log_request   channel, last_sequence, env
+            respond_async channel, last_sequence, env
+          end
         end
 
-        def cors_origin(env)
-          env['HTTP_ORIGIN']
-        end
+        class MultiplexingHandler < Handler
+          def wrap_frame(channel, message)
+            JSON.generate channel: channel, :message => message.payload, :last_sequence => message.sequence
+          end
 
-        def cors_headers(env)
-          # TODO seperate out CORS logic as an async middleware with a Goliath web server.
-          {'Access-Control-Allow-Origin' => cors_origin(env)}
+          def log_request(request, subscriptions, env)
+            if request.post?
+              Firehose.logger.debug "HTTP multiplexing POST, subscribing #{subscriptions.inspect}"
+            else
+              Firehose.logger.debug "HTTP multiplexing GET with query #{env["QUERY_STRING"].inspect}"
+            end
+          end
+
+          def handle_request(request, env)
+            subscriptions = Consumer.multiplex_subscriptions(request)
+            log_request request, subscriptions, env
+            subscriptions.each do |sub|
+              respond_async(sub[:channel], sub[:message_sequence], env)
+            end
+          end
         end
       end
     end
   end
-end
+end

data/lib/firehose/rack/consumer/web_socket.rb

@@ -1,5 +1,6 @@
 require 'faye/websocket'
 require 'json'
+require "rack/utils"
 
 module Firehose
   module Rack
@@ -8,7 +9,11 @@ module Firehose
         # Setup a handler for the websocket connection.
         def call(env)
           ws = Faye::WebSocket.new(env)
-          Handler.new(ws)
+          if Consumer.multiplexing_request?(env)
+            MultiplexingHandler.new(ws)
+          else
+            DefaultHandler.new(ws)
+          end
           ws.rack_response
         end
 
@@ -17,9 +22,6 @@ module Firehose
           Faye::WebSocket.websocket?(env)
         end
 
-        # Manages connection state for the web socket that's connected
-        # by the Consumer::WebSocket class. Deals with message sequence,
-        # connection, failures, and subscription state.
         class Handler
           def initialize(ws)
             @ws = ws
@@ -31,31 +33,36 @@ module Firehose
             @ws.onmessage = method :message
           end
 
-          # Subscribe the client to the channel on the server. Asks for
-          # the last sequence for clients that reconnect.
-          def subscribe(last_sequence)
-            @subscribed = true
-            @channel    = Server::Channel.new @req.path
-            @deferrable = @channel.next_message last_sequence
-            @deferrable.callback do |message, sequence|
-              Firehose.logger.debug "WS sent `#{message}` to `#{@req.path}` with sequence `#{sequence}`"
-              @ws.send self.class.wrap_frame(message, last_sequence)
-              subscribe sequence
-            end
-            @deferrable.errback do |e|
-              EM.next_tick { raise e.inspect } unless e == :disconnect
-            end
+          def parse_message(event)
+            JSON.parse(event.data, :symbolize_names => true) rescue {}
+          end
+
+          # Send a JSON message to the client
+          # Expects message to be a Hash
+          def send_message(message)
+            @ws.send JSON.generate(message)
           end
 
+          # Log errors if a socket fails. `close` will fire after this to clean up any
+          # remaining connectons.
+          def error(event)
+            Firehose.logger.error "WS connection `#{@req.path}` error. Message: `#{event.message.inspect}`"
+          end
+        end
+
+        # Manages connection state for the web socket that's connected
+        # by the Consumer::WebSocket class. Deals with message sequence,
+        # connection, failures, and subscription state.
+        class DefaultHandler < Handler
           # Manages messages sent from the connect client to the server. This is mostly
           # used to handle heart-beats that are designed to prevent the WebSocket connection
           # from timing out from inactivity.
           def message(event)
-            msg = JSON.parse(event.data, :symbolize_names => true) rescue {}
+            msg = parse_message(event)
             seq = msg[:message_sequence]
             if msg[:ping] == 'PING'
               Firehose.logger.debug "WS ping received, sending pong"
-              @ws.send JSON.generate :pong => 'PONG'
+              send_message pong: "PONG"
             elsif !@subscribed && seq.kind_of?(Integer)
               Firehose.logger.debug "Subscribing at message_sequence #{seq}"
               subscribe seq
@@ -77,16 +84,114 @@ module Firehose
             Firehose.logger.debug "WS connection `#{@req.path}` closing. Code: #{event.code.inspect}; Reason #{event.reason.inspect}"
           end
 
-          # Log errors if a socket fails. `close` will fire after this to clean up any
-          # remaining connectons.
-          def error(event)
-            Firehose.logger.error "WS connection `#{@req.path}` error. Message: `#{event.message.inspect}`"
+          # Subscribe the client to the channel on the server. Asks for
+          # the last sequence for clients that reconnect.
+          def subscribe(last_sequence)
+            @subscribed = true
+            @channel    = Server::Channel.new @req.path
+            @deferrable = @channel.next_messages last_sequence
+            @deferrable.callback do |messages|
+              messages.each do |message|
+                Firehose.logger.debug "WS sent `#{message.payload}` to `#{@req.path}` with sequence `#{message.sequence}`"
+                send_message message: message.payload, last_sequence: message.sequence
+              end
+              subscribe messages.last.sequence
+            end
+            @deferrable.errback do |e|
+              unless e == :disconnect
+                Firehose.logger.error "WS Error: #{e}"
+                EM.next_tick { raise e.inspect }
+              end
+            end
           end
+        end
 
-          # Wrap a message in a sequence so that the client can record this and give us
-          # the sequence when it reconnects.
-          def self.wrap_frame(message, last_sequence)
-            JSON.generate :message => message, :last_sequence => last_sequence
+        class MultiplexingHandler < Handler
+          class Subscription < Struct.new(:channel, :deferrable)
+            def close
+              deferrable.fail :disconnect
+              channel.unsubscribe(deferrable)
+            end
+          end
+
+          def initialize(ws)
+            super(ws)
+            @subscriptions = {}
+            subscribe_multiplexed Consumer.multiplex_subscriptions(@req)
+          end
+
+          def message(event)
+            msg = parse_message(event)
+
+            if subscriptions = msg[:multiplex_subscribe]
+              subscriptions = [subscriptions] unless subscriptions.is_a?(Array)
+              return subscribe_multiplexed(subscriptions)
+            end
+
+            if channel_names = msg[:multiplex_unsubscribe]
+              return unsubscribe(channel_names)
+            end
+
+            if msg[:ping] == 'PING'
+              Firehose.logger.debug "WS ping received, sending pong"
+              return send_message pong: "PONG"
+            end
+          end
+
+          def open(event)
+            Firehose.logger.debug "Multiplexing Websocket connected: #{@req.path}"
+          end
+
+          def close(event)
+            @subscriptions.each_value(&:close)
+            @subscriptions.clear
+          end
+
+          def subscribe_multiplexed(subscriptions)
+            subscriptions.each do |sub|
+              Firehose.logger.debug "Subscribing multiplexed to: #{sub}"
+
+              channel, sequence = sub[:channel], sub[:message_sequence]
+              next if channel.nil?
+
+              subscribe(channel, sequence.to_i)
+            end
+          end
+
+          # Subscribe the client to the channel on the server. Asks for
+          # the last sequence for clients that reconnect.
+          def subscribe(channel_name, last_sequence)
+            channel      = Server::Channel.new channel_name
+            deferrable   = channel.next_messages last_sequence
+            subscription = Subscription.new(channel, deferrable)
+
+            @subscriptions[channel_name] = subscription
+
+            deferrable.callback do |messages|
+              messages.each do |message|
+                send_message(
+                  channel: channel_name,
+                  message: message.payload,
+                  last_sequence: message.sequence
+                )
+                Firehose.logger.debug "WS sent `#{message.payload}` to `#{channel_name}` with sequence `#{message.sequence}`"
+              end
+              subscribe channel_name, messages.last.sequence
+            end
+
+            deferrable.errback do |e|
+              EM.next_tick { raise e.inspect } unless e == :disconnect
+            end
+          end
+
+          def unsubscribe(channel_names)
+            Firehose.logger.debug "Unsubscribing from channels: #{channel_names}"
+            Array(channel_names).each do |chan|
+              if sub = @subscriptions[chan]
+                sub.close
+                @subscriptions.delete(chan)
+              end
+            end
           end
         end
       end