firehose 1.3.6 → 2.1.0

data/lib/firehose/client/consumer.rb

@@ -1,94 +0,0 @@
-require 'json'
-
-# TODO - Spec this thing out. Unfortunately its not tested at all, mostly because the JSON client
-# is more important (and tested). Still, this should be tested, at least used by the specs
-# to test HTTP functionality of the server.
-module Firehose
-  module Client
-    module Consumer
-      # TODO - Test this libs. I had to throw these quickly into our app so that we could get 
-      #        some stress testing out of the way.
-      # TODO - Replace the integration test clients with these guys. You'll want to refactor each
-      #        transport to use on(:message), on(:conncect), and on(:disconnect) callbacks.
-      TransportNotSupportedError = Class.new(RuntimeError)
-
-      # Build up a benchmark client based on a given URI. Accepts ws:// and http:// for now.
-      def self.parse(uri)
-        case transport = ::URI.parse(uri).scheme
-        # TODO - Fix ws:// transport! See class WebSocket below to udnerstand
-        # why this doesn't work and support is dropped from the CLI.
-        # when 'ws'
-        #   Consumer::WebSocket.new(uri)
-        when 'http'
-          Consumer::HttpLongPoll.new(uri)
-        else
-          raise TransportNotSupportedError.new("Transport #{transport.inspect} not supported.")
-        end
-      end
-
-      # TODO - This won't even work. Dropping ws:// above until this is tested. This thing
-      # should be sending message sequences to Firehose.
-      # Connect to Firehose via WebSockets and consume messages.
-      class WebSocket
-        attr_reader :url, :logger
-
-        def initialize(url, logger = Firehose.logger)
-          @url, @logger = url, logger
-        end
-
-        def request
-          ws = Faye::WebSocket::Client.new(url)
-          ws.onmessage = lambda do |event|
-            logger.info "WS | #{event.data[0...40].inspect}"
-          end
-          ws.onclose = lambda do |event|
-            logger.info "WS | Closed"
-          end
-          ws.onerror do
-            logger.error "WS | Failed"
-          end
-        end
-      end
-
-      # Connect to Firehose via HTTP Long Polling and consume messages.
-      class HttpLongPoll
-        JITTER = 0.003
-
-        attr_reader :url, :logger
-
-        def initialize(url, logger = Firehose.logger)
-          @url, @logger = url, logger
-        end
-
-        def request(last_sequence=0)
-          http = EM::HttpRequest.new(url, :inactivity_timeout => 0).get(:query => {'last_message_sequence' => last_sequence})
-          http.callback do
-            case status = http.response_header.status
-            when 200
-              json = JSON.parse(http.response)
-              next_sequence = json['last_sequence'].to_i
-              message = json['message']
-
-              logger.info "HTTP 200 | Next Sequence: #{next_sequence} - #{message[0...40].inspect}"
-              EM::add_timer(jitter) { request next_sequence }
-            when 204
-              logger.info "HTTP 204 | Last Sequence #{last_sequence}"
-              EM::add_timer(jitter) { request last_sequence }
-            else
-              logger.error "HTTP #{status} | Failed"
-            end
-          end
-          http.errback do
-            logger.error "Connection Failed"
-          end
-        end
-
-      private
-        # Random jitter between long poll requests.
-        def jitter
-          rand*JITTER
-        end
-      end
-    end
-  end
-end

data/lib/firehose/client/producer.rb

@@ -1,114 +0,0 @@
-require "faraday"
-require "uri"
-
-module Firehose
-  module Client
-    module Producer
-      # Publish messages to Firehose via an HTTP interface.
-      class Http
-        # Exception gets raised when a 202 is _not_ received from the server after a message is published.
-        PublishError = Class.new(RuntimeError)
-        TimeoutError = Class.new(Faraday::Error::TimeoutError)
-        DEFAULT_TIMEOUT = 1 # How many seconds should we wait for a publish to take?
-        DEFAULT_ERROR_HANDLER = ->(e) { raise e }
-
-        # A DSL for publishing requests. This doesn't so much, but lets us call
-        # Firehose::Client::Producer::Http#publish('message').to('channel'). Slick eh? If you don't like it,
-        # just all Firehose::Client::Producer::Http#put('message', 'channel')
-        class Builder
-          def initialize(producer, message)
-            @producer, @message = producer, message
-            self
-          end
-
-          def to(channel, opts={}, &callback)
-            @producer.put(@message, channel, opts, &callback)
-          end
-        end
-
-        # URI for the Firehose server. This URI does not include the path of the channel.
-        attr_reader :uri, :timeout
-
-        def initialize(uri = Firehose::URI, timeout=DEFAULT_TIMEOUT)
-          @uri = ::URI.parse(uri.to_s)
-          @uri.scheme ||= 'http'
-          @timeout = timeout
-        end
-
-        # A DSL for publishing messages.
-        def publish(message)
-          Builder.new(self, message)
-        end
-
-        # Publish the message via HTTP.
-        def put(message, channel, opts, &block)
-          ttl         = opts[:ttl]
-          timeout     = opts[:timeout] || @timeout || DEFAULT_TIMEOUT
-          buffer_size = opts[:buffer_size]
-
-          response = conn.put do |req|
-            req.options[:timeout] = timeout
-            if conn.path_prefix.nil? || conn.path_prefix == '/'
-              # This avoids a double / if the channel starts with a / too (which is expected).
-              req.path = channel
-            else
-              if conn.path_prefix =~ /\/\Z/ || channel =~ /\A\//
-                req.path = [conn.path_prefix, channel].compact.join
-              else
-                # Add a / so the prefix and channel aren't just rammed together.
-                req.path = [conn.path_prefix, channel].compact.join('/')
-              end
-            end
-            req.body = message
-            req.headers['Cache-Control'] = "max-age=#{ttl.to_i}" if ttl
-            req.headers["X-Firehose-Buffer-Size"] = buffer_size.to_s if buffer_size
-          end
-          response.on_complete do
-            case response.status
-            when 202 # Fire off the callback if everything worked out OK.
-              block.call(response) if block
-            else
-              # don't pass along basic auth header, if present
-              response_data = response.inspect.gsub(/"Authorization"=>"Basic \S+"/, '"Authorization" => "Basic [HIDDEN]"')
-              endpoint = "#{uri}/#{channel}".gsub(/:\/\/\S+@/, "://")
-              error_handler.call PublishError.new("Could not publish #{message.inspect} to '#{endpoint}': #{response_data}")
-            end
-          end
-
-          # Hide Faraday with this Timeout exception, and through the error handler.
-          rescue Faraday::Error::TimeoutError => e
-            error_handler.call TimeoutError.new(e)
-        end
-
-        # Handle errors that could happen while publishing a message.
-        def on_error(&block)
-          @error_handler = block
-        end
-
-        # Raise an exception if an error occurs when connecting to the Firehose.
-        def error_handler
-          @error_handler || DEFAULT_ERROR_HANDLER
-        end
-
-        # What adapter should Firehose use to PUT the message? List of adapters is
-        # available at https://github.com/technoweenie/faraday.
-        def self.adapter=(adapter)
-          @adapter = adapter
-        end
-
-        # Use :net_http for the default Faraday adapter.
-        def self.adapter
-          @adapter ||= Faraday.default_adapter
-        end
-
-        private
-        # Build out a Faraday connection
-        def conn
-          @conn ||= Faraday.new(:url => uri.to_s) do |builder|
-            builder.adapter self.class.adapter
-          end
-        end
-      end
-    end
-  end
-end

data/lib/firehose/client.rb

@@ -1,10 +0,0 @@
-require 'em-http'
-require 'faye/websocket'
-
-module Firehose
-  # Ruby clients that connect to Firehose to either publish or consume messages.
-  module Client
-    autoload :Consumer, 'firehose/client/consumer'
-    autoload :Producer, 'firehose/client/producer'
-  end
-end

data/lib/firehose/logging.rb

@@ -1,32 +0,0 @@
-require 'logger'
-
-module Firehose
-  # Sets up logging
-  def self.logger
-    @logger ||= Logger.new($stdout)
-  end
-
-  def self.logger=(logger)
-    @logger = logger
-  end
-
-  self.logger.level = if ENV['LOG_LEVEL']
-    Logger.const_get(ENV['LOG_LEVEL'].upcase)
-  else
-    case ENV['RACK_ENV']
-      when 'test'         then Logger::ERROR
-      when 'development'  then Logger::DEBUG
-      else Logger::INFO
-    end
-  end
-
-  # TODO: Provide some way to allow this to be configured via an ENV variable.
-  self.logger.formatter = lambda do |severity, time, name, msg|
-    out_time = time.utc.strftime "%Y-%m-%d %H:%M:%S.%L"
-    "[#{out_time} ##$$] #{severity} : #{msg}\n"
-  end
-end
-
-# stdout gets "lost" in Foreman if this isn't here
-# https://github.com/ddollar/foreman/wiki/Missing-Output
-$stdout.sync = true if ENV['RACK_ENV'] == 'development' || ENV['SYNC_LOGGING']

data/lib/firehose/rack/app.rb

@@ -1,49 +0,0 @@
-module Firehose
-  module Rack
-    # Acts as the glue between the HTTP/WebSocket world and the Firehose::Server class,
-    # which talks directly to the Redis server. Also dispatches between HTTP and WebSocket
-    # transport handlers depending on the clients' request.
-    class App
-      def initialize
-        yield self if block_given?
-      end
-
-      def call(env)
-        # Cache the parsed request so we don't need to re-parse it when we pass
-        # control onto another app.
-        req     = env['parsed_request'] ||= ::Rack::Request.new(env)
-        method  = req.request_method
-
-        case method
-        when 'PUT'
-          # Firehose::Client::Publisher PUT's payloads to the server.
-          publisher.call(env)
-        when 'HEAD' 
-          # HEAD requests are used to prevent sockets from timing out
-          # from inactivity
-          ping.call(env)
-        else
-          # TODO - 'harden' this up with a GET request and throw a "Bad Request" 
-          # HTTP error code. I'd do it now but I'm in a plane and can't think of it.
-          consumer.call(env)
-        end
-      end
-
-      # The consumer pulls messages off of the backend and passes messages to the 
-      # connected HTTP or WebSocket client. This can be configured from the initialization
-      # method of the rack app.
-      def consumer
-        @consumer ||= Consumer.new
-      end
-
-      private
-      def publisher
-        @publisher ||= Publisher.new
-      end
-
-      def ping
-        @ping ||= Ping.new
-      end
-    end
-  end
-end

data/lib/firehose/rack/consumer/http_long_poll.rb

@@ -1,159 +0,0 @@
-require 'json'
-
-module Firehose
-  module Rack
-    class Consumer
-      class HttpLongPoll
-        # How long should we wait before closing out the consuming clients web connection
-        # for long polling? Most browsers timeout after a connection has been idle for 30s.
-        TIMEOUT = 20
-
-        attr_accessor :timeout
-
-        def initialize(timeout=TIMEOUT)
-          @timeout = timeout
-          yield self if block_given?
-        end
-
-        def call(env)
-          if Consumer.multiplexing_request?(env)
-            MultiplexingHandler.new(@timeout).call(env)
-          else
-            DefaultHandler.new(@timeout).call(env)
-          end
-        end
-
-        class Handler
-          include Firehose::Rack::Helpers
-
-          def initialize(timeout=TIMEOUT)
-            @timeout = timeout
-            yield self if block_given?
-          end
-
-          def call(env)
-            request = request(env)
-            method  = request.request_method
-
-            case method
-            # GET is how clients subscribe to the queue. When a messages comes in, we flush out a response,
-            # close down the requeust, and the client then reconnects.
-            when "GET"
-              handle_request(request, env)
-              return ASYNC_RESPONSE
-            # we use post messages for http long poll multiplexing
-            when "POST"
-              if Consumer.multiplexing_request?(env)
-                handle_request(request, env)
-                return ASYNC_RESPONSE
-              end
-            end
-
-            Firehose.logger.debug "HTTP #{method} not supported"
-            response(405, "#{method} not supported.", "Allow" => "GET")
-          end
-
-          private
-
-          # If the request is a CORS request, return those headers, otherwise don't worry 'bout it
-          def response_headers(env)
-            cors_origin(env) ? cors_headers(env) : {}
-          end
-
-          def cors_origin(env)
-            env['HTTP_ORIGIN']
-          end
-
-          def cors_headers(env)
-            # TODO seperate out CORS logic as an async middleware with a Goliath web server.
-            {'Access-Control-Allow-Origin' => cors_origin(env)}
-          end
-
-          def request(env)
-            env['parsed_request'] ||= ::Rack::Request.new(env)
-          end
-
-          def async_callback(env, code, message = "", headers = nil)
-            resp_headers = response_headers(env)
-
-            if headers
-              resp_headers.merge!(headers)
-            end
-
-            if cb = env["async.callback"]
-              cb.call response(code, message, resp_headers)
-            else
-              Firehose.logger.error "async.callback not set for response: #{message.inspect}"
-            end
-          end
-
-          def respond_async(channel, last_sequence, env)
-            EM.next_tick do
-              if last_sequence < 0
-                async_callback env, 400, "The last_message_sequence parameter may not be less than zero"
-              else
-                Server::Channel.new(channel).next_messages(last_sequence, :timeout => @timeout).callback do |messages|
-                  # TODO: Can we send all of these messages down in one request? Sending one message per
-                  # request is slow and inefficient. If we change the protocol (3.0?) we could batch the
-                  # messages and send them all down the pipe, then close the conneciton.
-                  message = messages.first
-                  async_callback env, 200, wrap_frame(channel, message)
-                end.errback do |e|
-                  if e == :timeout
-                    async_callback env, 204
-                  else
-                    Firehose.logger.error "Unexpected error when trying to GET last_sequence #{last_sequence} for path #{channel}: #{e.inspect}"
-                    async_callback env, 500, "Unexpected error"
-                  end
-                end
-              end
-            end
-          end
-        end
-
-        class DefaultHandler < Handler
-          def wrap_frame(channel, message)
-            JSON.generate :message => message.payload, :last_sequence => message.sequence
-          end
-
-          def log_request(path, last_sequence, env)
-            Firehose.logger.debug "HTTP GET with last_sequence #{last_sequence} for path #{path} with query #{env["QUERY_STRING"].inspect}"
-          end
-
-          def handle_request(request, env)
-            # Get the Last Message Sequence from the query string.
-            # Ideally we'd use an HTTP header, but android devices don't let us
-            # set any HTTP headers for CORS requests.
-            last_sequence = request.params['last_message_sequence'].to_i
-            channel       = request.path
-
-            log_request   channel, last_sequence, env
-            respond_async channel, last_sequence, env
-          end
-        end
-
-        class MultiplexingHandler < Handler
-          def wrap_frame(channel, message)
-            JSON.generate channel: channel, :message => message.payload, :last_sequence => message.sequence
-          end
-
-          def log_request(request, subscriptions, env)
-            if request.post?
-              Firehose.logger.debug "HTTP multiplexing POST, subscribing #{subscriptions.inspect}"
-            else
-              Firehose.logger.debug "HTTP multiplexing GET with query #{env["QUERY_STRING"].inspect}"
-            end
-          end
-
-          def handle_request(request, env)
-            subscriptions = Consumer.multiplex_subscriptions(request)
-            log_request request, subscriptions, env
-            subscriptions.each do |sub|
-              respond_async(sub[:channel], sub[:message_sequence], env)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/firehose/rack/consumer/web_socket.rb

@@ -1,200 +0,0 @@
-require 'faye/websocket'
-require 'json'
-require "rack/utils"
-
-module Firehose
-  module Rack
-    class Consumer
-      class WebSocket
-        # Setup a handler for the websocket connection.
-        def call(env)
-          ws = Faye::WebSocket.new(env)
-          if Consumer.multiplexing_request?(env)
-            MultiplexingHandler.new(ws)
-          else
-            DefaultHandler.new(ws)
-          end
-          ws.rack_response
-        end
-
-        # Determine if the rack request is a WebSocket request.
-        def self.request?(env)
-          Faye::WebSocket.websocket?(env)
-        end
-
-        class Handler
-          def initialize(ws)
-            @ws = ws
-            @req = ::Rack::Request.new ws.env
-            # Setup the event handlers from this class.
-            @ws.onopen    = method :open
-            @ws.onclose   = method :close
-            @ws.onerror   = method :error
-            @ws.onmessage = method :message
-          end
-
-          def parse_message(event)
-            JSON.parse(event.data, :symbolize_names => true) rescue {}
-          end
-
-          # Send a JSON message to the client
-          # Expects message to be a Hash
-          def send_message(message)
-            @ws.send JSON.generate(message)
-          end
-
-          # Log errors if a socket fails. `close` will fire after this to clean up any
-          # remaining connectons.
-          def error(event)
-            Firehose.logger.error "WS connection `#{@req.path}` error. Message: `#{event.message.inspect}`"
-          end
-        end
-
-        # Manages connection state for the web socket that's connected
-        # by the Consumer::WebSocket class. Deals with message sequence,
-        # connection, failures, and subscription state.
-        class DefaultHandler < Handler
-          # Manages messages sent from the connect client to the server. This is mostly
-          # used to handle heart-beats that are designed to prevent the WebSocket connection
-          # from timing out from inactivity.
-          def message(event)
-            msg = parse_message(event)
-            seq = msg[:message_sequence]
-            if msg[:ping] == 'PING'
-              Firehose.logger.debug "WS ping received, sending pong"
-              send_message pong: "PONG"
-            elsif !@subscribed && seq.kind_of?(Integer)
-              Firehose.logger.debug "Subscribing at message_sequence #{seq}"
-              subscribe seq
-            end
-          end
-
-          # Log a message that the client has connected.
-          def open(event)
-            Firehose.logger.debug "WebSocket subscribed to `#{@req.path}`. Waiting for message_sequence..."
-          end
-
-          # Log a message that hte client has disconnected and reset the state for the class. Clean
-          # up the subscribers to the channels.
-          def close(event)
-            if @deferrable
-              @deferrable.fail :disconnect
-              @channel.unsubscribe(@deferrable) if @channel
-            end
-            Firehose.logger.debug "WS connection `#{@req.path}` closing. Code: #{event.code.inspect}; Reason #{event.reason.inspect}"
-          end
-
-          # Subscribe the client to the channel on the server. Asks for
-          # the last sequence for clients that reconnect.
-          def subscribe(last_sequence)
-            @subscribed = true
-            @channel    = Server::Channel.new @req.path
-            @deferrable = @channel.next_messages last_sequence
-            @deferrable.callback do |messages|
-              messages.each do |message|
-                Firehose.logger.debug "WS sent `#{message.payload}` to `#{@req.path}` with sequence `#{message.sequence}`"
-                send_message message: message.payload, last_sequence: message.sequence
-              end
-              subscribe messages.last.sequence
-            end
-            @deferrable.errback do |e|
-              unless e == :disconnect
-                Firehose.logger.error "WS Error: #{e}"
-                EM.next_tick { raise e.inspect }
-              end
-            end
-          end
-        end
-
-        class MultiplexingHandler < Handler
-          class Subscription < Struct.new(:channel, :deferrable)
-            def close
-              deferrable.fail :disconnect
-              channel.unsubscribe(deferrable)
-            end
-          end
-
-          def initialize(ws)
-            super(ws)
-            @subscriptions = {}
-            subscribe_multiplexed Consumer.multiplex_subscriptions(@req)
-          end
-
-          def message(event)
-            msg = parse_message(event)
-
-            if subscriptions = msg[:multiplex_subscribe]
-              subscriptions = [subscriptions] unless subscriptions.is_a?(Array)
-              return subscribe_multiplexed(subscriptions)
-            end
-
-            if channel_names = msg[:multiplex_unsubscribe]
-              return unsubscribe(channel_names)
-            end
-
-            if msg[:ping] == 'PING'
-              Firehose.logger.debug "WS ping received, sending pong"
-              return send_message pong: "PONG"
-            end
-          end
-
-          def open(event)
-            Firehose.logger.debug "Multiplexing Websocket connected: #{@req.path}"
-          end
-
-          def close(event)
-            @subscriptions.each_value(&:close)
-            @subscriptions.clear
-          end
-
-          def subscribe_multiplexed(subscriptions)
-            subscriptions.each do |sub|
-              Firehose.logger.debug "Subscribing multiplexed to: #{sub}"
-
-              channel, sequence = sub[:channel], sub[:message_sequence]
-              next if channel.nil?
-
-              subscribe(channel, sequence.to_i)
-            end
-          end
-
-          # Subscribe the client to the channel on the server. Asks for
-          # the last sequence for clients that reconnect.
-          def subscribe(channel_name, last_sequence)
-            channel      = Server::Channel.new channel_name
-            deferrable   = channel.next_messages last_sequence
-            subscription = Subscription.new(channel, deferrable)
-
-            @subscriptions[channel_name] = subscription
-
-            deferrable.callback do |messages|
-              messages.each do |message|
-                send_message(
-                  channel: channel_name,
-                  message: message.payload,
-                  last_sequence: message.sequence
-                )
-                Firehose.logger.debug "WS sent `#{message.payload}` to `#{channel_name}` with sequence `#{message.sequence}`"
-              end
-              subscribe channel_name, messages.last.sequence
-            end
-
-            deferrable.errback do |e|
-              EM.next_tick { raise e.inspect } unless e == :disconnect
-            end
-          end
-
-          def unsubscribe(channel_names)
-            Firehose.logger.debug "Unsubscribing from channels: #{channel_names}"
-            Array(channel_names).each do |chan|
-              if sub = @subscriptions[chan]
-                sub.close
-                @subscriptions.delete(chan)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end