firehose 1.2.20 → 2.1.0

data/lib/firehose/client/consumer.rb

@@ -1,94 +0,0 @@
-require 'json'
-
-# TODO - Spec this thing out. Unfortunately its not tested at all, mostly because the JSON client
-# is more important (and tested). Still, this should be tested, at least used by the specs
-# to test HTTP functionality of the server.
-module Firehose
-  module Client
-    module Consumer
-      # TODO - Test this libs. I had to throw these quickly into our app so that we could get 
-      #        some stress testing out of the way.
-      # TODO - Replace the integration test clients with these guys. You'll want to refactor each
-      #        transport to use on(:message), on(:conncect), and on(:disconnect) callbacks.
-      TransportNotSupportedError = Class.new(RuntimeError)
-
-      # Build up a benchmark client based on a given URI. Accepts ws:// and http:// for now.
-      def self.parse(uri)
-        case transport = ::URI.parse(uri).scheme
-        # TODO - Fix ws:// transport! See class WebSocket below to udnerstand
-        # why this doesn't work and support is dropped from the CLI.
-        # when 'ws'
-        #   Consumer::WebSocket.new(uri)
-        when 'http'
-          Consumer::HttpLongPoll.new(uri)
-        else
-          raise TransportNotSupportedError.new("Transport #{transport.inspect} not supported.")
-        end
-      end
-
-      # TODO - This won't even work. Dropping ws:// above until this is tested. This thing
-      # should be sending message sequences to Firehose.
-      # Connect to Firehose via WebSockets and consume messages.
-      class WebSocket
-        attr_reader :url, :logger
-
-        def initialize(url, logger = Firehose.logger)
-          @url, @logger = url, logger
-        end
-
-        def request
-          ws = Faye::WebSocket::Client.new(url)
-          ws.onmessage = lambda do |event|
-            logger.info "WS | #{event.data[0...40].inspect}"
-          end
-          ws.onclose = lambda do |event|
-            logger.info "WS | Closed"
-          end
-          ws.onerror do
-            logger.error "WS | Failed"
-          end
-        end
-      end
-
-      # Connect to Firehose via HTTP Long Polling and consume messages.
-      class HttpLongPoll
-        JITTER = 0.003
-
-        attr_reader :url, :logger
-
-        def initialize(url, logger = Firehose.logger)
-          @url, @logger = url, logger
-        end
-
-        def request(last_sequence=0)
-          http = EM::HttpRequest.new(url, :inactivity_timeout => 0).get(:query => {'last_message_sequence' => last_sequence})
-          http.callback do
-            case status = http.response_header.status
-            when 200
-              json = JSON.parse(http.response)
-              next_sequence = json['last_sequence'].to_i
-              message = json['message']
-
-              logger.info "HTTP 200 | Next Sequence: #{next_sequence} - #{message[0...40].inspect}"
-              EM::add_timer(jitter) { request next_sequence }
-            when 204
-              logger.info "HTTP 204 | Last Sequence #{last_sequence}"
-              EM::add_timer(jitter) { request last_sequence }
-            else
-              logger.error "HTTP #{status} | Failed"
-            end
-          end
-          http.errback do
-            logger.error "Connection Failed"
-          end
-        end
-
-      private
-        # Random jitter between long poll requests.
-        def jitter
-          rand*JITTER
-        end
-      end
-    end
-  end
-end

data/lib/firehose/client/producer.rb

@@ -1,108 +0,0 @@
-require "faraday"
-require "uri"
-
-module Firehose
-  module Client
-    module Producer
-      # Publish messages to Firehose via an HTTP interface.
-      class Http
-        # Exception gets raised when a 202 is _not_ received from the server after a message is published.
-        PublishError = Class.new(RuntimeError)
-        TimeoutError = Class.new(Faraday::Error::TimeoutError)
-        DEFAULT_TIMEOUT = 1 # How many seconds should we wait for a publish to take?
-
-        # A DSL for publishing requests. This doesn't so much, but lets us call
-        # Firehose::Client::Producer::Http#publish('message').to('channel'). Slick eh? If you don't like it,
-        # just all Firehose::Client::Producer::Http#put('message', 'channel')
-        class Builder
-          def initialize(producer, message)
-            @producer, @message = producer, message
-            self
-          end
-
-          def to(channel, opts={}, &callback)
-            @producer.put(@message, channel, opts, &callback)
-          end
-        end
-
-        # URI for the Firehose server. This URI does not include the path of the channel.
-        attr_reader :uri, :timeout
-
-        def initialize(uri = Firehose::URI, timeout=DEFAULT_TIMEOUT)
-          @uri = ::URI.parse(uri.to_s)
-          @uri.scheme ||= 'http'
-          @timeout = timeout
-        end
-
-        # A DSL for publishing messages.
-        def publish(message)
-          Builder.new(self, message)
-        end
-
-        # Publish the message via HTTP.
-        def put(message, channel, opts, &block)
-          ttl = opts[:ttl]
-          timeout = opts[:timeout] || @timeout || DEFAULT_TIMEOUT
-
-          response = conn.put do |req|
-            req.options[:timeout] = timeout
-            if conn.path_prefix.nil? || conn.path_prefix == '/'
-              # This avoids a double / if the channel starts with a / too (which is expected).
-              req.path = channel
-            else
-              if conn.path_prefix =~ /\/\Z/ || channel =~ /\A\//
-                req.path = [conn.path_prefix, channel].compact.join
-              else
-                # Add a / so the prefix and channel aren't just rammed together.
-                req.path = [conn.path_prefix, channel].compact.join('/')
-              end
-            end
-            req.body = message
-            req.headers['Cache-Control'] = "max-age=#{ttl.to_i}" if ttl
-          end
-          response.on_complete do
-            case response.status
-            when 202 # Fire off the callback if everything worked out OK.
-              block.call(response) if block
-            else
-              error_handler.call PublishError.new("Could not publish #{message.inspect} to '#{uri.to_s}/#{channel}': #{response.inspect}")
-            end
-          end
-
-          # Hide Faraday with this Timeout exception, and through the error handler.
-          rescue Faraday::Error::TimeoutError => e
-            error_handler.call TimeoutError.new(e)
-        end
-
-        # Handle errors that could happen while publishing a message.
-        def on_error(&block)
-          @error_handler = block
-        end
-
-        # Raise an exception if an error occurs when connecting to the Firehose.
-        def error_handler
-          @error_handler || Proc.new{ |e| raise e }
-        end
-
-        # What adapter should Firehose use to PUT the message? List of adapters is
-        # available at https://github.com/technoweenie/faraday.
-        def self.adapter=(adapter)
-          @adapter = adapter
-        end
-
-        # Use :net_http for the default Faraday adapter.
-        def self.adapter
-          @adapter ||= Faraday.default_adapter
-        end
-
-        private
-        # Build out a Faraday connection
-        def conn
-          @conn ||= Faraday.new(:url => uri.to_s) do |builder|
-            builder.adapter self.class.adapter
-          end
-        end
-      end
-    end
-  end
-end

data/lib/firehose/client.rb

@@ -1,10 +0,0 @@
-require 'em-http'
-require 'faye/websocket'
-
-module Firehose
-  # Ruby clients that connect to Firehose to either publish or consume messages.
-  module Client
-    autoload :Consumer, 'firehose/client/consumer'
-    autoload :Producer, 'firehose/client/producer'
-  end
-end

data/lib/firehose/logging.rb

@@ -1,32 +0,0 @@
-require 'logger'
-
-module Firehose
-  # Sets up logging
-  def self.logger
-    @logger ||= Logger.new($stdout)
-  end
-
-  def self.logger=(logger)
-    @logger = logger
-  end
-
-  self.logger.level = if ENV['LOG_LEVEL']
-    Logger.const_get(ENV['LOG_LEVEL'].upcase)
-  else
-    case ENV['RACK_ENV']
-      when 'test'         then Logger::ERROR
-      when 'development'  then Logger::DEBUG
-      else Logger::INFO
-    end
-  end
-
-  # TODO: Provide some way to allow this to be configured via an ENV variable.
-  self.logger.formatter = lambda do |severity, time, name, msg|
-    out_time = time.utc.strftime "%Y-%m-%d %H:%M:%S.%L"
-    "[#{out_time} ##$$] #{severity} : #{msg}\n"
-  end
-end
-
-# stdout gets "lost" in Foreman if this isn't here
-# https://github.com/ddollar/foreman/wiki/Missing-Output
-$stdout.sync = true if ENV['RACK_ENV'] == 'development' || ENV['SYNC_LOGGING']

data/lib/firehose/rack/app.rb

@@ -1,49 +0,0 @@
-module Firehose
-  module Rack
-    # Acts as the glue between the HTTP/WebSocket world and the Firehose::Server class,
-    # which talks directly to the Redis server. Also dispatches between HTTP and WebSocket
-    # transport handlers depending on the clients' request.
-    class App
-      def initialize
-        yield self if block_given?
-      end
-
-      def call(env)
-        # Cache the parsed request so we don't need to re-parse it when we pass
-        # control onto another app.
-        req     = env['parsed_request'] ||= ::Rack::Request.new(env)
-        method  = req.request_method
-
-        case method
-        when 'PUT'
-          # Firehose::Client::Publisher PUT's payloads to the server.
-          publisher.call(env)
-        when 'HEAD' 
-          # HEAD requests are used to prevent sockets from timing out
-          # from inactivity
-          ping.call(env)
-        else
-          # TODO - 'harden' this up with a GET request and throw a "Bad Request" 
-          # HTTP error code. I'd do it now but I'm in a plane and can't think of it.
-          consumer.call(env)
-        end
-      end
-
-      # The consumer pulls messages off of the backend and passes messages to the 
-      # connected HTTP or WebSocket client. This can be configured from the initialization
-      # method of the rack app.
-      def consumer
-        @consumer ||= Consumer.new
-      end
-
-      private
-      def publisher
-        @publisher ||= Publisher.new
-      end
-
-      def ping
-        @ping ||= Ping.new
-      end
-    end
-  end
-end

data/lib/firehose/rack/consumer/http_long_poll.rb

@@ -1,86 +0,0 @@
-require 'json'
-
-module Firehose
-  module Rack
-    class Consumer
-      class HttpLongPoll
-        include Firehose::Rack::Helpers
-
-        # How long should we wait before closing out the consuming clients web connection
-        # for long polling? Most browsers timeout after a connection has been idle for 30s.
-        TIMEOUT = 20
-
-        # Configures the timeout for the 
-        attr_accessor :timeout
-
-        def initialize(timeout=TIMEOUT)
-          @timeout = timeout
-          yield self if block_given?
-        end
-
-        def call(env)
-          req     = env['parsed_request'] ||= ::Rack::Request.new(env)
-          path    = req.path
-          method  = req.request_method
-          # Get the Last Message Sequence from the query string.
-          # Ideally we'd use an HTTP header, but android devices don't let us
-          # set any HTTP headers for CORS requests.
-          last_sequence = req.params['last_message_sequence'].to_i
-
-          case method
-          # GET is how clients subscribe to the queue. When a messages comes in, we flush out a response,
-          # close down the requeust, and the client then reconnects.
-          when 'GET'
-            Firehose.logger.debug "HTTP GET with last_sequence #{last_sequence} for path #{path} with query #{env["QUERY_STRING"].inspect} and params #{req.params.inspect}"
-            EM.next_tick do
-
-              if last_sequence < 0
-                env['async.callback'].call response(400, "The last_message_sequence parameter may not be less than zero", response_headers(env))
-              else
-                Server::Channel.new(path).next_message(last_sequence, :timeout => timeout).callback do |message, sequence|
-                  env['async.callback'].call response(200, wrap_frame(message, sequence), response_headers(env))
-                end.errback do |e|
-                  if e == :timeout
-                    env['async.callback'].call response(204, '', response_headers(env))
-                  else
-                    Firehose.logger.error "Unexpected error when trying to GET last_sequence #{last_sequence} for path #{path}: #{e.inspect}"
-                    env['async.callback'].call response(500, 'Unexpected error', response_headers(env))
-                  end
-                end
-              end
-
-            end
-
-            # Tell the web server that this will be an async response.
-            ASYNC_RESPONSE
-
-          else
-            Firehose.logger.debug "HTTP #{method} not supported"
-            response(501, "#{method} not supported.")
-          end
-        end
-
-
-        private
-
-        def wrap_frame(message, last_sequence)
-          JSON.generate :message => message, :last_sequence => last_sequence
-        end
-
-        # If the request is a CORS request, return those headers, otherwise don't worry 'bout it
-        def response_headers(env)
-          cors_origin(env) ? cors_headers(env) : {}
-        end
-
-        def cors_origin(env)
-          env['HTTP_ORIGIN']
-        end
-
-        def cors_headers(env)
-          # TODO seperate out CORS logic as an async middleware with a Goliath web server.
-          {'Access-Control-Allow-Origin' => cors_origin(env)}
-        end
-      end
-    end
-  end
-end

data/lib/firehose/rack/consumer/web_socket.rb

@@ -1,95 +0,0 @@
-require 'faye/websocket'
-require 'json'
-
-module Firehose
-  module Rack
-    class Consumer
-      class WebSocket
-        # Setup a handler for the websocket connection.
-        def call(env)
-          ws = Faye::WebSocket.new(env)
-          Handler.new(ws)
-          ws.rack_response
-        end
-
-        # Determine if the rack request is a WebSocket request.
-        def self.request?(env)
-          Faye::WebSocket.websocket?(env)
-        end
-
-        # Manages connection state for the web socket that's connected
-        # by the Consumer::WebSocket class. Deals with message sequence,
-        # connection, failures, and subscription state.
-        class Handler
-          def initialize(ws)
-            @ws = ws
-            @req = ::Rack::Request.new ws.env
-            # Setup the event handlers from this class.
-            @ws.onopen    = method :open
-            @ws.onclose   = method :close
-            @ws.onerror   = method :error
-            @ws.onmessage = method :message
-          end
-
-          # Subscribe the client to the channel on the server. Asks for
-          # the last sequence for clients that reconnect.
-          def subscribe(last_sequence)
-            @subscribed = true
-            @channel    = Server::Channel.new @req.path
-            @deferrable = @channel.next_message last_sequence
-            @deferrable.callback do |message, sequence|
-              Firehose.logger.debug "WS sent `#{message}` to `#{@req.path}` with sequence `#{sequence}`"
-              @ws.send self.class.wrap_frame(message, last_sequence)
-              subscribe sequence
-            end
-            @deferrable.errback do |e|
-              EM.next_tick { raise e.inspect } unless e == :disconnect
-            end
-          end
-
-          # Manages messages sent from the connect client to the server. This is mostly
-          # used to handle heart-beats that are designed to prevent the WebSocket connection
-          # from timing out from inactivity.
-          def message(event)
-            msg = JSON.parse(event.data, :symbolize_names => true) rescue {}
-            seq = msg[:message_sequence]
-            if msg[:ping] == 'PING'
-              Firehose.logger.debug "WS ping received, sending pong"
-              @ws.send JSON.generate :pong => 'PONG'
-            elsif !@subscribed && seq.kind_of?(Integer)
-              Firehose.logger.debug "Subscribing at message_sequence #{seq}"
-              subscribe seq
-            end
-          end
-
-          # Log a message that the client has connected.
-          def open(event)
-            Firehose.logger.debug "WebSocket subscribed to `#{@req.path}`. Waiting for message_sequence..."
-          end
-
-          # Log a message that hte client has disconnected and reset the state for the class. Clean
-          # up the subscribers to the channels.
-          def close(event)
-            if @deferrable
-              @deferrable.fail :disconnect
-              @channel.unsubscribe(@deferrable) if @channel
-            end
-            Firehose.logger.debug "WS connection `#{@req.path}` closing. Code: #{event.code.inspect}; Reason #{event.reason.inspect}"
-          end
-
-          # Log errors if a socket fails. `close` will fire after this to clean up any
-          # remaining connectons.
-          def error(event)
-            Firehose.logger.error "WS connection `#{@req.path}` error. Message: `#{event.message.inspect}`"
-          end
-
-          # Wrap a message in a sequence so that the client can record this and give us
-          # the sequence when it reconnects.
-          def self.wrap_frame(message, last_sequence)
-            JSON.generate :message => message, :last_sequence => last_sequence
-          end
-        end
-      end
-    end
-  end
-end

data/lib/firehose/rack/consumer.rb

@@ -1,39 +0,0 @@
-require 'json'
-
-module Firehose
-  module Rack
-    # Handles a subscription request over HTTP or WebSockets depeding on its abilities and
-    # binds that to the Firehose::Server::Subscription class, which is bound to a channel that
-    # gets published to.
-    class Consumer
-      # Rack consumer transports
-      autoload :HttpLongPoll, 'firehose/rack/consumer/http_long_poll'
-      autoload :WebSocket,    'firehose/rack/consumer/web_socket'
-
-      # Let the client configure the consumer on initialization.
-      def initialize
-        yield self if block_given?
-      end
-
-      def call(env)
-        websocket_request?(env) ? websocket.call(env) : http_long_poll.call(env)
-      end
-
-      # Memoized instance of web socket that can be configured from the rack app.
-      def websocket
-        @web_socket ||= WebSocket.new
-      end
-
-      # Memoized instance of http long poll handler that can be configured from the rack app.
-      def http_long_poll
-        @http_long_poll ||= HttpLongPoll.new
-      end
-
-      private
-      # Determine if the incoming request is a websocket request.
-      def websocket_request?(env)
-        Firehose::Rack::Consumer::WebSocket.request?(env)
-      end
-    end
-  end
-end

data/lib/firehose/rack/ping.rb

@@ -1,88 +0,0 @@
-require 'em-hiredis'
-
-module Firehose
-  module Rack
-    # Allows the Firehose client to periodically "ping" the server
-    # so that the connection isn't timed out by browsers or proxies from
-    # inactivity.
-    class Ping
-      attr_reader :redis
-
-      def initialize(redis=nil)
-        @redis = redis
-      end
-
-      def call(env)
-        PingCheck.new(env, redis).call
-        ASYNC_RESPONSE
-      end
-
-      # Encapsulate this in a class so we aren't passing a bunch of variables around
-      class PingCheck
-        include Firehose::Rack::Helpers
-
-        attr_reader :req, :env, :key, :redis
-
-        TEST_VALUE = 'Firehose Healthcheck Test Value'
-        SECONDS_TO_EXPIRE = 60
-
-        def self.redis
-          @redis ||= EM::Hiredis.connect
-        end
-
-        def initialize(env, redis=nil)
-          @redis = redis || self.class.redis
-          @env   = env
-          @req   = env['parsed_request'] ||= ::Rack::Request.new(env)
-          @key   = "/firehose/ping/#{Time.now.to_i}/#{rand}"
-        end
-
-        def call
-          log req, 'started'
-          test_redis
-        end
-
-
-        private
-
-        def log(req, msg)
-          Firehose.logger.debug "HTTP PING request for path '#{req.path}': #{msg}"
-        end
-
-        def test_redis
-          redis.set(key, TEST_VALUE).
-            callback { expire_key }.
-            callback { read_and_respond }.
-            errback do |e|
-              log req, "failed with write value to redis: #{e.inspect}"
-              env['async.callback'].call response(500)
-            end
-        end
-
-        def expire_key
-          redis.expire(key, SECONDS_TO_EXPIRE).
-            errback do
-              log req, "failed to expire key #{key.inspect}. If this key is not manually deleted, it may cause a memory leak."
-            end
-        end
-
-        def read_and_respond
-          redis.get(key).
-            callback do |val|
-              if val == TEST_VALUE
-                log req, 'succeeded'
-                env['async.callback'].call response(200)
-              else
-                log req, "failed with unexpected value retrieved from redis: #{val.inspect}"
-                env['async.callback'].call response(500)
-              end
-            end.
-            errback do |e|
-              log req, "failed with read value from redis: #{e.inspect}"
-              env['async.callback'].call response(500)
-            end
-        end
-      end
-    end
-  end
-end

data/lib/firehose/rack/publisher.rb

@@ -1,53 +0,0 @@
-module Firehose
-  module Rack
-    class Publisher
-      include Firehose::Rack::Helpers
-
-      def call(env)
-        req     = env['parsed_request'] ||= ::Rack::Request.new(env)
-        path    = req.path
-        method  = req.request_method
-        cache_control = {}
-
-        # Parse out cache control directives from the Cache-Control header.
-        if cache_control_header = env['HTTP_CACHE_CONTROL']
-          cache_control = cache_control_header.split(',').map(&:strip).inject({}) do |memo, directive|
-            key, value = directive.split('=')
-            memo[key.downcase] = value
-            memo
-          end
-        end
-
-        # Read the max-age directive from the cache so that we can set a TTL on the redis key. This will
-        # prevent stale content from being served up to the client.
-        ttl = cache_control['max-age']
-
-        if method == 'PUT'
-          EM.next_tick do
-            body = env['rack.input'].read
-            Firehose.logger.debug "HTTP published #{body.inspect} to #{path.inspect} with ttl #{ttl.inspect}"
-            publisher.publish(path, body, :ttl => ttl).callback do
-              env['async.callback'].call [202, {'Content-Type' => 'text/plain', 'Content-Length' => '0'}, []]
-              env['async.callback'].call response(202, '', 'Content-Type' => 'text/plain')
-            end.errback do |e|
-              Firehose.logger.debug "Error publishing: #{e.inspect}"
-              env['async.callback'].call response(500, 'Error when trying to publish', 'Content-Type' => 'text/plain')
-            end
-          end
-
-          # Tell the web server that this will be an async response.
-          ASYNC_RESPONSE
-        else
-          Firehose.logger.debug "HTTP #{method} not supported"
-          msg = "#{method} not supported."
-          [501, {'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s}, [msg]]
-        end
-      end
-
-      private
-      def publisher
-        @publisher ||= Firehose::Server::Publisher.new
-      end
-    end
-  end
-end