firehose 1.3.6 → 2.1.0

data/lib/firehose/rack/consumer.rb

@@ -1,78 +0,0 @@
-require 'json'
-
-module Firehose
-  module Rack
-    # Handles a subscription request over HTTP or WebSockets depeding on its abilities and
-    # binds that to the Firehose::Server::Subscription class, which is bound to a channel that
-    # gets published to.
-    class Consumer
-      # Rack consumer transports
-      autoload :HttpLongPoll, 'firehose/rack/consumer/http_long_poll'
-      autoload :WebSocket,    'firehose/rack/consumer/web_socket'
-
-      MULTIPLEX_CHANNEL = "channels@firehose"
-
-      def self.multiplexing_request?(env)
-        env["PATH_INFO"].include? MULTIPLEX_CHANNEL
-      end
-
-      def self.multiplex_subscriptions(request)
-        if request.get?
-          query_string_subscriptions(request.env)
-        elsif request.post?
-          post_subscriptions(request)
-        end
-      end
-
-      def self.query_string_subscriptions(env)
-        query_params = ::Rack::Utils.parse_query(env["QUERY_STRING"])
-
-        query_params["subscribe"].to_s.split(",").map do |sub|
-          chan, last_sequence = sub.split("!")
-          last_sequence = last_sequence.to_i
-          last_sequence = 0 if last_sequence < 0
-          {
-            channel: chan,
-            message_sequence: last_sequence
-          }
-        end
-      end
-
-      def self.post_subscriptions(request)
-        body = request.body.read
-        subs = JSON.parse(body).map do |chan, last_sequence|
-          last_sequence = 0 if last_sequence < 0
-          {
-            channel: chan,
-            message_sequence: last_sequence
-          }
-        end
-      end
-
-      # Let the client configure the consumer on initialization.
-      def initialize
-        yield self if block_given?
-      end
-
-      def call(env)
-        websocket_request?(env) ? websocket.call(env) : http_long_poll.call(env)
-      end
-
-      # Memoized instance of web socket that can be configured from the rack app.
-      def websocket
-        @web_socket ||= WebSocket.new
-      end
-
-      # Memoized instance of http long poll handler that can be configured from the rack app.
-      def http_long_poll
-        @http_long_poll ||= HttpLongPoll.new
-      end
-
-      private
-      # Determine if the incoming request is a websocket request.
-      def websocket_request?(env)
-        Firehose::Rack::Consumer::WebSocket.request?(env)
-      end
-    end
-  end
-end

data/lib/firehose/rack/ping.rb

@@ -1,88 +0,0 @@
-require 'em-hiredis'
-
-module Firehose
-  module Rack
-    # Allows the Firehose client to periodically "ping" the server
-    # so that the connection isn't timed out by browsers or proxies from
-    # inactivity.
-    class Ping
-      attr_reader :redis
-
-      def initialize(redis=nil)
-        @redis = redis
-      end
-
-      def call(env)
-        PingCheck.new(env, redis).call
-        ASYNC_RESPONSE
-      end
-
-      # Encapsulate this in a class so we aren't passing a bunch of variables around
-      class PingCheck
-        include Firehose::Rack::Helpers
-
-        attr_reader :req, :env, :key, :redis
-
-        TEST_VALUE = 'Firehose Healthcheck Test Value'
-        SECONDS_TO_EXPIRE = 60
-
-        def self.redis
-          @redis ||= Firehose::Server.redis.connection
-        end
-
-        def initialize(env, redis=nil)
-          @redis = redis || self.class.redis
-          @env   = env
-          @req   = env['parsed_request'] ||= ::Rack::Request.new(env)
-          @key   = "/firehose/ping/#{Time.now.to_i}/#{rand}"
-        end
-
-        def call
-          log req, 'started'
-          test_redis
-        end
-
-
-        private
-
-        def log(req, msg)
-          Firehose.logger.debug "HTTP PING request for path '#{req.path}': #{msg}"
-        end
-
-        def test_redis
-          redis.set(key, TEST_VALUE).
-            callback { expire_key }.
-            callback { read_and_respond }.
-            errback do |e|
-              log req, "failed with write value to redis: #{e.inspect}"
-              env['async.callback'].call response(500)
-            end
-        end
-
-        def expire_key
-          redis.expire(key, SECONDS_TO_EXPIRE).
-            errback do
-              log req, "failed to expire key #{key.inspect}. If this key is not manually deleted, it may cause a memory leak."
-            end
-        end
-
-        def read_and_respond
-          redis.get(key).
-            callback do |val|
-              if val == TEST_VALUE
-                log req, 'succeeded'
-                env['async.callback'].call response(200)
-              else
-                log req, "failed with unexpected value retrieved from redis: #{val.inspect}"
-                env['async.callback'].call response(500)
-              end
-            end.
-            errback do |e|
-              log req, "failed with read value from redis: #{e.inspect}"
-              env['async.callback'].call response(500)
-            end
-        end
-      end
-    end
-  end
-end

data/lib/firehose/rack/publisher.rb

@@ -1,59 +0,0 @@
-require "rack/utils"
-
-module Firehose
-  module Rack
-    class Publisher
-      include Firehose::Rack::Helpers
-
-      def call(env)
-        req           = env['parsed_request'] ||= ::Rack::Request.new(env)
-        path          = req.path
-        method        = req.request_method
-        cache_control = {}
-
-        # Parse out cache control directives from the Cache-Control header.
-        if cache_control_header = env['HTTP_CACHE_CONTROL']
-          cache_control = cache_control_header.split(',').map(&:strip).inject({}) do |memo, directive|
-            key, value = directive.split('=')
-            memo[key.downcase] = value
-            memo
-          end
-        end
-
-        # Read the max-age directive from the cache so that we can set a TTL on the redis key. This will
-        # prevent stale content from being served up to the client.
-        ttl = cache_control['max-age']
-
-        if method == 'PUT'
-          EM.next_tick do
-            body = env['rack.input'].read
-            Firehose.logger.debug "HTTP published #{body.inspect} to #{path.inspect} with ttl #{ttl.inspect}"
-            opts = { :ttl => ttl }
-            if buffer_size = env["HTTP_X_FIREHOSE_BUFFER_SIZE"]
-              opts[:buffer_size] = buffer_size.to_i
-            end
-            publisher.publish(path, body, opts).callback do
-              env['async.callback'].call [202, {'Content-Type' => 'text/plain', 'Content-Length' => '0'}, []]
-              env['async.callback'].call response(202, '', 'Content-Type' => 'text/plain')
-            end.errback do |e|
-              Firehose.logger.debug "Error publishing: #{e.inspect}"
-              env['async.callback'].call response(500, 'Error when trying to publish', 'Content-Type' => 'text/plain')
-            end
-          end
-
-          # Tell the web server that this will be an async response.
-          ASYNC_RESPONSE
-        else
-          Firehose.logger.debug "HTTP #{method} not supported"
-          msg = "#{method} not supported."
-          [501, {'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s}, [msg]]
-        end
-      end
-
-      private
-      def publisher
-        @publisher ||= Firehose::Server::Publisher.new
-      end
-    end
-  end
-end

data/lib/firehose/rack.rb

@@ -1,32 +0,0 @@
-module Firehose
-  module Rack
-    autoload :Consumer,   'firehose/rack/consumer'
-    autoload :Publisher,  'firehose/rack/publisher'
-    autoload :Ping,       'firehose/rack/ping'
-    autoload :App,        'firehose/rack/app'
-
-    # Evented web servers recognize the -1 HTTP code as a response deferral, which
-    # is needed to stream responses via WebSockets or HTTP long polling.
-    ASYNC_RESPONSE = [-1, {}, []].freeze
-
-    # Normally we'd want to use a custom header to reduce the likelihood of some
-    # HTTP middleware clobbering the value. But Safari seems to ignore our CORS
-    # header instructions, so we are using 'pragma' because it is always allowed.
-    LAST_MESSAGE_SEQUENCE_HEADER = 'Pragma'
-
-    # Rack wants the header to start with HTTP, so we deal with that here.
-    RACK_LAST_MESSAGE_SEQUENCE_HEADER = "HTTP_#{LAST_MESSAGE_SEQUENCE_HEADER.upcase.gsub('-', '_')}"
-
-    # Disable CORS preflight caches for requests in development mode.
-    CORS_OPTIONS_MAX_AGE = ENV['RACK_ENV'] == 'development' ? '1' : '1728000'
-
-    module Helpers
-      # Calculates the content of a message body for the response so that HTTP Keep-Alive
-      # connections work.
-      def response(status, body='', headers={})
-        headers = {'Content-Length' => body.size.to_s}.merge(headers)
-        [status, headers, [body]]
-      end
-    end
-  end
-end

data/lib/firehose/rails.rb

@@ -1,7 +0,0 @@
-module Firehose
-  module Rails
-    class Engine < ::Rails::Engine
-      # enable rails to pick up lib/assets/* via sprockets.
-    end
-  end
-end

data/lib/firehose/server/app.rb

@@ -1,52 +0,0 @@
-module Firehose
-  module Server
-    # Configure servers that are booted with-out going through Rack. This is mostly used by
-    # the `firehose server` CLI command or for testing. Production configurations are likely
-    # to boot with custom rack configurations.
-    class App
-      def initialize(opts={})
-        @port   = opts[:port]   || Firehose::URI.port
-        @host   = opts[:host]   || Firehose::URI.host
-        @server = opts[:server] || :rainbows
-
-        Firehose.logger.info "Starting #{Firehose::VERSION} '#{Firehose::CODENAME}', in #{ENV['RACK_ENV']}"
-      end
-
-      def start
-        self.send("start_#{@server}")
-      end
-
-      private
-      # Boot the Firehose server with the Rainbows app server.
-      def start_rainbows
-        require 'rainbows'
-        Faye::WebSocket.load_adapter('rainbows')
-
-        rackup = Unicorn::Configurator::RACKUP
-        rackup[:port] = @port if @port
-        rackup[:host] = @host if @host
-        rackup[:set_listener] = true
-        opts = rackup[:options]
-        opts[:config_file] = File.expand_path('../../../../config/rainbows.rb', __FILE__)
-
-        server = Rainbows::HttpServer.new(Firehose::Rack::App.new, opts)
-        server.start.join
-      end
-
-      # Boot the Firehose server with the Thin app server.
-      def start_thin
-        require 'thin'
-
-        Faye::WebSocket.load_adapter('thin')
-
-        # TODO: See if we can just set Thin to use Firehose.logger instead of
-        #       printing out messages by itself.
-        Thin::Logging.silent = true if Firehose.logger.level == Logger::ERROR
-
-        server = Thin::Server.new(@host, @port) do
-          run Firehose::Rack::App.new
-        end.start
-      end
-    end
-  end
-end

data/lib/firehose/server/channel.rb

@@ -1,72 +0,0 @@
-module Firehose
-  module Server
-    # Connects to a specific channel on Redis and listens for messages to notify subscribers.
-    class Channel
-      attr_reader :channel_key, :list_key, :sequence_key
-      attr_reader :redis, :subscriber
-
-      def self.redis
-        @redis ||= Firehose::Server.redis.connection
-      end
-
-      def self.subscriber
-        @subscriber ||= Server::Subscriber.new(Firehose::Server.redis.connection)
-      end
-
-      def initialize(channel_key, redis=self.class.redis, subscriber=self.class.subscriber)
-        @redis        = redis
-        @subscriber   = subscriber
-        @channel_key  = channel_key
-        @list_key     = Server::Redis.key(channel_key, :list)
-        @sequence_key = Server::Redis.key(channel_key, :sequence)
-      end
-
-      def next_messages(consumer_sequence=nil, options={})
-        deferrable = EM::DefaultDeferrable.new
-        deferrable.errback {|e| EM.next_tick { raise e } unless [:timeout, :disconnect].include?(e) }
-
-        redis.multi
-          redis.get(sequence_key).
-            errback {|e| deferrable.fail e }
-          # Fetch entire list: http://stackoverflow.com/questions/10703019/redis-fetch-all-value-of-list-without-iteration-and-without-popping
-          redis.lrange(list_key, 0, -1).
-            errback {|e| deferrable.fail e }
-        redis.exec.callback do |(channel_sequence, message_list)|
-          # Reverse the messages so they can be correctly procesed by the MessageBuffer class. There's
-          # a patch in the message-buffer-redis branch that moves this concern into the Publisher LUA
-          # script. We kept it out of this for now because it represents a deployment risk and `reverse!`
-          # is a cheap operation in Ruby.
-          message_list.reverse!
-          buffer = MessageBuffer.new(message_list, channel_sequence, consumer_sequence)
-          if buffer.remaining_messages.empty?
-            Firehose.logger.debug "No messages in buffer, subscribing. sequence: `#{channel_sequence}` consumer_sequence: #{consumer_sequence}"
-            # Either this resource has never been seen before or we are all caught up.
-            # Subscribe and hope something gets published to this end-point.
-            subscribe(deferrable, options[:timeout])
-          else # Either the client is under water or caught up to head.
-            deferrable.succeed buffer.remaining_messages
-          end
-        end.errback {|e| deferrable.fail e }
-
-        deferrable
-      end
-
-      def unsubscribe(deferrable)
-        subscriber.unsubscribe channel_key, deferrable
-      end
-
-      private
-      def subscribe(deferrable, timeout=nil)
-        subscriber.subscribe(channel_key, deferrable)
-        if timeout
-          timer = EventMachine::Timer.new(timeout) do
-            deferrable.fail :timeout
-            unsubscribe deferrable
-          end
-          # Cancel the timer if when the deferrable succeeds
-          deferrable.callback { timer.cancel }
-        end
-      end
-    end
-  end
-end

data/lib/firehose/server/message_buffer.rb

@@ -1,59 +0,0 @@
-module Firehose
-  module Server
-    # Encapsulates a sequence of messages from the server along with their
-    # consumer_sequences calculate by offset.
-    class MessageBuffer
-      # Number of messages that Redis buffers for the client if its
-      # connection drops, then reconnects.
-      DEFAULT_SIZE = 100
-
-      Message = Struct.new(:payload, :sequence)
-
-      def initialize(message_list, channel_sequence, consumer_sequence = nil)
-        @message_list = message_list
-        @channel_sequence = channel_sequence.to_i
-        @consumer_sequence = consumer_sequence.to_i
-      end
-
-      def remaining_messages
-        messages.last(remaining_message_count)
-      end
-
-      private
-
-      def remaining_message_count
-        # Special case to always get the latest message.
-        return 1 unless @consumer_sequence > 0
-
-        count = @channel_sequence - @consumer_sequence
-
-        if count < 0
-          # UNEXPECTED: Somehow the sequence is ahead of the channel.
-          #             It is likely a bug in the consumer, but we'll assume
-          #             the consumer has all the messages.
-          0
-        elsif count > @message_list.size
-          # Consumer is under water since the last request. Just send the most recent message.
-          1
-        else
-          count
-        end
-      end
-
-      # Calculates the last_message_sequence per message.
-      # [a b c e f]
-      def messages
-        @messages ||= @message_list.map.with_index do |payload, index|
-          Message.new(payload, starting_channel_sequence + index)
-        end
-      end
-
-      # Channel sequence is   10
-      # Buffer size of        5
-      # Start of sequence in buffer ... which would be 6
-      def starting_channel_sequence
-        @channel_sequence - @message_list.size + 1
-      end
-    end
-  end
-end

data/lib/firehose/server/publisher.rb

@@ -1,133 +0,0 @@
-module Firehose
-  module Server
-    class Publisher
-      # Seconds that the message buffer should live before Redis expires it.
-      TTL = 60*60*24
-
-      # Delimited used to frame different parts of a message that's published
-      # over Firehose.
-      PAYLOAD_DELIMITER = "\n"
-
-      # Publish a message to a Firehose channel via Redis.
-      def publish(channel_key, message, opts={})
-        # How long should we hang on to the resource once is published?
-        ttl = (opts[:ttl] || TTL).to_i
-        buffer_size = (opts[:buffer_size] || MessageBuffer::DEFAULT_SIZE).to_i
-
-        # TODO hi-redis isn't that awesome... we have to setup an errback per even for wrong
-        # commands because of the lack of a method_missing whitelist. Perhaps implement a whitelist in
-        # em-hiredis or us a diff lib?
-        if (deferrable = opts[:deferrable]).nil?
-          deferrable = EM::DefaultDeferrable.new
-          deferrable.errback do |e|
-            # Handle missing Lua publishing script in cache
-            # (such as Redis restarting or someone executing SCRIPT FLUSH)
-            if e.message =~ /NOSCRIPT/
-              deferrable.succeed
-              EM.next_tick do
-                @publish_script_digest = nil
-                combined_opts = opts.merge :deferrable => deferrable
-                self.publish channel_key, message, combined_opts
-              end
-            else
-              EM.next_tick { raise e }
-            end
-          end
-        end
-
-        if @publish_script_digest.nil?
-          register_publish_script.errback do |e|
-            deferrable.fail e
-          end.callback do |digest|
-            @publish_script_digest = digest
-            Firehose.logger.debug "Registered Lua publishing script with Redis => #{digest}"
-            eval_publish_script channel_key, message, ttl, buffer_size, deferrable
-          end
-        else
-          eval_publish_script channel_key, message, ttl, buffer_size, deferrable
-        end
-
-        deferrable
-      end
-
-      private
-      def redis
-        @redis ||= Firehose::Server.redis.connection
-      end
-
-      # Serialize components of a message into something that can be dropped into Redis.
-      def self.to_payload(channel_key, sequence, message)
-        [channel_key, sequence, message].join(PAYLOAD_DELIMITER)
-      end
-
-      # Deserialize components of a message back into Ruby.
-      def self.from_payload(payload)
-        @payload_size ||= method(:to_payload).arity
-        payload.split(PAYLOAD_DELIMITER, @payload_size)
-      end
-
-      # TODO: Make this FAR more robust. Ideally we'd whitelist the permitted
-      #       characters and then escape or remove everything else.
-      #       See: http://en.wikibooks.org/wiki/Lua_Programming/How_to_Lua/escape_sequence
-      def lua_escape(str)
-        str.gsub(/\\/,'\\\\\\').gsub(/"/,'\"').gsub(/\n/,'\n').gsub(/\r/,'\r')
-      end
-
-      def register_publish_script
-        redis.script 'LOAD', REDIS_PUBLISH_SCRIPT
-      end
-
-      def eval_publish_script(channel_key, message, ttl, buffer_size, deferrable)
-        list_key = Server::Redis.key(channel_key, :list)
-        script_args = [
-          Server::Redis.key(channel_key, :sequence),
-          list_key,
-          Server::Redis.key(:channel_updates),
-          ttl,
-          message,
-          buffer_size,
-          PAYLOAD_DELIMITER,
-          channel_key
-        ]
-
-        redis.evalsha(
-          @publish_script_digest, script_args.length, *script_args
-        ).errback do |e|
-          deferrable.fail e
-        end.callback do |sequence|
-          Firehose.logger.debug "Redis stored/published `#{message}` to list `#{list_key}` with sequence `#{sequence}`"
-          deferrable.succeed
-        end
-      end
-
-      REDIS_PUBLISH_SCRIPT = <<-LUA
-        local sequence_key      = KEYS[1]
-        local list_key          = KEYS[2]
-        local channel_key       = KEYS[3]
-        local ttl               = KEYS[4]
-        local message           = KEYS[5]
-        local buffer_size       = KEYS[6]
-        local payload_delimiter = KEYS[7]
-        local firehose_resource = KEYS[8]
-
-        local current_sequence = redis.call('get', sequence_key)
-        if current_sequence == nil or current_sequence == false then
-          current_sequence = 0
-        end
-
-        local sequence = current_sequence + 1
-        local message_payload = firehose_resource .. payload_delimiter .. sequence .. payload_delimiter .. message
-
-        redis.call('set', sequence_key, sequence)
-        redis.call('expire', sequence_key, ttl)
-        redis.call('lpush', list_key, message)
-        redis.call('ltrim', list_key, 0, buffer_size - 1)
-        redis.call('expire', list_key, ttl)
-        redis.call('publish', channel_key, message_payload)
-
-        return sequence
-      LUA
-
-    end
-  end
-end

data/lib/firehose/server/redis.rb

@@ -1,32 +0,0 @@
-require "uri"
-
-module Firehose
-  module Server
-    # Manages redis configuration and connections.
-    class Redis
-      DEFAULT_URL = "redis://127.0.0.1:6379/0".freeze
-      KEY_DELIMITER = ":".freeze
-      ROOT_KEY = "firehose".freeze
-
-      attr_reader :url
-
-      def initialize(url = self.class.url)
-        @url = URI(url)
-      end
-
-      def connection
-        EM::Hiredis.connect(@url)
-      end
-
-      # Generates keys for all firehose interactions with Redis. Ensures a root
-      # key of `firehose`
-      def self.key(*segments)
-        segments.flatten.unshift(ROOT_KEY).join(KEY_DELIMITER)
-      end
-
-      def self.url
-        ENV.fetch("REDIS_URL", DEFAULT_URL)
-      end
-    end
-  end
-end

data/lib/firehose/server/subscriber.rb

@@ -1,51 +0,0 @@
-module Firehose
-  module Server
-    # Setups a connetion to Redis to listen for new resources...
-    class Subscriber
-      attr_reader :pubsub
-
-      def initialize(redis)
-        @pubsub = redis.pubsub
-        # TODO: Instead of just raising an exception, it would probably be better
-        #       for the errback to set some sort of 'disconnected' state. Then
-        #       whenever a deferrable was 'subscribed' we could instantly fail
-        #       the deferrable with whatever connection error we had.
-        #       An alternative which would have a similar result would be to
-        #       subscribe lazily (i.e. not until we have a deferrable to subscribe).
-        #       Then, if connecting failed, it'd be super easy to fail the deferrable
-        #       with the same error.
-        #       The final goal is to allow the failed deferrable bubble back up
-        #       so we can send back a nice, clean 500 error to the client.
-        channel_updates_key = Server::Redis.key('channel_updates')
-        pubsub.subscribe(channel_updates_key).
-          errback{|e| EM.next_tick { raise e } }.
-          callback { Firehose.logger.debug "Redis subscribed to `#{channel_updates_key}`" }
-        pubsub.on(:message) do |_, payload|
-          channel_key, channel_sequence, message = Server::Publisher.from_payload(payload)
-          messages = [ MessageBuffer::Message.new(message, channel_sequence.to_i) ]
-          if deferrables = subscriptions.delete(channel_key)
-            Firehose.logger.debug "Redis notifying #{deferrables.count} deferrable(s) at `#{channel_key}` with channel_sequence `#{channel_sequence}` and message `#{message}`"
-            deferrables.each do |deferrable|
-              Firehose.logger.debug "Sending message #{message} and channel_sequence #{channel_sequence} to client from subscriber"
-              deferrable.succeed messages
-            end
-          end
-        end
-      end
-
-      def subscribe(channel_key, deferrable)
-        subscriptions[channel_key].push deferrable
-      end
-
-      def unsubscribe(channel_key, deferrable)
-        subscriptions[channel_key].delete deferrable
-        subscriptions.delete(channel_key) if subscriptions[channel_key].empty?
-      end
-
-      private
-      def subscriptions
-        @subscriptions ||= Hash.new{|h,k| h[k] = []}
-      end
-    end
-  end
-end

data/lib/firehose/version.rb

@@ -1,4 +0,0 @@
-module Firehose
-  VERSION  = "1.3.6"
-  CODENAME = "Oops I did it again"
-end