firehose 1.2.20 → 2.1.0

data/lib/firehose/rack.rb

@@ -1,32 +0,0 @@
-module Firehose
-  module Rack
-    autoload :Consumer,   'firehose/rack/consumer'
-    autoload :Publisher,  'firehose/rack/publisher'
-    autoload :Ping,       'firehose/rack/ping'
-    autoload :App,        'firehose/rack/app'
-
-    # Evented web servers recognize the -1 HTTP code as a response deferral, which
-    # is needed to stream responses via WebSockets or HTTP long polling.
-    ASYNC_RESPONSE = [-1, {}, []].freeze
-
-    # Normally we'd want to use a custom header to reduce the likelihood of some
-    # HTTP middleware clobbering the value. But Safari seems to ignore our CORS
-    # header instructions, so we are using 'pragma' because it is always allowed.
-    LAST_MESSAGE_SEQUENCE_HEADER = 'Pragma'
-
-    # Rack wants the header to start with HTTP, so we deal with that here.
-    RACK_LAST_MESSAGE_SEQUENCE_HEADER = "HTTP_#{LAST_MESSAGE_SEQUENCE_HEADER.upcase.gsub('-', '_')}"
-
-    # Disable CORS preflight caches for requests in development mode.
-    CORS_OPTIONS_MAX_AGE = ENV['RACK_ENV'] == 'development' ? '1' : '1728000'
-
-    module Helpers
-      # Calculates the content of a message body for the response so that HTTP Keep-Alive
-      # connections work.
-      def response(status, body='', headers={})
-        headers = {'Content-Length' => body.size.to_s}.merge(headers)
-        [status, headers, [body]]
-      end
-    end
-  end
-end

data/lib/firehose/rails.rb

@@ -1,7 +0,0 @@
-module Firehose
-  module Rails
-    class Engine < ::Rails::Engine
-      # enable rails to pick up lib/assets/* via sprockets.
-    end
-  end
-end

data/lib/firehose/server/app.rb

@@ -1,52 +0,0 @@
-module Firehose
-  module Server
-    # Configure servers that are booted with-out going through Rack. This is mostly used by
-    # the `firehose server` CLI command or for testing. Production configurations are likely
-    # to boot with custom rack configurations.
-    class App
-      def initialize(opts={})
-        @port   = opts[:port]   || Firehose::URI.port
-        @host   = opts[:host]   || Firehose::URI.host
-        @server = opts[:server] || :rainbows
-
-        Firehose.logger.info "Starting #{Firehose::VERSION} '#{Firehose::CODENAME}', in #{ENV['RACK_ENV']}"
-      end
-
-      def start
-        self.send("start_#{@server}")
-      end
-
-      private
-      # Boot the Firehose server with the Rainbows app server.
-      def start_rainbows
-        require 'rainbows'
-        Faye::WebSocket.load_adapter('rainbows')
-
-        rackup = Unicorn::Configurator::RACKUP
-        rackup[:port] = @port if @port
-        rackup[:host] = @host if @host
-        rackup[:set_listener] = true
-        opts = rackup[:options]
-        opts[:config_file] = File.expand_path('../../../../config/rainbows.rb', __FILE__)
-
-        server = Rainbows::HttpServer.new(Firehose::Rack::App.new, opts)
-        server.start.join
-      end
-
-      # Boot the Firehose server with the Thin app server.
-      def start_thin
-        require 'thin'
-
-        Faye::WebSocket.load_adapter('thin')
-
-        # TODO: See if we can just set Thin to use Firehose.logger instead of
-        #       printing out messages by itself.
-        Thin::Logging.silent = true if Firehose.logger.level == Logger::ERROR
-
-        server = Thin::Server.new(@host, @port) do
-          run Firehose::Rack::App.new
-        end.start
-      end
-    end
-  end
-end

data/lib/firehose/server/channel.rb

@@ -1,80 +0,0 @@
-module Firehose
-  module Server
-    # Connects to a specific channel on Redis and listens for messages to notify subscribers.
-    class Channel
-      attr_reader :channel_key, :redis, :subscriber, :list_key, :sequence_key
-
-      def self.redis
-        @redis ||= EM::Hiredis.connect
-      end
-
-      def self.subscriber
-        @subscriber ||= Server::Subscriber.new(EM::Hiredis.connect)
-      end
-
-      def initialize(channel_key, redis=self.class.redis, subscriber=self.class.subscriber)
-        @channel_key, @redis, @subscriber = channel_key, redis, subscriber
-        @list_key, @sequence_key = Server.key(channel_key, :list), Server.key(channel_key, :sequence)
-      end
-
-      def next_message(last_sequence=nil, options={})
-        last_sequence = last_sequence.to_i
-
-        deferrable = EM::DefaultDeferrable.new
-        # TODO - Think this through a little harder... maybe some tests ol buddy!
-        deferrable.errback {|e| EM.next_tick { raise e } unless [:timeout, :disconnect].include?(e) }
-
-        # TODO: Use HSET so we don't have to pull 100 messages back every time.
-        redis.multi
-          redis.get(sequence_key).
-            errback {|e| deferrable.fail e }
-          redis.lrange(list_key, 0, Server::Publisher::MAX_MESSAGES).
-            errback {|e| deferrable.fail e }
-        redis.exec.callback do |(sequence, message_list)|
-          Firehose.logger.debug "exec returned: `#{sequence}` and `#{message_list.inspect}`"
-          sequence = sequence.to_i
-
-          if sequence.nil? || (diff = sequence - last_sequence) <= 0
-            Firehose.logger.debug "No message available yet, subscribing. sequence: `#{sequence}`"
-            # Either this resource has never been seen before or we are all caught up.
-            # Subscribe and hope something gets published to this end-point.
-            subscribe(deferrable, options[:timeout])
-          elsif last_sequence > 0 && diff < Server::Publisher::MAX_MESSAGES
-            # The client is kinda-sorta running behind, but has a chance to catch
-            # up. Catch them up FTW.
-            # But we won't "catch them up" if last_sequence was zero/nil because
-            # that implies the client is connecting for the 1st time.
-            message = message_list[diff-1]
-            Firehose.logger.debug "Sending old message `#{message}` and sequence `#{sequence}` to client directly. Client is `#{diff}` behind, at `#{last_sequence}`."
-            deferrable.succeed message, last_sequence + 1
-          else
-            # The client is hopelessly behind and underwater. Just reset
-            # their whole world with the lastest message.
-            message = message_list[0]
-            Firehose.logger.debug "Sending latest message `#{message}` and sequence `#{sequence}` to client directly."
-            deferrable.succeed message, sequence
-          end
-        end.errback {|e| deferrable.fail e }
-
-        deferrable
-      end
-
-      def unsubscribe(deferrable)
-        subscriber.unsubscribe channel_key, deferrable
-      end
-
-      private
-      def subscribe(deferrable, timeout=nil)
-        subscriber.subscribe(channel_key, deferrable)
-        if timeout
-          timer = EventMachine::Timer.new(timeout) do
-            deferrable.fail :timeout
-            unsubscribe deferrable
-          end
-          # Cancel the timer if when the deferrable succeeds
-          deferrable.callback { timer.cancel }
-        end
-      end
-    end
-  end
-end

data/lib/firehose/server/publisher.rb

@@ -1,134 +0,0 @@
-module Firehose
-  module Server
-    class Publisher
-      # Number of messages that Redis buffers for the client if its
-      # connection drops, then reconnects.
-      MAX_MESSAGES = 100
-
-      # Seconds that the message buffer should live before Redis expires it.
-      TTL = 60*60*24
-
-      # Delimited used to frame different parts of a message that's published
-      # over Firehose.
-      PAYLOAD_DELIMITER = "\n"
-
-      # Publish a message to a Firehose channel via Redis.
-      def publish(channel_key, message, opts={})
-        # How long should we hang on to the resource once is published?
-        ttl = (opts[:ttl] || TTL).to_i
-
-        # TODO hi-redis isn't that awesome... we have to setup an errback per even for wrong
-        # commands because of the lack of a method_missing whitelist. Perhaps implement a whitelist in
-        # em-hiredis or us a diff lib?
-        if (deferrable = opts[:deferrable]).nil?
-          deferrable = EM::DefaultDeferrable.new
-          deferrable.errback do |e|
-            # Handle missing Lua publishing script in cache
-            # (such as Redis restarting or someone executing SCRIPT FLUSH)
-            if e.message =~ /NOSCRIPT/
-              deferrable.succeed
-              EM.next_tick do
-                @publish_script_digest = nil
-                combined_opts = opts.merge :deferrable => deferrable
-                self.publish channel_key, message, combined_opts
-              end
-            else
-              EM.next_tick { raise e }
-            end
-          end
-        end
-
-        if @publish_script_digest.nil?
-          register_publish_script.errback do |e|
-            deferrable.fail e
-          end.callback do |digest|
-            @publish_script_digest = digest
-            Firehose.logger.debug "Registered Lua publishing script with Redis => #{digest}"
-            eval_publish_script channel_key, message, ttl, deferrable
-          end
-        else
-          eval_publish_script channel_key, message, ttl, deferrable
-        end
-
-        deferrable
-      end
-
-      private
-      def redis
-        @redis ||= EM::Hiredis.connect
-      end
-
-      # Serialize components of a message into something that can be dropped into Redis.
-      def self.to_payload(channel_key, sequence, message)
-        [channel_key, sequence, message].join(PAYLOAD_DELIMITER)
-      end
-
-      # Deserealize components of a message back into Ruby.
-      def self.from_payload(payload)
-        payload.split(PAYLOAD_DELIMITER, method(:to_payload).arity)
-      end
-
-      # TODO: Make this FAR more robust. Ideally we'd whitelist the permitted
-      #       characters and then escape or remove everything else.
-      #       See: http://en.wikibooks.org/wiki/Lua_Programming/How_to_Lua/escape_sequence
-      def lua_escape(str)
-        str.gsub(/\\/,'\\\\\\').gsub(/"/,'\"').gsub(/\n/,'\n').gsub(/\r/,'\r')
-      end
-
-      def register_publish_script
-        redis.script 'LOAD', REDIS_PUBLISH_SCRIPT
-      end
-
-      def eval_publish_script(channel_key, message, ttl, deferrable)
-        list_key = Server.key(channel_key, :list)
-        script_args = [
-          Server.key(channel_key, :sequence),
-          list_key,
-          Server.key(:channel_updates),
-          ttl,
-          message,
-          MAX_MESSAGES,
-          PAYLOAD_DELIMITER,
-          channel_key
-        ]
-        redis.evalsha(
-          @publish_script_digest, script_args.length, *script_args
-        ).errback do |e|
-          deferrable.fail e
-        end.callback do |sequence|
-          Firehose.logger.debug "Redis stored/published `#{message}` to list `#{list_key}` with sequence `#{sequence}`"
-          deferrable.succeed
-        end
-      end
-
-      REDIS_PUBLISH_SCRIPT = <<-LUA
-        local sequence_key      = KEYS[1]
-        local list_key          = KEYS[2]
-        local channel_key       = KEYS[3]
-        local ttl               = KEYS[4]
-        local message           = KEYS[5]
-        local max_messages      = KEYS[6]
-        local payload_delimiter = KEYS[7]
-        local firehose_resource = KEYS[8]
-
-        local current_sequence = redis.call('get', sequence_key)
-        if current_sequence == nil or current_sequence == false then
-          current_sequence = 0
-        end
-
-        local sequence = current_sequence + 1
-        local message_payload = firehose_resource .. payload_delimiter .. sequence .. payload_delimiter .. message
-
-        redis.call('set', sequence_key, sequence)
-        redis.call('expire', sequence_key, ttl)
-        redis.call('lpush', list_key, message)
-        redis.call('ltrim', list_key, 0, max_messages - 1)
-        redis.call('expire', list_key, ttl)
-        redis.call('publish', channel_key, message_payload)
-
-        return sequence
-      LUA
-
-    end
-  end
-end

data/lib/firehose/server/subscriber.rb

@@ -1,51 +0,0 @@
-module Firehose
-  module Server
-    # Setups a connetion to Redis to listen for new resources...
-    class Subscriber
-      attr_reader :pubsub
-
-      def initialize(redis)
-        @pubsub = redis.pubsub
-        # TODO: Instead of just raising an exception, it would probably be better
-        #       for the errback to set some sort of 'disconnected' state. Then
-        #       whenever a deferrable was 'subscribed' we could instantly fail
-        #       the deferrable with whatever connection error we had.
-        #       An alternative which would have a similar result would be to
-        #       subscribe lazily (i.e. not until we have a deferrable to subscribe).
-        #       Then, if connecting failed, it'd be super easy to fail the deferrable
-        #       with the same error.
-        #       The final goal is to allow the failed deferrable bubble back up
-        #       so we can send back a nice, clean 500 error to the client.
-        channel_updates_key = Server.key('channel_updates')
-        pubsub.subscribe(channel_updates_key).
-          errback{|e| EM.next_tick { raise e } }.
-          callback { Firehose.logger.debug "Redis subscribed to `#{channel_updates_key}`" }
-        pubsub.on(:message) do |_, payload|
-          channel_key, sequence, message = Server::Publisher.from_payload(payload)
-
-          if deferrables = subscriptions.delete(channel_key)
-            Firehose.logger.debug "Redis notifying #{deferrables.count} deferrable(s) at `#{channel_key}` with sequence `#{sequence}` and message `#{message}`"
-            deferrables.each do |deferrable|
-              Firehose.logger.debug "Sending message #{message} and sequence #{sequence} to client from subscriber"
-              deferrable.succeed message, sequence.to_i
-            end
-          end
-        end
-      end
-
-      def subscribe(channel_key, deferrable)
-        subscriptions[channel_key].push deferrable
-      end
-
-      def unsubscribe(channel_key, deferrable)
-        subscriptions[channel_key].delete deferrable
-        subscriptions.delete(channel_key) if subscriptions[channel_key].empty?
-      end
-
-      private
-      def subscriptions
-        @subscriptions ||= Hash.new{|h,k| h[k] = []}
-      end
-    end
-  end
-end

data/lib/firehose/version.rb

@@ -1,4 +0,0 @@
-module Firehose
-  VERSION  = "1.2.20"
-  CODENAME = "Onprogress of Doom"
-end

data/package.json

@@ -1,5 +0,0 @@
-{
-  "name": "firehose",
-  "version": "1.2.20",
-  "main": "lib/assets/javascripts/firehose.js.coffee"
-}

data/spec/integrations/integration_test_helper.rb

@@ -1,22 +0,0 @@
-require 'spec_helper'
-
-module IntegrationTestHelper
-  def start_server
-    @server_pid = fork do
-      Firehose::Server::App.new(:server => server, :port => uri.port).start
-    end
-
-    # Need to give the server a chance to boot up.
-    sleep 1
-  end
-
-  def stop_server
-    Process.kill 'INT', @server_pid
-  end
-
-
-  # Let isn't allowed in before(:all)
-  def uri
-    Firehose::URI
-  end
-end

data/spec/integrations/rainbows_spec.rb

@@ -1,7 +0,0 @@
-require 'spec_helper'
-require 'integrations/shared_examples'
-
-describe "rainbows" do
-  def server; :rainbows; end # Let isn't allowed in before(:all)
-  it_behaves_like 'Firehose::Rack::App'
-end

data/spec/integrations/shared_examples.rb

@@ -1,120 +0,0 @@
-require 'spec_helper'
-require 'integrations/integration_test_helper'
-require 'json'
-
-shared_examples_for 'Firehose::Rack::App' do
-  include EM::TestHelper
-  include IntegrationTestHelper
-
-  before(:all) do
-    Firehose::Client::Producer::Http.adapter = :em_http
-    start_server
-  end
-
-  after(:all) do
-    Firehose::Client::Producer::Http.adapter = nil
-    stop_server
-  end
-
-  before(:each) { WebMock.disable! }
-  after(:each)  { WebMock.enable! }
-
-  let(:app)       { Firehose::Rack::App.new }
-  let(:messages)  { (1..200).map{|n| "msg-#{n}" } }
-  let(:channel)   { "/firehose/integration/#{Time.now.to_i}" }
-  let(:http_url)  { "http://#{uri.host}:#{uri.port}#{channel}" }
-  let(:ws_url)    { "ws://#{uri.host}:#{uri.port}#{channel}" }
-
-  it "should pub-sub http and websockets" do
-    # Setup variables that we'll use after we turn off EM to validate our
-    # test assertions.
-    outgoing, received = messages.dup, Hash.new{|h,k| h[k] = []}
-
-    # Our WS and Http clients call this when they have received their messages to determine
-    # when to turn off EM and make the test assertion at the very bottom.
-    succeed = Proc.new do
-      # TODO: For some weird reason the `add_timer` call causes up to 20 seconds of delay after
-      #       the test finishes running. However, without it the test will randomly fail with a
-      #       "Redis disconnected" error.
-      em.add_timer(1) { em.stop } if received.values.all?{|arr| arr.size == messages.size }
-    end
-
-    # Setup a publisher
-    publish = Proc.new do
-      Firehose::Client::Producer::Http.new.publish(outgoing.shift).to(channel) do
-        # The random timer ensures that sometimes the clients will be behind
-        # and sometimes they will be caught up.
-        EM::add_timer(rand*0.005) { publish.call } unless outgoing.empty?
-      end
-    end
-
-    # Lets have an HTTP Long poll client
-    http_long_poll = Proc.new do |cid, last_sequence|
-      http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => last_sequence})
-      http.errback { em.stop }
-      http.callback do
-        frame = JSON.parse(http.response, :symbolize_names => true)
-        received[cid] << frame[:message]
-        if received[cid].size < messages.size
-          # Add some jitter so the clients aren't syncronized
-          EM::add_timer(rand*0.001) { http_long_poll.call cid, frame[:last_sequence] }
-        else
-          succeed.call cid
-        end
-      end
-    end
-
-    # And test a web socket client too, at the same time.
-    websocket = Proc.new do |cid|
-      ws = Faye::WebSocket::Client.new(ws_url)
-
-      ws.onopen = lambda do |event|
-        ws.send('{"message_sequence":0}')
-      end
-
-      ws.onmessage = lambda do |event|
-        frame = JSON.parse(event.data, :symbolize_names => true)
-        received[cid] << frame[:message]
-        succeed.call cid unless received[cid].size < messages.size
-      end
-
-      ws.onclose = lambda do |event|
-        ws = nil
-      end
-
-      ws.onerror = lambda do |event|
-        raise 'ws failed' + "\n" + event.inspect
-      end
-    end
-
-    # Great, we have all the pieces in order, lets run this thing in the reactor.
-    em 180 do
-      # Start the clients.
-      websocket.call(1)
-      websocket.call(2)
-      http_long_poll.call(3)
-      http_long_poll.call(4)
-
-      # Wait a sec to let our clients set up.
-      em.add_timer(1){ publish.call }
-    end
-
-    # When EM stops, these assertions will be made.
-    received.size.should == 4
-    received.values.each do |arr|
-      arr.should == messages
-    end
-  end
-
-
-  it "should return 400 error for long-polling when using http long polling and sequence header is < 0" do
-    em 5 do
-      http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => -1})
-      http.errback { |e| raise e.inspect }
-      http.callback do
-        http.response_header.status.should == 400
-        em.stop
-      end
-    end
-  end
-end

data/spec/integrations/thin_spec.rb

@@ -1,7 +0,0 @@
-require 'spec_helper'
-require 'integrations/shared_examples'
-
-describe "thin" do
-  def server; :thin; end # Let isn't allowed in before(:all)
-  it_behaves_like 'Firehose::Rack::App'
-end

data/spec/javascripts/firehose/base_spec.coffee

@@ -1,2 +0,0 @@
-describe 'Firehose namespace', ->
-  it 'exists', -> expect( window.Firehose ).toEqual jasmine.any Object

data/spec/javascripts/firehose/consumer_spec.coffee

@@ -1,45 +0,0 @@
-describe 'Firehose.Consumer', ->
-  beforeEach ->
-    @klass    = Firehose.Consumer
-    @instance = new @klass uri: 'test_uri', okInterval: 9999 # Stop crazy ajax loops during tests
-
-
-  #= Specs ===
-
-  describe 'new instance', ->
-    describe '@config', ->
-      beforeEach -> @subj = @instance.config
-
-      for callback in ['message','error','connected','disconnected','failed']
-        it "has a .#{callback} function", ->
-          expect( @subj[callback] ).toEqual jasmine.any Function
-
-      it 'has a params object', ->
-        expect( @subj.params ).toEqual jasmine.any Object
-
-      it 'has a parse function', ->
-        expect( @subj.parse ).toEqual jasmine.any Function
-
-  describe '#connect', ->
-    it 'runs without error', -> @instance.connect()
-
-    describe 'after #connect called', ->
-      beforeEach -> @instance.connect()
-
-      it 'creates a @transport instance', ->
-        expect( @instance.transport ).toEqual jasmine.any Firehose.Transport
-
-  describe '#stop', ->
-    describe 'when .connect called earlier', ->
-      beforeEach ->
-        @instance.connect()
-        sinon.spy @instance.transport, 'stop'
-
-      it 'calls stop once on @transport', ->
-        @instance.stop()
-        expect( @instance.transport.stop.calledOnce ).toBe true
-
-      it 'stops the upgrade timeout', ->
-        @instance.stop()
-        expect( @instance.upgradeTimeout? ).toBe false
-

data/spec/javascripts/firehose/long_poll_spec.coffee

@@ -1,44 +0,0 @@
-describe 'Firehose.LongPoll', ->
-  beforeEach ->
-    @klass    = Firehose.LongPoll
-    @instance = new @klass()
-
-  afterEach -> @instance.stop()
-
-  #= Specs ===
-
-  it '.ieSupported returns a Boolean', ->
-    expect( typeof @klass.ieSupported() ).toBe 'boolean'
-
-  it '.supported returns a Boolean', ->
-    expect( typeof @klass.supported() ).toBe 'boolean'
-
-  describe '#stop', ->
-    beforeEach ->
-      @instance._lastRequest     = $.ajax()
-      @instance._lastPingRequest = $.ajax()
-
-    it 'aborts the last request', ->
-      sinon.spy @instance._lastRequest, 'abort'
-      Object.freeze @instance # Prevents the delete call from modifying the request object, so we can verify the call
-      @instance.stop()
-      expect( @instance._lastRequest.abort.called ).toBe true
-
-    it 'deletes reference to the last request', ->
-      @instance.stop()
-      expect( @instance._lastRequest? ).toBe false
-
-    it 'aborts the last ping request', ->
-      sinon.spy @instance._lastPingRequest, 'abort'
-      Object.freeze @instance # Prevents the delete call from modifying the request object, so we can verify the call
-      @instance.stop()
-      expect( @instance._lastPingRequest.abort.called ).toBe true
-
-    it 'deletes reference to the last ping request', ->
-      @instance.stop()
-      expect( @instance._lastPingRequest? ).toBe false
-
-  describe 'callbacks', ->
-    for callback in ['_request','_success','_ping','_error']
-      it "has a #{callback} callback function", ->
-        expect( @instance[callback] ).toEqual jasmine.any Function

data/spec/javascripts/firehose/transport_spec.coffee

@@ -1,32 +0,0 @@
-describe 'Firehose.Transport', ->
-  beforeEach ->
-    @klass    = Firehose.Transport
-    @instance = new @klass()
-
-  it '.supported returns false', -> expect( @klass.supported() ).toBe false
-  it '.name throws error',       -> expect( -> @instance.name  ).toThrow()
-  it '.stop throws error',       -> expect( -> @instance.stop  ).toThrow()
-
-  describe 'constructor', ->
-    it 'returns an instance',     -> expect( @instance             ).toEqual jasmine.any @klass
-    it 'sets the @config object', -> expect( @instance.config      ).toEqual jasmine.any Object
-    it 'sets a retry delay',      -> expect( @instance._retryDelay ).toEqual jasmine.any Number
-
-  describe '#connect', ->
-    it 'returns itself', -> expect( @instance.connect() ).toBe @instance
-
-    describe 'when given a 500 delay', ->
-      beforeEach ->
-        @clock = sinon.useFakeTimers()
-        sinon.stub @instance, '_request'
-        @instance.connect 500
-
-      afterEach -> @clock.restore()
-
-      it 'does not call _request before the given delay', ->
-        @clock.tick 499
-        expect( @instance._request.called ).toBe false
-
-      it 'calls _request after the given delay', ->
-        @clock.tick 600
-        expect( @instance._request.called ).toBe true