firehose 0.1.1 → 0.2.alpha.2

data/lib/assets/javascripts/firehose/long_poll.js.coffee

@@ -1,4 +1,6 @@
 class Firehose.LongPoll extends Firehose.Transport
+  messageSequenceHeader: 'pragma'
+
   # CORS is supported in IE 8+
   @ieSupported: =>
     $.browser.msie and parseInt($.browser.version) > 7 and window.XDomainRequest
@@ -15,56 +17,43 @@ class Firehose.LongPoll extends Firehose.Transport
     # Protocol schema we should use for talking to WS server.
     @config.longPoll.url ||= "http:#{@config.uri}"
     # How many ms should we wait before timing out the AJAX connection?
-    @config.longPoll.timeout ||= 20000
+    @config.longPoll.timeout ||= 25000
 
     # TODO - What is @_lagTime for? Can't we just use the @_timeout value?
     # We use the lag time to make the client live longer than the server.
     @_lagTime = 5000
     @_timeout = @config.longPoll.timeout + @_lagTime
-    @_offlineTimer
     @_okInterval = 0
 
-    @registerIETransport()
-
-  registerIETransport: =>
-    if Firehose.LongPoll.ieSupported()
-      # TODO - Ask Steel what this is for. Looks like some kind of polygot fill, but I want
-      # to take the 'json' transport out and do that myself.
-      $.ajaxTransport 'json', (options, orignalOptions, jqXhr) ->
-        xdr = null
-        send: (_, callback) ->
-          xdr = new XDomainRequest()
-          xdr.onload = ->
-            statusCode = if xdr.responseText.length > 0 then 200 else 204
-            callback(statusCode, 'success', text: xdr.responseText)
-          
-          xdr.onerror = xdr.ontimeout = ->
-            callback(400, 'failed', text: xdr.responseText)
-
-          xdr.open(options.type, options.url)
-          xdr.send(options.data)
-        
-        abort: ->
-          if xdr
-            xdr.onerror = $.noop()
-            xdr.abort()
-
-      # also, override the support check
-      $.support.cors = true;
-    
+    @_isConnected = false
+   
   connect: (delay = 0) =>
-    @config.connected()
+    unless @_isConnected
+      @_isConnected = true
+      @config.connected()
     super(delay)
 
   _request: =>
+    # Set the Last Message Sequence in a query string.
+    # Ideally we'd use an HTTP header, but android devices don't let us
+    # set any HTTP headers for CORS requests.
+    data = @config.params
+    data.last_message_sequence = @_lastMessageSequence
+    # TODO: Some of these options will be deprecated in jQurey 1.8
+    #       See: http://api.jquery.com/jQuery.ajax/#jqXHR
     $.ajax @config.longPoll.url,
       crossDomain: true
-      cache: false
-      data: @config.params
+      data: data
       timeout: @_timeout
       success: @_success
       error: @_error
-  
+      complete: (jqXhr) =>
+        # Get the last sequence from the server if specified.
+        if jqXhr.status == 200
+          @_lastMessageSequence = jqXhr.getResponseHeader(@messageSequenceHeader)
+          if @_lastMessageSequence == null
+            console.log 'ERROR: Unable to get last message sequnce from header'
+
   _success: (data, status, jqXhr) =>
     # TODO we actually want to do this when the thing calls out... mmm right now it takes
     # up to 30s before we can call this thing.
@@ -76,16 +65,30 @@ class Firehose.LongPoll extends Firehose.Transport
       # 
       # Why did we use a 204 and not a 408? Because FireFox is really stupid about 400 level error
       # codes and would claims its a 0 error code, which we use for something else. Firefox is IE
-      # in thise case
+      # in this case
       @connect(@_okInterval)
     else
-      @config.message(@config.parse(data))
+      @config.message(@config.parse(jqXhr.responseText))
       @connect(@_okInterval)
 
+  _ping: =>
+    # Ping long poll server to verify internet connectivity
+    # jQuery CORS doesn't support timeouts and there is no way to access xhr2 object
+    # directly so we can't manually set a timeout.
+    $.ajax @config.longPoll.url,
+      method: 'HEAD'
+      crossDomain: true
+      data: @config.params
+      success: @config.connected
+
   # We need this custom handler to have the connection status
   # properly displayed
   _error: (jqXhr, status, error) =>
-    clearTimeout(@_offlineTimer)
+    @_isConnected = false
     @config.disconnected()
-    @_offlineTimer = setTimeout(@config.connected, @_retryDelay + @_lagTime)
-    @connect(@_retryDelay)
+    
+    # Ping the server to make sure this isn't a network connectivity error
+    setTimeout @_ping, @_retryDelay + @_lagTime
+
+    # Reconnect with delay
+    setTimeout @_request, @_retryDelay

data/lib/assets/javascripts/firehose/transport.js.coffee

@@ -32,7 +32,7 @@ class Firehose.Transport
   # Default connection established handler
   _open: (event) =>
     @_succeeded = true
-    @config.connected()
+    @config.connected(@)
 
   # Default connection closed handler
   _close: (event) =>

data/lib/assets/javascripts/firehose/web_socket.js.coffee

@@ -1,11 +1,13 @@
-class Firehose.WebSocket extends Firehose.Transport
-  @flashSupported: =>
-    $.browser.msie
+# Unfortunately this needs to a global variable. The only other option is to
+# hack into the internals of the web_socket.js plugin we are using.
+window.WEB_SOCKET_SWF_LOCATION = '/assets/firehose/WebSocketMain.swf' if !window.WEB_SOCKET_SWF_LOCATION
 
+class Firehose.WebSocket extends Firehose.Transport
   @supported: =>
     # Compatibility reference: http://caniuse.com/websockets
-    # Native websocket support + Flash web socket
-    !!(window.WebSocket || (window["MozWebSocket"] and window.MozWebSocket) || WebSocket.flashSupported())
+    # We don't need to explicitly check for Flash web socket or MozWebSocket
+    # because web_socket.js has already handled that.
+    !!(window.WebSocket)
 
   constructor: (args) ->
     super args
@@ -14,14 +16,6 @@ class Firehose.WebSocket extends Firehose.Transport
     @config.webSocket ||= {}
     # Protocol schema we should use for talking to WS server.
     @config.webSocket.url ||= "ws:#{@config.uri}?#{$.param(@config.params)}"
-    # Path of the swf WebSocket that we use in non-WS flash browsers.
-    @config.webSocket.swf_path ||= "/flash/firehose/WebSocketMain.swf"
-
-    # Set flash socket path for the WS SWF polyfill.
-    WebSocket.__swfLocation = @config.webSocket.swf_path
-
-    # Mozilla decided to have their own implementation of Web Sockets so detect for that.
-    window.WebSocket = window.MozWebSocket if window["MozWebSocket"] and window.MozWebSocket
 
   _request: =>
     @socket = new window.WebSocket(@config.webSocket.url)
@@ -49,4 +43,4 @@ class Firehose.WebSocket extends Firehose.Transport
       @socket.close()
       delete(@socket)
     
-    super
+    super

data/lib/firehose.rb

@@ -1,24 +1,19 @@
-require 'firehose/version'
-
-require 'em-hiredis'
-require 'logger'
+ENV['RACK_ENV'] ||= 'development'
 
-require 'firehose/rails' if defined?(::Rails::Engine)
+require 'firehose/version'
+require 'em-hiredis' # TODO Move this into a Redis module so that we can auto-load it. Lots of the CLI tools don't need this.
+require 'firehose/logging'
+require 'firehose/rails' if defined?(::Rails::Engine) # TODO Detect Sprockets instead of the jankin Rails::Engine test.
 
 module Firehose
-  autoload :Subscription, 'firehose/subscription'
+  autoload :Subscriber,   'firehose/subscriber'
   autoload :Publisher,    'firehose/publisher'
-  autoload :Producer,     'firehose/producer'
+  autoload :Producer,     'firehose/producer' # TODO Move this into the Firehose::Client namespace.
   autoload :Default,      'firehose/default'
   autoload :Rack,         'firehose/rack'
   autoload :CLI,          'firehose/cli'
-  
-  # Logging
-  def self.logger
-    @logger ||= Logger.new($stdout)
-  end
-
-  def self.logger=(logger)
-    @logger = logger
-  end
-end
+  autoload :Client,       'firehose/client'
+  autoload :Server,       'firehose/server'
+  autoload :Channel,      'firehose/channel'
+  autoload :SwfPolicyRequest, 'firehose/swf_policy_request'
+end

data/lib/firehose/channel.rb

@@ -0,0 +1,84 @@
+module Firehose
+  class Channel
+    attr_reader :channel_key, :redis, :subscriber, :list_key, :sequence_key
+
+    def self.redis
+      @redis ||= EM::Hiredis.connect
+    end
+
+    def self.subscriber
+      @subscriber ||= Subscriber.new(EM::Hiredis.connect)
+    end
+
+
+    def initialize(channel_key, redis=self.class.redis, subscriber=self.class.subscriber)
+      @channel_key, @redis, @subscriber = channel_key, redis, subscriber
+
+      @list_key, @sequence_key = key(channel_key, :list), key(channel_key, :sequence)
+    end
+
+    def next_message(last_sequence=nil, options={})
+      last_sequence = last_sequence.to_i
+
+      deferrable = EM::DefaultDeferrable.new
+      # TODO - Think this through a little harder... maybe some tests ol buddy!
+      deferrable.errback {|e| EM.next_tick { raise e } unless [:timeout, :disconnect].include?(e) }
+
+
+      # TODO: Use HSET so we don't have to pull 100 messages back every time.
+      redis.multi
+        redis.get(sequence_key).
+          errback {|e| deferrable.fail e }
+        redis.lrange(list_key, 0, Firehose::Publisher::MAX_MESSAGES).
+          errback {|e| deferrable.fail e }
+      redis.exec.callback do |(sequence, message_list)|
+        Firehose.logger.debug "exec returened: `#{sequence}` and `#{message_list.inspect}`"
+        sequence = sequence.to_i
+
+        if sequence.nil? || (diff = sequence - last_sequence) <= 0
+          Firehose.logger.debug "No message available yet, subscribing. sequence: `#{sequence}`"
+          # Either this resource has never been seen before or we are all caught up.
+          # Subscribe and hope something gets published to this end-point.
+          subscribe(deferrable, options[:timeout])
+        elsif last_sequence > 0 && diff < Firehose::Publisher::MAX_MESSAGES
+          # The client is kinda-sorta running behind, but has a chance to catch
+          # up. Catch them up FTW.
+          # But we won't "catch them up" if last_sequence was zero/nil because
+          # that implies the client is connecting for the 1st time.
+          message = message_list[diff-1]
+          Firehose.logger.debug "Sending old message `#{message}` and sequence `#{sequence}` to client directly. Client is `#{diff}` behind, at `#{last_sequence}`."
+          deferrable.succeed message, last_sequence + 1
+        else
+          # The client is hopelessly behind and underwater. Just reset
+          # their whole world with the lastest message.
+          message = message_list[0]
+          Firehose.logger.debug "Sending latest message `#{message}` and sequence `#{sequence}` to client directly."
+          deferrable.succeed message, sequence
+        end
+      end.errback {|e| deferrable.fail e }
+
+      deferrable
+    end
+
+    def unsubscribe(deferrable)
+      subscriber.unsubscribe channel_key, deferrable
+    end
+
+    private
+    def key(*segments)
+      segments.unshift(:firehose).join(':')
+    end
+
+    def subscribe(deferrable, timeout=nil)
+      subscriber.subscribe(channel_key, deferrable)
+      if timeout
+        timer = EventMachine::Timer.new(timeout) do
+          deferrable.fail :timeout
+          unsubscribe deferrable
+        end
+        # Cancel the timer if when the deferrable succeeds
+        deferrable.callback { timer.cancel }
+      end
+    end
+  end
+end

data/lib/firehose/cli.rb

@@ -1,28 +1,72 @@
 require 'thor'
-require 'thin'
+require 'eventmachine'
+require 'uri'
+
+# Enable native 
+EM.kqueue if EM.kqueue?
+EM.epoll  if EM.epoll?
 
 module Firehose
   class CLI < Thor
-    desc "version", "display the current version"
+    def initialize(*args)
+      super
+      # Disable buffering to $stdio for Firehose.logger
+      $stdout.sync = true
+    end
+
+    desc "version", "Display the current version."
     def version
       puts %[Firehose #{Firehose::VERSION} "#{Firehose::CODENAME}"]
     end
 
-    desc "server", "starts the firehose server"
-    method_option :port, :type => :numeric, :default => Firehose::Default::URI.port, :required => true, :aliases => '-p'
-    method_option :host, :type => :string, :default => '0.0.0.0', :required => true, :aliases => '-h'
-
+    desc "server", "Start an instance of a server."
+    method_option :port,   :type => :numeric, :default => ENV['PORT'] || Firehose::Default::URI.port, :required => false, :aliases => '-p'
+    method_option :host,   :type => :string,  :default => ENV['HOST'] || Firehose::Default::URI.host, :required => false, :aliases => '-h'
+    method_option :server, :type => :string,  :default => ENV['SERVER'] ||'rainbows', :required => false, :aliases => '-s'
     def server
-      server = Thin::Server.new(options[:host], options[:port]) do
-        run Firehose::Rack::App.new
-      end
-
       begin
-        server.start!
-      rescue
+        Firehose::Server.new(options).start
+      rescue => e
         Firehose.logger.error "#{e.message}: #{e.backtrace}"
-        raise
+        raise e
+      end
+    end
+
+    desc "consume URI", "Consume messages from a resource."
+    method_option :concurrency, :type => :numeric, :default => 1, :aliases => '-c'
+    def consume(uri)
+      EM.run do
+        options[:concurrency].times { Firehose::Client::Consumer.parse(uri).request }
       end
     end
+
+    desc "publish URI [PAYLOAD]", "Publish messages to a resource."
+    method_option :interval, :type => :numeric, :aliases => '-i'
+    method_option :times, :type => :numeric, :aliases => '-n'
+    def publish(uri, payload=nil)
+      payload     ||= $stdin.read
+      client      = Firehose::Producer.new(uri)
+      path        = URI.parse(uri).path
+      times       = options[:times]
+
+      EM.run do
+        # TODO I think this can be cleaned up so the top-level if/else can be ditched.
+        if interval = options[:interval]
+          # Publish messages at a forced interval.
+          EM.add_periodic_timer interval do 
+            client.publish(payload).to(path)
+            EM.stop if times && (times-=1).zero?
+          end
+        else
+          # Publish messages as soon as the last message was published.
+          worker = Proc.new do
+            client.publish(payload).to(path)
+            times && (times-=1).zero? ? EM.stop : worker.call
+          end
+          worker.call
+        end
+      end
+    end
+
   end
 end

data/lib/firehose/client.rb

@@ -0,0 +1,92 @@
+require 'em-http'
+require 'faye/websocket'
+
+module Firehose
+  # Ruby clients that connect to Firehose to either publish or consume messages.
+  module Client
+    # TODO - Move the Firehose producer.rb file/class in here and rename to Firehose::Client::Producer::Http.new() ..
+    module Producer
+    end
+
+    # TODO - Test this libs. I had to throw these quickly into our app so that we could get 
+    #        some stress testing out of the way.
+    # TODO - Replace the integration test clients with these guys. You'll want to refactor each
+    #        transport to use on(:message), on(:conncect), and on(:disconnect) callbacks.
+    module Consumer
+      TransportNotSupportedError = Class.new(RuntimeError)
+
+      # Build up a benchmark client based on a given URI. Accepts ws:// and http:// for now.
+      def self.parse(uri)
+        case transport = URI.parse(uri).scheme
+        when 'ws'
+          Consumer::WebSocket.new(uri)
+        when 'http'
+          Consumer::HttpLongPoll.new(uri)
+        else
+          raise TransportNotSupportedError.new("Transport #{transport.inspect} not supported.")
+        end
+      end
+
+      # Connect to Firehose via WebSockets and consume messages.
+      class WebSocket
+        attr_reader :url, :logger
+
+        def initialize(url, logger = Firehose.logger)
+          @url, @logger = url, logger
+        end
+
+        def request
+          ws = Faye::WebSocket::Client.new(url)
+          ws.onmessage = lambda do |event|
+            logger.info "WS | #{event.data[0...40].inspect}"
+          end
+          ws.onclose = lambda do |event|
+            logger.info "WS | Closed"
+          end
+          ws.onerror do
+            logger.error "WS | Failed"
+          end
+        end
+      end
+
+      # Connect to Firehose via HTTP Long Polling and consume messages.
+      class HttpLongPoll
+        JITTER = 0.003
+
+        attr_reader :url, :logger
+
+        def initialize(url, logger = Firehose.logger)
+          @url, @logger = url, logger
+        end
+
+        def request(last_sequence=0)
+          http = EM::HttpRequest.new(url, :inactivity_timeout => 0).get(:query => {'last_message_sequence' => last_sequence})
+          http.callback do
+            case status = http.response_header.status
+            when 200
+              next_sequence = http.response_header['Pragma'].to_i
+              logger.info "HTTP 200 | Next Sequence: #{next_sequence} - #{http.response[0...40].inspect}"
+              EM::add_timer(jitter) { request next_sequence }
+            when 204
+              logger.info "HTTP 204 | Last Sequence #{last_sequence}"
+              EM::add_timer(jitter) { request last_sequence }
+            else
+              logger.error "HTTP #{status} | Failed"
+            end
+          end
+          http.errback do
+            logger.error "Connection Failed"
+          end
+        end
+
+
+        private
+        # Random jitter between long poll requests.
+        def jitter
+          rand*JITTER
+        end
+      end
+
+    end
+  end
+end