firehose 1.1.1 → 1.2.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Yzc5MDg5MDIyZWJmYjY1ZTZjM2FiYTMzNDIxNjczYmYwZWUzNTI2Yg==
+  data.tar.gz: !binary |-
+    YjU0ODhhNDk3MWMwZTg2NTJkMDE3Y2M1MGE2YjY1OWQzN2ZkNjdkYg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    YTAwMDFmZDY3OWUxMjRhNDQ3ZDJkZmQzMmQyODZkZGFiZmUwMzQ0NWRkYjdm
+    OWFmMzVmOThmY2VmY2ZjNjJmMzViNzRiN2MyMjIzMDM0N2U0OGU0MDc0N2Iz
+    YjIwMDMyYWM3OTgzZDJmY2E1MDZmODZmY2QyYTVmYzEyYTExZmU=
+  data.tar.gz: !binary |-
+    M2ZlMTNiNGZjZTk1NDMyZDM4Y2EyY2M5MWFjYmEwY2E3YWQ1YmM2NTQ2OTU4
+    ZjEwNjgwZmQ1NjQyNDBmM2YyMWQwYzMxM2Y1MDc1NWQ1YzczM2M5ZTVmYWU3
+    YTk4ZjVmYTFhYTA3MDIwOGYzZjBkNjc3NDNmOWQ5OTA5NWFjNWM=

data/.rbenv-version

@@ -1 +1 @@
-1.9.3-p125
+1.9.3-p392

data/README.md

@@ -112,7 +112,7 @@ While you can certainly make your own PUT requests when publishing messages, Fir
 require 'firehose'
 require 'json'
 json = {'hello'=> 'world'}.to_json
-firehose = Firehose::Producer.new('//127.0.0.1:7474')
+firehose = Firehose::Client::Producer::Http.new('//127.0.0.1:7474')
 firehose.publish(json).to("/my/messages/path")
 ```
 

data/config/rainbows.rb

@@ -1,5 +1,6 @@
-# TODO - Dunno what a lot of this stuff is... Tune with benchmarks
-# http://rainbows.rubyforge.org/Rainbows/Configurator.html
+# Configuration from http://rainbows.rubyforge.org/Rainbows/Configurator.html. Don't juse
+# blindly copy and paste this configuration! Be sure you have read and understand the Rainbows
+# configuration documentation for your environment.
 
 Rainbows! do
   use                       :EventMachine   # concurrency model
@@ -16,5 +17,5 @@ working_directory ENV['WORKING_DIRECTORY'] if ENV['WORKING_DIRECTORY']
 logger Firehose.logger
 
 after_fork do |server, worker|
-  require 'rainbows_em_swf_policy'
+  require 'firehose/patches/rainbows'
 end if ENV['RACK_ENV'] == 'development'

data/lib/firehose.rb

@@ -1,24 +1,22 @@
 ENV['RACK_ENV'] ||= 'development' # TODO - Lets not rock out envs like its 1999.
 
+require 'uri'
 require 'firehose/version'
-require 'em-hiredis' # TODO Move this into a Redis module so that we can auto-load it. Lots of the CLI tools don't need this.
 require 'firehose/logging'
 
 # TODO - Figure if we need to have an if/else for Rails::Engine loading and Firehose::Assets::Sprockets.auto_detect
 require 'firehose/rails' if defined?(::Rails::Engine)
 
 module Firehose
-  autoload :Subscriber,   'firehose/subscriber'
-  autoload :Publisher,    'firehose/publisher'
-  autoload :Producer,     'firehose/producer' # TODO Move this into the Firehose::Client namespace.
-  autoload :Default,      'firehose/default'
+  autoload :Server,       'firehose/server'
+  autoload :Client,       'firehose/client'
   autoload :Assets,       'firehose/assets'
   autoload :Rack,         'firehose/rack'
   autoload :CLI,          'firehose/cli'
-  autoload :Client,       'firehose/client'
-  autoload :Server,       'firehose/server'
-  autoload :Channel,      'firehose/channel'
   autoload :SwfPolicyRequest, 'firehose/swf_policy_request'
+
+  # Default URI for the Firehose server. Consider the port "well-known" and bindable from other apps.
+  URI = URI.parse("//0.0.0.0:7474").freeze
 end
 
 # Detect if Sprockets is loaded. If it is, lets configure Firehose to use it!

data/lib/firehose/assets.rb

@@ -5,6 +5,7 @@ module Firehose
       File.join File.expand_path('../../assets', __FILE__), segs
     end
 
+    # Integrate Firehose ./lib/assets files into a sprocket-enabled environment.
     module Sprockets
       # Drop flash and javascript paths to Firehose assets into a sprockets environment.
       def self.configure(env)
@@ -22,11 +23,8 @@ module Firehose
 
       def self.manifest
         paths = []
-
         paths << Firehose::Assets.path('/flash/firehose/WebSocketMainInsecure.swf')
-
         paths << File.basename(Firehose::Assets.path('/javascripts/firehose/firehose.js.coffee'), '.coffee')
-
         paths
       end
     end

data/lib/firehose/cli.rb

@@ -2,7 +2,7 @@ require 'thor'
 require 'eventmachine'
 require 'uri'
 
-# Enable native 
+# Enable native async-io libs
 EM.kqueue if EM.kqueue?
 EM.epoll  if EM.epoll?
 
@@ -20,12 +20,12 @@ module Firehose
     end
 
     desc "server", "Start an instance of a server."
-    method_option :port,   :type => :numeric, :default => ENV['PORT'] || Firehose::Default::URI.port, :required => false, :aliases => '-p'
-    method_option :host,   :type => :string,  :default => ENV['HOST'] || Firehose::Default::URI.host, :required => false, :aliases => '-h'
+    method_option :port,   :type => :numeric, :default => ENV['PORT'] || Firehose::URI.port, :required => false, :aliases => '-p'
+    method_option :host,   :type => :string,  :default => ENV['HOST'] || Firehose::URI.host, :required => false, :aliases => '-h'
     method_option :server, :type => :string,  :default => ENV['SERVER'] ||'rainbows', :required => false, :aliases => '-s'
     def server
       begin
-        Firehose::Server.new(options).start
+        Firehose::Server::App.new(options).start
       rescue => e
         Firehose.logger.error "#{e.message}: #{e.backtrace}"
         raise e
@@ -44,11 +44,10 @@ module Firehose
     method_option :ttl, :type => :numeric, :aliases => '-t'
     method_option :times, :type => :numeric, :aliases => '-n', :default => 1
     method_option :interval, :type => :numeric, :aliases => '-i'
-
     def publish(uri, payload=nil)
       payload     ||= $stdin.read
-      client      = Firehose::Producer.new(uri)
-      path        = URI.parse(uri).path
+      client      = Firehose::Client::Producer::Http.new(uri)
+      path        = ::URI.parse(uri).path
       times       = options[:times]
       ttl         = options[:ttl]
 
@@ -70,6 +69,5 @@ module Firehose
         end
       end
     end
-
   end
 end

data/lib/firehose/client.rb

@@ -4,89 +4,7 @@ require 'faye/websocket'
 module Firehose
   # Ruby clients that connect to Firehose to either publish or consume messages.
   module Client
-    # TODO - Move the Firehose producer.rb file/class in here and rename to Firehose::Client::Producer::Http.new() ..
-    module Producer
-    end
-
-    # TODO - Test this libs. I had to throw these quickly into our app so that we could get 
-    #        some stress testing out of the way.
-    # TODO - Replace the integration test clients with these guys. You'll want to refactor each
-    #        transport to use on(:message), on(:conncect), and on(:disconnect) callbacks.
-    module Consumer
-      TransportNotSupportedError = Class.new(RuntimeError)
-
-      # Build up a benchmark client based on a given URI. Accepts ws:// and http:// for now.
-      def self.parse(uri)
-        case transport = URI.parse(uri).scheme
-        when 'ws'
-          Consumer::WebSocket.new(uri)
-        when 'http'
-          Consumer::HttpLongPoll.new(uri)
-        else
-          raise TransportNotSupportedError.new("Transport #{transport.inspect} not supported.")
-        end
-      end
-
-      # Connect to Firehose via WebSockets and consume messages.
-      class WebSocket
-        attr_reader :url, :logger
-
-        def initialize(url, logger = Firehose.logger)
-          @url, @logger = url, logger
-        end
-
-        def request
-          ws = Faye::WebSocket::Client.new(url)
-          ws.onmessage = lambda do |event|
-            logger.info "WS | #{event.data[0...40].inspect}"
-          end
-          ws.onclose = lambda do |event|
-            logger.info "WS | Closed"
-          end
-          ws.onerror do
-            logger.error "WS | Failed"
-          end
-        end
-      end
-
-      # Connect to Firehose via HTTP Long Polling and consume messages.
-      class HttpLongPoll
-        JITTER = 0.003
-
-        attr_reader :url, :logger
-
-        def initialize(url, logger = Firehose.logger)
-          @url, @logger = url, logger
-        end
-
-        def request(last_sequence=0)
-          http = EM::HttpRequest.new(url, :inactivity_timeout => 0).get(:query => {'last_message_sequence' => last_sequence})
-          http.callback do
-            case status = http.response_header.status
-            when 200
-              next_sequence = http.response_header['Pragma'].to_i
-              logger.info "HTTP 200 | Next Sequence: #{next_sequence} - #{http.response[0...40].inspect}"
-              EM::add_timer(jitter) { request next_sequence }
-            when 204
-              logger.info "HTTP 204 | Last Sequence #{last_sequence}"
-              EM::add_timer(jitter) { request last_sequence }
-            else
-              logger.error "HTTP #{status} | Failed"
-            end
-          end
-          http.errback do
-            logger.error "Connection Failed"
-          end
-        end
-
-
-        private
-        # Random jitter between long poll requests.
-        def jitter
-          rand*JITTER
-        end
-      end
-
-    end
+    autoload :Consumer, 'firehose/client/consumer'
+    autoload :Producer, 'firehose/client/producer'
   end
 end

data/lib/firehose/client/consumer.rb

@@ -0,0 +1,94 @@
+require 'json'
+
+# TODO - Spec this thing out. Unfortunately its not tested at all, mostly because the JSON client
+# is more important (and tested). Still, this should be tested, at least used by the specs
+# to test HTTP functionality of the server.
+module Firehose
+  module Client
+    module Consumer
+      # TODO - Test this libs. I had to throw these quickly into our app so that we could get 
+      #        some stress testing out of the way.
+      # TODO - Replace the integration test clients with these guys. You'll want to refactor each
+      #        transport to use on(:message), on(:conncect), and on(:disconnect) callbacks.
+      TransportNotSupportedError = Class.new(RuntimeError)
+
+      # Build up a benchmark client based on a given URI. Accepts ws:// and http:// for now.
+      def self.parse(uri)
+        case transport = ::URI.parse(uri).scheme
+        # TODO - Fix ws:// transport! See class WebSocket below to udnerstand
+        # why this doesn't work and support is dropped from the CLI.
+        # when 'ws'
+        #   Consumer::WebSocket.new(uri)
+        when 'http'
+          Consumer::HttpLongPoll.new(uri)
+        else
+          raise TransportNotSupportedError.new("Transport #{transport.inspect} not supported.")
+        end
+      end
+
+      # TODO - This won't even work. Dropping ws:// above until this is tested. This thing
+      # should be sending message sequences to Firehose.
+      # Connect to Firehose via WebSockets and consume messages.
+      class WebSocket
+        attr_reader :url, :logger
+
+        def initialize(url, logger = Firehose.logger)
+          @url, @logger = url, logger
+        end
+
+        def request
+          ws = Faye::WebSocket::Client.new(url)
+          ws.onmessage = lambda do |event|
+            logger.info "WS | #{event.data[0...40].inspect}"
+          end
+          ws.onclose = lambda do |event|
+            logger.info "WS | Closed"
+          end
+          ws.onerror do
+            logger.error "WS | Failed"
+          end
+        end
+      end
+
+      # Connect to Firehose via HTTP Long Polling and consume messages.
+      class HttpLongPoll
+        JITTER = 0.003
+
+        attr_reader :url, :logger
+
+        def initialize(url, logger = Firehose.logger)
+          @url, @logger = url, logger
+        end
+
+        def request(last_sequence=0)
+          http = EM::HttpRequest.new(url, :inactivity_timeout => 0).get(:query => {'last_message_sequence' => last_sequence})
+          http.callback do
+            case status = http.response_header.status
+            when 200
+              json = JSON.parse(http.response)
+              next_sequence = json['last_sequence'].to_i
+              message = json['message']
+
+              logger.info "HTTP 200 | Next Sequence: #{next_sequence} - #{message[0...40].inspect}"
+              EM::add_timer(jitter) { request next_sequence }
+            when 204
+              logger.info "HTTP 204 | Last Sequence #{last_sequence}"
+              EM::add_timer(jitter) { request last_sequence }
+            else
+              logger.error "HTTP #{status} | Failed"
+            end
+          end
+          http.errback do
+            logger.error "Connection Failed"
+          end
+        end
+
+      private
+        # Random jitter between long poll requests.
+        def jitter
+          rand*JITTER
+        end
+      end
+    end
+  end
+end

data/lib/firehose/client/producer.rb

@@ -0,0 +1,106 @@
+require "faraday"
+require "uri"
+
+module Firehose
+  module Client
+    module Producer
+      # Publish messages to Firehose via an HTTP interface.
+      class Http
+        # Exception gets raised when a 202 is _not_ received from the server after a message is published.
+        PublishError = Class.new(RuntimeError)
+        TimeoutError = Class.new(Faraday::Error::TimeoutError)
+        Timeout = 1 # How many seconds should we wait for a publish to take?
+
+        # A DSL for publishing requests. This doesn't so much, but lets us call
+        # Firehose::Client::Producer::Http#publish('message').to('channel'). Slick eh? If you don't like it,
+        # just all Firehose::Client::Producer::Http#put('message', 'channel')
+        class Builder
+          def initialize(producer, message)
+            @producer, @message = producer, message
+            self
+          end
+
+          def to(channel, opts={}, &callback)
+            @producer.put(@message, channel, opts, &callback)
+          end
+        end
+
+        # URI for the Firehose server. This URI does not include the path of the channel.
+        attr_reader :uri
+
+        def initialize(uri = Firehose::URI)
+          @uri = ::URI.parse(uri.to_s)
+          @uri.scheme ||= 'http'
+        end
+
+        # A DSL for publishing messages.
+        def publish(message)
+          Builder.new(self, message)
+        end
+
+        # Publish the message via HTTP.
+        def put(message, channel, opts, &block)
+          ttl = opts[:ttl]
+
+          response = conn.put do |req|
+            req.options[:timeout] = Timeout
+            if conn.path_prefix.nil? || conn.path_prefix == '/'
+              # This avoids a double / if the channel starts with a / too (which is expected).
+              req.path = channel
+            else
+              if conn.path_prefix =~ /\/\Z/ || channel =~ /\A\//
+                req.path = [conn.path_prefix, channel].compact.join
+              else
+                # Add a / so the prefix and channel aren't just rammed together.
+                req.path = [conn.path_prefix, channel].compact.join('/')
+              end
+            end
+            req.body = message
+            req.headers['Cache-Control'] = "max-age=#{ttl.to_i}" if ttl
+          end
+          response.on_complete do
+            case response.status
+            when 202 # Fire off the callback if everything worked out OK.
+              block.call(response) if block
+            else
+              error_handler.call PublishError.new("Could not publish #{message.inspect} to '#{uri.to_s}/#{channel}': #{response.inspect}")
+            end
+          end
+
+          # Hide Faraday with this Timeout exception, and through the error handler.
+          rescue Faraday::Error::TimeoutError => e
+            error_handler.call TimeoutError.new(e)
+        end
+
+        # Handle errors that could happen while publishing a message.
+        def on_error(&block)
+          @error_handler = block
+        end
+
+        # Raise an exception if an error occurs when connecting to the Firehose.
+        def error_handler
+          @error_handler || Proc.new{ |e| raise e }
+        end
+
+        # What adapter should Firehose use to PUT the message? List of adapters is
+        # available at https://github.com/technoweenie/faraday.
+        def self.adapter=(adapter)
+          @adapter = adapter
+        end
+
+        # Use :net_http for the default Faraday adapter.
+        def self.adapter
+          @adapter ||= Faraday.default_adapter
+        end
+
+        private
+        # Build out a Faraday connection
+        def conn
+          @conn ||= Faraday.new(:url => uri.to_s) do |builder|
+            builder.adapter self.class.adapter
+          end
+        end
+      end
+    end
+  end
+end

data/lib/firehose/logging.rb

@@ -1,8 +1,7 @@
-# Sets up logging
-
 require 'logger'
 
 module Firehose
+  # Sets up logging
   def self.logger
     @logger ||= Logger.new($stdout)
   end
@@ -28,8 +27,6 @@ module Firehose
   end
 end
 
-EM::Hiredis.logger = Firehose.logger
-
 # stdout gets "lost" in Foreman if this isn't here
 # https://github.com/ddollar/foreman/wiki/Missing-Output
 $stdout.sync = true if ENV['RACK_ENV'] == 'development' || ENV['SYNC_LOGGING']