firehose 0.1.1 → 0.2.alpha.2

data/lib/firehose/rack/publisher_app.rb

@@ -0,0 +1,40 @@
+module Firehose
+  module Rack
+    class PublisherApp
+      include Firehose::Rack::Helpers
+
+      def call(env)
+        req     = env['parsed_request'] ||= ::Rack::Request.new(env)
+        path    = req.path
+        method  = req.request_method
+
+        if method == 'PUT'
+          EM.next_tick do
+            body = env['rack.input'].read
+            Firehose.logger.debug "HTTP published `#{body}` to `#{path}`"
+            publisher.publish(path, body).callback do
+              env['async.callback'].call [202, {'Content-Type' => 'text/plain', 'Content-Length' => '0'}, []]
+              env['async.callback'].call response(202, '', 'Content-Type' => 'text/plain')
+            end.errback do |e|
+              Firehose.logger.debug "Error publishing: #{e.inspect}"
+              env['async.callback'].call response(500, 'Error when trying to publish', 'Content-Type' => 'text/plain')
+            end
+          end
+
+          # Tell the web server that this will be an async response.
+          ASYNC_RESPONSE
+        else
+          Firehose.logger.debug "HTTP #{method} not supported"
+          msg = "#{method} not supported."
+          [501, {'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s}, [msg]]
+        end
+      end
+
+
+      private
+      def publisher
+        @publisher ||= Firehose::Publisher.new
+      end
+    end
+  end
+end

data/lib/firehose/server.rb

@@ -0,0 +1,48 @@
+require 'faye/websocket'
+
+module Firehose
+  class Server
+    def initialize(opts={})
+      @port   = opts[:port]   || Firehose::Default::URI.port
+      @host   = opts[:host]   || Firehose::Default::URI.host
+      @server = opts[:server] || :rainbows
+
+      Firehose.logger.info "Starting #{Firehose::VERSION} '#{Firehose::CODENAME}', in #{ENV['RACK_ENV']}"
+    end
+
+    def start
+      self.send("start_#{@server}")
+    end
+
+  private
+    def start_rainbows
+      require 'rainbows'
+      Faye::WebSocket.load_adapter('rainbows')
+
+      rackup = Unicorn::Configurator::RACKUP
+      rackup[:port] = @port if @port
+      rackup[:host] = @host if @host
+      rackup[:set_listener] = true
+      opts = rackup[:options]
+      opts[:config_file] = File.expand_path('../../../config/rainbows.rb', __FILE__)
+
+      server = Rainbows::HttpServer.new(Firehose::Rack::App.new, opts)
+      server.start.join
+    end
+
+    def start_thin
+      require 'thin'
+      require 'thin_em_swf_policy' if ENV['RACK_ENV'] == 'development'
+
+      Faye::WebSocket.load_adapter('thin')
+
+      # TODO: See if we can just set Thin to use Firehose.logger instead of
+      #       printing out messages by itself.
+      Thin::Logging.silent = true if Firehose.logger.level == Logger::ERROR
+
+      server = Thin::Server.new(@host, @port) do
+        run Firehose::Rack::App.new
+      end.start
+    end
+  end
+end

data/lib/firehose/subscriber.rb

@@ -0,0 +1,54 @@
+module Firehose
+  # Setups a connetion to Redis to listen for new resources...
+  class Subscriber
+    attr_reader :redis
+
+    def initialize(redis)
+      @redis = redis
+
+      # TODO: Instead of just raising an exception, it would probably be better
+      #       for the errback to set some sort of 'disconnected' state. Then
+      #       whenever a deferrable was 'subscribed' we could instantly fail
+      #       the deferrable with whatever connection error we had.
+      #       An alternative which would have a similar result would be to
+      #       subscribe lazily (i.e. not until we have a deferrable to subscribe).
+      #       Then, if connecting failed, it'd be super easy to fail the deferrable
+      #       with the same error.
+      #       The final goal is to allow the failed deferrable bubble back up
+      #       so we can send back a nice, clean 500 error to the client.
+      redis.subscribe('firehose:channel_updates').
+        errback{|e| EM.next_tick { raise e } }.
+        callback { Firehose.logger.debug "Redis subscribed to `firehose:channel_updates`" }
+      redis.on(:message) do |_, payload|
+        channel_key, sequence, message = Firehose::Publisher.from_payload(payload)
+
+        if deferrables = subscriptions.delete(channel_key)
+          Firehose.logger.debug "Redis notifying #{deferrables.count} deferrable(s) at `#{channel_key}` with sequence `#{sequence}` and message `#{message}`"
+          deferrables.each do |deferrable|
+            Firehose.logger.debug "Sending message #{message} and sequence #{sequence} to client from subscriber"
+            deferrable.succeed message, sequence.to_i
+          end
+        end
+      end
+    end
+
+    def subscribe(channel_key, deferrable)
+      subscriptions[channel_key].push deferrable
+    end
+
+    def unsubscribe(channel_key, deferrable)
+      subscriptions[channel_key].delete deferrable
+      subscriptions.delete(channel_key) if subscriptions[channel_key].empty?
+    end
+
+
+    private
+    def subscriptions
+      @subscriptions ||= Hash.new{|h,k| h[k] = []}
+    end
+
+    def key(*segments)
+      segments.unshift(:firehose).join(':')
+    end
+  end
+end

data/lib/firehose/swf_policy_request.rb

@@ -0,0 +1,23 @@
+module Firehose
+  module SwfPolicyRequest
+
+    # Borrowed from: https://github.com/igrigorik/em-websocket/blob/3e7f7d7760cc23b9d1d34fc1c17bab4423b5d11a/lib/em-websocket/connection.rb#L104
+    def handle_swf_policy_request(data)
+      if data =~ /\A<policy-file-request\s*\/>/
+        Firehose.logger.debug "Received SWF Policy request: #{data.inspect}"
+        send_data policy
+        close_connection_after_writing
+        true
+      end
+    end
+
+    def policy
+      <<-EOS
+<?xml version="1.0"?>
+<cross-domain-policy>
+  <allow-access-from domain="*" to-ports="*"/>
+</cross-domain-policy>
+      EOS
+    end
+  end
+end

data/lib/firehose/version.rb

@@ -1,4 +1,4 @@
 module Firehose
-  VERSION = "0.1.1"
-  CODENAME = "Rails Ready"
+  VERSION = "0.2.alpha.2"
+  CODENAME = "Tricked-out Tools"
 end

data/lib/rainbows_em_swf_policy.rb

@@ -0,0 +1,33 @@
+# This file file monkeypatches Rainbows! to return a proper SWF policy file.
+# Enable this with something like this in your config/rainbows.rb file:
+#
+#     after_fork do |server, worker|
+#       require 'rainbows_em_swf_policy'
+#     end if ENV['RACK_ENV'] == 'development'
+#
+# You should only use this in development. It has not been well tested in a
+# production environment.
+#
+# NOTE: This only works if you are using Rainbows! with EventMachine.
+#
+# Some helpful links:
+# http://unicorn.bogomips.org/Unicorn/Configurator.html
+# http://www.adobe.com/devnet/flashplayer/articles/socket_policy_files.html
+# http://blog.vokle.com/index.php/2009/06/10/dealing-with-adobe-and-serving-socket-policy-servers-via-nginx-and-10-lines-of-code/
+
+require 'rainbows'
+# Ensure the class already exists so we are overwriting it.
+Rainbows::EventMachine::Client
+
+class Rainbows::EventMachine::Client
+  include Firehose::SwfPolicyRequest
+  alias_method :receive_data_without_swf_policy, :receive_data
+  # Borrowed from: https://github.com/igrigorik/em-websocket/blob/3e7f7d7760cc23b9d1d34fc1c17bab4423b5d11a/lib/em-websocket/connection.rb#L104
+  def receive_data(data)
+    if handle_swf_policy_request(data)
+      return false
+    else
+      receive_data_without_swf_policy(data)
+    end
+  end
+end

data/lib/thin_em_swf_policy.rb

@@ -0,0 +1,26 @@
+# This file file monkeypatches Thin to return a proper SWF policy file.
+#
+# You should only use this in development. It has not been well tested in a
+# production environment.
+#
+# NOTE: This only works if you are using Thin with EventMachine.
+#
+# Some helpful links:
+# http://www.adobe.com/devnet/flashplayer/articles/socket_policy_files.html
+# http://blog.vokle.com/index.php/2009/06/10/dealing-with-adobe-and-serving-socket-policy-servers-via-nginx-and-10-lines-of-code/
+
+require 'thin'
+# Ensure the class already exists so we are overwriting it.
+Thin::Connection
+
+class Thin::Connection
+  include Firehose::SwfPolicyRequest
+  alias_method :receive_data_without_swf_policy, :receive_data
+  def receive_data(data)
+    if handle_swf_policy_request(data)
+      return false
+    else
+      receive_data_without_swf_policy(data)
+    end
+  end
+end

data/spec/integrations/integration_test_helper.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+module IntegrationTestHelper
+  def start_server
+    @server_pid = fork do
+      Firehose::Server.new(:server => server, :port => uri.port).start
+    end
+
+    # Need to give the server a chance to boot up.
+    sleep 1
+  end
+
+  def stop_server
+    Process.kill 'INT', @server_pid
+  end
+end

data/spec/integrations/rainbows_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+require 'integrations/shared_examples'
+
+describe "rainbows" do
+  let(:server) { :rainbows }
+  it_behaves_like 'Firehose::Rack::App'
+end

data/spec/integrations/shared_examples.rb

@@ -0,0 +1,111 @@
+require 'spec_helper'
+require 'integrations/integration_test_helper'
+
+shared_examples_for 'Firehose::Rack::App' do
+  include EM::TestHelper
+  include IntegrationTestHelper
+
+  before(:all) do
+    Firehose::Producer.adapter = :em_http
+    start_server
+  end
+
+  after(:all) do
+    Firehose::Producer.adapter = nil
+    stop_server
+  end
+
+  before(:each) { WebMock.disable! }
+  after(:each) { WebMock.enable! }
+
+  let(:app)       { Firehose::Rack::App.new }
+  let(:messages)  { (1..2000).map{|n| "msg-#{n}" } }
+  let(:channel)   { "/firehose/integration/#{Time.now.to_i}" }
+  let(:uri)       { Firehose::Default::URI }
+  let(:http_url)  { "http://#{uri.host}:#{uri.port}#{channel}" }
+  let(:ws_url)    { "ws://#{uri.host}:#{uri.port}#{channel}" }
+
+  it "should pub-sub http and websockets" do
+    # Setup variables that we'll use after we turn off EM to validate our
+    # test assertions.
+    outgoing, received = messages.dup, Hash.new{|h,k| h[k] = []}
+
+    # Our WS and Http clients call this when they have received their messages to determine
+    # when to turn off EM and make the test assertion at the very bottom.
+    succeed = Proc.new do
+      # TODO: For some weird reason the `add_timer` call causes up to 20 seconds of delay after
+      #       the test finishes running. However, without it the test will randomly fail with a
+      #       "Redis disconnected" error.
+      em.add_timer(1) { em.stop } if received.values.all?{|arr| arr.size == messages.size }
+    end
+
+    # Setup a publisher
+    publish = Proc.new do
+      Firehose::Producer.new.publish(outgoing.shift).to(channel) do
+        # The random timer ensures that sometimes the clients will be behind
+        # and sometimes they will be caught up.
+        EM::add_timer(rand*0.005) { publish.call } unless outgoing.empty?
+      end
+    end
+
+    # Lets have an HTTP Long poll client
+    http_long_poll = Proc.new do |cid, last_sequence|
+      http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => last_sequence})
+      http.errback { em.stop }
+      http.callback do
+        received[cid] << http.response
+        if received[cid].size < messages.size
+          # Add some jitter so the clients aren't syncronized
+          EM::add_timer(rand*0.001) { http_long_poll.call cid, http.response_header['pragma'] }
+        else
+          succeed.call cid
+        end
+      end
+    end
+
+    # And test a web socket client too, at the same time.
+    websocket = Proc.new do |cid|
+      ws = Faye::WebSocket::Client.new(ws_url)
+      ws.onmessage = lambda do |event|
+        received[cid] << event.data
+        succeed.call cid unless received[cid].size < messages.size
+      end
+
+      ws.onclose = lambda do |event|
+        ws = nil
+      end
+
+      ws.onerror  { |e| raise 'ws failed' + "\n" + e.inspect }
+    end
+
+    # Great, we have all the pieces in order, lets run this thing in the reactor.
+    em 60 do
+      # Start the clients.
+      websocket.call(1)
+      websocket.call(2)
+      http_long_poll.call(3)
+      http_long_poll.call(4)
+
+      # Wait a sec to let our clients set up.
+      em.add_timer(1){ publish.call }
+    end
+
+    # When EM stops, these assertions will be made.
+    received.size.should == 4
+    received.values.each do |arr|
+      arr.should == messages
+    end
+  end
+
+
+  it "should return 400 error for long-polling when using http long polling and sequence header is < 0" do
+    em 5 do
+      http = EM::HttpRequest.new(http_url).get(:query => {'last_message_sequence' => -1})
+      http.errback { |e| raise e.inspect }
+      http.callback do
+        http.response_header.status.should == 400
+        em.stop
+      end
+    end
+  end
+end

data/spec/integrations/thin_spec.rb

@@ -1,81 +1,7 @@
 require 'spec_helper'
+require 'integrations/shared_examples'
 
-describe Firehose::Rack do
-  include EM::TestHelper
-
-  before(:all) do
-    Firehose::Producer.adapter = :em_http
-  end
-
-  after(:all) do
-    Firehose::Producer.adapter = nil
-  end
-
-  let(:app)       { Firehose::Rack::App.new }
-  let(:messages)  { (1..2000).map(&:to_s) }
-  let(:channel)   { "/firehose/integration/#{Time.now.to_i}" }
-  let(:uri)       { Firehose::Default::URI }
-  let(:http_url)  { "http://#{uri.host}:#{uri.port}#{channel}" }
-  let(:ws_url)    { "ws://#{uri.host}:#{uri.port}#{channel}" }
-
-  it "should pub-sub http and websockets" do
-    # Setup variables that we'll use after we turn off EM to validate our
-    # test assertions.
-    outgoing, received_http, received_ws = messages.dup, [], []
-
-    # Our WS and Http clients call this when they have received their messages to determine
-    # when to turn off EM and make the test assertion at the very bottom.
-    succeed = Proc.new do
-      em.stop if received_http.size == messages.size and received_ws.size == messages.size
-    end
-
-    # Setup a publisher
-    publish = Proc.new do
-      Firehose::Producer.new.publish(outgoing.pop).to(channel) do
-        publish.call unless outgoing.empty?
-      end
-    end
-
-    # Lets have an HTTP Long poll client
-    http_long_poll = Proc.new do
-      http = EM::HttpRequest.new(http_url).get(:query => {'cid' => 'alpha'})
-      http.errback { em.stop }
-      http.callback do
-        received_http << http.response
-        if received_http.size < messages.size
-          http_long_poll.call
-        else
-          succeed.call
-        end
-      end
-    end
-
-    # And test a web socket client too, at the same time.
-    websocket = Proc.new do
-      ws = EventMachine::WebSocketClient.connect("#{ws_url}?cid=bravo")
-      ws.errback  { em.stop }
-      ws.stream do |msg|
-        received_ws << msg
-        succeed.call unless received_ws.size < messages.size
-      end
-    end
-
-    # Great, we have all the pieces in order, lets run this thing in the reactor.
-    em do
-      # Start the server
-      server = ::Thin::Server.new('0.0.0.0', uri.port, app)
-      server.start
-
-      # Start the http_long_pollr.
-      http_long_poll.call
-      websocket.call
-
-      # Wait a sec to let our http_long_poll setup.
-      em.add_timer(1){ publish.call }
-    end
-
-    # When EM stops, these assertions will be made.
-    received_ws.should    =~ messages
-    received_http.should  =~ messages
-  end
-end
+describe "thin" do
+  let(:server) { :thin }
+  it_behaves_like 'Firehose::Rack::App'
+end