firehose 1.1.1 → 1.2.0

data/lib/firehose/server/publisher.rb

@@ -0,0 +1,134 @@
+module Firehose
+  module Server
+    class Publisher
+      # Number of messages that Redis buffers for the client if its
+      # connection drops, then reconnects.
+      MAX_MESSAGES = 100
+
+      # Seconds that the message buffer should live before Redis expires it.
+      TTL = 60*60*24
+
+      # Delimited used to frame different parts of a message that's published
+      # over Firehose.
+      PAYLOAD_DELIMITER = "\n"
+
+      # Publish a message to a Firehose channel via Redis.
+      def publish(channel_key, message, opts={})
+        # How long should we hang on to the resource once is published?
+        ttl = (opts[:ttl] || TTL).to_i
+
+        # TODO hi-redis isn't that awesome... we have to setup an errback per even for wrong
+        # commands because of the lack of a method_missing whitelist. Perhaps implement a whitelist in
+        # em-hiredis or us a diff lib?
+        if (deferrable = opts[:deferrable]).nil?
+          deferrable = EM::DefaultDeferrable.new
+          deferrable.errback do |e|
+            # Handle missing Lua publishing script in cache
+            # (such as Redis restarting or someone executing SCRIPT FLUSH)
+            if e.message =~ /NOSCRIPT/
+              deferrable.succeed
+              EM.next_tick do
+                @publish_script_digest = nil
+                combined_opts = opts.merge :deferrable => deferrable
+                self.publish channel_key, message, combined_opts
+              end
+            else
+              EM.next_tick { raise e }
+            end
+          end
+        end
+
+        if @publish_script_digest.nil?
+          register_publish_script.errback do |e|
+            deferrable.fail e
+          end.callback do |digest|
+            @publish_script_digest = digest
+            Firehose.logger.debug "Registered Lua publishing script with Redis => #{digest}"
+            eval_publish_script channel_key, message, ttl, deferrable
+          end
+        else
+          eval_publish_script channel_key, message, ttl, deferrable
+        end
+
+        deferrable
+      end
+
+      private
+      def redis
+        @redis ||= EM::Hiredis.connect
+      end
+
+      # Serialize components of a message into something that can be dropped into Redis.
+      def self.to_payload(channel_key, sequence, message)
+        [channel_key, sequence, message].join(PAYLOAD_DELIMITER)
+      end
+
+      # Deserealize components of a message back into Ruby.
+      def self.from_payload(payload)
+        payload.split(PAYLOAD_DELIMITER, method(:to_payload).arity)
+      end
+
+      # TODO: Make this FAR more robust. Ideally we'd whitelist the permitted
+      #       characters and then escape or remove everything else.
+      #       See: http://en.wikibooks.org/wiki/Lua_Programming/How_to_Lua/escape_sequence
+      def lua_escape(str)
+        str.gsub(/\\/,'\\\\\\').gsub(/"/,'\"').gsub(/\n/,'\n').gsub(/\r/,'\r')
+      end
+
+      def register_publish_script
+        redis.script 'LOAD', REDIS_PUBLISH_SCRIPT
+      end
+
+      def eval_publish_script(channel_key, message, ttl, deferrable)
+        list_key = Server.key(channel_key, :list)
+        script_args = [
+          Server.key(channel_key, :sequence),
+          list_key,
+          Server.key(:channel_updates),
+          ttl,
+          message,
+          MAX_MESSAGES,
+          PAYLOAD_DELIMITER,
+          channel_key
+        ]
+        redis.evalsha(
+          @publish_script_digest, script_args.length, *script_args
+        ).errback do |e|
+          deferrable.fail e
+        end.callback do |sequence|
+          Firehose.logger.debug "Redis stored/published `#{message}` to list `#{list_key}` with sequence `#{sequence}`"
+          deferrable.succeed
+        end
+      end
+
+      REDIS_PUBLISH_SCRIPT = <<-LUA
+        local sequence_key      = KEYS[1]
+        local list_key          = KEYS[2]
+        local channel_key       = KEYS[3]
+        local ttl               = KEYS[4]
+        local message           = KEYS[5]
+        local max_messages      = KEYS[6]
+        local payload_delimiter = KEYS[7]
+        local firehose_resource = KEYS[8]
+
+        local current_sequence = redis.call('get', sequence_key)
+        if current_sequence == nil or current_sequence == false then
+          current_sequence = 0
+        end
+
+        local sequence = current_sequence + 1
+        local message_payload = firehose_resource .. payload_delimiter .. sequence .. payload_delimiter .. message
+
+        redis.call('set', sequence_key, sequence)
+        redis.call('expire', sequence_key, ttl)
+        redis.call('lpush', list_key, message)
+        redis.call('ltrim', list_key, 0, max_messages - 1)
+        redis.call('expire', list_key, ttl)
+        redis.call('publish', channel_key, message_payload)
+
+        return sequence
+      LUA
+
+    end
+  end
+end

data/lib/firehose/server/subscriber.rb

@@ -0,0 +1,50 @@
+module Firehose
+  module Server
+    # Setups a connetion to Redis to listen for new resources...
+    class Subscriber
+      attr_reader :pubsub
+
+      def initialize(redis)
+        @pubsub = redis.pubsub
+        # TODO: Instead of just raising an exception, it would probably be better
+        #       for the errback to set some sort of 'disconnected' state. Then
+        #       whenever a deferrable was 'subscribed' we could instantly fail
+        #       the deferrable with whatever connection error we had.
+        #       An alternative which would have a similar result would be to
+        #       subscribe lazily (i.e. not until we have a deferrable to subscribe).
+        #       Then, if connecting failed, it'd be super easy to fail the deferrable
+        #       with the same error.
+        #       The final goal is to allow the failed deferrable bubble back up
+        #       so we can send back a nice, clean 500 error to the client.
+        pubsub.subscribe('firehose:channel_updates').
+          errback{|e| EM.next_tick { raise e } }.
+          callback { Firehose.logger.debug "Redis subscribed to `firehose:channel_updates`" }
+        pubsub.on(:message) do |_, payload|
+          channel_key, sequence, message = Server::Publisher.from_payload(payload)
+
+          if deferrables = subscriptions.delete(channel_key)
+            Firehose.logger.debug "Redis notifying #{deferrables.count} deferrable(s) at `#{channel_key}` with sequence `#{sequence}` and message `#{message}`"
+            deferrables.each do |deferrable|
+              Firehose.logger.debug "Sending message #{message} and sequence #{sequence} to client from subscriber"
+              deferrable.succeed message, sequence.to_i
+            end
+          end
+        end
+      end
+
+      def subscribe(channel_key, deferrable)
+        subscriptions[channel_key].push deferrable
+      end
+
+      def unsubscribe(channel_key, deferrable)
+        subscriptions[channel_key].delete deferrable
+        subscriptions.delete(channel_key) if subscriptions[channel_key].empty?
+      end
+
+      private
+      def subscriptions
+        @subscriptions ||= Hash.new{|h,k| h[k] = []}
+      end
+    end
+  end
+end

data/lib/firehose/version.rb

@@ -1,4 +1,4 @@
 module Firehose
-  VERSION  = "1.1.1"
-  CODENAME = "Radtastical Redis"
+  VERSION  = "1.2.0"
+  CODENAME = "Spring Cleaning"
 end

data/spec/integrations/integration_test_helper.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 module IntegrationTestHelper
   def start_server
     @server_pid = fork do
-      Firehose::Server.new(:server => server, :port => uri.port).start
+      Firehose::Server::App.new(:server => server, :port => uri.port).start
     end
 
     # Need to give the server a chance to boot up.
@@ -17,6 +17,6 @@ module IntegrationTestHelper
 
   # Let isn't allowed in before(:all)
   def uri
-    Firehose::Default::URI
+    Firehose::URI
   end
 end

data/spec/integrations/shared_examples.rb

@@ -7,12 +7,12 @@ shared_examples_for 'Firehose::Rack::App' do
   include IntegrationTestHelper
 
   before(:all) do
-    Firehose::Producer.adapter = :em_http
+    Firehose::Client::Producer::Http.adapter = :em_http
     start_server
   end
 
   after(:all) do
-    Firehose::Producer.adapter = nil
+    Firehose::Client::Producer::Http.adapter = nil
     stop_server
   end
 
@@ -41,7 +41,7 @@ shared_examples_for 'Firehose::Rack::App' do
 
     # Setup a publisher
     publish = Proc.new do
-      Firehose::Producer.new.publish(outgoing.shift).to(channel) do
+      Firehose::Client::Producer::Http.new.publish(outgoing.shift).to(channel) do
         # The random timer ensures that sometimes the clients will be behind
         # and sometimes they will be caught up.
         EM::add_timer(rand*0.005) { publish.call } unless outgoing.empty?

data/spec/lib/{producer_spec.rb → client/producer_spec.rb}

@@ -1,34 +1,34 @@
 require 'spec_helper'
 
-describe Firehose::Producer do
+describe Firehose::Client::Producer::Http do
   let(:channel)       { "/channel-awesome" }
-  let(:url)           { "#{Firehose::Default::URI}#{channel}"}
+  let(:url)           { "#{Firehose::URI}#{channel}"}
   let(:publish_stub)  { stub_request(:put, url) }
   let(:message)       { "hey dude" }
 
   before(:all) do
-    Firehose::Producer.adapter = :net_http
+    Firehose::Client::Producer::Http.adapter = :net_http
   end
 
   after(:all) do
-    Firehose::Producer.adapter = nil
+    Firehose::Client::Producer::Http.adapter = nil
   end
 
   it "should publish message to channel" do
     publish_stub.to_return(:body => "", :status => 202)
 
-    Firehose::Producer.new.publish(message).to(channel)
+    Firehose::Client::Producer::Http.new.publish(message).to(channel)
     WebMock.should have_requested(:put, url).with { |req| req.body == message }
   end
 
   context 'prefix is specified in URI' do
-    let(:firehose_uri) {"#{Firehose::Default::URI}/prefix"}
+    let(:firehose_uri) {"#{Firehose::URI}/prefix"}
     let(:url) { "#{firehose_uri}#{channel}"}
 
     it "should publish message to channel" do
       publish_stub.to_return(:body => "", :status => 202)
 
-      Firehose::Producer.new(firehose_uri).publish(message).to(channel)
+      Firehose::Client::Producer::Http.new(firehose_uri).publish(message).to(channel)
       WebMock.should have_requested(:put, url).with { |req| req.body == message }
     end
   end
@@ -37,7 +37,7 @@ describe Firehose::Producer do
     publish_stub.to_return(:body => "", :status => 202)
     ttl = 20
 
-    Firehose::Producer.new.publish(message).to(channel, :ttl => ttl)
+    Firehose::Client::Producer::Http.new.publish(message).to(channel, :ttl => ttl)
     WebMock.should have_requested(:put, url).with { |req| req.body == message and req.headers['Cache-Control'] == "max-age=#{ttl}" }
   end
 
@@ -46,14 +46,14 @@ describe Firehose::Producer do
       publish_stub.to_return(:body => "", :status => 500)
 
       lambda{
-        Firehose::Producer.new.publish(message).to(channel)
-      }.should raise_exception(Firehose::Producer::PublishError)
+        Firehose::Client::Producer::Http.new.publish(message).to(channel)
+      }.should raise_exception(Firehose::Client::Producer::Http::PublishError)
     end
 
     it "should use .error_handler if not 201" do
       publish_stub.to_return(:body => "", :status => 500)
 
-      producer = Firehose::Producer.new
+      producer = Firehose::Client::Producer::Http.new
       producer.on_error do |e|
         e.message.should =~ /could not publish.+to/i
       end
@@ -64,8 +64,8 @@ describe Firehose::Producer do
       publish_stub.to_timeout
 
       lambda{
-        Firehose::Producer.new.publish(message).to(channel)
-      }.should raise_exception(Firehose::Producer::TimeoutError)
+        Firehose::Client::Producer::Http.new.publish(message).to(channel)
+      }.should raise_exception(Firehose::Client::Producer::Http::TimeoutError)
     end
   end
 end

data/spec/lib/firehose_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe Firehose do
+  it "should have 7474 for default port" do
+    Firehose::URI.to_s.should == '//0.0.0.0:7474'
+  end
+end

data/spec/lib/rack/{consumer_app_spec.rb → consumer_spec.rb}

@@ -1,9 +1,9 @@
 require 'spec_helper'
 require 'rack/test'
 
-describe Firehose::Rack::ConsumerApp, :type => :request do
+describe Firehose::Rack::Consumer, :type => :request do
   include Rack::Test::Methods
-  let(:app) { Firehose::Rack::ConsumerApp.new }
+  let(:app) { Firehose::Rack::Consumer.new }
   let(:path) { "/test/path/#{Time.now.to_i}" }
 
   it "should have Content-Length on OPTIONS request"

data/spec/lib/rack/{ping_app_spec.rb → ping_spec.rb}

@@ -2,15 +2,15 @@ require 'spec_helper'
 require 'rack/test'
 require 'async_rack_test'
 
-describe Firehose::Rack::PingApp, :type => :request do
+describe Firehose::Rack::Ping, :type => :request do
   include AsyncRackTest::Methods
-  let(:app) { Firehose::Rack::PingApp.new dummy_redis }
+  let(:app) { Firehose::Rack::Ping.new dummy_redis }
   let(:path) { "/test/path/#{Time.now.to_i}" }
   let(:deferrable) { EM::DefaultDeferrable.new }
   let(:dummy_redis) { double 'redis', :set => deferrable, :get => deferrable, :expire => deferrable }
 
   context 'redis is available' do
-    before { deferrable.succeed Firehose::Rack::PingApp::PingCheck::TEST_VALUE }
+    before { deferrable.succeed Firehose::Rack::Ping::PingCheck::TEST_VALUE }
 
     it "should return 200" do
       ahead path

data/spec/lib/rack/{publisher_app_spec.rb → publisher_spec.rb}

@@ -2,9 +2,9 @@ require 'spec_helper'
 require 'rack/test'
 require 'async_rack_test'
 
-describe Firehose::Rack::PublisherApp, :type => :request do
+describe Firehose::Rack::Publisher, :type => :request do
   include AsyncRackTest::Methods
-  let(:app) { Firehose::Rack::PublisherApp.new }
+  let(:app) { Firehose::Rack::Publisher.new }
   let(:path) { "/test/path/#{Time.now.to_i}" }
   let(:deferrable) { EM::DefaultDeferrable.new }
 
@@ -27,7 +27,7 @@ describe Firehose::Rack::PublisherApp, :type => :request do
       body = "howdy dude!"
       ttl = '92'
 
-      Firehose::Publisher.any_instance.stub(:publish).with(path, body, :ttl => ttl).and_return(deferrable)
+      Firehose::Server::Publisher.any_instance.stub(:publish).with(path, body, :ttl => ttl).and_return(deferrable)
       aput path, body, 'HTTP_CACHE_CONTROL' => 'max-age=92'
     end
   end

data/spec/lib/server/app_spec.rb

@@ -0,0 +1 @@
+# The Firehose::Server::App class is tested via the spec/integrations suite.

data/spec/lib/{channel_spec.rb → server/channel_spec.rb}

@@ -1,13 +1,13 @@
 require 'spec_helper'
 
-describe Firehose::Channel do
+describe Firehose::Server::Channel do
   include EM::TestHelper
 
   let(:channel_key)     { '/bears/are/mean' }
-  let(:channel)         { Firehose::Channel.new(channel_key, EM::Hiredis.connect, subscriber) }
-  let(:subscriber)      { Firehose::Subscriber.new(EM::Hiredis.connect) }
+  let(:channel)         { Firehose::Server::Channel.new(channel_key, EM::Hiredis.connect, subscriber) }
+  let(:subscriber)      { Firehose::Server::Subscriber.new(EM::Hiredis.connect) }
   let(:message)         { 'Raaaarrrrrr!!!!' }
-  let(:publisher)       { Firehose::Publisher.new }
+  let(:publisher)       { Firehose::Server::Publisher.new }
 
   describe "#next_message" do
     it "should wait for message if message was not published before subscription" do

data/spec/lib/{publisher_spec.rb → server/publisher_spec.rb}

@@ -1,18 +1,18 @@
 require 'spec_helper'
 
-describe Firehose::Publisher do
+describe Firehose::Server::Publisher do
   include EM::TestHelper
 
-  let(:publisher)   { Firehose::Publisher.new }
+  let(:publisher)   { Firehose::Server::Publisher.new }
   let(:channel_key) { "/firehose/publisher/test/#{Time.now.to_i}" }
   let(:message)     { "howdy friends!" }
 
   it "should have 100 MAX_MESSAGES" do
-    Firehose::Publisher::MAX_MESSAGES.should == 100
+    Firehose::Server::Publisher::MAX_MESSAGES.should == 100
   end
 
   it "should have 1 day TTL" do
-    Firehose::Publisher::TTL.should == 86400
+    Firehose::Server::Publisher::TTL.should == 86400
   end
 
   describe "#publish" do
@@ -24,7 +24,7 @@ describe Firehose::Publisher do
           msg.should == "#{channel_key}\n1\n#{message}"
           em.next_tick { em.stop }
         }
-        Firehose::Publisher.new.publish channel_key, message
+        Firehose::Server::Publisher.new.publish channel_key, message
       end
     end
 
@@ -32,7 +32,7 @@ describe Firehose::Publisher do
       it "should publish messages with the '#{char.inspect}' character" do
         msg = [char, message, char].join
         em 1 do
-          Firehose::Publisher.new.publish(channel_key, msg).callback { em.stop }
+          Firehose::Server::Publisher.new.publish(channel_key, msg).callback { em.stop }
         end
         redis_exec('lpop', "firehose:#{channel_key}:list").should == msg
       end
@@ -40,19 +40,19 @@ describe Firehose::Publisher do
 
     it "should add message to list" do
       em do
-        Firehose::Publisher.new.publish(channel_key, message).callback { em.stop }
+        Firehose::Server::Publisher.new.publish(channel_key, message).callback { em.stop }
       end
       redis_exec('lpop', "firehose:#{channel_key}:list").should == message
     end
 
     it "should limit list to MAX_MESSAGES messages" do
       em do
-        Firehose::Publisher::MAX_MESSAGES.times do |n|
+        Firehose::Server::Publisher::MAX_MESSAGES.times do |n|
           publisher.publish(channel_key, message)
         end
         publisher.publish(channel_key, message).callback { em.stop }
       end
-      redis_exec('llen', "firehose:#{channel_key}:list").should == Firehose::Publisher::MAX_MESSAGES
+      redis_exec('llen', "firehose:#{channel_key}:list").should == Firehose::Server::Publisher::MAX_MESSAGES
     end
 
     it "should increment sequence" do