firehose 1.1.1 → 1.2.0

data/lib/firehose/channel.rb

@@ -1,84 +0,0 @@
-module Firehose
-  class Channel
-    attr_reader :channel_key, :redis, :subscriber, :list_key, :sequence_key
-
-    def self.redis
-      @redis ||= EM::Hiredis.connect
-    end
-
-    def self.subscriber
-      @subscriber ||= Subscriber.new(EM::Hiredis.connect)
-    end
-
-
-    def initialize(channel_key, redis=self.class.redis, subscriber=self.class.subscriber)
-      @channel_key, @redis, @subscriber = channel_key, redis, subscriber
-
-      @list_key, @sequence_key = key(channel_key, :list), key(channel_key, :sequence)
-    end
-
-    def next_message(last_sequence=nil, options={})
-      last_sequence = last_sequence.to_i
-
-      deferrable = EM::DefaultDeferrable.new
-      # TODO - Think this through a little harder... maybe some tests ol buddy!
-      deferrable.errback {|e| EM.next_tick { raise e } unless [:timeout, :disconnect].include?(e) }
-
-
-      # TODO: Use HSET so we don't have to pull 100 messages back every time.
-      redis.multi
-        redis.get(sequence_key).
-          errback {|e| deferrable.fail e }
-        redis.lrange(list_key, 0, Firehose::Publisher::MAX_MESSAGES).
-          errback {|e| deferrable.fail e }
-      redis.exec.callback do |(sequence, message_list)|
-        Firehose.logger.debug "exec returened: `#{sequence}` and `#{message_list.inspect}`"
-        sequence = sequence.to_i
-
-        if sequence.nil? || (diff = sequence - last_sequence) <= 0
-          Firehose.logger.debug "No message available yet, subscribing. sequence: `#{sequence}`"
-          # Either this resource has never been seen before or we are all caught up.
-          # Subscribe and hope something gets published to this end-point.
-          subscribe(deferrable, options[:timeout])
-        elsif last_sequence > 0 && diff < Firehose::Publisher::MAX_MESSAGES
-          # The client is kinda-sorta running behind, but has a chance to catch
-          # up. Catch them up FTW.
-          # But we won't "catch them up" if last_sequence was zero/nil because
-          # that implies the client is connecting for the 1st time.
-          message = message_list[diff-1]
-          Firehose.logger.debug "Sending old message `#{message}` and sequence `#{sequence}` to client directly. Client is `#{diff}` behind, at `#{last_sequence}`."
-          deferrable.succeed message, last_sequence + 1
-        else
-          # The client is hopelessly behind and underwater. Just reset
-          # their whole world with the lastest message.
-          message = message_list[0]
-          Firehose.logger.debug "Sending latest message `#{message}` and sequence `#{sequence}` to client directly."
-          deferrable.succeed message, sequence
-        end
-      end.errback {|e| deferrable.fail e }
-
-      deferrable
-    end
-
-    def unsubscribe(deferrable)
-      subscriber.unsubscribe channel_key, deferrable
-    end
-
-    private
-    def key(*segments)
-      segments.unshift(:firehose).join(':')
-    end
-
-    def subscribe(deferrable, timeout=nil)
-      subscriber.subscribe(channel_key, deferrable)
-      if timeout
-        timer = EventMachine::Timer.new(timeout) do
-          deferrable.fail :timeout
-          unsubscribe deferrable
-        end
-        # Cancel the timer if when the deferrable succeeds
-        deferrable.callback { timer.cancel }
-      end
-    end
-  end
-end

data/lib/firehose/default.rb

@@ -1,8 +0,0 @@
-require 'uri'
-
-module Firehose
-  module Default
-    # Default URI for the Firehose server. Consider the port "well-known" and bindable from other apps.
-    URI = URI.parse("//0.0.0.0:7474").freeze
-  end
-end

data/lib/firehose/producer.rb

@@ -1,104 +0,0 @@
-# TODO Move this into the Firehose::Client:Producer namespace and rename the class to Http (its an HTTP publisher dumby!)
-require "faraday"
-require "uri"
-
-module Firehose
-  # Publish messages to Firehose via an HTTP interface.
-  class Producer
-
-    # Exception gets raised when a 202 is _not_ received from the server after a message is published.
-    PublishError = Class.new(RuntimeError)
-    TimeoutError = Class.new(Faraday::Error::TimeoutError)
-    Timeout = 1 # How many seconds should we wait for a publish to take?
-
-    # A DSL for publishing requests. This doesn't so much, but lets us call
-    # Firehose::Producer#publish('message').to('channel'). Slick eh? If you don't like it,
-    # just all Firehose::Producer#put('message', 'channel')
-    class Builder
-      def initialize(producer, message)
-        @producer, @message = producer, message
-        self
-      end
-
-      def to(channel, opts={}, &callback)
-        @producer.put(@message, channel, opts, &callback)
-      end
-    end
-
-    # URI for the Firehose server. This URI does not include the path of the channel.
-    attr_reader :uri
-
-    def initialize(uri = Firehose::Default::URI)
-      @uri = URI.parse(uri.to_s)
-      @uri.scheme ||= 'http'
-    end
-
-    # A DSL for publishing messages.
-    def publish(message)
-      Builder.new(self, message)
-    end
-
-    # Publish the message via HTTP.
-    def put(message, channel, opts, &block)
-      ttl = opts[:ttl]
-
-      response = conn.put do |req|
-        req.options[:timeout] = Timeout
-        if conn.path_prefix.nil? || conn.path_prefix == '/'
-          # This avoids a double / if the channel starts with a / too (which is expected).
-          req.path = channel
-        else
-          if conn.path_prefix =~ /\/\Z/ || channel =~ /\A\//
-            req.path = [conn.path_prefix, channel].compact.join
-          else
-            # Add a / so the prefix and channel aren't just rammed together.
-            req.path = [conn.path_prefix, channel].compact.join('/')
-          end
-        end
-        req.body = message
-        req.headers['Cache-Control'] = "max-age=#{ttl.to_i}" if ttl
-      end
-      response.on_complete do
-        case response.status
-        when 202 # Fire off the callback if everything worked out OK.
-          block.call(response) if block
-        else
-          error_handler.call PublishError.new("Could not publish #{message.inspect} to '#{uri.to_s}/#{channel}': #{response.inspect}")
-        end
-      end
-
-      # Hide Faraday with this Timeout exception, and through the error handler.
-      rescue Faraday::Error::TimeoutError => e
-        error_handler.call TimeoutError.new(e)
-    end
-
-    # Handle errors that could happen while publishing a message.
-    def on_error(&block)
-      @error_handler = block
-    end
-
-    # Raise an exception if an error occurs when connecting to the Firehose.
-    def error_handler
-      @error_handler || Proc.new{ |e| raise e }
-    end
-
-    # What adapter should Firehose use to PUT the message? List of adapters is
-    # available at https://github.com/technoweenie/faraday.
-    def self.adapter=(adapter)
-      @adapter = adapter
-    end
-
-    # Use :net_http for the default Faraday adapter.
-    def self.adapter
-      @adapter ||= Faraday.default_adapter
-    end
-
-  private
-    # Build out a Faraday connection
-    def conn
-      @conn ||= Faraday.new(:url => uri.to_s) do |builder|
-        builder.adapter self.class.adapter
-      end
-    end
-  end
-end

data/lib/firehose/publisher.rb

@@ -1,127 +0,0 @@
-module Firehose
-  class Publisher
-    
-    MAX_MESSAGES = 100
-    TTL = 60*60*24  # 1 day of time, yay!
-    PAYLOAD_DELIMITER = "\n"
-
-    def publish(channel_key, message, opts={})
-      # How long should we hang on to the resource once is published?
-      ttl = (opts[:ttl] || TTL).to_i
-
-      # TODO hi-redis isn't that awesome... we have to setup an errback per even for wrong
-      # commands because of the lack of a method_missing whitelist. Perhaps implement a whitelist in
-      # em-hiredis or us a diff lib?
-      if (deferrable = opts[:deferrable]).nil?
-        deferrable = EM::DefaultDeferrable.new
-        deferrable.errback do |e|
-          # Handle missing Lua publishing script in cache
-          # (such as Redis restarting or someone executing SCRIPT FLUSH)
-          if e.message =~ /NOSCRIPT/
-            deferrable.succeed
-            EM.next_tick do
-              @publish_script_digest = nil
-              combined_opts = opts.merge :deferrable => deferrable
-              self.publish channel_key, message, combined_opts
-            end
-          else
-            EM.next_tick { raise e }
-          end
-        end
-      end
-
-      if @publish_script_digest.nil?
-        register_publish_script.errback do |e|
-          deferrable.fail e
-        end.callback do |digest|
-          @publish_script_digest = digest
-          Firehose.logger.debug "Registered Lua publishing script with Redis => #{digest}"
-          eval_publish_script channel_key, message, ttl, deferrable
-        end
-      else
-        eval_publish_script channel_key, message, ttl, deferrable
-      end
-
-      deferrable
-    end
-
-  private
-    def key(*segments)
-      segments.unshift(:firehose).join(':')
-    end
-
-    def redis
-      @redis ||= EM::Hiredis.connect
-    end
-
-    def self.to_payload(channel_key, sequence, message)
-      [channel_key, sequence, message].join(PAYLOAD_DELIMITER)
-    end
-
-    def self.from_payload(payload)
-      payload.split(PAYLOAD_DELIMITER, method(:to_payload).arity)
-    end
-
-    # TODO: Make this FAR more robust. Ideally we'd whitelist the permitted
-    #       characters and then escape or remove everything else.
-    #       See: http://en.wikibooks.org/wiki/Lua_Programming/How_to_Lua/escape_sequence
-    def lua_escape(str)
-      str.gsub(/\\/,'\\\\\\').gsub(/"/,'\"').gsub(/\n/,'\n').gsub(/\r/,'\r')
-    end
-
-    def register_publish_script
-      redis.script 'LOAD', REDIS_PUBLISH_SCRIPT
-    end
-
-    def eval_publish_script(channel_key, message, ttl, deferrable)
-      list_key = key(channel_key, :list)
-      script_args = [
-        key(channel_key, :sequence),
-        list_key,
-        key(:channel_updates),
-        ttl,
-        message,
-        MAX_MESSAGES,
-        PAYLOAD_DELIMITER,
-        channel_key
-      ]
-      redis.evalsha(
-        @publish_script_digest, script_args.length, *script_args
-      ).errback do |e|
-        deferrable.fail e
-      end.callback do |sequence|
-        Firehose.logger.debug "Redis stored/published `#{message}` to list `#{list_key}` with sequence `#{sequence}`"
-        deferrable.succeed
-      end
-    end
-
-    REDIS_PUBLISH_SCRIPT = <<-LUA
-      local sequence_key      = KEYS[1]
-      local list_key          = KEYS[2]
-      local channel_key       = KEYS[3]
-      local ttl               = KEYS[4]
-      local message           = KEYS[5]
-      local max_messages      = KEYS[6]
-      local payload_delimiter = KEYS[7]
-      local firehose_resource = KEYS[8]
-
-      local current_sequence = redis.call('get', sequence_key)
-      if current_sequence == nil or current_sequence == false then
-        current_sequence = 0
-      end
-
-      local sequence = current_sequence + 1
-      local message_payload = firehose_resource .. payload_delimiter .. sequence .. payload_delimiter .. message
-
-      redis.call('set', sequence_key, sequence)
-      redis.call('expire', sequence_key, ttl)
-      redis.call('lpush', list_key, message)
-      redis.call('ltrim', list_key, 0, max_messages - 1)
-      redis.call('expire', list_key, ttl)
-      redis.call('publish', channel_key, message_payload)
-
-      return sequence
-    LUA
-
-  end
-end

data/lib/firehose/subscriber.rb

@@ -1,54 +0,0 @@
-module Firehose
-  # Setups a connetion to Redis to listen for new resources...
-  class Subscriber
-    attr_reader :pubsub
-
-    def initialize(redis)
-      @pubsub = redis.pubsub
-
-      # TODO: Instead of just raising an exception, it would probably be better
-      #       for the errback to set some sort of 'disconnected' state. Then
-      #       whenever a deferrable was 'subscribed' we could instantly fail
-      #       the deferrable with whatever connection error we had.
-      #       An alternative which would have a similar result would be to
-      #       subscribe lazily (i.e. not until we have a deferrable to subscribe).
-      #       Then, if connecting failed, it'd be super easy to fail the deferrable
-      #       with the same error.
-      #       The final goal is to allow the failed deferrable bubble back up
-      #       so we can send back a nice, clean 500 error to the client.
-      pubsub.subscribe('firehose:channel_updates').
-        errback{|e| EM.next_tick { raise e } }.
-        callback { Firehose.logger.debug "Redis subscribed to `firehose:channel_updates`" }
-      pubsub.on(:message) do |_, payload|
-        channel_key, sequence, message = Firehose::Publisher.from_payload(payload)
-
-        if deferrables = subscriptions.delete(channel_key)
-          Firehose.logger.debug "Redis notifying #{deferrables.count} deferrable(s) at `#{channel_key}` with sequence `#{sequence}` and message `#{message}`"
-          deferrables.each do |deferrable|
-            Firehose.logger.debug "Sending message #{message} and sequence #{sequence} to client from subscriber"
-            deferrable.succeed message, sequence.to_i
-          end
-        end
-      end
-    end
-
-    def subscribe(channel_key, deferrable)
-      subscriptions[channel_key].push deferrable
-    end
-
-    def unsubscribe(channel_key, deferrable)
-      subscriptions[channel_key].delete deferrable
-      subscriptions.delete(channel_key) if subscriptions[channel_key].empty?
-    end
-
-
-    private
-    def subscriptions
-      @subscriptions ||= Hash.new{|h,k| h[k] = []}
-    end
-
-    def key(*segments)
-      segments.unshift(:firehose).join(':')
-    end
-  end
-end

data/lib/firehose/swf_policy_request.rb

@@ -1,23 +0,0 @@
-module Firehose
-  module SwfPolicyRequest
-
-    # Borrowed from: https://github.com/igrigorik/em-websocket/blob/3e7f7d7760cc23b9d1d34fc1c17bab4423b5d11a/lib/em-websocket/connection.rb#L104
-    def handle_swf_policy_request(data)
-      if data =~ /\A<policy-file-request\s*\/>/
-        Firehose.logger.debug "Received SWF Policy request: #{data.inspect}"
-        send_data policy
-        close_connection_after_writing
-        true
-      end
-    end
-
-    def policy
-      <<-EOS
-<?xml version="1.0"?>
-<cross-domain-policy>
-  <allow-access-from domain="*" to-ports="*"/>
-</cross-domain-policy>
-      EOS
-    end
-  end
-end

data/spec/lib/broker_spec.rb

@@ -1,30 +0,0 @@
-# require 'spec_helper'
-
-# describe Firehose::Broker do
-#   include EM::TestHelper
-
-#   let(:broker)  { Firehose::Broker.new }
-
-#   it "should unsubscibe consumers and remove them from the collection" do
-#     stats = nil
-
-#     em do
-#       broker.consumer('1').subscribe_to('/the-channel')
-#       broker.consumer('2').subscribe_to('/the-channel')
-#       broker.consumer('2').subscribe_to('/a-channel')
-
-#       em.add_timer(1) do
-#         stats = broker.stats
-#         broker.stop
-#         em.stop
-#       end
-#     end
-
-#     stats.should == {
-#       '1' => {'subscriptions' => ['/the-channel'] },
-#       '2' => {'subscriptions' => ['/the-channel', '/a-channel']}
-#     }
-
-#     broker.stats.should == {}
-#   end
-# end

data/spec/lib/consumer_spec.rb

@@ -1,66 +0,0 @@
-# require 'spec_helper'
-
-# describe Firehose::Consumer do
-#   include EM::TestHelper
-
-#   let(:consumer)  { Firehose::Consumer.new }
-#   let(:publisher) { Firehose::Publisher.new }
-#   let(:channel)   { '/papa-smurf' }
-#   let(:another_channel)  { '/mama-smurf' }
-
-#   describe "subscriptions" do
-#     it "should subscribe to channel" do
-#       sent, recieved = 'hi', nil
-
-#       em do
-#         consumer.subscribe_to channel do |msg|
-#           recieved = msg
-#           em.stop
-#         end
-#         em.add_timer(1) do
-#           publisher.publish(channel, sent)
-#         end
-#       end
-
-#       recieved.should == sent
-#     end
-
-#     it "should track subscriptions" do
-#       lambda{
-#         em do
-#           consumer.subscribe_to channel
-#           consumer.subscribe_to another_channel
-#           em.add_timer(1){ em.stop }
-#         end
-#       }.should change{ consumer.subscriptions.size }.by(2)
-#     end
-
-#     it "should only allow one subscription per channel" do
-#       lambda{
-#         em do
-#           3.times { consumer.subscribe_to channel }
-#           em.add_timer(1){ em.stop }
-#         end
-#       }.should change{ consumer.subscriptions.size }.by(1)
-#     end
-
-#     it "should unsubscribe from all channels" do
-#       subscribed_count, after_unsubscribe_count = 0, nil
-
-#       em do
-#         consumer.subscribe_to channel
-#         consumer.subscribe_to another_channel
-#         subscribed_count = consumer.subscriptions.size
-#         em.add_timer(1) do 
-#           consumer.unsubscribe
-#           em.add_timer(1) do 
-#             em.stop
-#           end
-#         end
-#       end
-
-#       subscribed_count.should == 2
-#       consumer.subscriptions.size.should == 0
-#     end
-#   end
-# end