fairway 0.0.4 → 0.0.5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fairway (0.0.3)
+    fairway (0.0.5)
       activesupport
       hiredis
       redis

data/README.markdown

@@ -1,123 +1,183 @@
-# Fairway (redis backbone)
+# Fairway - a fair way to queue messages in multi-user systems.
 
-## Responsibilities
+## Installation
 
-* Allow consumption of messages from various sources
-* Publish messages for services who are listening
-* adds messages to queues which services have registered
-* keeps track of which facets have messages queued
-* allows messages to be pulled off of queues in round-robin by facet
+Install the gem:
 
-### Queue structure
+    gem install fairway
 
-Once a service registers a queue, with a regex for what messages should be
-added to the queue, the backbone will begin pushing matching messages onto the queue.
+Then make sure you `bundle install`.
 
-In some cases, queuing systems can have problems in multi-user systems. If one user
-queues up a ton of messages, messages for other users may be delayed.
+## Configuration
 
-Driver solves that by allowing "facets" for each queue, which can be set to a facet
-for each user.
+    Fairway.configure do |config|
+      config.redis     = { host: "localhost", port: 6379 }
+      config.namespace = "fairway"
+    
+      config.facet do |message|
+        message[:user]
+      end
 
-Each facet is added to a list if it has messages waiting to be processed. This list
-is used to enforce a round-robin stategy for pulling messages off of the queue. This
-means we'll process one message for every facet which has messages queued, before
-looping back and processing additional messages.
+      config.register_queue("all_messages")
+    end
+    
+## What's a facet?
 
-### Redis LUA scripting
+In many queuing systems, if a single user manages to queue up a lot of messages/jobs at once,
+everyone else in the system has to wait.  Facets are a way of splitting up a single queue by
+user (or any other criteria) to ensure fair processing of each facet's jobs.
 
-Driver uses [LUA scripts](http://redis.io/commands/eval) inside of redis heavily. This is for a few reasons:
+When pulling off a faceted queue, facets are processed in a round-robin fashion, so you'll pull
+off one message for each facet which contains messages before doubling back and pulling
+additional messages from a given facet.
 
-* There is complex logic that can't be expressed in normal redis commands.
-* Each script contains many redis commands and it's important that these
-commands are processed atomically.  A LUA script does that.
-* Since the script is run inside of redis, once the script has started,
-there's very low latency for each redis command.  So, the script executes
-much faster than if we made each call independantly over the network.
+You can define how to facet your messages during configuration:
+
+    Fairway.configure do |config|
+      config.facet do |message|
+        message[:user]
+      end
+    end
 
-This means your Redis version must be `>= 2.6.0`
+Now, any message delivered by fairway, will use the `user` key of the message to determine
+which facet to use.
 
-### Usage
+You could also just have a queue for each user, but at scale, this can get crazy and many
+queuing systems don't perform well with thousands of queues.
 
- Add driver to your Gemfile
+## Queuing messages
 
-    gem 'driver', git: 'git@github.com:customerio/driver.git'
+In order to queue messages, you need to register a queue. You can register multiple queues,
+and each queue will receive delivered messages.
 
-Make sure to `bundle install`.
+Registering a queue is part of your fairway configuration:
 
-##### Configure driver
+    Fairway.configure do |config|
+      config.register_queue("myqueue")
+      config.register_queue("yourqueue")
+    end
 
-    Driver.configure do |config|
-      config.redis     = { host: "yourserver.com", port: 6379 }
-      config.namespace = "letsdrive"
+After configuring your queues, just create a fairway connection,
+and it'll handle persisting your queues in Redis:
 
-      config.facet do |message|
-        message[:user_id]
-      end
+    connection = Fairway::Connection.new
 
-      config.topic do |message|
-        "#{message[:user_id]}:#{message[:type]}"
-      end
-    end
+## Delivering messages
 
-If you don't configure, it'll default to:
+To add messages to your queues, you deliver them:
 
-    Driver.configure do |config|
-      config.redis     = { host: "localhost", port: 6379 }
-      config.namespace = nil
+    connection = Fairway::Connection.new
+    connection.deliver(type: "invite_friends", user: "bob", friends: ["nancy", "john"])
 
-      config.facet do |message|
-        message[:facet]
-      end
+Now, any registered queues will receive this message, faceted if you've defined
+a facet in your configuration.
 
-      config.topic do |message|
-        message[:topic]
-      end
+## Consuming messages from a queue
+
+Once you have messages on a queue, you can pull them off and process them:
+
+    connection = Fairway::Connection.new
+    queue      = Fairway::Queue.new(connection, "myqueue")
+    message    = queue.pull
+
+Behind the scenes, fairway uses a round-robin strategy to ensure equal weighting of
+any facets which contain messages.
+
+If there are no messages in any facets, `queue.pull` will return `nil`.
+
+## Channeling messages
+
+In many cases, you don't want all messages delivered to every queue. You'd like
+to filter which messages a queue receives.
+
+You can accomplish this with message channels. By default, all messages use the `default`
+channel. You can customize this by creating a `Fairway::ChanneledConnection` and
+a block which defines the channel for a given message:
+
+    conn = Fairway::Connection.new
+    conn = Fairway::ChanneledConnection.new(conn) do |message|
+      message[:type]
     end
 
-##### Create an instance of driver
+You can also register queues for a channel:
 
-    driver = Driver::Client.new
+    Fairway.configure do |config|
+      config.register_queue("invite_queue", "invite_friends")
+    end
+    
+Now, your queue will only receive messages which have the channel `invite_friends`.
 
-##### Send messages
+If you'd like to receive messages with channels that match a pattern:
 
-    driver.deliver(facet: 1, type: :page, name: "http://customer.io/blog", referrer: "http://customer.io")
+    Fairway.configure do |config|
+      config.register_queue("invite_queue", "invite_.*")
+    end
 
-You can pass any hash of data you'd like. Using the default configuration, this message will have a topic
-of `page`, which can useful if you'd like to listen for, or process, messages.
+Now, messages from the channels `invite_friends`, `invite_pets`, `invite_parents` will
+be delivered to the `invite_queue`.
 
-##### Listen for messages
+## Subscribing to messages
 
-If a message is sent in the middle of the forest, and no one is listening, was it ever really sent?
+To listen for messages without the overhead of queuing them, you can subscribe:
 
-You can listen for messages that are delivered by driver by subscribing to message topics:
+    connection = Fairway::Connection.new
 
-    driver.redis.psubscribe("page:*google*") do |on|
-      on.pmessage do |pattern, channel, message|
-        puts "[#{channel}] #{message}"
-      end
+    connection.subscribe do |message|
+      # Do something with each message
     end
 
-If you've configured your topic to be `"#{message[:type]}:#{message[:name]}`, this will listen for any page events with google in the name.
+If you'd like to only receive some messages, you can subscribe to just a particular channel:
 
-Now, if you deliver a message, it'll be printed out on the console.
+    connection = Fairway::Connection.new
 
-*Note:* redis psubscribe is blocking. So, you'll need multiple console windows open.
-One to deliver the message, and one to listen for them.
+    connection.subscribe("invite_*") do |message|
+      # Do something with each message which
+      # has a channel matching "invite_*"
+    end
 
-##### Create a queue
+## Fairway and Sidekiq
 
-Ok, so now you can listen to messages, but what if your listener dies and you miss all your important messages?
+Fairway isn't meant to be a robust system for processing queued messages/jobs. To more reliably
+process queued messages, we've integrated with [Sidekiq](http://sidekiq.org/).
 
-Not to worry, you can tell driver to queue up any messages you want to know about.
+    require 'fairway/sidekiq'
 
-    driver.register_queue("myqueue", ".*:page:.*google.*")
+    connection = Fairway::Connection.new
+    queues     = Fairway::Queue.new(connection, "myqueue", "yourqueue")
 
-Now driver will deliver all page events with google in the name to the queue named `myqueue`. To retrieve messages
-from your queue:
+    Sidekiq.options[:fetch] = Fairway::Sidekiq::Fetch.new do |fetch|
+      fetch.from :sidekiq, 2
+      fetch.from queues, 1 do |queue, message|
+        # translate message to normalized Sidekiq job, if needed
+        { "queue" => "fairway",
+          "class" => "FairwayMessageJob",
+          "args"  => [message],
+          "retry" => true }
+      end
+    end
 
-    message = driver.pull("myqueue")
+`fetch.from :sidekiq, 2` will fetch from sidekiq queues you have defined through the
+normal sidekiq configuration.
 
-This will return a message or nil (if no messages are queued).
+`fetch.from queues, 1` will pull messages from your fairway queue, and allow you to translate
+them into standard sidekiq jobs.
+
+The second parameters are fetch weights, so in the above example, we'll look for jobs first from
+your normal sidekiq queues twice as often as your fairway queues.
+
+## Queue structure
+
+TODO: low level description of what's going on? performance?
+
+## LUA scripting
+
+Fairway uses [LUA scripting](http://redis.io/commands/eval) heavily. This is for a few reasons:
+
+* There is complex logic that can't be expressed in normal redis commands.
+* Each script contains many redis commands and it's important that these
+commands are processed atomically.
+* Since the script is run inside of redis, once the script has started,
+there's very low latency for each redis command.  So, the script executes
+much faster than if we made each call independantly over the network.
 
-*Note:* `pull` is facet aware, and will rotate through all facets with queued messages.
+This means your must be using a Redis version `>= 2.6.0`

data/lib/fairway/config.rb

@@ -1,17 +1,18 @@
 module Fairway
   class Config
     attr_accessor :namespace
-    attr_reader :queues
+    attr_reader :defined_queues
 
     DEFAULT_FACET = "default"
 
     QueueDefinition = Struct.new(:name, :channel)
 
     def initialize
-      @redis_options = {}
-      @namespace = nil
-      @facet = lambda { |message| DEFAULT_FACET }
-      @queues = []
+      @redis_options  = {}
+      @namespace      = nil
+      @facet          = lambda { |message| DEFAULT_FACET }
+      @defined_queues = []
+
       yield self if block_given?
     end
 
@@ -24,7 +25,7 @@ module Fairway
     end
 
     def register_queue(name, channel = Connection::DEFAULT_CHANNEL)
-      @queues << QueueDefinition.new(name, channel)
+      @defined_queues << QueueDefinition.new(name, channel)
     end
 
     def redis=(options)
@@ -36,19 +37,11 @@ module Fairway
     end
 
     def scripts
-      @scripts ||= Scripts.new(raw_redis, scripts_namespace)
+      @scripts ||= Scripts.new(raw_redis, @namespace)
     end
 
   private
 
-    def scripts_namespace
-      if @namespace.blank?
-        ""
-      else
-        "#{@namespace}:"
-      end
-    end
-
     def raw_redis
       @raw_redis ||= Redis.new(@redis_options.merge(hiredis: true))
     end

data/lib/fairway/connection.rb

@@ -7,8 +7,16 @@ module Fairway
     def initialize(config = Fairway.config)
       @config = config
 
-      @config.queues.each do |queue|
-        scripts.fairway_register_queue(queue.name, queue.channel)
+      @config.defined_queues.each do |queue|
+        scripts.register_queue(queue.name, queue.channel)
+      end
+    end
+
+    def queues
+      @queues ||= begin
+        scripts.registered_queues.map do |name, _|
+          Queue.new(self, name)
+        end
       end
     end
 

data/lib/fairway/{queue_reader.rb → queue.rb}

@@ -1,7 +1,9 @@
 module Fairway
-  class QueueReader
+  class Queue
+    attr_reader :connection, :queue_names
+
     def initialize(connection, *queue_names)
-      @connection = connection
+      @connection  = connection
       @queue_names = [queue_names].flatten!
     end
 
@@ -12,5 +14,12 @@ module Fairway
     def pull
       @connection.scripts.fairway_pull(@queue_names)
     end
+
+    def ==(other)
+      other.respond_to?(:connection) &&
+      other.respond_to?(:queue_names) &&
+      connection == other.connection &&
+      queue_names == other.queue_names
+    end
   end
 end

data/lib/fairway/scripts.rb

@@ -12,9 +12,17 @@ module Fairway
       @namespace = namespace
     end
 
+    def register_queue(name, channel)
+      @redis.hset(registered_queues_key, name, channel)
+    end
+
+    def registered_queues
+      @redis.hgetall(registered_queues_key)
+    end
+
     def method_missing(method_name, *args)
       loaded = false
-      @redis.evalsha(script_sha(method_name), [@namespace], args)
+      @redis.evalsha(script_sha(method_name), [namespace], args)
     rescue Redis::CommandError => ex
       if ex.message.include?("NOSCRIPT") && !loaded
         @redis.script(:load, script_source(method_name))
@@ -27,6 +35,14 @@ module Fairway
 
   private
 
+    def registered_queues_key
+      "#{namespace}registered_queues"
+    end
+
+    def namespace
+      @namespace.blank? ? "" : "#{@namespace}:"
+    end
+
     def script_sha(name)
       self.class.script_shas[name] ||= Digest::SHA1.hexdigest(script_source(name))
     end
@@ -38,6 +54,5 @@ module Fairway
     def script_path(name)
       Pathname.new(__FILE__).dirname.join("../../redis/#{name}.lua")
     end
-
   end
 end

data/lib/fairway/sidekiq/{non_blocking_fetch.rb → basic_fetch.rb}

@@ -1,13 +1,15 @@
 module Fairway
   module Sidekiq
-    class NonBlockingFetch < ::Sidekiq::BasicFetch
+    class BasicFetch < ::Sidekiq::BasicFetch
       attr_reader :queues
 
       def initialize(options)
         @queues = options[:queues].map { |q| "queue:#{q}" }
       end
 
-      def retrieve_work
+      def retrieve_work(options = {})
+        options = { blocking: true }.merge(options)
+
         ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work #{queues_cmd}"
 
         work = ::Sidekiq.redis do |conn|
@@ -32,6 +34,7 @@ module Fairway
           work = UnitOfWork.new(*work)
         else
           ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got nil"
+          sleep 1 if options[:blocking]
         end
 
         work

data/lib/fairway/sidekiq/{queue_fetch.rb → fairway_fetch.rb}

@@ -1,18 +1,20 @@
-require "sidekiq/fetch"
-
 module Fairway
   module Sidekiq
-    class QueueFetch < ::Sidekiq::BasicFetch
-      def initialize(queue_reader, &block)
-        @queue_reader = queue_reader
+    class FairwayFetch < ::Sidekiq::BasicFetch
+      attr_reader :queues
+
+      def initialize(queues, &block)
+        @queues = queues
         @message_to_job = block if block_given?
       end
 
-      def retrieve_work
+      def retrieve_work(options = {})
+        options = { blocking: true }.merge(options)
+
         ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work"
         unit_of_work = nil
 
-        fairway_queue, work = @queue_reader.pull
+        fairway_queue, work = @queues.pull
 
         if work
           decoded_work = JSON.parse(work)
@@ -29,10 +31,15 @@ module Fairway
           ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got work"
         else
           ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got nil"
+          sleep 1 if options[:blocking]
         end
 
         unit_of_work
       end
+
+      def ==(other)
+        other.respond_to?(:queues) && queues == other.queues
+      end
     end
   end
 end

data/lib/fairway/sidekiq/fetch.rb

@@ -0,0 +1,59 @@
+module Fairway
+  module Sidekiq
+    class Fetch
+      class Fetches
+        attr_reader :list
+
+        def from(queue, weight = 1, &block)
+          if queue == :sidekiq
+            queue = BasicFetch.new(::Sidekiq.options)
+          else
+            queue = FairwayFetch.new(queue, &block)
+          end
+
+          weight.times do
+            list << queue
+          end
+        end
+
+        def list
+          @list ||= []
+        end
+      end
+
+      def initialize(&block)
+        yield(@fetches = Fetches.new)
+      end
+
+      def new(options)
+        self
+      end
+
+      def fetches
+        @fetches.list
+      end
+
+      def fetch_order
+        fetches.shuffle.uniq
+      end
+
+      def retrieve_work
+        ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work"
+
+        fetch_order.each do |fetch|
+          work = fetch.retrieve_work(blocking: false)
+
+          if work
+            ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got work"
+            return work
+          end
+        end
+
+        ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got nil"
+        sleep 1
+
+        return nil
+      end
+    end
+  end
+end

data/lib/fairway/sidekiq.rb

@@ -1,20 +1,14 @@
 require "sidekiq"
-require "sidekiq/manager"
+require "sidekiq/fetch"
 
-require "fairway/sidekiq/composite_fetch"
-require "fairway/sidekiq/fetcher"
-require "fairway/sidekiq/fetcher_factory"
-require "fairway/sidekiq/non_blocking_fetch"
-require "fairway/sidekiq/queue_fetch"
+require "fairway/sidekiq/fetch"
+require "fairway/sidekiq/basic_fetch"
+require "fairway/sidekiq/fairway_fetch"
 
-# conn         = Fairway::Connection.new
-# queue_reader = Fairway::QueueReader.new(conn, "fairway")
-#
-# queue_fetch  = Fairway::Sidekiq::QueueFetch.new(queue_reader) do |message|
-#   # Transform message into a sidekiq job
-#   message
-# end
+# conn  = Fairway::Connection.new
+# queues = Fairway::Queue.new(conn, "queue1", "queue2")
 # 
-# non_blocking_fetch = Fairway::Sidekiq::NonBlockingFetch.new(Sidekiq.options)
-# fetch              = Fairway::Sidekiq::CompositeFetch.new(queue_fetch => 1, non_blocking_fetch => 1)
-# Sidekiq.options[:fetcher] = Fairway::Sidekiq::FetcherFactory.new(fetch)
+# Sidekiq.options[:fetch] = Fairway::Sidekiq::Fetch.new do |fetch|
+#   fetch.from :sidekiq, 1
+#   fetch.from queues, 1
+# end

data/lib/fairway/version.rb

@@ -1,3 +1,3 @@
 module Fairway
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/fairway.rb

@@ -9,7 +9,7 @@ require "fairway/config"
 require "fairway/scripts"
 require "fairway/channeled_connection"
 require "fairway/connection"
-require "fairway/queue_reader"
+require "fairway/queue"
 
 module Fairway
   def self.config
@@ -17,6 +17,11 @@ module Fairway
   end
 
   def self.configure
-    yield(config) if block_given?
+    yield(config)
+  end
+
+  def self.reconfigure
+    @config = Config.new
+    yield(config)
   end
 end

data/spec/lib/fairway/connection_spec.rb

@@ -23,6 +23,16 @@ module Fairway
       end
     end
 
+    describe "#queues" do
+      it "returns a Queue for every currently registered queue" do
+        Fairway.config.redis.hset("registered_queues", "name", "channel")
+
+        connection.queues.should == [
+          Queue.new(connection, "name")
+        ]
+      end
+    end
+
     describe "#deliver" do
       it "publishes message over the message topic channel" do
         redis = Redis.new

data/spec/lib/fairway/fetch_spec.rb

@@ -0,0 +1,96 @@
+require "spec_helper"
+
+module Fairway::Sidekiq
+  describe Fetch do
+    describe "#initialize" do
+      it "accepts a block to define of fetches with priority" do
+        fetch = Fetch.new do |fetch|
+          fetch.from :fetchA, 10
+          fetch.from :fetchB, 1
+        end
+
+        fetchA = FairwayFetch.new(:fetchA)
+        fetchB = FairwayFetch.new(:fetchB)
+
+        fetch.fetches.should == [Array.new(10, fetchA), fetchB].flatten
+      end
+
+      it "instantiates a BasicFetch if you fetch from the keyword :sidekiq" do
+        fetch = Fetch.new do |fetch|
+          fetch.from :sidekiq, 1
+        end
+
+        fetch.fetches.length.should == 1
+        fetch.fetches.first.should be_instance_of(BasicFetch)
+      end
+
+      it "instantiates a FairwayFetch if you fetch from a queue object" do
+        queue = Fairway::Queue.new(Fairway::Connection.new, "fairway")
+
+        fetch = Fetch.new do |fetch|
+          fetch.from queue, 1
+        end
+
+        fetch.fetches.length.should == 1
+        fetch.fetches.first.should be_instance_of(FairwayFetch)
+      end
+    end
+
+    describe "#new" do
+      it "returns itself to match Sidekiq fetch API" do
+        fetch = Fetch.new do |fetch|
+          fetch.from :fetchA, 1
+        end
+
+        fetch.new({}).should == fetch
+      end
+    end
+
+    describe "#fetch_order" do
+      let(:fetch)  { Fetch.new { |f| f.from :fetchA, 10; f.from :fetchB, 1 } }
+
+      it "should shuffle and uniq fetches" do
+        fetch.fetches.should_receive(:shuffle).and_return(fetch.fetches)
+        fetch.fetch_order
+      end
+
+      it "should unique fetches list" do
+        fetch.fetches.length.should == 11
+        fetch.fetch_order.length.should == 2
+      end
+    end
+
+    describe "#retrieve_work" do
+      let(:work)   { mock(:work)  }
+      let(:fetchA) { mock(:fetch) }
+      let(:fetchB) { mock(:fetch) }
+      let(:fetch)  { Fetch.new { |f| f.from :fetchA, 10; f.from :fetchB, 1 } }
+
+      before do
+        fetch.stub(fetch_order: [fetchA, fetchB], sleep: nil)
+      end
+
+      it "returns work from the first fetch who has work" do
+        fetchA.stub(retrieve_work: work)
+        fetchB.should_not_receive(:retrieve_work)
+
+        fetch.retrieve_work.should == work
+      end
+
+      it "attempts to retrieve work from each fetch in a non blocking fashion" do
+        fetchA.should_receive(:retrieve_work).with(blocking: false)
+        fetchB.should_receive(:retrieve_work).with(blocking: false)
+        fetch.retrieve_work.should be_nil
+      end
+
+      it "sleeps if no work is found" do
+        fetch.should_receive(:sleep).with(1)
+
+        fetchA.stub(retrieve_work: nil)
+        fetchB.stub(retrieve_work: nil)
+
+        fetch.retrieve_work
+      end
+    end
+  end
+end

data/spec/lib/fairway/{queue_reader_spec.rb → queue_spec.rb}

@@ -1,7 +1,7 @@
 require "spec_helper"
 
 module Fairway
-  describe QueueReader do
+  describe Queue do
     let(:connection) do
       c = Connection.new(Fairway.config)
       ChanneledConnection.new(c) do |message|
@@ -12,31 +12,31 @@ module Fairway
 
     describe "#initialize" do
       it "requires a Connection and queue names" do
-        lambda { QueueReader.new }.should raise_error(ArgumentError)
+        lambda { Queue.new }.should raise_error(ArgumentError)
       end
     end
 
     describe "#length" do
-      let(:reader) { QueueReader.new(connection, "myqueue") }
+      let(:queue) { Queue.new(connection, "myqueue") }
 
       before do
         Fairway.config.register_queue("myqueue", "event:helloworld")
       end
 
       it "returns the number of queued messages across facets" do
-        reader.length.should == 0
+        queue.length.should == 0
 
         connection.deliver(message.merge(facet: 1, message: 1))
         connection.deliver(message.merge(facet: 1, message: 2))
         connection.deliver(message.merge(facet: 2, message: 3))
 
-        reader.length.should == 3
+        queue.length.should == 3
 
-        reader.pull
-        reader.pull
-        reader.pull
+        queue.pull
+        queue.pull
+        queue.pull
 
-        reader.length.should == 0
+        queue.length.should == 0
       end
     end
 
@@ -49,9 +49,9 @@ module Fairway
         connection.deliver(message1 = message.merge(message: 1))
         connection.deliver(message2 = message.merge(message: 2))
 
-        reader = QueueReader.new(connection, "myqueue")
-        reader.pull.should == ["myqueue", message1.to_json]
-        reader.pull.should == ["myqueue", message2.to_json]
+        queue = Queue.new(connection, "myqueue")
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
       end
 
       it "pulls from facets of the queue in a round-robin nature" do
@@ -59,27 +59,27 @@ module Fairway
         connection.deliver(message2 = message.merge(facet: 1, message: 2))
         connection.deliver(message3 = message.merge(facet: 2, message: 3))
 
-        reader = QueueReader.new(connection, "myqueue")
-        reader.pull.should == ["myqueue", message1.to_json]
-        reader.pull.should == ["myqueue", message3.to_json]
-        reader.pull.should == ["myqueue", message2.to_json]
+        queue = Queue.new(connection, "myqueue")
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message3.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
       end
 
       it "removes facet from active list if it becomes empty" do
         connection.deliver(message)
 
         Fairway.config.redis.smembers("myqueue:active_facets").should == ["1"]
-        reader = QueueReader.new(connection, "myqueue")
-        reader.pull
+        queue = Queue.new(connection, "myqueue")
+        queue.pull
         Fairway.config.redis.smembers("myqueue:active_facets").should be_empty
       end
 
       it "returns nil if there are no messages to retrieve" do
         connection.deliver(message)
 
-        reader = QueueReader.new(connection, "myqueue")
-        reader.pull.should == ["myqueue", message.to_json]
-        reader.pull.should be_nil
+        queue = Queue.new(connection, "myqueue")
+        queue.pull.should == ["myqueue", message.to_json]
+        queue.pull.should be_nil
       end
 
       context "pulling from multiple queues" do
@@ -92,14 +92,14 @@ module Fairway
           connection.deliver(message1 = message.merge(topic: "event:1"))
           connection.deliver(message2 = message.merge(topic: "event:2"))
 
-          reader = QueueReader.new(connection, "myqueue2", "myqueue1")
-          reader.pull.should == ["myqueue2", message2.to_json]
-          reader.pull.should == ["myqueue1", message1.to_json]
+          queue = Queue.new(connection, "myqueue2", "myqueue1")
+          queue.pull.should == ["myqueue2", message2.to_json]
+          queue.pull.should == ["myqueue1", message1.to_json]
         end
 
         it "returns nil if no queues have messages" do
-          reader = QueueReader.new(connection, "myqueue2", "myqueue1")
-          reader.pull.should be_nil
+          queue = Queue.new(connection, "myqueue2", "myqueue1")
+          queue.pull.should be_nil
         end
 
         it "pulls from facets of the queue in a round-robin nature" do
@@ -108,13 +108,28 @@ module Fairway
           connection.deliver(message3 = message.merge(facet: 2, topic: "event:1"))
           connection.deliver(message4 = message.merge(facet: 1, topic: "event:2"))
 
-          reader = QueueReader.new(connection, "myqueue2", "myqueue1")
-          reader.pull.should == ["myqueue2", message4.to_json]
-          reader.pull.should == ["myqueue1", message1.to_json]
-          reader.pull.should == ["myqueue1", message3.to_json]
-          reader.pull.should == ["myqueue1", message2.to_json]
+          queue = Queue.new(connection, "myqueue2", "myqueue1")
+          queue.pull.should == ["myqueue2", message4.to_json]
+          queue.pull.should == ["myqueue1", message1.to_json]
+          queue.pull.should == ["myqueue1", message3.to_json]
+          queue.pull.should == ["myqueue1", message2.to_json]
         end
       end
     end
+
+    describe "equality" do
+      it "should equal queues with same connection and queue names" do
+        Queue.new(connection, "a", "b", "c").should == Queue.new(connection, "a", "b", "c")
+      end
+
+      it "doesn't equal queues with different connection" do
+        new_conn = Connection.new(Fairway.config)
+        Queue.new(connection, "a", "b", "c").should_not == Queue.new(new_conn, "a", "b", "c")
+      end
+
+      it "doesn't equal queues with different queues" do
+        Queue.new(connection, "a", "b", "c").should_not == Queue.new(connection, "a", "b")
+      end
+    end
   end
 end

data/spec/lib/fairway/scripts_spec.rb

@@ -10,17 +10,36 @@ module Fairway
       end
     end
 
+    describe "#register_queue" do
+      let(:scripts) { Scripts.new(Redis.new, "foo") }
+
+      it "adds the queue and channel to the hash of registered queues" do
+        scripts.register_queue("name", "channel")
+        Redis.new.hgetall("foo:registered_queues").should == { "name" => "channel" }
+      end
+    end
+
+    describe "#registered_queue" do
+      let(:scripts) { Scripts.new(Redis.new, "foo") }
+
+      it "returns hash of all registered queues and their channels" do
+        Redis.new.hset("foo:registered_queues", "first", "channel1")
+        Redis.new.hset("foo:registered_queues", "second", "channel2")
+        scripts.registered_queues.should == { "first" => "channel1", "second" => "channel2" }
+      end
+    end
+
     describe "#method_missing" do
       let(:scripts) { Scripts.new(Redis.new, "foo") }
 
       it "runs the script" do
-        scripts.fairway_register_queue("namespace", "name", "topic")
+        scripts.fairway_pull("namespace", "name")
       end
 
       context "when the script does not exist" do
         it "loads the script" do
           Redis.new.script(:flush)
-          scripts.fairway_register_queue("namespace", "name", "topic")
+          scripts.fairway_pull("namespace", "name")
         end
       end
     end

data/spec/lib/fairway/sidekiq/{non_blocking_fetch_spec.rb → basic_fetch_spec.rb}

@@ -1,9 +1,9 @@
 require "spec_helper"
 
 module Fairway::Sidekiq
-  describe NonBlockingFetch do
+  describe BasicFetch do
     let(:queues) { [:critical, :critical, :default] }
-    let(:fetch)  { NonBlockingFetch.new(queues: queues) }
+    let(:fetch)  { BasicFetch.new(queues: queues) }
 
     it "accepts options with a list of queues and their weights" do
       fetch.queues.should == ["queue:critical", "queue:critical", "queue:default"]
@@ -26,6 +26,16 @@ module Fairway::Sidekiq
         unit_of_work.queue_name.should == "critical"
         unit_of_work.message.should == "critical"
       end
+
+      it "sleeps if no work is found" do
+        fetch.should_receive(:sleep).with(1)
+        fetch.retrieve_work
+      end
+
+      it "doesn't sleep if blocking option is false" do
+        fetch.should_not_receive(:sleep)
+        fetch.retrieve_work(blocking: false)
+      end
     end
   end
 end

data/spec/lib/fairway/sidekiq/{queue_fetch_spec.rb → fairway_fetch_spec.rb}

@@ -2,14 +2,13 @@ require "spec_helper"
 
 module Fairway
   module Sidekiq
-    describe QueueFetch do
-      let(:reader) { QueueReader.new(Connection.new, "fairway") }
+    describe FairwayFetch do
+      let(:queue) { Queue.new(Connection.new, "fairway") }
       let(:work)   { { queue: "golf_events", type: "swing", name: "putt" }.to_json }
+      let(:fetch)  { FairwayFetch.new(queue) }
 
-      it "requests work from the queue reader" do
-        fetch = QueueFetch.new(reader)
-
-        reader.stub(pull: ["fairway", work])
+      it "requests work from the queue queue" do
+        queue.stub(pull: ["fairway", work])
 
         unit_of_work = fetch.retrieve_work
         unit_of_work.queue_name.should == "golf_events"
@@ -17,19 +16,31 @@ module Fairway
       end
 
       it "allows transforming of the message into a job" do
-        fetch = QueueFetch.new(reader) do |fairway_queue, message|
+        fetch = FairwayFetch.new(queue) do |fairway_queue, message|
           {
             "queue" => "my_#{message["queue"]}",
             "class" => "GolfEventJob"
           }
         end
 
-        reader.stub(pull: ["fairway", work])
+        queue.stub(pull: ["fairway", work])
 
         unit_of_work = fetch.retrieve_work
         unit_of_work.queue_name.should == "my_golf_events"
         unit_of_work.message.should == { "queue" => "my_golf_events", "class" => "GolfEventJob" }.to_json
       end
+
+      it "sleeps if no work is found" do
+        fetch.should_receive(:sleep).with(1)
+        queue.stub(pull: nil)
+        fetch.retrieve_work
+      end
+
+      it "doesn't sleep if blocking option is false" do
+        fetch.should_not_receive(:sleep)
+        queue.stub(pull: nil)
+        fetch.retrieve_work(blocking: false)
+      end
     end
   end
 end

data/spec/spec_helper.rb

@@ -22,7 +22,7 @@ Dir[File.join(File.dirname(__FILE__), "support/**/*.rb")].each {|f| require f}
 
 RSpec.configure do |config|
   config.before(:each) do
-    Fairway.configure do |config|
+    Fairway.reconfigure do |config|
       config.namespace = "test:fairway"
       config.facet { |message| message[:facet] }
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fairway
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-15 00:00:00.000000000 Z
+date: 2013-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -94,27 +94,23 @@ files:
 - lib/fairway/channeled_connection.rb
 - lib/fairway/config.rb
 - lib/fairway/connection.rb
-- lib/fairway/queue_reader.rb
+- lib/fairway/queue.rb
 - lib/fairway/scripts.rb
 - lib/fairway/sidekiq.rb
-- lib/fairway/sidekiq/composite_fetch.rb
-- lib/fairway/sidekiq/fetcher.rb
-- lib/fairway/sidekiq/fetcher_factory.rb
-- lib/fairway/sidekiq/non_blocking_fetch.rb
-- lib/fairway/sidekiq/queue_fetch.rb
+- lib/fairway/sidekiq/basic_fetch.rb
+- lib/fairway/sidekiq/fairway_fetch.rb
+- lib/fairway/sidekiq/fetch.rb
 - lib/fairway/version.rb
 - redis/fairway_deliver.lua
 - redis/fairway_pull.lua
-- redis/fairway_register_queue.lua
 - spec/lib/fairway/channeled_connection_spec.rb
 - spec/lib/fairway/config_spec.rb
 - spec/lib/fairway/connection_spec.rb
-- spec/lib/fairway/queue_reader_spec.rb
+- spec/lib/fairway/fetch_spec.rb
+- spec/lib/fairway/queue_spec.rb
 - spec/lib/fairway/scripts_spec.rb
-- spec/lib/fairway/sidekiq/composite_fetch_spec.rb
-- spec/lib/fairway/sidekiq/fetcher_spec.rb
-- spec/lib/fairway/sidekiq/non_blocking_fetch_spec.rb
-- spec/lib/fairway/sidekiq/queue_fetch_spec.rb
+- spec/lib/fairway/sidekiq/basic_fetch_spec.rb
+- spec/lib/fairway/sidekiq/fairway_fetch_spec.rb
 - spec/lib/fairway/subscription_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/customerio/fairway
@@ -145,11 +141,10 @@ test_files:
 - spec/lib/fairway/channeled_connection_spec.rb
 - spec/lib/fairway/config_spec.rb
 - spec/lib/fairway/connection_spec.rb
-- spec/lib/fairway/queue_reader_spec.rb
+- spec/lib/fairway/fetch_spec.rb
+- spec/lib/fairway/queue_spec.rb
 - spec/lib/fairway/scripts_spec.rb
-- spec/lib/fairway/sidekiq/composite_fetch_spec.rb
-- spec/lib/fairway/sidekiq/fetcher_spec.rb
-- spec/lib/fairway/sidekiq/non_blocking_fetch_spec.rb
-- spec/lib/fairway/sidekiq/queue_fetch_spec.rb
+- spec/lib/fairway/sidekiq/basic_fetch_spec.rb
+- spec/lib/fairway/sidekiq/fairway_fetch_spec.rb
 - spec/lib/fairway/subscription_spec.rb
 - spec/spec_helper.rb

data/lib/fairway/sidekiq/composite_fetch.rb

@@ -1,37 +0,0 @@
-module Fairway
-  module Sidekiq
-    class CompositeFetch
-      attr_reader :fetches
-
-      def initialize(fetches)
-        @fetches = []
-
-        fetches.each do |fetch, weight|
-          [weight.to_i, 1].max.times do
-            @fetches << fetch
-          end
-        end
-      end
-
-      def fetch_order
-        fetches.shuffle.uniq
-      end
-
-      def retrieve_work
-        ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work"
-
-        fetch_order.each do |fetch|
-          work = fetch.retrieve_work
-
-          if work
-            ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got work"
-            return work
-          end
-        end
-
-        ::Sidekiq.logger.debug "#{self.class.name}#retrieve_work got nil"
-        return nil
-      end
-    end
-  end
-end

data/lib/fairway/sidekiq/fetcher.rb

@@ -1,37 +0,0 @@
-module Fairway
-  module Sidekiq
-    class Fetcher < ::Sidekiq::Fetcher
-      attr_reader :mgr, :strategy
-
-      def initialize(mgr, fetch)
-        @mgr = mgr
-        @strategy = fetch
-      end
-
-      def done!
-        @done = true
-      end
-
-      def fetch
-        watchdog('Fetcher#fetch died') do
-          return if @done
-
-          begin
-            work = @strategy.retrieve_work
-
-            if work
-              @mgr.async.assign(work)
-            else
-              after(TIMEOUT) { fetch }
-            end
-          rescue => ex
-            logger.error("Error fetching message: #{ex}")
-            logger.error(ex.backtrace.first)
-            sleep(TIMEOUT)
-            after(0) { fetch }
-          end
-        end
-      end
-    end
-  end
-end

data/lib/fairway/sidekiq/fetcher_factory.rb

@@ -1,23 +0,0 @@
-module Fairway
-  module Sidekiq
-    class FetcherFactory
-      def initialize(fetch)
-        @fetch = fetch
-      end
-
-      def done!
-        @fetcher.done!
-      end
-
-      def strategy
-        # This is only used for ::Sidekiq::BasicFetch.bulk_requeue
-        # which is the same for us.
-        ::Sidekiq::BasicFetch
-      end
-
-      def new(mgr, options)
-        @fetcher = Fetcher.new(mgr, @fetch)
-      end
-    end
-  end
-end

data/redis/fairway_register_queue.lua

@@ -1,5 +0,0 @@
-local hash          = KEYS[1] .. 'registered_queues';
-local queue_name    = ARGV[1];
-local queue_message = ARGV[2];
-
-redis.call('hset', hash, queue_name, queue_message);

data/spec/lib/fairway/sidekiq/composite_fetch_spec.rb

@@ -1,50 +0,0 @@
-require "spec_helper"
-
-module Fairway::Sidekiq
-  describe CompositeFetch do
-    describe "#initialize" do
-      it "accepts a hash of fetches with priority" do
-        fetcher = CompositeFetch.new(fetcherA: 10, fetcherB: 1)
-        fetcher.fetches.should == [Array.new(10, :fetcherA), :fetcherB].flatten
-      end
-    end
-
-    describe "#fetch_order" do
-      let(:fetcher)  { CompositeFetch.new(fetcherA: 10, fetcherB: 1) }
-
-      it "should shuffle and uniq fetches" do
-        fetcher.fetches.should_receive(:shuffle).and_return(fetcher.fetches)
-        fetcher.fetch_order
-      end
-
-      it "should unique fetches list" do
-        fetcher.fetches.length.should == 11
-        fetcher.fetch_order.length.should == 2
-      end
-    end
-
-    describe "#retrieve_work" do
-      let(:work)     { mock(:work) }
-      let(:fetcherA) { mock(:fetcher) }
-      let(:fetcherB) { mock(:fetcher) }
-      let(:fetcher)  { CompositeFetch.new(fetcherA => 10, fetcherB => 1) }
-
-      before do
-        fetcher.stub(fetch_order: [fetcherA, fetcherB])
-      end
-
-      it "returns work from the first fetcher who has work" do
-        fetcherA.stub(retrieve_work: work)
-        fetcherB.should_not_receive(:retrieve_work)
-
-        fetcher.retrieve_work.should == work
-      end
-
-      it "attempts to retrieve work from each fetcher if no work is found" do
-        fetcherA.should_receive(:retrieve_work)
-        fetcherB.should_receive(:retrieve_work)
-        fetcher.retrieve_work.should be_nil
-      end
-    end
-  end
-end

data/spec/lib/fairway/sidekiq/fetcher_spec.rb

@@ -1,31 +0,0 @@
-require "spec_helper"
-
-module Fairway::Sidekiq
-  describe Fetcher do
-    let(:manager) { mock(:manager) }
-    let(:fetch) { mock(:fetch) }
-
-    it "accepts a manager and a fetch strategy" do
-      fetcher = Fetcher.new(manager, fetch)
-      fetcher.mgr.should == manager
-      fetcher.strategy.should == fetch
-    end
-
-    describe "#fetch" do
-      let(:fetcher) { Fetcher.new(manager, fetch) }
-
-      it "retrieves work from fetch strategy" do
-        fetch.should_receive(:retrieve_work)
-        fetcher.fetch
-      end
-
-      it "tells manager to assign work if work is fetched" do
-        work = mock(:work)
-        fetch.stub(retrieve_work: work)
-        manager.stub(async: manager)
-        manager.should_receive(:assign).with(work)
-        fetcher.fetch
-      end
-    end
-  end
-end