fairway 0.1.0 → 0.1.1

data/Gemfile.lock

@@ -1,17 +1,17 @@
 PATH
   remote: .
   specs:
-    fairway (0.0.10)
+    fairway (0.1.0)
       activesupport
       hiredis
       redis
-      redis-namespace
+      redis-namespace (>= 1.3.0)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    activesupport (3.2.11)
-      i18n (~> 0.6)
+    activesupport (3.2.13)
+      i18n (= 0.6.1)
       multi_json (~> 1.0)
     celluloid (0.12.4)
       facter (>= 1.6.12)
@@ -22,8 +22,8 @@ GEM
     hiredis (0.4.5)
     i18n (0.6.1)
     multi_json (1.5.0)
-    redis (3.0.2)
-    redis-namespace (1.2.1)
+    redis (3.0.4)
+    redis-namespace (1.3.0)
       redis (~> 3.0.0)
     rspec (2.12.0)
       rspec-core (~> 2.12.0)

data/fairway.gemspec

@@ -20,5 +20,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency("activesupport")
   gem.add_dependency("redis")
   gem.add_dependency("hiredis")
-  gem.add_dependency("redis-namespace")
+  gem.add_dependency("redis-namespace", ">= 1.3.0")
 end

data/lib/fairway/facet.rb

@@ -0,0 +1,69 @@
+module Fairway
+  class Facet
+    class InvalidPriorityError < Exception; end
+
+    attr_reader :queue, :name
+
+    def initialize(queue, facet_name)
+      @queue = queue
+      @name  = facet_name
+    end
+
+    def length
+      each_queue do |queue|
+        redis.llen(facet_key(queue))
+      end.sum
+    end
+
+    def priority
+      each_queue do |queue|
+        (redis.hget(priority_key(queue), name) || 1).to_i
+      end
+    end
+
+    def priority=(priority)
+      validate_priority!(priority)
+
+      each_queue do |queue|
+        scripts.fairway_priority(queue, name, priority)
+      end
+    end
+
+    def facet_key(queue)
+      "#{queue}:#{name}"
+    end
+
+    def priority_key(queue)
+      "#{queue}:priorities"
+    end
+
+    def ==(other)
+      other.respond_to?(:queue) &&
+      other.respond_to?(:name) &&
+      queue == other.queue &&
+      name == other.name
+    end
+
+    private
+
+    def redis
+      queue.redis
+    end
+
+    def scripts
+      queue.scripts
+    end
+
+    def each_queue(&block)
+      queue.unique_queues.map do |queue|
+        yield(queue)
+      end
+    end
+
+    def validate_priority!(priority)
+      if priority.to_i.to_s != priority.to_s || priority.to_i < 0
+        raise InvalidPriorityError.new("Facet priority must be an integer >= 0")
+      end
+    end
+  end
+end

data/lib/fairway/queue.rb

@@ -8,21 +8,19 @@ module Fairway
     end
 
     def active_facets
-      each_queue do |queue|
+      facet_names = unique_queues.map do |queue|
         redis.smembers("#{queue}:active_facets")
       end.flatten.uniq
+      
+      facet_names.map do |name|
+        Facet.new(self, name)
+      end
     end
 
     def length
       redis.mget(unique_queues.map{|q| "#{q}:length" }).sum.to_i
     end
 
-    def facet_length(facet)
-      each_queue do |queue|
-        redis.llen("#{queue}:#{facet}")
-      end.sum
-    end
-
     def peek
       scripts.fairway_peek(@queue_names.shuffle.uniq)
     end
@@ -38,25 +36,23 @@ module Fairway
       queue_names == other.queue_names
     end
 
-    private
-
     def unique_queues
       @queue_names.uniq
     end
 
-    def each_queue(&block)
-      unique_queues.map do |queue|
-        yield(queue)
-      end
+    def queue_key
+      queue
+    end
+
+    def redis
+      @connection.redis
     end
 
     def scripts
       @connection.scripts
     end
 
-    def redis
-      @connection.redis
-    end
+    private
 
     def parse_queue_names(names)
       [].tap do |queues|

data/lib/fairway/sidekiq/basic_fetch.rb

@@ -26,7 +26,7 @@ module Fairway
             return nil;
           SCRIPT
 
-          conn.eval(script, queues_cmd.map{|q| "#{namespace}#{q}"})
+          conn.eval(script, queues_cmd)
         end
 
         if (work)
@@ -43,25 +43,6 @@ module Fairway
       def queues_cmd
         @queues.shuffle.uniq
       end
-
-      def namespace
-        @namespace ||= begin
-          namespaces = []
-
-          ::Sidekiq.redis do |conn|
-            wrapper = conn
-
-            while defined?(wrapper.redis)
-              namespaces.unshift(wrapper.namespace)
-              wrapper = wrapper.redis
-            end
-
-            namespace = namespaces.join(":")
-            namespace += ":" unless namespace.blank?
-            namespace
-          end
-        end
-      end
     end
   end
 end

data/lib/fairway/version.rb

@@ -1,3 +1,3 @@
 module Fairway
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/fairway.rb

@@ -10,6 +10,7 @@ require "fairway/scripts"
 require "fairway/channeled_connection"
 require "fairway/connection"
 require "fairway/queue"
+require "fairway/facet"
 
 module Fairway
   def self.config

data/redis/fairway_deliver.lua

@@ -3,24 +3,58 @@ local topic     = ARGV[1];
 local facet     = ARGV[2];
 local message   = ARGV[3];
 
+local k = function (queue, subkey)
+  return namespace .. queue .. ':' .. subkey;
+end
+
 local registered_queues_key = namespace .. 'registered_queues';
 local registered_queues     = redis.call('hgetall', registered_queues_key);
 
+-- Determine whether or not the message should
+-- be delivered to each registered queue.
 for i = 1, #registered_queues, 2 do
-  local queue_name    = registered_queues[i];
-  local queue_message = registered_queues[i+1];
+  local queue       = registered_queues[i];
+  local queue_topic = registered_queues[i+1];
 
-  if string.find(topic, queue_message) then
-    local active_facets = namespace .. queue_name .. ':active_facets';
-    local facet_queue   = namespace .. queue_name .. ':facet_queue';
+  -- If the message topic matches the queue topic,
+  -- we deliver the message to the queue.
+  if string.find(topic, queue_topic) then
+    local priorities    = k(queue, 'priorities');
+    local active_facets = k(queue, 'active_facets');
+    local round_robin   = k(queue, 'facet_queue');
+    local facet_pool    = k(queue, 'facet_pool');
 
-    redis.call('lpush', namespace .. queue_name .. ':' .. facet, message)
-    redis.call('incr', namespace .. queue_name .. ':length');
+    -- Delivering the message to a queue is as simple as
+    -- pushing it onto the facet's message list, and
+    -- incrementing the length of the queue itself.
+    redis.call('lpush', k(queue, facet), message)
+    redis.call('incr', k(queue, 'length'));
 
+    -- If the facet just became active, we need to add
+    -- the facet to the round-robin queue, so it's
+    -- messages will be processed.
     if redis.call('sadd', active_facets, facet) == 1 then
-      redis.call('lpush', facet_queue, facet);
+      local priority = tonumber(redis.call('hget', priorities, facet)) or 1
+
+      -- If the facet currently has a priority
+      -- we need to jump start the facet by adding
+      -- it to the round-robin queue and updating
+      -- the current priority.
+      if priority > 0 then
+        redis.call('lpush', round_robin, facet);
+        redis.call('hset', facet_pool, facet, 1);
+      
+      -- If the facet has no set priority, just set the
+      -- current priority to zero. Since the facet just
+      -- became active, we can be sure it's already zero
+      -- or undefined.
+      else
+        redis.call('hset', facet_pool, facet, 0);
+      end
     end
   end
 end
 
+-- For any clients listening over pub/sub,
+-- we should publish the message.
 redis.call('publish', namespace .. topic, message);

data/redis/fairway_priority.lua

@@ -0,0 +1,29 @@
+local namespace    = KEYS[1];
+local queue        = ARGV[1];
+local facet        = ARGV[2];
+local new_priority = tonumber(ARGV[3]);
+
+local k = function (queue, subkey)
+  return namespace .. queue .. ':' .. subkey;
+end
+
+local priorities  = k(queue, 'priorities');
+local round_robin = k(queue, 'facet_queue');
+local facet_pool  = k(queue, 'facet_pool');
+
+-- Find the current state of the facet for the queue
+local priority = tonumber(redis.call('hget', priorities, facet)) or 1;
+local current  = tonumber(redis.call('hget', facet_pool, facet));
+
+-- If priority is currently zero, we need to jump
+-- start the facet by adding it to the round-robin
+-- queue and updating the current priority.
+if new_priority > 0 and priority == 0 and current == 0 then
+  redis.call('lpush', round_robin, facet);
+  redis.call('hset', facet_pool, facet, 1);
+end
+
+-- Other than the 0 priority case, we can just
+-- set the new priority, and the real priority
+-- will update lazily on pull.
+redis.call('hset', priorities, facet, new_priority);

data/redis/fairway_pull.lua

@@ -1,25 +1,92 @@
 local namespace = KEYS[1];
 
-for index, queue_name in ipairs(ARGV) do
-  local active_facets = namespace .. queue_name .. ':active_facets';
-  local facet_queue   = namespace .. queue_name .. ':facet_queue';
+local k = function (queue, subkey)
+  return namespace .. queue .. ':' .. subkey;
+end
+
+-- Multiple queues can be passed through
+-- fairway_pull. We'll loop through all 
+-- provided queues, and return a message
+-- from the first one that isn't empty.
+for i, queue in ipairs(ARGV) do
+  local priorities    = k(queue, 'priorities');
+  local active_facets = k(queue, 'active_facets');
+  local round_robin   = k(queue, 'facet_queue');
+  local facet_pool    = k(queue, 'facet_pool');
 
-  local facet = redis.call('rpop', facet_queue);
+  -- Pull a facet from the round-robin list.
+  -- This list guarantees each active facet will have a
+  -- message pulled from the queue every time through..
+  local facet = redis.call('rpop', round_robin);
 
   if facet then
-    local message_queue = namespace .. queue_name .. ':' .. facet;
-    local message = redis.call('rpop', message_queue);
+    -- If we found an active facet, we know the facet
+    -- has at least one message available to be pulled
+    -- from it's message queue.
+    local messages = k(queue, facet);
+    local message  = redis.call('rpop', messages);
 
     if message then
-      redis.call('decr', namespace .. queue_name .. ':length');
+      redis.call('decr', k(queue, 'length'));
     end
 
-    if redis.call('llen', message_queue) == 0 then
+    local length = redis.call('llen', messages);
+
+    -- If the length of the facet's message queue
+    -- is empty, then it is no longer active as
+    -- it no longer has any messages.
+    if length == 0 then
+      -- We remove the facet from the set of active
+      -- facets and don't push the facet back on the
+      -- round-robin queue.
       redis.call('srem', active_facets, facet);
+     
+    -- If the facet still has messages to process,
+    -- it remains in the active facet set, and is
+    -- pushed back on the round-robin queue.
+    --
+    -- Additionally, the priority of the facet may
+    -- have changed, so we'll check and update the
+    -- current facet's priority if needed.
     else
-      redis.call('lpush', facet_queue, facet);
+      local priority = tonumber(redis.call('hget', priorities, facet)) or 1
+      local current  = tonumber(redis.call('hget', facet_pool, facet))
+
+      -- If the current priority is less than the
+      -- desired priority, let's increase the priority
+      -- by pushing the current facet on the round-robin
+      -- queue twice, and incrementing the current
+      -- priority.
+      --
+      -- Note: If there aren't enough messages left
+      -- on the facet, we don't increase priority.
+      if current < priority and length > current then
+        redis.call('lpush', round_robin, facet);
+        redis.call('lpush', round_robin, facet);
+        redis.call('hset', facet_pool, facet, current + 1);
+        
+      -- If the current priority is greater than the
+      -- desired priority, let's decrease the priority
+      -- by not pushing the current facet on the round-robin
+      -- queue, and decrementing the current priority.
+      --
+      -- Note: Also decrement priority if there aren't
+      -- enough messages for the current priority. This
+      -- ensures priority (entries in the round-robin queue)
+      -- never exceeds the number of messages for a given
+      -- facet.
+      elseif current > priority or current > length then
+        redis.call('hset', facet_pool, facet, current - 1);
+      
+      -- If the current priority is equals the
+      -- desired priority, let's maintain the current priority
+      -- by pushing the current facet on the round-robin
+      -- queue once.
+      else
+        redis.call('lpush', round_robin, facet);
+      end
     end
 
-    return {queue_name, message};
+    return {queue, message};
   end
 end

data/spec/lib/fairway/facet_spec.rb

@@ -0,0 +1,95 @@
+require "spec_helper"
+
+module Fairway
+  describe Facet do
+    let(:connection) { Connection.new }
+    let(:queue)      { Queue.new(connection, "myqueue") }
+    let(:message)    { { facet: 1, topic: "event:helloworld" } }
+
+    before do
+      Fairway.config.register_queue("myqueue")
+    end
+
+    describe "#length" do
+      it "returns number of messages queues for a given facet" do
+        connection.deliver(message.merge(facet: 1, message: 1))
+        connection.deliver(message.merge(facet: 1, message: 2))
+        connection.deliver(message.merge(facet: 2, message: 3))
+
+        Facet.new(queue, 1).length.should == 2
+        Facet.new(queue, 2).length.should == 1
+        Facet.new(queue, 3).length.should == 0
+      end
+    end
+
+    describe "#priority" do
+      let(:facet) { Facet.new(queue, 1) }
+
+      it "defaults all facets to a priority of 1" do
+        facet.priority.should == [1]
+      end
+
+      it "returns value of priority set" do
+        facet.priority = 4
+        facet.priority.should == [4]
+      end
+
+      context "multiple queues" do
+        let(:queue) { Queue.new(connection, "myqueue2", "myqueue1") }
+
+        it "returns priority for each queue" do
+          facet.priority.should == [1, 1]
+
+          Facet.new(Queue.new(connection, "myqueue1"), 1).priority = 2
+
+          facet.priority.should == [1, 2]
+        end
+      end
+    end
+
+    describe "#priority=" do
+      let(:facet) { Facet.new(queue, 1) }
+
+      it "allows positive integer priority" do
+        facet.priority = 5
+        facet.priority.should == [5]
+      end
+
+      it "doesn't allow non integer priority" do
+        lambda { facet.priority = "hello" }.should raise_error(Facet::InvalidPriorityError)
+      end
+
+      it "doesn't allow non integer priority" do
+        lambda { facet.priority = "1.23" }.should raise_error(Facet::InvalidPriorityError)
+      end
+
+      it "doesn't allow negative priority" do
+        lambda { facet.priority = -1 }.should raise_error(Facet::InvalidPriorityError)
+      end
+
+      context "multiple queues" do
+        let(:queue) { Queue.new(connection, "myqueue2", "myqueue1") }
+
+        it "sets priority on each queue" do
+          facet.priority = 2
+          facet.priority.should == [2, 2]
+        end
+      end
+    end
+
+    describe "equality" do
+      it "should equal facets with same queue and names" do
+        Facet.new(queue, "a").should == Facet.new(queue, "a")
+      end
+
+      it "doesn't equal facets with different queues" do
+        new_queue = Queue.new(connection, "otherqueue")
+        Facet.new(queue, "a").should_not == Queue.new(new_queue, "a")
+      end
+
+      it "doesn't equal queues with different queues" do
+        Facet.new(queue, "a").should_not == Facet.new(queue, "b")
+      end
+    end
+  end
+end

data/spec/lib/fairway/queue_spec.rb

@@ -47,7 +47,10 @@ module Fairway
 
         queue.pull
 
-        queue.active_facets.should == ["2", "3"]
+        queue.active_facets.should == [
+          Facet.new(queue, "2"),
+          Facet.new(queue, "3")
+        ]
       end
 
       context "multiple queues" do
@@ -64,39 +67,11 @@ module Fairway
           connection.deliver(message.merge(topic: "event:1", facet: 3, message: 2))
           connection.deliver(message.merge(topic: "event:2", facet: 3, message: 3))
 
-          queue.active_facets.should == ["1", "2", "3"]
-        end
-      end
-    end
-
-    describe "#facet_length" do
-      it "returns number of messages queues for a given facet" do
-        connection.deliver(message.merge(facet: 1, message: 1))
-        connection.deliver(message.merge(facet: 1, message: 2))
-        connection.deliver(message.merge(facet: 2, message: 3))
-
-        queue.facet_length(1).should == 2
-        queue.facet_length(2).should == 1
-        queue.facet_length(3).should == 0
-      end
-
-      context "multiple queues" do
-        let(:queue) { Queue.new(connection, "myqueue1", "myqueue2") }
-
-        before do
-          Fairway.config.register_queue("myqueue1", "event:1")
-          Fairway.config.register_queue("myqueue2", "event:2")
-        end
-
-        it "sums number of messages for facet across all queues" do
-          connection.deliver(message.merge(topic: "event:1", facet: 1, message: 1))
-          connection.deliver(message.merge(topic: "event:1", facet: 2, message: 2))
-          connection.deliver(message.merge(topic: "event:1", facet: 3, message: 2))
-          connection.deliver(message.merge(topic: "event:2", facet: 3, message: 3))
-
-          queue.facet_length(1).should == 1
-          queue.facet_length(2).should == 1
-          queue.facet_length(3).should == 2
+          queue.active_facets.should == [
+            Facet.new(queue, "1"),
+            Facet.new(queue, "2"),
+            Facet.new(queue, "3")
+          ]
         end
       end
     end
@@ -132,7 +107,7 @@ module Fairway
       it "removes facet from active list if it becomes empty" do
         connection.deliver(message)
 
-        queue.active_facets.should == ["1"]
+        queue.active_facets.should == [Facet.new(queue, "1")]
         queue.pull
         queue.active_facets.should be_empty
       end
@@ -220,6 +195,98 @@ module Fairway
       end
     end
 
+    describe "priority" do
+      let(:facet1)   { Facet.new(queue, 1) }
+      let(:message1) { message.merge(facet: 1) }
+      let(:message2) { message.merge(facet: 2) }
+
+      it "doesn't pull messages from a facet with priority of 0" do
+        facet1.priority = 0
+        connection.deliver(message1)
+        queue.pull.should be_nil
+      end
+
+      it "lazily adjusts priority on pull" do
+        connection.deliver(message1)
+        connection.deliver(message1)
+
+        facet1.priority = 0
+
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should be_nil
+      end
+
+      it "pulls more messages from facets with higher priority" do
+        connection.deliver(message1)
+        connection.deliver(message1)
+        connection.deliver(message2)
+        connection.deliver(message2)
+        connection.deliver(message2)
+        connection.deliver(message1)
+        connection.deliver(message1)
+        connection.deliver(message1)
+
+        facet1.priority = 2
+
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
+        queue.pull.should be_nil
+      end
+
+      it "only pulls messages from higher priority facets if enough messages exist" do
+        connection.deliver(message1)
+        connection.deliver(message1)
+        connection.deliver(message1)
+        connection.deliver(message2)
+        connection.deliver(message2)
+        connection.deliver(message2)
+        connection.deliver(message2)
+
+        facet1.priority = 10
+
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
+        queue.pull.should be_nil
+      end
+
+      it "properly increments priority from 0" do
+        facet1.priority = 0
+
+        connection.deliver(message1)
+
+        queue.pull.should be_nil
+
+        facet1.priority = 1
+
+        queue.pull.should == ["myqueue", message1.to_json]
+      end
+
+      it "rapidly changing priority from 0 to 1 works properly" do
+        connection.deliver(message1)
+
+        facet1.priority = 0
+        facet1.priority = 1
+
+        connection.deliver(message1)
+        connection.deliver(message2)
+
+        queue.pull.should == ["myqueue", message1.to_json]
+        queue.pull.should == ["myqueue", message2.to_json]
+        queue.pull.should == ["myqueue", message1.to_json]
+      end
+    end
+
     describe "equality" do
       it "should equal queues with same connection and queue names" do
         Queue.new(connection, "a", "b", "c").should == Queue.new(connection, "a", "b", "c")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fairway
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-07 00:00:00.000000000 Z
+date: 2013-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.0
 description: A fair way to queue work in multi-user systems.
 email:
 - john@customer.io
@@ -94,6 +94,7 @@ files:
 - lib/fairway/channeled_connection.rb
 - lib/fairway/config.rb
 - lib/fairway/connection.rb
+- lib/fairway/facet.rb
 - lib/fairway/queue.rb
 - lib/fairway/scripts.rb
 - lib/fairway/sidekiq.rb
@@ -103,10 +104,12 @@ files:
 - lib/fairway/version.rb
 - redis/fairway_deliver.lua
 - redis/fairway_peek.lua
+- redis/fairway_priority.lua
 - redis/fairway_pull.lua
 - spec/lib/fairway/channeled_connection_spec.rb
 - spec/lib/fairway/config_spec.rb
 - spec/lib/fairway/connection_spec.rb
+- spec/lib/fairway/facet_spec.rb
 - spec/lib/fairway/fetch_spec.rb
 - spec/lib/fairway/queue_spec.rb
 - spec/lib/fairway/scripts_spec.rb
@@ -142,6 +145,7 @@ test_files:
 - spec/lib/fairway/channeled_connection_spec.rb
 - spec/lib/fairway/config_spec.rb
 - spec/lib/fairway/connection_spec.rb
+- spec/lib/fairway/facet_spec.rb
 - spec/lib/fairway/fetch_spec.rb
 - spec/lib/fairway/queue_spec.rb
 - spec/lib/fairway/scripts_spec.rb