worker_roulette 0.1.0 → 0.1.1

data/README.md

@@ -121,12 +121,12 @@ end
 ## Performance
 Running the performance tests on my laptop, the numbers break down like this:
 ### Async Api
-  - Manual: ~4200 read-write round-trips / second
-  - Pubsub: ~5200 read-write round-trips / second
+  - Pubsub: ~5500 read-write round-trips / second
+  - Manual: ~4500 read-write round-trips / second
 
 ### Sync Api
-  - Manual: ~1600 read-write round-trips / second
-  - Pubsub: ~2000 read-write round-trips / second
+  - Pubsub: ~2700 read-write round-trips / second
+  - Manual: ~2500 read-write round-trips / second
 
 To run the perf tests yourself run `bundle exec spec:perf`
 

data/lib/worker_roulette.rb

@@ -33,8 +33,7 @@ module WorkerRoulette
   end
 
   def self.a_foreman(sender, channel = nil)
-    raise "WorkerRoulette not Started" unless @foreman_connection_pool
-    AForeman.new(sender, @foreman_connection_pool, channel)
+    foreman(sender, channel)
   end
 
   def self.a_tradesman(channel = nil)
@@ -58,6 +57,29 @@ module WorkerRoulette
     @redis_config.dup
   end
 
+  def self.dump(obj)
+    Oj.dump(obj)
+  rescue Oj::ParseError => e
+    {'error' => e, 'unparsable_string' => obj}
+  end
+
+  def self.load(json)
+    Oj.load(json)
+  rescue Oj::ParseError => e
+    {'error' => e, 'unparsable_string' => obj}
+  end
+
+  def self.job_board_key(namespace = nil)
+    "#{namespace + ':' if namespace}#{WorkerRoulette::JOB_BOARD}"
+  end
+
+  def self.sender_key(sender, namespace = nil)
+    "#{namespace + ':' if namespace}#{sender}"
+  end
+
+  def self.counter_key(sender, namespace = nil)
+    "#{namespace + ':' if namespace}counter_key"
+  end
 private
   def self.new_redis
     if @evented

data/lib/worker_roulette/a_tradesman.rb

@@ -8,13 +8,6 @@ module WorkerRoulette
       @redis_pubsub.subscribe(@channel)
     end
 
-    def work_orders!(&callback)
-      Lua.call(self.class.lua_drain_work_orders, [job_board_key, nil], [@namespace]) do |results|
-        @sender = (results.first || '').split(':').first
-        callback.call (results[1] || []).map {|work_order| Oj.load(work_order)} if callback
-      end
-    end
-
     def unsubscribe(&callback)
       deferable = @redis_pubsub.unsubscribe(@channel)
       deferable.callback do
@@ -38,30 +31,5 @@ module WorkerRoulette
         work_orders! {|work_orders| on_message_callback.call(work_orders, nil, nil)}
       end
     end
-
-    def self.lua_drain_work_orders
-      <<-HERE
-        local job_board_key     = KEYS[1]
-        local empty             = KEYS[2]
-        local namespace         = ARGV[1]
-
-        local function drain_work_orders(job_board_key, namespace)
-          local sender_key = redis.call('ZRANGE', job_board_key, 0, 0)[1]
-
-          if sender_key == false then
-            return {}
-          end
-
-          local results = {}
-          results[1] = sender_key
-          results[2] = redis.call('LRANGE', sender_key, 0, -1)
-          results[3] = redis.call('DEL', sender_key)
-          results[4] = redis.call('ZREM', job_board_key, sender_key)
-          return results
-        end
-
-        return drain_work_orders(job_board_key, namespace)
-      HERE
-    end
   end
 end

data/lib/worker_roulette/foreman.rb

@@ -1,7 +1,6 @@
 module WorkerRoulette
   class Foreman
     attr_reader :sender
-    COUNTER_KEY = 'counter_key'
 
     def initialize(sender, redis_pool, namespace = nil)
       @sender     = sender
@@ -10,41 +9,60 @@ module WorkerRoulette
       @channel    = namespace || WorkerRoulette::JOB_NOTIFICATIONS
     end
 
-    def job_board_key
-      @job_board_key ||= "#{@namespace + ':' if @namespace}#{WorkerRoulette::JOB_BOARD}"
+    def enqueue_work_order(work_order, headers = {}, &callback)
+      work_order = {'headers' => default_headers.merge(headers), 'payload' => work_order}
+      enqueue_work_order_without_headers(work_order, &callback)
     end
 
-    def sender_key
-      @sender_key = "#{@namespace + ':' if @namespace}#{@sender}"
+    def enqueue_work_order_without_headers(work_order, &callback)
+      Lua.call(self.class.lua_enqueue_work_orders, [counter_key, job_board_key, sender_key, @channel],
+                   [WorkerRoulette.dump(work_order),  WorkerRoulette::JOB_NOTIFICATIONS], &callback)
     end
 
-    def counter_key
-      @counter_key = "#{@namespace + ':' if @namespace}#{COUNTER_KEY}"
+    def job_board_key
+      @job_board_key ||= WorkerRoulette.job_board_key(@namespace)
     end
 
-    def enqueue_work_order_without_headers(work_order, &callback)
-      #Caveat Emptor: There is a race condition here, but it not serious;
-      #the count may be incremented again by another process before the sender
-      #is added to the job_queue. This is not a big deal bc it just means that
-      #the sender's queue will be processed one slot behind it's rightful place.
-      #This does not effect work_order ordering.
-      @redis_pool.with({}) do |redis|
-        @count = redis.incr(COUNTER_KEY)
-        redis.multi do
-          redis.zadd(job_board_key, @count, @sender)
-          redis.rpush(sender_key, Oj.dump(work_order))
-          redis.publish(@channel, WorkerRoulette::JOB_NOTIFICATIONS)
-        end
-      end
+    def counter_key
+      @counter_key ||= WorkerRoulette.counter_key(@namespace)
     end
 
-    def enqueue_work_order(work_order, headers = {}, &callback)
-      work_order = {'headers' => default_headers.merge(headers), 'payload' => work_order}
-      enqueue_work_order_without_headers(work_order, &callback)
+  private
+    def sender_key
+      @sender_key = WorkerRoulette.sender_key(sender, @namespace)
     end
 
     def default_headers
       Hash['sender' => sender]
     end
+
+    def self.lua_enqueue_work_orders
+      <<-HERE
+        local counter_key       = KEYS[1]
+        local job_board_key     = KEYS[2]
+        local sender_key        = KEYS[3]
+        local channel           = KEYS[4]
+
+        local work_order        = ARGV[1]
+        local job_notification  = ARGV[2]
+
+        local function enqueue_work_orders(work_order, job_notification)
+          local result    = sender_key .. ' updated'
+          local sender_on_job_board = redis.call('ZSCORE', job_board_key, sender_key)
+
+          if (sender_on_job_board == false) then
+            local count     = redis.call('INCR', counter_key)
+            local job_added = redis.call('ZADD',job_board_key, count, sender_key)
+            result    = sender_key .. ' added'
+          end
+
+          local work_added          = redis.call('RPUSH',sender_key, work_order)
+          local job_board_update    = redis.call('PUBLISH', channel, job_notification)
+          return result
+        end
+
+        return enqueue_work_orders(work_order, job_notification)
+      HERE
+    end
   end
 end

data/lib/worker_roulette/lua.rb

@@ -4,9 +4,7 @@ module WorkerRoulette
 
     def self.call(lua_script, keys_accessed = [], args = [], &callback)
       WorkerRoulette.tradesman_connection_pool.with do |redis|
-        results = redis.evalsha(sha(lua_script), keys_accessed.length, *keys_accessed, *args)
-        results.callback &callback
-        results.errback {self.eval(redis, lua_script, keys_accessed, args, &callback)}
+        results = evalsha(redis, lua_script, keys_accessed, args, &callback)
       end
     end
 
@@ -24,8 +22,25 @@ module WorkerRoulette
 
     def self.eval(redis, lua_script, keys_accessed, args, &callback)
       results = redis.eval(lua_script, keys_accessed.size, *keys_accessed, *args)
-      results.callback &callback
+      results.callback &callback if callback
       results.errback  {|err_msg| raise EM::Hiredis::RedisError.new(err_msg)}
     end
+
+    def self.evalsha(redis, lua_script, keys_accessed, args, &callback)
+      if redis.class == EM::Hiredis::Client
+        results = redis.evalsha(sha(lua_script), keys_accessed.length, *keys_accessed, *args)
+        results.callback &callback if callback
+        results.errback {self.eval(redis, lua_script, keys_accessed, args, &callback)}
+      else
+        begin
+          results = redis.evalsha(sha(lua_script), keys_accessed, args)
+        rescue Redis::CommandError
+          results = redis.eval(lua_script, keys_accessed, args)
+        ensure
+          return callback.call results if callback
+        end
+      end
+      results
+    end
   end
 end

data/lib/worker_roulette/tradesman.rb

@@ -8,14 +8,6 @@ module WorkerRoulette
       @channel     = namespace || WorkerRoulette::JOB_NOTIFICATIONS
     end
 
-    def job_board_key
-      @job_board_key ||= "#{@namespace + ':' if @namespace}#{WorkerRoulette::JOB_BOARD}"
-    end
-
-    def sender_key
-      @sender_key = "#{@namespace + ':' if @namespace}#{@sender}"
-    end
-
     def wait_for_work_orders(on_subscribe_callback = nil, &block)
       @pubsub_pool.with do |redis|
         redis.subscribe(@channel) do |on|
@@ -25,15 +17,12 @@ module WorkerRoulette
       end
     end
 
-    def work_orders!
-      @client_pool.with do |redis|
-        get_sender_for_next_job(redis)
-        results = redis.multi do
-          redis.lrange(sender_key, 0, -1)
-          redis.del(sender_key)
-          redis.zrem(job_board_key, sender_key)
-        end
-        ((results || []).first || []).map {|work_order| Oj.load(work_order)}
+    def work_orders!(&callback)
+      Lua.call(self.class.lua_drain_work_orders, [job_board_key, nil], [@namespace]) do |results|
+        @sender = (results.first || '').split(':').first
+        work = (results[1] || []).map {|work_order| WorkerRoulette.load(work_order)}
+        callback.call work if callback
+        work
       end
     end
 
@@ -41,7 +30,40 @@ module WorkerRoulette
       @pubsub_pool.with {|redis| redis.unsubscribe(@channel)}
     end
 
+    def job_board_key
+      @job_board_key ||= WorkerRoulette.job_board_key(@namespace)
+    end
+
   private
+    def sender_key
+      @sender_key = WorkerRoulette.sender_key(@sender, @namespace)
+    end
+
+    def self.lua_drain_work_orders
+      <<-HERE
+        local job_board_key     = KEYS[1]
+        local empty             = KEYS[2]
+        local namespace         = ARGV[1]
+
+        local function drain_work_orders(job_board_key, namespace)
+          local sender_key = redis.call('ZRANGE', job_board_key, 0, 0)[1]
+
+          if sender_key == false then
+            return {}
+          end
+
+          local results = {}
+          results[1] = sender_key
+          results[2] = redis.call('LRANGE', sender_key, 0, -1)
+          results[3] = redis.call('DEL', sender_key)
+          results[4] = redis.call('ZREM', job_board_key, sender_key)
+          return results
+        end
+
+        return drain_work_orders(job_board_key, namespace)
+      HERE
+    end
+
     def get_sender_for_next_job(redis)
       @sender = (redis.zrange(job_board_key, 0, 0) || []).first.to_s
     end

data/lib/worker_roulette/version.rb

@@ -1,3 +1,3 @@
 module WorkerRoulette
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/spec/benchmark/perf_test.rb

@@ -12,7 +12,7 @@ WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
 puts "Redis Connection Pool Size: #{REDIS_CONNECTION_POOL_SIZE}"
 
 Benchmark.bmbm do |x|
-  x.report "Time to insert and read #{ITERATIONS} large work_orders" do # ~1600 work_orders / second round trip; 50-50 read-write time; CPU and IO bound
+  x.report "Time to insert and read #{ITERATIONS} large work_orders" do # ~2500 work_orders / second round trip; 50-50 read-write time; CPU and IO bound
     WorkerRoulette.start(size: REDIS_CONNECTION_POOL_SIZE, evented: false)
     ITERATIONS.times do |iteration|
       sender = 'sender_' + iteration.to_s
@@ -33,7 +33,7 @@ EM::Hiredis.reconnect_timeout = 0.01
 WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
 
 Benchmark.bmbm do |x|
-  x.report "Time for tradesmans to enqueue_work_order and read #{ITERATIONS} large work_orders via pubsub" do # ~2000 work_orders / second round trip
+  x.report "Time for tradesmans to enqueue_work_order and read #{ITERATIONS} large work_orders via pubsub" do # ~2700 work_orders / second round trip
     WorkerRoulette.start(size: REDIS_CONNECTION_POOL_SIZE, evented: false)
     ITERATIONS.times do |iteration|
       p = -> do

data/spec/integration/evented_worker_roulette_spec.rb

@@ -9,7 +9,7 @@ describe WorkerRoulette do
   let(:hello_work_order) {Hash['payload' => "hello"]}
   let(:foreman_work_order) {Hash['payload' => "foreman"]}
   let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
-  let(:jsonized_work_orders_with_headers) {[Oj.dump(work_orders_with_headers)]}
+  let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
 
   let(:redis) {Redis.new(WorkerRoulette.redis_config)}
 
@@ -17,7 +17,7 @@ describe WorkerRoulette do
     WorkerRoulette.start(evented: true)
   end
 
-  context Foreman do
+  context "Evented Foreman" do
     let(:subject) {WorkerRoulette.a_foreman(sender)}
 
     it "should enqueue work" do
@@ -33,7 +33,7 @@ describe WorkerRoulette do
     it "should enqueue_work_order two work_orders in the sender's slot in the job board" do
       subject.enqueue_work_order(work_orders.first) do
         subject.enqueue_work_order(work_orders.last) do
-          redis.lrange(sender, 0, -1).should == work_orders.map {|m| Oj.dump(default_headers.merge({'payload' => m})) }
+          redis.lrange(sender, 0, -1).should == work_orders.map {|m| WorkerRoulette.dump(default_headers.merge({'payload' => m})) }
           done
         end
       end
@@ -41,7 +41,7 @@ describe WorkerRoulette do
 
     it "should enqueue_work_order an array of work_orders without headers in the sender's slot in the job board" do
       subject.enqueue_work_order_without_headers(work_orders) do
-        redis.lrange(sender, 0, -1).should == [Oj.dump(work_orders)]
+        redis.lrange(sender, 0, -1).should == [WorkerRoulette.dump(work_orders)]
         done
       end
     end
@@ -57,7 +57,7 @@ describe WorkerRoulette do
       extra_headers = {'foo' => 'bars'}
       subject.enqueue_work_order(work_orders, extra_headers) do
         work_orders_with_headers['headers'].merge!(extra_headers)
-        redis.lrange(sender, 0, -1).should == [Oj.dump(work_orders_with_headers)]
+        redis.lrange(sender, 0, -1).should == [WorkerRoulette.dump(work_orders_with_headers)]
         done
       end
     end
@@ -139,7 +139,7 @@ describe WorkerRoulette do
     end
   end
 
-  context Tradesman do
+  context "Evented Tradesman" do
     let(:foreman) {WorkerRoulette.a_foreman(sender)}
     let(:subject)  {WorkerRoulette.a_tradesman}
 
@@ -266,16 +266,10 @@ describe WorkerRoulette do
     end
   end
 
-  it "should return a hash with a string in the payload if OJ cannot parse the json" do
+  xit "should return a hash with a string in the payload if OJ cannot parse the json" do
 
   end
 
-  context "Potential Ack Success/Failure for Processing Queues" do
-    xit "should not delete the messages from the queue until they have been processed succcesfully"
-    xit "should checkout a readlock for a queue and put it back when its done processing; lock should expire after 5 minutes?"
-    xit "should retry doing work on a queue 3 times if it is locked (ex backoff)"
-  end
-
   context "Failure" do
     it "should not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; done; end
   end

data/spec/integration/worker_roulette_spec.rb

@@ -7,7 +7,7 @@ describe WorkerRoulette do
   let(:hello_work_order) {Hash['payload' => "hello"]}
   let(:foreman_work_order) {Hash['payload' => "foreman"]}
   let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
-  let(:jsonized_work_orders_with_headers) {[Oj.dump(work_orders_with_headers)]}
+  let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
 
   let(:redis) {Redis.new(WorkerRoulette.redis_config)}
 
@@ -26,20 +26,15 @@ describe WorkerRoulette do
       subject.sender.should == sender
     end
 
-    it "should be injected with a raw_redis_client so it can do is work" do
-      Redis.any_instance.should_receive(:rpush)
-      subject.enqueue_work_order(:whatever)
-    end
-
     it "should enqueue_work_order two work_orders in the sender's slot in the switchboard" do
-      subject.enqueue_work_order(work_orders.first)
-      subject.enqueue_work_order(work_orders.last)
-      redis.lrange(sender, 0, -1).should == work_orders.map {|m| Oj.dump(default_headers.merge({'payload' => m})) }
+      subject.enqueue_work_order(work_orders.first) {}
+      subject.enqueue_work_order(work_orders.last) {}
+      redis.lrange(sender, 0, -1).should == work_orders.map {|m| WorkerRoulette.dump(default_headers.merge({'payload' => m})) }
     end
 
     it "should enqueue_work_order an array of work_orders without headers in the sender's slot in the switchboard" do
       subject.enqueue_work_order_without_headers(work_orders)
-      redis.lrange(sender, 0, -1).should == [Oj.dump(work_orders)]
+      redis.lrange(sender, 0, -1).should == [WorkerRoulette.dump(work_orders)]
     end
 
     it "should enqueue_work_order an array of work_orders with default headers in the sender's slot in the switchboard" do
@@ -51,26 +46,24 @@ describe WorkerRoulette do
       extra_headers = {'foo' => 'bars'}
       subject.enqueue_work_order(work_orders, extra_headers)
       work_orders_with_headers['headers'].merge!(extra_headers)
-      redis.lrange(sender, 0, -1).should == [Oj.dump(work_orders_with_headers)]
+      redis.lrange(sender, 0, -1).should == [WorkerRoulette.dump(work_orders_with_headers)]
     end
 
     it "should post the sender's id to the job board with an order number" do
       subject.enqueue_work_order(work_orders.first)
-      subject.enqueue_work_order(work_orders.last)
-      redis.zrange(subject.job_board_key, 0, -1, with_scores: true).should == [[sender.to_s, work_orders.length.to_f]]
-    end
-
-    it "should post the sender_id and work_orders transactionally" do
-      Redis.any_instance.should_receive(:multi)
-      subject.enqueue_work_order(work_orders.first)
+      WorkerRoulette.foreman('other_forman').enqueue_work_order(work_orders.last)
+      redis.zrange(subject.job_board_key, 0, -1, with_scores: true).should == [[sender, 1.0], ["other_forman", 2.0]]
     end
 
-    it "should generate sequential order numbers" do
+    it "should generate a monotically increasing score for senders not on the job board, but not for senders already there" do
+      other_forman = WorkerRoulette.foreman('other_forman')
       redis.get(subject.counter_key).should == nil
       subject.enqueue_work_order(work_orders.first)
       redis.get(subject.counter_key).should == "1"
       subject.enqueue_work_order(work_orders.last)
-      redis.get(subject.counter_key).should == "2"
+      redis.get(subject.counter_key).should == "1"
+      other_forman.enqueue_work_order(work_orders.last)
+      redis.get(other_forman.counter_key).should == "2"
     end
 
     it "should publish a notification that a new job is ready" do
@@ -104,11 +97,6 @@ describe WorkerRoulette do
       subject.sender.should == sender
     end
 
-    it "should be injected with a redis client so it can do its work" do
-      Redis.any_instance.should_receive(:lrange).and_call_original
-      subject.work_orders!
-    end
-
     it "should drain one set of work_orders from the sender's slot in the switchboard" do
       subject.work_orders!.should == [work_orders_with_headers]
       subject.work_orders!.should == []
@@ -132,11 +120,6 @@ describe WorkerRoulette do
       redis.zrange(subject.job_board_key, 0, -1).should == [most_recent_sender]
     end
 
-    it "should get the sender and work_order list transactionally" do
-      Redis.any_instance.should_receive(:multi).and_call_original
-      subject.work_orders!
-    end
-
     it "should get the work_orders from the next queue when a new job is ready" do
       subject.work_orders!
       subject.should_receive(:work_orders!).and_call_original
@@ -168,9 +151,6 @@ describe WorkerRoulette do
       end
     end
 
-    it "should checkout a readlock for a queue and put it back when its done processing; lock should expire after 5 minutes?"
-    it "should eves drop on the job board"
-
     context "Failure" do
       it "should not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: worker_roulette
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -250,7 +250,6 @@ files:
 - README.md
 - Rakefile
 - lib/worker_roulette.rb
-- lib/worker_roulette/a_foreman.rb
 - lib/worker_roulette/a_tradesman.rb
 - lib/worker_roulette/foreman.rb
 - lib/worker_roulette/lua.rb

data/lib/worker_roulette/a_foreman.rb

@@ -1,40 +0,0 @@
-require_relative './foreman'
-module WorkerRoulette
-  class AForeman < Foreman
-    def enqueue_work_order_without_headers(work_order, &callback)
-      Lua.call(self.class.lua_enqueue_work_orders, [COUNTER_KEY, job_board_key, sender_key, @channel],
-                   [@sender, Oj.dump(work_order),  WorkerRoulette::JOB_NOTIFICATIONS], &callback)
-    end
-
-  private
-    def self.lua_enqueue_work_orders
-      <<-HERE
-        local counter_key       = KEYS[1]
-        local job_board_key     = KEYS[2]
-        local sender_key        = KEYS[3]
-        local channel           = KEYS[4]
-
-        local sender            = ARGV[1]
-        local work_order        = ARGV[2]
-        local job_notification  = ARGV[3]
-
-        local function enqueue_work_orders(sender, work_order, job_notification)
-          local result    = sender .. ' updated'
-          local sender_on_job_board = redis.call('ZSCORE', job_board_key, sender_key)
-
-          if (sender_on_job_board == false) then
-            local count     = redis.call('INCR', counter_key)
-            local job_added = redis.call('ZADD',job_board_key, count, sender_key)
-            result    = sender .. ' added'
-          end
-
-          local work_added          = redis.call('RPUSH',sender_key, work_order)
-          local job_board_update    = redis.call('PUBLISH', channel, job_notification)
-          return result
-        end
-
-        return enqueue_work_orders(sender, work_order, job_notification)
-      HERE
-    end
-  end
-end