nexia_worker_roulette 0.1.11

data/spec/benchmark/perf_test.rb

@@ -0,0 +1,137 @@
+require 'worker_roulette'
+require 'benchmark'
+require 'eventmachine'
+
+REDIS_CONNECTION_POOL_SIZE = 100
+ITERATIONS = 10_000
+
+work_order = {'ding dong' => "hello_foreman_" * 100}
+
+EM::Hiredis.reconnect_timeout = 0.01
+
+puts "Redis Connection Pool Size: #{REDIS_CONNECTION_POOL_SIZE}"
+
+times = Benchmark.bm do |x|
+  x.report "#{ITERATIONS} ASync Api Read/Writes" do
+    EM.run do
+      WorkerRoulette.start(evented: true)
+      WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
+      @total = 0
+      @tradesman = WorkerRoulette.tradesman
+
+      ITERATIONS.times do |iteration|
+        sender = 'sender_' + iteration.to_s
+        foreman = WorkerRoulette.foreman(sender)
+        foreman.enqueue_work_order(work_order) do
+          @tradesman.work_orders! do
+            @total += 1
+            EM.stop if @total == (ITERATIONS - 1)
+          end
+        end
+      end
+    end
+  end
+end
+puts "#{(ITERATIONS / times.first.real).to_i} ASync Api Read/Writes per second"
+puts "#################"
+puts
+
+WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
+
+times = Benchmark.bm do |x|
+  x.report "#{ITERATIONS * 2} ASync Api Polling Read/Writes" do
+    EM.run do
+      @processed = 0
+      @total     = 0
+      WorkerRoulette.start(evented: true)
+      WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
+      @total = 0
+      @tradesman = WorkerRoulette.tradesman
+      ITERATIONS.times do |iteration|
+        @start ||= Time.now
+        sender = 'sender_' + iteration.to_s
+        foreman = WorkerRoulette.foreman(sender)
+        foreman.enqueue_work_order(work_order)
+      end
+      @tradesman.wait_for_work_orders {@processed += 1; ((@stop = Time.now) && EM.add_timer(1){EM.stop}) if @processed == (ITERATIONS - 1)}
+    end
+  end
+end
+puts "#{ITERATIONS * 2 / (@stop - @start)} ASync Api Polling Read/Writes per second"
+puts "#################"
+puts
+WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
+
+WorkerRoulette.start(size: REDIS_CONNECTION_POOL_SIZE, evented: false)
+times = Benchmark.bm do |x|
+  puts x.class.name
+  x.report "#{ITERATIONS} Sync Api Writes" do
+    ITERATIONS.times do |iteration|
+      sender = 'sender_' + iteration.to_s
+      foreman = WorkerRoulette.foreman(sender)
+      foreman.enqueue_work_order(work_order)
+    end
+  end
+  WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
+end
+WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
+
+puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Writes per second"
+puts "#################"
+puts
+ITERATIONS.times do |iteration|
+  sender = 'sender_' + iteration.to_s
+  foreman = WorkerRoulette.foreman(sender)
+  foreman.enqueue_work_order(work_order)
+end
+
+times = Benchmark.bm do |x|
+  x.report "#{ITERATIONS} Sync Api Reads" do
+    ITERATIONS.times do |iteration|
+      sender = 'sender_' + iteration.to_s
+      tradesman = WorkerRoulette.tradesman
+      tradesman.work_orders!
+    end
+  end
+end
+puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Reads per second"
+puts "#################"
+puts
+WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
+
+times = Benchmark.bm do |x|
+  x.report "#{ITERATIONS} Sync Api Read/Writes" do
+    ITERATIONS.times do |iteration|
+      sender = 'sender_' + iteration.to_s
+      foreman = WorkerRoulette.foreman(sender)
+      foreman.enqueue_work_order(work_order)
+    end
+
+    ITERATIONS.times do |iteration|
+      sender = 'sender_' + iteration.to_s
+      tradesman = WorkerRoulette.tradesman
+      tradesman.work_orders!
+    end
+  end
+end
+puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Read/Writes per second"
+puts "#################"
+puts
+WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
+
+times = Benchmark.bm do |x|
+  x.report "#{ITERATIONS * 2} Sync Api Polling Read/Writes" do
+    WorkerRoulette.start(size: REDIS_CONNECTION_POOL_SIZE, evented: false)
+    ITERATIONS.times do |iteration|
+      sender = 'sender_' + iteration.to_s
+      foreman = WorkerRoulette.foreman(sender)
+      foreman.enqueue_work_order(work_order)
+      tradesman = WorkerRoulette.tradesman
+      tradesman.wait_for_work_orders {|m| m}
+    end
+  end
+end
+puts "#{(ITERATIONS * 2 / times.first.real).to_i} Sync Api Polling Read/Writes per second"
+puts "#################"
+puts
+WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}

data/spec/helpers/.gitkeep

@@ -0,0 +1 @@
+.gitkeep

data/spec/integration/evented_worker_roulette_spec.rb

@@ -0,0 +1,250 @@
+require "spec_helper"
+module WorkerRoulette
+  describe WorkerRoulette do
+    include EventedSpec::EMSpec
+
+    let(:sender) {'katie_80'}
+    let(:work_orders) {["hello", "foreman"]}
+    let(:default_headers) {Hash['headers' => {'sender' => sender}]}
+    let(:hello_work_order) {Hash['payload' => "hello"]}
+    let(:foreman_work_order) {Hash['payload' => "foreman"]}
+    let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
+    let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
+    let(:worker_roulette) { WorkerRoulette.start(evented: true) }
+    let(:redis) {Redis.new(worker_roulette.redis_config)}
+
+    context "Evented Foreman" do
+      let(:subject) {worker_roulette.foreman(sender)}
+
+      it "enqueues work" do
+        called = false
+        foreman = worker_roulette.foreman('foreman')
+        foreman.enqueue_work_order('some old fashion work') do |redis_response, stuff|
+          called = true
+        end
+        done(0.1) { expect(called).to be_truthy }
+      end
+
+      it "should enqueue_work_order two work_orders in the sender's slot in the job board" do
+        subject.enqueue_work_order(work_orders.first) do
+          subject.enqueue_work_order(work_orders.last) do
+            expected = work_orders.map { |m| WorkerRoulette.dump(default_headers.merge({'payload' => m})) }
+            expect(redis.lrange(sender, 0, -1)).to eq(expected)
+            done
+          end
+        end
+      end
+
+      it "should enqueue_work_order an array of work_orders without headers in the sender's slot in the job board" do
+        subject.enqueue_work_order_without_headers(work_orders) do
+          expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders)])
+          done
+        end
+      end
+
+      it "should enqueue_work_order an array of work_orders with default headers in the sender's slot in the job board" do
+        subject.enqueue_work_order(work_orders) do
+          expect(redis.lrange(sender, 0, -1)).to eq(jsonized_work_orders_with_headers)
+          done
+        end
+      end
+
+      it "should enqueue_work_order an array of work_orders with additional headers in the sender's slot in the job board" do
+        extra_headers = {'foo' => 'bars'}
+        subject.enqueue_work_order(work_orders, extra_headers) do
+          work_orders_with_headers['headers'].merge!(extra_headers)
+          expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders_with_headers)])
+          done
+        end
+      end
+
+      it "should post the sender's id to the job board with an order number" do
+        first_foreman      = worker_roulette.foreman('first_foreman')
+        first_foreman.enqueue_work_order('foo') do
+          subject.enqueue_work_order(work_orders.first) do
+            subject.enqueue_work_order(work_orders.last) do
+              expect(redis.zrange(subject.job_board_key, 0, -1, with_scores: true)).to eq([["first_foreman", 1.0], ["katie_80", 2.0]])
+              done
+            end
+          end
+        end
+      end
+
+      it "should generate a monotically increasing score for senders not on the job board, but not for senders already there" do
+        first_foreman = worker_roulette.foreman('first_foreman')
+        expect(redis.get(subject.counter_key)).to be_nil
+        first_foreman.enqueue_work_order(work_orders.first) do
+          expect(redis.get(subject.counter_key)).to eq("1")
+          first_foreman.enqueue_work_order(work_orders.last) do
+            expect(redis.get(subject.counter_key)).to eq("1")
+            subject.enqueue_work_order(work_orders.first) do
+              expect(redis.get(subject.counter_key)).to eq("2")
+              done
+            end
+          end
+        end
+      end
+    end
+
+    context "Evented Tradesman" do
+      let(:foreman) {worker_roulette.foreman(sender)}
+      let(:subject)  {worker_roulette.tradesman(nil, 0.01) }
+
+      it "should be working on behalf of a sender" do
+        foreman.enqueue_work_order(work_orders) do
+          subject.work_orders! do |r|
+            expect(subject.last_sender).to eq(sender)
+            done
+          end
+        end
+      end
+
+      it "should remove the lock from the last_sender's queue" do
+        most_recent_sender = 'most_recent_sender'
+        most_recent_foreman = worker_roulette.foreman(most_recent_sender)
+        other_foreman = worker_roulette.foreman('katie_80')
+
+        other_foreman.enqueue_work_order(work_orders) do
+          most_recent_foreman.enqueue_work_order(work_orders) do
+            expect(redis.keys("L*:*").length).to eq(0)
+            subject.work_orders! do
+              expect(redis.get("L*:katie_80")).to eq("1")
+              expect(redis.keys("L*:*").length).to eq(1)
+              subject.work_orders! do
+                expect(redis.keys("L*:*").length).to eq(1)
+                expect(redis.get("L*:most_recent_sender")).to eq("1")
+                subject.work_orders!
+                done(0.2) do
+                  expect(redis.keys("L*:*").length).to eq(0)
+                end
+              end
+            end
+          end
+        end
+      end
+
+      it "should drain one set of work_orders from the sender's slot in the job board" do
+        foreman.enqueue_work_order(work_orders) do
+          subject.work_orders! do |r|
+            expect(r).to eq([work_orders_with_headers])
+            subject.work_orders! do |r| expect(r).to be_empty
+              subject.work_orders! {|r| expect(r).to be_empty; done} #does not throw an error if queue is alreay empty
+            end
+          end
+        end
+      end
+
+      it "should take the oldest sender off the job board (FIFO)" do
+        foreman.enqueue_work_order(work_orders) do
+          oldest_sender = sender.to_s
+          most_recent_sender = 'most_recent_sender'
+          most_recent_foreman = worker_roulette.foreman(most_recent_sender)
+          most_recent_foreman.enqueue_work_order(work_orders) do
+            expect(redis.zrange(subject.job_board_key, 0, -1)).to eq([oldest_sender, most_recent_sender])
+            subject.work_orders! { expect(redis.zrange(subject.job_board_key, 0, -1)).to eq([most_recent_sender]); done }
+          end
+        end
+      end
+
+      it "should get the work_orders from the next queue when a new job is ready" do
+        #tradesman polls every so often, we care that it is called at least twice, but did not use
+        #the built in rspec syntax for that bc if the test ends while we're talking to redis, redis
+        #throws an Error. This way we ensure we call work_orders! at least twice and just stub the second
+        #call so as not to hurt redis' feelings.
+
+        expect(subject).to receive(:work_orders!).and_call_original
+        expect(subject).to receive(:work_orders!)
+
+        foreman.enqueue_work_order(work_orders) do
+          subject.wait_for_work_orders do |redis_work_orders|
+            expect(redis_work_orders).to eq([work_orders_with_headers])
+            expect(subject.last_sender).to match(/katie_80/)
+            done(0.1)
+          end
+        end
+      end
+
+      it "should publish and subscribe on custom channels" do
+        good_subscribed   = false
+        bad_subscribed    = false
+
+        tradesman         = worker_roulette.tradesman('good_channel', 0.001)
+        evil_tradesman    = worker_roulette.tradesman('bad_channel', 0.001)
+
+        good_foreman      = worker_roulette.foreman('foreman', 'good_channel')
+        bad_foreman       = worker_roulette.foreman('foreman', 'bad_channel')
+
+        #tradesman polls every so often, we care that it is called at least twice, but did not use
+        #the built in rspec syntax for that bc if the test ends while we're talking to redis, redis
+        #throws an Error. This way we ensure we call work_orders! at least twice and just stub the second
+        #call so as not to hurt redis' feelings.
+        expect(tradesman).to       receive(:work_orders!).and_call_original
+        expect(tradesman).to       receive(:work_orders!)
+
+        expect(evil_tradesman).to  receive(:work_orders!).and_call_original
+        expect(evil_tradesman).to  receive(:work_orders!)
+
+        good_foreman.enqueue_work_order('some old fashion work') do
+          bad_foreman.enqueue_work_order('evil biddings you should not carry out') do
+
+            tradesman.wait_for_work_orders do |good_work|
+              expect(good_work.to_s).to match("old fashion")
+              expect(good_work.to_s).not_to match("evil")
+            end
+
+            evil_tradesman.wait_for_work_orders do |bad_work|
+              expect(bad_work.to_s).not_to match("old fashion")
+              expect(bad_work.to_s).to match("evil")
+            end
+            done(0.1)
+
+          end
+        end
+      end
+
+      it "should pull off work orders for more than one sender" do
+        tradesman = worker_roulette.tradesman('good_channel')
+
+        good_foreman = worker_roulette.foreman('good_foreman', 'good_channel')
+        lazy_foreman = worker_roulette.foreman('lazy_foreman', 'good_channel')
+
+        got_good = false
+        got_lazy  = false
+        good_foreman.enqueue_work_order('do good work') do
+          tradesman.work_orders! do |r|
+            got_good = true
+            expect(r.first['payload']).to eq('do good work')
+          end
+        end
+        lazy_foreman.enqueue_work_order('just get it done') do
+          tradesman.work_orders! do |r|
+            got_lazy = true
+            expect(r.first['payload']).to eq('just get it done')
+          end
+        end
+
+        done(0.2) {expect(got_good && got_lazy).to eq(true)}
+      end
+    end
+
+    pending "should return a hash with a string in the payload if OJ cannot parse the json"
+
+    context "Failure" do
+      it "should not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; done; end
+    end
+
+    context "Concurrent Access" do
+      it "should not leak connections"
+
+      it "should be fork() proof" do
+        @subject = worker_roulette.tradesman
+        @subject.work_orders! do
+          fork do
+            @subject.work_orders!
+          end
+        end
+        done(1)
+      end
+    end
+  end
+end

data/spec/integration/worker_roulette_spec.rb

@@ -0,0 +1,172 @@
+require "spec_helper"
+module WorkerRoulette
+  describe WorkerRoulette do
+    let(:sender) {'katie_80'}
+    let(:work_orders) {["hello", "foreman"]}
+    let(:default_headers) {Hash['headers' => {'sender' => sender}]}
+    let(:hello_work_order) {Hash['payload' => "hello"]}
+    let(:foreman_work_order) {Hash['payload' => "foreman"]}
+    let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
+    let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
+    let(:worker_roulette) { WorkerRoulette.start }
+
+    let(:redis) {Redis.new(worker_roulette.redis_config)}
+
+    before do
+      redis.flushall
+    end
+
+    it "should exist" do
+      expect(worker_roulette).to be_instance_of(WorkerRoulette)
+    end
+
+    context Foreman do
+      let(:subject) {worker_roulette.foreman(sender)}
+
+      it "should be working on behalf of a sender" do
+        expect(subject.sender).to eq(sender)
+      end
+
+      it "should enqueue_work_order two work_orders in the sender's work queue" do
+        subject.enqueue_work_order(work_orders.first) {}
+        subject.enqueue_work_order(work_orders.last) {}
+        expect(redis.lrange(sender, 0, -1)).to eq(work_orders.map {|m| WorkerRoulette.dump(default_headers.merge({'payload' => m})) })
+      end
+
+      it "should enqueue_work_order an array of work_orders without headers in the sender's work queue" do
+        subject.enqueue_work_order_without_headers(work_orders)
+        expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders)])
+      end
+
+      it "should enqueue_work_order an array of work_orders with default headers in the sender's work queue" do
+        subject.enqueue_work_order(work_orders)
+        expect(redis.lrange(sender, 0, -1)).to eq(jsonized_work_orders_with_headers)
+      end
+
+      it "should enqueue_work_order an array of work_orders with additional headers in the sender's work queue" do
+        extra_headers = {'foo' => 'bars'}
+        subject.enqueue_work_order(work_orders, extra_headers)
+        work_orders_with_headers['headers'].merge!(extra_headers)
+        expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders_with_headers)])
+      end
+
+      it "should post the sender's id to the job board with an order number" do
+        subject.enqueue_work_order(work_orders.first)
+        worker_roulette.foreman('other_forman').enqueue_work_order(work_orders.last)
+        expect(redis.zrange(subject.job_board_key, 0, -1, with_scores: true)).to eq([[sender, 1.0], ["other_forman", 2.0]])
+      end
+
+      it "should generate a monotically increasing score for senders not on the job board, but not for senders already there" do
+        other_forman = worker_roulette.foreman('other_forman')
+        expect(redis.get(subject.counter_key)).to be_nil
+        subject.enqueue_work_order(work_orders.first)
+        expect(redis.get(subject.counter_key)).to eq("1")
+        subject.enqueue_work_order(work_orders.last)
+        expect(redis.get(subject.counter_key)).to eq("1")
+        other_forman.enqueue_work_order(work_orders.last)
+        expect(redis.get(other_forman.counter_key)).to eq("2")
+      end
+    end
+
+    context Tradesman do
+      let(:foreman) { worker_roulette.foreman(sender) }
+      let(:tradesman) { worker_roulette.tradesman }
+
+      before do
+        foreman.enqueue_work_order(work_orders)
+      end
+
+      context 'removing locks from queues' do
+        it "for the last_sender's queue" do
+          most_recent_sender = 'most_recent_sender'
+          most_recent_foreman = worker_roulette.foreman(most_recent_sender)
+          most_recent_foreman.enqueue_work_order(work_orders)
+          expect(redis.keys("L*:*").length).to eq(0)
+          tradesman.work_orders!
+          expect(redis.get("L*:katie_80")).to eq("1")
+          expect(redis.keys("L*:*").length).to eq(1)
+          tradesman.work_orders!
+          expect(redis.keys("L*:*").length).to eq(1)
+          expect(redis.get("L*:most_recent_sender")).to eq("1")
+          tradesman.work_orders!
+          expect(redis.keys("L*:*").length).to eq(0)
+        end
+      end
+
+      it "should have a last sender if it found messages" do
+        expect(tradesman.work_orders!.length).to eq(1)
+        expect(tradesman.last_sender).to eq(sender)
+      end
+
+      it "should not have a last sender if it found no messages" do
+        expect(tradesman.work_orders!.length).to eq(1)
+        expect(tradesman.work_orders!.length).to eq(0)
+        expect(tradesman.last_sender).to be_nil
+      end
+
+      it "should drain one set of work_orders from the sender's work queue" do
+        expect(tradesman.work_orders!).to eq([work_orders_with_headers])
+        expect(tradesman.work_orders!).to be_empty
+        expect(tradesman.work_orders!).to be_empty #does not throw an error if queue is already empty
+      end
+
+      it "should drain all the work_orders from the sender's work queue" do
+        foreman.enqueue_work_order(work_orders)
+        expect(tradesman.work_orders!).to eq([work_orders_with_headers, work_orders_with_headers])
+        expect(tradesman.work_orders!).to be_empty
+        expect(tradesman.work_orders!).to be_empty #does not throw an error if queue is already empty
+      end
+
+      it "should take the oldest sender off the job board (FIFO)" do
+        oldest_sender = sender.to_s
+        most_recent_sender = 'most_recent_sender'
+        most_recent_foreman = worker_roulette.foreman(most_recent_sender)
+        most_recent_foreman.enqueue_work_order(work_orders)
+        expect(redis.zrange(tradesman.job_board_key, 0, -1)).to eq([oldest_sender, most_recent_sender])
+        tradesman.work_orders!
+        expect(redis.zrange(tradesman.job_board_key, 0, -1)).to eq([most_recent_sender])
+      end
+
+      it "should get the work_orders from the next queue when a new job is ready, then poll for new work" do
+        tradesman.wait_for_work_orders do |redis_work_orders|
+          expect(redis_work_orders).to eq([work_orders_with_headers])
+          expect(tradesman.last_sender).to eq('katie_80')
+          allow(tradesman).to receive(:wait_for_work_orders)
+        end
+      end
+
+      it "should publish and subscribe on custom channels" do
+        tradesman         = worker_roulette.tradesman('good_channel')
+        expect(tradesman).to receive(:work_orders!).and_call_original
+
+        good_foreman      = worker_roulette.foreman('foreman', 'good_channel')
+        bad_foreman       = worker_roulette.foreman('foreman', 'bad_channel')
+
+        good_foreman.enqueue_work_order('some old fashion work')
+        bad_foreman.enqueue_work_order('evil biddings you should not carry out')
+
+        tradesman.wait_for_work_orders do |work|
+          expect(work.to_s).to match("some old fashion work")
+          expect(work.to_s).not_to match("evil")
+          expect(tradesman.last_sender).to eq('foreman')
+          allow(tradesman).to receive(:wait_for_work_orders)
+        end
+      end
+
+      context "Failure" do
+        it "should not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; end
+      end
+
+      context "Concurrent Access" do
+        it "should pool its connections" do
+          Array.new(100) do
+            Thread.new {worker_roulette.tradesman_connection_pool.with {|pooled_redis| pooled_redis.get("foo")}}
+          end.each(&:join)
+          worker_roulette.tradesman_connection_pool.with do |pooled_redis|
+            expect(pooled_redis.info["connected_clients"].to_i).to be > (worker_roulette.pool_size)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+module WorkerRoulette
+  require 'worker_roulette'
+  require 'evented-spec'
+  require 'rspec'
+  require 'pry'
+
+  require File.expand_path(File.join("..", "..", "lib", "worker_roulette.rb"), __FILE__)
+
+  Dir[File.join(File.dirname(__FILE__), 'helpers', '**/*.rb')].sort.each { |file| require file.gsub(".rb", "")}
+
+  EM::Hiredis.reconnect_timeout = 0.01
+
+  RSpec.configure do |c|
+    c.after(:each) do
+      Redis.new(WorkerRoulette.start.redis_config).flushdb
+    end
+  end
+end

data/spec/unit/evented_readlock_spec.rb

@@ -0,0 +1,108 @@
+require 'spec_helper'
+module WorkerRoulette
+  describe "Evented Read Lock" do
+    include EventedSpec::EMSpec
+
+    let(:redis) {Redis.new(WorkerRoulette.start.redis_config)}
+    let(:sender) {'katie_80'}
+    let(:work_orders) {"hellot"}
+    let(:lock_key) {"L*:#{sender}"}
+    let(:default_headers) {Hash['headers' => {'sender' => sender}]}
+    let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
+    let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
+    let(:worker_roulette) { WorkerRoulette.start(evented: true) }
+    let(:foreman) {worker_roulette.foreman(sender)}
+    let(:number_two) {worker_roulette.foreman('number_two')}
+    let(:subject) {worker_roulette.tradesman}
+    let(:subject_two) {worker_roulette.tradesman}
+    let(:lua) { Lua.new(worker_roulette.tradesman_connection_pool) }
+
+    em_before do
+      lua.clear_cache!
+      redis.script(:flush)
+      redis.flushdb
+    end
+
+    it "should lock a queue when it reads from it" do
+      evented_readlock_preconditions do
+        expect(redis.get(lock_key)).not_to be_nil
+        done
+      end
+    end
+
+    it "should set the lock to expire in 3 seconds" do
+      evented_readlock_preconditions do
+        expect(redis.ttl(lock_key)).to eq(3)
+        done
+      end
+    end
+
+    it "should not read a locked queue" do
+      evented_readlock_preconditions do
+        foreman.enqueue_work_order(work_orders) do #locked
+          subject_two.work_orders! { |work| expect(work).to be_empty; done}
+        end
+      end
+    end
+
+    it "should read from the first available queue that is not locked" do
+      evented_readlock_preconditions do
+        foreman.enqueue_work_order(work_orders) do    #locked
+          number_two.enqueue_work_order(work_orders) do  #unlocked
+            subject_two.work_orders!{|work| expect(work.first['headers']['sender']).to eq('number_two'); done}
+          end
+        end
+      end
+    end
+
+    it "should release its last lock when it asks for its next work order from another sender" do
+      evented_readlock_preconditions do
+        number_two.enqueue_work_order(work_orders) do #unlocked
+          expect(subject.last_sender).to eq(sender)
+          subject.work_orders! do |work|
+            expect(work.first['headers']['sender']).to eq('number_two')
+            expect(redis.get(lock_key)).to be_nil
+            done
+          end
+        end
+      end
+    end
+
+    it "should not release its lock when it asks for its next work order from the same sender" do
+      evented_readlock_preconditions do
+        foreman.enqueue_work_order(work_orders) do #locked
+          subject.work_orders! do |work|
+            expect(work).to eq([work_orders_with_headers])
+            expect(subject.last_sender).to eq(sender)
+            expect(redis.get(lock_key)).not_to be_nil
+            done
+          end
+        end
+      end
+    end
+
+    it "should not take out another lock if there is no work to do" do
+      evented_readlock_preconditions do
+        foreman.enqueue_work_order(work_orders) do #locked
+          subject.work_orders! do |work_order|
+            expect(work_order).to eq([work_orders_with_headers])
+            subject.work_orders! do |work|
+              expect(work).to be_empty
+              expect(redis.get(lock_key)).to be_nil
+              done
+            end
+          end
+        end
+      end
+    end
+
+    def evented_readlock_preconditions(&spec_block)
+      foreman.enqueue_work_order(work_orders) do
+        subject.work_orders! do |work|
+          expect(work).to eq([work_orders_with_headers])
+          spec_block.call
+        end
+      end
+    end
+  end
+end