nexia_worker_roulette 0.1.12 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f0f447f3680d7e1b31c9bb64288bfcd272e06e0b
-  data.tar.gz: 37a831cc363e6ecab5aee296d4fb0790e05f2900
+  metadata.gz: 22482d641cdf09efcb854a5a859f807399292ac5
+  data.tar.gz: 51850f1cd0f76e98a4a6c5bc8dc1c87c6f70eb98
 SHA512:
-  metadata.gz: f60bf31b75564128c79229d3504e4b7f1b22d79f20be3b9c4ed9845685c917d7fd7c478b277aead096a58dc4d2355f11b855edb411de12b2a130ba2bbb7bc0ab
-  data.tar.gz: 268b3765ffe7bc6dba03668a6a43184607640dfdd4a8376b157374bbbadc4a35c443f0155bc7e961c97f923aaf51dc377e004e89e8d6076924142ce0af4823c2
+  metadata.gz: 64d5fd31dde77cc677d0e3d293df55c7fb5353594e4755dd501cf4bffcf475df0845a208fbf183bd78a43d37b40088e30a2aee1767f68cd9169f9f09c31003e2
+  data.tar.gz: 04f9dfcec66f9e09e57bde1004df9e8c44925bb52da04c31d629eb761252c7505d64c5a79af4c51f57ff0c6e036178b903a46f2482218624beffeb0c9e8a4053

data/lib/worker_roulette/foreman.rb

@@ -1,6 +1,10 @@
+require_relative "preprocessor"
+
 module WorkerRoulette
   class Foreman
-    attr_reader :sender, :namespace, :channel
+    include Preprocessor
+
+    attr_reader :sender, :namespace, :channel, :preprocessors
 
     LUA_ENQUEUE_WORK_ORDERS = <<-HERE
       local counter_key       = KEYS[1]
@@ -32,22 +36,23 @@ module WorkerRoulette
       enqueue_work_orders(work_order, job_notification)
     HERE
 
-    def initialize(redis_pool, sender, namespace = nil)
-      @sender     = sender
-      @namespace  = namespace
-      @redis_pool = redis_pool
-      @channel    = namespace || WorkerRoulette::JOB_NOTIFICATIONS
-      @lua        = Lua.new(@redis_pool)
+    def initialize(redis_pool, sender, namespace = nil, preprocessors = [])
+      @redis_pool    = redis_pool
+      @sender        = sender
+      @preprocessors = preprocessors
+      @namespace     = namespace
+      @channel       = namespace || WorkerRoulette::JOB_NOTIFICATIONS
+      @lua           = Lua.new(@redis_pool)
     end
 
     def enqueue_work_order(work_order, headers = {}, &callback)
       work_order = {'headers' => default_headers.merge(headers), 'payload' => work_order}
-      enqueue_work_order_without_headers(work_order, &callback)
+      enqueue(work_order, &callback)
     end
 
-    def enqueue_work_order_without_headers(work_order, &callback)
+    def enqueue(work_order, &callback)
       @lua.call(LUA_ENQUEUE_WORK_ORDERS, [counter_key, job_board_key, sender_key, @channel],
-                [WorkerRoulette.dump(work_order),  WorkerRoulette::JOB_NOTIFICATIONS], &callback)
+                [WorkerRoulette.dump(preprocess(work_order, channel)),  WorkerRoulette::JOB_NOTIFICATIONS], &callback)
     end
 
     def job_board_key
@@ -65,7 +70,8 @@ module WorkerRoulette
     private
 
     def default_headers
-      Hash['sender' => sender]
+      { "sender" => sender }
     end
+
   end
 end

data/lib/worker_roulette/preprocessor.rb

@@ -0,0 +1,15 @@
+module WorkerRoulette
+  module Preprocessor
+    def preprocess(work_order, channel)
+      return work_order unless preprocessors.any?
+
+      class_name = self.class.name.split(/::/).last
+
+      preprocessors.inject(work_order) do |job, processor_module|
+        processor_class = processor_module.const_get(class_name)
+        processor = processor_class.new
+        processor.process(job, channel)
+      end
+    end
+  end
+end

data/lib/worker_roulette/queue_latency_tracker.rb

@@ -0,0 +1,47 @@
+module QueueLatencyTracker
+  GRANULARITY = 1_000_000
+
+  class Foreman
+    def process(work_order, _channel)
+      work_order['headers'].merge!("queued_at" => (Time.now.to_f * GRANULARITY).to_i)
+      work_order
+    end
+  end
+
+  class Tradesman
+    def process(work_order, channel)
+      send_latency(work_order["headers"]["queued_at"], channel)
+      work_order
+    end
+
+    def send_latency(queued_at, channel)
+      latency_ns = (Time.now.to_f * GRANULARITY).to_i - queued_at
+      logstash_send(latency_json(latency_ns / 1000.0, channel))
+    end
+
+    def logstash_send(json)
+      UDPSocket.new.send(json, 0, config[:logstash][:server_ip], config[:logstash][:port])
+    end
+
+    def latency_json(latency_ms, channel)
+      %({"server_name":"#{config[:server_name]}","queue_latency (ms)":#{latency_ms},"channel":"#{channel}"})
+    end
+
+    def config
+      QueueLatencyTracker.config
+    end
+  end
+
+  class << self
+    attr_reader :config
+    def configure(config)
+      @config = {
+        logstash: {
+          server_ip: config[:logstash_server_ip],
+          port: config[:logstash_port] },
+        server_name: config[:server_name]
+      }
+    end
+
+  end
+end

data/lib/worker_roulette/tradesman.rb

@@ -1,6 +1,9 @@
+require_relative "preprocessor"
+
 module WorkerRoulette
   class Tradesman
-    attr_reader :last_sender, :remaining_jobs, :timer
+    include Preprocessor
+    attr_reader :last_sender, :remaining_jobs, :timer, :preprocessors, :channel
 
     LUA_DRAIN_WORK_ORDERS = <<-HERE
       local empty_string      = ""
@@ -102,9 +105,10 @@ module WorkerRoulette
       return drain_work_orders_for_sender(job_board_key, sender_key)
     THERE
 
-    def initialize(redis_pool, evented, namespace = nil, polling_time = WorkerRoulette::DEFAULT_POLLING_TIME)
+    def initialize(redis_pool, evented, namespace = nil, polling_time = WorkerRoulette::DEFAULT_POLLING_TIME, preprocessors = [])
       @evented        = evented
       @polling_time   = polling_time
+      @preprocessors  = preprocessors
       @redis_pool     = redis_pool
       @namespace      = namespace
       @channel        = namespace || WorkerRoulette::JOB_NOTIFICATIONS
@@ -130,7 +134,7 @@ module WorkerRoulette
         work_orders     = results[1]
         @remaining_jobs = results[2]
         @last_sender    = sender_key.split(':').last
-        work            = work_orders.map {|work_order| WorkerRoulette.load(work_order)}
+        work = work_orders.map { |wo| preprocess(WorkerRoulette.load(wo), channel) }
         callback.call work if callback
         work
       end

data/lib/worker_roulette/version.rb

@@ -1,3 +1,3 @@
 module WorkerRoulette
-  VERSION = '0.1.12'
+  VERSION = '0.2.0'
 end

data/lib/worker_roulette.rb

@@ -13,6 +13,16 @@ module WorkerRoulette
     JOB_BOARD = "job_board"
     JOB_NOTIFICATIONS = "new_job_ready"
     DEFAULT_POLLING_TIME = 2
+    DEFAULT_REDIS_CONFIG = {
+      host: 'localhost',
+      port: 6379,
+      db: 14,
+      driver: :hiredis,
+      timeout: 5,
+      evented: false,
+      pool_size: 10,
+      polling_time: DEFAULT_POLLING_TIME
+    }
 
     def self.dump(obj)
       Oj.dump(obj)
@@ -44,25 +54,45 @@ module WorkerRoulette
     end
 
     private_class_method :new
+    attr_reader :preprocessors
 
     def initialize(config = {})
-      @redis_config               = { host: 'localhost', port: 6379, db: 14, driver: :hiredis, timeout: 5, evented: false, pool_size: 10 , polling_time: DEFAULT_POLLING_TIME}.merge(config)
-      @pool_config                = Hash[size: @redis_config.delete(:pool_size), timeout: @redis_config.delete(:timeout)]
+      @redis_config               = DEFAULT_REDIS_CONFIG.merge(config)
+      @pool_config                = { size: @redis_config.delete(:pool_size), timeout: @redis_config.delete(:timeout) }
       @evented                    = @redis_config.delete(:evented)
       @polling_time               = @redis_config.delete(:polling_time)
 
       @foreman_connection_pool    = ConnectionPool.new(@pool_config) {new_redis}
       @tradesman_connection_pool  = ConnectionPool.new(@pool_config) {new_redis}
+
+      @preprocessors = []
+
+      configure_latency_tracker(config.delete(:latency_tracker))
+    end
+
+    def configure_latency_tracker(config)
+      puts "WR:config: #{config}"
+      return unless config
+
+      QueueLatencyTracker.configure(
+        {
+          server_name: `hostname`.chomp,
+          logstash_server_ip: Resolv::DNS.new.getaddress(config[:logstash_server_name]).to_s,
+          logstash_port: config[:logstash_port]
+        }
+      )
+
+      preprocessors << QueueLatencyTracker
     end
 
     def foreman(sender, namespace = nil)
       raise "WorkerRoulette not Started" unless @foreman_connection_pool
-      Foreman.new(@foreman_connection_pool, sender, namespace)
+      Foreman.new(@foreman_connection_pool, sender, namespace, preprocessors)
     end
 
     def tradesman(namespace = nil, polling_time = DEFAULT_POLLING_TIME)
       raise "WorkerRoulette not Started" unless @tradesman_connection_pool
-      Tradesman.new(@tradesman_connection_pool, @evented, namespace, polling_time || @polling_time)
+      Tradesman.new(@tradesman_connection_pool, @evented, namespace, polling_time || @polling_time, preprocessors)
     end
 
     def tradesman_connection_pool

data/spec/integration/evented_worker_roulette_spec.rb

@@ -1,84 +1,90 @@
 require "spec_helper"
+
 module WorkerRoulette
   describe WorkerRoulette do
     include EventedSpec::EMSpec
 
-    let(:sender) {'katie_80'}
-    let(:work_orders) {["hello", "foreman"]}
-    let(:default_headers) {Hash['headers' => {'sender' => sender}]}
-    let(:hello_work_order) {Hash['payload' => "hello"]}
-    let(:foreman_work_order) {Hash['payload' => "foreman"]}
-    let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
-    let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
-    let(:worker_roulette) { WorkerRoulette.start(evented: true) }
-    let(:redis) {Redis.new(worker_roulette.redis_config)}
+    let(:sender)                            { "katie_80" }
+    let(:work_orders)                       { ["hello", "foreman"] }
+    let(:queued_at)                         { 1234567 }
+    let(:default_headers)                   { Hash["headers" => { "sender" => sender, "queued_at" => (queued_at.to_f * 1_000_000).to_i }] }
+    let(:hello_work_order)                  { Hash["payload" => "hello"] }
+    let(:foreman_work_order)                { Hash["payload" => "foreman"] }
+    let(:work_orders_with_headers)          { default_headers.merge({ "payload" => work_orders }) }
+    let(:jsonized_work_orders_with_headers) { [WorkerRoulette.dump(work_orders_with_headers)] }
+    let(:worker_roulette)                   { WorkerRoulette.start(evented: true) }
+    let(:redis)                             { Redis.new(worker_roulette.redis_config) }
+
+    before do
+      allow(Time).to receive(:now).and_return(queued_at)
+    end
 
     context "Evented Foreman" do
-      let(:subject) {worker_roulette.foreman(sender)}
+      subject(:foreman) {worker_roulette.foreman(sender)}
 
       it "enqueues work" do
         called = false
-        foreman = worker_roulette.foreman('foreman')
-        foreman.enqueue_work_order('some old fashion work') do |redis_response, stuff|
+        foreman = worker_roulette.foreman("foreman")
+        foreman.enqueue_work_order("some old fashion work") do |redis_response, stuff|
           called = true
         end
         done(0.1) { expect(called).to be_truthy }
       end
 
-      it "should enqueue_work_order two work_orders in the sender's slot in the job board" do
-        subject.enqueue_work_order(work_orders.first) do
-          subject.enqueue_work_order(work_orders.last) do
-            expected = work_orders.map { |m| WorkerRoulette.dump(default_headers.merge({'payload' => m})) }
+      it "enqueues two work_orders in the sender's slot in the job board" do
+        foreman.enqueue_work_order(work_orders.first) do
+          foreman.enqueue_work_order(work_orders.last) do
+            expected = work_orders.map { |m| WorkerRoulette.dump(default_headers.merge({"payload" => m})) }
             expect(redis.lrange(sender, 0, -1)).to eq(expected)
             done
           end
         end
       end
 
-      it "should enqueue_work_order an array of work_orders without headers in the sender's slot in the job board" do
-        subject.enqueue_work_order_without_headers(work_orders) do
+      it "enqueues an array of work_orders without headers in the sender's slot in the job board" do
+        foreman.enqueue(work_orders) do
           expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders)])
           done
         end
       end
 
-      it "should enqueue_work_order an array of work_orders with default headers in the sender's slot in the job board" do
-        subject.enqueue_work_order(work_orders) do
+      it "enqueues an array of work_orders with default headers in the sender's slot in the job board" do
+        foreman.enqueue_work_order(work_orders) do
           expect(redis.lrange(sender, 0, -1)).to eq(jsonized_work_orders_with_headers)
           done
         end
       end
 
-      it "should enqueue_work_order an array of work_orders with additional headers in the sender's slot in the job board" do
-        extra_headers = {'foo' => 'bars'}
-        subject.enqueue_work_order(work_orders, extra_headers) do
-          work_orders_with_headers['headers'].merge!(extra_headers)
+      it "enqueues an array of work_orders with additional headers in the sender's slot in the job board" do
+        extra_headers = {"foo" => "bars"}
+        foreman.enqueue_work_order(work_orders, extra_headers) do
+          work_orders_with_headers["headers"].merge!(extra_headers)
           expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders_with_headers)])
           done
         end
       end
 
-      it "should post the sender's id to the job board with an order number" do
-        first_foreman      = worker_roulette.foreman('first_foreman')
-        first_foreman.enqueue_work_order('foo') do
-          subject.enqueue_work_order(work_orders.first) do
-            subject.enqueue_work_order(work_orders.last) do
-              expect(redis.zrange(subject.job_board_key, 0, -1, with_scores: true)).to eq([["first_foreman", 1.0], ["katie_80", 2.0]])
+      it "posts the sender's id to the job board with an order number" do
+        first_foreman      = worker_roulette.foreman("first_foreman")
+        first_foreman.enqueue_work_order("foo") do
+          foreman.enqueue_work_order(work_orders.first) do
+            foreman.enqueue_work_order(work_orders.last) do
+              expect(redis.zrange(foreman.job_board_key, 0, -1, with_scores: true)).to eq([["first_foreman", 1.0], ["katie_80", 2.0]])
               done
             end
           end
         end
       end
 
-      it "should generate a monotically increasing score for senders not on the job board, but not for senders already there" do
-        first_foreman = worker_roulette.foreman('first_foreman')
-        expect(redis.get(subject.counter_key)).to be_nil
+      it "generates a monotically increasing score for senders not on the job board, but not for senders already there" do
+        first_foreman = worker_roulette.foreman("first_foreman")
+        expect(redis.get(foreman.counter_key)).to be_nil
         first_foreman.enqueue_work_order(work_orders.first) do
-          expect(redis.get(subject.counter_key)).to eq("1")
+          expect(redis.get(foreman.counter_key)).to eq("1")
           first_foreman.enqueue_work_order(work_orders.last) do
-            expect(redis.get(subject.counter_key)).to eq("1")
-            subject.enqueue_work_order(work_orders.first) do
-              expect(redis.get(subject.counter_key)).to eq("2")
+            expect(redis.get(foreman.counter_key)).to eq("1")
+            foreman.enqueue_work_order(work_orders.first) do
+              expect(redis.get(foreman.counter_key)).to eq("2")
               done
             end
           end
@@ -87,33 +93,33 @@ module WorkerRoulette
     end
 
     context "Evented Tradesman" do
-      let(:foreman) {worker_roulette.foreman(sender)}
-      let(:subject)  {worker_roulette.tradesman(nil, 0.01) }
+      let(:foreman)       { worker_roulette.foreman(sender) }
+      subject(:tradesman) { worker_roulette.tradesman(nil, 0.01) }
 
-      it "should be working on behalf of a sender" do
+      it "works on behalf of a sender" do
         foreman.enqueue_work_order(work_orders) do
-          subject.work_orders! do |r|
-            expect(subject.last_sender).to eq(sender)
+          tradesman.work_orders! do |r|
+            expect(tradesman.last_sender).to eq(sender)
             done
           end
         end
       end
 
-      it "should remove the lock from the last_sender's queue" do
-        most_recent_sender = 'most_recent_sender'
+      it "removes the lock from the last_sender's queue" do
+        most_recent_sender = "most_recent_sender"
         most_recent_foreman = worker_roulette.foreman(most_recent_sender)
-        other_foreman = worker_roulette.foreman('katie_80')
+        other_foreman = worker_roulette.foreman("katie_80")
 
         other_foreman.enqueue_work_order(work_orders) do
           most_recent_foreman.enqueue_work_order(work_orders) do
             expect(redis.keys("L*:*").length).to eq(0)
-            subject.work_orders! do
+            tradesman.work_orders! do
               expect(redis.get("L*:katie_80")).to eq("1")
               expect(redis.keys("L*:*").length).to eq(1)
-              subject.work_orders! do
+              tradesman.work_orders! do
                 expect(redis.keys("L*:*").length).to eq(1)
                 expect(redis.get("L*:most_recent_sender")).to eq("1")
-                subject.work_orders!
+                tradesman.work_orders!
                 done(0.2) do
                   expect(redis.keys("L*:*").length).to eq(0)
                 end
@@ -123,69 +129,66 @@ module WorkerRoulette
         end
       end
 
-      it "should drain one set of work_orders from the sender's slot in the job board" do
+      it "drains one set of work_orders from the sender's slot in the job board" do
         foreman.enqueue_work_order(work_orders) do
-          subject.work_orders! do |r|
-            expect(r).to eq([work_orders_with_headers])
-            subject.work_orders! do |r| expect(r).to be_empty
-              subject.work_orders! {|r| expect(r).to be_empty; done} #does not throw an error if queue is alreay empty
+          tradesman.work_orders! do |r0|
+            expect(r0).to eq([work_orders_with_headers])
+            tradesman.work_orders! do |r1| expect(r1).to be_empty
+              tradesman.work_orders! {|r2| expect(r2).to be_empty; done} #does not throw an error if queue is alreay empty
             end
           end
         end
       end
 
-      it "should take the oldest sender off the job board (FIFO)" do
+      it "takes the oldest sender off the job board (FIFO)" do
         foreman.enqueue_work_order(work_orders) do
           oldest_sender = sender.to_s
-          most_recent_sender = 'most_recent_sender'
+          most_recent_sender = "most_recent_sender"
           most_recent_foreman = worker_roulette.foreman(most_recent_sender)
           most_recent_foreman.enqueue_work_order(work_orders) do
-            expect(redis.zrange(subject.job_board_key, 0, -1)).to eq([oldest_sender, most_recent_sender])
-            subject.work_orders! { expect(redis.zrange(subject.job_board_key, 0, -1)).to eq([most_recent_sender]); done }
+            expect(redis.zrange(tradesman.job_board_key, 0, -1)).to eq([oldest_sender, most_recent_sender])
+            tradesman.work_orders! { expect(redis.zrange(tradesman.job_board_key, 0, -1)).to eq([most_recent_sender]); done }
           end
         end
       end
 
-      it "should get the work_orders from the next queue when a new job is ready" do
-        #tradesman polls every so often, we care that it is called at least twice, but did not use
-        #the built in rspec syntax for that bc if the test ends while we're talking to redis, redis
-        #throws an Error. This way we ensure we call work_orders! at least twice and just stub the second
-        #call so as not to hurt redis' feelings.
+      it "gets the work_orders from the next queue when a new job is ready" do
+        # tradesman polls every so often, we care that it is called at least twice, but did not use
+        # the built in rspec syntax for that bc if the test ends while we"re talking to redis, redis
+        # throws an Error. This way we ensure we call work_orders! at least twice and just stub the second
+        # call so as not to hurt redis" feelings.
 
-        expect(subject).to receive(:work_orders!).and_call_original
-        expect(subject).to receive(:work_orders!)
+        expect(tradesman).to receive(:work_orders!).and_call_original
+        expect(tradesman).to receive(:work_orders!)
 
         foreman.enqueue_work_order(work_orders) do
-          subject.wait_for_work_orders do |redis_work_orders|
+          tradesman.wait_for_work_orders do |redis_work_orders|
             expect(redis_work_orders).to eq([work_orders_with_headers])
-            expect(subject.last_sender).to match(/katie_80/)
+            expect(tradesman.last_sender).to match(/katie_80/)
             done(0.1)
           end
         end
       end
 
-      it "should publish and subscribe on custom channels" do
-        good_subscribed   = false
-        bad_subscribed    = false
-
-        tradesman         = worker_roulette.tradesman('good_channel', 0.001)
-        evil_tradesman    = worker_roulette.tradesman('bad_channel', 0.001)
+      it "publishes and subscribes on custom channels" do
+        tradesman         = worker_roulette.tradesman("good_channel", 0.001)
+        evil_tradesman    = worker_roulette.tradesman("bad_channel", 0.001)
 
-        good_foreman      = worker_roulette.foreman('foreman', 'good_channel')
-        bad_foreman       = worker_roulette.foreman('foreman', 'bad_channel')
+        good_foreman      = worker_roulette.foreman("foreman", "good_channel")
+        bad_foreman       = worker_roulette.foreman("foreman", "bad_channel")
 
-        #tradesman polls every so often, we care that it is called at least twice, but did not use
-        #the built in rspec syntax for that bc if the test ends while we're talking to redis, redis
-        #throws an Error. This way we ensure we call work_orders! at least twice and just stub the second
-        #call so as not to hurt redis' feelings.
+        # tradesman polls every so often, we care that it is called at least twice, but did not use
+        # the built in rspec syntax for that bc if the test ends while we"re talking to redis, redis
+        # throws an Error. This way we ensure we call work_orders! at least twice and just stub the second
+        # call so as not to hurt redis" feelings.
         expect(tradesman).to       receive(:work_orders!).and_call_original
         expect(tradesman).to       receive(:work_orders!)
 
         expect(evil_tradesman).to  receive(:work_orders!).and_call_original
         expect(evil_tradesman).to  receive(:work_orders!)
 
-        good_foreman.enqueue_work_order('some old fashion work') do
-          bad_foreman.enqueue_work_order('evil biddings you should not carry out') do
+        good_foreman.enqueue_work_order("some old fashion work") do
+          bad_foreman.enqueue_work_order("evil biddings you should not carry out") do
 
             tradesman.wait_for_work_orders do |good_work|
               expect(good_work.to_s).to match("old fashion")
@@ -202,24 +205,24 @@ module WorkerRoulette
         end
       end
 
-      it "should pull off work orders for more than one sender" do
-        tradesman = worker_roulette.tradesman('good_channel')
+      it "extracts work orders for more than one sender" do
+        tradesman = worker_roulette.tradesman("good_channel")
 
-        good_foreman = worker_roulette.foreman('good_foreman', 'good_channel')
-        lazy_foreman = worker_roulette.foreman('lazy_foreman', 'good_channel')
+        good_foreman = worker_roulette.foreman("good_foreman", "good_channel")
+        lazy_foreman = worker_roulette.foreman("lazy_foreman", "good_channel")
 
         got_good = false
         got_lazy  = false
-        good_foreman.enqueue_work_order('do good work') do
+        good_foreman.enqueue_work_order("do good work") do
           tradesman.work_orders! do |r|
             got_good = true
-            expect(r.first['payload']).to eq('do good work')
+            expect(r.first["payload"]).to eq("do good work")
           end
         end
-        lazy_foreman.enqueue_work_order('just get it done') do
+        lazy_foreman.enqueue_work_order("just get it done") do
           tradesman.work_orders! do |r|
             got_lazy = true
-            expect(r.first['payload']).to eq('just get it done')
+            expect(r.first["payload"]).to eq("just get it done")
           end
         end
 
@@ -227,20 +230,20 @@ module WorkerRoulette
       end
     end
 
-    pending "should return a hash with a string in the payload if OJ cannot parse the json"
+    pending "returns a hash with a string in the payload if OJ cannot parse the json"
 
     context "Failure" do
-      it "should not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; done; end
+      it "does not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; done; end
     end
 
     context "Concurrent Access" do
-      it "should not leak connections"
+      it "does not leak connections"
 
-      it "should be fork() proof" do
-        @subject = worker_roulette.tradesman
-        @subject.work_orders! do
+      it "is fork() proof" do
+        @tradesman = worker_roulette.tradesman
+        @tradesman.work_orders! do
           fork do
-            @subject.work_orders!
+            @tradesman.work_orders!
           end
         end
         done(1)

data/spec/integration/worker_roulette_spec.rb

@@ -5,68 +5,73 @@ module WorkerRoulette
   end
 
   describe WorkerRoulette do
-    let(:sender) {'katie_80'}
-    let(:work_orders) {["hello", "foreman"]}
-    let(:default_headers) {Hash['headers' => {'sender' => sender}]}
-    let(:hello_work_order) {Hash['payload' => "hello"]}
-    let(:foreman_work_order) {Hash['payload' => "foreman"]}
-    let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
-    let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
-    let(:worker_roulette) { WorkerRoulette.start }
-
-    let(:redis) {Redis.new(worker_roulette.redis_config)}
-
-    before do
+    let(:sender)                            { 'katie_80' }
+    let(:work_orders)                       { ["hello", "foreman"] }
+    let(:queued_at)                         { 1234567 }
+    let(:default_headers)                   { Hash["headers" => { "sender" => sender, "queued_at" => (queued_at.to_f * 1_000_000).to_i }] }
+    let(:hello_work_order)                  { Hash['payload' => "hello"] }
+    let(:foreman_work_order)                { Hash['payload' => "foreman"] }
+    let(:work_orders_with_headers)          { default_headers.merge({ 'payload' => work_orders }) }
+    let(:jsonized_work_orders_with_headers) { [WorkerRoulette.dump(work_orders_with_headers)] }
+    let(:worker_roulette)                   { WorkerRoulette.start }
+
+    let(:redis) { Redis.new(worker_roulette.redis_config) }
+
+    before :each do
       redis.flushall
     end
 
-    it "should exist" do
+    before :each do
+      allow(Time).to receive(:now).and_return(queued_at)
+    end
+
+    it "exists" do
       expect(worker_roulette).to be_instance_of(WorkerRoulette)
     end
 
     context Foreman do
-      let(:subject) {worker_roulette.foreman(sender)}
+      subject(:foreman) { worker_roulette.foreman(sender) }
 
-      it "should be working on behalf of a sender" do
-        expect(subject.sender).to eq(sender)
+      it "works on behalf of a sender" do
+        expect(foreman.sender).to eq(sender)
       end
 
-      it "should enqueue_work_order two work_orders in the sender's work queue" do
-        subject.enqueue_work_order(work_orders.first) {}
-        subject.enqueue_work_order(work_orders.last) {}
-        expect(redis.lrange(sender, 0, -1)).to eq(work_orders.map {|m| WorkerRoulette.dump(default_headers.merge({'payload' => m})) })
+      it "enqueues two work_orders in the sender's work queue" do
+        foreman.enqueue_work_order(work_orders.first) {}
+        foreman.enqueue_work_order(work_orders.last) {}
+        expect(redis.lrange(sender, 0, -1)).to eq(work_orders.map { |m| WorkerRoulette.dump(default_headers.merge({ 'payload' => m })) })
       end
 
-      it "should enqueue_work_order an array of work_orders without headers in the sender's work queue" do
-        subject.enqueue_work_order_without_headers(work_orders)
+      it "enqueues an array of work_orders without headers in the sender's work queue" do
+        foreman.enqueue(work_orders)
         expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders)])
       end
 
-      it "should enqueue_work_order an array of work_orders with default headers in the sender's work queue" do
-        subject.enqueue_work_order(work_orders)
+      it "enqueues an array of work_orders with default headers in the sender's work queue" do
+        foreman.enqueue_work_order(work_orders)
         expect(redis.lrange(sender, 0, -1)).to eq(jsonized_work_orders_with_headers)
       end
 
-      it "should enqueue_work_order an array of work_orders with additional headers in the sender's work queue" do
-        extra_headers = {'foo' => 'bars'}
-        subject.enqueue_work_order(work_orders, extra_headers)
+      it "enqueues an array of work_orders with additional headers in the sender's work queue" do
+        extra_headers = { 'foo' => 'bars' }
+        foreman.enqueue_work_order(work_orders, extra_headers)
         work_orders_with_headers['headers'].merge!(extra_headers)
         expect(redis.lrange(sender, 0, -1)).to eq([WorkerRoulette.dump(work_orders_with_headers)])
       end
 
-      it "should post the sender's id to the job board with an order number" do
-        subject.enqueue_work_order(work_orders.first)
+      it "posts the sender's id to the job board with an order number" do
+        foreman.enqueue_work_order(work_orders.first)
         worker_roulette.foreman('other_forman').enqueue_work_order(work_orders.last)
-        expect(redis.zrange(subject.job_board_key, 0, -1, with_scores: true)).to eq([[sender, 1.0], ["other_forman", 2.0]])
+        expect(redis.zrange(foreman.job_board_key, 0, -1, with_scores: true)).to eq([[sender, 1.0], ["other_forman", 2.0]])
       end
 
-      it "should generate a monotically increasing score for senders not on the job board, but not for senders already there" do
+      it "generates a monotically increasing score for senders not on the job board, but not for senders already there" do
         other_forman = worker_roulette.foreman('other_forman')
-        expect(redis.get(subject.counter_key)).to be_nil
-        subject.enqueue_work_order(work_orders.first)
-        expect(redis.get(subject.counter_key)).to eq("1")
-        subject.enqueue_work_order(work_orders.last)
-        expect(redis.get(subject.counter_key)).to eq("1")
+        expect(redis.get(foreman.counter_key)).to be_nil
+        foreman.enqueue_work_order(work_orders.first)
+        expect(redis.get(foreman.counter_key)).to eq("1")
+        foreman.enqueue_work_order(work_orders.last)
+        expect(redis.get(foreman.counter_key)).to eq("1")
         other_forman.enqueue_work_order(work_orders.last)
         expect(redis.get(other_forman.counter_key)).to eq("2")
       end
@@ -97,31 +102,31 @@ module WorkerRoulette
         end
       end
 
-      it "should have a last sender if it found messages" do
+      it "has a last sender if it found messages" do
         expect(tradesman.work_orders!.length).to eq(1)
         expect(tradesman.last_sender).to eq(sender)
       end
 
-      it "should not have a last sender if it found no messages" do
+      it "does not have a last sender if it found no messages" do
         expect(tradesman.work_orders!.length).to eq(1)
         expect(tradesman.work_orders!.length).to eq(0)
         expect(tradesman.last_sender).to be_nil
       end
 
-      it "should drain one set of work_orders from the sender's work queue" do
+      it "drains one set of work_orders from the sender's work queue" do
         expect(tradesman.work_orders!).to eq([work_orders_with_headers])
         expect(tradesman.work_orders!).to be_empty
         expect(tradesman.work_orders!).to be_empty #does not throw an error if queue is already empty
       end
 
-      it "should drain all the work_orders from the sender's work queue" do
+      it "drains all the work_orders from the sender's work queue" do
         foreman.enqueue_work_order(work_orders)
         expect(tradesman.work_orders!).to eq([work_orders_with_headers, work_orders_with_headers])
         expect(tradesman.work_orders!).to be_empty
         expect(tradesman.work_orders!).to be_empty #does not throw an error if queue is already empty
       end
 
-      it "should take the oldest sender off the job board (FIFO)" do
+      it "takes the oldest sender off the job board (FIFO)" do
         oldest_sender = sender.to_s
         most_recent_sender = 'most_recent_sender'
         most_recent_foreman = worker_roulette.foreman(most_recent_sender)
@@ -131,7 +136,7 @@ module WorkerRoulette
         expect(redis.zrange(tradesman.job_board_key, 0, -1)).to eq([most_recent_sender])
       end
 
-      it "should get the work_orders from the next queue when a new job is ready, then poll for new work" do
+      it "gets the work_orders from the next queue when a new job is ready, then poll for new work" do
         tradesman.wait_for_work_orders do |redis_work_orders|
           expect(redis_work_orders).to eq([work_orders_with_headers])
           expect(tradesman.last_sender).to eq('katie_80')
@@ -139,7 +144,14 @@ module WorkerRoulette
         end
       end
 
-      it "should publish and subscribe on custom channels" do
+      it "sees queued_at in the header" do
+        tradesman.wait_for_work_orders do |redis_work_orders|
+          expect(redis_work_orders.first["headers"]["queued_at"]).to_not be_nil
+          allow(tradesman).to receive(:wait_for_work_orders)
+        end
+      end
+
+      it "publishes and subscribes on custom channels" do
         tradesman         = worker_roulette.tradesman('good_channel')
         expect(tradesman).to receive(:work_orders!).and_call_original
 
@@ -178,13 +190,13 @@ module WorkerRoulette
       end
 
       context "Failure" do
-        it "should not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; end
+        it "does not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; end
       end
 
       context "Concurrent Access" do
-        it "should pool its connections" do
+        it "pools its connections" do
           Array.new(100) do
-            Thread.new {worker_roulette.tradesman_connection_pool.with {|pooled_redis| pooled_redis.get("foo")}}
+            Thread.new { worker_roulette.tradesman_connection_pool.with { |pooled_redis| pooled_redis.get("foo") } }
           end.each(&:join)
           worker_roulette.tradesman_connection_pool.with do |pooled_redis|
             expect(pooled_redis.info["connected_clients"].to_i).to be > (worker_roulette.pool_size)

data/spec/unit/evented_readlock_spec.rb

@@ -1,23 +1,25 @@
-require 'spec_helper'
+require "spec_helper"
 module WorkerRoulette
   describe "Evented Read Lock" do
     include EventedSpec::EMSpec
 
-    let(:redis) {Redis.new(WorkerRoulette.start.redis_config)}
-    let(:sender) {'katie_80'}
-    let(:work_orders) {"hellot"}
-    let(:lock_key) {"L*:#{sender}"}
-    let(:default_headers) {Hash['headers' => {'sender' => sender}]}
-    let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
-    let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
-    let(:worker_roulette) { WorkerRoulette.start(evented: true) }
-    let(:foreman) {worker_roulette.foreman(sender)}
-    let(:number_two) {worker_roulette.foreman('number_two')}
-    let(:subject) {worker_roulette.tradesman}
-    let(:subject_two) {worker_roulette.tradesman}
-    let(:lua) { Lua.new(worker_roulette.tradesman_connection_pool) }
+    let(:redis)                    { Redis.new(WorkerRoulette.start.redis_config) }
+    let(:sender)                   { "katie_80" }
+    let(:work_orders)              { "hellot" }
+    let(:lock_key)                 { "L*:#{sender}" }
+    let(:queued_at)                { 1234567 }
+    let(:default_headers)          { Hash["headers" => { "sender" => sender, "queued_at" => (queued_at.to_f * 1_000_000).to_i }] }
+    let(:work_orders_with_headers) { default_headers.merge({ "payload" => work_orders }) }
+    let(:worker_roulette)          { WorkerRoulette.start(evented: true) }
+    let(:foreman1)                 { worker_roulette.foreman(sender) }
+    let(:foreman2)                 { worker_roulette.foreman("foreman2") }
+    let(:tradesman2 )              { worker_roulette.tradesman }
+    let(:lua)                      { Lua.new(worker_roulette.tradesman_connection_pool) }
+
+    subject(:tradesman) {worker_roulette.tradesman}
 
     em_before do
+      allow(Time).to receive(:now).and_return(queued_at)
       lua.clear_cache!
       redis.script(:flush)
       redis.flushdb
@@ -39,17 +41,17 @@ module WorkerRoulette
 
     it "should not read a locked queue" do
       evented_readlock_preconditions do
-        foreman.enqueue_work_order(work_orders) do #locked
-          subject_two.work_orders! { |work| expect(work).to be_empty; done}
+        foreman1.enqueue_work_order(work_orders) do #locked
+          tradesman2.work_orders! { |work| expect(work).to be_empty; done}
         end
       end
     end
 
     it "should read from the first available queue that is not locked" do
       evented_readlock_preconditions do
-        foreman.enqueue_work_order(work_orders) do    #locked
-          number_two.enqueue_work_order(work_orders) do  #unlocked
-            subject_two.work_orders!{|work| expect(work.first['headers']['sender']).to eq('number_two'); done}
+        foreman1.enqueue_work_order(work_orders) do    #locked
+          foreman2.enqueue_work_order(work_orders) do  #unlocked
+            tradesman2.work_orders!{|work| expect(work.first["headers"]["sender"]).to eq("foreman2"); done}
           end
         end
       end
@@ -57,10 +59,10 @@ module WorkerRoulette
 
     it "should release its last lock when it asks for its next work order from another sender" do
       evented_readlock_preconditions do
-        number_two.enqueue_work_order(work_orders) do #unlocked
-          expect(subject.last_sender).to eq(sender)
-          subject.work_orders! do |work|
-            expect(work.first['headers']['sender']).to eq('number_two')
+        foreman2.enqueue_work_order(work_orders) do #unlocked
+          expect(tradesman.last_sender).to eq(sender)
+          tradesman.work_orders! do |work|
+            expect(work.first["headers"]["sender"]).to eq("foreman2")
             expect(redis.get(lock_key)).to be_nil
             done
           end
@@ -70,10 +72,9 @@ module WorkerRoulette
 
     it "should not release its lock when it asks for its next work order from the same sender" do
       evented_readlock_preconditions do
-        foreman.enqueue_work_order(work_orders) do #locked
-          subject.work_orders! do |work|
-            expect(work).to eq([work_orders_with_headers])
-            expect(subject.last_sender).to eq(sender)
+        foreman1.enqueue_work_order(work_orders) do #locked
+          tradesman.work_orders! do |work|
+            expect(tradesman.last_sender).to eq(sender)
             expect(redis.get(lock_key)).not_to be_nil
             done
           end
@@ -83,10 +84,9 @@ module WorkerRoulette
 
     it "should not take out another lock if there is no work to do" do
       evented_readlock_preconditions do
-        foreman.enqueue_work_order(work_orders) do #locked
-          subject.work_orders! do |work_order|
-            expect(work_order).to eq([work_orders_with_headers])
-            subject.work_orders! do |work|
+        foreman1.enqueue_work_order(work_orders) do #locked
+          tradesman.work_orders! do |work_order|
+            tradesman.work_orders! do |work|
               expect(work).to be_empty
               expect(redis.get(lock_key)).to be_nil
               done
@@ -97,9 +97,8 @@ module WorkerRoulette
     end
 
     def evented_readlock_preconditions(&spec_block)
-      foreman.enqueue_work_order(work_orders) do
-        subject.work_orders! do |work|
-          expect(work).to eq([work_orders_with_headers])
+      foreman1.enqueue_work_order(work_orders) do
+        tradesman.work_orders! do |work|
           spec_block.call
         end
       end

data/spec/unit/preprocessor_spec.rb

@@ -0,0 +1,51 @@
+require "spec_helper"
+
+describe WorkerRoulette::Preprocessor do
+  before { allow(subject).to receive(:preprocessors).and_return(preprocessors) }
+
+  class TestClass
+    include WorkerRoulette::Preprocessor
+    def preprocessors
+    end
+  end
+
+  module TestPreprocessor
+    class TestClass
+      def process(job, channel)
+      end
+    end
+  end
+
+  describe "#preprocess" do
+    let(:wo) { double("work_order") }
+    let(:result) { double("resulting_wo") }
+    let(:channel) { "aChannel" }
+    subject { TestClass.new }
+
+    context "with one preprocessor" do
+      let(:preprocessors) { [TestPreprocessor] }
+
+      it "calls the correct preprocessor with the correct args" do
+        expect_any_instance_of(TestPreprocessor::TestClass).to receive(:process).with(wo, channel)
+        subject.preprocess(wo, channel)
+      end
+
+      it "returns the value of the preprocessor" do
+        allow_any_instance_of(TestPreprocessor::TestClass).to receive(:process).and_return(result)
+        expect(subject.preprocess(wo, channel)).to eq(result)
+      end
+    end
+
+    context "with two preprocessors" do
+      let(:preprocessors) { [TestPreprocessor, TestPreprocessor] }
+      let(:intermediate) { double("intermediate_result") }
+
+      it "chains the preprocessors and returns the correct result" do
+        allow_any_instance_of(TestPreprocessor::TestClass).to receive(:process).with(wo, channel).and_return(intermediate)
+        allow_any_instance_of(TestPreprocessor::TestClass).to receive(:process).with(intermediate, channel).and_return(result)
+
+        expect(subject.preprocess(wo, channel)).to eq(result)
+      end
+    end
+  end
+end

data/spec/unit/queue_latency_tracker_spec.rb

@@ -0,0 +1,68 @@
+require "spec_helper"
+
+module QueueLatencyTracker
+  describe ".configure" do
+    let(:source_config) { { logstash_server_ip: ip, logstash_port: port, server_name: server_name } }
+    let(:ip)            { "1.2.3.4" }
+    let(:port)          { 123 }
+    let(:server_name)   { "server.example" }
+
+
+    it "stores the configuration" do
+      QueueLatencyTracker.configure(source_config)
+
+      expect(QueueLatencyTracker.config).to eq({
+        logstash: {
+          server_ip: ip,
+          port: port },
+        server_name: server_name
+      })
+    end
+  end
+
+  describe Foreman do
+    describe "#process" do
+      let(:channel)        { "a_channel" }
+      let(:queued_at)      { 1234567 }
+      let(:raw_work_order) { { "headers" => {}, "payload" => "aPayload" } }
+      let(:work_order)     { subject.process(raw_work_order, channel) }
+
+      before { allow(Time).to receive(:now).and_return(queued_at) }
+
+      it "sets queued_at to now using specified granularity" do
+        expect(work_order["headers"]["queued_at"]).to eq(queued_at * GRANULARITY)
+
+      end
+    end
+  end
+
+  describe Tradesman do
+    describe "#process" do
+      let(:queued_at)       { 1234567 * GRANULARITY }
+      let(:expected_json)   { %({"server_name":"#{server_name}","queue_latency (ms)":#{latency * 1000},"channel":"#{channel}"}) }
+      let(:ip)              { "1.2.3.4" }
+      let(:port)            { 123 }
+      let(:latency)         { 123.432 }
+      let(:server_name)     { "server.example" }
+      let(:channel)         { "a_channel" }
+      let(:headers)         { { "queued_at" => queued_at } }
+      let(:raw_work_order)  { { "headers" => headers, "payload" => "aPayload" } }
+      let(:logstash_config) { { server_ip: ip, port: port } }
+      let(:config)          { { logstash: logstash_config, server_name: server_name } }
+
+      before { allow(QueueLatencyTracker).to receive(:config).and_return(config) }
+      before { allow(Time).to receive(:now).and_return(queued_at / GRANULARITY + latency) }
+      before { allow_any_instance_of(UDPSocket).to receive(:send) }
+
+      it "passes the right json to logstash_send" do
+        expect_any_instance_of(UDPSocket).to receive(:send).with(expected_json, 0, ip, port)
+
+        subject.process(raw_work_order, channel)
+      end
+
+      it "returns the work order unchanged" do
+        expect(subject.process(raw_work_order, channel)).to eq(raw_work_order)
+      end
+    end
+  end
+end

data/spec/unit/readlock_spec.rb

@@ -1,70 +1,72 @@
-require 'spec_helper'
+require "spec_helper"
+
 module WorkerRoulette
  describe "Read Lock" do
-    let(:worker_roulette) {WorkerRoulette.start(evented: false)}
-    let(:redis) {Redis.new(worker_roulette.redis_config)}
-    let(:sender) {'katie_80'}
-    let(:work_orders) {"hellot"}
-    let(:lock_key) {"L*:#{sender}"}
-    let(:default_headers) {Hash['headers' => {'sender' => sender}]}
-    let(:work_orders_with_headers) {default_headers.merge({'payload' => work_orders})}
-    let(:jsonized_work_orders_with_headers) {[WorkerRoulette.dump(work_orders_with_headers)]}
-    let(:foreman) {worker_roulette.foreman(sender)}
-    let(:number_two) {worker_roulette.foreman('number_two')}
-    let(:subject) {worker_roulette.tradesman}
-    let(:subject_two) {worker_roulette.tradesman}
-    let(:lua) { Lua.new(worker_roulette.tradesman_connection_pool) }
+    let(:worker_roulette)          { WorkerRoulette.start(evented: false) }
+    let(:redis)                    { Redis.new(worker_roulette.redis_config) }
+    let(:sender)                   { "katie_80" }
+    let(:work_orders)              { "hello" }
+    let(:lock_key)                 { "L*:#{sender}" }
+    let(:queued_at)                { 1234567 }
+    let(:default_headers)          { Hash["headers" => { "sender" => sender, "queued_at" => (queued_at.to_f * 1_000_000).to_i }] }
+    let(:work_orders_with_headers) { default_headers.merge({ "payload" => work_orders }) }
+    let(:foreman1)                 { worker_roulette.foreman(sender) }
+    let(:foreman2)                 { worker_roulette.foreman("foreman2") }
+    let(:lua)                      { Lua.new(worker_roulette.tradesman_connection_pool) }
+    let(:tradesman_1)              { worker_roulette.tradesman }
+    let(:tradesman_2)              { worker_roulette.tradesman }
 
     before do
       lua.clear_cache!
       redis.script(:flush)
       redis.flushdb
-      foreman.enqueue_work_order(work_orders)
-      expect(subject.work_orders!).to eq([work_orders_with_headers])
+      allow(Time).to receive(:now).and_return(queued_at)
+      foreman1.enqueue_work_order(work_orders)
+      expect(tradesman_1.work_orders!).to eq([work_orders_with_headers])
     end
 
-    it "should lock a queue when it reads from it" do
+    it "locks a queue when it reads from it" do
       expect(redis.get(lock_key)).not_to be_nil
     end
 
-    it "should set the lock to expire in 3 second" do
+    it "sets the lock to expire in 3 seconds" do
       expect(redis.ttl(lock_key)).to eq(3)
     end
 
-    it "should not read a locked queue" do
-      foreman.enqueue_work_order(work_orders)    #locked
-      expect(subject_two.work_orders!).to be_empty
+    it "does not read a locked queue" do
+      foreman1.enqueue_work_order(work_orders)    #locked
+      expect(tradesman_2.work_orders!).to be_empty
     end
 
-    it "should read from the first available queue that is not locked" do
-       foreman.enqueue_work_order(work_orders)     #locked
-       number_two.enqueue_work_order(work_orders)  #unlocked
-       expect(subject_two.work_orders!.first['headers']['sender']).to eq('number_two')
+    it "reads from the first available queue that is not locked" do
+       foreman1.enqueue_work_order(work_orders)     #locked
+       foreman2.enqueue_work_order(work_orders)  #unlocked
+       expect(tradesman_2.work_orders!.first["headers"]["sender"]).to eq("foreman2")
     end
 
-    it "should release its previous lock when it asks for work from another sender" do
-      number_two.enqueue_work_order(work_orders)    #unlocked
-      expect(subject.last_sender).to eq(sender)
-      expect(subject.work_orders!.first['headers']['sender']).to eq('number_two')
+    it "releases its previous lock when it asks for work from another sender" do
+      foreman2.enqueue_work_order(work_orders)    #unlocked
+      expect(tradesman_1.last_sender).to eq(sender)
+      expect(tradesman_1.work_orders!.first["headers"]["sender"]).to eq("foreman2")
       expect(redis.get(lock_key)).to be_nil
     end
 
-    it "should not release its lock when it asks for work from the same sender" do
-      foreman.enqueue_work_order(work_orders)    #locked
-      expect(subject.work_orders!).to eq([work_orders_with_headers])
-      expect(subject.last_sender).to eq(sender)
+    it "does not release its lock when it asks for work from the same sender" do
+      foreman1.enqueue_work_order(work_orders)    #locked
+      expect(tradesman_1.work_orders!).to eq([work_orders_with_headers])
+      expect(tradesman_1.last_sender).to eq(sender)
 
-      foreman.enqueue_work_order(work_orders)    #locked
-      expect(subject.work_orders!).to eq([work_orders_with_headers])
-      expect(subject.last_sender).to eq(sender)
+      foreman1.enqueue_work_order(work_orders)    #locked
+      expect(tradesman_1.work_orders!).to eq([work_orders_with_headers])
+      expect(tradesman_1.last_sender).to eq(sender)
 
       expect(redis.get(lock_key)).not_to be_nil
     end
 
-    it "should release its previous lock if there is no work to do from the same sender" do
-      foreman.enqueue_work_order(work_orders)    #locked
-      expect(subject.work_orders!).to eq([work_orders_with_headers])
-      expect(subject.work_orders!).to be_empty
+    it "releases its previous lock if there is no work to do from the same sender" do
+      foreman1.enqueue_work_order(work_orders)    #locked
+      expect(tradesman_1.work_orders!).to eq([work_orders_with_headers])
+      expect(tradesman_1.work_orders!).to be_empty
       expect(redis.get(lock_key)).to be_nil
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nexia_worker_roulette
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.2.0
 platform: ruby
 authors:
 - Paul Saieg
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-22 00:00:00.000000000 Z
+date: 2015-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oj
@@ -211,6 +211,8 @@ files:
 - lib/worker_roulette.rb
 - lib/worker_roulette/foreman.rb
 - lib/worker_roulette/lua.rb
+- lib/worker_roulette/preprocessor.rb
+- lib/worker_roulette/queue_latency_tracker.rb
 - lib/worker_roulette/tradesman.rb
 - lib/worker_roulette/version.rb
 - spec/benchmark/irb_demo_runner.rb
@@ -221,6 +223,8 @@ files:
 - spec/spec_helper.rb
 - spec/unit/evented_readlock_spec.rb
 - spec/unit/lua_spec.rb
+- spec/unit/preprocessor_spec.rb
+- spec/unit/queue_latency_tracker_spec.rb
 - spec/unit/readlock_spec.rb
 - worker_roulette.gemspec
 homepage: https://github.com/nexiahome/worker_roulette
@@ -242,7 +246,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Pub Sub Queue for Redis that ensures ordered processing
@@ -255,4 +259,6 @@ test_files:
 - spec/spec_helper.rb
 - spec/unit/evented_readlock_spec.rb
 - spec/unit/lua_spec.rb
+- spec/unit/preprocessor_spec.rb
+- spec/unit/queue_latency_tracker_spec.rb
 - spec/unit/readlock_spec.rb