worker_roulette 0.1.7 → 0.1.9

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    M2EzZmZkYTllZGYwMjA1MDI1MzVmZDU5ODk1ZGRiNzYzMjQzMjQ1OQ==
+    MzJmOTEyZjdhMWMyYmIzODBiOWFmZTdmNDE5NGUzM2U4MDdlMGYxYg==
   data.tar.gz: !binary |-
-    NDZmNjhjMTJjZDJhMjJlNzI5MjE2MjllZjFkMGQ1ZDIxODExNTQ4NQ==
+    MDNhZWRmYjVkOWQ1MGEwYjk3NmQ5Njg5YjViMzIwZWFkYzU5NjMxMA==
 SHA512:
   metadata.gz: !binary |-
-    NjAyNTBlZDUxNmM5M2Y1MGMwMGYyNzdkNDUzNDEzMGQxNTI0ODY4NTlhY2U1
-    YjRiYjQ1MWI3YzMxNDdmZWM3ZjE1Y2ViZDg1M2FhYTA1ZjhmYzNhYWYwMjY2
-    ZThkODM4ZjY0MmE5NzlmMTdlZTRiMzRhMzUwMWFjMGJkM2NlYTU=
+    MGNhMjYxYTkxY2UwZGJmMDZjNTc1MzY0N2Y4MTYzMWJkZTE5N2RiYjZkNjIz
+    MzQ4NmVlYTU5ZjU2MGZmNWVhYTg5NDRiNWRlZTBlNzMxZDk2ZWE4OTE1OTg2
+    Y2Q1OTBhMmVhN2IwNmQ3OGQ3ZDA1MTI4MDdmMzA1MWJkZTZkYzc=
   data.tar.gz: !binary |-
-    ODUxMmQwNWNhOWI5NTBkMmQ2NzFlMzUzY2Q4NjZlZTYzZDE4ZmQ2NGZiODJk
-    ZWNkMzc3ZmUxY2UzOTk4OTJjZjlmNDZkMjU4N2Y3ZGVjMmUzYzIxOTdmMzg2
-    MDJlZjAzNGFiN2JjMGMzNWEwYWVlNmY3OTQ4MjE0NjliY2ViOTE=
+    Y2I0ZGFkOTI2OGNjODA5M2NmMTQxM2Q2MTNhN2IzMTVlZDZkNWI1ZjU2Nzll
+    OTUxNjhkOTFjYzU4NzQ3OTU5OTMxYjI3OGZmY2NiMzlhNmM0NTMwYTg5ZGZj
+    ZGE5MjliZWI0MTE2MjAzMWM5MTY3ODhmMzhlYjYyMmNhZTdjYWM=

data/README.md

@@ -1,99 +1,48 @@
 # WorkerRoulette
 
-WorkerRoulette is designed to allow large numbers of unique devices, processes, users, or whatever communicate over individual channels without messing up the order of their messages. WorkerRoulette was created to solve two otherwise hard problems. First, other messaging solutions (I'm looking at you RabbitMQ) are not designed to  handle very large numbers of queues (30,000+); because WorkerRoulette is built on top of Redis, we have successfully tested it running with millions of queues. Second, other messaging systems assume one (or more) of three things: 1. Your message consumers know the routing key of messages they are interested in processing; 2. Your messages can wait so that only one consumer is processed at a time; 3. You love to complicated write code to put your messages back in order. Sometimes, none of these things is true and that is where WorkerRoulette comes in.
+WorkerRoulette is designed to allow large numbers of unique devices, processes, users, or whatever communicate over individual channels without messing up the order of their messages. WorkerRoulette was created to solve two otherwise hard problems. First, other messaging solutions (I'm looking at you RabbitMQ) are not designed to  handle very large numbers of queues (millions); because WorkerRoulette is built on top of Redis, we have successfully tested it running with millions of queues. Second, other messaging systems assume one (or more) of three things: 1. Your message consumers know the routing key of messages they are interested in processing; 2. Your messages can wait so that only one consumer is processed at a time; 3. You love to complicated write code to put your messages back in order. Sometimes, none of these things is true and that is where WorkerRoulette comes in.
 
-WorkerRoulette lets you have thousands of competing consumers (distrubted over as many machines as you'd like) processing ordered messages from millions of totally unknown message providers. It does all this and ensures that the messages sent from each message provider are processed in exactly the order it sent them.
+WorkerRoulette lets you have thousands of competing consumers (distributed over as many machines as you'd like) processing ordered messages from millions of totally unknown message providers. It does all this and ensures that the messages sent from each message provider are processed in exactly the order it sent them.
 
-## Asynchronous Api (Evented)
+## Use
 ```ruby
 size_of_connection_pool = 100
 
 #Start it up
-#the config takes size for the connection pool size, evented to specify which api to use, then the normal redis config
-WorkerRoulette.start(size: size_of_connection_pool, evented: true, host: 'localhost', timeout: 5, db: 1)
-
-#Enqueue some work
-sender_id = :shady
-foreman = WorkerRoulette.a_foreman(sender_id)
-
-foreman.enqueue_work_order(['hello', 'foreman']) do |msg|
-  puts "work enqueued #{msg}"
-end
-
-#Pull it off
-tradesman = WorkerRoulette.a_tradesman
-tradesman.work_orders! do |work_orders| #drain the queue of the next available sender
-  work_orders.first # => ['hello', 'foreman']
-end
-
-#Enqueue some more from someone else
-other_sender_id = :the_real_slim_shady
-other_foreman = WorkerRoulette.a_foreman(other_sender_id)
-other_foreman.enqueue_work_order({'can you get me' => 'the number nine?'}) do |msg|
-  puts "work enqueued #{msg}"
-end
-
-#Have the same worker pull that off
-tradesman.work_orders! do |work_orders| #drain the queue of the next available sender
-  work_orders.first # => {'can you get me' => 'the number nine?'}
-end
-
-#Have your workers wait for work to come in
-on_subscribe_callback = -> do
-  puts "Huzzah! We're listening!"
-  foreman.enqueue_work_order('will I see you later?')
-  foreman.enqueue_work_order('can you give me back my dime?')
-end
-
-
-#And they will pull it off as it comes, as long as it comes
-#NB: This is NOT a blocking operation, so no worries
-tradesman.wait_for_work_orders(on_subscribe_callback) do |work_orders, message, channel| #drain the queue of the next available sender
-  work_orders # => ['will I see you later', 'can you give me back my dime?']
-  message # => 'new_job_ready'
-  channel # => '' #the name of the channel the message was published on, if one was used -- see below
-end
-```
-
-## Synchronous Api
-```ruby
-size_of_connection_pool = 100
-
-#Start it up
-#the config takes size for the connection pool size, evented to specify which api to use, then the normal redis config
-WorkerRoulette.start(size: size_of_connection_pool, evented: false, host: 'localhost', timeout: 5, db: 1)
+#the config takes size for the connection pool size, evented to specify whether to use evented redis or not,
+#polling_time dictates the number of seconds to wait before checking the job queue
+#(use higher numbers for high traffic systems), then the normal redis config
+WorkerRoulette.start(size: size_of_connection_pool, evented: false, host: 'localhost', timeout: 5, db: 1, polling_time: 2)
 
 #Enqueue some work
 sender_id = :shady
 foreman = WorkerRoulette.foreman(sender_id)
-foreman.enqueue_work_order(['hello', 'foreman'])
+foreman.enqueue_work_order('hello')
 
-#Pull it off
+#Pull it off (headers always include both the unique id of the sender)
 tradesman = WorkerRoulette.tradesman
 work_orders = tradesman.work_orders! #drain the queue of the next available sender
-work_orders.first # => ['hello', 'foreman']
+work_orders # => {'headers' => {'sender' => :shady}, 'payload' => ['hello']}
 
 #Enqueue some more from someone else
 other_sender_id = :the_real_slim_shady
-other_foreman = WorkerRoulette.foreman(other_sender_id)
-other_foreman.enqueue_work_order({'can you get me' => 'the number nine?'})
+other_foreman = WorkerRoulette.foreman(other_sender_id, 'some_namespace')
+other_foreman.enqueue_work_order({'can you get me' => 'the number nine?'}, {'custom' => 'headers'})
 
 #Have the same worker pull that off
 work_orders = tradesman.work_orders! #drain the queue of the next available sender
-work_orders.first # => {'can you get me' => 'the number nine?'}
+work_orders # => {'headers' => {'sender' => :the_real_slim_shady, 'custom' => 'headers'},
+            #     'payload' => [{'can you get me' => 'the number nine?'}]}
 
 #Have your workers wait for work to come in
-on_subscribe_callback = -> do
-  puts "Huzzah! We're listening!"
-  foreman.enqueue_work_order('will I see you later?')
-  foreman.enqueue_work_order('can you give me back my dime?')
-end
-
-
-#And they will pull it off as it comes, as long as it comes
-#NB: This IS a blocking operation
-tradesman.wait_for_work_orders(on_subscribe_callback) do |work_orders| #drain the queue of the next available sender
-  work_orders # => ['will I see you later', 'can you give me back my dime?']
+foreman.enqueue_work_order('will I see you later?')
+foreman.enqueue_work_order('can you give me back my dime?')
+
+#And they will pull it off as it comes, as long as it comes. If the job board is empty (ie there is now work to do),
+#this method will poll redis every "polling_time" seconds until it finds work,
+#at which point it will continue processing jobs until the job board is empty again
+tradesman.wait_for_work_orders do |work_orders| #drain the queue of the next available sender
+  work_orders.first # => ['will I see you later', 'can you give me back my dime?']
 end
 ```
 
@@ -106,33 +55,27 @@ tradesman.should_receive(:work_orders!).and_call_original
 good_foreman      = WorkerRoulette.foreman('foreman', 'good_channel')
 bad_foreman       = WorkerRoulette.foreman('foreman', 'bad_channel')
 
-publish  = -> do
-  good_foreman.enqueue_work_order('some old fashion work')
-  bad_foreman.enqueue_work_order('evil biddings you should not carry out')
-end
+good_foreman.enqueue_work_order('some old fashion work')
+bad_foreman.enqueue_work_order('evil biddings you should not carry out')
 
-tradesman.wait_for_work_orders(publish) do |work|
+tradesman.wait_for_work_orders do |work|
   work.to_s.should match("some old fashion work") #only got the work from the good foreman
   work.to_s.should_not match("evil")              #channels let us ignore the other's evil orders
-  tradesman.unsubscribe
 end
 ```
 
 ## Performance
 Running the performance tests on my laptop, the numbers break down like this:
-### Async Api
-  - Pubsub: ~5500 read-write round-trips / second
-  - Manual: ~4500 read-write round-trips / second
+### Evented
+  - Polling: ~11,500 read-write round-trips / second
+  - Manual:  ~6,000  read-write round-trips / second
 
-### Sync Api
-  - Pubsub: ~2700 read-write round-trips / second
-  - Manual: ~2500 read-write round-trips / second
+### Non-Evented
+  - Polling: ~10,000 read-write round-trips / second
+  - Manual:  ~5,500  read-write round-trips / second
 
 To run the perf tests yourself run `bundle exec spec:perf`
 
-## Redis Pubsub and Polling
-The `wait_for_work_orders` method works using Redis' pubsub mechanism. The advantage to this is that it is very fast and minimizes network traffic. The downside is that Redis' pubsub implementation is 'fire and forget', so any subscribers who are not listening at the moment the message is published will miss it. In order to compensate for this, WorkerRoulette's Async Api creates a backup timer (using EM.add_periodic_timer) that will poll redis every 20-25 seconds for new work. Since the timer is reset every time new work comes in, if you have an active publisher, the timer may never need to fire. It only serves a backup to make sure no work is left waiting in the queues because of network problems. Since there is no one polling mechanism that works for all situations in a synchrounous environment, this feature is only available through the Async Api.
-
 ## Redis Version
 WorkerRoulette uses Redis' lua scripting feature to acheive such high throughput and therefore requires a version of Redis that supports lua scripting (>= Redis 2.6)
 

data/lib/worker_roulette/foreman.rb

@@ -1,6 +1,6 @@
 module WorkerRoulette
   class Foreman
-    attr_reader :sender
+    attr_reader :sender, :namespace, :channel
 
     LUA_ENQUEUE_WORK_ORDERS = <<-HERE
         local counter_key       = KEYS[1]
@@ -16,25 +16,28 @@ module WorkerRoulette
         local zadd              = 'ZADD'
         local rpush             = 'RPUSH'
         local publish           = 'PUBLISH'
+        local zcard             = 'ZCARD'
+        local del               = 'DEL'
 
         local function enqueue_work_orders(work_order, job_notification)
+          redis_call(rpush, sender_key, work_order)
+
+          -- called when a work from a new sender is added
           if (redis_call(zscore, job_board_key, sender_key) == false) then
             local count     = redis_call(incr, counter_key)
             redis_call(zadd, job_board_key, count, sender_key)
           end
-
-          redis_call(rpush,sender_key, work_order)
-          redis_call(publish, channel, job_notification)
         end
 
         enqueue_work_orders(work_order, job_notification)
     HERE
 
-    def initialize(sender, redis_pool, namespace = nil)
+    def initialize(redis_pool, sender, namespace = nil)
       @sender     = sender
       @namespace  = namespace
       @redis_pool = redis_pool
       @channel    = namespace || WorkerRoulette::JOB_NOTIFICATIONS
+      @lua        = Lua.new(@redis_pool)
     end
 
     def enqueue_work_order(work_order, headers = {}, &callback)
@@ -43,7 +46,7 @@ module WorkerRoulette
     end
 
     def enqueue_work_order_without_headers(work_order, &callback)
-      Lua.call(self.class.lua_enqueue_work_orders, [counter_key, job_board_key, sender_key, @channel],
+      @lua.call(LUA_ENQUEUE_WORK_ORDERS, [counter_key, job_board_key, sender_key, @channel],
                [WorkerRoulette.dump(work_order),  WorkerRoulette::JOB_NOTIFICATIONS], &callback)
     end
 
@@ -55,18 +58,14 @@ module WorkerRoulette
       @counter_key ||= WorkerRoulette.counter_key(@namespace)
     end
 
-    private
-
     def sender_key
-      @sender_key = WorkerRoulette.sender_key(sender, @namespace)
+      @sender_key ||= WorkerRoulette.sender_key(@sender, @namespace)
     end
 
+    private
+
     def default_headers
       Hash['sender' => sender]
     end
-
-    def self.lua_enqueue_work_orders
-      LUA_ENQUEUE_WORK_ORDERS
-    end
   end
-end
+end

data/lib/worker_roulette/lua.rb

@@ -1,36 +1,40 @@
 module WorkerRoulette
-  module Lua
-    @cache = Hash.new
+  class Lua
+    Thread.main[:worker_roulette_lua_script_cache] = Hash.new
 
-    def self.call(lua_script, keys_accessed = [], args = [], &callback)
-      WorkerRoulette.tradesman_connection_pool.with do |redis|
+    def initialize(connection_pool)
+      @connection_pool = connection_pool
+    end
+
+    def call(lua_script, keys_accessed = [], args = [], &callback)
+      @connection_pool.with do |redis|
         results = evalsha(redis, lua_script, keys_accessed, args, &callback)
       end
     end
 
-    def self.sha(lua_script)
-      @cache[lua_script] ||= Digest::SHA1.hexdigest(lua_script)
+    def sha(lua_script)
+      Thread.main[:worker_roulette_lua_script_cache][lua_script] ||= Digest::SHA1.hexdigest(lua_script)
     end
 
-    def self.cache
-      @cache.dup
+    def cache
+      Thread.main[:worker_roulette_lua_script_cache].dup
     end
 
-    def self.clear_cache!
-      @cache = {}
+    def clear_cache!
+      Thread.main[:worker_roulette_lua_script_cache] = {}
     end
 
-    def self.eval(redis, lua_script, keys_accessed, args, &callback)
-      results = redis.eval(lua_script, keys_accessed.size, *keys_accessed, *args)
+    def eval(redis, lua_script, keys_accessed, args, &callback)
+      results = redis.eval(lua_script, keys_accessed.length, *keys_accessed, *args)
       results.callback &callback if callback
       results.errback  {|err_msg| raise EM::Hiredis::RedisError.new(err_msg)}
     end
 
-    def self.evalsha(redis, lua_script, keys_accessed, args, &callback)
+    def evalsha(redis, lua_script, keys_accessed, args, &callback)
       if redis.class == EM::Hiredis::Client
         results = redis.evalsha(sha(lua_script), keys_accessed.length, *keys_accessed, *args)
         results.callback &callback if callback
-        results.errback {self.eval(redis, lua_script, keys_accessed, args, &callback)}
+        results.errback {eval(redis, lua_script, keys_accessed, args, &callback)}
       else
         begin
           results = redis.evalsha(sha(lua_script), keys_accessed, args)
@@ -43,4 +47,4 @@ module WorkerRoulette
       results
     end
   end
-end
+end

data/lib/worker_roulette/tradesman.rb

@@ -1,98 +1,126 @@
+require 'timers'
+
 module WorkerRoulette
   class Tradesman
-    attr_reader :last_sender
+    attr_reader :last_sender, :remaining_jobs, :timer
 
     LUA_DRAIN_WORK_ORDERS = <<-HERE
-        local empty_string      = ""
-        local job_board_key     = KEYS[1]
-        local last_sender_key   = KEYS[2] or empty_string
-        local sender_key        = ARGV[1] or empty_string
-        local redis_call        = redis.call
-        local lock_key_prefix   = "L*:"
-        local lock_value        = "L"
-        local ex                = "EX"
-        local nx                = "NX"
-        local get               = "GET"
-        local set               = "SET"
-        local del               = "DEL"
-        local zrank             = "ZRANK"
-        local zrange            = "ZRANGE"
-
-        local function drain_work_orders(job_board_key, last_sender_key, sender_key)
-          if last_sender_key ~= empty_string then
-            local last_sender_lock_key = lock_key_prefix .. last_sender_key
-            redis_call(del, last_sender_lock_key)
-          end
-
-          if sender_key == empty_string then
-            sender_key = redis_call(zrange, job_board_key, 0, 0)[1] or empty_string
-            if sender_key == empty_string then
-              return {}
-            end
-          end
-
-          local lock_key = lock_key_prefix .. sender_key
-          local locked   = (redis_call(get, lock_key) == lock_value)
-
-          if not locked then
-            local results = {sender_key, redis_call('LRANGE', sender_key, 0, -1)}
-            redis_call(del, sender_key)
-            redis_call('ZREM', job_board_key, sender_key)
-            redis_call(set, lock_key, lock_value, ex, 1, nx)
-            return results
-          else
-            local sender_index    = redis_call(zrank, job_board_key, sender_key)
-            local next_index      = sender_index + 1
-            local next_sender_key = redis_call(zrange, job_board_key, next_index, next_index)[1]
-            if next_sender_key then
-              return drain_work_orders(job_board_key, empty_string, next_sender_key)
-            else
-              return {}
-            end
-          end
-        end
+    local empty_string      = ""
+    local job_board_key     = KEYS[1]
+    local last_sender_key   = KEYS[2] or empty_string
+    local sender_key        = ARGV[1] or empty_string
+    local redis_call        = redis.call
+    local lock_key_prefix   = "L*:"
+    local lock_value        = 1
+    local ex                = "EX"
+    local nx                = "NX"
+    local get               = "GET"
+    local set               = "SET"
+    local del               = "DEL"
+    local lrange            = "LRANGE"
+    local zrank             = "ZRANK"
+    local zrange            = "ZRANGE"
+    local zrem              = "ZREM"
+    local zcard             = 'ZCARD'
+
+    local function drain_work_orders(job_board_key, last_sender_key, sender_key)
 
-        return drain_work_orders(job_board_key, last_sender_key, empty_string)
-    HERE
+    --kill lock for last_sender_key
+    if last_sender_key ~= empty_string then
+      local last_sender_lock_key = lock_key_prefix .. last_sender_key
+      redis_call(del, last_sender_lock_key)
+    end
+
+    if sender_key == empty_string then
+      sender_key = redis_call(zrange, job_board_key, 0, 0)[1] or empty_string
 
-    def initialize(client_pool, pubsub_pool, namespace = nil)
-      @client_pool = client_pool
-      @pubsub_pool = pubsub_pool
-      @namespace   = namespace
-      @channel     = namespace || WorkerRoulette::JOB_NOTIFICATIONS
+      -- return if job_board is empty
+      if sender_key == empty_string then
+        return {empty_string, {}, 0}
+      end
     end
 
-    def wait_for_work_orders(on_subscribe_callback = nil, &block)
-      @pubsub_pool.with do |redis|
-        redis.subscribe(@channel) do |on|
-          on.subscribe {on_subscribe_callback.call if on_subscribe_callback}
-          on.message   {block.call(work_orders! + work_orders!) if block}
+    local lock_key       = lock_key_prefix .. sender_key
+    local was_not_locked = redis_call(set, lock_key, lock_value, ex, 3, nx)
+
+    if was_not_locked then
+      local work_orders    = redis_call(lrange, sender_key, 0, -1)
+      redis_call(del, sender_key)
+
+      redis_call(zrem, job_board_key, sender_key)
+      local remaining_jobs = redis_call(zcard, job_board_key) or 0
+
+      return {sender_key, work_orders, remaining_jobs}
+    else
+      local sender_index    = redis_call(zrank, job_board_key, sender_key)
+      local next_index      = sender_index + 1
+      local next_sender_key = redis_call(zrange, job_board_key, next_index, next_index)[1]
+      if next_sender_key then
+        return drain_work_orders(job_board_key, empty_string, next_sender_key)
+      else
+        -- return if job_board is empty
+        return {empty_string, {}, 0}
+      end
+    end
+  end
+
+  return drain_work_orders(job_board_key, last_sender_key, empty_string)
+  HERE
+
+    def initialize(redis_pool, evented, namespace = nil, polling_time = WorkerRoulette::DEFAULT_POLLING_TIME)
+      @evented        = evented
+      @polling_time   = polling_time
+      @redis_pool     = redis_pool
+      @namespace      = namespace
+      @channel        = namespace || WorkerRoulette::JOB_NOTIFICATIONS
+      @timer          = Timers::Group.new
+      @lua            = Lua.new(@redis_pool)
+      @remaining_jobs = 0
+    end
+
+    def wait_for_work_orders(&on_message_callback)
+      return unless on_message_callback
+      work_orders! do |work|
+        on_message_callback.call(work) if work.any?
+        if @evented
+          evented_drain_work_queue!(&on_message_callback)
+        else
+          non_evented_drain_work_queue!(&on_message_callback)
         end
       end
     end
 
     def work_orders!(&callback)
-      Lua.call(self.class.lua_drain_work_orders, [job_board_key, @last_sender], [nil]) do |results|
-        results ||= []
-        @last_sender = (results.first || '').split(':').first
-        work = (results[1] || []).map {|work_order| WorkerRoulette.load(work_order)}
+      @lua.call(LUA_DRAIN_WORK_ORDERS, [job_board_key, @last_sender], [nil]) do |results|
+        sender_key      = results[0]
+        work_orders     = results[1]
+        @remaining_jobs = results[2]
+        @last_sender    = sender_key.split(':').last
+        work            = work_orders.map {|work_order| WorkerRoulette.load(work_order)}
         callback.call work if callback
         work
       end
     end
 
-    def unsubscribe
-      @pubsub_pool.with {|redis| redis.unsubscribe(@channel)}
-    end
-
     def job_board_key
       @job_board_key ||= WorkerRoulette.job_board_key(@namespace)
     end
 
-    private
+  private
+    def evented_drain_work_queue!(&on_message_callback)
+      if remaining_jobs > 0
+        EM.next_tick {wait_for_work_orders(&on_message_callback)}
+      else
+        EM.add_timer(@polling_time) {wait_for_work_orders(&on_message_callback)}
+      end
+    end
 
-    def self.lua_drain_work_orders
-      LUA_DRAIN_WORK_ORDERS
+    def non_evented_drain_work_queue!(&on_message_callback)
+      if remaining_jobs > 0
+        wait_for_work_orders(&on_message_callback)
+      else
+        @timer.after(@polling_time) {wait_for_work_orders(&on_message_callback)}
+      end
     end
   end
 end

data/lib/worker_roulette/version.rb

@@ -1,3 +1,3 @@
 module WorkerRoulette
-  VERSION = "0.1.7"
+  VERSION = "0.1.9"
 end