worker_roulette 0.0.6 → 0.0.7

data/README.md

@@ -4,36 +4,68 @@ WorkerRoulette is designed to allow large numbers of unique devices, processes,
 
 WorkerRoulette lets you have thousands of competing consumers (distrubted over as many machines as you'd like) processing ordered messages from millions of totally unknown message providers. It does all this and ensures that the messages sent from each message provider are processed in exactly the order it sent them.
 
-## Usage
-    size_of_connection_pool = 100
-    redis_config = {host: 'localhost', timeout: 5, db: 1}
-
-    WorkerRoulette.start(size_of_connection_pool, redis_config)
-
-    sender_id = :shady
-    foreman = WorkerRoulette.foreman(sender_id)
-    foreman.enqueue_work_order(['hello', 'foreman'])
-
-    tradesman = WorkerRoulette.tradesman
-    messages = tradesman.work_orders! #drain the queue of the next available sender
-    messages.first # => ['hello', 'foreman']
-
-    other_sender_id = :the_real_slim_shady
-    other_foreman = WorkerRoulette.foreman(other_sender_id)
-    other_foreman.enqueue_work_order({'can you get me' => 'the number nine?'})
-
-    messages = tradesman.work_orders! #drain the queue of the next available sender
-    messages.first # => {'can you get me' => 'the number nine?'}
-
-    on_subscribe_callback = -> do
-      puts "Huzzah! We're listening!"
-      foreman.enqueue_work_order('will I see you later?')
-      foreman.enqueue_work_order('can you give me back my dime?')
-    end
-
-    tradesman.wait_for_work_orders(on_subscribe_callback) do |messages| #drain the queue of the next available sender
-      messages # => ['will I see you later', 'can you give me back my dime?']
-    end
+## General Usage
+```ruby
+size_of_connection_pool = 100
+redis_config = {host: 'localhost', timeout: 5, db: 1}
+
+#Start it up
+WorkerRoulette.start(size_of_connection_pool, redis_config)
+
+#Enqueue some work
+sender_id = :shady
+foreman = WorkerRoulette.foreman(sender_id)
+foreman.enqueue_work_order(['hello', 'foreman'])
+
+#Pull it off
+tradesman = WorkerRoulette.tradesman
+messages = tradesman.work_orders! #drain the queue of the next available sender
+messages.first # => ['hello', 'foreman']
+
+#Enqueue some more from someone else
+other_sender_id = :the_real_slim_shady
+other_foreman = WorkerRoulette.foreman(other_sender_id)
+other_foreman.enqueue_work_order({'can you get me' => 'the number nine?'})
+
+#Have the same worker pull that off
+messages = tradesman.work_orders! #drain the queue of the next available sender
+messages.first # => {'can you get me' => 'the number nine?'}
+
+#Have your workers wait for work to come in
+on_subscribe_callback = -> do
+  puts "Huzzah! We're listening!"
+  foreman.enqueue_work_order('will I see you later?')
+  foreman.enqueue_work_order('can you give me back my dime?')
+end
+
+
+#And they will pull it off as it comes, as long as it comes
+#(This is a blocking operation, so it is best in Threads or EventMachine.next_tick)
+tradesman.wait_for_work_orders(on_subscribe_callback) do |messages| #drain the queue of the next available sender
+  messages # => ['will I see you later', 'can you give me back my dime?']
+end
+```
+
+## Channels
+You can also namespace your work orders over a channel, in case you have several sorts of competing consumers who should not step on each other's toes:
+```ruby
+tradesman         = WorkerRoulette.tradesman('good_channel')
+tradesman.should_receive(:work_orders!).and_call_original
+
+good_foreman      = WorkerRoulette.foreman('foreman', 'good_channel')
+bad_foreman       = WorkerRoulette.foreman('foreman', 'bad_channel')
+
+publish  = -> do
+  good_foreman.enqueue_work_order('some old fashion work')
+  bad_foreman.enqueue_work_order('evil biddings you should not carry out') #channels let us ignore his evil orders
+end
+
+tradesman.wait_for_work_orders(publish) do |work|
+  work.to_s.should match("some old fashion work") #only got the work from the good foreman
+  tradesman.unsubscribe
+end
+
+```
 
 ##Caveat Emptor
 While WorkerRoulette does promise to keep the messages of each consumer processed in order by competing consumers, it does NOT guarantee the order in which the queues themselves will be processed. In general, work is processed in a FIFO order, but for performance reasons this has been left a loose FIFO. For example, if Abdul enqueue_work_orders some ordered messages ('1', '2', and '3') and then so do Mark and Wanda, Mark's messages may be processed first, then it would likely be Abdul's, and then Wanda's. However, even though Mark jumped the line, Abdul's messages will still be processed the order he enqueue_work_orderd them ('1', '2', then '3').

data/lib/worker_roulette/foreman.rb

@@ -3,17 +3,23 @@ module WorkerRoulette
     attr_reader :sender
     COUNTER_KEY = 'counter_key'
 
-    def initialize(sender, redis_pool)
-      @sender = sender
+    def initialize(sender, redis_pool, namespace = nil)
+      @sender     = sender
+      @namespace  = namespace
       @redis_pool = redis_pool
+      @channel    = namespace || WorkerRoulette::JOB_NOTIFICATIONS
     end
 
     def job_board_key
-      WorkerRoulette::JOB_BOARD
+      @job_board_key ||= "#{@namespace + ':' if @namespace}#{WorkerRoulette::JOB_BOARD}"
+    end
+
+    def sender_key
+      @sender_key ||= "#{@namespace + ':' if @namespace}#{@sender}"
     end
 
     def counter_key
-      COUNTER_KEY
+      @counter_key ||= "#{@namespace + ':' if @namespace}#{COUNTER_KEY}"
     end
 
     def enqueue_work_order_without_headers(work_order)
@@ -25,9 +31,9 @@ module WorkerRoulette
       @redis_pool.with({}) do |redis|
         @count = redis.incr(COUNTER_KEY)
         redis.multi do
-          redis.zadd(WorkerRoulette::JOB_BOARD, @count, sender)
-          redis.rpush(sender, Oj.dump(work_order))
-          redis.publish(WorkerRoulette::JOB_NOTIFICATIONS, WorkerRoulette::JOB_NOTIFICATIONS)
+          redis.zadd(job_board_key, @count, @sender)
+          redis.rpush(sender_key, Oj.dump(work_order))
+          redis.publish(@channel, WorkerRoulette::JOB_NOTIFICATIONS)
         end
       end
     end

data/lib/worker_roulette/tradesman.rb

@@ -1,18 +1,24 @@
 module WorkerRoulette
   class Tradesman
     attr_reader :sender
-    def initialize(client_pool, pubsub_pool)
+    def initialize(client_pool, pubsub_pool, namespace = nil)
       @client_pool = client_pool
       @pubsub_pool = pubsub_pool
+      @namespace   = namespace
+      @channel     = namespace || WorkerRoulette::JOB_NOTIFICATIONS
     end
 
     def job_board_key
-      WorkerRoulette::JOB_BOARD
+      @job_board_key ||= "#{@namespace + ':' if @namespace}#{WorkerRoulette::JOB_BOARD}"
+    end
+
+    def sender_key
+      @sender_key ||= "#{@namespace + ':' if @namespace}#{@sender}"
     end
 
     def wait_for_work_orders(on_subscribe_callback = nil, &block)
       @pubsub_pool.with({}) do |redis|
-        redis.subscribe(WorkerRoulette::JOB_NOTIFICATIONS) do |on|
+        redis.subscribe(@channel) do |on|
           on.subscribe {on_subscribe_callback.call if on_subscribe_callback}
           on.message   {block.call(work_orders!) if block}
         end
@@ -23,21 +29,21 @@ module WorkerRoulette
       @client_pool.with({}) do |redis|
         get_sender_for_next_job(redis)
         results = redis.multi do
-          redis.lrange(sender, 0, -1)
-          redis.del(sender)
-          redis.zrem(WorkerRoulette::JOB_BOARD, sender)
+          redis.lrange(sender_key, 0, -1)
+          redis.del(sender_key)
+          redis.zrem(job_board_key, sender_key)
         end
         ((results || []).first || []).map {|work_order| Oj.load(work_order)}
       end
     end
 
     def unsubscribe
-      @pubsub_pool.with({}) {|redis| redis.unsubscribe}
+      @pubsub_pool.with({}) {|redis| redis.unsubscribe(@channel)}
     end
 
   private
     def get_sender_for_next_job(redis)
-      @sender = (redis.zrange(WorkerRoulette::JOB_BOARD, 0, 0) || []).first.to_s
+      @sender = (redis.zrange(job_board_key, 0, 0) || []).first.to_s
     end
   end
 end

data/lib/worker_roulette/version.rb

@@ -1,3 +1,3 @@
 module WorkerRoulette
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/lib/worker_roulette.rb

@@ -22,14 +22,14 @@ module WorkerRoulette
     @pubsub_connection_pool     = connection_pool.new(@pool_config) {Redis.new(@redis_config)}
   end
 
-  def self.foreman(sender)
+  def self.foreman(sender, channel = nil)
     raise "WorkerRoulette not Started" unless @foreman_connection_pool
-    Foreman.new(sender, @foreman_connection_pool)
+    Foreman.new(sender, @foreman_connection_pool, channel)
   end
 
-  def self.tradesman
+  def self.tradesman(channel = nil)
     raise "WorkerRoulette not Started" unless @tradesman_connection_pool
-    Tradesman.new(@tradesman_connection_pool, @pubsub_connection_pool)
+    Tradesman.new(@tradesman_connection_pool, @pubsub_connection_pool, channel)
   end
 
   def self.tradesman_connection_pool

data/spec/integration/worker_roulette_spec.rb

@@ -137,15 +137,36 @@ describe WorkerRoulette do
       subject.work_orders!
     end
 
-    it "should get the work_orders from the next sender's slot when a new job is ready" do
+    it "should get the work_orders from the next queue when a new job is ready" do
       subject.work_orders!
       subject.should_receive(:work_orders!).and_call_original
-      publisher = -> {foreman.enqueue_work_order(work_orders); subject.unsubscribe; }
+
+      publisher = -> {puts :HOO.to_s; foreman.enqueue_work_order(work_orders); subject.unsubscribe}
+
       subject.wait_for_work_orders(publisher) do |redis_work_orders|
         redis_work_orders.should == [work_orders_with_headers]
       end
     end
 
+    it "should publish and subscribe on custom channels" do
+      tradesman         = WorkerRoulette.tradesman('good_channel')
+      tradesman.should_receive(:work_orders!).and_call_original
+
+      good_foreman      = WorkerRoulette.foreman('foreman', 'good_channel')
+      bad_foreman       = WorkerRoulette.foreman('foreman', 'bad_channel')
+
+
+      publish  = -> do
+        good_foreman.enqueue_work_order('some old fashion work')
+        bad_foreman.enqueue_work_order('evil biddings you should not carry out')
+        tradesman.unsubscribe
+      end
+
+      tradesman.wait_for_work_orders(publish) do |work|
+        work.to_s.should match("some old fashion work")
+      end
+    end
+
     it "should checkout a readlock for a queue and put it back when its done processing; lock should expire after 5 minutes?"
     it "should eves drop on the job board"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: worker_roulette
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors: