worker_roulette 0.1.7 → 0.1.9

data/lib/worker_roulette.rb

@@ -9,95 +9,88 @@ require "digest/sha1"
 Dir[File.join(File.dirname(__FILE__),'worker_roulette','**','*.rb')].sort.each { |file| require file.gsub(".rb", "")}
 
 module WorkerRoulette
-  JOB_BOARD = "job_board"
-  JOB_NOTIFICATIONS = "new_job_ready"
-
-  def self.start(config = {})
-    @redis_config               = { host: 'localhost', port: 6379, db: 14, driver: :hiredis, timeout: 5, evented: false, pool_size: 10 }.merge(config)
-    @pool_config                = Hash[size: @redis_config.delete(:pool_size), timeout: @redis_config.delete(:timeout)]
-    @evented                    = @redis_config.delete(:evented)
-
-    @foreman_connection_pool    = ConnectionPool.new(@pool_config) {new_redis}
-    @tradesman_connection_pool  = ConnectionPool.new(@pool_config) {new_redis}
-    @pubsub_connection_pool     = ConnectionPool.new(@pool_config) {new_redis_pubsub}
-  end
+  class WorkerRoulette
+    JOB_BOARD = "job_board"
+    JOB_NOTIFICATIONS = "new_job_ready"
+    DEFAULT_POLLING_TIME = 2
+
+    def self.dump(obj)
+      Oj.dump(obj)
+    rescue Oj::ParseError => e
+      {'error' => e, 'unparsable_string' => obj}
+    end
 
-  def self.foreman(sender, channel = nil)
-    raise "WorkerRoulette not Started" unless @foreman_connection_pool
-    Foreman.new(sender, @foreman_connection_pool, channel)
-  end
+    def self.load(json)
+      Oj.load(json)
+    rescue Oj::ParseError => e
+      {'error' => e, 'unparsable_string' => obj}
+    end
 
-  def self.tradesman(channel = nil)
-    raise "WorkerRoulette not Started" unless @tradesman_connection_pool
-    Tradesman.new(@tradesman_connection_pool, @pubsub_connection_pool, channel)
-  end
+    def self.job_board_key(namespace = nil)
+      "#{namespace + ':' if namespace}#{WorkerRoulette::JOB_BOARD}"
+    end
 
-  def self.a_foreman(sender, channel = nil)
-    foreman(sender, channel)
-  end
+    def self.sender_key(sender, namespace = nil)
+      "#{namespace + ':' if namespace}#{sender}"
+    end
 
-  def self.a_tradesman(channel = nil)
-    raise "WorkerRoulette not Started" unless @tradesman_connection_pool
-    ATradesman.new(@tradesman_connection_pool, @pubsub_connection_pool, channel)
-  end
+    def self.counter_key(sender, namespace = nil)
+      "#{namespace + ':' if namespace}counter_key"
+    end
 
-  def self.tradesman_connection_pool
-    @tradesman_connection_pool
-  end
+    def self.start(config = {})
+      instance = new(config)
+      instance
+    end
 
-  def self.pubsub_connection_pool
-    @pubsub_connection_pool
-  end
+    private_class_method :new
 
-  def self.pool_size
-    (@pool_config ||= {})[:size]
-  end
+    def initialize(config = {})
+      @redis_config               = { host: 'localhost', port: 6379, db: 14, driver: :hiredis, timeout: 5, evented: false, pool_size: 10 , polling_time: DEFAULT_POLLING_TIME}.merge(config)
+      @pool_config                = Hash[size: @redis_config.delete(:pool_size), timeout: @redis_config.delete(:timeout)]
+      @evented                    = @redis_config.delete(:evented)
+      @polling_time               = @redis_config.delete(:polling_time)
 
-  def self.redis_config
-    (@redis_config ||= {}).dup
-  end
-
-  def self.dump(obj)
-    Oj.dump(obj)
-  rescue Oj::ParseError => e
-    {'error' => e, 'unparsable_string' => obj}
-  end
+      @foreman_connection_pool    = ConnectionPool.new(@pool_config) {new_redis}
+      @tradesman_connection_pool  = ConnectionPool.new(@pool_config) {new_redis}
+    end
 
-  def self.load(json)
-    Oj.load(json)
-  rescue Oj::ParseError => e
-    {'error' => e, 'unparsable_string' => obj}
-  end
+    def foreman(sender, namespace = nil)
+      raise "WorkerRoulette not Started" unless @foreman_connection_pool
+      Foreman.new(@foreman_connection_pool, sender, namespace)
+    end
 
-  def self.job_board_key(namespace = nil)
-    "#{namespace + ':' if namespace}#{WorkerRoulette::JOB_BOARD}"
-  end
+    def tradesman(namespace = nil, polling_time = DEFAULT_POLLING_TIME)
+      raise "WorkerRoulette not Started" unless @tradesman_connection_pool
+      Tradesman.new(@tradesman_connection_pool, @evented, namespace, polling_time || @polling_time)
+    end
 
-  def self.sender_key(sender, namespace = nil)
-    "#{namespace + ':' if namespace}#{sender}"
-  end
+    def tradesman_connection_pool
+      @tradesman_connection_pool
+    end
 
-  def self.counter_key(sender, namespace = nil)
-    "#{namespace + ':' if namespace}counter_key"
-  end
+    def pool_size
+      (@pool_config ||= {})[:size]
+    end
 
-  private
+    def redis_config
+      (@redis_config ||= {}).dup
+    end
 
-  def self.new_redis
-    if @evented
-      require 'eventmachine'
-      redis = EM::Hiredis::Client.new(@redis_config[:host], @redis_config[:port], @redis_config[:password], @redis_config[:db])
-      redis.connect
-    else
-      Redis.new(@redis_config)
+    def polling_time
+      @polling_time
     end
-  end
 
-  def self.new_redis_pubsub
-    if @evented
-      new_redis.pubsub
-    else
-      new_redis
+    private
+
+    def new_redis
+      if @evented
+        require 'eventmachine'
+        redis = EM::Hiredis::Client.new(@redis_config[:host], @redis_config[:port], @redis_config[:password], @redis_config[:db])
+        redis.connect
+      else
+        Redis.new(@redis_config)
+      end
     end
   end
 end

data/spec/benchmark/irb_demo_runner.rb

@@ -0,0 +1,39 @@
+require 'worker_roulette'
+
+def pub(iterations)
+  instance = WorkerRoulette::WorkerRoulette.start(evented: false)
+  work_order = {'ding dong' => "hello_foreman_" * 100}
+  iterations.times do |iteration|
+    sender = 'sender_' + (30_000 * rand).to_i.to_s
+    foreman = instance.foreman(sender, 'good_channel')
+    foreman.enqueue_work_order(work_order)
+    puts "published: #{iteration}" if iteration % 10_000 == 0
+  end
+end
+
+def sub(iterations)
+  instance = WorkerRoulette::WorkerRoulette.start(evented: true)
+  @tradesman = instance.tradesman('good_channel')
+  @received = 0
+  @tradesman.wait_for_work_orders do |work|
+    @received += work.length
+    puts @received if @received % (iterations / 10) == 0
+  end
+end
+
+def start(action, iterations = 1_000_000)
+  EM.kqueue = true
+  socket_max = 50_000
+  EventMachine.set_descriptor_table_size(socket_max)
+
+  EM.run do
+    Signal.trap("INT") {
+      EM.stop_event_loop
+    }
+    Signal.trap("TERM") {
+      EM.stop_event_loop
+    }
+
+    self.send(action, iterations)
+  end
+end

data/spec/benchmark/perf_test.rb

@@ -17,11 +17,11 @@ times = Benchmark.bm do |x|
       WorkerRoulette.start(evented: true)
       WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
       @total = 0
-      @tradesman = WorkerRoulette.a_tradesman
+      @tradesman = WorkerRoulette.tradesman
 
       ITERATIONS.times do |iteration|
         sender = 'sender_' + iteration.to_s
-        foreman = WorkerRoulette.a_foreman(sender)
+        foreman = WorkerRoulette.foreman(sender)
         foreman.enqueue_work_order(work_order) do
           @tradesman.work_orders! do
             @total += 1
@@ -32,34 +32,32 @@ times = Benchmark.bm do |x|
     end
   end
 end
-puts "#{ITERATIONS / times.first.real} ASync Api Read/Writes per second"
+puts "#{(ITERATIONS / times.first.real).to_i} ASync Api Read/Writes per second"
 puts "#################"
 puts
 
 WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
 
 times = Benchmark.bm do |x|
-  x.report "#{ITERATIONS * 2} ASync Api Pubsub Read/Writes" do
+  x.report "#{ITERATIONS * 2} ASync Api Polling Read/Writes" do
     EM.run do
-      WorkerRoulette.start(evented: true)
       @processed = 0
       @total     = 0
       WorkerRoulette.start(evented: true)
       WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
       @total = 0
-      @tradesman = WorkerRoulette.a_tradesman
-      on_subscribe = ->(*args) do
-        ITERATIONS.times do |iteration|
-          sender = 'sender_' + iteration.to_s
-          foreman = WorkerRoulette.a_foreman(sender)
-          foreman.enqueue_work_order(work_order)
-        end
+      @tradesman = WorkerRoulette.tradesman
+      ITERATIONS.times do |iteration|
+        @start ||= Time.now
+        sender = 'sender_' + iteration.to_s
+        foreman = WorkerRoulette.foreman(sender)
+        foreman.enqueue_work_order(work_order)
       end
-      @tradesman.wait_for_work_orders(on_subscribe) {@processed += 1; EM.stop if @processed == (ITERATIONS - 1)}
+      @tradesman.wait_for_work_orders {@processed += 1; ((@stop = Time.now) && EM.add_timer(1){EM.stop}) if @processed == (ITERATIONS - 1)}
     end
   end
 end
-puts "#{ITERATIONS * 2 / times.first.real} ASync Api Pubsub Read/Writes per second"
+puts "#{ITERATIONS * 2 / (@stop - @start)} ASync Api Polling Read/Writes per second"
 puts "#################"
 puts
 WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
@@ -78,7 +76,7 @@ times = Benchmark.bm do |x|
 end
 WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
 
-puts "#{ITERATIONS / times.first.real} Sync Api Writes per second"
+puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Writes per second"
 puts "#################"
 puts
 ITERATIONS.times do |iteration|
@@ -96,7 +94,7 @@ times = Benchmark.bm do |x|
     end
   end
 end
-puts "#{ITERATIONS / times.first.real} Sync Api Reads per second"
+puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Reads per second"
 puts "#################"
 puts
 WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
@@ -116,26 +114,24 @@ times = Benchmark.bm do |x|
     end
   end
 end
-puts "#{ITERATIONS / times.first.real} Sync Api Read/Writes per second"
+puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Read/Writes per second"
 puts "#################"
 puts
 WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
 
 times = Benchmark.bm do |x|
-  x.report "#{ITERATIONS * 2} Sync Api Pubsub Read/Writes" do
+  x.report "#{ITERATIONS * 2} Sync Api Polling Read/Writes" do
     WorkerRoulette.start(size: REDIS_CONNECTION_POOL_SIZE, evented: false)
     ITERATIONS.times do |iteration|
-      p = -> do
-        sender = 'sender_' + iteration.to_s
-        foreman = WorkerRoulette.foreman(sender)
-        foreman.enqueue_work_order(work_order)
-      end
+      sender = 'sender_' + iteration.to_s
+      foreman = WorkerRoulette.foreman(sender)
+      foreman.enqueue_work_order(work_order)
       tradesman = WorkerRoulette.tradesman
-      tradesman.wait_for_work_orders(p) {|m| m; tradesman.unsubscribe}
+      tradesman.wait_for_work_orders {|m| m}
     end
   end
 end
-puts "#{ITERATIONS * 2 / times.first.real} Sync Api Pubsub Read/Writes per second"
+puts "#{(ITERATIONS * 2 / times.first.real).to_i} Sync Api Polling Read/Writes per second"
 puts "#################"
 puts
-WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
+WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}