worker_roulette 0.1.7 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,95 +9,88 @@ require "digest/sha1"
9
9
  Dir[File.join(File.dirname(__FILE__),'worker_roulette','**','*.rb')].sort.each { |file| require file.gsub(".rb", "")}
10
10
 
11
11
  module WorkerRoulette
12
- JOB_BOARD = "job_board"
13
- JOB_NOTIFICATIONS = "new_job_ready"
14
-
15
- def self.start(config = {})
16
- @redis_config = { host: 'localhost', port: 6379, db: 14, driver: :hiredis, timeout: 5, evented: false, pool_size: 10 }.merge(config)
17
- @pool_config = Hash[size: @redis_config.delete(:pool_size), timeout: @redis_config.delete(:timeout)]
18
- @evented = @redis_config.delete(:evented)
19
-
20
- @foreman_connection_pool = ConnectionPool.new(@pool_config) {new_redis}
21
- @tradesman_connection_pool = ConnectionPool.new(@pool_config) {new_redis}
22
- @pubsub_connection_pool = ConnectionPool.new(@pool_config) {new_redis_pubsub}
23
- end
12
+ class WorkerRoulette
13
+ JOB_BOARD = "job_board"
14
+ JOB_NOTIFICATIONS = "new_job_ready"
15
+ DEFAULT_POLLING_TIME = 2
16
+
17
+ def self.dump(obj)
18
+ Oj.dump(obj)
19
+ rescue Oj::ParseError => e
20
+ {'error' => e, 'unparsable_string' => obj}
21
+ end
24
22
 
25
- def self.foreman(sender, channel = nil)
26
- raise "WorkerRoulette not Started" unless @foreman_connection_pool
27
- Foreman.new(sender, @foreman_connection_pool, channel)
28
- end
23
+ def self.load(json)
24
+ Oj.load(json)
25
+ rescue Oj::ParseError => e
26
+ {'error' => e, 'unparsable_string' => obj}
27
+ end
29
28
 
30
- def self.tradesman(channel = nil)
31
- raise "WorkerRoulette not Started" unless @tradesman_connection_pool
32
- Tradesman.new(@tradesman_connection_pool, @pubsub_connection_pool, channel)
33
- end
29
+ def self.job_board_key(namespace = nil)
30
+ "#{namespace + ':' if namespace}#{WorkerRoulette::JOB_BOARD}"
31
+ end
34
32
 
35
- def self.a_foreman(sender, channel = nil)
36
- foreman(sender, channel)
37
- end
33
+ def self.sender_key(sender, namespace = nil)
34
+ "#{namespace + ':' if namespace}#{sender}"
35
+ end
38
36
 
39
- def self.a_tradesman(channel = nil)
40
- raise "WorkerRoulette not Started" unless @tradesman_connection_pool
41
- ATradesman.new(@tradesman_connection_pool, @pubsub_connection_pool, channel)
42
- end
37
+ def self.counter_key(sender, namespace = nil)
38
+ "#{namespace + ':' if namespace}counter_key"
39
+ end
43
40
 
44
- def self.tradesman_connection_pool
45
- @tradesman_connection_pool
46
- end
41
+ def self.start(config = {})
42
+ instance = new(config)
43
+ instance
44
+ end
47
45
 
48
- def self.pubsub_connection_pool
49
- @pubsub_connection_pool
50
- end
46
+ private_class_method :new
51
47
 
52
- def self.pool_size
53
- (@pool_config ||= {})[:size]
54
- end
48
+ def initialize(config = {})
49
+ @redis_config = { host: 'localhost', port: 6379, db: 14, driver: :hiredis, timeout: 5, evented: false, pool_size: 10 , polling_time: DEFAULT_POLLING_TIME}.merge(config)
50
+ @pool_config = Hash[size: @redis_config.delete(:pool_size), timeout: @redis_config.delete(:timeout)]
51
+ @evented = @redis_config.delete(:evented)
52
+ @polling_time = @redis_config.delete(:polling_time)
55
53
 
56
- def self.redis_config
57
- (@redis_config ||= {}).dup
58
- end
59
-
60
- def self.dump(obj)
61
- Oj.dump(obj)
62
- rescue Oj::ParseError => e
63
- {'error' => e, 'unparsable_string' => obj}
64
- end
54
+ @foreman_connection_pool = ConnectionPool.new(@pool_config) {new_redis}
55
+ @tradesman_connection_pool = ConnectionPool.new(@pool_config) {new_redis}
56
+ end
65
57
 
66
- def self.load(json)
67
- Oj.load(json)
68
- rescue Oj::ParseError => e
69
- {'error' => e, 'unparsable_string' => obj}
70
- end
58
+ def foreman(sender, namespace = nil)
59
+ raise "WorkerRoulette not Started" unless @foreman_connection_pool
60
+ Foreman.new(@foreman_connection_pool, sender, namespace)
61
+ end
71
62
 
72
- def self.job_board_key(namespace = nil)
73
- "#{namespace + ':' if namespace}#{WorkerRoulette::JOB_BOARD}"
74
- end
63
+ def tradesman(namespace = nil, polling_time = DEFAULT_POLLING_TIME)
64
+ raise "WorkerRoulette not Started" unless @tradesman_connection_pool
65
+ Tradesman.new(@tradesman_connection_pool, @evented, namespace, polling_time || @polling_time)
66
+ end
75
67
 
76
- def self.sender_key(sender, namespace = nil)
77
- "#{namespace + ':' if namespace}#{sender}"
78
- end
68
+ def tradesman_connection_pool
69
+ @tradesman_connection_pool
70
+ end
79
71
 
80
- def self.counter_key(sender, namespace = nil)
81
- "#{namespace + ':' if namespace}counter_key"
82
- end
72
+ def pool_size
73
+ (@pool_config ||= {})[:size]
74
+ end
83
75
 
84
- private
76
+ def redis_config
77
+ (@redis_config ||= {}).dup
78
+ end
85
79
 
86
- def self.new_redis
87
- if @evented
88
- require 'eventmachine'
89
- redis = EM::Hiredis::Client.new(@redis_config[:host], @redis_config[:port], @redis_config[:password], @redis_config[:db])
90
- redis.connect
91
- else
92
- Redis.new(@redis_config)
80
+ def polling_time
81
+ @polling_time
93
82
  end
94
- end
95
83
 
96
- def self.new_redis_pubsub
97
- if @evented
98
- new_redis.pubsub
99
- else
100
- new_redis
84
+ private
85
+
86
+ def new_redis
87
+ if @evented
88
+ require 'eventmachine'
89
+ redis = EM::Hiredis::Client.new(@redis_config[:host], @redis_config[:port], @redis_config[:password], @redis_config[:db])
90
+ redis.connect
91
+ else
92
+ Redis.new(@redis_config)
93
+ end
101
94
  end
102
95
  end
103
96
  end
@@ -0,0 +1,39 @@
1
+ require 'worker_roulette'
2
+
3
+ def pub(iterations)
4
+ instance = WorkerRoulette::WorkerRoulette.start(evented: false)
5
+ work_order = {'ding dong' => "hello_foreman_" * 100}
6
+ iterations.times do |iteration|
7
+ sender = 'sender_' + (30_000 * rand).to_i.to_s
8
+ foreman = instance.foreman(sender, 'good_channel')
9
+ foreman.enqueue_work_order(work_order)
10
+ puts "published: #{iteration}" if iteration % 10_000 == 0
11
+ end
12
+ end
13
+
14
+ def sub(iterations)
15
+ instance = WorkerRoulette::WorkerRoulette.start(evented: true)
16
+ @tradesman = instance.tradesman('good_channel')
17
+ @received = 0
18
+ @tradesman.wait_for_work_orders do |work|
19
+ @received += work.length
20
+ puts @received if @received % (iterations / 10) == 0
21
+ end
22
+ end
23
+
24
+ def start(action, iterations = 1_000_000)
25
+ EM.kqueue = true
26
+ socket_max = 50_000
27
+ EventMachine.set_descriptor_table_size(socket_max)
28
+
29
+ EM.run do
30
+ Signal.trap("INT") {
31
+ EM.stop_event_loop
32
+ }
33
+ Signal.trap("TERM") {
34
+ EM.stop_event_loop
35
+ }
36
+
37
+ self.send(action, iterations)
38
+ end
39
+ end
@@ -17,11 +17,11 @@ times = Benchmark.bm do |x|
17
17
  WorkerRoulette.start(evented: true)
18
18
  WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
19
19
  @total = 0
20
- @tradesman = WorkerRoulette.a_tradesman
20
+ @tradesman = WorkerRoulette.tradesman
21
21
 
22
22
  ITERATIONS.times do |iteration|
23
23
  sender = 'sender_' + iteration.to_s
24
- foreman = WorkerRoulette.a_foreman(sender)
24
+ foreman = WorkerRoulette.foreman(sender)
25
25
  foreman.enqueue_work_order(work_order) do
26
26
  @tradesman.work_orders! do
27
27
  @total += 1
@@ -32,34 +32,32 @@ times = Benchmark.bm do |x|
32
32
  end
33
33
  end
34
34
  end
35
- puts "#{ITERATIONS / times.first.real} ASync Api Read/Writes per second"
35
+ puts "#{(ITERATIONS / times.first.real).to_i} ASync Api Read/Writes per second"
36
36
  puts "#################"
37
37
  puts
38
38
 
39
39
  WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
40
40
 
41
41
  times = Benchmark.bm do |x|
42
- x.report "#{ITERATIONS * 2} ASync Api Pubsub Read/Writes" do
42
+ x.report "#{ITERATIONS * 2} ASync Api Polling Read/Writes" do
43
43
  EM.run do
44
- WorkerRoulette.start(evented: true)
45
44
  @processed = 0
46
45
  @total = 0
47
46
  WorkerRoulette.start(evented: true)
48
47
  WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
49
48
  @total = 0
50
- @tradesman = WorkerRoulette.a_tradesman
51
- on_subscribe = ->(*args) do
52
- ITERATIONS.times do |iteration|
53
- sender = 'sender_' + iteration.to_s
54
- foreman = WorkerRoulette.a_foreman(sender)
55
- foreman.enqueue_work_order(work_order)
56
- end
49
+ @tradesman = WorkerRoulette.tradesman
50
+ ITERATIONS.times do |iteration|
51
+ @start ||= Time.now
52
+ sender = 'sender_' + iteration.to_s
53
+ foreman = WorkerRoulette.foreman(sender)
54
+ foreman.enqueue_work_order(work_order)
57
55
  end
58
- @tradesman.wait_for_work_orders(on_subscribe) {@processed += 1; EM.stop if @processed == (ITERATIONS - 1)}
56
+ @tradesman.wait_for_work_orders {@processed += 1; ((@stop = Time.now) && EM.add_timer(1){EM.stop}) if @processed == (ITERATIONS - 1)}
59
57
  end
60
58
  end
61
59
  end
62
- puts "#{ITERATIONS * 2 / times.first.real} ASync Api Pubsub Read/Writes per second"
60
+ puts "#{ITERATIONS * 2 / (@stop - @start)} ASync Api Polling Read/Writes per second"
63
61
  puts "#################"
64
62
  puts
65
63
  WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
@@ -78,7 +76,7 @@ times = Benchmark.bm do |x|
78
76
  end
79
77
  WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
80
78
 
81
- puts "#{ITERATIONS / times.first.real} Sync Api Writes per second"
79
+ puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Writes per second"
82
80
  puts "#################"
83
81
  puts
84
82
  ITERATIONS.times do |iteration|
@@ -96,7 +94,7 @@ times = Benchmark.bm do |x|
96
94
  end
97
95
  end
98
96
  end
99
- puts "#{ITERATIONS / times.first.real} Sync Api Reads per second"
97
+ puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Reads per second"
100
98
  puts "#################"
101
99
  puts
102
100
  WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
@@ -116,26 +114,24 @@ times = Benchmark.bm do |x|
116
114
  end
117
115
  end
118
116
  end
119
- puts "#{ITERATIONS / times.first.real} Sync Api Read/Writes per second"
117
+ puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Read/Writes per second"
120
118
  puts "#################"
121
119
  puts
122
120
  WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
123
121
 
124
122
  times = Benchmark.bm do |x|
125
- x.report "#{ITERATIONS * 2} Sync Api Pubsub Read/Writes" do
123
+ x.report "#{ITERATIONS * 2} Sync Api Polling Read/Writes" do
126
124
  WorkerRoulette.start(size: REDIS_CONNECTION_POOL_SIZE, evented: false)
127
125
  ITERATIONS.times do |iteration|
128
- p = -> do
129
- sender = 'sender_' + iteration.to_s
130
- foreman = WorkerRoulette.foreman(sender)
131
- foreman.enqueue_work_order(work_order)
132
- end
126
+ sender = 'sender_' + iteration.to_s
127
+ foreman = WorkerRoulette.foreman(sender)
128
+ foreman.enqueue_work_order(work_order)
133
129
  tradesman = WorkerRoulette.tradesman
134
- tradesman.wait_for_work_orders(p) {|m| m; tradesman.unsubscribe}
130
+ tradesman.wait_for_work_orders {|m| m}
135
131
  end
136
132
  end
137
133
  end
138
- puts "#{ITERATIONS * 2 / times.first.real} Sync Api Pubsub Read/Writes per second"
134
+ puts "#{(ITERATIONS * 2 / times.first.real).to_i} Sync Api Polling Read/Writes per second"
139
135
  puts "#################"
140
136
  puts
141
- WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
137
+ WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}