worker_roulette 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/README.md +31 -88
- data/lib/worker_roulette/foreman.rb +13 -14
- data/lib/worker_roulette/lua.rb +19 -15
- data/lib/worker_roulette/tradesman.rb +100 -72
- data/lib/worker_roulette/version.rb +1 -1
- data/lib/worker_roulette.rb +66 -73
- data/spec/benchmark/irb_demo_runner.rb +39 -0
- data/spec/benchmark/perf_test.rb +22 -26
- data/spec/integration/evented_worker_roulette_spec.rb +165 -186
- data/spec/integration/worker_roulette_spec.rb +123 -145
- data/spec/spec_helper.rb +13 -12
- data/spec/unit/evented_readlock_spec.rb +22 -21
- data/spec/unit/lua_spec.rb +12 -14
- data/spec/unit/readlock_spec.rb +10 -11
- data/worker_roulette.gemspec +1 -0
- metadata +18 -5
- data/lib/worker_roulette/a_tradesman.rb +0 -46
- data/spec/unit/worker_roulette_spec.rb +0 -14
data/lib/worker_roulette.rb
CHANGED
@@ -9,95 +9,88 @@ require "digest/sha1"
|
|
9
9
|
Dir[File.join(File.dirname(__FILE__),'worker_roulette','**','*.rb')].sort.each { |file| require file.gsub(".rb", "")}
|
10
10
|
|
11
11
|
module WorkerRoulette
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
@pubsub_connection_pool = ConnectionPool.new(@pool_config) {new_redis_pubsub}
|
23
|
-
end
|
12
|
+
class WorkerRoulette
|
13
|
+
JOB_BOARD = "job_board"
|
14
|
+
JOB_NOTIFICATIONS = "new_job_ready"
|
15
|
+
DEFAULT_POLLING_TIME = 2
|
16
|
+
|
17
|
+
def self.dump(obj)
|
18
|
+
Oj.dump(obj)
|
19
|
+
rescue Oj::ParseError => e
|
20
|
+
{'error' => e, 'unparsable_string' => obj}
|
21
|
+
end
|
24
22
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
23
|
+
def self.load(json)
|
24
|
+
Oj.load(json)
|
25
|
+
rescue Oj::ParseError => e
|
26
|
+
{'error' => e, 'unparsable_string' => obj}
|
27
|
+
end
|
29
28
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
end
|
29
|
+
def self.job_board_key(namespace = nil)
|
30
|
+
"#{namespace + ':' if namespace}#{WorkerRoulette::JOB_BOARD}"
|
31
|
+
end
|
34
32
|
|
35
|
-
|
36
|
-
|
37
|
-
|
33
|
+
def self.sender_key(sender, namespace = nil)
|
34
|
+
"#{namespace + ':' if namespace}#{sender}"
|
35
|
+
end
|
38
36
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
37
|
+
def self.counter_key(sender, namespace = nil)
|
38
|
+
"#{namespace + ':' if namespace}counter_key"
|
39
|
+
end
|
43
40
|
|
44
|
-
|
45
|
-
|
46
|
-
|
41
|
+
def self.start(config = {})
|
42
|
+
instance = new(config)
|
43
|
+
instance
|
44
|
+
end
|
47
45
|
|
48
|
-
|
49
|
-
@pubsub_connection_pool
|
50
|
-
end
|
46
|
+
private_class_method :new
|
51
47
|
|
52
|
-
|
53
|
-
|
54
|
-
|
48
|
+
def initialize(config = {})
|
49
|
+
@redis_config = { host: 'localhost', port: 6379, db: 14, driver: :hiredis, timeout: 5, evented: false, pool_size: 10 , polling_time: DEFAULT_POLLING_TIME}.merge(config)
|
50
|
+
@pool_config = Hash[size: @redis_config.delete(:pool_size), timeout: @redis_config.delete(:timeout)]
|
51
|
+
@evented = @redis_config.delete(:evented)
|
52
|
+
@polling_time = @redis_config.delete(:polling_time)
|
55
53
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
def self.dump(obj)
|
61
|
-
Oj.dump(obj)
|
62
|
-
rescue Oj::ParseError => e
|
63
|
-
{'error' => e, 'unparsable_string' => obj}
|
64
|
-
end
|
54
|
+
@foreman_connection_pool = ConnectionPool.new(@pool_config) {new_redis}
|
55
|
+
@tradesman_connection_pool = ConnectionPool.new(@pool_config) {new_redis}
|
56
|
+
end
|
65
57
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
end
|
58
|
+
def foreman(sender, namespace = nil)
|
59
|
+
raise "WorkerRoulette not Started" unless @foreman_connection_pool
|
60
|
+
Foreman.new(@foreman_connection_pool, sender, namespace)
|
61
|
+
end
|
71
62
|
|
72
|
-
|
73
|
-
|
74
|
-
|
63
|
+
def tradesman(namespace = nil, polling_time = DEFAULT_POLLING_TIME)
|
64
|
+
raise "WorkerRoulette not Started" unless @tradesman_connection_pool
|
65
|
+
Tradesman.new(@tradesman_connection_pool, @evented, namespace, polling_time || @polling_time)
|
66
|
+
end
|
75
67
|
|
76
|
-
|
77
|
-
|
78
|
-
|
68
|
+
def tradesman_connection_pool
|
69
|
+
@tradesman_connection_pool
|
70
|
+
end
|
79
71
|
|
80
|
-
|
81
|
-
|
82
|
-
|
72
|
+
def pool_size
|
73
|
+
(@pool_config ||= {})[:size]
|
74
|
+
end
|
83
75
|
|
84
|
-
|
76
|
+
def redis_config
|
77
|
+
(@redis_config ||= {}).dup
|
78
|
+
end
|
85
79
|
|
86
|
-
|
87
|
-
|
88
|
-
require 'eventmachine'
|
89
|
-
redis = EM::Hiredis::Client.new(@redis_config[:host], @redis_config[:port], @redis_config[:password], @redis_config[:db])
|
90
|
-
redis.connect
|
91
|
-
else
|
92
|
-
Redis.new(@redis_config)
|
80
|
+
def polling_time
|
81
|
+
@polling_time
|
93
82
|
end
|
94
|
-
end
|
95
83
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
84
|
+
private
|
85
|
+
|
86
|
+
def new_redis
|
87
|
+
if @evented
|
88
|
+
require 'eventmachine'
|
89
|
+
redis = EM::Hiredis::Client.new(@redis_config[:host], @redis_config[:port], @redis_config[:password], @redis_config[:db])
|
90
|
+
redis.connect
|
91
|
+
else
|
92
|
+
Redis.new(@redis_config)
|
93
|
+
end
|
101
94
|
end
|
102
95
|
end
|
103
96
|
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'worker_roulette'
|
2
|
+
|
3
|
+
def pub(iterations)
|
4
|
+
instance = WorkerRoulette::WorkerRoulette.start(evented: false)
|
5
|
+
work_order = {'ding dong' => "hello_foreman_" * 100}
|
6
|
+
iterations.times do |iteration|
|
7
|
+
sender = 'sender_' + (30_000 * rand).to_i.to_s
|
8
|
+
foreman = instance.foreman(sender, 'good_channel')
|
9
|
+
foreman.enqueue_work_order(work_order)
|
10
|
+
puts "published: #{iteration}" if iteration % 10_000 == 0
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def sub(iterations)
|
15
|
+
instance = WorkerRoulette::WorkerRoulette.start(evented: true)
|
16
|
+
@tradesman = instance.tradesman('good_channel')
|
17
|
+
@received = 0
|
18
|
+
@tradesman.wait_for_work_orders do |work|
|
19
|
+
@received += work.length
|
20
|
+
puts @received if @received % (iterations / 10) == 0
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def start(action, iterations = 1_000_000)
|
25
|
+
EM.kqueue = true
|
26
|
+
socket_max = 50_000
|
27
|
+
EventMachine.set_descriptor_table_size(socket_max)
|
28
|
+
|
29
|
+
EM.run do
|
30
|
+
Signal.trap("INT") {
|
31
|
+
EM.stop_event_loop
|
32
|
+
}
|
33
|
+
Signal.trap("TERM") {
|
34
|
+
EM.stop_event_loop
|
35
|
+
}
|
36
|
+
|
37
|
+
self.send(action, iterations)
|
38
|
+
end
|
39
|
+
end
|
data/spec/benchmark/perf_test.rb
CHANGED
@@ -17,11 +17,11 @@ times = Benchmark.bm do |x|
|
|
17
17
|
WorkerRoulette.start(evented: true)
|
18
18
|
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
19
19
|
@total = 0
|
20
|
-
@tradesman = WorkerRoulette.
|
20
|
+
@tradesman = WorkerRoulette.tradesman
|
21
21
|
|
22
22
|
ITERATIONS.times do |iteration|
|
23
23
|
sender = 'sender_' + iteration.to_s
|
24
|
-
foreman = WorkerRoulette.
|
24
|
+
foreman = WorkerRoulette.foreman(sender)
|
25
25
|
foreman.enqueue_work_order(work_order) do
|
26
26
|
@tradesman.work_orders! do
|
27
27
|
@total += 1
|
@@ -32,34 +32,32 @@ times = Benchmark.bm do |x|
|
|
32
32
|
end
|
33
33
|
end
|
34
34
|
end
|
35
|
-
puts "#{ITERATIONS / times.first.real} ASync Api Read/Writes per second"
|
35
|
+
puts "#{(ITERATIONS / times.first.real).to_i} ASync Api Read/Writes per second"
|
36
36
|
puts "#################"
|
37
37
|
puts
|
38
38
|
|
39
39
|
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
40
40
|
|
41
41
|
times = Benchmark.bm do |x|
|
42
|
-
x.report "#{ITERATIONS * 2} ASync Api
|
42
|
+
x.report "#{ITERATIONS * 2} ASync Api Polling Read/Writes" do
|
43
43
|
EM.run do
|
44
|
-
WorkerRoulette.start(evented: true)
|
45
44
|
@processed = 0
|
46
45
|
@total = 0
|
47
46
|
WorkerRoulette.start(evented: true)
|
48
47
|
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
49
48
|
@total = 0
|
50
|
-
@tradesman = WorkerRoulette.
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
end
|
49
|
+
@tradesman = WorkerRoulette.tradesman
|
50
|
+
ITERATIONS.times do |iteration|
|
51
|
+
@start ||= Time.now
|
52
|
+
sender = 'sender_' + iteration.to_s
|
53
|
+
foreman = WorkerRoulette.foreman(sender)
|
54
|
+
foreman.enqueue_work_order(work_order)
|
57
55
|
end
|
58
|
-
@tradesman.wait_for_work_orders
|
56
|
+
@tradesman.wait_for_work_orders {@processed += 1; ((@stop = Time.now) && EM.add_timer(1){EM.stop}) if @processed == (ITERATIONS - 1)}
|
59
57
|
end
|
60
58
|
end
|
61
59
|
end
|
62
|
-
puts "#{ITERATIONS * 2 /
|
60
|
+
puts "#{ITERATIONS * 2 / (@stop - @start)} ASync Api Polling Read/Writes per second"
|
63
61
|
puts "#################"
|
64
62
|
puts
|
65
63
|
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
@@ -78,7 +76,7 @@ times = Benchmark.bm do |x|
|
|
78
76
|
end
|
79
77
|
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
80
78
|
|
81
|
-
puts "#{ITERATIONS / times.first.real} Sync Api Writes per second"
|
79
|
+
puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Writes per second"
|
82
80
|
puts "#################"
|
83
81
|
puts
|
84
82
|
ITERATIONS.times do |iteration|
|
@@ -96,7 +94,7 @@ times = Benchmark.bm do |x|
|
|
96
94
|
end
|
97
95
|
end
|
98
96
|
end
|
99
|
-
puts "#{ITERATIONS / times.first.real} Sync Api Reads per second"
|
97
|
+
puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Reads per second"
|
100
98
|
puts "#################"
|
101
99
|
puts
|
102
100
|
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
@@ -116,26 +114,24 @@ times = Benchmark.bm do |x|
|
|
116
114
|
end
|
117
115
|
end
|
118
116
|
end
|
119
|
-
puts "#{ITERATIONS / times.first.real} Sync Api Read/Writes per second"
|
117
|
+
puts "#{(ITERATIONS / times.first.real).to_i} Sync Api Read/Writes per second"
|
120
118
|
puts "#################"
|
121
119
|
puts
|
122
120
|
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
123
121
|
|
124
122
|
times = Benchmark.bm do |x|
|
125
|
-
x.report "#{ITERATIONS * 2} Sync Api
|
123
|
+
x.report "#{ITERATIONS * 2} Sync Api Polling Read/Writes" do
|
126
124
|
WorkerRoulette.start(size: REDIS_CONNECTION_POOL_SIZE, evented: false)
|
127
125
|
ITERATIONS.times do |iteration|
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
foreman.enqueue_work_order(work_order)
|
132
|
-
end
|
126
|
+
sender = 'sender_' + iteration.to_s
|
127
|
+
foreman = WorkerRoulette.foreman(sender)
|
128
|
+
foreman.enqueue_work_order(work_order)
|
133
129
|
tradesman = WorkerRoulette.tradesman
|
134
|
-
tradesman.wait_for_work_orders
|
130
|
+
tradesman.wait_for_work_orders {|m| m}
|
135
131
|
end
|
136
132
|
end
|
137
133
|
end
|
138
|
-
puts "#{ITERATIONS * 2 / times.first.real} Sync Api
|
134
|
+
puts "#{(ITERATIONS * 2 / times.first.real).to_i} Sync Api Polling Read/Writes per second"
|
139
135
|
puts "#################"
|
140
136
|
puts
|
141
|
-
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
137
|
+
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|