dat-worker-pool 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -1
- data/bench/report.rb +130 -0
- data/bench/report.txt +7 -0
- data/dat-worker-pool.gemspec +1 -1
- data/lib/dat-worker-pool.rb +38 -187
- data/lib/dat-worker-pool/default_queue.rb +60 -0
- data/lib/dat-worker-pool/locked_object.rb +60 -0
- data/lib/dat-worker-pool/queue.rb +71 -48
- data/lib/dat-worker-pool/runner.rb +196 -0
- data/lib/dat-worker-pool/version.rb +1 -1
- data/lib/dat-worker-pool/worker.rb +251 -72
- data/lib/dat-worker-pool/worker_pool_spy.rb +39 -53
- data/test/helper.rb +13 -0
- data/test/support/factory.rb +15 -0
- data/test/support/thread_spies.rb +83 -0
- data/test/system/dat-worker-pool_tests.rb +399 -0
- data/test/unit/dat-worker-pool_tests.rb +132 -255
- data/test/unit/default_queue_tests.rb +217 -0
- data/test/unit/locked_object_tests.rb +260 -0
- data/test/unit/queue_tests.rb +95 -72
- data/test/unit/runner_tests.rb +365 -0
- data/test/unit/worker_pool_spy_tests.rb +95 -102
- data/test/unit/worker_tests.rb +819 -153
- metadata +27 -12
- data/test/system/use_worker_pool_tests.rb +0 -34
@@ -1,321 +1,198 @@
|
|
1
1
|
require 'assert'
|
2
2
|
require 'dat-worker-pool'
|
3
3
|
|
4
|
+
require 'system_timer'
|
5
|
+
require 'dat-worker-pool/queue'
|
6
|
+
require 'dat-worker-pool/runner'
|
7
|
+
require 'dat-worker-pool/worker'
|
8
|
+
|
4
9
|
class DatWorkerPool
|
5
10
|
|
6
11
|
class UnitTests < Assert::Context
|
7
12
|
desc "DatWorkerPool"
|
8
13
|
setup do
|
9
|
-
@
|
10
|
-
end
|
11
|
-
subject{ @work_pool }
|
12
|
-
|
13
|
-
should have_readers :logger, :spawned, :queue
|
14
|
-
should have_readers :on_worker_error_callbacks
|
15
|
-
should have_readers :on_worker_start_callbacks, :on_worker_shutdown_callbacks
|
16
|
-
should have_readers :on_worker_sleep_callbacks, :on_worker_wakeup_callbacks
|
17
|
-
should have_readers :before_work_callbacks, :after_work_callbacks
|
18
|
-
should have_imeths :add_work, :start, :shutdown
|
19
|
-
should have_imeths :work_items, :waiting
|
20
|
-
should have_imeths :worker_available?, :all_spawned_workers_are_busy?
|
21
|
-
should have_imeths :reached_max_workers?
|
22
|
-
should have_imeths :queue_empty?
|
23
|
-
should have_imeths :on_queue_pop_callbacks, :on_queue_push_callbacks
|
24
|
-
should have_imeths :on_queue_pop, :on_queue_push
|
25
|
-
should have_imeths :on_worker_error
|
26
|
-
should have_imeths :on_worker_start, :on_worker_shutdown
|
27
|
-
should have_imeths :on_worker_sleep, :on_worker_wakeup
|
28
|
-
should have_imeths :before_work, :after_work
|
29
|
-
|
30
|
-
should "know its attributes" do
|
31
|
-
assert_instance_of ::Logger, subject.logger
|
32
|
-
assert_equal 0, subject.spawned
|
33
|
-
assert_instance_of Queue, subject.queue
|
14
|
+
@worker_pool_class = DatWorkerPool
|
34
15
|
end
|
16
|
+
subject{ @worker_pool_class }
|
35
17
|
|
36
|
-
should "
|
37
|
-
assert_equal
|
38
|
-
assert_equal
|
39
|
-
assert_equal 1, subject.on_worker_shutdown_callbacks.size
|
40
|
-
assert_instance_of Proc, subject.on_worker_shutdown_callbacks.first
|
41
|
-
assert_equal 1, subject.on_worker_sleep_callbacks.size
|
42
|
-
assert_instance_of Proc, subject.on_worker_sleep_callbacks.first
|
43
|
-
assert_equal 1, subject.on_worker_wakeup_callbacks.size
|
44
|
-
assert_instance_of Proc, subject.on_worker_wakeup_callbacks.first
|
45
|
-
assert_equal [], subject.before_work_callbacks
|
46
|
-
assert_equal [], subject.after_work_callbacks
|
47
|
-
end
|
48
|
-
|
49
|
-
should "demeter its queue's callbacks" do
|
50
|
-
callback = proc{ }
|
51
|
-
subject.on_queue_pop(&callback)
|
52
|
-
assert_equal [callback], subject.on_queue_pop_callbacks
|
53
|
-
callback = proc{ }
|
54
|
-
subject.on_queue_push(&callback)
|
55
|
-
assert_equal [callback], subject.on_queue_push_callbacks
|
18
|
+
should "know its default and min number of workers" do
|
19
|
+
assert_equal 1, DEFAULT_NUM_WORKERS
|
20
|
+
assert_equal 1, MIN_WORKERS
|
56
21
|
end
|
57
22
|
|
58
23
|
end
|
59
24
|
|
60
|
-
class
|
61
|
-
desc "
|
25
|
+
class InitSetupTests < UnitTests
|
26
|
+
desc "when init"
|
62
27
|
setup do
|
63
|
-
@
|
64
|
-
@
|
65
|
-
|
28
|
+
@num_workers = Factory.integer(4)
|
29
|
+
@logger = TEST_LOGGER
|
30
|
+
@queue = TestQueue.new
|
31
|
+
@worker_params = { Factory.string => Factory.string }
|
32
|
+
|
33
|
+
@runner_spy = RunnerSpy.new
|
34
|
+
Assert.stub(DatWorkerPool::Runner, :new) do |args|
|
35
|
+
@runner_spy.args = args
|
36
|
+
@runner_spy
|
37
|
+
end
|
66
38
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
assert_equal 1, @work_pool.spawned
|
78
|
-
assert_equal 0, @work_pool.waiting
|
79
|
-
assert_equal true, @work_pool.worker_available?
|
80
|
-
assert_equal true, @work_pool.all_spawned_workers_are_busy?
|
81
|
-
assert_equal false, @work_pool.reached_max_workers?
|
82
|
-
|
83
|
-
# an additional worker should be spawned
|
84
|
-
@work_pool.add_work 5
|
85
|
-
assert_equal 2, @work_pool.spawned
|
86
|
-
assert_equal 0, @work_pool.waiting
|
87
|
-
assert_equal false, @work_pool.worker_available?
|
88
|
-
assert_equal true, @work_pool.all_spawned_workers_are_busy?
|
89
|
-
assert_equal true, @work_pool.reached_max_workers?
|
90
|
-
|
91
|
-
# no additional workers are spawned, the work waits to be processed
|
92
|
-
@work_pool.add_work 5
|
93
|
-
assert_equal 2, @work_pool.spawned
|
94
|
-
assert_equal 0, @work_pool.waiting
|
95
|
-
assert_equal false, @work_pool.worker_available?
|
96
|
-
assert_equal true, @work_pool.all_spawned_workers_are_busy?
|
97
|
-
assert_equal true, @work_pool.reached_max_workers?
|
39
|
+
@worker_class = Class.new do
|
40
|
+
include DatWorkerPool::Worker
|
41
|
+
def work!(work_item); end
|
42
|
+
end
|
43
|
+
@options = {
|
44
|
+
:num_workers => @num_workers,
|
45
|
+
:logger => @logger,
|
46
|
+
:queue => @queue,
|
47
|
+
:worker_params => @worker_params
|
48
|
+
}
|
98
49
|
end
|
50
|
+
subject{ @worker_pool }
|
99
51
|
|
100
|
-
|
101
|
-
assert_equal 1, @work_pool.spawned
|
102
|
-
assert_equal 1, @work_pool.waiting
|
52
|
+
end
|
103
53
|
|
104
|
-
|
105
|
-
|
106
|
-
|
54
|
+
class InitTests < InitSetupTests
|
55
|
+
desc "when init"
|
56
|
+
setup do
|
57
|
+
@worker_pool = @worker_pool_class.new(@worker_class, @options)
|
58
|
+
end
|
59
|
+
subject{ @worker_pool }
|
107
60
|
|
108
|
-
|
61
|
+
should have_readers :queue
|
62
|
+
should have_imeths :start, :shutdown
|
63
|
+
should have_imeths :add_work, :push, :work_items
|
64
|
+
should have_imeths :available_worker_count, :worker_available?
|
109
65
|
|
110
|
-
|
111
|
-
assert_equal
|
66
|
+
should "know its attributes" do
|
67
|
+
assert_equal @queue, subject.queue
|
112
68
|
end
|
113
69
|
|
114
|
-
|
70
|
+
should "build a runner" do
|
71
|
+
exp = {
|
72
|
+
:num_workers => @num_workers,
|
73
|
+
:logger => @logger,
|
74
|
+
:queue => @queue,
|
75
|
+
:worker_class => @worker_class,
|
76
|
+
:worker_params => @worker_params
|
77
|
+
}
|
78
|
+
assert_equal exp, @runner_spy.args
|
79
|
+
end
|
115
80
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
@error_called = false
|
120
|
-
@start_called = false
|
121
|
-
@shutdown_called = false
|
122
|
-
@sleep_called = false
|
123
|
-
@wakeup_called = false
|
124
|
-
@before_work_called = false
|
125
|
-
@after_work_called = false
|
81
|
+
should "default its attributes" do
|
82
|
+
worker_pool = @worker_pool_class.new(@worker_class)
|
83
|
+
assert_instance_of DatWorkerPool::DefaultQueue, worker_pool.queue
|
126
84
|
|
127
|
-
|
128
|
-
|
129
|
-
end
|
130
|
-
@work_pool.on_worker_error{ @error_called = true }
|
131
|
-
@work_pool.on_worker_start{ @start_called = true }
|
132
|
-
@work_pool.on_worker_shutdown{ @shutdown_called = true }
|
133
|
-
@work_pool.on_worker_sleep{ @sleep_called = true }
|
134
|
-
@work_pool.on_worker_wakeup{ @wakeup_called = true }
|
135
|
-
@work_pool.before_work{ @before_work_called = true }
|
136
|
-
@work_pool.after_work{ @after_work_called = true }
|
85
|
+
assert_equal DEFAULT_NUM_WORKERS, @runner_spy.args[:num_workers]
|
86
|
+
assert_nil @runner_spy.args[:worker_params]
|
137
87
|
end
|
138
|
-
subject{ @work_pool }
|
139
88
|
|
140
|
-
should "
|
141
|
-
assert_false @start_called
|
142
|
-
assert_false @sleep_called
|
89
|
+
should "start its runner when its started" do
|
90
|
+
assert_false @runner_spy.start_called
|
143
91
|
subject.start
|
144
|
-
assert_true @start_called
|
145
|
-
assert_true @sleep_called
|
146
|
-
|
147
|
-
@sleep_called = false
|
148
|
-
assert_false @wakeup_called
|
149
|
-
assert_false @before_work_called
|
150
|
-
assert_false @after_work_called
|
151
|
-
subject.add_work 'work'
|
152
|
-
assert_true @wakeup_called
|
153
|
-
assert_true @before_work_called
|
154
|
-
assert_true @after_work_called
|
155
|
-
assert_true @sleep_called
|
156
|
-
|
157
|
-
@before_work_called = false
|
158
|
-
@after_work_called = false
|
159
|
-
assert_false @before_work_called
|
160
|
-
assert_false @error_called
|
161
|
-
assert_false @after_work_called
|
162
|
-
subject.add_work 'error'
|
163
|
-
assert_true @before_work_called
|
164
|
-
assert_true @error_called
|
165
|
-
assert_false @after_work_called
|
166
|
-
|
167
|
-
@wakeup_called = false
|
168
|
-
assert_false @shutdown_called
|
169
|
-
subject.shutdown
|
170
|
-
assert_true @wakeup_called
|
171
|
-
assert_true @shutdown_called
|
92
|
+
assert_true @runner_spy.start_called
|
172
93
|
end
|
173
94
|
|
174
|
-
|
95
|
+
should "shutdown its runner when its shutdown" do
|
96
|
+
assert_false @runner_spy.shutdown_called
|
97
|
+
subject.shutdown
|
98
|
+
assert_true @runner_spy.shutdown_called
|
99
|
+
assert_nil @runner_spy.shutdown_timeout
|
100
|
+
exp = "test/unit/dat-worker-pool_tests.rb"
|
101
|
+
assert_match exp, @runner_spy.shutdown_backtrace.first
|
175
102
|
|
176
|
-
|
177
|
-
|
178
|
-
|
103
|
+
timeout = Factory.integer
|
104
|
+
subject.shutdown(timeout)
|
105
|
+
assert_equal timeout, @runner_spy.shutdown_timeout
|
179
106
|
end
|
180
107
|
|
181
|
-
should "
|
182
|
-
assert_equal
|
183
|
-
@
|
184
|
-
assert_equal false, @work_pool.queue_empty?
|
108
|
+
should "demeter its runner" do
|
109
|
+
assert_equal @runner_spy.available_worker_count, subject.available_worker_count
|
110
|
+
assert_equal @runner_spy.worker_available?, subject.worker_available?
|
185
111
|
end
|
186
112
|
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
desc "add_work and process"
|
191
|
-
setup do
|
192
|
-
@result = nil
|
193
|
-
@work_pool = DatWorkerPool.new(1){|work| @result = (2 / work) }
|
194
|
-
@work_pool.start
|
113
|
+
should "raise an argument error if given an invalid worker class" do
|
114
|
+
assert_raises(ArgumentError){ @worker_pool_class.new(Module.new) }
|
115
|
+
assert_raises(ArgumentError){ @worker_pool_class.new(Class.new) }
|
195
116
|
end
|
196
117
|
|
197
|
-
should "
|
198
|
-
|
199
|
-
|
200
|
-
|
118
|
+
should "raise an argument error if given an invalid number of workers" do
|
119
|
+
assert_raises(ArgumentError) do
|
120
|
+
@worker_pool_class.new(@worker_class, {
|
121
|
+
:num_workers => [0, (Factory.integer * -1)].choice
|
122
|
+
})
|
123
|
+
end
|
201
124
|
end
|
202
125
|
|
203
|
-
|
204
|
-
subject.add_work 0
|
205
|
-
worker = subject.instance_variable_get("@workers").first
|
206
|
-
sleep 0.1
|
126
|
+
end
|
207
127
|
|
208
|
-
|
209
|
-
|
210
|
-
|
128
|
+
class StartedTests < InitTests
|
129
|
+
desc "and started"
|
130
|
+
setup do
|
131
|
+
@worker_pool.start
|
211
132
|
end
|
212
133
|
|
213
|
-
|
134
|
+
should "be able to add work onto its queue`" do
|
135
|
+
work_item = Factory.string
|
136
|
+
subject.add_work(work_item)
|
137
|
+
assert_equal work_item, @queue.work_items.last
|
214
138
|
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
@work_pool = DatWorkerPool.new(1, 2){ |work| sleep(work) }
|
139
|
+
work_item = Factory.string
|
140
|
+
subject.push(work_item)
|
141
|
+
assert_equal work_item, @queue.work_items.last
|
219
142
|
end
|
220
|
-
subject{ @work_pool }
|
221
143
|
|
222
|
-
should "
|
223
|
-
|
224
|
-
|
225
|
-
assert_false subject.queue.shutdown?
|
226
|
-
subject.shutdown
|
227
|
-
assert_true subject.queue.shutdown?
|
228
|
-
subject.start
|
229
|
-
assert_false subject.queue.shutdown?
|
144
|
+
should "not add `nil` work onto its queue" do
|
145
|
+
subject.add_work(nil)
|
146
|
+
assert_equal [], @queue.work_items
|
230
147
|
end
|
231
148
|
|
232
|
-
should "
|
233
|
-
|
234
|
-
subject.
|
235
|
-
assert_equal 0, subject.spawned
|
236
|
-
subject.start
|
237
|
-
assert_equal 1, subject.spawned
|
149
|
+
should "know its queue's work items" do
|
150
|
+
Factory.integer(3).times{ @queue.dwp_push(Factory.string) }
|
151
|
+
assert_equal @queue.work_items, subject.work_items
|
238
152
|
end
|
239
153
|
|
240
154
|
end
|
241
155
|
|
242
|
-
class
|
243
|
-
|
244
|
-
setup do
|
245
|
-
@mutex = Mutex.new
|
246
|
-
@finished = []
|
247
|
-
@work_pool = DatWorkerPool.new(1, 2, true) do |work|
|
248
|
-
sleep 1
|
249
|
-
@mutex.synchronize{ @finished << work }
|
250
|
-
end
|
251
|
-
@work_pool.start
|
252
|
-
@work_pool.add_work 'a'
|
253
|
-
@work_pool.add_work 'b'
|
254
|
-
@work_pool.add_work 'c'
|
255
|
-
end
|
156
|
+
class TestQueue
|
157
|
+
include DatWorkerPool::Queue
|
256
158
|
|
257
|
-
|
258
|
-
# make sure the workers haven't processed any work
|
259
|
-
assert_equal [], @finished
|
260
|
-
subject.shutdown(5)
|
159
|
+
attr_reader :work_items
|
261
160
|
|
262
|
-
|
263
|
-
|
264
|
-
assert_includes 'a', @finished
|
265
|
-
assert_includes 'b', @finished
|
266
|
-
assert_not_includes 'c', @finished
|
267
|
-
|
268
|
-
assert_equal 0, subject.spawned
|
269
|
-
assert_equal 0, subject.waiting
|
270
|
-
assert_includes 'c', subject.work_items
|
161
|
+
def initialize
|
162
|
+
@work_items = []
|
271
163
|
end
|
272
164
|
|
273
|
-
|
274
|
-
assert_equal [], @finished
|
275
|
-
subject.shutdown
|
276
|
-
assert_includes 'a', @finished
|
277
|
-
assert_includes 'b', @finished
|
278
|
-
end
|
165
|
+
private
|
279
166
|
|
280
|
-
|
281
|
-
|
282
|
-
subject.shutdown(0.1)
|
283
|
-
end
|
167
|
+
def push!(work_item)
|
168
|
+
@work_items << work_item
|
284
169
|
end
|
285
|
-
|
286
170
|
end
|
287
171
|
|
288
|
-
class
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
@
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
@mutex.synchronize{ @finished << error }
|
300
|
-
raise error # re-raise it otherwise worker won't shutdown
|
301
|
-
end
|
302
|
-
end
|
303
|
-
@work_pool.start
|
304
|
-
@work_pool.add_work 'a'
|
305
|
-
@work_pool.add_work 'b'
|
306
|
-
@work_pool.add_work 'c'
|
172
|
+
class RunnerSpy < DatWorkerPool::Runner
|
173
|
+
attr_accessor :args
|
174
|
+
attr_reader :start_called, :shutdown_called
|
175
|
+
attr_reader :shutdown_timeout, :shutdown_backtrace
|
176
|
+
|
177
|
+
def initialize
|
178
|
+
super({})
|
179
|
+
@start_called = false
|
180
|
+
@shutdown_called = false
|
181
|
+
@shutdown_timeout = nil
|
182
|
+
@shutdown_backtrace = nil
|
307
183
|
end
|
308
184
|
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
subject.shutdown(0.1)
|
313
|
-
assert_equal @max_workers, @finished.size
|
314
|
-
@finished.each do |error|
|
315
|
-
assert_instance_of DatWorkerPool::ShutdownError, error
|
316
|
-
end
|
185
|
+
def start
|
186
|
+
@args[:queue].dwp_start
|
187
|
+
@start_called = true
|
317
188
|
end
|
318
189
|
|
190
|
+
def shutdown(timeout, backtrace)
|
191
|
+
@args[:queue].dwp_shutdown
|
192
|
+
@shutdown_called = true
|
193
|
+
@shutdown_timeout = timeout
|
194
|
+
@shutdown_backtrace = backtrace
|
195
|
+
end
|
319
196
|
end
|
320
197
|
|
321
198
|
end
|