dat-worker-pool 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,321 +1,198 @@
1
1
  require 'assert'
2
2
  require 'dat-worker-pool'
3
3
 
4
+ require 'system_timer'
5
+ require 'dat-worker-pool/queue'
6
+ require 'dat-worker-pool/runner'
7
+ require 'dat-worker-pool/worker'
8
+
4
9
  class DatWorkerPool
5
10
 
6
11
  class UnitTests < Assert::Context
7
12
  desc "DatWorkerPool"
8
13
  setup do
9
- @work_pool = DatWorkerPool.new{ }
10
- end
11
- subject{ @work_pool }
12
-
13
- should have_readers :logger, :spawned, :queue
14
- should have_readers :on_worker_error_callbacks
15
- should have_readers :on_worker_start_callbacks, :on_worker_shutdown_callbacks
16
- should have_readers :on_worker_sleep_callbacks, :on_worker_wakeup_callbacks
17
- should have_readers :before_work_callbacks, :after_work_callbacks
18
- should have_imeths :add_work, :start, :shutdown
19
- should have_imeths :work_items, :waiting
20
- should have_imeths :worker_available?, :all_spawned_workers_are_busy?
21
- should have_imeths :reached_max_workers?
22
- should have_imeths :queue_empty?
23
- should have_imeths :on_queue_pop_callbacks, :on_queue_push_callbacks
24
- should have_imeths :on_queue_pop, :on_queue_push
25
- should have_imeths :on_worker_error
26
- should have_imeths :on_worker_start, :on_worker_shutdown
27
- should have_imeths :on_worker_sleep, :on_worker_wakeup
28
- should have_imeths :before_work, :after_work
29
-
30
- should "know its attributes" do
31
- assert_instance_of ::Logger, subject.logger
32
- assert_equal 0, subject.spawned
33
- assert_instance_of Queue, subject.queue
14
+ @worker_pool_class = DatWorkerPool
34
15
  end
16
+ subject{ @worker_pool_class }
35
17
 
36
- should "default its worker callbacks" do
37
- assert_equal [], subject.on_worker_error_callbacks
38
- assert_equal [], subject.on_worker_start_callbacks
39
- assert_equal 1, subject.on_worker_shutdown_callbacks.size
40
- assert_instance_of Proc, subject.on_worker_shutdown_callbacks.first
41
- assert_equal 1, subject.on_worker_sleep_callbacks.size
42
- assert_instance_of Proc, subject.on_worker_sleep_callbacks.first
43
- assert_equal 1, subject.on_worker_wakeup_callbacks.size
44
- assert_instance_of Proc, subject.on_worker_wakeup_callbacks.first
45
- assert_equal [], subject.before_work_callbacks
46
- assert_equal [], subject.after_work_callbacks
47
- end
48
-
49
- should "demeter its queue's callbacks" do
50
- callback = proc{ }
51
- subject.on_queue_pop(&callback)
52
- assert_equal [callback], subject.on_queue_pop_callbacks
53
- callback = proc{ }
54
- subject.on_queue_push(&callback)
55
- assert_equal [callback], subject.on_queue_push_callbacks
18
+ should "know its default and min number of workers" do
19
+ assert_equal 1, DEFAULT_NUM_WORKERS
20
+ assert_equal 1, MIN_WORKERS
56
21
  end
57
22
 
58
23
  end
59
24
 
60
- class WorkerBehaviorTests < UnitTests
61
- desc "workers"
25
+ class InitSetupTests < UnitTests
26
+ desc "when init"
62
27
  setup do
63
- @work_pool = DatWorkerPool.new(1, 2, true){ |work| sleep(work) }
64
- @work_pool.start
65
- end
28
+ @num_workers = Factory.integer(4)
29
+ @logger = TEST_LOGGER
30
+ @queue = TestQueue.new
31
+ @worker_params = { Factory.string => Factory.string }
32
+
33
+ @runner_spy = RunnerSpy.new
34
+ Assert.stub(DatWorkerPool::Runner, :new) do |args|
35
+ @runner_spy.args = args
36
+ @runner_spy
37
+ end
66
38
 
67
- should "be created as needed and only go up to the maximum number allowed" do
68
- # the minimum should be spawned and waiting
69
- assert_equal 1, @work_pool.spawned
70
- assert_equal 1, @work_pool.waiting
71
- assert_equal true, @work_pool.worker_available?
72
- assert_equal false, @work_pool.all_spawned_workers_are_busy?
73
- assert_equal false, @work_pool.reached_max_workers?
74
-
75
- # the minimum should be spawned, but no longer waiting
76
- @work_pool.add_work 5
77
- assert_equal 1, @work_pool.spawned
78
- assert_equal 0, @work_pool.waiting
79
- assert_equal true, @work_pool.worker_available?
80
- assert_equal true, @work_pool.all_spawned_workers_are_busy?
81
- assert_equal false, @work_pool.reached_max_workers?
82
-
83
- # an additional worker should be spawned
84
- @work_pool.add_work 5
85
- assert_equal 2, @work_pool.spawned
86
- assert_equal 0, @work_pool.waiting
87
- assert_equal false, @work_pool.worker_available?
88
- assert_equal true, @work_pool.all_spawned_workers_are_busy?
89
- assert_equal true, @work_pool.reached_max_workers?
90
-
91
- # no additional workers are spawned, the work waits to be processed
92
- @work_pool.add_work 5
93
- assert_equal 2, @work_pool.spawned
94
- assert_equal 0, @work_pool.waiting
95
- assert_equal false, @work_pool.worker_available?
96
- assert_equal true, @work_pool.all_spawned_workers_are_busy?
97
- assert_equal true, @work_pool.reached_max_workers?
39
+ @worker_class = Class.new do
40
+ include DatWorkerPool::Worker
41
+ def work!(work_item); end
42
+ end
43
+ @options = {
44
+ :num_workers => @num_workers,
45
+ :logger => @logger,
46
+ :queue => @queue,
47
+ :worker_params => @worker_params
48
+ }
98
49
  end
50
+ subject{ @worker_pool }
99
51
 
100
- should "go back to waiting when they finish working" do
101
- assert_equal 1, @work_pool.spawned
102
- assert_equal 1, @work_pool.waiting
52
+ end
103
53
 
104
- @work_pool.add_work 1
105
- assert_equal 1, @work_pool.spawned
106
- assert_equal 0, @work_pool.waiting
54
+ class InitTests < InitSetupTests
55
+ desc "when init"
56
+ setup do
57
+ @worker_pool = @worker_pool_class.new(@worker_class, @options)
58
+ end
59
+ subject{ @worker_pool }
107
60
 
108
- sleep 1 # allow the worker to run
61
+ should have_readers :queue
62
+ should have_imeths :start, :shutdown
63
+ should have_imeths :add_work, :push, :work_items
64
+ should have_imeths :available_worker_count, :worker_available?
109
65
 
110
- assert_equal 1, @work_pool.spawned
111
- assert_equal 1, @work_pool.waiting
66
+ should "know its attributes" do
67
+ assert_equal @queue, subject.queue
112
68
  end
113
69
 
114
- end
70
+ should "build a runner" do
71
+ exp = {
72
+ :num_workers => @num_workers,
73
+ :logger => @logger,
74
+ :queue => @queue,
75
+ :worker_class => @worker_class,
76
+ :worker_params => @worker_params
77
+ }
78
+ assert_equal exp, @runner_spy.args
79
+ end
115
80
 
116
- class WorkerCallbackTests < UnitTests
117
- desc "worker callbacks"
118
- setup do
119
- @error_called = false
120
- @start_called = false
121
- @shutdown_called = false
122
- @sleep_called = false
123
- @wakeup_called = false
124
- @before_work_called = false
125
- @after_work_called = false
81
+ should "default its attributes" do
82
+ worker_pool = @worker_pool_class.new(@worker_class)
83
+ assert_instance_of DatWorkerPool::DefaultQueue, worker_pool.queue
126
84
 
127
- @work_pool = DatWorkerPool.new(1) do |work|
128
- raise work if work == 'error'
129
- end
130
- @work_pool.on_worker_error{ @error_called = true }
131
- @work_pool.on_worker_start{ @start_called = true }
132
- @work_pool.on_worker_shutdown{ @shutdown_called = true }
133
- @work_pool.on_worker_sleep{ @sleep_called = true }
134
- @work_pool.on_worker_wakeup{ @wakeup_called = true }
135
- @work_pool.before_work{ @before_work_called = true }
136
- @work_pool.after_work{ @after_work_called = true }
85
+ assert_equal DEFAULT_NUM_WORKERS, @runner_spy.args[:num_workers]
86
+ assert_nil @runner_spy.args[:worker_params]
137
87
  end
138
- subject{ @work_pool }
139
88
 
140
- should "call the worker callbacks as workers wait or wakeup" do
141
- assert_false @start_called
142
- assert_false @sleep_called
89
+ should "start its runner when its started" do
90
+ assert_false @runner_spy.start_called
143
91
  subject.start
144
- assert_true @start_called
145
- assert_true @sleep_called
146
-
147
- @sleep_called = false
148
- assert_false @wakeup_called
149
- assert_false @before_work_called
150
- assert_false @after_work_called
151
- subject.add_work 'work'
152
- assert_true @wakeup_called
153
- assert_true @before_work_called
154
- assert_true @after_work_called
155
- assert_true @sleep_called
156
-
157
- @before_work_called = false
158
- @after_work_called = false
159
- assert_false @before_work_called
160
- assert_false @error_called
161
- assert_false @after_work_called
162
- subject.add_work 'error'
163
- assert_true @before_work_called
164
- assert_true @error_called
165
- assert_false @after_work_called
166
-
167
- @wakeup_called = false
168
- assert_false @shutdown_called
169
- subject.shutdown
170
- assert_true @wakeup_called
171
- assert_true @shutdown_called
92
+ assert_true @runner_spy.start_called
172
93
  end
173
94
 
174
- end
95
+ should "shutdown its runner when its shutdown" do
96
+ assert_false @runner_spy.shutdown_called
97
+ subject.shutdown
98
+ assert_true @runner_spy.shutdown_called
99
+ assert_nil @runner_spy.shutdown_timeout
100
+ exp = "test/unit/dat-worker-pool_tests.rb"
101
+ assert_match exp, @runner_spy.shutdown_backtrace.first
175
102
 
176
- class AddWorkWithNoWorkersTests < UnitTests
177
- setup do
178
- @work_pool = DatWorkerPool.new(0, 0){ |work| }
103
+ timeout = Factory.integer
104
+ subject.shutdown(timeout)
105
+ assert_equal timeout, @runner_spy.shutdown_timeout
179
106
  end
180
107
 
181
- should "return whether or not the queue is empty" do
182
- assert_equal true, @work_pool.queue_empty?
183
- @work_pool.add_work 'test'
184
- assert_equal false, @work_pool.queue_empty?
108
+ should "demeter its runner" do
109
+ assert_equal @runner_spy.available_worker_count, subject.available_worker_count
110
+ assert_equal @runner_spy.worker_available?, subject.worker_available?
185
111
  end
186
112
 
187
- end
188
-
189
- class AddWorkAndProcessItTests < UnitTests
190
- desc "add_work and process"
191
- setup do
192
- @result = nil
193
- @work_pool = DatWorkerPool.new(1){|work| @result = (2 / work) }
194
- @work_pool.start
113
+ should "raise an argument error if given an invalid worker class" do
114
+ assert_raises(ArgumentError){ @worker_pool_class.new(Module.new) }
115
+ assert_raises(ArgumentError){ @worker_pool_class.new(Class.new) }
195
116
  end
196
117
 
197
- should "have added the work and processed it by calling the passed block" do
198
- subject.add_work 2
199
- sleep 0.1 # ensure worker thread get's a chance to run
200
- assert_equal 1, @result
118
+ should "raise an argument error if given an invalid number of workers" do
119
+ assert_raises(ArgumentError) do
120
+ @worker_pool_class.new(@worker_class, {
121
+ :num_workers => [0, (Factory.integer * -1)].choice
122
+ })
123
+ end
201
124
  end
202
125
 
203
- should "swallow exceptions, so workers don't end unexpectedly" do
204
- subject.add_work 0
205
- worker = subject.instance_variable_get("@workers").first
206
- sleep 0.1
126
+ end
207
127
 
208
- assert_equal 1, subject.spawned
209
- assert_equal 1, subject.waiting
210
- assert worker.instance_variable_get("@thread").alive?
128
+ class StartedTests < InitTests
129
+ desc "and started"
130
+ setup do
131
+ @worker_pool.start
211
132
  end
212
133
 
213
- end
134
+ should "be able to add work onto its queue`" do
135
+ work_item = Factory.string
136
+ subject.add_work(work_item)
137
+ assert_equal work_item, @queue.work_items.last
214
138
 
215
- class StartTests < UnitTests
216
- desc "start"
217
- setup do
218
- @work_pool = DatWorkerPool.new(1, 2){ |work| sleep(work) }
139
+ work_item = Factory.string
140
+ subject.push(work_item)
141
+ assert_equal work_item, @queue.work_items.last
219
142
  end
220
- subject{ @work_pool }
221
143
 
222
- should "start its queue" do
223
- assert_false subject.queue.shutdown?
224
- subject.start
225
- assert_false subject.queue.shutdown?
226
- subject.shutdown
227
- assert_true subject.queue.shutdown?
228
- subject.start
229
- assert_false subject.queue.shutdown?
144
+ should "not add `nil` work onto its queue" do
145
+ subject.add_work(nil)
146
+ assert_equal [], @queue.work_items
230
147
  end
231
148
 
232
- should "keep workers from being spawned until its called" do
233
- assert_equal 0, subject.spawned
234
- subject.add_work 1
235
- assert_equal 0, subject.spawned
236
- subject.start
237
- assert_equal 1, subject.spawned
149
+ should "know its queue's work items" do
150
+ Factory.integer(3).times{ @queue.dwp_push(Factory.string) }
151
+ assert_equal @queue.work_items, subject.work_items
238
152
  end
239
153
 
240
154
  end
241
155
 
242
- class ShutdownTests < UnitTests
243
- desc "shutdown"
244
- setup do
245
- @mutex = Mutex.new
246
- @finished = []
247
- @work_pool = DatWorkerPool.new(1, 2, true) do |work|
248
- sleep 1
249
- @mutex.synchronize{ @finished << work }
250
- end
251
- @work_pool.start
252
- @work_pool.add_work 'a'
253
- @work_pool.add_work 'b'
254
- @work_pool.add_work 'c'
255
- end
156
+ class TestQueue
157
+ include DatWorkerPool::Queue
256
158
 
257
- should "allow any work that has been picked up to be processed" do
258
- # make sure the workers haven't processed any work
259
- assert_equal [], @finished
260
- subject.shutdown(5)
159
+ attr_reader :work_items
261
160
 
262
- # NOTE, the last work shouldn't have been processed, as it wasn't
263
- # picked up by a worker
264
- assert_includes 'a', @finished
265
- assert_includes 'b', @finished
266
- assert_not_includes 'c', @finished
267
-
268
- assert_equal 0, subject.spawned
269
- assert_equal 0, subject.waiting
270
- assert_includes 'c', subject.work_items
161
+ def initialize
162
+ @work_items = []
271
163
  end
272
164
 
273
- should "allow jobs to finish by not providing a shutdown timeout" do
274
- assert_equal [], @finished
275
- subject.shutdown
276
- assert_includes 'a', @finished
277
- assert_includes 'b', @finished
278
- end
165
+ private
279
166
 
280
- should "reraise shutdown errors in debug mode if workers take to long to finish" do
281
- assert_raises(DatWorkerPool::ShutdownError) do
282
- subject.shutdown(0.1)
283
- end
167
+ def push!(work_item)
168
+ @work_items << work_item
284
169
  end
285
-
286
170
  end
287
171
 
288
- class ForcedShutdownTests < UnitTests
289
- desc "forced shutdown"
290
- setup do
291
- @mutex = Mutex.new
292
- @finished = []
293
- @max_workers = 2
294
- # don't put leave the worker pool in debug mode
295
- @work_pool = DatWorkerPool.new(1, @max_workers, false) do |work|
296
- begin
297
- sleep 1
298
- rescue ShutdownError => error
299
- @mutex.synchronize{ @finished << error }
300
- raise error # re-raise it otherwise worker won't shutdown
301
- end
302
- end
303
- @work_pool.start
304
- @work_pool.add_work 'a'
305
- @work_pool.add_work 'b'
306
- @work_pool.add_work 'c'
172
+ class RunnerSpy < DatWorkerPool::Runner
173
+ attr_accessor :args
174
+ attr_reader :start_called, :shutdown_called
175
+ attr_reader :shutdown_timeout, :shutdown_backtrace
176
+
177
+ def initialize
178
+ super({})
179
+ @start_called = false
180
+ @shutdown_called = false
181
+ @shutdown_timeout = nil
182
+ @shutdown_backtrace = nil
307
183
  end
308
184
 
309
- should "force workers to shutdown if they take to long to finish" do
310
- # make sure the workers haven't processed any work
311
- assert_equal [], @finished
312
- subject.shutdown(0.1)
313
- assert_equal @max_workers, @finished.size
314
- @finished.each do |error|
315
- assert_instance_of DatWorkerPool::ShutdownError, error
316
- end
185
+ def start
186
+ @args[:queue].dwp_start
187
+ @start_called = true
317
188
  end
318
189
 
190
+ def shutdown(timeout, backtrace)
191
+ @args[:queue].dwp_shutdown
192
+ @shutdown_called = true
193
+ @shutdown_timeout = timeout
194
+ @shutdown_backtrace = backtrace
195
+ end
319
196
  end
320
197
 
321
198
  end