dat-worker-pool 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,108 +1,131 @@
1
1
  require 'assert'
2
2
  require 'dat-worker-pool/queue'
3
3
 
4
- class DatWorkerPool::Queue
4
+ module DatWorkerPool::Queue
5
5
 
6
6
  class UnitTests < Assert::Context
7
7
  desc "DatWorkerPool::Queue"
8
8
  setup do
9
- @queue = DatWorkerPool::Queue.new
10
- end
11
- subject{ @queue }
9
+ @queue_class = Class.new do
10
+ include DatWorkerPool::Queue
12
11
 
13
- should have_accessors :on_push_callbacks, :on_pop_callbacks
14
- should have_imeths :work_items, :push, :pop, :empty?
15
- should have_imeths :start, :shutdown, :shutdown?
12
+ attr_reader :start_called, :shutdown_called
13
+ attr_reader :push_called_with
14
+ attr_accessor :pop_result
16
15
 
17
- should "default its callbacks" do
18
- assert_equal [], subject.on_push_callbacks
19
- assert_equal [], subject.on_pop_callbacks
20
- end
16
+ def initialize
17
+ @start_called = false
18
+ @shutdown_called = false
19
+ @push_called_with = nil
20
+ @pop_result = nil
21
+ end
21
22
 
22
- should "allow pushing work items onto the queue with #push" do
23
- subject.push 'work'
24
- assert_equal [ 'work' ], subject.work_items
25
- end
23
+ def start!; @start_called = true; end
24
+ def shutdown!; @shutdown_called = true; end
26
25
 
27
- should "call its on push callback when work is pushed" do
28
- on_push_called = false
29
- subject.on_push_callbacks << proc{ on_push_called = true }
30
- subject.push 'work'
31
- assert_true on_push_called
26
+ def push!(*args); @push_called_with = args; end
27
+ def pop!; @pop_result; end
28
+ end
32
29
  end
30
+ subject{ @queue_class }
33
31
 
34
- should "raise an exception if trying to push work when shutdown" do
35
- subject.shutdown
36
- assert_raises(RuntimeError){ subject.push('work') }
32
+ should "raise a not implemented error for `dwp_push` and `dwp_pop` by default" do
33
+ queue_class = Class.new{ include DatWorkerPool::Queue }
34
+ queue = queue_class.new.tap(&:dwp_start)
35
+
36
+ assert_raises(NotImplementedError){ queue.dwp_push(Factory.string) }
37
+ assert_raises(NotImplementedError){ queue.dwp_pop }
37
38
  end
38
39
 
39
- should "pop work items off the queue with #pop" do
40
- subject.push 'work1'
41
- subject.push 'work2'
40
+ end
42
41
 
43
- assert_equal 2, subject.work_items.size
44
- assert_equal 'work1', subject.pop
45
- assert_equal 1, subject.work_items.size
46
- assert_equal 'work2', subject.pop
47
- assert_equal 0, subject.work_items.size
42
+ class InitTests < UnitTests
43
+ desc "when init"
44
+ setup do
45
+ @queue = @queue_class.new
48
46
  end
47
+ subject{ @queue }
48
+
49
+ should have_imeths :work_items
50
+ should have_imeths :dwp_start, :dwp_signal_shutdown, :dwp_shutdown
51
+ should have_imeths :running?, :shutdown?
52
+ should have_imeths :dwp_push, :dwp_pop
49
53
 
50
- should "call its on pop callback when work is popped" do
51
- subject.push 'work'
52
- on_pop_called = false
53
- subject.on_pop_callbacks << proc{ on_pop_called = true }
54
- subject.pop
55
- assert_true on_pop_called
54
+ should "raise a not implemented error using `work_items`" do
55
+ assert_raises(NotImplementedError){ subject.work_items }
56
56
  end
57
57
 
58
- should "return nothing with pop when the queue has been shutdown" do
59
- subject.push 'work1'
60
- subject.shutdown
61
- assert_nil subject.pop
58
+ should "set its flags using `dwp_start` and `dwp_shutdown`" do
59
+ assert_false subject.running?
60
+ assert_true subject.shutdown?
61
+ subject.dwp_start
62
+ assert_true subject.running?
63
+ assert_false subject.shutdown?
64
+ subject.dwp_shutdown
65
+ assert_false subject.running?
66
+ assert_true subject.shutdown?
62
67
  end
63
68
 
64
- should "return whether the queue is empty or not with #empty?" do
65
- assert subject.empty?
66
- subject.push 'work'
67
- assert_not subject.empty?
68
- subject.pop
69
- assert subject.empty?
69
+ should "call `start!` using `dwp_start`" do
70
+ assert_false subject.start_called
71
+ subject.dwp_start
72
+ assert_true subject.start_called
70
73
  end
71
74
 
72
- should "reset its shutdown flag when started" do
73
- assert_false subject.shutdown?
74
- subject.shutdown
75
- assert_true subject.shutdown?
76
- subject.start
77
- assert_false subject.shutdown?
75
+ should "set its shutdown flag using `dwp_signal_shutdown`" do
76
+ assert_false subject.running?
77
+ assert_false subject.shutdown_called
78
+ subject.dwp_start
79
+ assert_true subject.running?
80
+ assert_false subject.shutdown_called
81
+ subject.dwp_signal_shutdown
82
+ assert_false subject.running?
83
+ assert_false subject.shutdown_called
78
84
  end
79
85
 
80
- end
86
+ should "call `shutdown!` using `dwp_shutdown`" do
87
+ assert_false subject.shutdown_called
88
+ subject.dwp_shutdown
89
+ assert_true subject.shutdown_called
90
+ end
81
91
 
82
- class SignallingTests < UnitTests
83
- desc "mutex and condition variable behavior"
84
- setup do
85
- @thread = Thread.new do
86
- Thread.current['work_item'] = @queue.pop || 'got nothing'
87
- end
92
+ should "raise an error if `dwp_push` is called when the queue isn't running" do
93
+ assert_false subject.running?
94
+ assert_raise(RuntimeError){ subject.dwp_push(Factory.string) }
95
+ subject.dwp_start
96
+ assert_nothing_raised{ subject.dwp_push(Factory.string) }
97
+ subject.dwp_shutdown
98
+ assert_raise(RuntimeError){ subject.dwp_push(Factory.string) }
88
99
  end
89
100
 
90
- should "have threads wait for a work item to be added when using pop" do
91
- assert_equal "sleep", @thread.status
101
+ should "call `push!` using `dwp_push`" do
102
+ subject.dwp_start
103
+
104
+ work_item = Factory.string
105
+ subject.dwp_push(work_item)
106
+ assert_equal [work_item], subject.push_called_with
107
+
108
+ args = Factory.integer(3).times.map{ Factory.string }
109
+ subject.dwp_push(*args)
110
+ assert_equal args, subject.push_called_with
92
111
  end
93
112
 
94
- should "wakeup threads when work is pushed onto the queue" do
95
- subject.push 'some work'
96
- sleep 0.1
97
- assert !@thread.alive?
98
- assert_equal 'some work', @thread['work_item']
113
+ should "return nothing if `dwp_pop` is called when the queue isn't running" do
114
+ subject.pop_result = Factory.string
115
+ assert_false subject.running?
116
+ assert_nil subject.dwp_pop
117
+ subject.dwp_start
118
+ assert_not_nil subject.dwp_pop
119
+ subject.dwp_shutdown
120
+ assert_nil subject.dwp_pop
99
121
  end
100
122
 
101
- should "wakeup thread when the queue is shutdown" do
102
- subject.shutdown
103
- sleep 0.1
104
- assert !@thread.alive?
105
- assert_equal 'got nothing', @thread['work_item']
123
+ should "call `pop!` using `dwp_pop`" do
124
+ subject.dwp_start
125
+ subject.pop_result = Factory.string
126
+
127
+ value = subject.dwp_pop
128
+ assert_equal subject.pop_result, value
106
129
  end
107
130
 
108
131
  end
@@ -0,0 +1,365 @@
1
+ require 'assert'
2
+ require 'dat-worker-pool/runner'
3
+
4
+ require 'dat-worker-pool/default_queue'
5
+
6
+ class DatWorkerPool::Runner
7
+
8
+ class UnitTests < Assert::Context
9
+ desc "DatWorkerPool::Runner"
10
+ setup do
11
+ @runner_class = DatWorkerPool::Runner
12
+ end
13
+ subject{ @runner_class }
14
+
15
+ end
16
+
17
+ class InitTests < UnitTests
18
+ desc "when init"
19
+ setup do
20
+ # at least 2 workers, up to 4
21
+ @num_workers = Factory.integer(3) + 1
22
+ @logger = TEST_LOGGER || Logger.new("/dev/null")
23
+ @queue = DatWorkerPool::DefaultQueue.new
24
+ @worker_class = TestWorker
25
+ @worker_params = { Factory.string => Factory.string }
26
+
27
+ @workers = DatWorkerPool::LockedArray.new
28
+ Assert.stub(DatWorkerPool::LockedArray, :new){ @workers }
29
+
30
+ @available_workers_spy = DatWorkerPool::LockedSet.new
31
+ Assert.stub(DatWorkerPool::LockedSet, :new){ @available_workers_spy }
32
+
33
+ @options = {
34
+ :num_workers => @num_workers,
35
+ :logger => @logger,
36
+ :queue => @queue,
37
+ :worker_class => @worker_class,
38
+ :worker_params => @worker_params
39
+ }
40
+ @runner = @runner_class.new(@options)
41
+ end
42
+ teardown do
43
+ @runner.shutdown(0) rescue false
44
+ end
45
+ subject{ @runner }
46
+
47
+ should have_readers :num_workers, :worker_class, :worker_params
48
+ should have_readers :logger_proxy, :queue
49
+ should have_imeths :workers, :start, :shutdown
50
+ should have_imeths :available_worker_count, :worker_available?
51
+ should have_imeths :make_worker_available, :make_worker_unavailable
52
+ should have_imeths :worker_log
53
+
54
+ should "know its attributes" do
55
+ assert_equal @num_workers, subject.num_workers
56
+ assert_equal @worker_class, subject.worker_class
57
+ assert_equal @worker_params, subject.worker_params
58
+ assert_equal @queue, subject.queue
59
+
60
+ assert_instance_of LoggerProxy, subject.logger_proxy
61
+ assert_equal @logger, subject.logger_proxy.logger
62
+ end
63
+
64
+ should "default its logger" do
65
+ @options.delete(:logger)
66
+ runner = @runner_class.new(@options)
67
+ assert_instance_of NullLoggerProxy, runner.logger_proxy
68
+ end
69
+
70
+ should "know its workers" do
71
+ assert_equal @workers.values, subject.workers
72
+ @workers.push(Factory.string)
73
+ assert_equal @workers.values, subject.workers
74
+ end
75
+
76
+ should "start its queue when its started" do
77
+ assert_false @queue.running?
78
+ subject.start
79
+ assert_true @queue.running?
80
+ end
81
+
82
+ should "build and add workers when its started" do
83
+ subject.start
84
+
85
+ assert_equal @num_workers, subject.workers.size
86
+ subject.workers.each_with_index do |worker, n|
87
+ assert_equal subject, worker.dwp_runner
88
+ assert_equal @queue, worker.dwp_queue
89
+ assert_equal n + 1, worker.dwp_number
90
+ assert_true worker.dwp_running?
91
+ end
92
+ end
93
+
94
+ should "allow making workers available/unavailable" do
95
+ worker = @worker_class.new(@runner, @queue, Factory.integer(10))
96
+
97
+ assert_not_includes worker.object_id, @available_workers_spy.values
98
+ assert_false subject.worker_available?
99
+ subject.make_worker_available(worker)
100
+ assert_includes worker.object_id, @available_workers_spy.values
101
+ assert_true subject.worker_available?
102
+ subject.make_worker_unavailable(worker)
103
+ assert_not_includes worker.object_id, @available_workers_spy.values
104
+ assert_false subject.worker_available?
105
+ end
106
+
107
+ should "know how many workers are available" do
108
+ worker = @worker_class.new(@runner, @queue, Factory.integer(10))
109
+
110
+ assert_equal 0, subject.available_worker_count
111
+ subject.make_worker_available(worker)
112
+ assert_equal 1, subject.available_worker_count
113
+ subject.make_worker_unavailable(worker)
114
+ assert_equal 0, subject.available_worker_count
115
+ end
116
+
117
+ should "allow logging messages using `log`" do
118
+ logged_message = nil
119
+ Assert.stub(subject.logger_proxy, :runner_log) do |&mb|
120
+ logged_message = mb.call
121
+ end
122
+
123
+ text = Factory.text
124
+ subject.log{ text }
125
+ assert_equal text, logged_message
126
+ end
127
+
128
+ should "allow workers to log messages using `worker_log`" do
129
+ passed_worker = nil
130
+ logged_message = nil
131
+ Assert.stub(subject.logger_proxy, :worker_log) do |w, &mb|
132
+ passed_worker = w
133
+ logged_message = mb.call
134
+ end
135
+ worker = @worker_class.new(@runner, @queue, Factory.integer(10))
136
+
137
+ text = Factory.text
138
+ subject.worker_log(worker){ text }
139
+ assert_same worker, passed_worker
140
+ assert_equal text, logged_message
141
+ end
142
+
143
+ end
144
+
145
+ class ShutdownSetupTests < InitTests
146
+ desc "and started and shutdown"
147
+
148
+ end
149
+
150
+ class ShutdownTests < ShutdownSetupTests
151
+ setup do
152
+ @timeout_seconds = nil
153
+ @optional_timeout_called = false
154
+ # this acts as a spy but also keeps the shutdown from ever timing out
155
+ Assert.stub(OptionalTimeout, :new) do |secs, &block|
156
+ @timeout_seconds = secs
157
+ @optional_timeout_called = true
158
+ block.call
159
+ end
160
+
161
+ @options[:worker_class] = ShutdownSpyWorker
162
+ @runner = @runner_class.new(@options)
163
+ @runner.start
164
+ # we need a reference to the workers, the runners workers will get removed
165
+ # as they shutdown
166
+ @running_workers = @runner.workers.dup
167
+ end
168
+
169
+ should "optionally timeout when shutdown" do
170
+ subject.shutdown
171
+ assert_nil @timeout_seconds
172
+ assert_true @optional_timeout_called
173
+
174
+ @optional_timeout_called = false
175
+ seconds = Factory.integer
176
+ subject.shutdown(seconds)
177
+ assert_equal seconds, @timeout_seconds
178
+ assert_true @optional_timeout_called
179
+ end
180
+
181
+ should "shutdown all of its workers" do
182
+ @running_workers.each do |worker|
183
+ assert_false worker.dwp_shutdown?
184
+ end
185
+ subject.shutdown(Factory.boolean ? Factory.integer : nil)
186
+ @running_workers.each do |worker|
187
+ assert_true worker.dwp_shutdown?
188
+ end
189
+ end
190
+
191
+ should "shutdown its queue" do
192
+ assert_false @queue.shutdown?
193
+ subject.shutdown(Factory.boolean ? Factory.integer : nil)
194
+ assert_true @queue.shutdown?
195
+ end
196
+
197
+ should "join its workers waiting for them to finish" do
198
+ @running_workers.each do |worker|
199
+ assert_false worker.join_called
200
+ end
201
+ subject.shutdown(Factory.boolean ? Factory.integer : nil)
202
+ @running_workers.each do |worker|
203
+ assert_true worker.join_called
204
+ end
205
+ end
206
+
207
+ should "join all workers even if one raises an error when joined" do
208
+ @running_workers.choice.join_error = Factory.exception
209
+ subject.shutdown(Factory.boolean ? Factory.integer : nil)
210
+ @running_workers.each do |worker|
211
+ assert_true worker.join_called
212
+ end
213
+ end
214
+
215
+ should "remove workers as they finish" do
216
+ assert_false subject.workers.empty?
217
+ subject.shutdown(Factory.boolean ? Factory.integer : nil)
218
+ assert_true subject.workers.empty?
219
+ end
220
+
221
+ should "remove workers and make them unavailable even if they error" do
222
+ @running_workers.each{ |w| w.join_error = Factory.exception }
223
+
224
+ assert_false subject.workers.empty?
225
+ assert_false @available_workers_spy.empty?
226
+ subject.shutdown(Factory.boolean ? Factory.integer : nil)
227
+ assert_true subject.workers.empty?
228
+ assert_true @available_workers_spy.empty?
229
+ end
230
+
231
+ should "force its workers to shutdown if a timeout error occurs" do
232
+ Assert.stub(OptionalTimeout, :new){ raise TimeoutInterruptError }
233
+ subject.shutdown(Factory.integer)
234
+
235
+ @running_workers.each do |worker|
236
+ assert_instance_of DatWorkerPool::ShutdownError, worker.raised_error
237
+ assert_true worker.join_called
238
+ end
239
+ assert_true subject.workers.empty?
240
+ assert_true @available_workers_spy.empty?
241
+ end
242
+
243
+ should "force its workers to shutdown if a non-timeout error occurs" do
244
+ queue_exception = Factory.exception
245
+ Assert.stub(@queue, :dwp_shutdown){ raise queue_exception }
246
+
247
+ caught_exception = nil
248
+ begin
249
+ subject.shutdown(Factory.integer)
250
+ rescue StandardError => caught_exception
251
+ end
252
+ assert_same queue_exception, caught_exception
253
+
254
+ @running_workers.each do |worker|
255
+ assert_instance_of DatWorkerPool::ShutdownError, worker.raised_error
256
+ assert_true worker.join_called
257
+ end
258
+ assert_true subject.workers.empty?
259
+ assert_true @available_workers_spy.empty?
260
+ end
261
+
262
+ should "force shutdown all of its workers even if one raises an error when joining" do
263
+ Assert.stub(OptionalTimeout, :new){ raise TimeoutInterruptError }
264
+ error_class = Factory.boolean ? DatWorkerPool::ShutdownError : RuntimeError
265
+ @running_workers.choice.join_error = Factory.exception(error_class)
266
+ subject.shutdown(Factory.boolean ? Factory.integer : nil)
267
+
268
+ @running_workers.each do |worker|
269
+ assert_instance_of DatWorkerPool::ShutdownError, worker.raised_error
270
+ assert_true worker.join_called
271
+ end
272
+ assert_true subject.workers.empty?
273
+ assert_true @available_workers_spy.empty?
274
+ end
275
+
276
+ end
277
+
278
+ class LoggerProxyTests < UnitTests
279
+ desc "LoggerProxy"
280
+ setup do
281
+ @stringio = StringIO.new
282
+ @logger = Logger.new(@stringio)
283
+ @logger_proxy = LoggerProxy.new(@logger)
284
+ end
285
+ subject{ @logger_proxy }
286
+
287
+ should have_readers :logger
288
+ should have_imeths :runner_log, :worker_log
289
+
290
+ should "know its logger" do
291
+ assert_equal @logger, subject.logger
292
+ end
293
+
294
+ should "log a message block for a runner using `runner_log`" do
295
+ text = Factory.text
296
+ subject.runner_log{ text }
297
+ assert_match "[DWP] #{text}", @stringio.string
298
+ end
299
+
300
+ should "log a message block for a worker using `worker_log`" do
301
+ worker = FakeWorker.new(Factory.integer(10))
302
+ text = Factory.text
303
+ subject.worker_log(worker){ text }
304
+ assert_match "[DWP-#{worker.dwp_number}] #{text}", @stringio.string
305
+ end
306
+
307
+ end
308
+
309
+ class NullLoggerProxyTests < UnitTests
310
+ desc "NullLoggerProxy"
311
+ setup do
312
+ @null_logger_proxy = NullLoggerProxy.new
313
+ end
314
+ subject{ @null_logger_proxy }
315
+
316
+ should have_imeths :runner_log, :worker_log
317
+
318
+ end
319
+
320
+ class TestWorker
321
+ include DatWorkerPool::Worker
322
+
323
+ # for testing what is passed to the worker
324
+ attr_reader :dwp_runner, :dwp_queue
325
+ end
326
+
327
+ FakeWorker = Struct.new(:dwp_number)
328
+
329
+ class ShutdownSpyWorker < TestWorker
330
+ attr_reader :join_called
331
+ attr_accessor :join_error, :raised_error
332
+
333
+ # this pauses the shutdown so we can test that join or raise are called
334
+ # depending if we are doing a standard or forced shutdown; otherwise the
335
+ # worker threads can exit before join or raise ever gets called on them and
336
+ # then there is nothing to test
337
+ on_shutdown{ wait_for_join_or_raise }
338
+
339
+ def initialize(*args)
340
+ super
341
+ @mutex = Mutex.new
342
+ @cond_var = ConditionVariable.new
343
+ @join_called = false
344
+ @join_error = nil
345
+ @raised_error = nil
346
+ end
347
+
348
+ def dwp_join(*args)
349
+ @join_called = true
350
+ raise @join_error if @join_error
351
+ @mutex.synchronize{ @cond_var.broadcast }
352
+ end
353
+
354
+ def dwp_raise(error)
355
+ @raised_error = error
356
+ end
357
+
358
+ private
359
+
360
+ def wait_for_join_or_raise
361
+ @mutex.synchronize{ @cond_var.wait(@mutex) }
362
+ end
363
+ end
364
+
365
+ end