dat-worker-pool 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -1
- data/bench/report.rb +130 -0
- data/bench/report.txt +7 -0
- data/dat-worker-pool.gemspec +1 -1
- data/lib/dat-worker-pool.rb +38 -187
- data/lib/dat-worker-pool/default_queue.rb +60 -0
- data/lib/dat-worker-pool/locked_object.rb +60 -0
- data/lib/dat-worker-pool/queue.rb +71 -48
- data/lib/dat-worker-pool/runner.rb +196 -0
- data/lib/dat-worker-pool/version.rb +1 -1
- data/lib/dat-worker-pool/worker.rb +251 -72
- data/lib/dat-worker-pool/worker_pool_spy.rb +39 -53
- data/test/helper.rb +13 -0
- data/test/support/factory.rb +15 -0
- data/test/support/thread_spies.rb +83 -0
- data/test/system/dat-worker-pool_tests.rb +399 -0
- data/test/unit/dat-worker-pool_tests.rb +132 -255
- data/test/unit/default_queue_tests.rb +217 -0
- data/test/unit/locked_object_tests.rb +260 -0
- data/test/unit/queue_tests.rb +95 -72
- data/test/unit/runner_tests.rb +365 -0
- data/test/unit/worker_pool_spy_tests.rb +95 -102
- data/test/unit/worker_tests.rb +819 -153
- metadata +27 -12
- data/test/system/use_worker_pool_tests.rb +0 -34
@@ -0,0 +1,217 @@
|
|
1
|
+
require 'assert'
|
2
|
+
require 'dat-worker-pool/default_queue'
|
3
|
+
|
4
|
+
require 'thread'
|
5
|
+
require 'dat-worker-pool/queue'
|
6
|
+
require 'test/support/thread_spies'
|
7
|
+
|
8
|
+
class DatWorkerPool::DefaultQueue
|
9
|
+
|
10
|
+
class UnitTests < Assert::Context
|
11
|
+
desc "DatWorkerPool::DefaultQueue"
|
12
|
+
setup do
|
13
|
+
@queue_class = DatWorkerPool::DefaultQueue
|
14
|
+
end
|
15
|
+
subject{ @queue_class }
|
16
|
+
|
17
|
+
should "be a dat-worker-pool queue" do
|
18
|
+
assert_includes DatWorkerPool::Queue, subject
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
class InitTests < UnitTests
|
24
|
+
desc "when init"
|
25
|
+
setup do
|
26
|
+
@work_items = DatWorkerPool::LockedArray.new
|
27
|
+
Assert.stub(DatWorkerPool::LockedArray, :new){ @work_items }
|
28
|
+
|
29
|
+
@cond_var_spy = ConditionVariableSpy.new
|
30
|
+
Assert.stub(ConditionVariable, :new){ @cond_var_spy }
|
31
|
+
|
32
|
+
@queue = @queue_class.new
|
33
|
+
end
|
34
|
+
subject{ @queue }
|
35
|
+
|
36
|
+
should have_readers :on_push_callbacks, :on_pop_callbacks
|
37
|
+
should have_imeths :work_items, :empty?
|
38
|
+
should have_imeths :on_push, :on_pop
|
39
|
+
|
40
|
+
should "default its callbacks" do
|
41
|
+
assert_equal [], subject.on_push_callbacks
|
42
|
+
assert_equal [], subject.on_pop_callbacks
|
43
|
+
end
|
44
|
+
|
45
|
+
should "allow adding custom push and pop callbacks" do
|
46
|
+
callback = proc{ Factory.string }
|
47
|
+
subject.on_push(&callback)
|
48
|
+
assert_includes callback, subject.on_push_callbacks
|
49
|
+
|
50
|
+
callback = proc{ Factory.string }
|
51
|
+
subject.on_pop(&callback)
|
52
|
+
assert_includes callback, subject.on_pop_callbacks
|
53
|
+
end
|
54
|
+
|
55
|
+
should "know its work items and demeter them" do
|
56
|
+
assert_equal @work_items.values, subject.work_items
|
57
|
+
assert_true subject.empty?
|
58
|
+
|
59
|
+
@work_items.push(Factory.string)
|
60
|
+
assert_equal @work_items.values, subject.work_items
|
61
|
+
assert_false subject.empty?
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
class StartedTests < InitTests
|
67
|
+
desc "and started"
|
68
|
+
setup do
|
69
|
+
@queue.dwp_start
|
70
|
+
end
|
71
|
+
|
72
|
+
should "broadcast to all threads when shutdown" do
|
73
|
+
assert_false @cond_var_spy.broadcast_called
|
74
|
+
subject.dwp_shutdown
|
75
|
+
assert_true @cond_var_spy.broadcast_called
|
76
|
+
end
|
77
|
+
|
78
|
+
should "be able to add work items" do
|
79
|
+
work_item = Factory.string
|
80
|
+
subject.dwp_push(work_item)
|
81
|
+
assert_equal 1, subject.work_items.size
|
82
|
+
assert_equal work_item, subject.work_items.last
|
83
|
+
|
84
|
+
subject.dwp_push(work_item)
|
85
|
+
assert_equal 2, subject.work_items.size
|
86
|
+
assert_equal work_item, subject.work_items.last
|
87
|
+
end
|
88
|
+
|
89
|
+
should "signal threads waiting on its lock when adding work items" do
|
90
|
+
assert_false @cond_var_spy.signal_called
|
91
|
+
subject.dwp_push(Factory.string)
|
92
|
+
assert_true @cond_var_spy.signal_called
|
93
|
+
end
|
94
|
+
|
95
|
+
should "run on push callbacks when adding work items" do
|
96
|
+
on_push_queue = nil
|
97
|
+
on_push_work_item = nil
|
98
|
+
subject.on_push do |queue, work_item|
|
99
|
+
on_push_queue = queue
|
100
|
+
on_push_work_item = work_item
|
101
|
+
end
|
102
|
+
|
103
|
+
work_item = Factory.string
|
104
|
+
subject.dwp_push(work_item)
|
105
|
+
assert_equal subject, on_push_queue
|
106
|
+
assert_equal work_item, on_push_work_item
|
107
|
+
end
|
108
|
+
|
109
|
+
should "be able to pop work items" do
|
110
|
+
values = (Factory.integer(3) + 1).times.map{ Factory.string }
|
111
|
+
values.each{ |v| subject.dwp_push(v) }
|
112
|
+
|
113
|
+
assert_equal values.first, subject.dwp_pop
|
114
|
+
exp = values - [values.first]
|
115
|
+
assert_equal exp, subject.work_items
|
116
|
+
end
|
117
|
+
|
118
|
+
should "run on pop callbacks when popping work items" do
|
119
|
+
subject.dwp_push(Factory.string)
|
120
|
+
|
121
|
+
on_pop_queue = nil
|
122
|
+
on_pop_work_item = nil
|
123
|
+
subject.on_pop do |queue, work_item|
|
124
|
+
on_pop_queue = queue
|
125
|
+
on_pop_work_item = work_item
|
126
|
+
end
|
127
|
+
|
128
|
+
popped_work_item = subject.dwp_pop
|
129
|
+
assert_equal subject, on_pop_queue
|
130
|
+
assert_equal popped_work_item, on_pop_work_item
|
131
|
+
end
|
132
|
+
|
133
|
+
should "not run on pop callbacks when there isn't a work item" do
|
134
|
+
subject.dwp_push(nil)
|
135
|
+
|
136
|
+
on_pop_called = false
|
137
|
+
subject.on_pop{ on_pop_called = true }
|
138
|
+
subject.dwp_pop
|
139
|
+
assert_false on_pop_called
|
140
|
+
end
|
141
|
+
|
142
|
+
end
|
143
|
+
|
144
|
+
class ThreadTests < StartedTests
|
145
|
+
desc "with a thread using it"
|
146
|
+
setup do
|
147
|
+
@on_pop_called = false
|
148
|
+
@queue.on_pop{ @on_pop_called = true }
|
149
|
+
|
150
|
+
@thread = Thread.new{ Thread.current['popped_value'] = @queue.dwp_pop }
|
151
|
+
@thread.join(JOIN_SECONDS)
|
152
|
+
end
|
153
|
+
subject{ @thread }
|
154
|
+
|
155
|
+
should "sleep the thread if empty when popping work items" do
|
156
|
+
assert_equal 'sleep', subject.status
|
157
|
+
assert_equal @work_items.mutex, @cond_var_spy.wait_called_on
|
158
|
+
end
|
159
|
+
|
160
|
+
should "wakeup the thread (from waiting on pop) when work items are added" do
|
161
|
+
assert_equal 'sleep', subject.status
|
162
|
+
|
163
|
+
value = Factory.string
|
164
|
+
@queue.dwp_push(value)
|
165
|
+
|
166
|
+
assert_not subject.alive?
|
167
|
+
assert_equal value, subject['popped_value']
|
168
|
+
end
|
169
|
+
|
170
|
+
should "re-sleep the thread if woken up while the queue is empty" do
|
171
|
+
assert_equal 'sleep', subject.status
|
172
|
+
assert_equal 1, @cond_var_spy.wait_call_count
|
173
|
+
|
174
|
+
# wakeup the thread (like `push` does) but we don't want to add anything
|
175
|
+
# to the queue, its possible this can happen if another worker never
|
176
|
+
# sleeps and grabs the lock and work item before the thread being woken
|
177
|
+
# up
|
178
|
+
@work_items.with_lock{ @cond_var_spy.signal }
|
179
|
+
|
180
|
+
assert_equal 'sleep', subject.status
|
181
|
+
assert_equal 2, @cond_var_spy.wait_call_count
|
182
|
+
end
|
183
|
+
|
184
|
+
should "wakeup the thread (from waiting on pop) when its shutdown" do
|
185
|
+
assert_equal 'sleep', subject.status
|
186
|
+
assert_equal 1, @cond_var_spy.wait_call_count
|
187
|
+
|
188
|
+
@queue.dwp_shutdown
|
189
|
+
|
190
|
+
assert_not subject.alive?
|
191
|
+
assert_equal 1, @cond_var_spy.wait_call_count
|
192
|
+
end
|
193
|
+
|
194
|
+
should "not run on pop callbacks when shutdown" do
|
195
|
+
@queue.dwp_shutdown
|
196
|
+
assert_false @on_pop_called
|
197
|
+
end
|
198
|
+
|
199
|
+
should "not pop a work item when shutdown and not empty" do
|
200
|
+
assert_equal 'sleep', subject.status
|
201
|
+
|
202
|
+
# this is to simulate a specific situation where there is work on the
|
203
|
+
# queue and the queue gets shutdown while a worker is still sleeping (it
|
204
|
+
# hasn't gotten a chance to wakeup and pull work off the queue yet), if
|
205
|
+
# this happens we don't want it to pull work off the queue; to set up this
|
206
|
+
# scenario we can't use `push` because it will wakeup the thread; so this
|
207
|
+
# accesses the array directly and pushes an item on it
|
208
|
+
@work_items.push(Factory.string)
|
209
|
+
@queue.dwp_shutdown
|
210
|
+
|
211
|
+
assert_not subject.alive?
|
212
|
+
assert_nil subject['popped_value']
|
213
|
+
end
|
214
|
+
|
215
|
+
end
|
216
|
+
|
217
|
+
end
|
@@ -0,0 +1,260 @@
|
|
1
|
+
require 'assert'
|
2
|
+
require 'dat-worker-pool/locked_object'
|
3
|
+
|
4
|
+
require 'test/support/thread_spies'
|
5
|
+
|
6
|
+
class DatWorkerPool::LockedObject
|
7
|
+
|
8
|
+
class UnitTests < Assert::Context
|
9
|
+
setup do
|
10
|
+
@mutex_spy = MutexSpy.new
|
11
|
+
Assert.stub(Mutex, :new){ @mutex_spy }
|
12
|
+
end
|
13
|
+
|
14
|
+
end
|
15
|
+
|
16
|
+
class LockedObjectTests < UnitTests
|
17
|
+
desc "DatWorkerPool::LockedObject"
|
18
|
+
setup do
|
19
|
+
@passed_value = Factory.string
|
20
|
+
@locked_object = DatWorkerPool::LockedObject.new(@passed_value)
|
21
|
+
end
|
22
|
+
subject{ @locked_object }
|
23
|
+
|
24
|
+
should have_readers :mutex
|
25
|
+
should have_imeths :value, :set
|
26
|
+
should have_imeths :with_lock
|
27
|
+
|
28
|
+
should "know its value" do
|
29
|
+
assert_equal @passed_value, subject.value
|
30
|
+
end
|
31
|
+
|
32
|
+
should "default its value to `nil`" do
|
33
|
+
assert_nil DatWorkerPool::LockedObject.new.value
|
34
|
+
end
|
35
|
+
|
36
|
+
should "lock access to its value" do
|
37
|
+
assert_false @mutex_spy.synchronize_called
|
38
|
+
subject.value
|
39
|
+
assert_true @mutex_spy.synchronize_called
|
40
|
+
end
|
41
|
+
|
42
|
+
should "allow setting its value" do
|
43
|
+
new_value = Factory.string
|
44
|
+
subject.set(new_value)
|
45
|
+
assert_equal new_value, subject.value
|
46
|
+
end
|
47
|
+
|
48
|
+
should "lock access to its value when setting it" do
|
49
|
+
assert_false @mutex_spy.synchronize_called
|
50
|
+
subject.set(Factory.string)
|
51
|
+
assert_true @mutex_spy.synchronize_called
|
52
|
+
end
|
53
|
+
|
54
|
+
should "allow accessing its mutex and value with the lock using `with_lock`" do
|
55
|
+
yielded_mutex, yielded_value = [nil, nil]
|
56
|
+
assert_false @mutex_spy.synchronize_called
|
57
|
+
subject.with_lock{ |m, v| yielded_mutex, yielded_value = [m, v] }
|
58
|
+
assert_true @mutex_spy.synchronize_called
|
59
|
+
|
60
|
+
assert_equal subject.mutex, yielded_mutex
|
61
|
+
assert_equal subject.value, yielded_value
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
class LockedArrayTests < UnitTests
|
67
|
+
desc "DatWorkerPool::LockedArray"
|
68
|
+
setup do
|
69
|
+
@locked_array = DatWorkerPool::LockedArray.new
|
70
|
+
end
|
71
|
+
subject{ @locked_array }
|
72
|
+
|
73
|
+
should have_imeths :values
|
74
|
+
should have_imeths :first, :last
|
75
|
+
should have_imeths :size, :empty?
|
76
|
+
should have_imeths :push, :pop
|
77
|
+
should have_imeths :shift, :unshift
|
78
|
+
should have_imeths :delete
|
79
|
+
|
80
|
+
should "be a dat-worker-pool locked object" do
|
81
|
+
assert DatWorkerPool::LockedArray < DatWorkerPool::LockedObject
|
82
|
+
end
|
83
|
+
|
84
|
+
should "use an empty array for its value" do
|
85
|
+
assert_equal [], subject.value
|
86
|
+
end
|
87
|
+
|
88
|
+
should "alias its value method as values" do
|
89
|
+
assert_same subject.value, subject.values
|
90
|
+
end
|
91
|
+
|
92
|
+
should "know its first and last items" do
|
93
|
+
assert_nil subject.first
|
94
|
+
assert_nil subject.last
|
95
|
+
|
96
|
+
subject.value.push(Factory.string)
|
97
|
+
subject.value.push(Factory.string)
|
98
|
+
|
99
|
+
assert_equal subject.values.first, subject.first
|
100
|
+
assert_equal subject.values.last, subject.last
|
101
|
+
end
|
102
|
+
|
103
|
+
should "lock access to getting its first item" do
|
104
|
+
assert_false @mutex_spy.synchronize_called
|
105
|
+
subject.first
|
106
|
+
assert_true @mutex_spy.synchronize_called
|
107
|
+
end
|
108
|
+
|
109
|
+
should "lock access to getting its last item" do
|
110
|
+
assert_false @mutex_spy.synchronize_called
|
111
|
+
subject.last
|
112
|
+
assert_true @mutex_spy.synchronize_called
|
113
|
+
end
|
114
|
+
|
115
|
+
should "know its size" do
|
116
|
+
assert_equal 0, subject.size
|
117
|
+
subject.value.push(Factory.string)
|
118
|
+
assert_equal 1, subject.size
|
119
|
+
end
|
120
|
+
|
121
|
+
should "lock access to reading its size" do
|
122
|
+
assert_false @mutex_spy.synchronize_called
|
123
|
+
subject.size
|
124
|
+
assert_true @mutex_spy.synchronize_called
|
125
|
+
end
|
126
|
+
|
127
|
+
should "know if its empty or not" do
|
128
|
+
assert_true subject.empty?
|
129
|
+
subject.value.push(Factory.string)
|
130
|
+
assert_false subject.empty?
|
131
|
+
end
|
132
|
+
|
133
|
+
should "lock access to checking if its empty" do
|
134
|
+
assert_false @mutex_spy.synchronize_called
|
135
|
+
subject.empty?
|
136
|
+
assert_true @mutex_spy.synchronize_called
|
137
|
+
end
|
138
|
+
|
139
|
+
should "allow pushing and popping to its array" do
|
140
|
+
new_item = Factory.string
|
141
|
+
subject.push(new_item)
|
142
|
+
assert_same new_item, subject.value.last
|
143
|
+
|
144
|
+
result = subject.pop
|
145
|
+
assert_same new_item, result
|
146
|
+
end
|
147
|
+
|
148
|
+
should "lock accessing when pushing/popping" do
|
149
|
+
assert_false @mutex_spy.synchronize_called
|
150
|
+
subject.push(Factory.string)
|
151
|
+
assert_true @mutex_spy.synchronize_called
|
152
|
+
|
153
|
+
@mutex_spy.synchronize_called = false
|
154
|
+
subject.pop
|
155
|
+
assert_true @mutex_spy.synchronize_called
|
156
|
+
end
|
157
|
+
|
158
|
+
should "allow shifting and unshifting to its array" do
|
159
|
+
new_item = Factory.string
|
160
|
+
subject.unshift(new_item)
|
161
|
+
assert_same new_item, subject.value.first
|
162
|
+
|
163
|
+
result = subject.shift
|
164
|
+
assert_same new_item, result
|
165
|
+
end
|
166
|
+
|
167
|
+
should "lock accessing when shifting/unshifting" do
|
168
|
+
assert_false @mutex_spy.synchronize_called
|
169
|
+
subject.unshift(Factory.string)
|
170
|
+
assert_true @mutex_spy.synchronize_called
|
171
|
+
|
172
|
+
@mutex_spy.synchronize_called = false
|
173
|
+
subject.shift
|
174
|
+
assert_true @mutex_spy.synchronize_called
|
175
|
+
end
|
176
|
+
|
177
|
+
should "allow deleting items from its array" do
|
178
|
+
item = Factory.string
|
179
|
+
(Factory.integer(3) + 1).times{ subject.push(item) }
|
180
|
+
|
181
|
+
subject.delete(item)
|
182
|
+
assert_true subject.empty?
|
183
|
+
end
|
184
|
+
|
185
|
+
should "lock access to when deleting" do
|
186
|
+
assert_false @mutex_spy.synchronize_called
|
187
|
+
subject.delete(Factory.string)
|
188
|
+
assert_true @mutex_spy.synchronize_called
|
189
|
+
end
|
190
|
+
|
191
|
+
end
|
192
|
+
|
193
|
+
class LockedSetTests < UnitTests
|
194
|
+
desc "DatWorkerPool::LockedSet"
|
195
|
+
setup do
|
196
|
+
@locked_set = DatWorkerPool::LockedSet.new
|
197
|
+
end
|
198
|
+
subject{ @locked_set }
|
199
|
+
|
200
|
+
should have_imeths :values, :size, :empty?, :add, :remove
|
201
|
+
|
202
|
+
should "be a dat-worker-pool locked object" do
|
203
|
+
assert DatWorkerPool::LockedSet < DatWorkerPool::LockedObject
|
204
|
+
end
|
205
|
+
|
206
|
+
should "use an empty set for its value" do
|
207
|
+
assert_instance_of Set, subject.value
|
208
|
+
assert_true subject.value.empty?
|
209
|
+
end
|
210
|
+
|
211
|
+
should "alias its value method as values" do
|
212
|
+
assert_same subject.value, subject.values
|
213
|
+
end
|
214
|
+
|
215
|
+
should "know its size" do
|
216
|
+
assert_equal 0, subject.size
|
217
|
+
subject.value.add(Factory.string)
|
218
|
+
assert_equal 1, subject.size
|
219
|
+
end
|
220
|
+
|
221
|
+
should "lock access to reading its size" do
|
222
|
+
assert_false @mutex_spy.synchronize_called
|
223
|
+
subject.size
|
224
|
+
assert_true @mutex_spy.synchronize_called
|
225
|
+
end
|
226
|
+
|
227
|
+
should "know if its empty or not" do
|
228
|
+
assert_true subject.empty?
|
229
|
+
subject.value.add(Factory.string)
|
230
|
+
assert_false subject.empty?
|
231
|
+
end
|
232
|
+
|
233
|
+
should "lock access to checking if its empty" do
|
234
|
+
assert_false @mutex_spy.synchronize_called
|
235
|
+
subject.empty?
|
236
|
+
assert_true @mutex_spy.synchronize_called
|
237
|
+
end
|
238
|
+
|
239
|
+
should "allow adding and removing on its set" do
|
240
|
+
new_item = Factory.string
|
241
|
+
subject.add(new_item)
|
242
|
+
assert_includes new_item, subject.value
|
243
|
+
|
244
|
+
subject.remove(new_item)
|
245
|
+
assert_not_includes new_item, subject.value
|
246
|
+
end
|
247
|
+
|
248
|
+
should "lock accessing when pushing/popping" do
|
249
|
+
assert_false @mutex_spy.synchronize_called
|
250
|
+
subject.add(Factory.string)
|
251
|
+
assert_true @mutex_spy.synchronize_called
|
252
|
+
|
253
|
+
@mutex_spy.synchronize_called = false
|
254
|
+
subject.remove(Factory.string)
|
255
|
+
assert_true @mutex_spy.synchronize_called
|
256
|
+
end
|
257
|
+
|
258
|
+
end
|
259
|
+
|
260
|
+
end
|