workers 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +34 -43
- data/lib/workers/pool.rb +9 -5
- data/lib/workers/task.rb +4 -4
- data/lib/workers/task_group.rb +4 -3
- data/lib/workers/version.rb +1 -1
- data/lib/workers/worker.rb +8 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 50e412eae6d1bdf77fe6588ffc6a8d2412f739f7
|
4
|
+
data.tar.gz: ab20ac26b42dd524faa2d0d1171e6887de2e3bbf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d22b5c5fb81b2b5b7105c1d237b2d9b779af7b2ef9cad993cd80c9b012a851390d851ec976497f51d87a7cbc2265d31e3712c4d2b2d2572da65c7ed8fa9fbec7
|
7
|
+
data.tar.gz: 3c44a7b8cc2e9943e919cb02e231f2d25511b5b24a4df1be847ef7e9c3b7ae083d07d35b3f1f99c25d5e8784745953300301ebecce7b67c155de3182ca6ecd94
|
data/README.md
CHANGED
@@ -42,7 +42,7 @@ Any exceptions while mapping with cause the entire map method to fail.
|
|
42
42
|
If your block is prone to temporary failures (exceptions), you can retry it.
|
43
43
|
|
44
44
|
Workers.map([1, 2, 3, 4, 5], :max_tries => 100) do |i|
|
45
|
-
if rand <= 0.
|
45
|
+
if rand <= 0.8
|
46
46
|
puts "Sometimes I like to fail while computing #{i} * #{i}."
|
47
47
|
raise 'sad face'
|
48
48
|
end
|
@@ -53,7 +53,7 @@ If your block is prone to temporary failures (exceptions), you can retry it.
|
|
53
53
|
## Tasks
|
54
54
|
|
55
55
|
Tasks and task groups provide more flexibility than parallel map.
|
56
|
-
|
56
|
+
For example, you get to decide how you want to handle exceptions.
|
57
57
|
|
58
58
|
# Create a task group (it contains a pool of worker threads).
|
59
59
|
group = Workers::TaskGroup.new
|
@@ -61,8 +61,12 @@ The main benefit is that you get to decide how you want to handle exceptions.
|
|
61
61
|
# Add tasks to the group.
|
62
62
|
10.times do |i|
|
63
63
|
10.times do |j|
|
64
|
-
group.add do
|
64
|
+
group.add(:max_tries => 10) do
|
65
65
|
group.synchronize { puts "Computing #{i} * #{j}..." }
|
66
|
+
if rand <= 0.9
|
67
|
+
group.synchronize { puts "Sometimes I like to fail while computing #{i} * #{i}." }
|
68
|
+
raise 'sad face'
|
69
|
+
end
|
66
70
|
i * j # Last statement executed is the result of the task.
|
67
71
|
end
|
68
72
|
end
|
@@ -104,9 +108,9 @@ This method uses a mutex so you can serialize portions of your tasks that aren't
|
|
104
108
|
|
105
109
|
task = Workers::Task.new(
|
106
110
|
:logger => nil, # Ruby logger instance.
|
107
|
-
:
|
111
|
+
:on_perform => proc {}, # Required option. Block of code to run.
|
108
112
|
:args => [], # Array of arguments passed to the 'perform' block.
|
109
|
-
:
|
113
|
+
:on_finished => nil, # Callback to execute after attempting to run the task.
|
110
114
|
:max_tries => 1, # Number of times to try completing the task (without an exception).
|
111
115
|
)
|
112
116
|
|
@@ -116,32 +120,27 @@ This method uses a mutex so you can serialize portions of your tasks that aren't
|
|
116
120
|
|
117
121
|
The main purpose of the Worker class is to add an event system on top of Ruby's built-in Thread class.
|
118
122
|
This greatly simplifies inter-thread communication.
|
119
|
-
|
120
|
-
Failing to handle exceptions will result in dead workers so you must rescue them in your application code.
|
123
|
+
You must manually dispose of pools and workers so they get garbage collected.
|
121
124
|
|
122
125
|
# Initialize a worker pool.
|
123
|
-
pool = Workers::Pool.new
|
126
|
+
pool = Workers::Pool.new(:on_exception => proc { |e|
|
127
|
+
puts "A worker encountered an exception: #{e.class}: #{e.message}"
|
128
|
+
})
|
124
129
|
|
125
130
|
# Perform some work in the background.
|
126
131
|
100.times do
|
127
132
|
pool.perform do
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
rescue Exception => e
|
132
|
-
puts "Oh no, my hello world failed!"
|
133
|
-
end
|
133
|
+
sleep(rand(3))
|
134
|
+
raise 'sad face' if rand < 0.5
|
135
|
+
puts "Hello world from thread #{Thread.current.object_id}"
|
134
136
|
end
|
135
137
|
end
|
136
138
|
|
137
|
-
#
|
138
|
-
pool.
|
139
|
+
# Wait up to 30 seconds for the workers to cleanly shutdown (or forcefully kill them).
|
140
|
+
pool.dispose(30) do
|
139
141
|
puts "Worker thread #{Thread.current.object_id} is shutting down."
|
140
142
|
end
|
141
143
|
|
142
|
-
# Wait for the workers to shutdown.
|
143
|
-
pool.join
|
144
|
-
|
145
144
|
#### Advanced
|
146
145
|
|
147
146
|
The Worker class is designed to be customized through inheritence and its event system:
|
@@ -153,12 +152,8 @@ The Worker class is designed to be customized through inheritence and its event
|
|
153
152
|
def event_handler(event)
|
154
153
|
case event.command
|
155
154
|
when :my_custom
|
156
|
-
|
157
|
-
|
158
|
-
sleep(1)
|
159
|
-
rescue Exception => e
|
160
|
-
puts "This is a very sad program."
|
161
|
-
end
|
155
|
+
puts "Worker received custom event: #{event.data}"
|
156
|
+
sleep(1)
|
162
157
|
else
|
163
158
|
super(event)
|
164
159
|
end
|
@@ -166,27 +161,25 @@ The Worker class is designed to be customized through inheritence and its event
|
|
166
161
|
end
|
167
162
|
|
168
163
|
# Create a pool that uses your custom worker class.
|
169
|
-
pool = Workers::Pool.new(:worker_class => CustomWorker
|
164
|
+
pool = Workers::Pool.new(:worker_class => CustomWorker, :on_exception => proc { |e|
|
165
|
+
puts "A worker encountered an exception: #{e.class}: e.message}"
|
166
|
+
})
|
170
167
|
|
171
168
|
# Tell the workers to do some work using custom events.
|
172
169
|
100.times do |i|
|
173
170
|
pool.enqueue(:my_custom, i)
|
174
171
|
end
|
175
172
|
|
176
|
-
#
|
177
|
-
pool.
|
173
|
+
# Wait up to 30 seconds for the workers to cleanly shutdown (or forcefully kill them).
|
174
|
+
pool.dispose(30) do
|
178
175
|
puts "Worker thread #{Thread.current.object_id} is shutting down."
|
179
176
|
end
|
180
177
|
|
181
|
-
# Wait for the workers to shutdown.
|
182
|
-
pool.join
|
183
|
-
|
184
178
|
#### Without pools
|
185
179
|
|
186
180
|
In most cases you will be using a group of workers (a pool) as demonstrated above.
|
187
181
|
In certain cases, you may want to use a worker directly without the pool.
|
188
|
-
This
|
189
|
-
Note that you must handle exceptions yourself since you are working directly with the worker class:
|
182
|
+
This gives you direct access to a single event-driven thread that won't die on an exception.
|
190
183
|
|
191
184
|
# Create a single worker.
|
192
185
|
worker = Workers::Worker.new
|
@@ -194,23 +187,20 @@ Note that you must handle exceptions yourself since you are working directly wit
|
|
194
187
|
# Perform some work in the background.
|
195
188
|
25.times do |i|
|
196
189
|
worker.perform do
|
197
|
-
|
198
|
-
|
199
|
-
puts "Hello world from thread #{Thread.current.object_id}"
|
200
|
-
rescue Exception => e
|
201
|
-
puts "Oh no, my hello world failed!"
|
202
|
-
end
|
190
|
+
sleep(0.1)
|
191
|
+
puts "Hello world from thread #{Thread.current.object_id}"
|
203
192
|
end
|
204
193
|
end
|
205
194
|
|
206
|
-
#
|
207
|
-
worker.
|
195
|
+
# Wait up to 30 seconds for the worker to cleanly shutdown (or forcefully kill it).
|
196
|
+
worker.dispose(30)
|
208
197
|
|
209
198
|
#### Options
|
210
199
|
|
211
200
|
worker = Workers::Worker.new(
|
212
201
|
:logger => nil, # Ruby Logger instance.
|
213
|
-
:input_queue => nil
|
202
|
+
:input_queue => nil, # Ruby Queue used for input events.
|
203
|
+
:on_exception => nil # Callback to execute on exception (exception passed as only argument).
|
214
204
|
)
|
215
205
|
|
216
206
|
## Pools
|
@@ -240,6 +230,7 @@ Pools can be adjusted using the below methods:
|
|
240
230
|
:size => 20, # Number of threads to create.
|
241
231
|
:logger => nil, # Ruby Logger instance.
|
242
232
|
:worker_class => Workers::Worker # Class of worker to use for this pool.
|
233
|
+
:on_exception => nil # Callback to execute on exception (exception passed as only argument).
|
243
234
|
)
|
244
235
|
|
245
236
|
## Timers
|
@@ -346,7 +337,7 @@ Threads performing such IO will temporarily release the GIL and thus let another
|
|
346
337
|
|
347
338
|
#### MRI 1.8.x or older (not supported)
|
348
339
|
|
349
|
-
These old versions of Ruby
|
340
|
+
These old versions of Ruby use green threads (application layer threads) instead of operating system level threads.
|
350
341
|
I recommend you upgrade to a newer version as I haven't tested Workers with them.
|
351
342
|
They also aren't officially supported by the Ruby community at this point.
|
352
343
|
|
data/lib/workers/pool.rb
CHANGED
@@ -4,6 +4,8 @@ module Workers
|
|
4
4
|
|
5
5
|
DEFAULT_POOL_SIZE = 20
|
6
6
|
|
7
|
+
attr_accessor :on_exception
|
8
|
+
|
7
9
|
def initialize(options = {})
|
8
10
|
@logger = Workers::LogProxy.new(options[:logger])
|
9
11
|
@worker_class = options[:worker_class] || Workers::Worker
|
@@ -11,7 +13,7 @@ module Workers
|
|
11
13
|
@lock = Monitor.new
|
12
14
|
@workers = Set.new
|
13
15
|
@size = 0
|
14
|
-
@
|
16
|
+
@on_exception = options[:on_exception]
|
15
17
|
|
16
18
|
expand(options[:size] || Workers::Pool::DEFAULT_POOL_SIZE)
|
17
19
|
|
@@ -48,8 +50,11 @@ module Workers
|
|
48
50
|
results
|
49
51
|
end
|
50
52
|
|
51
|
-
def dispose(max_wait = nil)
|
52
|
-
shutdown
|
53
|
+
def dispose(max_wait = nil, &block)
|
54
|
+
shutdown do
|
55
|
+
block.call if block
|
56
|
+
end
|
57
|
+
|
53
58
|
join(max_wait)
|
54
59
|
end
|
55
60
|
|
@@ -66,8 +71,7 @@ module Workers
|
|
66
71
|
def expand(count)
|
67
72
|
@lock.synchronize do
|
68
73
|
count.times do
|
69
|
-
worker = @worker_class.new(:input_queue => @input_queue, :
|
70
|
-
:on_exception => @exception_callback, :logger => @logger)
|
74
|
+
worker = @worker_class.new(:input_queue => @input_queue, :on_exception => @on_exception, :logger => @logger)
|
71
75
|
@workers << worker
|
72
76
|
@size += 1
|
73
77
|
end
|
data/lib/workers/task.rb
CHANGED
@@ -12,8 +12,8 @@ module Workers
|
|
12
12
|
def initialize(options = {})
|
13
13
|
@logger = Workers::LogProxy.new(options[:logger])
|
14
14
|
@input = options[:input] || []
|
15
|
-
@
|
16
|
-
@
|
15
|
+
@on_perform = options[:on_perform] || raise(Workers::MissingCallbackError, 'on_perform callback is required.')
|
16
|
+
@on_finished = options[:on_finished]
|
17
17
|
@max_tries = options[:max_tries] || 1
|
18
18
|
@state = :initialized
|
19
19
|
@tries = 0
|
@@ -32,7 +32,7 @@ module Workers
|
|
32
32
|
@tries += 1
|
33
33
|
|
34
34
|
begin
|
35
|
-
@result = @
|
35
|
+
@result = @on_perform.call(@input)
|
36
36
|
@state = :succeeded
|
37
37
|
@exception = nil
|
38
38
|
rescue Exception => e
|
@@ -41,7 +41,7 @@ module Workers
|
|
41
41
|
end
|
42
42
|
end
|
43
43
|
|
44
|
-
@
|
44
|
+
@on_finished.call(self)
|
45
45
|
|
46
46
|
nil
|
47
47
|
end
|
data/lib/workers/task_group.rb
CHANGED
@@ -21,8 +21,8 @@ module Workers
|
|
21
21
|
def add(options = {}, &block)
|
22
22
|
state!(:initialized)
|
23
23
|
|
24
|
-
options[:
|
25
|
-
options[:
|
24
|
+
options[:on_finished] = method(:finished)
|
25
|
+
options[:on_perform] ||= block
|
26
26
|
|
27
27
|
@tasks << Workers::Task.new(options)
|
28
28
|
|
@@ -67,10 +67,11 @@ module Workers
|
|
67
67
|
|
68
68
|
if (failure = failures[0])
|
69
69
|
a = failure.input.inspect
|
70
|
+
c = failure.exception.class.to_s
|
70
71
|
m = failure.exception.message
|
71
72
|
b = failure.exception.backtrace.join("\n")
|
72
73
|
|
73
|
-
raise Workers::FailedTaskError, "
|
74
|
+
raise Workers::FailedTaskError, "#{failures.count} task(s) failed (Only the first failure is shown).\nARGS=#{a}, EXCEPTION=#{c}: #{m}\n#{b}\n----------\n"
|
74
75
|
end
|
75
76
|
|
76
77
|
tasks.map { |t| t.result }
|
data/lib/workers/version.rb
CHANGED
data/lib/workers/worker.rb
CHANGED
@@ -3,13 +3,13 @@ module Workers
|
|
3
3
|
include Workers::Helpers
|
4
4
|
|
5
5
|
attr_accessor :exception
|
6
|
+
attr_accessor :on_exception
|
6
7
|
|
7
8
|
def initialize(options = {})
|
8
9
|
@logger = Workers::LogProxy.new(options[:logger])
|
9
10
|
@input_queue = options[:input_queue] || Queue.new
|
10
11
|
@thread = Thread.new { start_event_loop }
|
11
|
-
@
|
12
|
-
@die_on_exception = options.include?(:die_on_exception) ? options[:die_on_exception] : true
|
12
|
+
@on_exception = options[:on_exception]
|
13
13
|
@run = true
|
14
14
|
|
15
15
|
nil
|
@@ -105,8 +105,12 @@ module Workers
|
|
105
105
|
|
106
106
|
def exception_handler(e)
|
107
107
|
@exception = e
|
108
|
-
|
109
|
-
|
108
|
+
|
109
|
+
begin
|
110
|
+
@on_exception.call(e) if @on_exception
|
111
|
+
rescue Exception => e2
|
112
|
+
STDERR.puts "Your #{self.class.to_s} exception handler raised an error. Fix your handler!\n#{e2.class.to_s}: #{e2.message}\n#{e2.backtrace.join("\n")}"
|
113
|
+
end
|
110
114
|
|
111
115
|
nil
|
112
116
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: workers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chad Remesch
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|