parallel 1.4.1 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/parallel.rb +112 -119
- data/lib/parallel/version.rb +1 -1
- metadata +10 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4ebdef905e948776369686271708c85a60b4c63c
|
4
|
+
data.tar.gz: da64471e0d337ca3a778670efed6e0c2eda454af
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9677d167fc5b5796630d9654bc5dd82b1e950aa0a69537dcdb1332326ac95399d91638827cb69a00be79123db43514586d30037a92a9572dba84ff63745b4ff1
|
7
|
+
data.tar.gz: 4dca43cfa0a92caf06ca45b95de2a3b28e781b6e0300b94c14fb4703a1f4a264d73c10c2d11b2e92c344e92c998afeb60245bd1aafb100c3e7d262ca88fd665f
|
data/lib/parallel.rb
CHANGED
@@ -16,8 +16,6 @@ module Parallel
|
|
16
16
|
|
17
17
|
Stop = Object.new
|
18
18
|
|
19
|
-
INTERRUPT_SIGNAL = :SIGINT
|
20
|
-
|
21
19
|
class ExceptionWrapper
|
22
20
|
attr_reader :exception
|
23
21
|
def initialize(exception)
|
@@ -65,31 +63,15 @@ module Parallel
|
|
65
63
|
end
|
66
64
|
end
|
67
65
|
|
68
|
-
class
|
69
|
-
def initialize(
|
70
|
-
@lambda = (
|
71
|
-
@
|
66
|
+
class JobFactory
|
67
|
+
def initialize(source, mutex)
|
68
|
+
@lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
|
69
|
+
@source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
|
72
70
|
@mutex = mutex
|
73
71
|
@index = -1
|
74
72
|
@stopped = false
|
75
73
|
end
|
76
74
|
|
77
|
-
def producer?
|
78
|
-
@lambda
|
79
|
-
end
|
80
|
-
|
81
|
-
def each_with_index(&block)
|
82
|
-
if producer?
|
83
|
-
loop do
|
84
|
-
item, index = self.next
|
85
|
-
break unless index
|
86
|
-
yield(item, index)
|
87
|
-
end
|
88
|
-
else
|
89
|
-
@items.each_with_index(&block)
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
75
|
def next
|
94
76
|
if producer?
|
95
77
|
# - index and item stay in sync
|
@@ -104,13 +86,17 @@ module Parallel
|
|
104
86
|
else
|
105
87
|
index = @mutex.synchronize { @index += 1 }
|
106
88
|
return if index >= size
|
107
|
-
item = @
|
89
|
+
item = @source[index]
|
108
90
|
end
|
109
91
|
[item, index]
|
110
92
|
end
|
111
93
|
|
112
94
|
def size
|
113
|
-
|
95
|
+
if producer?
|
96
|
+
Float::INFINITY
|
97
|
+
else
|
98
|
+
@source.size
|
99
|
+
end
|
114
100
|
end
|
115
101
|
|
116
102
|
def pack(item, index)
|
@@ -118,7 +104,13 @@ module Parallel
|
|
118
104
|
end
|
119
105
|
|
120
106
|
def unpack(data)
|
121
|
-
producer? ? data : [@
|
107
|
+
producer? ? data : [@source[data], data]
|
108
|
+
end
|
109
|
+
|
110
|
+
private
|
111
|
+
|
112
|
+
def producer?
|
113
|
+
@lambda
|
122
114
|
end
|
123
115
|
|
124
116
|
def queue_wrapper(array)
|
@@ -126,6 +118,68 @@ module Parallel
|
|
126
118
|
end
|
127
119
|
end
|
128
120
|
|
121
|
+
class UserInterruptHandler
|
122
|
+
INTERRUPT_SIGNAL = :SIGINT
|
123
|
+
|
124
|
+
class << self
|
125
|
+
# kill all these pids or threads if user presses Ctrl+c
|
126
|
+
def kill_on_ctrl_c(things, options)
|
127
|
+
return yield if RUBY_ENGINE == "jruby"
|
128
|
+
@to_be_killed ||= []
|
129
|
+
old_interrupt = nil
|
130
|
+
signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
|
131
|
+
|
132
|
+
if @to_be_killed.empty?
|
133
|
+
old_interrupt = trap_interrupt(signal) do
|
134
|
+
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
135
|
+
@to_be_killed.flatten.compact.each { |thing| kill(thing) }
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
@to_be_killed << things
|
140
|
+
|
141
|
+
yield
|
142
|
+
ensure
|
143
|
+
@to_be_killed.pop # free threads for GC and do not kill pids that could be used for new processes
|
144
|
+
restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
|
145
|
+
end
|
146
|
+
|
147
|
+
def kill(thing)
|
148
|
+
if thing.is_a?(Thread)
|
149
|
+
thing.kill
|
150
|
+
else
|
151
|
+
begin
|
152
|
+
Process.kill(:KILL, thing)
|
153
|
+
rescue Errno::ESRCH
|
154
|
+
# some linux systems already automatically killed the children at this point
|
155
|
+
# so we just ignore them not being there
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
private
|
161
|
+
|
162
|
+
def trap_interrupt(signal)
|
163
|
+
old = Signal.trap signal, 'IGNORE'
|
164
|
+
|
165
|
+
Signal.trap signal do
|
166
|
+
yield
|
167
|
+
if old == "DEFAULT"
|
168
|
+
raise Interrupt
|
169
|
+
else
|
170
|
+
old.call
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
old
|
175
|
+
end
|
176
|
+
|
177
|
+
def restore_interrupt(old, signal)
|
178
|
+
Signal.trap signal, old
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
129
183
|
class << self
|
130
184
|
def in_threads(options={:count => 2})
|
131
185
|
count, options = extract_count_from_options(options)
|
@@ -139,7 +193,7 @@ module Parallel
|
|
139
193
|
end
|
140
194
|
end
|
141
195
|
|
142
|
-
kill_on_ctrl_c(threads, options) { wait_for_threads(threads) }
|
196
|
+
UserInterruptHandler.kill_on_ctrl_c(threads, options) { wait_for_threads(threads) }
|
143
197
|
|
144
198
|
out
|
145
199
|
end
|
@@ -159,7 +213,7 @@ module Parallel
|
|
159
213
|
each(array, options.merge(:with_index => true), &block)
|
160
214
|
end
|
161
215
|
|
162
|
-
def map(
|
216
|
+
def map(source, options = {}, &block)
|
163
217
|
options[:mutex] = Mutex.new
|
164
218
|
|
165
219
|
if RUBY_PLATFORM =~ /java/ and not options[:in_processes]
|
@@ -173,24 +227,23 @@ module Parallel
|
|
173
227
|
if Process.respond_to?(:fork)
|
174
228
|
size = options[method] || processor_count
|
175
229
|
else
|
176
|
-
|
230
|
+
warn "Process.fork is not supported by this Ruby"
|
177
231
|
size = 0
|
178
232
|
end
|
179
233
|
end
|
180
234
|
|
181
|
-
|
182
|
-
|
183
|
-
size = [items.producer? ? size : items.size, size].min
|
235
|
+
job_factory = JobFactory.new(source, options[:mutex])
|
236
|
+
size = [job_factory.size, size].min
|
184
237
|
|
185
238
|
options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
|
186
|
-
add_progress_bar!(
|
239
|
+
add_progress_bar!(job_factory, options)
|
187
240
|
|
188
241
|
if size == 0
|
189
|
-
work_direct(
|
242
|
+
work_direct(job_factory, options, &block)
|
190
243
|
elsif method == :in_threads
|
191
|
-
work_in_threads(
|
244
|
+
work_in_threads(job_factory, options.merge(:count => size), &block)
|
192
245
|
else
|
193
|
-
work_in_processes(
|
246
|
+
work_in_processes(job_factory, options.merge(:count => size), &block)
|
194
247
|
end
|
195
248
|
end
|
196
249
|
|
@@ -200,9 +253,9 @@ module Parallel
|
|
200
253
|
|
201
254
|
private
|
202
255
|
|
203
|
-
def add_progress_bar!(
|
256
|
+
def add_progress_bar!(job_factory, options)
|
204
257
|
if progress_options = options[:progress]
|
205
|
-
raise "Progressbar
|
258
|
+
raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
|
206
259
|
require 'ruby-progressbar'
|
207
260
|
|
208
261
|
if progress_options.respond_to? :to_str
|
@@ -210,7 +263,7 @@ module Parallel
|
|
210
263
|
end
|
211
264
|
|
212
265
|
progress_options = {
|
213
|
-
total:
|
266
|
+
total: job_factory.size,
|
214
267
|
format: '%t |%E | %B | %a'
|
215
268
|
}.merge(progress_options)
|
216
269
|
|
@@ -223,10 +276,10 @@ module Parallel
|
|
223
276
|
end
|
224
277
|
end
|
225
278
|
|
226
|
-
|
227
|
-
def work_direct(items, options, &block)
|
279
|
+
def work_direct(job_factory, options, &block)
|
228
280
|
results = []
|
229
|
-
|
281
|
+
while set = job_factory.next
|
282
|
+
item, index = set
|
230
283
|
results << with_instrumentation(item, index, options) do
|
231
284
|
call_with_index(item, index, options, &block)
|
232
285
|
end
|
@@ -234,24 +287,20 @@ module Parallel
|
|
234
287
|
results
|
235
288
|
end
|
236
289
|
|
237
|
-
def work_in_threads(
|
290
|
+
def work_in_threads(job_factory, options, &block)
|
238
291
|
results = []
|
239
292
|
exception = nil
|
240
293
|
|
241
294
|
in_threads(options) do
|
242
|
-
# as long as there are more
|
243
|
-
|
244
|
-
break if exception
|
245
|
-
item, index = items.next
|
246
|
-
break unless index
|
247
|
-
|
295
|
+
# as long as there are more jobs, work on one of them
|
296
|
+
while !exception && set = job_factory.next
|
248
297
|
begin
|
298
|
+
item, index = set
|
249
299
|
results[index] = with_instrumentation item, index, options do
|
250
300
|
call_with_index(item, index, options, &block)
|
251
301
|
end
|
252
302
|
rescue StandardError => e
|
253
303
|
exception = e
|
254
|
-
break
|
255
304
|
end
|
256
305
|
end
|
257
306
|
end
|
@@ -259,12 +308,12 @@ module Parallel
|
|
259
308
|
handle_exception(exception, results)
|
260
309
|
end
|
261
310
|
|
262
|
-
def work_in_processes(
|
263
|
-
workers = create_workers(
|
311
|
+
def work_in_processes(job_factory, options, &blk)
|
312
|
+
workers = create_workers(job_factory, options, &blk)
|
264
313
|
results = []
|
265
314
|
exception = nil
|
266
315
|
|
267
|
-
kill_on_ctrl_c(workers.map(&:pid), options) do
|
316
|
+
UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
|
268
317
|
in_threads(options) do |i|
|
269
318
|
worker = workers[i]
|
270
319
|
worker.thread = Thread.current
|
@@ -272,19 +321,19 @@ module Parallel
|
|
272
321
|
begin
|
273
322
|
loop do
|
274
323
|
break if exception
|
275
|
-
item, index =
|
324
|
+
item, index = job_factory.next
|
276
325
|
break unless index
|
277
326
|
|
278
327
|
begin
|
279
328
|
results[index] = with_instrumentation item, index, options do
|
280
|
-
worker.work(
|
329
|
+
worker.work(job_factory.pack(item, index))
|
281
330
|
end
|
282
331
|
rescue StandardError => e
|
283
332
|
exception = e
|
284
333
|
if Parallel::Kill === exception
|
285
334
|
(workers - [worker]).each do |w|
|
286
|
-
|
287
|
-
|
335
|
+
UserInterruptHandler.kill(w.thread)
|
336
|
+
UserInterruptHandler.kill(w.pid)
|
288
337
|
end
|
289
338
|
end
|
290
339
|
end
|
@@ -299,18 +348,15 @@ module Parallel
|
|
299
348
|
handle_exception(exception, results)
|
300
349
|
end
|
301
350
|
|
302
|
-
def create_workers(
|
351
|
+
def create_workers(job_factory, options, &block)
|
303
352
|
workers = []
|
304
353
|
Array.new(options[:count]).each do
|
305
|
-
workers << worker(
|
354
|
+
workers << worker(job_factory, options.merge(:started_workers => workers), &block)
|
306
355
|
end
|
307
356
|
workers
|
308
357
|
end
|
309
358
|
|
310
|
-
def worker(
|
311
|
-
# use less memory on REE
|
312
|
-
GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
|
313
|
-
|
359
|
+
def worker(job_factory, options, &block)
|
314
360
|
child_read, parent_write = IO.pipe
|
315
361
|
parent_read, child_write = IO.pipe
|
316
362
|
|
@@ -321,7 +367,7 @@ module Parallel
|
|
321
367
|
parent_write.close
|
322
368
|
parent_read.close
|
323
369
|
|
324
|
-
process_incoming_jobs(child_read, child_write,
|
370
|
+
process_incoming_jobs(child_read, child_write, job_factory, options, &block)
|
325
371
|
ensure
|
326
372
|
child_read.close
|
327
373
|
child_write.close
|
@@ -334,10 +380,10 @@ module Parallel
|
|
334
380
|
Worker.new(parent_read, parent_write, pid)
|
335
381
|
end
|
336
382
|
|
337
|
-
def process_incoming_jobs(read, write,
|
338
|
-
|
383
|
+
def process_incoming_jobs(read, write, job_factory, options, &block)
|
384
|
+
until read.eof?
|
339
385
|
data = Marshal.load(read)
|
340
|
-
item, index =
|
386
|
+
item, index = job_factory.unpack(data)
|
341
387
|
result = begin
|
342
388
|
call_with_index(item, index, options, &block)
|
343
389
|
rescue StandardError => e
|
@@ -376,59 +422,6 @@ module Parallel
|
|
376
422
|
[count, options]
|
377
423
|
end
|
378
424
|
|
379
|
-
# kill all these pids or threads if user presses Ctrl+c
|
380
|
-
def kill_on_ctrl_c(things, options)
|
381
|
-
@to_be_killed ||= []
|
382
|
-
old_interrupt = nil
|
383
|
-
signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
|
384
|
-
|
385
|
-
if @to_be_killed.empty?
|
386
|
-
old_interrupt = trap_interrupt(signal) do
|
387
|
-
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
388
|
-
@to_be_killed.flatten.compact.each { |thing| kill_that_thing!(thing) }
|
389
|
-
end
|
390
|
-
end
|
391
|
-
|
392
|
-
@to_be_killed << things
|
393
|
-
|
394
|
-
yield
|
395
|
-
ensure
|
396
|
-
@to_be_killed.pop # free threads for GC and do not kill pids that could be used for new processes
|
397
|
-
restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
|
398
|
-
end
|
399
|
-
|
400
|
-
def trap_interrupt(signal)
|
401
|
-
old = Signal.trap signal, 'IGNORE'
|
402
|
-
|
403
|
-
Signal.trap signal do
|
404
|
-
yield
|
405
|
-
if old == "DEFAULT"
|
406
|
-
raise Interrupt
|
407
|
-
else
|
408
|
-
old.call
|
409
|
-
end
|
410
|
-
end
|
411
|
-
|
412
|
-
old
|
413
|
-
end
|
414
|
-
|
415
|
-
def restore_interrupt(old, signal)
|
416
|
-
Signal.trap signal, old
|
417
|
-
end
|
418
|
-
|
419
|
-
def kill_that_thing!(thing)
|
420
|
-
if thing.is_a?(Thread)
|
421
|
-
thing.kill
|
422
|
-
else
|
423
|
-
begin
|
424
|
-
Process.kill(:KILL, thing)
|
425
|
-
rescue Errno::ESRCH
|
426
|
-
# some linux systems already automatically killed the children at this point
|
427
|
-
# so we just ignore them not being there
|
428
|
-
end
|
429
|
-
end
|
430
|
-
end
|
431
|
-
|
432
425
|
def call_with_index(item, index, options, &block)
|
433
426
|
args = [item]
|
434
427
|
args << index if options[:with_index]
|
data/lib/parallel/version.rb
CHANGED
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-25 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
|
-
description:
|
13
|
+
description:
|
14
14
|
email: michael@grosser.it
|
15
15
|
executables: []
|
16
16
|
extensions: []
|
@@ -24,24 +24,24 @@ homepage: https://github.com/grosser/parallel
|
|
24
24
|
licenses:
|
25
25
|
- MIT
|
26
26
|
metadata: {}
|
27
|
-
post_install_message:
|
27
|
+
post_install_message:
|
28
28
|
rdoc_options: []
|
29
29
|
require_paths:
|
30
30
|
- lib
|
31
31
|
required_ruby_version: !ruby/object:Gem::Requirement
|
32
32
|
requirements:
|
33
|
-
- -
|
33
|
+
- - '>='
|
34
34
|
- !ruby/object:Gem::Version
|
35
35
|
version: 1.9.3
|
36
36
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
requirements: []
|
42
|
-
rubyforge_project:
|
43
|
-
rubygems_version: 2.
|
44
|
-
signing_key:
|
42
|
+
rubyforge_project:
|
43
|
+
rubygems_version: 2.1.9
|
44
|
+
signing_key:
|
45
45
|
specification_version: 4
|
46
46
|
summary: Run any kind of code in parallel processes
|
47
47
|
test_files: []
|