parallel 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/parallel.rb +112 -119
- data/lib/parallel/version.rb +1 -1
- metadata +10 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4ebdef905e948776369686271708c85a60b4c63c
|
4
|
+
data.tar.gz: da64471e0d337ca3a778670efed6e0c2eda454af
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9677d167fc5b5796630d9654bc5dd82b1e950aa0a69537dcdb1332326ac95399d91638827cb69a00be79123db43514586d30037a92a9572dba84ff63745b4ff1
|
7
|
+
data.tar.gz: 4dca43cfa0a92caf06ca45b95de2a3b28e781b6e0300b94c14fb4703a1f4a264d73c10c2d11b2e92c344e92c998afeb60245bd1aafb100c3e7d262ca88fd665f
|
data/lib/parallel.rb
CHANGED
@@ -16,8 +16,6 @@ module Parallel
|
|
16
16
|
|
17
17
|
Stop = Object.new
|
18
18
|
|
19
|
-
INTERRUPT_SIGNAL = :SIGINT
|
20
|
-
|
21
19
|
class ExceptionWrapper
|
22
20
|
attr_reader :exception
|
23
21
|
def initialize(exception)
|
@@ -65,31 +63,15 @@ module Parallel
|
|
65
63
|
end
|
66
64
|
end
|
67
65
|
|
68
|
-
class
|
69
|
-
def initialize(
|
70
|
-
@lambda = (
|
71
|
-
@
|
66
|
+
class JobFactory
|
67
|
+
def initialize(source, mutex)
|
68
|
+
@lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
|
69
|
+
@source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
|
72
70
|
@mutex = mutex
|
73
71
|
@index = -1
|
74
72
|
@stopped = false
|
75
73
|
end
|
76
74
|
|
77
|
-
def producer?
|
78
|
-
@lambda
|
79
|
-
end
|
80
|
-
|
81
|
-
def each_with_index(&block)
|
82
|
-
if producer?
|
83
|
-
loop do
|
84
|
-
item, index = self.next
|
85
|
-
break unless index
|
86
|
-
yield(item, index)
|
87
|
-
end
|
88
|
-
else
|
89
|
-
@items.each_with_index(&block)
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
75
|
def next
|
94
76
|
if producer?
|
95
77
|
# - index and item stay in sync
|
@@ -104,13 +86,17 @@ module Parallel
|
|
104
86
|
else
|
105
87
|
index = @mutex.synchronize { @index += 1 }
|
106
88
|
return if index >= size
|
107
|
-
item = @
|
89
|
+
item = @source[index]
|
108
90
|
end
|
109
91
|
[item, index]
|
110
92
|
end
|
111
93
|
|
112
94
|
def size
|
113
|
-
|
95
|
+
if producer?
|
96
|
+
Float::INFINITY
|
97
|
+
else
|
98
|
+
@source.size
|
99
|
+
end
|
114
100
|
end
|
115
101
|
|
116
102
|
def pack(item, index)
|
@@ -118,7 +104,13 @@ module Parallel
|
|
118
104
|
end
|
119
105
|
|
120
106
|
def unpack(data)
|
121
|
-
producer? ? data : [@
|
107
|
+
producer? ? data : [@source[data], data]
|
108
|
+
end
|
109
|
+
|
110
|
+
private
|
111
|
+
|
112
|
+
def producer?
|
113
|
+
@lambda
|
122
114
|
end
|
123
115
|
|
124
116
|
def queue_wrapper(array)
|
@@ -126,6 +118,68 @@ module Parallel
|
|
126
118
|
end
|
127
119
|
end
|
128
120
|
|
121
|
+
class UserInterruptHandler
|
122
|
+
INTERRUPT_SIGNAL = :SIGINT
|
123
|
+
|
124
|
+
class << self
|
125
|
+
# kill all these pids or threads if user presses Ctrl+c
|
126
|
+
def kill_on_ctrl_c(things, options)
|
127
|
+
return yield if RUBY_ENGINE == "jruby"
|
128
|
+
@to_be_killed ||= []
|
129
|
+
old_interrupt = nil
|
130
|
+
signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
|
131
|
+
|
132
|
+
if @to_be_killed.empty?
|
133
|
+
old_interrupt = trap_interrupt(signal) do
|
134
|
+
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
135
|
+
@to_be_killed.flatten.compact.each { |thing| kill(thing) }
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
@to_be_killed << things
|
140
|
+
|
141
|
+
yield
|
142
|
+
ensure
|
143
|
+
@to_be_killed.pop # free threads for GC and do not kill pids that could be used for new processes
|
144
|
+
restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
|
145
|
+
end
|
146
|
+
|
147
|
+
def kill(thing)
|
148
|
+
if thing.is_a?(Thread)
|
149
|
+
thing.kill
|
150
|
+
else
|
151
|
+
begin
|
152
|
+
Process.kill(:KILL, thing)
|
153
|
+
rescue Errno::ESRCH
|
154
|
+
# some linux systems already automatically killed the children at this point
|
155
|
+
# so we just ignore them not being there
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
private
|
161
|
+
|
162
|
+
def trap_interrupt(signal)
|
163
|
+
old = Signal.trap signal, 'IGNORE'
|
164
|
+
|
165
|
+
Signal.trap signal do
|
166
|
+
yield
|
167
|
+
if old == "DEFAULT"
|
168
|
+
raise Interrupt
|
169
|
+
else
|
170
|
+
old.call
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
old
|
175
|
+
end
|
176
|
+
|
177
|
+
def restore_interrupt(old, signal)
|
178
|
+
Signal.trap signal, old
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
129
183
|
class << self
|
130
184
|
def in_threads(options={:count => 2})
|
131
185
|
count, options = extract_count_from_options(options)
|
@@ -139,7 +193,7 @@ module Parallel
|
|
139
193
|
end
|
140
194
|
end
|
141
195
|
|
142
|
-
kill_on_ctrl_c(threads, options) { wait_for_threads(threads) }
|
196
|
+
UserInterruptHandler.kill_on_ctrl_c(threads, options) { wait_for_threads(threads) }
|
143
197
|
|
144
198
|
out
|
145
199
|
end
|
@@ -159,7 +213,7 @@ module Parallel
|
|
159
213
|
each(array, options.merge(:with_index => true), &block)
|
160
214
|
end
|
161
215
|
|
162
|
-
def map(
|
216
|
+
def map(source, options = {}, &block)
|
163
217
|
options[:mutex] = Mutex.new
|
164
218
|
|
165
219
|
if RUBY_PLATFORM =~ /java/ and not options[:in_processes]
|
@@ -173,24 +227,23 @@ module Parallel
|
|
173
227
|
if Process.respond_to?(:fork)
|
174
228
|
size = options[method] || processor_count
|
175
229
|
else
|
176
|
-
|
230
|
+
warn "Process.fork is not supported by this Ruby"
|
177
231
|
size = 0
|
178
232
|
end
|
179
233
|
end
|
180
234
|
|
181
|
-
|
182
|
-
|
183
|
-
size = [items.producer? ? size : items.size, size].min
|
235
|
+
job_factory = JobFactory.new(source, options[:mutex])
|
236
|
+
size = [job_factory.size, size].min
|
184
237
|
|
185
238
|
options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
|
186
|
-
add_progress_bar!(
|
239
|
+
add_progress_bar!(job_factory, options)
|
187
240
|
|
188
241
|
if size == 0
|
189
|
-
work_direct(
|
242
|
+
work_direct(job_factory, options, &block)
|
190
243
|
elsif method == :in_threads
|
191
|
-
work_in_threads(
|
244
|
+
work_in_threads(job_factory, options.merge(:count => size), &block)
|
192
245
|
else
|
193
|
-
work_in_processes(
|
246
|
+
work_in_processes(job_factory, options.merge(:count => size), &block)
|
194
247
|
end
|
195
248
|
end
|
196
249
|
|
@@ -200,9 +253,9 @@ module Parallel
|
|
200
253
|
|
201
254
|
private
|
202
255
|
|
203
|
-
def add_progress_bar!(
|
256
|
+
def add_progress_bar!(job_factory, options)
|
204
257
|
if progress_options = options[:progress]
|
205
|
-
raise "Progressbar
|
258
|
+
raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
|
206
259
|
require 'ruby-progressbar'
|
207
260
|
|
208
261
|
if progress_options.respond_to? :to_str
|
@@ -210,7 +263,7 @@ module Parallel
|
|
210
263
|
end
|
211
264
|
|
212
265
|
progress_options = {
|
213
|
-
total:
|
266
|
+
total: job_factory.size,
|
214
267
|
format: '%t |%E | %B | %a'
|
215
268
|
}.merge(progress_options)
|
216
269
|
|
@@ -223,10 +276,10 @@ module Parallel
|
|
223
276
|
end
|
224
277
|
end
|
225
278
|
|
226
|
-
|
227
|
-
def work_direct(items, options, &block)
|
279
|
+
def work_direct(job_factory, options, &block)
|
228
280
|
results = []
|
229
|
-
|
281
|
+
while set = job_factory.next
|
282
|
+
item, index = set
|
230
283
|
results << with_instrumentation(item, index, options) do
|
231
284
|
call_with_index(item, index, options, &block)
|
232
285
|
end
|
@@ -234,24 +287,20 @@ module Parallel
|
|
234
287
|
results
|
235
288
|
end
|
236
289
|
|
237
|
-
def work_in_threads(
|
290
|
+
def work_in_threads(job_factory, options, &block)
|
238
291
|
results = []
|
239
292
|
exception = nil
|
240
293
|
|
241
294
|
in_threads(options) do
|
242
|
-
# as long as there are more
|
243
|
-
|
244
|
-
break if exception
|
245
|
-
item, index = items.next
|
246
|
-
break unless index
|
247
|
-
|
295
|
+
# as long as there are more jobs, work on one of them
|
296
|
+
while !exception && set = job_factory.next
|
248
297
|
begin
|
298
|
+
item, index = set
|
249
299
|
results[index] = with_instrumentation item, index, options do
|
250
300
|
call_with_index(item, index, options, &block)
|
251
301
|
end
|
252
302
|
rescue StandardError => e
|
253
303
|
exception = e
|
254
|
-
break
|
255
304
|
end
|
256
305
|
end
|
257
306
|
end
|
@@ -259,12 +308,12 @@ module Parallel
|
|
259
308
|
handle_exception(exception, results)
|
260
309
|
end
|
261
310
|
|
262
|
-
def work_in_processes(
|
263
|
-
workers = create_workers(
|
311
|
+
def work_in_processes(job_factory, options, &blk)
|
312
|
+
workers = create_workers(job_factory, options, &blk)
|
264
313
|
results = []
|
265
314
|
exception = nil
|
266
315
|
|
267
|
-
kill_on_ctrl_c(workers.map(&:pid), options) do
|
316
|
+
UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
|
268
317
|
in_threads(options) do |i|
|
269
318
|
worker = workers[i]
|
270
319
|
worker.thread = Thread.current
|
@@ -272,19 +321,19 @@ module Parallel
|
|
272
321
|
begin
|
273
322
|
loop do
|
274
323
|
break if exception
|
275
|
-
item, index =
|
324
|
+
item, index = job_factory.next
|
276
325
|
break unless index
|
277
326
|
|
278
327
|
begin
|
279
328
|
results[index] = with_instrumentation item, index, options do
|
280
|
-
worker.work(
|
329
|
+
worker.work(job_factory.pack(item, index))
|
281
330
|
end
|
282
331
|
rescue StandardError => e
|
283
332
|
exception = e
|
284
333
|
if Parallel::Kill === exception
|
285
334
|
(workers - [worker]).each do |w|
|
286
|
-
|
287
|
-
|
335
|
+
UserInterruptHandler.kill(w.thread)
|
336
|
+
UserInterruptHandler.kill(w.pid)
|
288
337
|
end
|
289
338
|
end
|
290
339
|
end
|
@@ -299,18 +348,15 @@ module Parallel
|
|
299
348
|
handle_exception(exception, results)
|
300
349
|
end
|
301
350
|
|
302
|
-
def create_workers(
|
351
|
+
def create_workers(job_factory, options, &block)
|
303
352
|
workers = []
|
304
353
|
Array.new(options[:count]).each do
|
305
|
-
workers << worker(
|
354
|
+
workers << worker(job_factory, options.merge(:started_workers => workers), &block)
|
306
355
|
end
|
307
356
|
workers
|
308
357
|
end
|
309
358
|
|
310
|
-
def worker(
|
311
|
-
# use less memory on REE
|
312
|
-
GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
|
313
|
-
|
359
|
+
def worker(job_factory, options, &block)
|
314
360
|
child_read, parent_write = IO.pipe
|
315
361
|
parent_read, child_write = IO.pipe
|
316
362
|
|
@@ -321,7 +367,7 @@ module Parallel
|
|
321
367
|
parent_write.close
|
322
368
|
parent_read.close
|
323
369
|
|
324
|
-
process_incoming_jobs(child_read, child_write,
|
370
|
+
process_incoming_jobs(child_read, child_write, job_factory, options, &block)
|
325
371
|
ensure
|
326
372
|
child_read.close
|
327
373
|
child_write.close
|
@@ -334,10 +380,10 @@ module Parallel
|
|
334
380
|
Worker.new(parent_read, parent_write, pid)
|
335
381
|
end
|
336
382
|
|
337
|
-
def process_incoming_jobs(read, write,
|
338
|
-
|
383
|
+
def process_incoming_jobs(read, write, job_factory, options, &block)
|
384
|
+
until read.eof?
|
339
385
|
data = Marshal.load(read)
|
340
|
-
item, index =
|
386
|
+
item, index = job_factory.unpack(data)
|
341
387
|
result = begin
|
342
388
|
call_with_index(item, index, options, &block)
|
343
389
|
rescue StandardError => e
|
@@ -376,59 +422,6 @@ module Parallel
|
|
376
422
|
[count, options]
|
377
423
|
end
|
378
424
|
|
379
|
-
# kill all these pids or threads if user presses Ctrl+c
|
380
|
-
def kill_on_ctrl_c(things, options)
|
381
|
-
@to_be_killed ||= []
|
382
|
-
old_interrupt = nil
|
383
|
-
signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
|
384
|
-
|
385
|
-
if @to_be_killed.empty?
|
386
|
-
old_interrupt = trap_interrupt(signal) do
|
387
|
-
$stderr.puts 'Parallel execution interrupted, exiting ...'
|
388
|
-
@to_be_killed.flatten.compact.each { |thing| kill_that_thing!(thing) }
|
389
|
-
end
|
390
|
-
end
|
391
|
-
|
392
|
-
@to_be_killed << things
|
393
|
-
|
394
|
-
yield
|
395
|
-
ensure
|
396
|
-
@to_be_killed.pop # free threads for GC and do not kill pids that could be used for new processes
|
397
|
-
restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
|
398
|
-
end
|
399
|
-
|
400
|
-
def trap_interrupt(signal)
|
401
|
-
old = Signal.trap signal, 'IGNORE'
|
402
|
-
|
403
|
-
Signal.trap signal do
|
404
|
-
yield
|
405
|
-
if old == "DEFAULT"
|
406
|
-
raise Interrupt
|
407
|
-
else
|
408
|
-
old.call
|
409
|
-
end
|
410
|
-
end
|
411
|
-
|
412
|
-
old
|
413
|
-
end
|
414
|
-
|
415
|
-
def restore_interrupt(old, signal)
|
416
|
-
Signal.trap signal, old
|
417
|
-
end
|
418
|
-
|
419
|
-
def kill_that_thing!(thing)
|
420
|
-
if thing.is_a?(Thread)
|
421
|
-
thing.kill
|
422
|
-
else
|
423
|
-
begin
|
424
|
-
Process.kill(:KILL, thing)
|
425
|
-
rescue Errno::ESRCH
|
426
|
-
# some linux systems already automatically killed the children at this point
|
427
|
-
# so we just ignore them not being there
|
428
|
-
end
|
429
|
-
end
|
430
|
-
end
|
431
|
-
|
432
425
|
def call_with_index(item, index, options, &block)
|
433
426
|
args = [item]
|
434
427
|
args << index if options[:with_index]
|
data/lib/parallel/version.rb
CHANGED
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-25 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
|
-
description:
|
13
|
+
description:
|
14
14
|
email: michael@grosser.it
|
15
15
|
executables: []
|
16
16
|
extensions: []
|
@@ -24,24 +24,24 @@ homepage: https://github.com/grosser/parallel
|
|
24
24
|
licenses:
|
25
25
|
- MIT
|
26
26
|
metadata: {}
|
27
|
-
post_install_message:
|
27
|
+
post_install_message:
|
28
28
|
rdoc_options: []
|
29
29
|
require_paths:
|
30
30
|
- lib
|
31
31
|
required_ruby_version: !ruby/object:Gem::Requirement
|
32
32
|
requirements:
|
33
|
-
- -
|
33
|
+
- - '>='
|
34
34
|
- !ruby/object:Gem::Version
|
35
35
|
version: 1.9.3
|
36
36
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
requirements: []
|
42
|
-
rubyforge_project:
|
43
|
-
rubygems_version: 2.
|
44
|
-
signing_key:
|
42
|
+
rubyforge_project:
|
43
|
+
rubygems_version: 2.1.9
|
44
|
+
signing_key:
|
45
45
|
specification_version: 4
|
46
46
|
summary: Run any kind of code in parallel processes
|
47
47
|
test_files: []
|