parallel 1.14.0 → 1.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/parallel/processor_count.rb +27 -26
- data/lib/parallel/version.rb +2 -1
- data/lib/parallel.rb +97 -72
- metadata +9 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4dd167566b23262ceacbd640fda95c5e8d0d8149cc430f51ada53765a927cb18
|
4
|
+
data.tar.gz: 6a21c00b1109d5c665a471aec4a7af7d213ded2fc9123132b6576ee65ceb1758
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1f8a810c3b7fa77e8dd2b2539238b856b9b4ea40b5df49cf2cc4af2afbd2a73d9b6f34baab614c41e7b3b8bfe1917413bfbd014f647c5b4b96e7243d5433523d
|
7
|
+
data.tar.gz: 0bef1c7dc19b4738e228985512ee6cce1915f67ee29161b743821f69366d561ed8c3c5cbd4d9e6829342108fe9f10329563770d1978a660a04c58c0d9e2c8461
|
@@ -1,41 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'etc'
|
2
3
|
|
3
4
|
module Parallel
|
5
|
+
# TODO: inline this method into parallel.rb and kill physical_processor_count in next major release
|
4
6
|
module ProcessorCount
|
5
|
-
# Number of processors seen by the OS
|
7
|
+
# Number of processors seen by the OS, used for process scheduling
|
6
8
|
def processor_count
|
7
|
-
@processor_count ||=
|
8
|
-
Etc.nprocessors
|
9
|
-
end
|
9
|
+
@processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || Etc.nprocessors)
|
10
10
|
end
|
11
11
|
|
12
12
|
# Number of physical processor cores on the current system.
|
13
|
-
#
|
14
13
|
def physical_processor_count
|
15
14
|
@physical_processor_count ||= begin
|
16
|
-
ppc =
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
15
|
+
ppc =
|
16
|
+
case RbConfig::CONFIG["target_os"]
|
17
|
+
when /darwin[12]/
|
18
|
+
IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
|
19
|
+
when /linux/
|
20
|
+
cores = {} # unique physical ID / core ID combinations
|
21
|
+
phy = 0
|
22
|
+
IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
|
23
|
+
if ln.start_with?("physical")
|
24
|
+
phy = ln[/\d+/]
|
25
|
+
elsif ln.start_with?("core")
|
26
|
+
cid = "#{phy}:#{ln[/\d+/]}"
|
27
|
+
cores[cid] = true unless cores[cid]
|
28
|
+
end
|
28
29
|
end
|
30
|
+
cores.count
|
31
|
+
when /mswin|mingw/
|
32
|
+
require 'win32ole'
|
33
|
+
result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
|
34
|
+
"select NumberOfCores from Win32_Processor"
|
35
|
+
)
|
36
|
+
result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
|
37
|
+
else
|
38
|
+
processor_count
|
29
39
|
end
|
30
|
-
cores.count
|
31
|
-
when /mswin|mingw/
|
32
|
-
require 'win32ole'
|
33
|
-
result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
|
34
|
-
"select NumberOfCores from Win32_Processor")
|
35
|
-
result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
|
36
|
-
else
|
37
|
-
processor_count
|
38
|
-
end
|
39
40
|
# fall back to logical count if physical info is invalid
|
40
41
|
ppc > 0 ? ppc : processor_count
|
41
42
|
end
|
data/lib/parallel/version.rb
CHANGED
data/lib/parallel.rb
CHANGED
@@ -1,31 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'rbconfig'
|
2
3
|
require 'parallel/version'
|
3
4
|
require 'parallel/processor_count'
|
4
5
|
|
5
6
|
module Parallel
|
6
|
-
extend
|
7
|
+
extend ProcessorCount
|
8
|
+
|
9
|
+
Stop = Object.new.freeze
|
7
10
|
|
8
11
|
class DeadWorker < StandardError
|
9
12
|
end
|
10
13
|
|
11
14
|
class Break < StandardError
|
15
|
+
attr_reader :value
|
16
|
+
|
17
|
+
def initialize(value = nil)
|
18
|
+
super()
|
19
|
+
@value = value
|
20
|
+
end
|
12
21
|
end
|
13
22
|
|
14
|
-
class Kill <
|
23
|
+
class Kill < Break
|
15
24
|
end
|
16
25
|
|
17
26
|
class UndumpableException < StandardError
|
18
27
|
attr_reader :backtrace
|
28
|
+
|
19
29
|
def initialize(original)
|
20
30
|
super "#{original.class}: #{original.message}"
|
21
31
|
@backtrace = original.backtrace
|
22
32
|
end
|
23
33
|
end
|
24
34
|
|
25
|
-
Stop = Object.new
|
26
|
-
|
27
35
|
class ExceptionWrapper
|
28
36
|
attr_reader :exception
|
37
|
+
|
29
38
|
def initialize(exception)
|
30
39
|
# Remove the bindings stack added by the better_errors gem,
|
31
40
|
# because it cannot be marshalled
|
@@ -36,7 +45,7 @@ module Parallel
|
|
36
45
|
@exception =
|
37
46
|
begin
|
38
47
|
Marshal.dump(exception) && exception
|
39
|
-
rescue
|
48
|
+
rescue StandardError
|
40
49
|
UndumpableException.new(exception)
|
41
50
|
end
|
42
51
|
end
|
@@ -45,8 +54,11 @@ module Parallel
|
|
45
54
|
class Worker
|
46
55
|
attr_reader :pid, :read, :write
|
47
56
|
attr_accessor :thread
|
57
|
+
|
48
58
|
def initialize(read, write, pid)
|
49
|
-
@read
|
59
|
+
@read = read
|
60
|
+
@write = write
|
61
|
+
@pid = pid
|
50
62
|
end
|
51
63
|
|
52
64
|
def stop
|
@@ -73,7 +85,7 @@ module Parallel
|
|
73
85
|
rescue EOFError
|
74
86
|
raise DeadWorker
|
75
87
|
end
|
76
|
-
raise result.exception if ExceptionWrapper
|
88
|
+
raise result.exception if result.is_a?(ExceptionWrapper)
|
77
89
|
result
|
78
90
|
end
|
79
91
|
|
@@ -102,7 +114,7 @@ module Parallel
|
|
102
114
|
item, index = @mutex.synchronize do
|
103
115
|
return if @stopped
|
104
116
|
item = @lambda.call
|
105
|
-
@stopped = (item ==
|
117
|
+
@stopped = (item == Stop)
|
106
118
|
return if @stopped
|
107
119
|
[item, @index += 1]
|
108
120
|
end
|
@@ -140,7 +152,7 @@ module Parallel
|
|
140
152
|
end
|
141
153
|
|
142
154
|
def queue_wrapper(array)
|
143
|
-
array.respond_to?(:num_waiting) && array.respond_to?(:pop) &&
|
155
|
+
array.respond_to?(:num_waiting) && array.respond_to?(:pop) && -> { array.pop(false) }
|
144
156
|
end
|
145
157
|
end
|
146
158
|
|
@@ -156,7 +168,7 @@ module Parallel
|
|
156
168
|
|
157
169
|
if @to_be_killed.empty?
|
158
170
|
old_interrupt = trap_interrupt(signal) do
|
159
|
-
|
171
|
+
warn 'Parallel execution interrupted, exiting ...'
|
160
172
|
@to_be_killed.flatten.each { |pid| kill(pid) }
|
161
173
|
end
|
162
174
|
end
|
@@ -200,35 +212,44 @@ module Parallel
|
|
200
212
|
end
|
201
213
|
|
202
214
|
class << self
|
203
|
-
def in_threads(options={:
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
215
|
+
def in_threads(options = { count: 2 })
|
216
|
+
threads = []
|
217
|
+
count, = extract_count_from_options(options)
|
218
|
+
|
219
|
+
Thread.handle_interrupt(Exception => :never) do
|
220
|
+
Thread.handle_interrupt(Exception => :immediate) do
|
221
|
+
count.times do |i|
|
222
|
+
threads << Thread.new { yield(i) }
|
223
|
+
end
|
224
|
+
threads.map(&:value)
|
225
|
+
end
|
226
|
+
ensure
|
227
|
+
threads.each(&:kill)
|
228
|
+
end
|
208
229
|
end
|
209
230
|
|
210
231
|
def in_processes(options = {}, &block)
|
211
232
|
count, options = extract_count_from_options(options)
|
212
233
|
count ||= processor_count
|
213
|
-
map(0...count, options.merge(:
|
234
|
+
map(0...count, options.merge(in_processes: count), &block)
|
214
235
|
end
|
215
236
|
|
216
|
-
def each(array, options={}, &block)
|
217
|
-
map(array, options.merge(:
|
237
|
+
def each(array, options = {}, &block)
|
238
|
+
map(array, options.merge(preserve_results: false), &block)
|
218
239
|
end
|
219
240
|
|
220
241
|
def any?(*args, &block)
|
221
242
|
raise "You must provide a block when calling #any?" if block.nil?
|
222
|
-
!each(*args) { |*a| raise
|
243
|
+
!each(*args) { |*a| raise Kill if block.call(*a) }
|
223
244
|
end
|
224
245
|
|
225
246
|
def all?(*args, &block)
|
226
247
|
raise "You must provide a block when calling #all?" if block.nil?
|
227
|
-
!!each(*args) { |*a| raise
|
248
|
+
!!each(*args) { |*a| raise Kill unless block.call(*a) }
|
228
249
|
end
|
229
250
|
|
230
|
-
def each_with_index(array, options={}, &block)
|
231
|
-
each(array, options.merge(:
|
251
|
+
def each_with_index(array, options = {}, &block)
|
252
|
+
each(array, options.merge(with_index: true), &block)
|
232
253
|
end
|
233
254
|
|
234
255
|
def map(source, options = {}, &block)
|
@@ -236,8 +257,8 @@ module Parallel
|
|
236
257
|
options[:mutex] = Mutex.new
|
237
258
|
|
238
259
|
if options[:in_processes] && options[:in_threads]
|
239
|
-
raise ArgumentError
|
240
|
-
elsif RUBY_PLATFORM =~ /java/
|
260
|
+
raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
|
261
|
+
elsif RUBY_PLATFORM =~ (/java/) && !(options[:in_processes])
|
241
262
|
method = :in_threads
|
242
263
|
size = options[method] || processor_count
|
243
264
|
elsif options[:in_threads]
|
@@ -259,20 +280,22 @@ module Parallel
|
|
259
280
|
options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
|
260
281
|
add_progress_bar!(job_factory, options)
|
261
282
|
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
283
|
+
result =
|
284
|
+
if size == 0
|
285
|
+
work_direct(job_factory, options, &block)
|
286
|
+
elsif method == :in_threads
|
287
|
+
work_in_threads(job_factory, options.merge(count: size), &block)
|
288
|
+
else
|
289
|
+
work_in_processes(job_factory, options.merge(count: size), &block)
|
290
|
+
end
|
291
|
+
|
292
|
+
return result.value if result.is_a?(Break)
|
293
|
+
raise result if result.is_a?(Exception)
|
294
|
+
options[:return_results] ? result : source
|
272
295
|
end
|
273
296
|
|
274
|
-
def map_with_index(array, options={}, &block)
|
275
|
-
map(array, options.merge(:
|
297
|
+
def map_with_index(array, options = {}, &block)
|
298
|
+
map(array, options.merge(with_index: true), &block)
|
276
299
|
end
|
277
300
|
|
278
301
|
def flat_map(*args, &block)
|
@@ -283,6 +306,7 @@ module Parallel
|
|
283
306
|
Thread.current[:parallel_worker_number]
|
284
307
|
end
|
285
308
|
|
309
|
+
# TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
|
286
310
|
def worker_number=(worker_num)
|
287
311
|
Thread.current[:parallel_worker_number] = worker_num
|
288
312
|
end
|
@@ -325,10 +349,10 @@ module Parallel
|
|
325
349
|
call_with_index(item, index, options, &block)
|
326
350
|
end
|
327
351
|
end
|
328
|
-
rescue
|
352
|
+
rescue StandardError
|
329
353
|
exception = $!
|
330
354
|
end
|
331
|
-
|
355
|
+
exception || results
|
332
356
|
ensure
|
333
357
|
self.worker_number = nil
|
334
358
|
end
|
@@ -349,21 +373,17 @@ module Parallel
|
|
349
373
|
call_with_index(item, index, options, &block)
|
350
374
|
end
|
351
375
|
results_mutex.synchronize { results[index] = result }
|
352
|
-
rescue
|
376
|
+
rescue StandardError
|
353
377
|
exception = $!
|
354
378
|
end
|
355
379
|
end
|
356
380
|
end
|
357
381
|
|
358
|
-
|
382
|
+
exception || results
|
359
383
|
end
|
360
384
|
|
361
385
|
def work_in_processes(job_factory, options, &blk)
|
362
|
-
workers =
|
363
|
-
[] # we create workers per job and not beforehand
|
364
|
-
else
|
365
|
-
create_workers(job_factory, options, &blk)
|
366
|
-
end
|
386
|
+
workers = create_workers(job_factory, options, &blk)
|
367
387
|
results = []
|
368
388
|
results_mutex = Mutex.new # arrays are not thread-safe
|
369
389
|
exception = nil
|
@@ -371,6 +391,8 @@ module Parallel
|
|
371
391
|
UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
|
372
392
|
in_threads(options) do |i|
|
373
393
|
worker = workers[i]
|
394
|
+
worker.thread = Thread.current
|
395
|
+
worked = false
|
374
396
|
|
375
397
|
begin
|
376
398
|
loop do
|
@@ -379,44 +401,43 @@ module Parallel
|
|
379
401
|
break unless index
|
380
402
|
|
381
403
|
if options[:isolation]
|
382
|
-
worker = replace_worker(job_factory, workers, i, options, blk)
|
404
|
+
worker = replace_worker(job_factory, workers, i, options, blk) if worked
|
405
|
+
worked = true
|
406
|
+
worker.thread = Thread.current
|
383
407
|
end
|
384
408
|
|
385
|
-
worker.thread = Thread.current
|
386
|
-
|
387
409
|
begin
|
388
410
|
result = with_instrumentation item, index, options do
|
389
411
|
worker.work(job_factory.pack(item, index))
|
390
412
|
end
|
391
413
|
results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
|
392
|
-
rescue
|
393
|
-
exception =
|
394
|
-
if
|
414
|
+
rescue StandardError => e
|
415
|
+
exception = e
|
416
|
+
if exception.is_a?(Kill)
|
395
417
|
(workers - [worker]).each do |w|
|
396
|
-
w.thread
|
418
|
+
w.thread&.kill
|
397
419
|
UserInterruptHandler.kill(w.pid)
|
398
420
|
end
|
399
421
|
end
|
400
422
|
end
|
401
423
|
end
|
402
424
|
ensure
|
403
|
-
worker.stop
|
425
|
+
worker.stop
|
404
426
|
end
|
405
427
|
end
|
406
428
|
end
|
407
|
-
|
408
|
-
handle_exception(exception, results)
|
429
|
+
exception || results
|
409
430
|
end
|
410
431
|
|
411
|
-
def replace_worker(job_factory, workers,
|
432
|
+
def replace_worker(job_factory, workers, index, options, blk)
|
412
433
|
options[:mutex].synchronize do
|
413
434
|
# old worker is no longer used ... stop it
|
414
|
-
worker = workers[
|
415
|
-
worker.stop
|
435
|
+
worker = workers[index]
|
436
|
+
worker.stop
|
416
437
|
|
417
438
|
# create a new replacement worker
|
418
439
|
running = workers - [worker]
|
419
|
-
workers[
|
440
|
+
workers[index] = worker(job_factory, options.merge(started_workers: running, worker_number: index), &blk)
|
420
441
|
end
|
421
442
|
end
|
422
443
|
|
@@ -458,21 +479,25 @@ module Parallel
|
|
458
479
|
until read.eof?
|
459
480
|
data = Marshal.load(read)
|
460
481
|
item, index = job_factory.unpack(data)
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
482
|
+
|
483
|
+
result =
|
484
|
+
begin
|
485
|
+
call_with_index(item, index, options, &block)
|
486
|
+
# https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
|
487
|
+
rescue NoMemoryError, SignalException, Interrupt, SystemExit # rubocop:disable Lint/ShadowedException
|
488
|
+
raise $!
|
489
|
+
rescue Exception # # rubocop:disable Lint/RescueException
|
490
|
+
ExceptionWrapper.new($!)
|
491
|
+
end
|
492
|
+
|
493
|
+
begin
|
494
|
+
Marshal.dump(result, write)
|
495
|
+
rescue Errno::EPIPE
|
496
|
+
return # parent thread already dead
|
465
497
|
end
|
466
|
-
Marshal.dump(result, write)
|
467
498
|
end
|
468
499
|
end
|
469
500
|
|
470
|
-
def handle_exception(exception, results)
|
471
|
-
return nil if [Parallel::Break, Parallel::Kill].include? exception.class
|
472
|
-
raise exception if exception
|
473
|
-
results
|
474
|
-
end
|
475
|
-
|
476
501
|
# options is either a Integer or a Hash with :count
|
477
502
|
def extract_count_from_options(options)
|
478
503
|
if options.is_a?(Hash)
|
@@ -487,10 +512,10 @@ module Parallel
|
|
487
512
|
def call_with_index(item, index, options, &block)
|
488
513
|
args = [item]
|
489
514
|
args << index if options[:with_index]
|
515
|
+
results = block.call(*args)
|
490
516
|
if options[:return_results]
|
491
|
-
|
517
|
+
results
|
492
518
|
else
|
493
|
-
block.call(*args)
|
494
519
|
nil # avoid GC overhead of passing large results around
|
495
520
|
end
|
496
521
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.21.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-09-13 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email: michael@grosser.it
|
@@ -23,7 +23,11 @@ files:
|
|
23
23
|
homepage: https://github.com/grosser/parallel
|
24
24
|
licenses:
|
25
25
|
- MIT
|
26
|
-
metadata:
|
26
|
+
metadata:
|
27
|
+
bug_tracker_uri: https://github.com/grosser/parallel/issues
|
28
|
+
documentation_uri: https://github.com/grosser/parallel/blob/v1.21.0/Readme.md
|
29
|
+
source_code_uri: https://github.com/grosser/parallel/tree/v1.21.0
|
30
|
+
wiki_uri: https://github.com/grosser/parallel/wiki
|
27
31
|
post_install_message:
|
28
32
|
rdoc_options: []
|
29
33
|
require_paths:
|
@@ -32,15 +36,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
32
36
|
requirements:
|
33
37
|
- - ">="
|
34
38
|
- !ruby/object:Gem::Version
|
35
|
-
version: '2.
|
39
|
+
version: '2.5'
|
36
40
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
37
41
|
requirements:
|
38
42
|
- - ">="
|
39
43
|
- !ruby/object:Gem::Version
|
40
44
|
version: '0'
|
41
45
|
requirements: []
|
42
|
-
|
43
|
-
rubygems_version: 2.7.6
|
46
|
+
rubygems_version: 3.2.16
|
44
47
|
signing_key:
|
45
48
|
specification_version: 4
|
46
49
|
summary: Run any kind of code in parallel processes
|