parallel 1.20.1 → 1.22.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/parallel/processor_count.rb +25 -23
- data/lib/parallel/version.rb +2 -1
- data/lib/parallel.rb +136 -47
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7efbfe49c3df93ae88464b4bd4ce85d1aaccc9c173e1a81e7cf9100fe182982c
|
4
|
+
data.tar.gz: 7e82ff83bd44c96da760d83ab665baed41bc62d53f010987951c805cf41723e4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a81cae96dd4dcfa1d287ce3e5893c830de3cb8e305236db4170db81f94de021f187da0d21462b7df5e99310002c14984fbb00aef818936c13d0ae834c1c1b2e7
|
7
|
+
data.tar.gz: 79c099a5f1f8bdad3d9f187829213d242087779605f14fb421f0172c1de1a8f853b4f13045935410a989d845d08e736e9552c8e3b9a6c3763d2dfdd27ede7daf
|
@@ -1,39 +1,41 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
3
2
|
module Parallel
|
4
3
|
# TODO: inline this method into parallel.rb and kill physical_processor_count in next major release
|
5
4
|
module ProcessorCount
|
6
5
|
# Number of processors seen by the OS, used for process scheduling
|
7
6
|
def processor_count
|
7
|
+
require 'etc'
|
8
8
|
@processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || Etc.nprocessors)
|
9
9
|
end
|
10
10
|
|
11
11
|
# Number of physical processor cores on the current system.
|
12
12
|
def physical_processor_count
|
13
13
|
@physical_processor_count ||= begin
|
14
|
-
ppc =
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
14
|
+
ppc =
|
15
|
+
case RbConfig::CONFIG["target_os"]
|
16
|
+
when /darwin[12]/
|
17
|
+
IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
|
18
|
+
when /linux/
|
19
|
+
cores = {} # unique physical ID / core ID combinations
|
20
|
+
phy = 0
|
21
|
+
File.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
|
22
|
+
if ln.start_with?("physical")
|
23
|
+
phy = ln[/\d+/]
|
24
|
+
elsif ln.start_with?("core")
|
25
|
+
cid = "#{phy}:#{ln[/\d+/]}"
|
26
|
+
cores[cid] = true unless cores[cid]
|
27
|
+
end
|
26
28
|
end
|
29
|
+
cores.count
|
30
|
+
when /mswin|mingw/
|
31
|
+
require 'win32ole'
|
32
|
+
result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
|
33
|
+
"select NumberOfCores from Win32_Processor"
|
34
|
+
)
|
35
|
+
result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
|
36
|
+
else
|
37
|
+
processor_count
|
27
38
|
end
|
28
|
-
cores.count
|
29
|
-
when /mswin|mingw/
|
30
|
-
require 'win32ole'
|
31
|
-
result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
|
32
|
-
"select NumberOfCores from Win32_Processor")
|
33
|
-
result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
|
34
|
-
else
|
35
|
-
processor_count
|
36
|
-
end
|
37
39
|
# fall back to logical count if physical info is invalid
|
38
40
|
ppc > 0 ? ppc : processor_count
|
39
41
|
end
|
data/lib/parallel/version.rb
CHANGED
data/lib/parallel.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'rbconfig'
|
2
3
|
require 'parallel/version'
|
3
4
|
require 'parallel/processor_count'
|
@@ -12,7 +13,9 @@ module Parallel
|
|
12
13
|
|
13
14
|
class Break < StandardError
|
14
15
|
attr_reader :value
|
16
|
+
|
15
17
|
def initialize(value = nil)
|
18
|
+
super()
|
16
19
|
@value = value
|
17
20
|
end
|
18
21
|
end
|
@@ -22,6 +25,7 @@ module Parallel
|
|
22
25
|
|
23
26
|
class UndumpableException < StandardError
|
24
27
|
attr_reader :backtrace
|
28
|
+
|
25
29
|
def initialize(original)
|
26
30
|
super "#{original.class}: #{original.message}"
|
27
31
|
@backtrace = original.backtrace
|
@@ -30,6 +34,7 @@ module Parallel
|
|
30
34
|
|
31
35
|
class ExceptionWrapper
|
32
36
|
attr_reader :exception
|
37
|
+
|
33
38
|
def initialize(exception)
|
34
39
|
# Remove the bindings stack added by the better_errors gem,
|
35
40
|
# because it cannot be marshalled
|
@@ -40,7 +45,7 @@ module Parallel
|
|
40
45
|
@exception =
|
41
46
|
begin
|
42
47
|
Marshal.dump(exception) && exception
|
43
|
-
rescue
|
48
|
+
rescue StandardError
|
44
49
|
UndumpableException.new(exception)
|
45
50
|
end
|
46
51
|
end
|
@@ -49,8 +54,11 @@ module Parallel
|
|
49
54
|
class Worker
|
50
55
|
attr_reader :pid, :read, :write
|
51
56
|
attr_accessor :thread
|
57
|
+
|
52
58
|
def initialize(read, write, pid)
|
53
|
-
@read
|
59
|
+
@read = read
|
60
|
+
@write = write
|
61
|
+
@pid = pid
|
54
62
|
end
|
55
63
|
|
56
64
|
def stop
|
@@ -77,7 +85,7 @@ module Parallel
|
|
77
85
|
rescue EOFError
|
78
86
|
raise DeadWorker
|
79
87
|
end
|
80
|
-
raise result.exception if ExceptionWrapper
|
88
|
+
raise result.exception if result.is_a?(ExceptionWrapper)
|
81
89
|
result
|
82
90
|
end
|
83
91
|
|
@@ -144,7 +152,7 @@ module Parallel
|
|
144
152
|
end
|
145
153
|
|
146
154
|
def queue_wrapper(array)
|
147
|
-
array.respond_to?(:num_waiting) && array.respond_to?(:pop) &&
|
155
|
+
array.respond_to?(:num_waiting) && array.respond_to?(:pop) && -> { array.pop(false) }
|
148
156
|
end
|
149
157
|
end
|
150
158
|
|
@@ -160,7 +168,7 @@ module Parallel
|
|
160
168
|
|
161
169
|
if @to_be_killed.empty?
|
162
170
|
old_interrupt = trap_interrupt(signal) do
|
163
|
-
|
171
|
+
warn 'Parallel execution interrupted, exiting ...'
|
164
172
|
@to_be_killed.flatten.each { |pid| kill(pid) }
|
165
173
|
end
|
166
174
|
end
|
@@ -204,32 +212,30 @@ module Parallel
|
|
204
212
|
end
|
205
213
|
|
206
214
|
class << self
|
207
|
-
def in_threads(options={:
|
215
|
+
def in_threads(options = { count: 2 })
|
208
216
|
threads = []
|
209
|
-
count,
|
217
|
+
count, = extract_count_from_options(options)
|
210
218
|
|
211
219
|
Thread.handle_interrupt(Exception => :never) do
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
threads << Thread.new { yield(i) }
|
216
|
-
end
|
217
|
-
threads.map(&:value)
|
220
|
+
Thread.handle_interrupt(Exception => :immediate) do
|
221
|
+
count.times do |i|
|
222
|
+
threads << Thread.new { yield(i) }
|
218
223
|
end
|
219
|
-
|
220
|
-
threads.each(&:kill)
|
224
|
+
threads.map(&:value)
|
221
225
|
end
|
226
|
+
ensure
|
227
|
+
threads.each(&:kill)
|
222
228
|
end
|
223
229
|
end
|
224
230
|
|
225
231
|
def in_processes(options = {}, &block)
|
226
232
|
count, options = extract_count_from_options(options)
|
227
233
|
count ||= processor_count
|
228
|
-
map(0...count, options.merge(:
|
234
|
+
map(0...count, options.merge(in_processes: count), &block)
|
229
235
|
end
|
230
236
|
|
231
|
-
def each(array, options={}, &block)
|
232
|
-
map(array, options.merge(:
|
237
|
+
def each(array, options = {}, &block)
|
238
|
+
map(array, options.merge(preserve_results: false), &block)
|
233
239
|
end
|
234
240
|
|
235
241
|
def any?(*args, &block)
|
@@ -242,8 +248,8 @@ module Parallel
|
|
242
248
|
!!each(*args) { |*a| raise Kill unless block.call(*a) }
|
243
249
|
end
|
244
250
|
|
245
|
-
def each_with_index(array, options={}, &block)
|
246
|
-
each(array, options.merge(:
|
251
|
+
def each_with_index(array, options = {}, &block)
|
252
|
+
each(array, options.merge(with_index: true), &block)
|
247
253
|
end
|
248
254
|
|
249
255
|
def map(source, options = {}, &block)
|
@@ -251,13 +257,16 @@ module Parallel
|
|
251
257
|
options[:mutex] = Mutex.new
|
252
258
|
|
253
259
|
if options[:in_processes] && options[:in_threads]
|
254
|
-
raise ArgumentError
|
255
|
-
elsif RUBY_PLATFORM =~ /java/
|
260
|
+
raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
|
261
|
+
elsif RUBY_PLATFORM =~ (/java/) && !(options[:in_processes])
|
256
262
|
method = :in_threads
|
257
263
|
size = options[method] || processor_count
|
258
264
|
elsif options[:in_threads]
|
259
265
|
method = :in_threads
|
260
266
|
size = options[method]
|
267
|
+
elsif options[:in_ractors]
|
268
|
+
method = :in_ractors
|
269
|
+
size = options[method]
|
261
270
|
else
|
262
271
|
method = :in_processes
|
263
272
|
if Process.respond_to?(:fork)
|
@@ -278,9 +287,11 @@ module Parallel
|
|
278
287
|
if size == 0
|
279
288
|
work_direct(job_factory, options, &block)
|
280
289
|
elsif method == :in_threads
|
281
|
-
work_in_threads(job_factory, options.merge(:
|
290
|
+
work_in_threads(job_factory, options.merge(count: size), &block)
|
291
|
+
elsif method == :in_ractors
|
292
|
+
work_in_ractors(job_factory, options.merge(count: size), &block)
|
282
293
|
else
|
283
|
-
work_in_processes(job_factory, options.merge(:
|
294
|
+
work_in_processes(job_factory, options.merge(count: size), &block)
|
284
295
|
end
|
285
296
|
|
286
297
|
return result.value if result.is_a?(Break)
|
@@ -288,8 +299,8 @@ module Parallel
|
|
288
299
|
options[:return_results] ? result : source
|
289
300
|
end
|
290
301
|
|
291
|
-
def map_with_index(array, options={}, &block)
|
292
|
-
map(array, options.merge(:
|
302
|
+
def map_with_index(array, options = {}, &block)
|
303
|
+
map(array, options.merge(with_index: true), &block)
|
293
304
|
end
|
294
305
|
|
295
306
|
def flat_map(*args, &block)
|
@@ -343,7 +354,7 @@ module Parallel
|
|
343
354
|
call_with_index(item, index, options, &block)
|
344
355
|
end
|
345
356
|
end
|
346
|
-
rescue
|
357
|
+
rescue StandardError
|
347
358
|
exception = $!
|
348
359
|
end
|
349
360
|
exception || results
|
@@ -367,7 +378,7 @@ module Parallel
|
|
367
378
|
call_with_index(item, index, options, &block)
|
368
379
|
end
|
369
380
|
results_mutex.synchronize { results[index] = result }
|
370
|
-
rescue
|
381
|
+
rescue StandardError
|
371
382
|
exception = $!
|
372
383
|
end
|
373
384
|
end
|
@@ -376,6 +387,72 @@ module Parallel
|
|
376
387
|
exception || results
|
377
388
|
end
|
378
389
|
|
390
|
+
def work_in_ractors(job_factory, options)
|
391
|
+
exception = nil
|
392
|
+
results = []
|
393
|
+
results_mutex = Mutex.new # arrays are not thread-safe on jRuby
|
394
|
+
|
395
|
+
callback = options[:ractor]
|
396
|
+
if block_given? || !callback
|
397
|
+
raise ArgumentError, "pass the code you want to execute as `ractor: [ClassName, :method_name]`"
|
398
|
+
end
|
399
|
+
|
400
|
+
# build
|
401
|
+
ractors = Array.new(options.fetch(:count)) do
|
402
|
+
Ractor.new do
|
403
|
+
loop do
|
404
|
+
got = receive
|
405
|
+
(klass, method_name), item, index = got
|
406
|
+
break if index == :break
|
407
|
+
begin
|
408
|
+
Ractor.yield [nil, klass.send(method_name, item), item, index]
|
409
|
+
rescue StandardError => e
|
410
|
+
Ractor.yield [e, nil, item, index]
|
411
|
+
end
|
412
|
+
end
|
413
|
+
end
|
414
|
+
end
|
415
|
+
|
416
|
+
# start
|
417
|
+
ractors.dup.each do |ractor|
|
418
|
+
if set = job_factory.next
|
419
|
+
item, index = set
|
420
|
+
instrument_start item, index, options
|
421
|
+
ractor.send [callback, item, index]
|
422
|
+
else
|
423
|
+
ractor.send([[nil, nil], nil, :break]) # stop the ractor
|
424
|
+
ractors.delete ractor
|
425
|
+
end
|
426
|
+
end
|
427
|
+
|
428
|
+
# replace with new items
|
429
|
+
while set = job_factory.next
|
430
|
+
item_next, index_next = set
|
431
|
+
done, (exception, result, item, index) = Ractor.select(*ractors)
|
432
|
+
if exception
|
433
|
+
ractors.delete done
|
434
|
+
break
|
435
|
+
end
|
436
|
+
instrument_finish item, index, result, options
|
437
|
+
results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
|
438
|
+
|
439
|
+
instrument_start item_next, index_next, options
|
440
|
+
done.send([callback, item_next, index_next])
|
441
|
+
end
|
442
|
+
|
443
|
+
# finish
|
444
|
+
ractors.each do |ractor|
|
445
|
+
(new_exception, result, item, index) = ractor.take
|
446
|
+
exception ||= new_exception
|
447
|
+
next if new_exception
|
448
|
+
instrument_finish item, index, result, options
|
449
|
+
results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
|
450
|
+
ractor.send([[nil, nil], nil, :break]) # stop the ractor
|
451
|
+
end
|
452
|
+
|
453
|
+
exception || results
|
454
|
+
end
|
455
|
+
|
379
456
|
def work_in_processes(job_factory, options, &blk)
|
380
457
|
workers = create_workers(job_factory, options, &blk)
|
381
458
|
results = []
|
@@ -407,9 +484,9 @@ module Parallel
|
|
407
484
|
results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
|
408
485
|
rescue StandardError => e
|
409
486
|
exception = e
|
410
|
-
if Kill
|
487
|
+
if exception.is_a?(Kill)
|
411
488
|
(workers - [worker]).each do |w|
|
412
|
-
w.thread
|
489
|
+
w.thread&.kill
|
413
490
|
UserInterruptHandler.kill(w.pid)
|
414
491
|
end
|
415
492
|
end
|
@@ -420,18 +497,19 @@ module Parallel
|
|
420
497
|
end
|
421
498
|
end
|
422
499
|
end
|
500
|
+
|
423
501
|
exception || results
|
424
502
|
end
|
425
503
|
|
426
|
-
def replace_worker(job_factory, workers,
|
504
|
+
def replace_worker(job_factory, workers, index, options, blk)
|
427
505
|
options[:mutex].synchronize do
|
428
506
|
# old worker is no longer used ... stop it
|
429
|
-
worker = workers[
|
507
|
+
worker = workers[index]
|
430
508
|
worker.stop
|
431
509
|
|
432
510
|
# create a new replacement worker
|
433
511
|
running = workers - [worker]
|
434
|
-
workers[
|
512
|
+
workers[index] = worker(job_factory, options.merge(started_workers: running, worker_number: index), &blk)
|
435
513
|
end
|
436
514
|
end
|
437
515
|
|
@@ -473,14 +551,17 @@ module Parallel
|
|
473
551
|
until read.eof?
|
474
552
|
data = Marshal.load(read)
|
475
553
|
item, index = job_factory.unpack(data)
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
554
|
+
|
555
|
+
result =
|
556
|
+
begin
|
557
|
+
call_with_index(item, index, options, &block)
|
558
|
+
# https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
|
559
|
+
rescue NoMemoryError, SignalException, Interrupt, SystemExit # rubocop:disable Lint/ShadowedException
|
560
|
+
raise $!
|
561
|
+
rescue Exception # # rubocop:disable Lint/RescueException
|
562
|
+
ExceptionWrapper.new($!)
|
563
|
+
end
|
564
|
+
|
484
565
|
begin
|
485
566
|
Marshal.dump(result, write)
|
486
567
|
rescue Errno::EPIPE
|
@@ -503,21 +584,29 @@ module Parallel
|
|
503
584
|
def call_with_index(item, index, options, &block)
|
504
585
|
args = [item]
|
505
586
|
args << index if options[:with_index]
|
587
|
+
results = block.call(*args)
|
506
588
|
if options[:return_results]
|
507
|
-
|
589
|
+
results
|
508
590
|
else
|
509
|
-
block.call(*args)
|
510
591
|
nil # avoid GC overhead of passing large results around
|
511
592
|
end
|
512
593
|
end
|
513
594
|
|
514
595
|
def with_instrumentation(item, index, options)
|
515
|
-
|
516
|
-
on_finish = options[:finish]
|
517
|
-
options[:mutex].synchronize { on_start.call(item, index) } if on_start
|
596
|
+
instrument_start(item, index, options)
|
518
597
|
result = yield
|
519
|
-
|
598
|
+
instrument_finish(item, index, result, options)
|
520
599
|
result unless options[:preserve_results] == false
|
521
600
|
end
|
601
|
+
|
602
|
+
def instrument_finish(item, index, result, options)
|
603
|
+
return unless on_finish = options[:finish]
|
604
|
+
options[:mutex].synchronize { on_finish.call(item, index, result) }
|
605
|
+
end
|
606
|
+
|
607
|
+
def instrument_start(item, index, options)
|
608
|
+
return unless on_start = options[:start]
|
609
|
+
options[:mutex].synchronize { on_start.call(item, index) }
|
610
|
+
end
|
522
611
|
end
|
523
612
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.22.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-03-25 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email: michael@grosser.it
|
@@ -25,8 +25,8 @@ licenses:
|
|
25
25
|
- MIT
|
26
26
|
metadata:
|
27
27
|
bug_tracker_uri: https://github.com/grosser/parallel/issues
|
28
|
-
documentation_uri: https://github.com/grosser/parallel/blob/v1.
|
29
|
-
source_code_uri: https://github.com/grosser/parallel/tree/v1.
|
28
|
+
documentation_uri: https://github.com/grosser/parallel/blob/v1.22.1/Readme.md
|
29
|
+
source_code_uri: https://github.com/grosser/parallel/tree/v1.22.1
|
30
30
|
wiki_uri: https://github.com/grosser/parallel/wiki
|
31
31
|
post_install_message:
|
32
32
|
rdoc_options: []
|
@@ -36,14 +36,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
37
37
|
- - ">="
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '2.
|
39
|
+
version: '2.5'
|
40
40
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
41
41
|
requirements:
|
42
42
|
- - ">="
|
43
43
|
- !ruby/object:Gem::Version
|
44
44
|
version: '0'
|
45
45
|
requirements: []
|
46
|
-
rubygems_version: 3.1.
|
46
|
+
rubygems_version: 3.1.6
|
47
47
|
signing_key:
|
48
48
|
specification_version: 4
|
49
49
|
summary: Run any kind of code in parallel processes
|