parallel 1.20.1 → 1.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/parallel/processor_count.rb +24 -21
- data/lib/parallel/version.rb +2 -1
- data/lib/parallel.rb +52 -43
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4dd167566b23262ceacbd640fda95c5e8d0d8149cc430f51ada53765a927cb18
|
4
|
+
data.tar.gz: 6a21c00b1109d5c665a471aec4a7af7d213ded2fc9123132b6576ee65ceb1758
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1f8a810c3b7fa77e8dd2b2539238b856b9b4ea40b5df49cf2cc4af2afbd2a73d9b6f34baab614c41e7b3b8bfe1917413bfbd014f647c5b4b96e7243d5433523d
|
7
|
+
data.tar.gz: 0bef1c7dc19b4738e228985512ee6cce1915f67ee29161b743821f69366d561ed8c3c5cbd4d9e6829342108fe9f10329563770d1978a660a04c58c0d9e2c8461
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'etc'
|
2
3
|
|
3
4
|
module Parallel
|
@@ -11,29 +12,31 @@ module Parallel
|
|
11
12
|
# Number of physical processor cores on the current system.
|
12
13
|
def physical_processor_count
|
13
14
|
@physical_processor_count ||= begin
|
14
|
-
ppc =
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
15
|
+
ppc =
|
16
|
+
case RbConfig::CONFIG["target_os"]
|
17
|
+
when /darwin[12]/
|
18
|
+
IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
|
19
|
+
when /linux/
|
20
|
+
cores = {} # unique physical ID / core ID combinations
|
21
|
+
phy = 0
|
22
|
+
IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
|
23
|
+
if ln.start_with?("physical")
|
24
|
+
phy = ln[/\d+/]
|
25
|
+
elsif ln.start_with?("core")
|
26
|
+
cid = "#{phy}:#{ln[/\d+/]}"
|
27
|
+
cores[cid] = true unless cores[cid]
|
28
|
+
end
|
26
29
|
end
|
30
|
+
cores.count
|
31
|
+
when /mswin|mingw/
|
32
|
+
require 'win32ole'
|
33
|
+
result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
|
34
|
+
"select NumberOfCores from Win32_Processor"
|
35
|
+
)
|
36
|
+
result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
|
37
|
+
else
|
38
|
+
processor_count
|
27
39
|
end
|
28
|
-
cores.count
|
29
|
-
when /mswin|mingw/
|
30
|
-
require 'win32ole'
|
31
|
-
result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
|
32
|
-
"select NumberOfCores from Win32_Processor")
|
33
|
-
result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
|
34
|
-
else
|
35
|
-
processor_count
|
36
|
-
end
|
37
40
|
# fall back to logical count if physical info is invalid
|
38
41
|
ppc > 0 ? ppc : processor_count
|
39
42
|
end
|
data/lib/parallel/version.rb
CHANGED
data/lib/parallel.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'rbconfig'
|
2
3
|
require 'parallel/version'
|
3
4
|
require 'parallel/processor_count'
|
@@ -12,7 +13,9 @@ module Parallel
|
|
12
13
|
|
13
14
|
class Break < StandardError
|
14
15
|
attr_reader :value
|
16
|
+
|
15
17
|
def initialize(value = nil)
|
18
|
+
super()
|
16
19
|
@value = value
|
17
20
|
end
|
18
21
|
end
|
@@ -22,6 +25,7 @@ module Parallel
|
|
22
25
|
|
23
26
|
class UndumpableException < StandardError
|
24
27
|
attr_reader :backtrace
|
28
|
+
|
25
29
|
def initialize(original)
|
26
30
|
super "#{original.class}: #{original.message}"
|
27
31
|
@backtrace = original.backtrace
|
@@ -30,6 +34,7 @@ module Parallel
|
|
30
34
|
|
31
35
|
class ExceptionWrapper
|
32
36
|
attr_reader :exception
|
37
|
+
|
33
38
|
def initialize(exception)
|
34
39
|
# Remove the bindings stack added by the better_errors gem,
|
35
40
|
# because it cannot be marshalled
|
@@ -40,7 +45,7 @@ module Parallel
|
|
40
45
|
@exception =
|
41
46
|
begin
|
42
47
|
Marshal.dump(exception) && exception
|
43
|
-
rescue
|
48
|
+
rescue StandardError
|
44
49
|
UndumpableException.new(exception)
|
45
50
|
end
|
46
51
|
end
|
@@ -49,8 +54,11 @@ module Parallel
|
|
49
54
|
class Worker
|
50
55
|
attr_reader :pid, :read, :write
|
51
56
|
attr_accessor :thread
|
57
|
+
|
52
58
|
def initialize(read, write, pid)
|
53
|
-
@read
|
59
|
+
@read = read
|
60
|
+
@write = write
|
61
|
+
@pid = pid
|
54
62
|
end
|
55
63
|
|
56
64
|
def stop
|
@@ -77,7 +85,7 @@ module Parallel
|
|
77
85
|
rescue EOFError
|
78
86
|
raise DeadWorker
|
79
87
|
end
|
80
|
-
raise result.exception if ExceptionWrapper
|
88
|
+
raise result.exception if result.is_a?(ExceptionWrapper)
|
81
89
|
result
|
82
90
|
end
|
83
91
|
|
@@ -144,7 +152,7 @@ module Parallel
|
|
144
152
|
end
|
145
153
|
|
146
154
|
def queue_wrapper(array)
|
147
|
-
array.respond_to?(:num_waiting) && array.respond_to?(:pop) &&
|
155
|
+
array.respond_to?(:num_waiting) && array.respond_to?(:pop) && -> { array.pop(false) }
|
148
156
|
end
|
149
157
|
end
|
150
158
|
|
@@ -160,7 +168,7 @@ module Parallel
|
|
160
168
|
|
161
169
|
if @to_be_killed.empty?
|
162
170
|
old_interrupt = trap_interrupt(signal) do
|
163
|
-
|
171
|
+
warn 'Parallel execution interrupted, exiting ...'
|
164
172
|
@to_be_killed.flatten.each { |pid| kill(pid) }
|
165
173
|
end
|
166
174
|
end
|
@@ -204,32 +212,30 @@ module Parallel
|
|
204
212
|
end
|
205
213
|
|
206
214
|
class << self
|
207
|
-
def in_threads(options={:
|
215
|
+
def in_threads(options = { count: 2 })
|
208
216
|
threads = []
|
209
|
-
count,
|
217
|
+
count, = extract_count_from_options(options)
|
210
218
|
|
211
219
|
Thread.handle_interrupt(Exception => :never) do
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
threads << Thread.new { yield(i) }
|
216
|
-
end
|
217
|
-
threads.map(&:value)
|
220
|
+
Thread.handle_interrupt(Exception => :immediate) do
|
221
|
+
count.times do |i|
|
222
|
+
threads << Thread.new { yield(i) }
|
218
223
|
end
|
219
|
-
|
220
|
-
threads.each(&:kill)
|
224
|
+
threads.map(&:value)
|
221
225
|
end
|
226
|
+
ensure
|
227
|
+
threads.each(&:kill)
|
222
228
|
end
|
223
229
|
end
|
224
230
|
|
225
231
|
def in_processes(options = {}, &block)
|
226
232
|
count, options = extract_count_from_options(options)
|
227
233
|
count ||= processor_count
|
228
|
-
map(0...count, options.merge(:
|
234
|
+
map(0...count, options.merge(in_processes: count), &block)
|
229
235
|
end
|
230
236
|
|
231
|
-
def each(array, options={}, &block)
|
232
|
-
map(array, options.merge(:
|
237
|
+
def each(array, options = {}, &block)
|
238
|
+
map(array, options.merge(preserve_results: false), &block)
|
233
239
|
end
|
234
240
|
|
235
241
|
def any?(*args, &block)
|
@@ -242,8 +248,8 @@ module Parallel
|
|
242
248
|
!!each(*args) { |*a| raise Kill unless block.call(*a) }
|
243
249
|
end
|
244
250
|
|
245
|
-
def each_with_index(array, options={}, &block)
|
246
|
-
each(array, options.merge(:
|
251
|
+
def each_with_index(array, options = {}, &block)
|
252
|
+
each(array, options.merge(with_index: true), &block)
|
247
253
|
end
|
248
254
|
|
249
255
|
def map(source, options = {}, &block)
|
@@ -251,8 +257,8 @@ module Parallel
|
|
251
257
|
options[:mutex] = Mutex.new
|
252
258
|
|
253
259
|
if options[:in_processes] && options[:in_threads]
|
254
|
-
raise ArgumentError
|
255
|
-
elsif RUBY_PLATFORM =~ /java/
|
260
|
+
raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
|
261
|
+
elsif RUBY_PLATFORM =~ (/java/) && !(options[:in_processes])
|
256
262
|
method = :in_threads
|
257
263
|
size = options[method] || processor_count
|
258
264
|
elsif options[:in_threads]
|
@@ -278,9 +284,9 @@ module Parallel
|
|
278
284
|
if size == 0
|
279
285
|
work_direct(job_factory, options, &block)
|
280
286
|
elsif method == :in_threads
|
281
|
-
work_in_threads(job_factory, options.merge(:
|
287
|
+
work_in_threads(job_factory, options.merge(count: size), &block)
|
282
288
|
else
|
283
|
-
work_in_processes(job_factory, options.merge(:
|
289
|
+
work_in_processes(job_factory, options.merge(count: size), &block)
|
284
290
|
end
|
285
291
|
|
286
292
|
return result.value if result.is_a?(Break)
|
@@ -288,8 +294,8 @@ module Parallel
|
|
288
294
|
options[:return_results] ? result : source
|
289
295
|
end
|
290
296
|
|
291
|
-
def map_with_index(array, options={}, &block)
|
292
|
-
map(array, options.merge(:
|
297
|
+
def map_with_index(array, options = {}, &block)
|
298
|
+
map(array, options.merge(with_index: true), &block)
|
293
299
|
end
|
294
300
|
|
295
301
|
def flat_map(*args, &block)
|
@@ -343,7 +349,7 @@ module Parallel
|
|
343
349
|
call_with_index(item, index, options, &block)
|
344
350
|
end
|
345
351
|
end
|
346
|
-
rescue
|
352
|
+
rescue StandardError
|
347
353
|
exception = $!
|
348
354
|
end
|
349
355
|
exception || results
|
@@ -367,7 +373,7 @@ module Parallel
|
|
367
373
|
call_with_index(item, index, options, &block)
|
368
374
|
end
|
369
375
|
results_mutex.synchronize { results[index] = result }
|
370
|
-
rescue
|
376
|
+
rescue StandardError
|
371
377
|
exception = $!
|
372
378
|
end
|
373
379
|
end
|
@@ -407,9 +413,9 @@ module Parallel
|
|
407
413
|
results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
|
408
414
|
rescue StandardError => e
|
409
415
|
exception = e
|
410
|
-
if Kill
|
416
|
+
if exception.is_a?(Kill)
|
411
417
|
(workers - [worker]).each do |w|
|
412
|
-
w.thread
|
418
|
+
w.thread&.kill
|
413
419
|
UserInterruptHandler.kill(w.pid)
|
414
420
|
end
|
415
421
|
end
|
@@ -423,15 +429,15 @@ module Parallel
|
|
423
429
|
exception || results
|
424
430
|
end
|
425
431
|
|
426
|
-
def replace_worker(job_factory, workers,
|
432
|
+
def replace_worker(job_factory, workers, index, options, blk)
|
427
433
|
options[:mutex].synchronize do
|
428
434
|
# old worker is no longer used ... stop it
|
429
|
-
worker = workers[
|
435
|
+
worker = workers[index]
|
430
436
|
worker.stop
|
431
437
|
|
432
438
|
# create a new replacement worker
|
433
439
|
running = workers - [worker]
|
434
|
-
workers[
|
440
|
+
workers[index] = worker(job_factory, options.merge(started_workers: running, worker_number: index), &blk)
|
435
441
|
end
|
436
442
|
end
|
437
443
|
|
@@ -473,14 +479,17 @@ module Parallel
|
|
473
479
|
until read.eof?
|
474
480
|
data = Marshal.load(read)
|
475
481
|
item, index = job_factory.unpack(data)
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
482
|
+
|
483
|
+
result =
|
484
|
+
begin
|
485
|
+
call_with_index(item, index, options, &block)
|
486
|
+
# https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
|
487
|
+
rescue NoMemoryError, SignalException, Interrupt, SystemExit # rubocop:disable Lint/ShadowedException
|
488
|
+
raise $!
|
489
|
+
rescue Exception # # rubocop:disable Lint/RescueException
|
490
|
+
ExceptionWrapper.new($!)
|
491
|
+
end
|
492
|
+
|
484
493
|
begin
|
485
494
|
Marshal.dump(result, write)
|
486
495
|
rescue Errno::EPIPE
|
@@ -503,10 +512,10 @@ module Parallel
|
|
503
512
|
def call_with_index(item, index, options, &block)
|
504
513
|
args = [item]
|
505
514
|
args << index if options[:with_index]
|
515
|
+
results = block.call(*args)
|
506
516
|
if options[:return_results]
|
507
|
-
|
517
|
+
results
|
508
518
|
else
|
509
|
-
block.call(*args)
|
510
519
|
nil # avoid GC overhead of passing large results around
|
511
520
|
end
|
512
521
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.21.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-09-13 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email: michael@grosser.it
|
@@ -25,8 +25,8 @@ licenses:
|
|
25
25
|
- MIT
|
26
26
|
metadata:
|
27
27
|
bug_tracker_uri: https://github.com/grosser/parallel/issues
|
28
|
-
documentation_uri: https://github.com/grosser/parallel/blob/v1.
|
29
|
-
source_code_uri: https://github.com/grosser/parallel/tree/v1.
|
28
|
+
documentation_uri: https://github.com/grosser/parallel/blob/v1.21.0/Readme.md
|
29
|
+
source_code_uri: https://github.com/grosser/parallel/tree/v1.21.0
|
30
30
|
wiki_uri: https://github.com/grosser/parallel/wiki
|
31
31
|
post_install_message:
|
32
32
|
rdoc_options: []
|
@@ -36,14 +36,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
37
37
|
- - ">="
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '2.
|
39
|
+
version: '2.5'
|
40
40
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
41
41
|
requirements:
|
42
42
|
- - ">="
|
43
43
|
- !ruby/object:Gem::Version
|
44
44
|
version: '0'
|
45
45
|
requirements: []
|
46
|
-
rubygems_version: 3.
|
46
|
+
rubygems_version: 3.2.16
|
47
47
|
signing_key:
|
48
48
|
specification_version: 4
|
49
49
|
summary: Run any kind of code in parallel processes
|