parallel 1.21.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/parallel/version.rb +1 -1
- data/lib/parallel.rb +127 -7
- metadata +5 -6
- data/lib/parallel/processor_count.rb +0 -45
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7b8bb887652e89a339de17f31e8482aec8664426b2b12eb11566779f53153a56
|
4
|
+
data.tar.gz: e5fa1401f5748de7216a6d973d1ca1a2af514caeaec1dacafb153d2a5538334d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 481a4b04da349a2eb88cdcab724dc3c47efe757dcb61889a3105d0097fb3255ff6283470f6c2bc636c1bd053e274516fddfa33f8d362e8208f6f57638864ca48
|
7
|
+
data.tar.gz: 9093769c22e258cb921c1ab6ead1c7083fcc1d315b7bc268dff807c0fe1c4468b4316171e5f837283d7387ecdb01284f55763b60912c4bce7548aa9c869cc57b
|
data/lib/parallel/version.rb
CHANGED
data/lib/parallel.rb
CHANGED
@@ -1,11 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
require 'rbconfig'
|
3
3
|
require 'parallel/version'
|
4
|
-
require 'parallel/processor_count'
|
5
4
|
|
6
5
|
module Parallel
|
7
|
-
extend ProcessorCount
|
8
|
-
|
9
6
|
Stop = Object.new.freeze
|
10
7
|
|
11
8
|
class DeadWorker < StandardError
|
@@ -264,6 +261,9 @@ module Parallel
|
|
264
261
|
elsif options[:in_threads]
|
265
262
|
method = :in_threads
|
266
263
|
size = options[method]
|
264
|
+
elsif options[:in_ractors]
|
265
|
+
method = :in_ractors
|
266
|
+
size = options[method]
|
267
267
|
else
|
268
268
|
method = :in_processes
|
269
269
|
if Process.respond_to?(:fork)
|
@@ -285,6 +285,8 @@ module Parallel
|
|
285
285
|
work_direct(job_factory, options, &block)
|
286
286
|
elsif method == :in_threads
|
287
287
|
work_in_threads(job_factory, options.merge(count: size), &block)
|
288
|
+
elsif method == :in_ractors
|
289
|
+
work_in_ractors(job_factory, options.merge(count: size), &block)
|
288
290
|
else
|
289
291
|
work_in_processes(job_factory, options.merge(count: size), &block)
|
290
292
|
end
|
@@ -302,6 +304,49 @@ module Parallel
|
|
302
304
|
map(*args, &block).flatten(1)
|
303
305
|
end
|
304
306
|
|
307
|
+
def filter_map(*args, &block)
|
308
|
+
map(*args, &block).compact
|
309
|
+
end
|
310
|
+
|
311
|
+
# Number of physical processor cores on the current system.
|
312
|
+
def physical_processor_count
|
313
|
+
@physical_processor_count ||= begin
|
314
|
+
ppc =
|
315
|
+
case RbConfig::CONFIG["target_os"]
|
316
|
+
when /darwin[12]/
|
317
|
+
IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
|
318
|
+
when /linux/
|
319
|
+
cores = {} # unique physical ID / core ID combinations
|
320
|
+
phy = 0
|
321
|
+
File.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
|
322
|
+
if ln.start_with?("physical")
|
323
|
+
phy = ln[/\d+/]
|
324
|
+
elsif ln.start_with?("core")
|
325
|
+
cid = "#{phy}:#{ln[/\d+/]}"
|
326
|
+
cores[cid] = true unless cores[cid]
|
327
|
+
end
|
328
|
+
end
|
329
|
+
cores.count
|
330
|
+
when /mswin|mingw/
|
331
|
+
require 'win32ole'
|
332
|
+
result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
|
333
|
+
"select NumberOfCores from Win32_Processor"
|
334
|
+
)
|
335
|
+
result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
|
336
|
+
else
|
337
|
+
processor_count
|
338
|
+
end
|
339
|
+
# fall back to logical count if physical info is invalid
|
340
|
+
ppc > 0 ? ppc : processor_count
|
341
|
+
end
|
342
|
+
end
|
343
|
+
|
344
|
+
# Number of processors seen by the OS, used for process scheduling
|
345
|
+
def processor_count
|
346
|
+
require 'etc'
|
347
|
+
@processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || Etc.nprocessors)
|
348
|
+
end
|
349
|
+
|
305
350
|
def worker_number
|
306
351
|
Thread.current[:parallel_worker_number]
|
307
352
|
end
|
@@ -382,6 +427,72 @@ module Parallel
|
|
382
427
|
exception || results
|
383
428
|
end
|
384
429
|
|
430
|
+
def work_in_ractors(job_factory, options)
|
431
|
+
exception = nil
|
432
|
+
results = []
|
433
|
+
results_mutex = Mutex.new # arrays are not thread-safe on jRuby
|
434
|
+
|
435
|
+
callback = options[:ractor]
|
436
|
+
if block_given? || !callback
|
437
|
+
raise ArgumentError, "pass the code you want to execute as `ractor: [ClassName, :method_name]`"
|
438
|
+
end
|
439
|
+
|
440
|
+
# build
|
441
|
+
ractors = Array.new(options.fetch(:count)) do
|
442
|
+
Ractor.new do
|
443
|
+
loop do
|
444
|
+
got = receive
|
445
|
+
(klass, method_name), item, index = got
|
446
|
+
break if index == :break
|
447
|
+
begin
|
448
|
+
Ractor.yield [nil, klass.send(method_name, item), item, index]
|
449
|
+
rescue StandardError => e
|
450
|
+
Ractor.yield [e, nil, item, index]
|
451
|
+
end
|
452
|
+
end
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
456
|
+
# start
|
457
|
+
ractors.dup.each do |ractor|
|
458
|
+
if set = job_factory.next
|
459
|
+
item, index = set
|
460
|
+
instrument_start item, index, options
|
461
|
+
ractor.send [callback, item, index]
|
462
|
+
else
|
463
|
+
ractor.send([[nil, nil], nil, :break]) # stop the ractor
|
464
|
+
ractors.delete ractor
|
465
|
+
end
|
466
|
+
end
|
467
|
+
|
468
|
+
# replace with new items
|
469
|
+
while set = job_factory.next
|
470
|
+
item_next, index_next = set
|
471
|
+
done, (exception, result, item, index) = Ractor.select(*ractors)
|
472
|
+
if exception
|
473
|
+
ractors.delete done
|
474
|
+
break
|
475
|
+
end
|
476
|
+
instrument_finish item, index, result, options
|
477
|
+
results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
|
478
|
+
|
479
|
+
instrument_start item_next, index_next, options
|
480
|
+
done.send([callback, item_next, index_next])
|
481
|
+
end
|
482
|
+
|
483
|
+
# finish
|
484
|
+
ractors.each do |ractor|
|
485
|
+
(new_exception, result, item, index) = ractor.take
|
486
|
+
exception ||= new_exception
|
487
|
+
next if new_exception
|
488
|
+
instrument_finish item, index, result, options
|
489
|
+
results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
|
490
|
+
ractor.send([[nil, nil], nil, :break]) # stop the ractor
|
491
|
+
end
|
492
|
+
|
493
|
+
exception || results
|
494
|
+
end
|
495
|
+
|
385
496
|
def work_in_processes(job_factory, options, &blk)
|
386
497
|
workers = create_workers(job_factory, options, &blk)
|
387
498
|
results = []
|
@@ -426,6 +537,7 @@ module Parallel
|
|
426
537
|
end
|
427
538
|
end
|
428
539
|
end
|
540
|
+
|
429
541
|
exception || results
|
430
542
|
end
|
431
543
|
|
@@ -521,12 +633,20 @@ module Parallel
|
|
521
633
|
end
|
522
634
|
|
523
635
|
def with_instrumentation(item, index, options)
|
524
|
-
|
525
|
-
on_finish = options[:finish]
|
526
|
-
options[:mutex].synchronize { on_start.call(item, index) } if on_start
|
636
|
+
instrument_start(item, index, options)
|
527
637
|
result = yield
|
528
|
-
|
638
|
+
instrument_finish(item, index, result, options)
|
529
639
|
result unless options[:preserve_results] == false
|
530
640
|
end
|
641
|
+
|
642
|
+
def instrument_finish(item, index, result, options)
|
643
|
+
return unless on_finish = options[:finish]
|
644
|
+
options[:mutex].synchronize { on_finish.call(item, index, result) }
|
645
|
+
end
|
646
|
+
|
647
|
+
def instrument_start(item, index, options)
|
648
|
+
return unless on_start = options[:start]
|
649
|
+
options[:mutex].synchronize { on_start.call(item, index) }
|
650
|
+
end
|
531
651
|
end
|
532
652
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.23.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-04-18 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email: michael@grosser.it
|
@@ -18,15 +18,14 @@ extra_rdoc_files: []
|
|
18
18
|
files:
|
19
19
|
- MIT-LICENSE.txt
|
20
20
|
- lib/parallel.rb
|
21
|
-
- lib/parallel/processor_count.rb
|
22
21
|
- lib/parallel/version.rb
|
23
22
|
homepage: https://github.com/grosser/parallel
|
24
23
|
licenses:
|
25
24
|
- MIT
|
26
25
|
metadata:
|
27
26
|
bug_tracker_uri: https://github.com/grosser/parallel/issues
|
28
|
-
documentation_uri: https://github.com/grosser/parallel/blob/v1.
|
29
|
-
source_code_uri: https://github.com/grosser/parallel/tree/v1.
|
27
|
+
documentation_uri: https://github.com/grosser/parallel/blob/v1.23.0/Readme.md
|
28
|
+
source_code_uri: https://github.com/grosser/parallel/tree/v1.23.0
|
30
29
|
wiki_uri: https://github.com/grosser/parallel/wiki
|
31
30
|
post_install_message:
|
32
31
|
rdoc_options: []
|
@@ -43,7 +42,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
43
42
|
- !ruby/object:Gem::Version
|
44
43
|
version: '0'
|
45
44
|
requirements: []
|
46
|
-
rubygems_version: 3.
|
45
|
+
rubygems_version: 3.1.6
|
47
46
|
signing_key:
|
48
47
|
specification_version: 4
|
49
48
|
summary: Run any kind of code in parallel processes
|
@@ -1,45 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
require 'etc'
|
3
|
-
|
4
|
-
module Parallel
|
5
|
-
# TODO: inline this method into parallel.rb and kill physical_processor_count in next major release
|
6
|
-
module ProcessorCount
|
7
|
-
# Number of processors seen by the OS, used for process scheduling
|
8
|
-
def processor_count
|
9
|
-
@processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || Etc.nprocessors)
|
10
|
-
end
|
11
|
-
|
12
|
-
# Number of physical processor cores on the current system.
|
13
|
-
def physical_processor_count
|
14
|
-
@physical_processor_count ||= begin
|
15
|
-
ppc =
|
16
|
-
case RbConfig::CONFIG["target_os"]
|
17
|
-
when /darwin[12]/
|
18
|
-
IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
|
19
|
-
when /linux/
|
20
|
-
cores = {} # unique physical ID / core ID combinations
|
21
|
-
phy = 0
|
22
|
-
IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
|
23
|
-
if ln.start_with?("physical")
|
24
|
-
phy = ln[/\d+/]
|
25
|
-
elsif ln.start_with?("core")
|
26
|
-
cid = "#{phy}:#{ln[/\d+/]}"
|
27
|
-
cores[cid] = true unless cores[cid]
|
28
|
-
end
|
29
|
-
end
|
30
|
-
cores.count
|
31
|
-
when /mswin|mingw/
|
32
|
-
require 'win32ole'
|
33
|
-
result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
|
34
|
-
"select NumberOfCores from Win32_Processor"
|
35
|
-
)
|
36
|
-
result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
|
37
|
-
else
|
38
|
-
processor_count
|
39
|
-
end
|
40
|
-
# fall back to logical count if physical info is invalid
|
41
|
-
ppc > 0 ? ppc : processor_count
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|