parallel 1.12.1 → 1.26.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/lib/parallel/version.rb +2 -1
- data/lib/parallel.rb +286 -78
- metadata +13 -11
- data/lib/parallel/processor_count.rb +0 -93
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 404f38d083ffa4ed2163c3018f5243fe487b6154905afa83b59320380bd3629a
|
4
|
+
data.tar.gz: 832f558729b425fa3e2ad6b9a3d6418c44d0f4a2ecc180a398a28b2301cd5cab
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5947502dc01d242a8a1cef0e2133cc3df913107941f5ba5bd82032bda76de6358bc44e62848909a75f2bfa45c867ee099f454c2652555265d734a166c988d144
|
7
|
+
data.tar.gz: '08f69f6c82041f015dee319ccc42bb0b1e20fc3d58b4e4007c1636ce459a71b0625ebb3ec8c28549562c1ed3f99f4182da41d8741a42eeeb4a96cb51b9bff701'
|
data/lib/parallel/version.rb
CHANGED
data/lib/parallel.rb
CHANGED
@@ -1,31 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'rbconfig'
|
2
3
|
require 'parallel/version'
|
3
|
-
require 'parallel/processor_count'
|
4
4
|
|
5
5
|
module Parallel
|
6
|
-
|
6
|
+
Stop = Object.new.freeze
|
7
7
|
|
8
8
|
class DeadWorker < StandardError
|
9
9
|
end
|
10
10
|
|
11
11
|
class Break < StandardError
|
12
|
+
attr_reader :value
|
13
|
+
|
14
|
+
def initialize(value = nil)
|
15
|
+
super()
|
16
|
+
@value = value
|
17
|
+
end
|
12
18
|
end
|
13
19
|
|
14
|
-
class Kill <
|
20
|
+
class Kill < Break
|
15
21
|
end
|
16
22
|
|
17
23
|
class UndumpableException < StandardError
|
18
24
|
attr_reader :backtrace
|
25
|
+
|
19
26
|
def initialize(original)
|
20
|
-
super
|
27
|
+
super("#{original.class}: #{original.message}")
|
21
28
|
@backtrace = original.backtrace
|
22
29
|
end
|
23
30
|
end
|
24
31
|
|
25
|
-
Stop = Object.new
|
26
|
-
|
27
32
|
class ExceptionWrapper
|
28
33
|
attr_reader :exception
|
34
|
+
|
29
35
|
def initialize(exception)
|
30
36
|
# Remove the bindings stack added by the better_errors gem,
|
31
37
|
# because it cannot be marshalled
|
@@ -36,7 +42,7 @@ module Parallel
|
|
36
42
|
@exception =
|
37
43
|
begin
|
38
44
|
Marshal.dump(exception) && exception
|
39
|
-
rescue
|
45
|
+
rescue StandardError
|
40
46
|
UndumpableException.new(exception)
|
41
47
|
end
|
42
48
|
end
|
@@ -45,8 +51,11 @@ module Parallel
|
|
45
51
|
class Worker
|
46
52
|
attr_reader :pid, :read, :write
|
47
53
|
attr_accessor :thread
|
54
|
+
|
48
55
|
def initialize(read, write, pid)
|
49
|
-
@read
|
56
|
+
@read = read
|
57
|
+
@write = write
|
58
|
+
@pid = pid
|
50
59
|
end
|
51
60
|
|
52
61
|
def stop
|
@@ -73,7 +82,7 @@ module Parallel
|
|
73
82
|
rescue EOFError
|
74
83
|
raise DeadWorker
|
75
84
|
end
|
76
|
-
raise result.exception if ExceptionWrapper
|
85
|
+
raise result.exception if result.is_a?(ExceptionWrapper)
|
77
86
|
result
|
78
87
|
end
|
79
88
|
|
@@ -102,7 +111,7 @@ module Parallel
|
|
102
111
|
item, index = @mutex.synchronize do
|
103
112
|
return if @stopped
|
104
113
|
item = @lambda.call
|
105
|
-
@stopped = (item ==
|
114
|
+
@stopped = (item == Stop)
|
106
115
|
return if @stopped
|
107
116
|
[item, @index += 1]
|
108
117
|
end
|
@@ -140,7 +149,7 @@ module Parallel
|
|
140
149
|
end
|
141
150
|
|
142
151
|
def queue_wrapper(array)
|
143
|
-
array.respond_to?(:num_waiting) && array.respond_to?(:pop) &&
|
152
|
+
array.respond_to?(:num_waiting) && array.respond_to?(:pop) && -> { array.pop(false) }
|
144
153
|
end
|
145
154
|
end
|
146
155
|
|
@@ -156,7 +165,7 @@ module Parallel
|
|
156
165
|
|
157
166
|
if @to_be_killed.empty?
|
158
167
|
old_interrupt = trap_interrupt(signal) do
|
159
|
-
|
168
|
+
warn 'Parallel execution interrupted, exiting ...'
|
160
169
|
@to_be_killed.flatten.each { |pid| kill(pid) }
|
161
170
|
end
|
162
171
|
end
|
@@ -200,46 +209,61 @@ module Parallel
|
|
200
209
|
end
|
201
210
|
|
202
211
|
class << self
|
203
|
-
def in_threads(options={:
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
212
|
+
def in_threads(options = { count: 2 })
|
213
|
+
threads = []
|
214
|
+
count, = extract_count_from_options(options)
|
215
|
+
|
216
|
+
Thread.handle_interrupt(Exception => :never) do
|
217
|
+
Thread.handle_interrupt(Exception => :immediate) do
|
218
|
+
count.times do |i|
|
219
|
+
threads << Thread.new { yield(i) }
|
220
|
+
end
|
221
|
+
threads.map(&:value)
|
222
|
+
end
|
223
|
+
ensure
|
224
|
+
threads.each(&:kill)
|
225
|
+
end
|
208
226
|
end
|
209
227
|
|
210
228
|
def in_processes(options = {}, &block)
|
211
229
|
count, options = extract_count_from_options(options)
|
212
230
|
count ||= processor_count
|
213
|
-
map(0...count, options.merge(:
|
231
|
+
map(0...count, options.merge(in_processes: count), &block)
|
214
232
|
end
|
215
233
|
|
216
|
-
def each(array, options={}, &block)
|
217
|
-
map(array, options.merge(:
|
234
|
+
def each(array, options = {}, &block)
|
235
|
+
map(array, options.merge(preserve_results: false), &block)
|
218
236
|
end
|
219
237
|
|
220
238
|
def any?(*args, &block)
|
221
239
|
raise "You must provide a block when calling #any?" if block.nil?
|
222
|
-
!each(*args) { |*a| raise
|
240
|
+
!each(*args) { |*a| raise Kill if block.call(*a) }
|
223
241
|
end
|
224
242
|
|
225
243
|
def all?(*args, &block)
|
226
244
|
raise "You must provide a block when calling #all?" if block.nil?
|
227
|
-
!!each(*args) { |*a| raise
|
245
|
+
!!each(*args) { |*a| raise Kill unless block.call(*a) }
|
228
246
|
end
|
229
247
|
|
230
|
-
def each_with_index(array, options={}, &block)
|
231
|
-
each(array, options.merge(:
|
248
|
+
def each_with_index(array, options = {}, &block)
|
249
|
+
each(array, options.merge(with_index: true), &block)
|
232
250
|
end
|
233
251
|
|
234
252
|
def map(source, options = {}, &block)
|
253
|
+
options = options.dup
|
235
254
|
options[:mutex] = Mutex.new
|
236
255
|
|
237
|
-
if
|
256
|
+
if options[:in_processes] && options[:in_threads]
|
257
|
+
raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
|
258
|
+
elsif RUBY_PLATFORM =~ (/java/) && !(options[:in_processes])
|
238
259
|
method = :in_threads
|
239
260
|
size = options[method] || processor_count
|
240
261
|
elsif options[:in_threads]
|
241
262
|
method = :in_threads
|
242
263
|
size = options[method]
|
264
|
+
elsif options[:in_ractors]
|
265
|
+
method = :in_ractors
|
266
|
+
size = options[method]
|
243
267
|
else
|
244
268
|
method = :in_processes
|
245
269
|
if Process.respond_to?(:fork)
|
@@ -256,34 +280,109 @@ module Parallel
|
|
256
280
|
options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
|
257
281
|
add_progress_bar!(job_factory, options)
|
258
282
|
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
283
|
+
result =
|
284
|
+
if size == 0
|
285
|
+
work_direct(job_factory, options, &block)
|
286
|
+
elsif method == :in_threads
|
287
|
+
work_in_threads(job_factory, options.merge(count: size), &block)
|
288
|
+
elsif method == :in_ractors
|
289
|
+
work_in_ractors(job_factory, options.merge(count: size), &block)
|
290
|
+
else
|
291
|
+
work_in_processes(job_factory, options.merge(count: size), &block)
|
292
|
+
end
|
293
|
+
|
294
|
+
return result.value if result.is_a?(Break)
|
295
|
+
raise result if result.is_a?(Exception)
|
296
|
+
options[:return_results] ? result : source
|
297
|
+
end
|
298
|
+
|
299
|
+
def map_with_index(array, options = {}, &block)
|
300
|
+
map(array, options.merge(with_index: true), &block)
|
301
|
+
end
|
302
|
+
|
303
|
+
def flat_map(...)
|
304
|
+
map(...).flatten(1)
|
305
|
+
end
|
306
|
+
|
307
|
+
def filter_map(...)
|
308
|
+
map(...).compact
|
309
|
+
end
|
310
|
+
|
311
|
+
# Number of physical processor cores on the current system.
|
312
|
+
def physical_processor_count
|
313
|
+
@physical_processor_count ||= begin
|
314
|
+
ppc =
|
315
|
+
case RbConfig::CONFIG["target_os"]
|
316
|
+
when /darwin[12]/
|
317
|
+
IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
|
318
|
+
when /linux/
|
319
|
+
cores = {} # unique physical ID / core ID combinations
|
320
|
+
phy = 0
|
321
|
+
File.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
|
322
|
+
if ln.start_with?("physical")
|
323
|
+
phy = ln[/\d+/]
|
324
|
+
elsif ln.start_with?("core")
|
325
|
+
cid = "#{phy}:#{ln[/\d+/]}"
|
326
|
+
cores[cid] = true unless cores[cid]
|
327
|
+
end
|
328
|
+
end
|
329
|
+
cores.count
|
330
|
+
when /mswin|mingw/
|
331
|
+
physical_processor_count_windows
|
332
|
+
else
|
333
|
+
processor_count
|
334
|
+
end
|
335
|
+
# fall back to logical count if physical info is invalid
|
336
|
+
ppc > 0 ? ppc : processor_count
|
268
337
|
end
|
269
338
|
end
|
270
339
|
|
271
|
-
|
272
|
-
|
340
|
+
# Number of processors seen by the OS or value considering CPU quota if the process is inside a cgroup,
|
341
|
+
# used for process scheduling
|
342
|
+
def processor_count
|
343
|
+
@processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || available_processor_count)
|
273
344
|
end
|
274
345
|
|
275
346
|
def worker_number
|
276
347
|
Thread.current[:parallel_worker_number]
|
277
348
|
end
|
278
349
|
|
350
|
+
# TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
|
279
351
|
def worker_number=(worker_num)
|
280
352
|
Thread.current[:parallel_worker_number] = worker_num
|
281
353
|
end
|
282
354
|
|
283
355
|
private
|
284
356
|
|
357
|
+
def physical_processor_count_windows
|
358
|
+
# Get-CimInstance introduced in PowerShell 3 or earlier: https://learn.microsoft.com/en-us/previous-versions/powershell/module/cimcmdlets/get-ciminstance?view=powershell-3.0
|
359
|
+
result = run(
|
360
|
+
'powershell -command "Get-CimInstance -ClassName Win32_Processor -Property NumberOfCores ' \
|
361
|
+
'| Select-Object -Property NumberOfCores"'
|
362
|
+
)
|
363
|
+
if !result || $?.exitstatus != 0
|
364
|
+
# fallback to deprecated wmic for older systems
|
365
|
+
result = run("wmic cpu get NumberOfCores")
|
366
|
+
end
|
367
|
+
if !result || $?.exitstatus != 0
|
368
|
+
# Bail out if both commands returned something unexpected
|
369
|
+
warn "guessing pyhsical processor count"
|
370
|
+
processor_count
|
371
|
+
else
|
372
|
+
# powershell: "\nNumberOfCores\n-------------\n 4\n\n\n"
|
373
|
+
# wmic: "NumberOfCores \n\n4 \n\n\n\n"
|
374
|
+
result.scan(/\d+/).map(&:to_i).reduce(:+)
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
378
|
+
def run(command)
|
379
|
+
IO.popen(command, &:read)
|
380
|
+
rescue Errno::ENOENT
|
381
|
+
# Ignore
|
382
|
+
end
|
383
|
+
|
285
384
|
def add_progress_bar!(job_factory, options)
|
286
|
-
if progress_options = options[:progress]
|
385
|
+
if (progress_options = options[:progress])
|
287
386
|
raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
|
288
387
|
require 'ruby-progressbar'
|
289
388
|
|
@@ -312,16 +411,16 @@ module Parallel
|
|
312
411
|
results = []
|
313
412
|
exception = nil
|
314
413
|
begin
|
315
|
-
while set = job_factory.next
|
414
|
+
while (set = job_factory.next)
|
316
415
|
item, index = set
|
317
416
|
results << with_instrumentation(item, index, options) do
|
318
417
|
call_with_index(item, index, options, &block)
|
319
418
|
end
|
320
419
|
end
|
321
|
-
rescue
|
420
|
+
rescue StandardError
|
322
421
|
exception = $!
|
323
422
|
end
|
324
|
-
|
423
|
+
exception || results
|
325
424
|
ensure
|
326
425
|
self.worker_number = nil
|
327
426
|
end
|
@@ -335,28 +434,90 @@ module Parallel
|
|
335
434
|
in_threads(options) do |worker_num|
|
336
435
|
self.worker_number = worker_num
|
337
436
|
# as long as there are more jobs, work on one of them
|
338
|
-
while !exception && set = job_factory.next
|
437
|
+
while !exception && (set = job_factory.next)
|
339
438
|
begin
|
340
439
|
item, index = set
|
341
440
|
result = with_instrumentation item, index, options do
|
342
441
|
call_with_index(item, index, options, &block)
|
343
442
|
end
|
344
443
|
results_mutex.synchronize { results[index] = result }
|
345
|
-
rescue
|
444
|
+
rescue StandardError
|
346
445
|
exception = $!
|
347
446
|
end
|
348
447
|
end
|
349
448
|
end
|
350
449
|
|
351
|
-
|
450
|
+
exception || results
|
352
451
|
end
|
353
452
|
|
354
|
-
def
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
453
|
+
def work_in_ractors(job_factory, options)
|
454
|
+
exception = nil
|
455
|
+
results = []
|
456
|
+
results_mutex = Mutex.new # arrays are not thread-safe on jRuby
|
457
|
+
|
458
|
+
callback = options[:ractor]
|
459
|
+
if block_given? || !callback
|
460
|
+
raise ArgumentError, "pass the code you want to execute as `ractor: [ClassName, :method_name]`"
|
359
461
|
end
|
462
|
+
|
463
|
+
# build
|
464
|
+
ractors = Array.new(options.fetch(:count)) do
|
465
|
+
Ractor.new do
|
466
|
+
loop do
|
467
|
+
got = receive
|
468
|
+
(klass, method_name), item, index = got
|
469
|
+
break if index == :break
|
470
|
+
begin
|
471
|
+
Ractor.yield [nil, klass.send(method_name, item), item, index]
|
472
|
+
rescue StandardError => e
|
473
|
+
Ractor.yield [e, nil, item, index]
|
474
|
+
end
|
475
|
+
end
|
476
|
+
end
|
477
|
+
end
|
478
|
+
|
479
|
+
# start
|
480
|
+
ractors.dup.each do |ractor|
|
481
|
+
if (set = job_factory.next)
|
482
|
+
item, index = set
|
483
|
+
instrument_start item, index, options
|
484
|
+
ractor.send [callback, item, index]
|
485
|
+
else
|
486
|
+
ractor.send([[nil, nil], nil, :break]) # stop the ractor
|
487
|
+
ractors.delete ractor
|
488
|
+
end
|
489
|
+
end
|
490
|
+
|
491
|
+
# replace with new items
|
492
|
+
while (set = job_factory.next)
|
493
|
+
item_next, index_next = set
|
494
|
+
done, (exception, result, item, index) = Ractor.select(*ractors)
|
495
|
+
if exception
|
496
|
+
ractors.delete done
|
497
|
+
break
|
498
|
+
end
|
499
|
+
instrument_finish item, index, result, options
|
500
|
+
results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
|
501
|
+
|
502
|
+
instrument_start item_next, index_next, options
|
503
|
+
done.send([callback, item_next, index_next])
|
504
|
+
end
|
505
|
+
|
506
|
+
# finish
|
507
|
+
ractors.each do |ractor|
|
508
|
+
(new_exception, result, item, index) = ractor.take
|
509
|
+
exception ||= new_exception
|
510
|
+
next if new_exception
|
511
|
+
instrument_finish item, index, result, options
|
512
|
+
results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
|
513
|
+
ractor.send([[nil, nil], nil, :break]) # stop the ractor
|
514
|
+
end
|
515
|
+
|
516
|
+
exception || results
|
517
|
+
end
|
518
|
+
|
519
|
+
def work_in_processes(job_factory, options, &blk)
|
520
|
+
workers = create_workers(job_factory, options, &blk)
|
360
521
|
results = []
|
361
522
|
results_mutex = Mutex.new # arrays are not thread-safe
|
362
523
|
exception = nil
|
@@ -364,6 +525,8 @@ module Parallel
|
|
364
525
|
UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
|
365
526
|
in_threads(options) do |i|
|
366
527
|
worker = workers[i]
|
528
|
+
worker.thread = Thread.current
|
529
|
+
worked = false
|
367
530
|
|
368
531
|
begin
|
369
532
|
loop do
|
@@ -372,44 +535,44 @@ module Parallel
|
|
372
535
|
break unless index
|
373
536
|
|
374
537
|
if options[:isolation]
|
375
|
-
worker = replace_worker(job_factory, workers, i, options, blk)
|
538
|
+
worker = replace_worker(job_factory, workers, i, options, blk) if worked
|
539
|
+
worked = true
|
540
|
+
worker.thread = Thread.current
|
376
541
|
end
|
377
542
|
|
378
|
-
worker.thread = Thread.current
|
379
|
-
|
380
543
|
begin
|
381
544
|
result = with_instrumentation item, index, options do
|
382
545
|
worker.work(job_factory.pack(item, index))
|
383
546
|
end
|
384
547
|
results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
|
385
|
-
rescue
|
386
|
-
exception =
|
387
|
-
if
|
548
|
+
rescue StandardError => e
|
549
|
+
exception = e
|
550
|
+
if exception.is_a?(Kill)
|
388
551
|
(workers - [worker]).each do |w|
|
389
|
-
w.thread
|
552
|
+
w.thread&.kill
|
390
553
|
UserInterruptHandler.kill(w.pid)
|
391
554
|
end
|
392
555
|
end
|
393
556
|
end
|
394
557
|
end
|
395
558
|
ensure
|
396
|
-
worker.stop
|
559
|
+
worker.stop
|
397
560
|
end
|
398
561
|
end
|
399
562
|
end
|
400
563
|
|
401
|
-
|
564
|
+
exception || results
|
402
565
|
end
|
403
566
|
|
404
|
-
def replace_worker(job_factory, workers,
|
567
|
+
def replace_worker(job_factory, workers, index, options, blk)
|
405
568
|
options[:mutex].synchronize do
|
406
569
|
# old worker is no longer used ... stop it
|
407
|
-
worker = workers[
|
408
|
-
worker.stop
|
570
|
+
worker = workers[index]
|
571
|
+
worker.stop
|
409
572
|
|
410
573
|
# create a new replacement worker
|
411
574
|
running = workers - [worker]
|
412
|
-
workers[
|
575
|
+
workers[index] = worker(job_factory, options.merge(started_workers: running, worker_number: index), &blk)
|
413
576
|
end
|
414
577
|
end
|
415
578
|
|
@@ -451,21 +614,25 @@ module Parallel
|
|
451
614
|
until read.eof?
|
452
615
|
data = Marshal.load(read)
|
453
616
|
item, index = job_factory.unpack(data)
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
617
|
+
|
618
|
+
result =
|
619
|
+
begin
|
620
|
+
call_with_index(item, index, options, &block)
|
621
|
+
# https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
|
622
|
+
rescue NoMemoryError, SignalException, Interrupt, SystemExit # rubocop:disable Lint/ShadowedException
|
623
|
+
raise $!
|
624
|
+
rescue Exception # # rubocop:disable Lint/RescueException
|
625
|
+
ExceptionWrapper.new($!)
|
626
|
+
end
|
627
|
+
|
628
|
+
begin
|
629
|
+
Marshal.dump(result, write)
|
630
|
+
rescue Errno::EPIPE
|
631
|
+
return # parent thread already dead
|
458
632
|
end
|
459
|
-
Marshal.dump(result, write)
|
460
633
|
end
|
461
634
|
end
|
462
635
|
|
463
|
-
def handle_exception(exception, results)
|
464
|
-
return nil if [Parallel::Break, Parallel::Kill].include? exception.class
|
465
|
-
raise exception if exception
|
466
|
-
results
|
467
|
-
end
|
468
|
-
|
469
636
|
# options is either a Integer or a Hash with :count
|
470
637
|
def extract_count_from_options(options)
|
471
638
|
if options.is_a?(Hash)
|
@@ -480,21 +647,62 @@ module Parallel
|
|
480
647
|
def call_with_index(item, index, options, &block)
|
481
648
|
args = [item]
|
482
649
|
args << index if options[:with_index]
|
650
|
+
results = block.call(*args)
|
483
651
|
if options[:return_results]
|
484
|
-
|
652
|
+
results
|
485
653
|
else
|
486
|
-
block.call(*args)
|
487
654
|
nil # avoid GC overhead of passing large results around
|
488
655
|
end
|
489
656
|
end
|
490
657
|
|
491
658
|
def with_instrumentation(item, index, options)
|
492
|
-
|
493
|
-
on_finish = options[:finish]
|
494
|
-
options[:mutex].synchronize { on_start.call(item, index) } if on_start
|
659
|
+
instrument_start(item, index, options)
|
495
660
|
result = yield
|
496
|
-
|
661
|
+
instrument_finish(item, index, result, options)
|
497
662
|
result unless options[:preserve_results] == false
|
498
663
|
end
|
664
|
+
|
665
|
+
def instrument_finish(item, index, result, options)
|
666
|
+
return unless (on_finish = options[:finish])
|
667
|
+
return instrument_finish_in_order(item, index, result, options) if options[:finish_in_order]
|
668
|
+
options[:mutex].synchronize { on_finish.call(item, index, result) }
|
669
|
+
end
|
670
|
+
|
671
|
+
# yield results in the order of the input items
|
672
|
+
# needs to use `options` to store state between executions
|
673
|
+
# needs to use `done` index since a nil result would also be valid
|
674
|
+
def instrument_finish_in_order(item, index, result, options)
|
675
|
+
options[:mutex].synchronize do
|
676
|
+
# initialize our state
|
677
|
+
options[:finish_done] ||= []
|
678
|
+
options[:finish_expecting] ||= 0 # we wait for item at index 0
|
679
|
+
|
680
|
+
# store current result
|
681
|
+
options[:finish_done][index] = [item, result]
|
682
|
+
|
683
|
+
# yield all results that are now in order
|
684
|
+
break unless index == options[:finish_expecting]
|
685
|
+
index.upto(options[:finish_done].size).each do |i|
|
686
|
+
break unless (done = options[:finish_done][i])
|
687
|
+
options[:finish_done][i] = nil # allow GC to free this item and result
|
688
|
+
options[:finish].call(done[0], i, done[1])
|
689
|
+
options[:finish_expecting] += 1
|
690
|
+
end
|
691
|
+
end
|
692
|
+
end
|
693
|
+
|
694
|
+
def instrument_start(item, index, options)
|
695
|
+
return unless (on_start = options[:start])
|
696
|
+
options[:mutex].synchronize { on_start.call(item, index) }
|
697
|
+
end
|
698
|
+
|
699
|
+
def available_processor_count
|
700
|
+
gem 'concurrent-ruby', '>= 1.3.4'
|
701
|
+
require 'concurrent-ruby'
|
702
|
+
Concurrent.available_processor_count.floor
|
703
|
+
rescue LoadError
|
704
|
+
require 'etc'
|
705
|
+
Etc.nprocessors
|
706
|
+
end
|
499
707
|
end
|
500
708
|
end
|
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: parallel
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.26.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Grosser
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-08-16 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
|
-
description:
|
13
|
+
description:
|
14
14
|
email: michael@grosser.it
|
15
15
|
executables: []
|
16
16
|
extensions: []
|
@@ -18,13 +18,16 @@ extra_rdoc_files: []
|
|
18
18
|
files:
|
19
19
|
- MIT-LICENSE.txt
|
20
20
|
- lib/parallel.rb
|
21
|
-
- lib/parallel/processor_count.rb
|
22
21
|
- lib/parallel/version.rb
|
23
22
|
homepage: https://github.com/grosser/parallel
|
24
23
|
licenses:
|
25
24
|
- MIT
|
26
|
-
metadata:
|
27
|
-
|
25
|
+
metadata:
|
26
|
+
bug_tracker_uri: https://github.com/grosser/parallel/issues
|
27
|
+
documentation_uri: https://github.com/grosser/parallel/blob/v1.26.3/Readme.md
|
28
|
+
source_code_uri: https://github.com/grosser/parallel/tree/v1.26.3
|
29
|
+
wiki_uri: https://github.com/grosser/parallel/wiki
|
30
|
+
post_install_message:
|
28
31
|
rdoc_options: []
|
29
32
|
require_paths:
|
30
33
|
- lib
|
@@ -32,16 +35,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
32
35
|
requirements:
|
33
36
|
- - ">="
|
34
37
|
- !ruby/object:Gem::Version
|
35
|
-
version:
|
38
|
+
version: '2.7'
|
36
39
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
37
40
|
requirements:
|
38
41
|
- - ">="
|
39
42
|
- !ruby/object:Gem::Version
|
40
43
|
version: '0'
|
41
44
|
requirements: []
|
42
|
-
|
43
|
-
|
44
|
-
signing_key:
|
45
|
+
rubygems_version: 3.4.10
|
46
|
+
signing_key:
|
45
47
|
specification_version: 4
|
46
48
|
summary: Run any kind of code in parallel processes
|
47
49
|
test_files: []
|
@@ -1,93 +0,0 @@
|
|
1
|
-
if RUBY_VERSION.to_f >= 2.2
|
2
|
-
require 'etc'
|
3
|
-
end
|
4
|
-
|
5
|
-
module Parallel
|
6
|
-
module ProcessorCount
|
7
|
-
# Number of processors seen by the OS and used for process scheduling.
|
8
|
-
#
|
9
|
-
# * AIX: /usr/sbin/pmcycles (AIX 5+), /usr/sbin/lsdev
|
10
|
-
# * BSD: /sbin/sysctl
|
11
|
-
# * Cygwin: /proc/cpuinfo
|
12
|
-
# * Darwin: /usr/bin/hwprefs, /usr/sbin/sysctl
|
13
|
-
# * HP-UX: /usr/sbin/ioscan
|
14
|
-
# * IRIX: /usr/sbin/sysconf
|
15
|
-
# * Linux: /proc/cpuinfo
|
16
|
-
# * Minix 3+: /proc/cpuinfo
|
17
|
-
# * Solaris: /usr/sbin/psrinfo
|
18
|
-
# * Tru64 UNIX: /usr/sbin/psrinfo
|
19
|
-
# * UnixWare: /usr/sbin/psrinfo
|
20
|
-
#
|
21
|
-
def processor_count
|
22
|
-
@processor_count ||= begin
|
23
|
-
if defined?(Etc) && Etc.respond_to?(:nprocessors)
|
24
|
-
Etc.nprocessors
|
25
|
-
else
|
26
|
-
os_name = RbConfig::CONFIG["target_os"]
|
27
|
-
if os_name =~ /mingw|mswin/
|
28
|
-
require 'win32ole'
|
29
|
-
result = WIN32OLE.connect("winmgmts://").ExecQuery(
|
30
|
-
"select NumberOfLogicalProcessors from Win32_Processor")
|
31
|
-
result.to_enum.collect(&:NumberOfLogicalProcessors).reduce(:+)
|
32
|
-
elsif File.readable?("/proc/cpuinfo")
|
33
|
-
IO.read("/proc/cpuinfo").scan(/^processor/).size
|
34
|
-
elsif File.executable?("/usr/bin/hwprefs")
|
35
|
-
IO.popen("/usr/bin/hwprefs thread_count").read.to_i
|
36
|
-
elsif File.executable?("/usr/sbin/psrinfo")
|
37
|
-
IO.popen("/usr/sbin/psrinfo").read.scan(/^.*on-*line/).size
|
38
|
-
elsif File.executable?("/usr/sbin/ioscan")
|
39
|
-
IO.popen("/usr/sbin/ioscan -kC processor") do |out|
|
40
|
-
out.read.scan(/^.*processor/).size
|
41
|
-
end
|
42
|
-
elsif File.executable?("/usr/sbin/pmcycles")
|
43
|
-
IO.popen("/usr/sbin/pmcycles -m").read.count("\n")
|
44
|
-
elsif File.executable?("/usr/sbin/lsdev")
|
45
|
-
IO.popen("/usr/sbin/lsdev -Cc processor -S 1").read.count("\n")
|
46
|
-
elsif File.executable?("/usr/sbin/sysconf") and os_name =~ /irix/i
|
47
|
-
IO.popen("/usr/sbin/sysconf NPROC_ONLN").read.to_i
|
48
|
-
elsif File.executable?("/usr/sbin/sysctl")
|
49
|
-
IO.popen("/usr/sbin/sysctl -n hw.ncpu").read.to_i
|
50
|
-
elsif File.executable?("/sbin/sysctl")
|
51
|
-
IO.popen("/sbin/sysctl -n hw.ncpu").read.to_i
|
52
|
-
else
|
53
|
-
$stderr.puts "Unknown platform: " + RbConfig::CONFIG["target_os"]
|
54
|
-
$stderr.puts "Assuming 1 processor."
|
55
|
-
1
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
# Number of physical processor cores on the current system.
|
62
|
-
#
|
63
|
-
def physical_processor_count
|
64
|
-
@physical_processor_count ||= begin
|
65
|
-
ppc = case RbConfig::CONFIG["target_os"]
|
66
|
-
when /darwin1/
|
67
|
-
IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
|
68
|
-
when /linux/
|
69
|
-
cores = {} # unique physical ID / core ID combinations
|
70
|
-
phy = 0
|
71
|
-
IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
|
72
|
-
if ln.start_with?("physical")
|
73
|
-
phy = ln[/\d+/]
|
74
|
-
elsif ln.start_with?("core")
|
75
|
-
cid = phy + ":" + ln[/\d+/]
|
76
|
-
cores[cid] = true if not cores[cid]
|
77
|
-
end
|
78
|
-
end
|
79
|
-
cores.count
|
80
|
-
when /mswin|mingw/
|
81
|
-
require 'win32ole'
|
82
|
-
result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
|
83
|
-
"select NumberOfCores from Win32_Processor")
|
84
|
-
result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
|
85
|
-
else
|
86
|
-
processor_count
|
87
|
-
end
|
88
|
-
# fall back to logical count if physical info is invalid
|
89
|
-
ppc > 0 ? ppc : processor_count
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|