parallel 1.20.0 → 1.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1f41f840e7e5128ebbb8d4ea2955b5bc555925ac049184ae1d4323885a3980b9
4
- data.tar.gz: f9463997e0b890857d1909e6df6c8632b08eb028fc4015541631ca275d1fabfb
3
+ metadata.gz: 1b517593727e82ec4dd9a897612cfca6ed731f788b84bceee4166a455bbc339d
4
+ data.tar.gz: a47a4fbac65ebe5ecad57c3d2c78df64f261e3770637cacbb2063b91369538bf
5
5
  SHA512:
6
- metadata.gz: '0581d59c3d38af4fefdd761264899f9661a3c28e8a4d3c9683f7e93c2295313e53d518263a6b4d04fcbb9a57eb55dfe3854e8c27f9f4e378641823eea2aada72'
7
- data.tar.gz: f77d00250586ada81bddfc2dd47e4895a526215549cee3728f31a927dd617a3a4d2bca7e72d9c7ff3c481e3a4488933c490153d0ef7dec97dc6a5cdd6bc513ed
6
+ metadata.gz: a4e9dbbbf9c0cbff88e12750fd521ee3f65bd1c0de2ab5a18e1a9b7e96c413cd12cce0de985aef9bd4ca0eb8c5e438cf83ac59c88a74e67e38ef73cc7ac8fac9
7
+ data.tar.gz: 37ddce10ac73b9d722452693914bb889f2d638cbe52fe36ffe37673d6bba9d01601656568408eac29155619df8208892c51861e9033888e549bcaa467ce1f912
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'etc'
2
3
 
3
4
  module Parallel
@@ -11,29 +12,31 @@ module Parallel
11
12
  # Number of physical processor cores on the current system.
12
13
  def physical_processor_count
13
14
  @physical_processor_count ||= begin
14
- ppc = case RbConfig::CONFIG["target_os"]
15
- when /darwin1/
16
- IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
17
- when /linux/
18
- cores = {} # unique physical ID / core ID combinations
19
- phy = 0
20
- IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
21
- if ln.start_with?("physical")
22
- phy = ln[/\d+/]
23
- elsif ln.start_with?("core")
24
- cid = phy + ":" + ln[/\d+/]
25
- cores[cid] = true if not cores[cid]
15
+ ppc =
16
+ case RbConfig::CONFIG["target_os"]
17
+ when /darwin[12]/
18
+ IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
19
+ when /linux/
20
+ cores = {} # unique physical ID / core ID combinations
21
+ phy = 0
22
+ File.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
23
+ if ln.start_with?("physical")
24
+ phy = ln[/\d+/]
25
+ elsif ln.start_with?("core")
26
+ cid = "#{phy}:#{ln[/\d+/]}"
27
+ cores[cid] = true unless cores[cid]
28
+ end
26
29
  end
30
+ cores.count
31
+ when /mswin|mingw/
32
+ require 'win32ole'
33
+ result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
34
+ "select NumberOfCores from Win32_Processor"
35
+ )
36
+ result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
37
+ else
38
+ processor_count
27
39
  end
28
- cores.count
29
- when /mswin|mingw/
30
- require 'win32ole'
31
- result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
32
- "select NumberOfCores from Win32_Processor")
33
- result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
34
- else
35
- processor_count
36
- end
37
40
  # fall back to logical count if physical info is invalid
38
41
  ppc > 0 ? ppc : processor_count
39
42
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Parallel
2
- VERSION = Version = '1.20.0'
3
+ VERSION = Version = '1.22.0' # rubocop:disable Naming/ConstantName
3
4
  end
data/lib/parallel.rb CHANGED
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require 'rbconfig'
2
3
  require 'parallel/version'
3
4
  require 'parallel/processor_count'
@@ -12,7 +13,9 @@ module Parallel
12
13
 
13
14
  class Break < StandardError
14
15
  attr_reader :value
16
+
15
17
  def initialize(value = nil)
18
+ super()
16
19
  @value = value
17
20
  end
18
21
  end
@@ -22,6 +25,7 @@ module Parallel
22
25
 
23
26
  class UndumpableException < StandardError
24
27
  attr_reader :backtrace
28
+
25
29
  def initialize(original)
26
30
  super "#{original.class}: #{original.message}"
27
31
  @backtrace = original.backtrace
@@ -30,6 +34,7 @@ module Parallel
30
34
 
31
35
  class ExceptionWrapper
32
36
  attr_reader :exception
37
+
33
38
  def initialize(exception)
34
39
  # Remove the bindings stack added by the better_errors gem,
35
40
  # because it cannot be marshalled
@@ -40,7 +45,7 @@ module Parallel
40
45
  @exception =
41
46
  begin
42
47
  Marshal.dump(exception) && exception
43
- rescue
48
+ rescue StandardError
44
49
  UndumpableException.new(exception)
45
50
  end
46
51
  end
@@ -49,8 +54,11 @@ module Parallel
49
54
  class Worker
50
55
  attr_reader :pid, :read, :write
51
56
  attr_accessor :thread
57
+
52
58
  def initialize(read, write, pid)
53
- @read, @write, @pid = read, write, pid
59
+ @read = read
60
+ @write = write
61
+ @pid = pid
54
62
  end
55
63
 
56
64
  def stop
@@ -77,7 +85,7 @@ module Parallel
77
85
  rescue EOFError
78
86
  raise DeadWorker
79
87
  end
80
- raise result.exception if ExceptionWrapper === result
88
+ raise result.exception if result.is_a?(ExceptionWrapper)
81
89
  result
82
90
  end
83
91
 
@@ -144,7 +152,7 @@ module Parallel
144
152
  end
145
153
 
146
154
  def queue_wrapper(array)
147
- array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
155
+ array.respond_to?(:num_waiting) && array.respond_to?(:pop) && -> { array.pop(false) }
148
156
  end
149
157
  end
150
158
 
@@ -160,7 +168,7 @@ module Parallel
160
168
 
161
169
  if @to_be_killed.empty?
162
170
  old_interrupt = trap_interrupt(signal) do
163
- $stderr.puts 'Parallel execution interrupted, exiting ...'
171
+ warn 'Parallel execution interrupted, exiting ...'
164
172
  @to_be_killed.flatten.each { |pid| kill(pid) }
165
173
  end
166
174
  end
@@ -204,32 +212,30 @@ module Parallel
204
212
  end
205
213
 
206
214
  class << self
207
- def in_threads(options={:count => 2})
215
+ def in_threads(options = { count: 2 })
208
216
  threads = []
209
- count, _ = extract_count_from_options(options)
217
+ count, = extract_count_from_options(options)
210
218
 
211
219
  Thread.handle_interrupt(Exception => :never) do
212
- begin
213
- Thread.handle_interrupt(Exception => :immediate) do
214
- count.times do |i|
215
- threads << Thread.new { yield(i) }
216
- end
217
- threads.map(&:value)
220
+ Thread.handle_interrupt(Exception => :immediate) do
221
+ count.times do |i|
222
+ threads << Thread.new { yield(i) }
218
223
  end
219
- ensure
220
- threads.each(&:kill)
224
+ threads.map(&:value)
221
225
  end
226
+ ensure
227
+ threads.each(&:kill)
222
228
  end
223
229
  end
224
230
 
225
231
  def in_processes(options = {}, &block)
226
232
  count, options = extract_count_from_options(options)
227
233
  count ||= processor_count
228
- map(0...count, options.merge(:in_processes => count), &block)
234
+ map(0...count, options.merge(in_processes: count), &block)
229
235
  end
230
236
 
231
- def each(array, options={}, &block)
232
- map(array, options.merge(:preserve_results => false), &block)
237
+ def each(array, options = {}, &block)
238
+ map(array, options.merge(preserve_results: false), &block)
233
239
  end
234
240
 
235
241
  def any?(*args, &block)
@@ -242,8 +248,8 @@ module Parallel
242
248
  !!each(*args) { |*a| raise Kill unless block.call(*a) }
243
249
  end
244
250
 
245
- def each_with_index(array, options={}, &block)
246
- each(array, options.merge(:with_index => true), &block)
251
+ def each_with_index(array, options = {}, &block)
252
+ each(array, options.merge(with_index: true), &block)
247
253
  end
248
254
 
249
255
  def map(source, options = {}, &block)
@@ -251,13 +257,16 @@ module Parallel
251
257
  options[:mutex] = Mutex.new
252
258
 
253
259
  if options[:in_processes] && options[:in_threads]
254
- raise ArgumentError.new("Please specify only one of `in_processes` or `in_threads`.")
255
- elsif RUBY_PLATFORM =~ /java/ and not options[:in_processes]
260
+ raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
261
+ elsif RUBY_PLATFORM =~ (/java/) && !(options[:in_processes])
256
262
  method = :in_threads
257
263
  size = options[method] || processor_count
258
264
  elsif options[:in_threads]
259
265
  method = :in_threads
260
266
  size = options[method]
267
+ elsif options[:in_ractors]
268
+ method = :in_ractors
269
+ size = options[method]
261
270
  else
262
271
  method = :in_processes
263
272
  if Process.respond_to?(:fork)
@@ -278,9 +287,11 @@ module Parallel
278
287
  if size == 0
279
288
  work_direct(job_factory, options, &block)
280
289
  elsif method == :in_threads
281
- work_in_threads(job_factory, options.merge(:count => size), &block)
290
+ work_in_threads(job_factory, options.merge(count: size), &block)
291
+ elsif method == :in_ractors
292
+ work_in_ractors(job_factory, options.merge(count: size), &block)
282
293
  else
283
- work_in_processes(job_factory, options.merge(:count => size), &block)
294
+ work_in_processes(job_factory, options.merge(count: size), &block)
284
295
  end
285
296
 
286
297
  return result.value if result.is_a?(Break)
@@ -288,8 +299,8 @@ module Parallel
288
299
  options[:return_results] ? result : source
289
300
  end
290
301
 
291
- def map_with_index(array, options={}, &block)
292
- map(array, options.merge(:with_index => true), &block)
302
+ def map_with_index(array, options = {}, &block)
303
+ map(array, options.merge(with_index: true), &block)
293
304
  end
294
305
 
295
306
  def flat_map(*args, &block)
@@ -343,7 +354,7 @@ module Parallel
343
354
  call_with_index(item, index, options, &block)
344
355
  end
345
356
  end
346
- rescue
357
+ rescue StandardError
347
358
  exception = $!
348
359
  end
349
360
  exception || results
@@ -367,7 +378,7 @@ module Parallel
367
378
  call_with_index(item, index, options, &block)
368
379
  end
369
380
  results_mutex.synchronize { results[index] = result }
370
- rescue
381
+ rescue StandardError
371
382
  exception = $!
372
383
  end
373
384
  end
@@ -376,6 +387,72 @@ module Parallel
376
387
  exception || results
377
388
  end
378
389
 
390
+ def work_in_ractors(job_factory, options)
391
+ exception = nil
392
+ results = []
393
+ results_mutex = Mutex.new # arrays are not thread-safe on jRuby
394
+
395
+ callback = options[:ractor]
396
+ if block_given? || !callback
397
+ raise ArgumentError, "pass the code you want to execute as `ractor: [ClassName, :method_name]`"
398
+ end
399
+
400
+ # build
401
+ ractors = Array.new(options.fetch(:count)) do
402
+ Ractor.new do
403
+ loop do
404
+ got = receive
405
+ (klass, method_name), item, index = got
406
+ break if index == :break
407
+ begin
408
+ Ractor.yield [nil, klass.send(method_name, item), item, index]
409
+ rescue StandardError => e
410
+ Ractor.yield [e, nil, item, index]
411
+ end
412
+ end
413
+ end
414
+ end
415
+
416
+ # start
417
+ ractors.dup.each do |ractor|
418
+ if set = job_factory.next
419
+ item, index = set
420
+ instrument_start item, index, options
421
+ ractor.send [callback, item, index]
422
+ else
423
+ ractor.send([[nil, nil], nil, :break]) # stop the ractor
424
+ ractors.delete ractor
425
+ end
426
+ end
427
+
428
+ # replace with new items
429
+ while set = job_factory.next
430
+ item_next, index_next = set
431
+ done, (exception, result, item, index) = Ractor.select(*ractors)
432
+ if exception
433
+ ractors.delete done
434
+ break
435
+ end
436
+ instrument_finish item, index, result, options
437
+ results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
438
+
439
+ instrument_start item_next, index_next, options
440
+ done.send([callback, item_next, index_next])
441
+ end
442
+
443
+ # finish
444
+ ractors.each do |ractor|
445
+ (new_exception, result, item, index) = ractor.take
446
+ exception ||= new_exception
447
+ next if new_exception
448
+ instrument_finish item, index, result, options
449
+ results_mutex.synchronize { results[index] = (options[:preserve_results] == false ? nil : result) }
450
+ ractor.send([[nil, nil], nil, :break]) # stop the ractor
451
+ end
452
+
453
+ exception || results
454
+ end
455
+
379
456
  def work_in_processes(job_factory, options, &blk)
380
457
  workers = create_workers(job_factory, options, &blk)
381
458
  results = []
@@ -407,9 +484,9 @@ module Parallel
407
484
  results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
408
485
  rescue StandardError => e
409
486
  exception = e
410
- if Kill === exception
487
+ if exception.is_a?(Kill)
411
488
  (workers - [worker]).each do |w|
412
- w.thread.kill if w.thread
489
+ w.thread&.kill
413
490
  UserInterruptHandler.kill(w.pid)
414
491
  end
415
492
  end
@@ -420,18 +497,19 @@ module Parallel
420
497
  end
421
498
  end
422
499
  end
500
+
423
501
  exception || results
424
502
  end
425
503
 
426
- def replace_worker(job_factory, workers, i, options, blk)
504
+ def replace_worker(job_factory, workers, index, options, blk)
427
505
  options[:mutex].synchronize do
428
506
  # old worker is no longer used ... stop it
429
- worker = workers[i]
507
+ worker = workers[index]
430
508
  worker.stop
431
509
 
432
510
  # create a new replacement worker
433
511
  running = workers - [worker]
434
- workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk)
512
+ workers[index] = worker(job_factory, options.merge(started_workers: running, worker_number: index), &blk)
435
513
  end
436
514
  end
437
515
 
@@ -473,14 +551,17 @@ module Parallel
473
551
  until read.eof?
474
552
  data = Marshal.load(read)
475
553
  item, index = job_factory.unpack(data)
476
- result = begin
477
- call_with_index(item, index, options, &block)
478
- # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
479
- rescue NoMemoryError, SignalException, Interrupt, SystemExit
480
- raise $!
481
- rescue Exception
482
- ExceptionWrapper.new($!)
483
- end
554
+
555
+ result =
556
+ begin
557
+ call_with_index(item, index, options, &block)
558
+ # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
559
+ rescue NoMemoryError, SignalException, Interrupt, SystemExit # rubocop:disable Lint/ShadowedException
560
+ raise $!
561
+ rescue Exception # # rubocop:disable Lint/RescueException
562
+ ExceptionWrapper.new($!)
563
+ end
564
+
484
565
  begin
485
566
  Marshal.dump(result, write)
486
567
  rescue Errno::EPIPE
@@ -503,21 +584,29 @@ module Parallel
503
584
  def call_with_index(item, index, options, &block)
504
585
  args = [item]
505
586
  args << index if options[:with_index]
587
+ results = block.call(*args)
506
588
  if options[:return_results]
507
- block.call(*args)
589
+ results
508
590
  else
509
- block.call(*args)
510
591
  nil # avoid GC overhead of passing large results around
511
592
  end
512
593
  end
513
594
 
514
595
  def with_instrumentation(item, index, options)
515
- on_start = options[:start]
516
- on_finish = options[:finish]
517
- options[:mutex].synchronize { on_start.call(item, index) } if on_start
596
+ instrument_start(item, index, options)
518
597
  result = yield
519
- options[:mutex].synchronize { on_finish.call(item, index, result) } if on_finish
598
+ instrument_finish(item, index, result, options)
520
599
  result unless options[:preserve_results] == false
521
600
  end
601
+
602
+ def instrument_finish(item, index, result, options)
603
+ return unless on_finish = options[:finish]
604
+ options[:mutex].synchronize { on_finish.call(item, index, result) }
605
+ end
606
+
607
+ def instrument_start(item, index, options)
608
+ return unless on_start = options[:start]
609
+ options[:mutex].synchronize { on_start.call(item, index) }
610
+ end
522
611
  end
523
612
  end
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: parallel
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.20.0
4
+ version: 1.22.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michael Grosser
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-07 00:00:00.000000000 Z
11
+ date: 2022-03-21 00:00:00.000000000 Z
12
12
  dependencies: []
13
- description:
13
+ description:
14
14
  email: michael@grosser.it
15
15
  executables: []
16
16
  extensions: []
@@ -25,10 +25,10 @@ licenses:
25
25
  - MIT
26
26
  metadata:
27
27
  bug_tracker_uri: https://github.com/grosser/parallel/issues
28
- documentation_uri: https://github.com/grosser/parallel/blob/v1.20.0/Readme.md
29
- source_code_uri: https://github.com/grosser/parallel/tree/v1.20.0
28
+ documentation_uri: https://github.com/grosser/parallel/blob/v1.22.0/Readme.md
29
+ source_code_uri: https://github.com/grosser/parallel/tree/v1.22.0
30
30
  wiki_uri: https://github.com/grosser/parallel/wiki
31
- post_install_message:
31
+ post_install_message:
32
32
  rdoc_options: []
33
33
  require_paths:
34
34
  - lib
@@ -43,8 +43,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
43
43
  - !ruby/object:Gem::Version
44
44
  version: '0'
45
45
  requirements: []
46
- rubygems_version: 3.1.3
47
- signing_key:
46
+ rubygems_version: 3.3.3
47
+ signing_key:
48
48
  specification_version: 4
49
49
  summary: Run any kind of code in parallel processes
50
50
  test_files: []