parallel 1.14.0 → 1.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 587c7508b0e181ee03b60b9641ac9afa3feaa09a2fe3af0e6fc52d11ecf22338
4
- data.tar.gz: 9e4fafeb97a74277aa33276dbb4383ee5227b26553eca719af4a5ca4fd8daa0d
3
+ metadata.gz: 4dd167566b23262ceacbd640fda95c5e8d0d8149cc430f51ada53765a927cb18
4
+ data.tar.gz: 6a21c00b1109d5c665a471aec4a7af7d213ded2fc9123132b6576ee65ceb1758
5
5
  SHA512:
6
- metadata.gz: 411d4eb1d0f9387327c7778588eb2337280031739c4237b9123ab1ad995c7f9ec53312c2887e65df61203451061a9b318d2e23ea08a94ff727714ae2067a4c78
7
- data.tar.gz: 57719ea9cd17ce0b82470aeb1128344fda823ce7f28779aa2ff6cc4f7581583bf3db2e86418f59d9926fcc7e8124edb0643bf95930cdf6410f73d586ea0c47f9
6
+ metadata.gz: 1f8a810c3b7fa77e8dd2b2539238b856b9b4ea40b5df49cf2cc4af2afbd2a73d9b6f34baab614c41e7b3b8bfe1917413bfbd014f647c5b4b96e7243d5433523d
7
+ data.tar.gz: 0bef1c7dc19b4738e228985512ee6cce1915f67ee29161b743821f69366d561ed8c3c5cbd4d9e6829342108fe9f10329563770d1978a660a04c58c0d9e2c8461
@@ -1,41 +1,42 @@
1
+ # frozen_string_literal: true
1
2
  require 'etc'
2
3
 
3
4
  module Parallel
5
+ # TODO: inline this method into parallel.rb and kill physical_processor_count in next major release
4
6
  module ProcessorCount
5
- # Number of processors seen by the OS and used for process scheduling. It's just wrapper for Etc.nprocessors
7
+ # Number of processors seen by the OS, used for process scheduling
6
8
  def processor_count
7
- @processor_count ||= begin
8
- Etc.nprocessors
9
- end
9
+ @processor_count ||= Integer(ENV['PARALLEL_PROCESSOR_COUNT'] || Etc.nprocessors)
10
10
  end
11
11
 
12
12
  # Number of physical processor cores on the current system.
13
- #
14
13
  def physical_processor_count
15
14
  @physical_processor_count ||= begin
16
- ppc = case RbConfig::CONFIG["target_os"]
17
- when /darwin1/
18
- IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
19
- when /linux/
20
- cores = {} # unique physical ID / core ID combinations
21
- phy = 0
22
- IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
23
- if ln.start_with?("physical")
24
- phy = ln[/\d+/]
25
- elsif ln.start_with?("core")
26
- cid = phy + ":" + ln[/\d+/]
27
- cores[cid] = true if not cores[cid]
15
+ ppc =
16
+ case RbConfig::CONFIG["target_os"]
17
+ when /darwin[12]/
18
+ IO.popen("/usr/sbin/sysctl -n hw.physicalcpu").read.to_i
19
+ when /linux/
20
+ cores = {} # unique physical ID / core ID combinations
21
+ phy = 0
22
+ IO.read("/proc/cpuinfo").scan(/^physical id.*|^core id.*/) do |ln|
23
+ if ln.start_with?("physical")
24
+ phy = ln[/\d+/]
25
+ elsif ln.start_with?("core")
26
+ cid = "#{phy}:#{ln[/\d+/]}"
27
+ cores[cid] = true unless cores[cid]
28
+ end
28
29
  end
30
+ cores.count
31
+ when /mswin|mingw/
32
+ require 'win32ole'
33
+ result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
34
+ "select NumberOfCores from Win32_Processor"
35
+ )
36
+ result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
37
+ else
38
+ processor_count
29
39
  end
30
- cores.count
31
- when /mswin|mingw/
32
- require 'win32ole'
33
- result_set = WIN32OLE.connect("winmgmts://").ExecQuery(
34
- "select NumberOfCores from Win32_Processor")
35
- result_set.to_enum.collect(&:NumberOfCores).reduce(:+)
36
- else
37
- processor_count
38
- end
39
40
  # fall back to logical count if physical info is invalid
40
41
  ppc > 0 ? ppc : processor_count
41
42
  end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Parallel
2
- VERSION = Version = '1.14.0'
3
+ VERSION = Version = '1.21.0' # rubocop:disable Naming/ConstantName
3
4
  end
data/lib/parallel.rb CHANGED
@@ -1,31 +1,40 @@
1
+ # frozen_string_literal: true
1
2
  require 'rbconfig'
2
3
  require 'parallel/version'
3
4
  require 'parallel/processor_count'
4
5
 
5
6
  module Parallel
6
- extend Parallel::ProcessorCount
7
+ extend ProcessorCount
8
+
9
+ Stop = Object.new.freeze
7
10
 
8
11
  class DeadWorker < StandardError
9
12
  end
10
13
 
11
14
  class Break < StandardError
15
+ attr_reader :value
16
+
17
+ def initialize(value = nil)
18
+ super()
19
+ @value = value
20
+ end
12
21
  end
13
22
 
14
- class Kill < StandardError
23
+ class Kill < Break
15
24
  end
16
25
 
17
26
  class UndumpableException < StandardError
18
27
  attr_reader :backtrace
28
+
19
29
  def initialize(original)
20
30
  super "#{original.class}: #{original.message}"
21
31
  @backtrace = original.backtrace
22
32
  end
23
33
  end
24
34
 
25
- Stop = Object.new
26
-
27
35
  class ExceptionWrapper
28
36
  attr_reader :exception
37
+
29
38
  def initialize(exception)
30
39
  # Remove the bindings stack added by the better_errors gem,
31
40
  # because it cannot be marshalled
@@ -36,7 +45,7 @@ module Parallel
36
45
  @exception =
37
46
  begin
38
47
  Marshal.dump(exception) && exception
39
- rescue
48
+ rescue StandardError
40
49
  UndumpableException.new(exception)
41
50
  end
42
51
  end
@@ -45,8 +54,11 @@ module Parallel
45
54
  class Worker
46
55
  attr_reader :pid, :read, :write
47
56
  attr_accessor :thread
57
+
48
58
  def initialize(read, write, pid)
49
- @read, @write, @pid = read, write, pid
59
+ @read = read
60
+ @write = write
61
+ @pid = pid
50
62
  end
51
63
 
52
64
  def stop
@@ -73,7 +85,7 @@ module Parallel
73
85
  rescue EOFError
74
86
  raise DeadWorker
75
87
  end
76
- raise result.exception if ExceptionWrapper === result
88
+ raise result.exception if result.is_a?(ExceptionWrapper)
77
89
  result
78
90
  end
79
91
 
@@ -102,7 +114,7 @@ module Parallel
102
114
  item, index = @mutex.synchronize do
103
115
  return if @stopped
104
116
  item = @lambda.call
105
- @stopped = (item == Parallel::Stop)
117
+ @stopped = (item == Stop)
106
118
  return if @stopped
107
119
  [item, @index += 1]
108
120
  end
@@ -140,7 +152,7 @@ module Parallel
140
152
  end
141
153
 
142
154
  def queue_wrapper(array)
143
- array.respond_to?(:num_waiting) && array.respond_to?(:pop) && lambda { array.pop(false) }
155
+ array.respond_to?(:num_waiting) && array.respond_to?(:pop) && -> { array.pop(false) }
144
156
  end
145
157
  end
146
158
 
@@ -156,7 +168,7 @@ module Parallel
156
168
 
157
169
  if @to_be_killed.empty?
158
170
  old_interrupt = trap_interrupt(signal) do
159
- $stderr.puts 'Parallel execution interrupted, exiting ...'
171
+ warn 'Parallel execution interrupted, exiting ...'
160
172
  @to_be_killed.flatten.each { |pid| kill(pid) }
161
173
  end
162
174
  end
@@ -200,35 +212,44 @@ module Parallel
200
212
  end
201
213
 
202
214
  class << self
203
- def in_threads(options={:count => 2})
204
- count, _ = extract_count_from_options(options)
205
- Array.new(count) do |i|
206
- Thread.new { yield(i) }
207
- end.map!(&:value)
215
+ def in_threads(options = { count: 2 })
216
+ threads = []
217
+ count, = extract_count_from_options(options)
218
+
219
+ Thread.handle_interrupt(Exception => :never) do
220
+ Thread.handle_interrupt(Exception => :immediate) do
221
+ count.times do |i|
222
+ threads << Thread.new { yield(i) }
223
+ end
224
+ threads.map(&:value)
225
+ end
226
+ ensure
227
+ threads.each(&:kill)
228
+ end
208
229
  end
209
230
 
210
231
  def in_processes(options = {}, &block)
211
232
  count, options = extract_count_from_options(options)
212
233
  count ||= processor_count
213
- map(0...count, options.merge(:in_processes => count), &block)
234
+ map(0...count, options.merge(in_processes: count), &block)
214
235
  end
215
236
 
216
- def each(array, options={}, &block)
217
- map(array, options.merge(:preserve_results => false), &block)
237
+ def each(array, options = {}, &block)
238
+ map(array, options.merge(preserve_results: false), &block)
218
239
  end
219
240
 
220
241
  def any?(*args, &block)
221
242
  raise "You must provide a block when calling #any?" if block.nil?
222
- !each(*args) { |*a| raise Parallel::Kill if block.call(*a) }
243
+ !each(*args) { |*a| raise Kill if block.call(*a) }
223
244
  end
224
245
 
225
246
  def all?(*args, &block)
226
247
  raise "You must provide a block when calling #all?" if block.nil?
227
- !!each(*args) { |*a| raise Parallel::Kill unless block.call(*a) }
248
+ !!each(*args) { |*a| raise Kill unless block.call(*a) }
228
249
  end
229
250
 
230
- def each_with_index(array, options={}, &block)
231
- each(array, options.merge(:with_index => true), &block)
251
+ def each_with_index(array, options = {}, &block)
252
+ each(array, options.merge(with_index: true), &block)
232
253
  end
233
254
 
234
255
  def map(source, options = {}, &block)
@@ -236,8 +257,8 @@ module Parallel
236
257
  options[:mutex] = Mutex.new
237
258
 
238
259
  if options[:in_processes] && options[:in_threads]
239
- raise ArgumentError.new("Please specify only one of `in_processes` or `in_threads`.")
240
- elsif RUBY_PLATFORM =~ /java/ and not options[:in_processes]
260
+ raise ArgumentError, "Please specify only one of `in_processes` or `in_threads`."
261
+ elsif RUBY_PLATFORM =~ (/java/) && !(options[:in_processes])
241
262
  method = :in_threads
242
263
  size = options[method] || processor_count
243
264
  elsif options[:in_threads]
@@ -259,20 +280,22 @@ module Parallel
259
280
  options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
260
281
  add_progress_bar!(job_factory, options)
261
282
 
262
- results = if size == 0
263
- work_direct(job_factory, options, &block)
264
- elsif method == :in_threads
265
- work_in_threads(job_factory, options.merge(:count => size), &block)
266
- else
267
- work_in_processes(job_factory, options.merge(:count => size), &block)
268
- end
269
- if results
270
- options[:return_results] ? results : source
271
- end
283
+ result =
284
+ if size == 0
285
+ work_direct(job_factory, options, &block)
286
+ elsif method == :in_threads
287
+ work_in_threads(job_factory, options.merge(count: size), &block)
288
+ else
289
+ work_in_processes(job_factory, options.merge(count: size), &block)
290
+ end
291
+
292
+ return result.value if result.is_a?(Break)
293
+ raise result if result.is_a?(Exception)
294
+ options[:return_results] ? result : source
272
295
  end
273
296
 
274
- def map_with_index(array, options={}, &block)
275
- map(array, options.merge(:with_index => true), &block)
297
+ def map_with_index(array, options = {}, &block)
298
+ map(array, options.merge(with_index: true), &block)
276
299
  end
277
300
 
278
301
  def flat_map(*args, &block)
@@ -283,6 +306,7 @@ module Parallel
283
306
  Thread.current[:parallel_worker_number]
284
307
  end
285
308
 
309
+ # TODO: this does not work when doing threads in forks, so should remove and yield the number instead if needed
286
310
  def worker_number=(worker_num)
287
311
  Thread.current[:parallel_worker_number] = worker_num
288
312
  end
@@ -325,10 +349,10 @@ module Parallel
325
349
  call_with_index(item, index, options, &block)
326
350
  end
327
351
  end
328
- rescue
352
+ rescue StandardError
329
353
  exception = $!
330
354
  end
331
- handle_exception(exception, results)
355
+ exception || results
332
356
  ensure
333
357
  self.worker_number = nil
334
358
  end
@@ -349,21 +373,17 @@ module Parallel
349
373
  call_with_index(item, index, options, &block)
350
374
  end
351
375
  results_mutex.synchronize { results[index] = result }
352
- rescue
376
+ rescue StandardError
353
377
  exception = $!
354
378
  end
355
379
  end
356
380
  end
357
381
 
358
- handle_exception(exception, results)
382
+ exception || results
359
383
  end
360
384
 
361
385
  def work_in_processes(job_factory, options, &blk)
362
- workers = if options[:isolation]
363
- [] # we create workers per job and not beforehand
364
- else
365
- create_workers(job_factory, options, &blk)
366
- end
386
+ workers = create_workers(job_factory, options, &blk)
367
387
  results = []
368
388
  results_mutex = Mutex.new # arrays are not thread-safe
369
389
  exception = nil
@@ -371,6 +391,8 @@ module Parallel
371
391
  UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
372
392
  in_threads(options) do |i|
373
393
  worker = workers[i]
394
+ worker.thread = Thread.current
395
+ worked = false
374
396
 
375
397
  begin
376
398
  loop do
@@ -379,44 +401,43 @@ module Parallel
379
401
  break unless index
380
402
 
381
403
  if options[:isolation]
382
- worker = replace_worker(job_factory, workers, i, options, blk)
404
+ worker = replace_worker(job_factory, workers, i, options, blk) if worked
405
+ worked = true
406
+ worker.thread = Thread.current
383
407
  end
384
408
 
385
- worker.thread = Thread.current
386
-
387
409
  begin
388
410
  result = with_instrumentation item, index, options do
389
411
  worker.work(job_factory.pack(item, index))
390
412
  end
391
413
  results_mutex.synchronize { results[index] = result } # arrays are not threads safe on jRuby
392
- rescue
393
- exception = $!
394
- if Parallel::Kill === exception
414
+ rescue StandardError => e
415
+ exception = e
416
+ if exception.is_a?(Kill)
395
417
  (workers - [worker]).each do |w|
396
- w.thread.kill unless w.thread.nil?
418
+ w.thread&.kill
397
419
  UserInterruptHandler.kill(w.pid)
398
420
  end
399
421
  end
400
422
  end
401
423
  end
402
424
  ensure
403
- worker.stop if worker
425
+ worker.stop
404
426
  end
405
427
  end
406
428
  end
407
-
408
- handle_exception(exception, results)
429
+ exception || results
409
430
  end
410
431
 
411
- def replace_worker(job_factory, workers, i, options, blk)
432
+ def replace_worker(job_factory, workers, index, options, blk)
412
433
  options[:mutex].synchronize do
413
434
  # old worker is no longer used ... stop it
414
- worker = workers[i]
415
- worker.stop if worker
435
+ worker = workers[index]
436
+ worker.stop
416
437
 
417
438
  # create a new replacement worker
418
439
  running = workers - [worker]
419
- workers[i] = worker(job_factory, options.merge(started_workers: running, worker_number: i), &blk)
440
+ workers[index] = worker(job_factory, options.merge(started_workers: running, worker_number: index), &blk)
420
441
  end
421
442
  end
422
443
 
@@ -458,21 +479,25 @@ module Parallel
458
479
  until read.eof?
459
480
  data = Marshal.load(read)
460
481
  item, index = job_factory.unpack(data)
461
- result = begin
462
- call_with_index(item, index, options, &block)
463
- rescue
464
- ExceptionWrapper.new($!)
482
+
483
+ result =
484
+ begin
485
+ call_with_index(item, index, options, &block)
486
+ # https://github.com/rspec/rspec-support/blob/673133cdd13b17077b3d88ece8d7380821f8d7dc/lib/rspec/support.rb#L132-L140
487
+ rescue NoMemoryError, SignalException, Interrupt, SystemExit # rubocop:disable Lint/ShadowedException
488
+ raise $!
489
+ rescue Exception # # rubocop:disable Lint/RescueException
490
+ ExceptionWrapper.new($!)
491
+ end
492
+
493
+ begin
494
+ Marshal.dump(result, write)
495
+ rescue Errno::EPIPE
496
+ return # parent thread already dead
465
497
  end
466
- Marshal.dump(result, write)
467
498
  end
468
499
  end
469
500
 
470
- def handle_exception(exception, results)
471
- return nil if [Parallel::Break, Parallel::Kill].include? exception.class
472
- raise exception if exception
473
- results
474
- end
475
-
476
501
  # options is either a Integer or a Hash with :count
477
502
  def extract_count_from_options(options)
478
503
  if options.is_a?(Hash)
@@ -487,10 +512,10 @@ module Parallel
487
512
  def call_with_index(item, index, options, &block)
488
513
  args = [item]
489
514
  args << index if options[:with_index]
515
+ results = block.call(*args)
490
516
  if options[:return_results]
491
- block.call(*args)
517
+ results
492
518
  else
493
- block.call(*args)
494
519
  nil # avoid GC overhead of passing large results around
495
520
  end
496
521
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: parallel
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.14.0
4
+ version: 1.21.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michael Grosser
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-02-25 00:00:00.000000000 Z
11
+ date: 2021-09-13 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description:
14
14
  email: michael@grosser.it
@@ -23,7 +23,11 @@ files:
23
23
  homepage: https://github.com/grosser/parallel
24
24
  licenses:
25
25
  - MIT
26
- metadata: {}
26
+ metadata:
27
+ bug_tracker_uri: https://github.com/grosser/parallel/issues
28
+ documentation_uri: https://github.com/grosser/parallel/blob/v1.21.0/Readme.md
29
+ source_code_uri: https://github.com/grosser/parallel/tree/v1.21.0
30
+ wiki_uri: https://github.com/grosser/parallel/wiki
27
31
  post_install_message:
28
32
  rdoc_options: []
29
33
  require_paths:
@@ -32,15 +36,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
32
36
  requirements:
33
37
  - - ">="
34
38
  - !ruby/object:Gem::Version
35
- version: '2.2'
39
+ version: '2.5'
36
40
  required_rubygems_version: !ruby/object:Gem::Requirement
37
41
  requirements:
38
42
  - - ">="
39
43
  - !ruby/object:Gem::Version
40
44
  version: '0'
41
45
  requirements: []
42
- rubyforge_project:
43
- rubygems_version: 2.7.6
46
+ rubygems_version: 3.2.16
44
47
  signing_key:
45
48
  specification_version: 4
46
49
  summary: Run any kind of code in parallel processes