parallel 1.4.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. checksums.yaml +4 -4
  2. data/lib/parallel.rb +112 -119
  3. data/lib/parallel/version.rb +1 -1
  4. metadata +10 -10
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ca1dbd921b5d79d0ec3ece5ad278ba4e5f06416f
4
- data.tar.gz: aba073a0c18c20be741892f32ba0ff7c2421a50a
3
+ metadata.gz: 4ebdef905e948776369686271708c85a60b4c63c
4
+ data.tar.gz: da64471e0d337ca3a778670efed6e0c2eda454af
5
5
  SHA512:
6
- metadata.gz: 8ed2cc642c77c45eb234d18f9a39a53891d16a3f55cde102a91167e8e1b0a0fe1120f9e33da692b08fabd676265d8c471fc6668f64b714fc6f4d93410cea4723
7
- data.tar.gz: 74b5c077ffd1ae61e3bf4e1b8cf8f5662058b2bbe73adf8882f212a1c72a183545767eb86e2a7647ca1dbba422beabab5b76d5822dc1f3537a77be1f46ff4bb6
6
+ metadata.gz: 9677d167fc5b5796630d9654bc5dd82b1e950aa0a69537dcdb1332326ac95399d91638827cb69a00be79123db43514586d30037a92a9572dba84ff63745b4ff1
7
+ data.tar.gz: 4dca43cfa0a92caf06ca45b95de2a3b28e781b6e0300b94c14fb4703a1f4a264d73c10c2d11b2e92c344e92c998afeb60245bd1aafb100c3e7d262ca88fd665f
data/lib/parallel.rb CHANGED
@@ -16,8 +16,6 @@ module Parallel
16
16
 
17
17
  Stop = Object.new
18
18
 
19
- INTERRUPT_SIGNAL = :SIGINT
20
-
21
19
  class ExceptionWrapper
22
20
  attr_reader :exception
23
21
  def initialize(exception)
@@ -65,31 +63,15 @@ module Parallel
65
63
  end
66
64
  end
67
65
 
68
- class ItemWrapper
69
- def initialize(array, mutex)
70
- @lambda = (array.respond_to?(:call) && array) || queue_wrapper(array)
71
- @items = array.to_a unless @lambda # turn Range and other Enumerable-s into an Array
66
+ class JobFactory
67
+ def initialize(source, mutex)
68
+ @lambda = (source.respond_to?(:call) && source) || queue_wrapper(source)
69
+ @source = source.to_a unless @lambda # turn Range and other Enumerable-s into an Array
72
70
  @mutex = mutex
73
71
  @index = -1
74
72
  @stopped = false
75
73
  end
76
74
 
77
- def producer?
78
- @lambda
79
- end
80
-
81
- def each_with_index(&block)
82
- if producer?
83
- loop do
84
- item, index = self.next
85
- break unless index
86
- yield(item, index)
87
- end
88
- else
89
- @items.each_with_index(&block)
90
- end
91
- end
92
-
93
75
  def next
94
76
  if producer?
95
77
  # - index and item stay in sync
@@ -104,13 +86,17 @@ module Parallel
104
86
  else
105
87
  index = @mutex.synchronize { @index += 1 }
106
88
  return if index >= size
107
- item = @items[index]
89
+ item = @source[index]
108
90
  end
109
91
  [item, index]
110
92
  end
111
93
 
112
94
  def size
113
- @items.size
95
+ if producer?
96
+ Float::INFINITY
97
+ else
98
+ @source.size
99
+ end
114
100
  end
115
101
 
116
102
  def pack(item, index)
@@ -118,7 +104,13 @@ module Parallel
118
104
  end
119
105
 
120
106
  def unpack(data)
121
- producer? ? data : [@items[data], data]
107
+ producer? ? data : [@source[data], data]
108
+ end
109
+
110
+ private
111
+
112
+ def producer?
113
+ @lambda
122
114
  end
123
115
 
124
116
  def queue_wrapper(array)
@@ -126,6 +118,68 @@ module Parallel
126
118
  end
127
119
  end
128
120
 
121
+ class UserInterruptHandler
122
+ INTERRUPT_SIGNAL = :SIGINT
123
+
124
+ class << self
125
+ # kill all these pids or threads if user presses Ctrl+c
126
+ def kill_on_ctrl_c(things, options)
127
+ return yield if RUBY_ENGINE == "jruby"
128
+ @to_be_killed ||= []
129
+ old_interrupt = nil
130
+ signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
131
+
132
+ if @to_be_killed.empty?
133
+ old_interrupt = trap_interrupt(signal) do
134
+ $stderr.puts 'Parallel execution interrupted, exiting ...'
135
+ @to_be_killed.flatten.compact.each { |thing| kill(thing) }
136
+ end
137
+ end
138
+
139
+ @to_be_killed << things
140
+
141
+ yield
142
+ ensure
143
+ @to_be_killed.pop # free threads for GC and do not kill pids that could be used for new processes
144
+ restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
145
+ end
146
+
147
+ def kill(thing)
148
+ if thing.is_a?(Thread)
149
+ thing.kill
150
+ else
151
+ begin
152
+ Process.kill(:KILL, thing)
153
+ rescue Errno::ESRCH
154
+ # some linux systems already automatically killed the children at this point
155
+ # so we just ignore them not being there
156
+ end
157
+ end
158
+ end
159
+
160
+ private
161
+
162
+ def trap_interrupt(signal)
163
+ old = Signal.trap signal, 'IGNORE'
164
+
165
+ Signal.trap signal do
166
+ yield
167
+ if old == "DEFAULT"
168
+ raise Interrupt
169
+ else
170
+ old.call
171
+ end
172
+ end
173
+
174
+ old
175
+ end
176
+
177
+ def restore_interrupt(old, signal)
178
+ Signal.trap signal, old
179
+ end
180
+ end
181
+ end
182
+
129
183
  class << self
130
184
  def in_threads(options={:count => 2})
131
185
  count, options = extract_count_from_options(options)
@@ -139,7 +193,7 @@ module Parallel
139
193
  end
140
194
  end
141
195
 
142
- kill_on_ctrl_c(threads, options) { wait_for_threads(threads) }
196
+ UserInterruptHandler.kill_on_ctrl_c(threads, options) { wait_for_threads(threads) }
143
197
 
144
198
  out
145
199
  end
@@ -159,7 +213,7 @@ module Parallel
159
213
  each(array, options.merge(:with_index => true), &block)
160
214
  end
161
215
 
162
- def map(array, options = {}, &block)
216
+ def map(source, options = {}, &block)
163
217
  options[:mutex] = Mutex.new
164
218
 
165
219
  if RUBY_PLATFORM =~ /java/ and not options[:in_processes]
@@ -173,24 +227,23 @@ module Parallel
173
227
  if Process.respond_to?(:fork)
174
228
  size = options[method] || processor_count
175
229
  else
176
- $stderr.puts "Warning: Process.fork is not supported by this Ruby"
230
+ warn "Process.fork is not supported by this Ruby"
177
231
  size = 0
178
232
  end
179
233
  end
180
234
 
181
- items = ItemWrapper.new(array, options[:mutex])
182
-
183
- size = [items.producer? ? size : items.size, size].min
235
+ job_factory = JobFactory.new(source, options[:mutex])
236
+ size = [job_factory.size, size].min
184
237
 
185
238
  options[:return_results] = (options[:preserve_results] != false || !!options[:finish])
186
- add_progress_bar!(items, options)
239
+ add_progress_bar!(job_factory, options)
187
240
 
188
241
  if size == 0
189
- work_direct(items, options, &block)
242
+ work_direct(job_factory, options, &block)
190
243
  elsif method == :in_threads
191
- work_in_threads(items, options.merge(:count => size), &block)
244
+ work_in_threads(job_factory, options.merge(:count => size), &block)
192
245
  else
193
- work_in_processes(items, options.merge(:count => size), &block)
246
+ work_in_processes(job_factory, options.merge(:count => size), &block)
194
247
  end
195
248
  end
196
249
 
@@ -200,9 +253,9 @@ module Parallel
200
253
 
201
254
  private
202
255
 
203
- def add_progress_bar!(items, options)
256
+ def add_progress_bar!(job_factory, options)
204
257
  if progress_options = options[:progress]
205
- raise "Progressbar and producers don't mix" if items.producer?
258
+ raise "Progressbar can only be used with array like items" if job_factory.size == Float::INFINITY
206
259
  require 'ruby-progressbar'
207
260
 
208
261
  if progress_options.respond_to? :to_str
@@ -210,7 +263,7 @@ module Parallel
210
263
  end
211
264
 
212
265
  progress_options = {
213
- total: items.size,
266
+ total: job_factory.size,
214
267
  format: '%t |%E | %B | %a'
215
268
  }.merge(progress_options)
216
269
 
@@ -223,10 +276,10 @@ module Parallel
223
276
  end
224
277
  end
225
278
 
226
-
227
- def work_direct(items, options, &block)
279
+ def work_direct(job_factory, options, &block)
228
280
  results = []
229
- items.each_with_index do |item, index|
281
+ while set = job_factory.next
282
+ item, index = set
230
283
  results << with_instrumentation(item, index, options) do
231
284
  call_with_index(item, index, options, &block)
232
285
  end
@@ -234,24 +287,20 @@ module Parallel
234
287
  results
235
288
  end
236
289
 
237
- def work_in_threads(items, options, &block)
290
+ def work_in_threads(job_factory, options, &block)
238
291
  results = []
239
292
  exception = nil
240
293
 
241
294
  in_threads(options) do
242
- # as long as there are more items, work on one of them
243
- loop do
244
- break if exception
245
- item, index = items.next
246
- break unless index
247
-
295
+ # as long as there are more jobs, work on one of them
296
+ while !exception && set = job_factory.next
248
297
  begin
298
+ item, index = set
249
299
  results[index] = with_instrumentation item, index, options do
250
300
  call_with_index(item, index, options, &block)
251
301
  end
252
302
  rescue StandardError => e
253
303
  exception = e
254
- break
255
304
  end
256
305
  end
257
306
  end
@@ -259,12 +308,12 @@ module Parallel
259
308
  handle_exception(exception, results)
260
309
  end
261
310
 
262
- def work_in_processes(items, options, &blk)
263
- workers = create_workers(items, options, &blk)
311
+ def work_in_processes(job_factory, options, &blk)
312
+ workers = create_workers(job_factory, options, &blk)
264
313
  results = []
265
314
  exception = nil
266
315
 
267
- kill_on_ctrl_c(workers.map(&:pid), options) do
316
+ UserInterruptHandler.kill_on_ctrl_c(workers.map(&:pid), options) do
268
317
  in_threads(options) do |i|
269
318
  worker = workers[i]
270
319
  worker.thread = Thread.current
@@ -272,19 +321,19 @@ module Parallel
272
321
  begin
273
322
  loop do
274
323
  break if exception
275
- item, index = items.next
324
+ item, index = job_factory.next
276
325
  break unless index
277
326
 
278
327
  begin
279
328
  results[index] = with_instrumentation item, index, options do
280
- worker.work(items.pack(item, index))
329
+ worker.work(job_factory.pack(item, index))
281
330
  end
282
331
  rescue StandardError => e
283
332
  exception = e
284
333
  if Parallel::Kill === exception
285
334
  (workers - [worker]).each do |w|
286
- kill_that_thing!(w.thread)
287
- kill_that_thing!(w.pid)
335
+ UserInterruptHandler.kill(w.thread)
336
+ UserInterruptHandler.kill(w.pid)
288
337
  end
289
338
  end
290
339
  end
@@ -299,18 +348,15 @@ module Parallel
299
348
  handle_exception(exception, results)
300
349
  end
301
350
 
302
- def create_workers(items, options, &block)
351
+ def create_workers(job_factory, options, &block)
303
352
  workers = []
304
353
  Array.new(options[:count]).each do
305
- workers << worker(items, options.merge(:started_workers => workers), &block)
354
+ workers << worker(job_factory, options.merge(:started_workers => workers), &block)
306
355
  end
307
356
  workers
308
357
  end
309
358
 
310
- def worker(items, options, &block)
311
- # use less memory on REE
312
- GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
313
-
359
+ def worker(job_factory, options, &block)
314
360
  child_read, parent_write = IO.pipe
315
361
  parent_read, child_write = IO.pipe
316
362
 
@@ -321,7 +367,7 @@ module Parallel
321
367
  parent_write.close
322
368
  parent_read.close
323
369
 
324
- process_incoming_jobs(child_read, child_write, items, options, &block)
370
+ process_incoming_jobs(child_read, child_write, job_factory, options, &block)
325
371
  ensure
326
372
  child_read.close
327
373
  child_write.close
@@ -334,10 +380,10 @@ module Parallel
334
380
  Worker.new(parent_read, parent_write, pid)
335
381
  end
336
382
 
337
- def process_incoming_jobs(read, write, items, options, &block)
338
- while !read.eof?
383
+ def process_incoming_jobs(read, write, job_factory, options, &block)
384
+ until read.eof?
339
385
  data = Marshal.load(read)
340
- item, index = items.unpack(data)
386
+ item, index = job_factory.unpack(data)
341
387
  result = begin
342
388
  call_with_index(item, index, options, &block)
343
389
  rescue StandardError => e
@@ -376,59 +422,6 @@ module Parallel
376
422
  [count, options]
377
423
  end
378
424
 
379
- # kill all these pids or threads if user presses Ctrl+c
380
- def kill_on_ctrl_c(things, options)
381
- @to_be_killed ||= []
382
- old_interrupt = nil
383
- signal = options.fetch(:interrupt_signal, INTERRUPT_SIGNAL)
384
-
385
- if @to_be_killed.empty?
386
- old_interrupt = trap_interrupt(signal) do
387
- $stderr.puts 'Parallel execution interrupted, exiting ...'
388
- @to_be_killed.flatten.compact.each { |thing| kill_that_thing!(thing) }
389
- end
390
- end
391
-
392
- @to_be_killed << things
393
-
394
- yield
395
- ensure
396
- @to_be_killed.pop # free threads for GC and do not kill pids that could be used for new processes
397
- restore_interrupt(old_interrupt, signal) if @to_be_killed.empty?
398
- end
399
-
400
- def trap_interrupt(signal)
401
- old = Signal.trap signal, 'IGNORE'
402
-
403
- Signal.trap signal do
404
- yield
405
- if old == "DEFAULT"
406
- raise Interrupt
407
- else
408
- old.call
409
- end
410
- end
411
-
412
- old
413
- end
414
-
415
- def restore_interrupt(old, signal)
416
- Signal.trap signal, old
417
- end
418
-
419
- def kill_that_thing!(thing)
420
- if thing.is_a?(Thread)
421
- thing.kill
422
- else
423
- begin
424
- Process.kill(:KILL, thing)
425
- rescue Errno::ESRCH
426
- # some linux systems already automatically killed the children at this point
427
- # so we just ignore them not being there
428
- end
429
- end
430
- end
431
-
432
425
  def call_with_index(item, index, options, &block)
433
426
  args = [item]
434
427
  args << index if options[:with_index]
@@ -1,3 +1,3 @@
1
1
  module Parallel
2
- VERSION = Version = '1.4.1'
2
+ VERSION = Version = '1.5.0'
3
3
  end
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: parallel
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.1
4
+ version: 1.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michael Grosser
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-01 00:00:00.000000000 Z
11
+ date: 2015-05-25 00:00:00.000000000 Z
12
12
  dependencies: []
13
- description:
13
+ description:
14
14
  email: michael@grosser.it
15
15
  executables: []
16
16
  extensions: []
@@ -24,24 +24,24 @@ homepage: https://github.com/grosser/parallel
24
24
  licenses:
25
25
  - MIT
26
26
  metadata: {}
27
- post_install_message:
27
+ post_install_message:
28
28
  rdoc_options: []
29
29
  require_paths:
30
30
  - lib
31
31
  required_ruby_version: !ruby/object:Gem::Requirement
32
32
  requirements:
33
- - - ">="
33
+ - - '>='
34
34
  - !ruby/object:Gem::Version
35
35
  version: 1.9.3
36
36
  required_rubygems_version: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - '>='
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  requirements: []
42
- rubyforge_project:
43
- rubygems_version: 2.2.2
44
- signing_key:
42
+ rubyforge_project:
43
+ rubygems_version: 2.1.9
44
+ signing_key:
45
45
  specification_version: 4
46
46
  summary: Run any kind of code in parallel processes
47
47
  test_files: []