chantier 0.0.4 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0c4e130a470bcbd6888307a6130b049ada83ec34
4
- data.tar.gz: aac008cb10feec2ab5e804a5c6bccfb77c660bbf
3
+ metadata.gz: 748fbefb75937019db2ddbf9d8ff6ffcd88c1227
4
+ data.tar.gz: cacf4320b37f1891393ec55a8ed43d316825d21b
5
5
  SHA512:
6
- metadata.gz: 3daeffddbc5134d1247c2af7edeb109011f25072f4428d10805306c005f6236c74a1779dcf66993d8e741f5cfb1544d6b44ffae8e74b7944a52a324afa2f87e8
7
- data.tar.gz: d58a273d22173f4de4c07f49cf1102d0ee4b91c2c31a72b29ddf063f7a263a4b20fd6e5f361c425b0598ca92afe721fc25173d345674af5cb59bc2316444e436
6
+ metadata.gz: 0c14a5d0828f0ed0f4491abdb6e152f027e73d457869f90e4ec3bf9630deb819f0ffd3e710814bfa94b654dc4a25403efdf1fe27800aa4b164af123d0852ff42
7
+ data.tar.gz: b6382976248c36e1fa908dcd3f4b80162f4534c5392de7ceecf0687446d8a5aa2b727507f3bdf6ccff466f14ff9818aa36a25ba26bb6b455dfe72befd57d9369
data/chantier.gemspec CHANGED
@@ -2,16 +2,16 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: chantier 0.0.4 ruby lib
5
+ # stub: chantier 0.0.5 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "chantier"
9
- s.version = "0.0.4"
9
+ s.version = "0.0.5"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib"]
13
13
  s.authors = ["Julik Tarkhanov"]
14
- s.date = "2014-08-03"
14
+ s.date = "2014-08-05"
15
15
  s.description = " Process your jobs in parallel with a simple table of processes or threads "
16
16
  s.email = "me@julik.nl"
17
17
  s.extra_rdoc_files = [
data/lib/chantier.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  module Chantier
2
- VERSION = '0.0.4'
2
+ VERSION = '0.0.5'
3
3
  require_relative 'process_pool'
4
4
  require_relative 'process_pool_with_kill'
5
5
  require_relative 'thread_pool'
data/lib/process_pool.rb CHANGED
@@ -34,7 +34,13 @@ class Chantier::ProcessPool
34
34
  # the loop returns.
35
35
  SCHEDULER_SLEEP_SECONDS = (1.0 / 1000)
36
36
 
37
- def initialize(num_procs)
37
+ # Initializes a new ProcessPool with the given number of workers. If max_failures is
38
+ # given the fork_task method will raise an exception if more than N processes spawned
39
+ # have been terminated with a non-0 exit status.
40
+ def initialize(num_procs, max_failures: nil)
41
+ @max_failures = max_failures && max_failures.to_i
42
+ @non_zero_exits = 0
43
+
38
44
  raise "Need at least 1 slot, given #{num_procs.to_i}" unless num_procs.to_i > 0
39
45
  @pids = [nil] * num_procs.to_i
40
46
  @semaphore = Mutex.new
@@ -62,6 +68,10 @@ class Chantier::ProcessPool
62
68
  # becomes free. Once that happens, the given block will be forked off
63
69
  # and the method will return.
64
70
  def fork_task(&blk)
71
+ if @max_failures && @non_zero_exits > @max_failures
72
+ raise "Reached error limit of processes quitting with non-0 status - limit set at #{@max_failures}"
73
+ end
74
+
65
75
  destination_slot_idx = nil
66
76
 
67
77
  # Try to find a slot in the process table where this job can go
@@ -88,8 +98,13 @@ class Chantier::ProcessPool
88
98
  # process table
89
99
  Thread.new do
90
100
  Process.wait(task_pid) # This call will block until that process quites
91
- # Now we can remove that process from the process table
92
- @semaphore.synchronize { @pids[destination_slot_idx] = nil }
101
+ terminated_normally = $?.exited? && $?.exitstatus.zero?
102
+ @semaphore.synchronize do
103
+ # Now we can remove that process from the process table
104
+ @pids[destination_slot_idx] = nil
105
+ # and increment the error count if needed
106
+ @non_zero_exits += 1 unless terminated_normally
107
+ end
93
108
  end
94
109
 
95
110
  return task_pid
data/lib/thread_pool.rb CHANGED
@@ -34,10 +34,21 @@ class Chantier::ThreadPool
34
34
  # the loop returns.
35
35
  SCHEDULER_SLEEP_SECONDS = (1.0 / 500)
36
36
 
37
- def initialize(num_threads)
37
+ # Initializes a new ProcessPool with the given number of workers. If max_failures is
38
+ # given the fork_task method will raise an exception if more than N threads spawned
39
+ # have raised during execution.
40
+ def initialize(num_threads, max_failures: nil)
38
41
  raise "Need at least 1 slot, given #{num_threads.to_i}" unless num_threads.to_i > 0
39
42
  @threads = [nil] * num_threads.to_i
40
43
  @semaphore = Mutex.new
44
+
45
+ # Failure counters
46
+ @failure_count = 0
47
+ @max_failures = max_failures && max_failures.to_i
48
+
49
+ # Information on the last exception that happened
50
+ @aborted = false
51
+ @last_representative_exception = nil
41
52
  end
42
53
 
43
54
  # Distributes the elements in the given Enumerable to parallel workers,
@@ -61,6 +72,10 @@ class Chantier::ThreadPool
61
72
  # the thread it is called from until a slot in the thread table
62
73
  # becomes free.
63
74
  def fork_task(&blk)
75
+ if @last_representative_exception
76
+ raise "Reached error limit of #{@max_failures} (last error was #{@last_representative_exception.inspect})"
77
+ end
78
+
64
79
  destination_slot_idx = nil
65
80
 
66
81
  # Try to find a slot in the process table where this job can go
@@ -78,11 +93,11 @@ class Chantier::ThreadPool
78
93
 
79
94
  # No need to lock this because we already reserved that slot
80
95
  @threads[destination_slot_idx] = Thread.new do
81
- yield
82
- # Now we can remove that process from the process table
96
+ # Run the given block
97
+ run_block_with_exception_protection(&blk)
98
+ # ...and remove that process from the process table
83
99
  @semaphore.synchronize { @threads[destination_slot_idx] = nil }
84
100
  end
85
-
86
101
  end
87
102
 
88
103
  # Tells whether some processes are still churning
@@ -98,4 +113,20 @@ class Chantier::ThreadPool
98
113
  end
99
114
  end
100
115
  end
116
+
117
+ private
118
+
119
+ def run_block_with_exception_protection(&blk)
120
+ yield
121
+ rescue Exception => e
122
+ # Register the failure and decrement the counter. If we had more than N
123
+ # failures stop the machine completely by raising an exception in the caller.
124
+ @semaphore.synchronize do
125
+ @failure_count += 1
126
+ if @max_failures && (@failure_count > @max_failures)
127
+ @last_representative_exception = e
128
+ end
129
+ end
130
+ end
131
+
101
132
  end
@@ -50,6 +50,27 @@ describe Chantier::ProcessPool do
50
50
  Chantier::ProcessPool.new(10)
51
51
  end
52
52
 
53
+ context 'with failures' do
54
+ it 'raises after 4 failures' do
55
+ under_test = described_class.new(num_workers=3, max_failures: '4')
56
+ expect {
57
+ 15.times do
58
+ under_test.fork_task { raise "I am such a failure" }
59
+ end
60
+ }.to raise_error('Reached error limit of processes quitting with non-0 status - limit set at 4')
61
+ end
62
+
63
+ it 'runs through the jobs if max_failures is not given' do
64
+ under_test = described_class.new(num_workers=3)
65
+ 7.times {
66
+ under_test.fork_task { raise "I am such a failure" }
67
+ }
68
+ under_test.block_until_complete!
69
+ expect(true).to eq(true), "Should have gotten to this assertion without the Pool blocking"
70
+ end
71
+ end
72
+
73
+
53
74
  context 'with 1 slot' do
54
75
  let(:manager) { described_class.new(1) }
55
76
 
@@ -46,6 +46,26 @@ describe Chantier::ThreadPool do
46
46
  end
47
47
  end
48
48
 
49
+ context 'with failures' do
50
+ it 'raises after 4 failures' do
51
+ under_test = described_class.new(num_workers=3, max_failures: '4')
52
+ expect {
53
+ 15.times do
54
+ under_test.fork_task { raise "I am such a failure" }
55
+ end
56
+ }.to raise_error('Reached error limit of 4 (last error was #<RuntimeError: I am such a failure>)')
57
+ end
58
+
59
+ it 'runs through the jobs if max_failures is not given' do
60
+ under_test = described_class.new(num_workers=3)
61
+ 7.times {
62
+ under_test.fork_task { raise "I am such a failure" }
63
+ }
64
+ under_test.block_until_complete!
65
+ expect(true).to eq(true), "Should have gotten to this assertion without the Pool blocking"
66
+ end
67
+ end
68
+
49
69
  it 'gets instantiated with the given number of slots' do
50
70
  described_class.new(10)
51
71
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: chantier
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Julik Tarkhanov
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-08-03 00:00:00.000000000 Z
11
+ date: 2014-08-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rspec