chantier 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/chantier.gemspec +3 -3
- data/lib/chantier.rb +1 -1
- data/lib/process_pool.rb +18 -3
- data/lib/thread_pool.rb +35 -4
- data/spec/process_pool_spec.rb +21 -0
- data/spec/thread_pool_spec.rb +20 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 748fbefb75937019db2ddbf9d8ff6ffcd88c1227
|
4
|
+
data.tar.gz: cacf4320b37f1891393ec55a8ed43d316825d21b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0c14a5d0828f0ed0f4491abdb6e152f027e73d457869f90e4ec3bf9630deb819f0ffd3e710814bfa94b654dc4a25403efdf1fe27800aa4b164af123d0852ff42
|
7
|
+
data.tar.gz: b6382976248c36e1fa908dcd3f4b80162f4534c5392de7ceecf0687446d8a5aa2b727507f3bdf6ccff466f14ff9818aa36a25ba26bb6b455dfe72befd57d9369
|
data/chantier.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: chantier 0.0.
|
5
|
+
# stub: chantier 0.0.5 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "chantier"
|
9
|
-
s.version = "0.0.
|
9
|
+
s.version = "0.0.5"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Julik Tarkhanov"]
|
14
|
-
s.date = "2014-08-
|
14
|
+
s.date = "2014-08-05"
|
15
15
|
s.description = " Process your jobs in parallel with a simple table of processes or threads "
|
16
16
|
s.email = "me@julik.nl"
|
17
17
|
s.extra_rdoc_files = [
|
data/lib/chantier.rb
CHANGED
data/lib/process_pool.rb
CHANGED
@@ -34,7 +34,13 @@ class Chantier::ProcessPool
|
|
34
34
|
# the loop returns.
|
35
35
|
SCHEDULER_SLEEP_SECONDS = (1.0 / 1000)
|
36
36
|
|
37
|
-
|
37
|
+
# Initializes a new ProcessPool with the given number of workers. If max_failures is
|
38
|
+
# given the fork_task method will raise an exception if more than N processes spawned
|
39
|
+
# have been terminated with a non-0 exit status.
|
40
|
+
def initialize(num_procs, max_failures: nil)
|
41
|
+
@max_failures = max_failures && max_failures.to_i
|
42
|
+
@non_zero_exits = 0
|
43
|
+
|
38
44
|
raise "Need at least 1 slot, given #{num_procs.to_i}" unless num_procs.to_i > 0
|
39
45
|
@pids = [nil] * num_procs.to_i
|
40
46
|
@semaphore = Mutex.new
|
@@ -62,6 +68,10 @@ class Chantier::ProcessPool
|
|
62
68
|
# becomes free. Once that happens, the given block will be forked off
|
63
69
|
# and the method will return.
|
64
70
|
def fork_task(&blk)
|
71
|
+
if @max_failures && @non_zero_exits > @max_failures
|
72
|
+
raise "Reached error limit of processes quitting with non-0 status - limit set at #{@max_failures}"
|
73
|
+
end
|
74
|
+
|
65
75
|
destination_slot_idx = nil
|
66
76
|
|
67
77
|
# Try to find a slot in the process table where this job can go
|
@@ -88,8 +98,13 @@ class Chantier::ProcessPool
|
|
88
98
|
# process table
|
89
99
|
Thread.new do
|
90
100
|
Process.wait(task_pid) # This call will block until that process quites
|
91
|
-
|
92
|
-
@semaphore.synchronize
|
101
|
+
terminated_normally = $?.exited? && $?.exitstatus.zero?
|
102
|
+
@semaphore.synchronize do
|
103
|
+
# Now we can remove that process from the process table
|
104
|
+
@pids[destination_slot_idx] = nil
|
105
|
+
# and increment the error count if needed
|
106
|
+
@non_zero_exits += 1 unless terminated_normally
|
107
|
+
end
|
93
108
|
end
|
94
109
|
|
95
110
|
return task_pid
|
data/lib/thread_pool.rb
CHANGED
@@ -34,10 +34,21 @@ class Chantier::ThreadPool
|
|
34
34
|
# the loop returns.
|
35
35
|
SCHEDULER_SLEEP_SECONDS = (1.0 / 500)
|
36
36
|
|
37
|
-
|
37
|
+
# Initializes a new ProcessPool with the given number of workers. If max_failures is
|
38
|
+
# given the fork_task method will raise an exception if more than N threads spawned
|
39
|
+
# have raised during execution.
|
40
|
+
def initialize(num_threads, max_failures: nil)
|
38
41
|
raise "Need at least 1 slot, given #{num_threads.to_i}" unless num_threads.to_i > 0
|
39
42
|
@threads = [nil] * num_threads.to_i
|
40
43
|
@semaphore = Mutex.new
|
44
|
+
|
45
|
+
# Failure counters
|
46
|
+
@failure_count = 0
|
47
|
+
@max_failures = max_failures && max_failures.to_i
|
48
|
+
|
49
|
+
# Information on the last exception that happened
|
50
|
+
@aborted = false
|
51
|
+
@last_representative_exception = nil
|
41
52
|
end
|
42
53
|
|
43
54
|
# Distributes the elements in the given Enumerable to parallel workers,
|
@@ -61,6 +72,10 @@ class Chantier::ThreadPool
|
|
61
72
|
# the thread it is called from until a slot in the thread table
|
62
73
|
# becomes free.
|
63
74
|
def fork_task(&blk)
|
75
|
+
if @last_representative_exception
|
76
|
+
raise "Reached error limit of #{@max_failures} (last error was #{@last_representative_exception.inspect})"
|
77
|
+
end
|
78
|
+
|
64
79
|
destination_slot_idx = nil
|
65
80
|
|
66
81
|
# Try to find a slot in the process table where this job can go
|
@@ -78,11 +93,11 @@ class Chantier::ThreadPool
|
|
78
93
|
|
79
94
|
# No need to lock this because we already reserved that slot
|
80
95
|
@threads[destination_slot_idx] = Thread.new do
|
81
|
-
|
82
|
-
|
96
|
+
# Run the given block
|
97
|
+
run_block_with_exception_protection(&blk)
|
98
|
+
# ...and remove that process from the process table
|
83
99
|
@semaphore.synchronize { @threads[destination_slot_idx] = nil }
|
84
100
|
end
|
85
|
-
|
86
101
|
end
|
87
102
|
|
88
103
|
# Tells whether some processes are still churning
|
@@ -98,4 +113,20 @@ class Chantier::ThreadPool
|
|
98
113
|
end
|
99
114
|
end
|
100
115
|
end
|
116
|
+
|
117
|
+
private
|
118
|
+
|
119
|
+
def run_block_with_exception_protection(&blk)
|
120
|
+
yield
|
121
|
+
rescue Exception => e
|
122
|
+
# Register the failure and decrement the counter. If we had more than N
|
123
|
+
# failures stop the machine completely by raising an exception in the caller.
|
124
|
+
@semaphore.synchronize do
|
125
|
+
@failure_count += 1
|
126
|
+
if @max_failures && (@failure_count > @max_failures)
|
127
|
+
@last_representative_exception = e
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
101
132
|
end
|
data/spec/process_pool_spec.rb
CHANGED
@@ -50,6 +50,27 @@ describe Chantier::ProcessPool do
|
|
50
50
|
Chantier::ProcessPool.new(10)
|
51
51
|
end
|
52
52
|
|
53
|
+
context 'with failures' do
|
54
|
+
it 'raises after 4 failures' do
|
55
|
+
under_test = described_class.new(num_workers=3, max_failures: '4')
|
56
|
+
expect {
|
57
|
+
15.times do
|
58
|
+
under_test.fork_task { raise "I am such a failure" }
|
59
|
+
end
|
60
|
+
}.to raise_error('Reached error limit of processes quitting with non-0 status - limit set at 4')
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'runs through the jobs if max_failures is not given' do
|
64
|
+
under_test = described_class.new(num_workers=3)
|
65
|
+
7.times {
|
66
|
+
under_test.fork_task { raise "I am such a failure" }
|
67
|
+
}
|
68
|
+
under_test.block_until_complete!
|
69
|
+
expect(true).to eq(true), "Should have gotten to this assertion without the Pool blocking"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
|
53
74
|
context 'with 1 slot' do
|
54
75
|
let(:manager) { described_class.new(1) }
|
55
76
|
|
data/spec/thread_pool_spec.rb
CHANGED
@@ -46,6 +46,26 @@ describe Chantier::ThreadPool do
|
|
46
46
|
end
|
47
47
|
end
|
48
48
|
|
49
|
+
context 'with failures' do
|
50
|
+
it 'raises after 4 failures' do
|
51
|
+
under_test = described_class.new(num_workers=3, max_failures: '4')
|
52
|
+
expect {
|
53
|
+
15.times do
|
54
|
+
under_test.fork_task { raise "I am such a failure" }
|
55
|
+
end
|
56
|
+
}.to raise_error('Reached error limit of 4 (last error was #<RuntimeError: I am such a failure>)')
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'runs through the jobs if max_failures is not given' do
|
60
|
+
under_test = described_class.new(num_workers=3)
|
61
|
+
7.times {
|
62
|
+
under_test.fork_task { raise "I am such a failure" }
|
63
|
+
}
|
64
|
+
under_test.block_until_complete!
|
65
|
+
expect(true).to eq(true), "Should have gotten to this assertion without the Pool blocking"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
49
69
|
it 'gets instantiated with the given number of slots' do
|
50
70
|
described_class.new(10)
|
51
71
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: chantier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Julik Tarkhanov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-08-
|
11
|
+
date: 2014-08-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|