rbbt-util 5.28.8 → 5.28.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/workflow/util/orchestrator.rb +35 -10
- data/test/rbbt/workflow/util/test_orchestrator.rb +105 -18
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2fa2f71df9aef8810599ab3660c847c084dea75ccb2485d294cfa1962e341162
|
4
|
+
data.tar.gz: 7eb004ddf8bcf30b00325dc5251d9ed73a307de433c18c0847eb4d70a4bea1e2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6ba23322f9ac768f1782c4e26605c50171b6d35d1d3c2acc0ef195e26ca39af851a232e1581ca2ec220899b7780f35f8d0d36e1f61f0b0e6500c883999c3760b
|
7
|
+
data.tar.gz: afa0952f228a951ea78506ad0776fd96dbfbd2b91bac73310ee6807e4a7d0a74c7ca8f582880c5abd28bbf709c8be3d9dff1797056be756962091b2e97031f30
|
@@ -12,6 +12,7 @@ module Workflow
|
|
12
12
|
workload.merge!(job_workload(dep))
|
13
13
|
workload[job] += workload[dep]
|
14
14
|
workload[job] << dep
|
15
|
+
workload[job].uniq!
|
15
16
|
end
|
16
17
|
|
17
18
|
job.input_dependencies.each do |dep|
|
@@ -19,6 +20,7 @@ module Workflow
|
|
19
20
|
workload.merge!(job_workload(dep))
|
20
21
|
workload[job] += workload[dep]
|
21
22
|
workload[job] << dep
|
23
|
+
workload[job].uniq!
|
22
24
|
end
|
23
25
|
|
24
26
|
workload
|
@@ -32,7 +34,7 @@ module Workflow
|
|
32
34
|
return IndiferentHash.setup(rules["defaults"]) unless rules[workflow][task_name]
|
33
35
|
|
34
36
|
job_rules = IndiferentHash.setup(rules[workflow][task_name])
|
35
|
-
rules["defaults"].each{|k,v| job_rules[k]
|
37
|
+
rules["defaults"].each{|k,v| job_rules[k] = v if job_rules[k].nil? } if rules["defaults"]
|
36
38
|
job_rules
|
37
39
|
end
|
38
40
|
|
@@ -97,6 +99,7 @@ module Workflow
|
|
97
99
|
|
98
100
|
def release_resources(job)
|
99
101
|
if resources_used[job]
|
102
|
+
Log.debug "Orchestrator releasing resouces from #{job.path}"
|
100
103
|
resources_used[job].each do |resource,value|
|
101
104
|
next if resource == 'size'
|
102
105
|
resources_requested[resource] -= value.to_i
|
@@ -140,30 +143,51 @@ module Workflow
|
|
140
143
|
end
|
141
144
|
end
|
142
145
|
|
146
|
+
def erase_job_dependencies(job, rules, workload, top_level_jobs)
|
147
|
+
job.dependencies.each do |dep|
|
148
|
+
next if top_level_jobs.include? dep.path
|
149
|
+
next unless Orchestrator.job_rules(rules, dep)["erase"].to_s == 'true'
|
150
|
+
|
151
|
+
list = (workload.keys - [job]).collect{|pending| pending.dependencies}.flatten
|
152
|
+
next if list.include?(dep)
|
153
|
+
|
154
|
+
Log.high "Erasing #{dep.path} from #{job.path}"
|
155
|
+
job.archive_deps
|
156
|
+
job.copy_files_dir
|
157
|
+
job.dependencies = job.dependencies - [dep]
|
158
|
+
dep.clean
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
143
162
|
def process(rules, jobs)
|
144
163
|
begin
|
145
164
|
|
146
165
|
workload = jobs.inject({}){|acc,job| acc.merge!(Orchestrator.job_workload(job)) }
|
147
166
|
|
148
|
-
|
167
|
+
top_level_jobs = jobs.collect{|job| job.path }
|
168
|
+
while workload.any?
|
149
169
|
|
150
170
|
candidates = resources_used.keys + Orchestrator.candidates(workload, rules)
|
151
|
-
raise "No candidates" if candidates.empty?
|
171
|
+
raise "No candidates and no running jobs" if candidates.empty?
|
152
172
|
|
153
173
|
candidates.each do |job|
|
154
174
|
case
|
155
175
|
when (job.error? || job.aborted?)
|
156
|
-
|
157
|
-
job.
|
158
|
-
|
159
|
-
|
160
|
-
|
176
|
+
begin
|
177
|
+
if job.recoverable_error?
|
178
|
+
job.clean
|
179
|
+
raise TryAgain
|
180
|
+
else
|
181
|
+
next
|
182
|
+
end
|
183
|
+
ensure
|
184
|
+
Log.warn "Releases resources from failed job: #{job.path}"
|
185
|
+
release_resources(job)
|
161
186
|
end
|
162
|
-
release_resources(job)
|
163
187
|
when job.done?
|
164
188
|
Log.debug "Orchestrator done #{job.path}"
|
165
189
|
release_resources(job)
|
166
|
-
|
190
|
+
erase_job_dependencies(job, rules, workload, top_level_jobs)
|
167
191
|
|
168
192
|
when job.running?
|
169
193
|
next
|
@@ -180,6 +204,7 @@ module Workflow
|
|
180
204
|
next if k.done?
|
181
205
|
new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)}
|
182
206
|
end
|
207
|
+
workload = new_workload
|
183
208
|
sleep timer
|
184
209
|
end
|
185
210
|
rescue TryAgain
|
@@ -17,6 +17,7 @@ module TestWF
|
|
17
17
|
sleep(TestWF::MULT * (rand(10) + 2))
|
18
18
|
end
|
19
19
|
|
20
|
+
dep :a
|
20
21
|
dep :b
|
21
22
|
task :c => :text do
|
22
23
|
sleep(TestWF::MULT * (rand(10) + 2))
|
@@ -29,13 +30,13 @@ module TestWF
|
|
29
30
|
end
|
30
31
|
|
31
32
|
class TestClass < Test::Unit::TestCase
|
32
|
-
def
|
33
|
+
def test_orchestrate_resources
|
33
34
|
|
34
35
|
jobs =[]
|
35
36
|
|
36
37
|
num = 10
|
37
38
|
num.times do |i|
|
38
|
-
jobs.concat %w(
|
39
|
+
jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
|
39
40
|
end
|
40
41
|
jobs.each do |j| j.recursive_clean end
|
41
42
|
|
@@ -81,13 +82,13 @@ TestWF:
|
|
81
82
|
assert Misc.mean(second_cpus.values) < 30
|
82
83
|
end
|
83
84
|
|
84
|
-
def
|
85
|
+
def test_orchestrate_erase
|
85
86
|
|
86
87
|
jobs =[]
|
87
88
|
|
88
89
|
num = 10
|
89
90
|
num.times do |i|
|
90
|
-
jobs.concat %w(
|
91
|
+
jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
|
91
92
|
end
|
92
93
|
jobs.each do |j| j.recursive_clean end
|
93
94
|
|
@@ -98,9 +99,11 @@ default_resources:
|
|
98
99
|
IO: 1
|
99
100
|
TestWF:
|
100
101
|
a:
|
102
|
+
erase: true
|
101
103
|
resources:
|
102
104
|
cpus: 7
|
103
105
|
b:
|
106
|
+
erase: true
|
104
107
|
resources:
|
105
108
|
cpus: 2
|
106
109
|
c:
|
@@ -112,25 +115,109 @@ TestWF:
|
|
112
115
|
EOF
|
113
116
|
|
114
117
|
orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
|
115
|
-
Log.with_severity
|
118
|
+
Log.with_severity 3 do
|
116
119
|
orchestrator.process(rules, jobs)
|
117
120
|
end
|
118
121
|
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
122
|
+
jobs.each do |job|
|
123
|
+
assert job.step(:c).dependencies.empty?
|
124
|
+
assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
|
125
|
+
assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
|
126
|
+
end
|
127
|
+
|
128
|
+
end
|
129
|
+
|
130
|
+
def test_orchestrate_default
|
131
|
+
|
132
|
+
jobs =[]
|
133
|
+
|
134
|
+
num = 3
|
135
|
+
num.times do |i|
|
136
|
+
jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
|
137
|
+
end
|
138
|
+
jobs.each do |j| j.recursive_clean end
|
139
|
+
|
140
|
+
rules = YAML.load <<-EOF
|
141
|
+
defaults:
|
142
|
+
erase: true
|
143
|
+
log: 4
|
144
|
+
default_resources:
|
145
|
+
IO: 1
|
146
|
+
TestWF:
|
147
|
+
a:
|
148
|
+
erase: true
|
149
|
+
resources:
|
150
|
+
cpus: 7
|
151
|
+
b:
|
152
|
+
erase: true
|
153
|
+
resources:
|
154
|
+
cpus: 2
|
155
|
+
c:
|
156
|
+
erase: false
|
157
|
+
resources:
|
158
|
+
cpus: 10
|
159
|
+
d:
|
160
|
+
resources:
|
161
|
+
cpus: 15
|
162
|
+
EOF
|
163
|
+
|
164
|
+
orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
|
165
|
+
Log.with_severity 3 do
|
166
|
+
orchestrator.process(rules, jobs)
|
167
|
+
end
|
168
|
+
|
169
|
+
jobs.each do |job|
|
170
|
+
assert job.step(:c).dependencies.empty?
|
171
|
+
assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
|
172
|
+
assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
|
173
|
+
end
|
174
|
+
|
175
|
+
end
|
176
|
+
|
177
|
+
def test_orchestrate_top_level
|
178
|
+
|
179
|
+
jobs =[]
|
180
|
+
|
181
|
+
num = 3
|
182
|
+
num.times do |i|
|
183
|
+
jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
|
184
|
+
jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
|
185
|
+
end
|
186
|
+
jobs.each do |j| j.recursive_clean end
|
187
|
+
|
188
|
+
rules = YAML.load <<-EOF
|
189
|
+
defaults:
|
190
|
+
erase: true
|
191
|
+
log: 4
|
192
|
+
default_resources:
|
193
|
+
IO: 1
|
194
|
+
TestWF:
|
195
|
+
a:
|
196
|
+
resources:
|
197
|
+
cpus: 7
|
198
|
+
b:
|
199
|
+
resources:
|
200
|
+
cpus: 2
|
201
|
+
c:
|
202
|
+
resources:
|
203
|
+
cpus: 10
|
204
|
+
d:
|
205
|
+
resources:
|
206
|
+
cpus: 15
|
207
|
+
EOF
|
208
|
+
|
209
|
+
orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
|
210
|
+
Log.with_severity 3 do
|
211
|
+
orchestrator.process(rules, jobs)
|
212
|
+
end
|
213
|
+
|
214
|
+
jobs.each do |job|
|
215
|
+
next unless job.task_name.to_s == 'd'
|
216
|
+
assert job.step(:c).dependencies.empty?
|
217
|
+
assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
|
218
|
+
assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
|
130
219
|
end
|
131
220
|
|
132
|
-
assert Misc.mean(second_cpus.values) > 15
|
133
|
-
assert Misc.mean(second_cpus.values) < 30
|
134
221
|
end
|
135
222
|
end
|
136
223
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.28.
|
4
|
+
version: 5.28.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-11-
|
11
|
+
date: 2020-11-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|