bricolage 5.10.0 → 5.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bricolage/eventhandlers.rb +6 -1
- data/lib/bricolage/{jobflow.rb → jobnet.rb} +216 -192
- data/lib/bricolage/jobnetrunner.rb +17 -24
- data/lib/bricolage/taskqueue.rb +5 -9
- data/lib/bricolage/version.rb +1 -1
- data/test/home/Gemfile.lock +2 -2
- data/test/home/jobnet-test.rb +61 -0
- data/test/home/subsys/job1.job +1 -0
- data/test/home/subsys/job2.job +1 -0
- data/test/home/subsys/job3.job +1 -0
- data/test/home/subsys/job4.job +1 -0
- data/test/home/subsys/job5.job +1 -0
- data/test/home/subsys/job6.job +1 -0
- data/test/home/subsys/job7.job +1 -0
- data/test/home/subsys/net1.jobnet +3 -0
- data/test/home/subsys/net2.jobnet +2 -0
- metadata +13 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1ff4f8adcd84bf94e55cb1d285a3b5671210906e
|
4
|
+
data.tar.gz: b1f3f08b9c0e74a5e6d469ba197a29d08b51551f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: aec57849ed95f3c50bd0befdcaf66ddad3f6ab2aeb241e139f51e996a64e18ae63023e624a13f44daa9a5330d75d2928cc0df447f4ecae3c9055934b519647c5
|
7
|
+
data.tar.gz: dab80eab00b0e147249fcf6f53a6a75314a0b2161991d46fc0ffe21cedbc29fae3a6e0f00c6d95c60f3ddc26a5438c7d93061ca132a3714d0192b02abc63c593
|
@@ -40,7 +40,12 @@ module Bricolage
|
|
40
40
|
}
|
41
41
|
end
|
42
42
|
|
43
|
-
BeforeAllJobsEvent = Struct.new(:
|
43
|
+
BeforeAllJobsEvent = Struct.new(:jobnet_id, :queue)
|
44
|
+
class BeforeAllJobsEvent # reopen
|
45
|
+
def flow_id
|
46
|
+
jobnet_id
|
47
|
+
end
|
48
|
+
end
|
44
49
|
BeforeJobEvent = Struct.new(:job)
|
45
50
|
AfterJobEvent = Struct.new(:result)
|
46
51
|
AfterAllJobsEvent = Struct.new(:succeeded, :queue)
|
@@ -3,80 +3,100 @@ require 'tsort'
|
|
3
3
|
|
4
4
|
module Bricolage
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
6
|
+
# Represents "first" jobnet given by command line (e.g. bricolage-jobnet some.jobnet)
|
7
|
+
class RootJobNet
|
8
|
+
def RootJobNet.load(ctx, path)
|
9
|
+
root = new(JobNet::FileLoader.new(ctx), JobNet.load(path))
|
10
|
+
root.load_recursive
|
11
|
+
root.fix
|
12
|
+
root
|
13
13
|
end
|
14
14
|
|
15
|
-
def
|
16
|
-
|
17
|
-
|
18
|
-
|
15
|
+
def initialize(jobnet_loader, start_jobnet)
|
16
|
+
@jobnet_loader = jobnet_loader
|
17
|
+
@start_jobnet = start_jobnet
|
18
|
+
@jobnets = {start_jobnet.ref => start_jobnet}
|
19
|
+
@graph = nil
|
19
20
|
end
|
20
21
|
|
21
|
-
|
22
|
-
Parser.new(ref.subsystem).parse_stream(f, ref)
|
23
|
-
end
|
22
|
+
attr_reader :start_jobnet
|
24
23
|
|
25
|
-
def
|
26
|
-
|
24
|
+
def each_jobnet(&block)
|
25
|
+
@jobnets.each_value(&block)
|
27
26
|
end
|
28
27
|
|
29
|
-
|
30
|
-
|
31
|
-
def initialize(ref, location)
|
32
|
-
@ref = ref
|
33
|
-
@location = location
|
34
|
-
@flow = {} # Ref => [Ref] (src->dest)
|
35
|
-
@deps = {} # Ref => [Ref] (dest->src)
|
36
|
-
@subnets_resolved = false
|
28
|
+
def jobnets
|
29
|
+
@jobnets.values
|
37
30
|
end
|
38
31
|
|
39
|
-
def
|
40
|
-
|
32
|
+
def load_recursive
|
33
|
+
unresolved = [@start_jobnet]
|
34
|
+
until unresolved.empty?
|
35
|
+
loaded = []
|
36
|
+
unresolved.each do |net|
|
37
|
+
net.net_refs.each do |ref|
|
38
|
+
next if ref.jobnet
|
39
|
+
unless net = @jobnets[ref]
|
40
|
+
net = @jobnet_loader.load(ref)
|
41
|
+
@jobnets[ref] = net
|
42
|
+
loaded.push net
|
43
|
+
end
|
44
|
+
ref.jobnet = net
|
45
|
+
end
|
46
|
+
end
|
47
|
+
unresolved = loaded
|
48
|
+
end
|
41
49
|
end
|
42
50
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
51
|
+
def fix
|
52
|
+
each_jobnet do |net|
|
53
|
+
net.fix
|
54
|
+
end
|
55
|
+
@jobnets.freeze
|
56
|
+
@dag = JobDAG.build(jobnets)
|
47
57
|
end
|
48
58
|
|
49
|
-
def
|
50
|
-
|
51
|
-
(@deps[dest] ||= []).push src
|
59
|
+
def sequential_jobs
|
60
|
+
@dag.sequential_jobs
|
52
61
|
end
|
62
|
+
end
|
53
63
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
64
|
+
class JobDAG
|
65
|
+
def JobDAG.build(jobnets)
|
66
|
+
graph = new
|
67
|
+
jobnets.each do |net|
|
68
|
+
graph.merge! net
|
58
69
|
end
|
59
|
-
|
70
|
+
graph.fix
|
71
|
+
graph
|
60
72
|
end
|
61
73
|
|
62
|
-
def
|
74
|
+
def initialize
|
75
|
+
@deps = Hash.new { Array.new } # {JobRef => [JobRef]} (dest->srcs)
|
76
|
+
end
|
77
|
+
|
78
|
+
def to_hash
|
63
79
|
h = {}
|
64
|
-
@deps.each do |
|
65
|
-
h[
|
80
|
+
@deps.each do |dest, srcs|
|
81
|
+
h[dest.to_s] = srcs.map(&:to_s)
|
66
82
|
end
|
67
83
|
h
|
68
84
|
end
|
69
85
|
|
70
|
-
def
|
71
|
-
|
86
|
+
def merge!(net)
|
87
|
+
net.each_dependencies do |ref, deps|
|
88
|
+
@deps[ref] |= deps
|
89
|
+
end
|
72
90
|
end
|
73
91
|
|
74
|
-
def
|
75
|
-
|
92
|
+
def fix
|
93
|
+
@deps.freeze
|
94
|
+
check_cycle
|
95
|
+
check_orphan
|
76
96
|
end
|
77
97
|
|
78
98
|
def sequential_jobs
|
79
|
-
|
99
|
+
tsort.reject {|ref| ref.dummy? }
|
80
100
|
end
|
81
101
|
|
82
102
|
include TSort
|
@@ -89,153 +109,185 @@ module Bricolage
|
|
89
109
|
@deps.fetch(ref).each(&block)
|
90
110
|
end
|
91
111
|
|
92
|
-
def fix_graph
|
93
|
-
close_graph
|
94
|
-
check_cycle
|
95
|
-
check_orphan
|
96
|
-
end
|
97
|
-
|
98
|
-
def resolve_subnets(root_flow)
|
99
|
-
@deps.each_key do |ref|
|
100
|
-
next unless ref.net?
|
101
|
-
ref.flow = root_flow.subnet(ref)
|
102
|
-
end
|
103
|
-
@subnets_resolved = true
|
104
|
-
end
|
105
|
-
|
106
|
-
def subnets_resolved?
|
107
|
-
@subnets_resolved
|
108
|
-
end
|
109
|
-
|
110
112
|
private
|
111
113
|
|
112
|
-
def close_graph
|
113
|
-
@deps.values.flatten.uniq.each do |ref|
|
114
|
-
@deps[ref] ||= []
|
115
|
-
end
|
116
|
-
end
|
117
|
-
|
118
114
|
def check_cycle
|
119
115
|
each_strongly_connected_component do |refs|
|
120
116
|
unless refs.size == 1
|
121
117
|
cycle = (refs + [refs.first]).reverse.join(' -> ')
|
122
|
-
raise ParameterError, "found cycle in the
|
118
|
+
raise ParameterError, "found cycle in the jobnet: #{cycle}"
|
123
119
|
end
|
124
120
|
end
|
125
121
|
end
|
126
122
|
|
127
123
|
def check_orphan
|
128
124
|
orphan_nodes.each do |ref|
|
129
|
-
raise ParameterError, "found orphan job in the
|
125
|
+
raise ParameterError, "found orphan job in the jobnet: #{ref.location}: #{ref}"
|
130
126
|
end
|
131
127
|
end
|
132
128
|
|
133
129
|
def orphan_nodes
|
134
|
-
@deps.to_a.select {|ref, deps| deps.empty? and not ref.dummy?
|
130
|
+
@deps.to_a.select {|ref, deps| deps.empty? and not ref.dummy? }.map {|ref, *| ref }
|
135
131
|
end
|
136
132
|
end
|
137
133
|
|
138
|
-
class
|
139
|
-
def
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
134
|
+
class JobNet
|
135
|
+
def JobNet.load(path, ref = JobNetRef.for_path(path))
|
136
|
+
File.open(path) {|f|
|
137
|
+
Parser.new(ref).parse_stream(f)
|
138
|
+
}
|
139
|
+
rescue SystemCallError => err
|
140
|
+
raise ParameterError, "could not load jobnet: #{path} (#{err.message})"
|
144
141
|
end
|
145
142
|
|
146
|
-
def initialize(
|
147
|
-
@
|
148
|
-
|
149
|
-
@
|
143
|
+
def initialize(ref, location)
|
144
|
+
@ref = ref
|
145
|
+
@location = location
|
146
|
+
@flow = {} # Ref => [Ref] (src->dest)
|
147
|
+
@deps = {} # Ref => [Ref] (dest->src)
|
150
148
|
end
|
151
149
|
|
152
|
-
def
|
153
|
-
|
154
|
-
@subnets[flow.name] = flow
|
150
|
+
def inspect
|
151
|
+
"\#<#{self.class} #{ref}>"
|
155
152
|
end
|
156
153
|
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
flow = load_jobnet_auto(ref)
|
162
|
-
add_subnet flow
|
163
|
-
flow
|
164
|
-
end
|
154
|
+
attr_reader :ref
|
155
|
+
|
156
|
+
def start
|
157
|
+
@ref.start_ref
|
165
158
|
end
|
166
159
|
|
167
|
-
def
|
168
|
-
|
169
|
-
raise ParameterError, "undefined subnet: #{ref}" unless path.file?
|
170
|
-
JobFlow.load(path)
|
160
|
+
def end
|
161
|
+
@ref.end_ref
|
171
162
|
end
|
172
163
|
|
173
|
-
def
|
174
|
-
@
|
164
|
+
def name
|
165
|
+
@ref.to_s
|
175
166
|
end
|
176
167
|
|
177
|
-
def
|
178
|
-
|
179
|
-
@
|
168
|
+
def add_edge(src, dest)
|
169
|
+
(@flow[src] ||= []).push dest
|
170
|
+
(@deps[dest] ||= []).push src
|
180
171
|
end
|
181
172
|
|
182
|
-
def
|
183
|
-
|
184
|
-
|
173
|
+
def to_hash
|
174
|
+
h = {}
|
175
|
+
@flow.each do |src, dests|
|
176
|
+
h[src.to_s] = dests.map(&:to_s)
|
185
177
|
end
|
178
|
+
h
|
186
179
|
end
|
187
180
|
|
188
|
-
def
|
189
|
-
|
190
|
-
|
191
|
-
|
181
|
+
def to_deps_hash
|
182
|
+
h = {}
|
183
|
+
@deps.each do |dest, srcs|
|
184
|
+
h[dest.to_s] = srcs.map(&:to_s)
|
192
185
|
end
|
193
|
-
|
194
|
-
fix_graph
|
186
|
+
h
|
195
187
|
end
|
196
188
|
|
197
|
-
|
189
|
+
def refs
|
190
|
+
@flow.keys | @flow.values.flatten
|
191
|
+
end
|
198
192
|
|
199
|
-
def
|
200
|
-
|
201
|
-
while unresolved
|
202
|
-
unresolved = false
|
203
|
-
([self] + @subnets.values).each do |flow|
|
204
|
-
unless flow.subnets_resolved?
|
205
|
-
flow.resolve_subnets self
|
206
|
-
unresolved = true
|
207
|
-
end
|
208
|
-
end
|
209
|
-
end
|
193
|
+
def net_refs
|
194
|
+
@deps.keys.select {|ref| ref.net? }
|
210
195
|
end
|
211
196
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
197
|
+
# Adds dummy dependencies (@start and @end) to fix up all jobs
|
198
|
+
# into one DAG beginning with @start and ending with @end
|
199
|
+
def fix
|
200
|
+
refs.each do |ref|
|
201
|
+
next if ref.dummy?
|
202
|
+
unless @deps[ref]
|
203
|
+
(@flow[self.start] ||= []).push ref
|
204
|
+
@deps[ref] = [self.start]
|
205
|
+
end
|
206
|
+
unless @flow[ref]
|
207
|
+
(@flow[ref] ||= []).push self.end
|
208
|
+
(@deps[self.end] ||= []).push ref
|
219
209
|
end
|
220
210
|
end
|
211
|
+
@deps[self.start] ||= []
|
212
|
+
@flow.freeze
|
213
|
+
@deps.freeze
|
221
214
|
end
|
222
215
|
|
223
|
-
def
|
224
|
-
|
216
|
+
def each_dependencies
|
217
|
+
@deps.each do |ref, deps|
|
218
|
+
dest = (ref.net? ? ref.start : ref)
|
219
|
+
srcs = deps.map {|r| r.net? ? r.end : r }
|
220
|
+
yield dest, srcs
|
221
|
+
end
|
225
222
|
end
|
226
223
|
end
|
227
224
|
|
228
|
-
class
|
225
|
+
class JobNet # reopen as namespace
|
229
226
|
|
230
|
-
class
|
231
|
-
|
227
|
+
class FileLoader
|
228
|
+
def initialize(ctx)
|
229
|
+
@context = ctx
|
230
|
+
end
|
231
|
+
|
232
|
+
def load(ref)
|
233
|
+
path = @context.root_relative_path(ref.relative_path)
|
234
|
+
raise ParameterError, "undefined subnet: #{ref}" unless path.file?
|
235
|
+
JobNet.load(path, ref)
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
class Parser
|
240
|
+
def initialize(jobnet_ref)
|
241
|
+
@jobnet_ref = jobnet_ref
|
242
|
+
end
|
243
|
+
|
244
|
+
def parse_stream(f)
|
245
|
+
net = JobNet.new(@jobnet_ref, Location.for_file(f))
|
246
|
+
foreach_edge(f) do |src, dest|
|
247
|
+
net.add_edge src, dest
|
248
|
+
end
|
249
|
+
net
|
250
|
+
end
|
251
|
+
|
252
|
+
private
|
253
|
+
|
254
|
+
name = /\w[\w\-]*/
|
255
|
+
node_ref = %r<[@*]?(?:#{name}/)?#{name}>
|
256
|
+
START_PATTERN = /\A(#{node_ref})\z/
|
257
|
+
DEPEND_PATTERN = /\A(#{node_ref})?\s*->\s*(#{node_ref})\z/
|
258
|
+
|
259
|
+
def foreach_edge(f)
|
260
|
+
default_src = nil
|
261
|
+
f.each do |line|
|
262
|
+
text = line.sub(/\#.*/, '').strip
|
263
|
+
next if text.empty?
|
264
|
+
loc = Location.for_file(f)
|
265
|
+
|
266
|
+
if m = DEPEND_PATTERN.match(text)
|
267
|
+
src = (m[1] ? ref(m[1], loc) : default_src) or
|
268
|
+
raise ParameterError, "syntax error at #{loc}: '->' must follow any job"
|
269
|
+
dest = ref(m[2], loc)
|
270
|
+
yield src, dest
|
271
|
+
default_src = dest
|
272
|
+
|
273
|
+
elsif m = START_PATTERN.match(text)
|
274
|
+
dest = ref(m[1], loc)
|
275
|
+
yield @jobnet_ref.start_ref, dest
|
276
|
+
default_src = dest
|
277
|
+
|
278
|
+
else
|
279
|
+
raise ParameterError, "syntax error at #{loc}: #{line.strip.inspect}"
|
280
|
+
end
|
281
|
+
end
|
282
|
+
end
|
232
283
|
|
233
|
-
def
|
234
|
-
|
284
|
+
def ref(ref_str, location)
|
285
|
+
Ref.parse(ref_str, @jobnet_ref.subsystem, location)
|
235
286
|
end
|
287
|
+
end
|
236
288
|
|
289
|
+
class Ref
|
237
290
|
def Ref.parse(ref, curr_subsys = nil, location = Location.dummy)
|
238
|
-
return JobNetRef.new(nil, '', location) if ref == ROOT_FLOW_NAME
|
239
291
|
m = %r<\A(\*)?(?:(\w[\w\-]*)/)?(@?\w[\w\-]*)\z>.match(ref) or
|
240
292
|
raise ParameterError, "bad job name: #{ref.inspect}"
|
241
293
|
is_net, subsys, name = m.captures
|
@@ -287,14 +339,18 @@ module Bricolage
|
|
287
339
|
end
|
288
340
|
|
289
341
|
class JobNetRef < Ref
|
342
|
+
def JobNetRef.for_path(path)
|
343
|
+
new(path.parent.basename, path.basename('.jobnet'), Location.dummy)
|
344
|
+
end
|
345
|
+
|
290
346
|
def initialize(subsys, name, location)
|
291
347
|
super
|
292
|
-
@
|
348
|
+
@jobnet = nil
|
349
|
+
@start = nil
|
350
|
+
@end = nil
|
293
351
|
end
|
294
352
|
|
295
|
-
|
296
|
-
@flow = flow
|
297
|
-
end
|
353
|
+
attr_accessor :jobnet
|
298
354
|
|
299
355
|
def net?
|
300
356
|
true
|
@@ -307,6 +363,23 @@ module Bricolage
|
|
307
363
|
def relative_path
|
308
364
|
"#{subsystem}/#{name}.jobnet"
|
309
365
|
end
|
366
|
+
|
367
|
+
def start_ref
|
368
|
+
@start ||= JobRef.new(subsystem, "@#{name}@start", location)
|
369
|
+
end
|
370
|
+
|
371
|
+
def end_ref
|
372
|
+
@end ||= JobRef.new(subsystem, "@#{name}@end", location)
|
373
|
+
end
|
374
|
+
|
375
|
+
|
376
|
+
def start
|
377
|
+
@jobnet.start
|
378
|
+
end
|
379
|
+
|
380
|
+
def end
|
381
|
+
@jobnet.end
|
382
|
+
end
|
310
383
|
end
|
311
384
|
|
312
385
|
class Location
|
@@ -335,55 +408,6 @@ module Bricolage
|
|
335
408
|
end
|
336
409
|
end
|
337
410
|
|
338
|
-
class Parser
|
339
|
-
def initialize(subsys)
|
340
|
-
@subsys = subsys
|
341
|
-
end
|
342
|
-
|
343
|
-
def parse_stream(f, ref)
|
344
|
-
flow = JobFlow.new(ref, Location.for_file(f))
|
345
|
-
foreach_edge(f) do |src, dest|
|
346
|
-
flow.add_edge src, dest
|
347
|
-
end
|
348
|
-
flow
|
349
|
-
end
|
350
|
-
|
351
|
-
private
|
352
|
-
|
353
|
-
name = /\w[\w\-]*/
|
354
|
-
node_ref = %r<[@*]?(?:#{name}/)?#{name}>
|
355
|
-
START_PATTERN = /\A(#{node_ref})\z/
|
356
|
-
DEPEND_PATTERN = /\A(#{node_ref})?\s*->\s*(#{node_ref})\z/
|
357
|
-
|
358
|
-
def foreach_edge(f)
|
359
|
-
default_src = Ref.dummy
|
360
|
-
f.each do |line|
|
361
|
-
text = line.sub(/\#.*/, '').strip
|
362
|
-
next if text.empty?
|
363
|
-
loc = Location.for_file(f)
|
364
|
-
|
365
|
-
if m = DEPEND_PATTERN.match(text)
|
366
|
-
src = m[1] ? ref(m[1], loc) : default_src
|
367
|
-
dest = ref(m[2], loc)
|
368
|
-
yield src, dest
|
369
|
-
default_src = dest
|
370
|
-
|
371
|
-
elsif m = START_PATTERN.match(text)
|
372
|
-
dest = ref(m[1], loc)
|
373
|
-
yield Ref.dummy, dest
|
374
|
-
default_src = dest
|
375
|
-
|
376
|
-
else
|
377
|
-
raise ParameterError, "syntax error at #{loc}: #{line.strip.inspect}"
|
378
|
-
end
|
379
|
-
end
|
380
|
-
end
|
381
|
-
|
382
|
-
def ref(ref_str, location)
|
383
|
-
Ref.parse(ref_str, @subsys, location)
|
384
|
-
end
|
385
|
-
end
|
386
|
-
|
387
411
|
end
|
388
412
|
|
389
413
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'bricolage/application'
|
2
2
|
require 'bricolage/context'
|
3
|
-
require 'bricolage/
|
3
|
+
require 'bricolage/jobnet'
|
4
4
|
require 'bricolage/taskqueue'
|
5
5
|
require 'bricolage/job'
|
6
6
|
require 'bricolage/jobresult'
|
@@ -23,8 +23,8 @@ module Bricolage
|
|
23
23
|
def initialize
|
24
24
|
Signal.trap('PIPE', 'IGNORE')
|
25
25
|
@hooks = ::Bricolage
|
26
|
-
@
|
27
|
-
@
|
26
|
+
@jobnet_id = nil
|
27
|
+
@jobnet_start_time = Time.now
|
28
28
|
@log_path = nil
|
29
29
|
end
|
30
30
|
|
@@ -37,15 +37,15 @@ module Bricolage
|
|
37
37
|
@hooks.run_before_option_parsing_hooks(opts)
|
38
38
|
opts.parse ARGV
|
39
39
|
@ctx = Context.for_application(nil, opts.jobnet_file, environment: opts.environment, global_variables: opts.global_variables)
|
40
|
-
@
|
40
|
+
@jobnet_id = "#{opts.jobnet_file.dirname.basename}/#{opts.jobnet_file.basename('.jobnet')}"
|
41
41
|
@log_path = opts.log_path
|
42
|
-
|
42
|
+
jobnet = RootJobNet.load(@ctx, opts.jobnet_file)
|
43
43
|
queue = get_queue(opts)
|
44
44
|
if queue.locked?
|
45
45
|
raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help}"
|
46
46
|
end
|
47
47
|
unless queue.queued?
|
48
|
-
enqueue_jobs
|
48
|
+
enqueue_jobs jobnet, queue
|
49
49
|
logger.info "jobs are queued." if opts.queue_exist?
|
50
50
|
end
|
51
51
|
if opts.list_jobs?
|
@@ -79,25 +79,18 @@ module Bricolage
|
|
79
79
|
end
|
80
80
|
end
|
81
81
|
|
82
|
-
def enqueue_jobs(
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
seq += 1
|
88
|
-
end
|
82
|
+
def enqueue_jobs(jobnet, queue)
|
83
|
+
seq = 1
|
84
|
+
jobnet.sequential_jobs.each do |ref|
|
85
|
+
queue.enq JobTask.new(ref)
|
86
|
+
seq += 1
|
89
87
|
end
|
90
88
|
queue.save
|
91
89
|
end
|
92
90
|
|
93
91
|
def list_jobs(queue)
|
94
|
-
prev = nil
|
95
92
|
queue.each do |task|
|
96
|
-
if not prev or prev.jobnet != task.jobnet
|
97
|
-
puts "---- jobnet #{task.jobnet} ---"
|
98
|
-
end
|
99
93
|
puts task.job
|
100
|
-
prev = task
|
101
94
|
end
|
102
95
|
end
|
103
96
|
|
@@ -108,18 +101,18 @@ module Bricolage
|
|
108
101
|
end
|
109
102
|
|
110
103
|
def run_queue(queue)
|
111
|
-
@hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@
|
104
|
+
@hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@jobnet_id, queue))
|
112
105
|
queue.consume_each do |task|
|
113
106
|
result = execute_job(task.job, queue)
|
114
107
|
unless result.success?
|
115
|
-
logger.elapsed_time 'jobnet total: ', (Time.now - @
|
108
|
+
logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
|
116
109
|
logger.error "[job #{task.job}] #{result.message}"
|
117
110
|
@hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(false, queue))
|
118
111
|
exit result.status
|
119
112
|
end
|
120
113
|
end
|
121
114
|
@hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(true, queue))
|
122
|
-
logger.elapsed_time 'jobnet total: ', (Time.now - @
|
115
|
+
logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
|
123
116
|
logger.info "status all green"
|
124
117
|
end
|
125
118
|
|
@@ -142,11 +135,11 @@ module Bricolage
|
|
142
135
|
start_time = Time.now
|
143
136
|
@log_path.gsub(/%\{\w+\}/) {|var|
|
144
137
|
case var
|
145
|
-
when '%{
|
146
|
-
when '%{
|
138
|
+
when '%{jobnet_start_date}' then @jobnet_start_time.strftime('%Y%m%d')
|
139
|
+
when '%{jobnet_start_time}' then @jobnet_start_time.strftime('%Y%m%d_%H%M%S%L')
|
147
140
|
when '%{job_start_date}' then start_time.strftime('%Y%m%d')
|
148
141
|
when '%{job_start_time}' then start_time.strftime('%Y%m%d_%H%M%S%L')
|
149
|
-
when '%{flow}', '%{flow_id}' then @
|
142
|
+
when '%{jobnet}', '%{net}', '%{jobnet_id}', '%{net_id}', '%{flow}', '%{flow_id}' then @jobnet_id.gsub('/', '::')
|
150
143
|
when '%{subsystem}' then job_ref.subsystem
|
151
144
|
when '%{job}', '%{job_id}' then job_ref.name
|
152
145
|
else
|
data/lib/bricolage/taskqueue.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'bricolage/
|
1
|
+
require 'bricolage/jobnet'
|
2
2
|
require 'bricolage/exception'
|
3
3
|
require 'pathname'
|
4
4
|
|
@@ -133,23 +133,19 @@ module Bricolage
|
|
133
133
|
end
|
134
134
|
|
135
135
|
class JobTask
|
136
|
-
def initialize(
|
137
|
-
@jobnet = jobnet
|
138
|
-
@seq = seq
|
136
|
+
def initialize(job)
|
139
137
|
@job = job
|
140
138
|
end
|
141
139
|
|
142
|
-
attr_reader :jobnet
|
143
|
-
attr_reader :seq
|
144
140
|
attr_reader :job
|
145
141
|
|
146
142
|
def serialize
|
147
|
-
[@
|
143
|
+
[@job].join("\t")
|
148
144
|
end
|
149
145
|
|
150
146
|
def JobTask.deserialize(str)
|
151
|
-
|
152
|
-
new(
|
147
|
+
job, * = str.strip.split("\t")
|
148
|
+
new(JobNet::Ref.parse(job))
|
153
149
|
end
|
154
150
|
end
|
155
151
|
|
data/lib/bricolage/version.rb
CHANGED
data/test/home/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: ../..
|
3
3
|
specs:
|
4
|
-
bricolage (5.
|
4
|
+
bricolage (5.11.0)
|
5
5
|
aws-sdk (< 2)
|
6
6
|
mysql2
|
7
7
|
pg
|
@@ -23,7 +23,7 @@ GEM
|
|
23
23
|
json (1.8.3)
|
24
24
|
mini_portile (0.6.2)
|
25
25
|
msgpack (0.5.11)
|
26
|
-
mysql2 (0.4.
|
26
|
+
mysql2 (0.4.1)
|
27
27
|
nokogiri (1.6.6.2)
|
28
28
|
mini_portile (~> 0.6.0)
|
29
29
|
parallel (0.6.5)
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'bricolage/jobflow'
|
2
|
+
require 'pathname'
|
3
|
+
require 'stringio'
|
4
|
+
require 'forwardable'
|
5
|
+
require 'pp'
|
6
|
+
|
7
|
+
class ConstantLoader
|
8
|
+
def initialize(subnets)
|
9
|
+
@subnets = subnets
|
10
|
+
end
|
11
|
+
|
12
|
+
def load(ref)
|
13
|
+
@subnets[ref.to_s]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def root_flow(subnets)
|
18
|
+
h = {}
|
19
|
+
subnets.each do |flow|
|
20
|
+
h[flow.name] = flow
|
21
|
+
end
|
22
|
+
root = Bricolage::RootJobFlow.new(ConstantLoader.new(h))
|
23
|
+
subnets.each do |flow|
|
24
|
+
root.add_subnet flow
|
25
|
+
end
|
26
|
+
root.fix
|
27
|
+
root
|
28
|
+
end
|
29
|
+
|
30
|
+
class StringFile
|
31
|
+
def initialize(str, path)
|
32
|
+
@f = StringIO.new(str)
|
33
|
+
@path = path
|
34
|
+
end
|
35
|
+
|
36
|
+
attr_reader :path
|
37
|
+
|
38
|
+
extend Forwardable
|
39
|
+
def_delegators '@f', :each, :lineno
|
40
|
+
end
|
41
|
+
|
42
|
+
def make_flow(src, name)
|
43
|
+
path = "test/#{name}.jobnet"
|
44
|
+
ref = Bricolage::JobFlow.make_node_ref(Pathname(path))
|
45
|
+
Bricolage::JobFlow.parse_stream(StringFile.new(src, path), ref)
|
46
|
+
end
|
47
|
+
|
48
|
+
net1 = make_flow(<<-End, 'net1')
|
49
|
+
job1
|
50
|
+
-> *net2
|
51
|
+
-> job4
|
52
|
+
End
|
53
|
+
|
54
|
+
net2 = make_flow(<<-End, 'net2')
|
55
|
+
job3
|
56
|
+
-> job4
|
57
|
+
End
|
58
|
+
|
59
|
+
root_flow([net1, net2]).each_subnet do |flow|
|
60
|
+
pp flow.sequential_nodes
|
61
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
class: noop
|
@@ -0,0 +1 @@
|
|
1
|
+
class: noop
|
@@ -0,0 +1 @@
|
|
1
|
+
class: noop
|
@@ -0,0 +1 @@
|
|
1
|
+
class: noop
|
@@ -0,0 +1 @@
|
|
1
|
+
class: noop
|
@@ -0,0 +1 @@
|
|
1
|
+
class: noop
|
@@ -0,0 +1 @@
|
|
1
|
+
class: noop
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bricolage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Minero Aoki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-10-
|
11
|
+
date: 2015-10-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pg
|
@@ -137,7 +137,7 @@ files:
|
|
137
137
|
- lib/bricolage/job.rb
|
138
138
|
- lib/bricolage/jobclass.rb
|
139
139
|
- lib/bricolage/jobfile.rb
|
140
|
-
- lib/bricolage/
|
140
|
+
- lib/bricolage/jobnet.rb
|
141
141
|
- lib/bricolage/jobnetrunner.rb
|
142
142
|
- lib/bricolage/jobresult.rb
|
143
143
|
- lib/bricolage/logger.rb
|
@@ -168,12 +168,22 @@ files:
|
|
168
168
|
- test/home/data/20141002-1355_00.txt
|
169
169
|
- test/home/data/20141002-1355_01.txt
|
170
170
|
- test/home/data/20141002-1355_02.txt
|
171
|
+
- test/home/jobnet-test.rb
|
171
172
|
- test/home/put.sh
|
172
173
|
- test/home/revert.sh
|
173
174
|
- test/home/subsys/d.ct
|
175
|
+
- test/home/subsys/job1.job
|
176
|
+
- test/home/subsys/job2.job
|
177
|
+
- test/home/subsys/job3.job
|
178
|
+
- test/home/subsys/job4.job
|
179
|
+
- test/home/subsys/job5.job
|
180
|
+
- test/home/subsys/job6.job
|
181
|
+
- test/home/subsys/job7.job
|
174
182
|
- test/home/subsys/load_test.ct
|
175
183
|
- test/home/subsys/load_test.job
|
176
184
|
- test/home/subsys/migrate.job
|
185
|
+
- test/home/subsys/net1.jobnet
|
186
|
+
- test/home/subsys/net2.jobnet
|
177
187
|
- test/home/subsys/raw-vacuum.sql.job
|
178
188
|
- test/home/subsys/rebuild.sql.job
|
179
189
|
- test/home/subsys/search_backends.ct
|