bricolage 5.10.0 → 5.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 950cab4515b421cdfd9a29fba1fa7812d365647a
4
- data.tar.gz: 93badbfd8528d66c97f2cdb5bda2217172386bd8
3
+ metadata.gz: 1ff4f8adcd84bf94e55cb1d285a3b5671210906e
4
+ data.tar.gz: b1f3f08b9c0e74a5e6d469ba197a29d08b51551f
5
5
  SHA512:
6
- metadata.gz: 5bc1b110a47f6cf5f81a30595fe9b200b77a6b7918bf4b2f7a8064597b3af36e50cc73bf91d32e4def188c43dd2829a81a57184a87e3c82bcad054cdbf3da7a4
7
- data.tar.gz: e9cba607a0436f613093ae654b347d69c04024a5ac97860fc113d61e6a34e3c7f9f55afa856c584442a07ed1a02781dc2634d1d9707df725ea9d0aa30c89d666
6
+ metadata.gz: aec57849ed95f3c50bd0befdcaf66ddad3f6ab2aeb241e139f51e996a64e18ae63023e624a13f44daa9a5330d75d2928cc0df447f4ecae3c9055934b519647c5
7
+ data.tar.gz: dab80eab00b0e147249fcf6f53a6a75314a0b2161991d46fc0ffe21cedbc29fae3a6e0f00c6d95c60f3ddc26a5438c7d93061ca132a3714d0192b02abc63c593
@@ -40,7 +40,12 @@ module Bricolage
40
40
  }
41
41
  end
42
42
 
43
- BeforeAllJobsEvent = Struct.new(:flow_id, :queue)
43
+ BeforeAllJobsEvent = Struct.new(:jobnet_id, :queue)
44
+ class BeforeAllJobsEvent # reopen
45
+ def flow_id
46
+ jobnet_id
47
+ end
48
+ end
44
49
  BeforeJobEvent = Struct.new(:job)
45
50
  AfterJobEvent = Struct.new(:result)
46
51
  AfterAllJobsEvent = Struct.new(:succeeded, :queue)
@@ -3,80 +3,100 @@ require 'tsort'
3
3
 
4
4
  module Bricolage
5
5
 
6
- class JobFlow
7
- def JobFlow.load(path)
8
- File.open(path) {|f|
9
- parse_stream(f, make_node_ref(path))
10
- }
11
- rescue SystemCallError => err
12
- raise ParameterError, "could not load job flow: #{path} (#{err.message})"
6
+ # Represents "first" jobnet given by command line (e.g. bricolage-jobnet some.jobnet)
7
+ class RootJobNet
8
+ def RootJobNet.load(ctx, path)
9
+ root = new(JobNet::FileLoader.new(ctx), JobNet.load(path))
10
+ root.load_recursive
11
+ root.fix
12
+ root
13
13
  end
14
14
 
15
- def JobFlow.make_node_ref(path)
16
- subsys = path.parent.basename.to_s
17
- name = path.basename('.jobnet').to_s
18
- JobNetRef.new(subsys, name, Location.dummy)
15
+ def initialize(jobnet_loader, start_jobnet)
16
+ @jobnet_loader = jobnet_loader
17
+ @start_jobnet = start_jobnet
18
+ @jobnets = {start_jobnet.ref => start_jobnet}
19
+ @graph = nil
19
20
  end
20
21
 
21
- def JobFlow.parse_stream(f, ref)
22
- Parser.new(ref.subsystem).parse_stream(f, ref)
23
- end
22
+ attr_reader :start_jobnet
24
23
 
25
- def JobFlow.root
26
- RootJobFlow.new
24
+ def each_jobnet(&block)
25
+ @jobnets.each_value(&block)
27
26
  end
28
27
 
29
- ROOT_FLOW_NAME = '*'
30
-
31
- def initialize(ref, location)
32
- @ref = ref
33
- @location = location
34
- @flow = {} # Ref => [Ref] (src->dest)
35
- @deps = {} # Ref => [Ref] (dest->src)
36
- @subnets_resolved = false
28
+ def jobnets
29
+ @jobnets.values
37
30
  end
38
31
 
39
- def inspect
40
- "\#<#{self.class} #{ref}>"
32
+ def load_recursive
33
+ unresolved = [@start_jobnet]
34
+ until unresolved.empty?
35
+ loaded = []
36
+ unresolved.each do |net|
37
+ net.net_refs.each do |ref|
38
+ next if ref.jobnet
39
+ unless net = @jobnets[ref]
40
+ net = @jobnet_loader.load(ref)
41
+ @jobnets[ref] = net
42
+ loaded.push net
43
+ end
44
+ ref.jobnet = net
45
+ end
46
+ end
47
+ unresolved = loaded
48
+ end
41
49
  end
42
50
 
43
- attr_reader :ref
44
-
45
- def name
46
- ref.to_s
51
+ def fix
52
+ each_jobnet do |net|
53
+ net.fix
54
+ end
55
+ @jobnets.freeze
56
+ @dag = JobDAG.build(jobnets)
47
57
  end
48
58
 
49
- def add_edge(src, dest)
50
- (@flow[src] ||= []).push dest unless dest.net?
51
- (@deps[dest] ||= []).push src
59
+ def sequential_jobs
60
+ @dag.sequential_jobs
52
61
  end
62
+ end
53
63
 
54
- def flow_tree
55
- h = {}
56
- @flow.each do |src, dests|
57
- h[src.to_s] = dests.map {|d| d.to_s }
64
+ class JobDAG
65
+ def JobDAG.build(jobnets)
66
+ graph = new
67
+ jobnets.each do |net|
68
+ graph.merge! net
58
69
  end
59
- h
70
+ graph.fix
71
+ graph
60
72
  end
61
73
 
62
- def dependencies
74
+ def initialize
75
+ @deps = Hash.new { Array.new } # {JobRef => [JobRef]} (dest->srcs)
76
+ end
77
+
78
+ def to_hash
63
79
  h = {}
64
- @deps.each do |ref, deps|
65
- h[ref.to_s] = deps.map(&:to_s)
80
+ @deps.each do |dest, srcs|
81
+ h[dest.to_s] = srcs.map(&:to_s)
66
82
  end
67
83
  h
68
84
  end
69
85
 
70
- def dependent_flows
71
- @deps.values.flatten.select {|ref| ref.net? }.uniq
86
+ def merge!(net)
87
+ net.each_dependencies do |ref, deps|
88
+ @deps[ref] |= deps
89
+ end
72
90
  end
73
91
 
74
- def sequential_nodes
75
- tsort.reject {|ref| ref.dummy? }
92
+ def fix
93
+ @deps.freeze
94
+ check_cycle
95
+ check_orphan
76
96
  end
77
97
 
78
98
  def sequential_jobs
79
- sequential_nodes.reject {|ref| ref.net? }
99
+ tsort.reject {|ref| ref.dummy? }
80
100
  end
81
101
 
82
102
  include TSort
@@ -89,153 +109,185 @@ module Bricolage
89
109
  @deps.fetch(ref).each(&block)
90
110
  end
91
111
 
92
- def fix_graph
93
- close_graph
94
- check_cycle
95
- check_orphan
96
- end
97
-
98
- def resolve_subnets(root_flow)
99
- @deps.each_key do |ref|
100
- next unless ref.net?
101
- ref.flow = root_flow.subnet(ref)
102
- end
103
- @subnets_resolved = true
104
- end
105
-
106
- def subnets_resolved?
107
- @subnets_resolved
108
- end
109
-
110
112
  private
111
113
 
112
- def close_graph
113
- @deps.values.flatten.uniq.each do |ref|
114
- @deps[ref] ||= []
115
- end
116
- end
117
-
118
114
  def check_cycle
119
115
  each_strongly_connected_component do |refs|
120
116
  unless refs.size == 1
121
117
  cycle = (refs + [refs.first]).reverse.join(' -> ')
122
- raise ParameterError, "found cycle in the flow: #{cycle}"
118
+ raise ParameterError, "found cycle in the jobnet: #{cycle}"
123
119
  end
124
120
  end
125
121
  end
126
122
 
127
123
  def check_orphan
128
124
  orphan_nodes.each do |ref|
129
- raise ParameterError, "found orphan job in the flow: #{ref.location}: #{ref}"
125
+ raise ParameterError, "found orphan job in the jobnet: #{ref.location}: #{ref}"
130
126
  end
131
127
  end
132
128
 
133
129
  def orphan_nodes
134
- @deps.to_a.select {|ref, deps| deps.empty? and not ref.dummy? and not ref.net? }.map {|ref, *| ref }
130
+ @deps.to_a.select {|ref, deps| deps.empty? and not ref.dummy? }.map {|ref, *| ref }
135
131
  end
136
132
  end
137
133
 
138
- class RootJobFlow < JobFlow
139
- def RootJobFlow.load(ctx, path)
140
- flow = new(ctx)
141
- flow.add_subnet JobFlow.load(path)
142
- flow.fix
143
- flow
134
+ class JobNet
135
+ def JobNet.load(path, ref = JobNetRef.for_path(path))
136
+ File.open(path) {|f|
137
+ Parser.new(ref).parse_stream(f)
138
+ }
139
+ rescue SystemCallError => err
140
+ raise ParameterError, "could not load jobnet: #{path} (#{err.message})"
144
141
  end
145
142
 
146
- def initialize(ctx)
147
- @ctx = ctx
148
- super ROOT_FLOW_NAME, Location.dummy
149
- @subnets = {}
143
+ def initialize(ref, location)
144
+ @ref = ref
145
+ @location = location
146
+ @flow = {} # Ref => [Ref] (src->dest)
147
+ @deps = {} # Ref => [Ref] (dest->src)
150
148
  end
151
149
 
152
- def add_subnet(flow)
153
- raise ParameterError, "duplicated subnet definition: #{flow.name}" if @subnets.key?(flow.name)
154
- @subnets[flow.name] = flow
150
+ def inspect
151
+ "\#<#{self.class} #{ref}>"
155
152
  end
156
153
 
157
- def subnet(ref)
158
- if flow = @subnets[ref]
159
- flow
160
- else
161
- flow = load_jobnet_auto(ref)
162
- add_subnet flow
163
- flow
164
- end
154
+ attr_reader :ref
155
+
156
+ def start
157
+ @ref.start_ref
165
158
  end
166
159
 
167
- def load_jobnet_auto(ref)
168
- path = @ctx.root_relative_path(ref.relative_path)
169
- raise ParameterError, "undefined subnet: #{ref}" unless path.file?
170
- JobFlow.load(path)
160
+ def end
161
+ @ref.end_ref
171
162
  end
172
163
 
173
- def each_subnet(&block)
174
- @subnets.values.each(&block)
164
+ def name
165
+ @ref.to_s
175
166
  end
176
167
 
177
- def each_flow(&block)
178
- yield self
179
- @subnets.each_value(&block)
168
+ def add_edge(src, dest)
169
+ (@flow[src] ||= []).push dest
170
+ (@deps[dest] ||= []).push src
180
171
  end
181
172
 
182
- def each_subnet_sequence
183
- sequential_nodes.each do |ref|
184
- yield subnet(ref)
173
+ def to_hash
174
+ h = {}
175
+ @flow.each do |src, dests|
176
+ h[src.to_s] = dests.map(&:to_s)
185
177
  end
178
+ h
186
179
  end
187
180
 
188
- def fix
189
- resolve_subnet_references
190
- each_subnet do |flow|
191
- flow.fix_graph
181
+ def to_deps_hash
182
+ h = {}
183
+ @deps.each do |dest, srcs|
184
+ h[dest.to_s] = srcs.map(&:to_s)
192
185
  end
193
- add_jobnet_dependency_edges
194
- fix_graph
186
+ h
195
187
  end
196
188
 
197
- private
189
+ def refs
190
+ @flow.keys | @flow.values.flatten
191
+ end
198
192
 
199
- def resolve_subnet_references
200
- unresolved = true
201
- while unresolved
202
- unresolved = false
203
- ([self] + @subnets.values).each do |flow|
204
- unless flow.subnets_resolved?
205
- flow.resolve_subnets self
206
- unresolved = true
207
- end
208
- end
209
- end
193
+ def net_refs
194
+ @deps.keys.select {|ref| ref.net? }
210
195
  end
211
196
 
212
- def add_jobnet_dependency_edges
213
- each_subnet do |flow|
214
- # dummy dependency to ensure to execute all subnets
215
- add_edge Ref.dummy, flow.ref
216
- # jobnet -> jobnet dependency
217
- flow.dependent_flows.each do |dep_flow_ref|
218
- add_edge dep_flow_ref, flow.ref
197
+ # Adds dummy dependencies (@start and @end) to fix up all jobs
198
+ # into one DAG beginning with @start and ending with @end
199
+ def fix
200
+ refs.each do |ref|
201
+ next if ref.dummy?
202
+ unless @deps[ref]
203
+ (@flow[self.start] ||= []).push ref
204
+ @deps[ref] = [self.start]
205
+ end
206
+ unless @flow[ref]
207
+ (@flow[ref] ||= []).push self.end
208
+ (@deps[self.end] ||= []).push ref
219
209
  end
220
210
  end
211
+ @deps[self.start] ||= []
212
+ @flow.freeze
213
+ @deps.freeze
221
214
  end
222
215
 
223
- def check_orphan
224
- # should not check orphan for root.
216
+ def each_dependencies
217
+ @deps.each do |ref, deps|
218
+ dest = (ref.net? ? ref.start : ref)
219
+ srcs = deps.map {|r| r.net? ? r.end : r }
220
+ yield dest, srcs
221
+ end
225
222
  end
226
223
  end
227
224
 
228
- class JobFlow # reopen as namespace
225
+ class JobNet # reopen as namespace
229
226
 
230
- class Ref
231
- START_NAME = '@start'
227
+ class FileLoader
228
+ def initialize(ctx)
229
+ @context = ctx
230
+ end
231
+
232
+ def load(ref)
233
+ path = @context.root_relative_path(ref.relative_path)
234
+ raise ParameterError, "undefined subnet: #{ref}" unless path.file?
235
+ JobNet.load(path, ref)
236
+ end
237
+ end
238
+
239
+ class Parser
240
+ def initialize(jobnet_ref)
241
+ @jobnet_ref = jobnet_ref
242
+ end
243
+
244
+ def parse_stream(f)
245
+ net = JobNet.new(@jobnet_ref, Location.for_file(f))
246
+ foreach_edge(f) do |src, dest|
247
+ net.add_edge src, dest
248
+ end
249
+ net
250
+ end
251
+
252
+ private
253
+
254
+ name = /\w[\w\-]*/
255
+ node_ref = %r<[@*]?(?:#{name}/)?#{name}>
256
+ START_PATTERN = /\A(#{node_ref})\z/
257
+ DEPEND_PATTERN = /\A(#{node_ref})?\s*->\s*(#{node_ref})\z/
258
+
259
+ def foreach_edge(f)
260
+ default_src = nil
261
+ f.each do |line|
262
+ text = line.sub(/\#.*/, '').strip
263
+ next if text.empty?
264
+ loc = Location.for_file(f)
265
+
266
+ if m = DEPEND_PATTERN.match(text)
267
+ src = (m[1] ? ref(m[1], loc) : default_src) or
268
+ raise ParameterError, "syntax error at #{loc}: '->' must follow any job"
269
+ dest = ref(m[2], loc)
270
+ yield src, dest
271
+ default_src = dest
272
+
273
+ elsif m = START_PATTERN.match(text)
274
+ dest = ref(m[1], loc)
275
+ yield @jobnet_ref.start_ref, dest
276
+ default_src = dest
277
+
278
+ else
279
+ raise ParameterError, "syntax error at #{loc}: #{line.strip.inspect}"
280
+ end
281
+ end
282
+ end
232
283
 
233
- def Ref.dummy
234
- JobRef.new(nil, START_NAME, Location.dummy)
284
+ def ref(ref_str, location)
285
+ Ref.parse(ref_str, @jobnet_ref.subsystem, location)
235
286
  end
287
+ end
236
288
 
289
+ class Ref
237
290
  def Ref.parse(ref, curr_subsys = nil, location = Location.dummy)
238
- return JobNetRef.new(nil, '', location) if ref == ROOT_FLOW_NAME
239
291
  m = %r<\A(\*)?(?:(\w[\w\-]*)/)?(@?\w[\w\-]*)\z>.match(ref) or
240
292
  raise ParameterError, "bad job name: #{ref.inspect}"
241
293
  is_net, subsys, name = m.captures
@@ -287,14 +339,18 @@ module Bricolage
287
339
  end
288
340
 
289
341
  class JobNetRef < Ref
342
+ def JobNetRef.for_path(path)
343
+ new(path.parent.basename, path.basename('.jobnet'), Location.dummy)
344
+ end
345
+
290
346
  def initialize(subsys, name, location)
291
347
  super
292
- @flow = nil
348
+ @jobnet = nil
349
+ @start = nil
350
+ @end = nil
293
351
  end
294
352
 
295
- def flow=(flow)
296
- @flow = flow
297
- end
353
+ attr_accessor :jobnet
298
354
 
299
355
  def net?
300
356
  true
@@ -307,6 +363,23 @@ module Bricolage
307
363
  def relative_path
308
364
  "#{subsystem}/#{name}.jobnet"
309
365
  end
366
+
367
+ def start_ref
368
+ @start ||= JobRef.new(subsystem, "@#{name}@start", location)
369
+ end
370
+
371
+ def end_ref
372
+ @end ||= JobRef.new(subsystem, "@#{name}@end", location)
373
+ end
374
+
375
+
376
+ def start
377
+ @jobnet.start
378
+ end
379
+
380
+ def end
381
+ @jobnet.end
382
+ end
310
383
  end
311
384
 
312
385
  class Location
@@ -335,55 +408,6 @@ module Bricolage
335
408
  end
336
409
  end
337
410
 
338
- class Parser
339
- def initialize(subsys)
340
- @subsys = subsys
341
- end
342
-
343
- def parse_stream(f, ref)
344
- flow = JobFlow.new(ref, Location.for_file(f))
345
- foreach_edge(f) do |src, dest|
346
- flow.add_edge src, dest
347
- end
348
- flow
349
- end
350
-
351
- private
352
-
353
- name = /\w[\w\-]*/
354
- node_ref = %r<[@*]?(?:#{name}/)?#{name}>
355
- START_PATTERN = /\A(#{node_ref})\z/
356
- DEPEND_PATTERN = /\A(#{node_ref})?\s*->\s*(#{node_ref})\z/
357
-
358
- def foreach_edge(f)
359
- default_src = Ref.dummy
360
- f.each do |line|
361
- text = line.sub(/\#.*/, '').strip
362
- next if text.empty?
363
- loc = Location.for_file(f)
364
-
365
- if m = DEPEND_PATTERN.match(text)
366
- src = m[1] ? ref(m[1], loc) : default_src
367
- dest = ref(m[2], loc)
368
- yield src, dest
369
- default_src = dest
370
-
371
- elsif m = START_PATTERN.match(text)
372
- dest = ref(m[1], loc)
373
- yield Ref.dummy, dest
374
- default_src = dest
375
-
376
- else
377
- raise ParameterError, "syntax error at #{loc}: #{line.strip.inspect}"
378
- end
379
- end
380
- end
381
-
382
- def ref(ref_str, location)
383
- Ref.parse(ref_str, @subsys, location)
384
- end
385
- end
386
-
387
411
  end
388
412
 
389
413
  end
@@ -1,6 +1,6 @@
1
1
  require 'bricolage/application'
2
2
  require 'bricolage/context'
3
- require 'bricolage/jobflow'
3
+ require 'bricolage/jobnet'
4
4
  require 'bricolage/taskqueue'
5
5
  require 'bricolage/job'
6
6
  require 'bricolage/jobresult'
@@ -23,8 +23,8 @@ module Bricolage
23
23
  def initialize
24
24
  Signal.trap('PIPE', 'IGNORE')
25
25
  @hooks = ::Bricolage
26
- @flow_id = nil
27
- @flow_start_time = Time.now
26
+ @jobnet_id = nil
27
+ @jobnet_start_time = Time.now
28
28
  @log_path = nil
29
29
  end
30
30
 
@@ -37,15 +37,15 @@ module Bricolage
37
37
  @hooks.run_before_option_parsing_hooks(opts)
38
38
  opts.parse ARGV
39
39
  @ctx = Context.for_application(nil, opts.jobnet_file, environment: opts.environment, global_variables: opts.global_variables)
40
- @flow_id = "#{opts.jobnet_file.dirname.basename}/#{opts.jobnet_file.basename('.jobnet')}"
40
+ @jobnet_id = "#{opts.jobnet_file.dirname.basename}/#{opts.jobnet_file.basename('.jobnet')}"
41
41
  @log_path = opts.log_path
42
- flow = RootJobFlow.load(@ctx, opts.jobnet_file)
42
+ jobnet = RootJobNet.load(@ctx, opts.jobnet_file)
43
43
  queue = get_queue(opts)
44
44
  if queue.locked?
45
45
  raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help}"
46
46
  end
47
47
  unless queue.queued?
48
- enqueue_jobs flow, queue
48
+ enqueue_jobs jobnet, queue
49
49
  logger.info "jobs are queued." if opts.queue_exist?
50
50
  end
51
51
  if opts.list_jobs?
@@ -79,25 +79,18 @@ module Bricolage
79
79
  end
80
80
  end
81
81
 
82
- def enqueue_jobs(flow, queue)
83
- flow.each_subnet_sequence do |subnet|
84
- seq = 1
85
- subnet.sequential_jobs.each do |ref|
86
- queue.enq JobTask.new(subnet.ref, seq, ref)
87
- seq += 1
88
- end
82
+ def enqueue_jobs(jobnet, queue)
83
+ seq = 1
84
+ jobnet.sequential_jobs.each do |ref|
85
+ queue.enq JobTask.new(ref)
86
+ seq += 1
89
87
  end
90
88
  queue.save
91
89
  end
92
90
 
93
91
  def list_jobs(queue)
94
- prev = nil
95
92
  queue.each do |task|
96
- if not prev or prev.jobnet != task.jobnet
97
- puts "---- jobnet #{task.jobnet} ---"
98
- end
99
93
  puts task.job
100
- prev = task
101
94
  end
102
95
  end
103
96
 
@@ -108,18 +101,18 @@ module Bricolage
108
101
  end
109
102
 
110
103
  def run_queue(queue)
111
- @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@flow_id, queue))
104
+ @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@jobnet_id, queue))
112
105
  queue.consume_each do |task|
113
106
  result = execute_job(task.job, queue)
114
107
  unless result.success?
115
- logger.elapsed_time 'jobnet total: ', (Time.now - @flow_start_time)
108
+ logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
116
109
  logger.error "[job #{task.job}] #{result.message}"
117
110
  @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(false, queue))
118
111
  exit result.status
119
112
  end
120
113
  end
121
114
  @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(true, queue))
122
- logger.elapsed_time 'jobnet total: ', (Time.now - @flow_start_time)
115
+ logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
123
116
  logger.info "status all green"
124
117
  end
125
118
 
@@ -142,11 +135,11 @@ module Bricolage
142
135
  start_time = Time.now
143
136
  @log_path.gsub(/%\{\w+\}/) {|var|
144
137
  case var
145
- when '%{flow_start_date}' then @flow_start_time.strftime('%Y%m%d')
146
- when '%{flow_start_time}' then @flow_start_time.strftime('%Y%m%d_%H%M%S%L')
138
+ when '%{jobnet_start_date}' then @jobnet_start_time.strftime('%Y%m%d')
139
+ when '%{jobnet_start_time}' then @jobnet_start_time.strftime('%Y%m%d_%H%M%S%L')
147
140
  when '%{job_start_date}' then start_time.strftime('%Y%m%d')
148
141
  when '%{job_start_time}' then start_time.strftime('%Y%m%d_%H%M%S%L')
149
- when '%{flow}', '%{flow_id}' then @flow_id.gsub('/', '::')
142
+ when '%{jobnet}', '%{net}', '%{jobnet_id}', '%{net_id}', '%{flow}', '%{flow_id}' then @jobnet_id.gsub('/', '::')
150
143
  when '%{subsystem}' then job_ref.subsystem
151
144
  when '%{job}', '%{job_id}' then job_ref.name
152
145
  else
@@ -1,4 +1,4 @@
1
- require 'bricolage/jobflow'
1
+ require 'bricolage/jobnet'
2
2
  require 'bricolage/exception'
3
3
  require 'pathname'
4
4
 
@@ -133,23 +133,19 @@ module Bricolage
133
133
  end
134
134
 
135
135
  class JobTask
136
- def initialize(jobnet, seq, job)
137
- @jobnet = jobnet
138
- @seq = seq
136
+ def initialize(job)
139
137
  @job = job
140
138
  end
141
139
 
142
- attr_reader :jobnet
143
- attr_reader :seq
144
140
  attr_reader :job
145
141
 
146
142
  def serialize
147
- [@jobnet, @seq, @job].join("\t")
143
+ [@job].join("\t")
148
144
  end
149
145
 
150
146
  def JobTask.deserialize(str)
151
- jobnet, seq, job = str.strip.split("\t", 3)
152
- new(JobFlow::Ref.parse(jobnet), seq.to_i, JobFlow::Ref.parse(job))
147
+ job, * = str.strip.split("\t")
148
+ new(JobNet::Ref.parse(job))
153
149
  end
154
150
  end
155
151
 
@@ -1,4 +1,4 @@
1
1
  module Bricolage
2
2
  APPLICATION_NAME = 'Bricolage'
3
- VERSION = '5.10.0'
3
+ VERSION = '5.11.0'
4
4
  end
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: ../..
3
3
  specs:
4
- bricolage (5.9.6)
4
+ bricolage (5.11.0)
5
5
  aws-sdk (< 2)
6
6
  mysql2
7
7
  pg
@@ -23,7 +23,7 @@ GEM
23
23
  json (1.8.3)
24
24
  mini_portile (0.6.2)
25
25
  msgpack (0.5.11)
26
- mysql2 (0.4.0)
26
+ mysql2 (0.4.1)
27
27
  nokogiri (1.6.6.2)
28
28
  mini_portile (~> 0.6.0)
29
29
  parallel (0.6.5)
@@ -0,0 +1,61 @@
1
+ require 'bricolage/jobflow'
2
+ require 'pathname'
3
+ require 'stringio'
4
+ require 'forwardable'
5
+ require 'pp'
6
+
7
+ class ConstantLoader
8
+ def initialize(subnets)
9
+ @subnets = subnets
10
+ end
11
+
12
+ def load(ref)
13
+ @subnets[ref.to_s]
14
+ end
15
+ end
16
+
17
+ def root_flow(subnets)
18
+ h = {}
19
+ subnets.each do |flow|
20
+ h[flow.name] = flow
21
+ end
22
+ root = Bricolage::RootJobFlow.new(ConstantLoader.new(h))
23
+ subnets.each do |flow|
24
+ root.add_subnet flow
25
+ end
26
+ root.fix
27
+ root
28
+ end
29
+
30
+ class StringFile
31
+ def initialize(str, path)
32
+ @f = StringIO.new(str)
33
+ @path = path
34
+ end
35
+
36
+ attr_reader :path
37
+
38
+ extend Forwardable
39
+ def_delegators '@f', :each, :lineno
40
+ end
41
+
42
+ def make_flow(src, name)
43
+ path = "test/#{name}.jobnet"
44
+ ref = Bricolage::JobFlow.make_node_ref(Pathname(path))
45
+ Bricolage::JobFlow.parse_stream(StringFile.new(src, path), ref)
46
+ end
47
+
48
+ net1 = make_flow(<<-End, 'net1')
49
+ job1
50
+ -> *net2
51
+ -> job4
52
+ End
53
+
54
+ net2 = make_flow(<<-End, 'net2')
55
+ job3
56
+ -> job4
57
+ End
58
+
59
+ root_flow([net1, net2]).each_subnet do |flow|
60
+ pp flow.sequential_nodes
61
+ end
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1,3 @@
1
+ job1
2
+ -> *net2
3
+ -> job4
@@ -0,0 +1,2 @@
1
+ job2
2
+ -> job3
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bricolage
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.10.0
4
+ version: 5.11.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Minero Aoki
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-27 00:00:00.000000000 Z
11
+ date: 2015-10-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pg
@@ -137,7 +137,7 @@ files:
137
137
  - lib/bricolage/job.rb
138
138
  - lib/bricolage/jobclass.rb
139
139
  - lib/bricolage/jobfile.rb
140
- - lib/bricolage/jobflow.rb
140
+ - lib/bricolage/jobnet.rb
141
141
  - lib/bricolage/jobnetrunner.rb
142
142
  - lib/bricolage/jobresult.rb
143
143
  - lib/bricolage/logger.rb
@@ -168,12 +168,22 @@ files:
168
168
  - test/home/data/20141002-1355_00.txt
169
169
  - test/home/data/20141002-1355_01.txt
170
170
  - test/home/data/20141002-1355_02.txt
171
+ - test/home/jobnet-test.rb
171
172
  - test/home/put.sh
172
173
  - test/home/revert.sh
173
174
  - test/home/subsys/d.ct
175
+ - test/home/subsys/job1.job
176
+ - test/home/subsys/job2.job
177
+ - test/home/subsys/job3.job
178
+ - test/home/subsys/job4.job
179
+ - test/home/subsys/job5.job
180
+ - test/home/subsys/job6.job
181
+ - test/home/subsys/job7.job
174
182
  - test/home/subsys/load_test.ct
175
183
  - test/home/subsys/load_test.job
176
184
  - test/home/subsys/migrate.job
185
+ - test/home/subsys/net1.jobnet
186
+ - test/home/subsys/net2.jobnet
177
187
  - test/home/subsys/raw-vacuum.sql.job
178
188
  - test/home/subsys/rebuild.sql.job
179
189
  - test/home/subsys/search_backends.ct