bricolage 5.10.0 → 5.11.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 950cab4515b421cdfd9a29fba1fa7812d365647a
4
- data.tar.gz: 93badbfd8528d66c97f2cdb5bda2217172386bd8
3
+ metadata.gz: 1ff4f8adcd84bf94e55cb1d285a3b5671210906e
4
+ data.tar.gz: b1f3f08b9c0e74a5e6d469ba197a29d08b51551f
5
5
  SHA512:
6
- metadata.gz: 5bc1b110a47f6cf5f81a30595fe9b200b77a6b7918bf4b2f7a8064597b3af36e50cc73bf91d32e4def188c43dd2829a81a57184a87e3c82bcad054cdbf3da7a4
7
- data.tar.gz: e9cba607a0436f613093ae654b347d69c04024a5ac97860fc113d61e6a34e3c7f9f55afa856c584442a07ed1a02781dc2634d1d9707df725ea9d0aa30c89d666
6
+ metadata.gz: aec57849ed95f3c50bd0befdcaf66ddad3f6ab2aeb241e139f51e996a64e18ae63023e624a13f44daa9a5330d75d2928cc0df447f4ecae3c9055934b519647c5
7
+ data.tar.gz: dab80eab00b0e147249fcf6f53a6a75314a0b2161991d46fc0ffe21cedbc29fae3a6e0f00c6d95c60f3ddc26a5438c7d93061ca132a3714d0192b02abc63c593
@@ -40,7 +40,12 @@ module Bricolage
40
40
  }
41
41
  end
42
42
 
43
- BeforeAllJobsEvent = Struct.new(:flow_id, :queue)
43
+ BeforeAllJobsEvent = Struct.new(:jobnet_id, :queue)
44
+ class BeforeAllJobsEvent # reopen
45
+ def flow_id
46
+ jobnet_id
47
+ end
48
+ end
44
49
  BeforeJobEvent = Struct.new(:job)
45
50
  AfterJobEvent = Struct.new(:result)
46
51
  AfterAllJobsEvent = Struct.new(:succeeded, :queue)
@@ -3,80 +3,100 @@ require 'tsort'
3
3
 
4
4
  module Bricolage
5
5
 
6
- class JobFlow
7
- def JobFlow.load(path)
8
- File.open(path) {|f|
9
- parse_stream(f, make_node_ref(path))
10
- }
11
- rescue SystemCallError => err
12
- raise ParameterError, "could not load job flow: #{path} (#{err.message})"
6
+ # Represents "first" jobnet given by command line (e.g. bricolage-jobnet some.jobnet)
7
+ class RootJobNet
8
+ def RootJobNet.load(ctx, path)
9
+ root = new(JobNet::FileLoader.new(ctx), JobNet.load(path))
10
+ root.load_recursive
11
+ root.fix
12
+ root
13
13
  end
14
14
 
15
- def JobFlow.make_node_ref(path)
16
- subsys = path.parent.basename.to_s
17
- name = path.basename('.jobnet').to_s
18
- JobNetRef.new(subsys, name, Location.dummy)
15
+ def initialize(jobnet_loader, start_jobnet)
16
+ @jobnet_loader = jobnet_loader
17
+ @start_jobnet = start_jobnet
18
+ @jobnets = {start_jobnet.ref => start_jobnet}
19
+ @graph = nil
19
20
  end
20
21
 
21
- def JobFlow.parse_stream(f, ref)
22
- Parser.new(ref.subsystem).parse_stream(f, ref)
23
- end
22
+ attr_reader :start_jobnet
24
23
 
25
- def JobFlow.root
26
- RootJobFlow.new
24
+ def each_jobnet(&block)
25
+ @jobnets.each_value(&block)
27
26
  end
28
27
 
29
- ROOT_FLOW_NAME = '*'
30
-
31
- def initialize(ref, location)
32
- @ref = ref
33
- @location = location
34
- @flow = {} # Ref => [Ref] (src->dest)
35
- @deps = {} # Ref => [Ref] (dest->src)
36
- @subnets_resolved = false
28
+ def jobnets
29
+ @jobnets.values
37
30
  end
38
31
 
39
- def inspect
40
- "\#<#{self.class} #{ref}>"
32
+ def load_recursive
33
+ unresolved = [@start_jobnet]
34
+ until unresolved.empty?
35
+ loaded = []
36
+ unresolved.each do |net|
37
+ net.net_refs.each do |ref|
38
+ next if ref.jobnet
39
+ unless net = @jobnets[ref]
40
+ net = @jobnet_loader.load(ref)
41
+ @jobnets[ref] = net
42
+ loaded.push net
43
+ end
44
+ ref.jobnet = net
45
+ end
46
+ end
47
+ unresolved = loaded
48
+ end
41
49
  end
42
50
 
43
- attr_reader :ref
44
-
45
- def name
46
- ref.to_s
51
+ def fix
52
+ each_jobnet do |net|
53
+ net.fix
54
+ end
55
+ @jobnets.freeze
56
+ @dag = JobDAG.build(jobnets)
47
57
  end
48
58
 
49
- def add_edge(src, dest)
50
- (@flow[src] ||= []).push dest unless dest.net?
51
- (@deps[dest] ||= []).push src
59
+ def sequential_jobs
60
+ @dag.sequential_jobs
52
61
  end
62
+ end
53
63
 
54
- def flow_tree
55
- h = {}
56
- @flow.each do |src, dests|
57
- h[src.to_s] = dests.map {|d| d.to_s }
64
+ class JobDAG
65
+ def JobDAG.build(jobnets)
66
+ graph = new
67
+ jobnets.each do |net|
68
+ graph.merge! net
58
69
  end
59
- h
70
+ graph.fix
71
+ graph
60
72
  end
61
73
 
62
- def dependencies
74
+ def initialize
75
+ @deps = Hash.new { Array.new } # {JobRef => [JobRef]} (dest->srcs)
76
+ end
77
+
78
+ def to_hash
63
79
  h = {}
64
- @deps.each do |ref, deps|
65
- h[ref.to_s] = deps.map(&:to_s)
80
+ @deps.each do |dest, srcs|
81
+ h[dest.to_s] = srcs.map(&:to_s)
66
82
  end
67
83
  h
68
84
  end
69
85
 
70
- def dependent_flows
71
- @deps.values.flatten.select {|ref| ref.net? }.uniq
86
+ def merge!(net)
87
+ net.each_dependencies do |ref, deps|
88
+ @deps[ref] |= deps
89
+ end
72
90
  end
73
91
 
74
- def sequential_nodes
75
- tsort.reject {|ref| ref.dummy? }
92
+ def fix
93
+ @deps.freeze
94
+ check_cycle
95
+ check_orphan
76
96
  end
77
97
 
78
98
  def sequential_jobs
79
- sequential_nodes.reject {|ref| ref.net? }
99
+ tsort.reject {|ref| ref.dummy? }
80
100
  end
81
101
 
82
102
  include TSort
@@ -89,153 +109,185 @@ module Bricolage
89
109
  @deps.fetch(ref).each(&block)
90
110
  end
91
111
 
92
- def fix_graph
93
- close_graph
94
- check_cycle
95
- check_orphan
96
- end
97
-
98
- def resolve_subnets(root_flow)
99
- @deps.each_key do |ref|
100
- next unless ref.net?
101
- ref.flow = root_flow.subnet(ref)
102
- end
103
- @subnets_resolved = true
104
- end
105
-
106
- def subnets_resolved?
107
- @subnets_resolved
108
- end
109
-
110
112
  private
111
113
 
112
- def close_graph
113
- @deps.values.flatten.uniq.each do |ref|
114
- @deps[ref] ||= []
115
- end
116
- end
117
-
118
114
  def check_cycle
119
115
  each_strongly_connected_component do |refs|
120
116
  unless refs.size == 1
121
117
  cycle = (refs + [refs.first]).reverse.join(' -> ')
122
- raise ParameterError, "found cycle in the flow: #{cycle}"
118
+ raise ParameterError, "found cycle in the jobnet: #{cycle}"
123
119
  end
124
120
  end
125
121
  end
126
122
 
127
123
  def check_orphan
128
124
  orphan_nodes.each do |ref|
129
- raise ParameterError, "found orphan job in the flow: #{ref.location}: #{ref}"
125
+ raise ParameterError, "found orphan job in the jobnet: #{ref.location}: #{ref}"
130
126
  end
131
127
  end
132
128
 
133
129
  def orphan_nodes
134
- @deps.to_a.select {|ref, deps| deps.empty? and not ref.dummy? and not ref.net? }.map {|ref, *| ref }
130
+ @deps.to_a.select {|ref, deps| deps.empty? and not ref.dummy? }.map {|ref, *| ref }
135
131
  end
136
132
  end
137
133
 
138
- class RootJobFlow < JobFlow
139
- def RootJobFlow.load(ctx, path)
140
- flow = new(ctx)
141
- flow.add_subnet JobFlow.load(path)
142
- flow.fix
143
- flow
134
+ class JobNet
135
+ def JobNet.load(path, ref = JobNetRef.for_path(path))
136
+ File.open(path) {|f|
137
+ Parser.new(ref).parse_stream(f)
138
+ }
139
+ rescue SystemCallError => err
140
+ raise ParameterError, "could not load jobnet: #{path} (#{err.message})"
144
141
  end
145
142
 
146
- def initialize(ctx)
147
- @ctx = ctx
148
- super ROOT_FLOW_NAME, Location.dummy
149
- @subnets = {}
143
+ def initialize(ref, location)
144
+ @ref = ref
145
+ @location = location
146
+ @flow = {} # Ref => [Ref] (src->dest)
147
+ @deps = {} # Ref => [Ref] (dest->src)
150
148
  end
151
149
 
152
- def add_subnet(flow)
153
- raise ParameterError, "duplicated subnet definition: #{flow.name}" if @subnets.key?(flow.name)
154
- @subnets[flow.name] = flow
150
+ def inspect
151
+ "\#<#{self.class} #{ref}>"
155
152
  end
156
153
 
157
- def subnet(ref)
158
- if flow = @subnets[ref]
159
- flow
160
- else
161
- flow = load_jobnet_auto(ref)
162
- add_subnet flow
163
- flow
164
- end
154
+ attr_reader :ref
155
+
156
+ def start
157
+ @ref.start_ref
165
158
  end
166
159
 
167
- def load_jobnet_auto(ref)
168
- path = @ctx.root_relative_path(ref.relative_path)
169
- raise ParameterError, "undefined subnet: #{ref}" unless path.file?
170
- JobFlow.load(path)
160
+ def end
161
+ @ref.end_ref
171
162
  end
172
163
 
173
- def each_subnet(&block)
174
- @subnets.values.each(&block)
164
+ def name
165
+ @ref.to_s
175
166
  end
176
167
 
177
- def each_flow(&block)
178
- yield self
179
- @subnets.each_value(&block)
168
+ def add_edge(src, dest)
169
+ (@flow[src] ||= []).push dest
170
+ (@deps[dest] ||= []).push src
180
171
  end
181
172
 
182
- def each_subnet_sequence
183
- sequential_nodes.each do |ref|
184
- yield subnet(ref)
173
+ def to_hash
174
+ h = {}
175
+ @flow.each do |src, dests|
176
+ h[src.to_s] = dests.map(&:to_s)
185
177
  end
178
+ h
186
179
  end
187
180
 
188
- def fix
189
- resolve_subnet_references
190
- each_subnet do |flow|
191
- flow.fix_graph
181
+ def to_deps_hash
182
+ h = {}
183
+ @deps.each do |dest, srcs|
184
+ h[dest.to_s] = srcs.map(&:to_s)
192
185
  end
193
- add_jobnet_dependency_edges
194
- fix_graph
186
+ h
195
187
  end
196
188
 
197
- private
189
+ def refs
190
+ @flow.keys | @flow.values.flatten
191
+ end
198
192
 
199
- def resolve_subnet_references
200
- unresolved = true
201
- while unresolved
202
- unresolved = false
203
- ([self] + @subnets.values).each do |flow|
204
- unless flow.subnets_resolved?
205
- flow.resolve_subnets self
206
- unresolved = true
207
- end
208
- end
209
- end
193
+ def net_refs
194
+ @deps.keys.select {|ref| ref.net? }
210
195
  end
211
196
 
212
- def add_jobnet_dependency_edges
213
- each_subnet do |flow|
214
- # dummy dependency to ensure to execute all subnets
215
- add_edge Ref.dummy, flow.ref
216
- # jobnet -> jobnet dependency
217
- flow.dependent_flows.each do |dep_flow_ref|
218
- add_edge dep_flow_ref, flow.ref
197
+ # Adds dummy dependencies (@start and @end) to fix up all jobs
198
+ # into one DAG beginning with @start and ending with @end
199
+ def fix
200
+ refs.each do |ref|
201
+ next if ref.dummy?
202
+ unless @deps[ref]
203
+ (@flow[self.start] ||= []).push ref
204
+ @deps[ref] = [self.start]
205
+ end
206
+ unless @flow[ref]
207
+ (@flow[ref] ||= []).push self.end
208
+ (@deps[self.end] ||= []).push ref
219
209
  end
220
210
  end
211
+ @deps[self.start] ||= []
212
+ @flow.freeze
213
+ @deps.freeze
221
214
  end
222
215
 
223
- def check_orphan
224
- # should not check orphan for root.
216
+ def each_dependencies
217
+ @deps.each do |ref, deps|
218
+ dest = (ref.net? ? ref.start : ref)
219
+ srcs = deps.map {|r| r.net? ? r.end : r }
220
+ yield dest, srcs
221
+ end
225
222
  end
226
223
  end
227
224
 
228
- class JobFlow # reopen as namespace
225
+ class JobNet # reopen as namespace
229
226
 
230
- class Ref
231
- START_NAME = '@start'
227
+ class FileLoader
228
+ def initialize(ctx)
229
+ @context = ctx
230
+ end
231
+
232
+ def load(ref)
233
+ path = @context.root_relative_path(ref.relative_path)
234
+ raise ParameterError, "undefined subnet: #{ref}" unless path.file?
235
+ JobNet.load(path, ref)
236
+ end
237
+ end
238
+
239
+ class Parser
240
+ def initialize(jobnet_ref)
241
+ @jobnet_ref = jobnet_ref
242
+ end
243
+
244
+ def parse_stream(f)
245
+ net = JobNet.new(@jobnet_ref, Location.for_file(f))
246
+ foreach_edge(f) do |src, dest|
247
+ net.add_edge src, dest
248
+ end
249
+ net
250
+ end
251
+
252
+ private
253
+
254
+ name = /\w[\w\-]*/
255
+ node_ref = %r<[@*]?(?:#{name}/)?#{name}>
256
+ START_PATTERN = /\A(#{node_ref})\z/
257
+ DEPEND_PATTERN = /\A(#{node_ref})?\s*->\s*(#{node_ref})\z/
258
+
259
+ def foreach_edge(f)
260
+ default_src = nil
261
+ f.each do |line|
262
+ text = line.sub(/\#.*/, '').strip
263
+ next if text.empty?
264
+ loc = Location.for_file(f)
265
+
266
+ if m = DEPEND_PATTERN.match(text)
267
+ src = (m[1] ? ref(m[1], loc) : default_src) or
268
+ raise ParameterError, "syntax error at #{loc}: '->' must follow any job"
269
+ dest = ref(m[2], loc)
270
+ yield src, dest
271
+ default_src = dest
272
+
273
+ elsif m = START_PATTERN.match(text)
274
+ dest = ref(m[1], loc)
275
+ yield @jobnet_ref.start_ref, dest
276
+ default_src = dest
277
+
278
+ else
279
+ raise ParameterError, "syntax error at #{loc}: #{line.strip.inspect}"
280
+ end
281
+ end
282
+ end
232
283
 
233
- def Ref.dummy
234
- JobRef.new(nil, START_NAME, Location.dummy)
284
+ def ref(ref_str, location)
285
+ Ref.parse(ref_str, @jobnet_ref.subsystem, location)
235
286
  end
287
+ end
236
288
 
289
+ class Ref
237
290
  def Ref.parse(ref, curr_subsys = nil, location = Location.dummy)
238
- return JobNetRef.new(nil, '', location) if ref == ROOT_FLOW_NAME
239
291
  m = %r<\A(\*)?(?:(\w[\w\-]*)/)?(@?\w[\w\-]*)\z>.match(ref) or
240
292
  raise ParameterError, "bad job name: #{ref.inspect}"
241
293
  is_net, subsys, name = m.captures
@@ -287,14 +339,18 @@ module Bricolage
287
339
  end
288
340
 
289
341
  class JobNetRef < Ref
342
+ def JobNetRef.for_path(path)
343
+ new(path.parent.basename, path.basename('.jobnet'), Location.dummy)
344
+ end
345
+
290
346
  def initialize(subsys, name, location)
291
347
  super
292
- @flow = nil
348
+ @jobnet = nil
349
+ @start = nil
350
+ @end = nil
293
351
  end
294
352
 
295
- def flow=(flow)
296
- @flow = flow
297
- end
353
+ attr_accessor :jobnet
298
354
 
299
355
  def net?
300
356
  true
@@ -307,6 +363,23 @@ module Bricolage
307
363
  def relative_path
308
364
  "#{subsystem}/#{name}.jobnet"
309
365
  end
366
+
367
+ def start_ref
368
+ @start ||= JobRef.new(subsystem, "@#{name}@start", location)
369
+ end
370
+
371
+ def end_ref
372
+ @end ||= JobRef.new(subsystem, "@#{name}@end", location)
373
+ end
374
+
375
+
376
+ def start
377
+ @jobnet.start
378
+ end
379
+
380
+ def end
381
+ @jobnet.end
382
+ end
310
383
  end
311
384
 
312
385
  class Location
@@ -335,55 +408,6 @@ module Bricolage
335
408
  end
336
409
  end
337
410
 
338
- class Parser
339
- def initialize(subsys)
340
- @subsys = subsys
341
- end
342
-
343
- def parse_stream(f, ref)
344
- flow = JobFlow.new(ref, Location.for_file(f))
345
- foreach_edge(f) do |src, dest|
346
- flow.add_edge src, dest
347
- end
348
- flow
349
- end
350
-
351
- private
352
-
353
- name = /\w[\w\-]*/
354
- node_ref = %r<[@*]?(?:#{name}/)?#{name}>
355
- START_PATTERN = /\A(#{node_ref})\z/
356
- DEPEND_PATTERN = /\A(#{node_ref})?\s*->\s*(#{node_ref})\z/
357
-
358
- def foreach_edge(f)
359
- default_src = Ref.dummy
360
- f.each do |line|
361
- text = line.sub(/\#.*/, '').strip
362
- next if text.empty?
363
- loc = Location.for_file(f)
364
-
365
- if m = DEPEND_PATTERN.match(text)
366
- src = m[1] ? ref(m[1], loc) : default_src
367
- dest = ref(m[2], loc)
368
- yield src, dest
369
- default_src = dest
370
-
371
- elsif m = START_PATTERN.match(text)
372
- dest = ref(m[1], loc)
373
- yield Ref.dummy, dest
374
- default_src = dest
375
-
376
- else
377
- raise ParameterError, "syntax error at #{loc}: #{line.strip.inspect}"
378
- end
379
- end
380
- end
381
-
382
- def ref(ref_str, location)
383
- Ref.parse(ref_str, @subsys, location)
384
- end
385
- end
386
-
387
411
  end
388
412
 
389
413
  end
@@ -1,6 +1,6 @@
1
1
  require 'bricolage/application'
2
2
  require 'bricolage/context'
3
- require 'bricolage/jobflow'
3
+ require 'bricolage/jobnet'
4
4
  require 'bricolage/taskqueue'
5
5
  require 'bricolage/job'
6
6
  require 'bricolage/jobresult'
@@ -23,8 +23,8 @@ module Bricolage
23
23
  def initialize
24
24
  Signal.trap('PIPE', 'IGNORE')
25
25
  @hooks = ::Bricolage
26
- @flow_id = nil
27
- @flow_start_time = Time.now
26
+ @jobnet_id = nil
27
+ @jobnet_start_time = Time.now
28
28
  @log_path = nil
29
29
  end
30
30
 
@@ -37,15 +37,15 @@ module Bricolage
37
37
  @hooks.run_before_option_parsing_hooks(opts)
38
38
  opts.parse ARGV
39
39
  @ctx = Context.for_application(nil, opts.jobnet_file, environment: opts.environment, global_variables: opts.global_variables)
40
- @flow_id = "#{opts.jobnet_file.dirname.basename}/#{opts.jobnet_file.basename('.jobnet')}"
40
+ @jobnet_id = "#{opts.jobnet_file.dirname.basename}/#{opts.jobnet_file.basename('.jobnet')}"
41
41
  @log_path = opts.log_path
42
- flow = RootJobFlow.load(@ctx, opts.jobnet_file)
42
+ jobnet = RootJobNet.load(@ctx, opts.jobnet_file)
43
43
  queue = get_queue(opts)
44
44
  if queue.locked?
45
45
  raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help}"
46
46
  end
47
47
  unless queue.queued?
48
- enqueue_jobs flow, queue
48
+ enqueue_jobs jobnet, queue
49
49
  logger.info "jobs are queued." if opts.queue_exist?
50
50
  end
51
51
  if opts.list_jobs?
@@ -79,25 +79,18 @@ module Bricolage
79
79
  end
80
80
  end
81
81
 
82
- def enqueue_jobs(flow, queue)
83
- flow.each_subnet_sequence do |subnet|
84
- seq = 1
85
- subnet.sequential_jobs.each do |ref|
86
- queue.enq JobTask.new(subnet.ref, seq, ref)
87
- seq += 1
88
- end
82
+ def enqueue_jobs(jobnet, queue)
83
+ seq = 1
84
+ jobnet.sequential_jobs.each do |ref|
85
+ queue.enq JobTask.new(ref)
86
+ seq += 1
89
87
  end
90
88
  queue.save
91
89
  end
92
90
 
93
91
  def list_jobs(queue)
94
- prev = nil
95
92
  queue.each do |task|
96
- if not prev or prev.jobnet != task.jobnet
97
- puts "---- jobnet #{task.jobnet} ---"
98
- end
99
93
  puts task.job
100
- prev = task
101
94
  end
102
95
  end
103
96
 
@@ -108,18 +101,18 @@ module Bricolage
108
101
  end
109
102
 
110
103
  def run_queue(queue)
111
- @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@flow_id, queue))
104
+ @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@jobnet_id, queue))
112
105
  queue.consume_each do |task|
113
106
  result = execute_job(task.job, queue)
114
107
  unless result.success?
115
- logger.elapsed_time 'jobnet total: ', (Time.now - @flow_start_time)
108
+ logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
116
109
  logger.error "[job #{task.job}] #{result.message}"
117
110
  @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(false, queue))
118
111
  exit result.status
119
112
  end
120
113
  end
121
114
  @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(true, queue))
122
- logger.elapsed_time 'jobnet total: ', (Time.now - @flow_start_time)
115
+ logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
123
116
  logger.info "status all green"
124
117
  end
125
118
 
@@ -142,11 +135,11 @@ module Bricolage
142
135
  start_time = Time.now
143
136
  @log_path.gsub(/%\{\w+\}/) {|var|
144
137
  case var
145
- when '%{flow_start_date}' then @flow_start_time.strftime('%Y%m%d')
146
- when '%{flow_start_time}' then @flow_start_time.strftime('%Y%m%d_%H%M%S%L')
138
+ when '%{jobnet_start_date}' then @jobnet_start_time.strftime('%Y%m%d')
139
+ when '%{jobnet_start_time}' then @jobnet_start_time.strftime('%Y%m%d_%H%M%S%L')
147
140
  when '%{job_start_date}' then start_time.strftime('%Y%m%d')
148
141
  when '%{job_start_time}' then start_time.strftime('%Y%m%d_%H%M%S%L')
149
- when '%{flow}', '%{flow_id}' then @flow_id.gsub('/', '::')
142
+ when '%{jobnet}', '%{net}', '%{jobnet_id}', '%{net_id}', '%{flow}', '%{flow_id}' then @jobnet_id.gsub('/', '::')
150
143
  when '%{subsystem}' then job_ref.subsystem
151
144
  when '%{job}', '%{job_id}' then job_ref.name
152
145
  else
@@ -1,4 +1,4 @@
1
- require 'bricolage/jobflow'
1
+ require 'bricolage/jobnet'
2
2
  require 'bricolage/exception'
3
3
  require 'pathname'
4
4
 
@@ -133,23 +133,19 @@ module Bricolage
133
133
  end
134
134
 
135
135
  class JobTask
136
- def initialize(jobnet, seq, job)
137
- @jobnet = jobnet
138
- @seq = seq
136
+ def initialize(job)
139
137
  @job = job
140
138
  end
141
139
 
142
- attr_reader :jobnet
143
- attr_reader :seq
144
140
  attr_reader :job
145
141
 
146
142
  def serialize
147
- [@jobnet, @seq, @job].join("\t")
143
+ [@job].join("\t")
148
144
  end
149
145
 
150
146
  def JobTask.deserialize(str)
151
- jobnet, seq, job = str.strip.split("\t", 3)
152
- new(JobFlow::Ref.parse(jobnet), seq.to_i, JobFlow::Ref.parse(job))
147
+ job, * = str.strip.split("\t")
148
+ new(JobNet::Ref.parse(job))
153
149
  end
154
150
  end
155
151
 
@@ -1,4 +1,4 @@
1
1
  module Bricolage
2
2
  APPLICATION_NAME = 'Bricolage'
3
- VERSION = '5.10.0'
3
+ VERSION = '5.11.0'
4
4
  end
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: ../..
3
3
  specs:
4
- bricolage (5.9.6)
4
+ bricolage (5.11.0)
5
5
  aws-sdk (< 2)
6
6
  mysql2
7
7
  pg
@@ -23,7 +23,7 @@ GEM
23
23
  json (1.8.3)
24
24
  mini_portile (0.6.2)
25
25
  msgpack (0.5.11)
26
- mysql2 (0.4.0)
26
+ mysql2 (0.4.1)
27
27
  nokogiri (1.6.6.2)
28
28
  mini_portile (~> 0.6.0)
29
29
  parallel (0.6.5)
@@ -0,0 +1,61 @@
1
+ require 'bricolage/jobflow'
2
+ require 'pathname'
3
+ require 'stringio'
4
+ require 'forwardable'
5
+ require 'pp'
6
+
7
+ class ConstantLoader
8
+ def initialize(subnets)
9
+ @subnets = subnets
10
+ end
11
+
12
+ def load(ref)
13
+ @subnets[ref.to_s]
14
+ end
15
+ end
16
+
17
+ def root_flow(subnets)
18
+ h = {}
19
+ subnets.each do |flow|
20
+ h[flow.name] = flow
21
+ end
22
+ root = Bricolage::RootJobFlow.new(ConstantLoader.new(h))
23
+ subnets.each do |flow|
24
+ root.add_subnet flow
25
+ end
26
+ root.fix
27
+ root
28
+ end
29
+
30
+ class StringFile
31
+ def initialize(str, path)
32
+ @f = StringIO.new(str)
33
+ @path = path
34
+ end
35
+
36
+ attr_reader :path
37
+
38
+ extend Forwardable
39
+ def_delegators '@f', :each, :lineno
40
+ end
41
+
42
+ def make_flow(src, name)
43
+ path = "test/#{name}.jobnet"
44
+ ref = Bricolage::JobFlow.make_node_ref(Pathname(path))
45
+ Bricolage::JobFlow.parse_stream(StringFile.new(src, path), ref)
46
+ end
47
+
48
+ net1 = make_flow(<<-End, 'net1')
49
+ job1
50
+ -> *net2
51
+ -> job4
52
+ End
53
+
54
+ net2 = make_flow(<<-End, 'net2')
55
+ job3
56
+ -> job4
57
+ End
58
+
59
+ root_flow([net1, net2]).each_subnet do |flow|
60
+ pp flow.sequential_nodes
61
+ end
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1 @@
1
+ class: noop
@@ -0,0 +1,3 @@
1
+ job1
2
+ -> *net2
3
+ -> job4
@@ -0,0 +1,2 @@
1
+ job2
2
+ -> job3
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bricolage
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.10.0
4
+ version: 5.11.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Minero Aoki
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-27 00:00:00.000000000 Z
11
+ date: 2015-10-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pg
@@ -137,7 +137,7 @@ files:
137
137
  - lib/bricolage/job.rb
138
138
  - lib/bricolage/jobclass.rb
139
139
  - lib/bricolage/jobfile.rb
140
- - lib/bricolage/jobflow.rb
140
+ - lib/bricolage/jobnet.rb
141
141
  - lib/bricolage/jobnetrunner.rb
142
142
  - lib/bricolage/jobresult.rb
143
143
  - lib/bricolage/logger.rb
@@ -168,12 +168,22 @@ files:
168
168
  - test/home/data/20141002-1355_00.txt
169
169
  - test/home/data/20141002-1355_01.txt
170
170
  - test/home/data/20141002-1355_02.txt
171
+ - test/home/jobnet-test.rb
171
172
  - test/home/put.sh
172
173
  - test/home/revert.sh
173
174
  - test/home/subsys/d.ct
175
+ - test/home/subsys/job1.job
176
+ - test/home/subsys/job2.job
177
+ - test/home/subsys/job3.job
178
+ - test/home/subsys/job4.job
179
+ - test/home/subsys/job5.job
180
+ - test/home/subsys/job6.job
181
+ - test/home/subsys/job7.job
174
182
  - test/home/subsys/load_test.ct
175
183
  - test/home/subsys/load_test.job
176
184
  - test/home/subsys/migrate.job
185
+ - test/home/subsys/net1.jobnet
186
+ - test/home/subsys/net2.jobnet
177
187
  - test/home/subsys/raw-vacuum.sql.job
178
188
  - test/home/subsys/rebuild.sql.job
179
189
  - test/home/subsys/search_backends.ct