rukawa 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 999a3ff79a503ecea000dd2bed167c22a449f8cf
4
- data.tar.gz: 4fc79536e34bbe56be38e7761634ffaff46fa388
3
+ metadata.gz: cd835345eb1c1841c6dd7d6a907287eea0e359d6
4
+ data.tar.gz: 48eecf5784e84b8061edc309a657458567ea0743
5
5
  SHA512:
6
- metadata.gz: d5cf437492c3461a218b817834fbd0d50cdcb613b06a587b5770458acd18fc0945f1640cd19447eacae2357003a1e371a605f77f7646c6032596b4ed7fdcc10e
7
- data.tar.gz: b0c0649300511b87a1afe1d9ba04fbb2084169c718fdef5ef0f322333a7881204ff082717b871d495fd34f1f0da123c165d85213bd2f9dc29dc018dc169638ce
6
+ metadata.gz: 07caf395ab5d616dc6803fbb6da39465465aaa6bf295900792c4643721328b5f531abfe3b3e01a31dc8853aa4e11e7b19fdf254177ae8112ee2d8b0a66a723ba
7
+ data.tar.gz: 1b77d69131e8fc808294a7ef2e814b649c995a56a78e8422d6866127ad9bd229b9137ab4dea8a3bc645d4c05b885e5e7a2d0952d06731d3e9804d5cfa309c36b
data/.rspec CHANGED
@@ -1,2 +1,3 @@
1
1
  --format documentation
2
2
  --color
3
+ -r power_assert
data/README.md CHANGED
@@ -1,3 +1,4 @@
1
+ [![Stories in Ready](https://badge.waffle.io/joker1007/rukawa.png?label=ready&title=Ready)](https://waffle.io/joker1007/rukawa)
1
2
  # Rukawa
2
3
  [![Gem Version](https://badge.fury.io/rb/rukawa.svg)](https://badge.fury.io/rb/rukawa)
3
4
  [![Build Status](https://travis-ci.org/joker1007/rukawa.svg?branch=master)](https://travis-ci.org/joker1007/rukawa)
@@ -44,67 +45,71 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
44
45
  % cd rukawa/sample
45
46
 
46
47
  # load ./jobs/**/*.rb, ./job_net/**/*.rb automatically
47
- % bundle exec rukawa run SampleJobNet -r 1 -d result.dot
48
- +--------------+---------+
49
- | Job | Status |
50
- +--------------+---------+
51
- | Job1 | waiting |
52
- | Job2 | waiting |
53
- | Job3 | waiting |
54
- | Job4 | waiting |
55
- | InnerJobNet | waiting |
56
- | InnerJob3 | waiting |
57
- | InnerJob1 | waiting |
58
- | InnerJob2 | waiting |
59
- | Job8 | waiting |
60
- | Job5 | waiting |
61
- | Job6 | waiting |
62
- | Job7 | waiting |
63
- | InnerJobNet2 | waiting |
64
- | InnerJob4 | waiting |
65
- | InnerJob5 | waiting |
66
- | InnerJob6 | waiting |
67
- +--------------+---------+
68
- +--------------+----------+
69
- | Job | Status |
70
- +--------------+----------+
71
- | Job1 | finished |
72
- | Job2 | finished |
73
- | Job3 | finished |
74
- | Job4 | finished |
75
- | InnerJobNet | running |
76
- | InnerJob3 | running |
77
- | InnerJob1 | running |
78
- | InnerJob2 | waiting |
79
- | Job8 | waiting |
80
- | Job5 | error |
81
- | Job6 | error |
82
- | Job7 | error |
83
- | InnerJobNet2 | running |
84
- | InnerJob4 | running |
85
- | InnerJob5 | skipped |
86
- | InnerJob6 | waiting |
87
- +--------------+----------+
88
- +--------------+----------+
89
- | Job | Status |
90
- +--------------+----------+
91
- | Job1 | finished |
92
- | Job2 | finished |
93
- | Job3 | finished |
94
- | Job4 | finished |
95
- | InnerJobNet | error |
96
- | InnerJob3 | finished |
97
- | InnerJob1 | finished |
98
- | InnerJob2 | error |
99
- | Job8 | error |
100
- | Job5 | error |
101
- | Job6 | error |
102
- | Job7 | error |
103
- | InnerJobNet2 | finished |
104
- | InnerJob4 | finished |
105
- | InnerJob5 | skipped |
106
- | InnerJob6 | skipped |
107
- +--------------+----------+
48
+ % bundle exec rukawa run SampleJobNet -r 10 -c 10
49
+ +----------------+----------+
50
+ | Job | Status |
51
+ +----------------+----------+
52
+ | Job1 | finished |
53
+ | Job2 | waiting |
54
+ | Job3 | waiting |
55
+ | Job4 | waiting |
56
+ | InnerJobNet | waiting |
57
+ | InnerJob3 | waiting |
58
+ | InnerJob1 | waiting |
59
+ | InnerJob2 | waiting |
60
+ | Job8 | waiting |
61
+ | Job5 | waiting |
62
+ | Job6 | waiting |
63
+ | Job7 | waiting |
64
+ | InnerJobNet2 | waiting |
65
+ | InnerJob4 | waiting |
66
+ | InnerJob5 | waiting |
67
+ | InnerJob6 | waiting |
68
+ | InnerJobNet3 | waiting |
69
+ | InnerJob7 | waiting |
70
+ | InnerJob8 | waiting |
71
+ | InnerJob9 | waiting |
72
+ | InnerJob10 | waiting |
73
+ | InnerJobNet4 | waiting |
74
+ | InnerJob11 | waiting |
75
+ | InnerJob12 | waiting |
76
+ | InnerJob13 | waiting |
77
+ | NestedJobNet | waiting |
78
+ | NestedJob1 | waiting |
79
+ | NestedJob2 | waiting |
80
+ +----------------+----------+
81
+ +----------------+----------+
82
+ | Job | Status |
83
+ +----------------+----------+
84
+ | Job1 | finished |
85
+ | Job2 | finished |
86
+ | Job3 | finished |
87
+ | Job4 | finished |
88
+ | InnerJobNet | error |
89
+ | InnerJob3 | finished |
90
+ | InnerJob1 | finished |
91
+ | InnerJob2 | error |
92
+ | Job8 | aborted |
93
+ | Job5 | error |
94
+ | Job6 | aborted |
95
+ | Job7 | aborted |
96
+ | InnerJobNet2 | running |
97
+ | InnerJob4 | running |
98
+ | InnerJob5 | waiting |
99
+ | InnerJob6 | waiting |
100
+ | InnerJobNet3 | aborted |
101
+ | InnerJob7 | aborted |
102
+ | InnerJob8 | aborted |
103
+ | InnerJob9 | aborted |
104
+ | InnerJob10 | aborted |
105
+ | InnerJobNet4 | aborted |
106
+ | InnerJob11 | aborted |
107
+ | InnerJob12 | aborted |
108
+ | InnerJob13 | aborted |
109
+ | NestedJobNet | aborted |
110
+ | NestedJob1 | aborted |
111
+ | NestedJob2 | aborted |
112
+ +----------------+----------+
108
113
 
109
114
  # generate result graph image
110
115
  % dot -Tpng -o result.png result.dot
@@ -112,6 +117,108 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
112
117
 
113
118
  ![jobnet.png](https://raw.githubusercontent.com/joker1007/rukawa/master/sample/result.png)
114
119
 
120
+ `aborted` means that dependent job failure aborts following jobs.
121
+
122
+ ### Resume
123
+
124
+ Add `JOB_NAME` arguments to `run` command.
125
+
126
+ ```
127
+ % bundle exec rukawa run SampleJobNet Job8 InnerJob4 -r 5 -c 10
128
+ +----------------+----------+
129
+ | Job | Status |
130
+ +----------------+----------+
131
+ | Job1 | bypassed |
132
+ | Job2 | bypassed |
133
+ | Job3 | bypassed |
134
+ | Job4 | bypassed |
135
+ | InnerJobNet | bypassed |
136
+ | InnerJob3 | bypassed |
137
+ | InnerJob1 | bypassed |
138
+ | InnerJob2 | bypassed |
139
+ | Job8 | waiting |
140
+ | Job5 | bypassed |
141
+ | Job6 | bypassed |
142
+ | Job7 | bypassed |
143
+ | InnerJobNet2 | waiting |
144
+ | InnerJob4 | waiting |
145
+ | InnerJob5 | waiting |
146
+ | InnerJob6 | waiting |
147
+ | InnerJobNet3 | waiting |
148
+ | InnerJob7 | waiting |
149
+ | InnerJob8 | waiting |
150
+ | InnerJob9 | waiting |
151
+ | InnerJob10 | waiting |
152
+ | InnerJobNet4 | waiting |
153
+ | InnerJob11 | waiting |
154
+ | InnerJob12 | waiting |
155
+ | InnerJob13 | waiting |
156
+ | NestedJobNet | waiting |
157
+ | NestedJob1 | waiting |
158
+ | NestedJob2 | waiting |
159
+ +----------------+----------+
160
+ +----------------+----------+
161
+ | Job | Status |
162
+ +----------------+----------+
163
+ | Job1 | bypassed |
164
+ | Job2 | bypassed |
165
+ | Job3 | bypassed |
166
+ | Job4 | bypassed |
167
+ | InnerJobNet | bypassed |
168
+ | InnerJob3 | bypassed |
169
+ | InnerJob1 | bypassed |
170
+ | InnerJob2 | bypassed |
171
+ | Job8 | finished |
172
+ | Job5 | bypassed |
173
+ | Job6 | bypassed |
174
+ | Job7 | bypassed |
175
+ | InnerJobNet2 | skipped |
176
+ | InnerJob4 | finished |
177
+ | InnerJob5 | skipped |
178
+ | InnerJob6 | skipped |
179
+ | InnerJobNet3 | running |
180
+ | InnerJob7 | finished |
181
+ | InnerJob8 | running |
182
+ | InnerJob9 | waiting |
183
+ | InnerJob10 | waiting |
184
+ | InnerJobNet4 | waiting |
185
+ | InnerJob11 | waiting |
186
+ | InnerJob12 | waiting |
187
+ | InnerJob13 | waiting |
188
+ | NestedJobNet | waiting |
189
+ | NestedJob1 | waiting |
190
+ | NestedJob2 | waiting |
191
+ +----------------+----------+
192
+ ```
193
+
194
+ In above case, All jobs except `Job8` and `InnerJob4` and depending them are set `bypassed`.
195
+ `bypassed` means that job is not executed, and regarded as successful.
196
+ For example, a job depends two other jobs. Even if either job is `bypassed`, and another job is `finished`, it is executed.
197
+
198
+
199
+ ### Run Specific jobs
200
+
201
+ `run_job` command executes specified job forcely.
202
+
203
+ ```
204
+ % be rukawa run_job Job8 InnerJob6 NestedJob1 -c 3 -r 5
205
+ +------------+---------+
206
+ | Job | Status |
207
+ +------------+---------+
208
+ | Job8 | waiting |
209
+ | InnerJob6 | waiting |
210
+ | NestedJob1 | running |
211
+ +------------+---------+
212
+ +------------+----------+
213
+ | Job | Status |
214
+ +------------+----------+
215
+ | Job8 | finished |
216
+ | InnerJob6 | finished |
217
+ | NestedJob1 | finished |
218
+ +------------+----------+
219
+ ```
220
+
221
+ Main usage is manual reentering.
115
222
 
116
223
  ### Output jobnet graph (dot file)
117
224
 
@@ -124,18 +231,53 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
124
231
  ```
125
232
  % bundle exec rukawa help run
126
233
  Usage:
127
- rukawa run JOB_NET_NAME
234
+ rukawa run JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...
235
+
236
+ Options:
237
+ -c, [--concurrency=N] # Default: cpu count
238
+ [--variables=key:value]
239
+ [--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
240
+ [--job-dirs=one two three] # Load job directories
241
+ -b, [--batch], [--no-batch] # If batch mode, not display running status
242
+ -l, [--log=LOG]
243
+ # Default: ./rukawa.log
244
+ [--stdout], [--no-stdout] # Output log to stdout
245
+ -d, [--dot=DOT] # Output job status by dot format
246
+ -r, [--refresh-interval=N] # Refresh interval for running status information
247
+ # Default: 3
248
+
249
+ Run jobnet. If JOB_NET is set, resume from JOB_NAME
250
+
251
+
252
+ % bundle exec rukawa help run_job
253
+ Usage:
254
+ rukawa run_job JOB_NAME [JOB_NAME] ...
128
255
 
129
256
  Options:
130
257
  -c, [--concurrency=N] # Default: cpu count
131
258
  [--variables=key:value]
259
+ [--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
132
260
  [--job-dirs=one two three] # Load job directories
133
261
  -b, [--batch], [--no-batch] # If batch mode, not display running status
134
262
  -l, [--log=LOG]
135
263
  # Default: ./rukawa.log
264
+ [--stdout], [--no-stdout] # Output log to stdout
136
265
  -d, [--dot=DOT] # Output job status by dot format
137
266
  -r, [--refresh-interval=N] # Refresh interval for running status information
138
267
  # Default: 3
268
+
269
+ Run specific jobs.
270
+
271
+ % bundle exec rukawa help graph
272
+ Usage:
273
+ rukawa graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ... -o, --output=OUTPUT
274
+
275
+ Options:
276
+ [--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
277
+ [--job-dirs=one two three]
278
+ -o, --output=OUTPUT
279
+
280
+ Output jobnet graph. If JOB_NET is set, simulate resumed job sequence
139
281
  ```
140
282
 
141
283
  ## ToDo
@@ -3,6 +3,8 @@ require 'rukawa/state'
3
3
 
4
4
  module Rukawa
5
5
  class AbstractJob
6
+ attr_reader :parent_job_net
7
+
6
8
  class << self
7
9
  def skip_rules
8
10
  @skip_rules ||= []
@@ -17,8 +19,13 @@ module Rukawa
17
19
  self.class.to_s
18
20
  end
19
21
 
22
+ def inspect
23
+ to_s
24
+ end
25
+
20
26
  def skip?
21
- skip_rules.inject(false) do |cond, rule|
27
+ parent_skip = @parent_job_net ? @parent_job_net.skip? : false
28
+ parent_skip || skip_rules.inject(false) do |cond, rule|
22
29
  cond || rule.is_a?(Symbol) ? method(rule).call : rule.call(self)
23
30
  end
24
31
  end
@@ -26,5 +33,25 @@ module Rukawa
26
33
  def skip_rules
27
34
  self.class.skip_rules
28
35
  end
36
+
37
+ def elapsed_time_from(time = Time.now)
38
+ return finished_at - started_at if started_at && finished_at
39
+ return time - started_at if started_at
40
+
41
+ nil
42
+ end
43
+
44
+ def formatted_elapsed_time_from(time = Time.now)
45
+ sec = elapsed_time_from(time)
46
+ return "N/A" unless sec
47
+
48
+ hour = sec.to_i / 3600
49
+ min = sec.to_i / 60
50
+
51
+ hour_format = min > 0 ? "%dh " % hour : ""
52
+ min_format = min > 0 ? "%dm " % min : ""
53
+ sec_format = "#{sec.to_i}s"
54
+ "#{hour_format}#{min_format}#{sec_format}"
55
+ end
29
56
  end
30
57
  end
data/lib/rukawa/cli.rb CHANGED
@@ -34,19 +34,50 @@ module Rukawa
34
34
  exit 1 unless result
35
35
  end
36
36
 
37
- desc "graph JOB_NET_NAME", "Output jobnet graph"
37
+ desc "graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...", "Output jobnet graph. If JOB_NET is set, simulate resumed job sequence"
38
38
  method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
39
39
  method_option :job_dirs, type: :array, default: []
40
40
  method_option :output, aliases: "-o", type: :string, required: true
41
- def graph(job_net_name)
41
+ def graph(job_net_name, *job_name)
42
42
  load_config
43
43
  load_job_definitions
44
44
 
45
45
  job_net_class = Object.const_get(job_net_name)
46
- job_net = job_net_class.new(nil, options[:variables])
46
+ job_classes = job_name.map { |name| Object.const_get(name) }
47
+ job_net = job_net_class.new(nil, *job_classes)
47
48
  job_net.output_dot(options[:output])
48
49
  end
49
50
 
51
+ desc "run_job JOB_NAME [JOB_NAME] ...", "Run specific jobs."
52
+ method_option :concurrency, aliases: "-c", type: :numeric, default: nil, desc: "Default: cpu count"
53
+ method_option :variables, type: :hash, default: {}
54
+ method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
55
+ method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
56
+ method_option :batch, aliases: "-b", type: :boolean, default: false, desc: "If batch mode, not display running status"
57
+ method_option :log, aliases: "-l", type: :string, default: "./rukawa.log"
58
+ method_option :stdout, type: :boolean, default: false, desc: "Output log to stdout"
59
+ method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
60
+ method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
61
+ def run_job(*job_name)
62
+ load_config
63
+ Rukawa.configure do |c|
64
+ c.log_file = options[:stdout] ? $stdout : options[:log]
65
+ c.concurrency = options[:concurrency] if options[:concurrency]
66
+ end
67
+ load_job_definitions
68
+
69
+ job_classes = job_name.map { |name| Object.const_get(name) }
70
+ job_net_class = anonymous_job_net_class(*job_classes)
71
+ job_net = job_net_class.new(nil)
72
+ result = Runner.run(job_net, options[:batch], options[:refresh_interval])
73
+
74
+ if options[:dot]
75
+ job_net.output_dot(options[:dot])
76
+ end
77
+
78
+ exit 1 unless result
79
+ end
80
+
50
81
  private
51
82
 
52
83
  def load_config
@@ -71,5 +102,15 @@ module Rukawa
71
102
  Dir.glob(File.join(dir, "**/*.rb")) { |f| load f }
72
103
  end
73
104
  end
105
+
106
+ def anonymous_job_net_class(*job_classes)
107
+ Class.new(JobNet) do
108
+ self.singleton_class.send(:define_method, :dependencies) do
109
+ job_classes.map { |klass| [klass, []] }.to_h
110
+ end
111
+
112
+ define_method(:name) { "AnonymousJobNet" }
113
+ end
114
+ end
74
115
  end
75
116
  end
data/lib/rukawa/dag.rb CHANGED
@@ -1,8 +1,10 @@
1
1
  require 'set'
2
+ require 'tsort'
2
3
 
3
4
  module Rukawa
4
5
  class Dag
5
6
  include Enumerable
7
+ include TSort
6
8
 
7
9
  attr_reader :nodes, :jobs, :edges
8
10
 
@@ -47,6 +49,24 @@ module Rukawa
47
49
  end
48
50
  end
49
51
 
52
+ def tsort_each_node(&block)
53
+ @jobs.each(&block)
54
+ end
55
+
56
+ def tsort_each_child(node)
57
+ if block_given?
58
+ node.out_goings.each do |edge|
59
+ yield edge.to
60
+ end
61
+ else
62
+ Enumerator.new do |y|
63
+ node.out_goings.each do |edge|
64
+ y << edge.to
65
+ end
66
+ end
67
+ end
68
+ end
69
+
50
70
  private
51
71
 
52
72
  def tsortable_hash(hash)
data/lib/rukawa/job.rb CHANGED
@@ -4,7 +4,7 @@ require 'rukawa/abstract_job'
4
4
  module Rukawa
5
5
  class Job < AbstractJob
6
6
  attr_accessor :in_comings, :out_goings
7
- attr_reader :parent_job_net, :state
7
+ attr_reader :state, :started_at, :finished_at
8
8
 
9
9
  def initialize(parent_job_net)
10
10
  @parent_job_net = parent_job_net
@@ -13,6 +13,10 @@ module Rukawa
13
13
  set_state(:waiting)
14
14
  end
15
15
 
16
+ def set_state(name)
17
+ @state = Rukawa::State.get(name)
18
+ end
19
+
16
20
  def root?
17
21
  in_comings.select { |edge| edge.cluster == @parent_job_net }.empty?
18
22
  end
@@ -23,27 +27,10 @@ module Rukawa
23
27
 
24
28
  def dataflow
25
29
  return @dataflow if @dataflow
30
+ return @dataflow = bypass_dataflow if @state.bypassed?
26
31
 
27
32
  @dataflow = Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
28
- begin
29
- raise DependentJobFailure unless results.all? { |r| !r.nil? }
30
-
31
- if skip? || @parent_job_net.skip? || results.any? { |r| r == Rukawa::State.get(:skipped) }
32
- Rukawa.logger.info("Skip #{self.class}")
33
- set_state(:skipped)
34
- else
35
- Rukawa.logger.info("Start #{self.class}")
36
- set_state(:running)
37
- run
38
- Rukawa.logger.info("Finish #{self.class}")
39
- set_state(:finished)
40
- end
41
- rescue => e
42
- Rukawa.logger.error("Error #{self.class} by #{e}")
43
- set_state(:error)
44
- raise
45
- end
46
-
33
+ do_run(*results)
47
34
  @state
48
35
  end
49
36
  end
@@ -51,6 +38,27 @@ module Rukawa
51
38
  def run
52
39
  end
53
40
 
41
+ private def do_run(*results)
42
+ @started_at = Time.now
43
+ check_dependencies(results)
44
+
45
+ if skip? || results.any?(&:skipped?)
46
+ Rukawa.logger.info("Skip #{self.class}")
47
+ set_state(:skipped)
48
+ else
49
+ Rukawa.logger.info("Start #{self.class}")
50
+ set_state(:running)
51
+ run
52
+ Rukawa.logger.info("Finish #{self.class}")
53
+ set_state(:finished)
54
+ end
55
+ rescue => e
56
+ handle_error(e)
57
+ raise
58
+ ensure
59
+ @finished_at = Time.now
60
+ end
61
+
54
62
  def jobs_as_from
55
63
  [self]
56
64
  end
@@ -70,8 +78,23 @@ module Rukawa
70
78
  in_comings.map { |edge| edge.from.dataflow }
71
79
  end
72
80
 
73
- def set_state(name)
74
- @state = Rukawa::State.get(name)
81
+ def bypass_dataflow
82
+ Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
83
+ Rukawa.logger.info("Skip #{self.class}")
84
+ @state
85
+ end
86
+ end
87
+
88
+ def check_dependencies(results)
89
+ unless results.all? { |r| !r.nil? }
90
+ set_state(:aborted)
91
+ raise DependentJobFailure
92
+ end
93
+ end
94
+
95
+ def handle_error(e)
96
+ Rukawa.logger.error("Error #{self.class} by #{e}")
97
+ set_state(:error) unless e.is_a?(DependentJobFailure)
75
98
  end
76
99
 
77
100
  def store(key, value)
@@ -3,7 +3,7 @@ require 'rukawa/abstract_job'
3
3
  module Rukawa
4
4
  class JobNet < AbstractJob
5
5
  include Enumerable
6
- attr_reader :parent_job_net, :dag
6
+ attr_reader :dag
7
7
 
8
8
  class << self
9
9
  def dependencies
@@ -15,6 +15,29 @@ module Rukawa
15
15
  @parent_job_net = parent_job_net
16
16
  @dag = Dag.new
17
17
  @dag.build(self, self.class.dependencies)
18
+ @resume_job_classes = resume_job_classes
19
+
20
+ unless resume_job_classes.empty?
21
+ resume_targets = []
22
+ @dag.tsort_each_node do |node|
23
+ node.set_state(:bypassed)
24
+ resume_targets << node if resume_job_classes.include?(node.class)
25
+ end
26
+
27
+ resume_targets.each do |node|
28
+ @dag.each_strongly_connected_component_from(node) do |nodes|
29
+ nodes.each { |connected| connected.set_state(:waiting) }
30
+ end
31
+ end
32
+ end
33
+ end
34
+
35
+ def started_at
36
+ @dag.nodes.min_by { |j| j.started_at ? j.started_at.to_i : Float::INFINITY }.started_at
37
+ end
38
+
39
+ def finished_at
40
+ @dag.nodes.max_by { |j| j.finished_at.to_i }.finished_at
18
41
  end
19
42
 
20
43
  def toplevel?
@@ -26,13 +49,7 @@ module Rukawa
26
49
  end
27
50
 
28
51
  def dataflows
29
- flat_map do |j|
30
- if j.respond_to?(:dataflows)
31
- j.dataflows
32
- else
33
- [j.dataflow]
34
- end
35
- end
52
+ @dag.tsort.reverse.map(&:dataflow)
36
53
  end
37
54
 
38
55
  def state
@@ -48,7 +65,7 @@ module Rukawa
48
65
  def to_dot(subgraph = false)
49
66
  graphdef = subgraph ? "subgraph" : "digraph"
50
67
  buf = %Q|#{graphdef} "#{subgraph ? "cluster_" : ""}#{name}" {\n|
51
- buf += %Q{label = "#{name}";\n}
68
+ buf += %Q{label = "#{graph_label}";\n}
52
69
  buf += Rukawa.config.graph.attrs
53
70
  buf += Rukawa.config.graph.node.attrs
54
71
  buf += "color = blue;\n" if subgraph
@@ -77,5 +94,15 @@ module Rukawa
77
94
  def each(&block)
78
95
  @dag.each(&block)
79
96
  end
97
+
98
+ private
99
+
100
+ def graph_label
101
+ if @resume_job_classes.empty?
102
+ name
103
+ else
104
+ "#{name} resume from (#{@resume_job_classes.join(", ")})"
105
+ end
106
+ end
80
107
  end
81
108
  end
data/lib/rukawa/runner.rb CHANGED
@@ -24,6 +24,7 @@ module Rukawa
24
24
  Rukawa.logger.info("=== Finish Rukawa ===")
25
25
 
26
26
  display_table unless batch_mode
27
+ puts "Finished #{@root_job_net.name} in #{@root_job_net.formatted_elapsed_time_from}"
27
28
 
28
29
  errors = futures.map(&:reason).compact
29
30
 
@@ -41,7 +42,7 @@ module Rukawa
41
42
  private
42
43
 
43
44
  def display_table
44
- table = Terminal::Table.new headings: ["Job", "Status"] do |t|
45
+ table = Terminal::Table.new headings: ["Job", "Status", "Elapsed Time"] do |t|
45
46
  @root_job_net.each_with_index do |j|
46
47
  table_row(t, j)
47
48
  end
@@ -51,12 +52,12 @@ module Rukawa
51
52
 
52
53
  def table_row(table, job, level = 0)
53
54
  if job.is_a?(JobNet)
54
- table << [Paint["#{" " * level}#{job.class}", :bold, :underline], job.state.colored]
55
+ table << [Paint["#{" " * level}#{job.class}", :bold, :underline], Paint[job.state.colored, :bold, :underline], Paint[job.formatted_elapsed_time_from, :bold, :underline]]
55
56
  job.each do |inner_j|
56
57
  table_row(table, inner_j, level + 1)
57
58
  end
58
59
  else
59
- table << [Paint["#{" " * level}#{job.class}", :bold], job.state.colored]
60
+ table << [Paint["#{" " * level}#{job.class}", :bold], job.state.colored, job.formatted_elapsed_time_from]
60
61
  end
61
62
  end
62
63
  end
data/lib/rukawa/state.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  module Rukawa::State
2
2
  def self.get(name)
3
- const_get(name.to_s.capitalize)
3
+ const_get(name.to_s.split("_").map(&:capitalize).join)
4
4
  end
5
5
 
6
6
  module BaseExt
@@ -15,6 +15,12 @@ module Rukawa::State
15
15
  def merge(other)
16
16
  other
17
17
  end
18
+
19
+ %i(running? skipped? bypassed? error? aborted? waiting? finished?).each do |sym|
20
+ define_method(sym) do
21
+ false
22
+ end
23
+ end
18
24
  end
19
25
 
20
26
  module Running
@@ -27,6 +33,10 @@ module Rukawa::State
27
33
  def self.merge(_other)
28
34
  self
29
35
  end
36
+
37
+ def self.running?
38
+ true
39
+ end
30
40
  end
31
41
 
32
42
  module Skipped
@@ -43,6 +53,22 @@ module Rukawa::State
43
53
  other
44
54
  end
45
55
  end
56
+
57
+ def self.skipped?
58
+ true
59
+ end
60
+ end
61
+
62
+ module Bypassed
63
+ extend BaseExt
64
+
65
+ def self.color
66
+ :yellow
67
+ end
68
+
69
+ def self.bypassed?
70
+ true
71
+ end
46
72
  end
47
73
 
48
74
  module Error
@@ -59,6 +85,38 @@ module Rukawa::State
59
85
  self
60
86
  end
61
87
  end
88
+
89
+ def self.error?
90
+ true
91
+ end
92
+ end
93
+
94
+ module Aborted
95
+ extend BaseExt
96
+
97
+ def name
98
+ "aborted"
99
+ end
100
+
101
+ def self.color
102
+ :magenta
103
+ end
104
+
105
+ def self.merge(other)
106
+ if other == Running || other == Error
107
+ other
108
+ else
109
+ self
110
+ end
111
+ end
112
+
113
+ def self.error?
114
+ true
115
+ end
116
+
117
+ def self.aborted?
118
+ true
119
+ end
62
120
  end
63
121
 
64
122
  module Waiting
@@ -67,6 +125,10 @@ module Rukawa::State
67
125
  def self.color
68
126
  :default
69
127
  end
128
+
129
+ def self.waiting?
130
+ true
131
+ end
70
132
  end
71
133
 
72
134
  module Finished
@@ -75,5 +137,9 @@ module Rukawa::State
75
137
  def self.color
76
138
  :green
77
139
  end
140
+
141
+ def self.finished?
142
+ true
143
+ end
78
144
  end
79
145
  end
@@ -1,3 +1,3 @@
1
1
  module Rukawa
2
- VERSION = "0.2.1"
2
+ VERSION = "0.3.0"
3
3
  end
data/rukawa.gemspec CHANGED
@@ -26,4 +26,5 @@ Gem::Specification.new do |spec|
26
26
  spec.add_development_dependency "bundler", "~> 1.11"
27
27
  spec.add_development_dependency "rake", "~> 10.0"
28
28
  spec.add_development_dependency "rspec", "~> 3.0"
29
+ spec.add_development_dependency "rspec-power_assert"
29
30
  end
data/sample/result.dot ADDED
@@ -0,0 +1,86 @@
1
+ digraph "SampleJobNet" {
2
+ label = "SampleJobNet";
3
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
4
+ Job1 [style = filled,fillcolor = green];
5
+ Job2 [style = filled,fillcolor = green];
6
+ Job3 [style = filled,fillcolor = green];
7
+ Job4 [style = filled,fillcolor = green];
8
+ subgraph "cluster_InnerJobNet" {
9
+ label = "InnerJobNet";
10
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
11
+ color = blue;
12
+ InnerJob3 [style = filled,fillcolor = green];
13
+ InnerJob1 [style = filled,fillcolor = green];
14
+ InnerJob2 [style = filled,fillcolor = red];
15
+ "InnerJob1" -> "InnerJob2";
16
+ "InnerJob3" -> "InnerJob2";
17
+ }
18
+ Job8 [style = filled,fillcolor = magenta];
19
+ Job5 [style = filled,fillcolor = red];
20
+ Job6 [style = filled,fillcolor = magenta];
21
+ Job7 [style = filled,fillcolor = magenta];
22
+ subgraph "cluster_InnerJobNet2" {
23
+ label = "InnerJobNet2";
24
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
25
+ color = blue;
26
+ InnerJob4 [style = filled,fillcolor = green];
27
+ InnerJob5 [style = filled,fillcolor = yellow];
28
+ InnerJob6 [style = filled,fillcolor = yellow];
29
+ "InnerJob4" -> "InnerJob5";
30
+ "InnerJob4" -> "InnerJob6";
31
+ "InnerJob5" -> "InnerJob6";
32
+ }
33
+ subgraph "cluster_InnerJobNet3" {
34
+ label = "InnerJobNet3";
35
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
36
+ color = blue;
37
+ InnerJob7 [style = filled,fillcolor = magenta];
38
+ InnerJob8 [style = filled,fillcolor = magenta];
39
+ InnerJob9 [style = filled,fillcolor = magenta];
40
+ InnerJob10 [style = filled,fillcolor = magenta];
41
+ "InnerJob7" -> "InnerJob9";
42
+ "InnerJob8" -> "InnerJob9";
43
+ "InnerJob7" -> "InnerJob10";
44
+ "InnerJob8" -> "InnerJob10";
45
+ }
46
+ subgraph "cluster_InnerJobNet4" {
47
+ label = "InnerJobNet4";
48
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
49
+ color = blue;
50
+ InnerJob11 [style = filled,fillcolor = magenta];
51
+ InnerJob12 [style = filled,fillcolor = magenta];
52
+ InnerJob13 [style = filled,fillcolor = magenta];
53
+ subgraph "cluster_NestedJobNet" {
54
+ label = "NestedJobNet";
55
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
56
+ color = blue;
57
+ NestedJob1 [style = filled,fillcolor = magenta];
58
+ NestedJob2 [style = filled,fillcolor = magenta];
59
+ "NestedJob1" -> "NestedJob2";
60
+ }
61
+ "InnerJob11" -> "NestedJob1";
62
+ "InnerJob12" -> "NestedJob1";
63
+ }
64
+ "Job1" -> "Job2";
65
+ "Job1" -> "Job3";
66
+ "Job2" -> "Job4";
67
+ "Job3" -> "Job4";
68
+ "Job3" -> "InnerJob3";
69
+ "Job3" -> "InnerJob1";
70
+ "InnerJob2" -> "Job8";
71
+ "Job3" -> "Job5";
72
+ "Job4" -> "Job6";
73
+ "Job5" -> "Job6";
74
+ "Job6" -> "Job7";
75
+ "Job4" -> "InnerJob4";
76
+ "Job8" -> "InnerJob7";
77
+ "Job8" -> "InnerJob8";
78
+ "Job7" -> "InnerJob7";
79
+ "Job7" -> "InnerJob8";
80
+ "InnerJob9" -> "InnerJob11";
81
+ "InnerJob9" -> "InnerJob12";
82
+ "InnerJob9" -> "InnerJob13";
83
+ "InnerJob10" -> "InnerJob11";
84
+ "InnerJob10" -> "InnerJob12";
85
+ "InnerJob10" -> "InnerJob13";
86
+ }
data/sample/result.png CHANGED
Binary file
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rukawa
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - joker1007
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-03-22 00:00:00.000000000 Z
11
+ date: 2016-03-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
@@ -108,6 +108,20 @@ dependencies:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
110
  version: '3.0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rspec-power_assert
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
111
125
  description: Hyper simple job workflow engine
112
126
  email:
113
127
  - kakyoin.hierophant@gmail.com
@@ -140,6 +154,7 @@ files:
140
154
  - sample/job_nets/sample_job_net.rb
141
155
  - sample/jobnet.png
142
156
  - sample/jobs/sample_job.rb
157
+ - sample/result.dot
143
158
  - sample/result.png
144
159
  - sample/rukawa.rb
145
160
  homepage: https://github.com/joker1007/rukawa