rukawa 0.2.1 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 999a3ff79a503ecea000dd2bed167c22a449f8cf
4
- data.tar.gz: 4fc79536e34bbe56be38e7761634ffaff46fa388
3
+ metadata.gz: cd835345eb1c1841c6dd7d6a907287eea0e359d6
4
+ data.tar.gz: 48eecf5784e84b8061edc309a657458567ea0743
5
5
  SHA512:
6
- metadata.gz: d5cf437492c3461a218b817834fbd0d50cdcb613b06a587b5770458acd18fc0945f1640cd19447eacae2357003a1e371a605f77f7646c6032596b4ed7fdcc10e
7
- data.tar.gz: b0c0649300511b87a1afe1d9ba04fbb2084169c718fdef5ef0f322333a7881204ff082717b871d495fd34f1f0da123c165d85213bd2f9dc29dc018dc169638ce
6
+ metadata.gz: 07caf395ab5d616dc6803fbb6da39465465aaa6bf295900792c4643721328b5f531abfe3b3e01a31dc8853aa4e11e7b19fdf254177ae8112ee2d8b0a66a723ba
7
+ data.tar.gz: 1b77d69131e8fc808294a7ef2e814b649c995a56a78e8422d6866127ad9bd229b9137ab4dea8a3bc645d4c05b885e5e7a2d0952d06731d3e9804d5cfa309c36b
data/.rspec CHANGED
@@ -1,2 +1,3 @@
1
1
  --format documentation
2
2
  --color
3
+ -r power_assert
data/README.md CHANGED
@@ -1,3 +1,4 @@
1
+ [![Stories in Ready](https://badge.waffle.io/joker1007/rukawa.png?label=ready&title=Ready)](https://waffle.io/joker1007/rukawa)
1
2
  # Rukawa
2
3
  [![Gem Version](https://badge.fury.io/rb/rukawa.svg)](https://badge.fury.io/rb/rukawa)
3
4
  [![Build Status](https://travis-ci.org/joker1007/rukawa.svg?branch=master)](https://travis-ci.org/joker1007/rukawa)
@@ -44,67 +45,71 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
44
45
  % cd rukawa/sample
45
46
 
46
47
  # load ./jobs/**/*.rb, ./job_net/**/*.rb automatically
47
- % bundle exec rukawa run SampleJobNet -r 1 -d result.dot
48
- +--------------+---------+
49
- | Job | Status |
50
- +--------------+---------+
51
- | Job1 | waiting |
52
- | Job2 | waiting |
53
- | Job3 | waiting |
54
- | Job4 | waiting |
55
- | InnerJobNet | waiting |
56
- | InnerJob3 | waiting |
57
- | InnerJob1 | waiting |
58
- | InnerJob2 | waiting |
59
- | Job8 | waiting |
60
- | Job5 | waiting |
61
- | Job6 | waiting |
62
- | Job7 | waiting |
63
- | InnerJobNet2 | waiting |
64
- | InnerJob4 | waiting |
65
- | InnerJob5 | waiting |
66
- | InnerJob6 | waiting |
67
- +--------------+---------+
68
- +--------------+----------+
69
- | Job | Status |
70
- +--------------+----------+
71
- | Job1 | finished |
72
- | Job2 | finished |
73
- | Job3 | finished |
74
- | Job4 | finished |
75
- | InnerJobNet | running |
76
- | InnerJob3 | running |
77
- | InnerJob1 | running |
78
- | InnerJob2 | waiting |
79
- | Job8 | waiting |
80
- | Job5 | error |
81
- | Job6 | error |
82
- | Job7 | error |
83
- | InnerJobNet2 | running |
84
- | InnerJob4 | running |
85
- | InnerJob5 | skipped |
86
- | InnerJob6 | waiting |
87
- +--------------+----------+
88
- +--------------+----------+
89
- | Job | Status |
90
- +--------------+----------+
91
- | Job1 | finished |
92
- | Job2 | finished |
93
- | Job3 | finished |
94
- | Job4 | finished |
95
- | InnerJobNet | error |
96
- | InnerJob3 | finished |
97
- | InnerJob1 | finished |
98
- | InnerJob2 | error |
99
- | Job8 | error |
100
- | Job5 | error |
101
- | Job6 | error |
102
- | Job7 | error |
103
- | InnerJobNet2 | finished |
104
- | InnerJob4 | finished |
105
- | InnerJob5 | skipped |
106
- | InnerJob6 | skipped |
107
- +--------------+----------+
48
+ % bundle exec rukawa run SampleJobNet -r 10 -c 10
49
+ +----------------+----------+
50
+ | Job | Status |
51
+ +----------------+----------+
52
+ | Job1 | finished |
53
+ | Job2 | waiting |
54
+ | Job3 | waiting |
55
+ | Job4 | waiting |
56
+ | InnerJobNet | waiting |
57
+ | InnerJob3 | waiting |
58
+ | InnerJob1 | waiting |
59
+ | InnerJob2 | waiting |
60
+ | Job8 | waiting |
61
+ | Job5 | waiting |
62
+ | Job6 | waiting |
63
+ | Job7 | waiting |
64
+ | InnerJobNet2 | waiting |
65
+ | InnerJob4 | waiting |
66
+ | InnerJob5 | waiting |
67
+ | InnerJob6 | waiting |
68
+ | InnerJobNet3 | waiting |
69
+ | InnerJob7 | waiting |
70
+ | InnerJob8 | waiting |
71
+ | InnerJob9 | waiting |
72
+ | InnerJob10 | waiting |
73
+ | InnerJobNet4 | waiting |
74
+ | InnerJob11 | waiting |
75
+ | InnerJob12 | waiting |
76
+ | InnerJob13 | waiting |
77
+ | NestedJobNet | waiting |
78
+ | NestedJob1 | waiting |
79
+ | NestedJob2 | waiting |
80
+ +----------------+----------+
81
+ +----------------+----------+
82
+ | Job | Status |
83
+ +----------------+----------+
84
+ | Job1 | finished |
85
+ | Job2 | finished |
86
+ | Job3 | finished |
87
+ | Job4 | finished |
88
+ | InnerJobNet | error |
89
+ | InnerJob3 | finished |
90
+ | InnerJob1 | finished |
91
+ | InnerJob2 | error |
92
+ | Job8 | aborted |
93
+ | Job5 | error |
94
+ | Job6 | aborted |
95
+ | Job7 | aborted |
96
+ | InnerJobNet2 | running |
97
+ | InnerJob4 | running |
98
+ | InnerJob5 | waiting |
99
+ | InnerJob6 | waiting |
100
+ | InnerJobNet3 | aborted |
101
+ | InnerJob7 | aborted |
102
+ | InnerJob8 | aborted |
103
+ | InnerJob9 | aborted |
104
+ | InnerJob10 | aborted |
105
+ | InnerJobNet4 | aborted |
106
+ | InnerJob11 | aborted |
107
+ | InnerJob12 | aborted |
108
+ | InnerJob13 | aborted |
109
+ | NestedJobNet | aborted |
110
+ | NestedJob1 | aborted |
111
+ | NestedJob2 | aborted |
112
+ +----------------+----------+
108
113
 
109
114
  # generate result graph image
110
115
  % dot -Tpng -o result.png result.dot
@@ -112,6 +117,108 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
112
117
 
113
118
  ![jobnet.png](https://raw.githubusercontent.com/joker1007/rukawa/master/sample/result.png)
114
119
 
120
+ `aborted` means that dependent job failure aborts following jobs.
121
+
122
+ ### Resume
123
+
124
+ Add `JOB_NAME` arguments to `run` command.
125
+
126
+ ```
127
+ % bundle exec rukawa run SampleJobNet Job8 InnerJob4 -r 5 -c 10
128
+ +----------------+----------+
129
+ | Job | Status |
130
+ +----------------+----------+
131
+ | Job1 | bypassed |
132
+ | Job2 | bypassed |
133
+ | Job3 | bypassed |
134
+ | Job4 | bypassed |
135
+ | InnerJobNet | bypassed |
136
+ | InnerJob3 | bypassed |
137
+ | InnerJob1 | bypassed |
138
+ | InnerJob2 | bypassed |
139
+ | Job8 | waiting |
140
+ | Job5 | bypassed |
141
+ | Job6 | bypassed |
142
+ | Job7 | bypassed |
143
+ | InnerJobNet2 | waiting |
144
+ | InnerJob4 | waiting |
145
+ | InnerJob5 | waiting |
146
+ | InnerJob6 | waiting |
147
+ | InnerJobNet3 | waiting |
148
+ | InnerJob7 | waiting |
149
+ | InnerJob8 | waiting |
150
+ | InnerJob9 | waiting |
151
+ | InnerJob10 | waiting |
152
+ | InnerJobNet4 | waiting |
153
+ | InnerJob11 | waiting |
154
+ | InnerJob12 | waiting |
155
+ | InnerJob13 | waiting |
156
+ | NestedJobNet | waiting |
157
+ | NestedJob1 | waiting |
158
+ | NestedJob2 | waiting |
159
+ +----------------+----------+
160
+ +----------------+----------+
161
+ | Job | Status |
162
+ +----------------+----------+
163
+ | Job1 | bypassed |
164
+ | Job2 | bypassed |
165
+ | Job3 | bypassed |
166
+ | Job4 | bypassed |
167
+ | InnerJobNet | bypassed |
168
+ | InnerJob3 | bypassed |
169
+ | InnerJob1 | bypassed |
170
+ | InnerJob2 | bypassed |
171
+ | Job8 | finished |
172
+ | Job5 | bypassed |
173
+ | Job6 | bypassed |
174
+ | Job7 | bypassed |
175
+ | InnerJobNet2 | skipped |
176
+ | InnerJob4 | finished |
177
+ | InnerJob5 | skipped |
178
+ | InnerJob6 | skipped |
179
+ | InnerJobNet3 | running |
180
+ | InnerJob7 | finished |
181
+ | InnerJob8 | running |
182
+ | InnerJob9 | waiting |
183
+ | InnerJob10 | waiting |
184
+ | InnerJobNet4 | waiting |
185
+ | InnerJob11 | waiting |
186
+ | InnerJob12 | waiting |
187
+ | InnerJob13 | waiting |
188
+ | NestedJobNet | waiting |
189
+ | NestedJob1 | waiting |
190
+ | NestedJob2 | waiting |
191
+ +----------------+----------+
192
+ ```
193
+
194
+ In above case, All jobs except `Job8` and `InnerJob4` and depending them are set `bypassed`.
195
+ `bypassed` means that job is not executed, and regarded as successful.
196
+ For example, a job depends two other jobs. Even if either job is `bypassed`, and another job is `finished`, it is executed.
197
+
198
+
199
+ ### Run Specific jobs
200
+
201
+ `run_job` command executes specified job forcely.
202
+
203
+ ```
204
+ % be rukawa run_job Job8 InnerJob6 NestedJob1 -c 3 -r 5
205
+ +------------+---------+
206
+ | Job | Status |
207
+ +------------+---------+
208
+ | Job8 | waiting |
209
+ | InnerJob6 | waiting |
210
+ | NestedJob1 | running |
211
+ +------------+---------+
212
+ +------------+----------+
213
+ | Job | Status |
214
+ +------------+----------+
215
+ | Job8 | finished |
216
+ | InnerJob6 | finished |
217
+ | NestedJob1 | finished |
218
+ +------------+----------+
219
+ ```
220
+
221
+ Main usage is manual reentering.
115
222
 
116
223
  ### Output jobnet graph (dot file)
117
224
 
@@ -124,18 +231,53 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
124
231
  ```
125
232
  % bundle exec rukawa help run
126
233
  Usage:
127
- rukawa run JOB_NET_NAME
234
+ rukawa run JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...
235
+
236
+ Options:
237
+ -c, [--concurrency=N] # Default: cpu count
238
+ [--variables=key:value]
239
+ [--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
240
+ [--job-dirs=one two three] # Load job directories
241
+ -b, [--batch], [--no-batch] # If batch mode, not display running status
242
+ -l, [--log=LOG]
243
+ # Default: ./rukawa.log
244
+ [--stdout], [--no-stdout] # Output log to stdout
245
+ -d, [--dot=DOT] # Output job status by dot format
246
+ -r, [--refresh-interval=N] # Refresh interval for running status information
247
+ # Default: 3
248
+
249
+ Run jobnet. If JOB_NET is set, resume from JOB_NAME
250
+
251
+
252
+ % bundle exec rukawa help run_job
253
+ Usage:
254
+ rukawa run_job JOB_NAME [JOB_NAME] ...
128
255
 
129
256
  Options:
130
257
  -c, [--concurrency=N] # Default: cpu count
131
258
  [--variables=key:value]
259
+ [--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
132
260
  [--job-dirs=one two three] # Load job directories
133
261
  -b, [--batch], [--no-batch] # If batch mode, not display running status
134
262
  -l, [--log=LOG]
135
263
  # Default: ./rukawa.log
264
+ [--stdout], [--no-stdout] # Output log to stdout
136
265
  -d, [--dot=DOT] # Output job status by dot format
137
266
  -r, [--refresh-interval=N] # Refresh interval for running status information
138
267
  # Default: 3
268
+
269
+ Run specific jobs.
270
+
271
+ % bundle exec rukawa help graph
272
+ Usage:
273
+ rukawa graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ... -o, --output=OUTPUT
274
+
275
+ Options:
276
+ [--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
277
+ [--job-dirs=one two three]
278
+ -o, --output=OUTPUT
279
+
280
+ Output jobnet graph. If JOB_NET is set, simulate resumed job sequence
139
281
  ```
140
282
 
141
283
  ## ToDo
@@ -3,6 +3,8 @@ require 'rukawa/state'
3
3
 
4
4
  module Rukawa
5
5
  class AbstractJob
6
+ attr_reader :parent_job_net
7
+
6
8
  class << self
7
9
  def skip_rules
8
10
  @skip_rules ||= []
@@ -17,8 +19,13 @@ module Rukawa
17
19
  self.class.to_s
18
20
  end
19
21
 
22
+ def inspect
23
+ to_s
24
+ end
25
+
20
26
  def skip?
21
- skip_rules.inject(false) do |cond, rule|
27
+ parent_skip = @parent_job_net ? @parent_job_net.skip? : false
28
+ parent_skip || skip_rules.inject(false) do |cond, rule|
22
29
  cond || rule.is_a?(Symbol) ? method(rule).call : rule.call(self)
23
30
  end
24
31
  end
@@ -26,5 +33,25 @@ module Rukawa
26
33
  def skip_rules
27
34
  self.class.skip_rules
28
35
  end
36
+
37
+ def elapsed_time_from(time = Time.now)
38
+ return finished_at - started_at if started_at && finished_at
39
+ return time - started_at if started_at
40
+
41
+ nil
42
+ end
43
+
44
+ def formatted_elapsed_time_from(time = Time.now)
45
+ sec = elapsed_time_from(time)
46
+ return "N/A" unless sec
47
+
48
+ hour = sec.to_i / 3600
49
+ min = sec.to_i / 60
50
+
51
+ hour_format = min > 0 ? "%dh " % hour : ""
52
+ min_format = min > 0 ? "%dm " % min : ""
53
+ sec_format = "#{sec.to_i}s"
54
+ "#{hour_format}#{min_format}#{sec_format}"
55
+ end
29
56
  end
30
57
  end
data/lib/rukawa/cli.rb CHANGED
@@ -34,19 +34,50 @@ module Rukawa
34
34
  exit 1 unless result
35
35
  end
36
36
 
37
- desc "graph JOB_NET_NAME", "Output jobnet graph"
37
+ desc "graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...", "Output jobnet graph. If JOB_NET is set, simulate resumed job sequence"
38
38
  method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
39
39
  method_option :job_dirs, type: :array, default: []
40
40
  method_option :output, aliases: "-o", type: :string, required: true
41
- def graph(job_net_name)
41
+ def graph(job_net_name, *job_name)
42
42
  load_config
43
43
  load_job_definitions
44
44
 
45
45
  job_net_class = Object.const_get(job_net_name)
46
- job_net = job_net_class.new(nil, options[:variables])
46
+ job_classes = job_name.map { |name| Object.const_get(name) }
47
+ job_net = job_net_class.new(nil, *job_classes)
47
48
  job_net.output_dot(options[:output])
48
49
  end
49
50
 
51
+ desc "run_job JOB_NAME [JOB_NAME] ...", "Run specific jobs."
52
+ method_option :concurrency, aliases: "-c", type: :numeric, default: nil, desc: "Default: cpu count"
53
+ method_option :variables, type: :hash, default: {}
54
+ method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
55
+ method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
56
+ method_option :batch, aliases: "-b", type: :boolean, default: false, desc: "If batch mode, not display running status"
57
+ method_option :log, aliases: "-l", type: :string, default: "./rukawa.log"
58
+ method_option :stdout, type: :boolean, default: false, desc: "Output log to stdout"
59
+ method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
60
+ method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
61
+ def run_job(*job_name)
62
+ load_config
63
+ Rukawa.configure do |c|
64
+ c.log_file = options[:stdout] ? $stdout : options[:log]
65
+ c.concurrency = options[:concurrency] if options[:concurrency]
66
+ end
67
+ load_job_definitions
68
+
69
+ job_classes = job_name.map { |name| Object.const_get(name) }
70
+ job_net_class = anonymous_job_net_class(*job_classes)
71
+ job_net = job_net_class.new(nil)
72
+ result = Runner.run(job_net, options[:batch], options[:refresh_interval])
73
+
74
+ if options[:dot]
75
+ job_net.output_dot(options[:dot])
76
+ end
77
+
78
+ exit 1 unless result
79
+ end
80
+
50
81
  private
51
82
 
52
83
  def load_config
@@ -71,5 +102,15 @@ module Rukawa
71
102
  Dir.glob(File.join(dir, "**/*.rb")) { |f| load f }
72
103
  end
73
104
  end
105
+
106
+ def anonymous_job_net_class(*job_classes)
107
+ Class.new(JobNet) do
108
+ self.singleton_class.send(:define_method, :dependencies) do
109
+ job_classes.map { |klass| [klass, []] }.to_h
110
+ end
111
+
112
+ define_method(:name) { "AnonymousJobNet" }
113
+ end
114
+ end
74
115
  end
75
116
  end
data/lib/rukawa/dag.rb CHANGED
@@ -1,8 +1,10 @@
1
1
  require 'set'
2
+ require 'tsort'
2
3
 
3
4
  module Rukawa
4
5
  class Dag
5
6
  include Enumerable
7
+ include TSort
6
8
 
7
9
  attr_reader :nodes, :jobs, :edges
8
10
 
@@ -47,6 +49,24 @@ module Rukawa
47
49
  end
48
50
  end
49
51
 
52
+ def tsort_each_node(&block)
53
+ @jobs.each(&block)
54
+ end
55
+
56
+ def tsort_each_child(node)
57
+ if block_given?
58
+ node.out_goings.each do |edge|
59
+ yield edge.to
60
+ end
61
+ else
62
+ Enumerator.new do |y|
63
+ node.out_goings.each do |edge|
64
+ y << edge.to
65
+ end
66
+ end
67
+ end
68
+ end
69
+
50
70
  private
51
71
 
52
72
  def tsortable_hash(hash)
data/lib/rukawa/job.rb CHANGED
@@ -4,7 +4,7 @@ require 'rukawa/abstract_job'
4
4
  module Rukawa
5
5
  class Job < AbstractJob
6
6
  attr_accessor :in_comings, :out_goings
7
- attr_reader :parent_job_net, :state
7
+ attr_reader :state, :started_at, :finished_at
8
8
 
9
9
  def initialize(parent_job_net)
10
10
  @parent_job_net = parent_job_net
@@ -13,6 +13,10 @@ module Rukawa
13
13
  set_state(:waiting)
14
14
  end
15
15
 
16
+ def set_state(name)
17
+ @state = Rukawa::State.get(name)
18
+ end
19
+
16
20
  def root?
17
21
  in_comings.select { |edge| edge.cluster == @parent_job_net }.empty?
18
22
  end
@@ -23,27 +27,10 @@ module Rukawa
23
27
 
24
28
  def dataflow
25
29
  return @dataflow if @dataflow
30
+ return @dataflow = bypass_dataflow if @state.bypassed?
26
31
 
27
32
  @dataflow = Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
28
- begin
29
- raise DependentJobFailure unless results.all? { |r| !r.nil? }
30
-
31
- if skip? || @parent_job_net.skip? || results.any? { |r| r == Rukawa::State.get(:skipped) }
32
- Rukawa.logger.info("Skip #{self.class}")
33
- set_state(:skipped)
34
- else
35
- Rukawa.logger.info("Start #{self.class}")
36
- set_state(:running)
37
- run
38
- Rukawa.logger.info("Finish #{self.class}")
39
- set_state(:finished)
40
- end
41
- rescue => e
42
- Rukawa.logger.error("Error #{self.class} by #{e}")
43
- set_state(:error)
44
- raise
45
- end
46
-
33
+ do_run(*results)
47
34
  @state
48
35
  end
49
36
  end
@@ -51,6 +38,27 @@ module Rukawa
51
38
  def run
52
39
  end
53
40
 
41
+ private def do_run(*results)
42
+ @started_at = Time.now
43
+ check_dependencies(results)
44
+
45
+ if skip? || results.any?(&:skipped?)
46
+ Rukawa.logger.info("Skip #{self.class}")
47
+ set_state(:skipped)
48
+ else
49
+ Rukawa.logger.info("Start #{self.class}")
50
+ set_state(:running)
51
+ run
52
+ Rukawa.logger.info("Finish #{self.class}")
53
+ set_state(:finished)
54
+ end
55
+ rescue => e
56
+ handle_error(e)
57
+ raise
58
+ ensure
59
+ @finished_at = Time.now
60
+ end
61
+
54
62
  def jobs_as_from
55
63
  [self]
56
64
  end
@@ -70,8 +78,23 @@ module Rukawa
70
78
  in_comings.map { |edge| edge.from.dataflow }
71
79
  end
72
80
 
73
- def set_state(name)
74
- @state = Rukawa::State.get(name)
81
+ def bypass_dataflow
82
+ Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
83
+ Rukawa.logger.info("Skip #{self.class}")
84
+ @state
85
+ end
86
+ end
87
+
88
+ def check_dependencies(results)
89
+ unless results.all? { |r| !r.nil? }
90
+ set_state(:aborted)
91
+ raise DependentJobFailure
92
+ end
93
+ end
94
+
95
+ def handle_error(e)
96
+ Rukawa.logger.error("Error #{self.class} by #{e}")
97
+ set_state(:error) unless e.is_a?(DependentJobFailure)
75
98
  end
76
99
 
77
100
  def store(key, value)
@@ -3,7 +3,7 @@ require 'rukawa/abstract_job'
3
3
  module Rukawa
4
4
  class JobNet < AbstractJob
5
5
  include Enumerable
6
- attr_reader :parent_job_net, :dag
6
+ attr_reader :dag
7
7
 
8
8
  class << self
9
9
  def dependencies
@@ -15,6 +15,29 @@ module Rukawa
15
15
  @parent_job_net = parent_job_net
16
16
  @dag = Dag.new
17
17
  @dag.build(self, self.class.dependencies)
18
+ @resume_job_classes = resume_job_classes
19
+
20
+ unless resume_job_classes.empty?
21
+ resume_targets = []
22
+ @dag.tsort_each_node do |node|
23
+ node.set_state(:bypassed)
24
+ resume_targets << node if resume_job_classes.include?(node.class)
25
+ end
26
+
27
+ resume_targets.each do |node|
28
+ @dag.each_strongly_connected_component_from(node) do |nodes|
29
+ nodes.each { |connected| connected.set_state(:waiting) }
30
+ end
31
+ end
32
+ end
33
+ end
34
+
35
+ def started_at
36
+ @dag.nodes.min_by { |j| j.started_at ? j.started_at.to_i : Float::INFINITY }.started_at
37
+ end
38
+
39
+ def finished_at
40
+ @dag.nodes.max_by { |j| j.finished_at.to_i }.finished_at
18
41
  end
19
42
 
20
43
  def toplevel?
@@ -26,13 +49,7 @@ module Rukawa
26
49
  end
27
50
 
28
51
  def dataflows
29
- flat_map do |j|
30
- if j.respond_to?(:dataflows)
31
- j.dataflows
32
- else
33
- [j.dataflow]
34
- end
35
- end
52
+ @dag.tsort.reverse.map(&:dataflow)
36
53
  end
37
54
 
38
55
  def state
@@ -48,7 +65,7 @@ module Rukawa
48
65
  def to_dot(subgraph = false)
49
66
  graphdef = subgraph ? "subgraph" : "digraph"
50
67
  buf = %Q|#{graphdef} "#{subgraph ? "cluster_" : ""}#{name}" {\n|
51
- buf += %Q{label = "#{name}";\n}
68
+ buf += %Q{label = "#{graph_label}";\n}
52
69
  buf += Rukawa.config.graph.attrs
53
70
  buf += Rukawa.config.graph.node.attrs
54
71
  buf += "color = blue;\n" if subgraph
@@ -77,5 +94,15 @@ module Rukawa
77
94
  def each(&block)
78
95
  @dag.each(&block)
79
96
  end
97
+
98
+ private
99
+
100
+ def graph_label
101
+ if @resume_job_classes.empty?
102
+ name
103
+ else
104
+ "#{name} resume from (#{@resume_job_classes.join(", ")})"
105
+ end
106
+ end
80
107
  end
81
108
  end
data/lib/rukawa/runner.rb CHANGED
@@ -24,6 +24,7 @@ module Rukawa
24
24
  Rukawa.logger.info("=== Finish Rukawa ===")
25
25
 
26
26
  display_table unless batch_mode
27
+ puts "Finished #{@root_job_net.name} in #{@root_job_net.formatted_elapsed_time_from}"
27
28
 
28
29
  errors = futures.map(&:reason).compact
29
30
 
@@ -41,7 +42,7 @@ module Rukawa
41
42
  private
42
43
 
43
44
  def display_table
44
- table = Terminal::Table.new headings: ["Job", "Status"] do |t|
45
+ table = Terminal::Table.new headings: ["Job", "Status", "Elapsed Time"] do |t|
45
46
  @root_job_net.each_with_index do |j|
46
47
  table_row(t, j)
47
48
  end
@@ -51,12 +52,12 @@ module Rukawa
51
52
 
52
53
  def table_row(table, job, level = 0)
53
54
  if job.is_a?(JobNet)
54
- table << [Paint["#{" " * level}#{job.class}", :bold, :underline], job.state.colored]
55
+ table << [Paint["#{" " * level}#{job.class}", :bold, :underline], Paint[job.state.colored, :bold, :underline], Paint[job.formatted_elapsed_time_from, :bold, :underline]]
55
56
  job.each do |inner_j|
56
57
  table_row(table, inner_j, level + 1)
57
58
  end
58
59
  else
59
- table << [Paint["#{" " * level}#{job.class}", :bold], job.state.colored]
60
+ table << [Paint["#{" " * level}#{job.class}", :bold], job.state.colored, job.formatted_elapsed_time_from]
60
61
  end
61
62
  end
62
63
  end
data/lib/rukawa/state.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  module Rukawa::State
2
2
  def self.get(name)
3
- const_get(name.to_s.capitalize)
3
+ const_get(name.to_s.split("_").map(&:capitalize).join)
4
4
  end
5
5
 
6
6
  module BaseExt
@@ -15,6 +15,12 @@ module Rukawa::State
15
15
  def merge(other)
16
16
  other
17
17
  end
18
+
19
+ %i(running? skipped? bypassed? error? aborted? waiting? finished?).each do |sym|
20
+ define_method(sym) do
21
+ false
22
+ end
23
+ end
18
24
  end
19
25
 
20
26
  module Running
@@ -27,6 +33,10 @@ module Rukawa::State
27
33
  def self.merge(_other)
28
34
  self
29
35
  end
36
+
37
+ def self.running?
38
+ true
39
+ end
30
40
  end
31
41
 
32
42
  module Skipped
@@ -43,6 +53,22 @@ module Rukawa::State
43
53
  other
44
54
  end
45
55
  end
56
+
57
+ def self.skipped?
58
+ true
59
+ end
60
+ end
61
+
62
+ module Bypassed
63
+ extend BaseExt
64
+
65
+ def self.color
66
+ :yellow
67
+ end
68
+
69
+ def self.bypassed?
70
+ true
71
+ end
46
72
  end
47
73
 
48
74
  module Error
@@ -59,6 +85,38 @@ module Rukawa::State
59
85
  self
60
86
  end
61
87
  end
88
+
89
+ def self.error?
90
+ true
91
+ end
92
+ end
93
+
94
+ module Aborted
95
+ extend BaseExt
96
+
97
+ def name
98
+ "aborted"
99
+ end
100
+
101
+ def self.color
102
+ :magenta
103
+ end
104
+
105
+ def self.merge(other)
106
+ if other == Running || other == Error
107
+ other
108
+ else
109
+ self
110
+ end
111
+ end
112
+
113
+ def self.error?
114
+ true
115
+ end
116
+
117
+ def self.aborted?
118
+ true
119
+ end
62
120
  end
63
121
 
64
122
  module Waiting
@@ -67,6 +125,10 @@ module Rukawa::State
67
125
  def self.color
68
126
  :default
69
127
  end
128
+
129
+ def self.waiting?
130
+ true
131
+ end
70
132
  end
71
133
 
72
134
  module Finished
@@ -75,5 +137,9 @@ module Rukawa::State
75
137
  def self.color
76
138
  :green
77
139
  end
140
+
141
+ def self.finished?
142
+ true
143
+ end
78
144
  end
79
145
  end
@@ -1,3 +1,3 @@
1
1
  module Rukawa
2
- VERSION = "0.2.1"
2
+ VERSION = "0.3.0"
3
3
  end
data/rukawa.gemspec CHANGED
@@ -26,4 +26,5 @@ Gem::Specification.new do |spec|
26
26
  spec.add_development_dependency "bundler", "~> 1.11"
27
27
  spec.add_development_dependency "rake", "~> 10.0"
28
28
  spec.add_development_dependency "rspec", "~> 3.0"
29
+ spec.add_development_dependency "rspec-power_assert"
29
30
  end
data/sample/result.dot ADDED
@@ -0,0 +1,86 @@
1
+ digraph "SampleJobNet" {
2
+ label = "SampleJobNet";
3
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
4
+ Job1 [style = filled,fillcolor = green];
5
+ Job2 [style = filled,fillcolor = green];
6
+ Job3 [style = filled,fillcolor = green];
7
+ Job4 [style = filled,fillcolor = green];
8
+ subgraph "cluster_InnerJobNet" {
9
+ label = "InnerJobNet";
10
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
11
+ color = blue;
12
+ InnerJob3 [style = filled,fillcolor = green];
13
+ InnerJob1 [style = filled,fillcolor = green];
14
+ InnerJob2 [style = filled,fillcolor = red];
15
+ "InnerJob1" -> "InnerJob2";
16
+ "InnerJob3" -> "InnerJob2";
17
+ }
18
+ Job8 [style = filled,fillcolor = magenta];
19
+ Job5 [style = filled,fillcolor = red];
20
+ Job6 [style = filled,fillcolor = magenta];
21
+ Job7 [style = filled,fillcolor = magenta];
22
+ subgraph "cluster_InnerJobNet2" {
23
+ label = "InnerJobNet2";
24
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
25
+ color = blue;
26
+ InnerJob4 [style = filled,fillcolor = green];
27
+ InnerJob5 [style = filled,fillcolor = yellow];
28
+ InnerJob6 [style = filled,fillcolor = yellow];
29
+ "InnerJob4" -> "InnerJob5";
30
+ "InnerJob4" -> "InnerJob6";
31
+ "InnerJob5" -> "InnerJob6";
32
+ }
33
+ subgraph "cluster_InnerJobNet3" {
34
+ label = "InnerJobNet3";
35
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
36
+ color = blue;
37
+ InnerJob7 [style = filled,fillcolor = magenta];
38
+ InnerJob8 [style = filled,fillcolor = magenta];
39
+ InnerJob9 [style = filled,fillcolor = magenta];
40
+ InnerJob10 [style = filled,fillcolor = magenta];
41
+ "InnerJob7" -> "InnerJob9";
42
+ "InnerJob8" -> "InnerJob9";
43
+ "InnerJob7" -> "InnerJob10";
44
+ "InnerJob8" -> "InnerJob10";
45
+ }
46
+ subgraph "cluster_InnerJobNet4" {
47
+ label = "InnerJobNet4";
48
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
49
+ color = blue;
50
+ InnerJob11 [style = filled,fillcolor = magenta];
51
+ InnerJob12 [style = filled,fillcolor = magenta];
52
+ InnerJob13 [style = filled,fillcolor = magenta];
53
+ subgraph "cluster_NestedJobNet" {
54
+ label = "NestedJobNet";
55
+ graph [rankdir = LR,nodesep = 0.8,concentrate = true];
56
+ color = blue;
57
+ NestedJob1 [style = filled,fillcolor = magenta];
58
+ NestedJob2 [style = filled,fillcolor = magenta];
59
+ "NestedJob1" -> "NestedJob2";
60
+ }
61
+ "InnerJob11" -> "NestedJob1";
62
+ "InnerJob12" -> "NestedJob1";
63
+ }
64
+ "Job1" -> "Job2";
65
+ "Job1" -> "Job3";
66
+ "Job2" -> "Job4";
67
+ "Job3" -> "Job4";
68
+ "Job3" -> "InnerJob3";
69
+ "Job3" -> "InnerJob1";
70
+ "InnerJob2" -> "Job8";
71
+ "Job3" -> "Job5";
72
+ "Job4" -> "Job6";
73
+ "Job5" -> "Job6";
74
+ "Job6" -> "Job7";
75
+ "Job4" -> "InnerJob4";
76
+ "Job8" -> "InnerJob7";
77
+ "Job8" -> "InnerJob8";
78
+ "Job7" -> "InnerJob7";
79
+ "Job7" -> "InnerJob8";
80
+ "InnerJob9" -> "InnerJob11";
81
+ "InnerJob9" -> "InnerJob12";
82
+ "InnerJob9" -> "InnerJob13";
83
+ "InnerJob10" -> "InnerJob11";
84
+ "InnerJob10" -> "InnerJob12";
85
+ "InnerJob10" -> "InnerJob13";
86
+ }
data/sample/result.png CHANGED
Binary file
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rukawa
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - joker1007
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-03-22 00:00:00.000000000 Z
11
+ date: 2016-03-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
@@ -108,6 +108,20 @@ dependencies:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
110
  version: '3.0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rspec-power_assert
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
111
125
  description: Hyper simple job workflow engine
112
126
  email:
113
127
  - kakyoin.hierophant@gmail.com
@@ -140,6 +154,7 @@ files:
140
154
  - sample/job_nets/sample_job_net.rb
141
155
  - sample/jobnet.png
142
156
  - sample/jobs/sample_job.rb
157
+ - sample/result.dot
143
158
  - sample/result.png
144
159
  - sample/rukawa.rb
145
160
  homepage: https://github.com/joker1007/rukawa