rukawa 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/README.md +204 -62
- data/lib/rukawa/abstract_job.rb +28 -1
- data/lib/rukawa/cli.rb +44 -3
- data/lib/rukawa/dag.rb +20 -0
- data/lib/rukawa/job.rb +45 -22
- data/lib/rukawa/job_net.rb +36 -9
- data/lib/rukawa/runner.rb +4 -3
- data/lib/rukawa/state.rb +67 -1
- data/lib/rukawa/version.rb +1 -1
- data/rukawa.gemspec +1 -0
- data/sample/result.dot +86 -0
- data/sample/result.png +0 -0
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cd835345eb1c1841c6dd7d6a907287eea0e359d6
|
4
|
+
data.tar.gz: 48eecf5784e84b8061edc309a657458567ea0743
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 07caf395ab5d616dc6803fbb6da39465465aaa6bf295900792c4643721328b5f531abfe3b3e01a31dc8853aa4e11e7b19fdf254177ae8112ee2d8b0a66a723ba
|
7
|
+
data.tar.gz: 1b77d69131e8fc808294a7ef2e814b649c995a56a78e8422d6866127ad9bd229b9137ab4dea8a3bc645d4c05b885e5e7a2d0952d06731d3e9804d5cfa309c36b
|
data/.rspec
CHANGED
data/README.md
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
[](https://waffle.io/joker1007/rukawa)
|
1
2
|
# Rukawa
|
2
3
|
[](https://badge.fury.io/rb/rukawa)
|
3
4
|
[](https://travis-ci.org/joker1007/rukawa)
|
@@ -44,67 +45,71 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
|
|
44
45
|
% cd rukawa/sample
|
45
46
|
|
46
47
|
# load ./jobs/**/*.rb, ./job_net/**/*.rb automatically
|
47
|
-
% bundle exec rukawa run SampleJobNet -r
|
48
|
-
|
49
|
-
| Job
|
50
|
-
|
51
|
-
| Job1
|
52
|
-
| Job2
|
53
|
-
| Job3
|
54
|
-
| Job4
|
55
|
-
| InnerJobNet
|
56
|
-
| InnerJob3
|
57
|
-
| InnerJob1
|
58
|
-
| InnerJob2
|
59
|
-
| Job8
|
60
|
-
| Job5
|
61
|
-
| Job6
|
62
|
-
| Job7
|
63
|
-
| InnerJobNet2
|
64
|
-
| InnerJob4
|
65
|
-
| InnerJob5
|
66
|
-
| InnerJob6
|
67
|
-
|
68
|
-
|
69
|
-
|
|
70
|
-
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
80
|
-
|
81
|
-
|
|
82
|
-
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
88
|
-
|
89
|
-
|
|
90
|
-
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
48
|
+
% bundle exec rukawa run SampleJobNet -r 10 -c 10
|
49
|
+
+----------------+----------+
|
50
|
+
| Job | Status |
|
51
|
+
+----------------+----------+
|
52
|
+
| Job1 | finished |
|
53
|
+
| Job2 | waiting |
|
54
|
+
| Job3 | waiting |
|
55
|
+
| Job4 | waiting |
|
56
|
+
| InnerJobNet | waiting |
|
57
|
+
| InnerJob3 | waiting |
|
58
|
+
| InnerJob1 | waiting |
|
59
|
+
| InnerJob2 | waiting |
|
60
|
+
| Job8 | waiting |
|
61
|
+
| Job5 | waiting |
|
62
|
+
| Job6 | waiting |
|
63
|
+
| Job7 | waiting |
|
64
|
+
| InnerJobNet2 | waiting |
|
65
|
+
| InnerJob4 | waiting |
|
66
|
+
| InnerJob5 | waiting |
|
67
|
+
| InnerJob6 | waiting |
|
68
|
+
| InnerJobNet3 | waiting |
|
69
|
+
| InnerJob7 | waiting |
|
70
|
+
| InnerJob8 | waiting |
|
71
|
+
| InnerJob9 | waiting |
|
72
|
+
| InnerJob10 | waiting |
|
73
|
+
| InnerJobNet4 | waiting |
|
74
|
+
| InnerJob11 | waiting |
|
75
|
+
| InnerJob12 | waiting |
|
76
|
+
| InnerJob13 | waiting |
|
77
|
+
| NestedJobNet | waiting |
|
78
|
+
| NestedJob1 | waiting |
|
79
|
+
| NestedJob2 | waiting |
|
80
|
+
+----------------+----------+
|
81
|
+
+----------------+----------+
|
82
|
+
| Job | Status |
|
83
|
+
+----------------+----------+
|
84
|
+
| Job1 | finished |
|
85
|
+
| Job2 | finished |
|
86
|
+
| Job3 | finished |
|
87
|
+
| Job4 | finished |
|
88
|
+
| InnerJobNet | error |
|
89
|
+
| InnerJob3 | finished |
|
90
|
+
| InnerJob1 | finished |
|
91
|
+
| InnerJob2 | error |
|
92
|
+
| Job8 | aborted |
|
93
|
+
| Job5 | error |
|
94
|
+
| Job6 | aborted |
|
95
|
+
| Job7 | aborted |
|
96
|
+
| InnerJobNet2 | running |
|
97
|
+
| InnerJob4 | running |
|
98
|
+
| InnerJob5 | waiting |
|
99
|
+
| InnerJob6 | waiting |
|
100
|
+
| InnerJobNet3 | aborted |
|
101
|
+
| InnerJob7 | aborted |
|
102
|
+
| InnerJob8 | aborted |
|
103
|
+
| InnerJob9 | aborted |
|
104
|
+
| InnerJob10 | aborted |
|
105
|
+
| InnerJobNet4 | aborted |
|
106
|
+
| InnerJob11 | aborted |
|
107
|
+
| InnerJob12 | aborted |
|
108
|
+
| InnerJob13 | aborted |
|
109
|
+
| NestedJobNet | aborted |
|
110
|
+
| NestedJob1 | aborted |
|
111
|
+
| NestedJob2 | aborted |
|
112
|
+
+----------------+----------+
|
108
113
|
|
109
114
|
# generate result graph image
|
110
115
|
% dot -Tpng -o result.png result.dot
|
@@ -112,6 +117,108 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
|
|
112
117
|
|
113
118
|

|
114
119
|
|
120
|
+
`aborted` means that dependent job failure aborts following jobs.
|
121
|
+
|
122
|
+
### Resume
|
123
|
+
|
124
|
+
Add `JOB_NAME` arguments to `run` command.
|
125
|
+
|
126
|
+
```
|
127
|
+
% bundle exec rukawa run SampleJobNet Job8 InnerJob4 -r 5 -c 10
|
128
|
+
+----------------+----------+
|
129
|
+
| Job | Status |
|
130
|
+
+----------------+----------+
|
131
|
+
| Job1 | bypassed |
|
132
|
+
| Job2 | bypassed |
|
133
|
+
| Job3 | bypassed |
|
134
|
+
| Job4 | bypassed |
|
135
|
+
| InnerJobNet | bypassed |
|
136
|
+
| InnerJob3 | bypassed |
|
137
|
+
| InnerJob1 | bypassed |
|
138
|
+
| InnerJob2 | bypassed |
|
139
|
+
| Job8 | waiting |
|
140
|
+
| Job5 | bypassed |
|
141
|
+
| Job6 | bypassed |
|
142
|
+
| Job7 | bypassed |
|
143
|
+
| InnerJobNet2 | waiting |
|
144
|
+
| InnerJob4 | waiting |
|
145
|
+
| InnerJob5 | waiting |
|
146
|
+
| InnerJob6 | waiting |
|
147
|
+
| InnerJobNet3 | waiting |
|
148
|
+
| InnerJob7 | waiting |
|
149
|
+
| InnerJob8 | waiting |
|
150
|
+
| InnerJob9 | waiting |
|
151
|
+
| InnerJob10 | waiting |
|
152
|
+
| InnerJobNet4 | waiting |
|
153
|
+
| InnerJob11 | waiting |
|
154
|
+
| InnerJob12 | waiting |
|
155
|
+
| InnerJob13 | waiting |
|
156
|
+
| NestedJobNet | waiting |
|
157
|
+
| NestedJob1 | waiting |
|
158
|
+
| NestedJob2 | waiting |
|
159
|
+
+----------------+----------+
|
160
|
+
+----------------+----------+
|
161
|
+
| Job | Status |
|
162
|
+
+----------------+----------+
|
163
|
+
| Job1 | bypassed |
|
164
|
+
| Job2 | bypassed |
|
165
|
+
| Job3 | bypassed |
|
166
|
+
| Job4 | bypassed |
|
167
|
+
| InnerJobNet | bypassed |
|
168
|
+
| InnerJob3 | bypassed |
|
169
|
+
| InnerJob1 | bypassed |
|
170
|
+
| InnerJob2 | bypassed |
|
171
|
+
| Job8 | finished |
|
172
|
+
| Job5 | bypassed |
|
173
|
+
| Job6 | bypassed |
|
174
|
+
| Job7 | bypassed |
|
175
|
+
| InnerJobNet2 | skipped |
|
176
|
+
| InnerJob4 | finished |
|
177
|
+
| InnerJob5 | skipped |
|
178
|
+
| InnerJob6 | skipped |
|
179
|
+
| InnerJobNet3 | running |
|
180
|
+
| InnerJob7 | finished |
|
181
|
+
| InnerJob8 | running |
|
182
|
+
| InnerJob9 | waiting |
|
183
|
+
| InnerJob10 | waiting |
|
184
|
+
| InnerJobNet4 | waiting |
|
185
|
+
| InnerJob11 | waiting |
|
186
|
+
| InnerJob12 | waiting |
|
187
|
+
| InnerJob13 | waiting |
|
188
|
+
| NestedJobNet | waiting |
|
189
|
+
| NestedJob1 | waiting |
|
190
|
+
| NestedJob2 | waiting |
|
191
|
+
+----------------+----------+
|
192
|
+
```
|
193
|
+
|
194
|
+
In above case, All jobs except `Job8` and `InnerJob4` and depending them are set `bypassed`.
|
195
|
+
`bypassed` means that job is not executed, and regarded as successful.
|
196
|
+
For example, a job depends two other jobs. Even if either job is `bypassed`, and another job is `finished`, it is executed.
|
197
|
+
|
198
|
+
|
199
|
+
### Run Specific jobs
|
200
|
+
|
201
|
+
`run_job` command executes specified job forcely.
|
202
|
+
|
203
|
+
```
|
204
|
+
% be rukawa run_job Job8 InnerJob6 NestedJob1 -c 3 -r 5
|
205
|
+
+------------+---------+
|
206
|
+
| Job | Status |
|
207
|
+
+------------+---------+
|
208
|
+
| Job8 | waiting |
|
209
|
+
| InnerJob6 | waiting |
|
210
|
+
| NestedJob1 | running |
|
211
|
+
+------------+---------+
|
212
|
+
+------------+----------+
|
213
|
+
| Job | Status |
|
214
|
+
+------------+----------+
|
215
|
+
| Job8 | finished |
|
216
|
+
| InnerJob6 | finished |
|
217
|
+
| NestedJob1 | finished |
|
218
|
+
+------------+----------+
|
219
|
+
```
|
220
|
+
|
221
|
+
Main usage is manual reentering.
|
115
222
|
|
116
223
|
### Output jobnet graph (dot file)
|
117
224
|
|
@@ -124,18 +231,53 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
|
|
124
231
|
```
|
125
232
|
% bundle exec rukawa help run
|
126
233
|
Usage:
|
127
|
-
rukawa run JOB_NET_NAME
|
234
|
+
rukawa run JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...
|
235
|
+
|
236
|
+
Options:
|
237
|
+
-c, [--concurrency=N] # Default: cpu count
|
238
|
+
[--variables=key:value]
|
239
|
+
[--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
|
240
|
+
[--job-dirs=one two three] # Load job directories
|
241
|
+
-b, [--batch], [--no-batch] # If batch mode, not display running status
|
242
|
+
-l, [--log=LOG]
|
243
|
+
# Default: ./rukawa.log
|
244
|
+
[--stdout], [--no-stdout] # Output log to stdout
|
245
|
+
-d, [--dot=DOT] # Output job status by dot format
|
246
|
+
-r, [--refresh-interval=N] # Refresh interval for running status information
|
247
|
+
# Default: 3
|
248
|
+
|
249
|
+
Run jobnet. If JOB_NET is set, resume from JOB_NAME
|
250
|
+
|
251
|
+
|
252
|
+
% bundle exec rukawa help run_job
|
253
|
+
Usage:
|
254
|
+
rukawa run_job JOB_NAME [JOB_NAME] ...
|
128
255
|
|
129
256
|
Options:
|
130
257
|
-c, [--concurrency=N] # Default: cpu count
|
131
258
|
[--variables=key:value]
|
259
|
+
[--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
|
132
260
|
[--job-dirs=one two three] # Load job directories
|
133
261
|
-b, [--batch], [--no-batch] # If batch mode, not display running status
|
134
262
|
-l, [--log=LOG]
|
135
263
|
# Default: ./rukawa.log
|
264
|
+
[--stdout], [--no-stdout] # Output log to stdout
|
136
265
|
-d, [--dot=DOT] # Output job status by dot format
|
137
266
|
-r, [--refresh-interval=N] # Refresh interval for running status information
|
138
267
|
# Default: 3
|
268
|
+
|
269
|
+
Run specific jobs.
|
270
|
+
|
271
|
+
% bundle exec rukawa help graph
|
272
|
+
Usage:
|
273
|
+
rukawa graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ... -o, --output=OUTPUT
|
274
|
+
|
275
|
+
Options:
|
276
|
+
[--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
|
277
|
+
[--job-dirs=one two three]
|
278
|
+
-o, --output=OUTPUT
|
279
|
+
|
280
|
+
Output jobnet graph. If JOB_NET is set, simulate resumed job sequence
|
139
281
|
```
|
140
282
|
|
141
283
|
## ToDo
|
data/lib/rukawa/abstract_job.rb
CHANGED
@@ -3,6 +3,8 @@ require 'rukawa/state'
|
|
3
3
|
|
4
4
|
module Rukawa
|
5
5
|
class AbstractJob
|
6
|
+
attr_reader :parent_job_net
|
7
|
+
|
6
8
|
class << self
|
7
9
|
def skip_rules
|
8
10
|
@skip_rules ||= []
|
@@ -17,8 +19,13 @@ module Rukawa
|
|
17
19
|
self.class.to_s
|
18
20
|
end
|
19
21
|
|
22
|
+
def inspect
|
23
|
+
to_s
|
24
|
+
end
|
25
|
+
|
20
26
|
def skip?
|
21
|
-
|
27
|
+
parent_skip = @parent_job_net ? @parent_job_net.skip? : false
|
28
|
+
parent_skip || skip_rules.inject(false) do |cond, rule|
|
22
29
|
cond || rule.is_a?(Symbol) ? method(rule).call : rule.call(self)
|
23
30
|
end
|
24
31
|
end
|
@@ -26,5 +33,25 @@ module Rukawa
|
|
26
33
|
def skip_rules
|
27
34
|
self.class.skip_rules
|
28
35
|
end
|
36
|
+
|
37
|
+
def elapsed_time_from(time = Time.now)
|
38
|
+
return finished_at - started_at if started_at && finished_at
|
39
|
+
return time - started_at if started_at
|
40
|
+
|
41
|
+
nil
|
42
|
+
end
|
43
|
+
|
44
|
+
def formatted_elapsed_time_from(time = Time.now)
|
45
|
+
sec = elapsed_time_from(time)
|
46
|
+
return "N/A" unless sec
|
47
|
+
|
48
|
+
hour = sec.to_i / 3600
|
49
|
+
min = sec.to_i / 60
|
50
|
+
|
51
|
+
hour_format = min > 0 ? "%dh " % hour : ""
|
52
|
+
min_format = min > 0 ? "%dm " % min : ""
|
53
|
+
sec_format = "#{sec.to_i}s"
|
54
|
+
"#{hour_format}#{min_format}#{sec_format}"
|
55
|
+
end
|
29
56
|
end
|
30
57
|
end
|
data/lib/rukawa/cli.rb
CHANGED
@@ -34,19 +34,50 @@ module Rukawa
|
|
34
34
|
exit 1 unless result
|
35
35
|
end
|
36
36
|
|
37
|
-
desc "graph JOB_NET_NAME", "Output jobnet graph"
|
37
|
+
desc "graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...", "Output jobnet graph. If JOB_NET is set, simulate resumed job sequence"
|
38
38
|
method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
|
39
39
|
method_option :job_dirs, type: :array, default: []
|
40
40
|
method_option :output, aliases: "-o", type: :string, required: true
|
41
|
-
def graph(job_net_name)
|
41
|
+
def graph(job_net_name, *job_name)
|
42
42
|
load_config
|
43
43
|
load_job_definitions
|
44
44
|
|
45
45
|
job_net_class = Object.const_get(job_net_name)
|
46
|
-
|
46
|
+
job_classes = job_name.map { |name| Object.const_get(name) }
|
47
|
+
job_net = job_net_class.new(nil, *job_classes)
|
47
48
|
job_net.output_dot(options[:output])
|
48
49
|
end
|
49
50
|
|
51
|
+
desc "run_job JOB_NAME [JOB_NAME] ...", "Run specific jobs."
|
52
|
+
method_option :concurrency, aliases: "-c", type: :numeric, default: nil, desc: "Default: cpu count"
|
53
|
+
method_option :variables, type: :hash, default: {}
|
54
|
+
method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
|
55
|
+
method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
|
56
|
+
method_option :batch, aliases: "-b", type: :boolean, default: false, desc: "If batch mode, not display running status"
|
57
|
+
method_option :log, aliases: "-l", type: :string, default: "./rukawa.log"
|
58
|
+
method_option :stdout, type: :boolean, default: false, desc: "Output log to stdout"
|
59
|
+
method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
|
60
|
+
method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
|
61
|
+
def run_job(*job_name)
|
62
|
+
load_config
|
63
|
+
Rukawa.configure do |c|
|
64
|
+
c.log_file = options[:stdout] ? $stdout : options[:log]
|
65
|
+
c.concurrency = options[:concurrency] if options[:concurrency]
|
66
|
+
end
|
67
|
+
load_job_definitions
|
68
|
+
|
69
|
+
job_classes = job_name.map { |name| Object.const_get(name) }
|
70
|
+
job_net_class = anonymous_job_net_class(*job_classes)
|
71
|
+
job_net = job_net_class.new(nil)
|
72
|
+
result = Runner.run(job_net, options[:batch], options[:refresh_interval])
|
73
|
+
|
74
|
+
if options[:dot]
|
75
|
+
job_net.output_dot(options[:dot])
|
76
|
+
end
|
77
|
+
|
78
|
+
exit 1 unless result
|
79
|
+
end
|
80
|
+
|
50
81
|
private
|
51
82
|
|
52
83
|
def load_config
|
@@ -71,5 +102,15 @@ module Rukawa
|
|
71
102
|
Dir.glob(File.join(dir, "**/*.rb")) { |f| load f }
|
72
103
|
end
|
73
104
|
end
|
105
|
+
|
106
|
+
def anonymous_job_net_class(*job_classes)
|
107
|
+
Class.new(JobNet) do
|
108
|
+
self.singleton_class.send(:define_method, :dependencies) do
|
109
|
+
job_classes.map { |klass| [klass, []] }.to_h
|
110
|
+
end
|
111
|
+
|
112
|
+
define_method(:name) { "AnonymousJobNet" }
|
113
|
+
end
|
114
|
+
end
|
74
115
|
end
|
75
116
|
end
|
data/lib/rukawa/dag.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
require 'set'
|
2
|
+
require 'tsort'
|
2
3
|
|
3
4
|
module Rukawa
|
4
5
|
class Dag
|
5
6
|
include Enumerable
|
7
|
+
include TSort
|
6
8
|
|
7
9
|
attr_reader :nodes, :jobs, :edges
|
8
10
|
|
@@ -47,6 +49,24 @@ module Rukawa
|
|
47
49
|
end
|
48
50
|
end
|
49
51
|
|
52
|
+
def tsort_each_node(&block)
|
53
|
+
@jobs.each(&block)
|
54
|
+
end
|
55
|
+
|
56
|
+
def tsort_each_child(node)
|
57
|
+
if block_given?
|
58
|
+
node.out_goings.each do |edge|
|
59
|
+
yield edge.to
|
60
|
+
end
|
61
|
+
else
|
62
|
+
Enumerator.new do |y|
|
63
|
+
node.out_goings.each do |edge|
|
64
|
+
y << edge.to
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
50
70
|
private
|
51
71
|
|
52
72
|
def tsortable_hash(hash)
|
data/lib/rukawa/job.rb
CHANGED
@@ -4,7 +4,7 @@ require 'rukawa/abstract_job'
|
|
4
4
|
module Rukawa
|
5
5
|
class Job < AbstractJob
|
6
6
|
attr_accessor :in_comings, :out_goings
|
7
|
-
attr_reader :
|
7
|
+
attr_reader :state, :started_at, :finished_at
|
8
8
|
|
9
9
|
def initialize(parent_job_net)
|
10
10
|
@parent_job_net = parent_job_net
|
@@ -13,6 +13,10 @@ module Rukawa
|
|
13
13
|
set_state(:waiting)
|
14
14
|
end
|
15
15
|
|
16
|
+
def set_state(name)
|
17
|
+
@state = Rukawa::State.get(name)
|
18
|
+
end
|
19
|
+
|
16
20
|
def root?
|
17
21
|
in_comings.select { |edge| edge.cluster == @parent_job_net }.empty?
|
18
22
|
end
|
@@ -23,27 +27,10 @@ module Rukawa
|
|
23
27
|
|
24
28
|
def dataflow
|
25
29
|
return @dataflow if @dataflow
|
30
|
+
return @dataflow = bypass_dataflow if @state.bypassed?
|
26
31
|
|
27
32
|
@dataflow = Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
|
28
|
-
|
29
|
-
raise DependentJobFailure unless results.all? { |r| !r.nil? }
|
30
|
-
|
31
|
-
if skip? || @parent_job_net.skip? || results.any? { |r| r == Rukawa::State.get(:skipped) }
|
32
|
-
Rukawa.logger.info("Skip #{self.class}")
|
33
|
-
set_state(:skipped)
|
34
|
-
else
|
35
|
-
Rukawa.logger.info("Start #{self.class}")
|
36
|
-
set_state(:running)
|
37
|
-
run
|
38
|
-
Rukawa.logger.info("Finish #{self.class}")
|
39
|
-
set_state(:finished)
|
40
|
-
end
|
41
|
-
rescue => e
|
42
|
-
Rukawa.logger.error("Error #{self.class} by #{e}")
|
43
|
-
set_state(:error)
|
44
|
-
raise
|
45
|
-
end
|
46
|
-
|
33
|
+
do_run(*results)
|
47
34
|
@state
|
48
35
|
end
|
49
36
|
end
|
@@ -51,6 +38,27 @@ module Rukawa
|
|
51
38
|
def run
|
52
39
|
end
|
53
40
|
|
41
|
+
private def do_run(*results)
|
42
|
+
@started_at = Time.now
|
43
|
+
check_dependencies(results)
|
44
|
+
|
45
|
+
if skip? || results.any?(&:skipped?)
|
46
|
+
Rukawa.logger.info("Skip #{self.class}")
|
47
|
+
set_state(:skipped)
|
48
|
+
else
|
49
|
+
Rukawa.logger.info("Start #{self.class}")
|
50
|
+
set_state(:running)
|
51
|
+
run
|
52
|
+
Rukawa.logger.info("Finish #{self.class}")
|
53
|
+
set_state(:finished)
|
54
|
+
end
|
55
|
+
rescue => e
|
56
|
+
handle_error(e)
|
57
|
+
raise
|
58
|
+
ensure
|
59
|
+
@finished_at = Time.now
|
60
|
+
end
|
61
|
+
|
54
62
|
def jobs_as_from
|
55
63
|
[self]
|
56
64
|
end
|
@@ -70,8 +78,23 @@ module Rukawa
|
|
70
78
|
in_comings.map { |edge| edge.from.dataflow }
|
71
79
|
end
|
72
80
|
|
73
|
-
def
|
74
|
-
|
81
|
+
def bypass_dataflow
|
82
|
+
Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
|
83
|
+
Rukawa.logger.info("Skip #{self.class}")
|
84
|
+
@state
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def check_dependencies(results)
|
89
|
+
unless results.all? { |r| !r.nil? }
|
90
|
+
set_state(:aborted)
|
91
|
+
raise DependentJobFailure
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def handle_error(e)
|
96
|
+
Rukawa.logger.error("Error #{self.class} by #{e}")
|
97
|
+
set_state(:error) unless e.is_a?(DependentJobFailure)
|
75
98
|
end
|
76
99
|
|
77
100
|
def store(key, value)
|
data/lib/rukawa/job_net.rb
CHANGED
@@ -3,7 +3,7 @@ require 'rukawa/abstract_job'
|
|
3
3
|
module Rukawa
|
4
4
|
class JobNet < AbstractJob
|
5
5
|
include Enumerable
|
6
|
-
attr_reader :
|
6
|
+
attr_reader :dag
|
7
7
|
|
8
8
|
class << self
|
9
9
|
def dependencies
|
@@ -15,6 +15,29 @@ module Rukawa
|
|
15
15
|
@parent_job_net = parent_job_net
|
16
16
|
@dag = Dag.new
|
17
17
|
@dag.build(self, self.class.dependencies)
|
18
|
+
@resume_job_classes = resume_job_classes
|
19
|
+
|
20
|
+
unless resume_job_classes.empty?
|
21
|
+
resume_targets = []
|
22
|
+
@dag.tsort_each_node do |node|
|
23
|
+
node.set_state(:bypassed)
|
24
|
+
resume_targets << node if resume_job_classes.include?(node.class)
|
25
|
+
end
|
26
|
+
|
27
|
+
resume_targets.each do |node|
|
28
|
+
@dag.each_strongly_connected_component_from(node) do |nodes|
|
29
|
+
nodes.each { |connected| connected.set_state(:waiting) }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def started_at
|
36
|
+
@dag.nodes.min_by { |j| j.started_at ? j.started_at.to_i : Float::INFINITY }.started_at
|
37
|
+
end
|
38
|
+
|
39
|
+
def finished_at
|
40
|
+
@dag.nodes.max_by { |j| j.finished_at.to_i }.finished_at
|
18
41
|
end
|
19
42
|
|
20
43
|
def toplevel?
|
@@ -26,13 +49,7 @@ module Rukawa
|
|
26
49
|
end
|
27
50
|
|
28
51
|
def dataflows
|
29
|
-
|
30
|
-
if j.respond_to?(:dataflows)
|
31
|
-
j.dataflows
|
32
|
-
else
|
33
|
-
[j.dataflow]
|
34
|
-
end
|
35
|
-
end
|
52
|
+
@dag.tsort.reverse.map(&:dataflow)
|
36
53
|
end
|
37
54
|
|
38
55
|
def state
|
@@ -48,7 +65,7 @@ module Rukawa
|
|
48
65
|
def to_dot(subgraph = false)
|
49
66
|
graphdef = subgraph ? "subgraph" : "digraph"
|
50
67
|
buf = %Q|#{graphdef} "#{subgraph ? "cluster_" : ""}#{name}" {\n|
|
51
|
-
buf += %Q{label = "#{
|
68
|
+
buf += %Q{label = "#{graph_label}";\n}
|
52
69
|
buf += Rukawa.config.graph.attrs
|
53
70
|
buf += Rukawa.config.graph.node.attrs
|
54
71
|
buf += "color = blue;\n" if subgraph
|
@@ -77,5 +94,15 @@ module Rukawa
|
|
77
94
|
def each(&block)
|
78
95
|
@dag.each(&block)
|
79
96
|
end
|
97
|
+
|
98
|
+
private
|
99
|
+
|
100
|
+
def graph_label
|
101
|
+
if @resume_job_classes.empty?
|
102
|
+
name
|
103
|
+
else
|
104
|
+
"#{name} resume from (#{@resume_job_classes.join(", ")})"
|
105
|
+
end
|
106
|
+
end
|
80
107
|
end
|
81
108
|
end
|
data/lib/rukawa/runner.rb
CHANGED
@@ -24,6 +24,7 @@ module Rukawa
|
|
24
24
|
Rukawa.logger.info("=== Finish Rukawa ===")
|
25
25
|
|
26
26
|
display_table unless batch_mode
|
27
|
+
puts "Finished #{@root_job_net.name} in #{@root_job_net.formatted_elapsed_time_from}"
|
27
28
|
|
28
29
|
errors = futures.map(&:reason).compact
|
29
30
|
|
@@ -41,7 +42,7 @@ module Rukawa
|
|
41
42
|
private
|
42
43
|
|
43
44
|
def display_table
|
44
|
-
table = Terminal::Table.new headings: ["Job", "Status"] do |t|
|
45
|
+
table = Terminal::Table.new headings: ["Job", "Status", "Elapsed Time"] do |t|
|
45
46
|
@root_job_net.each_with_index do |j|
|
46
47
|
table_row(t, j)
|
47
48
|
end
|
@@ -51,12 +52,12 @@ module Rukawa
|
|
51
52
|
|
52
53
|
def table_row(table, job, level = 0)
|
53
54
|
if job.is_a?(JobNet)
|
54
|
-
table << [Paint["#{" " * level}#{job.class}", :bold, :underline], job.state.colored]
|
55
|
+
table << [Paint["#{" " * level}#{job.class}", :bold, :underline], Paint[job.state.colored, :bold, :underline], Paint[job.formatted_elapsed_time_from, :bold, :underline]]
|
55
56
|
job.each do |inner_j|
|
56
57
|
table_row(table, inner_j, level + 1)
|
57
58
|
end
|
58
59
|
else
|
59
|
-
table << [Paint["#{" " * level}#{job.class}", :bold], job.state.colored]
|
60
|
+
table << [Paint["#{" " * level}#{job.class}", :bold], job.state.colored, job.formatted_elapsed_time_from]
|
60
61
|
end
|
61
62
|
end
|
62
63
|
end
|
data/lib/rukawa/state.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Rukawa::State
|
2
2
|
def self.get(name)
|
3
|
-
const_get(name.to_s.capitalize)
|
3
|
+
const_get(name.to_s.split("_").map(&:capitalize).join)
|
4
4
|
end
|
5
5
|
|
6
6
|
module BaseExt
|
@@ -15,6 +15,12 @@ module Rukawa::State
|
|
15
15
|
def merge(other)
|
16
16
|
other
|
17
17
|
end
|
18
|
+
|
19
|
+
%i(running? skipped? bypassed? error? aborted? waiting? finished?).each do |sym|
|
20
|
+
define_method(sym) do
|
21
|
+
false
|
22
|
+
end
|
23
|
+
end
|
18
24
|
end
|
19
25
|
|
20
26
|
module Running
|
@@ -27,6 +33,10 @@ module Rukawa::State
|
|
27
33
|
def self.merge(_other)
|
28
34
|
self
|
29
35
|
end
|
36
|
+
|
37
|
+
def self.running?
|
38
|
+
true
|
39
|
+
end
|
30
40
|
end
|
31
41
|
|
32
42
|
module Skipped
|
@@ -43,6 +53,22 @@ module Rukawa::State
|
|
43
53
|
other
|
44
54
|
end
|
45
55
|
end
|
56
|
+
|
57
|
+
def self.skipped?
|
58
|
+
true
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
module Bypassed
|
63
|
+
extend BaseExt
|
64
|
+
|
65
|
+
def self.color
|
66
|
+
:yellow
|
67
|
+
end
|
68
|
+
|
69
|
+
def self.bypassed?
|
70
|
+
true
|
71
|
+
end
|
46
72
|
end
|
47
73
|
|
48
74
|
module Error
|
@@ -59,6 +85,38 @@ module Rukawa::State
|
|
59
85
|
self
|
60
86
|
end
|
61
87
|
end
|
88
|
+
|
89
|
+
def self.error?
|
90
|
+
true
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
module Aborted
|
95
|
+
extend BaseExt
|
96
|
+
|
97
|
+
def name
|
98
|
+
"aborted"
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.color
|
102
|
+
:magenta
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.merge(other)
|
106
|
+
if other == Running || other == Error
|
107
|
+
other
|
108
|
+
else
|
109
|
+
self
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def self.error?
|
114
|
+
true
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.aborted?
|
118
|
+
true
|
119
|
+
end
|
62
120
|
end
|
63
121
|
|
64
122
|
module Waiting
|
@@ -67,6 +125,10 @@ module Rukawa::State
|
|
67
125
|
def self.color
|
68
126
|
:default
|
69
127
|
end
|
128
|
+
|
129
|
+
def self.waiting?
|
130
|
+
true
|
131
|
+
end
|
70
132
|
end
|
71
133
|
|
72
134
|
module Finished
|
@@ -75,5 +137,9 @@ module Rukawa::State
|
|
75
137
|
def self.color
|
76
138
|
:green
|
77
139
|
end
|
140
|
+
|
141
|
+
def self.finished?
|
142
|
+
true
|
143
|
+
end
|
78
144
|
end
|
79
145
|
end
|
data/lib/rukawa/version.rb
CHANGED
data/rukawa.gemspec
CHANGED
data/sample/result.dot
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
digraph "SampleJobNet" {
|
2
|
+
label = "SampleJobNet";
|
3
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
4
|
+
Job1 [style = filled,fillcolor = green];
|
5
|
+
Job2 [style = filled,fillcolor = green];
|
6
|
+
Job3 [style = filled,fillcolor = green];
|
7
|
+
Job4 [style = filled,fillcolor = green];
|
8
|
+
subgraph "cluster_InnerJobNet" {
|
9
|
+
label = "InnerJobNet";
|
10
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
11
|
+
color = blue;
|
12
|
+
InnerJob3 [style = filled,fillcolor = green];
|
13
|
+
InnerJob1 [style = filled,fillcolor = green];
|
14
|
+
InnerJob2 [style = filled,fillcolor = red];
|
15
|
+
"InnerJob1" -> "InnerJob2";
|
16
|
+
"InnerJob3" -> "InnerJob2";
|
17
|
+
}
|
18
|
+
Job8 [style = filled,fillcolor = magenta];
|
19
|
+
Job5 [style = filled,fillcolor = red];
|
20
|
+
Job6 [style = filled,fillcolor = magenta];
|
21
|
+
Job7 [style = filled,fillcolor = magenta];
|
22
|
+
subgraph "cluster_InnerJobNet2" {
|
23
|
+
label = "InnerJobNet2";
|
24
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
25
|
+
color = blue;
|
26
|
+
InnerJob4 [style = filled,fillcolor = green];
|
27
|
+
InnerJob5 [style = filled,fillcolor = yellow];
|
28
|
+
InnerJob6 [style = filled,fillcolor = yellow];
|
29
|
+
"InnerJob4" -> "InnerJob5";
|
30
|
+
"InnerJob4" -> "InnerJob6";
|
31
|
+
"InnerJob5" -> "InnerJob6";
|
32
|
+
}
|
33
|
+
subgraph "cluster_InnerJobNet3" {
|
34
|
+
label = "InnerJobNet3";
|
35
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
36
|
+
color = blue;
|
37
|
+
InnerJob7 [style = filled,fillcolor = magenta];
|
38
|
+
InnerJob8 [style = filled,fillcolor = magenta];
|
39
|
+
InnerJob9 [style = filled,fillcolor = magenta];
|
40
|
+
InnerJob10 [style = filled,fillcolor = magenta];
|
41
|
+
"InnerJob7" -> "InnerJob9";
|
42
|
+
"InnerJob8" -> "InnerJob9";
|
43
|
+
"InnerJob7" -> "InnerJob10";
|
44
|
+
"InnerJob8" -> "InnerJob10";
|
45
|
+
}
|
46
|
+
subgraph "cluster_InnerJobNet4" {
|
47
|
+
label = "InnerJobNet4";
|
48
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
49
|
+
color = blue;
|
50
|
+
InnerJob11 [style = filled,fillcolor = magenta];
|
51
|
+
InnerJob12 [style = filled,fillcolor = magenta];
|
52
|
+
InnerJob13 [style = filled,fillcolor = magenta];
|
53
|
+
subgraph "cluster_NestedJobNet" {
|
54
|
+
label = "NestedJobNet";
|
55
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
56
|
+
color = blue;
|
57
|
+
NestedJob1 [style = filled,fillcolor = magenta];
|
58
|
+
NestedJob2 [style = filled,fillcolor = magenta];
|
59
|
+
"NestedJob1" -> "NestedJob2";
|
60
|
+
}
|
61
|
+
"InnerJob11" -> "NestedJob1";
|
62
|
+
"InnerJob12" -> "NestedJob1";
|
63
|
+
}
|
64
|
+
"Job1" -> "Job2";
|
65
|
+
"Job1" -> "Job3";
|
66
|
+
"Job2" -> "Job4";
|
67
|
+
"Job3" -> "Job4";
|
68
|
+
"Job3" -> "InnerJob3";
|
69
|
+
"Job3" -> "InnerJob1";
|
70
|
+
"InnerJob2" -> "Job8";
|
71
|
+
"Job3" -> "Job5";
|
72
|
+
"Job4" -> "Job6";
|
73
|
+
"Job5" -> "Job6";
|
74
|
+
"Job6" -> "Job7";
|
75
|
+
"Job4" -> "InnerJob4";
|
76
|
+
"Job8" -> "InnerJob7";
|
77
|
+
"Job8" -> "InnerJob8";
|
78
|
+
"Job7" -> "InnerJob7";
|
79
|
+
"Job7" -> "InnerJob8";
|
80
|
+
"InnerJob9" -> "InnerJob11";
|
81
|
+
"InnerJob9" -> "InnerJob12";
|
82
|
+
"InnerJob9" -> "InnerJob13";
|
83
|
+
"InnerJob10" -> "InnerJob11";
|
84
|
+
"InnerJob10" -> "InnerJob12";
|
85
|
+
"InnerJob10" -> "InnerJob13";
|
86
|
+
}
|
data/sample/result.png
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rukawa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- joker1007
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-03-
|
11
|
+
date: 2016-03-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '3.0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rspec-power_assert
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
111
125
|
description: Hyper simple job workflow engine
|
112
126
|
email:
|
113
127
|
- kakyoin.hierophant@gmail.com
|
@@ -140,6 +154,7 @@ files:
|
|
140
154
|
- sample/job_nets/sample_job_net.rb
|
141
155
|
- sample/jobnet.png
|
142
156
|
- sample/jobs/sample_job.rb
|
157
|
+
- sample/result.dot
|
143
158
|
- sample/result.png
|
144
159
|
- sample/rukawa.rb
|
145
160
|
homepage: https://github.com/joker1007/rukawa
|