rukawa 0.2.1 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/README.md +204 -62
- data/lib/rukawa/abstract_job.rb +28 -1
- data/lib/rukawa/cli.rb +44 -3
- data/lib/rukawa/dag.rb +20 -0
- data/lib/rukawa/job.rb +45 -22
- data/lib/rukawa/job_net.rb +36 -9
- data/lib/rukawa/runner.rb +4 -3
- data/lib/rukawa/state.rb +67 -1
- data/lib/rukawa/version.rb +1 -1
- data/rukawa.gemspec +1 -0
- data/sample/result.dot +86 -0
- data/sample/result.png +0 -0
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cd835345eb1c1841c6dd7d6a907287eea0e359d6
|
4
|
+
data.tar.gz: 48eecf5784e84b8061edc309a657458567ea0743
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 07caf395ab5d616dc6803fbb6da39465465aaa6bf295900792c4643721328b5f531abfe3b3e01a31dc8853aa4e11e7b19fdf254177ae8112ee2d8b0a66a723ba
|
7
|
+
data.tar.gz: 1b77d69131e8fc808294a7ef2e814b649c995a56a78e8422d6866127ad9bd229b9137ab4dea8a3bc645d4c05b885e5e7a2d0952d06731d3e9804d5cfa309c36b
|
data/.rspec
CHANGED
data/README.md
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
[![Stories in Ready](https://badge.waffle.io/joker1007/rukawa.png?label=ready&title=Ready)](https://waffle.io/joker1007/rukawa)
|
1
2
|
# Rukawa
|
2
3
|
[![Gem Version](https://badge.fury.io/rb/rukawa.svg)](https://badge.fury.io/rb/rukawa)
|
3
4
|
[![Build Status](https://travis-ci.org/joker1007/rukawa.svg?branch=master)](https://travis-ci.org/joker1007/rukawa)
|
@@ -44,67 +45,71 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
|
|
44
45
|
% cd rukawa/sample
|
45
46
|
|
46
47
|
# load ./jobs/**/*.rb, ./job_net/**/*.rb automatically
|
47
|
-
% bundle exec rukawa run SampleJobNet -r
|
48
|
-
|
49
|
-
| Job
|
50
|
-
|
51
|
-
| Job1
|
52
|
-
| Job2
|
53
|
-
| Job3
|
54
|
-
| Job4
|
55
|
-
| InnerJobNet
|
56
|
-
| InnerJob3
|
57
|
-
| InnerJob1
|
58
|
-
| InnerJob2
|
59
|
-
| Job8
|
60
|
-
| Job5
|
61
|
-
| Job6
|
62
|
-
| Job7
|
63
|
-
| InnerJobNet2
|
64
|
-
| InnerJob4
|
65
|
-
| InnerJob5
|
66
|
-
| InnerJob6
|
67
|
-
|
68
|
-
|
69
|
-
|
|
70
|
-
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
80
|
-
|
81
|
-
|
|
82
|
-
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
88
|
-
|
89
|
-
|
|
90
|
-
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
48
|
+
% bundle exec rukawa run SampleJobNet -r 10 -c 10
|
49
|
+
+----------------+----------+
|
50
|
+
| Job | Status |
|
51
|
+
+----------------+----------+
|
52
|
+
| Job1 | finished |
|
53
|
+
| Job2 | waiting |
|
54
|
+
| Job3 | waiting |
|
55
|
+
| Job4 | waiting |
|
56
|
+
| InnerJobNet | waiting |
|
57
|
+
| InnerJob3 | waiting |
|
58
|
+
| InnerJob1 | waiting |
|
59
|
+
| InnerJob2 | waiting |
|
60
|
+
| Job8 | waiting |
|
61
|
+
| Job5 | waiting |
|
62
|
+
| Job6 | waiting |
|
63
|
+
| Job7 | waiting |
|
64
|
+
| InnerJobNet2 | waiting |
|
65
|
+
| InnerJob4 | waiting |
|
66
|
+
| InnerJob5 | waiting |
|
67
|
+
| InnerJob6 | waiting |
|
68
|
+
| InnerJobNet3 | waiting |
|
69
|
+
| InnerJob7 | waiting |
|
70
|
+
| InnerJob8 | waiting |
|
71
|
+
| InnerJob9 | waiting |
|
72
|
+
| InnerJob10 | waiting |
|
73
|
+
| InnerJobNet4 | waiting |
|
74
|
+
| InnerJob11 | waiting |
|
75
|
+
| InnerJob12 | waiting |
|
76
|
+
| InnerJob13 | waiting |
|
77
|
+
| NestedJobNet | waiting |
|
78
|
+
| NestedJob1 | waiting |
|
79
|
+
| NestedJob2 | waiting |
|
80
|
+
+----------------+----------+
|
81
|
+
+----------------+----------+
|
82
|
+
| Job | Status |
|
83
|
+
+----------------+----------+
|
84
|
+
| Job1 | finished |
|
85
|
+
| Job2 | finished |
|
86
|
+
| Job3 | finished |
|
87
|
+
| Job4 | finished |
|
88
|
+
| InnerJobNet | error |
|
89
|
+
| InnerJob3 | finished |
|
90
|
+
| InnerJob1 | finished |
|
91
|
+
| InnerJob2 | error |
|
92
|
+
| Job8 | aborted |
|
93
|
+
| Job5 | error |
|
94
|
+
| Job6 | aborted |
|
95
|
+
| Job7 | aborted |
|
96
|
+
| InnerJobNet2 | running |
|
97
|
+
| InnerJob4 | running |
|
98
|
+
| InnerJob5 | waiting |
|
99
|
+
| InnerJob6 | waiting |
|
100
|
+
| InnerJobNet3 | aborted |
|
101
|
+
| InnerJob7 | aborted |
|
102
|
+
| InnerJob8 | aborted |
|
103
|
+
| InnerJob9 | aborted |
|
104
|
+
| InnerJob10 | aborted |
|
105
|
+
| InnerJobNet4 | aborted |
|
106
|
+
| InnerJob11 | aborted |
|
107
|
+
| InnerJob12 | aborted |
|
108
|
+
| InnerJob13 | aborted |
|
109
|
+
| NestedJobNet | aborted |
|
110
|
+
| NestedJob1 | aborted |
|
111
|
+
| NestedJob2 | aborted |
|
112
|
+
+----------------+----------+
|
108
113
|
|
109
114
|
# generate result graph image
|
110
115
|
% dot -Tpng -o result.png result.dot
|
@@ -112,6 +117,108 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
|
|
112
117
|
|
113
118
|
![jobnet.png](https://raw.githubusercontent.com/joker1007/rukawa/master/sample/result.png)
|
114
119
|
|
120
|
+
`aborted` means that dependent job failure aborts following jobs.
|
121
|
+
|
122
|
+
### Resume
|
123
|
+
|
124
|
+
Add `JOB_NAME` arguments to `run` command.
|
125
|
+
|
126
|
+
```
|
127
|
+
% bundle exec rukawa run SampleJobNet Job8 InnerJob4 -r 5 -c 10
|
128
|
+
+----------------+----------+
|
129
|
+
| Job | Status |
|
130
|
+
+----------------+----------+
|
131
|
+
| Job1 | bypassed |
|
132
|
+
| Job2 | bypassed |
|
133
|
+
| Job3 | bypassed |
|
134
|
+
| Job4 | bypassed |
|
135
|
+
| InnerJobNet | bypassed |
|
136
|
+
| InnerJob3 | bypassed |
|
137
|
+
| InnerJob1 | bypassed |
|
138
|
+
| InnerJob2 | bypassed |
|
139
|
+
| Job8 | waiting |
|
140
|
+
| Job5 | bypassed |
|
141
|
+
| Job6 | bypassed |
|
142
|
+
| Job7 | bypassed |
|
143
|
+
| InnerJobNet2 | waiting |
|
144
|
+
| InnerJob4 | waiting |
|
145
|
+
| InnerJob5 | waiting |
|
146
|
+
| InnerJob6 | waiting |
|
147
|
+
| InnerJobNet3 | waiting |
|
148
|
+
| InnerJob7 | waiting |
|
149
|
+
| InnerJob8 | waiting |
|
150
|
+
| InnerJob9 | waiting |
|
151
|
+
| InnerJob10 | waiting |
|
152
|
+
| InnerJobNet4 | waiting |
|
153
|
+
| InnerJob11 | waiting |
|
154
|
+
| InnerJob12 | waiting |
|
155
|
+
| InnerJob13 | waiting |
|
156
|
+
| NestedJobNet | waiting |
|
157
|
+
| NestedJob1 | waiting |
|
158
|
+
| NestedJob2 | waiting |
|
159
|
+
+----------------+----------+
|
160
|
+
+----------------+----------+
|
161
|
+
| Job | Status |
|
162
|
+
+----------------+----------+
|
163
|
+
| Job1 | bypassed |
|
164
|
+
| Job2 | bypassed |
|
165
|
+
| Job3 | bypassed |
|
166
|
+
| Job4 | bypassed |
|
167
|
+
| InnerJobNet | bypassed |
|
168
|
+
| InnerJob3 | bypassed |
|
169
|
+
| InnerJob1 | bypassed |
|
170
|
+
| InnerJob2 | bypassed |
|
171
|
+
| Job8 | finished |
|
172
|
+
| Job5 | bypassed |
|
173
|
+
| Job6 | bypassed |
|
174
|
+
| Job7 | bypassed |
|
175
|
+
| InnerJobNet2 | skipped |
|
176
|
+
| InnerJob4 | finished |
|
177
|
+
| InnerJob5 | skipped |
|
178
|
+
| InnerJob6 | skipped |
|
179
|
+
| InnerJobNet3 | running |
|
180
|
+
| InnerJob7 | finished |
|
181
|
+
| InnerJob8 | running |
|
182
|
+
| InnerJob9 | waiting |
|
183
|
+
| InnerJob10 | waiting |
|
184
|
+
| InnerJobNet4 | waiting |
|
185
|
+
| InnerJob11 | waiting |
|
186
|
+
| InnerJob12 | waiting |
|
187
|
+
| InnerJob13 | waiting |
|
188
|
+
| NestedJobNet | waiting |
|
189
|
+
| NestedJob1 | waiting |
|
190
|
+
| NestedJob2 | waiting |
|
191
|
+
+----------------+----------+
|
192
|
+
```
|
193
|
+
|
194
|
+
In above case, All jobs except `Job8` and `InnerJob4` and depending them are set `bypassed`.
|
195
|
+
`bypassed` means that job is not executed, and regarded as successful.
|
196
|
+
For example, a job depends two other jobs. Even if either job is `bypassed`, and another job is `finished`, it is executed.
|
197
|
+
|
198
|
+
|
199
|
+
### Run Specific jobs
|
200
|
+
|
201
|
+
`run_job` command executes specified job forcely.
|
202
|
+
|
203
|
+
```
|
204
|
+
% be rukawa run_job Job8 InnerJob6 NestedJob1 -c 3 -r 5
|
205
|
+
+------------+---------+
|
206
|
+
| Job | Status |
|
207
|
+
+------------+---------+
|
208
|
+
| Job8 | waiting |
|
209
|
+
| InnerJob6 | waiting |
|
210
|
+
| NestedJob1 | running |
|
211
|
+
+------------+---------+
|
212
|
+
+------------+----------+
|
213
|
+
| Job | Status |
|
214
|
+
+------------+----------+
|
215
|
+
| Job8 | finished |
|
216
|
+
| InnerJob6 | finished |
|
217
|
+
| NestedJob1 | finished |
|
218
|
+
+------------+----------+
|
219
|
+
```
|
220
|
+
|
221
|
+
Main usage is manual reentering.
|
115
222
|
|
116
223
|
### Output jobnet graph (dot file)
|
117
224
|
|
@@ -124,18 +231,53 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
|
|
124
231
|
```
|
125
232
|
% bundle exec rukawa help run
|
126
233
|
Usage:
|
127
|
-
rukawa run JOB_NET_NAME
|
234
|
+
rukawa run JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...
|
235
|
+
|
236
|
+
Options:
|
237
|
+
-c, [--concurrency=N] # Default: cpu count
|
238
|
+
[--variables=key:value]
|
239
|
+
[--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
|
240
|
+
[--job-dirs=one two three] # Load job directories
|
241
|
+
-b, [--batch], [--no-batch] # If batch mode, not display running status
|
242
|
+
-l, [--log=LOG]
|
243
|
+
# Default: ./rukawa.log
|
244
|
+
[--stdout], [--no-stdout] # Output log to stdout
|
245
|
+
-d, [--dot=DOT] # Output job status by dot format
|
246
|
+
-r, [--refresh-interval=N] # Refresh interval for running status information
|
247
|
+
# Default: 3
|
248
|
+
|
249
|
+
Run jobnet. If JOB_NET is set, resume from JOB_NAME
|
250
|
+
|
251
|
+
|
252
|
+
% bundle exec rukawa help run_job
|
253
|
+
Usage:
|
254
|
+
rukawa run_job JOB_NAME [JOB_NAME] ...
|
128
255
|
|
129
256
|
Options:
|
130
257
|
-c, [--concurrency=N] # Default: cpu count
|
131
258
|
[--variables=key:value]
|
259
|
+
[--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
|
132
260
|
[--job-dirs=one two three] # Load job directories
|
133
261
|
-b, [--batch], [--no-batch] # If batch mode, not display running status
|
134
262
|
-l, [--log=LOG]
|
135
263
|
# Default: ./rukawa.log
|
264
|
+
[--stdout], [--no-stdout] # Output log to stdout
|
136
265
|
-d, [--dot=DOT] # Output job status by dot format
|
137
266
|
-r, [--refresh-interval=N] # Refresh interval for running status information
|
138
267
|
# Default: 3
|
268
|
+
|
269
|
+
Run specific jobs.
|
270
|
+
|
271
|
+
% bundle exec rukawa help graph
|
272
|
+
Usage:
|
273
|
+
rukawa graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ... -o, --output=OUTPUT
|
274
|
+
|
275
|
+
Options:
|
276
|
+
[--config=CONFIG] # If this options is not set, try to load ./rukawa.rb
|
277
|
+
[--job-dirs=one two three]
|
278
|
+
-o, --output=OUTPUT
|
279
|
+
|
280
|
+
Output jobnet graph. If JOB_NET is set, simulate resumed job sequence
|
139
281
|
```
|
140
282
|
|
141
283
|
## ToDo
|
data/lib/rukawa/abstract_job.rb
CHANGED
@@ -3,6 +3,8 @@ require 'rukawa/state'
|
|
3
3
|
|
4
4
|
module Rukawa
|
5
5
|
class AbstractJob
|
6
|
+
attr_reader :parent_job_net
|
7
|
+
|
6
8
|
class << self
|
7
9
|
def skip_rules
|
8
10
|
@skip_rules ||= []
|
@@ -17,8 +19,13 @@ module Rukawa
|
|
17
19
|
self.class.to_s
|
18
20
|
end
|
19
21
|
|
22
|
+
def inspect
|
23
|
+
to_s
|
24
|
+
end
|
25
|
+
|
20
26
|
def skip?
|
21
|
-
|
27
|
+
parent_skip = @parent_job_net ? @parent_job_net.skip? : false
|
28
|
+
parent_skip || skip_rules.inject(false) do |cond, rule|
|
22
29
|
cond || rule.is_a?(Symbol) ? method(rule).call : rule.call(self)
|
23
30
|
end
|
24
31
|
end
|
@@ -26,5 +33,25 @@ module Rukawa
|
|
26
33
|
def skip_rules
|
27
34
|
self.class.skip_rules
|
28
35
|
end
|
36
|
+
|
37
|
+
def elapsed_time_from(time = Time.now)
|
38
|
+
return finished_at - started_at if started_at && finished_at
|
39
|
+
return time - started_at if started_at
|
40
|
+
|
41
|
+
nil
|
42
|
+
end
|
43
|
+
|
44
|
+
def formatted_elapsed_time_from(time = Time.now)
|
45
|
+
sec = elapsed_time_from(time)
|
46
|
+
return "N/A" unless sec
|
47
|
+
|
48
|
+
hour = sec.to_i / 3600
|
49
|
+
min = sec.to_i / 60
|
50
|
+
|
51
|
+
hour_format = min > 0 ? "%dh " % hour : ""
|
52
|
+
min_format = min > 0 ? "%dm " % min : ""
|
53
|
+
sec_format = "#{sec.to_i}s"
|
54
|
+
"#{hour_format}#{min_format}#{sec_format}"
|
55
|
+
end
|
29
56
|
end
|
30
57
|
end
|
data/lib/rukawa/cli.rb
CHANGED
@@ -34,19 +34,50 @@ module Rukawa
|
|
34
34
|
exit 1 unless result
|
35
35
|
end
|
36
36
|
|
37
|
-
desc "graph JOB_NET_NAME", "Output jobnet graph"
|
37
|
+
desc "graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...", "Output jobnet graph. If JOB_NET is set, simulate resumed job sequence"
|
38
38
|
method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
|
39
39
|
method_option :job_dirs, type: :array, default: []
|
40
40
|
method_option :output, aliases: "-o", type: :string, required: true
|
41
|
-
def graph(job_net_name)
|
41
|
+
def graph(job_net_name, *job_name)
|
42
42
|
load_config
|
43
43
|
load_job_definitions
|
44
44
|
|
45
45
|
job_net_class = Object.const_get(job_net_name)
|
46
|
-
|
46
|
+
job_classes = job_name.map { |name| Object.const_get(name) }
|
47
|
+
job_net = job_net_class.new(nil, *job_classes)
|
47
48
|
job_net.output_dot(options[:output])
|
48
49
|
end
|
49
50
|
|
51
|
+
desc "run_job JOB_NAME [JOB_NAME] ...", "Run specific jobs."
|
52
|
+
method_option :concurrency, aliases: "-c", type: :numeric, default: nil, desc: "Default: cpu count"
|
53
|
+
method_option :variables, type: :hash, default: {}
|
54
|
+
method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
|
55
|
+
method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
|
56
|
+
method_option :batch, aliases: "-b", type: :boolean, default: false, desc: "If batch mode, not display running status"
|
57
|
+
method_option :log, aliases: "-l", type: :string, default: "./rukawa.log"
|
58
|
+
method_option :stdout, type: :boolean, default: false, desc: "Output log to stdout"
|
59
|
+
method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
|
60
|
+
method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
|
61
|
+
def run_job(*job_name)
|
62
|
+
load_config
|
63
|
+
Rukawa.configure do |c|
|
64
|
+
c.log_file = options[:stdout] ? $stdout : options[:log]
|
65
|
+
c.concurrency = options[:concurrency] if options[:concurrency]
|
66
|
+
end
|
67
|
+
load_job_definitions
|
68
|
+
|
69
|
+
job_classes = job_name.map { |name| Object.const_get(name) }
|
70
|
+
job_net_class = anonymous_job_net_class(*job_classes)
|
71
|
+
job_net = job_net_class.new(nil)
|
72
|
+
result = Runner.run(job_net, options[:batch], options[:refresh_interval])
|
73
|
+
|
74
|
+
if options[:dot]
|
75
|
+
job_net.output_dot(options[:dot])
|
76
|
+
end
|
77
|
+
|
78
|
+
exit 1 unless result
|
79
|
+
end
|
80
|
+
|
50
81
|
private
|
51
82
|
|
52
83
|
def load_config
|
@@ -71,5 +102,15 @@ module Rukawa
|
|
71
102
|
Dir.glob(File.join(dir, "**/*.rb")) { |f| load f }
|
72
103
|
end
|
73
104
|
end
|
105
|
+
|
106
|
+
def anonymous_job_net_class(*job_classes)
|
107
|
+
Class.new(JobNet) do
|
108
|
+
self.singleton_class.send(:define_method, :dependencies) do
|
109
|
+
job_classes.map { |klass| [klass, []] }.to_h
|
110
|
+
end
|
111
|
+
|
112
|
+
define_method(:name) { "AnonymousJobNet" }
|
113
|
+
end
|
114
|
+
end
|
74
115
|
end
|
75
116
|
end
|
data/lib/rukawa/dag.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
require 'set'
|
2
|
+
require 'tsort'
|
2
3
|
|
3
4
|
module Rukawa
|
4
5
|
class Dag
|
5
6
|
include Enumerable
|
7
|
+
include TSort
|
6
8
|
|
7
9
|
attr_reader :nodes, :jobs, :edges
|
8
10
|
|
@@ -47,6 +49,24 @@ module Rukawa
|
|
47
49
|
end
|
48
50
|
end
|
49
51
|
|
52
|
+
def tsort_each_node(&block)
|
53
|
+
@jobs.each(&block)
|
54
|
+
end
|
55
|
+
|
56
|
+
def tsort_each_child(node)
|
57
|
+
if block_given?
|
58
|
+
node.out_goings.each do |edge|
|
59
|
+
yield edge.to
|
60
|
+
end
|
61
|
+
else
|
62
|
+
Enumerator.new do |y|
|
63
|
+
node.out_goings.each do |edge|
|
64
|
+
y << edge.to
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
50
70
|
private
|
51
71
|
|
52
72
|
def tsortable_hash(hash)
|
data/lib/rukawa/job.rb
CHANGED
@@ -4,7 +4,7 @@ require 'rukawa/abstract_job'
|
|
4
4
|
module Rukawa
|
5
5
|
class Job < AbstractJob
|
6
6
|
attr_accessor :in_comings, :out_goings
|
7
|
-
attr_reader :
|
7
|
+
attr_reader :state, :started_at, :finished_at
|
8
8
|
|
9
9
|
def initialize(parent_job_net)
|
10
10
|
@parent_job_net = parent_job_net
|
@@ -13,6 +13,10 @@ module Rukawa
|
|
13
13
|
set_state(:waiting)
|
14
14
|
end
|
15
15
|
|
16
|
+
def set_state(name)
|
17
|
+
@state = Rukawa::State.get(name)
|
18
|
+
end
|
19
|
+
|
16
20
|
def root?
|
17
21
|
in_comings.select { |edge| edge.cluster == @parent_job_net }.empty?
|
18
22
|
end
|
@@ -23,27 +27,10 @@ module Rukawa
|
|
23
27
|
|
24
28
|
def dataflow
|
25
29
|
return @dataflow if @dataflow
|
30
|
+
return @dataflow = bypass_dataflow if @state.bypassed?
|
26
31
|
|
27
32
|
@dataflow = Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
|
28
|
-
|
29
|
-
raise DependentJobFailure unless results.all? { |r| !r.nil? }
|
30
|
-
|
31
|
-
if skip? || @parent_job_net.skip? || results.any? { |r| r == Rukawa::State.get(:skipped) }
|
32
|
-
Rukawa.logger.info("Skip #{self.class}")
|
33
|
-
set_state(:skipped)
|
34
|
-
else
|
35
|
-
Rukawa.logger.info("Start #{self.class}")
|
36
|
-
set_state(:running)
|
37
|
-
run
|
38
|
-
Rukawa.logger.info("Finish #{self.class}")
|
39
|
-
set_state(:finished)
|
40
|
-
end
|
41
|
-
rescue => e
|
42
|
-
Rukawa.logger.error("Error #{self.class} by #{e}")
|
43
|
-
set_state(:error)
|
44
|
-
raise
|
45
|
-
end
|
46
|
-
|
33
|
+
do_run(*results)
|
47
34
|
@state
|
48
35
|
end
|
49
36
|
end
|
@@ -51,6 +38,27 @@ module Rukawa
|
|
51
38
|
def run
|
52
39
|
end
|
53
40
|
|
41
|
+
private def do_run(*results)
|
42
|
+
@started_at = Time.now
|
43
|
+
check_dependencies(results)
|
44
|
+
|
45
|
+
if skip? || results.any?(&:skipped?)
|
46
|
+
Rukawa.logger.info("Skip #{self.class}")
|
47
|
+
set_state(:skipped)
|
48
|
+
else
|
49
|
+
Rukawa.logger.info("Start #{self.class}")
|
50
|
+
set_state(:running)
|
51
|
+
run
|
52
|
+
Rukawa.logger.info("Finish #{self.class}")
|
53
|
+
set_state(:finished)
|
54
|
+
end
|
55
|
+
rescue => e
|
56
|
+
handle_error(e)
|
57
|
+
raise
|
58
|
+
ensure
|
59
|
+
@finished_at = Time.now
|
60
|
+
end
|
61
|
+
|
54
62
|
def jobs_as_from
|
55
63
|
[self]
|
56
64
|
end
|
@@ -70,8 +78,23 @@ module Rukawa
|
|
70
78
|
in_comings.map { |edge| edge.from.dataflow }
|
71
79
|
end
|
72
80
|
|
73
|
-
def
|
74
|
-
|
81
|
+
def bypass_dataflow
|
82
|
+
Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
|
83
|
+
Rukawa.logger.info("Skip #{self.class}")
|
84
|
+
@state
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def check_dependencies(results)
|
89
|
+
unless results.all? { |r| !r.nil? }
|
90
|
+
set_state(:aborted)
|
91
|
+
raise DependentJobFailure
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def handle_error(e)
|
96
|
+
Rukawa.logger.error("Error #{self.class} by #{e}")
|
97
|
+
set_state(:error) unless e.is_a?(DependentJobFailure)
|
75
98
|
end
|
76
99
|
|
77
100
|
def store(key, value)
|
data/lib/rukawa/job_net.rb
CHANGED
@@ -3,7 +3,7 @@ require 'rukawa/abstract_job'
|
|
3
3
|
module Rukawa
|
4
4
|
class JobNet < AbstractJob
|
5
5
|
include Enumerable
|
6
|
-
attr_reader :
|
6
|
+
attr_reader :dag
|
7
7
|
|
8
8
|
class << self
|
9
9
|
def dependencies
|
@@ -15,6 +15,29 @@ module Rukawa
|
|
15
15
|
@parent_job_net = parent_job_net
|
16
16
|
@dag = Dag.new
|
17
17
|
@dag.build(self, self.class.dependencies)
|
18
|
+
@resume_job_classes = resume_job_classes
|
19
|
+
|
20
|
+
unless resume_job_classes.empty?
|
21
|
+
resume_targets = []
|
22
|
+
@dag.tsort_each_node do |node|
|
23
|
+
node.set_state(:bypassed)
|
24
|
+
resume_targets << node if resume_job_classes.include?(node.class)
|
25
|
+
end
|
26
|
+
|
27
|
+
resume_targets.each do |node|
|
28
|
+
@dag.each_strongly_connected_component_from(node) do |nodes|
|
29
|
+
nodes.each { |connected| connected.set_state(:waiting) }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def started_at
|
36
|
+
@dag.nodes.min_by { |j| j.started_at ? j.started_at.to_i : Float::INFINITY }.started_at
|
37
|
+
end
|
38
|
+
|
39
|
+
def finished_at
|
40
|
+
@dag.nodes.max_by { |j| j.finished_at.to_i }.finished_at
|
18
41
|
end
|
19
42
|
|
20
43
|
def toplevel?
|
@@ -26,13 +49,7 @@ module Rukawa
|
|
26
49
|
end
|
27
50
|
|
28
51
|
def dataflows
|
29
|
-
|
30
|
-
if j.respond_to?(:dataflows)
|
31
|
-
j.dataflows
|
32
|
-
else
|
33
|
-
[j.dataflow]
|
34
|
-
end
|
35
|
-
end
|
52
|
+
@dag.tsort.reverse.map(&:dataflow)
|
36
53
|
end
|
37
54
|
|
38
55
|
def state
|
@@ -48,7 +65,7 @@ module Rukawa
|
|
48
65
|
def to_dot(subgraph = false)
|
49
66
|
graphdef = subgraph ? "subgraph" : "digraph"
|
50
67
|
buf = %Q|#{graphdef} "#{subgraph ? "cluster_" : ""}#{name}" {\n|
|
51
|
-
buf += %Q{label = "#{
|
68
|
+
buf += %Q{label = "#{graph_label}";\n}
|
52
69
|
buf += Rukawa.config.graph.attrs
|
53
70
|
buf += Rukawa.config.graph.node.attrs
|
54
71
|
buf += "color = blue;\n" if subgraph
|
@@ -77,5 +94,15 @@ module Rukawa
|
|
77
94
|
def each(&block)
|
78
95
|
@dag.each(&block)
|
79
96
|
end
|
97
|
+
|
98
|
+
private
|
99
|
+
|
100
|
+
def graph_label
|
101
|
+
if @resume_job_classes.empty?
|
102
|
+
name
|
103
|
+
else
|
104
|
+
"#{name} resume from (#{@resume_job_classes.join(", ")})"
|
105
|
+
end
|
106
|
+
end
|
80
107
|
end
|
81
108
|
end
|
data/lib/rukawa/runner.rb
CHANGED
@@ -24,6 +24,7 @@ module Rukawa
|
|
24
24
|
Rukawa.logger.info("=== Finish Rukawa ===")
|
25
25
|
|
26
26
|
display_table unless batch_mode
|
27
|
+
puts "Finished #{@root_job_net.name} in #{@root_job_net.formatted_elapsed_time_from}"
|
27
28
|
|
28
29
|
errors = futures.map(&:reason).compact
|
29
30
|
|
@@ -41,7 +42,7 @@ module Rukawa
|
|
41
42
|
private
|
42
43
|
|
43
44
|
def display_table
|
44
|
-
table = Terminal::Table.new headings: ["Job", "Status"] do |t|
|
45
|
+
table = Terminal::Table.new headings: ["Job", "Status", "Elapsed Time"] do |t|
|
45
46
|
@root_job_net.each_with_index do |j|
|
46
47
|
table_row(t, j)
|
47
48
|
end
|
@@ -51,12 +52,12 @@ module Rukawa
|
|
51
52
|
|
52
53
|
def table_row(table, job, level = 0)
|
53
54
|
if job.is_a?(JobNet)
|
54
|
-
table << [Paint["#{" " * level}#{job.class}", :bold, :underline], job.state.colored]
|
55
|
+
table << [Paint["#{" " * level}#{job.class}", :bold, :underline], Paint[job.state.colored, :bold, :underline], Paint[job.formatted_elapsed_time_from, :bold, :underline]]
|
55
56
|
job.each do |inner_j|
|
56
57
|
table_row(table, inner_j, level + 1)
|
57
58
|
end
|
58
59
|
else
|
59
|
-
table << [Paint["#{" " * level}#{job.class}", :bold], job.state.colored]
|
60
|
+
table << [Paint["#{" " * level}#{job.class}", :bold], job.state.colored, job.formatted_elapsed_time_from]
|
60
61
|
end
|
61
62
|
end
|
62
63
|
end
|
data/lib/rukawa/state.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Rukawa::State
|
2
2
|
def self.get(name)
|
3
|
-
const_get(name.to_s.capitalize)
|
3
|
+
const_get(name.to_s.split("_").map(&:capitalize).join)
|
4
4
|
end
|
5
5
|
|
6
6
|
module BaseExt
|
@@ -15,6 +15,12 @@ module Rukawa::State
|
|
15
15
|
def merge(other)
|
16
16
|
other
|
17
17
|
end
|
18
|
+
|
19
|
+
%i(running? skipped? bypassed? error? aborted? waiting? finished?).each do |sym|
|
20
|
+
define_method(sym) do
|
21
|
+
false
|
22
|
+
end
|
23
|
+
end
|
18
24
|
end
|
19
25
|
|
20
26
|
module Running
|
@@ -27,6 +33,10 @@ module Rukawa::State
|
|
27
33
|
def self.merge(_other)
|
28
34
|
self
|
29
35
|
end
|
36
|
+
|
37
|
+
def self.running?
|
38
|
+
true
|
39
|
+
end
|
30
40
|
end
|
31
41
|
|
32
42
|
module Skipped
|
@@ -43,6 +53,22 @@ module Rukawa::State
|
|
43
53
|
other
|
44
54
|
end
|
45
55
|
end
|
56
|
+
|
57
|
+
def self.skipped?
|
58
|
+
true
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
module Bypassed
|
63
|
+
extend BaseExt
|
64
|
+
|
65
|
+
def self.color
|
66
|
+
:yellow
|
67
|
+
end
|
68
|
+
|
69
|
+
def self.bypassed?
|
70
|
+
true
|
71
|
+
end
|
46
72
|
end
|
47
73
|
|
48
74
|
module Error
|
@@ -59,6 +85,38 @@ module Rukawa::State
|
|
59
85
|
self
|
60
86
|
end
|
61
87
|
end
|
88
|
+
|
89
|
+
def self.error?
|
90
|
+
true
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
module Aborted
|
95
|
+
extend BaseExt
|
96
|
+
|
97
|
+
def name
|
98
|
+
"aborted"
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.color
|
102
|
+
:magenta
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.merge(other)
|
106
|
+
if other == Running || other == Error
|
107
|
+
other
|
108
|
+
else
|
109
|
+
self
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def self.error?
|
114
|
+
true
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.aborted?
|
118
|
+
true
|
119
|
+
end
|
62
120
|
end
|
63
121
|
|
64
122
|
module Waiting
|
@@ -67,6 +125,10 @@ module Rukawa::State
|
|
67
125
|
def self.color
|
68
126
|
:default
|
69
127
|
end
|
128
|
+
|
129
|
+
def self.waiting?
|
130
|
+
true
|
131
|
+
end
|
70
132
|
end
|
71
133
|
|
72
134
|
module Finished
|
@@ -75,5 +137,9 @@ module Rukawa::State
|
|
75
137
|
def self.color
|
76
138
|
:green
|
77
139
|
end
|
140
|
+
|
141
|
+
def self.finished?
|
142
|
+
true
|
143
|
+
end
|
78
144
|
end
|
79
145
|
end
|
data/lib/rukawa/version.rb
CHANGED
data/rukawa.gemspec
CHANGED
data/sample/result.dot
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
digraph "SampleJobNet" {
|
2
|
+
label = "SampleJobNet";
|
3
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
4
|
+
Job1 [style = filled,fillcolor = green];
|
5
|
+
Job2 [style = filled,fillcolor = green];
|
6
|
+
Job3 [style = filled,fillcolor = green];
|
7
|
+
Job4 [style = filled,fillcolor = green];
|
8
|
+
subgraph "cluster_InnerJobNet" {
|
9
|
+
label = "InnerJobNet";
|
10
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
11
|
+
color = blue;
|
12
|
+
InnerJob3 [style = filled,fillcolor = green];
|
13
|
+
InnerJob1 [style = filled,fillcolor = green];
|
14
|
+
InnerJob2 [style = filled,fillcolor = red];
|
15
|
+
"InnerJob1" -> "InnerJob2";
|
16
|
+
"InnerJob3" -> "InnerJob2";
|
17
|
+
}
|
18
|
+
Job8 [style = filled,fillcolor = magenta];
|
19
|
+
Job5 [style = filled,fillcolor = red];
|
20
|
+
Job6 [style = filled,fillcolor = magenta];
|
21
|
+
Job7 [style = filled,fillcolor = magenta];
|
22
|
+
subgraph "cluster_InnerJobNet2" {
|
23
|
+
label = "InnerJobNet2";
|
24
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
25
|
+
color = blue;
|
26
|
+
InnerJob4 [style = filled,fillcolor = green];
|
27
|
+
InnerJob5 [style = filled,fillcolor = yellow];
|
28
|
+
InnerJob6 [style = filled,fillcolor = yellow];
|
29
|
+
"InnerJob4" -> "InnerJob5";
|
30
|
+
"InnerJob4" -> "InnerJob6";
|
31
|
+
"InnerJob5" -> "InnerJob6";
|
32
|
+
}
|
33
|
+
subgraph "cluster_InnerJobNet3" {
|
34
|
+
label = "InnerJobNet3";
|
35
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
36
|
+
color = blue;
|
37
|
+
InnerJob7 [style = filled,fillcolor = magenta];
|
38
|
+
InnerJob8 [style = filled,fillcolor = magenta];
|
39
|
+
InnerJob9 [style = filled,fillcolor = magenta];
|
40
|
+
InnerJob10 [style = filled,fillcolor = magenta];
|
41
|
+
"InnerJob7" -> "InnerJob9";
|
42
|
+
"InnerJob8" -> "InnerJob9";
|
43
|
+
"InnerJob7" -> "InnerJob10";
|
44
|
+
"InnerJob8" -> "InnerJob10";
|
45
|
+
}
|
46
|
+
subgraph "cluster_InnerJobNet4" {
|
47
|
+
label = "InnerJobNet4";
|
48
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
49
|
+
color = blue;
|
50
|
+
InnerJob11 [style = filled,fillcolor = magenta];
|
51
|
+
InnerJob12 [style = filled,fillcolor = magenta];
|
52
|
+
InnerJob13 [style = filled,fillcolor = magenta];
|
53
|
+
subgraph "cluster_NestedJobNet" {
|
54
|
+
label = "NestedJobNet";
|
55
|
+
graph [rankdir = LR,nodesep = 0.8,concentrate = true];
|
56
|
+
color = blue;
|
57
|
+
NestedJob1 [style = filled,fillcolor = magenta];
|
58
|
+
NestedJob2 [style = filled,fillcolor = magenta];
|
59
|
+
"NestedJob1" -> "NestedJob2";
|
60
|
+
}
|
61
|
+
"InnerJob11" -> "NestedJob1";
|
62
|
+
"InnerJob12" -> "NestedJob1";
|
63
|
+
}
|
64
|
+
"Job1" -> "Job2";
|
65
|
+
"Job1" -> "Job3";
|
66
|
+
"Job2" -> "Job4";
|
67
|
+
"Job3" -> "Job4";
|
68
|
+
"Job3" -> "InnerJob3";
|
69
|
+
"Job3" -> "InnerJob1";
|
70
|
+
"InnerJob2" -> "Job8";
|
71
|
+
"Job3" -> "Job5";
|
72
|
+
"Job4" -> "Job6";
|
73
|
+
"Job5" -> "Job6";
|
74
|
+
"Job6" -> "Job7";
|
75
|
+
"Job4" -> "InnerJob4";
|
76
|
+
"Job8" -> "InnerJob7";
|
77
|
+
"Job8" -> "InnerJob8";
|
78
|
+
"Job7" -> "InnerJob7";
|
79
|
+
"Job7" -> "InnerJob8";
|
80
|
+
"InnerJob9" -> "InnerJob11";
|
81
|
+
"InnerJob9" -> "InnerJob12";
|
82
|
+
"InnerJob9" -> "InnerJob13";
|
83
|
+
"InnerJob10" -> "InnerJob11";
|
84
|
+
"InnerJob10" -> "InnerJob12";
|
85
|
+
"InnerJob10" -> "InnerJob13";
|
86
|
+
}
|
data/sample/result.png
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rukawa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- joker1007
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-03-
|
11
|
+
date: 2016-03-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '3.0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rspec-power_assert
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
111
125
|
description: Hyper simple job workflow engine
|
112
126
|
email:
|
113
127
|
- kakyoin.hierophant@gmail.com
|
@@ -140,6 +154,7 @@ files:
|
|
140
154
|
- sample/job_nets/sample_job_net.rb
|
141
155
|
- sample/jobnet.png
|
142
156
|
- sample/jobs/sample_job.rb
|
157
|
+
- sample/result.dot
|
143
158
|
- sample/result.png
|
144
159
|
- sample/rukawa.rb
|
145
160
|
homepage: https://github.com/joker1007/rukawa
|