rukawa 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +2 -0
- data/.travis.yml +6 -0
- data/Gemfile +4 -0
- data/README.md +252 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/rukawa +6 -0
- data/lib/rukawa.rb +33 -0
- data/lib/rukawa/abstract_job.rb +30 -0
- data/lib/rukawa/cli.rb +57 -0
- data/lib/rukawa/configuration.rb +18 -0
- data/lib/rukawa/dag.rb +84 -0
- data/lib/rukawa/errors.rb +3 -0
- data/lib/rukawa/job.rb +82 -0
- data/lib/rukawa/job_net.rb +78 -0
- data/lib/rukawa/runner.rb +63 -0
- data/lib/rukawa/state.rb +79 -0
- data/lib/rukawa/version.rb +3 -0
- data/rukawa.gemspec +29 -0
- data/sample/job_nets/sample_job_net.rb +41 -0
- data/sample/jobnet.png +0 -0
- data/sample/jobs/sample_job.rb +54 -0
- data/sample/result.png +0 -0
- metadata +167 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e06e9e3a6f053f844f030ddbdd6d41b740ae4565
|
4
|
+
data.tar.gz: a6cefb0f982de9eb9b26407d364394e87183204b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 876ed17a6dcc459f8b97ceda257d79c33e99fade9d16edf7c7ef8e11cf715c1f7f3619e3b1cac0045616a6109fe1edf1b89fce6f2d221b1d957458e84be6db40
|
7
|
+
data.tar.gz: f1b491287e0541225161fe018115c0c8a1cf148c72f58e35e92eb6c32308c8fb6cef6e119a2814360d4f22112e55e10334fc3e17ef66acd4ecdbdbee05f53349
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,252 @@
|
|
1
|
+
# Rukawa
|
2
|
+
[](https://travis-ci.org/joker1007/rukawa)
|
3
|
+
[](https://codeclimate.com/github/joker1007/rukawa)
|
4
|
+
|
5
|
+
Rukawa = (流川)
|
6
|
+
|
7
|
+
This gem is workflow engine and this is hyper simple.
|
8
|
+
Job is defined by Ruby class.
|
9
|
+
Dependency of each jobs is defined by Hash.
|
10
|
+
|
11
|
+
## Installation
|
12
|
+
|
13
|
+
Add this line to your application's Gemfile:
|
14
|
+
|
15
|
+
```ruby
|
16
|
+
gem 'rukawa'
|
17
|
+
```
|
18
|
+
|
19
|
+
And then execute:
|
20
|
+
|
21
|
+
$ bundle
|
22
|
+
|
23
|
+
Or install it yourself as:
|
24
|
+
|
25
|
+
$ gem install rukawa
|
26
|
+
|
27
|
+
## Usage
|
28
|
+
|
29
|
+
### Job Definition
|
30
|
+
|
31
|
+
```rb
|
32
|
+
# jobs/sample_job.rb
|
33
|
+
|
34
|
+
module ExecuteLog
|
35
|
+
def self.store
|
36
|
+
@store ||= {}
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class SampleJob < Rukawa::Job
|
41
|
+
def run
|
42
|
+
sleep rand(5)
|
43
|
+
ExecuteLog.store[self.class] = Time.now
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class Job1 < SampleJob
|
48
|
+
end
|
49
|
+
class Job2 < SampleJob
|
50
|
+
end
|
51
|
+
class Job3 < SampleJob
|
52
|
+
end
|
53
|
+
class Job4 < SampleJob
|
54
|
+
end
|
55
|
+
class Job5 < SampleJob
|
56
|
+
def run
|
57
|
+
raise "job5 error"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
class Job6 < SampleJob
|
61
|
+
end
|
62
|
+
class Job7 < SampleJob
|
63
|
+
end
|
64
|
+
class Job8 < SampleJob
|
65
|
+
end
|
66
|
+
|
67
|
+
class InnerJob1 < SampleJob
|
68
|
+
end
|
69
|
+
|
70
|
+
class InnerJob2 < SampleJob
|
71
|
+
def run
|
72
|
+
raise "inner job2 error"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
class InnerJob3 < SampleJob
|
77
|
+
end
|
78
|
+
|
79
|
+
class InnerJob4 < SampleJob
|
80
|
+
end
|
81
|
+
|
82
|
+
class InnerJob5 < SampleJob
|
83
|
+
add_skip_rule ->(job) { job.is_a?(SampleJob) }
|
84
|
+
end
|
85
|
+
|
86
|
+
class InnerJob6 < SampleJob
|
87
|
+
end
|
88
|
+
```
|
89
|
+
|
90
|
+
### JobNet Definition
|
91
|
+
```rb
|
92
|
+
# job_nets/sample_job_net.rb
|
93
|
+
|
94
|
+
class InnerJobNet < Rukawa::JobNet
|
95
|
+
class << self
|
96
|
+
def dependencies
|
97
|
+
{
|
98
|
+
InnerJob3 => [],
|
99
|
+
InnerJob1 => [],
|
100
|
+
InnerJob2 => [InnerJob1, InnerJob3],
|
101
|
+
}
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
class InnerJobNet2 < Rukawa::JobNet
|
107
|
+
class << self
|
108
|
+
def dependencies
|
109
|
+
{
|
110
|
+
InnerJob4 => [],
|
111
|
+
InnerJob5 => [],
|
112
|
+
InnerJob6 => [InnerJob4, InnerJob5],
|
113
|
+
}
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
class SampleJobNet < Rukawa::JobNet
|
119
|
+
class << self
|
120
|
+
def dependencies
|
121
|
+
{
|
122
|
+
Job1 => [],
|
123
|
+
Job2 => [Job1], Job3 => [Job1],
|
124
|
+
Job4 => [Job2, Job3],
|
125
|
+
InnerJobNet => [Job3],
|
126
|
+
Job8 => [InnerJobNet],
|
127
|
+
Job5 => [Job3],
|
128
|
+
Job6 => [Job4, Job5],
|
129
|
+
Job7 => [Job6],
|
130
|
+
InnerJobNet2 => [Job4],
|
131
|
+
}
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
```
|
136
|
+
|
137
|
+

|
138
|
+
|
139
|
+
### Execution
|
140
|
+
|
141
|
+
```
|
142
|
+
% cd rukawa/sample
|
143
|
+
|
144
|
+
# load ./jobs/**/*.rb, ./job_net/**/*.rb automatically
|
145
|
+
% bundle exec rukawa run SampleJobNet -r 5 -d result.dot
|
146
|
+
+--------------+---------+
|
147
|
+
| Job | Status |
|
148
|
+
+--------------+---------+
|
149
|
+
| Job1 | waiting |
|
150
|
+
| Job2 | waiting |
|
151
|
+
| Job3 | waiting |
|
152
|
+
| Job4 | waiting |
|
153
|
+
| InnerJobNet | waiting |
|
154
|
+
| InnerJob3 | waiting |
|
155
|
+
| InnerJob1 | waiting |
|
156
|
+
| InnerJob2 | waiting |
|
157
|
+
| Job8 | waiting |
|
158
|
+
| Job5 | waiting |
|
159
|
+
| Job6 | waiting |
|
160
|
+
| Job7 | waiting |
|
161
|
+
| InnerJobNet2 | waiting |
|
162
|
+
| InnerJob4 | waiting |
|
163
|
+
| InnerJob5 | waiting |
|
164
|
+
| InnerJob6 | waiting |
|
165
|
+
+--------------+---------+
|
166
|
+
+--------------+----------+
|
167
|
+
| Job | Status |
|
168
|
+
+--------------+----------+
|
169
|
+
| Job1 | finished |
|
170
|
+
| Job2 | finished |
|
171
|
+
| Job3 | finished |
|
172
|
+
| Job4 | finished |
|
173
|
+
| InnerJobNet | running |
|
174
|
+
| InnerJob3 | running |
|
175
|
+
| InnerJob1 | running |
|
176
|
+
| InnerJob2 | waiting |
|
177
|
+
| Job8 | waiting |
|
178
|
+
| Job5 | error |
|
179
|
+
| Job6 | error |
|
180
|
+
| Job7 | error |
|
181
|
+
| InnerJobNet2 | running |
|
182
|
+
| InnerJob4 | running |
|
183
|
+
| InnerJob5 | skipped |
|
184
|
+
| InnerJob6 | waiting |
|
185
|
+
+--------------+----------+
|
186
|
+
+--------------+----------+
|
187
|
+
| Job | Status |
|
188
|
+
+--------------+----------+
|
189
|
+
| Job1 | finished |
|
190
|
+
| Job2 | finished |
|
191
|
+
| Job3 | finished |
|
192
|
+
| Job4 | finished |
|
193
|
+
| InnerJobNet | error |
|
194
|
+
| InnerJob3 | finished |
|
195
|
+
| InnerJob1 | finished |
|
196
|
+
| InnerJob2 | error |
|
197
|
+
| Job8 | error |
|
198
|
+
| Job5 | error |
|
199
|
+
| Job6 | error |
|
200
|
+
| Job7 | error |
|
201
|
+
| InnerJobNet2 | finished |
|
202
|
+
| InnerJob4 | finished |
|
203
|
+
| InnerJob5 | skipped |
|
204
|
+
| InnerJob6 | skipped |
|
205
|
+
+--------------+----------+
|
206
|
+
|
207
|
+
# generate result graph image
|
208
|
+
% dot -Tpng -o result.png result.dot
|
209
|
+
```
|
210
|
+
|
211
|
+

|
212
|
+
|
213
|
+
|
214
|
+
### Output jobnet graph (dot file)
|
215
|
+
|
216
|
+
```
|
217
|
+
% bundle exec rukawa graph -o SampleJobNet.dot SampleJobNet
|
218
|
+
% dot -Tpng -o SampleJobNet.png SampleJobNet.dot
|
219
|
+
```
|
220
|
+
|
221
|
+
### help
|
222
|
+
```
|
223
|
+
% bundle exec rukawa help run
|
224
|
+
Usage:
|
225
|
+
rukawa run JOB_NET_NAME
|
226
|
+
|
227
|
+
Options:
|
228
|
+
-c, [--concurrency=N] # Default: cpu count
|
229
|
+
[--variables=key:value]
|
230
|
+
[--job-dirs=one two three] # Load job directories
|
231
|
+
-b, [--batch], [--no-batch] # If batch mode, not display running status
|
232
|
+
-l, [--log=LOG]
|
233
|
+
# Default: ./rukawa.log
|
234
|
+
-d, [--dot=DOT] # Output job status by dot format
|
235
|
+
-r, [--refresh-interval=N] # Refresh interval for running status information
|
236
|
+
# Default: 3
|
237
|
+
```
|
238
|
+
|
239
|
+
## ToDo
|
240
|
+
- Write more tests
|
241
|
+
- Enable use variables
|
242
|
+
|
243
|
+
## Development
|
244
|
+
|
245
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. Run `bundle exec rukawa` to use the gem in this directory, ignoring other installed copies of this gem.
|
246
|
+
|
247
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
248
|
+
|
249
|
+
## Contributing
|
250
|
+
|
251
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/joker1007/rukawa.
|
252
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "rukawa"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
data/exe/rukawa
ADDED
data/lib/rukawa.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
require "concurrent"
|
2
|
+
|
3
|
+
module Rukawa
|
4
|
+
class << self
|
5
|
+
def logger
|
6
|
+
@logger ||= Logger.new(config.log_file)
|
7
|
+
end
|
8
|
+
|
9
|
+
def store
|
10
|
+
@store ||= Concurrent::Hash.new
|
11
|
+
end
|
12
|
+
|
13
|
+
def configure
|
14
|
+
yield config
|
15
|
+
end
|
16
|
+
|
17
|
+
def config
|
18
|
+
Configuration.instance
|
19
|
+
end
|
20
|
+
|
21
|
+
def executor
|
22
|
+
@executor ||= Concurrent::FixedThreadPool.new(config.concurrency)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
require "rukawa/version"
|
28
|
+
require 'rukawa/errors'
|
29
|
+
require 'rukawa/state'
|
30
|
+
require 'rukawa/configuration'
|
31
|
+
require 'rukawa/job_net'
|
32
|
+
require 'rukawa/job'
|
33
|
+
require 'rukawa/dag'
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'set'
|
2
|
+
require 'rukawa/state'
|
3
|
+
|
4
|
+
module Rukawa
|
5
|
+
class AbstractJob
|
6
|
+
class << self
|
7
|
+
def skip_rules
|
8
|
+
@skip_rules ||= []
|
9
|
+
end
|
10
|
+
|
11
|
+
def add_skip_rule(callable_or_symbol)
|
12
|
+
skip_rules.push(callable_or_symbol)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def name
|
17
|
+
self.class.to_s
|
18
|
+
end
|
19
|
+
|
20
|
+
def skip?
|
21
|
+
skip_rules.inject(false) do |cond, rule|
|
22
|
+
cond || rule.is_a?(Symbol) ? method(rule).call : rule.call(self)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def skip_rules
|
27
|
+
self.class.skip_rules
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/rukawa/cli.rb
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'rukawa/runner'
|
3
|
+
|
4
|
+
module Rukawa
|
5
|
+
class Cli < Thor
|
6
|
+
desc "run JOB_NET_NAME", "Run jobnet"
|
7
|
+
map "run" => "_run"
|
8
|
+
method_option :concurrency, aliases: "-c", type: :numeric, default: nil, desc: "Default: cpu count"
|
9
|
+
method_option :variables, type: :hash, default: {}
|
10
|
+
method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
|
11
|
+
method_option :batch, aliases: "-b", type: :boolean, default: false, desc: "If batch mode, not display running status"
|
12
|
+
method_option :log, aliases: "-l", type: :string, default: "./rukawa.log"
|
13
|
+
method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
|
14
|
+
method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
|
15
|
+
def _run(job_net_name)
|
16
|
+
Rukawa.configure do |c|
|
17
|
+
c.log_file = options[:log]
|
18
|
+
c.concurrency = options[:concurrency] if options[:concurrency]
|
19
|
+
end
|
20
|
+
load_job_definitions
|
21
|
+
|
22
|
+
job_net_class = Object.const_get(job_net_name)
|
23
|
+
job_net = job_net_class.new(options[:variables])
|
24
|
+
result = Runner.run(job_net, options[:batch], options[:refresh_interval])
|
25
|
+
|
26
|
+
if options[:dot]
|
27
|
+
job_net.output_dot(options[:dot])
|
28
|
+
end
|
29
|
+
|
30
|
+
exit 1 unless result
|
31
|
+
end
|
32
|
+
|
33
|
+
desc "graph JOB_NET_NAME", "Output jobnet graph"
|
34
|
+
method_option :job_dirs, type: :array, default: []
|
35
|
+
method_option :output, aliases: "-o", type: :string, required: true
|
36
|
+
def graph(job_net_name)
|
37
|
+
load_job_definitions
|
38
|
+
|
39
|
+
job_net_class = Object.const_get(job_net_name)
|
40
|
+
job_net = job_net_class.new(options[:variables])
|
41
|
+
job_net.output_dot(options[:output])
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def default_job_dirs
|
47
|
+
[File.join(Dir.pwd, "job_nets"), File.join(Dir.pwd, "jobs")]
|
48
|
+
end
|
49
|
+
|
50
|
+
def load_job_definitions
|
51
|
+
job_dirs = (default_job_dirs + options[:job_dirs]).map { |d| File.expand_path(d) }.uniq
|
52
|
+
job_dirs.each do |dir|
|
53
|
+
Dir.glob(File.join(dir, "**/*.rb")) { |f| load f }
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
require 'ostruct'
|
3
|
+
require 'delegate'
|
4
|
+
require 'concurrent'
|
5
|
+
|
6
|
+
module Rukawa
|
7
|
+
class Configuration < Delegator
|
8
|
+
include Singleton
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@config = OpenStruct.new(log_file: "./rukawa.log", concurrency: Concurrent.processor_count)
|
12
|
+
end
|
13
|
+
|
14
|
+
def __getobj__
|
15
|
+
@config
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
data/lib/rukawa/dag.rb
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'set'
|
2
|
+
|
3
|
+
module Rukawa
|
4
|
+
class Dag
|
5
|
+
include Enumerable
|
6
|
+
|
7
|
+
attr_reader :jobs, :edges
|
8
|
+
|
9
|
+
def initialize(job_net, dependencies)
|
10
|
+
deps = tsortable_hash(dependencies).tsort
|
11
|
+
@jobs = Set.new
|
12
|
+
@edges = Set.new
|
13
|
+
|
14
|
+
deps.each do |job_class|
|
15
|
+
job = job_class.new(job_net)
|
16
|
+
@jobs << job
|
17
|
+
|
18
|
+
dependencies[job_class].each do |depend_job_class|
|
19
|
+
depend_job = @jobs.find { |j| j.instance_of?(depend_job_class) }
|
20
|
+
|
21
|
+
depend_job.nodes_as_from.each do |from|
|
22
|
+
job.nodes_as_to.each do |to|
|
23
|
+
edge = Edge.new(from, to)
|
24
|
+
@edges << edge
|
25
|
+
from.out_goings << edge
|
26
|
+
to.in_comings << edge
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def each
|
34
|
+
if block_given?
|
35
|
+
@jobs.each { |j| yield j }
|
36
|
+
else
|
37
|
+
@jobs.each
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def roots
|
42
|
+
select(&:root?)
|
43
|
+
end
|
44
|
+
|
45
|
+
def leaves
|
46
|
+
select(&:leaf?)
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def tsortable_hash(hash)
|
52
|
+
class << hash
|
53
|
+
include TSort
|
54
|
+
alias :tsort_each_node :each_key
|
55
|
+
def tsort_each_child(node, &block)
|
56
|
+
fetch(node).each(&block)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
hash
|
60
|
+
end
|
61
|
+
|
62
|
+
class Edge
|
63
|
+
attr_reader :from, :to, :cluster
|
64
|
+
|
65
|
+
def initialize(from, to, cluster = nil)
|
66
|
+
@from, @to, @cluster = from, to, cluster
|
67
|
+
end
|
68
|
+
|
69
|
+
def inspect
|
70
|
+
"#{@from.name} -> #{@to.name}"
|
71
|
+
end
|
72
|
+
|
73
|
+
def ==(edge)
|
74
|
+
return false unless edge.is_a?(Edge)
|
75
|
+
from == edge.from && to == edge.to
|
76
|
+
end
|
77
|
+
alias :eql? :==
|
78
|
+
|
79
|
+
def hash
|
80
|
+
[from, to].hash
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
data/lib/rukawa/job.rb
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'concurrent'
|
2
|
+
require 'rukawa/abstract_job'
|
3
|
+
|
4
|
+
module Rukawa
|
5
|
+
class Job < AbstractJob
|
6
|
+
attr_accessor :in_comings, :out_goings
|
7
|
+
attr_reader :state
|
8
|
+
|
9
|
+
def initialize(job_net)
|
10
|
+
@job_net = job_net
|
11
|
+
@in_comings = Set.new
|
12
|
+
@out_goings = Set.new
|
13
|
+
set_state(:waiting)
|
14
|
+
end
|
15
|
+
|
16
|
+
def root?
|
17
|
+
in_comings.empty?
|
18
|
+
end
|
19
|
+
|
20
|
+
def leaf?
|
21
|
+
out_goings.empty?
|
22
|
+
end
|
23
|
+
|
24
|
+
def dataflow
|
25
|
+
return @dataflow if @dataflow
|
26
|
+
|
27
|
+
@dataflow = Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
|
28
|
+
begin
|
29
|
+
raise DependentJobFailure unless results.all? { |r| !r.nil? }
|
30
|
+
|
31
|
+
if skip? || @job_net.skip? || results.any? { |r| r == Rukawa::State.get(:skipped) }
|
32
|
+
Rukawa.logger.info("Skip #{self.class}")
|
33
|
+
set_state(:skipped)
|
34
|
+
else
|
35
|
+
Rukawa.logger.info("Start #{self.class}")
|
36
|
+
set_state(:running)
|
37
|
+
run
|
38
|
+
Rukawa.logger.info("Finish #{self.class}")
|
39
|
+
set_state(:finished)
|
40
|
+
end
|
41
|
+
rescue => e
|
42
|
+
Rukawa.logger.error("Error #{self.class} by #{e}")
|
43
|
+
set_state(:error)
|
44
|
+
raise
|
45
|
+
end
|
46
|
+
|
47
|
+
@state
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def run
|
52
|
+
end
|
53
|
+
|
54
|
+
def nodes_as_from
|
55
|
+
[self]
|
56
|
+
end
|
57
|
+
alias :nodes_as_to :nodes_as_from
|
58
|
+
|
59
|
+
def to_dot_def
|
60
|
+
if state == Rukawa::State::Waiting
|
61
|
+
""
|
62
|
+
else
|
63
|
+
"#{name} [color = #{state.color}];\n" unless state == Rukawa::State::Waiting
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def depend_dataflows
|
70
|
+
in_comings.map { |edge| edge.from.dataflow }
|
71
|
+
end
|
72
|
+
|
73
|
+
def set_state(name)
|
74
|
+
@state = Rukawa::State.get(name)
|
75
|
+
end
|
76
|
+
|
77
|
+
def store(key, value)
|
78
|
+
Rukawa.store[self.class] ||= Concurrent::Hash.new
|
79
|
+
Rukawa.store[self.class][key] = value
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'rukawa/abstract_job'
|
2
|
+
|
3
|
+
module Rukawa
|
4
|
+
class JobNet < AbstractJob
|
5
|
+
include Enumerable
|
6
|
+
attr_reader :dag
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def dependencies
|
10
|
+
raise NotImplementedError, "Please override"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(variables = {})
|
15
|
+
@variables = variables
|
16
|
+
@dag = Dag.new(self, self.class.dependencies)
|
17
|
+
end
|
18
|
+
|
19
|
+
def dataflows
|
20
|
+
flat_map do |j|
|
21
|
+
if j.respond_to?(:dataflows)
|
22
|
+
j.dataflows
|
23
|
+
else
|
24
|
+
[j.dataflow]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def state
|
30
|
+
inject(Rukawa::State::Waiting) do |state, j|
|
31
|
+
state.merge(j.state)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def output_dot(filename)
|
36
|
+
File.open(filename, 'w') { |f| f.write(to_dot) }
|
37
|
+
end
|
38
|
+
|
39
|
+
def nodes_as_from
|
40
|
+
leaves
|
41
|
+
end
|
42
|
+
|
43
|
+
def nodes_as_to
|
44
|
+
roots
|
45
|
+
end
|
46
|
+
|
47
|
+
def to_dot(subgraph = false)
|
48
|
+
graphdef = subgraph ? "subgraph" : "digraph"
|
49
|
+
buf = %Q|#{graphdef} "#{subgraph ? "cluster_" : ""}#{name}" {\n|
|
50
|
+
buf += %Q{label = "#{name}";\n}
|
51
|
+
buf += "color = blue;\n" if subgraph
|
52
|
+
dag.each do |j|
|
53
|
+
buf += j.to_dot_def
|
54
|
+
end
|
55
|
+
|
56
|
+
dag.edges.each do |edge|
|
57
|
+
buf += %Q|"#{edge.from.name}" -> "#{edge.to.name}";\n|
|
58
|
+
end
|
59
|
+
buf += "}\n"
|
60
|
+
end
|
61
|
+
|
62
|
+
def to_dot_def
|
63
|
+
to_dot(true)
|
64
|
+
end
|
65
|
+
|
66
|
+
def roots
|
67
|
+
@dag.roots
|
68
|
+
end
|
69
|
+
|
70
|
+
def leaves
|
71
|
+
@dag.leaves
|
72
|
+
end
|
73
|
+
|
74
|
+
def each(&block)
|
75
|
+
@dag.each(&block)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'terminal-table'
|
2
|
+
require 'paint'
|
3
|
+
|
4
|
+
module Rukawa
|
5
|
+
class Runner
|
6
|
+
DEFAULT_REFRESH_INTERVAL = 3
|
7
|
+
|
8
|
+
def self.run(job_net, batch_mode = false, refresh_interval = DEFAULT_REFRESH_INTERVAL)
|
9
|
+
new(job_net).run(batch_mode, refresh_interval)
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize(root_job_net)
|
13
|
+
@root_job_net = root_job_net
|
14
|
+
@errors = []
|
15
|
+
end
|
16
|
+
|
17
|
+
def run(batch_mode = false, refresh_interval = DEFAULT_REFRESH_INTERVAL)
|
18
|
+
Rukawa.logger.info("=== Start Rukawa ===")
|
19
|
+
futures = @root_job_net.dataflows.each(&:execute)
|
20
|
+
until futures.all?(&:complete?)
|
21
|
+
display_table unless batch_mode
|
22
|
+
sleep refresh_interval
|
23
|
+
end
|
24
|
+
Rukawa.logger.info("=== Finish Rukawa ===")
|
25
|
+
|
26
|
+
display_table unless batch_mode
|
27
|
+
|
28
|
+
errors = futures.map(&:reason).compact
|
29
|
+
|
30
|
+
unless errors.empty?
|
31
|
+
errors.each do |err|
|
32
|
+
next if err.is_a?(DependentJobFailure)
|
33
|
+
Rukawa.logger.error(err)
|
34
|
+
end
|
35
|
+
return false
|
36
|
+
end
|
37
|
+
|
38
|
+
true
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def display_table
|
44
|
+
table = Terminal::Table.new headings: ["Job", "Status"] do |t|
|
45
|
+
@root_job_net.each_with_index do |j|
|
46
|
+
table_row(t, j)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
puts table
|
50
|
+
end
|
51
|
+
|
52
|
+
def table_row(table, job, level = 0)
|
53
|
+
if job.is_a?(JobNet)
|
54
|
+
table << [Paint["#{" " * level}#{job.class}", :bold, :underline], job.state.colored]
|
55
|
+
job.each do |inner_j|
|
56
|
+
table_row(table, inner_j, level + 1)
|
57
|
+
end
|
58
|
+
else
|
59
|
+
table << [Paint["#{" " * level}#{job.class}", :bold], job.state.colored]
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
data/lib/rukawa/state.rb
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
module Rukawa::State
|
2
|
+
def self.get(name)
|
3
|
+
const_get(name.to_s.capitalize)
|
4
|
+
end
|
5
|
+
|
6
|
+
module BaseExt
|
7
|
+
def state_name
|
8
|
+
@state_name ||= to_s.gsub(/Rukawa::State::/, "").downcase
|
9
|
+
end
|
10
|
+
|
11
|
+
def colored
|
12
|
+
Paint[state_name.to_s, color]
|
13
|
+
end
|
14
|
+
|
15
|
+
def merge(other)
|
16
|
+
other
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
module Running
|
21
|
+
extend BaseExt
|
22
|
+
|
23
|
+
def self.color
|
24
|
+
:cyan
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.merge(_other)
|
28
|
+
self
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
module Skipped
|
33
|
+
extend BaseExt
|
34
|
+
|
35
|
+
def self.color
|
36
|
+
:yellow
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.merge(other)
|
40
|
+
if other == Finished
|
41
|
+
self
|
42
|
+
else
|
43
|
+
other
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
module Error
|
49
|
+
extend BaseExt
|
50
|
+
|
51
|
+
def self.color
|
52
|
+
:red
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.merge(other)
|
56
|
+
if other == Running
|
57
|
+
other
|
58
|
+
else
|
59
|
+
self
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
module Waiting
|
65
|
+
extend BaseExt
|
66
|
+
|
67
|
+
def self.color
|
68
|
+
:default
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
module Finished
|
73
|
+
extend BaseExt
|
74
|
+
|
75
|
+
def self.color
|
76
|
+
:green
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
data/rukawa.gemspec
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'rukawa/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "rukawa"
|
8
|
+
spec.version = Rukawa::VERSION
|
9
|
+
spec.authors = ["joker1007"]
|
10
|
+
spec.email = ["kakyoin.hierophant@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Hyper simple job workflow engine}
|
13
|
+
spec.description = %q{Hyper simple job workflow engine}
|
14
|
+
spec.homepage = "https://github.com/joker1007/rukawa"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
|
+
spec.bindir = "exe"
|
18
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_runtime_dependency "concurrent-ruby"
|
22
|
+
spec.add_runtime_dependency "thor"
|
23
|
+
spec.add_runtime_dependency "terminal-table"
|
24
|
+
spec.add_runtime_dependency "paint"
|
25
|
+
|
26
|
+
spec.add_development_dependency "bundler", "~> 1.11"
|
27
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
28
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
29
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
class InnerJobNet < Rukawa::JobNet
|
2
|
+
class << self
|
3
|
+
def dependencies
|
4
|
+
{
|
5
|
+
InnerJob3 => [],
|
6
|
+
InnerJob1 => [],
|
7
|
+
InnerJob2 => [InnerJob1, InnerJob3],
|
8
|
+
}
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class InnerJobNet2 < Rukawa::JobNet
|
14
|
+
class << self
|
15
|
+
def dependencies
|
16
|
+
{
|
17
|
+
InnerJob4 => [],
|
18
|
+
InnerJob5 => [],
|
19
|
+
InnerJob6 => [InnerJob4, InnerJob5],
|
20
|
+
}
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class SampleJobNet < Rukawa::JobNet
|
26
|
+
class << self
|
27
|
+
def dependencies
|
28
|
+
{
|
29
|
+
Job1 => [],
|
30
|
+
Job2 => [Job1], Job3 => [Job1],
|
31
|
+
Job4 => [Job2, Job3],
|
32
|
+
InnerJobNet => [Job3],
|
33
|
+
Job8 => [InnerJobNet],
|
34
|
+
Job5 => [Job3],
|
35
|
+
Job6 => [Job4, Job5],
|
36
|
+
Job7 => [Job6],
|
37
|
+
InnerJobNet2 => [Job4],
|
38
|
+
}
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/sample/jobnet.png
ADDED
Binary file
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module ExecuteLog
|
2
|
+
def self.store
|
3
|
+
@store ||= {}
|
4
|
+
end
|
5
|
+
end
|
6
|
+
|
7
|
+
class SampleJob < Rukawa::Job
|
8
|
+
def run
|
9
|
+
sleep rand(5)
|
10
|
+
ExecuteLog.store[self.class] = Time.now
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class Job1 < SampleJob
|
15
|
+
end
|
16
|
+
class Job2 < SampleJob
|
17
|
+
end
|
18
|
+
class Job3 < SampleJob
|
19
|
+
end
|
20
|
+
class Job4 < SampleJob
|
21
|
+
end
|
22
|
+
class Job5 < SampleJob
|
23
|
+
def run
|
24
|
+
raise "job5 error"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
class Job6 < SampleJob
|
28
|
+
end
|
29
|
+
class Job7 < SampleJob
|
30
|
+
end
|
31
|
+
class Job8 < SampleJob
|
32
|
+
end
|
33
|
+
|
34
|
+
class InnerJob1 < SampleJob
|
35
|
+
end
|
36
|
+
|
37
|
+
class InnerJob2 < SampleJob
|
38
|
+
def run
|
39
|
+
raise "inner job2 error"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
class InnerJob3 < SampleJob
|
44
|
+
end
|
45
|
+
|
46
|
+
class InnerJob4 < SampleJob
|
47
|
+
end
|
48
|
+
|
49
|
+
class InnerJob5 < SampleJob
|
50
|
+
add_skip_rule ->(job) { job.is_a?(SampleJob) }
|
51
|
+
end
|
52
|
+
|
53
|
+
class InnerJob6 < SampleJob
|
54
|
+
end
|
data/sample/result.png
ADDED
Binary file
|
metadata
ADDED
@@ -0,0 +1,167 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rukawa
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- joker1007
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-03-21 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: concurrent-ruby
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: thor
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: terminal-table
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: paint
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: bundler
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.11'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.11'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rake
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '10.0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '10.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rspec
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '3.0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '3.0'
|
111
|
+
description: Hyper simple job workflow engine
|
112
|
+
email:
|
113
|
+
- kakyoin.hierophant@gmail.com
|
114
|
+
executables:
|
115
|
+
- rukawa
|
116
|
+
extensions: []
|
117
|
+
extra_rdoc_files: []
|
118
|
+
files:
|
119
|
+
- ".gitignore"
|
120
|
+
- ".rspec"
|
121
|
+
- ".travis.yml"
|
122
|
+
- Gemfile
|
123
|
+
- README.md
|
124
|
+
- Rakefile
|
125
|
+
- bin/console
|
126
|
+
- bin/setup
|
127
|
+
- exe/rukawa
|
128
|
+
- lib/rukawa.rb
|
129
|
+
- lib/rukawa/abstract_job.rb
|
130
|
+
- lib/rukawa/cli.rb
|
131
|
+
- lib/rukawa/configuration.rb
|
132
|
+
- lib/rukawa/dag.rb
|
133
|
+
- lib/rukawa/errors.rb
|
134
|
+
- lib/rukawa/job.rb
|
135
|
+
- lib/rukawa/job_net.rb
|
136
|
+
- lib/rukawa/runner.rb
|
137
|
+
- lib/rukawa/state.rb
|
138
|
+
- lib/rukawa/version.rb
|
139
|
+
- rukawa.gemspec
|
140
|
+
- sample/job_nets/sample_job_net.rb
|
141
|
+
- sample/jobnet.png
|
142
|
+
- sample/jobs/sample_job.rb
|
143
|
+
- sample/result.png
|
144
|
+
homepage: https://github.com/joker1007/rukawa
|
145
|
+
licenses: []
|
146
|
+
metadata: {}
|
147
|
+
post_install_message:
|
148
|
+
rdoc_options: []
|
149
|
+
require_paths:
|
150
|
+
- lib
|
151
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
152
|
+
requirements:
|
153
|
+
- - ">="
|
154
|
+
- !ruby/object:Gem::Version
|
155
|
+
version: '0'
|
156
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
157
|
+
requirements:
|
158
|
+
- - ">="
|
159
|
+
- !ruby/object:Gem::Version
|
160
|
+
version: '0'
|
161
|
+
requirements: []
|
162
|
+
rubyforge_project:
|
163
|
+
rubygems_version: 2.5.1
|
164
|
+
signing_key:
|
165
|
+
specification_version: 4
|
166
|
+
summary: Hyper simple job workflow engine
|
167
|
+
test_files: []
|