tumugi 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +23 -2
- data/README.md +27 -1
- data/examples/fail_first_task.rb +20 -0
- data/examples/fail_intermediate_task.rb +21 -0
- data/examples/fail_last_task.rb +21 -0
- data/examples/task_parameter.rb +31 -0
- data/examples/tumugi_config.rb +4 -0
- data/lib/tumugi/application.rb +3 -0
- data/lib/tumugi/cli.rb +8 -0
- data/lib/tumugi/command/run.rb +48 -13
- data/lib/tumugi/config.rb +2 -1
- data/lib/tumugi/mixin/parameterizable.rb +85 -0
- data/lib/tumugi/parameter/converter.rb +58 -0
- data/lib/tumugi/parameter/error.rb +6 -0
- data/lib/tumugi/parameter/parameter.rb +81 -0
- data/lib/tumugi/parameter/parameter_proxy.rb +34 -0
- data/lib/tumugi/task.rb +7 -0
- data/lib/tumugi/task_definition.rb +36 -1
- data/lib/tumugi/version.rb +1 -1
- data/tumugi.gemspec +1 -0
- metadata +26 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c2ac1c29b7d48bd8d12b5f62b3cd177ca1f34109
|
|
4
|
+
data.tar.gz: 81f64f2898be4b90207549d547260fb333858c2d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ef38cfe6332a2d809899750b958683ccaef7890b563b7352150c36c75ae46051238e7c3235b64dbe407eb86744cb35b178e3a6a6b97884d9c842dfa54058c86f
|
|
7
|
+
data.tar.gz: 7d6a1fbcb68dafacbaf3adb0c60da3c132b345d1623ef3dd7ec34057a2bc0796a6af880c10d6c5ab7c3554a80ecda20f040ad5400e3c96865931c57b64abf0ca
|
data/CHANGELOG.md
CHANGED
|
@@ -1,12 +1,31 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
+
## [v0.3.0](https://github.com/tumugi/tumugi/tree/v0.3.0) (2016-05-07)
|
|
4
|
+
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.2.0...v0.3.0)
|
|
5
|
+
|
|
6
|
+
**Implemented enhancements:**
|
|
7
|
+
|
|
8
|
+
- Show result report [\#23](https://github.com/tumugi/tumugi/issues/23)
|
|
9
|
+
- Support parameter [\#21](https://github.com/tumugi/tumugi/issues/21)
|
|
10
|
+
|
|
11
|
+
**Fixed bugs:**
|
|
12
|
+
|
|
13
|
+
- Fix workflow dead locked when some task failed [\#25](https://github.com/tumugi/tumugi/pull/25) ([hakobera](https://github.com/hakobera))
|
|
14
|
+
|
|
15
|
+
**Merged pull requests:**
|
|
16
|
+
|
|
17
|
+
- Update README [\#26](https://github.com/tumugi/tumugi/pull/26) ([hakobera](https://github.com/hakobera))
|
|
18
|
+
- Implement workflow run result report [\#24](https://github.com/tumugi/tumugi/pull/24) ([hakobera](https://github.com/hakobera))
|
|
19
|
+
- Implement parameter feature [\#22](https://github.com/tumugi/tumugi/pull/22) ([hakobera](https://github.com/hakobera))
|
|
20
|
+
|
|
3
21
|
## [v0.2.0](https://github.com/tumugi/tumugi/tree/v0.2.0) (2016-05-02)
|
|
4
22
|
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.1.0...v0.2.0)
|
|
5
23
|
|
|
6
24
|
**Implemented enhancements:**
|
|
7
25
|
|
|
8
|
-
-
|
|
9
|
-
-
|
|
26
|
+
- \[Breaking Change\] Support plugin [\#15](https://github.com/tumugi/tumugi/issues/15)
|
|
27
|
+
- Add required\_ruby\_version to gemspec [\#19](https://github.com/tumugi/tumugi/pull/19) ([hakobera](https://github.com/hakobera))
|
|
28
|
+
- \[Breaking Change\] Change eval scope of output, run method [\#14](https://github.com/tumugi/tumugi/pull/14) ([hakobera](https://github.com/hakobera))
|
|
10
29
|
- \[Breaking Change\] Add Task\#logger and Task\#log method [\#12](https://github.com/tumugi/tumugi/pull/12) ([hakobera](https://github.com/hakobera))
|
|
11
30
|
- Update command description / Set file options mandatory [\#11](https://github.com/tumugi/tumugi/pull/11) ([hakobera](https://github.com/hakobera))
|
|
12
31
|
|
|
@@ -18,6 +37,8 @@
|
|
|
18
37
|
|
|
19
38
|
- Use bundler cache on travis [\#17](https://github.com/tumugi/tumugi/pull/17) ([hakobera](https://github.com/hakobera))
|
|
20
39
|
- Add gem version badge to README [\#16](https://github.com/tumugi/tumugi/pull/16) ([hakobera](https://github.com/hakobera))
|
|
40
|
+
- Prepare for release v0.2.0 [\#20](https://github.com/tumugi/tumugi/pull/20) ([hakobera](https://github.com/hakobera))
|
|
41
|
+
- \[Breaking Change\] Implement plugin architecture [\#18](https://github.com/tumugi/tumugi/pull/18) ([hakobera](https://github.com/hakobera))
|
|
21
42
|
- Add changelog of v0.1.0 [\#10](https://github.com/tumugi/tumugi/pull/10) ([hakobera](https://github.com/hakobera))
|
|
22
43
|
|
|
23
44
|
## [v0.1.0](https://github.com/tumugi/tumugi/tree/v0.1.0) (2016-04-28)
|
data/README.md
CHANGED
|
@@ -69,7 +69,33 @@ Save these code into `workflow.rb`,
|
|
|
69
69
|
then run this script by `tumugi` command like this:
|
|
70
70
|
|
|
71
71
|
```bash
|
|
72
|
-
$ tumugi workflow.rb task1
|
|
72
|
+
$ tumugi run -f workflow.rb task1
|
|
73
|
+
I, [2016-05-06T22:58:28.271234 #76156] INFO -- : start: task4
|
|
74
|
+
I, [2016-05-06T22:58:28.271310 #76156] INFO -- : run: task4
|
|
75
|
+
I, [2016-05-06T22:58:28.271386 #76156] INFO -- : task4#run
|
|
76
|
+
I, [2016-05-06T22:58:29.276218 #76156] INFO -- : completed: task4
|
|
77
|
+
I, [2016-05-06T22:58:29.276373 #76156] INFO -- : start: task2
|
|
78
|
+
I, [2016-05-06T22:58:29.276437 #76156] INFO -- : run: task2
|
|
79
|
+
I, [2016-05-06T22:58:29.276546 #76156] INFO -- : task2#run
|
|
80
|
+
I, [2016-05-06T22:58:29.276606 #76156] INFO -- : completed: task2
|
|
81
|
+
I, [2016-05-06T22:58:29.276650 #76156] INFO -- : start: task3
|
|
82
|
+
I, [2016-05-06T22:58:29.276688 #76156] INFO -- : run: task3
|
|
83
|
+
I, [2016-05-06T22:58:29.276733 #76156] INFO -- : task3#run
|
|
84
|
+
I, [2016-05-06T22:58:29.276765 #76156] INFO -- : completed: task3
|
|
85
|
+
I, [2016-05-06T22:58:29.276798 #76156] INFO -- : start: task1
|
|
86
|
+
I, [2016-05-06T22:58:29.276823 #76156] INFO -- : run: task1
|
|
87
|
+
I, [2016-05-06T22:58:29.276861 #76156] INFO -- : task1#run
|
|
88
|
+
I, [2016-05-06T22:58:29.276899 #76156] INFO -- : completed: task1
|
|
89
|
+
I, [2016-05-06T22:58:29.278919 #76156] INFO -- : Result report:
|
|
90
|
+
+-------+----------+------------+-----------+
|
|
91
|
+
| Task | Requires | Parameters | State |
|
|
92
|
+
+-------+----------+------------+-----------+
|
|
93
|
+
| task1 | task2 | | completed |
|
|
94
|
+
| | task3 | | |
|
|
95
|
+
| task3 | task4 | | completed |
|
|
96
|
+
| task2 | task4 | | completed |
|
|
97
|
+
| task4 | | | completed |
|
|
98
|
+
+-------+----------+------------+-----------+
|
|
73
99
|
```
|
|
74
100
|
|
|
75
101
|
## Development
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
task :task1 do
|
|
2
|
+
requires [:task2, :task3]
|
|
3
|
+
run { log 'task1#run' }
|
|
4
|
+
end
|
|
5
|
+
|
|
6
|
+
task :task2 do
|
|
7
|
+
requires [:task4]
|
|
8
|
+
run { log 'task2#run' }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
task :task3 do
|
|
12
|
+
requires [:task4]
|
|
13
|
+
run { log 'task3#run' }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
task :task4 do
|
|
17
|
+
run do
|
|
18
|
+
raise "error in #{id}"
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
task :task1 do
|
|
2
|
+
requires [:task2, :task3]
|
|
3
|
+
run { log 'task1#run' }
|
|
4
|
+
end
|
|
5
|
+
|
|
6
|
+
task :task2 do
|
|
7
|
+
requires [:task4]
|
|
8
|
+
run { raise "error in #{id}" }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
task :task3 do
|
|
12
|
+
requires [:task4]
|
|
13
|
+
run { log 'task3#run' }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
task :task4 do
|
|
17
|
+
run do
|
|
18
|
+
log 'task4#run'
|
|
19
|
+
sleep 1
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
task :task1 do
|
|
2
|
+
requires [:task2, :task3]
|
|
3
|
+
run { raise "error in #{id}" }
|
|
4
|
+
end
|
|
5
|
+
|
|
6
|
+
task :task2 do
|
|
7
|
+
requires [:task4]
|
|
8
|
+
run { log 'task2#run' }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
task :task3 do
|
|
12
|
+
requires [:task4]
|
|
13
|
+
run { log 'task3#run' }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
task :task4 do
|
|
17
|
+
run do
|
|
18
|
+
log 'task4#run'
|
|
19
|
+
sleep 1
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
task :task1 do
|
|
2
|
+
param :key1, required: true #=> 'value1', get value from CLI parameter
|
|
3
|
+
param :key2 #=> 'value2', get value from Environment variables
|
|
4
|
+
|
|
5
|
+
requires :task2
|
|
6
|
+
run do
|
|
7
|
+
log "key1=#{key1}" # You can get param as task property
|
|
8
|
+
log "key2=#{key2}"
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
task :task2 do
|
|
13
|
+
# If you want do disable auto binding feature, set `auto_bind: false`
|
|
14
|
+
param :key1, auto_bind: false #=> nil
|
|
15
|
+
param :key2
|
|
16
|
+
param_set :key2, 'value2' #=> 'value'
|
|
17
|
+
|
|
18
|
+
requires :task3
|
|
19
|
+
run do
|
|
20
|
+
log "key1=#{key1}"
|
|
21
|
+
log "key2=#{key2}"
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
task :task3 do
|
|
26
|
+
param :key3, default: 'default value' #=> 'default value', get value from :default when binding value does not found
|
|
27
|
+
|
|
28
|
+
run do
|
|
29
|
+
log "key3=#{key3}"
|
|
30
|
+
end
|
|
31
|
+
end
|
data/lib/tumugi/application.rb
CHANGED
data/lib/tumugi/cli.rb
CHANGED
|
@@ -9,6 +9,11 @@ module Tumugi
|
|
|
9
9
|
def common_options
|
|
10
10
|
option :file, aliases: '-f', desc: 'Task definition file name', required: true
|
|
11
11
|
option :config, aliases: '-c', desc: 'Configuration file name', default: 'tumugi.rb'
|
|
12
|
+
option :params, aliases: '-p', type: :hash, desc: 'Task parameters'
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def exit_on_failure?
|
|
16
|
+
true
|
|
12
17
|
end
|
|
13
18
|
end
|
|
14
19
|
|
|
@@ -44,6 +49,9 @@ module Tumugi
|
|
|
44
49
|
if config_file && File.exists?(config_file) && File.extname(config_file) == '.rb'
|
|
45
50
|
load(config_file)
|
|
46
51
|
end
|
|
52
|
+
|
|
53
|
+
params = options[:params]
|
|
54
|
+
Tumugi.application.params = params if params
|
|
47
55
|
end
|
|
48
56
|
end
|
|
49
57
|
end
|
data/lib/tumugi/command/run.rb
CHANGED
|
@@ -1,23 +1,37 @@
|
|
|
1
1
|
require 'parallel'
|
|
2
2
|
require 'retriable'
|
|
3
|
+
require 'terminal-table'
|
|
4
|
+
require 'thor'
|
|
5
|
+
|
|
6
|
+
require 'tumugi/mixin/listable'
|
|
3
7
|
|
|
4
8
|
module Tumugi
|
|
5
9
|
module Command
|
|
6
10
|
class Run
|
|
11
|
+
include Tumugi::Mixin::Listable
|
|
12
|
+
|
|
7
13
|
def execute(dag, options={})
|
|
8
14
|
workers = options[:workers] || Tumugi.config.workers
|
|
9
15
|
settings = { in_threads: workers }
|
|
16
|
+
logger = Tumugi.logger
|
|
10
17
|
|
|
11
|
-
|
|
12
|
-
|
|
18
|
+
logger.verbose! if options[:verbose]
|
|
19
|
+
logger.quiet! if options[:quiet]
|
|
13
20
|
|
|
14
21
|
Parallel.each(dag.tsort, settings) do |t|
|
|
15
|
-
|
|
16
|
-
until t.ready?
|
|
22
|
+
logger.info "start: #{t.id}"
|
|
23
|
+
until t.ready? || t.requires_failed?
|
|
17
24
|
sleep 1
|
|
18
25
|
end
|
|
19
|
-
|
|
20
|
-
|
|
26
|
+
|
|
27
|
+
if t.completed?
|
|
28
|
+
t.state = :skipped
|
|
29
|
+
logger.info "#{t.state}: #{t.id} is already completed"
|
|
30
|
+
elsif t.requires_failed?
|
|
31
|
+
t.state = :requires_failed
|
|
32
|
+
logger.info "#{t.state}: #{t.id} has failed requires task"
|
|
33
|
+
else
|
|
34
|
+
logger.info "run: #{t.id}"
|
|
21
35
|
t.state = :running
|
|
22
36
|
|
|
23
37
|
begin
|
|
@@ -25,18 +39,18 @@ module Tumugi
|
|
|
25
39
|
t.run
|
|
26
40
|
end
|
|
27
41
|
rescue => e
|
|
28
|
-
Tumugi.logger.info "failed: #{t.id}"
|
|
29
|
-
Tumugi.logger.error "#{e.message}"
|
|
30
42
|
t.state = :failed
|
|
43
|
+
logger.info "#{t.state}: #{t.id}"
|
|
44
|
+
logger.error "#{e.message}"
|
|
31
45
|
else
|
|
32
|
-
Tumugi.logger.info "completed: #{t.id}"
|
|
33
46
|
t.state = :completed
|
|
47
|
+
logger.info "#{t.state}: #{t.id}"
|
|
34
48
|
end
|
|
35
|
-
else
|
|
36
|
-
t.state = :skipped
|
|
37
|
-
Tumugi.logger.info "skip: #{t.id} is already completed"
|
|
38
49
|
end
|
|
39
50
|
end
|
|
51
|
+
|
|
52
|
+
show_result_report(dag)
|
|
53
|
+
raise ::Thor::Error.new("run failed") if dag.tsort.any? { |t| t.state == :failed }
|
|
40
54
|
end
|
|
41
55
|
|
|
42
56
|
private
|
|
@@ -55,8 +69,29 @@ module Tumugi
|
|
|
55
69
|
|
|
56
70
|
def on_retry
|
|
57
71
|
Proc.new do |exception, try, elapsed_time, next_interval|
|
|
58
|
-
|
|
72
|
+
if next_interval
|
|
73
|
+
Tumugi.logger.error "#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds and #{next_interval} seconds until the next try."
|
|
74
|
+
else
|
|
75
|
+
Tumugi.logger.error "#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds. Task failed."
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def show_result_report(dag)
|
|
81
|
+
headings = ['Task', 'Requires', 'Parameters', 'State']
|
|
82
|
+
table = Terminal::Table.new headings: headings do |t|
|
|
83
|
+
dag.tsort.reverse.map do |task|
|
|
84
|
+
proxy = task.class.merged_parameter_proxy
|
|
85
|
+
requires = list(task.requires).map do |r|
|
|
86
|
+
r.id
|
|
87
|
+
end
|
|
88
|
+
params = proxy.params.map do |name, _|
|
|
89
|
+
"#{name}=#{task.instance_variable_get("@#{name}")}"
|
|
90
|
+
end
|
|
91
|
+
t << [ task.id, requires.join("\n"), params.join("\n"), task.state ]
|
|
92
|
+
end
|
|
59
93
|
end
|
|
94
|
+
Tumugi.logger.info "Result report:\n#{table.to_s}"
|
|
60
95
|
end
|
|
61
96
|
end
|
|
62
97
|
end
|
data/lib/tumugi/config.rb
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
module Tumugi
|
|
2
2
|
class Config
|
|
3
|
-
attr_accessor :workers, :max_retry, :retry_interval
|
|
3
|
+
attr_accessor :workers, :max_retry, :retry_interval, :param_auto_bind_enabled
|
|
4
4
|
|
|
5
5
|
def initialize
|
|
6
6
|
@workers = 1
|
|
7
7
|
@max_retry = 3
|
|
8
8
|
@retry_interval = 300 #seconds
|
|
9
|
+
@param_auto_bind_enabled = true
|
|
9
10
|
end
|
|
10
11
|
end
|
|
11
12
|
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
require 'tumugi/parameter/error'
|
|
2
|
+
require 'tumugi/parameter/parameter_proxy'
|
|
3
|
+
|
|
4
|
+
module Tumugi
|
|
5
|
+
module Mixin
|
|
6
|
+
module Parameterizable
|
|
7
|
+
def self.included(mod)
|
|
8
|
+
mod.extend(ClassMethods)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def initialize
|
|
12
|
+
super()
|
|
13
|
+
proxy = self.class.merged_parameter_proxy
|
|
14
|
+
params = proxy.params
|
|
15
|
+
proxy.param_defaults.each do |name, value|
|
|
16
|
+
param = params[name]
|
|
17
|
+
param.overwrite_default(value) if param
|
|
18
|
+
end
|
|
19
|
+
params.each do |name, param|
|
|
20
|
+
unless proxy.param_auto_bind_enabled.nil?
|
|
21
|
+
param.task_param_auto_bind_enabled = proxy.param_auto_bind_enabled
|
|
22
|
+
end
|
|
23
|
+
instance_variable_set("@#{name}", param.get)
|
|
24
|
+
end
|
|
25
|
+
validate_params(params)
|
|
26
|
+
configure
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def validate_params(params)
|
|
30
|
+
params.each do |name, param|
|
|
31
|
+
if param.required? && instance_variable_get("@#{name}").nil?
|
|
32
|
+
raise Tumugi::Parameter::ParameterError.new("Parameter #{name} is required")
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def configure
|
|
38
|
+
# You can override in a subclass
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
module ClassMethods
|
|
42
|
+
def parameter_proxy_map
|
|
43
|
+
map = {}
|
|
44
|
+
self.define_singleton_method(:parameter_proxy_map) { map }
|
|
45
|
+
map
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def parameter_proxy(mod_name)
|
|
49
|
+
map = parameter_proxy_map
|
|
50
|
+
unless map[mod_name]
|
|
51
|
+
proxy = Tumugi::Parameter::ParameterProxy.new(mod_name)
|
|
52
|
+
map[mod_name] = proxy
|
|
53
|
+
end
|
|
54
|
+
map[mod_name]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def param(name, opts={})
|
|
58
|
+
parameter_proxy(proxy_id(self)).param(name, opts)
|
|
59
|
+
attr_accessor name
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def param_set(name, value)
|
|
63
|
+
parameter_proxy(proxy_id(self)).param_set(name, value)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def param_auto_bind_enabled(v)
|
|
67
|
+
parameter_proxy(proxy_id(self)).param_auto_bind_enabled = v
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def merged_parameter_proxy
|
|
71
|
+
parameterizable = ancestors.reverse.select{ |a| a.respond_to?(:parameter_proxy) }
|
|
72
|
+
parameterizable.map { |a| a.parameter_proxy(proxy_id(a)) }.reduce(:merge)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def dump
|
|
76
|
+
parameter_proxy_map[proxy_id(self)].dump
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def proxy_id(klass)
|
|
80
|
+
self.name || self.object_id.to_s
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
require 'time'
|
|
2
|
+
|
|
3
|
+
module Tumugi
|
|
4
|
+
module Parameter
|
|
5
|
+
class StringConverter
|
|
6
|
+
def convert(value, opts={})
|
|
7
|
+
value
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
class IntegerConverter
|
|
12
|
+
def convert(value, opts={})
|
|
13
|
+
Integer(value)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
class FloatConverter
|
|
18
|
+
def convert(value, opts={})
|
|
19
|
+
Float(value)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
class BoolConverter
|
|
24
|
+
def convert(value, opts={})
|
|
25
|
+
if value =~ /^(true|false)$/
|
|
26
|
+
value == 'true'
|
|
27
|
+
else
|
|
28
|
+
raise ArgumentError.new("Invalid value for Bool: '#{value}'")
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
class TimeConverter
|
|
34
|
+
def convert(value, opts={})
|
|
35
|
+
Time.parse(value)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
class Converter
|
|
40
|
+
CONVERTERS = {
|
|
41
|
+
string: StringConverter.new,
|
|
42
|
+
integer: IntegerConverter.new,
|
|
43
|
+
float: FloatConverter.new,
|
|
44
|
+
bool: BoolConverter.new,
|
|
45
|
+
time: TimeConverter.new,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
def self.convert(type, value)
|
|
49
|
+
converter = CONVERTERS[type]
|
|
50
|
+
if converter
|
|
51
|
+
converter.convert(value)
|
|
52
|
+
else
|
|
53
|
+
raise ArgumentError.new("Invalid type: #{type}")
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
require 'tumugi/parameter/converter'
|
|
2
|
+
require 'tumugi/parameter/error'
|
|
3
|
+
|
|
4
|
+
module Tumugi
|
|
5
|
+
module Parameter
|
|
6
|
+
class Parameter
|
|
7
|
+
attr_accessor :name, :task_param_auto_bind_enabled, :application_param_auto_bind_enabled
|
|
8
|
+
|
|
9
|
+
def initialize(name, opts={})
|
|
10
|
+
@name = name
|
|
11
|
+
@opts = opts
|
|
12
|
+
@application_param_auto_bind_enabled = Tumugi.config.param_auto_bind_enabled
|
|
13
|
+
@task_param_auto_bind_enabled = @application_param_auto_bind_enabled
|
|
14
|
+
validate
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def get
|
|
18
|
+
if auto_bind?
|
|
19
|
+
value = search_from_application_parameters
|
|
20
|
+
value = search_from_env if value.nil?
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
return value unless value.nil?
|
|
24
|
+
default_value
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def auto_bind?
|
|
28
|
+
if @opts[:auto_bind].nil?
|
|
29
|
+
if @task_param_auto_bind_enabled
|
|
30
|
+
true
|
|
31
|
+
else
|
|
32
|
+
false
|
|
33
|
+
end
|
|
34
|
+
else
|
|
35
|
+
@opts[:auto_bind]
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def required?
|
|
40
|
+
@opts[:required].nil? ? false : @opts[:required]
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def type
|
|
44
|
+
@opts[:type] || :string
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def default_value
|
|
48
|
+
@opts[:default] || nil
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def overwrite_default(value)
|
|
52
|
+
@opts[:required] = false
|
|
53
|
+
@opts[:default] = value
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def search_from_application_parameters
|
|
59
|
+
key = @name.to_s
|
|
60
|
+
value = Tumugi.application.params[key]
|
|
61
|
+
value ? Converter.convert(type, value) : nil
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def search_from_env
|
|
65
|
+
key = @name.to_s
|
|
66
|
+
value = nil
|
|
67
|
+
value = ENV[key] if ENV.has_key?(key)
|
|
68
|
+
value = ENV[key.upcase] if ENV.has_key?(key.upcase)
|
|
69
|
+
value ? Converter.convert(type, value) : nil
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
def validate
|
|
75
|
+
if required? && default_value != nil
|
|
76
|
+
raise ParameterError.new("When you set required: true, you cannot set default value")
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
require 'tumugi/parameter/parameter'
|
|
2
|
+
|
|
3
|
+
module Tumugi
|
|
4
|
+
module Parameter
|
|
5
|
+
class ParameterProxy
|
|
6
|
+
attr_accessor :name, :params, :param_defaults, :param_auto_bind_enabled
|
|
7
|
+
|
|
8
|
+
def initialize(name)
|
|
9
|
+
@name = name
|
|
10
|
+
@params = {}
|
|
11
|
+
@param_defaults = {}
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def merge(other)
|
|
15
|
+
merged = self.class.new(other.name)
|
|
16
|
+
merged.params = other.params.merge(self.params)
|
|
17
|
+
merged.param_defaults = other.param_defaults.merge(self.param_defaults)
|
|
18
|
+
merged
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def param(name, opts={})
|
|
22
|
+
@params[name] = Tumugi::Parameter::Parameter.new(name, opts)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def param_set(name, value)
|
|
26
|
+
@param_defaults[name] = value
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def dump
|
|
30
|
+
Marshal.dump(self)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
data/lib/tumugi/task.rb
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
require 'tumugi/mixin/listable'
|
|
2
2
|
require 'tumugi/mixin/task_helper'
|
|
3
|
+
require 'tumugi/mixin/parameterizable'
|
|
3
4
|
|
|
4
5
|
module Tumugi
|
|
5
6
|
class Task
|
|
7
|
+
include Tumugi::Mixin::Parameterizable
|
|
6
8
|
include Tumugi::Mixin::Listable
|
|
7
9
|
include Tumugi::Mixin::TaskHelper
|
|
8
10
|
|
|
9
11
|
attr_accessor :state # :pending, :running, :completed, :skipped
|
|
10
12
|
|
|
11
13
|
def initialize
|
|
14
|
+
super()
|
|
12
15
|
@state = :pending
|
|
13
16
|
end
|
|
14
17
|
|
|
@@ -72,6 +75,10 @@ module Tumugi
|
|
|
72
75
|
end
|
|
73
76
|
end
|
|
74
77
|
|
|
78
|
+
def requires_failed?
|
|
79
|
+
list(_requires).any? { |t| t.instance.state == :failed || t.instance.state == :requires_failed }
|
|
80
|
+
end
|
|
81
|
+
|
|
75
82
|
# Following methods are internal use only
|
|
76
83
|
|
|
77
84
|
def _requires
|
|
@@ -20,6 +20,8 @@ module Tumugi
|
|
|
20
20
|
def initialize(id, opts={})
|
|
21
21
|
@id = id
|
|
22
22
|
@opts = { type: Tumugi::Task }.merge(opts)
|
|
23
|
+
@params = {}
|
|
24
|
+
@param_defaults = {}
|
|
23
25
|
|
|
24
26
|
unless @opts[:type].is_a?(Class)
|
|
25
27
|
@opts[:type] = Tumugi::Plugin.lookup_task(@opts[:type])
|
|
@@ -30,6 +32,18 @@ module Tumugi
|
|
|
30
32
|
@task ||= create_task
|
|
31
33
|
end
|
|
32
34
|
|
|
35
|
+
def param(name, opts={})
|
|
36
|
+
@params[name] = opts
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def param_set(name, value)
|
|
40
|
+
@param_defaults[name] = value
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def param_auto_bind_enabled(v)
|
|
44
|
+
@param_auto_bind_enabled = v
|
|
45
|
+
end
|
|
46
|
+
|
|
33
47
|
def requires(tasks)
|
|
34
48
|
@required_tasks = tasks
|
|
35
49
|
end
|
|
@@ -64,10 +78,11 @@ module Tumugi
|
|
|
64
78
|
end
|
|
65
79
|
|
|
66
80
|
def define_task
|
|
67
|
-
task_class = Class.new(
|
|
81
|
+
task_class = Class.new(lookup_parent_task_class)
|
|
68
82
|
define_requires_method(task_class)
|
|
69
83
|
define_output_method(task_class)
|
|
70
84
|
define_run_method(task_class)
|
|
85
|
+
setup_params(task_class)
|
|
71
86
|
task_class
|
|
72
87
|
end
|
|
73
88
|
|
|
@@ -106,5 +121,25 @@ module Tumugi
|
|
|
106
121
|
end
|
|
107
122
|
end unless @run.nil?
|
|
108
123
|
end
|
|
124
|
+
|
|
125
|
+
def setup_params(task_class)
|
|
126
|
+
@params.each do |name, opts|
|
|
127
|
+
task_class.param(name, opts)
|
|
128
|
+
end
|
|
129
|
+
@param_defaults.each do |name, value|
|
|
130
|
+
task_class.param_set(name, value)
|
|
131
|
+
end
|
|
132
|
+
unless @param_auto_bind_enabled.nil?
|
|
133
|
+
task_class.param_auto_bind_enabled(@param_auto_bind_enabled)
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def lookup_parent_task_class
|
|
138
|
+
if @opts[:type].is_a?(Class)
|
|
139
|
+
@opts[:type]
|
|
140
|
+
else
|
|
141
|
+
Tumugi::Plugin.lookup_task(@opts[:type])
|
|
142
|
+
end
|
|
143
|
+
end
|
|
109
144
|
end
|
|
110
145
|
end
|
data/lib/tumugi/version.rb
CHANGED
data/tumugi.gemspec
CHANGED
|
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
|
|
|
23
23
|
spec.add_runtime_dependency 'parallel', '~> 1.8.0'
|
|
24
24
|
spec.add_runtime_dependency 'retriable', '~> 2.1'
|
|
25
25
|
spec.add_runtime_dependency 'ruby-graphviz', '~> 1.2.2'
|
|
26
|
+
spec.add_runtime_dependency 'terminal-table', '~> 1.5.2'
|
|
26
27
|
spec.add_runtime_dependency 'thor', '~> 0.19.1'
|
|
27
28
|
|
|
28
29
|
spec.add_development_dependency 'bundler', '~> 1.11'
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tumugi
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kazuyuki Honda
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-05-
|
|
11
|
+
date: 2016-05-06 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: parallel
|
|
@@ -52,6 +52,20 @@ dependencies:
|
|
|
52
52
|
- - "~>"
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
54
|
version: 1.2.2
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: terminal-table
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - "~>"
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: 1.5.2
|
|
62
|
+
type: :runtime
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - "~>"
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: 1.5.2
|
|
55
69
|
- !ruby/object:Gem::Dependency
|
|
56
70
|
name: thor
|
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -169,9 +183,14 @@ files:
|
|
|
169
183
|
- bin/setup
|
|
170
184
|
- examples/concurrent_task_run.rb
|
|
171
185
|
- examples/data_pipeline.rb
|
|
186
|
+
- examples/fail_first_task.rb
|
|
187
|
+
- examples/fail_intermediate_task.rb
|
|
188
|
+
- examples/fail_last_task.rb
|
|
172
189
|
- examples/simple.rb
|
|
173
190
|
- examples/target.rb
|
|
174
191
|
- examples/task_inheritance.rb
|
|
192
|
+
- examples/task_parameter.rb
|
|
193
|
+
- examples/tumugi_config.rb
|
|
175
194
|
- exe/tumugi
|
|
176
195
|
- lib/tumugi.rb
|
|
177
196
|
- lib/tumugi/application.rb
|
|
@@ -186,7 +205,12 @@ files:
|
|
|
186
205
|
- lib/tumugi/file_system_error.rb
|
|
187
206
|
- lib/tumugi/logger.rb
|
|
188
207
|
- lib/tumugi/mixin/listable.rb
|
|
208
|
+
- lib/tumugi/mixin/parameterizable.rb
|
|
189
209
|
- lib/tumugi/mixin/task_helper.rb
|
|
210
|
+
- lib/tumugi/parameter/converter.rb
|
|
211
|
+
- lib/tumugi/parameter/error.rb
|
|
212
|
+
- lib/tumugi/parameter/parameter.rb
|
|
213
|
+
- lib/tumugi/parameter/parameter_proxy.rb
|
|
190
214
|
- lib/tumugi/plugin.rb
|
|
191
215
|
- lib/tumugi/plugin/atomic_local_file.rb
|
|
192
216
|
- lib/tumugi/plugin/file_system_target.rb
|