tumugi 0.4.5 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +10 -16
- data/CHANGELOG.md +31 -2
- data/examples/concurrent_task_run.rb +1 -1
- data/examples/data_pipeline.rb +2 -2
- data/examples/target.rb +4 -4
- data/examples/task_inheritance.rb +1 -1
- data/examples/tumugi_config.rb +3 -3
- data/examples/tumugi_config_with_section.rb +5 -5
- data/lib/tumugi.rb +12 -4
- data/lib/tumugi/application.rb +12 -7
- data/lib/tumugi/cli.rb +13 -5
- data/lib/tumugi/command/run.rb +62 -32
- data/lib/tumugi/command/show.rb +5 -0
- data/lib/tumugi/config.rb +32 -19
- data/lib/tumugi/error.rb +15 -0
- data/lib/tumugi/file_system.rb +1 -1
- data/lib/tumugi/logger.rb +7 -1
- data/lib/tumugi/mixin/parameterizable.rb +2 -2
- data/lib/tumugi/parameter/parameter.rb +3 -14
- data/lib/tumugi/plugin.rb +1 -1
- data/lib/tumugi/plugin/target/local_file.rb +1 -1
- data/lib/tumugi/plugin/task/external.rb +13 -0
- data/lib/tumugi/target.rb +1 -1
- data/lib/tumugi/task.rb +5 -1
- data/lib/tumugi/version.rb +1 -1
- metadata +3 -3
- data/lib/tumugi/parameter/error.rb +0 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a8a889ceab4e85663023c13142fd19f4b427fb5c
|
4
|
+
data.tar.gz: 67ed6e06b39c20fe355b456cb1a84751b23544e6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ede2235c0f2c6810fce09093c78422ae152a2f0a3ccd3e6275856d5aabd5308f84846042467d032652c6bd5cfe96b6fd68428e32222f91d44af00630b225b2a6
|
7
|
+
data.tar.gz: 3841d5750ab943219623530d0032ddcf0532c21e6d3b4fc09e1cdf643f59c78e26cc1f9ab0e69893408e68d4ea19d150584a7cecb9848c0a17316f97176ce331
|
data/.travis.yml
CHANGED
@@ -1,23 +1,17 @@
|
|
1
1
|
language: ruby
|
2
|
-
|
2
|
+
cache: bundler
|
3
|
+
rvm:
|
4
|
+
- 2.1.10
|
5
|
+
- 2.2.5
|
6
|
+
- 2.3.1
|
7
|
+
- ruby-head
|
8
|
+
- jruby-9.0.5.0
|
9
|
+
before_install:
|
10
|
+
- gem install bundler
|
3
11
|
matrix:
|
4
|
-
include:
|
5
|
-
- rvm: 2.1.10
|
6
|
-
os: linux
|
7
|
-
- rvm: 2.2.5
|
8
|
-
os: linux
|
9
|
-
- rvm: 2.3.1
|
10
|
-
os: linux
|
11
|
-
- rvm: ruby-head
|
12
|
-
os: linux
|
13
|
-
- rvm: jruby-9.0.5.0
|
14
|
-
os: linux
|
15
12
|
allow_failures:
|
16
13
|
- rvm: ruby-head
|
17
|
-
|
18
|
-
cache: bundler
|
19
|
-
|
20
14
|
addons:
|
21
15
|
apt:
|
22
16
|
packages:
|
23
|
-
|
17
|
+
- graphviz
|
data/CHANGELOG.md
CHANGED
@@ -1,12 +1,41 @@
|
|
1
1
|
# Change Log
|
2
2
|
|
3
|
-
## [0.
|
4
|
-
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.4.
|
3
|
+
## [0.5.0](https://github.com/tumugi/tumugi/tree/0.5.0) (2016-05-26)
|
4
|
+
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.4.5...0.5.0)
|
5
|
+
|
6
|
+
**Implemented enhancements:**
|
7
|
+
|
8
|
+
- Add ExternalTask as bundled plugin [\#61](https://github.com/tumugi/tumugi/issues/61)
|
9
|
+
- \[Breaking Change\] Tumugi.configure [\#60](https://github.com/tumugi/tumugi/issues/60)
|
10
|
+
- Better task scheduling [\#57](https://github.com/tumugi/tumugi/issues/57)
|
11
|
+
- \[Breaking Change\] Disable parameter auto binding from ENV [\#56](https://github.com/tumugi/tumugi/issues/56)
|
12
|
+
- Timeout feature [\#54](https://github.com/tumugi/tumugi/issues/54)
|
13
|
+
- \[Breaking Change\] Restruct Error class [\#51](https://github.com/tumugi/tumugi/issues/51)
|
14
|
+
- Enhance logger [\#36](https://github.com/tumugi/tumugi/issues/36)
|
15
|
+
|
16
|
+
**Merged pull requests:**
|
17
|
+
|
18
|
+
- Call init first Tumugi::Logger instance access [\#67](https://github.com/tumugi/tumugi/pull/67) ([hakobera](https://github.com/hakobera))
|
19
|
+
- Better task scheduling. [\#66](https://github.com/tumugi/tumugi/pull/66) ([hakobera](https://github.com/hakobera))
|
20
|
+
- Support log output to file [\#65](https://github.com/tumugi/tumugi/pull/65) ([hakobera](https://github.com/hakobera))
|
21
|
+
- Add EternalTask as bundled plugin [\#64](https://github.com/tumugi/tumugi/pull/64) ([hakobera](https://github.com/hakobera))
|
22
|
+
- Add Tumugi.configure method to modify config. [\#63](https://github.com/tumugi/tumugi/pull/63) ([hakobera](https://github.com/hakobera))
|
23
|
+
- Implement timeout feature [\#62](https://github.com/tumugi/tumugi/pull/62) ([hakobera](https://github.com/hakobera))
|
24
|
+
- Disable parameter auto binding from ENV [\#59](https://github.com/tumugi/tumugi/pull/59) ([hakobera](https://github.com/hakobera))
|
25
|
+
- Restruct error handling [\#58](https://github.com/tumugi/tumugi/pull/58) ([hakobera](https://github.com/hakobera))
|
26
|
+
- Change CLI test to fix failed test with ruby-head [\#52](https://github.com/tumugi/tumugi/pull/52) ([hakobera](https://github.com/hakobera))
|
27
|
+
|
28
|
+
## [v0.4.5](https://github.com/tumugi/tumugi/tree/v0.4.5) (2016-05-16)
|
29
|
+
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.4.4...v0.4.5)
|
5
30
|
|
6
31
|
**Fixed bugs:**
|
7
32
|
|
8
33
|
- Fix required parameter validation [\#49](https://github.com/tumugi/tumugi/pull/49) ([hakobera](https://github.com/hakobera))
|
9
34
|
|
35
|
+
**Merged pull requests:**
|
36
|
+
|
37
|
+
- Prepare release for v0.4.5 [\#50](https://github.com/tumugi/tumugi/pull/50) ([hakobera](https://github.com/hakobera))
|
38
|
+
|
10
39
|
## [v0.4.4](https://github.com/tumugi/tumugi/tree/v0.4.4) (2016-05-13)
|
11
40
|
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.4.3...v0.4.4)
|
12
41
|
|
data/examples/data_pipeline.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
task :generate_data do
|
2
2
|
output do
|
3
|
-
target(:local_file, "
|
3
|
+
target(:local_file, "tmp/tumugi_data_#{Time.now.strftime('%Y-%m-%d')}.txt")
|
4
4
|
end
|
5
5
|
|
6
6
|
run do
|
@@ -16,7 +16,7 @@ task :sum do
|
|
16
16
|
requires :generate_data
|
17
17
|
|
18
18
|
output do
|
19
|
-
target(:local_file, "
|
19
|
+
target(:local_file, "tmp/tumugi_output_#{Time.now.strftime('%Y-%m-%d')}.txt")
|
20
20
|
end
|
21
21
|
|
22
22
|
run do
|
data/examples/target.rb
CHANGED
@@ -2,7 +2,7 @@ require 'tumugi/plugin/target/local_file'
|
|
2
2
|
|
3
3
|
task :task1 do
|
4
4
|
requires [:task2, :task3]
|
5
|
-
output target(:local_file, "
|
5
|
+
output target(:local_file, "tmp/tumugi_#{id}.txt")
|
6
6
|
run do
|
7
7
|
log 'task1#run'
|
8
8
|
output.open('w') {|f| f.puts('done') }
|
@@ -11,7 +11,7 @@ end
|
|
11
11
|
|
12
12
|
task :task2 do
|
13
13
|
requires [:task4]
|
14
|
-
output [target(:local_file, "
|
14
|
+
output [target(:local_file, "tmp/tumugi_#{id}.txt")]
|
15
15
|
run do
|
16
16
|
log 'task2#run'
|
17
17
|
output[0].open('w') {|f| f.puts('done') }
|
@@ -20,7 +20,7 @@ end
|
|
20
20
|
|
21
21
|
task :task3 do
|
22
22
|
requires [:task4]
|
23
|
-
output { target(:local_file, "
|
23
|
+
output { target(:local_file, "tmp/tumugi_#{id}.txt") }
|
24
24
|
run do
|
25
25
|
log 'task3#run'
|
26
26
|
output.open('w') {|f| f.puts('done') }
|
@@ -28,7 +28,7 @@ task :task3 do
|
|
28
28
|
end
|
29
29
|
|
30
30
|
task :task4 do
|
31
|
-
output Tumugi::Plugin::LocalFileTarget.new("
|
31
|
+
output Tumugi::Plugin::LocalFileTarget.new("tmp/tumugi_#{id}.txt")
|
32
32
|
run do
|
33
33
|
log 'task4#run'
|
34
34
|
output.open('w') {|f| f.puts('done') }
|
data/examples/tumugi_config.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
Tumugi.
|
2
|
-
|
3
|
-
|
1
|
+
Tumugi.configure do |config|
|
2
|
+
config.max_retry = 3
|
3
|
+
config.retry_interval = 1
|
4
4
|
end
|
@@ -1,8 +1,8 @@
|
|
1
|
-
Tumugi.
|
2
|
-
|
3
|
-
|
1
|
+
Tumugi.configure do |config|
|
2
|
+
config.max_retry = 3
|
3
|
+
config.retry_interval = 1
|
4
4
|
|
5
|
-
|
6
|
-
|
5
|
+
config.section('example') do |section|
|
6
|
+
section.key = 'value'
|
7
7
|
end
|
8
8
|
end
|
data/lib/tumugi.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'tumugi/application'
|
2
2
|
require 'tumugi/config'
|
3
|
+
require 'tumugi/error'
|
3
4
|
require 'tumugi/logger'
|
4
5
|
require 'tumugi/version'
|
5
6
|
|
@@ -9,14 +10,21 @@ module Tumugi
|
|
9
10
|
@application ||= Tumugi::Application.new
|
10
11
|
end
|
11
12
|
|
12
|
-
def
|
13
|
-
|
13
|
+
def configure(&block)
|
14
|
+
raise Tumugi::ConfigError.new 'Tumugi.configure must have block' unless block_given?
|
15
|
+
yield _config
|
16
|
+
nil
|
14
17
|
end
|
15
18
|
|
16
19
|
def config
|
20
|
+
raise Tumugi::ConfigError.new 'Tumugi.config with block is deprecated. Use Tumugi.configure instead.' if block_given?
|
21
|
+
_config.clone.freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def _config
|
17
27
|
@config ||= Tumugi::Config.new
|
18
|
-
yield @config if block_given?
|
19
|
-
@config
|
20
28
|
end
|
21
29
|
end
|
22
30
|
end
|
data/lib/tumugi/application.rb
CHANGED
@@ -16,7 +16,7 @@ module Tumugi
|
|
16
16
|
end
|
17
17
|
|
18
18
|
def execute(command, root_task_id, options)
|
19
|
-
process_common_options(options)
|
19
|
+
process_common_options(command, options)
|
20
20
|
load_workflow_file(options[:file])
|
21
21
|
dag = create_dag(root_task_id)
|
22
22
|
command_module = Kernel.const_get("Tumugi").const_get("Command")
|
@@ -44,7 +44,7 @@ module Tumugi
|
|
44
44
|
begin
|
45
45
|
load(file, true)
|
46
46
|
rescue LoadError => e
|
47
|
-
raise Tumugi::TumugiError, e
|
47
|
+
raise Tumugi::TumugiError.new("Workflow file load error: #{file}", e)
|
48
48
|
end
|
49
49
|
end
|
50
50
|
|
@@ -55,17 +55,22 @@ module Tumugi
|
|
55
55
|
dag
|
56
56
|
end
|
57
57
|
|
58
|
-
def process_common_options(options)
|
59
|
-
setup_logger(options)
|
58
|
+
def process_common_options(command, options)
|
59
|
+
setup_logger(command, options)
|
60
60
|
load_config(options)
|
61
61
|
set_params(options)
|
62
62
|
end
|
63
63
|
|
64
64
|
def logger
|
65
|
-
@logger ||= Tumugi.
|
65
|
+
@logger ||= Tumugi::Logger.instance
|
66
66
|
end
|
67
67
|
|
68
|
-
def setup_logger(options)
|
68
|
+
def setup_logger(command, options)
|
69
|
+
if command == :run && !options[:out].nil?
|
70
|
+
Tumugi::Logger.instance.init(options[:out])
|
71
|
+
else
|
72
|
+
Tumugi::Logger.instance.init
|
73
|
+
end
|
69
74
|
logger.verbose! if options[:verbose]
|
70
75
|
logger.quiet! if options[:quiet]
|
71
76
|
end
|
@@ -77,7 +82,7 @@ module Tumugi
|
|
77
82
|
load(config_file)
|
78
83
|
end
|
79
84
|
rescue LoadError => e
|
80
|
-
raise Tumugi::TumugiError, e
|
85
|
+
raise Tumugi::TumugiError.new("Config file load error: #{config_file}", e)
|
81
86
|
end
|
82
87
|
|
83
88
|
def set_params(options)
|
data/lib/tumugi/cli.rb
CHANGED
@@ -27,6 +27,7 @@ module Tumugi
|
|
27
27
|
desc "run TASK", "Run TASK in a workflow"
|
28
28
|
map "run" => "run_" # run is thor's reserved word, so this trick is needed
|
29
29
|
option :workers, aliases: '-w', type: :numeric, desc: 'Number of workers to run task concurrently'
|
30
|
+
option :out, aliases: '-o', desc: 'Output log filename. If not specified, log is write to STDOUT'
|
30
31
|
common_options
|
31
32
|
def run_(task)
|
32
33
|
execute(:run, task, options)
|
@@ -37,7 +38,9 @@ module Tumugi
|
|
37
38
|
option :out, aliases: '-o', desc: 'Output file name. If not specified, output result to STDOUT'
|
38
39
|
option :format, aliases: '-t', desc: 'Output file format. Only affected --out option is specified.', enum: ['dot', 'png', 'svg']
|
39
40
|
def show(task)
|
40
|
-
|
41
|
+
opts = options.dup
|
42
|
+
opts[:quiet] = true if opts[:out].nil?
|
43
|
+
execute(:show, task, opts.freeze)
|
41
44
|
end
|
42
45
|
|
43
46
|
private
|
@@ -45,14 +48,19 @@ module Tumugi
|
|
45
48
|
def execute(command, task, options)
|
46
49
|
success = Tumugi.application.execute(command, task, options)
|
47
50
|
unless success
|
48
|
-
|
51
|
+
logger.error "#{command} command failed"
|
49
52
|
raise Thor::Error, 'failed'
|
50
53
|
end
|
54
|
+
success
|
51
55
|
rescue => e
|
52
|
-
|
53
|
-
|
54
|
-
e.backtrace.each { |line|
|
56
|
+
logger.error "#{command} command failed"
|
57
|
+
logger.error e.message
|
58
|
+
e.backtrace.each { |line| logger.error line }
|
55
59
|
raise Thor::Error, 'failed'
|
56
60
|
end
|
61
|
+
|
62
|
+
def logger
|
63
|
+
Tumugi::Logger.instance
|
64
|
+
end
|
57
65
|
end
|
58
66
|
end
|
data/lib/tumugi/command/run.rb
CHANGED
@@ -2,7 +2,9 @@ require 'parallel'
|
|
2
2
|
require 'retriable'
|
3
3
|
require 'terminal-table'
|
4
4
|
require 'thor'
|
5
|
+
require 'timeout'
|
5
6
|
|
7
|
+
require 'tumugi/error'
|
6
8
|
require 'tumugi/mixin/listable'
|
7
9
|
|
8
10
|
module Tumugi
|
@@ -13,45 +15,69 @@ module Tumugi
|
|
13
15
|
def execute(dag, options={})
|
14
16
|
workers = options[:workers] || Tumugi.config.workers
|
15
17
|
settings = { in_threads: workers }
|
16
|
-
|
18
|
+
start(task_queue(dag), settings)
|
19
|
+
show_result_report(dag)
|
20
|
+
!dag.tsort.any? { |t| t.state == :failed }
|
21
|
+
end
|
17
22
|
|
18
|
-
|
19
|
-
logger.info "start: #{t.id}"
|
20
|
-
until t.ready? || t.requires_failed?
|
21
|
-
sleep 1
|
22
|
-
end
|
23
|
+
private
|
23
24
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
25
|
+
def task_queue(dag)
|
26
|
+
waiting_task_queue = Queue.new
|
27
|
+
available_task_queue = Queue.new
|
28
|
+
dag.tsort.each do |t|
|
29
|
+
if t.ready?
|
30
|
+
available_task_queue << t
|
30
31
|
else
|
31
|
-
|
32
|
-
|
32
|
+
waiting_task_queue << t
|
33
|
+
end
|
34
|
+
end
|
35
|
+
lambda {
|
36
|
+
task = (available_task_queue.empty? ? nil : available_task_queue.pop)
|
37
|
+
if task.nil?
|
38
|
+
task = (waiting_task_queue.empty? ? nil : waiting_task_queue.pop)
|
39
|
+
end
|
40
|
+
task || Parallel::Stop
|
41
|
+
}
|
42
|
+
end
|
33
43
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
44
|
+
def start(proc, settings)
|
45
|
+
Parallel.each(proc, settings) do |t|
|
46
|
+
logger.info "start: #{t.id}"
|
47
|
+
timeout = t.timeout || Tumugi.config.timeout
|
48
|
+
Timeout::timeout(timeout, Tumugi::TimeoutError) do
|
49
|
+
until t.ready? || t.requires_failed?
|
50
|
+
sleep 5
|
51
|
+
end
|
52
|
+
|
53
|
+
if t.completed?
|
54
|
+
t.state = :skipped
|
55
|
+
logger.info "#{t.state}: #{t.id} is already completed"
|
56
|
+
elsif t.requires_failed?
|
57
|
+
t.state = :requires_failed
|
58
|
+
logger.info "#{t.state}: #{t.id} has failed requires task"
|
42
59
|
else
|
43
|
-
|
44
|
-
|
60
|
+
logger.info "run: #{t.id}"
|
61
|
+
t.state = :running
|
62
|
+
|
63
|
+
begin
|
64
|
+
Retriable.retriable retry_options do
|
65
|
+
t.run
|
66
|
+
end
|
67
|
+
rescue => e
|
68
|
+
t.state = :failed
|
69
|
+
logger.info "#{t.state}: #{t.id}"
|
70
|
+
logger.error "#{e.message}"
|
71
|
+
logger.error e.backtrace.join("\n")
|
72
|
+
else
|
73
|
+
t.state = :completed
|
74
|
+
logger.info "#{t.state}: #{t.id}"
|
75
|
+
end
|
45
76
|
end
|
46
77
|
end
|
47
78
|
end
|
48
|
-
|
49
|
-
show_result_report(dag)
|
50
|
-
return !dag.tsort.any? { |t| t.state == :failed }
|
51
79
|
end
|
52
80
|
|
53
|
-
private
|
54
|
-
|
55
81
|
def retry_options
|
56
82
|
{
|
57
83
|
tries: Tumugi.config.max_retry,
|
@@ -67,9 +93,9 @@ module Tumugi
|
|
67
93
|
def on_retry
|
68
94
|
Proc.new do |exception, try, elapsed_time, next_interval|
|
69
95
|
if next_interval
|
70
|
-
|
96
|
+
logger.error "#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds and #{next_interval} seconds until the next try."
|
71
97
|
else
|
72
|
-
|
98
|
+
logger.error "#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds. Task failed."
|
73
99
|
end
|
74
100
|
end
|
75
101
|
end
|
@@ -88,7 +114,11 @@ module Tumugi
|
|
88
114
|
t << [ task.id, requires.join("\n"), params.join("\n"), task.state ]
|
89
115
|
end
|
90
116
|
end
|
91
|
-
|
117
|
+
logger.info "Result report:\n#{table.to_s}"
|
118
|
+
end
|
119
|
+
|
120
|
+
def logger
|
121
|
+
Tumugi::Logger.instance
|
92
122
|
end
|
93
123
|
end
|
94
124
|
end
|
data/lib/tumugi/command/show.rb
CHANGED
@@ -32,6 +32,7 @@ module Tumugi
|
|
32
32
|
end
|
33
33
|
|
34
34
|
if out
|
35
|
+
logger.info "output result to #{out}"
|
35
36
|
FileUtils.mkdir_p(File.dirname(out))
|
36
37
|
if format == 'dot'
|
37
38
|
File.write(out, g.to_s)
|
@@ -44,6 +45,10 @@ module Tumugi
|
|
44
45
|
|
45
46
|
return true
|
46
47
|
end
|
48
|
+
|
49
|
+
def logger
|
50
|
+
Tumugi::Logger.instance
|
51
|
+
end
|
47
52
|
end
|
48
53
|
end
|
49
54
|
end
|
data/lib/tumugi/config.rb
CHANGED
@@ -1,15 +1,18 @@
|
|
1
|
-
|
2
|
-
class ConfigError < StandardError
|
3
|
-
end
|
1
|
+
require 'tumugi/error'
|
4
2
|
|
3
|
+
module Tumugi
|
5
4
|
class Config
|
6
|
-
attr_accessor :workers
|
5
|
+
attr_accessor :workers
|
6
|
+
attr_accessor :max_retry
|
7
|
+
attr_accessor :retry_interval
|
8
|
+
attr_accessor :param_auto_bind_enabled
|
9
|
+
attr_accessor :timeout
|
7
10
|
|
8
11
|
@@sections ||= {}
|
9
12
|
|
10
13
|
def self.register_section(name, *args)
|
11
14
|
@@sections[name] = Struct.new(camelize(name), *args)
|
12
|
-
|
15
|
+
logger.debug "registered config section '#{name}' with '#{args}'"
|
13
16
|
end
|
14
17
|
|
15
18
|
def self.camelize(term)
|
@@ -19,33 +22,43 @@ module Tumugi
|
|
19
22
|
string
|
20
23
|
end
|
21
24
|
|
25
|
+
def self.logger
|
26
|
+
Tumugi::Logger.instance
|
27
|
+
end
|
28
|
+
|
22
29
|
def initialize
|
23
30
|
@workers = 1
|
24
31
|
@max_retry = 3
|
25
32
|
@retry_interval = 300 #seconds
|
26
33
|
@param_auto_bind_enabled = true
|
34
|
+
@timeout = 0 # meaning no timeout
|
35
|
+
|
27
36
|
@section_procs = {}
|
28
37
|
@section_instances = {}
|
29
38
|
end
|
30
39
|
|
31
40
|
def section(name, &block)
|
32
|
-
section_class = @@sections[name]
|
33
41
|
if block_given?
|
42
|
+
raise Tumugi::ConfigError.new('You cannot change section') if frozen?
|
34
43
|
@section_procs[name] ||= block
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
44
|
+
return nil
|
45
|
+
end
|
46
|
+
|
47
|
+
section_class = @@sections[name]
|
48
|
+
if section_class.nil?
|
49
|
+
raise ConfigError.new("Config section '#{name}' is not registered.")
|
50
|
+
end
|
51
|
+
|
52
|
+
if @section_instances[name].nil?
|
53
|
+
@section_instances[name] = section_class.new
|
54
|
+
begin
|
55
|
+
@section_procs[name].call(@section_instances[name])
|
56
|
+
rescue NoMethodError => e
|
57
|
+
logger.error "#{e.message}. Available attributes are #{@section_instances[name].members}"
|
58
|
+
raise e
|
59
|
+
end if @section_procs[name]
|
48
60
|
end
|
61
|
+
@section_instances[name].clone.freeze
|
49
62
|
end
|
50
63
|
end
|
51
64
|
end
|
data/lib/tumugi/error.rb
CHANGED
@@ -1,4 +1,19 @@
|
|
1
1
|
module Tumugi
|
2
2
|
class TumugiError < StandardError
|
3
|
+
attr_reader :reason
|
4
|
+
|
5
|
+
def initialize(message=nil, reason=nil)
|
6
|
+
super(message)
|
7
|
+
@reason = reason
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
class ConfigError < TumugiError
|
12
|
+
end
|
13
|
+
|
14
|
+
class ParameterError < TumugiError
|
15
|
+
end
|
16
|
+
|
17
|
+
class TimeoutError < TumugiError
|
3
18
|
end
|
4
19
|
end
|
data/lib/tumugi/file_system.rb
CHANGED
@@ -30,7 +30,7 @@ module Tumugi
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def rename(path, dest)
|
33
|
-
Tumugi.
|
33
|
+
Tumugi::Logger.instance.warn "File system #{self.class.name} client doesn't support atomic move."
|
34
34
|
raise FileAlreadyExistError if exist?(dest)
|
35
35
|
move(path, dest)
|
36
36
|
end
|
data/lib/tumugi/logger.rb
CHANGED
@@ -1,13 +1,19 @@
|
|
1
1
|
require 'logger'
|
2
2
|
require 'forwardable'
|
3
|
+
require 'singleton'
|
3
4
|
|
4
5
|
module Tumugi
|
5
6
|
class Logger
|
7
|
+
include Singleton
|
6
8
|
extend Forwardable
|
7
9
|
def_delegators :@logger, :debug, :error, :fatal, :info, :warn, :level
|
8
10
|
|
9
11
|
def initialize
|
10
|
-
|
12
|
+
init
|
13
|
+
end
|
14
|
+
|
15
|
+
def init(output=STDOUT)
|
16
|
+
@logger = ::Logger.new(output)
|
11
17
|
@logger.level = ::Logger::INFO
|
12
18
|
end
|
13
19
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'tumugi/
|
1
|
+
require 'tumugi/error'
|
2
2
|
require 'tumugi/parameter/parameter_proxy'
|
3
3
|
|
4
4
|
module Tumugi
|
@@ -30,7 +30,7 @@ module Tumugi
|
|
30
30
|
def validate_params(params)
|
31
31
|
params.each do |name, param|
|
32
32
|
if param.required? && instance_variable_get("@#{name}").nil?
|
33
|
-
raise Tumugi::
|
33
|
+
raise Tumugi::ParameterError.new("Parameter #{name} is required")
|
34
34
|
end
|
35
35
|
end
|
36
36
|
end
|
@@ -1,5 +1,5 @@
|
|
1
|
+
require 'tumugi/error'
|
1
2
|
require 'tumugi/parameter/converter'
|
2
|
-
require 'tumugi/parameter/error'
|
3
3
|
|
4
4
|
module Tumugi
|
5
5
|
module Parameter
|
@@ -17,11 +17,8 @@ module Tumugi
|
|
17
17
|
def get
|
18
18
|
if auto_bind?
|
19
19
|
value = search_from_application_parameters
|
20
|
-
value = search_from_env if value.nil?
|
21
20
|
end
|
22
|
-
|
23
|
-
return value unless value.nil?
|
24
|
-
default_value
|
21
|
+
value.nil? ? default_value : value
|
25
22
|
end
|
26
23
|
|
27
24
|
def auto_bind?
|
@@ -60,19 +57,11 @@ module Tumugi
|
|
60
57
|
value ? Converter.convert(type, value) : nil
|
61
58
|
end
|
62
59
|
|
63
|
-
def search_from_env
|
64
|
-
key = @name.to_s
|
65
|
-
value = nil
|
66
|
-
value = ENV[key] if ENV.has_key?(key)
|
67
|
-
value = ENV[key.upcase] if ENV.has_key?(key.upcase)
|
68
|
-
value ? Converter.convert(type, value) : nil
|
69
|
-
end
|
70
|
-
|
71
60
|
private
|
72
61
|
|
73
62
|
def validate
|
74
63
|
if required? && default_value != nil
|
75
|
-
raise ParameterError.new("When you set required: true, you cannot set default value")
|
64
|
+
raise Tumugi::ParameterError.new("When you set required: true, you cannot set default value")
|
76
65
|
end
|
77
66
|
end
|
78
67
|
end
|
data/lib/tumugi/plugin.rb
CHANGED
@@ -19,7 +19,7 @@ module Tumugi
|
|
19
19
|
raise "Invalid implementation as #{kind} plugin: '#{type}'. It must be a Class."
|
20
20
|
end
|
21
21
|
registry.register(type, value)
|
22
|
-
Tumugi.
|
22
|
+
Tumugi::Logger.instance.debug "registered #{kind} plugin '#{type}'"
|
23
23
|
nil
|
24
24
|
end
|
25
25
|
|
data/lib/tumugi/target.rb
CHANGED
data/lib/tumugi/task.rb
CHANGED
@@ -36,7 +36,7 @@ module Tumugi
|
|
36
36
|
end
|
37
37
|
|
38
38
|
def logger
|
39
|
-
@logger ||= Tumugi.
|
39
|
+
@logger ||= Tumugi::Logger.instance
|
40
40
|
end
|
41
41
|
|
42
42
|
def log(msg)
|
@@ -79,6 +79,10 @@ module Tumugi
|
|
79
79
|
list(_requires).any? { |t| t.instance.state == :failed || t.instance.state == :requires_failed }
|
80
80
|
end
|
81
81
|
|
82
|
+
def timeout
|
83
|
+
nil # meaning use default timeout
|
84
|
+
end
|
85
|
+
|
82
86
|
# Following methods are internal use only
|
83
87
|
|
84
88
|
def _requires
|
data/lib/tumugi/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tumugi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kazuyuki Honda
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-05-
|
11
|
+
date: 2016-05-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: parallel
|
@@ -211,7 +211,6 @@ files:
|
|
211
211
|
- lib/tumugi/mixin/parameterizable.rb
|
212
212
|
- lib/tumugi/mixin/task_helper.rb
|
213
213
|
- lib/tumugi/parameter/converter.rb
|
214
|
-
- lib/tumugi/parameter/error.rb
|
215
214
|
- lib/tumugi/parameter/parameter.rb
|
216
215
|
- lib/tumugi/parameter/parameter_proxy.rb
|
217
216
|
- lib/tumugi/plugin.rb
|
@@ -219,6 +218,7 @@ files:
|
|
219
218
|
- lib/tumugi/plugin/file_system_target.rb
|
220
219
|
- lib/tumugi/plugin/local_file_system.rb
|
221
220
|
- lib/tumugi/plugin/target/local_file.rb
|
221
|
+
- lib/tumugi/plugin/task/external.rb
|
222
222
|
- lib/tumugi/registry.rb
|
223
223
|
- lib/tumugi/target.rb
|
224
224
|
- lib/tumugi/task.rb
|