tumugi 0.4.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +10 -16
- data/CHANGELOG.md +31 -2
- data/examples/concurrent_task_run.rb +1 -1
- data/examples/data_pipeline.rb +2 -2
- data/examples/target.rb +4 -4
- data/examples/task_inheritance.rb +1 -1
- data/examples/tumugi_config.rb +3 -3
- data/examples/tumugi_config_with_section.rb +5 -5
- data/lib/tumugi.rb +12 -4
- data/lib/tumugi/application.rb +12 -7
- data/lib/tumugi/cli.rb +13 -5
- data/lib/tumugi/command/run.rb +62 -32
- data/lib/tumugi/command/show.rb +5 -0
- data/lib/tumugi/config.rb +32 -19
- data/lib/tumugi/error.rb +15 -0
- data/lib/tumugi/file_system.rb +1 -1
- data/lib/tumugi/logger.rb +7 -1
- data/lib/tumugi/mixin/parameterizable.rb +2 -2
- data/lib/tumugi/parameter/parameter.rb +3 -14
- data/lib/tumugi/plugin.rb +1 -1
- data/lib/tumugi/plugin/target/local_file.rb +1 -1
- data/lib/tumugi/plugin/task/external.rb +13 -0
- data/lib/tumugi/target.rb +1 -1
- data/lib/tumugi/task.rb +5 -1
- data/lib/tumugi/version.rb +1 -1
- metadata +3 -3
- data/lib/tumugi/parameter/error.rb +0 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a8a889ceab4e85663023c13142fd19f4b427fb5c
|
4
|
+
data.tar.gz: 67ed6e06b39c20fe355b456cb1a84751b23544e6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ede2235c0f2c6810fce09093c78422ae152a2f0a3ccd3e6275856d5aabd5308f84846042467d032652c6bd5cfe96b6fd68428e32222f91d44af00630b225b2a6
|
7
|
+
data.tar.gz: 3841d5750ab943219623530d0032ddcf0532c21e6d3b4fc09e1cdf643f59c78e26cc1f9ab0e69893408e68d4ea19d150584a7cecb9848c0a17316f97176ce331
|
data/.travis.yml
CHANGED
@@ -1,23 +1,17 @@
|
|
1
1
|
language: ruby
|
2
|
-
|
2
|
+
cache: bundler
|
3
|
+
rvm:
|
4
|
+
- 2.1.10
|
5
|
+
- 2.2.5
|
6
|
+
- 2.3.1
|
7
|
+
- ruby-head
|
8
|
+
- jruby-9.0.5.0
|
9
|
+
before_install:
|
10
|
+
- gem install bundler
|
3
11
|
matrix:
|
4
|
-
include:
|
5
|
-
- rvm: 2.1.10
|
6
|
-
os: linux
|
7
|
-
- rvm: 2.2.5
|
8
|
-
os: linux
|
9
|
-
- rvm: 2.3.1
|
10
|
-
os: linux
|
11
|
-
- rvm: ruby-head
|
12
|
-
os: linux
|
13
|
-
- rvm: jruby-9.0.5.0
|
14
|
-
os: linux
|
15
12
|
allow_failures:
|
16
13
|
- rvm: ruby-head
|
17
|
-
|
18
|
-
cache: bundler
|
19
|
-
|
20
14
|
addons:
|
21
15
|
apt:
|
22
16
|
packages:
|
23
|
-
|
17
|
+
- graphviz
|
data/CHANGELOG.md
CHANGED
@@ -1,12 +1,41 @@
|
|
1
1
|
# Change Log
|
2
2
|
|
3
|
-
## [0.
|
4
|
-
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.4.
|
3
|
+
## [0.5.0](https://github.com/tumugi/tumugi/tree/0.5.0) (2016-05-26)
|
4
|
+
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.4.5...0.5.0)
|
5
|
+
|
6
|
+
**Implemented enhancements:**
|
7
|
+
|
8
|
+
- Add ExternalTask as bundled plugin [\#61](https://github.com/tumugi/tumugi/issues/61)
|
9
|
+
- \[Breaking Change\] Tumugi.configure [\#60](https://github.com/tumugi/tumugi/issues/60)
|
10
|
+
- Better task scheduling [\#57](https://github.com/tumugi/tumugi/issues/57)
|
11
|
+
- \[Breaking Change\] Disable parameter auto binding from ENV [\#56](https://github.com/tumugi/tumugi/issues/56)
|
12
|
+
- Timeout feature [\#54](https://github.com/tumugi/tumugi/issues/54)
|
13
|
+
- \[Breaking Change\] Restruct Error class [\#51](https://github.com/tumugi/tumugi/issues/51)
|
14
|
+
- Enhance logger [\#36](https://github.com/tumugi/tumugi/issues/36)
|
15
|
+
|
16
|
+
**Merged pull requests:**
|
17
|
+
|
18
|
+
- Call init first Tumugi::Logger instance access [\#67](https://github.com/tumugi/tumugi/pull/67) ([hakobera](https://github.com/hakobera))
|
19
|
+
- Better task scheduling. [\#66](https://github.com/tumugi/tumugi/pull/66) ([hakobera](https://github.com/hakobera))
|
20
|
+
- Support log output to file [\#65](https://github.com/tumugi/tumugi/pull/65) ([hakobera](https://github.com/hakobera))
|
21
|
+
- Add EternalTask as bundled plugin [\#64](https://github.com/tumugi/tumugi/pull/64) ([hakobera](https://github.com/hakobera))
|
22
|
+
- Add Tumugi.configure method to modify config. [\#63](https://github.com/tumugi/tumugi/pull/63) ([hakobera](https://github.com/hakobera))
|
23
|
+
- Implement timeout feature [\#62](https://github.com/tumugi/tumugi/pull/62) ([hakobera](https://github.com/hakobera))
|
24
|
+
- Disable parameter auto binding from ENV [\#59](https://github.com/tumugi/tumugi/pull/59) ([hakobera](https://github.com/hakobera))
|
25
|
+
- Restruct error handling [\#58](https://github.com/tumugi/tumugi/pull/58) ([hakobera](https://github.com/hakobera))
|
26
|
+
- Change CLI test to fix failed test with ruby-head [\#52](https://github.com/tumugi/tumugi/pull/52) ([hakobera](https://github.com/hakobera))
|
27
|
+
|
28
|
+
## [v0.4.5](https://github.com/tumugi/tumugi/tree/v0.4.5) (2016-05-16)
|
29
|
+
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.4.4...v0.4.5)
|
5
30
|
|
6
31
|
**Fixed bugs:**
|
7
32
|
|
8
33
|
- Fix required parameter validation [\#49](https://github.com/tumugi/tumugi/pull/49) ([hakobera](https://github.com/hakobera))
|
9
34
|
|
35
|
+
**Merged pull requests:**
|
36
|
+
|
37
|
+
- Prepare release for v0.4.5 [\#50](https://github.com/tumugi/tumugi/pull/50) ([hakobera](https://github.com/hakobera))
|
38
|
+
|
10
39
|
## [v0.4.4](https://github.com/tumugi/tumugi/tree/v0.4.4) (2016-05-13)
|
11
40
|
[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.4.3...v0.4.4)
|
12
41
|
|
data/examples/data_pipeline.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
task :generate_data do
|
2
2
|
output do
|
3
|
-
target(:local_file, "
|
3
|
+
target(:local_file, "tmp/tumugi_data_#{Time.now.strftime('%Y-%m-%d')}.txt")
|
4
4
|
end
|
5
5
|
|
6
6
|
run do
|
@@ -16,7 +16,7 @@ task :sum do
|
|
16
16
|
requires :generate_data
|
17
17
|
|
18
18
|
output do
|
19
|
-
target(:local_file, "
|
19
|
+
target(:local_file, "tmp/tumugi_output_#{Time.now.strftime('%Y-%m-%d')}.txt")
|
20
20
|
end
|
21
21
|
|
22
22
|
run do
|
data/examples/target.rb
CHANGED
@@ -2,7 +2,7 @@ require 'tumugi/plugin/target/local_file'
|
|
2
2
|
|
3
3
|
task :task1 do
|
4
4
|
requires [:task2, :task3]
|
5
|
-
output target(:local_file, "
|
5
|
+
output target(:local_file, "tmp/tumugi_#{id}.txt")
|
6
6
|
run do
|
7
7
|
log 'task1#run'
|
8
8
|
output.open('w') {|f| f.puts('done') }
|
@@ -11,7 +11,7 @@ end
|
|
11
11
|
|
12
12
|
task :task2 do
|
13
13
|
requires [:task4]
|
14
|
-
output [target(:local_file, "
|
14
|
+
output [target(:local_file, "tmp/tumugi_#{id}.txt")]
|
15
15
|
run do
|
16
16
|
log 'task2#run'
|
17
17
|
output[0].open('w') {|f| f.puts('done') }
|
@@ -20,7 +20,7 @@ end
|
|
20
20
|
|
21
21
|
task :task3 do
|
22
22
|
requires [:task4]
|
23
|
-
output { target(:local_file, "
|
23
|
+
output { target(:local_file, "tmp/tumugi_#{id}.txt") }
|
24
24
|
run do
|
25
25
|
log 'task3#run'
|
26
26
|
output.open('w') {|f| f.puts('done') }
|
@@ -28,7 +28,7 @@ task :task3 do
|
|
28
28
|
end
|
29
29
|
|
30
30
|
task :task4 do
|
31
|
-
output Tumugi::Plugin::LocalFileTarget.new("
|
31
|
+
output Tumugi::Plugin::LocalFileTarget.new("tmp/tumugi_#{id}.txt")
|
32
32
|
run do
|
33
33
|
log 'task4#run'
|
34
34
|
output.open('w') {|f| f.puts('done') }
|
data/examples/tumugi_config.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
Tumugi.
|
2
|
-
|
3
|
-
|
1
|
+
Tumugi.configure do |config|
|
2
|
+
config.max_retry = 3
|
3
|
+
config.retry_interval = 1
|
4
4
|
end
|
@@ -1,8 +1,8 @@
|
|
1
|
-
Tumugi.
|
2
|
-
|
3
|
-
|
1
|
+
Tumugi.configure do |config|
|
2
|
+
config.max_retry = 3
|
3
|
+
config.retry_interval = 1
|
4
4
|
|
5
|
-
|
6
|
-
|
5
|
+
config.section('example') do |section|
|
6
|
+
section.key = 'value'
|
7
7
|
end
|
8
8
|
end
|
data/lib/tumugi.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'tumugi/application'
|
2
2
|
require 'tumugi/config'
|
3
|
+
require 'tumugi/error'
|
3
4
|
require 'tumugi/logger'
|
4
5
|
require 'tumugi/version'
|
5
6
|
|
@@ -9,14 +10,21 @@ module Tumugi
|
|
9
10
|
@application ||= Tumugi::Application.new
|
10
11
|
end
|
11
12
|
|
12
|
-
def
|
13
|
-
|
13
|
+
def configure(&block)
|
14
|
+
raise Tumugi::ConfigError.new 'Tumugi.configure must have block' unless block_given?
|
15
|
+
yield _config
|
16
|
+
nil
|
14
17
|
end
|
15
18
|
|
16
19
|
def config
|
20
|
+
raise Tumugi::ConfigError.new 'Tumugi.config with block is deprecated. Use Tumugi.configure instead.' if block_given?
|
21
|
+
_config.clone.freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def _config
|
17
27
|
@config ||= Tumugi::Config.new
|
18
|
-
yield @config if block_given?
|
19
|
-
@config
|
20
28
|
end
|
21
29
|
end
|
22
30
|
end
|
data/lib/tumugi/application.rb
CHANGED
@@ -16,7 +16,7 @@ module Tumugi
|
|
16
16
|
end
|
17
17
|
|
18
18
|
def execute(command, root_task_id, options)
|
19
|
-
process_common_options(options)
|
19
|
+
process_common_options(command, options)
|
20
20
|
load_workflow_file(options[:file])
|
21
21
|
dag = create_dag(root_task_id)
|
22
22
|
command_module = Kernel.const_get("Tumugi").const_get("Command")
|
@@ -44,7 +44,7 @@ module Tumugi
|
|
44
44
|
begin
|
45
45
|
load(file, true)
|
46
46
|
rescue LoadError => e
|
47
|
-
raise Tumugi::TumugiError, e
|
47
|
+
raise Tumugi::TumugiError.new("Workflow file load error: #{file}", e)
|
48
48
|
end
|
49
49
|
end
|
50
50
|
|
@@ -55,17 +55,22 @@ module Tumugi
|
|
55
55
|
dag
|
56
56
|
end
|
57
57
|
|
58
|
-
def process_common_options(options)
|
59
|
-
setup_logger(options)
|
58
|
+
def process_common_options(command, options)
|
59
|
+
setup_logger(command, options)
|
60
60
|
load_config(options)
|
61
61
|
set_params(options)
|
62
62
|
end
|
63
63
|
|
64
64
|
def logger
|
65
|
-
@logger ||= Tumugi.
|
65
|
+
@logger ||= Tumugi::Logger.instance
|
66
66
|
end
|
67
67
|
|
68
|
-
def setup_logger(options)
|
68
|
+
def setup_logger(command, options)
|
69
|
+
if command == :run && !options[:out].nil?
|
70
|
+
Tumugi::Logger.instance.init(options[:out])
|
71
|
+
else
|
72
|
+
Tumugi::Logger.instance.init
|
73
|
+
end
|
69
74
|
logger.verbose! if options[:verbose]
|
70
75
|
logger.quiet! if options[:quiet]
|
71
76
|
end
|
@@ -77,7 +82,7 @@ module Tumugi
|
|
77
82
|
load(config_file)
|
78
83
|
end
|
79
84
|
rescue LoadError => e
|
80
|
-
raise Tumugi::TumugiError, e
|
85
|
+
raise Tumugi::TumugiError.new("Config file load error: #{config_file}", e)
|
81
86
|
end
|
82
87
|
|
83
88
|
def set_params(options)
|
data/lib/tumugi/cli.rb
CHANGED
@@ -27,6 +27,7 @@ module Tumugi
|
|
27
27
|
desc "run TASK", "Run TASK in a workflow"
|
28
28
|
map "run" => "run_" # run is thor's reserved word, so this trick is needed
|
29
29
|
option :workers, aliases: '-w', type: :numeric, desc: 'Number of workers to run task concurrently'
|
30
|
+
option :out, aliases: '-o', desc: 'Output log filename. If not specified, log is write to STDOUT'
|
30
31
|
common_options
|
31
32
|
def run_(task)
|
32
33
|
execute(:run, task, options)
|
@@ -37,7 +38,9 @@ module Tumugi
|
|
37
38
|
option :out, aliases: '-o', desc: 'Output file name. If not specified, output result to STDOUT'
|
38
39
|
option :format, aliases: '-t', desc: 'Output file format. Only affected --out option is specified.', enum: ['dot', 'png', 'svg']
|
39
40
|
def show(task)
|
40
|
-
|
41
|
+
opts = options.dup
|
42
|
+
opts[:quiet] = true if opts[:out].nil?
|
43
|
+
execute(:show, task, opts.freeze)
|
41
44
|
end
|
42
45
|
|
43
46
|
private
|
@@ -45,14 +48,19 @@ module Tumugi
|
|
45
48
|
def execute(command, task, options)
|
46
49
|
success = Tumugi.application.execute(command, task, options)
|
47
50
|
unless success
|
48
|
-
|
51
|
+
logger.error "#{command} command failed"
|
49
52
|
raise Thor::Error, 'failed'
|
50
53
|
end
|
54
|
+
success
|
51
55
|
rescue => e
|
52
|
-
|
53
|
-
|
54
|
-
e.backtrace.each { |line|
|
56
|
+
logger.error "#{command} command failed"
|
57
|
+
logger.error e.message
|
58
|
+
e.backtrace.each { |line| logger.error line }
|
55
59
|
raise Thor::Error, 'failed'
|
56
60
|
end
|
61
|
+
|
62
|
+
def logger
|
63
|
+
Tumugi::Logger.instance
|
64
|
+
end
|
57
65
|
end
|
58
66
|
end
|
data/lib/tumugi/command/run.rb
CHANGED
@@ -2,7 +2,9 @@ require 'parallel'
|
|
2
2
|
require 'retriable'
|
3
3
|
require 'terminal-table'
|
4
4
|
require 'thor'
|
5
|
+
require 'timeout'
|
5
6
|
|
7
|
+
require 'tumugi/error'
|
6
8
|
require 'tumugi/mixin/listable'
|
7
9
|
|
8
10
|
module Tumugi
|
@@ -13,45 +15,69 @@ module Tumugi
|
|
13
15
|
def execute(dag, options={})
|
14
16
|
workers = options[:workers] || Tumugi.config.workers
|
15
17
|
settings = { in_threads: workers }
|
16
|
-
|
18
|
+
start(task_queue(dag), settings)
|
19
|
+
show_result_report(dag)
|
20
|
+
!dag.tsort.any? { |t| t.state == :failed }
|
21
|
+
end
|
17
22
|
|
18
|
-
|
19
|
-
logger.info "start: #{t.id}"
|
20
|
-
until t.ready? || t.requires_failed?
|
21
|
-
sleep 1
|
22
|
-
end
|
23
|
+
private
|
23
24
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
25
|
+
def task_queue(dag)
|
26
|
+
waiting_task_queue = Queue.new
|
27
|
+
available_task_queue = Queue.new
|
28
|
+
dag.tsort.each do |t|
|
29
|
+
if t.ready?
|
30
|
+
available_task_queue << t
|
30
31
|
else
|
31
|
-
|
32
|
-
|
32
|
+
waiting_task_queue << t
|
33
|
+
end
|
34
|
+
end
|
35
|
+
lambda {
|
36
|
+
task = (available_task_queue.empty? ? nil : available_task_queue.pop)
|
37
|
+
if task.nil?
|
38
|
+
task = (waiting_task_queue.empty? ? nil : waiting_task_queue.pop)
|
39
|
+
end
|
40
|
+
task || Parallel::Stop
|
41
|
+
}
|
42
|
+
end
|
33
43
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
44
|
+
def start(proc, settings)
|
45
|
+
Parallel.each(proc, settings) do |t|
|
46
|
+
logger.info "start: #{t.id}"
|
47
|
+
timeout = t.timeout || Tumugi.config.timeout
|
48
|
+
Timeout::timeout(timeout, Tumugi::TimeoutError) do
|
49
|
+
until t.ready? || t.requires_failed?
|
50
|
+
sleep 5
|
51
|
+
end
|
52
|
+
|
53
|
+
if t.completed?
|
54
|
+
t.state = :skipped
|
55
|
+
logger.info "#{t.state}: #{t.id} is already completed"
|
56
|
+
elsif t.requires_failed?
|
57
|
+
t.state = :requires_failed
|
58
|
+
logger.info "#{t.state}: #{t.id} has failed requires task"
|
42
59
|
else
|
43
|
-
|
44
|
-
|
60
|
+
logger.info "run: #{t.id}"
|
61
|
+
t.state = :running
|
62
|
+
|
63
|
+
begin
|
64
|
+
Retriable.retriable retry_options do
|
65
|
+
t.run
|
66
|
+
end
|
67
|
+
rescue => e
|
68
|
+
t.state = :failed
|
69
|
+
logger.info "#{t.state}: #{t.id}"
|
70
|
+
logger.error "#{e.message}"
|
71
|
+
logger.error e.backtrace.join("\n")
|
72
|
+
else
|
73
|
+
t.state = :completed
|
74
|
+
logger.info "#{t.state}: #{t.id}"
|
75
|
+
end
|
45
76
|
end
|
46
77
|
end
|
47
78
|
end
|
48
|
-
|
49
|
-
show_result_report(dag)
|
50
|
-
return !dag.tsort.any? { |t| t.state == :failed }
|
51
79
|
end
|
52
80
|
|
53
|
-
private
|
54
|
-
|
55
81
|
def retry_options
|
56
82
|
{
|
57
83
|
tries: Tumugi.config.max_retry,
|
@@ -67,9 +93,9 @@ module Tumugi
|
|
67
93
|
def on_retry
|
68
94
|
Proc.new do |exception, try, elapsed_time, next_interval|
|
69
95
|
if next_interval
|
70
|
-
|
96
|
+
logger.error "#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds and #{next_interval} seconds until the next try."
|
71
97
|
else
|
72
|
-
|
98
|
+
logger.error "#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds. Task failed."
|
73
99
|
end
|
74
100
|
end
|
75
101
|
end
|
@@ -88,7 +114,11 @@ module Tumugi
|
|
88
114
|
t << [ task.id, requires.join("\n"), params.join("\n"), task.state ]
|
89
115
|
end
|
90
116
|
end
|
91
|
-
|
117
|
+
logger.info "Result report:\n#{table.to_s}"
|
118
|
+
end
|
119
|
+
|
120
|
+
def logger
|
121
|
+
Tumugi::Logger.instance
|
92
122
|
end
|
93
123
|
end
|
94
124
|
end
|
data/lib/tumugi/command/show.rb
CHANGED
@@ -32,6 +32,7 @@ module Tumugi
|
|
32
32
|
end
|
33
33
|
|
34
34
|
if out
|
35
|
+
logger.info "output result to #{out}"
|
35
36
|
FileUtils.mkdir_p(File.dirname(out))
|
36
37
|
if format == 'dot'
|
37
38
|
File.write(out, g.to_s)
|
@@ -44,6 +45,10 @@ module Tumugi
|
|
44
45
|
|
45
46
|
return true
|
46
47
|
end
|
48
|
+
|
49
|
+
def logger
|
50
|
+
Tumugi::Logger.instance
|
51
|
+
end
|
47
52
|
end
|
48
53
|
end
|
49
54
|
end
|
data/lib/tumugi/config.rb
CHANGED
@@ -1,15 +1,18 @@
|
|
1
|
-
|
2
|
-
class ConfigError < StandardError
|
3
|
-
end
|
1
|
+
require 'tumugi/error'
|
4
2
|
|
3
|
+
module Tumugi
|
5
4
|
class Config
|
6
|
-
attr_accessor :workers
|
5
|
+
attr_accessor :workers
|
6
|
+
attr_accessor :max_retry
|
7
|
+
attr_accessor :retry_interval
|
8
|
+
attr_accessor :param_auto_bind_enabled
|
9
|
+
attr_accessor :timeout
|
7
10
|
|
8
11
|
@@sections ||= {}
|
9
12
|
|
10
13
|
def self.register_section(name, *args)
|
11
14
|
@@sections[name] = Struct.new(camelize(name), *args)
|
12
|
-
|
15
|
+
logger.debug "registered config section '#{name}' with '#{args}'"
|
13
16
|
end
|
14
17
|
|
15
18
|
def self.camelize(term)
|
@@ -19,33 +22,43 @@ module Tumugi
|
|
19
22
|
string
|
20
23
|
end
|
21
24
|
|
25
|
+
def self.logger
|
26
|
+
Tumugi::Logger.instance
|
27
|
+
end
|
28
|
+
|
22
29
|
def initialize
|
23
30
|
@workers = 1
|
24
31
|
@max_retry = 3
|
25
32
|
@retry_interval = 300 #seconds
|
26
33
|
@param_auto_bind_enabled = true
|
34
|
+
@timeout = 0 # meaning no timeout
|
35
|
+
|
27
36
|
@section_procs = {}
|
28
37
|
@section_instances = {}
|
29
38
|
end
|
30
39
|
|
31
40
|
def section(name, &block)
|
32
|
-
section_class = @@sections[name]
|
33
41
|
if block_given?
|
42
|
+
raise Tumugi::ConfigError.new('You cannot change section') if frozen?
|
34
43
|
@section_procs[name] ||= block
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
44
|
+
return nil
|
45
|
+
end
|
46
|
+
|
47
|
+
section_class = @@sections[name]
|
48
|
+
if section_class.nil?
|
49
|
+
raise ConfigError.new("Config section '#{name}' is not registered.")
|
50
|
+
end
|
51
|
+
|
52
|
+
if @section_instances[name].nil?
|
53
|
+
@section_instances[name] = section_class.new
|
54
|
+
begin
|
55
|
+
@section_procs[name].call(@section_instances[name])
|
56
|
+
rescue NoMethodError => e
|
57
|
+
logger.error "#{e.message}. Available attributes are #{@section_instances[name].members}"
|
58
|
+
raise e
|
59
|
+
end if @section_procs[name]
|
48
60
|
end
|
61
|
+
@section_instances[name].clone.freeze
|
49
62
|
end
|
50
63
|
end
|
51
64
|
end
|
data/lib/tumugi/error.rb
CHANGED
@@ -1,4 +1,19 @@
|
|
1
1
|
module Tumugi
|
2
2
|
class TumugiError < StandardError
|
3
|
+
attr_reader :reason
|
4
|
+
|
5
|
+
def initialize(message=nil, reason=nil)
|
6
|
+
super(message)
|
7
|
+
@reason = reason
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
class ConfigError < TumugiError
|
12
|
+
end
|
13
|
+
|
14
|
+
class ParameterError < TumugiError
|
15
|
+
end
|
16
|
+
|
17
|
+
class TimeoutError < TumugiError
|
3
18
|
end
|
4
19
|
end
|
data/lib/tumugi/file_system.rb
CHANGED
@@ -30,7 +30,7 @@ module Tumugi
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def rename(path, dest)
|
33
|
-
Tumugi.
|
33
|
+
Tumugi::Logger.instance.warn "File system #{self.class.name} client doesn't support atomic move."
|
34
34
|
raise FileAlreadyExistError if exist?(dest)
|
35
35
|
move(path, dest)
|
36
36
|
end
|
data/lib/tumugi/logger.rb
CHANGED
@@ -1,13 +1,19 @@
|
|
1
1
|
require 'logger'
|
2
2
|
require 'forwardable'
|
3
|
+
require 'singleton'
|
3
4
|
|
4
5
|
module Tumugi
|
5
6
|
class Logger
|
7
|
+
include Singleton
|
6
8
|
extend Forwardable
|
7
9
|
def_delegators :@logger, :debug, :error, :fatal, :info, :warn, :level
|
8
10
|
|
9
11
|
def initialize
|
10
|
-
|
12
|
+
init
|
13
|
+
end
|
14
|
+
|
15
|
+
def init(output=STDOUT)
|
16
|
+
@logger = ::Logger.new(output)
|
11
17
|
@logger.level = ::Logger::INFO
|
12
18
|
end
|
13
19
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'tumugi/
|
1
|
+
require 'tumugi/error'
|
2
2
|
require 'tumugi/parameter/parameter_proxy'
|
3
3
|
|
4
4
|
module Tumugi
|
@@ -30,7 +30,7 @@ module Tumugi
|
|
30
30
|
def validate_params(params)
|
31
31
|
params.each do |name, param|
|
32
32
|
if param.required? && instance_variable_get("@#{name}").nil?
|
33
|
-
raise Tumugi::
|
33
|
+
raise Tumugi::ParameterError.new("Parameter #{name} is required")
|
34
34
|
end
|
35
35
|
end
|
36
36
|
end
|
@@ -1,5 +1,5 @@
|
|
1
|
+
require 'tumugi/error'
|
1
2
|
require 'tumugi/parameter/converter'
|
2
|
-
require 'tumugi/parameter/error'
|
3
3
|
|
4
4
|
module Tumugi
|
5
5
|
module Parameter
|
@@ -17,11 +17,8 @@ module Tumugi
|
|
17
17
|
def get
|
18
18
|
if auto_bind?
|
19
19
|
value = search_from_application_parameters
|
20
|
-
value = search_from_env if value.nil?
|
21
20
|
end
|
22
|
-
|
23
|
-
return value unless value.nil?
|
24
|
-
default_value
|
21
|
+
value.nil? ? default_value : value
|
25
22
|
end
|
26
23
|
|
27
24
|
def auto_bind?
|
@@ -60,19 +57,11 @@ module Tumugi
|
|
60
57
|
value ? Converter.convert(type, value) : nil
|
61
58
|
end
|
62
59
|
|
63
|
-
def search_from_env
|
64
|
-
key = @name.to_s
|
65
|
-
value = nil
|
66
|
-
value = ENV[key] if ENV.has_key?(key)
|
67
|
-
value = ENV[key.upcase] if ENV.has_key?(key.upcase)
|
68
|
-
value ? Converter.convert(type, value) : nil
|
69
|
-
end
|
70
|
-
|
71
60
|
private
|
72
61
|
|
73
62
|
def validate
|
74
63
|
if required? && default_value != nil
|
75
|
-
raise ParameterError.new("When you set required: true, you cannot set default value")
|
64
|
+
raise Tumugi::ParameterError.new("When you set required: true, you cannot set default value")
|
76
65
|
end
|
77
66
|
end
|
78
67
|
end
|
data/lib/tumugi/plugin.rb
CHANGED
@@ -19,7 +19,7 @@ module Tumugi
|
|
19
19
|
raise "Invalid implementation as #{kind} plugin: '#{type}'. It must be a Class."
|
20
20
|
end
|
21
21
|
registry.register(type, value)
|
22
|
-
Tumugi.
|
22
|
+
Tumugi::Logger.instance.debug "registered #{kind} plugin '#{type}'"
|
23
23
|
nil
|
24
24
|
end
|
25
25
|
|
data/lib/tumugi/target.rb
CHANGED
data/lib/tumugi/task.rb
CHANGED
@@ -36,7 +36,7 @@ module Tumugi
|
|
36
36
|
end
|
37
37
|
|
38
38
|
def logger
|
39
|
-
@logger ||= Tumugi.
|
39
|
+
@logger ||= Tumugi::Logger.instance
|
40
40
|
end
|
41
41
|
|
42
42
|
def log(msg)
|
@@ -79,6 +79,10 @@ module Tumugi
|
|
79
79
|
list(_requires).any? { |t| t.instance.state == :failed || t.instance.state == :requires_failed }
|
80
80
|
end
|
81
81
|
|
82
|
+
def timeout
|
83
|
+
nil # meaning use default timeout
|
84
|
+
end
|
85
|
+
|
82
86
|
# Following methods are internal use only
|
83
87
|
|
84
88
|
def _requires
|
data/lib/tumugi/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tumugi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kazuyuki Honda
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-05-
|
11
|
+
date: 2016-05-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: parallel
|
@@ -211,7 +211,6 @@ files:
|
|
211
211
|
- lib/tumugi/mixin/parameterizable.rb
|
212
212
|
- lib/tumugi/mixin/task_helper.rb
|
213
213
|
- lib/tumugi/parameter/converter.rb
|
214
|
-
- lib/tumugi/parameter/error.rb
|
215
214
|
- lib/tumugi/parameter/parameter.rb
|
216
215
|
- lib/tumugi/parameter/parameter_proxy.rb
|
217
216
|
- lib/tumugi/plugin.rb
|
@@ -219,6 +218,7 @@ files:
|
|
219
218
|
- lib/tumugi/plugin/file_system_target.rb
|
220
219
|
- lib/tumugi/plugin/local_file_system.rb
|
221
220
|
- lib/tumugi/plugin/target/local_file.rb
|
221
|
+
- lib/tumugi/plugin/task/external.rb
|
222
222
|
- lib/tumugi/registry.rb
|
223
223
|
- lib/tumugi/target.rb
|
224
224
|
- lib/tumugi/task.rb
|