tumugi 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.travis.yml +3 -0
- data/CHANGELOG.md +44 -2
- data/README.md +5 -91
- data/Rakefile +0 -4
- data/examples/task_inheritance.rb +9 -0
- data/examples/task_parameter.rb +4 -5
- data/lib/tumugi.rb +6 -3
- data/lib/tumugi/cli.rb +31 -6
- data/lib/tumugi/command/new.rb +64 -0
- data/lib/tumugi/command/run.rb +14 -104
- data/lib/tumugi/config.rb +2 -2
- data/lib/tumugi/dag_result_reporter.rb +37 -0
- data/lib/tumugi/data/new/Gemfile.erb +4 -0
- data/lib/tumugi/data/new/README.md.erb +59 -0
- data/lib/tumugi/data/new/Rakefile.erb +10 -0
- data/lib/tumugi/data/new/examples/example.rb.erb +4 -0
- data/lib/tumugi/data/new/gemspec.erb +28 -0
- data/lib/tumugi/data/new/gitignore.erb +6 -0
- data/lib/tumugi/data/new/lib/tumugi/plugin/target/target.rb.erb +20 -0
- data/lib/tumugi/data/new/lib/tumugi/plugin/task/task.rb.erb +22 -0
- data/lib/tumugi/data/new/test/plugin/target/target_test.rb.erb +9 -0
- data/lib/tumugi/data/new/test/plugin/task/task_test.rb.erb +27 -0
- data/lib/tumugi/data/new/test/test.rb.erb +7 -0
- data/lib/tumugi/data/new/test/test_helper.rb.erb +8 -0
- data/lib/tumugi/executor/local_executor.rb +128 -0
- data/lib/tumugi/logger.rb +31 -2
- data/lib/tumugi/mixin/parameterizable.rb +11 -2
- data/lib/tumugi/parameter/parameter.rb +3 -3
- data/lib/tumugi/parameter/parameter_proxy.rb +1 -1
- data/lib/tumugi/task.rb +84 -3
- data/lib/tumugi/task_definition.rb +37 -18
- data/lib/tumugi/test/helper.rb +46 -0
- data/lib/tumugi/version.rb +1 -1
- data/lib/tumugi/{application.rb → workflow.rb} +11 -4
- data/tumugi.gemspec +3 -3
- metadata +39 -23
data/lib/tumugi/config.rb
CHANGED
@@ -11,7 +11,7 @@ module Tumugi
|
|
11
11
|
|
12
12
|
def self.register_section(name, *args)
|
13
13
|
@@sections[name] = Struct.new(camelize(name), *args)
|
14
|
-
logger.debug "registered config section '#{name}' with '#{args}'"
|
14
|
+
logger.debug { "registered config section '#{name}' with '#{args}'" }
|
15
15
|
end
|
16
16
|
|
17
17
|
def self.camelize(term)
|
@@ -29,7 +29,7 @@ module Tumugi
|
|
29
29
|
@workers = 1
|
30
30
|
@max_retry = 3
|
31
31
|
@retry_interval = 300 #seconds
|
32
|
-
@timeout =
|
32
|
+
@timeout = nil # meaning no timeout
|
33
33
|
|
34
34
|
@section_procs = {}
|
35
35
|
@section_instances = {}
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'terminal-table'
|
2
|
+
|
3
|
+
require 'tumugi/mixin/listable'
|
4
|
+
|
5
|
+
module Tumugi
|
6
|
+
class DAGResultReporter
|
7
|
+
include Tumugi::Mixin::Listable
|
8
|
+
|
9
|
+
def show(dag)
|
10
|
+
headings = ['Task', 'Requires', 'Parameters', 'State']
|
11
|
+
Terminal::Table.new title: "Workflow Result", headings: headings do |t|
|
12
|
+
dag.tsort.map.with_index do |task, index|
|
13
|
+
proxy = task.class.merged_parameter_proxy
|
14
|
+
requires = list(task.requires).map do |r|
|
15
|
+
r.id
|
16
|
+
end
|
17
|
+
params = proxy.params.map do |name, _|
|
18
|
+
"#{name}=#{truncate(task.send(name.to_sym).to_s, 25)}"
|
19
|
+
end
|
20
|
+
t << :separator if index != 0
|
21
|
+
t << [ task.id, requires.join("\n"), params.join("\n"), task.state ]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def truncate(text, length)
|
29
|
+
return nil if text.nil?
|
30
|
+
if text.length <= length
|
31
|
+
text
|
32
|
+
else
|
33
|
+
text[0, length].concat('...')
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# <%= name %> plugin for [tumugi](https://github.com/tumugi/tumugi)
|
2
|
+
|
3
|
+
TODO: Write short description here and <%= full_project_name %>.gemspec file.
|
4
|
+
|
5
|
+
## Target
|
6
|
+
|
7
|
+
### Tumugi::Plugin::<%= name.capitalize %>Target
|
8
|
+
|
9
|
+
TODO: Write description about `Tumugi::Plugin::<%= name.capitalize %>Target`.
|
10
|
+
|
11
|
+
## Task
|
12
|
+
|
13
|
+
### Tumugi::Plugin::<%= name.capitalize %>Task
|
14
|
+
|
15
|
+
TODO: Write description about `Tumugi::Plugin::<%= name.capitalize %>Task`.
|
16
|
+
|
17
|
+
#### Parameters
|
18
|
+
|
19
|
+
- **param1** description (string, required)
|
20
|
+
- **param2** description (integer, default: 1)
|
21
|
+
|
22
|
+
#### Example
|
23
|
+
|
24
|
+
```rb
|
25
|
+
task :task1, type: :bigquery_dataset do
|
26
|
+
param_set :dataset_id, 'test'
|
27
|
+
end
|
28
|
+
```
|
29
|
+
|
30
|
+
### Config Section
|
31
|
+
|
32
|
+
TODO: Write description about config section named "<%= name %>"
|
33
|
+
if this plugin has config section.
|
34
|
+
|
35
|
+
#### Example
|
36
|
+
|
37
|
+
```rb
|
38
|
+
Tumugi.configure do |config|
|
39
|
+
config.section("<%= name %>") do |section|
|
40
|
+
section.config1 = "xxx"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
```
|
44
|
+
|
45
|
+
## Development
|
46
|
+
|
47
|
+
After checking out the repo, run `bundle install` to install dependencies.
|
48
|
+
Then run `bundle exec rake test` to run the tests.
|
49
|
+
|
50
|
+
Run this plugin with tumugi, run
|
51
|
+
|
52
|
+
```
|
53
|
+
$ bundle exec tumugi run -f examples/example.rb main
|
54
|
+
```
|
55
|
+
|
56
|
+
## License
|
57
|
+
|
58
|
+
The gem is available as open source under the terms of the [Apache License
|
59
|
+
Version 2.0](http://www.apache.org/licenses/).
|
@@ -0,0 +1,28 @@
|
|
1
|
+
lib = File.expand_path('../lib', __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
|
4
|
+
Gem::Specification.new do |spec|
|
5
|
+
spec.name = "<%= full_project_name %>"
|
6
|
+
spec.version = "0.1.0"
|
7
|
+
# TODO set this: spec.authors = ["Your Name"]
|
8
|
+
# TODO set this: spec.email = ["your-email@example.com"]
|
9
|
+
|
10
|
+
spec.summary = "<%= name %> plugin for tumugi"
|
11
|
+
# TODO set this: spec.homepage = "https://github.com/YOUR_ACCOUNT/<%= full_project_name %>"
|
12
|
+
spec.license = "Apache License Version 2.0"
|
13
|
+
|
14
|
+
spec.required_ruby_version = '>= 2.1'
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
|
+
spec.bindir = "exe"
|
18
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_runtime_dependency "tumugi", ">= <%= tumugi_version %>"
|
22
|
+
#spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
|
23
|
+
|
24
|
+
spec.add_development_dependency 'bundler', '~> 1.11'
|
25
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
26
|
+
spec.add_development_dependency 'test-unit', '~> 3.1'
|
27
|
+
spec.add_development_dependency 'test-unit-rr'
|
28
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'tumugi'
|
2
|
+
|
3
|
+
module Tumugi
|
4
|
+
module Plugin
|
5
|
+
class <%= name.capitalize %>Target < Tumugi::Target
|
6
|
+
Tumugi::Plugin.register_target('<%= name %>', self)
|
7
|
+
|
8
|
+
attr_reader :value
|
9
|
+
|
10
|
+
def initialize(value)
|
11
|
+
@value = value
|
12
|
+
end
|
13
|
+
|
14
|
+
def exist?
|
15
|
+
# TODO: Implemente this method
|
16
|
+
true
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'tumugi'
|
2
|
+
require 'tumugi/plugin/target/<%= name %>'
|
3
|
+
|
4
|
+
module Tumugi
|
5
|
+
module Plugin
|
6
|
+
class <%= name.capitalize %>Task < Tumugi::Task
|
7
|
+
Tumugi::Plugin.register_task('<%= name %>', self)
|
8
|
+
|
9
|
+
param :param1, type: :string, default: ""
|
10
|
+
|
11
|
+
def output
|
12
|
+
@output ||= Tumugi::Plugin::<%= name.capitalize %>Target.new(param1)
|
13
|
+
end
|
14
|
+
|
15
|
+
def run
|
16
|
+
# TODO: Implemente this method
|
17
|
+
log "run #{self.class.name}"
|
18
|
+
log "param1: #{param1}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
require_relative '../../test_helper'
|
2
|
+
require 'tumugi/plugin/target/<%= name %>'
|
3
|
+
|
4
|
+
class Tumugi::Plugin::<%= name.capitalize %>TargetTest < Test::Unit::TestCase
|
5
|
+
test "exist? should return true if exists" do
|
6
|
+
target = Tumugi::Plugin::<%= name.capitalize %>Target.new("value")
|
7
|
+
assert_true(target.exist?)
|
8
|
+
end
|
9
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require_relative '../../test_helper'
|
2
|
+
require 'tumugi/plugin/task/<%= name %>'
|
3
|
+
|
4
|
+
class Tumugi::Plugin::<%= name.capitalize %>TaskTest < Test::Unit::TestCase
|
5
|
+
setup do
|
6
|
+
@klass = Class.new(Tumugi::Plugin::<%= name.capitalize %>Task)
|
7
|
+
@klass.set(:param1, "value")
|
8
|
+
end
|
9
|
+
|
10
|
+
test "should set correctly" do
|
11
|
+
task = @klass.new
|
12
|
+
assert_equal("value", task.param1)
|
13
|
+
end
|
14
|
+
|
15
|
+
test "#output" do
|
16
|
+
task = @klass.new
|
17
|
+
output = task.output
|
18
|
+
assert_true(output.is_a? Tumugi::Plugin::<%= name.capitalize %>Target)
|
19
|
+
end
|
20
|
+
|
21
|
+
test "#run" do
|
22
|
+
task = @klass.new
|
23
|
+
output = task.output
|
24
|
+
task.run
|
25
|
+
assert_equal("value", output.value)
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
require 'much-timeout'
|
2
|
+
require 'concurrent'
|
3
|
+
|
4
|
+
require 'tumugi'
|
5
|
+
require 'tumugi/error'
|
6
|
+
|
7
|
+
Concurrent.use_stdlib_logger(Logger::DEBUG)
|
8
|
+
|
9
|
+
module Tumugi
|
10
|
+
module Executor
|
11
|
+
class LocalExecutor
|
12
|
+
def initialize(dag, logger=nil, worker_num: 1)
|
13
|
+
@dag = dag
|
14
|
+
@main_task = dag.tsort.last
|
15
|
+
@logger = logger || Tumugi::Logger.instance
|
16
|
+
@options = { worker_num: worker_num }
|
17
|
+
@mutex = Mutex.new
|
18
|
+
end
|
19
|
+
|
20
|
+
def execute
|
21
|
+
pool = Concurrent::ThreadPoolExecutor.new(
|
22
|
+
min_threads: @options[:worker_num],
|
23
|
+
max_threads: @options[:worker_num]
|
24
|
+
)
|
25
|
+
|
26
|
+
setup_task_queue(@dag)
|
27
|
+
loop do
|
28
|
+
task = dequeue_task
|
29
|
+
break if task.nil?
|
30
|
+
|
31
|
+
Concurrent::Future.execute(executor: pool) do
|
32
|
+
if !task.runnable?(Time.now)
|
33
|
+
info "not_runnable: #{task.id}"
|
34
|
+
enqueue_task(task)
|
35
|
+
else
|
36
|
+
begin
|
37
|
+
info "start: #{task.id}"
|
38
|
+
task.trigger!(:start)
|
39
|
+
MuchTimeout.optional_timeout(task_timeout(task), Tumugi::TimeoutError) do
|
40
|
+
task.run
|
41
|
+
end
|
42
|
+
task.trigger!(:complete)
|
43
|
+
info "#{task.state}: #{task.id}"
|
44
|
+
rescue => e
|
45
|
+
handle_error(task, e)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
pool.shutdown
|
52
|
+
pool.wait_for_termination
|
53
|
+
|
54
|
+
@dag.tsort.all? { |t| t.success? }
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def task_timeout(task)
|
60
|
+
timeout = task.timeout || Tumugi.config.timeout
|
61
|
+
timeout = nil if !timeout.nil? && timeout == 0 # for backward compatibility
|
62
|
+
timeout
|
63
|
+
end
|
64
|
+
|
65
|
+
def setup_task_queue(dag)
|
66
|
+
@queue = []
|
67
|
+
dag.tsort.each { |t| enqueue_task(t) }
|
68
|
+
@queue
|
69
|
+
end
|
70
|
+
|
71
|
+
def dequeue_task
|
72
|
+
loop do
|
73
|
+
task = @mutex.synchronize {
|
74
|
+
debug { "queue: #{@queue.map(&:id)}" }
|
75
|
+
@queue.shift
|
76
|
+
}
|
77
|
+
|
78
|
+
if task.nil?
|
79
|
+
if @main_task.finished?
|
80
|
+
break nil
|
81
|
+
else
|
82
|
+
sleep(0.1)
|
83
|
+
end
|
84
|
+
else
|
85
|
+
debug { "dequeue: #{task.id}" }
|
86
|
+
|
87
|
+
if task.requires_failed?
|
88
|
+
task.trigger!(:requires_fail)
|
89
|
+
info "#{task.state}: #{task.id} has failed requires task"
|
90
|
+
elsif task.completed?
|
91
|
+
task.trigger!(:skip)
|
92
|
+
info "#{task.state}: #{task.id} is already completed"
|
93
|
+
else
|
94
|
+
break task
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def enqueue_task(task)
|
101
|
+
debug { "enqueue: #{task.id}" }
|
102
|
+
@mutex.synchronize { @queue.push(task) }
|
103
|
+
end
|
104
|
+
|
105
|
+
def handle_error(task, err)
|
106
|
+
if task.retry
|
107
|
+
task.trigger!(:pend)
|
108
|
+
@logger.error "#{err.class}: '#{err.message}' - #{task.tries} tries and wait #{task.retry_interval} seconds until the next try."
|
109
|
+
enqueue_task(task)
|
110
|
+
else
|
111
|
+
task.trigger!(:fail)
|
112
|
+
@logger.error "#{err.class}: '#{err.message}' - #{task.tries} tries and reached max retry count, so task #{task.id} failed."
|
113
|
+
info "#{task.state}: #{task.id}"
|
114
|
+
@logger.error "#{err.message}"
|
115
|
+
@logger.debug { err.backtrace.join("\n") }
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def info(message)
|
120
|
+
@logger.info "#{message}, thread: #{Thread.current.object_id}"
|
121
|
+
end
|
122
|
+
|
123
|
+
def debug(&block)
|
124
|
+
@logger.debug { "#{block.call}, thread: #{Thread.current.object_id}" }
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
data/lib/tumugi/logger.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
-
require 'logger'
|
2
1
|
require 'forwardable'
|
2
|
+
require 'json'
|
3
|
+
require 'logger'
|
3
4
|
require 'singleton'
|
4
5
|
|
5
6
|
module Tumugi
|
@@ -7,14 +8,20 @@ module Tumugi
|
|
7
8
|
include Singleton
|
8
9
|
extend Forwardable
|
9
10
|
def_delegators :@logger, :debug, :error, :fatal, :info, :warn, :level
|
11
|
+
attr_accessor :workflow_id
|
10
12
|
|
11
13
|
def initialize
|
14
|
+
@formatters = {
|
15
|
+
text: text_formatter,
|
16
|
+
json: json_formatter
|
17
|
+
}
|
12
18
|
init
|
13
19
|
end
|
14
20
|
|
15
|
-
def init(output
|
21
|
+
def init(output: STDOUT, format: :text)
|
16
22
|
@logger = ::Logger.new(output)
|
17
23
|
@logger.level = ::Logger::INFO
|
24
|
+
@logger.formatter = @formatters[format]
|
18
25
|
end
|
19
26
|
|
20
27
|
def verbose!
|
@@ -24,5 +31,27 @@ module Tumugi
|
|
24
31
|
def quiet!
|
25
32
|
@logger = ::Logger.new(nil)
|
26
33
|
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def text_formatter
|
38
|
+
Proc.new { |severity, datetime, progname, msg|
|
39
|
+
if !workflow_id.nil?
|
40
|
+
"#{datetime} #{severity} [#{workflow_id}] #{msg}\n"
|
41
|
+
else
|
42
|
+
"#{datetime} #{severity} #{msg}\n"
|
43
|
+
end
|
44
|
+
}
|
45
|
+
end
|
46
|
+
|
47
|
+
def json_formatter
|
48
|
+
Proc.new { |severity, datetime, progname, msg|
|
49
|
+
hash = { time: datetime, severity: severity, message: msg }
|
50
|
+
if !workflow_id.nil?
|
51
|
+
hash[:workflow] = workflow_id
|
52
|
+
end
|
53
|
+
"#{JSON.generate(hash)}\n"
|
54
|
+
}
|
55
|
+
end
|
27
56
|
end
|
28
57
|
end
|