tumugi 0.5.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +3 -0
  4. data/CHANGELOG.md +44 -2
  5. data/README.md +5 -91
  6. data/Rakefile +0 -4
  7. data/examples/task_inheritance.rb +9 -0
  8. data/examples/task_parameter.rb +4 -5
  9. data/lib/tumugi.rb +6 -3
  10. data/lib/tumugi/cli.rb +31 -6
  11. data/lib/tumugi/command/new.rb +64 -0
  12. data/lib/tumugi/command/run.rb +14 -104
  13. data/lib/tumugi/config.rb +2 -2
  14. data/lib/tumugi/dag_result_reporter.rb +37 -0
  15. data/lib/tumugi/data/new/Gemfile.erb +4 -0
  16. data/lib/tumugi/data/new/README.md.erb +59 -0
  17. data/lib/tumugi/data/new/Rakefile.erb +10 -0
  18. data/lib/tumugi/data/new/examples/example.rb.erb +4 -0
  19. data/lib/tumugi/data/new/gemspec.erb +28 -0
  20. data/lib/tumugi/data/new/gitignore.erb +6 -0
  21. data/lib/tumugi/data/new/lib/tumugi/plugin/target/target.rb.erb +20 -0
  22. data/lib/tumugi/data/new/lib/tumugi/plugin/task/task.rb.erb +22 -0
  23. data/lib/tumugi/data/new/test/plugin/target/target_test.rb.erb +9 -0
  24. data/lib/tumugi/data/new/test/plugin/task/task_test.rb.erb +27 -0
  25. data/lib/tumugi/data/new/test/test.rb.erb +7 -0
  26. data/lib/tumugi/data/new/test/test_helper.rb.erb +8 -0
  27. data/lib/tumugi/executor/local_executor.rb +128 -0
  28. data/lib/tumugi/logger.rb +31 -2
  29. data/lib/tumugi/mixin/parameterizable.rb +11 -2
  30. data/lib/tumugi/parameter/parameter.rb +3 -3
  31. data/lib/tumugi/parameter/parameter_proxy.rb +1 -1
  32. data/lib/tumugi/task.rb +84 -3
  33. data/lib/tumugi/task_definition.rb +37 -18
  34. data/lib/tumugi/test/helper.rb +46 -0
  35. data/lib/tumugi/version.rb +1 -1
  36. data/lib/tumugi/{application.rb → workflow.rb} +11 -4
  37. data/tumugi.gemspec +3 -3
  38. metadata +39 -23
@@ -11,7 +11,7 @@ module Tumugi
11
11
 
12
12
  def self.register_section(name, *args)
13
13
  @@sections[name] = Struct.new(camelize(name), *args)
14
- logger.debug "registered config section '#{name}' with '#{args}'"
14
+ logger.debug { "registered config section '#{name}' with '#{args}'" }
15
15
  end
16
16
 
17
17
  def self.camelize(term)
@@ -29,7 +29,7 @@ module Tumugi
29
29
  @workers = 1
30
30
  @max_retry = 3
31
31
  @retry_interval = 300 #seconds
32
- @timeout = 0 # meaning no timeout
32
+ @timeout = nil # meaning no timeout
33
33
 
34
34
  @section_procs = {}
35
35
  @section_instances = {}
@@ -0,0 +1,37 @@
1
+ require 'terminal-table'
2
+
3
+ require 'tumugi/mixin/listable'
4
+
5
+ module Tumugi
6
+ class DAGResultReporter
7
+ include Tumugi::Mixin::Listable
8
+
9
+ def show(dag)
10
+ headings = ['Task', 'Requires', 'Parameters', 'State']
11
+ Terminal::Table.new title: "Workflow Result", headings: headings do |t|
12
+ dag.tsort.map.with_index do |task, index|
13
+ proxy = task.class.merged_parameter_proxy
14
+ requires = list(task.requires).map do |r|
15
+ r.id
16
+ end
17
+ params = proxy.params.map do |name, _|
18
+ "#{name}=#{truncate(task.send(name.to_sym).to_s, 25)}"
19
+ end
20
+ t << :separator if index != 0
21
+ t << [ task.id, requires.join("\n"), params.join("\n"), task.state ]
22
+ end
23
+ end
24
+ end
25
+
26
+ private
27
+
28
+ def truncate(text, length)
29
+ return nil if text.nil?
30
+ if text.length <= length
31
+ text
32
+ else
33
+ text[0, length].concat('...')
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in <%= full_project_name %>.gemspec
4
+ gemspec
@@ -0,0 +1,59 @@
1
+ # <%= name %> plugin for [tumugi](https://github.com/tumugi/tumugi)
2
+
3
+ TODO: Write short description here and <%= full_project_name %>.gemspec file.
4
+
5
+ ## Target
6
+
7
+ ### Tumugi::Plugin::<%= name.capitalize %>Target
8
+
9
+ TODO: Write description about `Tumugi::Plugin::<%= name.capitalize %>Target`.
10
+
11
+ ## Task
12
+
13
+ ### Tumugi::Plugin::<%= name.capitalize %>Task
14
+
15
+ TODO: Write description about `Tumugi::Plugin::<%= name.capitalize %>Task`.
16
+
17
+ #### Parameters
18
+
19
+ - **param1** description (string, required)
20
+ - **param2** description (integer, default: 1)
21
+
22
+ #### Example
23
+
24
+ ```rb
25
+ task :task1, type: :bigquery_dataset do
26
+ param_set :dataset_id, 'test'
27
+ end
28
+ ```
29
+
30
+ ### Config Section
31
+
32
+ TODO: Write description about config section named "<%= name %>"
33
+ if this plugin has config section.
34
+
35
+ #### Example
36
+
37
+ ```rb
38
+ Tumugi.configure do |config|
39
+ config.section("<%= name %>") do |section|
40
+ section.config1 = "xxx"
41
+ end
42
+ end
43
+ ```
44
+
45
+ ## Development
46
+
47
+ After checking out the repo, run `bundle install` to install dependencies.
48
+ Then run `bundle exec rake test` to run the tests.
49
+
50
+ Run this plugin with tumugi, run
51
+
52
+ ```
53
+ $ bundle exec tumugi run -f examples/example.rb main
54
+ ```
55
+
56
+ ## License
57
+
58
+ The gem is available as open source under the terms of the [Apache License
59
+ Version 2.0](http://www.apache.org/licenses/).
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+
4
+ Rake::TestTask.new(:test) do |t|
5
+ t.libs << "test"
6
+ t.libs << "lib"
7
+ t.test_files = FileList['test/**/*_test.rb']
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,4 @@
1
+ task main, type: :<%= name %> do
2
+ param1 'value1'
3
+ output { target(:<%= name %>, param1) }
4
+ end
@@ -0,0 +1,28 @@
1
+ lib = File.expand_path('../lib', __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+
4
+ Gem::Specification.new do |spec|
5
+ spec.name = "<%= full_project_name %>"
6
+ spec.version = "0.1.0"
7
+ # TODO set this: spec.authors = ["Your Name"]
8
+ # TODO set this: spec.email = ["your-email@example.com"]
9
+
10
+ spec.summary = "<%= name %> plugin for tumugi"
11
+ # TODO set this: spec.homepage = "https://github.com/YOUR_ACCOUNT/<%= full_project_name %>"
12
+ spec.license = "Apache License Version 2.0"
13
+
14
+ spec.required_ruby_version = '>= 2.1'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
+ spec.bindir = "exe"
18
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_runtime_dependency "tumugi", ">= <%= tumugi_version %>"
22
+ #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
23
+
24
+ spec.add_development_dependency 'bundler', '~> 1.11'
25
+ spec.add_development_dependency 'rake', '~> 10.0'
26
+ spec.add_development_dependency 'test-unit', '~> 3.1'
27
+ spec.add_development_dependency 'test-unit-rr'
28
+ end
@@ -0,0 +1,6 @@
1
+ *~
2
+ /.bundle/
3
+ /Gemfile.lock
4
+ /pkg/
5
+ /tmp/
6
+ .ruby-version
@@ -0,0 +1,20 @@
1
+ require 'tumugi'
2
+
3
+ module Tumugi
4
+ module Plugin
5
+ class <%= name.capitalize %>Target < Tumugi::Target
6
+ Tumugi::Plugin.register_target('<%= name %>', self)
7
+
8
+ attr_reader :value
9
+
10
+ def initialize(value)
11
+ @value = value
12
+ end
13
+
14
+ def exist?
15
+ # TODO: Implemente this method
16
+ true
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,22 @@
1
+ require 'tumugi'
2
+ require 'tumugi/plugin/target/<%= name %>'
3
+
4
+ module Tumugi
5
+ module Plugin
6
+ class <%= name.capitalize %>Task < Tumugi::Task
7
+ Tumugi::Plugin.register_task('<%= name %>', self)
8
+
9
+ param :param1, type: :string, default: ""
10
+
11
+ def output
12
+ @output ||= Tumugi::Plugin::<%= name.capitalize %>Target.new(param1)
13
+ end
14
+
15
+ def run
16
+ # TODO: Implemente this method
17
+ log "run #{self.class.name}"
18
+ log "param1: #{param1}"
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,9 @@
1
+ require_relative '../../test_helper'
2
+ require 'tumugi/plugin/target/<%= name %>'
3
+
4
+ class Tumugi::Plugin::<%= name.capitalize %>TargetTest < Test::Unit::TestCase
5
+ test "exist? should return true if exists" do
6
+ target = Tumugi::Plugin::<%= name.capitalize %>Target.new("value")
7
+ assert_true(target.exist?)
8
+ end
9
+ end
@@ -0,0 +1,27 @@
1
+ require_relative '../../test_helper'
2
+ require 'tumugi/plugin/task/<%= name %>'
3
+
4
+ class Tumugi::Plugin::<%= name.capitalize %>TaskTest < Test::Unit::TestCase
5
+ setup do
6
+ @klass = Class.new(Tumugi::Plugin::<%= name.capitalize %>Task)
7
+ @klass.set(:param1, "value")
8
+ end
9
+
10
+ test "should set correctly" do
11
+ task = @klass.new
12
+ assert_equal("value", task.param1)
13
+ end
14
+
15
+ test "#output" do
16
+ task = @klass.new
17
+ output = task.output
18
+ assert_true(output.is_a? Tumugi::Plugin::<%= name.capitalize %>Target)
19
+ end
20
+
21
+ test "#run" do
22
+ task = @klass.new
23
+ output = task.output
24
+ task.run
25
+ assert_equal("value", output.value)
26
+ end
27
+ end
@@ -0,0 +1,7 @@
1
+ require_relative './test_helper'
2
+
3
+ class Tumugi::<%= name.capitalize %>Test < Test::Unit::TestCase
4
+ test 'run success' do
5
+ assert_run_success("examples/example.rb", "main")
6
+ end
7
+ end
@@ -0,0 +1,8 @@
1
+ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
2
+
3
+ require 'test/unit'
4
+ require 'test/unit/rr'
5
+
6
+ require 'tumugi'
7
+ require 'tumugi/test/helper'
8
+ include Tumugi::Test::Helpers
@@ -0,0 +1,128 @@
1
+ require 'much-timeout'
2
+ require 'concurrent'
3
+
4
+ require 'tumugi'
5
+ require 'tumugi/error'
6
+
7
+ Concurrent.use_stdlib_logger(Logger::DEBUG)
8
+
9
+ module Tumugi
10
+ module Executor
11
+ class LocalExecutor
12
+ def initialize(dag, logger=nil, worker_num: 1)
13
+ @dag = dag
14
+ @main_task = dag.tsort.last
15
+ @logger = logger || Tumugi::Logger.instance
16
+ @options = { worker_num: worker_num }
17
+ @mutex = Mutex.new
18
+ end
19
+
20
+ def execute
21
+ pool = Concurrent::ThreadPoolExecutor.new(
22
+ min_threads: @options[:worker_num],
23
+ max_threads: @options[:worker_num]
24
+ )
25
+
26
+ setup_task_queue(@dag)
27
+ loop do
28
+ task = dequeue_task
29
+ break if task.nil?
30
+
31
+ Concurrent::Future.execute(executor: pool) do
32
+ if !task.runnable?(Time.now)
33
+ info "not_runnable: #{task.id}"
34
+ enqueue_task(task)
35
+ else
36
+ begin
37
+ info "start: #{task.id}"
38
+ task.trigger!(:start)
39
+ MuchTimeout.optional_timeout(task_timeout(task), Tumugi::TimeoutError) do
40
+ task.run
41
+ end
42
+ task.trigger!(:complete)
43
+ info "#{task.state}: #{task.id}"
44
+ rescue => e
45
+ handle_error(task, e)
46
+ end
47
+ end
48
+ end
49
+ end
50
+
51
+ pool.shutdown
52
+ pool.wait_for_termination
53
+
54
+ @dag.tsort.all? { |t| t.success? }
55
+ end
56
+
57
+ private
58
+
59
+ def task_timeout(task)
60
+ timeout = task.timeout || Tumugi.config.timeout
61
+ timeout = nil if !timeout.nil? && timeout == 0 # for backward compatibility
62
+ timeout
63
+ end
64
+
65
+ def setup_task_queue(dag)
66
+ @queue = []
67
+ dag.tsort.each { |t| enqueue_task(t) }
68
+ @queue
69
+ end
70
+
71
+ def dequeue_task
72
+ loop do
73
+ task = @mutex.synchronize {
74
+ debug { "queue: #{@queue.map(&:id)}" }
75
+ @queue.shift
76
+ }
77
+
78
+ if task.nil?
79
+ if @main_task.finished?
80
+ break nil
81
+ else
82
+ sleep(0.1)
83
+ end
84
+ else
85
+ debug { "dequeue: #{task.id}" }
86
+
87
+ if task.requires_failed?
88
+ task.trigger!(:requires_fail)
89
+ info "#{task.state}: #{task.id} has failed requires task"
90
+ elsif task.completed?
91
+ task.trigger!(:skip)
92
+ info "#{task.state}: #{task.id} is already completed"
93
+ else
94
+ break task
95
+ end
96
+ end
97
+ end
98
+ end
99
+
100
+ def enqueue_task(task)
101
+ debug { "enqueue: #{task.id}" }
102
+ @mutex.synchronize { @queue.push(task) }
103
+ end
104
+
105
+ def handle_error(task, err)
106
+ if task.retry
107
+ task.trigger!(:pend)
108
+ @logger.error "#{err.class}: '#{err.message}' - #{task.tries} tries and wait #{task.retry_interval} seconds until the next try."
109
+ enqueue_task(task)
110
+ else
111
+ task.trigger!(:fail)
112
+ @logger.error "#{err.class}: '#{err.message}' - #{task.tries} tries and reached max retry count, so task #{task.id} failed."
113
+ info "#{task.state}: #{task.id}"
114
+ @logger.error "#{err.message}"
115
+ @logger.debug { err.backtrace.join("\n") }
116
+ end
117
+ end
118
+
119
+ def info(message)
120
+ @logger.info "#{message}, thread: #{Thread.current.object_id}"
121
+ end
122
+
123
+ def debug(&block)
124
+ @logger.debug { "#{block.call}, thread: #{Thread.current.object_id}" }
125
+ end
126
+ end
127
+ end
128
+ end
@@ -1,5 +1,6 @@
1
- require 'logger'
2
1
  require 'forwardable'
2
+ require 'json'
3
+ require 'logger'
3
4
  require 'singleton'
4
5
 
5
6
  module Tumugi
@@ -7,14 +8,20 @@ module Tumugi
7
8
  include Singleton
8
9
  extend Forwardable
9
10
  def_delegators :@logger, :debug, :error, :fatal, :info, :warn, :level
11
+ attr_accessor :workflow_id
10
12
 
11
13
  def initialize
14
+ @formatters = {
15
+ text: text_formatter,
16
+ json: json_formatter
17
+ }
12
18
  init
13
19
  end
14
20
 
15
- def init(output=STDOUT)
21
+ def init(output: STDOUT, format: :text)
16
22
  @logger = ::Logger.new(output)
17
23
  @logger.level = ::Logger::INFO
24
+ @logger.formatter = @formatters[format]
18
25
  end
19
26
 
20
27
  def verbose!
@@ -24,5 +31,27 @@ module Tumugi
24
31
  def quiet!
25
32
  @logger = ::Logger.new(nil)
26
33
  end
34
+
35
+ private
36
+
37
+ def text_formatter
38
+ Proc.new { |severity, datetime, progname, msg|
39
+ if !workflow_id.nil?
40
+ "#{datetime} #{severity} [#{workflow_id}] #{msg}\n"
41
+ else
42
+ "#{datetime} #{severity} #{msg}\n"
43
+ end
44
+ }
45
+ end
46
+
47
+ def json_formatter
48
+ Proc.new { |severity, datetime, progname, msg|
49
+ hash = { time: datetime, severity: severity, message: msg }
50
+ if !workflow_id.nil?
51
+ hash[:workflow] = workflow_id
52
+ end
53
+ "#{JSON.generate(hash)}\n"
54
+ }
55
+ end
27
56
  end
28
57
  end