tumugi 0.5.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +3 -0
  4. data/CHANGELOG.md +44 -2
  5. data/README.md +5 -91
  6. data/Rakefile +0 -4
  7. data/examples/task_inheritance.rb +9 -0
  8. data/examples/task_parameter.rb +4 -5
  9. data/lib/tumugi.rb +6 -3
  10. data/lib/tumugi/cli.rb +31 -6
  11. data/lib/tumugi/command/new.rb +64 -0
  12. data/lib/tumugi/command/run.rb +14 -104
  13. data/lib/tumugi/config.rb +2 -2
  14. data/lib/tumugi/dag_result_reporter.rb +37 -0
  15. data/lib/tumugi/data/new/Gemfile.erb +4 -0
  16. data/lib/tumugi/data/new/README.md.erb +59 -0
  17. data/lib/tumugi/data/new/Rakefile.erb +10 -0
  18. data/lib/tumugi/data/new/examples/example.rb.erb +4 -0
  19. data/lib/tumugi/data/new/gemspec.erb +28 -0
  20. data/lib/tumugi/data/new/gitignore.erb +6 -0
  21. data/lib/tumugi/data/new/lib/tumugi/plugin/target/target.rb.erb +20 -0
  22. data/lib/tumugi/data/new/lib/tumugi/plugin/task/task.rb.erb +22 -0
  23. data/lib/tumugi/data/new/test/plugin/target/target_test.rb.erb +9 -0
  24. data/lib/tumugi/data/new/test/plugin/task/task_test.rb.erb +27 -0
  25. data/lib/tumugi/data/new/test/test.rb.erb +7 -0
  26. data/lib/tumugi/data/new/test/test_helper.rb.erb +8 -0
  27. data/lib/tumugi/executor/local_executor.rb +128 -0
  28. data/lib/tumugi/logger.rb +31 -2
  29. data/lib/tumugi/mixin/parameterizable.rb +11 -2
  30. data/lib/tumugi/parameter/parameter.rb +3 -3
  31. data/lib/tumugi/parameter/parameter_proxy.rb +1 -1
  32. data/lib/tumugi/task.rb +84 -3
  33. data/lib/tumugi/task_definition.rb +37 -18
  34. data/lib/tumugi/test/helper.rb +46 -0
  35. data/lib/tumugi/version.rb +1 -1
  36. data/lib/tumugi/{application.rb → workflow.rb} +11 -4
  37. data/tumugi.gemspec +3 -3
  38. metadata +39 -23
@@ -1,4 +1,5 @@
1
1
  require 'tumugi/error'
2
+ require 'tumugi/logger'
2
3
  require 'tumugi/parameter/parameter_proxy'
3
4
 
4
5
  module Tumugi
@@ -56,6 +57,9 @@ module Tumugi
56
57
  end
57
58
 
58
59
  def param(name, opts={})
60
+ if instance_methods.include?(name)
61
+ raise Tumugi::ParameterError.new("Parameter '#{name}' cannot use, because it is already defined or override instance method.")
62
+ end
59
63
  parameter_proxy(proxy_id).param(name, opts)
60
64
  attr_writer name
61
65
  define_method(name) do
@@ -70,8 +74,13 @@ module Tumugi
70
74
  end
71
75
  end
72
76
 
77
+ def set(name, value)
78
+ parameter_proxy(proxy_id).set(name, value)
79
+ end
80
+
73
81
  def param_set(name, value)
74
- parameter_proxy(proxy_id).param_set(name, value)
82
+ Tumugi::Logger.instance.warn("'param_set' is deprecated and will be removed in a future release. Use 'set' instead.")
83
+ set(name, value)
75
84
  end
76
85
 
77
86
  def param_auto_bind_enabled(v)
@@ -84,7 +93,7 @@ module Tumugi
84
93
  end
85
94
 
86
95
  def dump
87
- parameter_proxy_map[proxy_id].dump
96
+ parameter_proxy(proxy_id).dump
88
97
  end
89
98
 
90
99
  def proxy_id
@@ -14,7 +14,7 @@ module Tumugi
14
14
 
15
15
  def get
16
16
  if auto_bind?
17
- value = search_from_application_parameters
17
+ value = search_from_workflow_parameters
18
18
  end
19
19
  value.nil? ? default_value : value
20
20
  end
@@ -45,9 +45,9 @@ module Tumugi
45
45
 
46
46
  private
47
47
 
48
- def search_from_application_parameters
48
+ def search_from_workflow_parameters
49
49
  key = @name.to_s
50
- value = Tumugi.application.params[key]
50
+ value = Tumugi.workflow.params[key]
51
51
  value ? Converter.convert(type, value) : nil
52
52
  end
53
53
 
@@ -22,7 +22,7 @@ module Tumugi
22
22
  @params[name] = Tumugi::Parameter::Parameter.new(name, opts)
23
23
  end
24
24
 
25
- def param_set(name, value)
25
+ def set(name, value)
26
26
  @param_defaults[name] = value
27
27
  end
28
28
 
@@ -8,11 +8,25 @@ module Tumugi
8
8
  include Tumugi::Mixin::Listable
9
9
  include Tumugi::Mixin::TaskHelper
10
10
 
11
- attr_accessor :state # :pending, :running, :completed, :skipped
11
+ attr_reader :visible_at, :tries, :max_retry, :retry_interval
12
+
13
+ AVAILABLE_STATES = [
14
+ :pending,
15
+ :running,
16
+ :completed,
17
+ :skipped,
18
+ :failed,
19
+ :requires_failed,
20
+ ]
12
21
 
13
22
  def initialize
14
23
  super()
24
+ @visible_at = Time.now
25
+ @tries = 0
26
+ @max_retry = Tumugi.config.max_retry
27
+ @retry_interval = Tumugi.config.retry_interval
15
28
  @state = :pending
29
+ @lock = Mutex.new
16
30
  end
17
31
 
18
32
  def id
@@ -69,20 +83,79 @@ module Tumugi
69
83
  def completed?
70
84
  outputs = list(output)
71
85
  if outputs.empty?
72
- @state == :completed || @state == :skipped
86
+ success?
73
87
  else
74
88
  outputs.all?(&:exist?)
75
89
  end
76
90
  end
77
91
 
78
92
  def requires_failed?
79
- list(_requires).any? { |t| t.instance.state == :failed || t.instance.state == :requires_failed }
93
+ list(_requires).any? { |t| t.instance.finished? && !t.instance.success? }
94
+ end
95
+
96
+ def runnable?(now)
97
+ ready? && visible?(now)
98
+ end
99
+
100
+ def success?
101
+ case state
102
+ when :completed, :skipped
103
+ true
104
+ else
105
+ false
106
+ end
107
+ end
108
+
109
+ def finished?
110
+ case state
111
+ when :completed, :skipped, :failed, :requires_failed
112
+ true
113
+ else
114
+ false
115
+ end
80
116
  end
81
117
 
82
118
  def timeout
83
119
  nil # meaning use default timeout
84
120
  end
85
121
 
122
+ def retry
123
+ @tries += 1
124
+ @visible_at += @retry_interval
125
+ retriable?
126
+ end
127
+
128
+ def state
129
+ @lock.synchronize { @state }
130
+ end
131
+
132
+ def trigger!(event)
133
+ @lock.synchronize do
134
+ s = case event
135
+ when :skip
136
+ :skipped
137
+ when :start
138
+ :running
139
+ when :pend
140
+ :pending
141
+ when :complete
142
+ :completed
143
+ when :fail
144
+ :failed
145
+ when :requires_fail
146
+ :requires_failed
147
+ else
148
+ raise Tumugi::TumugiError.new("Invalid event: #{event}")
149
+ end
150
+
151
+ if not AVAILABLE_STATES.include?(s)
152
+ raise Tumugi::TumugiError.new("Invalid state: #{s}")
153
+ end
154
+
155
+ @state = s
156
+ end
157
+ end
158
+
86
159
  # Following methods are internal use only
87
160
 
88
161
  def _requires
@@ -106,5 +179,13 @@ module Tumugi
106
179
  _requires.instance._output
107
180
  end
108
181
  end
182
+
183
+ def visible?(now)
184
+ now >= @visible_at
185
+ end
186
+
187
+ def retriable?
188
+ @tries <= @max_retry
189
+ end
109
190
  end
110
191
  end
@@ -1,5 +1,6 @@
1
- require 'tumugi/task'
1
+ require 'tumugi/logger'
2
2
  require 'tumugi/plugin'
3
+ require 'tumugi/task'
3
4
  require 'tumugi/mixin/listable'
4
5
  require 'tumugi/mixin/task_helper'
5
6
 
@@ -11,7 +12,7 @@ module Tumugi
11
12
  def self.define(id, opts={}, &block)
12
13
  td = Tumugi::TaskDefinition.new(id, opts)
13
14
  td.instance_eval(&block) if block_given?
14
- Tumugi.application.add_task(id, td)
15
+ Tumugi.workflow.add_task(id, td)
15
16
  td
16
17
  end
17
18
 
@@ -22,10 +23,7 @@ module Tumugi
22
23
  @opts = { type: Tumugi::Task }.merge(opts)
23
24
  @params = {}
24
25
  @param_defaults = {}
25
-
26
- unless @opts[:type].is_a?(Class)
27
- @opts[:type] = Tumugi::Plugin.lookup_task(@opts[:type])
28
- end
26
+ define_parent_parameter_methods
29
27
  end
30
28
 
31
29
  def instance
@@ -33,11 +31,21 @@ module Tumugi
33
31
  end
34
32
 
35
33
  def param(name, opts={})
34
+ define_parameter_method(name)
36
35
  @params[name] = opts
37
36
  end
38
37
 
39
- def param_set(name, value)
40
- @param_defaults[name] = value
38
+ def set(name, value=nil, &block)
39
+ if block_given?
40
+ @param_defaults[name] = block
41
+ else
42
+ @param_defaults[name] = value
43
+ end
44
+ end
45
+
46
+ def param_set(name, value=nil, &block)
47
+ Tumugi::Logger.instance.warn("'param_set' is deprecated and will be removed in a future release. Use 'set' instead.")
48
+ set(name, value, &block)
41
49
  end
42
50
 
43
51
  def param_auto_bind_enabled(v)
@@ -68,6 +76,14 @@ module Tumugi
68
76
  task.instance_eval(&@run)
69
77
  end
70
78
 
79
+ def parent_task_class
80
+ if @opts[:type].is_a?(Class)
81
+ @opts[:type]
82
+ else
83
+ Tumugi::Plugin.lookup_task(@opts[:type])
84
+ end
85
+ end
86
+
71
87
  private
72
88
 
73
89
  def create_task
@@ -78,7 +94,7 @@ module Tumugi
78
94
  end
79
95
 
80
96
  def define_task
81
- task_class = Class.new(lookup_parent_task_class)
97
+ task_class = Class.new(parent_task_class)
82
98
  define_requires_method(task_class)
83
99
  define_output_method(task_class)
84
100
  define_run_method(task_class)
@@ -94,11 +110,11 @@ module Tumugi
94
110
  if reqs.nil?
95
111
  []
96
112
  elsif reqs.is_a?(Array)
97
- reqs.map { |t| Tumugi.application.find_task(t) }
113
+ reqs.map { |t| Tumugi.workflow.find_task(t) }
98
114
  elsif reqs.is_a?(Hash)
99
- Hash[reqs.map { |k, t| [k, Tumugi.application.find_task(t)] }]
115
+ Hash[reqs.map { |k, t| [k, Tumugi.workflow.find_task(t)] }]
100
116
  else
101
- Tumugi.application.find_task(reqs)
117
+ Tumugi.workflow.find_task(reqs)
102
118
  end
103
119
  end
104
120
  end
@@ -127,18 +143,21 @@ module Tumugi
127
143
  task_class.param(name, opts)
128
144
  end
129
145
  @param_defaults.each do |name, value|
130
- task_class.param_set(name, value)
146
+ task_class.set(name, value)
131
147
  end
132
148
  unless @param_auto_bind_enabled.nil?
133
149
  task_class.param_auto_bind_enabled(@param_auto_bind_enabled)
134
150
  end
135
151
  end
136
152
 
137
- def lookup_parent_task_class
138
- if @opts[:type].is_a?(Class)
139
- @opts[:type]
140
- else
141
- Tumugi::Plugin.lookup_task(@opts[:type])
153
+ def define_parameter_method(name)
154
+ instance_eval("def self.#{name}(value=nil, &block); set(:#{name}, value, &block); end")
155
+ end
156
+
157
+ def define_parent_parameter_methods
158
+ proxy = parent_task_class.merged_parameter_proxy
159
+ proxy.params.each do |name, _|
160
+ define_parameter_method(name)
142
161
  end
143
162
  end
144
163
  end
@@ -0,0 +1,46 @@
1
+ require 'test/unit'
2
+ require 'test/unit/rr'
3
+
4
+ require 'tumugi'
5
+ require 'tumugi/cli'
6
+
7
+ module Tumugi
8
+ module Test
9
+ module Helpers
10
+ def capture_stdout
11
+ out = StringIO.new
12
+ $stdout = out
13
+ yield
14
+ return out.string
15
+ ensure
16
+ $stdout = STDOUT
17
+ end
18
+ end
19
+
20
+ class TumugiTestCase < ::Test::Unit::TestCase
21
+ def invoke(command, workflow_file, task, options)
22
+ Tumugi::CLI.new.invoke(command, [task], { file: workflow_file, quiet: true }.merge(options))
23
+ end
24
+
25
+ def assert_run_success(workflow_file, task, options={})
26
+ assert_true(invoke(:run_, workflow_file, task, { workers: 2 }.merge(options)))
27
+ end
28
+
29
+ def assert_run_fail(workflow_file, task, options={})
30
+ assert_raise(Thor::Error) do
31
+ invoke(:run_, workflow_file, task, options)
32
+ end
33
+ end
34
+
35
+ def assert_show_success(workflow_file, task, options={})
36
+ assert_true(invoke(:show, workflow_file, task, options))
37
+ end
38
+
39
+ def assert_show_fail(workflow_file, task, options={})
40
+ assert_raise(Thor::Error) do
41
+ invoke(:show, workflow_file, task, options)
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -1,3 +1,3 @@
1
1
  module Tumugi
2
- VERSION = "0.5.3"
2
+ VERSION = "0.6.0"
3
3
  end
@@ -6,11 +6,15 @@ require 'tumugi/target'
6
6
  require 'tumugi/command/run'
7
7
  require 'tumugi/command/show'
8
8
 
9
+ require 'securerandom'
10
+
9
11
  module Tumugi
10
- class Application
12
+ class Workflow
13
+ attr_reader :id
11
14
  attr_accessor :params
12
15
 
13
16
  def initialize
17
+ @id = SecureRandom.uuid
14
18
  @tasks = {}
15
19
  @params = {}
16
20
  end
@@ -42,6 +46,7 @@ module Tumugi
42
46
  end
43
47
 
44
48
  begin
49
+ logger.info "Load workflow from #{file}"
45
50
  load(file, true)
46
51
  rescue LoadError => e
47
52
  raise Tumugi::TumugiError.new("Workflow file load error: #{file}", e)
@@ -66,18 +71,20 @@ module Tumugi
66
71
  end
67
72
 
68
73
  def setup_logger(command, options)
74
+ log_format = (options[:log_format] || :text).to_sym
69
75
  if command == :run && !options[:out].nil?
70
- Tumugi::Logger.instance.init(options[:out])
76
+ logger.init(output: options[:out], format: log_format)
71
77
  else
72
- Tumugi::Logger.instance.init
78
+ logger.init(format: log_format)
73
79
  end
74
80
  logger.verbose! if options[:verbose]
75
81
  logger.quiet! if options[:quiet]
82
+ logger.workflow_id = id
76
83
  end
77
84
 
78
85
  def load_config(options)
79
86
  config_file = options[:config]
80
- if config_file && File.exists?(config_file) && File.extname(config_file) == '.rb'
87
+ if config_file && File.exist?(config_file) && File.extname(config_file) == '.rb'
81
88
  logger.info "Load config from #{config_file}"
82
89
  load(config_file)
83
90
  end
@@ -20,8 +20,9 @@ Gem::Specification.new do |spec|
20
20
 
21
21
  spec.required_ruby_version = '>= 2.1'
22
22
 
23
- spec.add_runtime_dependency 'parallel', '~> 1.8.0'
24
- spec.add_runtime_dependency 'retriable', '~> 2.1'
23
+ spec.add_runtime_dependency 'concurrent-ruby', '~> 1.0.2'
24
+ spec.add_runtime_dependency 'erubis', '~> 2.7.0'
25
+ spec.add_runtime_dependency 'much-timeout', '~> 0.1.1'
25
26
  spec.add_runtime_dependency 'ruby-graphviz', '~> 1.2.2'
26
27
  spec.add_runtime_dependency 'terminal-table', '~> 1.5.2'
27
28
  spec.add_runtime_dependency 'thor', '~> 0.19.1'
@@ -31,5 +32,4 @@ Gem::Specification.new do |spec|
31
32
  spec.add_development_dependency 'test-unit', '~> 3.1'
32
33
  spec.add_development_dependency 'test-unit-rr'
33
34
  spec.add_development_dependency 'coveralls'
34
- spec.add_development_dependency 'github_changelog_generator'
35
35
  end
metadata CHANGED
@@ -1,43 +1,57 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tumugi
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.3
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kazuyuki Honda
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-06-14 00:00:00.000000000 Z
11
+ date: 2016-07-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: parallel
14
+ name: concurrent-ruby
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: 1.8.0
19
+ version: 1.0.2
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: 1.8.0
26
+ version: 1.0.2
27
27
  - !ruby/object:Gem::Dependency
28
- name: retriable
28
+ name: erubis
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '2.1'
33
+ version: 2.7.0
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '2.1'
40
+ version: 2.7.0
41
+ - !ruby/object:Gem::Dependency
42
+ name: much-timeout
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.1.1
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: 0.1.1
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: ruby-graphviz
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -150,20 +164,6 @@ dependencies:
150
164
  - - ">="
151
165
  - !ruby/object:Gem::Version
152
166
  version: '0'
153
- - !ruby/object:Gem::Dependency
154
- name: github_changelog_generator
155
- requirement: !ruby/object:Gem::Requirement
156
- requirements:
157
- - - ">="
158
- - !ruby/object:Gem::Version
159
- version: '0'
160
- type: :development
161
- prerelease: false
162
- version_requirements: !ruby/object:Gem::Requirement
163
- requirements:
164
- - - ">="
165
- - !ruby/object:Gem::Version
166
- version: '0'
167
167
  description:
168
168
  email:
169
169
  - hakobera@gmail.com
@@ -195,15 +195,29 @@ files:
195
195
  - examples/tumugi_config_with_section.rb
196
196
  - exe/tumugi
197
197
  - lib/tumugi.rb
198
- - lib/tumugi/application.rb
199
198
  - lib/tumugi/atomic_file.rb
200
199
  - lib/tumugi/cli.rb
200
+ - lib/tumugi/command/new.rb
201
201
  - lib/tumugi/command/run.rb
202
202
  - lib/tumugi/command/show.rb
203
203
  - lib/tumugi/config.rb
204
204
  - lib/tumugi/dag.rb
205
+ - lib/tumugi/dag_result_reporter.rb
206
+ - lib/tumugi/data/new/Gemfile.erb
207
+ - lib/tumugi/data/new/README.md.erb
208
+ - lib/tumugi/data/new/Rakefile.erb
209
+ - lib/tumugi/data/new/examples/example.rb.erb
210
+ - lib/tumugi/data/new/gemspec.erb
211
+ - lib/tumugi/data/new/gitignore.erb
212
+ - lib/tumugi/data/new/lib/tumugi/plugin/target/target.rb.erb
213
+ - lib/tumugi/data/new/lib/tumugi/plugin/task/task.rb.erb
214
+ - lib/tumugi/data/new/test/plugin/target/target_test.rb.erb
215
+ - lib/tumugi/data/new/test/plugin/task/task_test.rb.erb
216
+ - lib/tumugi/data/new/test/test.rb.erb
217
+ - lib/tumugi/data/new/test/test_helper.rb.erb
205
218
  - lib/tumugi/dsl.rb
206
219
  - lib/tumugi/error.rb
220
+ - lib/tumugi/executor/local_executor.rb
207
221
  - lib/tumugi/file_system.rb
208
222
  - lib/tumugi/logger.rb
209
223
  - lib/tumugi/mixin/listable.rb
@@ -222,7 +236,9 @@ files:
222
236
  - lib/tumugi/target.rb
223
237
  - lib/tumugi/task.rb
224
238
  - lib/tumugi/task_definition.rb
239
+ - lib/tumugi/test/helper.rb
225
240
  - lib/tumugi/version.rb
241
+ - lib/tumugi/workflow.rb
226
242
  - tumugi.gemspec
227
243
  homepage: https://github.com/tumugi/tumugi
228
244
  licenses: