tumugi 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c2ac1c29b7d48bd8d12b5f62b3cd177ca1f34109
4
- data.tar.gz: 81f64f2898be4b90207549d547260fb333858c2d
3
+ metadata.gz: 9c59de7fccebbe6c040ae40cbf9c50acb0efd0e9
4
+ data.tar.gz: 92a74c2333d2b346c85c4536d93fd96faa502cb6
5
5
  SHA512:
6
- metadata.gz: ef38cfe6332a2d809899750b958683ccaef7890b563b7352150c36c75ae46051238e7c3235b64dbe407eb86744cb35b178e3a6a6b97884d9c842dfa54058c86f
7
- data.tar.gz: 7d6a1fbcb68dafacbaf3adb0c60da3c132b345d1623ef3dd7ec34057a2bc0796a6af880c10d6c5ab7c3554a80ecda20f040ad5400e3c96865931c57b64abf0ca
6
+ metadata.gz: b49c7ab0b98ad42c33bddeee8ee2653a66d4679874170a368cac814b2a96f5deb68fe6373234e3b615da8c17fc93fa2d41b9b1543b621c5014e6ab201753333f
7
+ data.tar.gz: 5736bd6b520a2b42a959bcf75fe3a198a34458899ec1b6b5dc4abaf8987e62d34ff574c59e94fee365b7aff6a90d65c2afb3b42e403f36a05c28ca7b977e44c5
data/.travis.yml CHANGED
@@ -1,11 +1,22 @@
1
1
  language: ruby
2
- rvm:
3
- - 2.0.0-p648
4
- - 2.1.10
5
- - 2.2.5
6
- - 2.3.1
7
- - jruby-9.0.5.0
2
+
3
+ matrix:
4
+ include:
5
+ - rvm: 2.1.10
6
+ os: linux
7
+ - rvm: 2.2.5
8
+ os: linux
9
+ - rvm: 2.3.1
10
+ os: linux
11
+ - rvm: ruby-head
12
+ os: linux
13
+ - rvm: jruby-9.0.5.0
14
+ os: linux
15
+ allow_failures:
16
+ - rvm: ruby-head
17
+
8
18
  cache: bundler
19
+
9
20
  addons:
10
21
  apt:
11
22
  packages:
data/CHANGELOG.md CHANGED
@@ -1,6 +1,21 @@
1
1
  # Change Log
2
2
 
3
- ## [v0.3.0](https://github.com/tumugi/tumugi/tree/v0.3.0) (2016-05-07)
3
+ ## [0.4.0](https://github.com/tumugi/tumugi/tree/0.4.0) (2016-05-09)
4
+ [Full Changelog](https://github.com/tumugi/tumugi/compare/v0.3.0...0.4.0)
5
+
6
+ **Implemented enhancements:**
7
+
8
+ - \[Breaking Change\] Remove support ruby 2.0 [\#33](https://github.com/tumugi/tumugi/issues/33)
9
+ - Add config section [\#31](https://github.com/tumugi/tumugi/issues/31)
10
+
11
+ **Merged pull requests:**
12
+
13
+ - Refactoring error handling [\#37](https://github.com/tumugi/tumugi/pull/37) ([hakobera](https://github.com/hakobera))
14
+ - Change config section spec [\#35](https://github.com/tumugi/tumugi/pull/35) ([hakobera](https://github.com/hakobera))
15
+ - Remove support Ruby 2.0 [\#34](https://github.com/tumugi/tumugi/pull/34) ([hakobera](https://github.com/hakobera))
16
+ - Implement config section [\#32](https://github.com/tumugi/tumugi/pull/32) ([hakobera](https://github.com/hakobera))
17
+
18
+ ## [v0.3.0](https://github.com/tumugi/tumugi/tree/v0.3.0) (2016-05-06)
4
19
  [Full Changelog](https://github.com/tumugi/tumugi/compare/v0.2.0...v0.3.0)
5
20
 
6
21
  **Implemented enhancements:**
@@ -14,6 +29,7 @@
14
29
 
15
30
  **Merged pull requests:**
16
31
 
32
+ - Prepare for release v0.3.0 [\#27](https://github.com/tumugi/tumugi/pull/27) ([hakobera](https://github.com/hakobera))
17
33
  - Update README [\#26](https://github.com/tumugi/tumugi/pull/26) ([hakobera](https://github.com/hakobera))
18
34
  - Implement workflow run result report [\#24](https://github.com/tumugi/tumugi/pull/24) ([hakobera](https://github.com/hakobera))
19
35
  - Implement parameter feature [\#22](https://github.com/tumugi/tumugi/pull/22) ([hakobera](https://github.com/hakobera))
data/README.md CHANGED
@@ -6,7 +6,7 @@ Tumugi is a ruby library to build, run and manage complex workflows. Tumugi enab
6
6
 
7
7
  ## Installation
8
8
 
9
- **Tumugi only support Ruby 2.0+**
9
+ **Tumugi only support Ruby 2.1 and above**
10
10
 
11
11
  Add this line to your application's Gemfile:
12
12
 
@@ -0,0 +1,8 @@
1
+ ::Tumugi::Config.register_section('example', :key)
2
+
3
+ task :task1 do
4
+ run do
5
+ log 'task1#run'
6
+ log "config section example.key=#{Tumugi.config.section('example').key}"
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ Tumugi.config do |c|
2
+ c.max_retry = 3
3
+ c.retry_interval = 1
4
+
5
+ c.section('example') do |s|
6
+ s.key = 'value'
7
+ end
8
+ end
data/lib/tumugi.rb CHANGED
@@ -15,9 +15,7 @@ module Tumugi
15
15
 
16
16
  def config
17
17
  @config ||= Tumugi::Config.new
18
- if block_given?
19
- yield @config
20
- end
18
+ yield @config if block_given?
21
19
  @config
22
20
  end
23
21
  end
@@ -1,5 +1,6 @@
1
1
  require 'tumugi/dag'
2
2
  require 'tumugi/dsl'
3
+ require 'tumugi/error'
3
4
  require 'tumugi/plugin'
4
5
  require 'tumugi/target'
5
6
  require 'tumugi/command/run'
@@ -15,7 +16,8 @@ module Tumugi
15
16
  end
16
17
 
17
18
  def execute(command, root_task_id, options)
18
- load(options[:file], true)
19
+ process_common_options(options)
20
+ load_workflow_file(options[:file])
19
21
  dag = create_dag(root_task_id)
20
22
  command_module = Kernel.const_get("Tumugi").const_get("Command")
21
23
  cmd = command_module.const_get("#{command.to_s.capitalize}").new
@@ -28,17 +30,55 @@ module Tumugi
28
30
 
29
31
  def find_task(id)
30
32
  task = @tasks[id.to_s]
31
- raise "Task not found: #{id}" if task.nil?
33
+ raise Tumugi::Error, "Task not found: #{id}" if task.nil?
32
34
  task
33
35
  end
34
36
 
35
37
  private
36
38
 
39
+ def load_workflow_file(file)
40
+ load(file, true)
41
+ rescue LoadError => e
42
+ raise Tumugi::Error, e
43
+ end
44
+
37
45
  def create_dag(id)
38
46
  dag = Tumugi::DAG.new
39
47
  task = find_task(id)
40
48
  dag.add_task(task)
41
49
  dag
42
50
  end
51
+
52
+ def process_common_options(options)
53
+ setup_logger(options)
54
+ load_config(options)
55
+ set_params(options)
56
+ end
57
+
58
+ def logger
59
+ @logger ||= Tumugi.logger
60
+ end
61
+
62
+ def setup_logger(options)
63
+ logger.verbose! if options[:verbose]
64
+ logger.quiet! if options[:quiet]
65
+ end
66
+
67
+ def load_config(options)
68
+ config_file = options[:config]
69
+ if config_file && File.exists?(config_file) && File.extname(config_file) == '.rb'
70
+ logger.info "Load config from #{config_file}"
71
+ load(config_file)
72
+ end
73
+ rescue LoadError => e
74
+ raise Tumugi::Error, e
75
+ end
76
+
77
+ def set_params(options)
78
+ if options[:params]
79
+ @params = options[:params]
80
+ logger.info "Parameters: #{@params}"
81
+ end
82
+ end
43
83
  end
44
84
  end
data/lib/tumugi/cli.rb CHANGED
@@ -8,8 +8,10 @@ module Tumugi
8
8
  class << self
9
9
  def common_options
10
10
  option :file, aliases: '-f', desc: 'Task definition file name', required: true
11
- option :config, aliases: '-c', desc: 'Configuration file name', default: 'tumugi.rb'
11
+ option :config, aliases: '-c', desc: 'Configuration file name', default: 'tumugi_config.rb'
12
12
  option :params, aliases: '-p', type: :hash, desc: 'Task parameters'
13
+ option :quiet, type: :boolean, desc: 'Suppress log', default: false
14
+ option :verbose, type: :boolean, desc: 'Show verbose log', default: false
13
15
  end
14
16
 
15
17
  def exit_on_failure?
@@ -25,12 +27,9 @@ module Tumugi
25
27
  desc "run TASK", "Run TASK in a workflow"
26
28
  map "run" => "run_" # run is thor's reserved word, so this trick is needed
27
29
  option :workers, aliases: '-w', type: :numeric, desc: 'Number of workers to run task concurrently'
28
- option :quiet, type: :boolean, desc: 'Suppress log', default: false
29
- option :verbose, type: :boolean, desc: 'Show verbose log', default: false
30
30
  common_options
31
31
  def run_(task)
32
- process_common_options
33
- Tumugi.application.execute(:run, task, options)
32
+ execute(:run, task, options)
34
33
  end
35
34
 
36
35
  desc "show TASK", "Show DAG of TASK in a workflow"
@@ -38,20 +37,22 @@ module Tumugi
38
37
  option :out, aliases: '-o', desc: 'Output file name. If not specified, output result to STDOUT'
39
38
  option :format, aliases: '-t', desc: 'Output file format. Only affected --out option is specified.', enum: ['dot', 'png', 'svg']
40
39
  def show(task)
41
- process_common_options
42
- Tumugi.application.execute(:show, task, options)
40
+ execute(:show, task, options)
43
41
  end
44
42
 
45
43
  private
46
44
 
47
- def process_common_options
48
- config_file = options[:config]
49
- if config_file && File.exists?(config_file) && File.extname(config_file) == '.rb'
50
- load(config_file)
45
+ def execute(command, task, options)
46
+ success = Tumugi.application.execute(command, task, options)
47
+ unless success
48
+ Tumugi.logger.error "#{command} command failed"
49
+ raise Thor::Error, 'failed'
51
50
  end
52
-
53
- params = options[:params]
54
- Tumugi.application.params = params if params
51
+ rescue => e
52
+ Tumugi.logger.error "#{command} command failed"
53
+ Tumugi.logger.error "Stacktrace:"
54
+ e.backtrace.each { |line| Tumugi.logger.error line }
55
+ raise Thor::Error, 'failed'
55
56
  end
56
57
  end
57
58
  end
@@ -15,9 +15,6 @@ module Tumugi
15
15
  settings = { in_threads: workers }
16
16
  logger = Tumugi.logger
17
17
 
18
- logger.verbose! if options[:verbose]
19
- logger.quiet! if options[:quiet]
20
-
21
18
  Parallel.each(dag.tsort, settings) do |t|
22
19
  logger.info "start: #{t.id}"
23
20
  until t.ready? || t.requires_failed?
@@ -50,7 +47,7 @@ module Tumugi
50
47
  end
51
48
 
52
49
  show_result_report(dag)
53
- raise ::Thor::Error.new("run failed") if dag.tsort.any? { |t| t.state == :failed }
50
+ return !dag.tsort.any? { |t| t.state == :failed }
54
51
  end
55
52
 
56
53
  private
@@ -41,6 +41,8 @@ module Tumugi
41
41
  else
42
42
  print g
43
43
  end
44
+
45
+ return true
44
46
  end
45
47
  end
46
48
  end
data/lib/tumugi/config.rb CHANGED
@@ -1,12 +1,51 @@
1
1
  module Tumugi
2
+ class ConfigError < StandardError
3
+ end
4
+
2
5
  class Config
3
6
  attr_accessor :workers, :max_retry, :retry_interval, :param_auto_bind_enabled
4
7
 
8
+ @@sections ||= {}
9
+
10
+ def self.register_section(name, *args)
11
+ @@sections[name] = Struct.new(camelize(name), *args)
12
+ Tumugi.logger.debug "registered config section '#{name}' with '#{args}'"
13
+ end
14
+
15
+ def self.camelize(term)
16
+ string = term.to_s
17
+ string = string.sub(/^[a-z\d]*/) { $&.capitalize }
18
+ string.gsub!(/(?:_|(\/))([a-z\d]*)/) { $2.capitalize }
19
+ string
20
+ end
21
+
5
22
  def initialize
6
23
  @workers = 1
7
24
  @max_retry = 3
8
25
  @retry_interval = 300 #seconds
9
26
  @param_auto_bind_enabled = true
27
+ @section_procs = {}
28
+ @section_instances = {}
29
+ end
30
+
31
+ def section(name, &block)
32
+ section_class = @@sections[name]
33
+ if block_given?
34
+ @section_procs[name] ||= block
35
+ elsif section_class.nil?
36
+ raise ConfigError, "Config section '#{name}' is not registered."
37
+ else
38
+ @section_instances[name] ||= section_class.new
39
+ if @section_procs[name]
40
+ begin
41
+ @section_procs[name].call(@section_instances[name])
42
+ rescue NoMethodError => e
43
+ Tumugi.logger.error "#{e.message}. Available attributes are #{@section_instances[name].members}"
44
+ raise e
45
+ end
46
+ end
47
+ @section_instances[name]
48
+ end
10
49
  end
11
50
  end
12
51
  end
@@ -0,0 +1,4 @@
1
+ module Tumugi
2
+ class TumugiError < StandardError
3
+ end
4
+ end
data/lib/tumugi/target.rb CHANGED
@@ -2,6 +2,10 @@ require 'tumugi'
2
2
 
3
3
  module Tumugi
4
4
  class Target
5
+ def initialize
6
+ super()
7
+ end
8
+
5
9
  def exist?
6
10
  raise NotImplementedError, "You must implement #{self.class}##{__method__}"
7
11
  end
@@ -1,3 +1,3 @@
1
1
  module Tumugi
2
- VERSION = "0.3.0"
2
+ VERSION = "0.4.0"
3
3
  end
data/tumugi.gemspec CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
18
18
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
19
19
  spec.require_paths = ['lib']
20
20
 
21
- spec.required_ruby_version = '>= 2.0'
21
+ spec.required_ruby_version = '>= 2.1'
22
22
 
23
23
  spec.add_runtime_dependency 'parallel', '~> 1.8.0'
24
24
  spec.add_runtime_dependency 'retriable', '~> 2.1'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tumugi
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kazuyuki Honda
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-05-06 00:00:00.000000000 Z
11
+ date: 2016-05-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: parallel
@@ -182,6 +182,7 @@ files:
182
182
  - bin/console
183
183
  - bin/setup
184
184
  - examples/concurrent_task_run.rb
185
+ - examples/config_section.rb
185
186
  - examples/data_pipeline.rb
186
187
  - examples/fail_first_task.rb
187
188
  - examples/fail_intermediate_task.rb
@@ -191,6 +192,7 @@ files:
191
192
  - examples/task_inheritance.rb
192
193
  - examples/task_parameter.rb
193
194
  - examples/tumugi_config.rb
195
+ - examples/tumugi_config_with_section.rb
194
196
  - exe/tumugi
195
197
  - lib/tumugi.rb
196
198
  - lib/tumugi/application.rb
@@ -201,6 +203,7 @@ files:
201
203
  - lib/tumugi/config.rb
202
204
  - lib/tumugi/dag.rb
203
205
  - lib/tumugi/dsl.rb
206
+ - lib/tumugi/error.rb
204
207
  - lib/tumugi/file_system.rb
205
208
  - lib/tumugi/file_system_error.rb
206
209
  - lib/tumugi/logger.rb
@@ -234,7 +237,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
234
237
  requirements:
235
238
  - - ">="
236
239
  - !ruby/object:Gem::Version
237
- version: '2.0'
240
+ version: '2.1'
238
241
  required_rubygems_version: !ruby/object:Gem::Requirement
239
242
  requirements:
240
243
  - - ">="