tumugi 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c2ac1c29b7d48bd8d12b5f62b3cd177ca1f34109
-  data.tar.gz: 81f64f2898be4b90207549d547260fb333858c2d
+  metadata.gz: 9c59de7fccebbe6c040ae40cbf9c50acb0efd0e9
+  data.tar.gz: 92a74c2333d2b346c85c4536d93fd96faa502cb6
 SHA512:
-  metadata.gz: ef38cfe6332a2d809899750b958683ccaef7890b563b7352150c36c75ae46051238e7c3235b64dbe407eb86744cb35b178e3a6a6b97884d9c842dfa54058c86f
-  data.tar.gz: 7d6a1fbcb68dafacbaf3adb0c60da3c132b345d1623ef3dd7ec34057a2bc0796a6af880c10d6c5ab7c3554a80ecda20f040ad5400e3c96865931c57b64abf0ca
+  metadata.gz: b49c7ab0b98ad42c33bddeee8ee2653a66d4679874170a368cac814b2a96f5deb68fe6373234e3b615da8c17fc93fa2d41b9b1543b621c5014e6ab201753333f
+  data.tar.gz: 5736bd6b520a2b42a959bcf75fe3a198a34458899ec1b6b5dc4abaf8987e62d34ff574c59e94fee365b7aff6a90d65c2afb3b42e403f36a05c28ca7b977e44c5

data/.travis.yml

@@ -1,11 +1,22 @@
 language: ruby
-rvm:
-  - 2.0.0-p648
-  - 2.1.10
-  - 2.2.5
-  - 2.3.1
-  - jruby-9.0.5.0
+
+matrix:
+  include:
+    - rvm: 2.1.10
+      os: linux
+    - rvm: 2.2.5
+      os: linux
+    - rvm: 2.3.1
+      os: linux
+    - rvm: ruby-head
+      os: linux
+    - rvm: jruby-9.0.5.0
+      os: linux
+  allow_failures:
+    - rvm: ruby-head
+
 cache: bundler
+
 addons:
   apt:
     packages:

data/CHANGELOG.md

@@ -1,6 +1,21 @@
 # Change Log
 
-## [v0.3.0](https://github.com/tumugi/tumugi/tree/v0.3.0) (2016-05-07)
+## [0.4.0](https://github.com/tumugi/tumugi/tree/0.4.0) (2016-05-09)
+[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.3.0...0.4.0)
+
+**Implemented enhancements:**
+
+- \[Breaking Change\] Remove support ruby 2.0 [\#33](https://github.com/tumugi/tumugi/issues/33)
+- Add config section [\#31](https://github.com/tumugi/tumugi/issues/31)
+
+**Merged pull requests:**
+
+- Refactoring error handling [\#37](https://github.com/tumugi/tumugi/pull/37) ([hakobera](https://github.com/hakobera))
+- Change config section spec [\#35](https://github.com/tumugi/tumugi/pull/35) ([hakobera](https://github.com/hakobera))
+- Remove support Ruby 2.0 [\#34](https://github.com/tumugi/tumugi/pull/34) ([hakobera](https://github.com/hakobera))
+- Implement config section [\#32](https://github.com/tumugi/tumugi/pull/32) ([hakobera](https://github.com/hakobera))
+
+## [v0.3.0](https://github.com/tumugi/tumugi/tree/v0.3.0) (2016-05-06)
 [Full Changelog](https://github.com/tumugi/tumugi/compare/v0.2.0...v0.3.0)
 
 **Implemented enhancements:**
@@ -14,6 +29,7 @@
 
 **Merged pull requests:**
 
+- Prepare for release v0.3.0 [\#27](https://github.com/tumugi/tumugi/pull/27) ([hakobera](https://github.com/hakobera))
 - Update README [\#26](https://github.com/tumugi/tumugi/pull/26) ([hakobera](https://github.com/hakobera))
 - Implement workflow run result report [\#24](https://github.com/tumugi/tumugi/pull/24) ([hakobera](https://github.com/hakobera))
 - Implement parameter feature [\#22](https://github.com/tumugi/tumugi/pull/22) ([hakobera](https://github.com/hakobera))

data/README.md

@@ -6,7 +6,7 @@ Tumugi is a ruby library to build, run and manage complex workflows. Tumugi enab
 
 ## Installation
 
-**Tumugi only support Ruby 2.0+**
+**Tumugi only support Ruby 2.1 and above**
 
 Add this line to your application's Gemfile:
 

data/examples/config_section.rb

@@ -0,0 +1,8 @@
+::Tumugi::Config.register_section('example', :key)
+
+task :task1 do
+  run do
+    log 'task1#run'
+    log "config section example.key=#{Tumugi.config.section('example').key}"
+  end
+end

data/examples/tumugi_config_with_section.rb

@@ -0,0 +1,8 @@
+Tumugi.config do |c|
+  c.max_retry = 3
+  c.retry_interval = 1
+
+  c.section('example') do |s|
+    s.key = 'value'
+  end
+end

data/lib/tumugi.rb

@@ -15,9 +15,7 @@ module Tumugi
 
     def config
       @config ||= Tumugi::Config.new
-      if block_given?
-        yield @config
-      end
+      yield @config if block_given?
       @config
     end
   end

data/lib/tumugi/application.rb

@@ -1,5 +1,6 @@
 require 'tumugi/dag'
 require 'tumugi/dsl'
+require 'tumugi/error'
 require 'tumugi/plugin'
 require 'tumugi/target'
 require 'tumugi/command/run'
@@ -15,7 +16,8 @@ module Tumugi
     end
 
     def execute(command, root_task_id, options)
-      load(options[:file], true)
+      process_common_options(options)
+      load_workflow_file(options[:file])
       dag = create_dag(root_task_id)
       command_module = Kernel.const_get("Tumugi").const_get("Command")
       cmd = command_module.const_get("#{command.to_s.capitalize}").new
@@ -28,17 +30,55 @@ module Tumugi
 
     def find_task(id)
       task = @tasks[id.to_s]
-      raise "Task not found: #{id}" if task.nil?
+      raise Tumugi::Error, "Task not found: #{id}" if task.nil?
       task
     end
 
     private
 
+    def load_workflow_file(file)
+      load(file, true)
+    rescue LoadError => e
+      raise Tumugi::Error, e
+    end
+
     def create_dag(id)
       dag = Tumugi::DAG.new
       task = find_task(id)
       dag.add_task(task)
       dag
     end
+
+    def process_common_options(options)
+      setup_logger(options)
+      load_config(options)
+      set_params(options)
+    end
+
+    def logger
+      @logger ||= Tumugi.logger
+    end
+
+    def setup_logger(options)
+      logger.verbose! if options[:verbose]
+      logger.quiet! if options[:quiet]
+    end
+
+    def load_config(options)
+      config_file = options[:config]
+      if config_file && File.exists?(config_file) && File.extname(config_file) == '.rb'
+        logger.info "Load config from #{config_file}"
+        load(config_file)
+      end
+    rescue LoadError => e
+      raise Tumugi::Error, e
+    end
+
+    def set_params(options)
+      if options[:params]
+        @params = options[:params]
+        logger.info "Parameters: #{@params}"
+      end
+    end
   end
 end

data/lib/tumugi/cli.rb

@@ -8,8 +8,10 @@ module Tumugi
     class << self
       def common_options
         option :file, aliases: '-f', desc: 'Task definition file name', required: true
-        option :config, aliases: '-c', desc: 'Configuration file name', default: 'tumugi.rb'
+        option :config, aliases: '-c', desc: 'Configuration file name', default: 'tumugi_config.rb'
         option :params, aliases: '-p', type: :hash, desc: 'Task parameters'
+        option :quiet, type: :boolean, desc: 'Suppress log', default: false
+        option :verbose, type: :boolean, desc: 'Show verbose log', default: false
       end
 
       def exit_on_failure?
@@ -25,12 +27,9 @@ module Tumugi
     desc "run TASK", "Run TASK in a workflow"
     map "run" => "run_" # run is thor's reserved word, so this trick is needed
     option :workers, aliases: '-w', type: :numeric, desc: 'Number of workers to run task concurrently'
-    option :quiet, type: :boolean, desc: 'Suppress log', default: false
-    option :verbose, type: :boolean, desc: 'Show verbose log', default: false
     common_options
     def run_(task)
-      process_common_options
-      Tumugi.application.execute(:run, task, options)
+      execute(:run, task, options)
     end
 
     desc "show TASK", "Show DAG of TASK in a workflow"
@@ -38,20 +37,22 @@ module Tumugi
     option :out, aliases: '-o', desc: 'Output file name. If not specified, output result to STDOUT'
     option :format, aliases: '-t', desc: 'Output file format. Only affected --out option is specified.', enum: ['dot', 'png', 'svg']
     def show(task)
-      process_common_options
-      Tumugi.application.execute(:show, task, options)
+      execute(:show, task, options)
     end
 
     private
 
-    def process_common_options
-      config_file = options[:config]
-      if config_file && File.exists?(config_file) && File.extname(config_file) == '.rb'
-        load(config_file)
+    def execute(command, task, options)
+      success = Tumugi.application.execute(command, task, options)
+      unless success
+        Tumugi.logger.error "#{command} command failed"
+        raise Thor::Error, 'failed'
       end
-
-      params = options[:params]
-      Tumugi.application.params = params if params
+    rescue => e
+      Tumugi.logger.error "#{command} command failed"
+      Tumugi.logger.error "Stacktrace:"
+      e.backtrace.each { |line| Tumugi.logger.error line }
+      raise Thor::Error, 'failed'
     end
   end
 end

data/lib/tumugi/command/run.rb

@@ -15,9 +15,6 @@ module Tumugi
         settings = { in_threads: workers }
         logger = Tumugi.logger
 
-        logger.verbose! if options[:verbose]
-        logger.quiet! if options[:quiet]
-
         Parallel.each(dag.tsort, settings) do |t|
           logger.info "start: #{t.id}"
           until t.ready? || t.requires_failed?
@@ -50,7 +47,7 @@ module Tumugi
         end
 
         show_result_report(dag)
-        raise ::Thor::Error.new("run failed") if dag.tsort.any? { |t| t.state == :failed }
+        return !dag.tsort.any? { |t| t.state == :failed }
       end
 
       private

data/lib/tumugi/command/show.rb

@@ -41,6 +41,8 @@ module Tumugi
         else
           print g
         end
+
+        return true
       end
     end
   end

data/lib/tumugi/config.rb

@@ -1,12 +1,51 @@
 module Tumugi
+  class ConfigError < StandardError
+  end
+
   class Config
     attr_accessor :workers, :max_retry, :retry_interval, :param_auto_bind_enabled
 
+    @@sections ||= {}
+
+    def self.register_section(name, *args)
+      @@sections[name] = Struct.new(camelize(name), *args)
+      Tumugi.logger.debug "registered config section '#{name}' with '#{args}'"
+    end
+
+    def self.camelize(term)
+      string = term.to_s
+      string = string.sub(/^[a-z\d]*/) { $&.capitalize }
+      string.gsub!(/(?:_|(\/))([a-z\d]*)/) { $2.capitalize }
+      string
+    end
+
     def initialize
       @workers = 1
       @max_retry = 3
       @retry_interval = 300 #seconds
       @param_auto_bind_enabled = true
+      @section_procs = {}
+      @section_instances = {}
+    end
+
+    def section(name, &block)
+      section_class = @@sections[name]
+      if block_given?
+        @section_procs[name] ||= block
+      elsif section_class.nil?
+        raise ConfigError, "Config section '#{name}' is not registered."
+      else
+        @section_instances[name] ||= section_class.new
+        if @section_procs[name]
+          begin
+            @section_procs[name].call(@section_instances[name])
+          rescue NoMethodError => e
+            Tumugi.logger.error "#{e.message}. Available attributes are #{@section_instances[name].members}"
+            raise e
+          end
+        end
+        @section_instances[name]
+      end
     end
   end
 end

data/lib/tumugi/error.rb

@@ -0,0 +1,4 @@
+module Tumugi
+  class TumugiError < StandardError
+  end
+end

data/lib/tumugi/target.rb

@@ -2,6 +2,10 @@ require 'tumugi'
 
 module Tumugi
   class Target
+    def initialize
+      super()
+    end
+
     def exist?
       raise NotImplementedError, "You must implement #{self.class}##{__method__}"
     end

data/lib/tumugi/version.rb

@@ -1,3 +1,3 @@
 module Tumugi
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/tumugi.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.0'
+  spec.required_ruby_version = '>= 2.1'
 
   spec.add_runtime_dependency 'parallel', '~> 1.8.0'
   spec.add_runtime_dependency 'retriable', '~> 2.1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tumugi
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-06 00:00:00.000000000 Z
+date: 2016-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -182,6 +182,7 @@ files:
 - bin/console
 - bin/setup
 - examples/concurrent_task_run.rb
+- examples/config_section.rb
 - examples/data_pipeline.rb
 - examples/fail_first_task.rb
 - examples/fail_intermediate_task.rb
@@ -191,6 +192,7 @@ files:
 - examples/task_inheritance.rb
 - examples/task_parameter.rb
 - examples/tumugi_config.rb
+- examples/tumugi_config_with_section.rb
 - exe/tumugi
 - lib/tumugi.rb
 - lib/tumugi/application.rb
@@ -201,6 +203,7 @@ files:
 - lib/tumugi/config.rb
 - lib/tumugi/dag.rb
 - lib/tumugi/dsl.rb
+- lib/tumugi/error.rb
 - lib/tumugi/file_system.rb
 - lib/tumugi/file_system_error.rb
 - lib/tumugi/logger.rb
@@ -234,7 +237,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.0'
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="