tumugi 0.6.3 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5377ec144cacd5bddd1555300c31f7794b628dbc
-  data.tar.gz: aa97c713302feba8c3735ab4876bc1616953bb3a
+  metadata.gz: e1245ddebd2d69768433b3cfdea9ea0c3e39cff8
+  data.tar.gz: f40eded5362a22f951caf384411387758fb668ce
 SHA512:
-  metadata.gz: 09197a49b7d0b0b59a04ef6f038c2f4990dfdd61a54683d578b894203b5f48a34fdb99465359cbd48f10c556190b61e63833f08d2f5ceea69595e028dcd8cc57
-  data.tar.gz: 4397186e5d06c7e21531835f0de0d98ecc3f4237ee40cf927155ec1b29dc58fcf0e69c8d5dbee0d2ea61433ce3a26296ef3b8743099867b2223f3086a944c9c3
+  metadata.gz: e1d78046c13e29e2f0cff749e54cfb020256de583c5561c9664edbabfcdae7a45fc64918b5cb4582a0ea7321e8f87932e0470365f142e12c792b82e432d94bff
+  data.tar.gz: b6745f25485a5aaa1e6ad494e468640209839fe77a7952ba955f888554f41764ccd31c6c983b04e6030a82e2e040f51af01d8f8d7c51b097ec62016ea5eb0bd0

data/.travis.yml

@@ -6,7 +6,7 @@ rvm:
   - 2.3.1
   - ruby-head
   - jruby-9.0.5.0
-  - jruby-9.1.2.0
+  - jruby-9.1.5.0
   - jruby-head
 before_install:
   - gem install bundler

data/CHANGELOG.md

@@ -1,6 +1,28 @@
 # Change Log
 
-## [v0.6.3](https://github.com/tumugi/tumugi/tree/v0.6.3) (2016-08-14)
+## [v0.7.0](https://github.com/tumugi/tumugi/tree/v0.7.0) (2016-11-01)
+[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.6.3...v0.7.0)
+
+**Implemented enhancements:**
+
+- Force run all tasks in a workflow [\#125](https://github.com/tumugi/tumugi/issues/125)
+- Custom callback on success/retry/failure for each task [\#122](https://github.com/tumugi/tumugi/issues/122)
+- Add secret option for parameter [\#117](https://github.com/tumugi/tumugi/issues/117)
+- Add log about elapsed time for each tasks and workflow [\#116](https://github.com/tumugi/tumugi/issues/116)
+- \[Breaking Change\] Introduce subcommand of `tumugi new` [\#111](https://github.com/tumugi/tumugi/issues/111)
+
+**Merged pull requests:**
+
+- Introduce success/failure event callback [\#128](https://github.com/tumugi/tumugi/pull/128) ([hakobera](https://github.com/hakobera))
+- Support force run all tasks [\#127](https://github.com/tumugi/tumugi/pull/127) ([hakobera](https://github.com/hakobera))
+- Write workflow and task elapsed time to log [\#124](https://github.com/tumugi/tumugi/pull/124) ([hakobera](https://github.com/hakobera))
+- Filter coverage target files [\#121](https://github.com/tumugi/tumugi/pull/121) ([hakobera](https://github.com/hakobera))
+- Test with jruby-9.1.5.0 on travis [\#120](https://github.com/tumugi/tumugi/pull/120) ([hakobera](https://github.com/hakobera))
+- Introduce new sub commands [\#119](https://github.com/tumugi/tumugi/pull/119) ([hakobera](https://github.com/hakobera))
+- Add param secret option [\#118](https://github.com/tumugi/tumugi/pull/118) ([hakobera](https://github.com/hakobera))
+- Bumpup version to 0.7.0.dev [\#115](https://github.com/tumugi/tumugi/pull/115) ([hakobera](https://github.com/hakobera))
+
+## [v0.6.3](https://github.com/tumugi/tumugi/tree/v0.6.3) (2016-08-13)
 [Full Changelog](https://github.com/tumugi/tumugi/compare/v0.6.2...v0.6.3)
 
 **Implemented enhancements:**
@@ -12,6 +34,10 @@
 - Fix plugin template [\#112](https://github.com/tumugi/tumugi/pull/112) ([hakobera](https://github.com/hakobera))
 - Fix commad failed error message [\#110](https://github.com/tumugi/tumugi/pull/110) ([hakobera](https://github.com/hakobera))
 
+**Merged pull requests:**
+
+- Prepare release for 0.6.3 [\#114](https://github.com/tumugi/tumugi/pull/114) ([hakobera](https://github.com/hakobera))
+
 ## [v0.6.2](https://github.com/tumugi/tumugi/tree/v0.6.2) (2016-08-02)
 [Full Changelog](https://github.com/tumugi/tumugi/compare/v0.6.1...v0.6.2)
 

data/examples/event_callbacks.rb

@@ -0,0 +1,37 @@
+task :task1 do
+  requires [:task2, :task3]
+  run { log 'task1#run' }
+end
+
+task :task2 do
+  requires [:task4]
+  run { raise "error in #{id}" }
+  on_retry do
+    log 'task2#on_retry'
+  end
+  on_failure do
+    log 'task2#on_failure'
+  end
+end
+
+task :task3 do
+  requires [:task4]
+  run { log 'task3#run' }
+  on_success do
+    log 'task3#on_success'
+  end
+end
+
+task :task4 do
+  run do
+    log 'task4#run'
+    sleep 1
+  end
+  on_success do
+    log 'task4#on_success'
+  end
+end
+
+on_failure do
+  log 'task1#on_failure'
+end

data/lib/tumugi/cli.rb

@@ -30,6 +30,7 @@ module Tumugi
     map "run" => "run_" # run is thor's reserved word, so this trick is needed
     option :workers, aliases: '-w', type: :numeric, desc: 'Number of workers to run task concurrently'
     option :out, aliases: '-o', desc: 'Output log filename. If not specified, log is write to STDOUT'
+    option :all, aliases: '-a', type: :boolean, desc: 'Run all tasks even if target exists', default: false
     common_options
     def run_(task)
       execute(:run, task, options)
@@ -45,9 +46,14 @@ module Tumugi
       execute(:show, task, opts.freeze)
     end
 
-    desc "new PLUGIN_NAME", "Create new plugin project"
-    def new(name)
-      generate_plugin(name, options)
+    desc "new type NAME", "Create a new template of type. type is 'plugin' or 'project'"
+    def new(type, name)
+      generate_project(type, name, options)
+    end
+
+    desc "init", "Create an tumugi project template"
+    def init(path='')
+      generate_project('project', path, force: true)
     end
 
     private
@@ -55,6 +61,7 @@ module Tumugi
     def execute(command, task, options)
       logger.info "tumugi v#{Tumugi::VERSION}"
       args = { task: task, options: options }
+      logger.info "status: running, command: #{command}, args: #{args}"
       success = Tumugi.workflow.execute(command, task, options)
       unless success
         raise Thor::Error, "execute finished, but failed"
@@ -64,10 +71,11 @@ module Tumugi
       handle_error(command, e, args)
     end
 
-    def generate_plugin(name, options)
+    def generate_project(type, name, options)
       logger.info "tumugi v#{Tumugi::VERSION}"
-      args = { name: name, options: options }
-      Tumugi::Command::New.new.execute(name, options)
+      args = { type: type, name: name, options: options }
+      logger.info "status: running, command: new, args: #{args}"
+      Tumugi::Command::New.new.execute(type, name, options)
       logger.info "status: success, command: new, args: #{args}"
     rescue => e
       handle_error("new", e, args)

data/lib/tumugi/command/new.rb

@@ -1,71 +1,27 @@
-require 'fileutils'
-require 'erubis'
+require_relative './new/plugin_generator'
+require_relative './new/project_generator'
 
 module Tumugi
   module Command
     class New
-      def execute(name, options={})
-        full_project_name = "tumugi-plugin-#{name}"
-        templates = [
-          [ "Gemfile.erb", "Gemfile" ],
-          [ "gemspec.erb", "#{full_project_name}.gemspec" ],
-          [ "gitignore.erb", ".gitignore" ],
-          [ "Rakefile.erb", "Rakefile" ],
-          [ "README.md.erb", "README.md" ],
-          [ "examples/example.rb.erb", "examples/example.rb" ],
-          [ "lib/tumugi/plugin/target/target.rb.erb", "lib/tumugi/plugin/target/#{name}.rb" ],
-          [ "lib/tumugi/plugin/task/task.rb.erb", "lib/tumugi/plugin/task/#{name}.rb" ],
-          [ "test/test_helper.rb.erb", "test/test_helper.rb" ],
-          [ "test/test.rb.erb", "test/#{name}_test.rb" ],
-          [ "test/plugin/target/target_test.rb.erb", "test/plugin/target/#{name}_test.rb" ],
-          [ "test/plugin/task/task_test.rb.erb", "test/plugin/task/#{name}_test.rb" ],
-        ]
-
-        @data_dir = "#{File.expand_path(File.dirname(__FILE__))}/../data/new"
-        @dest_dir = "#{(options[:path] || '.')}/#{full_project_name}"
-        if File.exist?(@dest_dir)
-          puts "#{@dest_dir} is already exists. Please delete it first"
-          return false
-        end
-
-        puts "Create #{@dest_dir}"
-        FileUtils.mkdir_p(@dest_dir)
-
-        templates.each do |value|
-          src_file, dest_file = value
-          eruby = Erubis::Eruby.new(File.read(src_path(src_file)))
-          eruby.filename = src_path(src_file)
-          context = {
-            full_project_name: full_project_name,
-            name: name,
-            tumugi_version: Tumugi::VERSION,
-          }
-          puts "  Create #{dest_path(dest_file)}"
-          FileUtils.mkdir_p(File.dirname(dest_path(dest_file)))
-          File.write(dest_path(dest_file), eruby.result(context))
-        end
-
-        puts ""
-        puts "Plugin template is successfully generated."
-        puts "Next steps:"
-        puts ""
-        puts "  $ cd #{full_project_name}"
-        puts "  $ git init"
-        puts "  $ bundle install"
-        puts "  $ bundle exec rake"
-        puts ""
-
-        return true
+      def execute(type, name, options={})
+        generator = create_generator(type, name, options)
+        generator.generate
       end
 
-      private
-
-      def src_path(file)
-        "#{@data_dir}/#{file}"
+      def logger
+        @logger ||= Tumugi::ScopedLogger.new("tumugi-new")
       end
 
-      def dest_path(file)
-        "#{@dest_dir}/#{file}"
+      def create_generator(type, name, options)
+        case type
+        when "plugin"
+          PluginGenerator.new(name, options)
+        when "project"
+          ProjectGenerator.new(name, options)
+        else
+          raise Tumugi::TumugiError.new("Unsupported type of new sub command: #{type}")
+        end
       end
     end
   end

data/lib/tumugi/command/new/generator.rb

@@ -0,0 +1,76 @@
+require 'fileutils'
+require 'erubis'
+
+module Tumugi
+  module Command
+    class New
+      class Generator
+        attr_reader :name, :options
+
+        def initialize(name, options={})
+          @name = name
+          @options = options
+        end
+
+        def generate
+          if File.exist?(dest_dir) && !options[:force]
+            logger.error "#{dest_dir} is already exists. Please delete it first"
+            return false
+          end
+
+          logger.info "Create #{dest_dir}"
+          FileUtils.mkdir_p(dest_dir)
+
+          templates.each do |value|
+            src_file, dest_file = value
+            eruby = Erubis::Eruby.new(File.read(src_path(src_file)))
+            eruby.filename = src_path(src_file)
+            logger.info "  Create #{dest_path(dest_file)}"
+            FileUtils.mkdir_p(File.dirname(dest_path(dest_file)))
+            File.write(dest_path(dest_file), eruby.result(context))
+          end
+
+          unless post_messages.empty?
+            post_messages.each{|msg| logger.info msg }
+          end
+
+          true
+        end
+
+        def data_dir
+          nil
+        end
+
+        def dest_dir
+          nil
+        end
+
+        def templates
+          []
+        end
+
+        def context
+          {}
+        end
+
+        def post_messages
+          []
+        end
+
+        def logger
+          @logger ||= Tumugi::ScopedLogger.new("tumugi-new")
+        end
+
+        private
+
+        def src_path(file)
+          File.join(data_dir, file)
+        end
+
+        def dest_path(file)
+          File.join(dest_dir, file)
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/command/new/plugin_generator.rb

@@ -0,0 +1,60 @@
+require_relative 'generator'
+
+module Tumugi
+  module Command
+    class New
+      class PluginGenerator < Generator
+        def data_dir
+          "#{File.expand_path(File.dirname(__FILE__))}/../../data/new/plugin"
+        end
+
+        def dest_dir
+          File.join(options[:path] || '.', full_project_name)
+        end
+
+        def templates
+          [
+            [ "Gemfile.erb", "Gemfile" ],
+            [ "gemspec.erb", "#{full_project_name}.gemspec" ],
+            [ "gitignore.erb", ".gitignore" ],
+            [ "Rakefile.erb", "Rakefile" ],
+            [ "README.md.erb", "README.md" ],
+            [ "examples/example.rb.erb", "examples/example.rb" ],
+            [ "lib/tumugi/plugin/target/target.rb.erb", "lib/tumugi/plugin/target/#{name}.rb" ],
+            [ "lib/tumugi/plugin/task/task.rb.erb", "lib/tumugi/plugin/task/#{name}.rb" ],
+            [ "test/test_helper.rb.erb", "test/test_helper.rb" ],
+            [ "test/test.rb.erb", "test/#{name}_test.rb" ],
+            [ "test/plugin/target/target_test.rb.erb", "test/plugin/target/#{name}_test.rb" ],
+            [ "test/plugin/task/task_test.rb.erb", "test/plugin/task/#{name}_test.rb" ],
+          ]
+        end
+
+        def full_project_name
+          "tumugi-plugin-#{name}"
+        end
+
+        def context
+          {
+            full_project_name: full_project_name,
+            name: name,
+            tumugi_version: Tumugi::VERSION,
+          }
+        end
+
+        def post_messages
+          [
+            "",
+            "Plugin template is successfully generated.",
+            "Next steps:",
+            "",
+            "  $ cd #{full_project_name}",
+            "  $ git init",
+            "  $ bundle install",
+            "  $ bundle exec rake",
+            "",
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/command/new/project_generator.rb

@@ -0,0 +1,45 @@
+require_relative 'generator'
+
+module Tumugi
+  module Command
+    class New
+      class ProjectGenerator < Generator
+        def data_dir
+          "#{File.expand_path(File.dirname(__FILE__))}/../../data/new/project"
+        end
+
+        def dest_dir
+          File.join(options[:path] || '.', name)
+        end
+
+        def templates
+          [
+            [ "Gemfile.erb", "Gemfile" ],
+            [ "workflow.rb.erb", "workflow.rb" ],
+            [ "tumugi_config.rb.erb", "tumugi_config.rb" ],
+          ]
+        end
+
+        def context
+          {
+            name: name,
+            tumugi_version: Tumugi::VERSION,
+          }
+        end
+
+        def post_messages
+          [
+            "",
+            "Project template is successfully generated.",
+            "Next steps:",
+            "",
+            name.empty? ? "" : "  $ cd #{name}",
+            "  $ bundle install",
+            "  $ bundle exec tumugi -f workflow.rb main",
+            "",
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/command/run.rb

@@ -6,21 +6,14 @@ module Tumugi
     class Run
       def execute(dag, options={})
         worker_num = options[:workers] || Tumugi.config.workers
-        executor = Tumugi::Executor::LocalExecutor.new(dag, worker_num: worker_num)
-        result = start(executor)
+        executor = Tumugi::Executor::LocalExecutor.new(dag, worker_num: worker_num, run_all: options[:all])
+        result = executor.execute
         show_result_report(dag)
         result
       end
 
       private
 
-      def start(executor)
-        logger.info "workflow_start: #{Tumugi.workflow.id}"
-        result = executor.execute
-        logger.info "workflow_end: #{Tumugi.workflow.id}"
-        result
-      end
-
       def show_result_report(dag)
         reporter = Tumugi::DAGResultReporter.new
         report = reporter.show(dag)

data/lib/tumugi/dag_result_reporter.rb

@@ -1,5 +1,4 @@
 require 'terminal-table'
-
 require 'tumugi/mixin/listable'
 
 module Tumugi
@@ -7,18 +6,19 @@ module Tumugi
     include Tumugi::Mixin::Listable
 
     def show(dag)
-      headings = ['Task', 'Requires', 'Parameters', 'State']
+      headings = ['Task', 'Requires', 'Parameters', 'State', 'Elapsed']
       Terminal::Table.new title: "Workflow Result", headings: headings do |t|
         dag.tsort.map.with_index do |task, index|
           proxy = task.class.merged_parameter_proxy
           requires = list(task.requires).map do |r|
             r.id
           end
-          params = proxy.params.map do |name, _|
-            "#{name}=#{truncate(task.send(name.to_sym).to_s, 25)}"
+          params = proxy.params.map do |name, param|
+            val = param.secret? ? '***' : truncate(task.send(name.to_sym).to_s, 25)
+            "#{name}=#{val}"
           end
           t << :separator if index != 0
-          t << [ task.id, requires.join("\n"), params.join("\n"), task.state ]
+          t << [ task.id, requires.join("\n"), params.join("\n"), task.state, task.elapsed_time ]
         end
       end
     end

data/lib/tumugi/data/new/project/Gemfile.erb

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'tumugi', '~> <%= tumugi_version %>'

data/lib/tumugi/data/new/project/tumugi_config.rb.erb

@@ -0,0 +1,33 @@
+Tumugi.configure do |config|
+  ##############################################################################
+  # Number of worker threads for parallel execution, default value is 1.
+  # You can override this value by CLI '-w' option like:
+  #
+  # $ tumugi run -f workflow.rb main -w 2
+  ##############################################################################
+  # config.workers = 1
+
+  ##############################################################################
+  # Max retry count for each task, default value is 3.
+  ##############################################################################
+  # config.max_retry = 3
+
+  ##############################################################################
+  # Retry interval seconds for each task, default value is 300 seconds.
+  ##############################################################################
+  # config.retry_interval = 300
+
+  ##############################################################################
+  # Timeout seconds for each task, default value is nil, it means no timeout.
+  ##############################################################################
+  # config.timeout = nil
+
+  ##############################################################################
+  # If you use plugins and plugins support config section,
+  # define it like following.
+  ##############################################################################
+
+  # config.section("section_name") do |section|
+  #   section.key = "value"
+  # end
+end

data/lib/tumugi/data/new/project/workflow.rb.erb

@@ -0,0 +1,3 @@
+task :main do
+  run { log "Hello, tumugi!" }
+end

data/lib/tumugi/event.rb

@@ -0,0 +1,11 @@
+module Tumugi
+  module Event
+    SUCCESS = :success
+    FAILURE = :failure
+    RETRY = :retry
+
+    def self.all
+      self.constants.map{|name| self.const_get(name) }
+    end
+  end
+end

data/lib/tumugi/executor/local_executor.rb

@@ -7,17 +7,18 @@ require 'tumugi/error'
 module Tumugi
   module Executor
     class LocalExecutor
-      def initialize(dag, worker_num: 1)
+      def initialize(dag, worker_num: 1, run_all: false)
         @dag = dag
         @main_task = dag.tsort.last
-        @options = { worker_num: worker_num }
+        @worker_num = worker_num
+        @run_all = run_all
         @mutex = Mutex.new
       end
 
       def execute
         pool = Concurrent::ThreadPoolExecutor.new(
-          min_threads: @options[:worker_num],
-          max_threads: @options[:worker_num]
+          min_threads: @worker_num,
+          max_threads: @worker_num
         )
 
         setup_task_queue(@dag)
@@ -27,7 +28,7 @@ module Tumugi
 
           Concurrent::Future.execute(executor: pool) do
             if !task.runnable?(Time.now)
-              logger.trace { "task_cannot_run: #{task.id}" }
+              logger.trace { "task_not_runnable: #{task.id}" }
               enqueue_task(task)
             else
               begin
@@ -37,7 +38,8 @@ module Tumugi
                   task.run
                 end
                 task.trigger!(:complete)
-                logger.info { "task_#{task.state}: #{task.id}" }
+                logger.info { "task_#{task.state}: #{task.id}, elapsed_time: #{task.elapsed_time}" }
+                task.on_success
               rescue => e
                 handle_error(task, e)
               end
@@ -83,10 +85,10 @@ module Tumugi
 
             if task.requires_failed?
               task.trigger!(:requires_fail)
-              logger.info { "task_#{task.state}: #{task.id} has failed requires task" }
-            elsif task.completed?
+              logger.info { "task_#{task.state}: #{task.id} has failed requires task, elapsed_time: #{task.elapsed_time}" }
+            elsif task.completed? && !@run_all
               task.trigger!(:skip)
-              logger.info { "task_#{task.state}: #{task.id} is already completed" }
+              logger.info { "task_#{task.state}: #{task.id} is already completed, elapsed_time: #{task.elapsed_time}" }
             else
               break task
             end
@@ -102,15 +104,17 @@ module Tumugi
       def handle_error(task, err)
         if task.retry
           task.trigger!(:pend)
+          logger.info { "task_#{task.state}: #{task.id} failed, elapsed_time: #{task.elapsed_time}" }
           logger.error { "#{err.class}: '#{err.message}' - #{task.tries} tries and wait #{task.retry_interval} seconds until the next try." }
           enqueue_task(task)
+          task.on_retry
         else
           task.trigger!(:fail)
+          logger.info { "task_#{task.state}: #{task.id} failed, elapsed_time: #{task.elapsed_time}" }
           logger.error { "#{err.class}: '#{err.message}' - #{task.tries} tries and reached max retry count, so task #{task.id} failed." }
-          logger.error { "#{err.message}" }
-          logger.error { err.backtrace.join("\n") }
+          task.on_failure
         end
-        logger.info { "task_#{task.state}: error handeling done for #{task.id}" }
+        logger.error { err.backtrace.join("\n") }
       end
 
       def logger

data/lib/tumugi/mixin/human_readable.rb

@@ -0,0 +1,16 @@
+module Tumugi
+  module Mixin
+    module HumanReadable
+      def human_readable_time(seconds)
+        [[60, :s], [60, :m], [10000, :h]].map{|count, name|
+          if seconds > 0
+            seconds, n = seconds.divmod(count)
+            "#{sprintf('%02d', n)}"
+          else
+            '00'
+          end
+        }.compact.reverse.join(':')
+      end
+    end
+  end
+end

data/lib/tumugi/parameter/parameter.rb

@@ -20,15 +20,11 @@ module Tumugi
       end
 
       def auto_bind?
-        if @opts[:auto_bind].nil?
-          false
-        else
-          @opts[:auto_bind]
-        end
+        option_as_bool(:auto_bind)
       end
 
       def required?
-        @opts[:required].nil? ? false : @opts[:required]
+        option_as_bool(:required)
       end
 
       def type
@@ -43,6 +39,10 @@ module Tumugi
         self.class.new(@name, @opts.merge(required: false, default: value))
       end
 
+      def secret?
+        option_as_bool(:secret)
+      end
+
       private
 
       def search_from_workflow_parameters
@@ -51,13 +51,15 @@ module Tumugi
         value ? Converter.convert(type, value) : nil
       end
 
-      private
-
       def validate
         if required? && !default_value.nil?
           raise Tumugi::ParameterError.new("When you set required: true, you cannot set default value")
         end
       end
+
+      def option_as_bool(key)
+        @opts[key].nil? ? false : @opts[key]
+      end
     end
   end
 end

data/lib/tumugi/task.rb

@@ -1,15 +1,18 @@
+require 'tumugi/event'
 require 'tumugi/logger/scoped_logger'
 require 'tumugi/mixin/listable'
 require 'tumugi/mixin/task_helper'
 require 'tumugi/mixin/parameterizable'
+require 'tumugi/mixin/human_readable'
 
 module Tumugi
   class Task
     include Tumugi::Mixin::Parameterizable
     include Tumugi::Mixin::Listable
     include Tumugi::Mixin::TaskHelper
+    include Tumugi::Mixin::HumanReadable
 
-    attr_reader :visible_at, :tries, :max_retry, :retry_interval
+    attr_reader :visible_at, :tries, :max_retry, :retry_interval, :elapsed_time
 
     AVAILABLE_STATES = [
       :pending,
@@ -28,6 +31,8 @@ module Tumugi
       @retry_interval = Tumugi.config.retry_interval
       @state = :pending
       @lock = Mutex.new
+      @_elapsed_times = []
+      @elapsed_time = '00:00:00'
     end
 
     def id
@@ -98,6 +103,27 @@ module Tumugi
       ready? && visible?(now)
     end
 
+    def ready?
+      list(_requires).all? { |t| t.instance.completed? }
+    end
+
+    def completed?
+      outputs = list(output)
+      if outputs.empty?
+        success?
+      else
+        outputs.all?(&:exist?)
+      end
+    end
+
+    def requires_failed?
+      list(_requires).any? { |t| t.instance.finished? && !t.instance.success? }
+    end
+
+    def runnable?(now)
+      ready? && visible?(now)
+    end
+
     def success?
       case state
       when :completed, :skipped
@@ -107,15 +133,19 @@ module Tumugi
       end
     end
 
-    def finished?
+    def failure?
       case state
-      when :completed, :skipped, :failed, :requires_failed
+      when :failed, :requires_failed
         true
       else
         false
       end
     end
 
+    def finished?
+      success? or failure?
+    end
+
     def timeout
       nil # meaning use default timeout
     end
@@ -132,6 +162,9 @@ module Tumugi
 
     def trigger!(event)
       @lock.synchronize do
+        now = Time.now
+        @_elapsed_times[tries] ||= { start: now }
+
         s = case event
             when :skip
               :skipped
@@ -153,10 +186,20 @@ module Tumugi
           raise Tumugi::TumugiError.new("Invalid state: #{s}")
         end
 
+        @_elapsed_times[tries][:end] = now
+        @elapsed_time = human_readable_time((@_elapsed_times[tries][:end] - @_elapsed_times[tries][:start]).to_i)
         @state = s
       end
     end
 
+    # Event callbacks
+    Event.all.each do |event|
+      class_eval <<-EOS
+        def on_#{event}
+        end
+      EOS
+    end
+
     # Following methods are internal use only
 
     def _requires

data/lib/tumugi/task_definition.rb

@@ -1,3 +1,4 @@
+require 'tumugi/event'
 require 'tumugi/plugin'
 require 'tumugi/task'
 require 'tumugi/logger/logger'
@@ -64,6 +65,14 @@ module Tumugi
       @run = block
     end
 
+    Event.all.each do |event|
+      class_eval <<-EOS
+        def on_#{event}(&block)
+          @on_#{event} ||= block
+        end
+      EOS
+    end
+
     def output_eval(task)
       @out ||= @outputs.is_a?(Proc) ? task.instance_eval(&@outputs) : @outputs
     end
@@ -72,10 +81,6 @@ module Tumugi
       @required_tasks
     end
 
-    def run_block(task)
-      task.instance_eval(&@run)
-    end
-
     def parent_task_class
       if @opts[:type].is_a?(Class)
         @opts[:type]
@@ -84,6 +89,15 @@ module Tumugi
       end
     end
 
+    def run_block(task)
+      task.instance_eval(&@run)
+    end
+
+    def event_block(task, event)
+      callback = instance_variable_get("@on_#{event}")
+      task.instance_eval(&callback)
+    end
+
     private
 
     def create_task
@@ -98,6 +112,7 @@ module Tumugi
       define_requires_method(task_class)
       define_output_method(task_class)
       define_run_method(task_class)
+      define_event_callback_methods(task_class)
       setup_params(task_class)
       task_class
     end
@@ -138,6 +153,19 @@ module Tumugi
       end unless @run.nil?
     end
 
+    def define_event_callback_methods(task_class)
+      td = self
+      Event.all.each do |event|
+        if instance_variable_get("@on_#{event}")
+          task_class.class_eval do
+            define_method(:"on_#{event}") do
+              td.event_block(self, event)
+            end
+          end
+        end
+      end
+    end
+
     def setup_params(task_class)
       @params.each do |name, opts|
         task_class.param(name, opts)

data/lib/tumugi/version.rb

@@ -1,3 +1,3 @@
 module Tumugi
-  VERSION = "0.6.3"
+  VERSION = "0.7.0"
 end

data/lib/tumugi/workflow.rb

@@ -5,11 +5,14 @@ require 'tumugi/plugin'
 require 'tumugi/target'
 require 'tumugi/command/run'
 require 'tumugi/command/show'
+require 'tumugi/mixin/human_readable'
 
 require 'securerandom'
 
 module Tumugi
   class Workflow
+    include Tumugi::Mixin::HumanReadable
+
     attr_reader :id
     attr_accessor :params
 
@@ -22,12 +25,14 @@ module Tumugi
     end
 
     def execute(command, root_task_id, options)
+      @start_time = Time.now
+      logger.info "start id: #{id}"
       process_common_options(command, options)
       load_workflow_file(options[:file])
-      dag = create_dag(root_task_id)
-      command_module = Kernel.const_get("Tumugi").const_get("Command")
-      cmd = command_module.const_get("#{command.to_s.capitalize}").new
-      cmd.execute(dag, options)
+      result = execute_command(command, root_task_id, options)
+      @end_time = Time.now
+      logger.info "end id: #{id}, elapsed_time: #{elapsed_time}"
+      result
     end
 
     def add_task(id, task)
@@ -55,13 +60,6 @@ module Tumugi
       end
     end
 
-    def create_dag(id)
-      dag = Tumugi::DAG.new
-      task = find_task(id)
-      dag.add_task(task)
-      dag
-    end
-
     def process_common_options(command, options)
       setup_logger(command, options)
       load_config(options)
@@ -111,5 +109,23 @@ module Tumugi
         logger.info "Parameters: #{@params}"
       end
     end
+
+    def elapsed_time
+      human_readable_time((@end_time - @start_time).to_i)
+    end
+
+    def execute_command(command, root_task_id, options)
+      dag = create_dag(root_task_id)
+      command_module = Kernel.const_get("Tumugi").const_get("Command")
+      cmd = command_module.const_get("#{command.to_s.capitalize}").new
+      cmd.execute(dag, options)
+    end
+
+    def create_dag(id)
+      dag = Tumugi::DAG.new
+      task = find_task(id)
+      dag.add_task(task)
+      dag
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tumugi
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.7.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-13 00:00:00.000000000 Z
+date: 2016-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -184,6 +184,7 @@ files:
 - examples/concurrent_task_run.rb
 - examples/config_section.rb
 - examples/data_pipeline.rb
+- examples/event_callbacks.rb
 - examples/fail_first_task.rb
 - examples/fail_intermediate_task.rb
 - examples/fail_last_task.rb
@@ -198,29 +199,37 @@ files:
 - lib/tumugi/atomic_file.rb
 - lib/tumugi/cli.rb
 - lib/tumugi/command/new.rb
+- lib/tumugi/command/new/generator.rb
+- lib/tumugi/command/new/plugin_generator.rb
+- lib/tumugi/command/new/project_generator.rb
 - lib/tumugi/command/run.rb
 - lib/tumugi/command/show.rb
 - lib/tumugi/config.rb
 - lib/tumugi/dag.rb
 - lib/tumugi/dag_result_reporter.rb
-- lib/tumugi/data/new/Gemfile.erb
-- lib/tumugi/data/new/README.md.erb
-- lib/tumugi/data/new/Rakefile.erb
-- lib/tumugi/data/new/examples/example.rb.erb
-- lib/tumugi/data/new/gemspec.erb
-- lib/tumugi/data/new/gitignore.erb
-- lib/tumugi/data/new/lib/tumugi/plugin/target/target.rb.erb
-- lib/tumugi/data/new/lib/tumugi/plugin/task/task.rb.erb
-- lib/tumugi/data/new/test/plugin/target/target_test.rb.erb
-- lib/tumugi/data/new/test/plugin/task/task_test.rb.erb
-- lib/tumugi/data/new/test/test.rb.erb
-- lib/tumugi/data/new/test/test_helper.rb.erb
+- lib/tumugi/data/new/plugin/Gemfile.erb
+- lib/tumugi/data/new/plugin/README.md.erb
+- lib/tumugi/data/new/plugin/Rakefile.erb
+- lib/tumugi/data/new/plugin/examples/example.rb.erb
+- lib/tumugi/data/new/plugin/gemspec.erb
+- lib/tumugi/data/new/plugin/gitignore.erb
+- lib/tumugi/data/new/plugin/lib/tumugi/plugin/target/target.rb.erb
+- lib/tumugi/data/new/plugin/lib/tumugi/plugin/task/task.rb.erb
+- lib/tumugi/data/new/plugin/test/plugin/target/target_test.rb.erb
+- lib/tumugi/data/new/plugin/test/plugin/task/task_test.rb.erb
+- lib/tumugi/data/new/plugin/test/test.rb.erb
+- lib/tumugi/data/new/plugin/test/test_helper.rb.erb
+- lib/tumugi/data/new/project/Gemfile.erb
+- lib/tumugi/data/new/project/tumugi_config.rb.erb
+- lib/tumugi/data/new/project/workflow.rb.erb
 - lib/tumugi/dsl.rb
 - lib/tumugi/error.rb
+- lib/tumugi/event.rb
 - lib/tumugi/executor/local_executor.rb
 - lib/tumugi/file_system.rb
 - lib/tumugi/logger/logger.rb
 - lib/tumugi/logger/scoped_logger.rb
+- lib/tumugi/mixin/human_readable.rb
 - lib/tumugi/mixin/listable.rb
 - lib/tumugi/mixin/parameterizable.rb
 - lib/tumugi/mixin/task_helper.rb