tumugi 0.5.3 → 0.6.0

data/lib/tumugi/config.rb

@@ -11,7 +11,7 @@ module Tumugi
 
     def self.register_section(name, *args)
       @@sections[name] = Struct.new(camelize(name), *args)
-      logger.debug "registered config section '#{name}' with '#{args}'"
+      logger.debug { "registered config section '#{name}' with '#{args}'" }
     end
 
     def self.camelize(term)
@@ -29,7 +29,7 @@ module Tumugi
       @workers = 1
       @max_retry = 3
       @retry_interval = 300 #seconds
-      @timeout = 0 # meaning no timeout
+      @timeout = nil # meaning no timeout
 
       @section_procs = {}
       @section_instances = {}

data/lib/tumugi/dag_result_reporter.rb

@@ -0,0 +1,37 @@
+require 'terminal-table'
+
+require 'tumugi/mixin/listable'
+
+module Tumugi
+  class DAGResultReporter
+    include Tumugi::Mixin::Listable
+
+    def show(dag)
+      headings = ['Task', 'Requires', 'Parameters', 'State']
+      Terminal::Table.new title: "Workflow Result", headings: headings do |t|
+        dag.tsort.map.with_index do |task, index|
+          proxy = task.class.merged_parameter_proxy
+          requires = list(task.requires).map do |r|
+            r.id
+          end
+          params = proxy.params.map do |name, _|
+            "#{name}=#{truncate(task.send(name.to_sym).to_s, 25)}"
+          end
+          t << :separator if index != 0
+          t << [ task.id, requires.join("\n"), params.join("\n"), task.state ]
+        end
+      end
+    end
+
+    private
+
+    def truncate(text, length)
+      return nil if text.nil?
+      if text.length <= length
+        text
+      else
+        text[0, length].concat('...')
+      end
+    end
+  end
+end

data/lib/tumugi/data/new/Gemfile.erb

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in <%= full_project_name %>.gemspec
+gemspec

data/lib/tumugi/data/new/README.md.erb

@@ -0,0 +1,59 @@
+# <%= name %> plugin for [tumugi](https://github.com/tumugi/tumugi)
+
+TODO: Write short description here and <%= full_project_name %>.gemspec file.
+
+## Target
+
+### Tumugi::Plugin::<%= name.capitalize %>Target
+
+TODO: Write description about `Tumugi::Plugin::<%= name.capitalize %>Target`.
+
+## Task
+
+### Tumugi::Plugin::<%= name.capitalize %>Task
+
+TODO: Write description about `Tumugi::Plugin::<%= name.capitalize %>Task`.
+
+#### Parameters
+
+- **param1** description (string, required)
+- **param2** description (integer, default: 1)
+
+#### Example
+
+```rb
+task :task1, type: :bigquery_dataset do
+  param_set :dataset_id, 'test'
+end
+```
+
+### Config Section
+
+TODO: Write description about config section named "<%= name %>"
+if this plugin has config section.
+
+#### Example
+
+```rb
+Tumugi.configure do |config|
+  config.section("<%= name %>") do |section|
+    section.config1 = "xxx"
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bundle install` to install dependencies.
+Then run `bundle exec rake test` to run the tests.
+
+Run this plugin with tumugi, run
+
+```
+$ bundle exec tumugi run -f examples/example.rb main
+```
+
+## License
+
+The gem is available as open source under the terms of the [Apache License
+Version 2.0](http://www.apache.org/licenses/).

data/lib/tumugi/data/new/Rakefile.erb

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/lib/tumugi/data/new/examples/example.rb.erb

@@ -0,0 +1,4 @@
+task main, type: :<%= name %> do
+  param1 'value1'
+  output { target(:<%= name %>, param1) }
+end

data/lib/tumugi/data/new/gemspec.erb

@@ -0,0 +1,28 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "<%= full_project_name %>"
+  spec.version       = "0.1.0"
+  # TODO set this: spec.authors       = ["Your Name"]
+  # TODO set this: spec.email         = ["your-email@example.com"]
+
+  spec.summary       = "<%= name %> plugin for tumugi"
+  # TODO set this: spec.homepage      = "https://github.com/YOUR_ACCOUNT/<%= full_project_name %>"
+  spec.license       = "Apache License Version 2.0"
+
+  spec.required_ruby_version = '>= 2.1'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "tumugi", ">= <%= tumugi_version %>"
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+
+  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'test-unit', '~> 3.1'
+  spec.add_development_dependency 'test-unit-rr'
+end

data/lib/tumugi/data/new/gitignore.erb

@@ -0,0 +1,6 @@
+*~
+/.bundle/
+/Gemfile.lock
+/pkg/
+/tmp/
+.ruby-version

data/lib/tumugi/data/new/lib/tumugi/plugin/target/target.rb.erb

@@ -0,0 +1,20 @@
+require 'tumugi'
+
+module Tumugi
+  module Plugin
+    class <%= name.capitalize %>Target < Tumugi::Target
+      Tumugi::Plugin.register_target('<%= name %>', self)
+
+      attr_reader :value
+
+      def initialize(value)
+        @value = value
+      end
+
+      def exist?
+        # TODO: Implemente this method
+        true
+      end
+    end
+  end
+end

data/lib/tumugi/data/new/lib/tumugi/plugin/task/task.rb.erb

@@ -0,0 +1,22 @@
+require 'tumugi'
+require 'tumugi/plugin/target/<%= name %>'
+
+module Tumugi
+  module Plugin
+    class <%= name.capitalize %>Task < Tumugi::Task
+      Tumugi::Plugin.register_task('<%= name %>', self)
+
+      param :param1, type: :string, default: ""
+
+      def output
+        @output ||= Tumugi::Plugin::<%= name.capitalize %>Target.new(param1)
+      end
+
+      def run
+        # TODO: Implemente this method
+        log "run #{self.class.name}"
+        log "param1: #{param1}"
+      end
+    end
+  end
+end

data/lib/tumugi/data/new/test/plugin/target/target_test.rb.erb

@@ -0,0 +1,9 @@
+require_relative '../../test_helper'
+require 'tumugi/plugin/target/<%= name %>'
+
+class Tumugi::Plugin::<%= name.capitalize %>TargetTest < Test::Unit::TestCase
+  test "exist? should return true if exists" do
+    target = Tumugi::Plugin::<%= name.capitalize %>Target.new("value")
+    assert_true(target.exist?)
+  end
+end

data/lib/tumugi/data/new/test/plugin/task/task_test.rb.erb

@@ -0,0 +1,27 @@
+require_relative '../../test_helper'
+require 'tumugi/plugin/task/<%= name %>'
+
+class Tumugi::Plugin::<%= name.capitalize %>TaskTest < Test::Unit::TestCase
+  setup do
+    @klass = Class.new(Tumugi::Plugin::<%= name.capitalize %>Task)
+    @klass.set(:param1, "value")
+  end
+
+  test "should set correctly" do
+    task = @klass.new
+    assert_equal("value", task.param1)
+  end
+
+  test "#output" do
+    task = @klass.new
+    output = task.output
+    assert_true(output.is_a? Tumugi::Plugin::<%= name.capitalize %>Target)
+  end
+
+  test "#run" do
+    task = @klass.new
+    output = task.output
+    task.run
+    assert_equal("value", output.value)
+  end
+end

data/lib/tumugi/data/new/test/test.rb.erb

@@ -0,0 +1,7 @@
+require_relative './test_helper'
+
+class Tumugi::<%= name.capitalize %>Test < Test::Unit::TestCase
+  test 'run success' do
+    assert_run_success("examples/example.rb", "main")
+  end
+end

data/lib/tumugi/data/new/test/test_helper.rb.erb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+
+require 'test/unit'
+require 'test/unit/rr'
+
+require 'tumugi'
+require 'tumugi/test/helper'
+include Tumugi::Test::Helpers

data/lib/tumugi/executor/local_executor.rb

@@ -0,0 +1,128 @@
+require 'much-timeout'
+require 'concurrent'
+
+require 'tumugi'
+require 'tumugi/error'
+
+Concurrent.use_stdlib_logger(Logger::DEBUG)
+
+module Tumugi
+  module Executor
+    class LocalExecutor
+      def initialize(dag, logger=nil, worker_num: 1)
+        @dag = dag
+        @main_task = dag.tsort.last
+        @logger = logger || Tumugi::Logger.instance
+        @options = { worker_num: worker_num }
+        @mutex = Mutex.new
+      end
+
+      def execute
+        pool = Concurrent::ThreadPoolExecutor.new(
+          min_threads: @options[:worker_num],
+          max_threads: @options[:worker_num]
+        )
+
+        setup_task_queue(@dag)
+        loop do
+          task = dequeue_task
+          break if task.nil?
+
+          Concurrent::Future.execute(executor: pool) do
+            if !task.runnable?(Time.now)
+              info "not_runnable: #{task.id}"
+              enqueue_task(task)
+            else
+              begin
+                info "start: #{task.id}"
+                task.trigger!(:start)
+                MuchTimeout.optional_timeout(task_timeout(task), Tumugi::TimeoutError) do
+                  task.run
+                end
+                task.trigger!(:complete)
+                info "#{task.state}: #{task.id}"
+              rescue => e
+                handle_error(task, e)
+              end
+            end
+          end
+        end
+
+        pool.shutdown
+        pool.wait_for_termination
+
+        @dag.tsort.all? { |t| t.success? }
+      end
+
+      private
+
+      def task_timeout(task)
+        timeout = task.timeout || Tumugi.config.timeout
+        timeout = nil if !timeout.nil? && timeout == 0 # for backward compatibility
+        timeout
+      end
+
+      def setup_task_queue(dag)
+        @queue = []
+        dag.tsort.each { |t| enqueue_task(t) }
+        @queue
+      end
+
+      def dequeue_task
+        loop do
+          task = @mutex.synchronize {
+            debug { "queue: #{@queue.map(&:id)}" }
+            @queue.shift
+          }
+
+          if task.nil?
+            if @main_task.finished?
+              break nil
+            else
+              sleep(0.1)
+            end
+          else
+            debug { "dequeue: #{task.id}" }
+
+            if task.requires_failed?
+              task.trigger!(:requires_fail)
+              info "#{task.state}: #{task.id} has failed requires task"
+            elsif task.completed?
+              task.trigger!(:skip)
+              info "#{task.state}: #{task.id} is already completed"
+            else
+              break task
+            end
+          end
+        end
+      end
+
+      def enqueue_task(task)
+        debug { "enqueue: #{task.id}" }
+        @mutex.synchronize { @queue.push(task) }
+      end
+
+      def handle_error(task, err)
+        if task.retry
+          task.trigger!(:pend)
+          @logger.error "#{err.class}: '#{err.message}' - #{task.tries} tries and wait #{task.retry_interval} seconds until the next try."
+          enqueue_task(task)
+        else
+          task.trigger!(:fail)
+          @logger.error "#{err.class}: '#{err.message}' - #{task.tries} tries and reached max retry count, so task #{task.id} failed."
+          info "#{task.state}: #{task.id}"
+          @logger.error "#{err.message}"
+          @logger.debug { err.backtrace.join("\n") }
+        end
+      end
+
+      def info(message)
+        @logger.info "#{message}, thread: #{Thread.current.object_id}"
+      end
+
+      def debug(&block)
+        @logger.debug { "#{block.call}, thread: #{Thread.current.object_id}" }
+      end
+    end
+  end
+end

data/lib/tumugi/logger.rb

@@ -1,5 +1,6 @@
-require 'logger'
 require 'forwardable'
+require 'json'
+require 'logger'
 require 'singleton'
 
 module Tumugi
@@ -7,14 +8,20 @@ module Tumugi
     include Singleton
     extend Forwardable
     def_delegators :@logger, :debug, :error, :fatal, :info, :warn, :level
+    attr_accessor :workflow_id
 
     def initialize
+      @formatters = {
+        text: text_formatter,
+        json: json_formatter
+      }
       init
     end
 
-    def init(output=STDOUT)
+    def init(output: STDOUT, format: :text)
       @logger = ::Logger.new(output)
       @logger.level = ::Logger::INFO
+      @logger.formatter = @formatters[format]
     end
 
     def verbose!
@@ -24,5 +31,27 @@ module Tumugi
     def quiet!
       @logger = ::Logger.new(nil)
     end
+
+    private
+
+    def text_formatter
+      Proc.new { |severity, datetime, progname, msg|
+        if !workflow_id.nil?
+          "#{datetime} #{severity} [#{workflow_id}] #{msg}\n"
+        else
+          "#{datetime} #{severity} #{msg}\n"
+        end
+      }
+    end
+
+    def json_formatter
+      Proc.new { |severity, datetime, progname, msg|
+        hash = { time: datetime, severity: severity, message: msg }
+        if !workflow_id.nil?
+          hash[:workflow] = workflow_id
+        end
+        "#{JSON.generate(hash)}\n"
+      }
+    end
   end
 end