tumugi 0.1.0

data/examples/target.rb

@@ -0,0 +1,51 @@
+require 'tumugi/target/file_target'
+
+task :task1 do
+  requires [:task2, :task3]
+
+  output do |task|
+    Tumugi::Target::FileTarget.new("/tmp/#{task.id}.txt")
+  end
+
+  run do |task|
+    puts 'task1#run'
+    File.write(task.output.path, 'done')
+  end
+end
+
+task :task2 do
+  requires [:task4]
+
+  output do |task|
+    Tumugi::Target::FileTarget.new("/tmp/#{task.id}.txt")
+  end
+
+  run do |task|
+    puts 'task2#run'
+    File.write(task.output.path, 'done')
+  end
+end
+
+task :task3 do
+  requires [:task4]
+
+  output do |task|
+    Tumugi::Target::FileTarget.new("/tmp/#{task.id}.txt")
+  end
+
+  run do |task|
+    puts 'task3#run'
+    File.write(task.output.path, 'done')
+  end
+end
+
+task :task4 do
+  output do
+    Tumugi::Target::FileTarget.new('/tmp/task4.txt')
+  end
+
+  run do |task|
+    puts 'task4#run'
+    File.write(task.output.path, 'done')
+  end
+end

data/examples/task_inheritance.rb

@@ -0,0 +1,27 @@
+require 'tumugi/target/file_target'
+
+class FileTask < Tumugi::Task
+  def output
+    Tumugi::Target::FileTarget.new("/tmp/#{self.id}.txt")
+  end
+
+  def run
+    puts "#{self.id}#run"
+    File.write(output.path, 'done')
+  end
+end
+
+task :task1, type: FileTask do
+  requires [:task2, :task3]
+end
+
+task :task2, type: FileTask do
+  requires [:task4]
+end
+
+task :task3, type: FileTask do
+  requires [:task4]
+end
+
+task :task4, type: FileTask do
+end

data/exe/tumugi

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'tumugi/cli'
+Tumugi::CLI.start(ARGV)

data/lib/tumugi.rb

@@ -0,0 +1,4 @@
+require 'tumugi/version'
+
+module Tumugi
+end

data/lib/tumugi/application.rb

@@ -0,0 +1,40 @@
+require "active_support/all"
+
+require 'tumugi/dag'
+require 'tumugi/dsl'
+require 'tumugi/command/run'
+require 'tumugi/command/show'
+
+module Tumugi
+  class Application
+    def initialize
+      @tasks = {}
+    end
+
+    def execute(command, root_task_id, options)
+      load(options[:file], true)
+      dag = create_dag(root_task_id)
+      cmd = "Tumugi::Command::#{command.to_s.classify}".constantize.new
+      cmd.execute(dag, options)
+    end
+
+    def add_task(id, task)
+      @tasks[id.to_s] = task
+    end
+
+    def find_task(id)
+      task = @tasks[id.to_s]
+      raise "Task not found: #{id}" if task.nil?
+      task
+    end
+
+    private
+
+    def create_dag(id)
+      dag = Tumugi::DAG.new
+      task = find_task(id)
+      dag.add_task(task)
+      dag
+    end
+  end
+end

data/lib/tumugi/cli.rb

@@ -0,0 +1,50 @@
+require 'thor'
+require 'tumugi'
+require 'tumugi/tumugi_module'
+
+module Tumugi
+  class CLI < Thor
+    package_name "tumugi"
+
+    class << self
+      def common_options
+        option :file, aliases: '-f', desc: 'Task definition file name'
+        option :config, aliases: '-c', desc: 'Configuration file name', default: 'tumugi.rb'
+      end
+    end
+
+    desc "version", "Show version"
+    def version
+      puts "tumugi v#{Tumugi::VERSION}"
+    end
+
+    desc "run", "Run workflow"
+    map "run" => "run_" # run is thor's reserved word, so this trick is needed
+    option :workers, aliases: '-w', type: :numeric, desc: 'Number of workers to run task concurrently'
+    option :quiet, type: :boolean, desc: 'Suppress log', default: false
+    option :verbose, type: :boolean, desc: 'Show verbose log', default: false
+    common_options
+    def run_(task)
+      process_common_options
+      Tumugi.application.execute(:run, task, options)
+    end
+
+    desc "show", "Show DAG of workflow"
+    common_options
+    option :out, aliases: '-o', desc: 'Output file name. If not specified, output result to STDOUT'
+    option :format, aliases: '-t', desc: 'Output file format. Only affected --out option is specified.', enum: ['dot', 'png', 'svg']
+    def show(task)
+      process_common_options
+      Tumugi.application.execute(:show, task, options)
+    end
+
+    private
+
+    def process_common_options
+      config_file = options[:config]
+      if config_file && File.exists?(config_file) && File.extname(config_file) == '.rb'
+        load(config_file)
+      end
+    end
+  end
+end

data/lib/tumugi/command/run.rb

@@ -0,0 +1,63 @@
+require 'parallel'
+require 'retriable'
+
+module Tumugi
+  module Command
+    class Run
+      def execute(dag, options={})
+        workers = options[:workers] || Tumugi.config.workers
+        settings = { in_threads: workers }
+
+        Tumugi.logger.verbose! if options[:verbose]
+        Tumugi.logger.quiet! if options[:quiet]
+
+        Parallel.each(dag.tsort, settings) do |t|
+          Tumugi.logger.info "start: #{t.id}"
+          until t.ready?
+            sleep 1
+          end
+          unless t.completed?
+            Tumugi.logger.info "run: #{t.id}"
+            t.state = :running
+
+            begin
+              Retriable.retriable retry_options do
+                t.run
+              end
+            rescue => e
+              Tumugi.logger.info "failed: #{t.id}"
+              Tumugi.logger.error "#{e.message}"
+              t.state = :failed
+            else
+              Tumugi.logger.info "completed: #{t.id}"
+              t.state = :completed
+            end
+          else
+            t.state = :skipped
+            Tumugi.logger.info "skip: #{t.id} is already completed"
+          end
+        end
+      end
+
+      private
+
+      def retry_options
+        {
+          tries: Tumugi.config.max_retry,
+          base_interval: Tumugi.config.retry_interval,
+          max_interval: Tumugi.config.retry_interval * Tumugi.config.max_retry,
+          max_elapsed_time: Tumugi.config.retry_interval * Tumugi.config.max_retry,
+          multiplier: 1.0,
+          rand_factor: 0.0,
+          on_retry: on_retry
+        }
+      end
+
+      def on_retry
+        Proc.new do |exception, try, elapsed_time, next_interval|
+          Tumugi.logger.error "#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds and #{next_interval} seconds until the next try."
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/command/show.rb

@@ -0,0 +1,40 @@
+require 'gviz'
+require 'tmpdir'
+require 'fileutils'
+
+module Tumugi
+  module Command
+    class Show
+      @@supported_formats = ['dot', 'png', 'jpg', 'svg', 'pdf']
+
+      def execute(dag, options={})
+        out = options[:out]
+        if out
+          ext = File.extname(options[:out])
+          format = ext[1..-1] if ext.start_with?('.')
+          raise "#{format} is not supported format" unless @@supported_formats.include?(format)
+        else
+          format = options[:format]
+        end
+
+        graph = Graph do
+          dag.tsort.each do |task|
+            node task.id
+            route task.id => task._requires.map {|t| t.id}
+          end
+        end
+
+        if out.present?
+          Dir.mktmpdir do |dir|
+            file_base_path = "#{File.dirname(dir)}/#{File.basename(out, '.*')}"
+            graph.save(file_base_path, format == 'dot' ? nil : format)
+            FileUtils.mkdir_p(File.dirname(out))
+            FileUtils.copy("#{file_base_path}.#{format}", out)
+          end
+        else
+          print graph
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/config.rb

@@ -0,0 +1,11 @@
+module Tumugi
+  class Config
+    attr_accessor :workers, :max_retry, :retry_interval
+
+    def initialize
+      @workers = 1
+      @max_retry = 3
+      @retry_interval = 300 #seconds
+    end
+  end
+end

data/lib/tumugi/dag.rb

@@ -0,0 +1,35 @@
+require 'tsort'
+require 'tumugi/helper'
+
+module Tumugi
+  class DAG
+    include TSort
+    include Tumugi::Helper
+
+    attr_reader :tasks
+
+    def initialize
+      @tasks = {}
+    end
+
+    def tsort_each_node(&block)
+      @tasks.each_key(&block)
+    end
+
+    def tsort_each_child(node, &block)
+      @tasks.fetch(node).each(&block)
+    end
+
+    def add_task(task)
+      t = task.instance
+      unless @tasks[t]
+        reqs = list(t._requires).map {|r| r.instance }
+        @tasks[t] = reqs
+        reqs.each do |r|
+          add_task(r)
+        end
+      end
+      task
+    end
+  end
+end

data/lib/tumugi/dsl.rb

@@ -0,0 +1,15 @@
+# Tumugi DSL functions.
+
+require 'tumugi/task_definition'
+
+module Tumugi
+  module DSL
+    def task(*args, &block)
+      Tumugi::TaskDefinition.define(*args, &block)
+    end
+  end
+end
+
+# Extend the main object with the DSL commands.
+# This allows top-level calls to task, etc.
+extend Tumugi::DSL

data/lib/tumugi/helper.rb

@@ -0,0 +1,15 @@
+module Tumugi
+  module Helper
+    def list(obj)
+      if obj.nil?
+        []
+      elsif obj.is_a?(Array)
+        obj
+      elsif obj.is_a?(Hash)
+        obj.map { |k,v| v }
+      else
+        [obj]
+      end
+    end
+  end
+end

data/lib/tumugi/logger.rb

@@ -0,0 +1,21 @@
+require 'forwardable'
+
+module Tumugi
+  class Logger
+    extend Forwardable
+    def_delegators :@logger, :debug, :error, :fatal, :info, :warn, :level
+
+    def initialize
+      @logger = ::Logger.new(STDOUT)
+      @logger.level = ::Logger::INFO
+    end
+
+    def verbose!
+      @logger.level = ::Logger::DEBUG
+    end
+
+    def quiet!
+      @logger = ::Logger.new(nil)
+    end
+  end
+end

data/lib/tumugi/target/base.rb

@@ -0,0 +1,9 @@
+module Tumugi
+  module Target
+    class Base
+      def exist?
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+    end
+  end
+end

data/lib/tumugi/target/file_target.rb

@@ -0,0 +1,17 @@
+require 'tumugi/target/base'
+
+module Tumugi
+  module Target
+    class FileTarget < Base
+      attr_reader :path
+
+      def initialize(path)
+        @path = path
+      end
+
+      def exist?
+        ::File.exist?(path)
+      end
+    end
+  end
+end

data/lib/tumugi/task.rb

@@ -0,0 +1,86 @@
+require 'tumugi/helper'
+
+module Tumugi
+  class Task
+    include Tumugi::Helper
+
+    attr_accessor :state # :pending, :running, :completed, :skipped
+
+    def initialize
+      @logger = Tumugi.logger
+      @state = :pending
+    end
+
+    def id
+      @id ||= self.class.name
+    end
+
+    def id=(s)
+      @id = s
+    end
+
+    def eql?(other)
+      self.hash == other.hash
+    end
+
+    def hash
+      self.id.hash
+    end
+
+    def instance
+      self
+    end
+
+    # If you need to define task dependencies, override in subclass
+    def requires
+      []
+    end
+
+    def input
+      @input ||= _input
+    end
+
+    # If you need to define output of task to skip alredy done task,
+    # override in subclass. If not, a task run always.
+    def output
+      []
+    end
+
+    def run
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def ready?
+      list(_requires).all? { |t| t.instance.completed? }
+    end
+
+    def completed?
+      outputs = list(output)
+      !outputs.empty? && outputs.all?(&:exist?)
+    end
+
+    # Following methods are internal use only
+
+    def _requires
+      @_requires ||= requires
+    end
+
+    def _output
+      @_output ||= output
+    end
+
+    private
+
+    def _input
+      if _requires.nil?
+        []
+      elsif _requires.is_a?(Array)
+        _requires.map { |t| t.instance._output }
+      elsif _requires.is_a?(Hash)
+        Hash[_requires.map { |k, t| [k, t.instance._output] }]
+      else
+        _requires.instance._output
+      end
+    end
+  end
+end