tumugi 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a8780828db349af4d7d5a31db4d85704f5c07d27
-  data.tar.gz: c3e7ea47a53da76b5d9dbae3e1ae723bd43a89df
+  metadata.gz: fe91c5dd14b1c75b8562de885608d52d9f8ff927
+  data.tar.gz: 7ce1c778ca4614bb5a9a4be2e04ac1d75dc59a5b
 SHA512:
-  metadata.gz: 2ea11f3233b611657f645e4b4f3e1e6ee390f27b7cad70b9f0fa383800d574ec6b48acc774adb759bcde80bb60cbde4bcb17833af57fa24886d44c6c76d69aa1
-  data.tar.gz: fffbd00ab9c2a5efac1c3af410b25f80751b1a4fe37af17e667a74d72aa69a8c0e8fa5ba28d2157d841562ec5a38e76a0262780b863dfe5b08a2b1193f825a6a
+  metadata.gz: 5005eed43b4ec6261c0870c89afe93609c516719541f67dd9ef683f1b0742b6ed8e8dedb574eb578485612856d3096aab64198ef55c8e8cefead786d3f538663
+  data.tar.gz: 3208bf2657f87d7f76dd39a568e9ec356c549ddfca95986e8f879d3ac0b88b2c017ab150159b3cec7b653c37e9fdd86cda56414f1df817f228879bf061c18cee

data/.travis.yml

@@ -1,11 +1,12 @@
 language: ruby
 rvm:
   - 2.0.0-p648
-  - 2.1.8
-  - 2.2.4
-  - 2.3.0
+  - 2.1.10
+  - 2.2.5
+  - 2.3.1
   - jruby-9.0.5.0
-sudo: required
-before_install:
-  - sudo apt-get install graphviz
-  - gem install bundler -v 1.11.2
+cache: bundler
+addons:
+  apt:
+    packages:
+    - graphviz

data/CHANGELOG.md

@@ -0,0 +1,43 @@
+# Change Log
+
+## [v0.2.0](https://github.com/tumugi/tumugi/tree/v0.2.0) (2016-05-02)
+[Full Changelog](https://github.com/tumugi/tumugi/compare/v0.1.0...v0.2.0)
+
+**Implemented enhancements:**
+
+- Implement plugin architecture [\#18](https://github.com/tumugi/tumugi/pull/18) ([hakobera](https://github.com/hakobera))
+- \[Breaking change\] Change eval scope of output, run method [\#14](https://github.com/tumugi/tumugi/pull/14) ([hakobera](https://github.com/hakobera))
+- \[Breaking Change\] Add Task\#logger and Task\#log method  [\#12](https://github.com/tumugi/tumugi/pull/12) ([hakobera](https://github.com/hakobera))
+- Update command description / Set file options mandatory [\#11](https://github.com/tumugi/tumugi/pull/11) ([hakobera](https://github.com/hakobera))
+
+**Fixed bugs:**
+
+- Fix show command cannot handle task which id include underscore [\#13](https://github.com/tumugi/tumugi/pull/13) ([hakobera](https://github.com/hakobera))
+
+**Merged pull requests:**
+
+- Use bundler cache on travis [\#17](https://github.com/tumugi/tumugi/pull/17) ([hakobera](https://github.com/hakobera))
+- Add gem version badge to README [\#16](https://github.com/tumugi/tumugi/pull/16) ([hakobera](https://github.com/hakobera))
+- Add changelog of v0.1.0 [\#10](https://github.com/tumugi/tumugi/pull/10) ([hakobera](https://github.com/hakobera))
+
+## [v0.1.0](https://github.com/tumugi/tumugi/tree/v0.1.0) (2016-04-28)
+**Implemented enhancements:**
+
+- Retry when task failed [\#8](https://github.com/tumugi/tumugi/pull/8) ([hakobera](https://github.com/hakobera))
+- Support run tasks concurrently [\#7](https://github.com/tumugi/tumugi/pull/7) ([hakobera](https://github.com/hakobera))
+- Visualize [\#6](https://github.com/tumugi/tumugi/pull/6) ([hakobera](https://github.com/hakobera))
+- Support task inheritance [\#5](https://github.com/tumugi/tumugi/pull/5) ([hakobera](https://github.com/hakobera))
+- First DSL implementation [\#3](https://github.com/tumugi/tumugi/pull/3) ([hakobera](https://github.com/hakobera))
+
+**Closed issues:**
+
+- Features and Tasks for v0.1 [\#4](https://github.com/tumugi/tumugi/issues/4)
+
+**Merged pull requests:**
+
+- Create gem [\#2](https://github.com/tumugi/tumugi/pull/2) ([hakobera](https://github.com/hakobera))
+- Add initial spec [\#1](https://github.com/tumugi/tumugi/pull/1) ([hakobera](https://github.com/hakobera))
+
+
+
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/tumugi/tumugi.svg)](https://travis-ci.org/tumugi/tumugi) [![Code Climate](https://codeclimate.com/github/tumugi/tumugi/badges/gpa.svg)](https://codeclimate.com/github/tumugi/tumugi) [![Coverage Status](https://coveralls.io/repos/tumugi/tumugi/badge.svg?branch=master&service=github)](https://coveralls.io/github/tumugi/tumugi?branch=master)
+[![Build Status](https://travis-ci.org/tumugi/tumugi.svg)](https://travis-ci.org/tumugi/tumugi) [![Code Climate](https://codeclimate.com/github/tumugi/tumugi/badges/gpa.svg)](https://codeclimate.com/github/tumugi/tumugi) [![Coverage Status](https://coveralls.io/repos/tumugi/tumugi/badge.svg?branch=master&service=github)](https://coveralls.io/github/tumugi/tumugi?branch=master) [![Gem Version](https://badge.fury.io/rb/tumugi.svg)](https://badge.fury.io/rb/tumugi)
 
 # Tumugi
 

data/Rakefile

@@ -1,5 +1,6 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
+require "github_changelog_generator/task"
 
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
@@ -7,4 +8,7 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList['test/**/*_test.rb']
 end
 
+GitHubChangelogGenerator::RakeTask.new :changelog do |config|
+end
+
 task :default => :test

data/examples/concurrent_task_run.rb

@@ -1,27 +1,26 @@
-require 'tumugi/target/file_target'
-
 class FileTask < Tumugi::Task
+  Tumugi::Plugin.register_task(:local_file, self)
+
   def output
-    Tumugi::Target::FileTarget.new("/tmp/tumugi_#{self.id}.txt")
+    target(:local_file, "/tmp/tumugi_#{self.id}.txt")
   end
 
   def run
-    @logger.info "#{self.id}#run"
-    sleep 5
-    File.write(output.path, 'done')
-    @logger.info "#{self.id}#done"
+    log 'sleep 2 seconds'
+    sleep 2
+    output.open('w') {|f| f.puts('done') }
   end
 end
 
-task :task1, type: FileTask do
+task :task1, type: :local_file do
   requires (1..10).map {|i| :"task2_#{i}"}
 end
 
 (1..10).each do |i|
-  task :"task2_#{i}", type: FileTask do
+  task :"task2_#{i}", type: :local_file do
     requires [:task3]
   end
 end
 
-task :task3, type: FileTask do
+task :task3, type: :local_file do
 end

data/examples/data_pipeline.rb

@@ -1,12 +1,10 @@
-require 'tumugi/target/file_target'
-
 task :generate_data do
   output do
-    Tumugi::Target::FileTarget.new("/tmp/data_#{Time.now.strftime('%Y-%m-%d')}.txt")
+    target(:local_file, "/tmp/tumugi_data_#{Time.now.strftime('%Y-%m-%d')}.txt")
   end
 
-  run do |task|
-    File.open(task.output.path, "w") do |f|
+  run do
+    output.open('w') do |f|
       (1..10).each do |i|
         f.puts i
       end
@@ -18,14 +16,14 @@ task :sum do
   requires :generate_data
 
   output do
-    Tumugi::Target::FileTarget.new("/tmp/output_#{Time.now.strftime('%Y-%m-%d')}.txt")
+    target(:local_file, "/tmp/tumugi_output_#{Time.now.strftime('%Y-%m-%d')}.txt")
   end
 
-  run do |task|
+  run do
     sum = 0
-    File.foreach(task.input.path) do |line|
+    input.open do |line|
       sum += line.to_i
     end
-    File.write(task.output.path, sum)
+    output { |f| f.puts(sum) }
   end
 end

data/examples/simple.rb

@@ -1,21 +1,21 @@
 task :task1 do
   requires [:task2, :task3]
-  run { puts 'task1#run' }
+  run { log 'task1#run' }
 end
 
 task :task2 do
   requires [:task4]
-  run { puts 'task2#run' }
+  run { log 'task2#run' }
 end
 
 task :task3 do
   requires [:task4]
-  run { puts 'task3#run' }
+  run { log 'task3#run' }
 end
 
 task :task4 do
   run do
-    puts 'task4#run'
+    log 'task4#run'
     sleep 1
   end
 end

data/examples/target.rb

@@ -1,51 +1,36 @@
-require 'tumugi/target/file_target'
+require 'tumugi/plugin/target/local_file'
 
 task :task1 do
   requires [:task2, :task3]
-
-  output do |task|
-    Tumugi::Target::FileTarget.new("/tmp/#{task.id}.txt")
-  end
-
-  run do |task|
-    puts 'task1#run'
-    File.write(task.output.path, 'done')
+  output target(:local_file, "/tmp/tumugi_#{id}.txt")
+  run do
+    log 'task1#run'
+    output.open('w') {|f| f.puts('done') }
   end
 end
 
 task :task2 do
   requires [:task4]
-
-  output do |task|
-    Tumugi::Target::FileTarget.new("/tmp/#{task.id}.txt")
-  end
-
-  run do |task|
-    puts 'task2#run'
-    File.write(task.output.path, 'done')
+  output [target(:local_file, "/tmp/tumugi_#{id}.txt")]
+  run do
+    log 'task2#run'
+    output[0].open('w') {|f| f.puts('done') }
   end
 end
 
 task :task3 do
   requires [:task4]
-
-  output do |task|
-    Tumugi::Target::FileTarget.new("/tmp/#{task.id}.txt")
-  end
-
-  run do |task|
-    puts 'task3#run'
-    File.write(task.output.path, 'done')
+  output { target(:local_file, "/tmp/tumugi_#{id}.txt") }
+  run do
+    log 'task3#run'
+    output.open('w') {|f| f.puts('done') }
   end
 end
 
 task :task4 do
-  output do
-    Tumugi::Target::FileTarget.new('/tmp/task4.txt')
-  end
-
-  run do |task|
-    puts 'task4#run'
-    File.write(task.output.path, 'done')
+  output Tumugi::Plugin::LocalFileTarget.new("/tmp/tumugi_#{id}.txt")
+  run do
+    log 'task4#run'
+    output.open('w') {|f| f.puts('done') }
   end
 end

data/examples/task_inheritance.rb

@@ -1,24 +1,28 @@
-require 'tumugi/target/file_target'
-
 class FileTask < Tumugi::Task
+  Tumugi::Plugin.register_task(:file, self)
+
   def output
-    Tumugi::Target::FileTarget.new("/tmp/#{self.id}.txt")
+    target(:local_file, "/tmp/#{id}.txt")
   end
 
   def run
-    puts "#{self.id}#run"
-    File.write(output.path, 'done')
+    log "#{id}#run"
+    output.open('w') do |f|
+      f.puts('done')
+    end
   end
 end
 
-task :task1, type: FileTask do
+# Task type can specified by task plugin ID
+task :task1, type: :file do
   requires [:task2, :task3]
 end
 
-task :task2, type: FileTask do
+task :task2, type: :file do
   requires [:task4]
 end
 
+# You can also specify type by Class object
 task :task3, type: FileTask do
   requires [:task4]
 end

data/lib/tumugi.rb

@@ -1,4 +1,24 @@
+require 'tumugi/application'
+require 'tumugi/config'
+require 'tumugi/logger'
 require 'tumugi/version'
 
 module Tumugi
+  class << self
+    def application
+      @application ||= Tumugi::Application.new
+    end
+
+    def logger
+      @logger ||= Tumugi::Logger.new
+    end
+
+    def config
+      @config ||= Tumugi::Config.new
+      if block_given?
+        yield @config
+      end
+      @config
+    end
+  end
 end

data/lib/tumugi/application.rb

@@ -1,7 +1,7 @@
-require "active_support/all"
-
 require 'tumugi/dag'
 require 'tumugi/dsl'
+require 'tumugi/plugin'
+require 'tumugi/target'
 require 'tumugi/command/run'
 require 'tumugi/command/show'
 
@@ -14,7 +14,8 @@ module Tumugi
     def execute(command, root_task_id, options)
       load(options[:file], true)
       dag = create_dag(root_task_id)
-      cmd = "Tumugi::Command::#{command.to_s.classify}".constantize.new
+      command_module = Kernel.const_get("Tumugi").const_get("Command")
+      cmd = command_module.const_get("#{command.to_s.capitalize}").new
       cmd.execute(dag, options)
     end
 

data/lib/tumugi/atomic_file.rb

@@ -0,0 +1,49 @@
+require 'tempfile'
+require 'forwardable'
+
+module Tumugi
+  class AtomicFile
+    extend Forwardable
+    def_delegators :@temp_file,
+      :bin_mode?, :print, :printf, :putc, :puts, :write,
+      :seek, :set_encoding, :sync, :sync=, :sysseek,
+      :syswrite, :write, :write_nonblock
+
+    def initialize(path)
+      @path = path
+    end
+
+    attr_reader :path
+
+    def open(&block)
+      if block_given?
+        Tempfile.open(basename) do |fp|
+          @temp_file = fp
+          block.call(self)
+          close
+        end
+      else
+        @temp_file = Tempfile.open(basename)
+      end
+      self
+    end
+
+    def close
+      if @temp_file
+        move_to_final_destination(@temp_file)
+        @temp_file.close
+        @temp_file = nil
+      end
+    end
+
+    def move_to_final_destination(temp_file)
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    private
+
+    def basename
+      @basename ||= File.basename(@path)
+    end
+  end
+end

data/lib/tumugi/cli.rb

@@ -1,6 +1,5 @@
 require 'thor'
 require 'tumugi'
-require 'tumugi/tumugi_module'
 
 module Tumugi
   class CLI < Thor
@@ -8,7 +7,7 @@ module Tumugi
 
     class << self
       def common_options
-        option :file, aliases: '-f', desc: 'Task definition file name'
+        option :file, aliases: '-f', desc: 'Task definition file name', required: true
         option :config, aliases: '-c', desc: 'Configuration file name', default: 'tumugi.rb'
       end
     end
@@ -18,7 +17,7 @@ module Tumugi
       puts "tumugi v#{Tumugi::VERSION}"
     end
 
-    desc "run", "Run workflow"
+    desc "run TASK", "Run TASK in a workflow"
     map "run" => "run_" # run is thor's reserved word, so this trick is needed
     option :workers, aliases: '-w', type: :numeric, desc: 'Number of workers to run task concurrently'
     option :quiet, type: :boolean, desc: 'Suppress log', default: false
@@ -29,7 +28,7 @@ module Tumugi
       Tumugi.application.execute(:run, task, options)
     end
 
-    desc "show", "Show DAG of workflow"
+    desc "show TASK", "Show DAG of TASK in a workflow"
     common_options
     option :out, aliases: '-o', desc: 'Output file name. If not specified, output result to STDOUT'
     option :format, aliases: '-t', desc: 'Output file format. Only affected --out option is specified.', enum: ['dot', 'png', 'svg']

data/lib/tumugi/command/show.rb

@@ -1,10 +1,13 @@
-require 'gviz'
+require 'graphviz'
 require 'tmpdir'
 require 'fileutils'
+require 'tumugi/mixin/listable'
 
 module Tumugi
   module Command
     class Show
+      include Tumugi::Mixin::Listable
+
       @@supported_formats = ['dot', 'png', 'jpg', 'svg', 'pdf']
 
       def execute(dag, options={})
@@ -17,22 +20,26 @@ module Tumugi
           format = options[:format]
         end
 
-        graph = Graph do
-          dag.tsort.each do |task|
-            node task.id
-            route task.id => task._requires.map {|t| t.id}
+        g = GraphViz.new(:G, type: :digraph, rankdir: "RL")
+        tasks = dag.tsort
+        tasks.each do |task|
+          g.add_nodes(task.id.to_s)
+        end
+        tasks.each do |task|
+          list(task._requires).each do |req|
+            g.add_edge(g.get_node(req.id.to_s), g.get_node(task.id.to_s))
           end
         end
 
-        if out.present?
-          Dir.mktmpdir do |dir|
-            file_base_path = "#{File.dirname(dir)}/#{File.basename(out, '.*')}"
-            graph.save(file_base_path, format == 'dot' ? nil : format)
-            FileUtils.mkdir_p(File.dirname(out))
-            FileUtils.copy("#{file_base_path}.#{format}", out)
+        if out
+          FileUtils.mkdir_p(File.dirname(out))
+          if format == 'dot'
+            File.write(out, g.to_s)
+          else
+            g.output(format => out)
           end
         else
-          print graph
+          print g
         end
       end
     end

data/lib/tumugi/dag.rb

@@ -1,10 +1,10 @@
 require 'tsort'
-require 'tumugi/helper'
+require 'tumugi/mixin/listable'
 
 module Tumugi
   class DAG
     include TSort
-    include Tumugi::Helper
+    include Tumugi::Mixin::Listable
 
     attr_reader :tasks
 

data/lib/tumugi/file_system.rb

@@ -0,0 +1,38 @@
+require 'tumugi'
+require 'tumugi/file_system_error'
+
+module Tumugi
+  # This class defines interfaces of file system
+  # such as local file, Amazon S3, Google Cloud Storage
+  class FileSystem
+    def exist?(path)
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def remove(path, recursive: true)
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def mkdir(path, parents: true, raise_if_exist: false)
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def directory?(path)
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def entries(path)
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def move(src, dest, raise_if_exist: false)
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def rename(path, dest)
+      Tumugi.logger.warn "File system #{self.class.name} client doesn't support atomic move."
+      raise FileAlreadyExistError if exist?(dest)
+      move(path, dest)
+    end
+  end
+end

data/lib/tumugi/file_system_error.rb

@@ -0,0 +1,13 @@
+module Tumugi
+  class FileSystemError < StandardError
+  end
+
+  class FileAlreadyExistError < FileSystemError
+  end
+
+  class MissingParentDirectoryError < FileSystemError
+  end
+
+  class NotADirectoryError < FileSystemError
+  end
+end

data/lib/tumugi/logger.rb

@@ -1,3 +1,4 @@
+require 'logger'
 require 'forwardable'
 
 module Tumugi

data/lib/tumugi/mixin/listable.rb

@@ -0,0 +1,17 @@
+module Tumugi
+  module Mixin
+    module Listable
+      def list(obj)
+        if obj.nil?
+          []
+        elsif obj.is_a?(Array)
+          obj
+        elsif obj.is_a?(Hash)
+          obj.map { |k,v| v }
+        else
+          [obj]
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/mixin/task_helper.rb

@@ -0,0 +1,12 @@
+require 'tumugi/plugin'
+
+module Tumugi
+  module Mixin
+    module TaskHelper
+      def target(type, *args)
+        klass = Tumugi::Plugin.lookup_target(type)
+        klass.new(*args)
+      end
+    end
+  end
+end

data/lib/tumugi/plugin.rb

@@ -0,0 +1,43 @@
+require 'tumugi'
+require 'tumugi/registry'
+
+module Tumugi
+  module Plugin
+    TARGET_REGISTRY = Registry.new(:target, 'tumugi/plugin/target/')
+    TASK_REGISTRY   = Registry.new(:task,   'tumugi/plugin/task/')
+
+    def self.register_target(type, klass)
+      register_impl('target', TARGET_REGISTRY, type, klass)
+    end
+
+    def self.register_task(type, klass)
+      register_impl('task', TASK_REGISTRY, type, klass)
+    end
+
+    def self.register_impl(kind, registry, type, value)
+      if !value.is_a?(Class)
+        raise "Invalid implementation as #{kind} plugin: '#{type}'. It must be a Class."
+      end
+      registry.register(type, value)
+      Tumugi.logger.debug "registered #{kind} plugin '#{type}'"
+      nil
+    end
+
+    def self.lookup_target(type)
+      lookup_impl('target', TARGET_REGISTRY, type)
+    end
+
+    def self.lookup_task(type)
+      lookup_impl('task', TASK_REGISTRY, type)
+    end
+
+    def self.lookup_impl(kind, registry, type)
+      obj = registry.lookup(type)
+      if obj.is_a?(Class)
+        obj
+      else
+        raise "#{kind} plugin '#{type}' is not a Class"
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/atomic_local_file.rb

@@ -0,0 +1,12 @@
+require 'fileutils'
+require 'tumugi/atomic_file'
+
+module Tumugi
+  module Plugin
+    class AtomicLocalFile < AtomicFile
+      def move_to_final_destination(temp_file)
+        FileUtils.move(temp_file, path)
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/file_system_target.rb

@@ -0,0 +1,30 @@
+require 'tumugi/target'
+require 'tumugi/file_system'
+require 'tumugi/file_system_error'
+
+module Tumugi
+  module Plugin
+    class FileSystemTarget < Target
+      attr_reader :path
+
+      def initialize(path)
+        @path = path
+      end
+
+      def fs
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
+      def open(mode="r")
+      end
+
+      def exist?
+        fs.exist?(@path)
+      end
+
+      def remove
+        fs.remove(@path)
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/local_file_system.rb

@@ -0,0 +1,62 @@
+require 'fileutils'
+require 'tumugi/file_system'
+require 'tumugi/file_system_error'
+
+module Tumugi
+  module Plugin
+    class LocalFileSystem
+      def exist?(path)
+        File.exist?(path)
+      end
+
+      def remove(path, recursive: true)
+        if recursive && directory?(path)
+          FileUtils.rm_r(path)
+        else
+          FileUtils.remove_file(path)
+        end
+      end
+
+      def mkdir(path, parents: true, raise_if_exist: false)
+        if File.exist?(path)
+          if raise_if_exist
+            raise FileAlreadyExistError
+          elsif !directory?(path)
+            raise NotADirectoryError
+          else
+            return
+          end
+        end
+
+        if parents
+          FileUtils.mkdir_p(path)
+        else
+          if File.exist?(File.expand_path("..", path))
+            FileUtils.mkdir(path)
+          else
+            raise MissingParentDirectoryError
+          end
+        end
+      end
+
+      def directory?(path)
+        File.directory?(path)
+      end
+
+      def entries(path)
+        if directory?(path)
+          Dir.glob(File.join(path, '*'))
+        else
+          raise NotADirectoryError
+        end
+      end
+
+      def move(src, dest, raise_if_exist: false)
+        if File.exist?(dest) && raise_if_exist
+          raise FileAlreadyExistError
+        end
+        FileUtils.mv(src, dest, force: true)
+      end
+    end
+  end
+end

data/lib/tumugi/plugin/target/local_file.rb

@@ -0,0 +1,25 @@
+require 'tumugi/plugin/atomic_local_file'
+require 'tumugi/plugin/file_system_target'
+require 'tumugi/plugin/local_file_system'
+
+module Tumugi
+  module Plugin
+    class LocalFileTarget < FileSystemTarget
+      Plugin.register_target(:local_file, self)
+
+      def fs
+        @fs ||= LocalFileSystem.new
+      end
+
+      def open(mode="r", &block)
+        if mode.include? 'r'
+          File.open(path, mode, &block)
+        elsif mode.include? 'w'
+          AtomicLocalFile.new(path).open(&block)
+        else
+          raise 'Invalid mode: #{mode}'
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/registry.rb

@@ -0,0 +1,56 @@
+module Tumugi
+  class Registry
+    DEFAULT_PLUGIN_PATH = File.expand_path('plugin', __FILE__)
+
+    def initialize(kind, search_prefix)
+      @kind = kind
+      @search_prefix = search_prefix
+      @map = {}
+      @paths = [DEFAULT_PLUGIN_PATH]
+    end
+
+    attr_reader :kind, :paths
+
+    def register(type, value)
+      type = type.to_sym
+      @map[type] = value
+    end
+
+    def lookup(type)
+      t = type.to_sym
+      return @map[t] if @map.has_key?(t)
+      search(type)
+      return @map[t] if @map.has_key?(t)
+      raise "Unknown #{@kind} plugin '#{type}'"
+    end
+
+    def search(type)
+      path = "#{@search_prefix}#{type}"
+
+      # prefer LOAD_PATH than gems
+      [@paths, $LOAD_PATH].each do |paths|
+        files = paths.map { |lp|
+          lpath = File.expand_path(File.join(lp, "#{path}.rb"))
+          File.exist?(lpath) ? lpath : nil
+        }.compact
+        unless files.empty?
+          require files.sort.last
+          return
+        end
+      end
+
+      specs = Gem::Specification.find_all { |spec|
+        spec.contains_requirable_file? path
+      }
+
+      # prefer newer version
+      specs = specs.sort_by { |spec| spec.version }
+      if spec = specs.last
+        spec.require_paths.each do |lib|
+          file = "#{spec.full_gem_path}/#{lib}/#{path}"
+          return file
+        end
+      end
+    end
+  end
+end

data/lib/tumugi/target.rb

@@ -0,0 +1,17 @@
+require 'tumugi'
+
+module Tumugi
+  class Target
+    def exist?
+      raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+    end
+
+    def logger
+      @logger ||= Tumugi.logger
+    end
+
+    def log(msg)
+      logger.info(msg)
+    end
+  end
+end

data/lib/tumugi/task.rb

@@ -1,13 +1,14 @@
-require 'tumugi/helper'
+require 'tumugi/mixin/listable'
+require 'tumugi/mixin/task_helper'
 
 module Tumugi
   class Task
-    include Tumugi::Helper
+    include Tumugi::Mixin::Listable
+    include Tumugi::Mixin::TaskHelper
 
     attr_accessor :state # :pending, :running, :completed, :skipped
 
     def initialize
-      @logger = Tumugi.logger
       @state = :pending
     end
 
@@ -31,6 +32,14 @@ module Tumugi
       self
     end
 
+    def logger
+      @logger ||= Tumugi.logger
+    end
+
+    def log(msg)
+      logger.info(msg)
+    end
+
     # If you need to define task dependencies, override in subclass
     def requires
       []
@@ -56,7 +65,11 @@ module Tumugi
 
     def completed?
       outputs = list(output)
-      !outputs.empty? && outputs.all?(&:exist?)
+      if outputs.empty?
+        @state == :completed || @state == :skipped
+      else
+        outputs.all?(&:exist?)
+      end
     end
 
     # Following methods are internal use only

data/lib/tumugi/task_definition.rb

@@ -1,12 +1,16 @@
 require 'tumugi/task'
+require 'tumugi/plugin'
+require 'tumugi/mixin/listable'
+require 'tumugi/mixin/task_helper'
 
 module Tumugi
   class TaskDefinition
-    include Tumugi::Helper
+    include Tumugi::Mixin::Listable
+    include Tumugi::Mixin::TaskHelper
 
     def self.define(id, opts={}, &block)
       td = Tumugi::TaskDefinition.new(id, opts)
-      td.instance_eval(&block)
+      td.instance_eval(&block) if block_given?
       Tumugi.application.add_task(id, td)
       td
     end
@@ -16,6 +20,10 @@ module Tumugi
     def initialize(id, opts={})
       @id = id
       @opts = { type: Tumugi::Task }.merge(opts)
+
+      unless @opts[:type].is_a?(Class)
+        @opts[:type] = Tumugi::Plugin.lookup_task(@opts[:type])
+      end
     end
 
     def instance
@@ -35,7 +43,7 @@ module Tumugi
     end
 
     def output_eval(task)
-      @out ||= @outputs.is_a?(Proc) ? @outputs.call(task) : @outputs
+      @out ||= @outputs.is_a?(Proc) ? task.instance_eval(&@outputs) : @outputs
     end
 
     def required_tasks
@@ -43,13 +51,14 @@ module Tumugi
     end
 
     def run_block(task)
-      @run.call(task)
+      task.instance_eval(&@run)
     end
 
     private
 
     def create_task
       task = define_task.new
+      raise "Invalid type: '#{@opts[:type]}'" unless task.is_a?(Tumugi::Task)
       task.id = @id
       task
     end

data/lib/tumugi/version.rb

@@ -1,3 +1,3 @@
 module Tumugi
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/tumugi.gemspec

@@ -18,15 +18,17 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'activesupport', '~> 4.2'
-  spec.add_runtime_dependency 'gviz', '~> 0.3.5'
-  spec.add_runtime_dependency 'thor', '~> 0.19.1'
+  spec.required_ruby_version = '>= 2.0'
+
   spec.add_runtime_dependency 'parallel', '~> 1.8.0'
   spec.add_runtime_dependency 'retriable', '~> 2.1'
+  spec.add_runtime_dependency 'ruby-graphviz', '~> 1.2.2'
+  spec.add_runtime_dependency 'thor', '~> 0.19.1'
 
   spec.add_development_dependency 'bundler', '~> 1.11'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'test-unit', '~> 3.1'
   spec.add_development_dependency 'test-unit-rr'
   spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'github_changelog_generator'
 end

metadata

@@ -1,85 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: tumugi
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-28 00:00:00.000000000 Z
+date: 2016-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
-  name: gviz
+  name: retriable
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.5
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.5
+        version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: ruby-graphviz
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.19.1
+        version: 1.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.19.1
+        version: 1.2.2
 - !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.8.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.8.0
-- !ruby/object:Gem::Dependency
-  name: retriable
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.19.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.19.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: github_changelog_generator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - hakobera@gmail.com
@@ -160,6 +160,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
@@ -174,19 +175,27 @@ files:
 - exe/tumugi
 - lib/tumugi.rb
 - lib/tumugi/application.rb
+- lib/tumugi/atomic_file.rb
 - lib/tumugi/cli.rb
 - lib/tumugi/command/run.rb
 - lib/tumugi/command/show.rb
 - lib/tumugi/config.rb
 - lib/tumugi/dag.rb
 - lib/tumugi/dsl.rb
-- lib/tumugi/helper.rb
+- lib/tumugi/file_system.rb
+- lib/tumugi/file_system_error.rb
 - lib/tumugi/logger.rb
-- lib/tumugi/target/base.rb
-- lib/tumugi/target/file_target.rb
+- lib/tumugi/mixin/listable.rb
+- lib/tumugi/mixin/task_helper.rb
+- lib/tumugi/plugin.rb
+- lib/tumugi/plugin/atomic_local_file.rb
+- lib/tumugi/plugin/file_system_target.rb
+- lib/tumugi/plugin/local_file_system.rb
+- lib/tumugi/plugin/target/local_file.rb
+- lib/tumugi/registry.rb
+- lib/tumugi/target.rb
 - lib/tumugi/task.rb
 - lib/tumugi/task_definition.rb
-- lib/tumugi/tumugi_module.rb
 - lib/tumugi/version.rb
 - tumugi.gemspec
 homepage: https://github.com/tumugi/tumugi
@@ -201,7 +210,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/tumugi/helper.rb

@@ -1,15 +0,0 @@
-module Tumugi
-  module Helper
-    def list(obj)
-      if obj.nil?
-        []
-      elsif obj.is_a?(Array)
-        obj
-      elsif obj.is_a?(Hash)
-        obj.map { |k,v| v }
-      else
-        [obj]
-      end
-    end
-  end
-end

data/lib/tumugi/target/base.rb

@@ -1,9 +0,0 @@
-module Tumugi
-  module Target
-    class Base
-      def exist?
-        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
-      end
-    end
-  end
-end

data/lib/tumugi/target/file_target.rb

@@ -1,17 +0,0 @@
-require 'tumugi/target/base'
-
-module Tumugi
-  module Target
-    class FileTarget < Base
-      attr_reader :path
-
-      def initialize(path)
-        @path = path
-      end
-
-      def exist?
-        ::File.exist?(path)
-      end
-    end
-  end
-end

data/lib/tumugi/tumugi_module.rb

@@ -1,23 +0,0 @@
-require 'tumugi/application'
-require 'tumugi/config'
-require 'tumugi/logger'
-
-module Tumugi
-  class << self
-    def application
-      @application ||= Tumugi::Application.new
-    end
-
-    def logger
-      @logger ||= Tumugi::Logger.new
-    end
-
-    def config
-      @config ||= Tumugi::Config.new
-      if block_given?
-        yield @config
-      end
-      @config
-    end
-  end
-end