katte 0.0.1.3

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.ruby-version

@@ -0,0 +1 @@
+1.9.3-p194

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in katte.gemspec
+gemspec
+
+group :test, :development do
+  gem 'rspec'
+  gem 'debugger'
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Ojima Hikaru
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,107 @@
+# Katte
+
+Katte は以下の特徴を備えたバッチ処理管理ツールです。
+
+- 実行可能なタスク定義
+- タスクの依存関係の解決
+- 実行コマンドや出力コマンドを拡張可能
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'katte'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install katte
+
+## Turtorial
+
+### はじめの一歩
+
+`recipes/` 以下にタスクの定義を記述します。
+
+    $ vim recipes/first_task.sh
+
+    echo ${date}
+
+`$ bin/katte run -d yyyy-mm-dd` を実行すると、`result/first_task.log` 以下
+に結果が出力されます。
+
+拡張子から実行コマンドを判別します。
+実行コマンドは、bash(.sh) / ruby(.rb) / Rscript(.R) / hive(.sql) をデ
+フォルトでサポートしています。
+
+### 実行オプションを記述する
+
+レシピの先頭に実行オプションを記述できます。
+
+    # require: second_task
+    # output : file, stdio
+    # period : week
+
+    echo good day
+
+オプションの意味は以下の通りです。
+
+<dl>
+  <dt>`require`</dt><dd>指定したタスクの完了を待って実行します。タスクは
+  `recipes/` からの相対パスで指定します。</dd>
+  <dt>`output`</dt><dd>出力方法を指定します。デフォルトで `file` と
+  `stdio` が指定可能です。</dd>
+  <dt>`period`</dt><dd>実行周期を指定します。`day` / `week` / `month`
+  が指定可能です。</dd>
+</dl>
+
+### 実行コマンドを拡張する
+
+`Katte::Plugins::FileType` を継承したクラスをロードすることで、実行コ
+マンドを拡張できます。
+
+`Katte::Plugins::Output` を継承したクラスをロードすることで、出力方法を
+拡張できます。
+
+記述例は `lib/katte/plugins/file_type/bash.rb` /
+`lib/katte/plugins/output/file.rb` を参考にしてください。
+
+### カスタムレシピを記述する
+
+rubyレシピに `custom` オプションを指定することで、タスクの実行タイミン
+グを柔軟に制御することができます。
+
+    $ vim custom_recipe.rb
+
+    # option: custom
+    loop {
+      if FileTest.file?('/some/important/file')
+        tag 'important'
+        break
+      elsif FileTest.file?('/some/critical/file')
+        tag 'critical'
+        break
+      end
+      sleep 1000
+    }
+
+    done
+
+    $ vim custom_recipe/sub_task.sh
+
+    # require: custom_recipe:important
+    cat /some/important/file
+
+`tag <tag>` で次のタスクの実行を開始します。
+`done` でこのタスクの実行を終了します。
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/katte

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.expand_path(File.join(__FILE__, '../../lib')))
+
+APP_PATH = File.expand_path('../../', __FILE__)
+
+require 'katte/command'

data/katte.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'katte/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "katte"
+  gem.version       = Katte::VERSION
+  gem.authors       = ["Ojima Hikaru"]
+  gem.email         = ["ojiam.h@gmail.com"]
+  gem.description   = %q{Batch job management tool.}
+  gem.summary       = %q{Batch job management tool.}
+  gem.homepage      = "https://github.com/ojima-h/katte"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

data/lib/katte.rb

@@ -0,0 +1,50 @@
+require 'find'
+require 'logger'
+
+class Katte
+  def self.new(params = {})
+    @instance ||= super(params)
+  end
+  def self.app(params = {})
+    @instance ||= new(params)
+  end
+
+  attr_reader :env
+  attr_reader :config
+  attr_reader :logger
+  attr_reader :options
+
+  def initialize(options = {})
+    @env     = Environment.new(options)
+    @options = options
+    @config  = Katte::Config.config
+
+    @logger = if STDOUT.tty?
+                Logger.new(STDOUT)
+              else
+                Logger.new(File.join(@config.log_root, 'katte.log'), 'daily')
+              end
+    @logger.level = Logger::WARN if config.mode == 'test'
+  end
+
+  def run
+    (@runner ||= Katte::Runner.new).run
+  end
+
+  def exec(recipe_path)
+    node_factory = config.factory || Katte::Recipe::NodeFactory.new
+    node = node_factory.load(recipe_path)
+
+    node.file_type.execute(node)
+  end
+end
+
+require 'katte/node'
+require "katte/version"
+require 'katte/environment'
+require 'katte/config'
+require 'katte/plugins'
+require 'katte/filter'
+require 'katte/driver'
+require 'katte/recipe'
+require 'katte/runner'

data/lib/katte/command.rb

@@ -0,0 +1,33 @@
+require 'katte'
+require 'date'
+require 'optparse'
+
+class Katte
+  class Command
+    opt = OptionParser.new
+    opt.on('-d date') {|v| options[:datetime] = v }
+    opt.on('-v')      { options[:verbose] = true }
+    opt.on('-h')      { print_help_message; exit }
+    opt.banner = "katte [options] [files]"
+    opt.parse!(ARGV)
+
+    files = ARGV.select(&FileTest.method(:file?))
+                .map(&File.method(:absolute_path))
+    options = {
+      :date  => (Date.today - 1).strftime("%Y-%m-%d"),
+      :files => files,
+    }
+
+    app = Katte.new(options)
+
+    if files.empty?
+      app.run
+    else
+      files.each {|file| app.exec file }
+    end
+
+    def print_help_message
+      print opt.help
+    end
+  end
+end

data/lib/katte/config.rb

@@ -0,0 +1,21 @@
+require 'yaml'
+
+class Katte
+  class Config
+    config_klass = Struct.new(:mode,
+                              :recipes_root,
+                              :result_root,
+                              :log_root,
+                              :factory)
+    @config = config_klass.new
+
+    @config.mode         = ENV['KATTE_MODE'] || 'production'
+    @config.recipes_root = File.join(APP_PATH, 'recipes')
+    @config.result_root  = File.join(APP_PATH, 'result')
+    @config.log_root     = File.join(APP_PATH, 'log')
+
+    def self.config
+      self == Katte::Config ? @config : Katte::Config.config
+    end
+  end
+end

data/lib/katte/driver.rb

@@ -0,0 +1,71 @@
+class Katte
+  class Driver
+    def self.run(nodes)
+      core = Core.new(nodes)
+      core.run
+      core.summary
+    end
+
+    class Core
+      attr_reader :summary
+      def initialize(nodes, options = {})
+        @nodes = nodes
+        @nodes_list  = Hash[nodes.map {|node| [node.name, node.parents.length] }]
+        @nodes_index = Hash[nodes.map {|node| [node.name, node] }]
+
+        @queue = Queue.new
+
+        @summary = {:success => [], :fail => [], :skip => []}
+      end
+
+      %w(done fail next skip finish).each do |method|
+        define_method(method) do |*args|
+          @queue.push [:"_#{method}", *args]
+        end
+      end
+
+      def run
+        return if @nodes_list.empty?
+
+        run_nodes(@nodes.select{|n| n.parents.length.zero? })
+
+        loop {
+          method, *args = @queue.pop
+
+          break if method == :_finish
+
+          self.send(method, *args)
+        }
+      end
+
+      private
+      def run_nodes(nodes)
+        nodes.each {|node| node.run(self) }
+      end
+
+      def _done(node, *args)
+        @nodes_list.delete(node.name)
+        return finish if @nodes_list.empty?
+
+        node.children.each do |child|
+          @nodes_list[child.name] -= 1
+          child.run(self) if @nodes_list[child.name].zero?
+        end
+      end
+      def _fail(node, *args)
+        @nodes_list.delete(node.name)
+        node.descendants.each {|dec| @nodes_list.delete(dec.name) }
+
+        finish if @nodes_list.empty?
+      end
+      def _next(node, child, *args)
+        @nodes_list[child] -= 1
+        @nodes_index[child].run(self) if @nodes_list[child].zero?
+      end
+      def _skip(node, *args)
+        node.descendants.each {|dec| @nodes_list.delete(dec) }
+        finish if @nodes_list.empty?
+      end
+    end
+  end
+end

data/lib/katte/environment.rb

@@ -0,0 +1,14 @@
+class Katte
+  class Environment
+    attr_accessor :datetime
+    def initialize(options = {})
+      @datetime = options[:datetime] ? DateTime.parse(options[:datetime]) : DateTime.now
+    end
+
+    def to_hash
+      @hash ||= {
+        'date' => @datetime.strftime('%Y-%m-%d'),
+      }
+    end
+  end
+end

data/lib/katte/filter.rb

@@ -0,0 +1,22 @@
+require 'date'
+
+class Katte
+  class Filter
+    def initialize(options = {})
+      @datetime = options[:datetime] || DateTime.now
+    end
+
+    def call(params)
+      check_date(params)
+    end
+
+    private
+    def check_date(params)
+      case params[:period]
+      when 'day'   then true
+      when 'week'  then @datetime.wday == 1 # monday
+      when 'month' then @datetime.day  == 1
+      end
+    end
+  end
+end

data/lib/katte/node.rb

@@ -0,0 +1,5 @@
+module Katte::Node
+end
+
+require 'katte/node/base'
+require 'katte/node/collection'

data/lib/katte/node/base.rb

@@ -0,0 +1,48 @@
+module Katte::Node
+  module Base
+    class AbstractNodeException < Exception; end
+
+    def name
+      raise AbstractNodeException, "method #name must be implemented"
+    end
+    def requires
+      @requires ||= []
+    end
+
+    # execution filter
+    # return whether this node to run
+    def run?
+      false
+    end
+    def run(driver)
+      driver.skip(self)
+    end
+    
+    # accessor for children
+    def children
+      @children ||= []
+    end
+    def add_child(node, *params)
+      children << node
+    end
+
+    # accessor for parents
+    def parents
+      @parents ||= []
+    end
+    def add_parent(node, *params)
+      parents << node
+    end
+
+    def descendants
+      Enumerator.new {|enum| _descendants(enum) }
+    end
+    def _descendants(enum)
+      children.each {|node|
+        enum << node
+        node._descendants(enum)
+      }
+      enum
+    end
+  end
+end