katte 0.0.1.3

data/lib/katte/node/collection.rb

@@ -0,0 +1,47 @@
+class Katte::Node::Collection
+  include Enumerable
+
+  def each &proc
+    @nodes.values.each &proc
+  end
+  
+  def initialize
+    @nodes  = {}
+  end
+
+  def all
+    @nodes.values
+  end
+  def find(node_name)
+    @nodes[node_name]
+  end
+
+  def add(node, params = {})
+    @nodes[node.name] = node
+  end
+  alias :<< :add
+
+  def connect
+    cache ||= {} # connection cache
+
+    @nodes.values.each do |node|
+      # connect self to parents if exist
+      node.requires.each do |req, prm|
+        if @nodes[req]
+          node.add_parent(@nodes[req], *prm)
+          @nodes[req].add_child(node, *prm)
+        else
+          (cache[req] ||= []) << [node, prm]
+        end
+      end
+
+      # connect children to self
+      if children = cache.delete(node.name)
+        children.each {|c, prm|
+          c.add_parent(node, *prm)
+          node.add_child(c, *prm)
+        }
+      end
+    end
+  end
+end

data/lib/katte/plugins.rb

@@ -0,0 +1,11 @@
+require 'katte/plugins/base'
+require 'katte/plugins/file_type'
+require 'katte/plugins/output'
+require 'katte/plugins/node'
+
+module Katte::Plugins
+end
+
+Dir[File.expand_path('../plugins/file_type/*.rb', __FILE__),
+    File.expand_path('../plugins/output/*.rb'   , __FILE__),
+    File.expand_path('../plugins/node/*.rb'     , __FILE__)].each {|p| require p }

data/lib/katte/plugins/base.rb

@@ -0,0 +1,38 @@
+module Katte::Plugins
+  module Base
+    def class_methods
+      @class_methods ||= Module.new
+    end
+    private :class_methods
+
+    def included(klass)
+      klass.extend class_methods
+    end
+
+    def index(keyword = nil)
+      return @index unless keyword
+      @index = keyword
+    end
+    def define_keyword(keyword)
+      klass = self
+
+      class_methods.send(:define_method, keyword) {|value|
+        define_method(keyword) { value }
+
+        if klass.index == keyword
+          klass.register(value, self.new)
+        end
+      }
+    end
+
+    def plugins
+      @plugins ||= {}
+    end
+    def register(index, plugin)
+      plugins[index] = plugin
+    end
+    def find(index)
+      plugins[index]
+    end
+  end
+end

data/lib/katte/plugins/file_type.rb

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+module Katte::Plugins::FileType
+  extend Katte::Plugins::Base
+
+  define_keyword :comment_by
+  define_keyword :extname
+  index :extname
+
+  def simple_exec(node, program, *args)
+    node.open {|out, err|
+      system(Katte.app.env.to_hash, program, *args, :out => out, :err => err)
+    }
+  end
+
+  def parse(path)
+    c = Regexp.escape(comment_by)
+    @comment_pattern   ||= /^#{c}|^\s*$/
+    @directive_pattern ||= %r{
+        ^#{c}\s*                    # comment
+        (?<key>\w+)\s*:\s*          # key
+        (?<value>[^(\s]+)           # value
+        (\((?<params>[^)]*)\))?     # param (optional)
+        $
+      }x
+
+    directive = Hash.new {|h,k| h[k] = [] }
+    open(path) do |io|
+      while line = io.gets
+        line.chomp!
+        break unless     @comment_pattern.match(line)
+        next  unless m = @directive_pattern.match(line)
+
+        key    = m[:key]
+        value  = m[:value].strip
+        params = m[:params] && m[:params].split(',').map(&:strip).map(&method(:convert_variable))
+
+        directive[key] << [value, params]
+      end
+    end
+    directive
+  end
+
+  def convert_variable(value)
+    # #{xx} というパターンを env['xx'] で置換する
+    value.gsub(/\#\{((?:\\\{|[^\{])+)\}/) {|m| Katte.app.env.to_hash[$1]}
+  end
+end

data/lib/katte/plugins/file_type/bash.rb

@@ -0,0 +1,9 @@
+class Katte::Plugins::FileType::Bash
+  include Katte::Plugins::FileType
+  extname    'sh'
+  comment_by '#'
+
+  def execute(node)
+    simple_exec(node, 'bash', node.path)
+  end
+end

data/lib/katte/plugins/file_type/debug.rb

@@ -0,0 +1,14 @@
+class Katte::Plugins::FileType::Debug
+  include Katte::Plugins::FileType
+
+  extname :debug
+
+  class Abort < StandardError; end
+
+  def execute(node)
+    node.options['callback'].each {|cb| cb.call(node) }
+    return true
+  rescue Abort => e
+    return false
+  end
+end

data/lib/katte/plugins/file_type/hive.rb

@@ -0,0 +1,9 @@
+class Katte::Plugins::FileType::Hive
+  include Katte::Plugins::FileType
+  extname    'sql'
+  comment_by '--'
+
+  def execute(node)
+    simple_exec(node, 'hive', '-f', node.path)
+  end
+end

data/lib/katte/plugins/file_type/r.rb

@@ -0,0 +1,9 @@
+class Katte::Plugins::FileType::R
+  include Katte::Plugins::FileType
+  extname    'R'
+  comment_by '#'
+
+  def execute(node)
+    simple_exec(node, 'Rscript', node.path)
+  end
+end

data/lib/katte/plugins/file_type/ruby.rb

@@ -0,0 +1,9 @@
+class Katte::Plugins::FileType::Rubyn
+  include Katte::Plugins::FileType
+  extname    'rb'
+  comment_by '#'
+
+  def execute(node)
+    simple_exec(node, 'ruby', node.path)
+  end
+end

data/lib/katte/plugins/node.rb

@@ -0,0 +1,6 @@
+module Katte::Plugins::Node
+  extend Katte::Plugins::Base
+
+  define_keyword :name
+  index :name
+end

data/lib/katte/plugins/node/debug.rb

@@ -0,0 +1,11 @@
+class Katte::Plugins::Node::Debug
+  include Katte::Node::Base
+  include Katte::Plugins::Node
+
+  name 'debug'
+
+  def run(driver)
+    children.each {|child| driver.next(self, child.name) }
+    driver.done(self)
+  end
+end

data/lib/katte/plugins/node/file.rb

@@ -0,0 +1,45 @@
+class Katte::Plugins::Node::File
+  include Katte::Node::Base
+  include Katte::Plugins::Node
+
+  name 'file'
+
+  def duration; 30; end
+  
+  def add_child(node, *params)
+    file = params.first
+    return unless file
+    add_watch_list(file, node)
+  end
+
+  def run(driver)
+    until watching_files.empty?
+      update_watch_list do |file, nodes|
+        if FileTest.exist? file
+          nodes.each {|n| driver.next(self, n.name) }
+          true
+        else
+          false
+        end
+      end
+
+      watching_files.each {|f| Katte.app.logger.info("wating: #{f}") }
+      sleep duration * 60 unless watching_files.empty?
+    end
+    
+    driver.done(self)
+  end
+
+  private
+  def watching_files
+    (@watch_list || {}).keys
+  end
+  def add_watch_list(file, node)
+    @watch_list ||= {}
+    (@watch_list[file] ||= []) << node
+  end
+  def update_watch_list(&proc)
+    @watch_list ||= {}
+    @watch_list = @watch_list.reject(&proc)
+  end
+end

data/lib/katte/plugins/output.rb

@@ -0,0 +1,13 @@
+module Katte::Plugins::Output
+  extend Katte::Plugins::Base
+
+  define_keyword :name
+  index :name
+
+  def out(node, output)
+    output
+  end
+  def err(node, output)
+    output
+  end
+end

data/lib/katte/plugins/output/debug.rb

@@ -0,0 +1,14 @@
+class Katte::Plugins::Output::Debug
+  include Katte::Plugins::Output
+  name :debug
+
+  def history
+    @history ||= Queue.new
+  end
+
+  def out(node, data)
+    history.push(node: node, out: data)
+
+    data
+  end
+end

data/lib/katte/plugins/output/file.rb

@@ -0,0 +1,31 @@
+require 'fileutils'
+
+class Katte::Plugins::Output::File_
+  include Katte::Plugins::Output
+  name :file
+
+  def out(node, data)
+    return data if data.empty?
+
+    file = File.join(Katte.app.config.result_root,
+                     node.name + '.out',
+                     Katte.app.env.to_hash['date'] + ".txt")
+
+    FileUtils.makedirs(File.dirname(file))
+
+    File.open(file, 'w') {|f| f.print data }
+  end
+
+  def err(node, data)
+    return data if data.empty?
+
+    file = File.join(Katte.app.config.log_root,
+                     'recipes',
+                     node.name + '.log',
+                     Katte.app.env.to_hash['date'] + ".txt")
+
+    FileUtils.makedirs(File.dirname(file))
+
+    File.open(file, 'a') {|f| f.print data }
+  end
+end

data/lib/katte/plugins/output/stderr.rb

@@ -0,0 +1,9 @@
+class Katte::Plugins::Output::Stderr
+  include Katte::Plugins::Output
+  name :stderr
+
+  def err(node, data)
+    STDERR.puts data
+    data
+  end
+end

data/lib/katte/plugins/output/stdoe.rb

@@ -0,0 +1,14 @@
+class Katte::Plugins::Output::Stdoe
+  include Katte::Plugins::Output
+  name :stdoe
+
+  def out(node, data)
+    STDIN.puts data
+    data
+  end
+
+  def err(node, data)
+    STDERR.puts data
+    data
+  end
+end

data/lib/katte/recipe.rb

@@ -0,0 +1,8 @@
+class Katte
+  module Recipe
+  end
+end
+
+require 'katte/recipe/file_type'
+require 'katte/recipe/node'     
+require 'katte/recipe/node_factory'

data/lib/katte/recipe/file_type.rb

@@ -0,0 +1,29 @@
+module Katte::Recipe::FileType
+  class <<self
+    def table
+      @table ||= {}
+    end
+    private :table
+      
+    def register(file_type, extname = nil)
+      extname ||= file_type.extname
+      table[extname] = file_type
+    end
+
+    def find(extname)
+      table[extname] || Katte::Plugins::FileType.find(extname)
+    end
+    def register_builtin_file_types
+      Dir[File.expand_path('../plugins/file_type/*.rb', __FILE__)].each {|p|
+        require p
+        name = File.basename(p, ".rb")
+        klass = "katte/plugins/file_type/#{name}".camelize.constantize
+      }
+      register(Katte::Plugins::FileType::Bash.new)
+      register(Katte::Plugins::FileType::Debug.new)
+      register(Katte::Plugins::FileType::Hive.new)
+      register(Katte::Plugins::FileType::R.new)
+      register(Katte::Plugins::FileType::Ruby.new)
+    end
+  end
+end

data/lib/katte/recipe/node.rb

@@ -0,0 +1,71 @@
+require 'katte/node/base'
+require 'katte/recipe/node_factory'
+require 'katte/thread_pool'
+
+module Katte::Recipe
+  class Node
+    include Katte::Node::Base
+
+    %w(name path file_type output period options).each {|attr|
+      attr_accessor attr
+    }
+
+    def initialize(params)
+      @name         = params[:name]
+      @path         = params[:path]
+      @requires     = params[:require]
+      @file_type    = params[:file_type]
+      @output       = params[:output]       || []
+      @period       = params[:period]       || 'day'
+      @options      = params[:options]      || {}
+
+      @parents      = []
+      @children     = []
+    end
+
+    def open
+      out_r, out_w = IO.pipe
+      err_r, err_w = IO.pipe
+
+      result = yield out_w, err_w
+
+      [out_w, err_w].each {|io| io.close unless io.closed? }
+      out_a, err_a = out_r.to_a.join, err_r.to_a.join
+      [out_r, err_r].each {|io| io.close unless io.closed? }
+
+      Katte.app.logger.warn("#{self.name} -- result is empty") if out_a.empty?
+
+      @output.reduce(out_a) {|stream, o| o.out(self, stream) }
+      @output.reduce(err_a) {|stream, o| o.err(self, stream) }
+
+      result
+    ensure
+      [out_r, out_w, err_r, err_w].each {|io| io.close unless io.closed? }
+    end
+
+    def run(driver)
+      unless @file_type
+        Katte.app.logger.error("no file type specified for %s" % @name)
+        return
+      end
+
+      Katte::ThreadPool.instance.push {
+        Katte.app.logger.info("[start] #{self.name}")
+        result = file_type.execute(self)
+        Katte.app.logger.info(sprintf("[%s] #{self.name}", (result ? "success" : "fail")))
+
+        result ? driver.done(self) : driver.fail(self)
+      }
+    end
+    def descendants
+      Enumerator.new {|enum| _descendants(enum) }
+    end
+    def _descendants(enum)
+      children.each {|node|
+        enum << node
+        node._descendants(enum)
+      }
+      enum
+    end
+  end
+end