katte 0.0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +17 -0
- data/.ruby-version +1 -0
- data/Gemfile +9 -0
- data/LICENSE.txt +22 -0
- data/README.md +107 -0
- data/Rakefile +1 -0
- data/bin/katte +7 -0
- data/katte.gemspec +19 -0
- data/lib/katte.rb +50 -0
- data/lib/katte/command.rb +33 -0
- data/lib/katte/config.rb +21 -0
- data/lib/katte/driver.rb +71 -0
- data/lib/katte/environment.rb +14 -0
- data/lib/katte/filter.rb +22 -0
- data/lib/katte/node.rb +5 -0
- data/lib/katte/node/base.rb +48 -0
- data/lib/katte/node/collection.rb +47 -0
- data/lib/katte/plugins.rb +11 -0
- data/lib/katte/plugins/base.rb +38 -0
- data/lib/katte/plugins/file_type.rb +47 -0
- data/lib/katte/plugins/file_type/bash.rb +9 -0
- data/lib/katte/plugins/file_type/debug.rb +14 -0
- data/lib/katte/plugins/file_type/hive.rb +9 -0
- data/lib/katte/plugins/file_type/r.rb +9 -0
- data/lib/katte/plugins/file_type/ruby.rb +9 -0
- data/lib/katte/plugins/node.rb +6 -0
- data/lib/katte/plugins/node/debug.rb +11 -0
- data/lib/katte/plugins/node/file.rb +45 -0
- data/lib/katte/plugins/output.rb +13 -0
- data/lib/katte/plugins/output/debug.rb +14 -0
- data/lib/katte/plugins/output/file.rb +31 -0
- data/lib/katte/plugins/output/stderr.rb +9 -0
- data/lib/katte/plugins/output/stdoe.rb +14 -0
- data/lib/katte/recipe.rb +8 -0
- data/lib/katte/recipe/file_type.rb +29 -0
- data/lib/katte/recipe/node.rb +71 -0
- data/lib/katte/recipe/node_factory.rb +57 -0
- data/lib/katte/runner.rb +66 -0
- data/lib/katte/runner/callback.rb +36 -0
- data/lib/katte/thread_pool.rb +41 -0
- data/lib/katte/version.rb +3 -0
- data/spec/katte/config_spec.rb +16 -0
- data/spec/katte/driver_spec.rb +58 -0
- data/spec/katte/filter_spec.rb +19 -0
- data/spec/katte/node/base_spec.rb +20 -0
- data/spec/katte/node/collection_spec.rb +36 -0
- data/spec/katte/plugins/file_type_spec.rb +59 -0
- data/spec/katte/plugins/output_spec.rb +31 -0
- data/spec/katte/recipe/node_factory_spec.rb +20 -0
- data/spec/katte/runner_spec.rb +34 -0
- data/spec/katte/thread_pool_spec.rb +47 -0
- data/spec/recipes/custom/sample_1.sh +3 -0
- data/spec/recipes/custom/sample_2.sh +3 -0
- data/spec/recipes/test/sample.sh +5 -0
- data/spec/recipes/test/sample/sub.sh +3 -0
- data/spec/spec_helper.rb +7 -0
- metadata +117 -0
data/.gitignore
ADDED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.9.3-p194
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Ojima Hikaru
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
# Katte
|
2
|
+
|
3
|
+
Katte は以下の特徴を備えたバッチ処理管理ツールです。
|
4
|
+
|
5
|
+
- 実行可能なタスク定義
|
6
|
+
- タスクの依存関係の解決
|
7
|
+
- 実行コマンドや出力コマンドを拡張可能
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Add this line to your application's Gemfile:
|
12
|
+
|
13
|
+
gem 'katte'
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install katte
|
22
|
+
|
23
|
+
## Turtorial
|
24
|
+
|
25
|
+
### はじめの一歩
|
26
|
+
|
27
|
+
`recipes/` 以下にタスクの定義を記述します。
|
28
|
+
|
29
|
+
$ vim recipes/first_task.sh
|
30
|
+
|
31
|
+
echo ${date}
|
32
|
+
|
33
|
+
`$ bin/katte run -d yyyy-mm-dd` を実行すると、`result/first_task.log` 以下
|
34
|
+
に結果が出力されます。
|
35
|
+
|
36
|
+
拡張子から実行コマンドを判別します。
|
37
|
+
実行コマンドは、bash(.sh) / ruby(.rb) / Rscript(.R) / hive(.sql) をデ
|
38
|
+
フォルトでサポートしています。
|
39
|
+
|
40
|
+
### 実行オプションを記述する
|
41
|
+
|
42
|
+
レシピの先頭に実行オプションを記述できます。
|
43
|
+
|
44
|
+
# require: second_task
|
45
|
+
# output : file, stdio
|
46
|
+
# period : week
|
47
|
+
|
48
|
+
echo good day
|
49
|
+
|
50
|
+
オプションの意味は以下の通りです。
|
51
|
+
|
52
|
+
<dl>
|
53
|
+
<dt>`require`</dt><dd>指定したタスクの完了を待って実行します。タスクは
|
54
|
+
`recipes/` からの相対パスで指定します。</dd>
|
55
|
+
<dt>`output`</dt><dd>出力方法を指定します。デフォルトで `file` と
|
56
|
+
`stdio` が指定可能です。</dd>
|
57
|
+
<dt>`period`</dt><dd>実行周期を指定します。`day` / `week` / `month`
|
58
|
+
が指定可能です。</dd>
|
59
|
+
</dl>
|
60
|
+
|
61
|
+
### 実行コマンドを拡張する
|
62
|
+
|
63
|
+
`Katte::Plugins::FileType` を継承したクラスをロードすることで、実行コ
|
64
|
+
マンドを拡張できます。
|
65
|
+
|
66
|
+
`Katte::Plugins::Output` を継承したクラスをロードすることで、出力方法を
|
67
|
+
拡張できます。
|
68
|
+
|
69
|
+
記述例は `lib/katte/plugins/file_type/bash.rb` /
|
70
|
+
`lib/katte/plugins/output/file.rb` を参考にしてください。
|
71
|
+
|
72
|
+
### カスタムレシピを記述する
|
73
|
+
|
74
|
+
rubyレシピに `custom` オプションを指定することで、タスクの実行タイミン
|
75
|
+
グを柔軟に制御することができます。
|
76
|
+
|
77
|
+
$ vim custom_recipe.rb
|
78
|
+
|
79
|
+
# option: custom
|
80
|
+
loop {
|
81
|
+
if FileTest.file?('/some/important/file')
|
82
|
+
tag 'important'
|
83
|
+
break
|
84
|
+
elsif FileTest.file?('/some/critical/file')
|
85
|
+
tag 'critical'
|
86
|
+
break
|
87
|
+
end
|
88
|
+
sleep 1000
|
89
|
+
}
|
90
|
+
|
91
|
+
done
|
92
|
+
|
93
|
+
$ vim custom_recipe/sub_task.sh
|
94
|
+
|
95
|
+
# require: custom_recipe:important
|
96
|
+
cat /some/important/file
|
97
|
+
|
98
|
+
`tag <tag>` で次のタスクの実行を開始します。
|
99
|
+
`done` でこのタスクの実行を終了します。
|
100
|
+
|
101
|
+
## Contributing
|
102
|
+
|
103
|
+
1. Fork it
|
104
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
105
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
106
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
107
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/bin/katte
ADDED
data/katte.gemspec
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'katte/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = "katte"
|
8
|
+
gem.version = Katte::VERSION
|
9
|
+
gem.authors = ["Ojima Hikaru"]
|
10
|
+
gem.email = ["ojiam.h@gmail.com"]
|
11
|
+
gem.description = %q{Batch job management tool.}
|
12
|
+
gem.summary = %q{Batch job management tool.}
|
13
|
+
gem.homepage = "https://github.com/ojima-h/katte"
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split($/)
|
16
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
|
+
gem.require_paths = ["lib"]
|
19
|
+
end
|
data/lib/katte.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'find'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
class Katte
|
5
|
+
def self.new(params = {})
|
6
|
+
@instance ||= super(params)
|
7
|
+
end
|
8
|
+
def self.app(params = {})
|
9
|
+
@instance ||= new(params)
|
10
|
+
end
|
11
|
+
|
12
|
+
attr_reader :env
|
13
|
+
attr_reader :config
|
14
|
+
attr_reader :logger
|
15
|
+
attr_reader :options
|
16
|
+
|
17
|
+
def initialize(options = {})
|
18
|
+
@env = Environment.new(options)
|
19
|
+
@options = options
|
20
|
+
@config = Katte::Config.config
|
21
|
+
|
22
|
+
@logger = if STDOUT.tty?
|
23
|
+
Logger.new(STDOUT)
|
24
|
+
else
|
25
|
+
Logger.new(File.join(@config.log_root, 'katte.log'), 'daily')
|
26
|
+
end
|
27
|
+
@logger.level = Logger::WARN if config.mode == 'test'
|
28
|
+
end
|
29
|
+
|
30
|
+
def run
|
31
|
+
(@runner ||= Katte::Runner.new).run
|
32
|
+
end
|
33
|
+
|
34
|
+
def exec(recipe_path)
|
35
|
+
node_factory = config.factory || Katte::Recipe::NodeFactory.new
|
36
|
+
node = node_factory.load(recipe_path)
|
37
|
+
|
38
|
+
node.file_type.execute(node)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
require 'katte/node'
|
43
|
+
require "katte/version"
|
44
|
+
require 'katte/environment'
|
45
|
+
require 'katte/config'
|
46
|
+
require 'katte/plugins'
|
47
|
+
require 'katte/filter'
|
48
|
+
require 'katte/driver'
|
49
|
+
require 'katte/recipe'
|
50
|
+
require 'katte/runner'
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'katte'
|
2
|
+
require 'date'
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
class Katte
|
6
|
+
class Command
|
7
|
+
opt = OptionParser.new
|
8
|
+
opt.on('-d date') {|v| options[:datetime] = v }
|
9
|
+
opt.on('-v') { options[:verbose] = true }
|
10
|
+
opt.on('-h') { print_help_message; exit }
|
11
|
+
opt.banner = "katte [options] [files]"
|
12
|
+
opt.parse!(ARGV)
|
13
|
+
|
14
|
+
files = ARGV.select(&FileTest.method(:file?))
|
15
|
+
.map(&File.method(:absolute_path))
|
16
|
+
options = {
|
17
|
+
:date => (Date.today - 1).strftime("%Y-%m-%d"),
|
18
|
+
:files => files,
|
19
|
+
}
|
20
|
+
|
21
|
+
app = Katte.new(options)
|
22
|
+
|
23
|
+
if files.empty?
|
24
|
+
app.run
|
25
|
+
else
|
26
|
+
files.each {|file| app.exec file }
|
27
|
+
end
|
28
|
+
|
29
|
+
def print_help_message
|
30
|
+
print opt.help
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/katte/config.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
class Katte
|
4
|
+
class Config
|
5
|
+
config_klass = Struct.new(:mode,
|
6
|
+
:recipes_root,
|
7
|
+
:result_root,
|
8
|
+
:log_root,
|
9
|
+
:factory)
|
10
|
+
@config = config_klass.new
|
11
|
+
|
12
|
+
@config.mode = ENV['KATTE_MODE'] || 'production'
|
13
|
+
@config.recipes_root = File.join(APP_PATH, 'recipes')
|
14
|
+
@config.result_root = File.join(APP_PATH, 'result')
|
15
|
+
@config.log_root = File.join(APP_PATH, 'log')
|
16
|
+
|
17
|
+
def self.config
|
18
|
+
self == Katte::Config ? @config : Katte::Config.config
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/katte/driver.rb
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
class Katte
|
2
|
+
class Driver
|
3
|
+
def self.run(nodes)
|
4
|
+
core = Core.new(nodes)
|
5
|
+
core.run
|
6
|
+
core.summary
|
7
|
+
end
|
8
|
+
|
9
|
+
class Core
|
10
|
+
attr_reader :summary
|
11
|
+
def initialize(nodes, options = {})
|
12
|
+
@nodes = nodes
|
13
|
+
@nodes_list = Hash[nodes.map {|node| [node.name, node.parents.length] }]
|
14
|
+
@nodes_index = Hash[nodes.map {|node| [node.name, node] }]
|
15
|
+
|
16
|
+
@queue = Queue.new
|
17
|
+
|
18
|
+
@summary = {:success => [], :fail => [], :skip => []}
|
19
|
+
end
|
20
|
+
|
21
|
+
%w(done fail next skip finish).each do |method|
|
22
|
+
define_method(method) do |*args|
|
23
|
+
@queue.push [:"_#{method}", *args]
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def run
|
28
|
+
return if @nodes_list.empty?
|
29
|
+
|
30
|
+
run_nodes(@nodes.select{|n| n.parents.length.zero? })
|
31
|
+
|
32
|
+
loop {
|
33
|
+
method, *args = @queue.pop
|
34
|
+
|
35
|
+
break if method == :_finish
|
36
|
+
|
37
|
+
self.send(method, *args)
|
38
|
+
}
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
def run_nodes(nodes)
|
43
|
+
nodes.each {|node| node.run(self) }
|
44
|
+
end
|
45
|
+
|
46
|
+
def _done(node, *args)
|
47
|
+
@nodes_list.delete(node.name)
|
48
|
+
return finish if @nodes_list.empty?
|
49
|
+
|
50
|
+
node.children.each do |child|
|
51
|
+
@nodes_list[child.name] -= 1
|
52
|
+
child.run(self) if @nodes_list[child.name].zero?
|
53
|
+
end
|
54
|
+
end
|
55
|
+
def _fail(node, *args)
|
56
|
+
@nodes_list.delete(node.name)
|
57
|
+
node.descendants.each {|dec| @nodes_list.delete(dec.name) }
|
58
|
+
|
59
|
+
finish if @nodes_list.empty?
|
60
|
+
end
|
61
|
+
def _next(node, child, *args)
|
62
|
+
@nodes_list[child] -= 1
|
63
|
+
@nodes_index[child].run(self) if @nodes_list[child].zero?
|
64
|
+
end
|
65
|
+
def _skip(node, *args)
|
66
|
+
node.descendants.each {|dec| @nodes_list.delete(dec) }
|
67
|
+
finish if @nodes_list.empty?
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
class Katte
|
2
|
+
class Environment
|
3
|
+
attr_accessor :datetime
|
4
|
+
def initialize(options = {})
|
5
|
+
@datetime = options[:datetime] ? DateTime.parse(options[:datetime]) : DateTime.now
|
6
|
+
end
|
7
|
+
|
8
|
+
def to_hash
|
9
|
+
@hash ||= {
|
10
|
+
'date' => @datetime.strftime('%Y-%m-%d'),
|
11
|
+
}
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/katte/filter.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'date'
|
2
|
+
|
3
|
+
class Katte
|
4
|
+
class Filter
|
5
|
+
def initialize(options = {})
|
6
|
+
@datetime = options[:datetime] || DateTime.now
|
7
|
+
end
|
8
|
+
|
9
|
+
def call(params)
|
10
|
+
check_date(params)
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
def check_date(params)
|
15
|
+
case params[:period]
|
16
|
+
when 'day' then true
|
17
|
+
when 'week' then @datetime.wday == 1 # monday
|
18
|
+
when 'month' then @datetime.day == 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/katte/node.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
module Katte::Node
|
2
|
+
module Base
|
3
|
+
class AbstractNodeException < Exception; end
|
4
|
+
|
5
|
+
def name
|
6
|
+
raise AbstractNodeException, "method #name must be implemented"
|
7
|
+
end
|
8
|
+
def requires
|
9
|
+
@requires ||= []
|
10
|
+
end
|
11
|
+
|
12
|
+
# execution filter
|
13
|
+
# return whether this node to run
|
14
|
+
def run?
|
15
|
+
false
|
16
|
+
end
|
17
|
+
def run(driver)
|
18
|
+
driver.skip(self)
|
19
|
+
end
|
20
|
+
|
21
|
+
# accessor for children
|
22
|
+
def children
|
23
|
+
@children ||= []
|
24
|
+
end
|
25
|
+
def add_child(node, *params)
|
26
|
+
children << node
|
27
|
+
end
|
28
|
+
|
29
|
+
# accessor for parents
|
30
|
+
def parents
|
31
|
+
@parents ||= []
|
32
|
+
end
|
33
|
+
def add_parent(node, *params)
|
34
|
+
parents << node
|
35
|
+
end
|
36
|
+
|
37
|
+
def descendants
|
38
|
+
Enumerator.new {|enum| _descendants(enum) }
|
39
|
+
end
|
40
|
+
def _descendants(enum)
|
41
|
+
children.each {|node|
|
42
|
+
enum << node
|
43
|
+
node._descendants(enum)
|
44
|
+
}
|
45
|
+
enum
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|