katte 0.0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. data/.gitignore +17 -0
  2. data/.ruby-version +1 -0
  3. data/Gemfile +9 -0
  4. data/LICENSE.txt +22 -0
  5. data/README.md +107 -0
  6. data/Rakefile +1 -0
  7. data/bin/katte +7 -0
  8. data/katte.gemspec +19 -0
  9. data/lib/katte.rb +50 -0
  10. data/lib/katte/command.rb +33 -0
  11. data/lib/katte/config.rb +21 -0
  12. data/lib/katte/driver.rb +71 -0
  13. data/lib/katte/environment.rb +14 -0
  14. data/lib/katte/filter.rb +22 -0
  15. data/lib/katte/node.rb +5 -0
  16. data/lib/katte/node/base.rb +48 -0
  17. data/lib/katte/node/collection.rb +47 -0
  18. data/lib/katte/plugins.rb +11 -0
  19. data/lib/katte/plugins/base.rb +38 -0
  20. data/lib/katte/plugins/file_type.rb +47 -0
  21. data/lib/katte/plugins/file_type/bash.rb +9 -0
  22. data/lib/katte/plugins/file_type/debug.rb +14 -0
  23. data/lib/katte/plugins/file_type/hive.rb +9 -0
  24. data/lib/katte/plugins/file_type/r.rb +9 -0
  25. data/lib/katte/plugins/file_type/ruby.rb +9 -0
  26. data/lib/katte/plugins/node.rb +6 -0
  27. data/lib/katte/plugins/node/debug.rb +11 -0
  28. data/lib/katte/plugins/node/file.rb +45 -0
  29. data/lib/katte/plugins/output.rb +13 -0
  30. data/lib/katte/plugins/output/debug.rb +14 -0
  31. data/lib/katte/plugins/output/file.rb +31 -0
  32. data/lib/katte/plugins/output/stderr.rb +9 -0
  33. data/lib/katte/plugins/output/stdoe.rb +14 -0
  34. data/lib/katte/recipe.rb +8 -0
  35. data/lib/katte/recipe/file_type.rb +29 -0
  36. data/lib/katte/recipe/node.rb +71 -0
  37. data/lib/katte/recipe/node_factory.rb +57 -0
  38. data/lib/katte/runner.rb +66 -0
  39. data/lib/katte/runner/callback.rb +36 -0
  40. data/lib/katte/thread_pool.rb +41 -0
  41. data/lib/katte/version.rb +3 -0
  42. data/spec/katte/config_spec.rb +16 -0
  43. data/spec/katte/driver_spec.rb +58 -0
  44. data/spec/katte/filter_spec.rb +19 -0
  45. data/spec/katte/node/base_spec.rb +20 -0
  46. data/spec/katte/node/collection_spec.rb +36 -0
  47. data/spec/katte/plugins/file_type_spec.rb +59 -0
  48. data/spec/katte/plugins/output_spec.rb +31 -0
  49. data/spec/katte/recipe/node_factory_spec.rb +20 -0
  50. data/spec/katte/runner_spec.rb +34 -0
  51. data/spec/katte/thread_pool_spec.rb +47 -0
  52. data/spec/recipes/custom/sample_1.sh +3 -0
  53. data/spec/recipes/custom/sample_2.sh +3 -0
  54. data/spec/recipes/test/sample.sh +5 -0
  55. data/spec/recipes/test/sample/sub.sh +3 -0
  56. data/spec/spec_helper.rb +7 -0
  57. metadata +117 -0
@@ -0,0 +1,57 @@
1
+ require 'pathname'
2
+
3
+ module Katte::Recipe
4
+ class NodeFactory
5
+ @@after_create_hook = nil
6
+ def self.after_create(&proc)
7
+ @@after_create_hook = proc
8
+ end
9
+
10
+ def initialize
11
+ pattern_regexp = File.join(Katte.app.config.recipes_root, '(?<name>.+?)\.(?<ext>\w+)')
12
+ @path_pattern = /^#{pattern_regexp}$/
13
+
14
+ @nodes = {}
15
+ end
16
+
17
+ def load(path)
18
+ return unless FileTest.file? path
19
+ return unless m = @path_pattern.match(path)
20
+
21
+ name, ext = m[:name], m[:ext]
22
+
23
+ file_type = Katte::Recipe::FileType.find(ext)
24
+
25
+ directive = file_type.parse(path)
26
+
27
+ requires = directive['require']
28
+ output = directive['output'].map {|o| Katte::Plugins.output[o.first.to_sym]}
29
+ period = (directive['period'].empty? ? 'day' : directive['period'].last)
30
+ options = Hash[directive['option'].map {|k, v| [k, v || true]}]
31
+
32
+ params = {
33
+ :name => name,
34
+ :path => path,
35
+ :require => requires,
36
+ :file_type => file_type,
37
+ :output => output,
38
+ :period => period,
39
+ :options => options,
40
+ }
41
+
42
+ if Katte.app.config.mode == 'test'
43
+ params[:output] = [Katte::Plugins::Output.find(:debug)]
44
+ end
45
+
46
+ create(params)
47
+ end
48
+
49
+ def create(params)
50
+ node = Katte::Recipe::Node.new params
51
+
52
+ @@after_create_hook.call(node, params) if @@after_create_hook
53
+
54
+ node
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,66 @@
1
+ class Katte
2
+ class Runner
3
+ require 'katte/runner/callback'
4
+ include Katte::Runner::Callback
5
+
6
+ def run
7
+ load_nodes
8
+
9
+ connect_nodes
10
+
11
+ execute
12
+
13
+ print_summary
14
+ end
15
+
16
+ def initialize
17
+ end
18
+
19
+ def load_nodes
20
+ @nodes = Katte::Node::Collection.new
21
+ (builtin_nodes + recipe_nodes).each {|node| @nodes << node }
22
+
23
+ call_after_callbacks(:load_nodes, @nodes)
24
+ end
25
+ def builtin_nodes
26
+ Plugins::Node.plugins.values
27
+ end
28
+ def recipe_nodes
29
+ node_factory = Katte.app.config.factory || Katte::Recipe::NodeFactory.new
30
+ Find.find(Katte.app.config.recipes_root).select {|file|
31
+ File.file? file
32
+ }.map {|file|
33
+ node_factory.load(file)
34
+ }
35
+ end
36
+
37
+ def connect_nodes
38
+ @nodes.connect
39
+ end
40
+
41
+ def execute
42
+ @summary = Driver.run(@nodes)
43
+ end
44
+
45
+ def print_summary
46
+ return if Katte.app.config.mode == 'test'
47
+
48
+ summray_log_file = File.join(Katte.app.config.log_root, 'summary.log')
49
+ File.open(summray_log_file, 'w') do |file|
50
+ file.print <<-EOF
51
+ Summary:
52
+ success: #{@summary[:success].length}
53
+ fail: #{@summary[:fail].length}
54
+ skip: #{@summary[:skip].length}
55
+ EOF
56
+ end
57
+
58
+ failed_log_file = File.join(Katte.app.config.log_root, 'failed.log')
59
+ File.open(failed_log_file, 'w') do |file|
60
+ @summary[:fail].each do |node|
61
+ file.puts node.name
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,36 @@
1
+ module Katte::Runner::Callback
2
+ module ClassMethods
3
+ def before(task_name, &proc)
4
+ before_callbacks[task_name] ||= []
5
+ before_callbacks[task_name] << proc
6
+ end
7
+ def after(task_name, &proc)
8
+ after_callbacks[task_name] ||= []
9
+ after_callbacks[task_name] << proc
10
+ end
11
+
12
+ def after_callbacks
13
+ @after_callbacks ||= {}
14
+ end
15
+ def before_callbacks
16
+ @before_callbacks ||= {}
17
+ end
18
+ end
19
+
20
+ def self.included(klass)
21
+ klass.extend ClassMethods
22
+ end
23
+
24
+ def call_before_callbacks(task_name, *params)
25
+ return unless self.class.before_callbacks[task_name]
26
+ self.class.before_callbacks[task_name].each do |proc|
27
+ proc.call(*params)
28
+ end
29
+ end
30
+ def call_after_callbacks(task_name, *params)
31
+ return unless self.class.after_callbacks[task_name]
32
+ self.class.after_callbacks[task_name].each do |proc|
33
+ proc.call(*params)
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,41 @@
1
+ require 'thread'
2
+
3
+ class Katte
4
+ class ThreadPool
5
+ def self.instance
6
+ @instance ||= new.tap(&:run)
7
+ end
8
+
9
+ attr_reader :threads
10
+ def initialize(threads_num = 4, logger = Katte.app.logger)
11
+ @queue = Queue.new
12
+ @threads_num = threads_num
13
+ @logger = logger
14
+ end
15
+
16
+ def run
17
+ procedure = Proc.new {
18
+ loop {
19
+ begin
20
+ @queue.pop.call
21
+ rescue => e
22
+ @logger.error(e)
23
+ end
24
+ }
25
+ }
26
+ @threads ||= @threads_num.times.map { Thread.start &procedure }
27
+ end
28
+
29
+ def push &procedure
30
+ @queue.push procedure
31
+ end
32
+
33
+ def stop
34
+ @threads.each &:kill
35
+ end
36
+
37
+ def join
38
+ @threads.each {|t| t.join}
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,3 @@
1
+ class Katte
2
+ VERSION = "0.0.1.3"
3
+ end
@@ -0,0 +1,16 @@
1
+ require 'spec_helper'
2
+
3
+ class Katte
4
+ describe Config do
5
+ it "set params to Katte::Config" do
6
+ config_bak = Katte::Config.config.recipes_root
7
+
8
+ config = Class.new(Katte::Config) do |klass|
9
+ klass.config.recipes_root = 'dummy'
10
+ end
11
+
12
+ expect(Katte::Config.config.recipes_root).to eq 'dummy'
13
+ Katte::Config.config.recipes_root = config_bak
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,58 @@
1
+ require 'spec_helper'
2
+
3
+ class Katte
4
+ describe Driver do
5
+ it 'excecute each node according to dependency graph' do
6
+ call_log = []
7
+ callback = Proc.new {|node| call_log << node.name}
8
+
9
+ debug_plugin = Katte::Plugins::FileType::Debug.new
10
+
11
+ factory = Katte::Recipe::NodeFactory.new
12
+ node_collection = Katte::Node::Collection.new
13
+ node_collection << factory.create(:name => 'test_1',
14
+ :file_type => debug_plugin,
15
+ :options => {'callback' => [callback]})
16
+ node_collection << factory.create(:name => 'test_2',
17
+ :file_type => debug_plugin,
18
+ :require => ['test_1'],
19
+ :options => {'callback' => [callback]})
20
+ node_collection << factory.create(:name => 'test_3',
21
+ :file_type => debug_plugin,
22
+ :require => ['test_2'],
23
+ :options => {'callback' => [callback]})
24
+
25
+ node_collection.connect
26
+
27
+ Driver.run(node_collection)
28
+
29
+ expect(call_log).to eq %w(test_1 test_2 test_3)
30
+ end
31
+
32
+ it 'skip nodes when parent node failed' do
33
+ call_log = []
34
+ callback = Proc.new {|node| call_log << node.name }
35
+ failure_callback = Proc.new{|node| raise Katte::Plugins::FileType::Debug::Abort}
36
+ debug_plugin = Katte::Plugins::FileType::Debug.new
37
+
38
+ factory = Katte::Recipe::NodeFactory.new
39
+ node_collection = Katte::Node::Collection.new
40
+ node_collection << factory.create(:name => 'test_1',
41
+ :file_type => debug_plugin,
42
+ :options => {'callback' => [failure_callback]})
43
+ node_collection << factory.create(:name => 'test_2',
44
+ :file_type => debug_plugin,
45
+ :require => ['test_1'],
46
+ :options => {'callback' => [callback]})
47
+ node_collection << factory.create(:name => 'test_3',
48
+ :file_type => debug_plugin,
49
+ :options => {'callback' => [callback]})
50
+
51
+ node_collection.connect
52
+
53
+ Driver.run(node_collection)
54
+
55
+ expect(call_log).to eq ["test_3"]
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,19 @@
1
+ require 'spec_helper'
2
+
3
+ class Katte
4
+ describe Filter do
5
+ describe ".call" do
6
+ it "validate date" do
7
+ filter = Filter.new(datetime: DateTime.parse('2013-01-01'))
8
+ expect(filter.call(period: 'day')).to be_true
9
+ expect(filter.call(period: 'week')).to be_false
10
+ expect(filter.call(period: 'month')).to be_true
11
+
12
+ filter = Filter.new(datetime: DateTime.parse('2013-01-07'))
13
+ expect(filter.call(period: 'day')).to be_true
14
+ expect(filter.call(period: 'week')).to be_true
15
+ expect(filter.call(period: 'month')).to be_false
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec_helper'
2
+ module Katte::Recipe
3
+ describe Node do
4
+ describe "#descendants" do
5
+ it "collect all descendants" do
6
+ factory = Katte::Recipe::NodeFactory.new
7
+ node_collection = Katte::Node::Collection.new
8
+ node_collection << factory.create(name: "a", :require => ["b"])
9
+ node_collection << factory.create(name: "b", :require => ["c"])
10
+ node_collection << factory.create(name: "c")
11
+
12
+ node_collection.connect
13
+
14
+ node_c = node_collection.find("c")
15
+
16
+ expect(node_c.descendants.map(&:name)).to eq ["b", "a"]
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,36 @@
1
+ require 'spec_helper'
2
+
3
+ module Katte::Node
4
+ describe Collection do
5
+ before :all do
6
+ node_class = Class.new do
7
+ include Katte::Node::Base
8
+
9
+ def initialize(params)
10
+ @name = params[:name]
11
+ @requires = params[:require]
12
+ end
13
+ def name; @name; end
14
+ end
15
+
16
+ @node_collection = Katte::Node::Collection.new
17
+ @node_collection << node_class.new(name: "a")
18
+ @node_collection << node_class.new(name: "b")
19
+ @node_collection << node_class.new(name: "c", require: ["b", "e"])
20
+ @node_collection << node_class.new(name: "d", require: ["b"])
21
+
22
+ @node_collection.connect
23
+ end
24
+
25
+ describe '#connect' do
26
+ it "ignores unregistered nodes" do
27
+ expect(@node_collection.find("c").parents.map(&:name)).not_to include "e"
28
+ expect(@node_collection.find("c").parents).not_to include "e"
29
+ end
30
+ it "connect parents and childs" do
31
+ expect(@node_collection.find("b").children.map(&:name)).to eq %w(c d)
32
+ expect(@node_collection.find("c").parents.map(&:name)).to eq %w(b)
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,59 @@
1
+ require 'spec_helper'
2
+ require 'tempfile'
3
+
4
+ module Katte::Plugins
5
+ describe FileType do
6
+ describe "#simple_exec" do
7
+ before :all do
8
+ @file_type_class = Class.new {|klass|
9
+ include FileType
10
+ }
11
+ end
12
+
13
+ before(:each) { Katte::Plugins::Output.find(:debug).history.clear }
14
+ it "execute shell script" do
15
+ node = Katte::Recipe::Node.new(:name => 'test/sample',
16
+ :path => File.expand_path('../../../recipes/test/sample.sh', __FILE__),
17
+ :output => [Katte::Plugins::Output.find(:debug)])
18
+
19
+ file_type = @file_type_class.new
20
+ file_type.simple_exec(node, 'bash', node.path)
21
+
22
+ output = Katte::Plugins::Output.find(:debug).history.pop
23
+ result = output[:out]
24
+
25
+ expect(result).to eq "0\n"
26
+ end
27
+ end
28
+
29
+ describe '#parse' do
30
+ before :all do
31
+ @recipe_path = Tempfile.open('sample_recipe') do |f|
32
+ f.print <<-EOF
33
+ # require: path/recipe(xxx)
34
+ # option: conf1(p1, p2)
35
+ # option: conf2()
36
+ # option: conf3
37
+
38
+ echo hello
39
+ EOF
40
+ f.path
41
+ end
42
+ end
43
+ after(:all) { File.delete @recipe_path if File.exists? @recipe_path}
44
+
45
+ it "parse recipe file" do
46
+ file_type = Katte::Plugins::FileType::Bash.new
47
+ directive = file_type.parse(@recipe_path)
48
+
49
+ expect(directive).to have_key 'require'
50
+ expect(directive).to have_key 'option'
51
+
52
+ expect(directive['require']).to eq [['path/recipe', ['xxx']]]
53
+ expect(directive['option']).to eq [['conf1', ['p1', 'p2']],
54
+ ['conf2', []],
55
+ ['conf3', nil]]
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,31 @@
1
+ require 'spec_helper'
2
+
3
+ module Katte::Plugins
4
+ describe Output do
5
+ before :all do
6
+ klasses = 2.times.map {|i|
7
+ Class.new {|klass|
8
+ include Katte::Plugins::Output
9
+ name :"test_#{i}"
10
+ attr_reader :result
11
+ def out(node, data)
12
+ (@result ||= []) << data
13
+ data
14
+ end
15
+ }
16
+ }
17
+ @plugins = klasses.map &:new
18
+ end
19
+
20
+ it "puts to all output plugins" do
21
+ node = Katte::Recipe::Node.new(output: @plugins)
22
+ node.open {|out, err|
23
+ out.puts "a"
24
+ }
25
+
26
+ @plugins.all? {|plugin|
27
+ expect(plugin.result).to eq ["a\n"]
28
+ }
29
+ end
30
+ end
31
+ end