cawcaw 0.0.3 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +11 -7
- data/{README.rdoc → README.md} +15 -11
- data/Rakefile +9 -0
- data/VERSION +1 -1
- data/bin/cawcaw +2 -0
- data/cawcaw.gemspec +37 -22
- data/lib/cawcaw/command/postgresql/table.rb +133 -0
- data/lib/cawcaw/command/postgresql.rb +17 -0
- data/lib/cawcaw/core/common.rb +22 -46
- data/lib/cawcaw.rb +4 -1
- data/spec/bin/cawcaw/hadoop/dfs_spec.rb +6 -6
- data/spec/bin/cawcaw/postgresql/table_spec.rb +30 -0
- data/spec/bin/cawcaw/rabbitmq/queue_count_spec.rb +2 -2
- data/spec/spec_helper.rb +5 -3
- metadata +94 -27
data/Gemfile
CHANGED
@@ -3,21 +3,25 @@ source "http://rubygems.org"
|
|
3
3
|
# Example:
|
4
4
|
# gem "activesupport", ">= 2.3.5"
|
5
5
|
|
6
|
-
gem "
|
6
|
+
gem "activerecord", "~> 3.2.11"
|
7
|
+
gem "activerecord-postgresql-adapter", "~> 0.0.1"
|
8
|
+
gem "bunnish", "~> 0.1.2"
|
7
9
|
|
8
10
|
# Add dependencies to develop your gem here.
|
9
11
|
# Include everything needed to run rake, tests, features, etc.
|
10
12
|
group :development do
|
11
|
-
gem "rspec", "
|
13
|
+
gem "rspec", "~> 2.12.0"
|
14
|
+
gem "yard", "~> 0.8.3"
|
15
|
+
gem "redcarpet", "~> 2.2.2"
|
12
16
|
gem "rdoc", "~> 3.12"
|
13
|
-
gem "bundler", "
|
17
|
+
gem "bundler", "~> 1.2.3"
|
14
18
|
gem "jeweler", "~> 1.8.4"
|
15
19
|
if RUBY_VERSION <= '1.8.7'
|
16
|
-
gem "rcov", "
|
20
|
+
gem "rcov", "~> 1.0.0"
|
17
21
|
else
|
18
|
-
gem "simplecov", "
|
19
|
-
gem "simplecov-rcov", "
|
22
|
+
gem "simplecov", "~> 0.7.1"
|
23
|
+
gem "simplecov-rcov", "~> 0.2.3"
|
20
24
|
end
|
21
|
-
gem "ci_reporter", "
|
25
|
+
gem "ci_reporter", "~> 1.8.3"
|
22
26
|
gem "flog", ">= 3.2.0"
|
23
27
|
end
|
data/{README.rdoc → README.md}
RENAMED
@@ -1,28 +1,32 @@
|
|
1
|
-
|
1
|
+
# Cawcaw
|
2
2
|
|
3
3
|
Cawcaw supports munin plugin for Hadoop/RabbitMQ.
|
4
4
|
|
5
|
-
|
5
|
+
## Supported Ruby versions and implementations
|
6
6
|
Cawcaw should work identically on:
|
7
7
|
|
8
8
|
* Ruby 1.9.3
|
9
9
|
* Ruby 1.9.2
|
10
10
|
* Ruby 1.8.7
|
11
11
|
|
12
|
-
|
12
|
+
## Install
|
13
13
|
|
14
14
|
You can install cawcaw by gem.
|
15
|
-
gem install cawcaw
|
16
15
|
|
17
|
-
|
16
|
+
gem install cawcaw
|
18
17
|
|
19
|
-
|
20
|
-
$ cawcaw rabbitmq queue-count <queue-name>
|
18
|
+
## Usage
|
21
19
|
|
22
|
-
|
23
|
-
|
20
|
+
### Draw munin graph of Hadoop HDFS size
|
21
|
+
cawcaw hadoop dfs <hdfs-path[,hdfs-path...]> [autoconf|config]
|
24
22
|
|
25
|
-
|
23
|
+
### Draw munin graph of PostgreSQL table record size
|
24
|
+
cawcaw postgresql table <table-name[,table-name...]> [autoconf|config]
|
25
|
+
|
26
|
+
### Draw munin graph of RabbitMQ queue message count
|
27
|
+
cawcaw rabbitmq queue-count <queue-name[,queue-name...]> [autoconf|config]
|
28
|
+
|
29
|
+
## Contributing to cawcaw
|
26
30
|
|
27
31
|
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
|
28
32
|
* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
|
@@ -32,7 +36,7 @@ You can install cawcaw by gem.
|
|
32
36
|
* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
|
33
37
|
* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
|
34
38
|
|
35
|
-
|
39
|
+
## Copyright
|
36
40
|
|
37
41
|
Copyright (c) 2012 Kenji Hara. See LICENSE.txt for
|
38
42
|
further details.
|
data/Rakefile
CHANGED
@@ -48,6 +48,15 @@ Rake::RDocTask.new do |rdoc|
|
|
48
48
|
rdoc.rdoc_files.include("lib/**/*.rb")
|
49
49
|
end
|
50
50
|
|
51
|
+
## YARD
|
52
|
+
require "yard"
|
53
|
+
require "yard/rake/yardoc_task"
|
54
|
+
YARD::Rake::YardocTask.new do |t|
|
55
|
+
t.files = ["lib/**/*.rb"]
|
56
|
+
t.options = []
|
57
|
+
t.options << "--debug" << "--verbose" if $trace
|
58
|
+
end
|
59
|
+
|
51
60
|
require "ci/reporter/rake/rspec" # use this if you're using RSpec
|
52
61
|
|
53
62
|
if RUBY_VERSION <= '1.8.7'
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0
|
1
|
+
0.1.0
|
data/bin/cawcaw
CHANGED
data/cawcaw.gemspec
CHANGED
@@ -5,24 +5,24 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "cawcaw"
|
8
|
-
s.version = "0.0
|
8
|
+
s.version = "0.1.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Kenji Hara"]
|
12
|
-
s.date = "2013-01-
|
12
|
+
s.date = "2013-01-19"
|
13
13
|
s.description = "cawcaw RubyGem"
|
14
14
|
s.email = "haracane@gmail.com"
|
15
15
|
s.executables = ["cawcaw"]
|
16
16
|
s.extra_rdoc_files = [
|
17
17
|
"LICENSE.txt",
|
18
|
-
"README.
|
18
|
+
"README.md"
|
19
19
|
]
|
20
20
|
s.files = [
|
21
21
|
".document",
|
22
22
|
".rspec",
|
23
23
|
"Gemfile",
|
24
24
|
"LICENSE.txt",
|
25
|
-
"README.
|
25
|
+
"README.md",
|
26
26
|
"Rakefile",
|
27
27
|
"VERSION",
|
28
28
|
"bin/cawcaw",
|
@@ -30,10 +30,13 @@ Gem::Specification.new do |s|
|
|
30
30
|
"lib/cawcaw.rb",
|
31
31
|
"lib/cawcaw/command/hadoop.rb",
|
32
32
|
"lib/cawcaw/command/hadoop/dfs.rb",
|
33
|
+
"lib/cawcaw/command/postgresql.rb",
|
34
|
+
"lib/cawcaw/command/postgresql/table.rb",
|
33
35
|
"lib/cawcaw/command/rabbitmq.rb",
|
34
36
|
"lib/cawcaw/command/rabbitmq/queue_count.rb",
|
35
37
|
"lib/cawcaw/core/common.rb",
|
36
38
|
"spec/bin/cawcaw/hadoop/dfs_spec.rb",
|
39
|
+
"spec/bin/cawcaw/postgresql/table_spec.rb",
|
37
40
|
"spec/bin/cawcaw/rabbitmq/queue_count_spec.rb",
|
38
41
|
"spec/lib/cawcaw_spec.rb",
|
39
42
|
"spec/spec_helper.rb"
|
@@ -48,35 +51,47 @@ Gem::Specification.new do |s|
|
|
48
51
|
s.specification_version = 3
|
49
52
|
|
50
53
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
51
|
-
s.add_runtime_dependency(%q<
|
52
|
-
s.
|
54
|
+
s.add_runtime_dependency(%q<activerecord>, ["~> 3.2.11"])
|
55
|
+
s.add_runtime_dependency(%q<activerecord-postgresql-adapter>, ["~> 0.0.1"])
|
56
|
+
s.add_runtime_dependency(%q<bunnish>, ["~> 0.1.2"])
|
57
|
+
s.add_development_dependency(%q<rspec>, ["~> 2.12.0"])
|
58
|
+
s.add_development_dependency(%q<yard>, ["~> 0.8.3"])
|
59
|
+
s.add_development_dependency(%q<redcarpet>, ["~> 2.2.2"])
|
53
60
|
s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
|
54
|
-
s.add_development_dependency(%q<bundler>, ["
|
61
|
+
s.add_development_dependency(%q<bundler>, ["~> 1.2.3"])
|
55
62
|
s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
|
56
|
-
s.add_development_dependency(%q<simplecov>, ["
|
57
|
-
s.add_development_dependency(%q<simplecov-rcov>, ["
|
58
|
-
s.add_development_dependency(%q<ci_reporter>, ["
|
63
|
+
s.add_development_dependency(%q<simplecov>, ["~> 0.7.1"])
|
64
|
+
s.add_development_dependency(%q<simplecov-rcov>, ["~> 0.2.3"])
|
65
|
+
s.add_development_dependency(%q<ci_reporter>, ["~> 1.8.3"])
|
59
66
|
s.add_development_dependency(%q<flog>, [">= 3.2.0"])
|
60
67
|
else
|
61
|
-
s.add_dependency(%q<
|
62
|
-
s.add_dependency(%q<
|
68
|
+
s.add_dependency(%q<activerecord>, ["~> 3.2.11"])
|
69
|
+
s.add_dependency(%q<activerecord-postgresql-adapter>, ["~> 0.0.1"])
|
70
|
+
s.add_dependency(%q<bunnish>, ["~> 0.1.2"])
|
71
|
+
s.add_dependency(%q<rspec>, ["~> 2.12.0"])
|
72
|
+
s.add_dependency(%q<yard>, ["~> 0.8.3"])
|
73
|
+
s.add_dependency(%q<redcarpet>, ["~> 2.2.2"])
|
63
74
|
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
64
|
-
s.add_dependency(%q<bundler>, ["
|
75
|
+
s.add_dependency(%q<bundler>, ["~> 1.2.3"])
|
65
76
|
s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
|
66
|
-
s.add_dependency(%q<simplecov>, ["
|
67
|
-
s.add_dependency(%q<simplecov-rcov>, ["
|
68
|
-
s.add_dependency(%q<ci_reporter>, ["
|
77
|
+
s.add_dependency(%q<simplecov>, ["~> 0.7.1"])
|
78
|
+
s.add_dependency(%q<simplecov-rcov>, ["~> 0.2.3"])
|
79
|
+
s.add_dependency(%q<ci_reporter>, ["~> 1.8.3"])
|
69
80
|
s.add_dependency(%q<flog>, [">= 3.2.0"])
|
70
81
|
end
|
71
82
|
else
|
72
|
-
s.add_dependency(%q<
|
73
|
-
s.add_dependency(%q<
|
83
|
+
s.add_dependency(%q<activerecord>, ["~> 3.2.11"])
|
84
|
+
s.add_dependency(%q<activerecord-postgresql-adapter>, ["~> 0.0.1"])
|
85
|
+
s.add_dependency(%q<bunnish>, ["~> 0.1.2"])
|
86
|
+
s.add_dependency(%q<rspec>, ["~> 2.12.0"])
|
87
|
+
s.add_dependency(%q<yard>, ["~> 0.8.3"])
|
88
|
+
s.add_dependency(%q<redcarpet>, ["~> 2.2.2"])
|
74
89
|
s.add_dependency(%q<rdoc>, ["~> 3.12"])
|
75
|
-
s.add_dependency(%q<bundler>, ["
|
90
|
+
s.add_dependency(%q<bundler>, ["~> 1.2.3"])
|
76
91
|
s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
|
77
|
-
s.add_dependency(%q<simplecov>, ["
|
78
|
-
s.add_dependency(%q<simplecov-rcov>, ["
|
79
|
-
s.add_dependency(%q<ci_reporter>, ["
|
92
|
+
s.add_dependency(%q<simplecov>, ["~> 0.7.1"])
|
93
|
+
s.add_dependency(%q<simplecov-rcov>, ["~> 0.2.3"])
|
94
|
+
s.add_dependency(%q<ci_reporter>, ["~> 1.8.3"])
|
80
95
|
s.add_dependency(%q<flog>, [">= 3.2.0"])
|
81
96
|
end
|
82
97
|
end
|
@@ -0,0 +1,133 @@
|
|
1
|
+
module Cawcaw
|
2
|
+
module Command
|
3
|
+
module Postgresql
|
4
|
+
module Table
|
5
|
+
def self.run(argv, input_stream=$stdin, output_stream=$stdout)
|
6
|
+
|
7
|
+
params = Cawcaw.parse_opts(argv)
|
8
|
+
|
9
|
+
params[:graph_title] ||= "PostgreSQL Records"
|
10
|
+
params[:graph_args] ||= "--base 1000"
|
11
|
+
params[:graph_vlabel] ||= "records"
|
12
|
+
params[:graph_category] ||= "postgresql"
|
13
|
+
params[:graph_info] ||= "PostgreSQL record size"
|
14
|
+
|
15
|
+
bytes_flag = (params[:graph_vlabel] == "bytes")
|
16
|
+
|
17
|
+
params[:adapter] = "postgresql"
|
18
|
+
params[:host] ||= "localhost"
|
19
|
+
params[:port] ||= 5432
|
20
|
+
|
21
|
+
params[:label_warning] ||= 50000000
|
22
|
+
params[:label_critical] ||= 100000000
|
23
|
+
|
24
|
+
if params[:stdin_flag]
|
25
|
+
table_paths = []
|
26
|
+
while line = input_stream.gets
|
27
|
+
line.chomp!
|
28
|
+
table_paths.push line
|
29
|
+
end
|
30
|
+
else
|
31
|
+
table_paths = argv.shift
|
32
|
+
table_paths = table_paths.split(/,/) unless table_paths.nil?
|
33
|
+
table_paths ||= []
|
34
|
+
table_paths.delete("")
|
35
|
+
end
|
36
|
+
command = argv.shift
|
37
|
+
|
38
|
+
if table_paths == []
|
39
|
+
Cawcaw.logger.error("PostgreSQL table name is not set")
|
40
|
+
return 1
|
41
|
+
end
|
42
|
+
|
43
|
+
case command
|
44
|
+
when "autoconf"
|
45
|
+
output_stream.puts "yes"
|
46
|
+
when "config"
|
47
|
+
output_stream.puts <<-EOF
|
48
|
+
graph_title #{params[:graph_title]}
|
49
|
+
graph_args #{params[:graph_args]}
|
50
|
+
graph_vlabel #{params[:graph_vlabel]}
|
51
|
+
graph_category #{params[:graph_category]}
|
52
|
+
graph_info #{params[:graph_info]}
|
53
|
+
EOF
|
54
|
+
|
55
|
+
label_draw = "AREA"
|
56
|
+
|
57
|
+
table_paths.each do |table_path|
|
58
|
+
label = table_path.gsub(/[^a-zA-Z0-9]/, "_").downcase
|
59
|
+
output_stream.puts <<-EOF
|
60
|
+
#{label}.label #{table_path}
|
61
|
+
#{label}.info #{table_path} size
|
62
|
+
#{label}.draw #{label_draw}
|
63
|
+
#{label}.warning #{params[:label_warning]}
|
64
|
+
#{label}.critical #{params[:label_critical]}
|
65
|
+
EOF
|
66
|
+
label_draw = "STACK"
|
67
|
+
end
|
68
|
+
else
|
69
|
+
|
70
|
+
ActiveRecord::Base.establish_connection(params)
|
71
|
+
|
72
|
+
full_table_paths = {}
|
73
|
+
table_paths.each do |table_path|
|
74
|
+
table_path_pair = table_path.split(/\./)
|
75
|
+
if /[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)?/ !~ table_path
|
76
|
+
Cawcaw.logger.error("PostgreSQL table name '#{table_path}' is invalid")
|
77
|
+
return 1
|
78
|
+
end
|
79
|
+
table_name = table_path_pair.pop
|
80
|
+
schema_name = table_path_pair.pop || "public"
|
81
|
+
full_table_paths[table_path] = "#{schema_name}.#{table_name}"
|
82
|
+
end
|
83
|
+
|
84
|
+
if params[:stats_flag]
|
85
|
+
count_sql = <<-EOF
|
86
|
+
select nspname || '.' || relname as full_table_path, reltuples, relpages
|
87
|
+
from pg_class
|
88
|
+
inner join pg_namespace
|
89
|
+
on pg_class.relnamespace = pg_namespace.oid
|
90
|
+
where relkind='r'
|
91
|
+
and (
|
92
|
+
#{
|
93
|
+
table_paths.map{|table_path|
|
94
|
+
"(nspname || '.' || relname) = #{ActiveRecord::Base.sanitize(full_table_paths[table_path])}"
|
95
|
+
}.join(" or ")
|
96
|
+
}
|
97
|
+
)
|
98
|
+
EOF
|
99
|
+
else
|
100
|
+
count_sql = <<-EOF
|
101
|
+
#{
|
102
|
+
table_paths.map{|table_path|
|
103
|
+
"(select #{ActiveRecord::Base.sanitize(full_table_paths[table_path])} as full_table_path, count(*) as reltuples from #{full_table_paths[table_path]})"
|
104
|
+
}.join(" union ")
|
105
|
+
}
|
106
|
+
EOF
|
107
|
+
end
|
108
|
+
|
109
|
+
|
110
|
+
results = ActiveRecord::Base.connection.execute count_sql
|
111
|
+
table_sizes = {}
|
112
|
+
results.each do |record|
|
113
|
+
table_path = record["full_table_path"]
|
114
|
+
if bytes_flag
|
115
|
+
table_sizes[table_path] = record["relpages"].to_i * 8
|
116
|
+
else
|
117
|
+
table_sizes[table_path] = record["reltuples"].to_i
|
118
|
+
end
|
119
|
+
end
|
120
|
+
table_paths.each do |table_path|
|
121
|
+
label = table_path.gsub(/[^a-zA-Z0-9]/, "_").downcase
|
122
|
+
table_size = table_sizes[full_table_paths[table_path]]
|
123
|
+
output_stream.puts "#{label}.value #{table_size}"
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
return 0
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Cawcaw
|
2
|
+
module Command
|
3
|
+
module Postgresql
|
4
|
+
def self.run(argv, input_stream=$stdin, output_stream=$stdout)
|
5
|
+
command = argv.shift
|
6
|
+
case command
|
7
|
+
when "table"
|
8
|
+
return Cawcaw::Command::Postgresql::Table.run(argv, input_stream, output_stream)
|
9
|
+
else
|
10
|
+
STDERR.puts "invalid postgresql command: '#{command}'"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
require "cawcaw/command/postgresql/table"
|
data/lib/cawcaw/core/common.rb
CHANGED
@@ -2,71 +2,47 @@ module Cawcaw
|
|
2
2
|
module Core
|
3
3
|
module Common
|
4
4
|
def self.parse_opts(argv)
|
5
|
-
|
6
|
-
graph_args = nil
|
7
|
-
graph_vlabel = nil
|
8
|
-
graph_category = nil
|
9
|
-
graph_info = nil
|
5
|
+
ret = {}
|
10
6
|
|
11
|
-
hadoop_working_directory = "/user/#{ENV["USER"]}"
|
12
|
-
hadoop_command = "hadoop"
|
7
|
+
ret[:hadoop_working_directory] = "/user/#{ENV["USER"]}"
|
8
|
+
ret[:hadoop_command] = "hadoop"
|
13
9
|
|
14
|
-
stdin_flag = false
|
15
|
-
verbose_flag = false
|
10
|
+
ret[:stdin_flag] = false
|
11
|
+
ret[:verbose_flag] = false
|
16
12
|
|
17
13
|
next_argv = []
|
18
14
|
|
19
15
|
while 0 < argv.size do
|
20
16
|
val = argv.shift
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
when
|
31
|
-
|
32
|
-
when
|
33
|
-
|
34
|
-
when
|
35
|
-
label_critical = argv.shift.to_i
|
36
|
-
when "--hadoop-command"
|
37
|
-
hadoop_command = argv.shift
|
38
|
-
when "--hadoop-wdir"
|
17
|
+
if val[0..1] == "--"
|
18
|
+
label = val[2..-1].gsub(/-/,"_").to_sym
|
19
|
+
else
|
20
|
+
label = nil
|
21
|
+
end
|
22
|
+
case label
|
23
|
+
when :graph_title, :graph_args, :graph_vlabel, :graph_category, :graph_info, \
|
24
|
+
:hadoop_command, :adapter, :host, :database, :username, :password, :encoding
|
25
|
+
ret[label] = argv.shift
|
26
|
+
when :label_warning, :label_critical, :port
|
27
|
+
ret[label] = ret[label].to_i
|
28
|
+
when :stdin, :verbose, :stats
|
29
|
+
ret["#{label}_flag".to_sym] = true
|
30
|
+
when :hadoop_wdir
|
39
31
|
hadoop_working_directory = argv.shift
|
40
|
-
hadoop_working_directory = hadoop_working_directory.gsub(/\/$/, "")
|
41
|
-
when "--stdin"
|
42
|
-
stdin_flag = true
|
43
|
-
when "--verbose"
|
44
|
-
verbose_flag = true
|
32
|
+
ret[:hadoop_working_directory] = hadoop_working_directory.gsub(/\/$/, "")
|
45
33
|
else
|
46
34
|
next_argv.push val
|
47
35
|
end
|
48
36
|
end
|
49
37
|
argv.push(*next_argv)
|
50
38
|
|
51
|
-
if verbose_flag then
|
39
|
+
if ret[:verbose_flag] then
|
52
40
|
Cawcaw.logger.level = Logger::INFO
|
53
41
|
else
|
54
42
|
Cawcaw.logger.level = Logger::WARN
|
55
43
|
end
|
56
44
|
|
57
|
-
return
|
58
|
-
:graph_title=>graph_title,
|
59
|
-
:graph_args=>graph_args,
|
60
|
-
:graph_vlabel=>graph_vlabel,
|
61
|
-
:graph_category=>graph_category,
|
62
|
-
:graph_info=>graph_info,
|
63
|
-
:label_warning=>label_warning,
|
64
|
-
:label_critical=>label_critical,
|
65
|
-
:hadoop_command=>hadoop_command,
|
66
|
-
:hadoop_working_directory=>hadoop_working_directory,
|
67
|
-
:stdin_flag=>stdin_flag,
|
68
|
-
:verbose_flag=>verbose_flag
|
69
|
-
}
|
45
|
+
return ret
|
70
46
|
end
|
71
47
|
end
|
72
48
|
end
|
data/lib/cawcaw.rb
CHANGED
@@ -1,11 +1,14 @@
|
|
1
|
+
require "rubygems"
|
1
2
|
require "logger"
|
2
3
|
require "uri"
|
4
|
+
require "active_record"
|
3
5
|
require "bunnish"
|
4
6
|
|
5
7
|
module Cawcaw
|
6
8
|
end
|
7
9
|
|
8
10
|
require "cawcaw/command/hadoop"
|
11
|
+
require "cawcaw/command/postgresql"
|
9
12
|
require "cawcaw/command/rabbitmq"
|
10
13
|
require "cawcaw/core/common"
|
11
14
|
|
@@ -18,7 +21,7 @@ module Cawcaw
|
|
18
21
|
if @logger.nil?
|
19
22
|
@logger = (rails_logger || default_logger)
|
20
23
|
@logger.formatter = proc { |severity, datetime, progname, msg|
|
21
|
-
datetime.strftime("[%Y-%m-%d %H:%M:%S](#{severity})#{msg}\n")
|
24
|
+
datetime.strftime("[%Y-%m-%d %H:%M:%S](#{severity}) #{msg}\n")
|
22
25
|
}
|
23
26
|
end
|
24
27
|
return @logger
|
@@ -10,7 +10,7 @@ describe "bin/cawcaw hadoop dfs" do
|
|
10
10
|
end
|
11
11
|
context "when hdfs path is absolute" do
|
12
12
|
it "should output hadoop dfs config" do
|
13
|
-
result =
|
13
|
+
result = `ruby -I lib ./bin/cawcaw hadoop dfs /user/#{ENV["USER"]}/cawcaw-test config`
|
14
14
|
hash = {}
|
15
15
|
result.split(/\r?\n/).each do |line|
|
16
16
|
record = line.split(/ /, 2)
|
@@ -31,7 +31,7 @@ describe "bin/cawcaw hadoop dfs" do
|
|
31
31
|
end
|
32
32
|
|
33
33
|
it "should output hadoop dfs value" do
|
34
|
-
result =
|
34
|
+
result = `ruby -I lib ./bin/cawcaw hadoop dfs /user/#{ENV["USER"]}/cawcaw-test`
|
35
35
|
hash = {}
|
36
36
|
result.split(/\r?\n/).each do |line|
|
37
37
|
record = line.split(/ /, 2)
|
@@ -45,7 +45,7 @@ describe "bin/cawcaw hadoop dfs" do
|
|
45
45
|
context "when hdfs path is relative" do
|
46
46
|
context "with hadoop-wdir option" do
|
47
47
|
it "should output hadoop dfs config" do
|
48
|
-
result =
|
48
|
+
result = `ruby -I lib ./bin/cawcaw hadoop dfs --hadoop-wdir /user #{ENV["USER"]}/cawcaw-test config`
|
49
49
|
hash = {}
|
50
50
|
result.split(/\r?\n/).each do |line|
|
51
51
|
record = line.split(/ /, 2)
|
@@ -66,7 +66,7 @@ describe "bin/cawcaw hadoop dfs" do
|
|
66
66
|
end
|
67
67
|
|
68
68
|
it "should output hadoop dfs value" do
|
69
|
-
result =
|
69
|
+
result = `ruby -I lib ./bin/cawcaw hadoop dfs --hadoop-wdir /user #{ENV["USER"]}/cawcaw-test`
|
70
70
|
hash = {}
|
71
71
|
result.split(/\r?\n/).each do |line|
|
72
72
|
record = line.split(/ /, 2)
|
@@ -80,7 +80,7 @@ describe "bin/cawcaw hadoop dfs" do
|
|
80
80
|
|
81
81
|
context "without hadoop-wdir option" do
|
82
82
|
it "should output hadoop dfs config" do
|
83
|
-
result =
|
83
|
+
result = `ruby -I lib ./bin/cawcaw hadoop dfs cawcaw-test config`
|
84
84
|
hash = {}
|
85
85
|
result.split(/\r?\n/).each do |line|
|
86
86
|
record = line.split(/ /, 2)
|
@@ -101,7 +101,7 @@ describe "bin/cawcaw hadoop dfs" do
|
|
101
101
|
end
|
102
102
|
|
103
103
|
it "should output hadoop dfs value" do
|
104
|
-
result =
|
104
|
+
result = `ruby -I lib ./bin/cawcaw hadoop dfs cawcaw-test`
|
105
105
|
hash = {}
|
106
106
|
result.split(/\r?\n/).each do |line|
|
107
107
|
record = line.split(/ /, 2)
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe "bin/cawcaw postgresql table" do
|
4
|
+
before :all do
|
5
|
+
CawcawName.establish_connection({
|
6
|
+
:adapter=>"postgresql",
|
7
|
+
:host=>"localhost",
|
8
|
+
:port=>5432,
|
9
|
+
:username=>"postgres",
|
10
|
+
:database=>"cawcaw"
|
11
|
+
})
|
12
|
+
CawcawName.connection.create_table(:cawcaw_names) do |t|
|
13
|
+
t.column :name, :string
|
14
|
+
end
|
15
|
+
CawcawName.create(:name=>"foo")
|
16
|
+
CawcawName.create(:name=>"bar")
|
17
|
+
CawcawName.create(:name=>"baz")
|
18
|
+
end
|
19
|
+
|
20
|
+
after :all do
|
21
|
+
CawcawName.connection.drop_table(:cawcaw_names)
|
22
|
+
end
|
23
|
+
context "when table path is absolute" do
|
24
|
+
it "should output record size" do
|
25
|
+
result = `ruby -I lib ./bin/cawcaw postgresql table --username postgres --database cawcaw cawcaw_names`
|
26
|
+
record = result.chomp.split(/ /)
|
27
|
+
record.should == ["cawcaw_names.value", "3"]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -6,7 +6,7 @@ describe "bin/cawcaw rabbitmq queue-count" do
|
|
6
6
|
system("echo \"foo\nbar\nbaz\" | bunnish publish cawcaw-test-queue")
|
7
7
|
end
|
8
8
|
it "should output rabbitmq queue-count config" do
|
9
|
-
result =
|
9
|
+
result = `ruby -I lib ./bin/cawcaw rabbitmq queue-count cawcaw-test-queue config`
|
10
10
|
hash = {}
|
11
11
|
result.split(/\r?\n/).each do |line|
|
12
12
|
record = line.split(/ /, 2)
|
@@ -27,7 +27,7 @@ describe "bin/cawcaw rabbitmq queue-count" do
|
|
27
27
|
end
|
28
28
|
|
29
29
|
it "should output rabbitmq queue-count value" do
|
30
|
-
result =
|
30
|
+
result = `ruby -I lib ./bin/cawcaw rabbitmq queue-count cawcaw-test-queue`
|
31
31
|
hash = {}
|
32
32
|
result.split(/\r?\n/).each do |line|
|
33
33
|
record = line.split(/ /, 2)
|
data/spec/spec_helper.rb
CHANGED
@@ -22,13 +22,11 @@ end
|
|
22
22
|
|
23
23
|
module Cawcaw
|
24
24
|
CAWCAW_HOME = File.expand_path(File.dirname(__FILE__) + "/..")
|
25
|
-
BIN_DIR = "#{CAWCAW_HOME}/bin"
|
26
|
-
LIB_DIR = "#{CAWCAW_HOME}/lib"
|
27
|
-
RUBY_CMD = "/usr/bin/env ruby -I #{LIB_DIR}"
|
28
25
|
REDIRECT = {}
|
29
26
|
end
|
30
27
|
|
31
28
|
Cawcaw.logger = Logger.new(STDERR)
|
29
|
+
ActiveRecord::Base.logger = Cawcaw.logger
|
32
30
|
if File.exist?('/tmp/cawcaw.debug') then
|
33
31
|
Cawcaw.logger.level = Logger::DEBUG
|
34
32
|
Cawcaw::REDIRECT[:stdout] = nil
|
@@ -38,3 +36,7 @@ else
|
|
38
36
|
Cawcaw::REDIRECT[:stdout] = "> /dev/null"
|
39
37
|
Cawcaw::REDIRECT[:stderr] = "2> /dev/null"
|
40
38
|
end
|
39
|
+
|
40
|
+
class CawcawName < ActiveRecord::Base
|
41
|
+
end
|
42
|
+
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cawcaw
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,14 +9,46 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: activerecord
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 3.2.11
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 3.2.11
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: activerecord-postgresql-adapter
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 0.0.1
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 0.0.1
|
14
46
|
- !ruby/object:Gem::Dependency
|
15
47
|
name: bunnish
|
16
48
|
requirement: !ruby/object:Gem::Requirement
|
17
49
|
none: false
|
18
50
|
requirements:
|
19
|
-
- -
|
51
|
+
- - ~>
|
20
52
|
- !ruby/object:Gem::Version
|
21
53
|
version: 0.1.2
|
22
54
|
type: :runtime
|
@@ -24,7 +56,7 @@ dependencies:
|
|
24
56
|
version_requirements: !ruby/object:Gem::Requirement
|
25
57
|
none: false
|
26
58
|
requirements:
|
27
|
-
- -
|
59
|
+
- - ~>
|
28
60
|
- !ruby/object:Gem::Version
|
29
61
|
version: 0.1.2
|
30
62
|
- !ruby/object:Gem::Dependency
|
@@ -32,17 +64,49 @@ dependencies:
|
|
32
64
|
requirement: !ruby/object:Gem::Requirement
|
33
65
|
none: false
|
34
66
|
requirements:
|
35
|
-
- -
|
67
|
+
- - ~>
|
36
68
|
- !ruby/object:Gem::Version
|
37
|
-
version: 2.
|
69
|
+
version: 2.12.0
|
38
70
|
type: :development
|
39
71
|
prerelease: false
|
40
72
|
version_requirements: !ruby/object:Gem::Requirement
|
41
73
|
none: false
|
42
74
|
requirements:
|
43
|
-
- -
|
75
|
+
- - ~>
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 2.12.0
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: yard
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ~>
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: 0.8.3
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ~>
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: 0.8.3
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: redcarpet
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ~>
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: 2.2.2
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ~>
|
44
108
|
- !ruby/object:Gem::Version
|
45
|
-
version: 2.
|
109
|
+
version: 2.2.2
|
46
110
|
- !ruby/object:Gem::Dependency
|
47
111
|
name: rdoc
|
48
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -64,17 +128,17 @@ dependencies:
|
|
64
128
|
requirement: !ruby/object:Gem::Requirement
|
65
129
|
none: false
|
66
130
|
requirements:
|
67
|
-
- -
|
131
|
+
- - ~>
|
68
132
|
- !ruby/object:Gem::Version
|
69
|
-
version: 1.
|
133
|
+
version: 1.2.3
|
70
134
|
type: :development
|
71
135
|
prerelease: false
|
72
136
|
version_requirements: !ruby/object:Gem::Requirement
|
73
137
|
none: false
|
74
138
|
requirements:
|
75
|
-
- -
|
139
|
+
- - ~>
|
76
140
|
- !ruby/object:Gem::Version
|
77
|
-
version: 1.
|
141
|
+
version: 1.2.3
|
78
142
|
- !ruby/object:Gem::Dependency
|
79
143
|
name: jeweler
|
80
144
|
requirement: !ruby/object:Gem::Requirement
|
@@ -96,49 +160,49 @@ dependencies:
|
|
96
160
|
requirement: !ruby/object:Gem::Requirement
|
97
161
|
none: false
|
98
162
|
requirements:
|
99
|
-
- -
|
163
|
+
- - ~>
|
100
164
|
- !ruby/object:Gem::Version
|
101
|
-
version:
|
165
|
+
version: 0.7.1
|
102
166
|
type: :development
|
103
167
|
prerelease: false
|
104
168
|
version_requirements: !ruby/object:Gem::Requirement
|
105
169
|
none: false
|
106
170
|
requirements:
|
107
|
-
- -
|
171
|
+
- - ~>
|
108
172
|
- !ruby/object:Gem::Version
|
109
|
-
version:
|
173
|
+
version: 0.7.1
|
110
174
|
- !ruby/object:Gem::Dependency
|
111
175
|
name: simplecov-rcov
|
112
176
|
requirement: !ruby/object:Gem::Requirement
|
113
177
|
none: false
|
114
178
|
requirements:
|
115
|
-
- -
|
179
|
+
- - ~>
|
116
180
|
- !ruby/object:Gem::Version
|
117
|
-
version:
|
181
|
+
version: 0.2.3
|
118
182
|
type: :development
|
119
183
|
prerelease: false
|
120
184
|
version_requirements: !ruby/object:Gem::Requirement
|
121
185
|
none: false
|
122
186
|
requirements:
|
123
|
-
- -
|
187
|
+
- - ~>
|
124
188
|
- !ruby/object:Gem::Version
|
125
|
-
version:
|
189
|
+
version: 0.2.3
|
126
190
|
- !ruby/object:Gem::Dependency
|
127
191
|
name: ci_reporter
|
128
192
|
requirement: !ruby/object:Gem::Requirement
|
129
193
|
none: false
|
130
194
|
requirements:
|
131
|
-
- -
|
195
|
+
- - ~>
|
132
196
|
- !ruby/object:Gem::Version
|
133
|
-
version: 1.
|
197
|
+
version: 1.8.3
|
134
198
|
type: :development
|
135
199
|
prerelease: false
|
136
200
|
version_requirements: !ruby/object:Gem::Requirement
|
137
201
|
none: false
|
138
202
|
requirements:
|
139
|
-
- -
|
203
|
+
- - ~>
|
140
204
|
- !ruby/object:Gem::Version
|
141
|
-
version: 1.
|
205
|
+
version: 1.8.3
|
142
206
|
- !ruby/object:Gem::Dependency
|
143
207
|
name: flog
|
144
208
|
requirement: !ruby/object:Gem::Requirement
|
@@ -162,13 +226,13 @@ executables:
|
|
162
226
|
extensions: []
|
163
227
|
extra_rdoc_files:
|
164
228
|
- LICENSE.txt
|
165
|
-
- README.
|
229
|
+
- README.md
|
166
230
|
files:
|
167
231
|
- .document
|
168
232
|
- .rspec
|
169
233
|
- Gemfile
|
170
234
|
- LICENSE.txt
|
171
|
-
- README.
|
235
|
+
- README.md
|
172
236
|
- Rakefile
|
173
237
|
- VERSION
|
174
238
|
- bin/cawcaw
|
@@ -176,10 +240,13 @@ files:
|
|
176
240
|
- lib/cawcaw.rb
|
177
241
|
- lib/cawcaw/command/hadoop.rb
|
178
242
|
- lib/cawcaw/command/hadoop/dfs.rb
|
243
|
+
- lib/cawcaw/command/postgresql.rb
|
244
|
+
- lib/cawcaw/command/postgresql/table.rb
|
179
245
|
- lib/cawcaw/command/rabbitmq.rb
|
180
246
|
- lib/cawcaw/command/rabbitmq/queue_count.rb
|
181
247
|
- lib/cawcaw/core/common.rb
|
182
248
|
- spec/bin/cawcaw/hadoop/dfs_spec.rb
|
249
|
+
- spec/bin/cawcaw/postgresql/table_spec.rb
|
183
250
|
- spec/bin/cawcaw/rabbitmq/queue_count_spec.rb
|
184
251
|
- spec/lib/cawcaw_spec.rb
|
185
252
|
- spec/spec_helper.rb
|
@@ -198,7 +265,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
198
265
|
version: '0'
|
199
266
|
segments:
|
200
267
|
- 0
|
201
|
-
hash: -
|
268
|
+
hash: -3162639150070516761
|
202
269
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
203
270
|
none: false
|
204
271
|
requirements:
|