meteorlog 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 29de27158912619561019ddd020283b84d17ba3c
4
+ data.tar.gz: 994519363b62e6df8367bfb5f4c691dc7582759d
5
+ SHA512:
6
+ metadata.gz: 942ac817e7daab22123cfc96aef611ee079d2752fe7e6a5efe5193b16e0d286274c5e610047c419cd9238fdcd14e3ac1767066e2fa5a336a877be1703d629b92
7
+ data.tar.gz: f719f786635167f42ef4fc6673fc5b3c9835549ec56e2e07f24d2312af47718dcf3847701de3571c050b5a08e6e3a3e0c2bb2a4c095b09bbf730979cb5f2e27f
data/.gitignore ADDED
@@ -0,0 +1,20 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ test.rb
19
+ Logsfile
20
+ *.swp
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --colour
2
+ --require spec_helper
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in meteorlog.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Genki Sugawara
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,84 @@
1
+ # Meteorlog
2
+
3
+ Meteorlog is a tool to manage CloudWatch Logs.
4
+
5
+ It defines the state of CloudWatch Logs using DSL, and updates CloudWatch Logs according to DSL.
6
+
7
+ [![Gem Version](https://badge.fury.io/rb/meteorlog.svg)](http://badge.fury.io/rb/meteorlog)
8
+
9
+ ## Installation
10
+
11
+ Add this line to your application's Gemfile:
12
+
13
+ gem 'meteorlog'
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install meteorlog
22
+
23
+ ## Usage
24
+
25
+ ```sh
26
+ export AWS_ACCESS_KEY_ID='...'
27
+ export AWS_SECRET_ACCESS_KEY='...'
28
+ export AWS_REGION='us-east-1'
29
+ meteorlog -e -o Logsfile # export CloudWatch Logs
30
+ vi Logsfile
31
+ meteorlog -a --dry-run
32
+ meteorlog -a # apply `Logsfile` to CloudWatch Logs
33
+ ```
34
+
35
+ ## Help
36
+
37
+ ```
38
+ Usage: meteorlog [options]
39
+ -p, --profile PROFILE_NAME
40
+ -k, --access-key ACCESS_KEY
41
+ -s, --secret-key SECRET_KEY
42
+ -r, --region REGION
43
+ -a, --apply
44
+ -f, --file FILE
45
+ --dry-run
46
+ -e, --export
47
+ -o, --output FILE
48
+ --no-color
49
+ --debug
50
+ ```
51
+
52
+ ## Logsfile example
53
+
54
+ ```ruby
55
+ require 'other/logsfile'
56
+
57
+ log_group "/var/log/messages" do
58
+ log_stream "my-stream"
59
+
60
+ metric_filter "MyAppAccessCount" do
61
+ metric :name=>"EventCount", :namespace=>"YourNamespace", :value=>"1"
62
+ end
63
+
64
+ metric_filter "MyAppAccessCount2" do
65
+ # see http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/FilterAndPatternSyntax.html
66
+ filter_pattern '[ip, user, username, timestamp, request, status_code, bytes > 1000]'
67
+ metric :name=>"EventCount2", :namespace=>"YourNamespace2", :value=>"2"
68
+ end
69
+ end
70
+
71
+ log_group "/var/log/maillog" do
72
+ log_stream "my-stream2"
73
+
74
+ metric_filter "MyAppAccessCount" do
75
+ filter_pattern '[..., status_code, bytes]'
76
+ metric :name=>"EventCount3", :namespace=>"YourNamespace", :value=>"1"
77
+ end
78
+
79
+ metric_filter "MyAppAccessCount2" do
80
+ filter_pattern '[ip, user, username, timestamp, request = *html*, status_code = 4*, bytes]'
81
+ metric :name=>"EventCount4", :namespace=>"YourNamespace2", :value=>"2"
82
+ end
83
+ end
84
+ ```
data/Rakefile ADDED
@@ -0,0 +1,5 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new('spec')
5
+ task :default => :spec
data/bin/meteorlog ADDED
@@ -0,0 +1,108 @@
1
+ #!/usr/bin/env ruby
2
+ $: << File.expand_path("#{File.dirname __FILE__}/../lib")
3
+ require 'rubygems'
4
+ require 'meteorlog'
5
+ require 'optparse'
6
+
7
+ Version = Meteorlog::VERSION
8
+ DEFAULT_FILENAME = 'Logsfile'
9
+
10
+ mode = nil
11
+ file = DEFAULT_FILENAME
12
+ output_file = '-'
13
+
14
+ options = {
15
+ :dry_run => false,
16
+ :color => true,
17
+ :debug => false,
18
+ }
19
+
20
+ ARGV.options do |opt|
21
+ begin
22
+ access_key = nil
23
+ secret_key = nil
24
+ region = nil
25
+ profile_name = nil
26
+
27
+ opt.on('-p', '--profile PROFILE_NAME') {|v| profile_name = v }
28
+ opt.on('-k', '--access-key ACCESS_KEY') {|v| access_key = v }
29
+ opt.on('-s', '--secret-key SECRET_KEY') {|v| secret_key = v }
30
+ opt.on('-r', '--region REGION') {|v| region = v }
31
+ opt.on('-a', '--apply') { mode = :apply }
32
+ opt.on('-f', '--file FILE') {|v| file = v }
33
+ opt.on('', '--dry-run') { options[:dry_run] = true }
34
+ opt.on('-e', '--export') { mode = :export }
35
+ opt.on('-o', '--output FILE') {|v| output_file = v }
36
+ opt.on('' , '--no-color') { options[:color] = false }
37
+ opt.on('' , '--debug') { options[:debug] = true }
38
+ opt.parse!
39
+
40
+ aws_opts = {}
41
+
42
+ if access_key and secret_key
43
+ aws_opts.update(
44
+ :access_key_id => access_key,
45
+ :secret_access_key => secret_key
46
+ )
47
+ elsif profile_name
48
+ credentials = Aws::SharedCredentials.new(:profile_name => profile_name)
49
+ aws_opts[:credentials] = credentials
50
+ elsif (access_key and !secret_key) or (!access_key and secret_key) or mode.nil?
51
+ puts opt.help
52
+ exit 1
53
+ end
54
+
55
+ aws_opts[:region] = region if region
56
+ Aws.config.update(aws_opts)
57
+ rescue => e
58
+ $stderr.puts("[ERROR] #{e.message}")
59
+ exit 1
60
+ end
61
+ end
62
+
63
+ String.colorize = options[:color]
64
+
65
+ if options[:debug]
66
+ AWS.config({
67
+ :http_wire_trace => true,
68
+ :logger => Meteorlog::Logger.instance,
69
+ })
70
+ end
71
+
72
+ begin
73
+ logger = Meteorlog::Logger.instance
74
+ logger.set_debug(options[:debug])
75
+ client = Meteorlog::Client.new(options)
76
+
77
+ case mode
78
+ when :export
79
+ if output_file == '-'
80
+ logger.info('# Export Logs')
81
+ puts client.export(options)
82
+ else
83
+ logger.info("Export Logs to `#{output_file}`")
84
+ open(output_file, 'wb') {|f| f.puts client.export(options) }
85
+ end
86
+ when :apply
87
+ unless File.exist?(file)
88
+ raise "No Logsfile found (looking for: #{file})"
89
+ end
90
+
91
+ msg = "Apply `#{file}` to CloudWatch Logs"
92
+ msg << ' (dry-run)' if options[:dry_run]
93
+ logger.info(msg)
94
+
95
+ updated = client.apply(file)
96
+
97
+ logger.info('No change'.intense_blue) unless updated
98
+ else
99
+ raise 'must not happen'
100
+ end
101
+ rescue => e
102
+ if options[:debug]
103
+ raise e
104
+ else
105
+ $stderr.puts("[ERROR] #{e.message}".red)
106
+ exit 1
107
+ end
108
+ end
@@ -0,0 +1,104 @@
1
+ class Meteorlog::Client
2
+ include Meteorlog::Logger::Helper
3
+ include Meteorlog::Utils
4
+
5
+ def initialize(options = {})
6
+ @options = options
7
+ @cloud_watch_logs = Aws::CloudWatchLogs::Client.new
8
+ end
9
+
10
+ def export(opts = {})
11
+ exported = Meteorlog::Exporter.export(@cloud_watch_logs, @options.merge(opts))
12
+ Meteorlog::DSL.convert(exported, @options.merge(opts))
13
+ end
14
+
15
+ def apply(file)
16
+ walk(file)
17
+ end
18
+
19
+ private
20
+
21
+ def walk(file)
22
+ dsl = load_file(file)
23
+ dsl_log_groups = collect_to_hash(dsl.log_groups, :log_group_name)
24
+ aws = Meteorlog::Wrapper.wrap(@cloud_watch_logs, @options)
25
+ aws_log_groups = collect_to_hash(aws.log_groups, :log_group_name)
26
+
27
+ dsl_log_groups.each do |log_group_name, dsl_log_group|
28
+ aws_log_group = aws_log_groups[log_group_name]
29
+
30
+ unless aws_log_group
31
+ aws_log_group = aws.log_groups.create(log_group_name)
32
+ aws_log_groups[log_group_name] = aws_log_group
33
+ end
34
+ end
35
+
36
+ dsl_log_groups.each do |log_group_name, dsl_log_group|
37
+ aws_log_group = aws_log_groups.delete(log_group_name)
38
+ walk_log_group(dsl_log_group, aws_log_group)
39
+ end
40
+
41
+ aws_log_groups.each do |log_group_name, aws_log_group|
42
+ aws_log_group.delete
43
+ end
44
+
45
+ aws.modified?
46
+ end
47
+
48
+ def walk_log_group(dsl_log_group, aws_log_group)
49
+ walk_log_streams(dsl_log_group.log_streams, aws_log_group.log_streams)
50
+ walk_metric_filters(dsl_log_group.metric_filters, aws_log_group.metric_filters)
51
+ end
52
+
53
+ def walk_log_streams(dsl_log_streams, aws_log_streams)
54
+ collection_api = aws_log_streams
55
+ dsl_log_streams = collect_to_hash(dsl_log_streams, :log_stream_name)
56
+ aws_log_streams = collect_to_hash(aws_log_streams, :log_stream_name)
57
+
58
+ dsl_log_streams.each do |log_stream_name, dsl_log_stream|
59
+ aws_log_stream = aws_log_streams.delete(log_stream_name)
60
+
61
+ unless aws_log_stream
62
+ collection_api.create(log_stream_name)
63
+ end
64
+ end
65
+
66
+ aws_log_streams.each do |log_stream_name, aws_log_stream|
67
+ aws_log_stream.delete
68
+ end
69
+ end
70
+
71
+ def walk_metric_filters(dsl_metric_filters, aws_metric_filters)
72
+ collection_api = aws_metric_filters
73
+ dsl_metric_filters = collect_to_hash(dsl_metric_filters, :filter_name)
74
+ aws_metric_filters = collect_to_hash(aws_metric_filters, :filter_name)
75
+
76
+ dsl_metric_filters.each do |filter_name, dsl_metric_filter|
77
+ aws_metric_filter = aws_metric_filters.delete(filter_name)
78
+
79
+ if aws_metric_filter
80
+ unless aws_metric_filter.eql?(dsl_metric_filter)
81
+ aws_metric_filter.update(dsl_metric_filter)
82
+ end
83
+ else
84
+ collection_api.create(filter_name, dsl_metric_filter)
85
+ end
86
+ end
87
+
88
+ aws_metric_filters.each do |filter_name, aws_metric_filter|
89
+ aws_metric_filter.delete
90
+ end
91
+ end
92
+
93
+ def load_file(file)
94
+ if file.kind_of?(String)
95
+ open(file) do |f|
96
+ Meteorlog::DSL.parse(f.read, file)
97
+ end
98
+ elsif file.respond_to?(:read)
99
+ Meteorlog::DSL.parse(file.read, file.path)
100
+ else
101
+ raise TypeError, "can't convert #{file} into File"
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,33 @@
1
+ class Meteorlog::DSL::Context::LogGroup
2
+ include Meteorlog::DSL::Validator
3
+
4
+ attr_reader :result
5
+
6
+ def initialize(name, &block)
7
+ @error_identifier = "LogGroup `#{name}`"
8
+ @result = OpenStruct.new(
9
+ :log_group_name => name,
10
+ :log_streams => [],
11
+ :metric_filters => [],
12
+ )
13
+ instance_eval(&block)
14
+ end
15
+
16
+ def log_stream(name)
17
+ _required(:log_stream_name, name)
18
+ _validate("LogStream `#{name}` is already defined") do
19
+ @result.log_streams.all? {|i| i.log_stream_name != name }
20
+ end
21
+
22
+ @result.log_streams << OpenStruct.new(:log_stream_name => name)
23
+ end
24
+
25
+ def metric_filter(name, &block)
26
+ _required(:filter_name, name)
27
+ _validate("MetricFilter `#{name}` is already defined") do
28
+ @result.metric_filters.all? {|i| i.filter_name != name }
29
+ end
30
+
31
+ @result.metric_filters << Meteorlog::DSL::Context::MetricFilter.new(name, @result.log_group_name, &block).result
32
+ end
33
+ end
@@ -0,0 +1,36 @@
1
+ class Meteorlog::DSL::Context::MetricFilter
2
+ include Meteorlog::DSL::Validator
3
+
4
+ attr_reader :result
5
+
6
+ def initialize(name, log_group_name, &block)
7
+ @error_identifier = "LogGroup `#{log_group_name}` > MetricFilter `#{name}`"
8
+ @result = OpenStruct.new(
9
+ :filter_name => name,
10
+ :metric_transformations => [],
11
+ )
12
+ instance_eval(&block)
13
+ end
14
+
15
+ def filter_pattern(pattern)
16
+ _call_once(:filter_pattern)
17
+ _required(:filter_pattern, pattern)
18
+ @result.filter_pattern = pattern.to_s
19
+ end
20
+
21
+ def metric(attrs)
22
+ metric_attrs = {}
23
+ _expected_type(attrs, Hash)
24
+ {
25
+ :name => :metric_name,
26
+ :namespace => :metric_namespace,
27
+ :value => :metric_value
28
+ }.each do |attr_name, metric_name|
29
+ _required("metric[#{attr_name}]", attrs[attr_name])
30
+ metric_attrs[metric_name] = attrs[attr_name].to_s
31
+ end
32
+ _expected_length(metric_attrs, 3)
33
+
34
+ @result.metric_transformations << metric_attrs
35
+ end
36
+ end
@@ -0,0 +1,45 @@
1
+ class Meteorlog::DSL::Context
2
+ include Meteorlog::DSL::Validator
3
+
4
+ class << self
5
+ def eval(dsl, path, opts = {})
6
+ self.new(path, opts) {
7
+ eval(dsl, binding, path)
8
+ }
9
+ end
10
+ end # of class methods
11
+
12
+ attr_reader :result
13
+
14
+ def initialize(path, options = {}, &block)
15
+ @path = path
16
+ @options = options
17
+ @result = OpenStruct.new(:log_groups => [])
18
+ @log_group_names = []
19
+ instance_eval(&block)
20
+ end
21
+
22
+ private
23
+
24
+ def require(file)
25
+ logsfile = File.expand_path(File.join(File.dirname(@path), file))
26
+
27
+ if File.exist?(logsfile)
28
+ instance_eval(File.read(logsfile), logsfile)
29
+ elsif File.exist?(logsfile + '.rb')
30
+ instance_eval(File.read(logsfile + '.rb'), logsfile + '.rb')
31
+ else
32
+ Kernel.require(file)
33
+ end
34
+ end
35
+
36
+ def log_group(name, &block)
37
+ _required(:log_group_name, name)
38
+ _validate("LogGroup `#{name}` is already defined") do
39
+ not @log_group_names.include?(name)
40
+ end
41
+
42
+ @result.log_groups << Meteorlog::DSL::Context::LogGroup.new(name, &block).result
43
+ @log_group_names << name
44
+ end
45
+ end
@@ -0,0 +1,78 @@
1
+ class Meteorlog::DSL::Converter
2
+ class << self
3
+ def convert(exported, opts = {})
4
+ self.new(exported, opts).convert
5
+ end
6
+ end # of class methods
7
+
8
+ def initialize(exported, options = {})
9
+ @exported = exported
10
+ @options = options
11
+ end
12
+
13
+ def convert
14
+ @exported.each.map {|log_group_name, log_group_attrs|
15
+ output_log_group(log_group_name, log_group_attrs)
16
+ }.join("\n")
17
+ end
18
+
19
+ private
20
+
21
+ def output_log_group(log_group_name, log_group_attrs)
22
+ log_group_name = log_group_name.inspect
23
+ buf = []
24
+ streams = log_group_attrs[:log_streams]
25
+ buf << output_streams(streams, :prefix => ' ') if streams
26
+ metric_filters = (log_group_attrs[:metric_filters] || [])
27
+ buf << output_metric_filters(metric_filters, :prefix => ' ') unless metric_filters.empty?
28
+
29
+ <<-EOS
30
+ log_group #{log_group_name} do
31
+ #{buf.join("\n\n ")}
32
+ end
33
+ EOS
34
+ end
35
+
36
+ def output_streams(streams, opts = {})
37
+ prefix = opts[:prefix]
38
+
39
+ streams.map {|stream|
40
+ "log_stream #{stream.inspect}"
41
+ }.join("\n#{prefix}")
42
+ end
43
+
44
+ def output_metric_filters(metric_filters, opts = {})
45
+ prefix = opts[:prefix]
46
+
47
+ metric_filters.map {|metric_filter_name, metric_filter_attrs|
48
+ metric_filter_name = metric_filter_name.inspect
49
+ filter_pattern = metric_filter_attrs[:filter_pattern]
50
+ metrics = metric_filter_attrs[:metric_transformations] || []
51
+
52
+ buf = []
53
+ buf << "metric_filter #{metric_filter_name} do"
54
+ buf << " filter_pattern #{filter_pattern.inspect}" if filter_pattern
55
+ buf << " " + output_metrics(metrics, :prefix => prefix + ' ') unless metrics.empty?
56
+ buf << "end"
57
+ buf.join("\n#{prefix}")
58
+ }.join("\n\n#{prefix}")
59
+ end
60
+
61
+ def output_metrics(metrics, opts = {})
62
+ prefix = opts[:prefix]
63
+
64
+ metrics.map {|metric|
65
+ metric_attrs = unbrace({
66
+ :name => metric[:metric_name],
67
+ :namespace => metric[:metric_namespace],
68
+ :value => metric[:metric_value],
69
+ }.inspect)
70
+
71
+ "metric #{metric_attrs}"
72
+ }.join("\n#{prefix}")
73
+ end
74
+
75
+ def unbrace(str)
76
+ str.sub(/\A\s*\{/, '').sub(/\}\s*\z/, '')
77
+ end
78
+ end
@@ -0,0 +1,52 @@
1
+ module Meteorlog::DSL::Validator
2
+ def _required(name, value)
3
+ invalid = false
4
+
5
+ if value
6
+ case value
7
+ when String
8
+ invalid = value.strip.empty?
9
+ when Array, Hash
10
+ invalid = value.empty?
11
+ end
12
+ elsif value.nil?
13
+ invalid = true
14
+ end
15
+
16
+ raise _identify("`#{name}` is required") if invalid
17
+ end
18
+
19
+ def _call_once(method_name)
20
+ @called ||= []
21
+
22
+ if @called.include?(method_name)
23
+ raise _identify("`#{method_name}` is already defined")
24
+ end
25
+
26
+ @called << method_name
27
+ end
28
+
29
+ def _expected_type(value, *types)
30
+ unless types.any? {|t| value.kind_of?(t) }
31
+ raise _identify("Invalid type: #{value}")
32
+ end
33
+ end
34
+
35
+ def _expected_length(value, length)
36
+ if value.length != length
37
+ raise _identify("Invalid length: #{value}")
38
+ end
39
+ end
40
+
41
+ def _validate(errmsg)
42
+ raise _identify(errmsg) unless yield
43
+ end
44
+
45
+ def _identify(errmsg)
46
+ if @error_identifier
47
+ errmsg = "#{@error_identifier}: #{errmsg}"
48
+ end
49
+
50
+ return errmsg
51
+ end
52
+ end
@@ -0,0 +1,11 @@
1
+ class Meteorlog::DSL
2
+ class << self
3
+ def convert(exported, opts = {})
4
+ Meteorlog::DSL::Converter.convert(exported, opts)
5
+ end
6
+
7
+ def parse(dsl, path, opts = {})
8
+ Meteorlog::DSL::Context.eval(dsl, path, opts).result
9
+ end
10
+ end # of class methods
11
+ end
@@ -0,0 +1,68 @@
1
+ class Meteorlog::Exporter
2
+ class << self
3
+ def export(cloud_watch_logs, opts = {})
4
+ self.new(cloud_watch_logs, opts).export
5
+ end
6
+ end # of class methods
7
+
8
+ def initialize(cloud_watch_logs, options = {})
9
+ @cloud_watch_logs = cloud_watch_logs
10
+ @options = options
11
+ end
12
+
13
+ def export
14
+ result = {}
15
+
16
+ log_groups = @cloud_watch_logs.describe_log_groups.each.inject([]) {|r, i| r + i.log_groups }
17
+
18
+ log_groups.each do |log_group|
19
+ export_log_graoup(log_group, result)
20
+ end
21
+
22
+ return result
23
+ end
24
+
25
+ private
26
+
27
+ def export_log_graoup(log_group, result)
28
+ log_group_name = log_group.log_group_name
29
+ log_streams = @cloud_watch_logs.describe_log_streams(
30
+ :log_group_name => log_group_name).log_streams
31
+ metric_filters = @cloud_watch_logs.describe_metric_filters(
32
+ :log_group_name => log_group_name).metric_filters
33
+
34
+ result[log_group_name] = {
35
+ :log_streams => export_log_streams(log_streams),
36
+ :metric_filters => export_metric_filters(metric_filters)
37
+ }
38
+ end
39
+
40
+ def export_log_streams(log_streams)
41
+ log_streams.map {|log_stream| log_stream.log_stream_name }
42
+ end
43
+
44
+ def export_metric_filters(metric_filters)
45
+ result = {}
46
+
47
+ metric_filters.each do |metric_filter|
48
+ filter_name = metric_filter.filter_name
49
+ metric_transformations = metric_filter.metric_transformations.map do |metric_transformation|
50
+ {
51
+ :metric_namespace => metric_transformation.metric_namespace,
52
+ :metric_name => metric_transformation.metric_name,
53
+ :metric_value => metric_transformation.metric_value,
54
+ }
55
+ end
56
+
57
+ result[filter_name] = {
58
+ :metric_transformations => metric_transformations
59
+ }
60
+
61
+ if metric_filter.filter_pattern
62
+ result[filter_name][:filter_pattern] = metric_filter.filter_pattern
63
+ end
64
+ end
65
+
66
+ return result
67
+ end
68
+ end
@@ -0,0 +1,25 @@
1
+ class String
2
+ @@colorize = false
3
+
4
+ class << self
5
+ def colorize=(value)
6
+ @@colorize = value
7
+ end
8
+
9
+ def colorize
10
+ @@colorize
11
+ end
12
+ end # of class methods
13
+
14
+ Term::ANSIColor::Attribute.named_attributes.map do |attribute|
15
+ class_eval(<<-EOS, __FILE__, __LINE__ + 1)
16
+ def #{attribute.name}
17
+ if @@colorize
18
+ Term::ANSIColor.send(#{attribute.name.inspect}, self)
19
+ else
20
+ self
21
+ end
22
+ end
23
+ EOS
24
+ end
25
+ end
@@ -0,0 +1,27 @@
1
+ class Meteorlog::Logger < ::Logger
2
+ include Singleton
3
+
4
+ def initialize
5
+ super($stdout)
6
+
7
+ self.formatter = proc do |severity, datetime, progname, msg|
8
+ "#{msg}\n"
9
+ end
10
+
11
+ self.level = Logger::INFO
12
+ end
13
+
14
+ def set_debug(value)
15
+ self.level = value ? Logger::DEBUG : Logger::INFO
16
+ end
17
+
18
+ module Helper
19
+ def log(level, message, color, log_id = nil)
20
+ message = "[#{level.to_s.upcase}] #{message}" unless level == :info
21
+ message << ": #{log_id}" if log_id
22
+ message << ' (dry-run)' if @options && @options[:dry_run]
23
+ logger = (@options && @options[:logger]) || Meteorlog::Logger.instance
24
+ logger.send(level, message.send(color))
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,25 @@
1
+ module Meteorlog::Utils
2
+ def collect_to_hash(collection, *key_attrs)
3
+ opts = key_attrs.last.kind_of?(Hash) ? key_attrs.pop : {}
4
+ hash = {}
5
+
6
+ collection.each do |item|
7
+ if block_given?
8
+ key = yield(item)
9
+ else
10
+ key = key_attrs.map {|k| item.send(k) }
11
+ key = key.first if key_attrs.length == 1
12
+ end
13
+
14
+ if opts[:has_many]
15
+ hash[key] ||= []
16
+ hash[key] << item
17
+ else
18
+ hash[key] = item
19
+ end
20
+ end
21
+
22
+ return hash
23
+ end
24
+ module_function :collect_to_hash
25
+ end
@@ -0,0 +1,3 @@
1
+ module Meteorlog
2
+ VERSION = '0.0.1'
3
+ end
@@ -0,0 +1,17 @@
1
+ class Meteorlog::Wrapper::CloudWatchLogs
2
+ def initialize(cloud_watch_logs, options = {})
3
+ @cloud_watch_logs = cloud_watch_logs
4
+ @options = options.dup
5
+ end
6
+
7
+ def log_groups
8
+ Meteorlog::Wrapper::LogGroupCollection.new(
9
+ @cloud_watch_logs,
10
+ @cloud_watch_logs.describe_log_groups.each.inject([]) {|r, i| r + i.log_groups },
11
+ @options)
12
+ end
13
+
14
+ def modified?
15
+ !!@options[:modified]
16
+ end
17
+ end
@@ -0,0 +1,47 @@
1
+ class Meteorlog::Wrapper::LogGroup
2
+ extend Forwardable
3
+ include Meteorlog::Logger::Helper
4
+
5
+ def_delegator :@log_group, :log_group_name
6
+
7
+ def initialize(cloud_watch_logs, log_group, options = {})
8
+ @cloud_watch_logs = cloud_watch_logs
9
+ @log_group = log_group
10
+ @options = options
11
+ end
12
+
13
+ def log_streams
14
+ if @log_group.respond_to?(:log_streams)
15
+ lss = @log_group.log_streams
16
+ else
17
+ lss = @cloud_watch_logs.describe_log_streams(
18
+ :log_group_name => log_group_name
19
+ ).each.inject([]) {|r, i| r + i.log_streams }
20
+ end
21
+
22
+ Meteorlog::Wrapper::LogStreamCollection.new(
23
+ @cloud_watch_logs, lss, @log_group, @options)
24
+ end
25
+
26
+ def metric_filters
27
+ if @log_group.respond_to?(:metric_filters)
28
+ mfs = @log_group.metric_filters
29
+ else
30
+ mfs = @cloud_watch_logs.describe_metric_filters(
31
+ :log_group_name => log_group_name
32
+ ).each.inject([]) {|r, i| r + i.metric_filters }
33
+ end
34
+
35
+ Meteorlog::Wrapper::MetricFilterCollection.new(
36
+ @cloud_watch_logs, mfs, @log_group, @options)
37
+ end
38
+
39
+ def delete
40
+ log(:info, 'Delete LogGroup', :red, self.log_group_name)
41
+
42
+ unless @options[:dry_run]
43
+ @cloud_watch_logs.delete_log_group(:log_group_name => self.log_group_name)
44
+ @options[:modified] = true
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,31 @@
1
+ class Meteorlog::Wrapper::LogGroupCollection
2
+ include Meteorlog::Logger::Helper
3
+
4
+ def initialize(cloud_watch_logs, log_groups, options = {})
5
+ @cloud_watch_logs = cloud_watch_logs
6
+ @log_groups = log_groups
7
+ @options = options
8
+ end
9
+
10
+ def each
11
+ @log_groups.each do |log_group|
12
+ yield(Meteorlog::Wrapper::LogGroup.new(
13
+ @cloud_watch_logs, log_group, @options))
14
+ end
15
+ end
16
+
17
+ def create(name, opts = {})
18
+ log(:info, 'Create LogGroup', :cyan, name)
19
+
20
+ unless @options[:dry_run]
21
+ @cloud_watch_logs.create_log_group(:log_group_name => name)
22
+ @options[:modified] = true
23
+ end
24
+
25
+ log_group = OpenStruct.new(
26
+ :log_group_name => name, :log_streams => [], :metric_filters => [])
27
+
28
+ Meteorlog::Wrapper::LogGroup.new(
29
+ @cloud_watch_logs, log_group, @options)
30
+ end
31
+ end
@@ -0,0 +1,25 @@
1
+ class Meteorlog::Wrapper::LogStream
2
+ extend Forwardable
3
+ include Meteorlog::Logger::Helper
4
+
5
+ def_delegator :@log_stream, :log_stream_name
6
+ def_delegator :@log_group, :log_group_name
7
+
8
+ def initialize(cloud_watch_logs, log_stream, log_group, options = {})
9
+ @cloud_watch_logs = cloud_watch_logs
10
+ @log_stream = log_stream
11
+ @log_group = log_group
12
+ @options = options
13
+ end
14
+
15
+ def delete
16
+ log(:info, 'Delete LogStream', :red, "#{self.log_group_name} > #{self.log_stream_name}")
17
+
18
+ unless @options[:dry_run]
19
+ @cloud_watch_logs.delete_log_stream(
20
+ :log_group_name => self.log_group_name,
21
+ :log_stream_name => self.log_stream_name)
22
+ @options[:modified] = true
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,36 @@
1
+ class Meteorlog::Wrapper::LogStreamCollection
2
+ extend Forwardable
3
+ include Meteorlog::Logger::Helper
4
+
5
+ def_delegator :@log_group, :log_group_name
6
+
7
+ def initialize(cloud_watch_logs, log_streams, log_group, options = {})
8
+ @cloud_watch_logs = cloud_watch_logs
9
+ @log_streams = log_streams
10
+ @log_group = log_group
11
+ @options = options
12
+ end
13
+
14
+ def each
15
+ @log_streams.each do |log_stream|
16
+ yield(Meteorlog::Wrapper::LogStream.new(
17
+ @cloud_watch_logs, log_stream, @log_group, @options))
18
+ end
19
+ end
20
+
21
+ def create(name, opts = {})
22
+ log(:info, 'Create LogStream', :cyan, "#{self.log_group_name} > #{name}")
23
+
24
+ unless @options[:dry_run]
25
+ @cloud_watch_logs.create_log_stream(
26
+ :log_group_name => self.log_group_name,
27
+ :log_stream_name => name)
28
+ @options[:modified] = true
29
+ end
30
+
31
+ log_stream = OpenStruct.new(:log_stream_name => name)
32
+
33
+ Meteorlog::Wrapper::LogStream.new(
34
+ @cloud_watch_logs, log_stream, @log_group, @options)
35
+ end
36
+ end
@@ -0,0 +1,80 @@
1
+ class Meteorlog::Wrapper::MetricFilter
2
+ extend Forwardable
3
+ include Meteorlog::Logger::Helper
4
+
5
+ DEFAULT_VALUES = {}
6
+
7
+ def_delegators :@metric_filter,
8
+ :filter_name, :filter_pattern, :metric_transformations
9
+ def_delegator :@log_group, :log_group_name
10
+
11
+ def initialize(cloud_watch_logs, metric_filter, log_group, options = {})
12
+ @cloud_watch_logs = cloud_watch_logs
13
+ @metric_filter = metric_filter
14
+ @log_group = log_group
15
+ @options = options
16
+ end
17
+
18
+ def eql?(dsl)
19
+ diff(dsl).empty?
20
+ end
21
+
22
+ def update(dsl)
23
+ delta = diff(dsl)
24
+ log(:info, 'Update MetricFilter', :green, "#{self.log_group_name} > #{self.filter_name}: #{format_delta(delta)}")
25
+
26
+ unless @options[:dry_run]
27
+ @cloud_watch_logs.put_metric_filter(
28
+ :log_group_name => self.log_group_name,
29
+ :filter_name => self.filter_name,
30
+ :filter_pattern => dsl[:filter_pattern] || '',
31
+ :metric_transformations => dsl[:metric_transformations])
32
+ @options[:modified] = true
33
+ end
34
+ end
35
+
36
+ def delete
37
+ log(:info, 'Delete MetricFilter', :red, "#{self.log_group_name} > #{self.filter_name}")
38
+
39
+ unless @options[:dry_run]
40
+ @cloud_watch_logs.delete_metric_filter(
41
+ :log_group_name => self.log_group_name,
42
+ :filter_name => self.filter_name)
43
+ @options[:modified] = true
44
+ end
45
+ end
46
+
47
+ private
48
+
49
+ def diff(dsl)
50
+ delta = {}
51
+
52
+ [
53
+ [:filter_pattern, self.filter_pattern, dsl[:filter_pattern]],
54
+ [:metric_transformations,
55
+ self.metric_transformations.map {|i| i.to_h }, dsl[:metric_transformations]],
56
+ ].each do |name, self_value, dsl_value|
57
+ if normalize(name, self_value) != normalize(name, dsl_value)
58
+ delta[name] = {:old => self_value, :new => dsl_value}
59
+ end
60
+ end
61
+
62
+ return delta
63
+ end
64
+
65
+ def format_delta(delta)
66
+ delta.map {|name, values|
67
+ "#{name}(#{values[:old].inspect}=>#{values[:new].inspect})"
68
+ }.join(', ')
69
+ end
70
+
71
+ def normalize(name, value)
72
+ if [Array, Hash].any? {|c| value.kind_of?(c) }
73
+ value.sort
74
+ elsif DEFAULT_VALUES.has_key?(name) and value.nil?
75
+ DEFAULT_VALUES[name]
76
+ else
77
+ value
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,40 @@
1
+ class Meteorlog::Wrapper::MetricFilterCollection
2
+ extend Forwardable
3
+ include Meteorlog::Logger::Helper
4
+
5
+ def_delegator :@log_group, :log_group_name
6
+
7
+ def initialize(cloud_watch_logs, metric_filters, log_group, options = {})
8
+ @cloud_watch_logs = cloud_watch_logs
9
+ @metric_filters = metric_filters
10
+ @log_group = log_group
11
+ @options = options
12
+ end
13
+
14
+ def each
15
+ @metric_filters.each do |metric_filter|
16
+ yield(Meteorlog::Wrapper::MetricFilter.new(
17
+ @cloud_watch_logs, metric_filter, @log_group, @options))
18
+ end
19
+ end
20
+
21
+ def create(name, opts = {})
22
+ log(:info, 'Create MetricFilter', :cyan, "#{self.log_group_name} > #{name}")
23
+
24
+ unless @options[:dry_run]
25
+ @cloud_watch_logs.put_metric_filter(
26
+ :log_group_name => self.log_group_name,
27
+ :filter_name => name,
28
+ :filter_pattern => opts[:filter_pattern] || '',
29
+ :metric_transformations => opts[:metric_transformations])
30
+ @options[:modified] = true
31
+ end
32
+
33
+ metric_filter = OpenStruct.new(
34
+ :filter_name => name,
35
+ :filter_pattern => opts[:filter_pattern],
36
+ :metric_transformations => opts[:metric_transformations])
37
+
38
+ Meteorlog::Wrapper::MetricFilter.new(metric_filter, @log_group, @options)
39
+ end
40
+ end
@@ -0,0 +1,7 @@
1
+ class Meteorlog::Wrapper
2
+ class << self
3
+ def wrap(cwl, opts = {})
4
+ Meteorlog::Wrapper::CloudWatchLogs.new(cwl, opts)
5
+ end
6
+ end # of class methods
7
+ end
data/lib/meteorlog.rb ADDED
@@ -0,0 +1,31 @@
1
+ module Meteorlog; end
2
+
3
+ require 'forwardable'
4
+ require 'json'
5
+ require 'logger'
6
+ require 'ostruct'
7
+ require 'singleton'
8
+
9
+ require 'aws-sdk-core'
10
+ require 'term/ansicolor'
11
+
12
+ require 'meteorlog/logger'
13
+ require 'meteorlog/utils'
14
+ require 'meteorlog/client'
15
+ require 'meteorlog/dsl'
16
+ require 'meteorlog/dsl/validator'
17
+ require 'meteorlog/dsl/context'
18
+ require 'meteorlog/dsl/context/log_group'
19
+ require 'meteorlog/dsl/context/metric_filter'
20
+ require 'meteorlog/dsl/converter'
21
+ require 'meteorlog/exporter'
22
+ require 'meteorlog/ext/string_ext'
23
+ require 'meteorlog/version'
24
+ require 'meteorlog/wrapper'
25
+ require 'meteorlog/wrapper/cloud_watch_logs'
26
+ require 'meteorlog/wrapper/log_group'
27
+ require 'meteorlog/wrapper/log_group_collection'
28
+ require 'meteorlog/wrapper/log_stream'
29
+ require 'meteorlog/wrapper/log_stream_collection'
30
+ require 'meteorlog/wrapper/metric_filter'
31
+ require 'meteorlog/wrapper/metric_filter_collection'
data/meteorlog.gemspec ADDED
@@ -0,0 +1,27 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'meteorlog/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'meteorlog'
8
+ spec.version = Meteorlog::VERSION
9
+ spec.authors = ['Genki Sugawara']
10
+ spec.email = ['sgwr_dts@yahoo.co.jp']
11
+ spec.description = "Meteorlog is a tool to manage CloudWatch Logs. It defines the state of CloudWatch Logs using DSL, and updates CloudWatch Logs according to DSL."
12
+ spec.summary = "Meteorlog is a tool to manage CloudWatch Logs."
13
+ spec.homepage = 'https://github.com/winebarrel/meteorlog'
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_dependency 'aws-sdk-core', '>= 2.0.0.rc8'
22
+ spec.add_dependency "json"
23
+ spec.add_dependency "term-ansicolor"
24
+ spec.add_development_dependency 'bundler', '~> 1.5'
25
+ spec.add_development_dependency 'rake'
26
+ spec.add_development_dependency 'rspec', '>= 3.0.0'
27
+ end
@@ -0,0 +1,5 @@
1
+ describe Meteorlog do
2
+ it do
3
+ expect(1).to eq(1)
4
+ end
5
+ end
@@ -0,0 +1 @@
1
+ require 'meteorlog'
metadata ADDED
@@ -0,0 +1,163 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: meteorlog
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Genki Sugawara
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-08-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk-core
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 2.0.0.rc8
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '>='
25
+ - !ruby/object:Gem::Version
26
+ version: 2.0.0.rc8
27
+ - !ruby/object:Gem::Dependency
28
+ name: json
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: term-ansicolor
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: '1.5'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ~>
67
+ - !ruby/object:Gem::Version
68
+ version: '1.5'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rake
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '>='
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '>='
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rspec
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '>='
88
+ - !ruby/object:Gem::Version
89
+ version: 3.0.0
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '>='
95
+ - !ruby/object:Gem::Version
96
+ version: 3.0.0
97
+ description: Meteorlog is a tool to manage CloudWatch Logs. It defines the state of
98
+ CloudWatch Logs using DSL, and updates CloudWatch Logs according to DSL.
99
+ email:
100
+ - sgwr_dts@yahoo.co.jp
101
+ executables:
102
+ - meteorlog
103
+ extensions: []
104
+ extra_rdoc_files: []
105
+ files:
106
+ - .gitignore
107
+ - .rspec
108
+ - Gemfile
109
+ - LICENSE.txt
110
+ - README.md
111
+ - Rakefile
112
+ - bin/meteorlog
113
+ - lib/meteorlog.rb
114
+ - lib/meteorlog/client.rb
115
+ - lib/meteorlog/dsl.rb
116
+ - lib/meteorlog/dsl/context.rb
117
+ - lib/meteorlog/dsl/context/log_group.rb
118
+ - lib/meteorlog/dsl/context/metric_filter.rb
119
+ - lib/meteorlog/dsl/converter.rb
120
+ - lib/meteorlog/dsl/validator.rb
121
+ - lib/meteorlog/exporter.rb
122
+ - lib/meteorlog/ext/string_ext.rb
123
+ - lib/meteorlog/logger.rb
124
+ - lib/meteorlog/utils.rb
125
+ - lib/meteorlog/version.rb
126
+ - lib/meteorlog/wrapper.rb
127
+ - lib/meteorlog/wrapper/cloud_watch_logs.rb
128
+ - lib/meteorlog/wrapper/log_group.rb
129
+ - lib/meteorlog/wrapper/log_group_collection.rb
130
+ - lib/meteorlog/wrapper/log_stream.rb
131
+ - lib/meteorlog/wrapper/log_stream_collection.rb
132
+ - lib/meteorlog/wrapper/metric_filter.rb
133
+ - lib/meteorlog/wrapper/metric_filter_collection.rb
134
+ - meteorlog.gemspec
135
+ - spec/meteorlog_spec.rb
136
+ - spec/spec_helper.rb
137
+ homepage: https://github.com/winebarrel/meteorlog
138
+ licenses:
139
+ - MIT
140
+ metadata: {}
141
+ post_install_message:
142
+ rdoc_options: []
143
+ require_paths:
144
+ - lib
145
+ required_ruby_version: !ruby/object:Gem::Requirement
146
+ requirements:
147
+ - - '>='
148
+ - !ruby/object:Gem::Version
149
+ version: '0'
150
+ required_rubygems_version: !ruby/object:Gem::Requirement
151
+ requirements:
152
+ - - '>='
153
+ - !ruby/object:Gem::Version
154
+ version: '0'
155
+ requirements: []
156
+ rubyforge_project:
157
+ rubygems_version: 2.0.14
158
+ signing_key:
159
+ specification_version: 4
160
+ summary: Meteorlog is a tool to manage CloudWatch Logs.
161
+ test_files:
162
+ - spec/meteorlog_spec.rb
163
+ - spec/spec_helper.rb