cloudwatchtographite 0.0.0.pre1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,205 @@
1
+ # _*_ coding: utf-8 _*_
2
+ # == Synopsis
3
+ # CloudwatchToGraphite retrieves metrics from the Amazon CloudWatch APIs
4
+ # and passes them on to a graphite server
5
+ #
6
+ # == Author
7
+ # S. Zachariah Sprackett <zac@sprackett.com>
8
+ #
9
+ # == License
10
+ # The MIT License (MIT)
11
+ #
12
+ # == Copyright
13
+ # Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
14
+ #
15
+ require 'time'
16
+ require 'hashifiable'
17
+
18
+ module CloudwatchToGraphite
19
+ UNITS = [
20
+ 'None',
21
+ 'Seconds',
22
+ 'Microseconds',
23
+ 'Milliseconds',
24
+ 'Bytes',
25
+ 'Kilobytes',
26
+ 'Megabytes',
27
+ 'Gigabytes',
28
+ 'Terabytes',
29
+ 'Bits',
30
+ 'Kilobits',
31
+ 'Megabits',
32
+ 'Gigabits',
33
+ 'Terabits',
34
+ 'Gigabytes/Second',
35
+ 'Terabytes/Second',
36
+ 'Bits/Second',
37
+ 'Kilobits/Second',
38
+ 'Megabits/Second',
39
+ 'Gigabits/Second',
40
+ 'Terabits/Second',
41
+ 'Count/Second'
42
+ ]
43
+
44
+ STATISTICS = [
45
+ 'Minimum',
46
+ 'Maximum',
47
+ 'Sum',
48
+ 'Average',
49
+ 'SampleCount'
50
+ ]
51
+
52
+ SETTER_MAPPINGS = {
53
+ 'namespace' => :Namespace=,
54
+ 'metricname' => :MetricName=,
55
+ 'statistics' => :Statistics=,
56
+ 'starttime' => :StartTime=,
57
+ 'endtime' => :EndTime=,
58
+ 'period' => :Period=,
59
+ 'dimensions' => :Dimensions=,
60
+ 'unit' => :Unit=
61
+ }
62
+
63
+ # A hashable representation of an AWS CloudWatch metric
64
+ #
65
+ class MetricDefinition
66
+ attr_reader :Namespace, :MetricName, :Statistics, :Unit, :Period
67
+ extend Hashifiable
68
+ hashify 'Namespace', 'MetricName', 'Statistics', 'StartTime', \
69
+ 'EndTime', 'Period', 'Dimensions'#, 'Unit'
70
+
71
+ def initialize
72
+ @Unit = UNITS[0]
73
+ @Dimensions = []
74
+ @Statistics = []
75
+ @Period = 60
76
+ end
77
+
78
+ def Namespace=(n)
79
+ Validator::string_shorter_than(n, 256)
80
+ @Namespace=n
81
+ end
82
+
83
+ def MetricName=(n)
84
+ Validator::string_shorter_than(n, 256)
85
+ @MetricName=n
86
+ end
87
+
88
+ def StartTime=(time)
89
+ raise ArgumentTypeError unless time.kind_of?(Time)
90
+ @StartTime=time
91
+ end
92
+
93
+ def StartTime
94
+ if @StartTime.nil?
95
+ (Time.now-600).iso8601
96
+ else
97
+ @StartTime.iso8601
98
+ end
99
+ end
100
+
101
+ def EndTime=(time)
102
+ raise ArgumentTypeError unless time.kind_of?(Time)
103
+ @EndTime=time
104
+ end
105
+
106
+ def EndTime
107
+ if @EndTime.nil?
108
+ Time.now.iso8601
109
+ else
110
+ @EndTime.iso8601
111
+ end
112
+ end
113
+
114
+ def Period=(n)
115
+ raise ArgumentTypeError unless n.kind_of? Integer
116
+ @Period = n
117
+ end
118
+
119
+ def Unit=(n)
120
+ raise ArgumentTypeError unless UNITS.include? n
121
+ @Unit = n
122
+ end
123
+
124
+ def Dimensions
125
+ @Dimensions.map(&:to_h)
126
+ end
127
+
128
+ def add_statistic(n)
129
+ raise ArgumentTypeError unless STATISTICS.include? n
130
+ if not @Statistics.include? n
131
+ @Statistics.push(n)
132
+ end
133
+ end
134
+
135
+ def add_dimension(n)
136
+ if not n.kind_of?(MetricDimension)
137
+ raise ArgumentTypeError
138
+ elsif @Dimensions.length >= 10
139
+ raise TooManyDimensionError
140
+ end
141
+ @Dimensions.push(n)
142
+ end
143
+
144
+ def valid?
145
+ if @Namespace.nil? or @MetricName.nil? or @Statistics.empty? \
146
+ or @Dimensions.empty? or @Unit.nil?
147
+ false
148
+ else
149
+ true
150
+ end
151
+ end
152
+
153
+ def graphite_path(stat)
154
+ path = "%s.%s.%s.%s" % [
155
+ self.Namespace,
156
+ self.MetricName,
157
+ stat,
158
+ @Dimensions.join('.')
159
+ ]
160
+ path.gsub('/', '.').downcase
161
+ end
162
+
163
+ def self.create_and_fill(definition)
164
+ md = MetricDefinition.new
165
+ definition.each_key do |k|
166
+ populate_metric_definition(
167
+ md, k, definition[k]
168
+ )
169
+ end
170
+ raise ParseError unless md.valid?
171
+ return md
172
+ end
173
+
174
+ def self.populate_metric_definition(md, key, value)
175
+ case key
176
+ when 'namespace', 'metricname', 'period', 'unit'
177
+ md.send(SETTER_MAPPINGS[key], value)
178
+ when 'starttime', 'endtime'
179
+ begin
180
+ md.send(SETTER_MAPPINGS[key], Time.parse(value))
181
+ rescue ArgumentTypeError
182
+ raise ParseError
183
+ end
184
+ when 'statistics'
185
+ populate_statistics(md, value)
186
+ when 'dimensions'
187
+ populate_dimensions_from_hashes(md, value)
188
+ else
189
+ raise ParseError
190
+ end
191
+ end
192
+
193
+ def self.populate_statistics(md, statistics)
194
+ Array(statistics).each do |stat|
195
+ md.add_statistic(stat)
196
+ end
197
+ end
198
+
199
+ def self.populate_dimensions_from_hashes(md, dimensions)
200
+ Array(dimensions).each do |dimension|
201
+ md.add_dimension(MetricDimension.create_from_hash(dimension))
202
+ end
203
+ end
204
+ end
205
+ end
@@ -0,0 +1,48 @@
1
+ # _*_ coding: utf-8 _*_
2
+ # == Synopsis
3
+ # CloudwatchToGraphite retrieves metrics from the Amazon CloudWatch APIs
4
+ # and passes them on to a graphite server
5
+ #
6
+ # == Author
7
+ # S. Zachariah Sprackett <zac@sprackett.com>
8
+ #
9
+ # == License
10
+ # The MIT License (MIT)
11
+ #
12
+ # == Copyright
13
+ # Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
14
+ #
15
+ module CloudwatchToGraphite
16
+ # A hashable representation of an AWS CloudWatch metric dimension
17
+ class MetricDimension
18
+ attr_reader :Name, :Value
19
+ extend Hashifiable
20
+ hashify 'Name', 'Value'
21
+
22
+ def Name=(n)
23
+ Validator::string_shorter_than(n, 256)
24
+ @Name=n
25
+ end
26
+
27
+ def Value=(n)
28
+ Validator::string_shorter_than(n, 256)
29
+ @Value=n
30
+ end
31
+
32
+ def self.create(name, value)
33
+ md = MetricDimension.new
34
+ md.Name = name
35
+ md.Value = value
36
+ md
37
+ end
38
+
39
+ def self.create_from_hash(dhash)
40
+ Validator::hash_with_keys(dhash, ['name', 'value'])
41
+
42
+ MetricDimension::create(
43
+ dhash['name'],
44
+ dhash['value']
45
+ )
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,47 @@
1
+ # _*_ coding: utf-8 _*_
2
+ # == Synopsis
3
+ # CloudwatchToGraphite retrieves metrics from the Amazon CloudWatch APIs
4
+ # and passes them on to a graphite server
5
+ #
6
+ # == Author
7
+ # S. Zachariah Sprackett <zac@sprackett.com>
8
+ #
9
+ # == License
10
+ # The MIT License (MIT)
11
+ #
12
+ # == Copyright
13
+ # Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
14
+ #
15
+ module CloudwatchToGraphite
16
+ # static methods to validate arguments
17
+ class Validator
18
+ def self.string(n)
19
+ raise ArgumentTypeError unless n.kind_of?(String)
20
+ end
21
+
22
+ def self.string_shorter_than(n, length)
23
+ string(n)
24
+ raise ArgumentLengthError unless n.length < length
25
+ end
26
+
27
+ def self.string_longer_than(n, length)
28
+ string(n)
29
+ raise ArgumentLengthError unless n.length > length
30
+ end
31
+
32
+ def self.hash_with_key(h, key)
33
+ raise ArgumentTypeError unless h.kind_of?(Hash) and h.has_key?(key)
34
+ end
35
+
36
+ def self.hash_with_keys(h, keys)
37
+ keys.each do |k|
38
+ hash_with_key(h,k)
39
+ end
40
+ end
41
+
42
+ def self.hash_with_key_of_type(h, key, type)
43
+ hash_with_key(h, key)
44
+ raise ArgumentTypeError unless h[key].kind_of?(type)
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,25 @@
1
+ # _*_ coding: utf-8 _*_
2
+ # == Synopsis
3
+ # CloudwatchToGraphite retrieves metrics from the Amazon CloudWatch APIs
4
+ # and passes them on to a graphite server
5
+ #
6
+ # == Author
7
+ # S. Zachariah Sprackett <zac@sprackett.com>
8
+ #
9
+ # == License
10
+ # The MIT License (MIT)
11
+ #
12
+ # == Copyright
13
+ # Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
14
+ #
15
+ module CloudwatchToGraphite
16
+ class VERSION
17
+ MAJOR = 0
18
+ MINOR = 0
19
+ PATCH = 0
20
+ BUILD = 'pre1'
21
+
22
+ STRING = [MAJOR, MINOR, PATCH].compact.join('.') \
23
+ + (BUILD.empty? ? '' : ".#{BUILD}")
24
+ end
25
+ end
@@ -0,0 +1,185 @@
1
+ # _*_ coding: utf-8 _*_
2
+ #
3
+ # Author:: S. Zachariah Sprackett <zac@sprackett.com>
4
+ # License:: The MIT License (MIT)
5
+ # Copyright:: Copyright (C) 2013 - S. Zachariah Sprackett <zac@sprackett.com>
6
+ #
7
+ require_relative './cloudwatchtographite/exception'
8
+ require_relative './cloudwatchtographite/version'
9
+ require_relative './cloudwatchtographite/metricdefinition'
10
+ require_relative './cloudwatchtographite/metricdimension'
11
+ require_relative './cloudwatchtographite/loadmetrics'
12
+ require_relative './cloudwatchtographite/validator'
13
+ require 'socket'
14
+ require 'fog'
15
+ require 'pp'
16
+
17
+ module CloudwatchToGraphite
18
+ # This class is responsible for retrieving metrics from CloudWatch and
19
+ # sending the results to a Graphite server.
20
+ class Base
21
+ attr_accessor :protocol
22
+ attr_accessor :graphite_server
23
+ attr_accessor :graphite_port
24
+ attr_reader :carbon_prefix
25
+
26
+ # Initialize the CloudwatchToGraphite::Base object.
27
+ # aws_access_key:: The AWS user key
28
+ # aws_secret_key:: The AWS secret
29
+ # region:: The AWS region (eg: us-west-1)
30
+ # verbose:: boolean to enable verbose output
31
+ #
32
+ def initialize(aws_access_key, aws_secret_key, region, verbose=false)
33
+ @protocol = 'udp'
34
+ @carbon_prefix = 'cloudwatch'
35
+ @graphite_server = 'localhost'
36
+ @graphite_port = 2003
37
+ @verbose = verbose
38
+
39
+ debug_log "Fog setting up for region #{region}"
40
+
41
+ @cloudwatch = Fog::AWS::CloudWatch.new(
42
+ :aws_access_key_id => aws_access_key,
43
+ :aws_secret_access_key => aws_secret_key,
44
+ :region => region
45
+ )
46
+ end
47
+
48
+ # Send data to a Graphite server via the UDP protocol
49
+ # contents:: a string or array containing the contents to send
50
+ #
51
+ def send_udp(contents)
52
+ sock = nil
53
+ contents = contents.join("\n") if contents.kind_of?(Array)
54
+
55
+ debug_log "Attempting to send #{contents.length} bytes " +
56
+ "to #{@graphite_server}:#{@graphite_port} via udp"
57
+
58
+ begin
59
+ sock = UDPSocket.open
60
+ sock.send(contents, 0, @graphite_server, @graphite_port)
61
+ retval = true
62
+ rescue Exception => e
63
+ debug_log "Caught exception! [#{e}]"
64
+ retval = false
65
+ ensure
66
+ sock.close if sock
67
+ end
68
+ retval
69
+ end
70
+
71
+ # Send data to a Graphite server via the TCP protocol
72
+ # contents:: a string or array containing the contents to send
73
+ #
74
+ def send_tcp(contents)
75
+ sock = nil
76
+ contents = contents.join("\n") if contents.kind_of?(Array)
77
+
78
+ debug_log "Attempting to send #{contents.length} bytes " +
79
+ "to #{@graphite_server}:#{@graphite_port} via tcp"
80
+
81
+ retval = false
82
+ begin
83
+ sock = TCPSocket.open(@graphite_server, @graphite_port)
84
+ sock.print(contents)
85
+ retval = true
86
+ rescue Exception => e
87
+ debug_log "Caught exception! [#{e}]"
88
+ ensure
89
+ sock.close if sock
90
+ end
91
+ retval
92
+ end
93
+
94
+ def retrieve_datapoints(metrics)
95
+ ret = []
96
+ Array(metrics).each do |m|
97
+ begin
98
+ ret.concat retrieve_one_datapoint(m)
99
+ rescue Excon::Errors::SocketError, Excon::Errors::BadRequest => e
100
+ warn "[Error in CloudWatch call] #{e.message}"
101
+ rescue Excon::Errors::Forbidden
102
+ warn "[Error in CloudWatch call] permission denied - check keys!"
103
+ end
104
+ end
105
+ ret
106
+ end
107
+
108
+ def retrieve_one_datapoint(metric)
109
+ debug_log "Sending to CloudWatch:"
110
+ debug_object metric.to_h
111
+ data_points = @cloudwatch.get_metric_statistics(
112
+ metric.to_h
113
+ ).body['GetMetricStatisticsResult']['Datapoints']
114
+ debug_log "Received from CloudWatch:"
115
+ debug_object data_points
116
+
117
+ return retrieve_statistics(metric, order_data_points(data_points))
118
+ end
119
+
120
+ def retrieve_statistics(metric, data_points)
121
+ ret = []
122
+ metric.Statistics.each do |stat|
123
+ name = "#{@carbon_prefix}.#{metric.graphite_path(stat)}"
124
+ data_points.each do |d|
125
+ ret.push "#{name} #{d[stat]} #{d['Timestamp'].utc.to_i}"
126
+ end
127
+ end
128
+ debug_log "Returning Statistics:"
129
+ debug_object ret
130
+ ret
131
+ end
132
+
133
+ def fetch_and_forward(metrics)
134
+ results = retrieve_datapoints(metrics)
135
+ if results.length == 0
136
+ false
137
+ else
138
+ case @protocol
139
+ when 'tcp'
140
+ send_tcp(results)
141
+ when 'udp'
142
+ send_udp(results)
143
+ else
144
+ debug_log "Unknown protocol #{@protocol}"
145
+ raise ProtocolError
146
+ end
147
+ end
148
+ end
149
+
150
+ # set the carbon prefix
151
+ # p:: the string prefix to use
152
+ def carbon_prefix=(p)
153
+ Validator::string_longer_than(p, 0)
154
+ @carbon_prefix=p
155
+ end
156
+
157
+ private
158
+ def order_data_points(data_points)
159
+ if data_points.nil?
160
+ data_points = []
161
+ else
162
+ data_points = Array(data_points)
163
+ end
164
+
165
+ if data_points.length == 0
166
+ warn "No data points!"
167
+ data_points
168
+ else
169
+ data_points = data_points.sort_by {|array| array['Timestamp'] }
170
+ end
171
+ end
172
+
173
+ def debug_log(s)
174
+ if @verbose
175
+ warn s
176
+ end
177
+ end
178
+
179
+ def debug_object(s)
180
+ if @verbose
181
+ warn PP.pp(s, "")
182
+ end
183
+ end
184
+ end
185
+ end
@@ -0,0 +1,28 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ describe CloudwatchToGraphite::Base do
4
+ before :each do
5
+ @base = CloudwatchToGraphite::Base.new 'foo', 'bar', 'us-east-1', false
6
+ end
7
+
8
+ describe ".new" do
9
+ it "takes four parameters and returns a CloudwatchToGraphite::Base" do
10
+ @base.should be_an_instance_of CloudwatchToGraphite::Base
11
+ end
12
+ end
13
+
14
+ describe ".fetch_and_forward" do
15
+ end
16
+
17
+ describe ".carbon_prefix=" do
18
+ it "allows setting a prefix for carbon" do
19
+ expect { @base.carbon_prefix = 123 }.to \
20
+ raise_error(CloudwatchToGraphite::ArgumentTypeError)
21
+ expect { @base.carbon_prefix = '' }.to \
22
+ raise_error(CloudwatchToGraphite::ArgumentLengthError)
23
+ @base.carbon_prefix = "the_prefix"
24
+ @base.carbon_prefix.should eql "the_prefix"
25
+ end
26
+ end
27
+ end
28
+
@@ -0,0 +1,12 @@
1
+ FactoryGirl.define do
2
+ factory :metricdefinition, class: CloudwatchToGraphite::MetricDefinition do
3
+ sequence(:MetricName) { |n| "metricname#{n}" }
4
+ sequence(:Namespace) { |n| "namespace#{n}" }
5
+ Period 90
6
+ Unit 'None'
7
+ after(:build) do |definition, evaluator|
8
+ definition.add_statistic('Sum')
9
+ definition.add_statistic('Average')
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,6 @@
1
+ FactoryGirl.define do
2
+ factory :metricdimension, class: CloudwatchToGraphite::MetricDimension do
3
+ sequence(:Name) { |n| "name#{n}" }
4
+ sequence(:Value) { |n| "value#{n}" }
5
+ end
6
+ end
@@ -0,0 +1,27 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ describe CloudwatchToGraphite::LoadMetrics do
4
+
5
+ describe "#load_content" do
6
+ it "it should not load invalid metrics" do
7
+ content = {'shmetrics' => 'foo'}
8
+ expect {
9
+ CloudwatchToGraphite::LoadMetrics::load_content(content)
10
+ }.to raise_error(CloudwatchToGraphite::ArgumentTypeError)
11
+ end
12
+
13
+ #it "it should load valid metrics" do
14
+ # content = {
15
+ # 'metrics' => [ {
16
+ # 'namespace' => 'thenamespace',
17
+ # 'metricname' => 'themetricname',
18
+ # 'statistics' => 'Average',
19
+ # 'period' => 1,
20
+ # 'dimensions' => [ { 'name' => 'thename', 'value' => 'thevalue' }]
21
+ # }]
22
+ # }
23
+ # metrics = CloudwatchToGraphite::LoadMetrics::load_content(content)
24
+ # metrics.should be_an(Array)
25
+ #end
26
+ end
27
+ end