logstash-input-cloudwatch 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +9 -0
- data/docs/index.asciidoc +266 -0
- data/lib/logstash/inputs/cloudwatch.rb +82 -31
- data/logstash-input-cloudwatch.gemspec +18 -13
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9b00ee77394f455a6f10c3520133ad01a07ba862
|
4
|
+
data.tar.gz: c6c12406db0d518f93a07b0c3303cf97f3d137c9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 080344bbd37d1533e166e93855d283e7d3355f6b045155946076d0e8ffc4b364b84ddee3beed4d4337094297e957fe4f902ae3a8f48b89e79fb09076822df9bc
|
7
|
+
data.tar.gz: b851865091d5ea2aaf0e7937bf9d47ff48d9a901a43933bd4c2f3df3e9c2c320bfd776499ea4d26bc4b373a4d1e62ae89e7fe142bfa1683f4b15c945204ba982
|
data/Gemfile
CHANGED
@@ -1,2 +1,11 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
|
+
|
2
3
|
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,266 @@
|
|
1
|
+
:plugin: cloudwatch
|
2
|
+
:type: input
|
3
|
+
|
4
|
+
///////////////////////////////////////////
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
6
|
+
///////////////////////////////////////////
|
7
|
+
:version: %VERSION%
|
8
|
+
:release_date: %RELEASE_DATE%
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
10
|
+
:include_path: ../../../../logstash/docs/include
|
11
|
+
///////////////////////////////////////////
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
13
|
+
///////////////////////////////////////////
|
14
|
+
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
16
|
+
|
17
|
+
=== Cloudwatch input plugin
|
18
|
+
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
|
21
|
+
==== Description
|
22
|
+
|
23
|
+
Pull events from the Amazon Web Services CloudWatch API.
|
24
|
+
|
25
|
+
To use this plugin, you *must* have an AWS account, and the following policy
|
26
|
+
|
27
|
+
Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
|
28
|
+
A sample policy for EC2 metrics is as follows:
|
29
|
+
[source,json]
|
30
|
+
{
|
31
|
+
"Version": "2012-10-17",
|
32
|
+
"Statement": [
|
33
|
+
{
|
34
|
+
"Sid": "Stmt1444715676000",
|
35
|
+
"Effect": "Allow",
|
36
|
+
"Action": [
|
37
|
+
"cloudwatch:GetMetricStatistics",
|
38
|
+
"cloudwatch:ListMetrics"
|
39
|
+
],
|
40
|
+
"Resource": "*"
|
41
|
+
},
|
42
|
+
{
|
43
|
+
"Sid": "Stmt1444716576170",
|
44
|
+
"Effect": "Allow",
|
45
|
+
"Action": [
|
46
|
+
"ec2:DescribeInstances"
|
47
|
+
],
|
48
|
+
"Resource": "*"
|
49
|
+
}
|
50
|
+
]
|
51
|
+
}
|
52
|
+
|
53
|
+
See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
|
54
|
+
|
55
|
+
# Configuration Example
|
56
|
+
[source, ruby]
|
57
|
+
input {
|
58
|
+
cloudwatch {
|
59
|
+
namespace => "AWS/EC2"
|
60
|
+
metrics => [ "CPUUtilization" ]
|
61
|
+
filters => { "tag:Group" => "API-Production" }
|
62
|
+
region => "us-east-1"
|
63
|
+
}
|
64
|
+
}
|
65
|
+
|
66
|
+
input {
|
67
|
+
cloudwatch {
|
68
|
+
namespace => "AWS/EBS"
|
69
|
+
metrics => ["VolumeQueueLength"]
|
70
|
+
filters => { "tag:Monitoring" => "Yes" }
|
71
|
+
region => "us-east-1"
|
72
|
+
}
|
73
|
+
}
|
74
|
+
|
75
|
+
input {
|
76
|
+
cloudwatch {
|
77
|
+
namespace => "AWS/RDS"
|
78
|
+
metrics => ["CPUUtilization", "CPUCreditUsage"]
|
79
|
+
filters => { "EngineName" => "mysql" } # Only supports EngineName, DatabaseClass and DBInstanceIdentifier
|
80
|
+
region => "us-east-1"
|
81
|
+
}
|
82
|
+
}
|
83
|
+
|
84
|
+
|
85
|
+
[id="plugins-{type}s-{plugin}-options"]
|
86
|
+
==== Cloudwatch Input Configuration Options
|
87
|
+
|
88
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
89
|
+
|
90
|
+
[cols="<,<,<",options="header",]
|
91
|
+
|=======================================================================
|
92
|
+
|Setting |Input type|Required
|
93
|
+
| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
|
94
|
+
| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
|
95
|
+
| <<plugins-{type}s-{plugin}-combined>> |<<boolean,boolean>>|No
|
96
|
+
| <<plugins-{type}s-{plugin}-filters>> |<<array,array>>|Yes
|
97
|
+
| <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
|
98
|
+
| <<plugins-{type}s-{plugin}-metrics>> |<<array,array>>|No
|
99
|
+
| <<plugins-{type}s-{plugin}-namespace>> |<<string,string>>|No
|
100
|
+
| <<plugins-{type}s-{plugin}-period>> |<<number,number>>|No
|
101
|
+
| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
|
102
|
+
| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
|
103
|
+
| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
|
104
|
+
| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
|
105
|
+
| <<plugins-{type}s-{plugin}-statistics>> |<<array,array>>|No
|
106
|
+
| <<plugins-{type}s-{plugin}-use_ssl>> |<<boolean,boolean>>|No
|
107
|
+
|=======================================================================
|
108
|
+
|
109
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
110
|
+
input plugins.
|
111
|
+
|
112
|
+
|
113
|
+
|
114
|
+
[id="plugins-{type}s-{plugin}-access_key_id"]
|
115
|
+
===== `access_key_id`
|
116
|
+
|
117
|
+
* Value type is <<string,string>>
|
118
|
+
* There is no default value for this setting.
|
119
|
+
|
120
|
+
This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
|
121
|
+
|
122
|
+
1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
|
123
|
+
2. External credentials file specified by `aws_credentials_file`
|
124
|
+
3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
|
125
|
+
4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
|
126
|
+
5. IAM Instance Profile (available when running inside EC2)
|
127
|
+
|
128
|
+
[id="plugins-{type}s-{plugin}-aws_credentials_file"]
|
129
|
+
===== `aws_credentials_file`
|
130
|
+
|
131
|
+
* Value type is <<string,string>>
|
132
|
+
* There is no default value for this setting.
|
133
|
+
|
134
|
+
Path to YAML file containing a hash of AWS credentials.
|
135
|
+
This file will only be loaded if `access_key_id` and
|
136
|
+
`secret_access_key` aren't set. The contents of the
|
137
|
+
file should look like this:
|
138
|
+
|
139
|
+
[source,ruby]
|
140
|
+
----------------------------------
|
141
|
+
:access_key_id: "12345"
|
142
|
+
:secret_access_key: "54321"
|
143
|
+
----------------------------------
|
144
|
+
|
145
|
+
|
146
|
+
[id="plugins-{type}s-{plugin}-combined"]
|
147
|
+
===== `combined`
|
148
|
+
|
149
|
+
* Value type is <<boolean,boolean>>
|
150
|
+
* Default value is `false`
|
151
|
+
|
152
|
+
Use this for namespaces that need to combine the dimensions like S3 and SNS.
|
153
|
+
|
154
|
+
[id="plugins-{type}s-{plugin}-filters"]
|
155
|
+
===== `filters`
|
156
|
+
|
157
|
+
* This is a required setting.
|
158
|
+
* Value type is <<array,array>>
|
159
|
+
* There is no default value for this setting.
|
160
|
+
|
161
|
+
Specify the filters to apply when fetching resources:
|
162
|
+
|
163
|
+
This needs to follow the AWS convention of specifiying filters.
|
164
|
+
Instances: { 'instance-id' => 'i-12344321' }
|
165
|
+
Tags: { "tag:Environment" => "Production" }
|
166
|
+
Volumes: { 'attachment.status' => 'attached' }
|
167
|
+
Each namespace uniquely support certian dimensions. Please consult the documentation
|
168
|
+
to ensure you're using valid filters.
|
169
|
+
|
170
|
+
[id="plugins-{type}s-{plugin}-interval"]
|
171
|
+
===== `interval`
|
172
|
+
|
173
|
+
* Value type is <<number,number>>
|
174
|
+
* Default value is `900`
|
175
|
+
|
176
|
+
Set how frequently CloudWatch should be queried
|
177
|
+
|
178
|
+
The default, `900`, means check every 15 minutes. Setting this value too low
|
179
|
+
(generally less than 300) results in no metrics being returned from CloudWatch.
|
180
|
+
|
181
|
+
[id="plugins-{type}s-{plugin}-metrics"]
|
182
|
+
===== `metrics`
|
183
|
+
|
184
|
+
* Value type is <<array,array>>
|
185
|
+
* Default value is `["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"]`
|
186
|
+
|
187
|
+
Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
|
188
|
+
for the available metrics for other namespaces.
|
189
|
+
|
190
|
+
[id="plugins-{type}s-{plugin}-namespace"]
|
191
|
+
===== `namespace`
|
192
|
+
|
193
|
+
* Value type is <<string,string>>
|
194
|
+
* Default value is `"AWS/EC2"`
|
195
|
+
|
196
|
+
If undefined, LogStash will complain, even if codec is unused.
|
197
|
+
The service namespace of the metrics to fetch.
|
198
|
+
|
199
|
+
The default is for the EC2 service. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
|
200
|
+
for valid values.
|
201
|
+
|
202
|
+
[id="plugins-{type}s-{plugin}-period"]
|
203
|
+
===== `period`
|
204
|
+
|
205
|
+
* Value type is <<number,number>>
|
206
|
+
* Default value is `300`
|
207
|
+
|
208
|
+
Set the granularity of the returned datapoints.
|
209
|
+
|
210
|
+
Must be at least 60 seconds and in multiples of 60.
|
211
|
+
|
212
|
+
[id="plugins-{type}s-{plugin}-proxy_uri"]
|
213
|
+
===== `proxy_uri`
|
214
|
+
|
215
|
+
* Value type is <<string,string>>
|
216
|
+
* There is no default value for this setting.
|
217
|
+
|
218
|
+
URI to proxy server if required
|
219
|
+
|
220
|
+
[id="plugins-{type}s-{plugin}-region"]
|
221
|
+
===== `region`
|
222
|
+
|
223
|
+
* Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
|
224
|
+
* Default value is `"us-east-1"`
|
225
|
+
|
226
|
+
The AWS Region
|
227
|
+
|
228
|
+
[id="plugins-{type}s-{plugin}-secret_access_key"]
|
229
|
+
===== `secret_access_key`
|
230
|
+
|
231
|
+
* Value type is <<string,string>>
|
232
|
+
* There is no default value for this setting.
|
233
|
+
|
234
|
+
The AWS Secret Access Key
|
235
|
+
|
236
|
+
[id="plugins-{type}s-{plugin}-session_token"]
|
237
|
+
===== `session_token`
|
238
|
+
|
239
|
+
* Value type is <<string,string>>
|
240
|
+
* There is no default value for this setting.
|
241
|
+
|
242
|
+
The AWS Session token for temporary credential
|
243
|
+
|
244
|
+
[id="plugins-{type}s-{plugin}-statistics"]
|
245
|
+
===== `statistics`
|
246
|
+
|
247
|
+
* Value type is <<array,array>>
|
248
|
+
* Default value is `["SampleCount", "Average", "Minimum", "Maximum", "Sum"]`
|
249
|
+
|
250
|
+
Specify the statistics to fetch for each namespace
|
251
|
+
|
252
|
+
[id="plugins-{type}s-{plugin}-use_ssl"]
|
253
|
+
===== `use_ssl`
|
254
|
+
|
255
|
+
* Value type is <<boolean,boolean>>
|
256
|
+
* Default value is `true`
|
257
|
+
|
258
|
+
Make sure we require the V1 classes when including this module.
|
259
|
+
require 'aws-sdk' will load v2 classes.
|
260
|
+
Should we require (true) or disable (false) using SSL for communicating with the AWS API
|
261
|
+
The AWS SDK for Ruby defaults to SSL so we preserve that
|
262
|
+
|
263
|
+
|
264
|
+
|
265
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
266
|
+
include::{include_path}/{type}.asciidoc[]
|
@@ -8,10 +8,12 @@ require "aws-sdk"
|
|
8
8
|
|
9
9
|
# Pull events from the Amazon Web Services CloudWatch API.
|
10
10
|
#
|
11
|
-
# To use this plugin, you *must* have an AWS account, and the following policy
|
11
|
+
# To use this plugin, you *must* have an AWS account, and the following policy.
|
12
12
|
#
|
13
13
|
# Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
|
14
|
+
#
|
14
15
|
# A sample policy for EC2 metrics is as follows:
|
16
|
+
#
|
15
17
|
# [source,json]
|
16
18
|
# {
|
17
19
|
# "Version": "2012-10-17",
|
@@ -78,11 +80,15 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
78
80
|
|
79
81
|
# The service namespace of the metrics to fetch.
|
80
82
|
#
|
81
|
-
# The default is for the EC2 service.
|
83
|
+
# The default is for the EC2 service.
|
84
|
+
#
|
85
|
+
# See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
|
82
86
|
# for valid values.
|
83
87
|
config :namespace, :validate => :string, :default => 'AWS/EC2'
|
84
88
|
|
85
|
-
# Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific.
|
89
|
+
# Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific.
|
90
|
+
#
|
91
|
+
# See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
|
86
92
|
# for the available metrics for other namespaces.
|
87
93
|
config :metrics, :validate => :array, :default => [ 'CPUUtilization', 'DiskReadOps', 'DiskWriteOps', 'NetworkIn', 'NetworkOut' ]
|
88
94
|
|
@@ -102,39 +108,41 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
102
108
|
|
103
109
|
# Specify the filters to apply when fetching resources:
|
104
110
|
#
|
111
|
+
# Instances: { 'instance-id' => 'i-12344321' }
|
112
|
+
# Tags: { 'tag:Environment' => 'Production' }
|
113
|
+
# Volumes: { 'attachment.status' => 'attached' }
|
114
|
+
#
|
105
115
|
# This needs to follow the AWS convention of specifiying filters.
|
106
|
-
#
|
107
|
-
#
|
108
|
-
# Volumes: { 'attachment.status' => 'attached' }
|
109
|
-
# Each namespace uniquely support certian dimensions. Please consult the documentation
|
116
|
+
#
|
117
|
+
# Each namespace uniquely supports certain dimensions. Consult the documentation
|
110
118
|
# to ensure you're using valid filters.
|
111
119
|
config :filters, :validate => :array, :required => true
|
112
120
|
|
113
121
|
# Use this for namespaces that need to combine the dimensions like S3 and SNS.
|
114
122
|
config :combined, :validate => :boolean, :default => false
|
115
123
|
|
116
|
-
public
|
117
124
|
def aws_service_endpoint(region)
|
118
125
|
{ region: region }
|
119
126
|
end
|
120
127
|
|
121
|
-
public
|
122
128
|
def register
|
123
129
|
AWS.config(:logger => @logger)
|
124
130
|
|
125
131
|
raise 'Interval needs to be higher than period' unless @interval >= @period
|
126
|
-
raise 'Interval must be divisible by
|
132
|
+
raise 'Interval must be divisible by period' unless @interval % @period == 0
|
127
133
|
|
128
134
|
@last_check = Time.now
|
129
135
|
end # def register
|
130
136
|
|
137
|
+
# Runs the poller to get metrics for the provided namespace
|
138
|
+
#
|
139
|
+
# @param queue [Array] Logstash queue
|
131
140
|
def run(queue)
|
132
141
|
Stud.interval(@interval) do
|
133
142
|
@logger.info('Polling CloudWatch API')
|
134
143
|
|
135
144
|
raise 'No metrics to query' unless metrics_for(@namespace).count > 0
|
136
145
|
|
137
|
-
# For every metric
|
138
146
|
metrics_for(@namespace).each do |metric|
|
139
147
|
@logger.info "Polling metric #{metric}"
|
140
148
|
@logger.info "Filters: #{aws_filters}"
|
@@ -144,6 +152,11 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
144
152
|
end # def run
|
145
153
|
|
146
154
|
private
|
155
|
+
|
156
|
+
# Gets metrics from provided resources.
|
157
|
+
#
|
158
|
+
# @param queue [Array] Logstash queue
|
159
|
+
# @param metric [String] Metric name
|
147
160
|
def from_resources(queue, metric)
|
148
161
|
# For every dimension in the metric
|
149
162
|
resources.each_pair do |dimension, dim_resources|
|
@@ -151,10 +164,13 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
151
164
|
dim_resources = *dim_resources
|
152
165
|
dim_resources.each do |resource|
|
153
166
|
@logger.info "Polling resource #{dimension}: #{resource}"
|
167
|
+
|
154
168
|
options = metric_options(@namespace, metric)
|
155
169
|
options[:dimensions] = [ { name: dimension, value: resource } ]
|
170
|
+
|
156
171
|
datapoints = clients['CloudWatch'].get_metric_statistics(options)
|
157
172
|
@logger.debug "DPs: #{datapoints.data}"
|
173
|
+
|
158
174
|
# For every event in the resource
|
159
175
|
datapoints[:datapoints].each do |event|
|
160
176
|
event.merge! options
|
@@ -167,25 +183,36 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
167
183
|
end
|
168
184
|
end
|
169
185
|
|
170
|
-
|
186
|
+
# Gets metrics from provided filter options
|
187
|
+
#
|
188
|
+
# @param queue [Array] Logstash queue
|
189
|
+
# @param metric [String] Metric name
|
171
190
|
def from_filters(queue, metric)
|
172
191
|
options = metric_options(@namespace, metric)
|
173
192
|
options[:dimensions] = aws_filters
|
174
193
|
@logger.info "Dim: #{options[:dimensions]}"
|
194
|
+
|
175
195
|
datapoints = clients['CloudWatch'].get_metric_statistics(options)
|
176
196
|
@logger.debug "DPs: #{datapoints.data}"
|
197
|
+
|
177
198
|
datapoints[:datapoints].each do |event|
|
178
199
|
event.merge! options
|
200
|
+
|
179
201
|
aws_filters.each do |dimension|
|
180
202
|
event[dimension[:name].to_sym] = dimension[:value]
|
181
203
|
end
|
204
|
+
|
182
205
|
event = LogStash::Event.new(cleanup(event))
|
183
206
|
decorate(event)
|
184
207
|
queue << event
|
185
208
|
end
|
186
209
|
end
|
187
210
|
|
188
|
-
|
211
|
+
# Cleans up an event to remove unneeded fields and format time
|
212
|
+
#
|
213
|
+
# @param event [Hash] Raw event
|
214
|
+
#
|
215
|
+
# @return [Hash] Cleaned event
|
189
216
|
def cleanup(event)
|
190
217
|
event.delete :statistics
|
191
218
|
event.delete :dimensions
|
@@ -195,35 +222,50 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
195
222
|
LogStash::Util.stringify_symbols(event)
|
196
223
|
end
|
197
224
|
|
198
|
-
|
225
|
+
# Dynamic AWS client instantiator for retrieving the proper client
|
226
|
+
# for the provided namespace
|
227
|
+
#
|
228
|
+
# @return [Hash]
|
199
229
|
def clients
|
200
|
-
@clients ||= Hash.new do |
|
201
|
-
|
202
|
-
|
203
|
-
cls = AWS.const_get(
|
204
|
-
|
230
|
+
@clients ||= Hash.new do |client_hash, namespace|
|
231
|
+
namespace = namespace[4..-1] if namespace[0..3] == 'AWS/'
|
232
|
+
namespace = 'EC2' if namespace == 'EBS'
|
233
|
+
cls = AWS.const_get(namespace)
|
234
|
+
client_hash[namespace] = cls::Client.new(aws_options_hash)
|
205
235
|
end
|
206
236
|
end
|
207
237
|
|
208
|
-
|
238
|
+
# Gets metrics for a provided namespace based on the union of available and
|
239
|
+
# found metrics
|
240
|
+
#
|
241
|
+
# @param namespace [String] Namespace to retrieve metrics for
|
242
|
+
#
|
243
|
+
# @return [Hash]
|
209
244
|
def metrics_for(namespace)
|
210
245
|
metrics_available[namespace] & @metrics
|
211
246
|
end
|
212
247
|
|
213
|
-
|
248
|
+
# Gets available metrics for a given namespace
|
249
|
+
#
|
250
|
+
# @return [Hash]
|
214
251
|
def metrics_available
|
215
|
-
@metrics_available ||= Hash.new do |
|
216
|
-
|
252
|
+
@metrics_available ||= Hash.new do |metrics_hash, namespace|
|
253
|
+
metrics_hash[namespace] = []
|
217
254
|
|
218
|
-
|
219
|
-
|
220
|
-
h[k].push metrics[:metric_name]
|
255
|
+
clients['CloudWatch'].list_metrics({ namespace: namespace })[:metrics].each do |metrics|
|
256
|
+
metrics_hash[namespace].push metrics[:metric_name]
|
221
257
|
end
|
222
|
-
|
258
|
+
|
259
|
+
metrics_hash[namespace]
|
223
260
|
end
|
224
261
|
end
|
225
262
|
|
226
|
-
|
263
|
+
# Gets options for querying against Cloudwatch for a given metric and namespace
|
264
|
+
#
|
265
|
+
# @param namespace [String] Namespace to query in
|
266
|
+
# @param metric [String] Metric to query for
|
267
|
+
#
|
268
|
+
# @return [Hash]
|
227
269
|
def metric_options(namespace, metric)
|
228
270
|
{
|
229
271
|
namespace: namespace,
|
@@ -235,7 +277,9 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
235
277
|
}
|
236
278
|
end
|
237
279
|
|
238
|
-
|
280
|
+
# Filters used in querying the AWS SDK for resources
|
281
|
+
#
|
282
|
+
# @return [Array]
|
239
283
|
def aws_filters
|
240
284
|
@filters.collect do |key, value|
|
241
285
|
if @combined
|
@@ -247,21 +291,28 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
247
291
|
end
|
248
292
|
end
|
249
293
|
|
250
|
-
|
294
|
+
# Gets resources based on the provided namespace
|
295
|
+
#
|
296
|
+
# @see http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/CW_Support_For_AWS.html
|
297
|
+
#
|
298
|
+
# @return [Array]
|
251
299
|
def resources
|
252
|
-
# See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/CW_Support_For_AWS.html
|
253
300
|
case @namespace
|
254
301
|
when 'AWS/EC2'
|
255
302
|
instances = clients[@namespace].describe_instances(filters: aws_filters)[:reservation_set].collect do |r|
|
256
303
|
r[:instances_set].collect{ |i| i[:instance_id] }
|
257
304
|
end.flatten
|
305
|
+
|
258
306
|
@logger.debug "AWS/EC2 Instances: #{instances}"
|
307
|
+
|
259
308
|
{ 'InstanceId' => instances }
|
260
309
|
when 'AWS/EBS'
|
261
310
|
volumes = clients[@namespace].describe_volumes(filters: aws_filters)[:volume_set].collect do |a|
|
262
311
|
a[:attachment_set].collect{ |v| v[:volume_id] }
|
263
312
|
end.flatten
|
313
|
+
|
264
314
|
@logger.debug "AWS/EBS Volumes: #{volumes}"
|
315
|
+
|
265
316
|
{ 'VolumeId' => volumes }
|
266
317
|
else
|
267
318
|
@filters
|
@@ -1,27 +1,32 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
|
-
s.name
|
3
|
-
s.version = '2.0.
|
4
|
-
s.licenses
|
5
|
-
s.summary
|
6
|
-
s.description
|
7
|
-
s.authors
|
8
|
-
s.email
|
9
|
-
s.homepage
|
2
|
+
s.name = 'logstash-input-cloudwatch'
|
3
|
+
s.version = '2.0.1'
|
4
|
+
s.licenses = ['Apache License (2.0)']
|
5
|
+
s.summary = "Retrieve stats from AWS CloudWatch."
|
6
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
|
+
s.authors = ["Jurgens du Toit"]
|
8
|
+
s.email = 'jrgns@eagerelk.com'
|
9
|
+
s.homepage = "http://eagerelk.com"
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
# Files
|
13
13
|
s.files = Dir[
|
14
|
-
'lib/**/*',
|
15
|
-
'spec/**/*',
|
16
|
-
'vendor/**/*',
|
17
14
|
'*.gemspec',
|
18
15
|
'*.md',
|
19
16
|
'CONTRIBUTORS',
|
17
|
+
'docs/**/*',
|
20
18
|
'Gemfile',
|
19
|
+
'lib/**/*',
|
21
20
|
'LICENSE',
|
22
|
-
'NOTICE.TXT'
|
21
|
+
'NOTICE.TXT',
|
22
|
+
'spec/**/*',
|
23
|
+
'vendor/**/*',
|
24
|
+
'vendor/jar-dependencies/**/*.jar',
|
25
|
+
'vendor/jar-dependencies/**/*.rb',
|
26
|
+
'VERSION',
|
23
27
|
]
|
24
|
-
|
28
|
+
|
29
|
+
# Tests
|
25
30
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
26
31
|
|
27
32
|
# Special flag to let us know this is actually a logstash plugin
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-cloudwatch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jurgens du Toit
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -97,6 +97,7 @@ files:
|
|
97
97
|
- Gemfile
|
98
98
|
- LICENSE
|
99
99
|
- README.md
|
100
|
+
- docs/index.asciidoc
|
100
101
|
- lib/logstash/inputs/cloudwatch.rb
|
101
102
|
- logstash-input-cloudwatch.gemspec
|
102
103
|
- spec/inputs/cloudwatch_spec.rb
|