logstash-input-cloudwatch 2.1.1 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -2
- data/README.md +1 -1
- data/docs/index.asciidoc +7 -3
- data/lib/logstash/inputs/cloudwatch.rb +41 -27
- data/logstash-input-cloudwatch.gemspec +2 -2
- data/spec/inputs/cloudwatch_spec.rb +54 -48
- data/spec/integration/cloudwatch_spec.rb +26 -0
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 62466bcf82aa9ead9c7b348572f537a814ab3061a7d8788186679a8f8e9f212c
|
4
|
+
data.tar.gz: ca93c360fd186bb7b1e81da1541b0b9670be8a5e626d296676cd437e3c9e1d0e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ac855312c07e22f78f8451b17d837746f10d4bd5bf3656b53800c2853dd033d676876fb4a1eb2827552fd31f4df057602a611f3dd1adff0290a9213264147c9d
|
7
|
+
data.tar.gz: 17a4eb451e7a5acd26c392e64aa7917a51bbf5b15080e4a6015092488e889c55928e19712d4562e856e8e5c8f99d7dfdb362afe6328fcde93ef4626634770317
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
## 2.2.2
|
2
|
+
- Added ability to use AWS/EC2 namespace without requiring filters
|
3
|
+
|
4
|
+
## 2.2.1
|
5
|
+
- Fixed README.md link to request metric support to point to this repo [#34](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/34)
|
6
|
+
|
7
|
+
## 2.2.0
|
8
|
+
- Changed to use the underlying version of the AWS SDK to v2. [#32](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/32)
|
9
|
+
- Fixed License definition in gemspec to be valid SPDX identifier [#32](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/32)
|
10
|
+
- Fixed fatal error when using secret key attribute in config [#30](https://github.com/logstash-plugins/logstash-input-cloudwatch/issues/30)
|
11
|
+
|
1
12
|
## 2.1.1
|
2
13
|
- Docs: Set the default_codec doc attribute.
|
3
14
|
|
@@ -15,5 +26,5 @@
|
|
15
26
|
- Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
|
16
27
|
# 1.1.1
|
17
28
|
- New dependency requirements for logstash-core for the 5.0 release
|
18
|
-
## 1.1.0
|
19
|
-
- Moved from jrgns/logstash-input-cloudwatch to logstash-plugins
|
29
|
+
## 1.1.0
|
30
|
+
- Moved from jrgns/logstash-input-cloudwatch to logstash-plugins
|
data/README.md
CHANGED
@@ -70,6 +70,6 @@ Just note that the below configuration doesn't contain the AWS API access inform
|
|
70
70
|
|
71
71
|
See AWS Developer Guide for more information on [namespaces and metrics][2].
|
72
72
|
|
73
|
-
[1]: https://github.com/
|
73
|
+
[1]: https://github.com/logstash-plugins/logstash-input-cloudwatch/labels/metric%20support
|
74
74
|
[2]: http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
|
75
75
|
[3]: http://aws.amazon.com/iam/
|
data/docs/index.asciidoc
CHANGED
@@ -53,7 +53,7 @@ A sample policy for EC2 metrics is as follows:
|
|
53
53
|
|
54
54
|
See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
|
55
55
|
|
56
|
-
|
56
|
+
===== Configuration Example
|
57
57
|
[source, ruby]
|
58
58
|
input {
|
59
59
|
cloudwatch {
|
@@ -64,6 +64,7 @@ See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
|
|
64
64
|
}
|
65
65
|
}
|
66
66
|
|
67
|
+
[source, ruby]
|
67
68
|
input {
|
68
69
|
cloudwatch {
|
69
70
|
namespace => "AWS/EBS"
|
@@ -73,6 +74,7 @@ See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
|
|
73
74
|
}
|
74
75
|
}
|
75
76
|
|
77
|
+
[source, ruby]
|
76
78
|
input {
|
77
79
|
cloudwatch {
|
78
80
|
namespace => "AWS/RDS"
|
@@ -95,7 +97,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
95
97
|
| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
|
96
98
|
| <<plugins-{type}s-{plugin}-combined>> |<<boolean,boolean>>|No
|
97
99
|
| <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
|
98
|
-
| <<plugins-{type}s-{plugin}-filters>> |<<array,array>>|
|
100
|
+
| <<plugins-{type}s-{plugin}-filters>> |<<array,array>>|See <<plugins-{type}s-{plugin}-filters,note>>
|
99
101
|
| <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
|
100
102
|
| <<plugins-{type}s-{plugin}-metrics>> |<<array,array>>|No
|
101
103
|
| <<plugins-{type}s-{plugin}-namespace>> |<<string,string>>|No
|
@@ -168,10 +170,12 @@ guaranteed to work correctly with the AWS SDK.
|
|
168
170
|
[id="plugins-{type}s-{plugin}-filters"]
|
169
171
|
===== `filters`
|
170
172
|
|
171
|
-
* This
|
173
|
+
* This setting can be required or optional. See note below.
|
172
174
|
* Value type is <<array,array>>
|
173
175
|
* There is no default value for this setting.
|
174
176
|
|
177
|
+
NOTE: This setting is optional when the namespace is `AWS/EC2` - otherwise this is a required field.
|
178
|
+
|
175
179
|
Specify the filters to apply when fetching resources:
|
176
180
|
|
177
181
|
This needs to follow the AWS convention of specifiying filters.
|
@@ -71,7 +71,7 @@ require "aws-sdk"
|
|
71
71
|
#
|
72
72
|
|
73
73
|
class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
74
|
-
include LogStash::PluginMixins::AwsConfig
|
74
|
+
include LogStash::PluginMixins::AwsConfig::V2
|
75
75
|
|
76
76
|
config_name "cloudwatch"
|
77
77
|
|
@@ -116,7 +116,7 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
116
116
|
#
|
117
117
|
# Each namespace uniquely supports certain dimensions. Consult the documentation
|
118
118
|
# to ensure you're using valid filters.
|
119
|
-
config :filters, :validate => :array
|
119
|
+
config :filters, :validate => :array
|
120
120
|
|
121
121
|
# Use this for namespaces that need to combine the dimensions like S3 and SNS.
|
122
122
|
config :combined, :validate => :boolean, :default => false
|
@@ -126,14 +126,22 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
126
126
|
end
|
127
127
|
|
128
128
|
def register
|
129
|
-
AWS.config(:logger => @logger)
|
130
|
-
|
131
129
|
raise 'Interval needs to be higher than period' unless @interval >= @period
|
132
130
|
raise 'Interval must be divisible by period' unless @interval % @period == 0
|
131
|
+
raise "Filters must be defined for when using #{@namespace} namespace" if @filters.nil? && filters_required?(@namespace)
|
133
132
|
|
134
133
|
@last_check = Time.now
|
135
134
|
end # def register
|
136
135
|
|
136
|
+
def filters_required?(namespace)
|
137
|
+
case namespace
|
138
|
+
when 'AWS/EC2'
|
139
|
+
false
|
140
|
+
else
|
141
|
+
true
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
137
145
|
# Runs the poller to get metrics for the provided namespace
|
138
146
|
#
|
139
147
|
# @param queue [Array] Logstash queue
|
@@ -143,10 +151,15 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
143
151
|
|
144
152
|
raise 'No metrics to query' unless metrics_for(@namespace).count > 0
|
145
153
|
|
154
|
+
# For every metric
|
146
155
|
metrics_for(@namespace).each do |metric|
|
147
156
|
@logger.debug "Polling metric #{metric}"
|
148
|
-
@
|
149
|
-
|
157
|
+
if @filters.nil?
|
158
|
+
from_resources(queue, metric)
|
159
|
+
else
|
160
|
+
@logger.debug "Filters: #{aws_filters}"
|
161
|
+
@combined ? from_filters(queue, metric) : from_resources(queue, metric)
|
162
|
+
end
|
150
163
|
end
|
151
164
|
end # loop
|
152
165
|
end # def run
|
@@ -170,12 +183,11 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
170
183
|
|
171
184
|
datapoints = clients['CloudWatch'].get_metric_statistics(options)
|
172
185
|
@logger.debug "DPs: #{datapoints.data}"
|
173
|
-
|
174
186
|
# For every event in the resource
|
175
|
-
datapoints[:datapoints].each do |
|
176
|
-
|
177
|
-
|
178
|
-
event = LogStash::Event.new(cleanup(
|
187
|
+
datapoints[:datapoints].each do |datapoint|
|
188
|
+
event_hash = datapoint.to_hash.update(options)
|
189
|
+
event_hash[dimension.to_sym] = resource
|
190
|
+
event = LogStash::Event.new(cleanup(event_hash))
|
179
191
|
decorate(event)
|
180
192
|
queue << event
|
181
193
|
end
|
@@ -195,14 +207,13 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
195
207
|
datapoints = clients['CloudWatch'].get_metric_statistics(options)
|
196
208
|
@logger.debug "DPs: #{datapoints.data}"
|
197
209
|
|
198
|
-
datapoints[:datapoints].each do |
|
199
|
-
|
200
|
-
|
210
|
+
datapoints[:datapoints].each do |datapoint|
|
211
|
+
event_hash = datapoint.to_hash.update(options)
|
201
212
|
aws_filters.each do |dimension|
|
202
|
-
|
213
|
+
event_hash[dimension[:name].to_sym] = dimension[:value]
|
203
214
|
end
|
204
215
|
|
205
|
-
event = LogStash::Event.new(cleanup(
|
216
|
+
event = LogStash::Event.new(cleanup(event_hash))
|
206
217
|
decorate(event)
|
207
218
|
queue << event
|
208
219
|
end
|
@@ -230,8 +241,9 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
230
241
|
@clients ||= Hash.new do |client_hash, namespace|
|
231
242
|
namespace = namespace[4..-1] if namespace[0..3] == 'AWS/'
|
232
243
|
namespace = 'EC2' if namespace == 'EBS'
|
233
|
-
cls =
|
234
|
-
|
244
|
+
cls = Aws.const_get(namespace)
|
245
|
+
# TODO: Move logger configuration into mixin.
|
246
|
+
client_hash[namespace] = cls::Client.new(aws_options_hash.merge(:logger => @logger))
|
235
247
|
end
|
236
248
|
end
|
237
249
|
|
@@ -255,7 +267,6 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
255
267
|
clients['CloudWatch'].list_metrics({ namespace: namespace })[:metrics].each do |metrics|
|
256
268
|
metrics_hash[namespace].push metrics[:metric_name]
|
257
269
|
end
|
258
|
-
|
259
270
|
metrics_hash[namespace]
|
260
271
|
end
|
261
272
|
end
|
@@ -298,17 +309,15 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
298
309
|
# @return [Array]
|
299
310
|
def resources
|
300
311
|
case @namespace
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
@logger.debug "AWS/EC2 Instances: #{instances}"
|
312
|
+
when 'AWS/EC2'
|
313
|
+
instances = clients[@namespace].describe_instances(filter_options)[:reservations].collect do |r|
|
314
|
+
r[:instances].collect{ |i| i[:instance_id] }
|
315
|
+
end.flatten
|
307
316
|
|
308
317
|
{ 'InstanceId' => instances }
|
309
318
|
when 'AWS/EBS'
|
310
|
-
volumes = clients[@namespace].describe_volumes(filters: aws_filters)[:
|
311
|
-
a[:
|
319
|
+
volumes = clients[@namespace].describe_volumes(filters: aws_filters)[:volumes].collect do |a|
|
320
|
+
a[:attachments].collect{ |v| v[:volume_id] }
|
312
321
|
end.flatten
|
313
322
|
|
314
323
|
@logger.debug "AWS/EBS Volumes: #{volumes}"
|
@@ -318,4 +327,9 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
|
|
318
327
|
@filters
|
319
328
|
end
|
320
329
|
end
|
330
|
+
|
331
|
+
def filter_options
|
332
|
+
@filters.nil? ? {} : { :filters => aws_filters }
|
333
|
+
end
|
334
|
+
|
321
335
|
end # class LogStash::Inputs::CloudWatch
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-cloudwatch'
|
3
|
-
s.version = '2.
|
4
|
-
s.licenses = ['Apache
|
3
|
+
s.version = '2.2.2'
|
4
|
+
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Pulls events from the Amazon Web Services CloudWatch API "
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
7
|
s.authors = ["Jurgens du Toit"]
|
@@ -3,64 +3,70 @@ require 'logstash/inputs/cloudwatch'
|
|
3
3
|
require 'aws-sdk'
|
4
4
|
|
5
5
|
describe LogStash::Inputs::CloudWatch do
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
end
|
10
|
-
|
11
|
-
describe '#register' do
|
12
|
-
let(:config) {
|
13
|
-
{
|
6
|
+
subject { LogStash::Inputs::CloudWatch.new(config) }
|
7
|
+
let(:config) {
|
8
|
+
{
|
14
9
|
'access_key_id' => '1234',
|
15
10
|
'secret_access_key' => 'secret',
|
16
|
-
'
|
17
|
-
'filters' => { 'instance-id' => 'i-12344321' },
|
11
|
+
'metrics' => [ 'CPUUtilization' ],
|
18
12
|
'region' => 'us-east-1'
|
19
|
-
}
|
20
13
|
}
|
21
|
-
|
14
|
+
}
|
22
15
|
|
23
|
-
|
24
|
-
|
25
|
-
|
16
|
+
|
17
|
+
before do
|
18
|
+
Aws.config[:stub_responses] = true
|
19
|
+
Thread.abort_on_exception = true
|
26
20
|
end
|
27
21
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
22
|
+
shared_examples_for 'it requires filters' do
|
23
|
+
context 'without filters' do
|
24
|
+
it "raises an error" do
|
25
|
+
expect { subject.register }.to raise_error(StandardError)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
context 'with filters' do
|
30
|
+
let (:config) { super().merge('filters' => { 'tag:Monitoring' => 'Yes' })}
|
31
|
+
|
32
|
+
it "registers succesfully" do
|
33
|
+
expect { subject.register }.to_not raise_error
|
34
|
+
end
|
35
|
+
end
|
39
36
|
end
|
40
37
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
38
|
+
shared_examples_for 'it does not require filters' do
|
39
|
+
context 'without filters' do
|
40
|
+
it "registers succesfully" do
|
41
|
+
expect { subject.register }.to_not raise_error
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
context 'with filters' do
|
46
|
+
let (:config) { super().merge('filters' => { 'tag:Monitoring' => 'Yes' })}
|
47
|
+
|
48
|
+
it "registers succesfully" do
|
49
|
+
expect { subject.register }.to_not raise_error
|
50
|
+
end
|
51
|
+
end
|
52
52
|
end
|
53
53
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
54
|
+
describe '#register' do
|
55
|
+
|
56
|
+
context "EC2 namespace" do
|
57
|
+
let(:config) { super().merge('namespace' => 'AWS/EC2') }
|
58
|
+
it_behaves_like 'it does not require filters'
|
59
|
+
end
|
60
|
+
|
61
|
+
context "EBS namespace" do
|
62
|
+
let(:config) { super().merge('namespace' => 'AWS/EBS') }
|
63
|
+
it_behaves_like 'it requires filters'
|
64
|
+
end
|
65
|
+
|
66
|
+
context "RDS namespace" do
|
67
|
+
let(:config) { super().merge('namespace' => 'AWS/RDS') }
|
68
|
+
it_behaves_like 'it requires filters'
|
69
|
+
end
|
70
|
+
|
65
71
|
end
|
66
72
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require "logstash/inputs/cloudwatch"
|
3
|
+
require "aws-sdk"
|
4
|
+
|
5
|
+
describe LogStash::Inputs::CloudWatch, :integration => true do
|
6
|
+
|
7
|
+
let(:settings) { { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
|
8
|
+
"secret_access_key" => LogStash::Util::Password.new(ENV['AWS_SECRET_ACCESS_KEY']),
|
9
|
+
"region" => ENV["AWS_REGION"] || "us-east-1",
|
10
|
+
"namespace" => "AWS/S3",
|
11
|
+
'filters' => { "BucketName" => "*"},
|
12
|
+
'metrics' => ["BucketSizeBytes","NumberOfObjects"]
|
13
|
+
|
14
|
+
}}
|
15
|
+
|
16
|
+
def metrics_for(settings)
|
17
|
+
cw = LogStash::Inputs::CloudWatch.new(settings)
|
18
|
+
cw.register
|
19
|
+
cw.send('metrics_for', settings['namespace'])
|
20
|
+
end
|
21
|
+
|
22
|
+
#
|
23
|
+
it "should not raise a type error when using a password" do
|
24
|
+
expect{metrics_for(settings)}.not_to raise_error
|
25
|
+
end
|
26
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-cloudwatch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jurgens du Toit
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -103,9 +103,10 @@ files:
|
|
103
103
|
- lib/logstash/inputs/cloudwatch.rb
|
104
104
|
- logstash-input-cloudwatch.gemspec
|
105
105
|
- spec/inputs/cloudwatch_spec.rb
|
106
|
+
- spec/integration/cloudwatch_spec.rb
|
106
107
|
homepage: http://eagerelk.com
|
107
108
|
licenses:
|
108
|
-
- Apache
|
109
|
+
- Apache-2.0
|
109
110
|
metadata:
|
110
111
|
logstash_plugin: 'true'
|
111
112
|
logstash_group: input
|
@@ -125,9 +126,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
125
126
|
version: '0'
|
126
127
|
requirements: []
|
127
128
|
rubyforge_project:
|
128
|
-
rubygems_version: 2.6.
|
129
|
+
rubygems_version: 2.6.13
|
129
130
|
signing_key:
|
130
131
|
specification_version: 4
|
131
132
|
summary: Pulls events from the Amazon Web Services CloudWatch API
|
132
133
|
test_files:
|
133
134
|
- spec/inputs/cloudwatch_spec.rb
|
135
|
+
- spec/integration/cloudwatch_spec.rb
|