logstash-output-cloudwatch 3.0.4 → 3.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +8 -1
- data/docs/index.asciidoc +317 -0
- data/logstash-output-cloudwatch.gemspec +2 -2
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5b80229009d2a02d4db6f01a42d11fcee2265919
|
4
|
+
data.tar.gz: d5b20fa08def241e33724cf46ea2ad4732bc76ba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 191ba5084ba7b60b99f8e9beee4a8d61854ad85f4f04aaf0a501318023406d2fafe9633df842e1a443e3ed1af6f96d86022e54b9f8111923f96a4aebb50de6c1
|
7
|
+
data.tar.gz: 88772fc8b5da22162f3b0be236c47405782f30c5a6bc2408dac845a09014ae7b443e5958d0e240c374b2f76a5821af60ba91faa7b3f6e7b0ca25e078fca7913b
|
data/Gemfile
CHANGED
@@ -1,4 +1,11 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
2
|
|
3
|
-
# Specify your gem's dependencies in logstash-mass_effect.gemspec
|
4
3
|
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,317 @@
|
|
1
|
+
:plugin: cloudwatch
|
2
|
+
:type: output
|
3
|
+
|
4
|
+
///////////////////////////////////////////
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
6
|
+
///////////////////////////////////////////
|
7
|
+
:version: %VERSION%
|
8
|
+
:release_date: %RELEASE_DATE%
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
10
|
+
:include_path: ../../../../logstash/docs/include
|
11
|
+
///////////////////////////////////////////
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
13
|
+
///////////////////////////////////////////
|
14
|
+
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
16
|
+
|
17
|
+
=== Cloudwatch output plugin
|
18
|
+
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
|
21
|
+
==== Description
|
22
|
+
|
23
|
+
This output lets you aggregate and send metric data to AWS CloudWatch
|
24
|
+
|
25
|
+
==== Summary:
|
26
|
+
This plugin is intended to be used on a logstash indexer agent (but that
|
27
|
+
is not the only way, see below.) In the intended scenario, one cloudwatch
|
28
|
+
output plugin is configured, on the logstash indexer node, with just AWS API
|
29
|
+
credentials, and possibly a region and/or a namespace. The output looks
|
30
|
+
for fields present in events, and when it finds them, it uses them to
|
31
|
+
calculate aggregate statistics. If the `metricname` option is set in this
|
32
|
+
output, then any events which pass through it will be aggregated & sent to
|
33
|
+
CloudWatch, but that is not recommended. The intended use is to NOT set the
|
34
|
+
metricname option here, and instead to add a `CW_metricname` field (and other
|
35
|
+
fields) to only the events you want sent to CloudWatch.
|
36
|
+
|
37
|
+
When events pass through this output they are queued for background
|
38
|
+
aggregation and sending, which happens every minute by default. The
|
39
|
+
queue has a maximum size, and when it is full aggregated statistics will be
|
40
|
+
sent to CloudWatch ahead of schedule. Whenever this happens a warning
|
41
|
+
message is written to logstash's log. If you see this you should increase
|
42
|
+
the `queue_size` configuration option to avoid the extra API calls. The queue
|
43
|
+
is emptied every time we send data to CloudWatch.
|
44
|
+
|
45
|
+
Note: when logstash is stopped the queue is destroyed before it can be processed.
|
46
|
+
This is a known limitation of logstash and will hopefully be addressed in a
|
47
|
+
future version.
|
48
|
+
|
49
|
+
==== Details:
|
50
|
+
There are two ways to configure this plugin, and they can be used in
|
51
|
+
combination: event fields & per-output defaults
|
52
|
+
|
53
|
+
Event Field configuration...
|
54
|
+
You add fields to your events in inputs & filters and this output reads
|
55
|
+
those fields to aggregate events. The names of the fields read are
|
56
|
+
configurable via the `field_*` options.
|
57
|
+
|
58
|
+
Per-output defaults...
|
59
|
+
You set universal defaults in this output plugin's configuration, and
|
60
|
+
if an event does not have a field for that option then the default is
|
61
|
+
used.
|
62
|
+
|
63
|
+
Notice, the event fields take precedence over the per-output defaults.
|
64
|
+
|
65
|
+
At a minimum events must have a "metric name" to be sent to CloudWatch.
|
66
|
+
This can be achieved either by providing a default here OR by adding a
|
67
|
+
`CW_metricname` field. By default, if no other configuration is provided
|
68
|
+
besides a metric name, then events will be counted (Unit: Count, Value: 1)
|
69
|
+
by their metric name (either a default or from their `CW_metricname` field)
|
70
|
+
|
71
|
+
Other fields which can be added to events to modify the behavior of this
|
72
|
+
plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and
|
73
|
+
`CW_dimensions`. All of these field names are configurable in
|
74
|
+
this output. You can also set per-output defaults for any of them.
|
75
|
+
See below for details.
|
76
|
+
|
77
|
+
Read more about http://aws.amazon.com/cloudwatch/[AWS CloudWatch],
|
78
|
+
and the specific of API endpoint this output uses,
|
79
|
+
http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html[PutMetricData]
|
80
|
+
|
81
|
+
[id="plugins-{type}s-{plugin}-options"]
|
82
|
+
==== Cloudwatch Output Configuration Options
|
83
|
+
|
84
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
85
|
+
|
86
|
+
[cols="<,<,<",options="header",]
|
87
|
+
|=======================================================================
|
88
|
+
|Setting |Input type|Required
|
89
|
+
| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
|
90
|
+
| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
|
91
|
+
| <<plugins-{type}s-{plugin}-batch_size>> |<<number,number>>|No
|
92
|
+
| <<plugins-{type}s-{plugin}-dimensions>> |<<hash,hash>>|No
|
93
|
+
| <<plugins-{type}s-{plugin}-field_dimensions>> |<<string,string>>|No
|
94
|
+
| <<plugins-{type}s-{plugin}-field_metricname>> |<<string,string>>|No
|
95
|
+
| <<plugins-{type}s-{plugin}-field_namespace>> |<<string,string>>|No
|
96
|
+
| <<plugins-{type}s-{plugin}-field_unit>> |<<string,string>>|No
|
97
|
+
| <<plugins-{type}s-{plugin}-field_value>> |<<string,string>>|No
|
98
|
+
| <<plugins-{type}s-{plugin}-metricname>> |<<string,string>>|No
|
99
|
+
| <<plugins-{type}s-{plugin}-namespace>> |<<string,string>>|No
|
100
|
+
| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
|
101
|
+
| <<plugins-{type}s-{plugin}-queue_size>> |<<number,number>>|No
|
102
|
+
| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
|
103
|
+
| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
|
104
|
+
| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
|
105
|
+
| <<plugins-{type}s-{plugin}-timeframe>> |<<string,string>>|No
|
106
|
+
| <<plugins-{type}s-{plugin}-unit>> |<<string,string>>, one of `["Seconds", "Microseconds", "Milliseconds", "Bytes", "Kilobytes", "Megabytes", "Gigabytes", "Terabytes", "Bits", "Kilobits", "Megabits", "Gigabits", "Terabits", "Percent", "Count", "Bytes/Second", "Kilobytes/Second", "Megabytes/Second", "Gigabytes/Second", "Terabytes/Second", "Bits/Second", "Kilobits/Second", "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", "None"]`|No
|
107
|
+
| <<plugins-{type}s-{plugin}-value>> |<<string,string>>|No
|
108
|
+
|=======================================================================
|
109
|
+
|
110
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
111
|
+
output plugins.
|
112
|
+
|
113
|
+
|
114
|
+
|
115
|
+
[id="plugins-{type}s-{plugin}-access_key_id"]
|
116
|
+
===== `access_key_id`
|
117
|
+
|
118
|
+
* Value type is <<string,string>>
|
119
|
+
* There is no default value for this setting.
|
120
|
+
|
121
|
+
This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
|
122
|
+
|
123
|
+
1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
|
124
|
+
2. External credentials file specified by `aws_credentials_file`
|
125
|
+
3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
|
126
|
+
4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
|
127
|
+
5. IAM Instance Profile (available when running inside EC2)
|
128
|
+
|
129
|
+
[id="plugins-{type}s-{plugin}-aws_credentials_file"]
|
130
|
+
===== `aws_credentials_file`
|
131
|
+
|
132
|
+
* Value type is <<string,string>>
|
133
|
+
* There is no default value for this setting.
|
134
|
+
|
135
|
+
Path to YAML file containing a hash of AWS credentials.
|
136
|
+
This file will only be loaded if `access_key_id` and
|
137
|
+
`secret_access_key` aren't set. The contents of the
|
138
|
+
file should look like this:
|
139
|
+
|
140
|
+
[source,ruby]
|
141
|
+
----------------------------------
|
142
|
+
:access_key_id: "12345"
|
143
|
+
:secret_access_key: "54321"
|
144
|
+
----------------------------------
|
145
|
+
|
146
|
+
|
147
|
+
[id="plugins-{type}s-{plugin}-batch_size"]
|
148
|
+
===== `batch_size`
|
149
|
+
|
150
|
+
* Value type is <<number,number>>
|
151
|
+
* Default value is `20`
|
152
|
+
|
153
|
+
How many data points can be given in one call to the CloudWatch API
|
154
|
+
|
155
|
+
[id="plugins-{type}s-{plugin}-dimensions"]
|
156
|
+
===== `dimensions`
|
157
|
+
|
158
|
+
* Value type is <<hash,hash>>
|
159
|
+
* There is no default value for this setting.
|
160
|
+
|
161
|
+
The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field
|
162
|
+
|
163
|
+
[id="plugins-{type}s-{plugin}-field_dimensions"]
|
164
|
+
===== `field_dimensions`
|
165
|
+
|
166
|
+
* Value type is <<string,string>>
|
167
|
+
* Default value is `"CW_dimensions"`
|
168
|
+
|
169
|
+
The name of the field used to set the dimensions on an event metric
|
170
|
+
The field named here, if present in an event, must have an array of
|
171
|
+
one or more key & value pairs, for example...
|
172
|
+
`add_field => [ "CW_dimensions", "Environment", "CW_dimensions", "prod" ]`
|
173
|
+
or, equivalently...
|
174
|
+
`add_field => [ "CW_dimensions", "Environment" ]`
|
175
|
+
`add_field => [ "CW_dimensions", "prod" ]`
|
176
|
+
|
177
|
+
[id="plugins-{type}s-{plugin}-field_metricname"]
|
178
|
+
===== `field_metricname`
|
179
|
+
|
180
|
+
* Value type is <<string,string>>
|
181
|
+
* Default value is `"CW_metricname"`
|
182
|
+
|
183
|
+
The name of the field used to set the metric name on an event
|
184
|
+
The author of this plugin recommends adding this field to events in inputs &
|
185
|
+
filters rather than using the per-output default setting so that one output
|
186
|
+
plugin on your logstash indexer can serve all events (which of course had
|
187
|
+
fields set on your logstash shippers.)
|
188
|
+
|
189
|
+
[id="plugins-{type}s-{plugin}-field_namespace"]
|
190
|
+
===== `field_namespace`
|
191
|
+
|
192
|
+
* Value type is <<string,string>>
|
193
|
+
* Default value is `"CW_namespace"`
|
194
|
+
|
195
|
+
The name of the field used to set a different namespace per event
|
196
|
+
Note: Only one namespace can be sent to CloudWatch per API call
|
197
|
+
so setting different namespaces will increase the number of API calls
|
198
|
+
and those cost money.
|
199
|
+
|
200
|
+
[id="plugins-{type}s-{plugin}-field_unit"]
|
201
|
+
===== `field_unit`
|
202
|
+
|
203
|
+
* Value type is <<string,string>>
|
204
|
+
* Default value is `"CW_unit"`
|
205
|
+
|
206
|
+
The name of the field used to set the unit on an event metric
|
207
|
+
|
208
|
+
[id="plugins-{type}s-{plugin}-field_value"]
|
209
|
+
===== `field_value`
|
210
|
+
|
211
|
+
* Value type is <<string,string>>
|
212
|
+
* Default value is `"CW_value"`
|
213
|
+
|
214
|
+
The name of the field used to set the value (float) on an event metric
|
215
|
+
|
216
|
+
[id="plugins-{type}s-{plugin}-metricname"]
|
217
|
+
===== `metricname`
|
218
|
+
|
219
|
+
* Value type is <<string,string>>
|
220
|
+
* There is no default value for this setting.
|
221
|
+
|
222
|
+
The default metric name to use for events which do not have a `CW_metricname` field.
|
223
|
+
Beware: If this is provided then all events which pass through this output will be aggregated and
|
224
|
+
sent to CloudWatch, so use this carefully. Furthermore, when providing this option, you
|
225
|
+
will probably want to also restrict events from passing through this output using event
|
226
|
+
type, tag, and field matching
|
227
|
+
|
228
|
+
[id="plugins-{type}s-{plugin}-namespace"]
|
229
|
+
===== `namespace`
|
230
|
+
|
231
|
+
* Value type is <<string,string>>
|
232
|
+
* Default value is `"Logstash"`
|
233
|
+
|
234
|
+
The default namespace to use for events which do not have a `CW_namespace` field
|
235
|
+
|
236
|
+
[id="plugins-{type}s-{plugin}-proxy_uri"]
|
237
|
+
===== `proxy_uri`
|
238
|
+
|
239
|
+
* Value type is <<string,string>>
|
240
|
+
* There is no default value for this setting.
|
241
|
+
|
242
|
+
URI to proxy server if required
|
243
|
+
|
244
|
+
[id="plugins-{type}s-{plugin}-queue_size"]
|
245
|
+
===== `queue_size`
|
246
|
+
|
247
|
+
* Value type is <<number,number>>
|
248
|
+
* Default value is `10000`
|
249
|
+
|
250
|
+
How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule
|
251
|
+
Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls
|
252
|
+
|
253
|
+
[id="plugins-{type}s-{plugin}-region"]
|
254
|
+
===== `region`
|
255
|
+
|
256
|
+
* Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
|
257
|
+
* Default value is `"us-east-1"`
|
258
|
+
|
259
|
+
The AWS Region
|
260
|
+
|
261
|
+
[id="plugins-{type}s-{plugin}-secret_access_key"]
|
262
|
+
===== `secret_access_key`
|
263
|
+
|
264
|
+
* Value type is <<string,string>>
|
265
|
+
* There is no default value for this setting.
|
266
|
+
|
267
|
+
The AWS Secret Access Key
|
268
|
+
|
269
|
+
[id="plugins-{type}s-{plugin}-session_token"]
|
270
|
+
===== `session_token`
|
271
|
+
|
272
|
+
* Value type is <<string,string>>
|
273
|
+
* There is no default value for this setting.
|
274
|
+
|
275
|
+
The AWS Session token for temporary credential
|
276
|
+
|
277
|
+
[id="plugins-{type}s-{plugin}-timeframe"]
|
278
|
+
===== `timeframe`
|
279
|
+
|
280
|
+
* Value type is <<string,string>>
|
281
|
+
* Default value is `"1m"`
|
282
|
+
|
283
|
+
Constants
|
284
|
+
aggregate_key members
|
285
|
+
Units
|
286
|
+
How often to send data to CloudWatch
|
287
|
+
This does not affect the event timestamps, events will always have their
|
288
|
+
actual timestamp (to-the-minute) sent to CloudWatch.
|
289
|
+
|
290
|
+
We only call the API if there is data to send.
|
291
|
+
|
292
|
+
See the Rufus Scheduler docs for an https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler[explanation of allowed values]
|
293
|
+
|
294
|
+
[id="plugins-{type}s-{plugin}-unit"]
|
295
|
+
===== `unit`
|
296
|
+
|
297
|
+
* Value can be any of: `Seconds`, `Microseconds`, `Milliseconds`, `Bytes`, `Kilobytes`, `Megabytes`, `Gigabytes`, `Terabytes`, `Bits`, `Kilobits`, `Megabits`, `Gigabits`, `Terabits`, `Percent`, `Count`, `Bytes/Second`, `Kilobytes/Second`, `Megabytes/Second`, `Gigabytes/Second`, `Terabytes/Second`, `Bits/Second`, `Kilobits/Second`, `Megabits/Second`, `Gigabits/Second`, `Terabits/Second`, `Count/Second`, `None`
|
298
|
+
* Default value is `"Count"`
|
299
|
+
|
300
|
+
The default unit to use for events which do not have a `CW_unit` field
|
301
|
+
If you set this option you should probably set the "value" option along with it
|
302
|
+
|
303
|
+
[id="plugins-{type}s-{plugin}-value"]
|
304
|
+
===== `value`
|
305
|
+
|
306
|
+
* Value type is <<string,string>>
|
307
|
+
* Default value is `"1"`
|
308
|
+
|
309
|
+
The default value to use for events which do not have a `CW_value` field
|
310
|
+
If provided, this must be a string which can be converted to a float, for example...
|
311
|
+
"1", "2.34", ".5", and "0.67"
|
312
|
+
If you set this option you should probably set the `unit` option along with it
|
313
|
+
|
314
|
+
|
315
|
+
|
316
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
317
|
+
include::{include_path}/{type}.asciidoc[]
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-cloudwatch'
|
4
|
-
s.version = '3.0.
|
4
|
+
s.version = '3.0.5'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "This output lets you aggregate and send metric data to AWS CloudWatch"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
|
11
11
|
s.require_paths = ["lib"]
|
12
12
|
|
13
13
|
# Files
|
14
|
-
s.files = Dir[
|
14
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
15
15
|
|
16
16
|
# Tests
|
17
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-cloudwatch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,6 +84,7 @@ files:
|
|
84
84
|
- LICENSE
|
85
85
|
- NOTICE.TXT
|
86
86
|
- README.md
|
87
|
+
- docs/index.asciidoc
|
87
88
|
- lib/logstash/outputs/cloudwatch.rb
|
88
89
|
- logstash-output-cloudwatch.gemspec
|
89
90
|
- spec/outputs/cloudwatch_spec.rb
|