logstash-output-cloudwatch 3.0.4 → 3.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06ba2d836bb306142cd381fb4b2d0e7a820d573f
-  data.tar.gz: 616e1a673b73112bdd7892198111c43822945d6d
+  metadata.gz: 5b80229009d2a02d4db6f01a42d11fcee2265919
+  data.tar.gz: d5b20fa08def241e33724cf46ea2ad4732bc76ba
 SHA512:
-  metadata.gz: cdd220a70723b17954ea5015985734d05f729d45168f49d5a6764d50e306195a69f15b06385ad90e78e485f68ca46c59ef4a956cd4d96f19a5dc0fa4cf684a9e
-  data.tar.gz: e61785036233e12668ba4bb8e0f2e336d1248a501c755a71d1cad5c4e62c04b34ca4cb3d2e54ac8e1a1afc7e42553ac19fbe210f36bb32784078992b2bcb9869
+  metadata.gz: 191ba5084ba7b60b99f8e9beee4a8d61854ad85f4f04aaf0a501318023406d2fafe9633df842e1a443e3ed1af6f96d86022e54b9f8111923f96a4aebb50de6c1
+  data.tar.gz: 88772fc8b5da22162f3b0be236c47405782f30c5a6bc2408dac845a09014ae7b443e5958d0e240c374b2f76a5821af60ba91faa7b3f6e7b0ca25e078fca7913b

data/Gemfile

@@ -1,4 +1,11 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,317 @@
+:plugin: cloudwatch
+:type: output
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Cloudwatch output plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This output lets you aggregate and send metric data to AWS CloudWatch
+
+==== Summary:
+This plugin is intended to be used on a logstash indexer agent (but that
+is not the only way, see below.)  In the intended scenario, one cloudwatch
+output plugin is configured, on the logstash indexer node, with just AWS API
+credentials, and possibly a region and/or a namespace.  The output looks
+for fields present in events, and when it finds them, it uses them to
+calculate aggregate statistics.  If the `metricname` option is set in this
+output, then any events which pass through it will be aggregated & sent to
+CloudWatch, but that is not recommended.  The intended use is to NOT set the
+metricname option here, and instead to add a `CW_metricname` field (and other
+fields) to only the events you want sent to CloudWatch.
+
+When events pass through this output they are queued for background
+aggregation and sending, which happens every minute by default.  The
+queue has a maximum size, and when it is full aggregated statistics will be
+sent to CloudWatch ahead of schedule. Whenever this happens a warning
+message is written to logstash's log.  If you see this you should increase
+the `queue_size` configuration option to avoid the extra API calls.  The queue
+is emptied every time we send data to CloudWatch.
+
+Note: when logstash is stopped the queue is destroyed before it can be processed.
+This is a known limitation of logstash and will hopefully be addressed in a
+future version.
+
+==== Details:
+There are two ways to configure this plugin, and they can be used in
+combination: event fields & per-output defaults
+
+Event Field configuration...
+You add fields to your events in inputs & filters and this output reads
+those fields to aggregate events.  The names of the fields read are
+configurable via the `field_*` options.
+
+Per-output defaults...
+You set universal defaults in this output plugin's configuration, and
+if an event does not have a field for that option then the default is
+used.
+
+Notice, the event fields take precedence over the per-output defaults.
+
+At a minimum events must have a "metric name" to be sent to CloudWatch.
+This can be achieved either by providing a default here OR by adding a
+`CW_metricname` field. By default, if no other configuration is provided
+besides a metric name, then events will be counted (Unit: Count, Value: 1)
+by their metric name (either a default or from their `CW_metricname` field)
+
+Other fields which can be added to events to modify the behavior of this
+plugin are, `CW_namespace`, `CW_unit`, `CW_value`, and
+`CW_dimensions`.  All of these field names are configurable in
+this output.  You can also set per-output defaults for any of them.
+See below for details.
+
+Read more about http://aws.amazon.com/cloudwatch/[AWS CloudWatch],
+and the specific of API endpoint this output uses,
+http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/API_PutMetricData.html[PutMetricData]
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Cloudwatch Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-batch_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-dimensions>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-field_dimensions>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-field_metricname>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-field_namespace>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-field_unit>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-field_value>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-metricname>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-namespace>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-queue_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-timeframe>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-unit>> |<<string,string>>, one of `["Seconds", "Microseconds", "Milliseconds", "Bytes", "Kilobytes", "Megabytes", "Gigabytes", "Terabytes", "Bits", "Kilobits", "Megabits", "Gigabits", "Terabits", "Percent", "Count", "Bytes/Second", "Kilobytes/Second", "Megabytes/Second", "Gigabytes/Second", "Terabytes/Second", "Bits/Second", "Kilobits/Second", "Megabits/Second", "Gigabits/Second", "Terabits/Second", "Count/Second", "None"]`|No
+| <<plugins-{type}s-{plugin}-value>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+2. External credentials file specified by `aws_credentials_file`
+3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+5. IAM Instance Profile (available when running inside EC2)
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-batch_size"]
+===== `batch_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `20`
+
+How many data points can be given in one call to the CloudWatch API
+
+[id="plugins-{type}s-{plugin}-dimensions"]
+===== `dimensions` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+The default dimensions [ name, value, ... ] to use for events which do not have a `CW_dimensions` field
+
+[id="plugins-{type}s-{plugin}-field_dimensions"]
+===== `field_dimensions` 
+
+  * Value type is <<string,string>>
+  * Default value is `"CW_dimensions"`
+
+The name of the field used to set the dimensions on an event metric
+The field named here, if present in an event, must have an array of
+one or more key & value pairs, for example...
+    `add_field => [ "CW_dimensions", "Environment", "CW_dimensions", "prod" ]`
+or, equivalently...
+    `add_field => [ "CW_dimensions", "Environment" ]`
+    `add_field => [ "CW_dimensions", "prod" ]`
+
+[id="plugins-{type}s-{plugin}-field_metricname"]
+===== `field_metricname` 
+
+  * Value type is <<string,string>>
+  * Default value is `"CW_metricname"`
+
+The name of the field used to set the metric name on an event
+The author of this plugin recommends adding this field to events in inputs &
+filters rather than using the per-output default setting so that one output
+plugin on your logstash indexer can serve all events (which of course had
+fields set on your logstash shippers.)
+
+[id="plugins-{type}s-{plugin}-field_namespace"]
+===== `field_namespace` 
+
+  * Value type is <<string,string>>
+  * Default value is `"CW_namespace"`
+
+The name of the field used to set a different namespace per event
+Note: Only one namespace can be sent to CloudWatch per API call
+so setting different namespaces will increase the number of API calls
+and those cost money.
+
+[id="plugins-{type}s-{plugin}-field_unit"]
+===== `field_unit` 
+
+  * Value type is <<string,string>>
+  * Default value is `"CW_unit"`
+
+The name of the field used to set the unit on an event metric
+
+[id="plugins-{type}s-{plugin}-field_value"]
+===== `field_value` 
+
+  * Value type is <<string,string>>
+  * Default value is `"CW_value"`
+
+The name of the field used to set the value (float) on an event metric
+
+[id="plugins-{type}s-{plugin}-metricname"]
+===== `metricname` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The default metric name to use for events which do not have a `CW_metricname` field.
+Beware: If this is provided then all events which pass through this output will be aggregated and
+sent to CloudWatch, so use this carefully.  Furthermore, when providing this option, you
+will probably want to also restrict events from passing through this output using event
+type, tag, and field matching
+
+[id="plugins-{type}s-{plugin}-namespace"]
+===== `namespace` 
+
+  * Value type is <<string,string>>
+  * Default value is `"Logstash"`
+
+The default namespace to use for events which do not have a `CW_namespace` field
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-queue_size"]
+===== `queue_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `10000`
+
+How many events to queue before forcing a call to the CloudWatch API ahead of `timeframe` schedule
+Set this to the number of events-per-timeframe you will be sending to CloudWatch to avoid extra API calls
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+[id="plugins-{type}s-{plugin}-timeframe"]
+===== `timeframe` 
+
+  * Value type is <<string,string>>
+  * Default value is `"1m"`
+
+Constants
+aggregate_key members
+Units
+How often to send data to CloudWatch
+This does not affect the event timestamps, events will always have their
+actual timestamp (to-the-minute) sent to CloudWatch.
+
+We only call the API if there is data to send.
+
+See the Rufus Scheduler docs for an https://github.com/jmettraux/rufus-scheduler#the-time-strings-understood-by-rufus-scheduler[explanation of allowed values]
+
+[id="plugins-{type}s-{plugin}-unit"]
+===== `unit` 
+
+  * Value can be any of: `Seconds`, `Microseconds`, `Milliseconds`, `Bytes`, `Kilobytes`, `Megabytes`, `Gigabytes`, `Terabytes`, `Bits`, `Kilobits`, `Megabits`, `Gigabits`, `Terabits`, `Percent`, `Count`, `Bytes/Second`, `Kilobytes/Second`, `Megabytes/Second`, `Gigabytes/Second`, `Terabytes/Second`, `Bits/Second`, `Kilobits/Second`, `Megabits/Second`, `Gigabits/Second`, `Terabits/Second`, `Count/Second`, `None`
+  * Default value is `"Count"`
+
+The default unit to use for events which do not have a `CW_unit` field
+If you set this option you should probably set the "value" option along with it
+
+[id="plugins-{type}s-{plugin}-value"]
+===== `value` 
+
+  * Value type is <<string,string>>
+  * Default value is `"1"`
+
+The default value to use for events which do not have a `CW_value` field
+If provided, this must be a string which can be converted to a float, for example...
+    "1", "2.34", ".5", and "0.67"
+If you set this option you should probably set the `unit` option along with it
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/logstash-output-cloudwatch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-cloudwatch'
-  s.version         = '3.0.4'
+  s.version         = '3.0.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output lets you aggregate and send metric data to AWS CloudWatch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-cloudwatch
 version: !ruby/object:Gem::Version
-  version: 3.0.4
+  version: 3.0.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-16 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/outputs/cloudwatch.rb
 - logstash-output-cloudwatch.gemspec
 - spec/outputs/cloudwatch_spec.rb