logstash-input-cloudwatch 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49bf38d2bb591bfd80a7b586f1f824215f7166ad
-  data.tar.gz: 38710ee89440aabde152e85bfe8f9734b20ee6c1
+  metadata.gz: 9b00ee77394f455a6f10c3520133ad01a07ba862
+  data.tar.gz: c6c12406db0d518f93a07b0c3303cf97f3d137c9
 SHA512:
-  metadata.gz: da633ee271f1989e8a9ce25f690c3182abb2ba92e0447fd7f488080c430691111d5c67ae3c9c643afa2481c906b89c511613108395789372bfd6faca3c6f8e8c
-  data.tar.gz: e0f3a68b078b67f4ca337b1ed1dae95d35684769d756012f5887b4c37a0a424a939f47fc03472f532ec226854ff466605f3b69d83dba301ec9ead3da3810eeae
+  metadata.gz: 080344bbd37d1533e166e93855d283e7d3355f6b045155946076d0e8ffc4b364b84ddee3beed4d4337094297e957fe4f902ae3a8f48b89e79fb09076822df9bc
+  data.tar.gz: b851865091d5ea2aaf0e7937bf9d47ff48d9a901a43933bd4c2f3df3e9c2c320bfd776499ea4d26bc4b373a4d1e62ae89e7fe142bfa1683f4b15c945204ba982

data/Gemfile

@@ -1,2 +1,11 @@
 source 'https://rubygems.org'
+
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,266 @@
+:plugin: cloudwatch
+:type: input
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Cloudwatch input plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Pull events from the Amazon Web Services CloudWatch API.
+
+To use this plugin, you *must* have an AWS account, and the following policy
+
+Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
+A sample policy for EC2 metrics is as follows:
+[source,json]
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Sid": "Stmt1444715676000",
+                "Effect": "Allow",
+                "Action": [
+                    "cloudwatch:GetMetricStatistics",
+                    "cloudwatch:ListMetrics"
+                ],
+                "Resource": "*"
+            },
+            {
+                "Sid": "Stmt1444716576170",
+                "Effect": "Allow",
+                "Action": [
+                    "ec2:DescribeInstances"
+                ],
+                "Resource": "*"
+            }
+        ]
+    }
+
+See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
+
+# Configuration Example
+[source, ruby]
+    input {
+      cloudwatch {
+        namespace => "AWS/EC2"
+        metrics => [ "CPUUtilization" ]
+        filters => { "tag:Group" => "API-Production" }
+        region => "us-east-1"
+      }
+    }
+
+    input {
+      cloudwatch {
+        namespace => "AWS/EBS"
+        metrics => ["VolumeQueueLength"]
+        filters => { "tag:Monitoring" => "Yes" }
+        region => "us-east-1"
+      }
+    }
+
+    input {
+      cloudwatch {
+        namespace => "AWS/RDS"
+        metrics => ["CPUUtilization", "CPUCreditUsage"]
+        filters => { "EngineName" => "mysql" } # Only supports EngineName, DatabaseClass and DBInstanceIdentifier
+        region => "us-east-1"
+      }
+    }
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Cloudwatch Input Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-access_key_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-combined>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-filters>> |<<array,array>>|Yes
+| <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-metrics>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-namespace>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-period>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-region>> |<<string,string>>, one of `["us-east-1", "us-east-2", "us-west-1", "us-west-2", "eu-central-1", "eu-west-1", "eu-west-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ap-northeast-2", "sa-east-1", "us-gov-west-1", "cn-north-1", "ap-south-1", "ca-central-1"]`|No
+| <<plugins-{type}s-{plugin}-secret_access_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-statistics>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-use_ssl>> |<<boolean,boolean>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+input plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-access_key_id"]
+===== `access_key_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+
+1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+2. External credentials file specified by `aws_credentials_file`
+3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+5. IAM Instance Profile (available when running inside EC2)
+
+[id="plugins-{type}s-{plugin}-aws_credentials_file"]
+===== `aws_credentials_file` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Path to YAML file containing a hash of AWS credentials.
+This file will only be loaded if `access_key_id` and
+`secret_access_key` aren't set. The contents of the
+file should look like this:
+
+[source,ruby]
+----------------------------------
+    :access_key_id: "12345"
+    :secret_access_key: "54321"
+----------------------------------
+
+
+[id="plugins-{type}s-{plugin}-combined"]
+===== `combined` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Use this for namespaces that need to combine the dimensions like S3 and SNS.
+
+[id="plugins-{type}s-{plugin}-filters"]
+===== `filters` 
+
+  * This is a required setting.
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+Specify the filters to apply when fetching resources:
+
+This needs to follow the AWS convention of specifiying filters.
+Instances: { 'instance-id' => 'i-12344321' }
+Tags: { "tag:Environment" => "Production" }
+Volumes: { 'attachment.status' => 'attached' }
+Each namespace uniquely support certian dimensions. Please consult the documentation
+to ensure you're using valid filters.
+
+[id="plugins-{type}s-{plugin}-interval"]
+===== `interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `900`
+
+Set how frequently CloudWatch should be queried
+
+The default, `900`, means check every 15 minutes. Setting this value too low
+(generally less than 300) results in no metrics being returned from CloudWatch.
+
+[id="plugins-{type}s-{plugin}-metrics"]
+===== `metrics` 
+
+  * Value type is <<array,array>>
+  * Default value is `["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"]`
+
+Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
+for the available metrics for other namespaces.
+
+[id="plugins-{type}s-{plugin}-namespace"]
+===== `namespace` 
+
+  * Value type is <<string,string>>
+  * Default value is `"AWS/EC2"`
+
+If undefined, LogStash will complain, even if codec is unused.
+The service namespace of the metrics to fetch.
+
+The default is for the EC2 service. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
+for valid values.
+
+[id="plugins-{type}s-{plugin}-period"]
+===== `period` 
+
+  * Value type is <<number,number>>
+  * Default value is `300`
+
+Set the granularity of the returned datapoints.
+
+Must be at least 60 seconds and in multiples of 60.
+
+[id="plugins-{type}s-{plugin}-proxy_uri"]
+===== `proxy_uri` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+URI to proxy server if required
+
+[id="plugins-{type}s-{plugin}-region"]
+===== `region` 
+
+  * Value can be any of: `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`, `eu-central-1`, `eu-west-1`, `eu-west-2`, `ap-southeast-1`, `ap-southeast-2`, `ap-northeast-1`, `ap-northeast-2`, `sa-east-1`, `us-gov-west-1`, `cn-north-1`, `ap-south-1`, `ca-central-1`
+  * Default value is `"us-east-1"`
+
+The AWS Region
+
+[id="plugins-{type}s-{plugin}-secret_access_key"]
+===== `secret_access_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Secret Access Key
+
+[id="plugins-{type}s-{plugin}-session_token"]
+===== `session_token` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The AWS Session token for temporary credential
+
+[id="plugins-{type}s-{plugin}-statistics"]
+===== `statistics` 
+
+  * Value type is <<array,array>>
+  * Default value is `["SampleCount", "Average", "Minimum", "Maximum", "Sum"]`
+
+Specify the statistics to fetch for each namespace
+
+[id="plugins-{type}s-{plugin}-use_ssl"]
+===== `use_ssl` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Make sure we require the V1 classes when including this module.
+require 'aws-sdk' will load v2 classes.
+Should we require (true) or disable (false) using SSL for communicating with the AWS API
+The AWS SDK for Ruby defaults to SSL so we preserve that
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/cloudwatch.rb

@@ -8,10 +8,12 @@ require "aws-sdk"
 
 # Pull events from the Amazon Web Services CloudWatch API.
 #
-# To use this plugin, you *must* have an AWS account, and the following policy
+# To use this plugin, you *must* have an AWS account, and the following policy.
 #
 # Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
+#
 # A sample policy for EC2 metrics is as follows:
+#
 # [source,json]
 #     {
 #         "Version": "2012-10-17",
@@ -78,11 +80,15 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
 
   # The service namespace of the metrics to fetch.
   #
-  # The default is for the EC2 service. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
+  # The default is for the EC2 service.
+  #
+  # See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
   # for valid values.
   config :namespace, :validate => :string, :default => 'AWS/EC2'
 
-  # Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific. See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
+  # Specify the metrics to fetch for the namespace. The defaults are AWS/EC2 specific.
+  #
+  # See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
   # for the available metrics for other namespaces.
   config :metrics, :validate => :array, :default => [ 'CPUUtilization', 'DiskReadOps', 'DiskWriteOps', 'NetworkIn', 'NetworkOut' ]
 
@@ -102,39 +108,41 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
 
   # Specify the filters to apply when fetching resources:
   #
+  #     Instances: { 'instance-id' => 'i-12344321' }
+  #     Tags:      { 'tag:Environment' => 'Production' }
+  #     Volumes:   { 'attachment.status' => 'attached' }
+  #
   # This needs to follow the AWS convention of specifiying filters.
-  # Instances: { 'instance-id' => 'i-12344321' }
-  # Tags: { "tag:Environment" => "Production" }
-  # Volumes: { 'attachment.status' => 'attached' }
-  # Each namespace uniquely support certian dimensions. Please consult the documentation
+  #
+  # Each namespace uniquely supports certain dimensions. Consult the documentation
   # to ensure you're using valid filters.
   config :filters, :validate => :array, :required => true
 
   # Use this for namespaces that need to combine the dimensions like S3 and SNS.
   config :combined, :validate => :boolean, :default => false
 
-  public
   def aws_service_endpoint(region)
     { region: region }
   end
 
-  public
   def register
     AWS.config(:logger => @logger)
 
     raise 'Interval needs to be higher than period' unless @interval >= @period
-    raise 'Interval must be divisible by peruid' unless @interval % @period == 0
+    raise 'Interval must be divisible by period' unless @interval % @period == 0
 
     @last_check = Time.now
   end # def register
 
+  # Runs the poller to get metrics for the provided namespace
+  #
+  # @param queue [Array] Logstash queue
   def run(queue)
     Stud.interval(@interval) do
       @logger.info('Polling CloudWatch API')
 
       raise 'No metrics to query' unless metrics_for(@namespace).count > 0
 
-      # For every metric
       metrics_for(@namespace).each do |metric|
         @logger.info "Polling metric #{metric}"
         @logger.info "Filters: #{aws_filters}"
@@ -144,6 +152,11 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
   end # def run
 
   private
+
+  # Gets metrics from provided resources.
+  #
+  # @param queue  [Array]  Logstash queue
+  # @param metric [String] Metric name
   def from_resources(queue, metric)
     # For every dimension in the metric
     resources.each_pair do |dimension, dim_resources|
@@ -151,10 +164,13 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
       dim_resources = *dim_resources
       dim_resources.each do |resource|
         @logger.info "Polling resource #{dimension}: #{resource}"
+
         options = metric_options(@namespace, metric)
         options[:dimensions] = [ { name: dimension, value: resource } ]
+
         datapoints = clients['CloudWatch'].get_metric_statistics(options)
         @logger.debug "DPs: #{datapoints.data}"
+
         # For every event in the resource
         datapoints[:datapoints].each do |event|
           event.merge! options
@@ -167,25 +183,36 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
     end
   end
 
-  private
+  # Gets metrics from provided filter options
+  #
+  # @param queue  [Array]  Logstash queue
+  # @param metric [String] Metric name
   def from_filters(queue, metric)
     options = metric_options(@namespace, metric)
     options[:dimensions] = aws_filters
     @logger.info "Dim: #{options[:dimensions]}"
+
     datapoints = clients['CloudWatch'].get_metric_statistics(options)
     @logger.debug "DPs: #{datapoints.data}"
+
     datapoints[:datapoints].each do |event|
       event.merge! options
+
       aws_filters.each do |dimension|
         event[dimension[:name].to_sym] = dimension[:value]
       end
+
       event = LogStash::Event.new(cleanup(event))
       decorate(event)
       queue << event
     end
   end
 
-  private
+  # Cleans up an event to remove unneeded fields and format time
+  #
+  # @param event [Hash] Raw event
+  #
+  # @return [Hash] Cleaned event
   def cleanup(event)
     event.delete :statistics
     event.delete :dimensions
@@ -195,35 +222,50 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
     LogStash::Util.stringify_symbols(event)
   end
 
-  private
+  # Dynamic AWS client instantiator for retrieving the proper client
+  # for the provided namespace
+  #
+  # @return [Hash]
   def clients
-    @clients ||= Hash.new do |h, k|
-      k = k[4..-1] if k[0..3] == 'AWS/'
-      k = 'EC2' if k == 'EBS'
-      cls = AWS.const_get(k)
-      h[k] = cls::Client.new(aws_options_hash)
+    @clients ||= Hash.new do |client_hash, namespace|
+      namespace = namespace[4..-1] if namespace[0..3] == 'AWS/'
+      namespace = 'EC2' if namespace == 'EBS'
+      cls = AWS.const_get(namespace)
+      client_hash[namespace] = cls::Client.new(aws_options_hash)
     end
   end
 
-  private
+  # Gets metrics for a provided namespace based on the union of available and
+  # found metrics
+  #
+  # @param namespace [String] Namespace to retrieve metrics for
+  #
+  # @return [Hash]
   def metrics_for(namespace)
     metrics_available[namespace] & @metrics
   end
 
-  private
+  # Gets available metrics for a given namespace
+  #
+  # @return [Hash]
   def metrics_available
-    @metrics_available ||= Hash.new do |h, k|
-      h[k] = []
+    @metrics_available ||= Hash.new do |metrics_hash, namespace|
+      metrics_hash[namespace] = []
 
-      options = { namespace: k }
-      clients['CloudWatch'].list_metrics(options)[:metrics].each do |metrics|
-        h[k].push metrics[:metric_name]
+      clients['CloudWatch'].list_metrics({ namespace: namespace })[:metrics].each do |metrics|
+        metrics_hash[namespace].push metrics[:metric_name]
       end
-      h[k]
+
+      metrics_hash[namespace]
     end
   end
 
-  private
+  # Gets options for querying against Cloudwatch for a given metric and namespace
+  #
+  # @param namespace [String] Namespace to query in
+  # @param metric    [String] Metric to query for
+  #
+  # @return [Hash]
   def metric_options(namespace, metric)
     {
       namespace: namespace,
@@ -235,7 +277,9 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
     }
   end
 
-  private
+  # Filters used in querying the AWS SDK for resources
+  #
+  # @return [Array]
   def aws_filters
     @filters.collect do |key, value|
       if @combined
@@ -247,21 +291,28 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
     end
   end
 
-  private
+  # Gets resources based on the provided namespace
+  #
+  # @see http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/CW_Support_For_AWS.html
+  #
+  # @return [Array]
   def resources
-    # See http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/CW_Support_For_AWS.html
     case @namespace
     when 'AWS/EC2'
       instances = clients[@namespace].describe_instances(filters: aws_filters)[:reservation_set].collect do |r|
         r[:instances_set].collect{ |i| i[:instance_id] }
       end.flatten
+
       @logger.debug "AWS/EC2 Instances: #{instances}"
+
       { 'InstanceId' => instances }
     when 'AWS/EBS'
       volumes = clients[@namespace].describe_volumes(filters: aws_filters)[:volume_set].collect do |a|
         a[:attachment_set].collect{ |v| v[:volume_id] }
       end.flatten
+
       @logger.debug "AWS/EBS Volumes: #{volumes}"
+
       { 'VolumeId' => volumes }
     else
       @filters

data/logstash-input-cloudwatch.gemspec

@@ -1,27 +1,32 @@
 Gem::Specification.new do |s|
-  s.name = 'logstash-input-cloudwatch'
-  s.version         = '2.0.0'
-  s.licenses = ['Apache License (2.0)']
-  s.summary = "Retrieve stats from AWS CloudWatch."
-  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
-  s.authors = ["Jurgens du Toit"]
-  s.email = 'jrgns@eagerelk.com'
-  s.homepage = "http://eagerelk.com"
+  s.name          = 'logstash-input-cloudwatch'
+  s.version         = '2.0.1'
+  s.licenses      = ['Apache License (2.0)']
+  s.summary       = "Retrieve stats from AWS CloudWatch."
+  s.description   = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors       = ["Jurgens du Toit"]
+  s.email         = 'jrgns@eagerelk.com'
+  s.homepage      = "http://eagerelk.com"
   s.require_paths = ["lib"]
 
   # Files
   s.files = Dir[
-    'lib/**/*',
-    'spec/**/*',
-    'vendor/**/*',
     '*.gemspec',
     '*.md',
     'CONTRIBUTORS',
+    'docs/**/*',
     'Gemfile',
+    'lib/**/*',
     'LICENSE',
-    'NOTICE.TXT'
+    'NOTICE.TXT',
+    'spec/**/*',
+    'vendor/**/*',
+    'vendor/jar-dependencies/**/*.jar',
+    'vendor/jar-dependencies/**/*.rb',
+    'VERSION',
   ]
-   # Tests
+
+  # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-cloudwatch
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Jurgens du Toit
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-22 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -97,6 +97,7 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- docs/index.asciidoc
 - lib/logstash/inputs/cloudwatch.rb
 - logstash-input-cloudwatch.gemspec
 - spec/inputs/cloudwatch_spec.rb