logstash-input-cloudwatch 2.1.1 → 2.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73adc950ac53eba1c1eb8c01b57b3dc5092d73ffa09d9cfa341269f57e2f5e3d
-  data.tar.gz: 2518385dbf20aba8a433517bd694e22cb665dae9be39a0b2d257838f9d4ac290
+  metadata.gz: 62466bcf82aa9ead9c7b348572f537a814ab3061a7d8788186679a8f8e9f212c
+  data.tar.gz: ca93c360fd186bb7b1e81da1541b0b9670be8a5e626d296676cd437e3c9e1d0e
 SHA512:
-  metadata.gz: eec7f5bf3b2fa83b4f7aaf05b261b041d80cad0b4f1a9173906dc9c7a15deb750791fc9d08043a2403912aa4bc827999a333a1d5b2e7b75326e07d99f035db1b
-  data.tar.gz: 225a0d3511b3598e795b0f6c324e1a76f2b81931c3aea6e6a3901bf30332298c7eb0784adac56d413a86a612c7900fae1b430a4731ce718d4c8e3d547ba737b4
+  metadata.gz: ac855312c07e22f78f8451b17d837746f10d4bd5bf3656b53800c2853dd033d676876fb4a1eb2827552fd31f4df057602a611f3dd1adff0290a9213264147c9d
+  data.tar.gz: 17a4eb451e7a5acd26c392e64aa7917a51bbf5b15080e4a6015092488e889c55928e19712d4562e856e8e5c8f99d7dfdb362afe6328fcde93ef4626634770317

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 2.2.2
+  - Added ability to use AWS/EC2 namespace without requiring filters
+
+## 2.2.1
+  - Fixed README.md link to request metric support to point to this repo [#34](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/34)
+
+## 2.2.0
+  - Changed to use the underlying version of the AWS SDK to v2. [#32](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/32)
+  - Fixed License definition in gemspec to be valid SPDX identifier [#32](https://github.com/logstash-plugins/logstash-input-cloudwatch/pull/32)
+  - Fixed fatal error when using secret key attribute in config [#30](https://github.com/logstash-plugins/logstash-input-cloudwatch/issues/30)
+
 ## 2.1.1
   - Docs: Set the default_codec doc attribute.
 
@@ -15,5 +26,5 @@
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 1.1.1
   - New dependency requirements for logstash-core for the 5.0 release
-## 1.1.0
- - Moved from jrgns/logstash-input-cloudwatch to logstash-plugins
+## 1.1.0
+ - Moved from jrgns/logstash-input-cloudwatch to logstash-plugins

data/README.md

@@ -70,6 +70,6 @@ Just note that the below configuration doesn't contain the AWS API access inform
 
 See AWS Developer Guide for more information on [namespaces and metrics][2].
 
-[1]: https://github.com/EagerELK/logstash-input-cloudwatch/labels/metric%20support
+[1]: https://github.com/logstash-plugins/logstash-input-cloudwatch/labels/metric%20support
 [2]: http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/aws-namespaces.html
 [3]: http://aws.amazon.com/iam/

data/docs/index.asciidoc

@@ -53,7 +53,7 @@ A sample policy for EC2 metrics is as follows:
 
 See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
 
-# Configuration Example
+===== Configuration Example
 [source, ruby]
     input {
       cloudwatch {
@@ -64,6 +64,7 @@ See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
       }
     }
 
+[source, ruby]
     input {
       cloudwatch {
         namespace => "AWS/EBS"
@@ -73,6 +74,7 @@ See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
       }
     }
 
+[source, ruby]
     input {
       cloudwatch {
         namespace => "AWS/RDS"
@@ -95,7 +97,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-aws_credentials_file>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-combined>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-filters>> |<<array,array>>|Yes
+| <<plugins-{type}s-{plugin}-filters>> |<<array,array>>|See <<plugins-{type}s-{plugin}-filters,note>>
 | <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-metrics>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-namespace>> |<<string,string>>|No
@@ -168,10 +170,12 @@ guaranteed to work correctly with the AWS SDK.
 [id="plugins-{type}s-{plugin}-filters"]
 ===== `filters` 
 
-  * This is a required setting.
+  * This setting can be required or optional. See note below.
   * Value type is <<array,array>>
   * There is no default value for this setting.
 
+NOTE: This setting is optional when the namespace is `AWS/EC2` - otherwise this is a required field.
+
 Specify the filters to apply when fetching resources:
 
 This needs to follow the AWS convention of specifiying filters.

data/lib/logstash/inputs/cloudwatch.rb

@@ -71,7 +71,7 @@ require "aws-sdk"
 #
 
 class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
-  include LogStash::PluginMixins::AwsConfig
+  include LogStash::PluginMixins::AwsConfig::V2
 
   config_name "cloudwatch"
 
@@ -116,7 +116,7 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
   #
   # Each namespace uniquely supports certain dimensions. Consult the documentation
   # to ensure you're using valid filters.
-  config :filters, :validate => :array, :required => true
+  config :filters, :validate => :array
 
   # Use this for namespaces that need to combine the dimensions like S3 and SNS.
   config :combined, :validate => :boolean, :default => false
@@ -126,14 +126,22 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
   end
 
   def register
-    AWS.config(:logger => @logger)
-
     raise 'Interval needs to be higher than period' unless @interval >= @period
     raise 'Interval must be divisible by period' unless @interval % @period == 0
+    raise "Filters must be defined for when using #{@namespace} namespace" if @filters.nil? && filters_required?(@namespace)
 
     @last_check = Time.now
   end # def register
 
+  def filters_required?(namespace)
+    case namespace
+    when 'AWS/EC2'
+      false
+    else
+      true
+    end
+  end
+
   # Runs the poller to get metrics for the provided namespace
   #
   # @param queue [Array] Logstash queue
@@ -143,10 +151,15 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
 
       raise 'No metrics to query' unless metrics_for(@namespace).count > 0
 
+      # For every metric
       metrics_for(@namespace).each do |metric|
         @logger.debug "Polling metric #{metric}"
-        @logger.debug "Filters: #{aws_filters}"
-        @combined ? from_filters(queue, metric) : from_resources(queue, metric)
+        if @filters.nil?
+          from_resources(queue, metric)
+        else
+          @logger.debug "Filters: #{aws_filters}"
+          @combined ? from_filters(queue, metric) : from_resources(queue, metric)
+        end
       end
     end # loop
   end # def run
@@ -170,12 +183,11 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
 
         datapoints = clients['CloudWatch'].get_metric_statistics(options)
         @logger.debug "DPs: #{datapoints.data}"
-
         # For every event in the resource
-        datapoints[:datapoints].each do |event|
-          event.merge! options
-          event[dimension.to_sym] = resource
-          event = LogStash::Event.new(cleanup(event))
+        datapoints[:datapoints].each do |datapoint|
+          event_hash = datapoint.to_hash.update(options)
+          event_hash[dimension.to_sym] = resource
+          event = LogStash::Event.new(cleanup(event_hash))
           decorate(event)
           queue << event
         end
@@ -195,14 +207,13 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
     datapoints = clients['CloudWatch'].get_metric_statistics(options)
     @logger.debug "DPs: #{datapoints.data}"
 
-    datapoints[:datapoints].each do |event|
-      event.merge! options
-
+    datapoints[:datapoints].each do |datapoint|
+      event_hash = datapoint.to_hash.update(options)
       aws_filters.each do |dimension|
-        event[dimension[:name].to_sym] = dimension[:value]
+        event_hash[dimension[:name].to_sym] = dimension[:value]
       end
 
-      event = LogStash::Event.new(cleanup(event))
+      event = LogStash::Event.new(cleanup(event_hash))
       decorate(event)
       queue << event
     end
@@ -230,8 +241,9 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
     @clients ||= Hash.new do |client_hash, namespace|
       namespace = namespace[4..-1] if namespace[0..3] == 'AWS/'
       namespace = 'EC2' if namespace == 'EBS'
-      cls = AWS.const_get(namespace)
-      client_hash[namespace] = cls::Client.new(aws_options_hash)
+      cls = Aws.const_get(namespace)
+      # TODO: Move logger configuration into mixin.
+      client_hash[namespace] = cls::Client.new(aws_options_hash.merge(:logger => @logger))
     end
   end
 
@@ -255,7 +267,6 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
       clients['CloudWatch'].list_metrics({ namespace: namespace })[:metrics].each do |metrics|
         metrics_hash[namespace].push metrics[:metric_name]
       end
-
       metrics_hash[namespace]
     end
   end
@@ -298,17 +309,15 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
   # @return [Array]
   def resources
     case @namespace
-    when 'AWS/EC2'
-      instances = clients[@namespace].describe_instances(filters: aws_filters)[:reservation_set].collect do |r|
-        r[:instances_set].collect{ |i| i[:instance_id] }
-      end.flatten
-
-      @logger.debug "AWS/EC2 Instances: #{instances}"
+      when 'AWS/EC2'
+        instances = clients[@namespace].describe_instances(filter_options)[:reservations].collect do |r|
+          r[:instances].collect{ |i| i[:instance_id] }
+        end.flatten
 
       { 'InstanceId' => instances }
     when 'AWS/EBS'
-      volumes = clients[@namespace].describe_volumes(filters: aws_filters)[:volume_set].collect do |a|
-        a[:attachment_set].collect{ |v| v[:volume_id] }
+      volumes = clients[@namespace].describe_volumes(filters: aws_filters)[:volumes].collect do |a|
+        a[:attachments].collect{ |v| v[:volume_id] }
       end.flatten
 
       @logger.debug "AWS/EBS Volumes: #{volumes}"
@@ -318,4 +327,9 @@ class LogStash::Inputs::CloudWatch < LogStash::Inputs::Base
       @filters
     end
   end
+
+  def filter_options
+    @filters.nil? ? {} : { :filters => aws_filters }
+  end
+
 end # class LogStash::Inputs::CloudWatch

data/logstash-input-cloudwatch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-cloudwatch'
-  s.version         = '2.1.1'
-  s.licenses      = ['Apache License (2.0)']
+  s.version         = '2.2.2'
+  s.licenses      = ['Apache-2.0']
   s.summary       = "Pulls events from the Amazon Web Services CloudWatch API "
   s.description   = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors       = ["Jurgens du Toit"]

data/spec/inputs/cloudwatch_spec.rb

@@ -3,64 +3,70 @@ require 'logstash/inputs/cloudwatch'
 require 'aws-sdk'
 
 describe LogStash::Inputs::CloudWatch do
-  before do
-    AWS.stub!
-    Thread.abort_on_exception = true
-  end
-
-  describe '#register' do
-    let(:config) {
-      {
+  subject { LogStash::Inputs::CloudWatch.new(config) }
+  let(:config) {
+    {
         'access_key_id' => '1234',
         'secret_access_key' => 'secret',
-        'namespace' => 'AWS/EC2',
-        'filters' => { 'instance-id' => 'i-12344321' },
+        'metrics' => [ 'CPUUtilization' ],
         'region' => 'us-east-1'
-      }
     }
-    subject { LogStash::Inputs::CloudWatch.new(config) }
+  }
 
-    it "registers succesfully" do
-      expect { subject.register }.to_not raise_error
-    end
+
+  before do
+    Aws.config[:stub_responses] = true
+    Thread.abort_on_exception = true
   end
 
-  context "EC2 events" do
-    let(:config) {
-      {
-        'access_key_id' => '1234',
-        'secret_access_key' => 'secret',
-        'namespace' => 'AWS/EC2',
-        'metrics' => [ 'CPUUtilization' ],
-        'filters' => { 'tag:Monitoring' => 'Yes' },
-        'region' => 'us-east-1'
-      }
-    }
+  shared_examples_for 'it requires filters' do
+    context 'without filters' do
+      it "raises an error" do
+        expect { subject.register }.to raise_error(StandardError)
+      end
+    end
+
+    context 'with filters' do
+      let (:config) { super().merge('filters' => { 'tag:Monitoring' => 'Yes' })}
+
+      it "registers succesfully" do
+        expect { subject.register }.to_not raise_error
+      end
+    end
   end
 
-  context "EBS events" do
-    let(:config) {
-      {
-        'access_key_id' => '1234',
-        'secret_access_key' => 'secret',
-        'namespace' => 'AWS/EBS',
-        'metrics' => [ 'VolumeQueueLength' ],
-        'filters' => { 'tag:Monitoring' => 'Yes' },
-        'region' => 'us-east-1'
-      }
-    }
+  shared_examples_for 'it does not require filters' do
+    context 'without filters' do
+      it "registers succesfully" do
+        expect { subject.register }.to_not raise_error
+      end
+    end
+
+    context 'with filters' do
+      let (:config) { super().merge('filters' => { 'tag:Monitoring' => 'Yes' })}
+
+      it "registers succesfully" do
+        expect { subject.register }.to_not raise_error
+      end
+    end
   end
 
-  context "RDS events" do
-    let(:config) {
-      {
-        'access_key_id' => '1234',
-        'secret_access_key' => 'secret',
-        'namespace' => 'AWS/RDS',
-        'metrics' => [ 'CPUUtilization', 'CPUCreditUsage' ],
-        'filters' => { 'EngineName' => 'mysql' },
-        'region' => 'us-east-1'
-      }
-    }
+  describe '#register' do
+
+    context "EC2 namespace" do
+      let(:config) { super().merge('namespace' => 'AWS/EC2') }
+      it_behaves_like 'it does not require filters'
+    end
+
+    context "EBS namespace" do
+      let(:config) { super().merge('namespace' => 'AWS/EBS') }
+      it_behaves_like 'it requires filters'
+    end
+
+    context "RDS namespace" do
+      let(:config) { super().merge('namespace' => 'AWS/RDS') }
+      it_behaves_like 'it requires filters'
+    end
+
   end
 end

data/spec/integration/cloudwatch_spec.rb

@@ -0,0 +1,26 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/cloudwatch"
+require "aws-sdk"
+
+describe LogStash::Inputs::CloudWatch, :integration => true do
+
+  let(:settings)  {  { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
+                       "secret_access_key" => LogStash::Util::Password.new(ENV['AWS_SECRET_ACCESS_KEY']),
+                       "region" => ENV["AWS_REGION"] || "us-east-1",
+                       "namespace" => "AWS/S3",
+                       'filters' => { "BucketName" => "*"},
+                       'metrics' => ["BucketSizeBytes","NumberOfObjects"]
+
+  }}
+
+  def metrics_for(settings)
+    cw = LogStash::Inputs::CloudWatch.new(settings)
+    cw.register
+    cw.send('metrics_for', settings['namespace'])
+  end
+
+  #
+  it "should not raise a type error when using a password" do
+    expect{metrics_for(settings)}.not_to raise_error
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-cloudwatch
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.2
 platform: ruby
 authors:
 - Jurgens du Toit
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2018-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -103,9 +103,10 @@ files:
 - lib/logstash/inputs/cloudwatch.rb
 - logstash-input-cloudwatch.gemspec
 - spec/inputs/cloudwatch_spec.rb
+- spec/integration/cloudwatch_spec.rb
 homepage: http://eagerelk.com
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: input
@@ -125,9 +126,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Pulls events from the Amazon Web Services CloudWatch API
 test_files:
 - spec/inputs/cloudwatch_spec.rb
+- spec/integration/cloudwatch_spec.rb