logstash-mixin-aws 0.1.10 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0891ea54a75d4c21a591479312ff7b9ccec09507
-  data.tar.gz: 664552672c4bde14792cf9c12cdbcc51c3b630d4
+  metadata.gz: f00cc7ea12d56fa9b3c604ee68cef128f7c9718b
+  data.tar.gz: 9b84affbeb7ded1589cb9fd42b3aebf94c188d5d
 SHA512:
-  metadata.gz: 55a85d65e77113eb8c32dc8d1dd493f9aabe8b802cbab7d5f6e056fd7c9359a0b6a84622a4c332b5a2bf4dfdcf2cc0e58b02b6d6998205336636101ea3828b3f
-  data.tar.gz: 59c159646ae8b1a9139c921ba8d2624be249b11578eb3bf041221bc2743a5d3c33c327c8d3899686970a87fad9ab4dc6d0fe18501bb11de4e4fdbad8074416ab
+  metadata.gz: 2994e1df54cfb124c2fad4b4f27b61a7180c516ad7f602ef415968879104540e407dd16d5a74854462706ebb3c6cf1145ba790a16b16930f35c92bf5fdd71f12
+  data.tar.gz: bf8695dac26cd81bb4efc6f8b93b959de2ecac3634689e32f73166b4986e40ab51a89303e7e8d986ce717c3f5a2e061b144271ff0e7f6ed019a24aa74abd7bcc

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# 1.0.0
+  * Allow to use either V1 or V2 of the `AWS-SDK` in your plugins. Fixes: https://github.com/logstash-plugins/logstash-mixin-aws/issues/8

data/README.md

@@ -13,7 +13,7 @@ Logstash provides infrastructure to automatically generate documentation for thi
 
 ## Need Help?
 
-Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
 
 ## Developing
 

data/lib/logstash/plugin_mixins/aws_config.rb

@@ -1,93 +1,20 @@
 # encoding: utf-8
 require "logstash/config/mixin"
 
+# This module provides helper for the `AWS-SDK` v1,
+# and it will be deprecated in the near future, please use the V2 module
+# for any new development.
 module LogStash::PluginMixins::AwsConfig
+  require "logstash/plugin_mixins/aws_config/v1"
+  require "logstash/plugin_mixins/aws_config/v2"
 
-  @logger = Cabin::Channel.get(LogStash)
+  US_EAST_1 = "us-east-1"
+  REGIONS_ENDPOINT = [US_EAST_1, "us-west-1", "us-west-2", "eu-central-1",
+                      "eu-west-1", "ap-southeast-1", "ap-southeast-2",
+                      "ap-northeast-1", "sa-east-1", "us-gov-west-1", "cn-north-1"]
 
-  # This method is called when someone includes this module
   def self.included(base)
     # Add these methods to the 'base' given.
-    base.extend(self)
-    base.setup_aws_config
+    base.send(:include, V1)
   end
-
-  US_EAST_1 = "us-east-1"
-
-  public
-  def setup_aws_config
-    # The AWS Region
-    config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2", "eu-central-1",
-                                  "eu-west-1", "ap-southeast-1", "ap-southeast-2",
-                                  "ap-northeast-1", "sa-east-1", "us-gov-west-1",
-                                  "cn-north-1"], :default => US_EAST_1
-
-    # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order...
-    # 1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
-    # 2. External credentials file specified by `aws_credentials_file`
-    # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
-    # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
-    # 5. IAM Instance Profile (available when running inside EC2)
-    config :access_key_id, :validate => :string
-
-    # The AWS Secret Access Key
-    config :secret_access_key, :validate => :string
-
-    # The AWS Session token for temprory credential
-    config :session_token, :validate => :string
-
-    # Should we require (true) or disable (false) using SSL for communicating with the AWS API
-    # The AWS SDK for Ruby defaults to SSL so we preserve that
-    config :use_ssl, :validate => :boolean, :default => true
-
-    # URI to proxy server if required
-    config :proxy_uri, :validate => :string
-
-    # Path to YAML file containing a hash of AWS credentials.
-    # This file will only be loaded if `access_key_id` and
-    # `secret_access_key` aren't set. The contents of the
-    # file should look like this:
-    #
-    #     :access_key_id: "12345"
-    #     :secret_access_key: "54321"
-    #
-    config :aws_credentials_file, :validate => :string
-  end
-
-  public
-  def aws_options_hash
-    opts = {}
-
-    if @access_key_id.is_a?(NilClass) ^ @secret_access_key.is_a?(NilClass)
-      @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.")
-    end
-
-    if @access_key_id && @secret_access_key
-      opts = {
-        :access_key_id => @access_key_id,
-        :secret_access_key => @secret_access_key
-      }
-      opts[:session_token] = @session_token if @session_token
-    elsif @aws_credentials_file
-      opts = YAML.load_file(@aws_credentials_file)
-    end
-
-    opts[:proxy_uri] = @proxy_uri if @proxy_uri
-    opts[:use_ssl] = @use_ssl
-
-
-    # The AWS SDK for Ruby doesn't know how to make an endpoint hostname from a region
-    # for example us-west-1 -> foosvc.us-west-1.amazonaws.com
-    # So our plugins need to know how to generate their endpoints from a region
-    # Furthermore, they need to know the symbol required to set that value in the AWS SDK
-    # Classes using this module must implement aws_service_endpoint(region:string)
-    # which must return a hash with one key, the aws sdk for ruby config symbol of the service
-    # endpoint, which has a string value of the service endpoint hostname
-    # for example, CloudWatch, { :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com" }
-    # For a list, see https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/core/configuration.rb
-    opts.merge!(self.aws_service_endpoint(@region))
-
-    return opts
-  end # def aws_options_hash
-
 end

data/lib/logstash/plugin_mixins/aws_config/generic.rb

@@ -0,0 +1,38 @@
+module LogStash::PluginMixins::AwsConfig::Generic
+  def self.included(base)
+    base.extend(self)
+    base.generic_aws_config
+  end
+
+  def generic_aws_config
+    # The AWS Region
+    config :region, :validate => LogStash::PluginMixins::AwsConfig::REGIONS_ENDPOINT, :default => LogStash::PluginMixins::AwsConfig::US_EAST_1 
+
+    # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order...
+    # 1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config
+    # 2. External credentials file specified by `aws_credentials_file`
+    # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+    # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+    # 5. IAM Instance Profile (available when running inside EC2)
+    config :access_key_id, :validate => :string
+
+    # The AWS Secret Access Key
+    config :secret_access_key, :validate => :string
+
+    # The AWS Session token for temprory credential
+    config :session_token, :validate => :string
+
+    # URI to proxy server if required
+    config :proxy_uri, :validate => :string
+
+    # Path to YAML file containing a hash of AWS credentials.
+    # This file will only be loaded if `access_key_id` and
+    # `secret_access_key` aren't set. The contents of the
+    # file should look like this:
+    #
+    #     :access_key_id: "12345"
+    #     :secret_access_key: "54321"
+    #
+    config :aws_credentials_file, :validate => :string
+  end
+end

data/lib/logstash/plugin_mixins/aws_config/v1.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+require "logstash/plugin_mixins/aws_config/generic"
+
+module LogStash::PluginMixins::AwsConfig::V1
+  def self.included(base)
+    # Make sure we require the V1 classes when including this module.
+    # require 'aws-sdk' will load v2 classes.
+    require "aws-sdk-v1"
+    base.extend(self)
+    base.send(:include, LogStash::PluginMixins::AwsConfig::Generic)
+    base.setup_aws_config
+  end
+
+  public
+  def setup_aws_config
+    # Should we require (true) or disable (false) using SSL for communicating with the AWS API
+    # The AWS SDK for Ruby defaults to SSL so we preserve that
+    config :use_ssl, :validate => :boolean, :default => true
+  end
+
+  public
+  def aws_options_hash
+    opts = {}
+
+    if @access_key_id.is_a?(NilClass) ^ @secret_access_key.is_a?(NilClass)
+      @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.")
+    end
+
+    if @access_key_id && @secret_access_key
+      opts = {
+        :access_key_id => @access_key_id,
+        :secret_access_key => @secret_access_key
+      }
+      opts[:session_token] = @session_token if @session_token
+    elsif @aws_credentials_file
+      opts = YAML.load_file(@aws_credentials_file)
+    end
+
+    opts[:proxy_uri] = @proxy_uri if @proxy_uri
+    opts[:use_ssl] = @use_ssl
+
+
+    # The AWS SDK for Ruby doesn't know how to make an endpoint hostname from a region
+    # for example us-west-1 -> foosvc.us-west-1.amazonaws.com
+    # So our plugins need to know how to generate their endpoints from a region
+    # Furthermore, they need to know the symbol required to set that value in the AWS SDK
+    # Classes using this module must implement aws_service_endpoint(region:string)
+    # which must return a hash with one key, the aws sdk for ruby config symbol of the service
+    # endpoint, which has a string value of the service endpoint hostname
+    # for example, CloudWatch, { :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com" }
+    # For a list, see https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/core/configuration.rb
+    opts.merge!(self.aws_service_endpoint(@region))
+
+    return opts
+  end # def aws_options_hash
+end

data/lib/logstash/plugin_mixins/aws_config/v2.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+require "logstash/plugin_mixins/aws_config/generic"
+
+module LogStash::PluginMixins::AwsConfig::V2
+  def self.included(base)
+    base.extend(self)
+    base.send(:include, LogStash::PluginMixins::AwsConfig::Generic)
+  end
+
+  public
+  def aws_options_hash
+    opts = {}
+
+    if @access_key_id.is_a?(NilClass) ^ @secret_access_key.is_a?(NilClass)
+      @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.")
+    end
+
+    opts[:credentials] = credentials if credentials
+
+    opts[:proxy_uri] = @proxy_uri if @proxy_uri
+
+    # The AWS SDK for Ruby doesn't know how to make an endpoint hostname from a region
+    # for example us-west-1 -> foosvc.us-west-1.amazonaws.com
+    # So our plugins need to know how to generate their endpoints from a region
+    # Furthermore, they need to know the symbol required to set that value in the AWS SDK
+    # Classes using this module must implement aws_service_endpoint(region:string)
+    # which must return a hash with one key, the aws sdk for ruby config symbol of the service
+    # endpoint, which has a string value of the service endpoint hostname
+    # for example, CloudWatch, { :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com" }
+    # For a list, see https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/core/configuration.rb
+    if self.respond_to?(:aws_service_endpoint)
+      opts.merge!(self.aws_service_endpoint(@region))
+    else
+      opts.merge!({ :region => @region })
+    end
+
+    return opts
+  end
+
+  private
+  def credentials
+    @creds ||= begin
+                 if @access_key_id && @secret_access_key
+                   credentials_opts = {
+                     :access_key_id => @access_key_id,
+                     :secret_access_key => @secret_access_key
+                   }
+
+                   credentials_opts[:session_token] = @session_token if @session_token
+                 elsif @aws_credentials_file
+                   credentials_opts = YAML.load_file(@aws_credentials_file)
+                 end
+
+                 if credentials_opts
+                   Aws::Credentials.new(credentials_opts[:access_key_id],
+                                        credentials_opts[:secret_access_key],
+                                        credentials_opts[:session_token])
+                 end
+               end
+  end
+end

data/logstash-mixin-aws.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-mixin-aws'
-  s.version         = '0.1.10'
+  s.version         = '1.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "AWS mixins to provide a unified interface for Amazon Webservice"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -18,7 +18,8 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'aws-sdk', '~> 1.61.0'
+  s.add_runtime_dependency 'aws-sdk-v1', '>= 1.61.0'
+  s.add_runtime_dependency 'aws-sdk', '~> 2.1.0'
   s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/plugin_mixin/aws_config_spec.rb

@@ -3,9 +3,20 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/plugin_mixins/aws_config"
 require 'aws-sdk'
 
-class DummyInputAwsConfig < LogStash::Inputs::Base
-  include LogStash::PluginMixins::AwsConfig
+class DummyInputAwsConfigV2 < LogStash::Inputs::Base
+  include LogStash::PluginMixins::AwsConfig::V2
 
+  def aws_service_endpoint(region)
+    { :dummy_input_aws_config_region => "#{region}.awswebservice.local" }
+  end
+end
+
+class DummyInputAwsConfigV2NoRegionMethod < LogStash::Inputs::Base
+  include LogStash::PluginMixins::AwsConfig::V2
+end
+
+class DummyInputAwsConfigV1 < LogStash::Inputs::Base
+  include LogStash::PluginMixins::AwsConfig
 
   def aws_service_endpoint(region)
     { :dummy_input_aws_config_region => "#{region}.awswebservice.local" }
@@ -13,10 +24,9 @@ class DummyInputAwsConfig < LogStash::Inputs::Base
 end
 
 describe LogStash::PluginMixins::AwsConfig do
-
   let(:settings) { {} }
 
-  subject { DummyInputAwsConfig.new(settings).aws_options_hash }
+  subject { DummyInputAwsConfigV1.new(settings).aws_options_hash }
 
   describe 'config credential' do
 
@@ -24,8 +34,7 @@ describe LogStash::PluginMixins::AwsConfig do
       let(:settings) { { 'aws_credentials_file' => File.join(File.dirname(__FILE__), '..', 'fixtures/aws_credentials_file_sample_test.yml') } }
 
       it 'should support reading configuration from a yaml file' do
-        subject[:access_key_id].should == '1234'
-        subject[:secret_access_key].should == 'secret'
+        expect(subject).to include(:access_key_id => "1234", :secret_access_key => "secret")
       end
     end
 
@@ -34,9 +43,9 @@ describe LogStash::PluginMixins::AwsConfig do
         let(:settings) { { 'access_key_id' => '1234', 'secret_access_key' => 'secret', 'session_token' => 'session_token' } }
 
         it "should support passing as key, value, and session_token" do
-          subject[:access_key_id].should == settings['access_key_id']
-          subject[:secret_access_key].should == settings['secret_access_key']
-          subject[:session_token].should == settings['session_token']
+          expect(subject[:access_key_id]).to eq(settings["access_key_id"])
+          expect(subject[:secret_access_key]).to eq(settings["secret_access_key"])
+          expect(subject[:session_token]).to eq(settings["session_token"])
         end
       end
 
@@ -44,21 +53,19 @@ describe LogStash::PluginMixins::AwsConfig do
         let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret' } }
 
         it 'should support passing credentials as key, value' do
-          subject[:access_key_id].should == settings['access_key_id']
-          subject[:secret_access_key].should == settings['secret_access_key']
+          expect(subject[:access_key_id]).to eq(settings['access_key_id'])
+          expect(subject[:secret_access_key]).to eq(settings['secret_access_key'])
         end
       end
     end
-
   end
 
   describe 'config region' do
-
     context 'region provided' do
       let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret', 'region' => 'us-west-2' } }
 
       it 'should use provided region to generate the endpoint configuration' do
-        subject[:dummy_input_aws_config_region].should == "us-west-2.awswebservice.local"
+        expect(subject[:dummy_input_aws_config_region]).to eq("us-west-2.awswebservice.local")
       end
     end
 
@@ -66,7 +73,7 @@ describe LogStash::PluginMixins::AwsConfig do
       let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret'} }
 
       it 'should use default region to generate the endpoint configuration' do
-        subject[:dummy_input_aws_config_region].should == "us-east-1.awswebservice.local"
+        expect(subject[:dummy_input_aws_config_region]).to eq("us-east-1.awswebservice.local")
       end
     end
   end
@@ -78,3 +85,90 @@ describe LogStash::PluginMixins::AwsConfig do
     end
   end
 end
+
+describe LogStash::PluginMixins::AwsConfig::V2 do
+  let(:settings) { {} }
+
+  subject { DummyInputAwsConfigV2.new(settings).aws_options_hash }
+
+  describe 'config credential' do
+    subject { DummyInputAwsConfigV2.new(settings).aws_options_hash[:credentials] }
+
+    context 'in credential file' do
+      let(:settings) { { 'aws_credentials_file' => File.join(File.dirname(__FILE__), '..', 'fixtures/aws_credentials_file_sample_test.yml') } }
+
+      it 'should support reading configuration from a yaml file' do
+        expect(subject.access_key_id).to eq("1234")
+        expect(subject.secret_access_key).to eq("secret")
+      end
+    end
+
+    context 'inline' do
+      context 'temporary credential' do
+        let(:settings) { { 'access_key_id' => '1234', 'secret_access_key' => 'secret', 'session_token' => 'session_token' } }
+
+        it "should support passing as key, value, and session_token" do
+          expect(subject.access_key_id).to eq(settings['access_key_id'])
+          expect(subject.secret_access_key).to eq(settings['secret_access_key'])
+          expect(subject.session_token).to eq(settings['session_token'])
+        end
+      end
+
+      context 'normal credential' do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret' } }
+
+        it 'should support passing credentials as key, value' do
+          expect(subject.access_key_id).to eq(settings['access_key_id'])
+          expect(subject.secret_access_key).to eq(settings['secret_access_key'])
+        end
+      end
+    end
+  end
+
+  describe 'config region' do
+    context "when the class implement `#aws_service_endpoint`" do
+      context 'region provided' do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret', 'region' => 'us-west-2' } }
+
+        it 'should use provided region to generate the endpoint configuration' do
+          expect(subject).to include(:dummy_input_aws_config_region => "us-west-2.awswebservice.local")
+        end
+      end
+
+      context "region not provided" do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret'} }
+
+        it 'should use default region to generate the endpoint configuration' do
+          expect(subject).to include(:dummy_input_aws_config_region => "us-east-1.awswebservice.local")
+        end
+      end
+    end
+
+    context "when the classe doesn't implement `#aws_service_endpoint`" do
+      subject { DummyInputAwsConfigV2NoRegionMethod.new(settings).aws_options_hash }
+
+      context 'region provided' do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret', 'region' => 'us-west-2' } }
+
+        it 'should use provided region to generate the endpoint configuration' do
+          expect(subject[:region]).to eq("us-west-2")
+        end
+      end
+
+      context "region not provided" do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret'} }
+
+        it 'should use default region to generate the endpoint configuration' do
+          expect(subject[:region]).to eq("us-east-1")
+        end
+      end
+    end
+  end
+
+  context 'when we arent providing credentials' do
+    let(:settings) { {} }
+    it 'should always return a hash' do
+      expect(subject).to eq({ :dummy_input_aws_config_region => "us-east-1.awswebservice.local" })  
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-mixin-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 1.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-07 00:00:00.000000000 Z
+date: 2015-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -44,18 +44,32 @@ dependencies:
         version: '0'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-v1
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.61.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.61.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.61.0
+        version: 2.1.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.61.0
+        version: 2.1.0
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -79,12 +93,16 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - lib/logstash/plugin_mixins/aws_config.rb
+- lib/logstash/plugin_mixins/aws_config/generic.rb
+- lib/logstash/plugin_mixins/aws_config/v1.rb
+- lib/logstash/plugin_mixins/aws_config/v2.rb
 - logstash-mixin-aws.gemspec
 - spec/fixtures/aws_credentials_file_sample_test.yml
 - spec/fixtures/aws_temporary_credentials_file_sample_test.yml