logstash-integration-aws 0.1.0.pre

data/spec/outputs/s3_spec.rb

@@ -0,0 +1,232 @@
+# encoding: utf-8
+require "logstash/outputs/s3"
+require "logstash/event"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe LogStash::Outputs::S3 do
+  let(:prefix) { "super/%{server}" }
+  let(:region) { "us-east-1" }
+  let(:bucket_name) { "mybucket" }
+  let(:options) { { "region" => region,
+                    "bucket" => bucket_name,
+                    "prefix" => prefix,
+                    "restore" => false,
+                    "access_key_id" => "access_key_id",
+                    "secret_access_key" => "secret_access_key"
+  } }
+  let(:client) { Aws::S3::Client.new(stub_responses: true) }
+  let(:mock_bucket) { Aws::S3::Bucket.new(:name => bucket_name, :stub_responses => true, :client => client) }
+  let(:event) { LogStash::Event.new({ "server" => "overwatch" }) }
+  let(:event_encoded) { "super hype" }
+  let(:events_and_encoded) { { event => event_encoded } }
+
+  subject { described_class.new(options) }
+
+  before do
+    allow_any_instance_of(LogStash::Outputs::S3::WriteBucketPermissionValidator).to receive(:valid?).and_return(true)
+  end
+
+  context "#register configuration validation" do
+    describe "signature version" do
+      it "should set the signature version if specified" do
+        ["v2", "v4"].each do |version|
+          s3 = described_class.new(options.merge({ "signature_version" => version }))
+          expect(s3.full_options).to include(:signature_version => version)
+        end
+      end
+
+      it "should omit the option completely if not specified" do
+        s3 = described_class.new(options)
+        expect(s3.full_options.has_key?(:signature_version)).to eql(false)
+      end
+    end
+
+    describe "Access control list" do
+      context "when configured" do
+        ["private", "public-read", "public-read-write", "authenticated-read", "aws-exec-read", "bucket-owner-read", "bucket-owner-full-control", "log-delivery-write"].each do |permission|
+          it "should return the configured ACL permissions: #{permission}" do
+            s3 = described_class.new(options.merge({ "canned_acl" => permission }))
+            expect(s3.upload_options).to include(:acl => permission)
+          end
+        end
+      end
+
+      context "when not configured" do
+        it "uses private as the default" do
+          s3 = described_class.new(options)
+          expect(s3.upload_options).to include(:acl => "private")
+        end
+      end
+    end
+
+    describe "Multipart upload threshold" do
+      context "when configured" do
+        it "should use the configured threshold" do
+          threshold = 1 * 1024 * 1024
+          s3 = described_class.new(options.merge({ "upload_multipart_threshold" => threshold }))
+          expect(s3.upload_options).to include(:multipart_threshold => threshold)
+        end
+      end
+
+      context "when not configured" do
+        it "should use 15MB as the default" do
+          s3 = described_class.new(options)
+          expect(s3.upload_options).to include(:multipart_threshold => 15 * 1024 * 1024)
+        end
+      end
+    end
+
+    describe "Service Side Encryption" do
+
+      context "when configured" do
+          it "should be configure" do
+            s3 = described_class.new(options.merge({ "server_side_encryption" => true }))
+            expect(s3.upload_options).to include(:server_side_encryption => "AES256")
+          end
+        end
+
+      context "when algorithm is configured" do
+        ["AES256", "aws:kms"].each do |sse|
+          it "should return the configured SSE: #{sse}" do
+            s3 = described_class.new(options.merge({ "server_side_encryption" => true, "server_side_encryption_algorithm" => sse }))
+            expect(s3.upload_options).to include(:server_side_encryption => sse)
+          end
+        end
+      end
+
+      context "when using SSE with KMS and custom key" do
+        it "should return the configured KMS key" do
+          s3 = described_class.new(options.merge({ "server_side_encryption" => true, "server_side_encryption_algorithm" => "aws:kms",  "ssekms_key_id" => "test"}))
+          expect(s3.upload_options).to include(:server_side_encryption => "aws:kms")
+          expect(s3.upload_options).to include(:ssekms_key_id => "test")
+        end
+      end
+
+      context "when using SSE with KMS but no custom key" do
+        it "should return the configured KMS key" do
+          s3 = described_class.new(options.merge({ "server_side_encryption" => true, "server_side_encryption_algorithm" => "aws:kms"}))
+          expect(s3.upload_options).to include(:server_side_encryption => "aws:kms")
+          expect(s3.upload_options).to include(:ssekms_key_id => nil)
+        end
+      end
+
+      context "when not configured" do
+          it "should not be configured" do
+            s3 = described_class.new(options)
+            expect(s3.upload_options).to include(:server_side_encryption => nil)
+            expect(s3.upload_options).to include(:ssekms_key_id => nil)
+          end
+      end
+    end
+
+    describe "Storage Class" do
+      context "when configured" do
+        ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"].each do |storage_class|
+          it "should return the configured storage class: #{storage_class}" do
+            s3 = described_class.new(options.merge({ "storage_class" => storage_class }))
+            expect(s3.upload_options).to include(:storage_class => storage_class)
+          end
+        end
+      end
+
+      context "when not configured" do
+        it "uses STANDARD as the default" do
+          s3 = described_class.new(options)
+          expect(s3.upload_options).to include(:storage_class => "STANDARD")
+        end
+      end
+    end
+
+    describe "temporary directory" do
+      let(:temporary_directory) { Stud::Temporary.pathname }
+      let(:options) { super().merge({ "temporary_directory" => temporary_directory }) }
+
+      it "creates the directory when it doesn't exist" do
+        expect(Dir.exist?(temporary_directory)).to be_falsey
+        subject.register
+        expect(Dir.exist?(temporary_directory)).to be_truthy
+      end
+
+      it "raises an error if we cannot write to the directory" do
+        expect(LogStash::Outputs::S3::WritableDirectoryValidator).to receive(:valid?).with(temporary_directory).and_return(false)
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    it "validates the prefix" do
+      s3 = described_class.new(options.merge({ "prefix" => "`no\><^" }))
+      expect { s3.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    describe "additional_settings" do
+      context "supported settings" do
+        let(:additional_settings) do
+          { "additional_settings" => { "force_path_style" => 'true', "ssl_verify_peer" => 'false', "profile" => 'logstash' } }
+        end
+
+        it "validates the prefix" do
+          expect(Aws::S3::Bucket).to receive(:new).twice.
+              with(anything, hash_including(:force_path_style => true, :ssl_verify_peer => false, :profile => 'logstash')).
+              and_call_original
+          described_class.new(options.merge(additional_settings)).register
+        end
+      end
+      context "when using a non existing setting" do
+        let(:additional_settings) do
+          { "additional_settings" => { "doesnt_exist" => true } }
+        end
+
+        it "raises an error" do
+          plugin = described_class.new(options.merge(additional_settings))
+          expect { plugin.register }.to raise_error(ArgumentError)
+        end
+      end
+    end
+
+    it "allow to not validate credentials" do
+      s3 = described_class.new(options.merge({"validate_credentials_on_root_bucket" => false}))
+      expect_any_instance_of(LogStash::Outputs::S3::WriteBucketPermissionValidator).not_to receive(:valid?).with(any_args)
+      s3.register
+    end
+  end
+
+  context "receiving events" do
+    before do
+      allow(subject).to receive(:bucket_resource).and_return(mock_bucket)
+      subject.register
+    end
+
+    after do
+      subject.close
+    end
+
+    it "uses `Event#sprintf` for the prefix" do
+      expect(event).to receive(:sprintf).with(prefix).and_return("super/overwatch")
+      subject.multi_receive_encoded(events_and_encoded)
+    end
+  end
+
+  describe "aws service" do
+    context 'us-east-1' do
+      let(:region) { 'us-east-1' }
+      it "sets endpoint" do
+        expect( subject.send(:bucket_resource).client.config.endpoint.to_s ).to eql 'https://s3.us-east-1.amazonaws.com'
+      end
+    end
+
+    context 'ap-east-1' do
+      let(:region) { 'ap-east-1' }
+      it "sets endpoint" do
+        expect( subject.send(:bucket_resource).client.config.endpoint.to_s ).to eql 'https://s3.ap-east-1.amazonaws.com'
+      end
+    end
+
+    context 'cn-northwest-1' do
+      let(:region) { 'cn-northwest-1' }
+      it "sets endpoint" do
+        expect( subject.send(:bucket_resource).client.config.endpoint.to_s ).to eql 'https://s3.cn-northwest-1.amazonaws.com.cn'
+      end
+    end
+  end
+end

data/spec/outputs/sns_spec.rb

@@ -0,0 +1,160 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/outputs/sns'
+require 'logstash/event'
+require "logstash/plugin_mixins/aws_config"
+
+require "aws-sdk-sns"
+
+describe LogStash::Outputs::Sns do
+  let(:arn) { "arn:aws:sns:us-east-1:999999999:logstash-test-sns-topic" }
+  let(:sns_subject) { "The Plain in Spain" }
+  let(:sns_message) { "That's where the rain falls, plainly." }
+  let(:mock_client) { double("Aws::SNS::Client") }
+  let(:instance) {
+    allow(Aws::SNS::Client).to receive(:new).and_return(mock_client)
+    inst = LogStash::Outputs::Sns.new
+    allow(inst).to receive(:publish_boot_message_arn).and_return(nil)
+    inst.register
+    inst
+  }
+
+  describe "receiving an event" do
+    let(:expected_subject) { double("expected_subject")}
+    subject {
+      inst = instance
+      allow(inst).to receive(:send_sns_message).with(any_args)
+      allow(inst).to receive(:event_subject).
+                       with(any_args).
+                       and_return(expected_subject)
+      inst.receive(event)
+      inst
+    }
+
+    shared_examples("publishing correctly") do
+      it "should send a message to the correct ARN if the event has 'arn' set" do
+        expect(subject).to have_received(:send_sns_message).with(arn, anything, anything)
+      end
+
+      it "should send the message" do
+        expect(subject).to have_received(:send_sns_message).with(anything, anything, expected_message)
+      end
+
+      it "should send the subject" do
+        expect(subject).to have_received(:send_sns_message).with(anything, expected_subject, anything)
+      end
+    end
+
+    describe "with an explicit message" do
+      let(:expected_subject) { sns_subject }
+      let(:expected_message) { sns_message }
+      let(:event) { LogStash::Event.new("sns" => arn, "sns_subject" => sns_subject,
+                                        "sns_message" => sns_message) }
+      include_examples("publishing correctly")
+    end
+
+    describe "without an explicit message" do
+      # Testing codecs sucks. It'd be nice if codecs had to implement some sort of encode_sync method
+      let(:expected_message) {
+        c = subject.codec.clone
+        result = nil;
+        c.on_event {|event, encoded| result = encoded }
+        c.encode(event)
+        result
+      }
+      let(:event) { LogStash::Event.new("sns" => arn, "sns_subject" => sns_subject) }
+
+      include_examples("publishing correctly")
+    end
+  end
+
+  describe "determining the subject" do
+    it "should return 'sns_subject' when set" do
+      event = LogStash::Event.new("sns_subject" => "foo")
+      expect(subject.send(:event_subject, event)).to eql("foo")
+    end
+
+    it "should return the sns subject as JSON if not a string" do
+      event = LogStash::Event.new("sns_subject" => ["foo", "bar"])
+      expect(subject.send(:event_subject, event)).to eql(LogStash::Json.dump(["foo", "bar"]))
+    end
+
+    it "should return the host if 'sns_subject' not set" do
+      event = LogStash::Event.new("host" => "foo")
+      expect(subject.send(:event_subject, event)).to eql("foo")
+    end
+
+    it "should return the host name (ECS compatibility) if 'sns_subject' not set" do
+      event = LogStash::Event.new
+      event.set('[host][hostname]', 'server1')
+      expect(subject.send(:event_subject, event)).to eql('server1')
+    end
+
+    it "should return the host IP (ECS compatibility) if 'sns_subject' not set" do
+      event = LogStash::Event.new
+      event.set('host.geo.name', 'Vychodne Pobrezie')
+      expect(subject.send(:event_subject, event)).to eql(LogStash::Outputs::Sns::NO_SUBJECT)
+    end
+
+    it "should return no subject when no info in host object (ECS compatibility) if 'sns_subject' not set" do
+      event = LogStash::Event.new
+      event.set('[host][ip]', '192.168.1.111')
+      expect(subject.send(:event_subject, event)).to eql('192.168.1.111')
+    end
+
+    it "should return 'NO SUBJECT' when subject cannot be determined" do
+      event = LogStash::Event.new("foo" => "bar")
+      expect(subject.send(:event_subject, event)).to eql(LogStash::Outputs::Sns::NO_SUBJECT)
+    end
+  end
+
+  describe "sending an SNS notification" do
+    let(:good_publish_args) {
+      {
+        :topic_arn => arn,
+        :subject => sns_subject,
+        :message => sns_message
+      }
+    }
+    let(:long_message) { "A" * (LogStash::Outputs::Sns::MAX_MESSAGE_SIZE_IN_BYTES + 1) }
+    let(:long_subject) { "S" * (LogStash::Outputs::Sns::MAX_SUBJECT_SIZE_IN_CHARACTERS + 1) }
+    subject { instance }
+
+    it "should raise an ArgumentError if no arn is provided" do
+      expect {
+        subject.send(:send_sns_message, nil, sns_subject, sns_message)
+      }.to raise_error(ArgumentError)
+    end
+
+    it "should send a well formed message through to SNS" do
+      expect(mock_client).to receive(:publish).with(good_publish_args)
+      subject.send(:send_sns_message, arn, sns_subject, sns_message)
+    end
+
+    it "should attempt to publish a boot message" do
+      expect(subject).to have_received(:publish_boot_message_arn).once
+      x = case "foo"
+            when "bar"
+              "hello"
+          end
+    end
+
+    it "should truncate long messages before sending" do
+      max_size = LogStash::Outputs::Sns::MAX_MESSAGE_SIZE_IN_BYTES
+      expect(mock_client).to receive(:publish) {|args|
+                               expect(args[:message].bytesize).to eql(max_size)
+                             }
+
+      subject.send(:send_sns_message, arn, sns_subject, long_message)
+    end
+
+    it "should truncate long subjects before sending" do
+      max_size = LogStash::Outputs::Sns::MAX_SUBJECT_SIZE_IN_CHARACTERS
+      expect(mock_client).to receive(:publish) {|args|
+                               expect(args[:subject].bytesize).to eql(max_size)
+                             }
+
+      subject.send(:send_sns_message, arn, long_subject, sns_message)
+    end
+  end
+end

data/spec/plugin_mixin/aws_config_spec.rb

@@ -0,0 +1,217 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin_mixins/aws_config"
+require 'timecop'
+
+class DummyInputAwsConfigV2 < LogStash::Inputs::Base
+  include LogStash::PluginMixins::AwsConfig::V2
+
+  def aws_service_endpoint(region)
+    { :dummy_input_aws_config_region => "#{region}.awswebservice.local" }
+  end
+end
+
+class DummyInputAwsConfigV2NoRegionMethod < LogStash::Inputs::Base
+  include LogStash::PluginMixins::AwsConfig::V2
+end
+
+describe LogStash::PluginMixins::AwsConfig::V2 do
+  let(:settings) { {} }
+
+  subject { DummyInputAwsConfigV2.new(settings).aws_options_hash }
+
+  describe 'config credential' do
+    subject { DummyInputAwsConfigV2.new(settings).aws_options_hash[:credentials] }
+
+    context 'in credential file' do
+      let(:settings) { { 'aws_credentials_file' => File.join(File.dirname(__FILE__), '..', 'fixtures/aws_credentials_file_sample_test.yml') } }
+
+      it 'should support reading configuration from a yaml file' do
+        expect(subject.access_key_id).to eq("1234")
+        expect(subject.secret_access_key).to eq("secret")   
+      end
+    end
+
+    context 'inline' do
+      context 'temporary credential' do
+        let(:settings) { { 'access_key_id' => '1234', 'secret_access_key' => 'secret', 'session_token' => 'session_token' } }
+
+        it "should support passing as key, value, and session_token" do
+          expect(subject.access_key_id).to eq(settings['access_key_id'])
+          expect(subject.secret_access_key).to eq(settings['secret_access_key'])
+          expect(subject.session_token).to eq(settings['session_token'])
+        end
+      end
+
+      context 'normal credential' do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret' } }
+
+        it 'should support passing credentials as key, value' do
+          expect(subject.access_key_id).to eq(settings['access_key_id'])
+          expect(subject.secret_access_key).to eq(settings['secret_access_key'])
+        end
+      end
+
+      context 'role arn is provided' do
+        let(:settings) { { 'role_arn' => 'arn:aws:iam::012345678910:role/foo', 'region' => 'us-west-2' } }
+        let(:sts_double) { instance_double(Aws::STS::Client) }
+        let(:now) { Time.now }
+        let(:expiration) { Time.at(now.to_i + 3600) }
+        let(:temp_credentials) {
+          double(credentials:
+                  double(
+                    access_key_id: '1234',
+                    secret_access_key: 'secret',
+                    session_token: 'session_token',
+                    expiration: expiration.to_s,
+                  )
+                )
+        }
+        let(:new_temp_credentials) {
+          double(credentials:
+                  double(
+                    access_key_id: '5678',
+                    secret_access_key: 'secret1',
+                    session_token: 'session_token1',
+                    expiration: expiration.to_s,
+                  )
+                )
+        }
+
+        before do
+          allow(Aws::STS::Client).to receive(:new).and_return(sts_double)
+          allow(sts_double).to receive(:assume_role)  {
+            if Time.now < expiration
+              temp_credentials
+            else
+              new_temp_credentials
+            end
+          }
+        end
+
+        it 'supports passing role_arn' do
+          Timecop.freeze(now) do
+            expect(subject.credentials.access_key_id).to eq('1234')
+            expect(subject.credentials.secret_access_key).to eq('secret')
+            expect(subject.credentials.session_token).to eq('session_token')
+          end
+        end
+
+        it 'rotates the keys once they expire' do
+          Timecop.freeze(Time.at(expiration.to_i + 100)) do
+            expect(subject.credentials.access_key_id).to eq('5678')
+            expect(subject.credentials.secret_access_key).to eq('secret1')
+            expect(subject.credentials.session_token).to eq('session_token1')
+          end
+        end       
+      end
+
+      context 'role arn with credentials' do
+
+        let(:settings) do
+          {
+              'role_arn' => 'arn:aws:iam::012345678910:role/foo',
+              'region' => 'us-west-2',
+
+              'access_key_id' => '12345678',
+              'secret_access_key' => 'secret',
+              'session_token' => 'session_token',
+
+              'proxy_uri' => 'http://a-proxy.net:1234'
+          }
+        end
+
+        let(:aws_options_hash) { DummyInputAwsConfigV2NoRegionMethod.new(settings).aws_options_hash }
+
+        before do
+          allow_any_instance_of(Aws::AssumeRoleCredentials).to receive(:refresh) # called from #initialize
+        end
+
+        it 'uses credentials' do
+          subject = aws_options_hash[:credentials]
+          expect( subject ).to be_a Aws::AssumeRoleCredentials
+          expect( subject.client ).to be_a Aws::STS::Client
+          expect( credentials = subject.client.config.credentials ).to be_a Aws::Credentials
+          expect( credentials.access_key_id ).to eql '12345678'
+        end
+
+        it 'sets up proxy on client and region' do
+          subject = aws_options_hash[:credentials]
+          expect( subject.client.config.http_proxy ).to eql 'http://a-proxy.net:1234'
+          expect( subject.client.config.region ).to eql 'us-west-2' # probably redundant (kept for backwards compat)
+        end
+
+        it 'sets up proxy top level' do # setting in on the client isn't enough!
+          expect( aws_options_hash[:http_proxy] ).to eql 'http://a-proxy.net:1234'
+        end
+
+        it 'sets up region top-level' do
+          # NOTE: this one is required for real with role_arn :
+          expect( aws_options_hash[:region] ).to eql 'us-west-2'
+        end
+
+      end
+    end
+  end
+
+  describe 'config proxy' do
+    let(:proxy) { "http://localhost:1234"  }
+    let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret', 'region' => 'us-west-2', 'proxy_uri' => proxy } }
+
+    it "should set the http_proxy option" do
+      expect(subject[:http_proxy]).to eql(proxy)
+    end
+
+    it "should not set the legacy http proxy option" do
+      expect(subject[:proxy_uri]).not_to eql(proxy)
+    end
+  end
+
+  describe 'config region' do
+    context "when the class implement `#aws_service_endpoint`" do
+      context 'region provided' do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret', 'region' => 'us-west-2' } }
+
+        it 'should use provided region to generate the endpoint configuration' do
+          expect(subject).to include(:dummy_input_aws_config_region => "us-west-2.awswebservice.local")
+        end
+      end
+
+      context "region not provided" do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret'} }
+
+        it 'should use default region to generate the endpoint configuration' do
+          expect(subject).to include(:dummy_input_aws_config_region => "us-east-1.awswebservice.local")
+        end
+      end
+    end
+
+    context "when the classe doesn't implement `#aws_service_endpoint`" do
+      subject { DummyInputAwsConfigV2NoRegionMethod.new(settings).aws_options_hash }
+
+      context 'region provided' do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret', 'region' => 'us-west-2' } }
+
+        it 'should use provided region to generate the endpoint configuration' do
+          expect(subject[:region]).to eq("us-west-2")
+        end
+      end
+
+      context "region not provided" do
+        let(:settings) { { 'access_key_id' => '1234',  'secret_access_key' => 'secret'} }
+
+        it 'should use default region to generate the endpoint configuration' do
+          expect(subject[:region]).to eq("us-east-1")
+        end
+      end
+    end
+  end
+
+  context 'when we arent providing credentials' do
+    let(:settings) { {} }
+    it 'should always return a hash' do
+      expect(subject).to eq({ :dummy_input_aws_config_region => "us-east-1.awswebservice.local" })  
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/sqs'
+require 'logstash/logging/logger'
+require_relative 'support/helpers'
+
+LogStash::Logging::Logger::configure_logging("debug") if ENV["DEBUG"]