logstash-integration-aws 0.1.0.pre

data/spec/inputs/sqs_spec.rb

@@ -0,0 +1,324 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/devutils/rspec/shared_examples"
+require "logstash/inputs/sqs"
+require "logstash/errors"
+require "logstash/event"
+require "logstash/json"
+require "aws-sdk-sqs"
+require "ostruct"
+
+describe LogStash::Inputs::SQS do
+  let(:queue_name) { "the-infinite-pandora-box" }
+  let(:queue_url) { "https://sqs.test.local/#{queue_name}" }
+  let(:config) do
+    {
+      "region" => "us-east-1",
+      "access_key_id" => "123",
+      "secret_access_key" => "secret",
+      "queue" => queue_name
+    }
+  end
+
+  let(:input) { LogStash::Inputs::SQS.new(config) }
+  let(:decoded_message) { { "bonjour" => "awesome" }  }
+  let(:encoded_message)  { double("sqs_message", :body => LogStash::Json::dump(decoded_message)) }
+
+  subject { input }
+
+  let(:mock_sqs) { Aws::SQS::Client.new({ :stub_responses => true }) }
+
+
+  context "with invalid credentials" do
+    before do
+      expect(Aws::SQS::Client).to receive(:new).and_return(mock_sqs)
+      expect(mock_sqs).to receive(:get_queue_url).with({ :queue_name => queue_name }) { raise Aws::SQS::Errors::ServiceError.new("bad-something", "bad token") }
+    end
+
+    it "raises a Configuration error if the credentials are bad" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "valid credentials" do
+    let(:queue) { [] }
+
+    it "doesn't raise an error with valid credentials" do
+      expect(Aws::SQS::Client).to receive(:new).and_return(mock_sqs)
+      expect(mock_sqs).to receive(:get_queue_url).with({ :queue_name => queue_name }).and_return({:queue_url => queue_url })
+      expect { subject.register }.not_to raise_error
+    end
+
+    context "when queue_aws_account_id option is specified" do
+      let(:queue_account_id) { "123456789012" }
+      let(:config) do
+        {
+          "region" => "us-east-1",
+          "access_key_id" => "123",
+          "secret_access_key" => "secret",
+          "queue" => queue_name,
+          "queue_owner_aws_account_id" => queue_account_id
+        }
+      end
+      it "passes the option to sqs client" do
+        expect(Aws::SQS::Client).to receive(:new).and_return(mock_sqs)
+        expect(mock_sqs).to receive(:get_queue_url).with({ :queue_name => queue_name, :queue_owner_aws_account_id => queue_account_id }).and_return({:queue_url => queue_url })
+        expect { subject.register }.not_to raise_error
+      end
+    end
+
+    describe "additional_settings" do
+      context "supported settings" do
+        let(:config) {
+          {
+            "additional_settings" => { "force_path_style" => 'true', "ssl_verify_peer" => 'false', "profile" => 'logstash' },
+            "queue" => queue_name
+          }
+        }
+
+        it 'should instantiate Aws::SQS clients with force_path_style set' do
+          expect(Aws::SQS::Client).to receive(:new).and_return(mock_sqs)
+          # mock a remote call to retrieve the queue URL
+          expect(mock_sqs).to receive(:get_queue_url).with({ :queue_name => queue_name }).and_return({:queue_url => queue_url })
+
+          expect(subject.aws_options_hash).to include({:force_path_style => true, :ssl_verify_peer => false, :profile => 'logstash'})
+
+          expect { subject.register }.not_to raise_error
+        end
+      end
+
+      context "unsupported settings" do
+        let(:config) {
+          {
+            "additional_settings" => { "stub_responses" => 'true', "invalid_option" => "invalid" },
+            "queue" => queue_name
+          }
+        }
+
+        it 'must fail with ArgumentError' do
+          expect {subject.register}.to raise_error(ArgumentError, /invalid_option/)
+        end
+      end
+
+    end
+
+    context "when interrupting the plugin" do
+      before do
+        expect(Aws::SQS::Client).to receive(:new).and_return(mock_sqs)
+        expect(mock_sqs).to receive(:get_queue_url).with({ :queue_name => queue_name }).and_return({:queue_url => queue_url })
+        expect(subject).to receive(:poller).and_return(mock_sqs).at_least(:once)
+
+        # We have to make sure we create a bunch of events
+        # so we actually really try to stop the plugin.
+        # 
+        # rspec's `and_yield` allow you to define a fix amount of possible 
+        # yielded values and doesn't allow you to create infinite loop.
+        # And since we are actually creating thread we need to make sure
+        # we have enough work to keep the thread working until we kill it..
+        #
+        # I haven't found a way to make it rspec friendly
+        mock_sqs.instance_eval do
+          def poll(polling_options = {})
+            loop do
+              yield [OpenStruct.new(:body => LogStash::Json::dump({ "message" => "hello world"}))], OpenStruct.new
+            end
+          end
+        end
+      end
+
+      it_behaves_like "an interruptible input plugin"
+    end
+
+    context "enrich event" do
+      let(:event) { LogStash::Event.new }
+
+      let(:message_id) { "123" }
+      let(:md5_of_body) { "dr strange" }
+      let(:sent_timestamp) { LogStash::Timestamp.new }
+      let(:epoch_timestamp) { (sent_timestamp.utc.to_f * 1000).to_i }
+
+      let(:id_field) { "my_id_field" }
+      let(:md5_field) { "my_md5_field" }
+      let(:sent_timestamp_field) { "my_sent_timestamp_field" }
+
+      let(:message) do
+        double("message", :message_id => message_id, :md5_of_body => md5_of_body, :attributes => { LogStash::Inputs::SQS::SENT_TIMESTAMP  => epoch_timestamp } )
+      end 
+
+      subject { input.add_sqs_data(event, message) }
+
+      context "when the option is specified" do
+        let(:config) do
+          {
+            "region" => "us-east-1",
+            "access_key_id" => "123",
+            "secret_access_key" => "secret",
+            "queue" => queue_name,
+            "id_field" => id_field,
+            "md5_field" => md5_field,
+            "sent_timestamp_field" => sent_timestamp_field
+          }
+        end
+
+        it "add the `message_id`" do
+          expect(subject.get(id_field)).to eq(message_id)
+        end
+
+        it "add the `md5_of_body`" do
+          expect(subject.get(md5_field)).to eq(md5_of_body)
+        end
+
+        it "add the `sent_timestamp`" do
+          expect(subject.get(sent_timestamp_field).to_i).to eq(sent_timestamp.to_i)
+        end
+      end
+
+      context "when the option isn't specified" do
+        it "doesnt add the `message_id`" do
+          expect(subject).not_to include(id_field)
+        end
+
+        it "doesnt add the `md5_of_body`" do
+          expect(subject).not_to include(md5_field)
+        end
+
+        it "doesnt add the `sent_timestamp`" do
+          expect(subject).not_to include(sent_timestamp_field)
+        end
+      end
+    end
+
+    context "when decoding body" do
+      subject { LogStash::Inputs::SQS::new(config.merge({ "codec" => "json" })) }
+
+      it "uses the specified codec" do
+        subject.handle_message(encoded_message, queue)
+        expect(queue.pop.get("bonjour")).to eq(decoded_message["bonjour"])
+      end
+    end
+
+    context "receiving messages" do
+      before do 
+        expect(subject).to receive(:poller).and_return(mock_sqs).at_least(:once)
+      end
+
+      it "creates logstash event" do
+        expect(mock_sqs).to receive(:poll).with(anything()).and_yield([encoded_message], double("stats"))
+        subject.run(queue)
+        expect(queue.pop.get("bonjour")).to eq(decoded_message["bonjour"])
+      end
+
+      context 'can create multiple events' do
+        require "logstash/codecs/json_lines"
+        let(:config) { super().merge({ "codec" => "json_lines" }) }
+        let(:first_message) { { "sequence" => "first" } }
+        let(:second_message) { { "sequence" => "second" } }
+        let(:encoded_message)  { double("sqs_message", :body => "#{LogStash::Json::dump(first_message)}\n#{LogStash::Json::dump(second_message)}\n") }
+
+        it 'creates multiple events' do
+          expect(mock_sqs).to receive(:poll).with(anything()).and_yield([encoded_message], double("stats"))
+          subject.run(queue)
+          events = queue.map{ |e|e.get('sequence')}
+          expect(events).to match_array([first_message['sequence'], second_message['sequence']])
+        end
+      end
+    end
+
+    context "on errors" do
+      let(:payload) { "Hello world" }
+
+      before do
+        expect(subject).to receive(:poller).and_return(mock_sqs).at_least(:once)
+      end
+
+      context "SQS error" do
+        it "retry to fetch messages" do
+          # change the poller implementation to raise SQS errors.
+          had_error = false
+          
+          # actually using the child of `Object` to do an expectation of `#sleep`
+          expect(subject).to receive(:sleep).with(LogStash::Inputs::SQS::BACKOFF_SLEEP_TIME)
+          expect(mock_sqs).to receive(:poll).with(anything()).at_most(2) do
+            unless had_error
+              had_error = true
+              raise Aws::SQS::Errors::ServiceError.new("testing", "testing exception")
+            end
+
+            queue << payload
+          end
+
+          subject.run(queue)
+
+          expect(queue.size).to eq(1)
+          expect(queue.pop).to eq(payload)
+        end
+      end
+
+      context "SQS error (retries)" do
+
+        it "retry to fetch messages" do
+          sleep_time = LogStash::Inputs::SQS::BACKOFF_SLEEP_TIME
+          expect(subject).to receive(:sleep).with(sleep_time)
+          expect(subject).to receive(:sleep).with(sleep_time * 2)
+          expect(subject).to receive(:sleep).with(sleep_time * 4)
+
+          error_count = 0
+          expect(mock_sqs).to receive(:poll).with(anything()).at_most(4) do
+            error_count += 1
+            if error_count <= 3
+              raise Aws::SQS::Errors::QueueDoesNotExist.new("testing", "testing exception (#{error_count})")
+            end
+
+            queue << payload
+          end
+
+          subject.run(queue)
+
+          expect(queue.size).to eq(1)
+          expect(queue.pop).to eq(payload)
+        end
+
+      end
+
+      context "networking error" do
+
+        before(:all) { require 'seahorse/client/networking_error' }
+
+        it "retry to fetch messages" do
+          sleep_time = LogStash::Inputs::SQS::BACKOFF_SLEEP_TIME
+          expect(subject).to receive(:sleep).with(sleep_time).twice
+
+          error_count = 0
+          expect(mock_sqs).to receive(:poll).with(anything()).at_most(5) do
+            error_count += 1
+            if error_count == 1
+              raise Seahorse::Client::NetworkingError.new(Net::OpenTimeout.new, 'timeout')
+            end
+            if error_count == 3
+              raise Seahorse::Client::NetworkingError.new(SocketError.new('spec-error'))
+            end
+
+            queue << payload
+          end
+
+          subject.run(queue)
+          subject.run(queue)
+
+          expect(queue.size).to eq(2)
+          expect(queue.pop).to eq(payload)
+        end
+
+      end
+
+      context "other error" do
+        it "stops executing the code and raise the exception" do
+          expect(mock_sqs).to receive(:poll).with(anything()).at_most(2) do
+            raise RuntimeError
+          end
+
+          expect { subject.run(queue) }.to raise_error(RuntimeError)
+        end
+      end
+    end
+  end
+end

data/spec/integration/cloudwatch_spec.rb

@@ -0,0 +1,25 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/cloudwatch"
+
+describe LogStash::Inputs::CloudWatch, :integration => true do
+
+  let(:settings)  {  { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
+                       "secret_access_key" => LogStash::Util::Password.new(ENV['AWS_SECRET_ACCESS_KEY']),
+                       "region" => ENV["AWS_REGION"] || "us-east-1",
+                       "namespace" => "AWS/S3",
+                       'filters' => { "BucketName" => "*"},
+                       'metrics' => ["BucketSizeBytes","NumberOfObjects"]
+
+  }}
+
+  def metrics_for(settings)
+    cw = LogStash::Inputs::CloudWatch.new(settings)
+    cw.register
+    cw.send('metrics_for', settings['namespace'])
+  end
+
+  #
+  it "should not raise a type error when using a password" do
+    expect{metrics_for(settings)}.not_to raise_error
+  end
+end

data/spec/integration/dynamic_prefix_spec.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Dynamic Prefix", :integration => true do
+  include_context "setup plugin"
+
+  let(:options) { main_options.merge({ "rotation_strategy" => "size" }) }
+  let(:sandbox) { "test" }
+
+  before do
+    clean_remote_files(sandbox)
+    subject.register
+    subject.multi_receive_encoded(batch)
+    subject.close
+  end
+
+  context "With field string" do
+    let(:prefix) { "/#{sandbox}/%{server}/%{language}" }
+    let(:batch) do
+      b = {}
+      e1 = LogStash::Event.new({ "server" => "es1", "language" => "ruby"})
+      b[e1] = "es1-ruby"
+      e2 = LogStash::Event.new({ "server" => "es2", "language" => "java"})
+      b[e2] = "es2-ruby"
+      b
+    end
+
+    it "creates a specific quantity of files" do
+      expect(bucket_resource.objects(:prefix => sandbox).count).to eq(batch.size)
+    end
+
+    it "creates specific keys" do
+      re = Regexp.union(/^es1\/ruby\/ls.s3.sashimi/, /^es2\/java\/ls.s3.sashimi/)
+
+      bucket_resource.objects(:prefix => sandbox) do |obj|
+        expect(obj.key).to match(re)
+      end
+    end
+
+    it "Persists all events" do
+      download_directory = Stud::Temporary.pathname
+
+      FileUtils.rm_rf(download_directory)
+      FileUtils.mkdir_p(download_directory)
+
+      counter = 0
+      bucket_resource.objects(:prefix => sandbox).each do |object|
+        target = File.join(download_directory, "#{counter}.txt")
+        object.get(:response_target => target)
+        counter += 1
+      end
+      expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(batch.size)
+    end
+  end
+
+  context "with unsupported char" do
+    let(:prefix) { "/#{sandbox}/%{server}/%{language}" }
+    let(:batch) do
+      b = {}
+      e1 = LogStash::Event.new({ "server" => "e>s1", "language" => "ruby"})
+      b[e1] = "es2-ruby"
+      b
+    end
+
+    it "convert them to underscore" do
+      re = Regexp.union(/^e_s1\/ruby\/ls.s3.sashimi/)
+
+      bucket_resource.objects(:prefix => sandbox) do |obj|
+        expect(obj.key).to match(re)
+      end
+    end
+  end
+
+  context "with dates" do
+    let(:prefix) { "/#{sandbox}/%{+YYYY-MM-d}" }
+
+    let(:batch) do
+      b = {}
+      e1 = LogStash::Event.new({ "server" => "e>s1", "language" => "ruby"})
+      b[e1] = "es2-ruby"
+      b
+    end
+
+    it "creates dated path" do
+      re = /^#{sandbox}\/\d{4}-\d{2}-\d{1,2}\/ls\.s3\./
+      expect(bucket_resource.objects(:prefix => sandbox).first.key).to match(re)
+    end
+  end
+end

data/spec/integration/gzip_file_spec.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Gzip File Time rotation with constant write", :integration => true do
+  include_context "setup plugin"
+
+  let(:time_file) { 0.004 }
+  let(:options) { main_options.merge({ "encoding" => "gzip",
+                                       "rotation_strategy" => "time" }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:minimum_number_of_time_rotation) { 3 }
+  let(:batch_step) { (number_of_events / minimum_number_of_time_rotation).ceil }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+
+    # simulate batch read/write
+    batch.each_slice(batch_step) do |batch_time|
+      batch_time.each_slice(batch_size) do |smaller_batch|
+        subject.multi_receive_encoded(smaller_batch)
+      end
+      sleep(1)
+    end
+
+    subject.close
+  end
+
+  it "creates multiples files" do
+    # using close will upload the current file
+    expect(bucket_resource.objects(:prefix => prefix).count).to be_between(minimum_number_of_time_rotation, minimum_number_of_time_rotation + 1).inclusive
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.gz")
+      object.get(:response_target => target)
+      counter += 1
+    end
+
+    expect(Dir.glob(File.join(download_directory, "**", "*.gz")).inject(0) { |sum, f| sum + Zlib::GzipReader.new(File.open(f)).readlines.size }).to eq(number_of_events)
+  end
+end

data/spec/integration/gzip_size_rotation_spec.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Gzip Size rotation", :integration => true do
+  include_context "setup plugin"
+
+  let(:document_size) { 20 * 1024 } # in bytes
+
+  let(:options) do
+    main_options.merge({
+      "encoding" => "gzip",
+      "size_file" => document_size,
+      "rotation_strategy" => "size" })
+  end
+
+  let(:number_of_events) { 1_000_000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" * 20 }
+  let(:batch) do
+    b = {}
+    batch_size.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:number_of_files) { number_of_events / 50000 }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+    (number_of_events/batch_size).times do
+      subject.multi_receive_encoded(batch)
+    end
+    subject.close
+  end
+
+  it "Rotates the files based on size" do
+    f = bucket_resource.objects(:prefix => prefix).first
+    expect(f.size).to be_between(document_size, document_size * 2).inclusive
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt.gz")
+      object.get(:response_target => target)
+      counter += 1
+    end
+
+    expect(Dir.glob(File.join(download_directory, "**", "*.gz")).inject(0) do |sum, f|
+      sum + Zlib::GzipReader.new(File.open(f)).readlines.size
+    end).to eq(number_of_events)
+  end
+end

data/spec/integration/outputs/sqs_spec.rb

@@ -0,0 +1,98 @@
+# encoding: utf-8
+
+require_relative '../../spec_helper'
+require 'logstash/event'
+require 'logstash/json'
+require 'securerandom'
+
+describe LogStash::Outputs::SQS, :integration => true do
+  let(:config) do
+    {
+      'queue' => @queue_name,
+    }
+  end
+  subject { described_class.new(config) }
+
+  # Create an SQS queue with a random name.
+  before(:all) do
+    @sqs = Aws::SQS::Client.new
+    @queue_name = "logstash-output-sqs-#{SecureRandom.hex}"
+    @queue_url = @sqs.create_queue(:queue_name => @queue_name)[:queue_url]
+  end
+
+  # Destroy the SQS queue which was created in `before(:all)`.
+  after(:all) do
+    @sqs.delete_queue(:queue_url => @queue_url)
+  end
+
+  describe '#register' do
+    context 'with invalid credentials' do
+      let(:config) do
+        super().merge({
+          'access_key_id' => 'bad_access',
+          'secret_access_key' => 'bad_secret_key',
+        })
+      end
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with a nonexistent queue' do
+      let(:config) { super().merge('queue' => 'nonexistent-queue') }
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with a nonpermissive account id' do
+      let(:config) { super().merge('queue_owner_aws_account_id' => '123456789012')}
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with valid configuration' do
+      it 'does not raise an error' do
+        expect { subject.register }.not_to raise_error
+      end
+    end
+  end
+
+  describe '#multi_receive_encoded' do
+    let(:sample_count) { 10 }
+    let(:sample_event) { LogStash::Event.new('message' => 'This is a message') }
+    let(:sample_events) do
+      sample_count.times.map do
+        [sample_event, LogStash::Json.dump(sample_event)]
+      end
+    end
+
+    before do
+      subject.register
+    end
+
+    after do
+      subject.close
+    end
+
+    context 'with batching disabled' do
+      let(:config) { super().merge('batch_events' => 1) }
+
+      it 'publishes to SQS' do
+        subject.multi_receive_encoded(sample_events)
+        expect(receive_all_messages(@queue_url).count).to eq(sample_events.count)
+      end
+    end
+
+    context 'with batching enabled (default)' do
+      it 'publishes to SQS' do
+        subject.multi_receive_encoded(sample_events)
+        expect(receive_all_messages(@queue_url).count).to eq(sample_events.count)
+      end
+    end
+  end
+end