logstash-integration-aws 0.1.0.pre

data/spec/support/helpers.rb

@@ -0,0 +1,119 @@
+def fetch_events(settings)
+  queue = []
+  s3 = LogStash::Inputs::S3.new(settings)
+  s3.register
+  s3.process_files(queue)
+  queue
+end
+
+# delete_files(prefix)
+def upload_file(local_file, remote_name)
+  bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
+  bucket.object(remote_name).upload_file(file)
+end
+
+def delete_remote_files(prefix)
+  bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  bucket.objects(:prefix => prefix).each { |object| object.delete }
+end
+
+def list_remote_files(prefix, target_bucket = ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  bucket = s3object.bucket(target_bucket)
+  bucket.objects(:prefix => prefix).collect(&:key)
+end
+
+def create_bucket(name)
+  s3object.bucket(name).create
+end
+
+def delete_bucket(name)
+  s3object.bucket(name).objects.map(&:delete)
+  s3object.bucket(name).delete
+end
+
+def s3object
+  Aws::S3::Resource.new
+end
+
+class TestInfiniteS3Object
+  def initialize(s3_obj)
+    @s3_obj = s3_obj
+  end
+
+  def each
+    counter = 1
+
+    loop do
+      yield @s3_obj
+      counter +=1
+    end
+  end
+end
+
+def push_sqs_event(message)
+  client = Aws::SQS::Client.new
+  queue_url = client.get_queue_url(:queue_name => ENV["SQS_QUEUE_NAME"])
+
+  client.send_message({
+    queue_url: queue_url.queue_url,
+    message_body: message,
+  })
+end
+
+shared_context "setup plugin" do
+  let(:temporary_directory) { Stud::Temporary.pathname }
+
+  let(:bucket) { ENV["AWS_LOGSTASH_TEST_BUCKET"] }
+  let(:access_key_id) {  ENV["AWS_ACCESS_KEY_ID"] }
+  let(:secret_access_key) { ENV["AWS_SECRET_ACCESS_KEY"] }
+  let(:size_file) { 100 }
+  let(:time_file) { 100 }
+  let(:tags) { [] }
+  let(:prefix) { "home" }
+  let(:region) { ENV['AWS_REGION'] }
+
+  let(:main_options) do
+    {
+      "bucket" => bucket,
+      "prefix" => prefix,
+      "temporary_directory" => temporary_directory,
+      "access_key_id" => access_key_id,
+      "secret_access_key" => secret_access_key,
+      "size_file" => size_file,
+      "time_file" => time_file,
+      "region" => region,
+      "tags" => []
+    }
+  end
+
+  let(:client_credentials) { Aws::Credentials.new(access_key_id, secret_access_key) }
+  let(:bucket_resource) { Aws::S3::Bucket.new(bucket, { :credentials => client_credentials, :region => region }) }
+
+  subject { LogStash::Outputs::S3.new(options) }
+end
+
+def clean_remote_files(prefix = "")
+  bucket_resource.objects(:prefix => prefix).each do |object|
+    object.delete
+  end
+end
+
+# Retrieve all available messages from the specified queue.
+#
+# Rather than utilizing `Aws::SQS::QueuePoller` directly in order to poll an
+# SQS queue for messages, this method retrieves and returns all messages that
+# are able to be received from the specified SQS queue.
+def receive_all_messages(queue_url, options = {})
+  options[:idle_timeout] ||= 0
+  options[:max_number_of_messages] ||= 10
+
+  messages = []
+  poller = Aws::SQS::QueuePoller.new(queue_url, options)
+
+  poller.poll do |received_messages|
+    messages.concat(received_messages)
+  end
+
+  messages
+end

data/spec/unit/outputs/sqs_spec.rb

@@ -0,0 +1,247 @@
+# encoding: utf-8
+
+require_relative '../../spec_helper'
+require 'logstash/errors'
+require 'logstash/event'
+require 'logstash/json'
+
+describe LogStash::Outputs::SQS do
+  let(:config) do
+    {
+      'queue' => queue_name,
+      'region' => region,
+    }
+  end
+  let(:queue_name) { 'my-sqs-queue' }
+  let(:queue_url) { "https://sqs.#{region}.amazonaws.com/123456789012/#{queue_name}" }
+  let(:region) { 'us-east-1' }
+
+  let(:sqs) { Aws::SQS::Client.new(:stub_responses => true) }
+  subject { described_class.new(config) }
+
+  describe '#register' do
+    context 'with a batch size that is too large' do
+      let(:config) { super().merge('batch_events' => 100) }
+
+      before do
+        allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
+      end
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with a batch size that is too small' do
+      let(:config) { super().merge('batch_events' => 0) }
+
+      before do
+        allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
+      end
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'without a queue' do
+      let(:config) { Hash.new }
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with a nonexistent queue' do
+      before do
+        expect(Aws::SQS::Client).to receive(:new).and_return(sqs)
+        expect(sqs).to receive(:get_queue_url).with(:queue_name => queue_name) do
+          raise Aws::SQS::Errors::NonExistentQueue.new(nil, 'The specified queue does not exist for this wsdl version.')
+        end
+      end
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with a valid queue' do
+      before do
+        expect(Aws::SQS::Client).to receive(:new).and_return(sqs)
+        expect(sqs).to receive(:get_queue_url).with(:queue_name => queue_name).and_return(:queue_url => queue_url)
+      end
+
+      it 'does not raise an error' do
+        expect { subject.register }.not_to raise_error
+      end
+    end
+  end
+
+  describe '#multi_receive_encoded' do
+    before do
+      expect(Aws::SQS::Client).to receive(:new).and_return(sqs)
+      expect(sqs).to receive(:get_queue_url).with(:queue_name => queue_name).and_return(:queue_url => queue_url)
+      subject.register
+    end
+
+    after do
+      subject.close
+    end
+
+    let(:sample_count) { 10 }
+    let(:sample_event) { LogStash::Event.new('message' => 'This is a message') }
+    let(:sample_event_encoded) { LogStash::Json.dump(sample_event) }
+    let(:sample_events) do
+      sample_count.times.map do
+        [sample_event, sample_event_encoded]
+      end
+    end
+
+    context 'with batching disabled' do
+      let(:config) do
+        super().merge({
+          'batch_events' => 1,
+        })
+      end
+
+      it 'should call send_message' do
+        expect(sqs).to receive(:send_message).with(:queue_url => queue_url, :message_body => sample_event_encoded).exactly(sample_count).times
+        subject.multi_receive_encoded(sample_events)
+      end
+
+      it 'should not call send_message_batch' do
+        expect(sqs).not_to receive(:send_message_batch)
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+
+    context 'with batching enabled' do
+      let(:batch_events) { 3 }
+      let(:config) do
+        super().merge({
+          'batch_events' => batch_events,
+        })
+      end
+
+      let(:sample_batches) do
+        sample_events.each_slice(batch_events).each_with_index.map do |sample_batch, batch_index|
+          sample_batch.each_with_index.map do |encoded_event, index|
+            event, encoded = encoded_event
+            {
+              :id => (batch_index * batch_events + index).to_s,
+              :message_body => encoded,
+            }
+          end
+        end
+      end
+
+      it 'should call send_message_batch' do
+        expect(sqs).to receive(:send_message_batch).at_least(:once)
+        subject.multi_receive_encoded(sample_events)
+      end
+
+      it 'should batch events' do
+        sample_batches.each do |batch_entries|
+          expect(sqs).to receive(:send_message_batch).with(:queue_url => queue_url, :entries => batch_entries)
+        end
+
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+
+    context 'with empty payload' do
+      let(:sample_count) { 0 }
+
+      it 'does not raise an error' do
+        expect { subject.multi_receive_encoded(sample_events) }.not_to raise_error
+      end
+
+      it 'should not send any messages' do
+        expect(sqs).not_to receive(:send_message)
+        expect(sqs).not_to receive(:send_message_batch)
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+
+    context 'with event exceeding maximum size' do
+      let(:config) { super().merge('message_max_size' => message_max_size) }
+      let(:message_max_size) { 1024 }
+
+      let(:sample_count) { 1 }
+      let(:sample_event) { LogStash::Event.new('message' => '.' * message_max_size) }
+
+      it 'should drop event' do
+        expect(sqs).not_to receive(:send_message)
+        expect(sqs).not_to receive(:send_message_batch)
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+
+
+
+    context 'with large batch' do
+      let(:batch_events) { 4 }
+      let(:config) do
+        super().merge({
+          'batch_events' => batch_events,
+          'message_max_size' => message_max_size,
+        })
+      end
+      let(:message_max_size) { 1024 }
+
+      let(:sample_events) do
+        # This is the overhead associating with transmitting each message. The
+        # overhead is caused by metadata (such as the `message` field name and
+        # the `@timestamp` field) as well as additional characters as a result
+        # of encoding the event.
+        overhead = 90
+
+        events = [
+          LogStash::Event.new('message' => 'a' * (0.6 * message_max_size - overhead)),
+          LogStash::Event.new('message' => 'b' * (0.5 * message_max_size - overhead)),
+          LogStash::Event.new('message' => 'c' * (0.5 * message_max_size - overhead)),
+          LogStash::Event.new('message' => 'd' * (0.4 * message_max_size - overhead)),
+        ]
+
+        events.map do |event|
+          [event, LogStash::Json.dump(event)]
+        end
+      end
+
+      let(:sample_batches) do
+        [
+          [
+            {
+              :id => '0',
+              :message_body => sample_events[0][1],
+            },
+          ],
+          [
+            {
+              :id => '1',
+              :message_body => sample_events[1][1],
+            },
+            {
+              :id => '2',
+              :message_body => sample_events[2][1],
+            },
+          ],
+          [
+            {
+              :id => '3',
+              :message_body => sample_events[3][1],
+            },
+          ],
+        ]
+      end
+
+      it 'should split events into smaller batches' do
+        sample_batches.each do |entries|
+          expect(sqs).to receive(:send_message_batch).with(:queue_url => queue_url, :entries => entries)
+        end
+
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+  end
+end