logstash-output-sqs 3.0.2 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5099a8853aa054d817e2a5e96d48f9d0e9f0c900
-  data.tar.gz: b9220f0a03ac674117ebc0eecfd5b970051297fd
+  metadata.gz: 4a12c855c89c8c21ea8ce285b37a6634d49edc2e
+  data.tar.gz: f2f7443979136609b06cba6f4fad00a7d279b882
 SHA512:
-  metadata.gz: 7558caee7aa481c71ae522900f8a000df55715553acf5e6660894ffb3f556a1aa6b067be9473662ce5e8146c62162e165e03d7ec8b1d9314a46c3bdb761ac52e
-  data.tar.gz: d766b61f6235c8996552671f9ba2815b6e80240c3d389e372427d3450ba80ef360ba88be923f49338d9beec06e74f30780eaca9330ada4183b5fac419ea5f94c
+  metadata.gz: 3b0fef2f371a8d541e45690cb958461237ec7413ddee50585427e56ee9aefd42b1f4a53fb2be3b911ebf627f78f241f42fdfc444b3fbb0cad4f12b28340ec529
+  data.tar.gz: fb92b8e9f07c111a7d0cc7228bba59059a77b8bc7ab1adea5429b0586047d614cc6292d7a49408c717d780f6806d11603494bd081c75aaaeb1ddff278bd92db4

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 4.0.0
+  - Add unit and integration tests.
+  - Adjust the sample IAM policy in the documentation, removing actions which are not actually required by the plugin. Specifically, the following actions are not required: `sqs:ChangeMessageVisibility`, `sqs:ChangeMessageVisibilityBatch`, `sqs:GetQueueAttributes` and `sqs:ListQueues`.
+  - Dynamically adjust the batch message size. SQS allows up to 10 messages to be published in a single batch, however the total size of the batch is limited to 256KiB (see [Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)). This plugin will now dynamically adjust the number of events included in each batch to ensure that the total batch size does not exceed `message_max_size`. Note that any single messages which exceeds the 256KiB size limit will be dropped.
+  - Move to the new concurrency model, `:shared`.
+  - The `batch_timeout` parameter has been deprecated because it no longer has any effect.
+  - The individual (non-batch) mode of operation (i.e. `batch => false`) has been deprecated. Batch mode is vastly more performant and we do not believe that there are any use cases which require non-batch mode. You can emulate non-batch mode by setting `batch_events => 1`, although this will call `sqs:SendMessageBatch` with a batch size of 1 rather than calling `sqs:SendMessage`.
+  - The plugin now implements `#multi_receive_encoded` and no longer uses `Stud::Buffer`.
+  - Update the AWS SDK to version 2.
+
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 
@@ -13,7 +23,7 @@
 # 2.0.3
   - New dependency requirements for logstash-core for the 5.0 release
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/CONTRIBUTORS

@@ -6,12 +6,13 @@ Contributors:
 * James Turnbull (jamtur01)
 * John Price (awesometown)
 * Jordan Sissel (jordansissel)
+* Joshua Spence (joshuaspence)
+* Jurgens du Toit (jrgns)
 * Kurt Hurtado (kurtado)
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Sean Laurent (organicveggie)
 * Toby Collier (tobyw4n)
-* Jurgens du Toit (jrgns)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/outputs/sqs.rb

@@ -1,136 +1,175 @@
 # encoding: utf-8
-require "logstash/outputs/base"
-require "logstash/namespace"
-require "logstash/plugin_mixins/aws_config"
-require "stud/buffer"
-require "digest/sha2"
 
-# Push events to an Amazon Web Services Simple Queue Service (SQS) queue.
+require 'aws-sdk'
+require 'logstash/errors'
+require 'logstash/namespace'
+require 'logstash/outputs/base'
+require 'logstash/outputs/sqs/patch'
+require 'logstash/plugin_mixins/aws_config'
+
+# Forcibly load all modules marked to be lazily loaded.
 #
-# SQS is a simple, scalable queue system that is part of the 
-# Amazon Web Services suite of tools.
+# It is recommended that this is called prior to launching threads. See
+# https://aws.amazon.com/blogs/developer/threading-with-the-aws-sdk-for-ruby/.
+Aws.eager_autoload!
+
+# Push events to an Amazon Web Services (AWS) Simple Queue Service (SQS) queue.
 #
-# Although SQS is similar to other queuing systems like AMQP, it
-# uses a custom API and requires that you have an AWS account.
-# See http://aws.amazon.com/sqs/ for more details on how SQS works,
-# what the pricing schedule looks like and how to setup a queue.
+# SQS is a simple, scalable queue system that is part of the Amazon Web
+# Services suite of tools. Although SQS is similar to other queuing systems
+# such as Advanced Message Queuing Protocol (AMQP), it uses a custom API and
+# requires that you have an AWS account. See http://aws.amazon.com/sqs/ for
+# more details on how SQS works, what the pricing schedule looks like and how
+# to setup a queue.
 #
-# To use this plugin, you *must*:
+# The "consumer" identity must have the following permissions on the queue:
 #
-#  * Have an AWS account
-#  * Setup an SQS queue
-#  * Create an identify that has access to publish messages to the queue.
+#   * `sqs:GetQueueUrl`
+#   * `sqs:SendMessage`
+#   * `sqs:SendMessageBatch`
 #
-# The "consumer" identity must have the following permissions on the queue:
+# Typically, you should setup an IAM policy, create a user and apply the IAM
+# policy to the user. See http://aws.amazon.com/iam/ for more details on
+# setting up AWS identities. A sample policy is as follows:
+#
+# [source,json]
+# {
+#   "Version": "2012-10-17",
+#   "Statement": [
+#     {
+#       "Effect": "Allow",
+#       "Action": [
+#         "sqs:GetQueueUrl",
+#         "sqs:SendMessage",
+#         "sqs:SendMessageBatch"
+#       ],
+#       "Resource": "arn:aws:sqs:us-east-1:123456789012:my-sqs-queue"
+#     }
+#   ]
+# }
 #
-#  * sqs:ChangeMessageVisibility
-#  * sqs:ChangeMessageVisibilityBatch
-#  * sqs:GetQueueAttributes
-#  * sqs:GetQueueUrl
-#  * sqs:ListQueues
-#  * sqs:SendMessage
-#  * sqs:SendMessageBatch
+# = Batch Publishing
+# This output publishes messages to SQS in batches in order to optimize event
+# throughput and increase performance. This is done using the
+# [`SendMessageBatch`](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html)
+# API. When publishing messages to SQS in batches, the following service limits
+# must be respected (see
+# [Limits in Amazon SQS](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html)):
 #
-# Typically, you should setup an IAM policy, create a user and apply the IAM policy to the user.
-# A sample policy is as follows:
-# [source,ruby]
-#      {
-#        "Statement": [
-#          {
-#            "Sid": "Stmt1347986764948",
-#            "Action": [
-#              "sqs:ChangeMessageVisibility",
-#              "sqs:ChangeMessageVisibilityBatch",
-#              "sqs:GetQueueAttributes",
-#              "sqs:GetQueueUrl",
-#              "sqs:ListQueues",
-#              "sqs:SendMessage",
-#              "sqs:SendMessageBatch"
-#            ],
-#            "Effect": "Allow",
-#            "Resource": [
-#              "arn:aws:sqs:us-east-1:200850199751:Logstash"
-#            ]
-#          }
-#        ]
-#      }
+#   * The maximum allowed individual message size is 256KiB.
+#   * The maximum total payload size (i.e. the sum of the sizes of all
+#     individual messages within a batch) is also 256KiB.
 #
-# See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
+# This plugin will dynamically adjust the size of the batch published to SQS in
+# order to ensure that the total payload size does not exceed 256KiB.
+#
+# WARNING: This output cannot currently handle messages larger than 256KiB. Any
+# single message exceeding this size will be dropped.
 #
 class LogStash::Outputs::SQS < LogStash::Outputs::Base
-  include LogStash::PluginMixins::AwsConfig
-  include Stud::Buffer
+  include LogStash::PluginMixins::AwsConfig::V2
 
-  config_name "sqs"
+  config_name 'sqs'
+  default :codec, 'json'
 
-  # Name of SQS queue to push messages into. Note that this is just the name of the queue, not the URL or ARN.
-  config :queue, :validate => :string, :required => true
+  concurrency :shared
 
-  # Set to true if you want send messages to SQS in batches with `batch_send`
-  # from the amazon sdk
-  config :batch, :validate => :boolean, :default => true
+  # Set to `true` to send messages to SQS in batches (with the
+  # `SendMessageBatch` API) or `false` to send messages to SQS individually
+  # (with the `SendMessage` API). The size of the batch is configurable via
+  # `batch_events`.
+  config :batch, :validate => :boolean, :default => true, :deprecated => true
 
-  # If `batch` is set to true, the number of events we queue up for a `batch_send`.
+  # The number of events to be sent in each batch. Set this to `1` to disable
+  # the batch sending of messages.
   config :batch_events, :validate => :number, :default => 10
 
-  # If `batch` is set to true, the maximum amount of time between `batch_send` commands when there are pending events to flush.
-  config :batch_timeout, :validate => :number, :default => 5
+  config :batch_timeout, :validate => :number, :deprecated => 'This setting no longer has any effect.'
 
-  public
-  def aws_service_endpoint(region)
-    return {
-        :sqs_endpoint => "sqs.#{region}.amazonaws.com"
-    }
-  end
+  # The maximum number of bytes for any message sent to SQS. Messages exceeding
+  # this size will be dropped. See
+  # http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/limits-messages.html.
+  config :message_max_size, :validate => :bytes, :default => '256KiB'
 
-  public 
+  # The name of the target SQS queue. Note that this is just the name of the
+  # queue, not the URL or ARN.
+  config :queue, :validate => :string, :required => true
+
+  public
   def register
-    require "aws-sdk"
-
-    @sqs = AWS::SQS.new(aws_options_hash)
-
-    if @batch
-      if @batch_events > 10
-        raise RuntimeError.new(
-          "AWS only allows a batch_events parameter of 10 or less"
-        )
-      elsif @batch_events <= 1
-        raise RuntimeError.new(
-          "batch_events parameter must be greater than 1 (or its not a batch)"
-        )
-      end
-      buffer_initialize(
-        :max_items => @batch_events,
-        :max_interval => @batch_timeout,
-        :logger => @logger
-      )
-    end
+    @sqs = Aws::SQS::Client.new(aws_options_hash)
+
+    if @batch_events > 10
+      raise LogStash::ConfigurationError, 'The maximum batch size is 10 events'
+    elsif @batch_events < 1
+      raise LogStash::ConfigurationError, 'The batch size must be greater than 0'
+   end
 
     begin
-      @logger.debug("Connecting to AWS SQS queue '#{@queue}'...")
-      @sqs_queue = @sqs.queues.named(@queue)
-      @logger.info("Connected to AWS SQS queue '#{@queue}' successfully.")
-    rescue Exception => e
-      @logger.error("Unable to access SQS queue '#{@queue}': #{e.to_s}")
-    end # begin/rescue
-  end # def register
+      @logger.debug('Connecting to SQS queue', :queue => @queue, :region => region)
+      @queue_url = @sqs.get_queue_url(:queue_name => @queue)[:queue_url]
+      @logger.info('Connected to SQS queue successfully', :queue => @queue, :region => region)
+    rescue Aws::SQS::Errors::ServiceError => e
+      @logger.error('Failed to connect to SQS', :error => e)
+      raise LogStash::ConfigurationError, 'Verify the SQS queue name and your credentials'
+    end
+  end
 
   public
-  def receive(event)
-    if @batch
-      buffer_receive(event.to_json)
-      return
+  def multi_receive_encoded(encoded_events)
+    if @batch and @batch_events > 1
+      multi_receive_encoded_batch(encoded_events)
+    else
+      multi_receive_encoded_single(encoded_events)
     end
-    @sqs_queue.send_message(event.to_json)
-  end # def receive
+  end
 
-  # called from Stud::Buffer#buffer_flush when there are events to flush
-  def flush(events, close=false)
-    @sqs_queue.batch_send(events)
+  private
+  def multi_receive_encoded_batch(encoded_events)
+    bytes = 0
+    entries = []
+
+    # Split the events into multiple batches to ensure that no single batch
+    # exceeds `@message_max_size` bytes.
+    encoded_events.each_with_index do |encoded_event, index|
+      event, encoded = encoded_event
+
+      if encoded.bytesize > @message_max_size
+        @logger.warn('Message exceeds maximum length and will be dropped', :message_size => encoded.bytesize)
+        next
+      end
+
+      if entries.size >= @batch_events or (bytes + encoded.bytesize) > @message_max_size
+        send_message_batch(entries)
+
+        bytes = 0
+        entries = []
+      end
+
+      bytes += encoded.bytesize
+      entries.push(:id => index.to_s, :message_body => encoded)
+    end
+
+    send_message_batch(entries) unless entries.empty?
   end
 
-  public
-  def close
-    buffer_flush(:final => true)
-  end # def close
+  private
+  def multi_receive_encoded_single(encoded_events)
+    encoded_events.each do |encoded_event|
+      event, encoded = encoded_event
+
+      if encoded.bytesize > @message_max_size
+        @logger.warn('Message exceeds maximum length and will be dropped', :message_size => encoded.bytesize)
+        next
+      end
+
+      @sqs.send_message(:queue_url => @queue_url, :message_body => encoded)
+    end
+  end
+
+  private
+  def send_message_batch(entries)
+    @logger.debug("Publishing #{entries.size} messages to SQS", :queue_url => @queue_url, :entries => entries)
+    @sqs.send_message_batch(:queue_url => @queue_url, :entries => entries)
+  end
 end

data/lib/logstash/outputs/sqs/patch.rb

@@ -0,0 +1,21 @@
+# This patch was stolen from logstash-plugins/logstash-output-s3#102.
+#
+# This patch is a workaround for a JRuby issue which has been fixed in JRuby
+# 9000, but not in JRuby 1.7. See https://github.com/jruby/jruby/issues/3645
+# and https://github.com/jruby/jruby/issues/3920. This is necessary because the
+# `aws-sdk` is doing tricky name discovery to generate the correct error class.
+#
+# As per https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960,
+# this patch may be short-lived anyway.
+require 'aws-sdk'
+
+begin
+  old_stderr = $stderr
+  $stderr = StringIO.new
+
+  module Aws
+    const_set(:SQS, Aws::SQS)
+  end
+ensure
+  $stderr = old_stderr
+end

data/logstash-output-sqs.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-sqs'
-  s.version         = '3.0.2'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Push events to an Amazon Web Services Simple Queue Service (SQS) queue."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,12 +20,10 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency "logstash-mixin-aws", ">= 1.0.0"
-
-  s.add_runtime_dependency 'aws-sdk'
-  s.add_runtime_dependency 'stud'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
+  s.add_runtime_dependency 'logstash-mixin-aws', '>= 1.0.0'
 
+  s.add_development_dependency 'logstash-codec-json'
   s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/integration/outputs/sqs_spec.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+
+require_relative '../../spec_helper'
+require 'logstash/event'
+require 'logstash/json'
+require 'securerandom'
+
+describe LogStash::Outputs::SQS, :integration => true do
+  let(:config) do
+    {
+      'queue' => @queue_name,
+    }
+  end
+  subject { described_class.new(config) }
+
+  # Create an SQS queue with a random name.
+  before(:all) do
+    @sqs = Aws::SQS::Client.new
+    @queue_name = "logstash-output-sqs-#{SecureRandom.hex}"
+    @queue_url = @sqs.create_queue(:queue_name => @queue_name)[:queue_url]
+  end
+
+  # Destroy the SQS queue which was created in `before(:all)`.
+  after(:all) do
+    @sqs.delete_queue(:queue_url => @queue_url)
+  end
+
+  describe '#register' do
+    context 'with invalid credentials' do
+      let(:config) do
+        super.merge({
+          'access_key_id' => 'bad_access',
+          'secret_access_key' => 'bad_secret_key',
+        })
+      end
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with a nonexistent queue' do
+      let(:config) { super.merge('queue' => 'nonexistent-queue') }
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with valid configuration' do
+      it 'does not raise an error' do
+        expect { subject.register }.not_to raise_error
+      end
+    end
+  end
+
+  describe '#multi_receive_encoded' do
+    let(:sample_count) { 10 }
+    let(:sample_event) { LogStash::Event.new('message' => 'This is a message') }
+    let(:sample_events) do
+      sample_count.times.map do
+        [sample_event, LogStash::Json.dump(sample_event)]
+      end
+    end
+
+    before do
+      subject.register
+    end
+
+    after do
+      subject.close
+    end
+
+    context 'with batching disabled' do
+      let(:config) { super.merge('batch' => false) }
+
+      it 'publishes to SQS' do
+        subject.multi_receive_encoded(sample_events)
+        expect(receive_all_messages(@queue_url).count).to eq(sample_events.count)
+      end
+    end
+
+    context 'with batching enabled' do
+      let(:config) { super.merge('batch' => true) }
+
+      it 'publishes to SQS' do
+        subject.multi_receive_encoded(sample_events)
+        expect(receive_all_messages(@queue_url).count).to eq(sample_events.count)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+require 'aws-sdk'
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/sqs'
+require_relative 'supports/helpers'
+
+LogStash::Logging::Logger::configure_logging('debug') if ENV['DEBUG']

data/spec/supports/helpers.rb

@@ -0,0 +1,20 @@
+require 'aws-sdk'
+
+# Retrieve all available messages from the specified queue.
+#
+# Rather than utilizing `Aws::SQS::QueuePoller` directly in order to poll an
+# SQS queue for messages, this method retrieves and returns all messages that
+# are able to be received from the specified SQS queue.
+def receive_all_messages(queue_url, options = {})
+  options[:idle_timeout] ||= 0
+  options[:max_number_of_messages] ||= 10
+
+  messages = []
+  poller = Aws::SQS::QueuePoller.new(queue_url, options)
+
+  poller.poll do |received_messages|
+    messages.concat(received_messages)
+  end
+
+  messages
+end

data/spec/unit/outputs/sqs_spec.rb

@@ -0,0 +1,263 @@
+# encoding: utf-8
+
+require_relative '../../spec_helper'
+require 'logstash/errors'
+require 'logstash/event'
+require 'logstash/json'
+
+describe LogStash::Outputs::SQS do
+  let(:config) do
+    {
+      'queue' => queue_name,
+      'region' => region,
+    }
+  end
+  let(:queue_name) { 'my-sqs-queue' }
+  let(:queue_url) { "https://sqs.#{region}.amazonaws.com/123456789012/#{queue_name}" }
+  let(:region) { 'us-east-1' }
+
+  let(:sqs) { Aws::SQS::Client.new(:stub_responses => true) }
+  subject { described_class.new(config) }
+
+  describe '#register' do
+    context 'with a batch size that is too large' do
+      let(:config) { super.merge('batch_events' => 100) }
+
+      before do
+        allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
+      end
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with a batch size that is too small' do
+      let(:config) { super.merge('batch_events' => 0) }
+
+      before do
+        allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
+      end
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'without a queue' do
+      let(:config) { Hash.new }
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with a nonexistent queue' do
+      before do
+        expect(Aws::SQS::Client).to receive(:new).and_return(sqs)
+        expect(sqs).to receive(:get_queue_url).with(:queue_name => queue_name) do
+          raise Aws::SQS::Errors::NonExistentQueue.new(nil, 'The specified queue does not exist for this wsdl version.')
+        end
+      end
+
+      it 'raises a configuration error' do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with a valid queue' do
+      before do
+        expect(Aws::SQS::Client).to receive(:new).and_return(sqs)
+        expect(sqs).to receive(:get_queue_url).with(:queue_name => queue_name).and_return(:queue_url => queue_url)
+      end
+
+      it 'does not raise an error' do
+        expect { subject.register }.not_to raise_error
+      end
+    end
+  end
+
+  describe '#multi_receive_encoded' do
+    before do
+      expect(Aws::SQS::Client).to receive(:new).and_return(sqs)
+      expect(sqs).to receive(:get_queue_url).with(:queue_name => queue_name).and_return(:queue_url => queue_url)
+      subject.register
+    end
+
+    after do
+      subject.close
+    end
+
+    let(:sample_count) { 10 }
+    let(:sample_event) { LogStash::Event.new('message' => 'This is a message') }
+    let(:sample_event_encoded) { LogStash::Json.dump(sample_event) }
+    let(:sample_events) do
+      sample_count.times.map do
+        [sample_event, sample_event_encoded]
+      end
+    end
+
+    context 'with batching disabled using the `batch` parameter' do
+      let(:config) { super.merge('batch' => false) }
+
+      it 'should call send_message' do
+        expect(sqs).to receive(:send_message).with(:queue_url => queue_url, :message_body => sample_event_encoded).exactly(sample_count).times
+        subject.multi_receive_encoded(sample_events)
+      end
+
+      it 'should not call send_message_batch' do
+        expect(sqs).not_to receive(:send_message_batch)
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+
+    context 'with batching disabled' do
+      let(:config) do
+        super.merge({
+          'batch' => true,
+          'batch_events' => 1,
+        })
+      end
+
+      it 'should call send_message' do
+        expect(sqs).to receive(:send_message).with(:queue_url => queue_url, :message_body => sample_event_encoded).exactly(sample_count).times
+        subject.multi_receive_encoded(sample_events)
+      end
+
+      it 'should not call send_message_batch' do
+        expect(sqs).not_to receive(:send_message_batch)
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+
+    context 'with batching enabled' do
+      let(:batch_events) { 3 }
+      let(:config) do
+        super.merge({
+          'batch' => true,
+          'batch_events' => batch_events,
+        })
+      end
+
+      let(:sample_batches) do
+        sample_events.each_slice(batch_events).each_with_index.map do |sample_batch, batch_index|
+          sample_batch.each_with_index.map do |encoded_event, index|
+            event, encoded = encoded_event
+            {
+              :id => (batch_index * batch_events + index).to_s,
+              :message_body => encoded,
+            }
+          end
+        end
+      end
+
+      it 'should call send_message_batch' do
+        expect(sqs).to receive(:send_message_batch).at_least(:once)
+        subject.multi_receive_encoded(sample_events)
+      end
+
+      it 'should batch events' do
+        sample_batches.each do |batch_entries|
+          expect(sqs).to receive(:send_message_batch).with(:queue_url => queue_url, :entries => batch_entries)
+        end
+
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+
+    context 'with empty payload' do
+      let(:sample_count) { 0 }
+
+      it 'does not raise an error' do
+        expect { subject.multi_receive_encoded(sample_events) }.not_to raise_error
+      end
+
+      it 'should not send any messages' do
+        expect(sqs).not_to receive(:send_message)
+        expect(sqs).not_to receive(:send_message_batch)
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+
+    context 'with event exceeding maximum size' do
+      let(:config) { super.merge('message_max_size' => message_max_size) }
+      let(:message_max_size) { 1024 }
+
+      let(:sample_count) { 1 }
+      let(:sample_event) { LogStash::Event.new('message' => '.' * message_max_size) }
+
+      it 'should drop event' do
+        expect(sqs).not_to receive(:send_message)
+        expect(sqs).not_to receive(:send_message_batch)
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+
+
+
+    context 'with large batch' do
+      let(:batch_events) { 4 }
+      let(:config) do
+        super.merge({
+          'batch_events' => batch_events,
+          'message_max_size' => message_max_size,
+        })
+      end
+      let(:message_max_size) { 1024 }
+
+      let(:sample_events) do
+        # This is the overhead associating with transmitting each message. The
+        # overhead is caused by metadata (such as the `message` field name and
+        # the `@timestamp` field) as well as additional characters as a result
+        # of encoding the event.
+        overhead = 69
+
+        events = [
+          LogStash::Event.new('message' => 'a' * (0.6 * message_max_size - overhead)),
+          LogStash::Event.new('message' => 'b' * (0.5 * message_max_size - overhead)),
+          LogStash::Event.new('message' => 'c' * (0.5 * message_max_size - overhead)),
+          LogStash::Event.new('message' => 'd' * (0.4 * message_max_size - overhead)),
+        ]
+
+        events.map do |event|
+          [event, LogStash::Json.dump(event)]
+        end
+      end
+
+      let(:sample_batches) do
+        [
+          [
+            {
+              :id => '0',
+              :message_body => sample_events[0][1],
+            },
+          ],
+          [
+            {
+              :id => '1',
+              :message_body => sample_events[1][1],
+            },
+            {
+              :id => '2',
+              :message_body => sample_events[2][1],
+            },
+          ],
+          [
+            {
+              :id => '3',
+              :message_body => sample_events[3][1],
+            },
+          ],
+        ]
+      end
+
+      it 'should split events into smaller batches' do
+        sample_batches.each do |entries|
+          expect(sqs).to receive(:send_message_batch).with(:queue_url => queue_url, :entries => entries)
+        end
+
+        subject.multi_receive_encoded(sample_events)
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-sqs
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2017-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,23 +50,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: aws-sdk
+  name: logstash-codec-json
   prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: stud
-  prerelease: false
-  type: :runtime
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -99,8 +85,12 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/outputs/sqs.rb
+- lib/logstash/outputs/sqs/patch.rb
 - logstash-output-sqs.gemspec
-- spec/outputs/sqs_spec.rb
+- spec/integration/outputs/sqs_spec.rb
+- spec/spec_helper.rb
+- spec/supports/helpers.rb
+- spec/unit/outputs/sqs_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -123,9 +113,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Push events to an Amazon Web Services Simple Queue Service (SQS) queue.
 test_files:
-- spec/outputs/sqs_spec.rb
+- spec/integration/outputs/sqs_spec.rb
+- spec/spec_helper.rb
+- spec/supports/helpers.rb
+- spec/unit/outputs/sqs_spec.rb

data/spec/outputs/sqs_spec.rb

@@ -1,5 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require 'logstash/outputs/sqs'
-
-