logstash-input-sqs 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 08fba64b2923f5a1ae87c399907d524df37b46de
-  data.tar.gz: 793f5d98eb640d2a03d30505951090965731ca84
+  metadata.gz: 2a6fdc994c8c6ebd9175705b27655d59b05077bf
+  data.tar.gz: 27f71b1d9c8c5b3fb3e5f0ff88e89d612dbf68a5
 SHA512:
-  metadata.gz: 8cce064c2c706433cb508415402a9f282cad3602b2bf2d8282fd6837093f3686f26bd16e420fe3dff11174fedd2da4a95f592b37c26eef330404c9a7645c9fe3
-  data.tar.gz: 6e4f0ed8a0815a2fbc3f7870a9fff19018730d93b67dbc975ece14e49c9af41570fb50870be39e3ad703619bb520b7f2f0fa3323040a975d2296cb9085d7a4d8
+  metadata.gz: 144dc569bcb78514c165eb2cc4089e2f90069b778537ad6dc501edd03a537f46fda0b9074ca24b01bf2c348b0687f4fb05c4943a6c8cd3edc87b55ba484a44ff
+  data.tar.gz: 5dab8585276679b08119fd525a02b38a428060cce3d02068a6663facf0c5c065f885c99ded701de7c92503819e30c28c6fae12c031c008c070d0455db703c6c3

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# 1.1.0
+- AWS ruby SDK v2 upgrade
+- Replaces aws-sdk dependencies with mixin-aws
+- Removes unnecessary de-allocation
+- Move the code into smaller methods to allow easier mocking and testing
+- Add the option to configure polling frequency
+- Adding a monkey patch to make sure `LogStash::ShutdownSignal` doesn't get catch by AWS RetryError.

data/lib/logstash/inputs/sqs.rb

@@ -1,9 +1,10 @@
 # encoding: utf-8
+#
 require "logstash/inputs/threadable"
 require "logstash/namespace"
 require "logstash/timestamp"
 require "logstash/plugin_mixins/aws_config"
-require "digest/sha2"
+require "logstash/errors"
 
 # Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
 #
@@ -58,7 +59,15 @@ require "digest/sha2"
 # See http://aws.amazon.com/iam/ for more details on setting up AWS identities.
 #
 class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
-  include LogStash::PluginMixins::AwsConfig
+  include LogStash::PluginMixins::AwsConfig::V2
+
+  MAX_TIME_BEFORE_GIVING_UP = 60
+  MAX_MESSAGES_TO_FETCH = 10 # Between 1-10 in the AWS-SDK doc
+  SENT_TIMESTAMP = "SentTimestamp"
+  SQS_ATTRIBUTES = [SENT_TIMESTAMP]
+  BACKOFF_SLEEP_TIME = 1
+  BACKOFF_FACTOR = 2
+  DEFAULT_POLLING_FREQUENCY = 20
 
   config_name "sqs"
 
@@ -73,104 +82,114 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
   # Name of the event field in which to store the SQS message MD5 checksum
   config :md5_field, :validate => :string
 
-  # Name of the event field in which to store the  SQS message Sent Timestamp
+  # Name of the event field in which to store the SQS message Sent Timestamp
   config :sent_timestamp_field, :validate => :string
 
-  public
-  def aws_service_endpoint(region)
-    return {
-        :sqs_endpoint => "sqs.#{region}.amazonaws.com"
-    }
-  end
+  # Polling frequency, default is 20 seconds
+  config :polling_frequency, :validate => :number, :default => DEFAULT_POLLING_FREQUENCY
+
+  attr_reader :poller
 
-  public
   def register
-    @logger.info("Registering SQS input", :queue => @queue)
     require "aws-sdk"
+    @logger.info("Registering SQS input", :queue => @queue)
 
-    @sqs = AWS::SQS.new(aws_options_hash)
+    monkey_patch_aws_retryable_plugin!
+    setup_queue
+  end
 
-    begin
-      @logger.debug("Connecting to AWS SQS queue", :queue => @queue)
-      @sqs_queue = @sqs.queues.named(@queue)
-      @logger.info("Connected to AWS SQS queue successfully.", :queue => @queue)
-    rescue Exception => e
-      @logger.error("Unable to access SQS queue.", :error => e.to_s, :queue => @queue)
-      throw e
-    end # begin/rescue
-  end # def register
-
-  public
-  def run(output_queue)
-    @logger.debug("Polling SQS queue", :queue => @queue)
+  def setup_queue
+    aws_sqs_client = Aws::SQS::Client.new(aws_options_hash)
+    queue_url = aws_sqs_client.get_queue_url(:queue_name =>  @queue)[:queue_url]
+    @poller = Aws::SQS::QueuePoller.new(queue_url, :client => aws_sqs_client)
+  rescue Aws::SQS::Errors::ServiceError => e
+    @logger.error("Cannot establish connection to Amazon SQS", :error => e)
+    raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
+  end
 
-    receive_opts = {
-        :limit => 10,
-        :visibility_timeout => 30,
-        :attributes => [:sent_at]
+  def polling_options
+    { 
+      :max_number_of_messages => MAX_MESSAGES_TO_FETCH,
+      :attribute_names => SQS_ATTRIBUTES,
+      :wait_time_seconds => @polling_frequency
     }
+  end
 
-    continue_polling = true
-    while running? && continue_polling
-      continue_polling = run_with_backoff(60, 1) do
-        @sqs_queue.receive_message(receive_opts) do |message|
-          if message
-            @codec.decode(message.body) do |event|
-              decorate(event)
-              if @id_field
-                event[@id_field] = message.id
-              end
-              if @md5_field
-                event[@md5_field] = message.md5
-              end
-              if @sent_timestamp_field
-                event[@sent_timestamp_field] = LogStash::Timestamp.new(message.sent_timestamp).utc
-              end
-              @logger.debug? && @logger.debug("Processed SQS message", :message_id => message.id, :message_md5 => message.md5, :sent_timestamp => message.sent_timestamp, :queue => @queue)
-              output_queue << event
-              message.delete
-            end # codec.decode
-          end # valid SQS message
-        end # receive_message
-      end # run_with_backoff
-    end # polling loop
-  end # def run
-
-  def teardown
-    @sqs_queue = nil
-    finished
-  end # def teardown
+  def decode_event(message)
+    @codec.decode(message.body) do |event|
+      return event
+    end
+  end
+  
+  def add_sqs_data(event, message)
+    event[@id_field] = message.message_id if @id_field
+    event[@md5_field] = message.md5_of_body if @md5_field
+    event[@sent_timestamp_field] = convert_epoch_to_timestamp(message.attributes[SENT_TIMESTAMP]) if @sent_timestamp_field
+
+    return event
+  end
+
+  def handle_message(message)
+    event = decode_event(message)
+    add_sqs_data(event, message)
+    decorate(event)
+    return event
+  end
+
+  def run(output_queue)
+    @logger.debug("Polling SQS queue", :polling_options => polling_options)
+
+    run_with_backoff do
+      poller.poll(polling_options) do |messages, stats|
+        messages.each do |message|
+          output_queue << handle_message(message)
+        end
+
+        @logger.debug("SQS Stats:", :request_count => stats.request_count,
+                      :messages_count => stats.message_count,
+                      :last_message_received_at => stats.last_message_received_at) if @logger.debug?
+      end
+    end
+  end
 
   private
   # Runs an AWS request inside a Ruby block with an exponential backoff in case
-  # we exceed the allowed AWS RequestLimit.
+  # we experience a ServiceError.
   #
   # @param [Integer] max_time maximum amount of time to sleep before giving up.
   # @param [Integer] sleep_time the initial amount of time to sleep before retrying.
   # @param [Block] block Ruby code block to execute.
-  def run_with_backoff(max_time, sleep_time, &block)
-    if sleep_time > max_time
-      @logger.error("AWS::EC2::Errors::RequestLimitExceeded ... failed.", :queue => @queue)
-      return false
-    end # retry limit exceeded
+  def run_with_backoff(max_time = MAX_TIME_BEFORE_GIVING_UP, sleep_time = BACKOFF_SLEEP_TIME, &block)
+    next_sleep = sleep_time
 
     begin
       block.call
-    rescue AWS::EC2::Errors::RequestLimitExceeded
-      @logger.info("AWS::EC2::Errors::RequestLimitExceeded ... retrying SQS request", :queue => @queue, :sleep_time => sleep_time)
-      sleep sleep_time
-      run_with_backoff(max_time, sleep_time * 2, &block)
-    rescue AWS::EC2::Errors::InstanceLimitExceeded
-      @logger.warn("AWS::EC2::Errors::InstanceLimitExceeded ... aborting SQS message retreival.")
-      return false
-    rescue AWS::SQS::Errors::InternalError
-      @logger.info("AWS::SQS::Errors::AWS Internal Error ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time)
-      sleep sleep_time
-      run_with_backoff(max_time, sleep_time * 2, &block)
-    rescue Exception => bang
-      @logger.error("Error reading SQS queue.", :error => bang, :queue => @queue)
-      return false
-    end # begin/rescue
-    return true
-  end # def run_with_backoff
+      next_sleep = sleep_time
+    rescue Aws::SQS::Errors::ServiceError => e
+      @logger.warn("Aws::SQS::Errors::ServiceError ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time, :error => e)
+      sleep(next_sleep)
+      next_sleep =  next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR 
+
+      retry
+    rescue LogStash::ShutdownSignal
+      # The pipeline is currently shutting down.
+      # we can safely rescue and return, all unacked sqs messages will be resend
+      # when the pipeline is up again.
+    end
+  end
+
+  def convert_epoch_to_timestamp(time)
+    LogStash::Timestamp.at(time.to_i / 1000)
+  end
+
+  def monkey_patch_aws_retryable_plugin!
+    Aws::Plugins::RetryErrors::ErrorInspector.module_eval do
+      def networking?
+        !@error.is_a?(LogStash::ShutdownSignal) &&
+          (@error.is_a?(Seahorse::Client::NetworkingError) ||
+           Aws::Plugins::RetryErrors::ErrorInspector::NETWORKING_ERRORS.include?(@name) ||
+           @http_status_code == 0)
+      end
+    end
+  end
 end # class LogStash::Inputs::SQS

data/logstash-input-sqs.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |s|
-
   s.name            = 'logstash-input-sqs'
-  s.version         = '1.0.0'
+  s.version         = '1.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Pull events from an Amazon Web Services Simple Queue Service (SQS) queue."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -23,7 +22,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
 
   s.add_runtime_dependency 'logstash-codec-json'
-  s.add_runtime_dependency 'aws-sdk'
+  s.add_runtime_dependency "logstash-mixin-aws", ">= 1.0.0"
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/inputs/sqs_spec.rb

@@ -1 +1,170 @@
-require "logstash/devutils/rspec/spec_helper"
+# encoding: utf-8
+require "logstash/inputs/sqs"
+require "logstash/errors"
+require "logstash/event"
+require "logstash/json"
+require "aws-sdk"
+require "spec_helper"
+
+describe LogStash::Inputs::SQS do
+  let(:queue_name) { "the-infinite-pandora-box" }
+  let(:queue_url) { "https://sqs.test.local/#{queue}" }
+  let(:options) do
+    {
+      "region" => "us-east-1",
+      "access_key_id" => "123",
+      "secret_access_key" => "secret",
+      "queue" => queue_name
+    }
+  end
+
+  let(:input) { LogStash::Inputs::SQS.new(options) }
+
+  let(:decoded_message) { { "bonjour" => "awesome" }  }
+  let(:encoded_message)  { double("sqs_message", :body => LogStash::Json::dump(decoded_message)) }
+
+  subject { input }
+
+  let(:mock_sqs) { Aws::SQS::Client.new({ :stub_responses => true }) }
+
+  context "with invalid credentials" do
+    before do
+      expect(Aws::SQS::Client).to receive(:new).and_return(mock_sqs)
+      expect(mock_sqs).to receive(:get_queue_url).with({ :queue_name => queue_name }) { raise Aws::SQS::Errors::ServiceError.new("bad-something", "bad token") }
+    end
+
+    it "raises a Configuration error if the credentials are bad" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "valid credentials" do
+    let(:queue) { [] }
+
+    it "doesn't raise an error with valid credentials" do
+      expect(Aws::SQS::Client).to receive(:new).and_return(mock_sqs)
+      expect(mock_sqs).to receive(:get_queue_url).with({ :queue_name => queue_name }).and_return({:queue_url => queue_url })
+      expect { subject.register }.not_to raise_error
+    end
+
+    context "enrich event" do
+      let(:event) { LogStash::Event.new }
+
+      let(:message_id) { "123" }
+      let(:md5_of_body) { "dr strange" }
+      let(:sent_timestamp) { LogStash::Timestamp.new }
+      let(:epoch_timestamp) { (sent_timestamp.utc.to_f * 1000).to_i }
+
+      let(:id_field) { "my_id_field" }
+      let(:md5_field) { "my_md5_field" }
+      let(:sent_timestamp_field) { "my_sent_timestamp_field" }
+
+      let(:message) do
+        double("message", :message_id => message_id, :md5_of_body => md5_of_body, :attributes => { LogStash::Inputs::SQS::SENT_TIMESTAMP  => epoch_timestamp } )
+      end 
+
+      subject { input.add_sqs_data(event, message) }
+
+      context "when the option is specified" do
+        let(:options) do
+          {
+            "region" => "us-east-1",
+            "access_key_id" => "123",
+            "secret_access_key" => "secret",
+            "queue" => queue_name,
+            "id_field" => id_field,
+            "md5_field" => md5_field,
+            "sent_timestamp_field" => sent_timestamp_field
+          }
+        end
+
+        it "add the `message_id`" do
+          expect(subject[id_field]).to eq(message_id)
+        end
+
+        it "add the `md5_of_body`" do
+          expect(subject[md5_field]).to eq(md5_of_body)
+        end
+
+        it "add the `sent_timestamp`" do
+          expect(subject[sent_timestamp_field].to_i).to eq(sent_timestamp.to_i)
+        end
+      end
+
+      context "when the option isn't specified" do
+        it "doesnt add the `message_id`" do
+          expect(subject).not_to include(id_field)
+        end
+
+        it "doesnt add the `md5_of_body`" do
+          expect(subject).not_to include(md5_field)
+        end
+
+        it "doesnt add the `sent_timestamp`" do
+          expect(subject).not_to include(sent_timestamp_field)
+        end
+      end
+    end
+
+    context "when decoding body" do
+      subject { LogStash::Inputs::SQS::new(options.merge({ "codec" => "json" })) }
+
+      it "uses the specified codec" do
+        expect(subject.decode_event(encoded_message)["bonjour"]).to eq(decoded_message["bonjour"])
+      end
+    end
+
+    context "receiving messages" do
+      before do 
+        expect(subject).to receive(:poller).and_return(mock_sqs).at_least(:once)
+      end
+
+      it "creates logstash event" do
+        expect(mock_sqs).to receive(:poll).with(anything()).and_yield([encoded_message], double("stats"))
+        subject.run(queue)
+        expect(queue.pop["bonjour"]).to eq(decoded_message["bonjour"])
+      end
+    end
+
+    context "on errors" do
+      let(:payload) { "Hello world" }
+
+      before do
+        expect(subject).to receive(:poller).and_return(mock_sqs).at_least(:once)
+      end
+
+      context "SQS errors" do
+        it "retry to fetch messages" do
+          # change the poller implementation to raise SQS errors.
+          had_error = false
+          
+          # actually using the child of `Object` to do an expectation of `#sleep`
+          expect(subject).to receive(:sleep).with(LogStash::Inputs::SQS::BACKOFF_SLEEP_TIME)
+          expect(mock_sqs).to receive(:poll).with(anything()).at_most(2) do
+            unless had_error
+              had_error = true
+              raise Aws::SQS::Errors::ServiceError.new("testing", "testing exception")
+            end
+
+            queue << payload
+          end
+
+          subject.run(queue)
+
+          expect(queue.size).to eq(1)
+          expect(queue.pop).to eq(payload)
+        end
+      end
+
+      context "other errors" do
+        it "stops executing the code and raise the exception" do
+          expect(mock_sqs).to receive(:poll).with(anything()).at_most(2) do
+            raise RuntimeError
+          end
+
+          expect { subject.run(queue) }.to raise_error(RuntimeError)
+        end
+      end
+    end
+  end
+end

data/spec/integration/sqs_spec.rb

@@ -0,0 +1,111 @@
+# encoding: utf-8
+require "logstash/inputs/sqs"
+require "logstash/event"
+require "logstash/json"
+require "aws-sdk"
+require "spec_helper"
+require_relative "../support/helpers"
+require "thread"
+
+Thread.abort_on_exception = true
+
+describe "LogStash::Inputs::SQS integration", :integration => true do
+  let(:decoded_message) { { "drstrange" => "is-he-really-that-strange" } }
+  let(:encoded_message) { LogStash::Json.dump(decoded_message) }
+  let(:queue) { Queue.new }
+
+  let(:input) { LogStash::Inputs::SQS.new(options) }
+
+  context "with invalid credentials" do
+    let(:options) do
+      {
+        "queue" => "do-not-exist",
+        "access_key_id" => "bad_access",
+        "secret_access_key" => "bad_secret_key",
+        "region" => ENV["AWS_REGION"]
+      }
+    end
+
+    subject { input }
+
+    it "raises a Configuration error if the credentials are bad" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "with valid credentials" do
+    let(:options) do
+      {
+        "queue" => ENV["SQS_QUEUE_NAME"],
+        "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
+        "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
+        "region" => ENV["AWS_REGION"]
+      }
+    end
+
+    before :each do
+      push_sqs_event(encoded_message)
+      input.register
+      @server = Thread.new { input.run(queue) }
+    end
+
+    after do
+      @server.kill
+    end
+
+    subject { queue.pop }
+
+    it "creates logstash events" do
+      expect(subject["drstrange"]).to eq(decoded_message["drstrange"])
+    end
+
+    context "when the optionals fields are not specified" do
+      let(:id_field) { "my_id_field" }
+      let(:md5_field) { "my_md5_field" }
+      let(:sent_timestamp_field) { "my_sent_timestamp_field" }
+
+      it "add the `message_id`" do
+        expect(subject[id_field]).to be_nil
+      end
+
+      it "add the `md5_of_body`" do
+        expect(subject[md5_field]).to be_nil
+      end
+
+      it "add the `sent_timestamp`" do
+        expect(subject[sent_timestamp_field]).to be_nil
+      end
+
+    end
+
+    context "when the optionals fields are specified" do
+      let(:id_field) { "my_id_field" }
+      let(:md5_field) { "my_md5_field" }
+      let(:sent_timestamp_field) { "my_sent_timestamp_field" }
+
+      let(:options) do
+        {
+          "queue" => ENV["SQS_QUEUE_NAME"],
+          "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
+          "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
+          "region" => ENV["AWS_REGION"],
+          "id_field" => id_field,
+          "md5_field" => md5_field,
+          "sent_timestamp_field" => sent_timestamp_field
+        }
+      end
+
+      it "add the `message_id`" do
+        expect(subject[id_field]).not_to be_nil
+      end
+
+      it "add the `md5_of_body`" do
+        expect(subject[md5_field]).not_to be_nil
+      end
+
+      it "add the `sent_timestamp`" do
+        expect(subject[sent_timestamp_field]).not_to be_nil
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

data/spec/support/helpers.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+def push_sqs_event(message)
+  client = Aws::SQS::Client.new
+  queue_url = client.get_queue_url(:queue_name => ENV["SQS_QUEUE_NAME"])
+
+  client.send_message({
+    queue_url: queue_url.queue_url,
+    message_body: message,
+  })
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-sqs
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -45,17 +45,17 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: logstash-mixin-aws
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -89,6 +89,9 @@ files:
 - lib/logstash/inputs/sqs.rb
 - logstash-input-sqs.gemspec
 - spec/inputs/sqs_spec.rb
+- spec/integration/sqs_spec.rb
+- spec/spec_helper.rb
+- spec/support/helpers.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -111,9 +114,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Pull events from an Amazon Web Services Simple Queue Service (SQS) queue.
 test_files:
 - spec/inputs/sqs_spec.rb
+- spec/integration/sqs_spec.rb
+- spec/spec_helper.rb
+- spec/support/helpers.rb