logstash-input-sqs 3.1.1 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 015c2627d3dc9ef31ed1f584d0a5c93f8be4ad6e109f459685d1f7fde9f7232d
-  data.tar.gz: 72e1686588f790fd3465d18d2c09a0912f6fa7bbe1f92601e4967192a3c4083f
+  metadata.gz: b3db0942f9be76324ff9694a6e419e810775b0ce1eacf1ba29e933250dcfd87f
+  data.tar.gz: 326cb0268d8ab36b790b8fe72a757dfd1da4d9fdb593b96c61de775279b7e8c6
 SHA512:
-  metadata.gz: 79556cf77d93fea45b38a95f690c18f1ed46a3fa975bcad97fbb564e09c31fedb619ea013af5d497f037481881fc2865317fec242f778f0787d7f79e372f0aaa
-  data.tar.gz: b147aab99d3d73e2119f8c56646f641470e95b3bb66d0571343dc0bbe99105df3ad71b293e032d6b39152b2bd6421ab39bee8ed4d63cca0aabb2cdfd3b89a884
+  metadata.gz: 6a789dfbaa97c0642fa6d692ed3c5dc5430e25156635e9daa1817b5e3e4e1731add22148ab7ff6290d6bb0dfedf31d441f331bd09d4f24ec71bec52fd6e56494
+  data.tar.gz: 4f6ac286a62edd84f54a15d301f2096f701f461ad2a3ab2dfdac8d9e359c07345f7714fe103a168e42f1af6fd11b31b55c84a7b600bd6332945c8ed571abaf75

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.1.2
+  - Added support for multiple events inside same message from SQS [#48](https://github.com/logstash-plugins/logstash-input-sqs/pull/48/files) 
+
 ## 3.1.1
   - Docs: Set the default_codec doc attribute.
 

data/Gemfile

@@ -9,3 +9,7 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+
+if RUBY_VERSION == "1.9.3"
+  gem 'rake', '12.2.1'
+end

data/docs/index.asciidoc

@@ -36,7 +36,7 @@ To use this plugin, you *must*:
 
  * Have an AWS account
  * Setup an SQS queue
- * Create an identify that has access to consume messages from the queue.
+ * Create an identity that has access to consume messages from the queue.
 
 The "consumer" identity must have the following permissions on the queue:
 
@@ -251,4 +251,4 @@ The AWS Session token for temporary credential
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:default_codec!:

data/lib/logstash/inputs/sqs.rb

@@ -121,25 +121,19 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
     }
   end
 
-  def decode_event(message)
-    @codec.decode(message.body) do |event|
-      return event
-    end
-  end
-  
   def add_sqs_data(event, message)
     event.set(@id_field, message.message_id) if @id_field
     event.set(@md5_field, message.md5_of_body) if @md5_field
     event.set(@sent_timestamp_field, convert_epoch_to_timestamp(message.attributes[SENT_TIMESTAMP])) if @sent_timestamp_field
-
-    return event
+    event
   end
 
-  def handle_message(message)
-    event = decode_event(message)
-    add_sqs_data(event, message)
-    decorate(event)
-    return event
+  def handle_message(message, output_queue)
+    @codec.decode(message.body) do |event|
+      add_sqs_data(event, message)
+      decorate(event)
+      output_queue << event
+    end
   end
 
   def run(output_queue)
@@ -148,11 +142,7 @@ class LogStash::Inputs::SQS < LogStash::Inputs::Threadable
     run_with_backoff do
       poller.poll(polling_options) do |messages, stats|
         break if stop?
-
-        messages.each do |message|
-          output_queue << handle_message(message)
-        end
-
+        messages.each {|message| handle_message(message, output_queue) }
         @logger.debug("SQS Stats:", :request_count => stats.request_count,
                       :received_message_count => stats.received_message_count,
                       :last_message_received_at => stats.last_message_received_at) if @logger.debug?

data/logstash-input-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-sqs'
-  s.version         = '3.1.1'
+  s.version         = '3.1.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Pulls events from an Amazon Web Services Simple Queue Service queue"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,5 +25,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-aws', '>= 4.3.0'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency "logstash-codec-json_lines"
 end
 

data/spec/inputs/sqs_spec.rb

@@ -138,7 +138,8 @@ describe LogStash::Inputs::SQS do
       subject { LogStash::Inputs::SQS::new(config.merge({ "codec" => "json" })) }
 
       it "uses the specified codec" do
-        expect(subject.decode_event(encoded_message).get("bonjour")).to eq(decoded_message["bonjour"])
+        subject.handle_message(encoded_message, queue)
+        expect(queue.pop.get("bonjour")).to eq(decoded_message["bonjour"])
       end
     end
 
@@ -152,6 +153,21 @@ describe LogStash::Inputs::SQS do
         subject.run(queue)
         expect(queue.pop.get("bonjour")).to eq(decoded_message["bonjour"])
       end
+
+      context 'can create multiple events' do
+        require "logstash/codecs/json_lines"
+        let(:config) { super.merge({ "codec" => "json_lines" }) }
+        let(:first_message) { { "sequence" => "first" } }
+        let(:second_message) { { "sequence" => "second" } }
+        let(:encoded_message)  { double("sqs_message", :body => "#{LogStash::Json::dump(first_message)}\n#{LogStash::Json::dump(second_message)}\n") }
+
+        it 'creates multiple events' do
+          expect(mock_sqs).to receive(:poll).with(anything()).and_yield([encoded_message], double("stats"))
+          subject.run(queue)
+          events = queue.map{ |e|e.get('sequence')}
+          expect(events).to match_array([first_message['sequence'], second_message['sequence']])
+        end
+      end
     end
 
     context "on errors" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-sqs
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.1.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2018-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +72,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json_lines
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -116,7 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Pulls events from an Amazon Web Services Simple Queue Service queue