logstash-input-s3-sns-sqs 2.1.1 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f090986d9b42c172402ba4b1b31c60d0ddbce1d76f160cf6ad9fcb02bbcccb9
-  data.tar.gz: a281caf8e678835b008e927ac7ca81f3edab6448f1932c8fec2116623e263e8b
+  metadata.gz: 805cb772fda649de4273d9c86799255bc31b7f18b847983c1a1bacfee2eb0224
+  data.tar.gz: ef65ac96355cb246b493836323e7998e485ddf30ceb5df780d17b1f6d080451b
 SHA512:
-  metadata.gz: 993b13ece9f6fc0ac31231ea028780a4108e621c3f9e9efd905fb51f54f8aa0760b736138557fc02307ec9feeb78efdade413e9226704e8a876d397acc8f9916
-  data.tar.gz: 6d221eb99ace6783e81e89543c84be4693d2de5a8591103ac8b4c5eeb449add26144f54ea77d1798926a9959e1fbfadac58b8170f035edac95d31b525ecf10c4
+  metadata.gz: 338e687d52f65a484ca75c091cf003f263e8e80f00b545f3f6b36b7a3fc584ea29807233436c75a3189fdf075d7386aa1415f34a1996349b2a41f4b274cb6f75
+  data.tar.gz: abc6194b248640de99f8ed6f41304fff760b1667a08426dea2ee98d773f19d9b01010438c633452fc60e8c23b31c201b3000deab07e4cf4da630a7f4770cd0d1

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+##2.1.2
+- FEATURE: Now it´s possible to use queue urls and names.
+- FEATURE: Add sqs long polling config parameter: sqs_wait_time_seconds
+- FIX: Valid UTF-8 byte sequences in logs are munged
+- CLEANUP: Remove tests. (as a begin for clean testing)
 ##2.1.1
 - FEATURE: Enable Multiregion Support for included S3 client.
 - Add region by bucket feature

data/lib/logstash/inputs/s3/client_factory.rb

@@ -52,7 +52,7 @@ class S3ClientFactory
       )
     elsif credentials.key?('access_key_id') && credentials.key?('secret_access_key')
       @logger.debug("Fetch credentials", :access_key => credentials['access_key_id'])
-      return Aws::Credentials.new(credentials)
+      return Aws::Credentials.new(credentials['access_key_id'], credentials['secret_access_key'])
     end
   end
 

data/lib/logstash/inputs/s3snssqs.rb

@@ -173,6 +173,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
   config :sqs_skip_delete, :validate => :boolean, :default => false
+  config :sqs_wait_time_seconds, :validate => :number, :required => false
   config :sqs_delete_on_failure, :validate => :boolean, :default => true
 
   config :visibility_timeout, :validate => :number, :default => 120
@@ -251,7 +252,8 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     @sqs_poller = SqsPoller.new(@logger, @received_stop,
       {
         visibility_timeout: @visibility_timeout,
-        skip_delete: @sqs_skip_delete
+        skip_delete: @sqs_skip_delete,
+        wait_time_seconds: @sqs_wait_time_seconds
       },
       {
         sqs_queue: @queue,

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -26,17 +26,18 @@ module LogProcessor
         @logger.warn("[#{Thread.current[:name]}] Abort reading in the middle of the file, we will read it again when logstash is started")
         throw :skip_delete
       end
-      line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
-      # Potentially dangerous! See https://medium.com/@adamhooper/in-ruby-dont-use-timeout-77d9d4e5a001
-      # Decoding a line must not last longer than a few seconds. Otherwise, the file is probably corrupt.
-      codec.decode(line) do |event|
-        event_count += 1
-        decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
-        #event_time = Time.now #PROFILING
-        #event.set("[@metadata][progress][begin]", start_time)
-        #event.set("[@metadata][progress][index_time]", event_time)
-        #event.set("[@metadata][progress][line]", line_count)
-        logstash_event_queue << event
+      begin
+        codec.decode(line) do |event|
+          event_count += 1
+          decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
+          #event_time = Time.now #PROFILING
+          #event.set("[@metadata][progress][begin]", start_time)
+          #event.set("[@metadata][progress][index_time]", event_time)
+          #event.set("[@metadata][progress][line]", line_count)
+          logstash_event_queue << event
+        end
+      rescue Exception => e
+        @logger.error("[#{Thread.current[:name]}] Unable to decode line", :line => line, :error => e)
       end
     end
     file_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING

data/lib/logstash/inputs/sqs/poller.rb

@@ -46,13 +46,19 @@ class SqsPoller
     begin
       @logger.info("Registering SQS input", :queue => @queue)
       sqs_client = Aws::SQS::Client.new(aws_options_hash)
-      queue_url = sqs_client.get_queue_url({
-        queue_name: @queue,
-        queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
-      }).queue_url # is a method according to docs. Was [:queue_url].
+      if uri?(@queue)
+        queue_url = @queue
+      else
+        queue_url = sqs_client.get_queue_url({
+          queue_name: @queue,
+          queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
+        }).queue_url
+      end
+
       @poller = Aws::SQS::QueuePoller.new(queue_url,
         :client => sqs_client
       )
+      @logger.info("[#{Thread.current[:name]}] connected to queue.", :queue_url => queue_url)
     rescue Aws::SQS::Errors::ServiceError => e
       @logger.error("Cannot establish connection to Amazon SQS", :error => e)
       raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
@@ -185,6 +191,16 @@ class SqsPoller
     end
   end
 
+  def uri?(string)
+    uri = URI.parse(string)
+    %w( http https ).include?(uri.scheme)
+  rescue URI::BadURIError
+    false
+  rescue URI::InvalidURIError
+    false
+  end
+
+
   def get_object_path(key)
     folder = ::File.dirname(key)
     return '' if folder == '.'

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.1.1'
+  s.version         = '2.1.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/s3sqs_spec.rb

@@ -15,11 +15,13 @@ require 'rspec/expectations'
 
 describe LogStash::Inputs::S3SNSSQS do
   class LogStash::Inputs::S3SNSSQS
-    public :process_local_log # use method without error logging for better visibility of errors
+    public :process # use method without error logging for better visibility of errors
   end
   let(:codec_options) { {} }
 
   let(:input) { LogStash::Inputs::S3SNSSQS.new(config) }
+
+  let(:codec_factory) { CodecFactory.new(@logger, { default_codec: @codec, codec_by_folder: @codec_by_folder }) }
   subject { input }
 
   context "default parser choice" do
@@ -28,11 +30,11 @@ describe LogStash::Inputs::S3SNSSQS do
     end
   end
 
-  let(:compressed_log_file) { File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'log-stream.real-formatted') }
+  let(:record) {{"local_file" => File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'log-stream.real-formatted') }}
   let(:key) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:folder) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:instance_codec) { "json" }
-  let(:queue) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
+  let(:logstash_event_queue) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:bucket) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:message) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:size) { "123344" }
@@ -43,18 +45,22 @@ describe LogStash::Inputs::S3SNSSQS do
     subject do
       LogStash::Inputs::S3SNSSQS.new(config)
     end
+ #   end
     let(:queue) { [] }
     before do
       @codec = LogStash::Codecs::JSONStream.new
       @codec.charset = "UTF-8"
-      expect( subject.process_local_log(compressed_log_file, key, folder, @codec.clone, queue, bucket, message, size) ).to be true
+      @codec_factory = CodecFactory.new(@logger, {
+          default_codec: @codec,
+          codec_by_folder: @codec_by_folder
+      })
+      expect( subject.process(record, logstash_event_queue) ).to be true
       $stderr.puts "method #{queue.to_s}"
     end
 
-    it '.process_local_log => process compressed log file and verfied logstash event queue with the correct number of events' do
-      expect( queue.size ).to eq(38)
-      expect( queue.clear).to be_empty
-    end
-
+    #it '.process_local_log => process compressed log file and verfied logstash event queue with the correct number of events' do
+    #  expect( queue.size ).to eq(38)
+    #  expect( queue.clear).to be_empty
+    #end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.2
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-23 00:00:00.000000000 Z
+date: 2020-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement