logstash-input-s3-sns-sqs 2.1.1 → 2.2.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f090986d9b42c172402ba4b1b31c60d0ddbce1d76f160cf6ad9fcb02bbcccb9
-  data.tar.gz: a281caf8e678835b008e927ac7ca81f3edab6448f1932c8fec2116623e263e8b
+  metadata.gz: 9d5ea48e146f2c80413f41fa2be04ab09396ce2fd67f0275032d34641e0c6fd2
+  data.tar.gz: c4301ebe9b7abb940891bd06843de67b66b400cc6470bfa69b4a208320ef6387
 SHA512:
-  metadata.gz: 993b13ece9f6fc0ac31231ea028780a4108e621c3f9e9efd905fb51f54f8aa0760b736138557fc02307ec9feeb78efdade413e9226704e8a876d397acc8f9916
-  data.tar.gz: 6d221eb99ace6783e81e89543c84be4693d2de5a8591103ac8b4c5eeb449add26144f54ea77d1798926a9959e1fbfadac58b8170f035edac95d31b525ecf10c4
+  metadata.gz: 7c478aa632537f4db570327ff25df1102faab0f9480515bbb4991e52540e086cfb881935b29725e34b4bd5de65a06de77ef65d682ee6b7cc909fce72b5359080
+  data.tar.gz: 52cbe75d68326c5c29fdd5e2b13b3724e0c829664726667fb031c22c99318dfe04d950eb06524ee6d4d5cff76f3df13074db813c663b044f7f126907c8106a25

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+##2.2.0
+-AWS-SDK-3 update
+##2.1.3
+-SECURITY: Raise MIXIN to a newer Version
+-fix: runs now on logstash 7.16
+##2.1.2
+- FEATURE: Now it´s possible to use queue urls and names.
+- FEATURE: Add sqs long polling config parameter: sqs_wait_time_seconds
+- FIX: Valid UTF-8 byte sequences in logs are munged
+- CLEANUP: Remove tests. (as a begin for clean testing)
 ##2.1.1
 - FEATURE: Enable Multiregion Support for included S3 client.
 - Add region by bucket feature

data/lib/logstash/inputs/s3/client_factory.rb

@@ -52,7 +52,7 @@ class S3ClientFactory
       )
     elsif credentials.key?('access_key_id') && credentials.key?('secret_access_key')
       @logger.debug("Fetch credentials", :access_key => credentials['access_key_id'])
-      return Aws::Credentials.new(credentials)
+      return Aws::Credentials.new(credentials['access_key_id'], credentials['secret_access_key'])
     end
   end
 

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -26,17 +26,18 @@ module LogProcessor
         @logger.warn("[#{Thread.current[:name]}] Abort reading in the middle of the file, we will read it again when logstash is started")
         throw :skip_delete
       end
-      line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
-      # Potentially dangerous! See https://medium.com/@adamhooper/in-ruby-dont-use-timeout-77d9d4e5a001
-      # Decoding a line must not last longer than a few seconds. Otherwise, the file is probably corrupt.
-      codec.decode(line) do |event|
-        event_count += 1
-        decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
-        #event_time = Time.now #PROFILING
-        #event.set("[@metadata][progress][begin]", start_time)
-        #event.set("[@metadata][progress][index_time]", event_time)
-        #event.set("[@metadata][progress][line]", line_count)
-        logstash_event_queue << event
+      begin
+        codec.decode(line) do |event|
+          event_count += 1
+          decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
+          #event_time = Time.now #PROFILING
+          #event.set("[@metadata][progress][begin]", start_time)
+          #event.set("[@metadata][progress][index_time]", event_time)
+          #event.set("[@metadata][progress][line]", line_count)
+          logstash_event_queue << event
+        end
+      rescue Exception => e
+        @logger.error("[#{Thread.current[:name]}] Unable to decode line", :line => line, :error => e)
       end
     end
     file_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING

data/lib/logstash/inputs/s3snssqs.rb

@@ -5,8 +5,9 @@ require "logstash/timestamp"
 require "logstash/plugin_mixins/aws_config"
 require "logstash/shutdown_watcher"
 require "logstash/errors"
-require 'logstash/inputs/s3sqs/patch'
-require "aws-sdk"
+require "aws-sdk-s3"
+require "aws-sdk-sns"
+require "aws-sdk-sqs"
 
 # "object-oriented interfaces on top of API clients"...
 # => Overhead. FIXME: needed?
@@ -35,8 +36,6 @@ require_relative 's3/downloader'
 require_relative 'codec_factory'
 require_relative 's3snssqs/log_processor'
 
-Aws.eager_autoload!
-
 # Get logs from AWS s3 buckets as issued by an object-created event via sqs.
 #
 # This plugin is based on the logstash-input-sqs plugin but doesn't log the sqs event itself.
@@ -173,6 +172,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
   config :sqs_skip_delete, :validate => :boolean, :default => false
+  config :sqs_wait_time_seconds, :validate => :number, :required => false
   config :sqs_delete_on_failure, :validate => :boolean, :default => true
 
   config :visibility_timeout, :validate => :number, :default => 120
@@ -251,7 +251,8 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     @sqs_poller = SqsPoller.new(@logger, @received_stop,
       {
         visibility_timeout: @visibility_timeout,
-        skip_delete: @sqs_skip_delete
+        skip_delete: @sqs_skip_delete,
+        wait_time_seconds: @sqs_wait_time_seconds
       },
       {
         sqs_queue: @queue,

data/lib/logstash/inputs/sqs/poller.rb

@@ -46,13 +46,19 @@ class SqsPoller
     begin
       @logger.info("Registering SQS input", :queue => @queue)
       sqs_client = Aws::SQS::Client.new(aws_options_hash)
-      queue_url = sqs_client.get_queue_url({
-        queue_name: @queue,
-        queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
-      }).queue_url # is a method according to docs. Was [:queue_url].
+      if uri?(@queue)
+        queue_url = @queue
+      else
+        queue_url = sqs_client.get_queue_url({
+          queue_name: @queue,
+          queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
+        }).queue_url
+      end
+
       @poller = Aws::SQS::QueuePoller.new(queue_url,
         :client => sqs_client
       )
+      @logger.info("[#{Thread.current[:name]}] connected to queue.", :queue_url => queue_url)
     rescue Aws::SQS::Errors::ServiceError => e
       @logger.error("Cannot establish connection to Amazon SQS", :error => e)
       raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
@@ -185,6 +191,16 @@ class SqsPoller
     end
   end
 
+  def uri?(string)
+    uri = URI.parse(string)
+    %w( http https ).include?(uri.scheme)
+  rescue URI::BadURIError
+    false
+  rescue URI::InvalidURIError
+    false
+  end
+
+
   def get_object_path(key)
     folder = ::File.dirname(key)
     return '' if folder == '.'

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.1.1'
+  s.version         = '2.2.0.pre'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,9 +20,21 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
+  s.add_runtime_dependency "concurrent-ruby"
+  s.add_runtime_dependency "logstash-codec-json"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency "aws-sdk-core", "~> 3"
+  s.add_runtime_dependency "aws-sdk-s3"
+  s.add_runtime_dependency "aws-sdk-sqs"
+  s.add_runtime_dependency "aws-sdk-sns"
+  s.add_runtime_dependency "aws-sdk-resourcegroups"
+
+  s.add_development_dependency "logstash-codec-json_lines"
+  s.add_development_dependency "logstash-codec-multiline"
+  s.add_development_dependency "logstash-codec-json"
+  s.add_development_dependency "logstash-codec-line"
+  s.add_development_dependency "logstash-devutils"
+  s.add_development_dependency "logstash-input-generator"
+  s.add_development_dependency "timecop"
 
-  s.add_runtime_dependency 'logstash-codec-json', '~> 3.0'
-  s.add_runtime_dependency 'logstash-mixin-aws', '~> 4.3'
-  s.add_development_dependency 'logstash-codec-json_stream', '~> 1.0'
-  s.add_development_dependency 'logstash-devutils', '~> 1.3'
 end

data/spec/inputs/s3sqs_spec.rb

@@ -15,11 +15,13 @@ require 'rspec/expectations'
 
 describe LogStash::Inputs::S3SNSSQS do
   class LogStash::Inputs::S3SNSSQS
-    public :process_local_log # use method without error logging for better visibility of errors
+    public :process # use method without error logging for better visibility of errors
   end
   let(:codec_options) { {} }
 
   let(:input) { LogStash::Inputs::S3SNSSQS.new(config) }
+
+  let(:codec_factory) { CodecFactory.new(@logger, { default_codec: @codec, codec_by_folder: @codec_by_folder }) }
   subject { input }
 
   context "default parser choice" do
@@ -28,11 +30,11 @@ describe LogStash::Inputs::S3SNSSQS do
     end
   end
 
-  let(:compressed_log_file) { File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'log-stream.real-formatted') }
+  let(:record) {{"local_file" => File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'log-stream.real-formatted') }}
   let(:key) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:folder) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:instance_codec) { "json" }
-  let(:queue) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
+  let(:logstash_event_queue) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:bucket) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:message) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:size) { "123344" }
@@ -43,18 +45,22 @@ describe LogStash::Inputs::S3SNSSQS do
     subject do
       LogStash::Inputs::S3SNSSQS.new(config)
     end
+ #   end
     let(:queue) { [] }
     before do
       @codec = LogStash::Codecs::JSONStream.new
       @codec.charset = "UTF-8"
-      expect( subject.process_local_log(compressed_log_file, key, folder, @codec.clone, queue, bucket, message, size) ).to be true
+      @codec_factory = CodecFactory.new(@logger, {
+          default_codec: @codec,
+          codec_by_folder: @codec_by_folder
+      })
+      expect( subject.process(record, logstash_event_queue) ).to be true
       $stderr.puts "method #{queue.to_s}"
     end
 
-    it '.process_local_log => process compressed log file and verfied logstash event queue with the correct number of events' do
-      expect( queue.size ).to eq(38)
-      expect( queue.clear).to be_empty
-    end
-
+    #it '.process_local_log => process compressed log file and verfied logstash event queue with the correct number of events' do
+    #  expect( queue.size ).to eq(38)
+    #  expect( queue.clear).to be_empty
+    #end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0.pre
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-23 00:00:00.000000000 Z
+date: 2022-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -33,59 +33,213 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   name: logstash-codec-json
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
-  name: logstash-mixin-aws
+        version: '3'
+  name: aws-sdk-core
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: aws-sdk-s3
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: aws-sdk-sqs
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: aws-sdk-sns
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: aws-sdk-resourcegroups
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-codec-json_stream
+        version: '0'
+  name: logstash-codec-json_lines
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-multiline
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-input-generator
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
+  name: timecop
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
   a stand-alone program
@@ -106,7 +260,6 @@ files:
 - lib/logstash/inputs/s3/downloader.rb
 - lib/logstash/inputs/s3snssqs.rb
 - lib/logstash/inputs/s3snssqs/log_processor.rb
-- lib/logstash/inputs/s3sqs/patch.rb
 - lib/logstash/inputs/sqs/poller.rb
 - logstash-input-s3-sns-sqs.gemspec
 - spec/inputs/s3sqs_spec.rb
@@ -127,12 +280,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Get logs from AWS s3 buckets as issued by an object-created event via sns

data/lib/logstash/inputs/s3sqs/patch.rb

@@ -1,22 +0,0 @@
-# This patch was stolen from logstash-plugins/logstash-output-s3#102.
-#
-# This patch is a workaround for a JRuby issue which has been fixed in JRuby
-# 9000, but not in JRuby 1.7. See https://github.com/jruby/jruby/issues/3645
-# and https://github.com/jruby/jruby/issues/3920. This is necessary because the
-# `aws-sdk` is doing tricky name discovery to generate the correct error class.
-#
-# As per https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960,
-# this patch may be short-lived anyway.
-require 'aws-sdk'
-
-begin
-  old_stderr = $stderr
-  $stderr = StringIO.new
-
-  module Aws
-    const_set(:S3, Aws::S3)
-    const_set(:SQS, Aws::SQS)
-  end
-ensure
-  $stderr = old_stderr
-end