logstash-input-s3-sns-sqs 2.0.9 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2de335bdaa2405c6548eab14d6cf2d7e2432fa854b07da4b4a907878a0d3c2b6
-  data.tar.gz: ee5c2054793e6fee1a8ba150ced23228d30615cc048ee52031a77d849fd0eeb8
+  metadata.gz: c42ccc1885923bc79a1f546a7be3b4ef2b1840c301905f0219e1cddf89266b9c
+  data.tar.gz: 755b037e0a94be66863bf05f6ee6ea20816663de671de879c1642c7ba4a4cec0
 SHA512:
-  metadata.gz: cb42c6fe5d559594d96fe2191b79ce04135d9d6991495f2effc3364ccbc3b0b65f5725d4729625eddbfa14c119d8cc6bed5071972b5eb7a183119d5132efa376
-  data.tar.gz: 2509fc5c56bd8dba8d2049809f49a5cbf91b69851b85178da8918c1e38133650ba0eb86901bd86ccb0913d85e089642b51cb45548195b7f30ffded113932123f
+  metadata.gz: 982cd8b60721b89f56e9e013126ce39fc95c5d0747ab73d98224af2626182f789a6dcde8317e3539612dfac7824d75ab149bda75e0cf75435a06bdd2291e7a0a
+  data.tar.gz: fbd220e56e392510d2bbe44b413afe458e7f0a8eb82c32cfa5024b8fc56629c61315e83ed31bb0ae581afe212a92f0cd4ccc6fd16bb32a27a996b09efd64bdd6

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+##2.1.3
+-SECURITY: Raise MIXIN to a newer Version
+##2.1.2
+- FEATURE: Now it´s possible to use queue urls and names.
+- FEATURE: Add sqs long polling config parameter: sqs_wait_time_seconds
+- FIX: Valid UTF-8 byte sequences in logs are munged
+- CLEANUP: Remove tests. (as a begin for clean testing)
+##2.1.1
+- FEATURE: Enable Multiregion Support for included S3 client.
+- Add region by bucket feature
+##2.1.0
+- FEATURE: Add S3 metadata -> config :include_object_properties
+- FEATURE: Watch for threads in exception state and restart...
 ##2.0.9
 -gzip dectection should return false for files smaller than gzip_signiture_bytes
 ##2.0.8

data/lib/logstash/inputs/s3/client_factory.rb

@@ -10,6 +10,7 @@ class S3ClientFactory
     @aws_options_hash.merge!(@s3_default_options) unless @s3_default_options.empty?
     @sts_client = Aws::STS::Client.new(region: options[:aws_region])
     @credentials_by_bucket = options[:s3_credentials_by_bucket]
+    @region_by_bucket = options[:s3_region_by_bucket]
     @logger.debug("Credentials by Bucket", :credentials => @credentials_by_bucket)
     @default_session_name = options[:s3_role_session_name]
     @clients_by_bucket = {}
@@ -24,6 +25,9 @@ class S3ClientFactory
         unless @credentials_by_bucket[bucket_name].nil?
           options.merge!(credentials: get_s3_auth(@credentials_by_bucket[bucket_name]))
         end
+        unless @region_by_bucket[bucket_name].nil?
+          options.merge!(region: @region_by_bucket[bucket_name])
+        end
         @clients_by_bucket[bucket_symbol] = Aws::S3::Client.new(options)
         @logger.debug("Created a new S3 Client", :bucket_name => bucket_name, :client => @clients_by_bucket[bucket_symbol], :used_options => options)
       end
@@ -48,7 +52,7 @@ class S3ClientFactory
       )
     elsif credentials.key?('access_key_id') && credentials.key?('secret_access_key')
       @logger.debug("Fetch credentials", :access_key => credentials['access_key_id'])
-      return Aws::Credentials.new(credentials)
+      return Aws::Credentials.new(credentials['access_key_id'], credentials['secret_access_key'])
     end
   end
 

data/lib/logstash/inputs/s3/downloader.rb

@@ -9,6 +9,7 @@ class S3Downloader
     @stopped = stop_semaphore
     @factory = options[:s3_client_factory]
     @delete_on_success = options[:delete_on_success]
+    @include_object_properties = options[:include_object_properties]
   end
 
   def copy_s3object_to_disk(record)
@@ -21,6 +22,7 @@ class S3Downloader
           key: record[:key],
           response_target: record[:local_file]
         )
+        record[:s3_data] = response.to_h.keep_if { |key| @include_object_properties.include?(key) }
       end
     rescue Aws::S3::Errors::ServiceError => e
       @logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -26,17 +26,18 @@ module LogProcessor
         @logger.warn("[#{Thread.current[:name]}] Abort reading in the middle of the file, we will read it again when logstash is started")
         throw :skip_delete
       end
-      line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
-      # Potentially dangerous! See https://medium.com/@adamhooper/in-ruby-dont-use-timeout-77d9d4e5a001
-      # Decoding a line must not last longer than a few seconds. Otherwise, the file is probably corrupt.
-      codec.decode(line) do |event|
-        event_count += 1
-        decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
-        #event_time = Time.now #PROFILING
-        #event.set("[@metadata][progress][begin]", start_time)
-        #event.set("[@metadata][progress][index_time]", event_time)
-        #event.set("[@metadata][progress][line]", line_count)
-        logstash_event_queue << event
+      begin
+        codec.decode(line) do |event|
+          event_count += 1
+          decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
+          #event_time = Time.now #PROFILING
+          #event.set("[@metadata][progress][begin]", start_time)
+          #event.set("[@metadata][progress][index_time]", event_time)
+          #event.set("[@metadata][progress][line]", line_count)
+          logstash_event_queue << event
+        end
+      rescue Exception => e
+        @logger.error("[#{Thread.current[:name]}] Unable to decode line", :line => line, :error => e)
       end
     end
     file_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
@@ -45,7 +46,7 @@ module LogProcessor
     # ensure any stateful codecs (such as multi-line ) are flushed to the queue
     codec.flush do |event|
       event_count += 1
-      decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
+      decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
       @logger.debug("[#{Thread.current[:name]}] Flushing an incomplete event", :event => event.to_s)
       logstash_event_queue << event
     end
@@ -55,7 +56,7 @@ module LogProcessor
 
   private
 
-  def decorate_event(event, metadata, type, key, bucket, folder)
+  def decorate_event(event, metadata, type, key, bucket, s3_data)
     if event_is_metadata?(event)
       @logger.debug('Updating the current cloudfront metadata', :event => event)
       update_metadata(metadata, event)
@@ -67,9 +68,11 @@ module LogProcessor
       event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
       event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
 
+      event.set("[@metadata][s3]", s3_data)
       event.set("[@metadata][s3][object_key]", key)
       event.set("[@metadata][s3][bucket_name]", bucket)
       event.set("[@metadata][s3][object_folder]", get_object_folder(key))
+
     end
   end
 

data/lib/logstash/inputs/s3snssqs.rb

@@ -161,6 +161,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Session name to use when assuming an IAM role
   config :s3_role_session_name, :validate => :string, :default => "logstash"
   config :delete_on_success, :validate => :boolean, :default => false
+  # Whether or not to include the S3 object's properties (last_modified, content_type, metadata)
+  # into each Event at [@metadata][s3]. Regardless of this setting, [@metdata][s3][key] will always
+  # be present.
+  config :include_object_properties, :validate => :array, :default => [:last_modified, :content_type, :metadata]
 
   ### sqs
   # Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
@@ -169,9 +173,11 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
   config :sqs_skip_delete, :validate => :boolean, :default => false
+  config :sqs_wait_time_seconds, :validate => :number, :required => false
+  config :sqs_delete_on_failure, :validate => :boolean, :default => true
+
   config :visibility_timeout, :validate => :number, :default => 120
   config :max_processing_time, :validate => :number, :default => 8000
-
   ### system
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
   # To run in multiple threads use this
@@ -188,6 +194,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
     @id ||= "Unknown" #Use INPUT{ id => name} for thread identifier
     @credentials_by_bucket = hash_key_is_regex({})
+    @region_by_bucket = hash_key_is_regex({})
     # create the bucket=>folder=>codec lookup from config options
     @codec_by_folder = hash_key_is_regex({})
     @type_by_folder = hash_key_is_regex({})
@@ -222,6 +229,9 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       if options.key?('credentials')
         @credentials_by_bucket[bucket] = options['credentials']
       end
+      if options.key?('region')
+        @region_by_bucket[bucket] = options['region']
+      end
       if options.key?('folders')
         # make these hashes do key lookups using regex matching
         folders = hash_key_is_regex({})
@@ -242,24 +252,28 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     @sqs_poller = SqsPoller.new(@logger, @received_stop,
       {
         visibility_timeout: @visibility_timeout,
-        skip_delete: @sqs_skip_delete
+        skip_delete: @sqs_skip_delete,
+        wait_time_seconds: @sqs_wait_time_seconds
       },
       {
         sqs_queue: @queue,
         queue_owner_aws_account_id: @queue_owner_aws_account_id,
         from_sns: @from_sns,
-        max_processing_time: @max_processing_time
+        max_processing_time: @max_processing_time,
+        sqs_delete_on_failure: @sqs_delete_on_failure
       },
       aws_options_hash)
     @s3_client_factory = S3ClientFactory.new(@logger, {
       aws_region: @region,
       s3_default_options: @s3_default_options,
       s3_credentials_by_bucket: @credentials_by_bucket,
+      s3_region_by_bucket: @region_by_bucket,
       s3_role_session_name: @s3_role_session_name
     }, aws_options_hash)
     @s3_downloader = S3Downloader.new(@logger, @received_stop, {
       s3_client_factory: @s3_client_factory,
-      delete_on_success: @delete_on_success
+      delete_on_success: @delete_on_success,
+      include_object_properties: @include_object_properties
     })
     @codec_factory = CodecFactory.new(@logger, {
       default_codec: @codec,
@@ -273,18 +287,21 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   # startup
   def run(logstash_event_queue)
-    #LogStash::ShutdownWatcher.abort_threshold(30)
-    # start them
-    @queue_mutex = Mutex.new
-    #@consumer_threads= 1
-    @worker_threads = @consumer_threads.times.map do |thread_id|
-      t = run_worker_thread(logstash_event_queue, thread_id)
-      #make thead start async to prevent polling the same message from sqs
-      sleep 0.5
-      t
+    @control_threads = @consumer_threads.times.map do |thread_id|
+      Thread.new do
+        restart_count = 0
+        while not stop?
+          #make thead start async to prevent polling the same message from sqs
+          sleep 0.5
+          worker_thread = run_worker_thread(logstash_event_queue, thread_id)
+          worker_thread.join
+          restart_count += 1
+          thread_id = "#{thread_id}_#{restart_count}"
+          @logger.info("[control_thread] restarting a thread #{thread_id}... ", :thread => worker_thread.inspect)
+        end
+      end
     end
-    # and wait (possibly infinitely) for them to shut down
-    @worker_threads.each { |t| t.join }
+    @control_threads.each { |t| t.join }
   end
 
   # shutdown
@@ -311,7 +328,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # --- END plugin interface ------------------------------------------#
 
   private
-
   def run_worker_thread(queue, thread_id)
     Thread.new do
       LogStash::Util.set_thread_name("Worker #{@id}/#{thread_id}")

data/lib/logstash/inputs/sqs/poller.rb

@@ -41,17 +41,24 @@ class SqsPoller
     @queue = client_options[:sqs_queue]
     @from_sns = client_options[:from_sns]
     @max_processing_time = client_options[:max_processing_time]
+    @sqs_delete_on_failure = client_options[:sqs_delete_on_failure]
     @options = DEFAULT_OPTIONS.merge(poller_options)
     begin
       @logger.info("Registering SQS input", :queue => @queue)
       sqs_client = Aws::SQS::Client.new(aws_options_hash)
-      queue_url = sqs_client.get_queue_url({
-        queue_name: @queue,
-        queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
-      }).queue_url # is a method according to docs. Was [:queue_url].
+      if uri?(@queue)
+        queue_url = @queue
+      else
+        queue_url = sqs_client.get_queue_url({
+          queue_name: @queue,
+          queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
+        }).queue_url
+      end
+
       @poller = Aws::SQS::QueuePoller.new(queue_url,
         :client => sqs_client
       )
+      @logger.info("[#{Thread.current[:name]}] connected to queue.", :queue_url => queue_url)
     rescue Aws::SQS::Errors::ServiceError => e
       @logger.error("Cannot establish connection to Amazon SQS", :error => e)
       raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
@@ -86,6 +93,7 @@ class SqsPoller
         poller_thread = Thread.current
         extender = Thread.new do
           while new_visibility < @max_processing_time do
+
             sleep message_backoff
             begin
               @poller.change_message_visibility_timeout(message, new_visibility)
@@ -98,8 +106,8 @@ class SqsPoller
             end
           end
           @logger.error("[#{Thread.current[:name]}] Maximum visibility reached! We will delete this message from queue!")
-          @poller.delete_message(message)
-          poller_thread.raise "[#{poller_thread[:name]}] Maximum visibility reached...!".freeze
+          @poller.delete_message(message) if @sqs_delete_on_failure
+          poller_thread.kill
         end
         extender[:name] = "#{Thread.current[:name]}/extender" #PROFILING
         failed = false
@@ -183,6 +191,16 @@ class SqsPoller
     end
   end
 
+  def uri?(string)
+    uri = URI.parse(string)
+    %w( http https ).include?(uri.scheme)
+  rescue URI::BadURIError
+    false
+  rescue URI::InvalidURIError
+    false
+  end
+
+
   def get_object_path(key)
     folder = ::File.dirname(key)
     return '' if folder == '.'

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.0.9'
+  s.version         = '2.1.3'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
 
   s.add_runtime_dependency 'logstash-codec-json', '~> 3.0'
-  s.add_runtime_dependency 'logstash-mixin-aws', '~> 4.3'
+  s.add_runtime_dependency 'logstash-mixin-aws', '>= 4.3'
   s.add_development_dependency 'logstash-codec-json_stream', '~> 1.0'
   s.add_development_dependency 'logstash-devutils', '~> 1.3'
 end

data/spec/inputs/s3sqs_spec.rb

@@ -15,11 +15,13 @@ require 'rspec/expectations'
 
 describe LogStash::Inputs::S3SNSSQS do
   class LogStash::Inputs::S3SNSSQS
-    public :process_local_log # use method without error logging for better visibility of errors
+    public :process # use method without error logging for better visibility of errors
   end
   let(:codec_options) { {} }
 
   let(:input) { LogStash::Inputs::S3SNSSQS.new(config) }
+
+  let(:codec_factory) { CodecFactory.new(@logger, { default_codec: @codec, codec_by_folder: @codec_by_folder }) }
   subject { input }
 
   context "default parser choice" do
@@ -28,11 +30,11 @@ describe LogStash::Inputs::S3SNSSQS do
     end
   end
 
-  let(:compressed_log_file) { File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'log-stream.real-formatted') }
+  let(:record) {{"local_file" => File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'log-stream.real-formatted') }}
   let(:key) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:folder) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:instance_codec) { "json" }
-  let(:queue) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
+  let(:logstash_event_queue) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:bucket) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:message) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:size) { "123344" }
@@ -43,18 +45,22 @@ describe LogStash::Inputs::S3SNSSQS do
     subject do
       LogStash::Inputs::S3SNSSQS.new(config)
     end
+ #   end
     let(:queue) { [] }
     before do
       @codec = LogStash::Codecs::JSONStream.new
       @codec.charset = "UTF-8"
-      expect( subject.process_local_log(compressed_log_file, key, folder, @codec.clone, queue, bucket, message, size) ).to be true
+      @codec_factory = CodecFactory.new(@logger, {
+          default_codec: @codec,
+          codec_by_folder: @codec_by_folder
+      })
+      expect( subject.process(record, logstash_event_queue) ).to be true
       $stderr.puts "method #{queue.to_s}"
     end
 
-    it '.process_local_log => process compressed log file and verfied logstash event queue with the correct number of events' do
-      expect( queue.size ).to eq(38)
-      expect( queue.clear).to be_empty
-    end
-
+    #it '.process_local_log => process compressed log file and verfied logstash event queue with the correct number of events' do
+    #  expect( queue.size ).to eq(38)
+    #  expect( queue.clear).to be_empty
+    #end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.0.9
+  version: 2.1.3
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-21 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -47,7 +47,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.3'
   name: logstash-mixin-aws
@@ -55,7 +55,7 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.3'
 - !ruby/object:Gem::Dependency