logstash-input-s3-sns-sqs 2.0.7 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 129cf226f1cc14ddcd1b0ffa54dbc3f3472eafe574efeac163d15827e7226e9a
-  data.tar.gz: eba5acd15e497fe5eb4ac6ee64a817a132ae4bb308601f4013e42da3e344e1cc
+  metadata.gz: 805cb772fda649de4273d9c86799255bc31b7f18b847983c1a1bacfee2eb0224
+  data.tar.gz: ef65ac96355cb246b493836323e7998e485ddf30ceb5df780d17b1f6d080451b
 SHA512:
-  metadata.gz: 7293553ce571cd1347cd970852a83a88a1c51d2a29fe0e992df212cf4807a909e0ea8eb46663d2043cf3778a65c39006626b013308735f5e14cf1a1e1fcf69cc
-  data.tar.gz: 4f4068a601e0f930167cd988b404f844a3420e3938db30d8c9b768216fca900f077e8658c2ad8bff54e4ff18b7217d4e0f316aaf6df0312176c3294bd047af10
+  metadata.gz: 338e687d52f65a484ca75c091cf003f263e8e80f00b545f3f6b36b7a3fc584ea29807233436c75a3189fdf075d7386aa1415f34a1996349b2a41f4b274cb6f75
+  data.tar.gz: abc6194b248640de99f8ed6f41304fff760b1667a08426dea2ee98d773f19d9b01010438c633452fc60e8c23b31c201b3000deab07e4cf4da630a7f4770cd0d1

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+##2.1.2
+- FEATURE: Now it´s possible to use queue urls and names.
+- FEATURE: Add sqs long polling config parameter: sqs_wait_time_seconds
+- FIX: Valid UTF-8 byte sequences in logs are munged
+- CLEANUP: Remove tests. (as a begin for clean testing)
+##2.1.1
+- FEATURE: Enable Multiregion Support for included S3 client.
+- Add region by bucket feature
+##2.1.0
+- FEATURE: Add S3 metadata -> config :include_object_properties
+- FEATURE: Watch for threads in exception state and restart...
+##2.0.9
+-gzip dectection should return false for files smaller than gzip_signiture_bytes
+##2.0.8
+-fix nil class error
 ##2.0.7
 -fix gem error
 ##2.0.6

data/lib/logstash/inputs/mime/magic_gzip_validator.rb

@@ -11,7 +11,7 @@ class MagicGzipValidator
     # signature.
     if file.stat.size < minimum_bytes_for_determining_signature
       puts "File too small to calculate signature"
-      throw :skip_delete
+      return false
     end
 
     @file = file

data/lib/logstash/inputs/s3/client_factory.rb

@@ -10,6 +10,7 @@ class S3ClientFactory
     @aws_options_hash.merge!(@s3_default_options) unless @s3_default_options.empty?
     @sts_client = Aws::STS::Client.new(region: options[:aws_region])
     @credentials_by_bucket = options[:s3_credentials_by_bucket]
+    @region_by_bucket = options[:s3_region_by_bucket]
     @logger.debug("Credentials by Bucket", :credentials => @credentials_by_bucket)
     @default_session_name = options[:s3_role_session_name]
     @clients_by_bucket = {}
@@ -24,6 +25,9 @@ class S3ClientFactory
         unless @credentials_by_bucket[bucket_name].nil?
           options.merge!(credentials: get_s3_auth(@credentials_by_bucket[bucket_name]))
         end
+        unless @region_by_bucket[bucket_name].nil?
+          options.merge!(region: @region_by_bucket[bucket_name])
+        end
         @clients_by_bucket[bucket_symbol] = Aws::S3::Client.new(options)
         @logger.debug("Created a new S3 Client", :bucket_name => bucket_name, :client => @clients_by_bucket[bucket_symbol], :used_options => options)
       end
@@ -48,7 +52,7 @@ class S3ClientFactory
       )
     elsif credentials.key?('access_key_id') && credentials.key?('secret_access_key')
       @logger.debug("Fetch credentials", :access_key => credentials['access_key_id'])
-      return Aws::Credentials.new(credentials)
+      return Aws::Credentials.new(credentials['access_key_id'], credentials['secret_access_key'])
     end
   end
 

data/lib/logstash/inputs/s3/downloader.rb

@@ -9,6 +9,7 @@ class S3Downloader
     @stopped = stop_semaphore
     @factory = options[:s3_client_factory]
     @delete_on_success = options[:delete_on_success]
+    @include_object_properties = options[:include_object_properties]
   end
 
   def copy_s3object_to_disk(record)
@@ -21,6 +22,7 @@ class S3Downloader
           key: record[:key],
           response_target: record[:local_file]
         )
+        record[:s3_data] = response.to_h.keep_if { |key| @include_object_properties.include?(key) }
       end
     rescue Aws::S3::Errors::ServiceError => e
       @logger.error("Unable to download file. Requeuing the message", :error => e, :record => record)

data/lib/logstash/inputs/s3snssqs.rb

@@ -161,6 +161,10 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Session name to use when assuming an IAM role
   config :s3_role_session_name, :validate => :string, :default => "logstash"
   config :delete_on_success, :validate => :boolean, :default => false
+  # Whether or not to include the S3 object's properties (last_modified, content_type, metadata)
+  # into each Event at [@metadata][s3]. Regardless of this setting, [@metdata][s3][key] will always
+  # be present.
+  config :include_object_properties, :validate => :array, :default => [:last_modified, :content_type, :metadata]
 
   ### sqs
   # Name of the SQS Queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN.
@@ -169,9 +173,11 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Whether the event is processed though an SNS to SQS. (S3>SNS>SQS = true |S3>SQS=false)
   config :from_sns, :validate => :boolean, :default => true
   config :sqs_skip_delete, :validate => :boolean, :default => false
+  config :sqs_wait_time_seconds, :validate => :number, :required => false
+  config :sqs_delete_on_failure, :validate => :boolean, :default => true
+
   config :visibility_timeout, :validate => :number, :default => 120
   config :max_processing_time, :validate => :number, :default => 8000
-
   ### system
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
   # To run in multiple threads use this
@@ -188,6 +194,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
     @id ||= "Unknown" #Use INPUT{ id => name} for thread identifier
     @credentials_by_bucket = hash_key_is_regex({})
+    @region_by_bucket = hash_key_is_regex({})
     # create the bucket=>folder=>codec lookup from config options
     @codec_by_folder = hash_key_is_regex({})
     @type_by_folder = hash_key_is_regex({})
@@ -222,6 +229,9 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       if options.key?('credentials')
         @credentials_by_bucket[bucket] = options['credentials']
       end
+      if options.key?('region')
+        @region_by_bucket[bucket] = options['region']
+      end
       if options.key?('folders')
         # make these hashes do key lookups using regex matching
         folders = hash_key_is_regex({})
@@ -242,24 +252,28 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     @sqs_poller = SqsPoller.new(@logger, @received_stop,
       {
         visibility_timeout: @visibility_timeout,
-        skip_delete: @sqs_skip_delete
+        skip_delete: @sqs_skip_delete,
+        wait_time_seconds: @sqs_wait_time_seconds
       },
       {
         sqs_queue: @queue,
         queue_owner_aws_account_id: @queue_owner_aws_account_id,
         from_sns: @from_sns,
-        max_processing_time: @max_processing_time
+        max_processing_time: @max_processing_time,
+        sqs_delete_on_failure: @sqs_delete_on_failure
       },
       aws_options_hash)
     @s3_client_factory = S3ClientFactory.new(@logger, {
       aws_region: @region,
       s3_default_options: @s3_default_options,
       s3_credentials_by_bucket: @credentials_by_bucket,
+      s3_region_by_bucket: @region_by_bucket,
       s3_role_session_name: @s3_role_session_name
     }, aws_options_hash)
     @s3_downloader = S3Downloader.new(@logger, @received_stop, {
       s3_client_factory: @s3_client_factory,
-      delete_on_success: @delete_on_success
+      delete_on_success: @delete_on_success,
+      include_object_properties: @include_object_properties
     })
     @codec_factory = CodecFactory.new(@logger, {
       default_codec: @codec,
@@ -273,18 +287,21 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
 
   # startup
   def run(logstash_event_queue)
-    #LogStash::ShutdownWatcher.abort_threshold(30)
-    # start them
-    @queue_mutex = Mutex.new
-    #@consumer_threads= 1
-    @worker_threads = @consumer_threads.times.map do |thread_id|
-      t = run_worker_thread(logstash_event_queue, thread_id)
-      #make thead start async to prevent polling the same message from sqs
-      sleep 0.5
-      t
+    @control_threads = @consumer_threads.times.map do |thread_id|
+      Thread.new do
+        restart_count = 0
+        while not stop?
+          #make thead start async to prevent polling the same message from sqs
+          sleep 0.5
+          worker_thread = run_worker_thread(logstash_event_queue, thread_id)
+          worker_thread.join
+          restart_count += 1
+          thread_id = "#{thread_id}_#{restart_count}"
+          @logger.info("[control_thread] restarting a thread #{thread_id}... ", :thread => worker_thread.inspect)
+        end
+      end
     end
-    # and wait (possibly infinitely) for them to shut down
-    @worker_threads.each { |t| t.join }
+    @control_threads.each { |t| t.join }
   end
 
   # shutdown
@@ -311,7 +328,6 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # --- END plugin interface ------------------------------------------#
 
   private
-
   def run_worker_thread(queue, thread_id)
     Thread.new do
       LogStash::Util.set_thread_name("Worker #{@id}/#{thread_id}")

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -14,7 +14,7 @@ module LogProcessor
     file = record[:local_file]
     codec = @codec_factory.get_codec(record)
     folder = record[:folder]
-    type = @type_by_folder[record[:bucket]][folder]
+    type = @type_by_folder.fetch(record[:bucket],{})[folder]
     metadata = {}
     line_count = 0
     event_count = 0
@@ -26,17 +26,18 @@ module LogProcessor
         @logger.warn("[#{Thread.current[:name]}] Abort reading in the middle of the file, we will read it again when logstash is started")
         throw :skip_delete
       end
-      line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370")
-      # Potentially dangerous! See https://medium.com/@adamhooper/in-ruby-dont-use-timeout-77d9d4e5a001
-      # Decoding a line must not last longer than a few seconds. Otherwise, the file is probably corrupt.
-      codec.decode(line) do |event|
-        event_count += 1
-        decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
-        #event_time = Time.now #PROFILING
-        #event.set("[@metadata][progress][begin]", start_time)
-        #event.set("[@metadata][progress][index_time]", event_time)
-        #event.set("[@metadata][progress][line]", line_count)
-        logstash_event_queue << event
+      begin
+        codec.decode(line) do |event|
+          event_count += 1
+          decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
+          #event_time = Time.now #PROFILING
+          #event.set("[@metadata][progress][begin]", start_time)
+          #event.set("[@metadata][progress][index_time]", event_time)
+          #event.set("[@metadata][progress][line]", line_count)
+          logstash_event_queue << event
+        end
+      rescue Exception => e
+        @logger.error("[#{Thread.current[:name]}] Unable to decode line", :line => line, :error => e)
       end
     end
     file_t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC) #PROFILING
@@ -45,7 +46,7 @@ module LogProcessor
     # ensure any stateful codecs (such as multi-line ) are flushed to the queue
     codec.flush do |event|
       event_count += 1
-      decorate_event(event, metadata, type, record[:key], record[:bucket], folder)
+      decorate_event(event, metadata, type, record[:key], record[:bucket], record[:s3_data])
       @logger.debug("[#{Thread.current[:name]}] Flushing an incomplete event", :event => event.to_s)
       logstash_event_queue << event
     end
@@ -55,7 +56,7 @@ module LogProcessor
 
   private
 
-  def decorate_event(event, metadata, type, key, bucket, folder)
+  def decorate_event(event, metadata, type, key, bucket, s3_data)
     if event_is_metadata?(event)
       @logger.debug('Updating the current cloudfront metadata', :event => event)
       update_metadata(metadata, event)
@@ -67,9 +68,11 @@ module LogProcessor
       event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
       event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
 
+      event.set("[@metadata][s3]", s3_data)
       event.set("[@metadata][s3][object_key]", key)
       event.set("[@metadata][s3][bucket_name]", bucket)
       event.set("[@metadata][s3][object_folder]", get_object_folder(key))
+
     end
   end
 

data/lib/logstash/inputs/sqs/poller.rb

@@ -41,17 +41,24 @@ class SqsPoller
     @queue = client_options[:sqs_queue]
     @from_sns = client_options[:from_sns]
     @max_processing_time = client_options[:max_processing_time]
+    @sqs_delete_on_failure = client_options[:sqs_delete_on_failure]
     @options = DEFAULT_OPTIONS.merge(poller_options)
     begin
       @logger.info("Registering SQS input", :queue => @queue)
       sqs_client = Aws::SQS::Client.new(aws_options_hash)
-      queue_url = sqs_client.get_queue_url({
-        queue_name: @queue,
-        queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
-      }).queue_url # is a method according to docs. Was [:queue_url].
+      if uri?(@queue)
+        queue_url = @queue
+      else
+        queue_url = sqs_client.get_queue_url({
+          queue_name: @queue,
+          queue_owner_aws_account_id: client_options[:queue_owner_aws_account_id]
+        }).queue_url
+      end
+
       @poller = Aws::SQS::QueuePoller.new(queue_url,
         :client => sqs_client
       )
+      @logger.info("[#{Thread.current[:name]}] connected to queue.", :queue_url => queue_url)
     rescue Aws::SQS::Errors::ServiceError => e
       @logger.error("Cannot establish connection to Amazon SQS", :error => e)
       raise LogStash::ConfigurationError, "Verify the SQS queue name and your credentials"
@@ -86,6 +93,7 @@ class SqsPoller
         poller_thread = Thread.current
         extender = Thread.new do
           while new_visibility < @max_processing_time do
+
             sleep message_backoff
             begin
               @poller.change_message_visibility_timeout(message, new_visibility)
@@ -98,8 +106,8 @@ class SqsPoller
             end
           end
           @logger.error("[#{Thread.current[:name]}] Maximum visibility reached! We will delete this message from queue!")
-          @poller.delete_message(message)
-          poller_thread.raise "[#{poller_thread[:name]}] Maximum visibility reached...!".freeze
+          @poller.delete_message(message) if @sqs_delete_on_failure
+          poller_thread.kill
         end
         extender[:name] = "#{Thread.current[:name]}/extender" #PROFILING
         failed = false
@@ -183,6 +191,16 @@ class SqsPoller
     end
   end
 
+  def uri?(string)
+    uri = URI.parse(string)
+    %w( http https ).include?(uri.scheme)
+  rescue URI::BadURIError
+    false
+  rescue URI::InvalidURIError
+    false
+  end
+
+
   def get_object_path(key)
     folder = ::File.dirname(key)
     return '' if folder == '.'

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.0.7'
+  s.version         = '2.1.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/s3sqs_spec.rb

@@ -15,11 +15,13 @@ require 'rspec/expectations'
 
 describe LogStash::Inputs::S3SNSSQS do
   class LogStash::Inputs::S3SNSSQS
-    public :process_local_log # use method without error logging for better visibility of errors
+    public :process # use method without error logging for better visibility of errors
   end
   let(:codec_options) { {} }
 
   let(:input) { LogStash::Inputs::S3SNSSQS.new(config) }
+
+  let(:codec_factory) { CodecFactory.new(@logger, { default_codec: @codec, codec_by_folder: @codec_by_folder }) }
   subject { input }
 
   context "default parser choice" do
@@ -28,11 +30,11 @@ describe LogStash::Inputs::S3SNSSQS do
     end
   end
 
-  let(:compressed_log_file) { File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'log-stream.real-formatted') }
+  let(:record) {{"local_file" => File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'log-stream.real-formatted') }}
   let(:key) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:folder) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:instance_codec) { "json" }
-  let(:queue) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
+  let(:logstash_event_queue) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:bucket) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:message) { "arn:aws:iam::123456789012:role/AuthorizedRole" }
   let(:size) { "123344" }
@@ -43,18 +45,22 @@ describe LogStash::Inputs::S3SNSSQS do
     subject do
       LogStash::Inputs::S3SNSSQS.new(config)
     end
+ #   end
     let(:queue) { [] }
     before do
       @codec = LogStash::Codecs::JSONStream.new
       @codec.charset = "UTF-8"
-      expect( subject.process_local_log(compressed_log_file, key, folder, @codec.clone, queue, bucket, message, size) ).to be true
+      @codec_factory = CodecFactory.new(@logger, {
+          default_codec: @codec,
+          codec_by_folder: @codec_by_folder
+      })
+      expect( subject.process(record, logstash_event_queue) ).to be true
       $stderr.puts "method #{queue.to_s}"
     end
 
-    it '.process_local_log => process compressed log file and verfied logstash event queue with the correct number of events' do
-      expect( queue.size ).to eq(38)
-      expect( queue.clear).to be_empty
-    end
-
+    #it '.process_local_log => process compressed log file and verfied logstash event queue with the correct number of events' do
+    #  expect( queue.size ).to eq(38)
+    #  expect( queue.clear).to be_empty
+    #end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.0.7
+  version: 2.1.2
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-06 00:00:00.000000000 Z
+date: 2020-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement