logstash-input-s3-sns-sqs 2.2.0.pre → 2.2.1.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d5ea48e146f2c80413f41fa2be04ab09396ce2fd67f0275032d34641e0c6fd2
-  data.tar.gz: c4301ebe9b7abb940891bd06843de67b66b400cc6470bfa69b4a208320ef6387
+  metadata.gz: 045e651e9ce6ed3e99a4f06b52be476e30ef7501b5e53a5bf97bcf0118c463d6
+  data.tar.gz: 25b9a355da317dec8b235fd84ce0cb04aec7fec9843edbbd418aa7f0cb570237
 SHA512:
-  metadata.gz: 7c478aa632537f4db570327ff25df1102faab0f9480515bbb4991e52540e086cfb881935b29725e34b4bd5de65a06de77ef65d682ee6b7cc909fce72b5359080
-  data.tar.gz: 52cbe75d68326c5c29fdd5e2b13b3724e0c829664726667fb031c22c99318dfe04d950eb06524ee6d4d5cff76f3df13074db813c663b044f7f126907c8106a25
+  metadata.gz: 51a683ac50b21cd5e27c7dbcf73a494adeb3c5f04c7f590b38e3cb3730c308d331a8ab475a06e0df92204172841c62563ea29f92a1304932e15e3fe828c6ea0b
+  data.tar.gz: a8957a59f15f3d134f8c1fb7e60f1bbfa0461ceca58ec4569d58419634309bef27e780f6b43a084533690ccde3c040ec7549286aef49ecfc9bdee71da83c4bfa

data/lib/logstash/inputs/s3/downloader.rb

@@ -9,6 +9,7 @@ class S3Downloader
     @stopped = stop_semaphore
     @factory = options[:s3_client_factory]
     @delete_on_success = options[:delete_on_success]
+    @move_to_bucket = options[:move_to_bucket]
     @include_object_properties = options[:include_object_properties]
   end
 
@@ -40,7 +41,7 @@ class S3Downloader
   end
 
   def cleanup_s3object(record)
-    return unless @delete_on_success
+    return unless @delete_on_success || @move_to_bucket
     begin
       @factory.get_s3_client(record[:bucket]) do |s3|
         s3.delete_object(bucket: record[:bucket], key: record[:key])
@@ -50,6 +51,17 @@ class S3Downloader
     end
   end
 
+  def move_s3object(record)
+    return unless @move_to_bucket
+    begin
+      @factory.get_s3_client(@move_to_bucket) do |s3|
+        s3.copy_object(bucket: @move_to_bucket, copy_source: "/#{record[:bucket]}/#{record[:key]}", key: record[:key])
+      end
+    rescue Exception => e
+      @logger.warn("Failed to move s3 object", :record => record, :error => e)
+    end
+  end
+
   def stop?
     @stopped.value
   end

data/lib/logstash/inputs/s3snssqs/log_processor.rb

@@ -77,8 +77,7 @@ module LogProcessor
   end
 
   def gzip?(filename)
-    return true if filename.end_with?('.gz','.gzip')
-    MagicGzipValidator.new(File.new(filename, 'rb')).valid?
+    filename.end_with?('.gz','.gzip') && MagicGzipValidator.new(File.new(filename, 'rb')).valid?
   rescue Exception => e
     @logger.warn("Problem while gzip detection", :error => e)
   end

data/lib/logstash/inputs/s3snssqs.rb

@@ -26,7 +26,7 @@ java_import java.io.FileInputStream
 java_import java.io.BufferedReader
 java_import java.util.zip.GZIPInputStream
 java_import java.util.zip.ZipException
-import java.lang.StringBuilder
+java_import java.lang.StringBuilder
 
 # our helper classes
 # these may go into this file for brevity...
@@ -160,6 +160,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # Session name to use when assuming an IAM role
   config :s3_role_session_name, :validate => :string, :default => "logstash"
   config :delete_on_success, :validate => :boolean, :default => false
+  config :move_to_bucket, :validate => :string, :default => nil
   # Whether or not to include the S3 object's properties (last_modified, content_type, metadata)
   # into each Event at [@metadata][s3]. Regardless of this setting, [@metdata][s3][key] will always
   # be present.
@@ -272,6 +273,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     @s3_downloader = S3Downloader.new(@logger, @received_stop, {
       s3_client_factory: @s3_client_factory,
       delete_on_success: @delete_on_success,
+      move_to_bucket: @move_to_bucket,
       include_object_properties: @include_object_properties
     })
     @codec_factory = CodecFactory.new(@logger, {
@@ -343,6 +345,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
           @s3_downloader.cleanup_local_object(record)
           # re-throw if necessary:
           throw :skip_delete unless completed
+          @s3_downloader.move_s3object(record)
           @s3_downloader.cleanup_s3object(record)
         end
       end

data/lib/logstash/plugin_mixins/aws_config/generic.rb

@@ -0,0 +1,61 @@
+module LogStash::PluginMixins::AwsConfig::Generic
+  def self.included(base)
+    base.extend(self)
+    base.generic_aws_config
+  end
+
+  def generic_aws_config
+    # The AWS Region
+    config :region, :validate => :string, :default => LogStash::PluginMixins::AwsConfig::US_EAST_1
+
+    # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order:
+    #
+    # 1. Static configuration, using `access_key_id` and `secret_access_key` params in the logstash plugin config
+    # 2. External credentials file specified by `aws_credentials_file`
+    # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+    # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`
+    # 5. IAM Instance Profile (available when running inside EC2)
+    config :access_key_id, :validate => :string
+
+    # The AWS Secret Access Key
+    config :secret_access_key, :validate => :password
+
+    # The AWS Session token for temporary credential
+    config :session_token, :validate => :password
+
+    # URI to proxy server if required
+    config :proxy_uri, :validate => :string
+
+    # Custom endpoint to connect to s3
+    config :endpoint, :validate => :string
+
+    # The AWS IAM Role to assume, if any.
+    # This is used to generate temporary credentials typically for cross-account access.
+    # See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html for more information.
+    # When `role_arn` is set, AWS (`access_key_id`/`secret_access_key`) credentials still get used if they're configured.
+    config :role_arn, :validate => :string
+
+    # Session name to use when assuming an IAM role
+    config :role_session_name, :validate => :string, :default => "logstash"
+
+    # Path to YAML file containing a hash of AWS credentials.
+    # This file will only be loaded if `access_key_id` and
+    # `secret_access_key` aren't set. The contents of the
+    # file should look like this:
+    #
+    # [source,ruby]
+    # ----------------------------------
+    #     :access_key_id: "12345"
+    #     :secret_access_key: "54321"
+    # ----------------------------------
+    #
+    config :aws_credentials_file, :validate => :string
+
+    # By default, this plugin uses cert available to OpenSSL provided by OS
+    # when verifying SSL peer certificates.
+    # For cases where the default cert is unavailable, e.g. Windows,
+    # you can use the bundled ca certificate provided by AWS SDK
+    # by setting `use_aws_bundled_ca` to true
+    config :use_aws_bundled_ca, :validate => :boolean, :default => false
+  end
+end

data/lib/logstash/plugin_mixins/aws_config/v2.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+require "logstash/plugin_mixins/aws_config/generic"
+
+module LogStash::PluginMixins::AwsConfig::V2
+  def self.included(base)
+    base.extend(self)
+    base.send(:include, LogStash::PluginMixins::AwsConfig::Generic)
+  end
+
+  public
+  def aws_options_hash
+    opts = {}
+
+    opts[:http_proxy] = @proxy_uri if @proxy_uri
+
+    if @role_arn
+      credentials = assume_role(opts.dup)
+      opts[:credentials] = credentials
+    else
+      credentials = aws_credentials
+      opts[:credentials] = credentials if credentials
+    end
+
+    if self.respond_to?(:aws_service_endpoint)
+      # used by CloudWatch to basically do the same as bellow (returns { region: region })
+      opts.merge!(self.aws_service_endpoint(@region))
+    else
+      # NOTE: setting :region works with the aws sdk (resolves correct endpoint)
+      opts[:region] = @region
+    end
+
+    opts[:endpoint] = @endpoint unless @endpoint.nil?
+
+    if respond_to?(:additional_settings)
+      opts = symbolize_keys_and_cast_true_false(additional_settings).merge(opts)
+    end
+
+    if @use_aws_bundled_ca
+      aws_core_library = Gem.loaded_specs['aws-sdk-core']&.full_gem_path or fail("AWS Core library not available")
+      opts[:ssl_ca_bundle] = File.expand_path('ca-bundle.crt', aws_core_library).tap do |aws_core_ca_bundle|
+        fail("AWS Core CA bundle not found") unless File.exists?(aws_core_ca_bundle)
+      end
+    end
+
+    return opts
+  end
+
+  private
+
+  def aws_credentials
+    if @access_key_id && @secret_access_key
+      Aws::Credentials.new(@access_key_id, @secret_access_key.value, @session_token ? @session_token.value : nil)
+    elsif @access_key_id.nil? ^ @secret_access_key.nil?
+      @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.")
+      secret_access_key = @secret_access_key ? @secret_access_key.value : nil
+      Aws::Credentials.new(@access_key_id, secret_access_key, @session_token ? @session_token.value : nil)
+    elsif @aws_credentials_file
+      credentials_opts = YAML.load_file(@aws_credentials_file)
+      credentials_opts.default_proc = lambda { |hash, key| hash.fetch(key.to_s, nil) }
+      Aws::Credentials.new(credentials_opts[:access_key_id],
+                           credentials_opts[:secret_access_key],
+                           credentials_opts[:session_token])
+    else
+      nil # AWS client will read ENV or ~/.aws/credentials
+    end
+  end
+  alias credentials aws_credentials
+
+  def assume_role(opts = {})
+    unless opts.key?(:credentials)
+      credentials = aws_credentials
+      opts[:credentials] = credentials if credentials
+    end
+
+    # for a regional endpoint :region is always required by AWS
+    opts[:region] = @region
+
+    Aws::AssumeRoleCredentials.new(
+        :client => Aws::STS::Client.new(opts),
+        :role_arn => @role_arn,
+        :role_session_name => @role_session_name
+    )
+  end
+
+  def symbolize_keys_and_cast_true_false(hash)
+    case hash
+    when Hash
+      symbolized = {}
+      hash.each { |key, value| symbolized[key.to_sym] = symbolize_keys_and_cast_true_false(value) }
+      symbolized
+    when 'true'
+      true
+    when 'false'
+      false
+    else
+      hash
+    end
+  end
+
+end

data/lib/logstash/plugin_mixins/aws_config.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require "logstash/config/mixin"
+
+module LogStash::PluginMixins::AwsConfig
+  require "logstash/plugin_mixins/aws_config/v2"
+
+  US_EAST_1 = "us-east-1"
+end

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '2.2.0.pre'
+  s.version         = '2.2.1.pre'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,8 +19,9 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency "concurrent-ruby"
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
   s.add_runtime_dependency "logstash-codec-json"
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_runtime_dependency "aws-sdk-core", "~> 3"
@@ -31,9 +32,8 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "logstash-codec-json_lines"
   s.add_development_dependency "logstash-codec-multiline"
-  s.add_development_dependency "logstash-codec-json"
   s.add_development_dependency "logstash-codec-line"
-  s.add_development_dependency "logstash-devutils"
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
   s.add_development_dependency "logstash-input-generator"
   s.add_development_dependency "timecop"
 

metadata

@@ -1,35 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 2.2.0.pre
+  version: 2.2.1.pre
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-13 00:00:00.000000000 Z
+date: 2023-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.1.12
-    - - "<="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.99'
+        version: '2.0'
   name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.1.12
-    - - "<="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.99'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -44,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -170,20 +178,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-codec-json
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -203,7 +197,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.16
   name: logstash-devutils
   prerelease: false
   type: :development
@@ -211,7 +205,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.16
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -261,6 +255,9 @@ files:
 - lib/logstash/inputs/s3snssqs.rb
 - lib/logstash/inputs/s3snssqs/log_processor.rb
 - lib/logstash/inputs/sqs/poller.rb
+- lib/logstash/plugin_mixins/aws_config.rb
+- lib/logstash/plugin_mixins/aws_config/generic.rb
+- lib/logstash/plugin_mixins/aws_config/v2.rb
 - logstash-input-s3-sns-sqs.gemspec
 - spec/inputs/s3sqs_spec.rb
 homepage: https://github.com/cherweg/logstash-input-s3-sns-sqs
@@ -284,7 +281,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.29
 signing_key:
 specification_version: 4
 summary: Get logs from AWS s3 buckets as issued by an object-created event via sns