fluent-plugin-s3 1.6.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 155f0d28d700caf2e5c817843abec96a26f301404d2bf521c6cd383068fdeb47
-  data.tar.gz: '0701182695128c6812b28179685658a8b570332a2f747146e157e859d61d0898'
+  metadata.gz: fe9afa0a2351dfffc6ea9afa0c2f3598a29b19a55fbfa22f58f3d17b606a3113
+  data.tar.gz: e5813910a178d16ed61c2a8782e2adeb394d0d256acfbcfdbead05069d789163
 SHA512:
-  metadata.gz: 47460a080d212895f1a121da9dc547b08281df9c9ae1c2644dce582c63cad4b971aeefcf287553bf8f983dd4b2e628acfd6119b2dd3b725e5d0125e58fe5c043
-  data.tar.gz: a10f906a8db7c81f80dad1257a1aaf38a9a432ede35f12df6e21df2e138fd6662668192c6382cba9b647dd4e8427c9697f8b4188acef5280fd0278f24ef87b26
+  metadata.gz: e3ca35df25975b7b57244c88d3145ab8fb5521054b863e030846a867bae6cbc9a69343f586f01b3a29d233eea787cf0ab540ff77edc0423d583394e29a192a7c
+  data.tar.gz: d46deabb1425e03250c9bc24578ad8dacef71cc445335ac9e4122b11c6ad91cf7a73f92b63d3688848ff35d427cb6371777f0011c94c45a898a6063edc1e3d9c

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,72 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README](https://github.com/fluent/fluent-plugin-s3#readme) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - fluent-plugin-s3 version:
+        - aws-sdk-s3 version:
+        - aws-sdk-sqs version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://discuss.fluentd.org/
+    about: I have questions about fluent-plugin-kafka. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README.md](https://github.com/fluent/fluent-plugin-s3/blob/master/README.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/workflows/linux.yml

@@ -1,14 +1,16 @@
 name: linux
 on:
-  - push
-  - pull_request
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
 jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        ruby: [ '3.1', '3.0', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/stale-actions.yml

@@ -0,0 +1,22 @@
+name: "Mark or close stale issues and PRs"
+on:
+  schedule:
+  - cron: "00 10 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        days-before-stale: 90
+        days-before-close: 30
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
+        close-issue-message: "This issue was automatically closed because of stale in 30 days"
+        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        stale-pr-label: "stale"
+        stale-issue-label: "stale"
+        exempt-issue-labels: "bug,help wanted,enhancement"
+        exempt-pr-labels: "bug,help wanted,enhancement"

data/ChangeLog

@@ -1,3 +1,19 @@
+Release 1.7.1 - 2022/07/15
+
+  * in_s3: Add `match_regexp` parameter to selectively download S3 files based on the object key
+  * out_s3: Support `ssl_ca_bundle` and `ssl_ca_directory` parameter
+
+Release 1.7.0 - 2022/06/14
+
+  * in_s3: Allow multi workers
+  * in_s3: Support alternative AWS key ID and secret for SQS
+  * out_s3: Add warning for multi workers
+  * out_s3: Support object tagging
+
+Release 1.6.1 - 2021/08/19
+
+  * in_s3/out_s3: Don't raise error when s3_endpoint is used for VPC endpoint (GitHub: #384)
+
 Release 1.6.0 - 2021/04/08
 
   * out_s3: Add support for Parquet compressor. Use `<compress>` section to configure columnify command behavior.

data/README.md

@@ -20,6 +20,8 @@ the former one is stored in "20110102.gz" file, and latter one in
 SQS queue on the region same as S3 bucket.
 We must setup SQS queue and S3 event notification before use this plugin.
 
+:warning: Be sure to keep a close eye on S3 costs, as a few user have reported [unexpectedly high costs](https://github.com/fluent/fluent-plugin-s3/issues/160).
+
 ## Requirements
 
 | fluent-plugin-s3  | fluentd | ruby |

data/VERSION

@@ -1 +1 @@
-1.6.0
+1.7.1

data/docs/input.md

@@ -18,6 +18,7 @@ See also [Configuration: credentials](credentials.md) for common comprehensive p
       s3_bucket YOUR_S3_BUCKET_NAME
       s3_region ap-northeast-1
       add_object_metadata true
+      match_regexp production_.*
 
       <sqs>
         queue_name YOUR_SQS_QUEUE_NAME
@@ -28,6 +29,10 @@ See also [Configuration: credentials](credentials.md) for common comprehensive p
 
 Whether or not object metadata should be added to the record. Defaults to `false`. See below for details.
 
+## match_regexp
+
+If provided, process the S3 object only if its keys matches the regular expression
+
 ## s3_bucket (required)
 
 S3 bucket name.
@@ -77,6 +82,14 @@ SQS queue name. Need to create SQS queue on the region same as S3 bucket.
 
 SQS Owner Account ID
 
+### aws_key_id
+
+Alternative aws key id for SQS
+
+### aws_sec_key
+
+Alternative aws key secret for SQS
+
 ### skip_delete
 
 When true, messages are not deleted after polling block. Default is false.

data/docs/output.md

@@ -81,6 +81,14 @@ This fixes the following error often seen in Windows:
 
     SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed (Seahorse::Client::NetworkingError)
 
+## ssl_ca_bundle
+
+Full path to the SSL certificate authority bundle file that should be used when verifying peer certificates. If you do not pass `ssl_ca_bundle` or `ssl_ca_directory` the the system default will be used if available.
+
+## ssl_ca_directory
+
+Full path of the directory that contains the unbundled SSL certificate authority files for verifying peer certificates. If you do not pass `ssl_ca_bundle` or `ssl_ca_directory` the the system default will be used if available.
+
 ## ssl_verify_peer
 
 Verify SSL certificate of the endpoint. Default is true. Set false when you want to ignore the endpoint SSL certificate.
@@ -152,6 +160,8 @@ NOTE: ${hostname} placeholder is deprecated since v0.8. You can get same result
 
 Above two configurations are same. The important point is wrapping `""` is needed for `#{Socket.gethostname}`.
 
+NOTE: If `check_object` is set to `false`, Ensure the value of `s3_object_key_format` must be unique in each write, If not, existing file will be overwritten.
+
 ## force_path_style
 
 :force_path_style (Boolean) — default: false — When set to true, the
@@ -322,6 +332,8 @@ Use UTC instead of local time.
 
 Set storage class. Possible values are `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA` from [Ruby SDK](http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#storage_class-instance_method).
 
+Note that reduced redundancy is [not reccomended](https://serverfault.com/a/1010951/512362).
+
 ## reduced_redundancy
 
 Use S3 reduced redundancy storage for 33% cheaper pricing. Default is
@@ -434,6 +446,10 @@ It would be useful when you use S3 compatible storage that accepts only signatur
 
 Given a threshold to treat events as delay, output warning logs if delayed events were put into s3.
 
+## tagging
+
+The S3 tag-set for the object. The tag-set must be encoded as URL Query parameters. (For example, "Key1=Value1").
+
 ## \<bucket_lifecycle_rule\> section
 
 Specify one or more lifecycle rules for the bucket

data/fluent-plugin-s3.gemspec

@@ -23,4 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
   gem.add_development_dependency "timecop"
+  # aws-sdk-core requires one of ox, oga, libxml, nokogiri or rexml,
+  # and rexml is no longer default gem as of Ruby 3.0.
+  gem.add_development_dependency "rexml"
 end

data/lib/fluent/plugin/in_s3.rb

@@ -90,6 +90,8 @@ module Fluent::Plugin
     config_param :check_apikey_on_start, :bool, default: true
     desc "URI of proxy environment"
     config_param :proxy_uri, :string, default: nil
+    desc "Optional RegEx to match incoming messages"
+    config_param :match_regexp, :regexp, default: nil
 
     config_section :sqs, required: true, multi: false do
       desc "SQS queue name"
@@ -98,6 +100,10 @@ module Fluent::Plugin
       config_param :queue_owner_aws_account_id, :string, default: nil
       desc "Use 's3_region' instead"
       config_param :endpoint, :string, default: nil
+      desc "AWS access key id for SQS user"
+      config_param :aws_key_id, :string, default: nil, secret: true
+      desc "AWS secret key for SQS user."
+      config_param :aws_sec_key, :string, default: nil, secret: true
       desc "Skip message deletion"
       config_param :skip_delete, :bool, default: false
       desc "The long polling interval."
@@ -115,10 +121,15 @@ module Fluent::Plugin
 
     attr_reader :bucket
 
+    def reject_s3_endpoint?
+      @s3_endpoint && !@s3_endpoint.end_with?('vpce.amazonaws.com') &&
+        @s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) }
+    end
+
     def configure(conf)
       super
 
-      if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
+      if reject_s3_endpoint?
         raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
       end
 
@@ -131,6 +142,14 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "sqs/queue_name is required"
       end
 
+      if !!@aws_key_id ^ !!@aws_sec_key
+        raise Fluent::ConfigError, "aws_key_id or aws_sec_key is missing"
+      end
+
+      if !!@sqs.aws_key_id ^ !!@sqs.aws_sec_key
+        raise Fluent::ConfigError, "sqs/aws_key_id or sqs/aws_sec_key is missing"
+      end
+
       Aws.use_bundled_cert! if @use_bundled_cert
 
       @extractor = EXTRACTOR_REGISTRY.lookup(@store_as).new(log: log)
@@ -139,6 +158,10 @@ module Fluent::Plugin
       @parser = parser_create(conf: parser_config, default_type: DEFAULT_PARSE_TYPE)
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def start
       super
 
@@ -183,7 +206,12 @@ module Fluent::Plugin
             body = Yajl.load(message.body)
             log.debug(body)
             next unless body["Records"] # skip test queue
-
+            if @match_regexp
+              s3 = body["Records"].first["s3"]
+              raw_key = s3["object"]["key"]
+              key = CGI.unescape(raw_key)
+              next unless @match_regexp.match?(key) 
+            end
             process(body)
           rescue => e
             log.warn(error: e)
@@ -270,6 +298,10 @@ module Fluent::Plugin
       options[:region] = @s3_region if @s3_region
       options[:endpoint] = @sqs.endpoint if @sqs.endpoint
       options[:http_proxy] = @proxy_uri if @proxy_uri
+      if @sqs.aws_key_id && @sqs.aws_sec_key
+        options[:access_key_id] = @sqs.aws_key_id
+        options[:secret_access_key] = @sqs.aws_sec_key
+      end
       log.on_trace do
         options[:http_wire_trace] = true
         options[:logger] = log

data/lib/fluent/plugin/out_s3.rb

@@ -97,6 +97,10 @@ module Fluent::Plugin
     config_param :enable_dual_stack, :bool, default: false
     desc "If false, the certificate of endpoint will not be verified"
     config_param :ssl_verify_peer, :bool, :default => true
+    desc "Full path to the SSL certificate authority bundle file that should be used when verifying peer certificates. If unspecified, defaults to the system CA if available."
+    config_param :ssl_ca_bundle, :string, :default => nil
+    desc "Full path of the directory that contains the unbundled SSL certificate authority files for verifying peer certificates. If you do not pass ssl_ca_bundle or ssl_ca_directory the the system default will be used if available."
+    config_param :ssl_ca_directory, :string, :default => nil
     desc "The format of S3 object keys"
     config_param :s3_object_key_format, :string, default: "%{path}%{time_slice}_%{index}.%{file_extension}"
     desc "If true, the bucket name is always left in the request URI and never moved to the host as a sub-domain"
@@ -147,6 +151,8 @@ module Fluent::Plugin
     config_param :signature_version, :string, default: nil # use nil to follow SDK default configuration
     desc "Given a threshold to treat events as delay, output warning logs if delayed events were put into s3"
     config_param :warn_for_delay, :time, default: nil
+    desc "Arbitrary S3 tag-set for the object"
+    config_param :tagging, :string, default: nil
     desc "Arbitrary S3 metadata headers to set for the object"
     config_param :s3_metadata, :hash, default: nil
     config_section :bucket_lifecycle_rule, param_name: :bucket_lifecycle_rules, multi: true do
@@ -173,6 +179,11 @@ module Fluent::Plugin
 
     MAX_HEX_RANDOM_LENGTH = 16
 
+    def reject_s3_endpoint?
+      @s3_endpoint && !@s3_endpoint.end_with?('vpce.amazonaws.com') &&
+        @s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) }
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :buffer, :formatter, :inject)
 
@@ -180,7 +191,7 @@ module Fluent::Plugin
 
       Aws.use_bundled_cert! if @use_bundled_cert
 
-      if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
+      if reject_s3_endpoint?
         raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
       end
 
@@ -242,6 +253,8 @@ module Fluent::Plugin
       options[:compute_checksums] = @compute_checksums unless @compute_checksums.nil?
       options[:signature_version] = @signature_version unless @signature_version.nil?
       options[:ssl_verify_peer] = @ssl_verify_peer
+      options[:ssl_ca_bundle] = @ssl_ca_bundle if @ssl_ca_bundle
+      options[:ssl_ca_directory] = @ssl_ca_directory if @ssl_ca_directory
       log.on_trace do
         options[:http_wire_trace] = true
         options[:logger] = log
@@ -355,6 +368,7 @@ module Fluent::Plugin
         put_options[:grant_read] = @grant_read if @grant_read
         put_options[:grant_read_acp] = @grant_read_acp if @grant_read_acp
         put_options[:grant_write_acp] = @grant_write_acp if @grant_write_acp
+        put_options[:tagging] = @tagging if @tagging
 
         if @s3_metadata
           put_options[:metadata] = {}
@@ -456,8 +470,9 @@ module Fluent::Plugin
         log.warn "The default value of s3_object_key_format will use ${chunk_id} instead of %{index} to avoid object conflict in v2"
       end
 
-      if (@buffer_config.flush_thread_count > 1) && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
-        log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
+      is_working_on_parallel = @buffer_config.flush_thread_count > 1 || system_config.workers > 1
+      if is_working_on_parallel && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
+        log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads or multiple workers. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
       end
     end
 

data/test/test_in_s3.rb

@@ -84,7 +84,7 @@ class S3InputTest < Test::Unit::TestCase
 
     def test_unknown_store_as
       config = CONFIG + "\nstore_as unknown"
-      assert_raise(Fluent::ConfigError) do
+      assert_raise(Fluent::NotFoundPluginError) do
         create_driver(config)
       end
     end
@@ -115,14 +115,19 @@ class S3InputTest < Test::Unit::TestCase
   end
 
 
-  def test_s3_endpoint_with_valid_endpoint
-    d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
-    assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
+  data('Normal endpoint' => 'riak-cs.example.com',
+       'VPCE endpoint' => 'vpce.amazonaws.com',
+       'FIPS endpoint' => 'fips.xxx.amazonaws.com',
+       'GOV endpoint' => 'gov.xxx.amazonaws.com')
+  def test_s3_endpoint_with_valid_endpoint(endpoint)
+    d = create_driver(CONFIG + "s3_endpoint #{endpoint}")
+    assert_equal endpoint, d.instance.s3_endpoint
   end
 
   data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
        'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
-       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
+       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com',
+       'Invalid VPCE' => 'vpce.xxx.amazonaws.com')
   def test_s3_endpoint_with_invalid_endpoint(endpoint)
     assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
       create_driver(CONFIG + "s3_endpoint #{endpoint}")
@@ -148,6 +153,104 @@ EOS
     }
   end
 
+  def test_sqs_with_invalid_keys_missing_secret_key
+    assert_raise(Fluent::ConfigError, "sqs/aws_key_id or sqs/aws_sec_key is missing") {
+      conf = <<"EOS"
+aws_key_id test_key_id
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+  aws_key_id sqs_test_key_id
+</sqs>
+EOS
+        create_driver(conf)
+      }
+    end
+
+  def test_sqs_with_invalid_aws_keys_missing_key_id
+    assert_raise(Fluent::ConfigError, "sqs/aws_key_id or sqs/aws_sec_key is missing") {
+      conf = <<"EOS"
+aws_key_id test_key_id
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+  aws_sec_key sqs_test_sec_key
+</sqs>
+EOS
+      create_driver(conf)
+    }
+  end
+
+  def test_sqs_with_valid_aws_keys_complete_pair
+    conf = <<"EOS"
+aws_key_id test_key_id
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+  aws_key_id sqs_test_key_id
+  aws_sec_key sqs_test_sec_key
+</sqs>
+EOS
+    d = create_driver(conf)
+    assert_equal 'sqs_test_key_id', d.instance.sqs.aws_key_id
+    assert_equal 'sqs_test_sec_key', d.instance.sqs.aws_sec_key
+  end
+
+  def test_with_invalid_aws_keys_missing_secret_key
+    assert_raise(Fluent::ConfigError, "aws_key_id or aws_sec_key is missing") {
+      conf = <<"EOS"
+aws_key_id test_key_id
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+</sqs>
+EOS
+      create_driver(conf)
+    }
+  end
+
+  def test_with_invalid_aws_keys_missing_key_id
+    assert_raise(Fluent::ConfigError, "aws_key_id or aws_sec_key is missing") {
+      conf = <<"EOS"
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+</sqs>
+EOS
+      create_driver(conf)
+    }
+  end
+
+  def test_with_valid_aws_keys_complete_pair
+    conf = <<"EOS"
+aws_key_id test_key_id
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+</sqs>
+EOS
+    d = create_driver(conf)
+    assert_equal 'test_key_id', d.instance.aws_key_id
+    assert_equal 'test_sec_key', d.instance.aws_sec_key
+  end
+
   Struct.new("StubResponse", :queue_url)
   Struct.new("StubMessage", :message_id, :receipt_handle, :body)
 
@@ -510,4 +613,68 @@ EOS
     ]
     assert_equal(expected_records, events.map {|_tag, _time, record| record })
   end
+
+  def test_regexp_matching
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nmatch_regexp .*_key?")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa bbb ccc") }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    assert_equal({ "message" => "aaa bbb ccc" }, events.first[2])
+  end
+
+  def test_regexp_not_matching
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nmatch_regexp live?_key")
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    assert_nothing_raised do
+      d.run {}
+    end
+  end
 end

data/test/test_out_s3.rb

@@ -52,102 +52,147 @@ class S3OutputTest < Test::Unit::TestCase
     end.configure(conf)
   end
 
-  def test_configure
-    d = create_driver
-    assert_equal 'test_key_id', d.instance.aws_key_id
-    assert_equal 'test_sec_key', d.instance.aws_sec_key
-    assert_equal 'test_bucket', d.instance.s3_bucket
-    assert_equal 'log', d.instance.path
-    assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
-    assert_equal false, d.instance.force_path_style
-    assert_equal nil, d.instance.compute_checksums
-    assert_equal nil, d.instance.signature_version
-    assert_equal true, d.instance.check_bucket
-    assert_equal true, d.instance.check_object
-  end
-
-  def test_s3_endpoint_with_valid_endpoint
-    d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
-    assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
-  end
-
-  data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
-       'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
-       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
-  def test_s3_endpoint_with_invalid_endpoint(endpoint)
-    assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
-      create_driver(CONFIG + "s3_endpoint #{endpoint}")
-    }
-  end
+  sub_test_case "configure" do
+    def test_configure
+      d = create_driver
+      assert_equal 'test_key_id', d.instance.aws_key_id
+      assert_equal 'test_sec_key', d.instance.aws_sec_key
+      assert_equal 'test_bucket', d.instance.s3_bucket
+      assert_equal 'log', d.instance.path
+      assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
+      assert_equal false, d.instance.force_path_style
+      assert_equal nil, d.instance.compute_checksums
+      assert_equal nil, d.instance.signature_version
+      assert_equal true, d.instance.check_bucket
+      assert_equal true, d.instance.check_object
+    end
 
-  def test_configure_with_mime_type_json
-    conf = CONFIG.clone
-    conf << "\nstore_as json\n"
-    d = create_driver(conf)
-    assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
-  end
+    def test_s3_endpoint_with_valid_endpoint
+      d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
+      assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
+    end
 
-  def test_configure_with_mime_type_text
-    conf = CONFIG.clone
-    conf << "\nstore_as text\n"
-    d = create_driver(conf)
-    assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
-  end
+    data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
+         'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
+         'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
+    def test_s3_endpoint_with_invalid_endpoint(endpoint)
+      assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
+        create_driver(CONFIG + "s3_endpoint #{endpoint}")
+      }
+    end
 
-  def test_configure_with_mime_type_lzo
-    conf = CONFIG.clone
-    conf << "\nstore_as lzo\n"
-    d = create_driver(conf)
-    assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
-  rescue => e
-    # TODO: replace code with disable lzop command
-    assert(e.is_a?(Fluent::ConfigError))
-  end
+    def test_configure_with_mime_type_json
+      conf = CONFIG.clone
+      conf << "\nstore_as json\n"
+      d = create_driver(conf)
+      assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
+    end
 
-  def test_configure_with_path_style
-    conf = CONFIG.clone
-    conf << "\nforce_path_style true\n"
-    d = create_driver(conf)
-    assert d.instance.force_path_style
-  end
+    def test_configure_with_mime_type_text
+      conf = CONFIG.clone
+      conf << "\nstore_as text\n"
+      d = create_driver(conf)
+      assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
+    end
 
-  def test_configure_with_compute_checksums
-    conf = CONFIG.clone
-    conf << "\ncompute_checksums false\n"
-    d = create_driver(conf)
-    assert_equal false, d.instance.compute_checksums
-  end
+    def test_configure_with_mime_type_lzo
+      conf = CONFIG.clone
+      conf << "\nstore_as lzo\n"
+      d = create_driver(conf)
+      assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
+    rescue => e
+      # TODO: replace code with disable lzop command
+      assert(e.is_a?(Fluent::ConfigError))
+    end
 
-  def test_configure_with_hex_random_length
-    conf = CONFIG.clone
-    assert_raise Fluent::ConfigError do
-      create_driver(conf + "\nhex_random_length 17\n")
+    def test_configure_with_path_style
+      conf = CONFIG.clone
+      conf << "\nforce_path_style true\n"
+      d = create_driver(conf)
+      assert d.instance.force_path_style
     end
-    assert_nothing_raised do
-      create_driver(conf + "\nhex_random_length 16\n")
+
+    def test_configure_with_compute_checksums
+      conf = CONFIG.clone
+      conf << "\ncompute_checksums false\n"
+      d = create_driver(conf)
+      assert_equal false, d.instance.compute_checksums
     end
-  end
 
-  def test_configure_with_no_check_on_s3
-    conf = CONFIG.clone
-    conf << "\ncheck_bucket false\ncheck_object false\n"
-    d = create_driver(conf)
-    assert_equal false, d.instance.check_bucket
-    assert_equal false, d.instance.check_object
-  end
+    def test_configure_with_hex_random_length
+      conf = CONFIG.clone
+      assert_raise Fluent::ConfigError do
+        create_driver(conf + "\nhex_random_length 17\n")
+      end
+      assert_nothing_raised do
+        create_driver(conf + "\nhex_random_length 16\n")
+      end
+    end
+
+    def test_configure_with_no_check_on_s3
+      conf = CONFIG.clone
+      conf << "\ncheck_bucket false\ncheck_object false\n"
+      d = create_driver(conf)
+      assert_equal false, d.instance.check_bucket
+      assert_equal false, d.instance.check_object
+    end
+
+    def test_configure_with_grant
+      conf = CONFIG.clone
+      conf << "\grant_full_control id='0123456789'\ngrant_read id='1234567890'\ngrant_read_acp id='2345678901'\ngrant_write_acp id='3456789012'\n"
+      d = create_driver(conf)
+      assert_equal "id='0123456789'", d.instance.grant_full_control
+      assert_equal "id='1234567890'", d.instance.grant_read
+      assert_equal "id='2345678901'", d.instance.grant_read_acp
+      assert_equal "id='3456789012'", d.instance.grant_write_acp
+    end
 
-  def test_configure_with_grant
-    conf = CONFIG.clone
-    conf << "\grant_full_control id='0123456789'\ngrant_read id='1234567890'\ngrant_read_acp id='2345678901'\ngrant_write_acp id='3456789012'\n"
-    d = create_driver(conf)
-    assert_equal "id='0123456789'", d.instance.grant_full_control
-    assert_equal "id='1234567890'", d.instance.grant_read
-    assert_equal "id='2345678901'", d.instance.grant_read_acp
-    assert_equal "id='3456789012'", d.instance.grant_write_acp
+    CONFIG_WITH_OBJECTKEY_DEFAULT = %[
+      s3_object_key_format "%{path}%{time_slice}_%{index}.%{file_extension}"
+      aws_key_id test_key_id
+      aws_sec_key test_sec_key
+      s3_bucket test_bucket
+      path log
+      utc
+      buffer_type memory
+      time_slice_format %Y%m%d-%H
+    ]
+
+    CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD = %[
+      s3_object_key_format "%{path}%{time_slice}_${chunk_id}.%{file_extension}"
+      aws_key_id test_key_id
+      aws_sec_key test_sec_key
+      s3_bucket test_bucket
+      path log
+      utc
+      buffer_type memory
+      time_slice_format %Y%m%d-%H
+    ]
+
+    data("non_objectkey", {"expected_warning_num" => 1, "conf" => CONFIG, "workers" => 1, "with_multi_buffers" => false})
+    data("non_objectkey-multi_buffer", {"expected_warning_num" => 2, "conf" => CONFIG, "workers" => 1, "with_multi_buffers" => true})
+    data("non_objectkey-multi_worker", {"expected_warning_num" => 2, "conf" => CONFIG, "workers" => 2, "with_multi_buffers" => false})
+    data("default_objectkey", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 1, "with_multi_buffers" => false})
+    data("default_objectkey-multi_buffer", {"expected_warning_num" => 1, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 1, "with_multi_buffers" => true})
+    data("default_objectkey-multi_worker", {"expected_warning_num" => 1, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 2, "with_multi_buffers" => false})
+    data("fixed_objectkey", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => false})
+    data("fixed_objectkey-multi_buffer", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => true})
+    data("fixed_objectkey-multi_worker", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 2, "with_multi_buffers" => false})
+    def test_configure_warning_on_parallel(data)
+      conf = data["conf"].clone
+      if data["with_multi_buffers"]
+        conf << "\n<buffer>\n@type memory\nflush_thread_count 2\n</buffer>\n"
+      end
+      assert_rr do
+        d = Fluent::Test::Driver::Output.new(Fluent::Plugin::S3Output, opts: {"workers": data["workers"]})
+        mock(d.instance.log).warn(anything).times(data["expected_warning_num"])
+        d.configure(conf)
+      end
+    end
   end
 
   def test_format
@@ -517,9 +562,9 @@ EOC
 
   def test_assume_role_credentials
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: anything){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: anything }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -540,9 +585,9 @@ EOC
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     sts_client = Aws::STS::Client.new(region: 'ap-northeast-1')
     mock(Aws::STS::Client).new(region: 'ap-northeast-1'){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -565,9 +610,9 @@ EOC
     sts_client = Aws::STS::Client.new(region: 'ap-northeast-1', credentials: expected_credentials)
     mock(Aws::Credentials).new("test_key_id", "test_sec_key") { expected_credentials }
     mock(Aws::STS::Client).new(region: 'ap-northeast-1', credentials: expected_credentials){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client } ){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE
@@ -592,10 +637,10 @@ EOC
     expected_sts_http_proxy = 'http://example.com'
     sts_client = Aws::STS::Client.new(region: expected_region, http_proxy: expected_sts_http_proxy)
     mock(Aws::STS::Client).new(region:expected_region, http_proxy: expected_sts_http_proxy){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client,
-                                         sts_http_proxy: expected_sts_http_proxy){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client,
+                                           sts_http_proxy: expected_sts_http_proxy }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -619,10 +664,10 @@ EOC
     expected_sts_http_proxy = 'http://example.com'
     sts_client = Aws::STS::Client.new(region: "us-east-1", http_proxy: expected_sts_http_proxy)
     mock(Aws::STS::Client).new(region: "us-east-1", http_proxy: expected_sts_http_proxy){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client,
-                                         sts_http_proxy: expected_sts_http_proxy){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client,
+                                           sts_http_proxy: expected_sts_http_proxy }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -645,10 +690,10 @@ EOC
     expected_sts_endpoint_url = 'http://example.com'
     sts_client = Aws::STS::Client.new(region: "us-east-1", endpoint: expected_sts_endpoint_url)
     mock(Aws::STS::Client).new(region: "us-east-1", endpoint: expected_sts_endpoint_url){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client,
-                                         sts_endpoint_url: expected_sts_endpoint_url){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client,
+                                           sts_endpoint_url: expected_sts_endpoint_url }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -671,9 +716,9 @@ EOC
     expected_sts_region = 'ap-south-1'
     sts_client = Aws::STS::Client.new(region: expected_sts_region)
     mock(Aws::STS::Client).new(region: expected_sts_region){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -694,10 +739,12 @@ EOC
   def test_web_identity_credentials
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     mock(Aws::AssumeRoleWebIdentityCredentials).new(
-      role_arn: "test_arn",
-      role_session_name: "test_session",
-      web_identity_token_file: "test_file",
-      client: anything
+      {
+        role_arn: "test_arn",
+        role_session_name: "test_session",
+        web_identity_token_file: "test_file",
+        client: anything
+      }
     ){
       expected_credentials
     }
@@ -722,10 +769,12 @@ EOC
     sts_client = Aws::STS::Client.new(region: 'us-east-1')
     mock(Aws::STS::Client).new(region: 'us-east-1'){ sts_client }
     mock(Aws::AssumeRoleWebIdentityCredentials).new(
-      role_arn: "test_arn",
-      role_session_name: "test_session",
-      web_identity_token_file: "test_file",
-      client: sts_client
+      {
+        role_arn: "test_arn",
+        role_session_name: "test_session",
+        web_identity_token_file: "test_file",
+        client: sts_client
+      }
     ){
       expected_credentials
     }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.7.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-09 00:00:00.000000000 Z
+date: 2022-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -115,13 +115,31 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Amazon S3 output plugin for Fluentd event collector
 email: frsyuki@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/workflows/linux.yml"
+- ".github/workflows/stale-actions.yml"
 - ".gitignore"
 - AUTHORS
 - ChangeLog
@@ -168,7 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector