fluent-plugin-s3 1.6.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 155f0d28d700caf2e5c817843abec96a26f301404d2bf521c6cd383068fdeb47
-  data.tar.gz: '0701182695128c6812b28179685658a8b570332a2f747146e157e859d61d0898'
+  metadata.gz: fbb79ec06cc2a084f5043825a68b9e15b3d3eed9008f0906ff6fedede8659a22
+  data.tar.gz: '09eb263937f9f5dd397296579426a744f701eee6ba45d4a2b05b0f5c7990fe5b'
 SHA512:
-  metadata.gz: 47460a080d212895f1a121da9dc547b08281df9c9ae1c2644dce582c63cad4b971aeefcf287553bf8f983dd4b2e628acfd6119b2dd3b725e5d0125e58fe5c043
-  data.tar.gz: a10f906a8db7c81f80dad1257a1aaf38a9a432ede35f12df6e21df2e138fd6662668192c6382cba9b647dd4e8427c9697f8b4188acef5280fd0278f24ef87b26
+  metadata.gz: 5a0685a1d250f12608bf432c4157d6a1b9cd7a627161cf2278c703ae07cce9fe858d5468bbe5056d060f40914570ea9ec39bbec04828c1dbf193737edf72193a
+  data.tar.gz: 0caf64221c557536e5e3d1d6f21f28cfdce5fc865a4a4d5b55ec490fd716701675b49999faf81b9e030348abbd68e265df690154f2aac155613efbfcf4ba144d

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,72 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README](https://github.com/fluent/fluent-plugin-s3#readme) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - fluent-plugin-s3 version:
+        - aws-sdk-s3 version:
+        - aws-sdk-sqs version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://discuss.fluentd.org/
+    about: I have questions about fluent-plugin-kafka. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README.md](https://github.com/fluent/fluent-plugin-s3/blob/master/README.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/workflows/stale-actions.yml

@@ -0,0 +1,22 @@
+name: "Mark or close stale issues and PRs"
+on:
+  schedule:
+  - cron: "00 10 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        days-before-stale: 90
+        days-before-close: 30
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
+        close-issue-message: "This issue was automatically closed because of stale in 30 days"
+        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        stale-pr-label: "stale"
+        stale-issue-label: "stale"
+        exempt-issue-labels: "bug,help wanted"
+        exempt-pr-labels: "bug,help wanted"

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 1.6.1 - 2021/08/19
+
+  * in_s3/out_s3: Don't raise error when s3_endpoint is used for VPC endpoint (GitHub: #384)
+
 Release 1.6.0 - 2021/04/08
 
   * out_s3: Add support for Parquet compressor. Use `<compress>` section to configure columnify command behavior.

data/README.md

@@ -20,6 +20,8 @@ the former one is stored in "20110102.gz" file, and latter one in
 SQS queue on the region same as S3 bucket.
 We must setup SQS queue and S3 event notification before use this plugin.
 
+:warning: Be sure to keep a close eye on S3 costs, as a few user have reported [unexpectedly high costs](https://github.com/fluent/fluent-plugin-s3/issues/160).
+
 ## Requirements
 
 | fluent-plugin-s3  | fluentd | ruby |

data/VERSION

@@ -1 +1 @@
-1.6.0
+1.6.1

data/docs/output.md

@@ -152,6 +152,8 @@ NOTE: ${hostname} placeholder is deprecated since v0.8. You can get same result
 
 Above two configurations are same. The important point is wrapping `""` is needed for `#{Socket.gethostname}`.
 
+NOTE: If `check_object` is set to `false`, Ensure the value of `s3_object_key_format` must be unique in each write, If not, existing file will be overwritten.
+
 ## force_path_style
 
 :force_path_style (Boolean) — default: false — When set to true, the
@@ -322,6 +324,8 @@ Use UTC instead of local time.
 
 Set storage class. Possible values are `STANDARD`, `REDUCED_REDUNDANCY`, `STANDARD_IA` from [Ruby SDK](http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#storage_class-instance_method).
 
+Note that reduced redundancy is [not reccomended](https://serverfault.com/a/1010951/512362).
+
 ## reduced_redundancy
 
 Use S3 reduced redundancy storage for 33% cheaper pricing. Default is

data/lib/fluent/plugin/in_s3.rb

@@ -115,10 +115,15 @@ module Fluent::Plugin
 
     attr_reader :bucket
 
+    def reject_s3_endpoint?
+      @s3_endpoint && !@s3_endpoint.end_with?('vpce.amazonaws.com') &&
+        @s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) }
+    end
+
     def configure(conf)
       super
 
-      if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
+      if reject_s3_endpoint?
         raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
       end
 

data/lib/fluent/plugin/out_s3.rb

@@ -173,6 +173,11 @@ module Fluent::Plugin
 
     MAX_HEX_RANDOM_LENGTH = 16
 
+    def reject_s3_endpoint?
+      @s3_endpoint && !@s3_endpoint.end_with?('vpce.amazonaws.com') &&
+        @s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) }
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :buffer, :formatter, :inject)
 
@@ -180,7 +185,7 @@ module Fluent::Plugin
 
       Aws.use_bundled_cert! if @use_bundled_cert
 
-      if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
+      if reject_s3_endpoint?
         raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
       end
 

data/test/test_in_s3.rb

@@ -115,14 +115,19 @@ class S3InputTest < Test::Unit::TestCase
   end
 
 
-  def test_s3_endpoint_with_valid_endpoint
-    d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
-    assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
+  data('Normal endpoint' => 'riak-cs.example.com',
+       'VPCE endpoint' => 'vpce.amazonaws.com',
+       'FIPS endpoint' => 'fips.xxx.amazonaws.com',
+       'GOV endpoint' => 'gov.xxx.amazonaws.com')
+  def test_s3_endpoint_with_valid_endpoint(endpoint)
+    d = create_driver(CONFIG + "s3_endpoint #{endpoint}")
+    assert_equal endpoint, d.instance.s3_endpoint
   end
 
   data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
        'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
-       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
+       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com',
+       'Invalid VPCE' => 'vpce.xxx.amazonaws.com')
   def test_s3_endpoint_with_invalid_endpoint(endpoint)
     assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
       create_driver(CONFIG + "s3_endpoint #{endpoint}")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-09 00:00:00.000000000 Z
+date: 2021-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -121,7 +121,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/workflows/linux.yml"
+- ".github/workflows/stale-actions.yml"
 - ".gitignore"
 - AUTHORS
 - ChangeLog
@@ -168,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector