fluent-plugin-s3 1.7.1 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe9afa0a2351dfffc6ea9afa0c2f3598a29b19a55fbfa22f58f3d17b606a3113
-  data.tar.gz: e5813910a178d16ed61c2a8782e2adeb394d0d256acfbcfdbead05069d789163
+  metadata.gz: 8c72b8fcce805d79141dededa99e19fe802a27b169f4f39b6f842dfc2829daab
+  data.tar.gz: eb62e090ec39010bf6f2598c827d875ae0032dccc14b71607dcd92fa80082d0e
 SHA512:
-  metadata.gz: e3ca35df25975b7b57244c88d3145ab8fb5521054b863e030846a867bae6cbc9a69343f586f01b3a29d233eea787cf0ab540ff77edc0423d583394e29a192a7c
-  data.tar.gz: d46deabb1425e03250c9bc24578ad8dacef71cc445335ac9e4122b11c6ad91cf7a73f92b63d3688848ff35d427cb6371777f0011c94c45a898a6063edc1e3d9c
+  metadata.gz: 53e0911c336861aa4dddb73e9f0f5f58fa78d8d8434a217e55f2532fbb264c7ec7e894cc9f1cfd1abc0452f065954099658420bf435dd5bfe7a8a402ecab0e25
+  data.tar.gz: 3916876756d629413c80a6a24f428ff6a453e284a235d0805853b4b01b7bfb6f7866426342c5a598d00a53fe3ee2daf46ac90f4ba04577f762bc3fbabfef1e7f

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -1,5 +1,6 @@
 name: Bug Report
 description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
 body:
   - type: markdown
     attributes:

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -1,5 +1,6 @@
 name: Feature request
 description: Suggest an idea for this project
+labels: "waiting-for-triage"
 body:
   - type: markdown
     attributes:

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'monthly'

data/.github/workflows/linux.yml

@@ -10,12 +10,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.1', '3.0', '2.7' ]
+        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
@@ -23,6 +23,6 @@ jobs:
       env:
         CI: true
       run: |
-        gem install bundler rake
+        gem install rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -7,16 +7,18 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v3
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        days-before-stale: 90
-        days-before-close: 30
-        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
-        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
-        close-issue-message: "This issue was automatically closed because of stale in 30 days"
-        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        days-before-stale: 30
+        days-before-close: 7
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this issue will be closed in 7 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this PR will be closed in 7 days"
+        close-issue-message: "This issue was automatically closed because of stale in 7 days"
+        close-pr-message: "This PR was automatically closed because of stale in 7 days"
         stale-pr-label: "stale"
         stale-issue-label: "stale"
-        exempt-issue-labels: "bug,help wanted,enhancement"
-        exempt-pr-labels: "bug,help wanted,enhancement"
+        exempt-issue-labels: "waiting-for-triage,bug,help wanted,enhancement"
+        exempt-pr-labels: "waiting-for-triage,bug,help wanted,enhancement"
+        exempt-all-assignees: true
+        exempt-all-milestones: true

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 1.8.0 - 2024/11/06
+
+  * out_s3: Add zstd compression support
+
+Release 1.7.2 - 2022/10/19
+
+  * in_s3: Add `event_bridge_mode` parameter
+  * out_s3: Fix `s3_object_key_format` check to allow `%{hex_random}` as well as `%{uuid_flush}` or `${chunk_id}`
+
 Release 1.7.1 - 2022/07/15
 
   * in_s3: Add `match_regexp` parameter to selectively download S3 files based on the object key

data/VERSION

@@ -1 +1 @@
-1.7.1
+1.8.0

data/docs/credentials.md

@@ -126,7 +126,7 @@ See also:
 
 ### retries
 
-Number of times to retry when retrieving credentials. Default is 5.
+Number of times to retry when retrieving credentials. Default is nil.
 
 ### ip_address
 

data/docs/howto.md

@@ -7,7 +7,7 @@ downstream processors to better identify the source of a given record.
 
 # IAM Policy
 
-The following is an example for a IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket-test, etc.).
+The following is an example for a IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket/test, etc.).
 
     {
       "Version": "2012-10-17",

data/docs/input.md

@@ -101,3 +101,7 @@ The long polling interval. Default is 20.
 ### retry_error_interval
 
 Interval to retry polling SQS if polling unsuccessful, in seconds. Default is 300.
+
+### event_bridge_mode
+When true, Amazon S3 Event Notification should be configured using the EventBridge integration. Default is false.
+See [Configure S3 event notification using EventBridge](https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html) for additional information.

data/fluent-plugin-s3.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "fluentd", [">= 0.14.22", "< 2"]
   gem.add_dependency "aws-sdk-s3", "~> 1.60"
   gem.add_dependency "aws-sdk-sqs", "~> 1.23"
+  gem.add_dependency 'zstd-ruby'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", ">= 1.0.3"

data/lib/fluent/plugin/in_s3.rb

@@ -110,6 +110,8 @@ module Fluent::Plugin
       config_param :wait_time_seconds, :integer, default: 20
       desc "Polling error retry interval."
       config_param :retry_error_interval, :integer, default: 300
+      desc "Event bridge mode"
+      config_param :event_bridge_mode, :bool, default: false
     end
 
     desc "Tag string"
@@ -205,10 +207,9 @@ module Fluent::Plugin
           begin
             body = Yajl.load(message.body)
             log.debug(body)
-            next unless body["Records"] # skip test queue
+            next unless is_valid_queue(body) # skip test queue
             if @match_regexp
-              s3 = body["Records"].first["s3"]
-              raw_key = s3["object"]["key"]
+              raw_key = get_raw_key(body)
               key = CGI.unescape(raw_key)
               next unless @match_regexp.match?(key) 
             end
@@ -226,6 +227,24 @@ module Fluent::Plugin
       end
     end
 
+    def is_valid_queue(body)
+      if @sqs.event_bridge_mode
+        log.debug("checking for eventbridge property")
+        !!body["detail"]
+      else 
+        log.debug("checking for Records property")
+        !!body["Records"]
+      end
+    end
+
+    def get_raw_key(body)
+      if @sqs.event_bridge_mode
+        body["detail"]["object"]["key"]
+      else
+        body["Records"].first["s3"]["object"]["key"]
+      end     
+    end
+
     def setup_credentials
       options = {}
       credentials_options = {}
@@ -318,8 +337,7 @@ module Fluent::Plugin
     end
 
     def process(body)
-      s3 = body["Records"].first["s3"]
-      raw_key = s3["object"]["key"]
+      raw_key = get_raw_key(body)
       key = CGI.unescape(raw_key)
 
       io = @bucket.object(key).get.body

data/lib/fluent/plugin/out_s3.rb

@@ -471,8 +471,8 @@ module Fluent::Plugin
       end
 
       is_working_on_parallel = @buffer_config.flush_thread_count > 1 || system_config.workers > 1
-      if is_working_on_parallel && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
-        log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads or multiple workers. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
+      if is_working_on_parallel && ['${chunk_id}', '%{uuid_flush}', '%{hex_random}'].none? { |key| @s3_object_key_format.include?(key) }
+        log.warn "No ${chunk_id}, %{uuid_flush} or %{hex_random} in s3_object_key_format with multiple flush threads or multiple workers. Recommend to set ${chunk_id}, %{uuid_flush} or %{hex_random} to avoid data lost by object conflict"
       end
     end
 

data/lib/fluent/plugin/s3_compressor_zstd.rb

@@ -0,0 +1,30 @@
+require 'zstd-ruby'
+
+module Fluent::Plugin
+  class S3Output
+    class ZstdCompressor < Compressor
+      S3Output.register_compressor('zstd', self)
+
+      config_section :compress, param_name: :compress_config, init: true, multi: false do
+        desc "Compression level for zstd (1-22)"
+        config_param :level, :integer, default: 3
+      end
+
+      def ext
+        'zst'.freeze
+      end
+
+      def content_type
+        'application/x-zst'.freeze
+      end
+
+      def compress(chunk, tmp)
+        compressed = Zstd.compress(chunk.read, level: @compress_config.level)
+        tmp.write(compressed)
+      rescue => e
+        log.warn "zstd compression failed: #{e.message}"
+        raise
+      end
+    end
+  end
+end

data/test/test_in_s3.rb

@@ -166,9 +166,9 @@ buffer_type memory
   aws_key_id sqs_test_key_id
 </sqs>
 EOS
-        create_driver(conf)
-      }
-    end
+      create_driver(conf)
+    }
+  end
 
   def test_sqs_with_invalid_aws_keys_missing_key_id
     assert_raise(Fluent::ConfigError, "sqs/aws_key_id or sqs/aws_sec_key is missing") {
@@ -677,4 +677,50 @@ EOS
       d.run {}
     end
   end
+
+  def test_event_bridge_mode
+    setup_mocks
+    d = create_driver("
+      aws_key_id test_key_id
+      aws_sec_key test_sec_key
+      s3_bucket test_bucket
+      buffer_type memory
+      check_apikey_on_start false
+      store_as text
+      format none
+      <sqs>
+        event_bridge_mode true
+        queue_name test_queue
+        queue_owner_aws_account_id 123456789123
+      </sqs>
+    ")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa") }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "detail" => {
+        "object" => {
+          "key" => "test_key"
+        }
+      }
+    }
+    
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    assert_equal({ "message" => "aaa" }, events.first[2])
+  end
+
 end

data/test/test_out_s3.rb

@@ -109,6 +109,18 @@ class S3OutputTest < Test::Unit::TestCase
       assert(e.is_a?(Fluent::ConfigError))
     end
 
+    data('level default' => nil,
+         'level 1' => 1)
+    def test_configure_with_mime_type_zstd(level)
+      conf = CONFIG.clone
+      conf << "\nstore_as zstd\n"
+      conf << "\n<compress>\nlevel #{level}\n</compress>\n" if level
+      d = create_driver(conf)
+      assert_equal 'zst', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'application/x-zst', d.instance.instance_variable_get(:@compressor).content_type
+      assert_equal (level || 3), d.instance.instance_variable_get(:@compressor).instance_variable_get(:@compress_config).level
+    end
+
     def test_configure_with_path_style
       conf = CONFIG.clone
       conf << "\nforce_path_style true\n"
@@ -456,6 +468,33 @@ EOC
     FileUtils.rm_f(s3_local_file_path)
   end
 
+  def test_write_with_zstd
+    setup_mocks(true)
+    s3_local_file_path = "/tmp/s3-test.zst"
+
+    expected_s3path = "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.zst"
+
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: expected_s3path)
+
+    config = CONFIG_TIME_SLICE + "\nstore_as zstd\n"
+    d = create_time_sliced_driver(config)
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, { "a" => 1 })
+      d.feed(time, { "a" => 2 })
+    end
+
+    File.open(s3_local_file_path, 'rb') do |file|
+      compressed_data = file.read
+      uncompressed_data = Zstd.decompress(compressed_data)
+      expected_data = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                      %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+      assert_equal expected_data, uncompressed_data
+    end
+    FileUtils.rm_f(s3_local_file_path)
+  end
+
   class MockResponse
     attr_reader :data
 

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.8.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 - Masahiro Nakagawa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-15 00:00:00.000000000 Z
+date: 2024-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -59,6 +59,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.23'
+- !ruby/object:Gem::Dependency
+  name: zstd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -138,6 +152,7 @@ files:
 - ".github/ISSUE_TEMPLATE/bug_report.yaml"
 - ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/feature_request.yaml"
+- ".github/dependabot.yml"
 - ".github/workflows/linux.yml"
 - ".github/workflows/stale-actions.yml"
 - ".gitignore"
@@ -162,6 +177,7 @@ files:
 - lib/fluent/plugin/s3_compressor_lzma2.rb
 - lib/fluent/plugin/s3_compressor_lzo.rb
 - lib/fluent/plugin/s3_compressor_parquet.rb
+- lib/fluent/plugin/s3_compressor_zstd.rb
 - lib/fluent/plugin/s3_extractor_gzip_command.rb
 - lib/fluent/plugin/s3_extractor_lzma2.rb
 - lib/fluent/plugin/s3_extractor_lzo.rb
@@ -171,7 +187,7 @@ homepage: https://github.com/fluent/fluent-plugin-s3
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -186,8 +202,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key: 
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector
 test_files: []