fluent-plugin-s3 1.7.2 → 1.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1450176cd690d79412b7522ce8e67163c2f3c9c1f87941e57d6b3f3410bdf330
-  data.tar.gz: ee66229b8cd65ef2507feb5229d54322ed2f076e2daefb3f32e3b0e2ba47c634
+  metadata.gz: f4d7487f86806d6cf37414ff6e6654658bc42e93c4608029b5e10aba358558f0
+  data.tar.gz: b93f2ad6a077006a8185485a8d3239d9c9eafb68fcb1eb19bdcb95b94f3da7d0
 SHA512:
-  metadata.gz: f0428c1da0ef734a65d86aec878374636cf4bfd10ed2ab06578d3d8a9cc8ee607e62b4a85c05144599dec2167995dbd836b70cb7cfb66f38fc38d5e82dc1073a
-  data.tar.gz: bb4d7820ff8fa7fa3e58092b69fc78bb4804d92854206fb61c4efa21b415519eff682cf64763a9ec237f30f8997ab20b0b395bb8f1f17a8c88a9c63b3381b02b
+  metadata.gz: 1357ea41f8d7878ce4f71b40ef7bd8b53d8d57216264df2ffa5a433b46132628d48d5b8bbb7ffd52436d1f703a1072ec2e3e99d6a509ae6dd955c57934463534
+  data.tar.gz: ee4c93f4c3423e7e55962980ad8980b819fa3f206096e5ffd463a658946f3ab82079ad3650f7d88775d53471ebdc3b07bc1cc8a7c20123c04bcea5b0abdf0161

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -1,5 +1,6 @@
 name: Bug Report
 description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
 body:
   - type: markdown
     attributes:

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -1,5 +1,6 @@
 name: Feature request
 description: Suggest an idea for this project
+labels: "waiting-for-triage"
 body:
   - type: markdown
     attributes:

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'monthly'

data/.github/workflows/linux.yml

@@ -10,12 +10,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.1', '3.0', '2.7' ]
+        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
@@ -23,6 +23,6 @@ jobs:
       env:
         CI: true
       run: |
-        gem install bundler rake
+        gem install rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -7,16 +7,18 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v3
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        days-before-stale: 90
-        days-before-close: 30
-        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
-        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
-        close-issue-message: "This issue was automatically closed because of stale in 30 days"
-        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        days-before-stale: 30
+        days-before-close: 7
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this issue will be closed in 7 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this PR will be closed in 7 days"
+        close-issue-message: "This issue was automatically closed because of stale in 7 days"
+        close-pr-message: "This PR was automatically closed because of stale in 7 days"
         stale-pr-label: "stale"
         stale-issue-label: "stale"
-        exempt-issue-labels: "bug,help wanted,enhancement"
-        exempt-pr-labels: "bug,help wanted,enhancement"
+        exempt-issue-labels: "waiting-for-triage,bug,help wanted,enhancement"
+        exempt-pr-labels: "waiting-for-triage,bug,help wanted,enhancement"
+        exempt-all-assignees: true
+        exempt-all-milestones: true

data/ChangeLog

@@ -1,3 +1,11 @@
+Release 1.8.1 - 2024/11/15
+
+  * dependency: Change zstd-ruby dependency optional. Install zstd-ruby manually if you want to enable Zstd compression feature.
+
+Release 1.8.0 - 2024/11/06
+
+  * out_s3: Add zstd compression support
+
 Release 1.7.2 - 2022/10/19
 
   * in_s3: Add `event_bridge_mode` parameter

data/VERSION

@@ -1 +1 @@
-1.7.2
+1.8.1

data/docs/credentials.md

@@ -126,7 +126,7 @@ See also:
 
 ### retries
 
-Number of times to retry when retrieving credentials. Default is 5.
+Number of times to retry when retrieving credentials. Default is nil.
 
 ### ip_address
 

data/fluent-plugin-s3.gemspec

@@ -26,4 +26,5 @@ Gem::Specification.new do |gem|
   # aws-sdk-core requires one of ox, oga, libxml, nokogiri or rexml,
   # and rexml is no longer default gem as of Ruby 3.0.
   gem.add_development_dependency "rexml"
+  gem.add_development_dependency "zstd-ruby"
 end

data/lib/fluent/plugin/s3_compressor_zstd.rb

@@ -0,0 +1,36 @@
+module Fluent::Plugin
+  class S3Output
+    class ZstdCompressor < Compressor
+      S3Output.register_compressor('zstd', self)
+
+      config_section :compress, param_name: :compress_config, init: true, multi: false do
+        desc "Compression level for zstd (1-22)"
+        config_param :level, :integer, default: 3
+      end
+
+      def initialize(opts = {})
+        super
+        require 'zstd-ruby'
+      rescue LoadError => e
+        log.error "failed to load zstd-ruby gem. You need to manually install 'zstd-ruby' gem to use 'zstd'.", error: e.message
+        raise Fluent::ConfigError, "failed to load 'zstd-ruby' gem"
+      end
+
+      def ext
+        'zst'.freeze
+      end
+
+      def content_type
+        'application/x-zst'.freeze
+      end
+
+      def compress(chunk, tmp)
+        compressed = Zstd.compress(chunk.read, level: @compress_config.level)
+        tmp.write(compressed)
+      rescue => e
+        log.warn "zstd compression failed: #{e.message}"
+        raise
+      end
+    end
+  end
+end

data/test/test_out_s3.rb

@@ -109,6 +109,18 @@ class S3OutputTest < Test::Unit::TestCase
       assert(e.is_a?(Fluent::ConfigError))
     end
 
+    data('level default' => nil,
+         'level 1' => 1)
+    def test_configure_with_mime_type_zstd(level)
+      conf = CONFIG.clone
+      conf << "\nstore_as zstd\n"
+      conf << "\n<compress>\nlevel #{level}\n</compress>\n" if level
+      d = create_driver(conf)
+      assert_equal 'zst', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'application/x-zst', d.instance.instance_variable_get(:@compressor).content_type
+      assert_equal (level || 3), d.instance.instance_variable_get(:@compressor).instance_variable_get(:@compress_config).level
+    end
+
     def test_configure_with_path_style
       conf = CONFIG.clone
       conf << "\nforce_path_style true\n"
@@ -456,6 +468,33 @@ EOC
     FileUtils.rm_f(s3_local_file_path)
   end
 
+  def test_write_with_zstd
+    setup_mocks(true)
+    s3_local_file_path = "/tmp/s3-test.zst"
+
+    expected_s3path = "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.zst"
+
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: expected_s3path)
+
+    config = CONFIG_TIME_SLICE + "\nstore_as zstd\n"
+    d = create_time_sliced_driver(config)
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, { "a" => 1 })
+      d.feed(time, { "a" => 2 })
+    end
+
+    File.open(s3_local_file_path, 'rb') do |file|
+      compressed_data = file.read
+      uncompressed_data = Zstd.decompress(compressed_data)
+      expected_data = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                      %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+      assert_equal expected_data, uncompressed_data
+    end
+    FileUtils.rm_f(s3_local_file_path)
+  end
+
   class MockResponse
     attr_reader :data
 

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 1.8.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 - Masahiro Nakagawa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-19 00:00:00.000000000 Z
+date: 2024-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -129,6 +129,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zstd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Amazon S3 output plugin for Fluentd event collector
 email: frsyuki@gmail.com
 executables: []
@@ -138,6 +152,7 @@ files:
 - ".github/ISSUE_TEMPLATE/bug_report.yaml"
 - ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/feature_request.yaml"
+- ".github/dependabot.yml"
 - ".github/workflows/linux.yml"
 - ".github/workflows/stale-actions.yml"
 - ".gitignore"
@@ -162,6 +177,7 @@ files:
 - lib/fluent/plugin/s3_compressor_lzma2.rb
 - lib/fluent/plugin/s3_compressor_lzo.rb
 - lib/fluent/plugin/s3_compressor_parquet.rb
+- lib/fluent/plugin/s3_compressor_zstd.rb
 - lib/fluent/plugin/s3_extractor_gzip_command.rb
 - lib/fluent/plugin/s3_extractor_lzma2.rb
 - lib/fluent/plugin/s3_extractor_lzo.rb
@@ -171,7 +187,7 @@ homepage: https://github.com/fluent/fluent-plugin-s3
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -186,8 +202,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key: 
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector
 test_files: []