fluent-plugin-s3 1.8.0 → 1.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c72b8fcce805d79141dededa99e19fe802a27b169f4f39b6f842dfc2829daab
-  data.tar.gz: eb62e090ec39010bf6f2598c827d875ae0032dccc14b71607dcd92fa80082d0e
+  metadata.gz: 217eb830b5223b315c3315bfe2eaa0e9ad177ad8b4f9fc90b0575ac6bd30dcfc
+  data.tar.gz: 77895d2eca5be5449db57d5a8d69e87b7b1e3da8cbee5bdffe2f0afa56d4fcb6
 SHA512:
-  metadata.gz: 53e0911c336861aa4dddb73e9f0f5f58fa78d8d8434a217e55f2532fbb264c7ec7e894cc9f1cfd1abc0452f065954099658420bf435dd5bfe7a8a402ecab0e25
-  data.tar.gz: 3916876756d629413c80a6a24f428ff6a453e284a235d0805853b4b01b7bfb6f7866426342c5a598d00a53fe3ee2daf46ac90f4ba04577f762bc3fbabfef1e7f
+  metadata.gz: 04557ccde6e3d9f3a2d3ab4a5cc8c87b7d1dedddc09baa858239be4d59d071b5b4afb43f5cf685c936d09a13208d916683fc298beb89f168f44fdad30a456f8a
+  data.tar.gz: bcc7742d25f8c7f2108ad77753fb5b864f8fb0d644ad0d832a6fe26143b5b938bd03436695c43181b7b66d0f538421454f87e87fa73aa3c4907fd86979aa6b01

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 1.8.2 - 2024/12/18
+
+  * out_s3: Add more logging to identify unexpected error of Tempfile#close.
+  * out_s3: Support `checksum_algorithm` parameter to validate the data with checksum (CRC32, CRC32C, SHA1 and SHA256) during upload/download.
+
+Release 1.8.1 - 2024/11/15
+
+  * dependency: Change zstd-ruby dependency optional. Install zstd-ruby manually if you want to enable Zstd compression feature.
+
 Release 1.8.0 - 2024/11/06
 
   * out_s3: Add zstd compression support

data/VERSION

@@ -1 +1 @@
-1.8.0
+1.8.2

data/docs/output.md

@@ -430,6 +430,18 @@ Specifies the AWS KMS key ID to use for object encryption.
 
 Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.
 
+## checksum_algorithm
+
+AWS allows to calculate the integrity checksum server side. The additional checksum is
+used to validate the data during upload or download. The following 4 SHA and CRC algorithms are supported:
+
+* CRC32
+* CRC32C
+* SHA1
+* SHA256
+
+For more info refer to [object integrity](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html).
+
 ## compute_checksums
 
 AWS SDK uses MD5 for API request/response by default. On FIPS enabled environment,

data/fluent-plugin-s3.gemspec

@@ -19,7 +19,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency "fluentd", [">= 0.14.22", "< 2"]
   gem.add_dependency "aws-sdk-s3", "~> 1.60"
   gem.add_dependency "aws-sdk-sqs", "~> 1.23"
-  gem.add_dependency 'zstd-ruby'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
@@ -27,4 +26,5 @@ Gem::Specification.new do |gem|
   # aws-sdk-core requires one of ox, oga, libxml, nokogiri or rexml,
   # and rexml is no longer default gem as of Ruby 3.0.
   gem.add_development_dependency "rexml"
+  gem.add_development_dependency "zstd-ruby"
 end

data/lib/fluent/plugin/out_s3.rb

@@ -154,6 +154,8 @@ module Fluent::Plugin
     desc "Arbitrary S3 tag-set for the object"
     config_param :tagging, :string, default: nil
     desc "Arbitrary S3 metadata headers to set for the object"
+    config_param :checksum_algorithm, :string, default: nil
+    desc "Indicates the algorithm you want Amazon S3 to use to create the checksum for the object (CRC32,CRC32C,SHA1,SHA256)"
     config_param :s3_metadata, :hash, default: nil
     config_section :bucket_lifecycle_rule, param_name: :bucket_lifecycle_rules, multi: true do
       desc "A unique ID for this rule"
@@ -368,6 +370,7 @@ module Fluent::Plugin
         put_options[:grant_read] = @grant_read if @grant_read
         put_options[:grant_read_acp] = @grant_read_acp if @grant_read_acp
         put_options[:grant_write_acp] = @grant_write_acp if @grant_write_acp
+        put_options[:checksum_algorithm] = @checksum_algorithm if @checksum_algorithm
         put_options[:tagging] = @tagging if @tagging
 
         if @s3_metadata
@@ -378,6 +381,8 @@ module Fluent::Plugin
         end
         @bucket.object(s3path).put(put_options)
 
+        log.debug "out_s3: completed to write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to s3://#{@s3_bucket}/#{s3path}"
+
         @values_for_s3_object_chunk.delete(chunk.unique_id)
 
         if @warn_for_delay
@@ -386,7 +391,11 @@ module Fluent::Plugin
           end
         end
       ensure
-        tmp.close(true) rescue nil
+        begin
+          tmp.close(true)
+        rescue => e
+          log.info "out_s3: Tempfile#close caused unexpected error", error: e
+        end
       end
     end
 

data/lib/fluent/plugin/s3_compressor_zstd.rb

@@ -1,5 +1,3 @@
-require 'zstd-ruby'
-
 module Fluent::Plugin
   class S3Output
     class ZstdCompressor < Compressor
@@ -10,6 +8,14 @@ module Fluent::Plugin
         config_param :level, :integer, default: 3
       end
 
+      def initialize(opts = {})
+        super
+        require 'zstd-ruby'
+      rescue LoadError => e
+        log.error "failed to load zstd-ruby gem. You need to manually install 'zstd-ruby' gem to use 'zstd'.", error: e.message
+        raise Fluent::ConfigError, "failed to load 'zstd-ruby' gem"
+      end
+
       def ext
         'zst'.freeze
       end
@@ -27,4 +33,4 @@ module Fluent::Plugin
       end
     end
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.8.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-13 00:00:00.000000000 Z
+date: 2024-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -59,20 +59,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.23'
-- !ruby/object:Gem::Dependency
-  name: zstd-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -143,6 +129,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: zstd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Amazon S3 output plugin for Fluentd event collector
 email: frsyuki@gmail.com
 executables: []