fluent-plugin-s3 1.1.8 → 1.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 706ac1da3860e8310ea63b3a0acdb3d819f327ed
-  data.tar.gz: a5dc2d7ed59f8af1f1ecbba02cbb6b8b49b9f256
+  metadata.gz: 9946aa9a708d9fe24ef1a4582334bea3683efb1b
+  data.tar.gz: ff8eb6d8aca6f8fbcdc7d8dd20b354419e248150
 SHA512:
-  metadata.gz: de3ef963574a32ea2ccb517e7ea6e6a8e17b8489e2b8809f8ba3a243d695f4d0e14a5e91d238b93fa703c892a487bf0f9a172870ac249cef438b98e5edd707e9
-  data.tar.gz: c2e79fb312a87fddda7c5d8dfef25448c59d3923a89f03c904e93891aa203a13c3f174ba087d3da8edc69b5146cc0ee137b6fd9a1a7b727e68dbe0710a24cde0
+  metadata.gz: aa61687fed5f22827e94fde41098336c3ae3935352e294f70d490ccca5763563c5d1426f3a0a55335ab2b8d5e7a49575bdc762c644b77f57fac73bee4d06dd09
+  data.tar.gz: 43958485397cf76a352352312890c4445c26122c81f0b251c6d42704e60bc28087963b1fce4f0e3f311cc62e92ad7b6fb3c33e1c3dce51f17cc4eff31c4f5bd9

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 1.1.9 - 2019/03/26
+
+  * out_s3: Add enable_transfer_acceleration parameter
+  * Update fluentd dependency to v0.14.22 or later
+
 Release 1.1.8 - 2019/01/28
 
   * in_s3: Restart SQS polling when the error happens

data/README.md

@@ -253,6 +253,10 @@ recommend using `s3_region` instead of `s3_endpoint`.
 endpoint for S3 compatible services. For example, Riak CS based storage or
 something. This option doesn't work on S3, use `s3_region` instead.
 
+**enable_transfer_acceleration**
+
+Enable [S3 Transfer Acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html) for uploads. **IMPORTANT**: For this to work, you must first enable this feature on your destination S3 bucket.
+
 **use_bundled_cert**
 
 For cases where the default SSL certificate is unavailable (e.g. Windows), you can set this option to true in order to use the AWS SDK bundled certificate. Default is false.
@@ -406,29 +410,16 @@ Create S3 bucket if it does not exists. Default is true.
 
 Check mentioned bucket if it exists in AWS or not. Default is true.
 
-When it is false,
-	fluentd will not check aws s3 for the existence of the mentioned bucket. This is the
-	case where bucket will be pre-created before running fluentd.
+When it is false, fluentd will not check aws s3 for the existence of the mentioned bucket.
+This is the case where bucket will be pre-created before running fluentd.
 
 **check_object**
 
 Check object before creation if it exists or not. Default is true.
 
-When it is false,
-	s3_object_key_format will be %{path}%{time_slice}_%{hms_slice}.%{file_extension} by default
-	where, hms_slice will be time-slice in hhmmss format, so that each object will be unique.
-	Example object name, assuming it is created on 2016/16/11 3:30:54 PM
-		20161611_153054.txt (extension can be anything as per user's choice)
-
-**Example when check_bucket=false and check_object=false**
-
-When the mentioned configuration will be made, fluentd will work with the
-minimum IAM poilcy, like:
-				"Statement": [{
-					"Effect": "Allow",
-					"Action": "s3:PutObject",
-					"Resource": ["*"]
-				}]
+When it is false, s3_object_key_format will be %{path}%{time_slice}_%{hms_slice}.%{file_extension} by default where,
+hms_slice will be time-slice in hhmmss format, so that each object will be unique.
+Example object name, assuming it is created on 2016/16/11 3:30:54 PM 20161611_153054.txt (extension can be anything as per user's choice)
 
 **check_apikey_on_start**
 
@@ -656,8 +647,7 @@ Interval to retry polling SQS if polling unsuccessful, in seconds. Default is 30
 
 ## IAM Policy
 
-The following is an example for a minimal IAM policy needed to write to an s3
-bucket (matches my-s3bucket/logs, my-s3bucket-test, etc.).
+The following is an example for a IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket-test, etc.).
 
     {
       "Version": "2012-10-17",
@@ -672,15 +662,18 @@ bucket (matches my-s3bucket/logs, my-s3bucket-test, etc.).
         {
           "Effect": "Allow",
           "Action": [
-            "s3:PutObject"
+            "s3:PutObject",
+            "s3:GetObject"
           ],
           "Resource": "arn:aws:s3:::my-s3bucket/*"
         }
       ]
     }
 
-Note that the bucket must already exist and **auto_create_bucket** has no
-effect in this case.
+Note that the bucket must already exist and **auto_create_bucket** has no effect in this case.
+
+`s3:GetObject` is needed for object check to avoid object overwritten.
+If you set `check_object false`, `s3:GetObject` is not needed.
 
 Refer to the [AWS
 documentation](http://docs.aws.amazon.com/IAM/latest/UserGuide/ExampleIAMPolicies.html) for example policies.
@@ -690,6 +683,19 @@ roles](http://docs.aws.amazon.com/IAM/latest/UserGuide/WorkingWithRoles.html)
 with a properly configured IAM policy are preferred over embedding access keys
 on EC2 instances.
 
+### Example when `check_bucket false` and `check_object false`
+
+When the mentioned configuration will be made, fluentd will work with the
+minimum IAM poilcy, like:
+
+
+    "Statement": [{
+      "Effect": "Allow",
+      "Action": "s3:PutObject",
+      "Resource": ["*"]
+    }]
+
+
 ## Use your (de)compression algorithm
 
 s3 plugin has pluggable compression mechanizm like Fluentd's input / output

data/VERSION

@@ -1 +1 @@
-1.1.8
+1.1.9

data/fluent-plugin-s3.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 0.14.2", "< 2"]
+  gem.add_dependency "fluentd", [">= 0.14.22", "< 2"]
   gem.add_dependency "aws-sdk-s3", "~> 1.0"
   gem.add_dependency "aws-sdk-sqs", "~> 1.0"
   gem.add_development_dependency "rake", ">= 0.9.2"

data/lib/fluent/plugin/out_s3.rb

@@ -68,6 +68,8 @@ module Fluent::Plugin
     config_param :s3_region, :string, default: ENV["AWS_REGION"] || "us-east-1"
     desc "Use 's3_region' instead"
     config_param :s3_endpoint, :string, default: nil
+    desc "If true, S3 Transfer Acceleration will be enabled for uploads. IMPORTANT: You must first enable this feature on your destination S3 bucket"
+    config_param :enable_transfer_acceleration, :bool, default: false
     desc "If false, the certificate of endpoint will not be verified"
     config_param :ssl_verify_peer, :bool, :default => true
     desc "The format of S3 object keys"
@@ -198,6 +200,7 @@ module Fluent::Plugin
       options = setup_credentials
       options[:region] = @s3_region if @s3_region
       options[:endpoint] = @s3_endpoint if @s3_endpoint
+      options[:use_accelerate_endpoint] = @enable_transfer_acceleration
       options[:http_proxy] = @proxy_uri if @proxy_uri
       options[:force_path_style] = @force_path_style
       options[:compute_checksums] = @compute_checksums unless @compute_checksums.nil?
@@ -251,7 +254,7 @@ module Fluent::Plugin
           s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) do |matched_key|
             values_for_s3_object_key_pre.fetch(matched_key, matched_key)
           end
-          s3path = extract_placeholders(s3path, metadata)
+          s3path = extract_placeholders(s3path, chunk)
           s3path = s3path.gsub(%r(%{[^}]+}), values_for_s3_object_key_post)
           if (i > 0) && (s3path == previous_path)
             if @overwrite
@@ -289,7 +292,7 @@ module Fluent::Plugin
         s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) do |matched_key|
           values_for_s3_object_key_pre.fetch(matched_key, matched_key)
         end
-        s3path = extract_placeholders(s3path, metadata)
+        s3path = extract_placeholders(s3path, chunk)
         s3path = s3path.gsub(%r(%{[^}]+}), values_for_s3_object_key_post)
       end
 
@@ -319,7 +322,7 @@ module Fluent::Plugin
         if @s3_metadata
           put_options[:metadata] = {}
           @s3_metadata.each do |k, v|
-            put_options[:metadata][k] = extract_placeholders(v, metadata).gsub(%r(%{[^}]+}), {"%{index}" => sprintf(@index_format, i - 1)})
+            put_options[:metadata][k] = extract_placeholders(v, chunk).gsub(%r(%{[^}]+}), {"%{index}" => sprintf(@index_format, i - 1)})
           end
         end
         @bucket.object(s3path).put(put_options)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.1.8
+  version: 1.1.9
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-28 00:00:00.000000000 Z
+date: 2019-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -17,7 +17,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.2
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -27,7 +27,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.2
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'