fluent-plugin-s3 1.4.0 → 1.6.1

data/docs/v0.12.md

@@ -0,0 +1,52 @@
+# Configuration: Output (v0.12 style)
+
+Here is a sample configuration for old fluentd v0.12. It works with fluentd v1 too but not recommended for it.
+
+    <match pattern>
+      @type s3
+
+      aws_key_id YOUR_AWS_KEY_ID
+      aws_sec_key YOUR_AWS_SECRET_KEY
+      s3_bucket YOUR_S3_BUCKET_NAME
+      s3_region ap-northeast-1
+
+      path logs/
+      s3_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
+      buffer_path /var/log/fluent/s3
+      time_slice_format %Y%m%d-%H
+      time_slice_wait 10m
+      utc
+      format json
+    </match>
+
+If you want to embed tag in [`path`](output.md#path) / [`s3_object_key_format`](output.md#s3_object_key_format), you need to use `fluent-plugin-forest` plugin.
+
+The following explanations are about the differences with v1. Other parameters are same with v1, see [Configuration: Output](output.md) for them.
+
+## format (for v0.12)
+
+    @format json
+    include_time_key true
+    time_key log_time # default is time
+
+This parameter is for v0.12. Use [`<format>`](https://docs.fluentd.org/configuration/format-section) and [`<inject>`](https://docs.fluentd.org/configuration/inject-section) for v1.
+
+## buffer_path (for v0.12)
+
+path prefix of the files to buffer logs.
+
+This parameter is for v0.12. Use [`<buffer>`](https://docs.fluentd.org/configuration/buffer-section)'s `path` in v1.
+
+## time_slice_format (for v0.12)
+
+Format of the time used as the file name. Default is '%Y%m%d'. Use
+'%Y%m%d%H' to split files hourly.
+
+This parameter is for v0.12. Use [buffer placeholder](https://docs.fluentd.org/configuration/buffer-section#placeholders) for [`path`](output.md#path) / [`s3_object_key_format`](output.md#s3_object_key_format) in v1.
+
+## time_slice_wait (for v0.12)
+
+The time to wait old logs. Default is 10 minutes. Specify larger value if
+old logs may reach.
+
+This parameter is for v0.12. Use [`<buffer>`](https://docs.fluentd.org/configuration/buffer-section)'s `timekey_wait` in v1.

data/lib/fluent/plugin/in_s3.rb

@@ -115,10 +115,15 @@ module Fluent::Plugin
 
     attr_reader :bucket
 
+    def reject_s3_endpoint?
+      @s3_endpoint && !@s3_endpoint.end_with?('vpce.amazonaws.com') &&
+        @s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) }
+    end
+
     def configure(conf)
       super
 
-      if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
+      if reject_s3_endpoint?
         raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
       end
 

data/lib/fluent/plugin/out_s3.rb

@@ -173,6 +173,11 @@ module Fluent::Plugin
 
     MAX_HEX_RANDOM_LENGTH = 16
 
+    def reject_s3_endpoint?
+      @s3_endpoint && !@s3_endpoint.end_with?('vpce.amazonaws.com') &&
+        @s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) }
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :buffer, :formatter, :inject)
 
@@ -180,7 +185,7 @@ module Fluent::Plugin
 
       Aws.use_bundled_cert! if @use_bundled_cert
 
-      if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
+      if reject_s3_endpoint?
         raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
       end
 
@@ -473,11 +478,10 @@ module Fluent::Plugin
       options = {}
       credentials_options = {}
       case
-      when @aws_key_id && @aws_sec_key
-        options[:access_key_id] = @aws_key_id
-        options[:secret_access_key] = @aws_sec_key
       when @assume_role_credentials
         c = @assume_role_credentials
+        iam_user_credentials = @aws_key_id && @aws_sec_key ? Aws::Credentials.new(@aws_key_id, @aws_sec_key) : nil
+        region = c.sts_region || @s3_region
         credentials_options[:role_arn] = c.role_arn
         credentials_options[:role_session_name] = c.role_session_name
         credentials_options[:policy] = c.policy if c.policy
@@ -486,21 +490,35 @@ module Fluent::Plugin
         credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
         credentials_options[:sts_http_proxy] = c.sts_http_proxy if c.sts_http_proxy
         if c.sts_http_proxy && c.sts_endpoint_url
-          credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
-        elsif @region && c.sts_http_proxy
-          credentials_options[:client] = Aws::STS::Client.new(region: @region, http_proxy: c.sts_http_proxy)
-        elsif @region && c.sts_endpoint_url
-          credentials_options[:client] = Aws::STS::Client.new(region: @region, endpoint: c.sts_endpoint_url)
+          credentials_options[:client] = if iam_user_credentials
+                                           Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url, credentials: iam_user_credentials)
+                                         else
+                                           Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
+                                         end
         elsif c.sts_http_proxy
-          credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy)
+          credentials_options[:client] = if iam_user_credentials
+                                           Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, credentials: iam_user_credentials)
+                                         else
+                                           Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy)
+                                         end
         elsif c.sts_endpoint_url
-          credentials_options[:client] = Aws::STS::Client.new(endpoint: c.sts_endpoint_url)
-        elsif c.sts_region
-          credentials_options[:client] = Aws::STS::Client.new(region: c.sts_region)
-        elsif @s3_region
-          credentials_options[:client] = Aws::STS::Client.new(region: @s3_region)
+          credentials_options[:client] = if iam_user_credentials
+                                           Aws::STS::Client.new(region: region, endpoint: c.sts_endpoint_url, credentials: iam_user_credentials)
+                                         else
+                                           Aws::STS::Client.new(region: region, endpoint: c.sts_endpoint_url)
+                                         end
+        else
+          credentials_options[:client] = if iam_user_credentials
+                                           Aws::STS::Client.new(region: region, credentials: iam_user_credentials)
+                                         else
+                                           Aws::STS::Client.new(region: region)
+                                         end
         end
+
         options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+      when @aws_key_id && @aws_sec_key
+        options[:access_key_id] = @aws_key_id
+        options[:secret_access_key] = @aws_sec_key
       when @web_identity_credentials
         c = @web_identity_credentials
         credentials_options[:role_arn] = c.role_arn

data/lib/fluent/plugin/s3_compressor_parquet.rb

@@ -0,0 +1,83 @@
+require "open3"
+
+module Fluent::Plugin
+  class S3Output
+    class ParquetCompressor < Compressor
+      S3Output.register_compressor("parquet", self)
+
+      config_section :compress, multi: false do
+        desc "parquet compression codec"
+        config_param :parquet_compression_codec, :enum, list: [:uncompressed, :snappy, :gzip, :lzo, :brotli, :lz4, :zstd], default: :snappy
+        desc "parquet file page size"
+        config_param :parquet_page_size, :size, default: 8192
+        desc "parquet file row group size"
+        config_param :parquet_row_group_size, :size, default: 128 * 1024 * 1024
+        desc "record data format type"
+        config_param :record_type, :enum, list: [:avro, :csv, :jsonl, :msgpack, :tsv, :json], default: :msgpack
+        desc "schema type"
+        config_param :schema_type, :enum, list: [:avro, :bigquery], default: :avro
+        desc "path to schema file"
+        config_param :schema_file, :string
+      end
+
+      def configure(conf)
+        super
+        check_command("columnify", "-h")
+
+        if [:lzo, :brotli, :lz4].include?(@compress.parquet_compression_codec)
+          raise Fluent::ConfigError, "unsupported compression codec: #{@compress.parquet_compression_codec}"
+        end
+
+        @parquet_compression_codec = @compress.parquet_compression_codec.to_s.upcase
+        if @compress.record_type == :json
+          @record_type = :jsonl
+        else
+          @record_type = @compress.record_type
+        end
+      end
+
+      def ext
+        "parquet".freeze
+      end
+
+      def content_type
+        "application/octet-stream".freeze
+      end
+
+      def compress(chunk, tmp)
+        chunk_is_file = @buffer_type == "file"
+        path = if chunk_is_file
+                 chunk.path
+               else
+                 w = Tempfile.new("chunk-parquet-tmp")
+                 w.binmode
+                 chunk.write_to(w)
+                 w.close
+                 w.path
+               end
+        stdout, stderr, status = columnify(path, tmp.path)
+        unless status.success?
+          raise Fluent::UnrecoverableError, "failed to execute columnify command. stdout=#{stdout} stderr=#{stderr} status=#{status.inspect}"
+        end
+      ensure
+        unless chunk_is_file
+          w.close(true) rescue nil
+        end
+      end
+
+      private
+
+      def columnify(src_path, dst_path)
+        Open3.capture3("columnify",
+                       "-parquetCompressionCodec", @parquet_compression_codec,
+                       "-parquetPageSize", @compress.parquet_page_size.to_s,
+                       "-parquetRowGroupSize", @compress.parquet_row_group_size.to_s,
+                       "-recordType", @record_type.to_s,
+                       "-schemaType", @compress.schema_type.to_s,
+                       "-schemaFile", @compress.schema_file,
+                       "-output", dst_path,
+                       src_path)
+      end
+    end
+  end
+end

data/test/test_in_s3.rb

@@ -115,14 +115,19 @@ class S3InputTest < Test::Unit::TestCase
   end
 
 
-  def test_s3_endpoint_with_valid_endpoint
-    d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
-    assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
+  data('Normal endpoint' => 'riak-cs.example.com',
+       'VPCE endpoint' => 'vpce.amazonaws.com',
+       'FIPS endpoint' => 'fips.xxx.amazonaws.com',
+       'GOV endpoint' => 'gov.xxx.amazonaws.com')
+  def test_s3_endpoint_with_valid_endpoint(endpoint)
+    d = create_driver(CONFIG + "s3_endpoint #{endpoint}")
+    assert_equal endpoint, d.instance.s3_endpoint
   end
 
   data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
        'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
-       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
+       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com',
+       'Invalid VPCE' => 'vpce.xxx.amazonaws.com')
   def test_s3_endpoint_with_invalid_endpoint(endpoint)
     assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
       create_driver(CONFIG + "s3_endpoint #{endpoint}")

data/test/test_out_s3.rb

@@ -560,6 +560,137 @@ EOC
     assert_equal(expected_credentials, credentials)
   end
 
+  def test_assume_role_with_iam_credentials
+    expected_credentials = Aws::Credentials.new("test_key_id", "test_sec_key")
+    sts_client = Aws::STS::Client.new(region: 'ap-northeast-1', credentials: expected_credentials)
+    mock(Aws::Credentials).new("test_key_id", "test_sec_key") { expected_credentials }
+    mock(Aws::STS::Client).new(region: 'ap-northeast-1', credentials: expected_credentials){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE
+    config += %[
+      s3_region ap-northeast-1
+
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_region_and_sts_http_proxy
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_region = "ap-northeast-1"
+    expected_sts_http_proxy = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: expected_region, http_proxy: expected_sts_http_proxy)
+    mock(Aws::STS::Client).new(region:expected_region, http_proxy: expected_sts_http_proxy){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client,
+                                         sts_http_proxy: expected_sts_http_proxy){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      s3_region #{expected_region}
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_http_proxy #{expected_sts_http_proxy}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_sts_http_proxy
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_http_proxy = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: "us-east-1", http_proxy: expected_sts_http_proxy)
+    mock(Aws::STS::Client).new(region: "us-east-1", http_proxy: expected_sts_http_proxy){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client,
+                                         sts_http_proxy: expected_sts_http_proxy){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_http_proxy #{expected_sts_http_proxy}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_sts_endpoint_url
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_endpoint_url = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: "us-east-1", endpoint: expected_sts_endpoint_url)
+    mock(Aws::STS::Client).new(region: "us-east-1", endpoint: expected_sts_endpoint_url){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client,
+                                         sts_endpoint_url: expected_sts_endpoint_url){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_endpoint_url #{expected_sts_endpoint_url}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_sts_region
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_region = 'ap-south-1'
+    sts_client = Aws::STS::Client.new(region: expected_sts_region)
+    mock(Aws::STS::Client).new(region: expected_sts_region){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_region #{expected_sts_region}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
   def test_web_identity_credentials
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     mock(Aws::AssumeRoleWebIdentityCredentials).new(

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.6.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-03 00:00:00.000000000 Z
+date: 2021-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -121,8 +121,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
+- ".github/workflows/linux.yml"
+- ".github/workflows/stale-actions.yml"
 - ".gitignore"
-- ".travis.yml"
 - AUTHORS
 - ChangeLog
 - Gemfile
@@ -131,6 +135,11 @@ files:
 - Rakefile
 - VERSION
 - appveyor.yml
+- docs/credentials.md
+- docs/howto.md
+- docs/input.md
+- docs/output.md
+- docs/v0.12.md
 - fluent-plugin-s3.gemspec
 - lib/fluent/log-ext.rb
 - lib/fluent/plugin/in_s3.rb
@@ -138,6 +147,7 @@ files:
 - lib/fluent/plugin/s3_compressor_gzip_command.rb
 - lib/fluent/plugin/s3_compressor_lzma2.rb
 - lib/fluent/plugin/s3_compressor_lzo.rb
+- lib/fluent/plugin/s3_compressor_parquet.rb
 - lib/fluent/plugin/s3_extractor_gzip_command.rb
 - lib/fluent/plugin/s3_extractor_lzma2.rb
 - lib/fluent/plugin/s3_extractor_lzo.rb
@@ -162,10 +172,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector
-test_files:
-- test/test_in_s3.rb
-- test/test_out_s3.rb
+test_files: []