fluent-plugin-s3 1.3.3 → 1.6.0

data/docs/v0.12.md

@@ -0,0 +1,52 @@
+# Configuration: Output (v0.12 style)
+
+Here is a sample configuration for old fluentd v0.12. It works with fluentd v1 too but not recommended for it.
+
+    <match pattern>
+      @type s3
+
+      aws_key_id YOUR_AWS_KEY_ID
+      aws_sec_key YOUR_AWS_SECRET_KEY
+      s3_bucket YOUR_S3_BUCKET_NAME
+      s3_region ap-northeast-1
+
+      path logs/
+      s3_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
+      buffer_path /var/log/fluent/s3
+      time_slice_format %Y%m%d-%H
+      time_slice_wait 10m
+      utc
+      format json
+    </match>
+
+If you want to embed tag in [`path`](output.md#path) / [`s3_object_key_format`](output.md#s3_object_key_format), you need to use `fluent-plugin-forest` plugin.
+
+The following explanations are about the differences with v1. Other parameters are same with v1, see [Configuration: Output](output.md) for them.
+
+## format (for v0.12)
+
+    @format json
+    include_time_key true
+    time_key log_time # default is time
+
+This parameter is for v0.12. Use [`<format>`](https://docs.fluentd.org/configuration/format-section) and [`<inject>`](https://docs.fluentd.org/configuration/inject-section) for v1.
+
+## buffer_path (for v0.12)
+
+path prefix of the files to buffer logs.
+
+This parameter is for v0.12. Use [`<buffer>`](https://docs.fluentd.org/configuration/buffer-section)'s `path` in v1.
+
+## time_slice_format (for v0.12)
+
+Format of the time used as the file name. Default is '%Y%m%d'. Use
+'%Y%m%d%H' to split files hourly.
+
+This parameter is for v0.12. Use [buffer placeholder](https://docs.fluentd.org/configuration/buffer-section#placeholders) for [`path`](output.md#path) / [`s3_object_key_format`](output.md#s3_object_key_format) in v1.
+
+## time_slice_wait (for v0.12)
+
+The time to wait old logs. Default is 10 minutes. Specify larger value if
+old logs may reach.
+
+This parameter is for v0.12. Use [`<buffer>`](https://docs.fluentd.org/configuration/buffer-section)'s `timekey_wait` in v1.

data/lib/fluent/plugin/in_s3.rb

@@ -192,7 +192,7 @@ module Fluent::Plugin
           end
         end
       rescue => e
-        log.warn("SQS Polling Failed. Retry in #{@sqs.retry_error_interval} seconds")
+        log.warn("SQS Polling Failed. Retry in #{@sqs.retry_error_interval} seconds", error: e)
         sleep(@sqs.retry_error_interval)
         retry
       end

data/lib/fluent/plugin/out_s3.rb

@@ -5,6 +5,7 @@ require 'aws-sdk-s3'
 require 'zlib'
 require 'time'
 require 'tempfile'
+require 'securerandom'
 
 module Fluent::Plugin
   class S3Output < Output
@@ -41,6 +42,10 @@ module Fluent::Plugin
       config_param :external_id, :string, default: nil, secret: true
       desc "The region of the STS endpoint to use."
       config_param :sts_region, :string, default: nil
+      desc "A http proxy url for requests to aws sts service"
+      config_param :sts_http_proxy, :string, default: nil, secret: true
+      desc "A url for a regional sts api endpoint, the default is global"
+      config_param :sts_endpoint_url, :string, default: nil
     end
     # See the following link for additional params that could be added:
     # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
@@ -380,7 +385,7 @@ module Fluent::Plugin
     end
 
     def uuid_random
-      ::UUIDTools::UUID.random_create.to_s
+      SecureRandom.uuid
     end
 
     # This is stolen from Fluentd
@@ -437,17 +442,6 @@ module Fluent::Plugin
       }
 
       if @s3_object_key_format.include?('%{uuid_flush}')
-        # test uuidtools works or not
-        begin
-          require 'uuidtools'
-        rescue LoadError
-          raise Fluent::ConfigError, "uuidtools gem not found. Install uuidtools gem first"
-        end
-        begin
-          uuid_random
-        rescue => e
-          raise Fluent::ConfigError, "Generating uuid doesn't work. Can't use %{uuid_flush} on this environment. #{e}"
-        end
         @uuid_flush_enabled = true
       end
 
@@ -479,22 +473,47 @@ module Fluent::Plugin
       options = {}
       credentials_options = {}
       case
-      when @aws_key_id && @aws_sec_key
-        options[:access_key_id] = @aws_key_id
-        options[:secret_access_key] = @aws_sec_key
       when @assume_role_credentials
         c = @assume_role_credentials
+        iam_user_credentials = @aws_key_id && @aws_sec_key ? Aws::Credentials.new(@aws_key_id, @aws_sec_key) : nil
+        region = c.sts_region || @s3_region
         credentials_options[:role_arn] = c.role_arn
         credentials_options[:role_session_name] = c.role_session_name
         credentials_options[:policy] = c.policy if c.policy
         credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
         credentials_options[:external_id] = c.external_id if c.external_id
-        if c.sts_region
-          credentials_options[:client] = Aws::STS::Client.new(region: c.sts_region)
-        elsif @s3_region
-          credentials_options[:client] = Aws::STS::Client.new(region: @s3_region)
+        credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
+        credentials_options[:sts_http_proxy] = c.sts_http_proxy if c.sts_http_proxy
+        if c.sts_http_proxy && c.sts_endpoint_url
+          credentials_options[:client] = if iam_user_credentials
+                                           Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url, credentials: iam_user_credentials)
+                                         else
+                                           Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
+                                         end
+        elsif c.sts_http_proxy
+          credentials_options[:client] = if iam_user_credentials
+                                           Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, credentials: iam_user_credentials)
+                                         else
+                                           Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy)
+                                         end
+        elsif c.sts_endpoint_url
+          credentials_options[:client] = if iam_user_credentials
+                                           Aws::STS::Client.new(region: region, endpoint: c.sts_endpoint_url, credentials: iam_user_credentials)
+                                         else
+                                           Aws::STS::Client.new(region: region, endpoint: c.sts_endpoint_url)
+                                         end
+        else
+          credentials_options[:client] = if iam_user_credentials
+                                           Aws::STS::Client.new(region: region, credentials: iam_user_credentials)
+                                         else
+                                           Aws::STS::Client.new(region: region)
+                                         end
         end
+
         options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+      when @aws_key_id && @aws_sec_key
+        options[:access_key_id] = @aws_key_id
+        options[:secret_access_key] = @aws_sec_key
       when @web_identity_credentials
         c = @web_identity_credentials
         credentials_options[:role_arn] = c.role_arn

data/lib/fluent/plugin/s3_compressor_parquet.rb

@@ -0,0 +1,83 @@
+require "open3"
+
+module Fluent::Plugin
+  class S3Output
+    class ParquetCompressor < Compressor
+      S3Output.register_compressor("parquet", self)
+
+      config_section :compress, multi: false do
+        desc "parquet compression codec"
+        config_param :parquet_compression_codec, :enum, list: [:uncompressed, :snappy, :gzip, :lzo, :brotli, :lz4, :zstd], default: :snappy
+        desc "parquet file page size"
+        config_param :parquet_page_size, :size, default: 8192
+        desc "parquet file row group size"
+        config_param :parquet_row_group_size, :size, default: 128 * 1024 * 1024
+        desc "record data format type"
+        config_param :record_type, :enum, list: [:avro, :csv, :jsonl, :msgpack, :tsv, :json], default: :msgpack
+        desc "schema type"
+        config_param :schema_type, :enum, list: [:avro, :bigquery], default: :avro
+        desc "path to schema file"
+        config_param :schema_file, :string
+      end
+
+      def configure(conf)
+        super
+        check_command("columnify", "-h")
+
+        if [:lzo, :brotli, :lz4].include?(@compress.parquet_compression_codec)
+          raise Fluent::ConfigError, "unsupported compression codec: #{@compress.parquet_compression_codec}"
+        end
+
+        @parquet_compression_codec = @compress.parquet_compression_codec.to_s.upcase
+        if @compress.record_type == :json
+          @record_type = :jsonl
+        else
+          @record_type = @compress.record_type
+        end
+      end
+
+      def ext
+        "parquet".freeze
+      end
+
+      def content_type
+        "application/octet-stream".freeze
+      end
+
+      def compress(chunk, tmp)
+        chunk_is_file = @buffer_type == "file"
+        path = if chunk_is_file
+                 chunk.path
+               else
+                 w = Tempfile.new("chunk-parquet-tmp")
+                 w.binmode
+                 chunk.write_to(w)
+                 w.close
+                 w.path
+               end
+        stdout, stderr, status = columnify(path, tmp.path)
+        unless status.success?
+          raise Fluent::UnrecoverableError, "failed to execute columnify command. stdout=#{stdout} stderr=#{stderr} status=#{status.inspect}"
+        end
+      ensure
+        unless chunk_is_file
+          w.close(true) rescue nil
+        end
+      end
+
+      private
+
+      def columnify(src_path, dst_path)
+        Open3.capture3("columnify",
+                       "-parquetCompressionCodec", @parquet_compression_codec,
+                       "-parquetPageSize", @compress.parquet_page_size.to_s,
+                       "-parquetRowGroupSize", @compress.parquet_row_group_size.to_s,
+                       "-recordType", @record_type.to_s,
+                       "-schemaType", @compress.schema_type.to_s,
+                       "-schemaFile", @compress.schema_file,
+                       "-output", dst_path,
+                       src_path)
+      end
+    end
+  end
+end

data/test/test_out_s3.rb

@@ -9,7 +9,6 @@ require 'test/unit/rr'
 require 'zlib'
 require 'fileutils'
 require 'timecop'
-require 'uuidtools'
 require 'ostruct'
 
 include Fluent::Test::Helpers
@@ -349,17 +348,11 @@ EOC
 
   def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
 
-    begin
-      require 'uuidtools'
-    rescue LoadError
-      pend("uuidtools not found. skip this test")
-    end
-
     # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
     setup_mocks(true)
 
     uuid = "5755e23f-9b54-42d8-8818-2ea38c6f279e"
-    stub(::UUIDTools::UUID).random_create{ uuid }
+    stub(::SecureRandom).uuid{ uuid }
 
     s3_local_file_path = "/tmp/s3-test.txt"
     s3path = "log/events/ts=20110102-13/events_0-#{uuid}.gz"
@@ -567,6 +560,137 @@ EOC
     assert_equal(expected_credentials, credentials)
   end
 
+  def test_assume_role_with_iam_credentials
+    expected_credentials = Aws::Credentials.new("test_key_id", "test_sec_key")
+    sts_client = Aws::STS::Client.new(region: 'ap-northeast-1', credentials: expected_credentials)
+    mock(Aws::Credentials).new("test_key_id", "test_sec_key") { expected_credentials }
+    mock(Aws::STS::Client).new(region: 'ap-northeast-1', credentials: expected_credentials){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE
+    config += %[
+      s3_region ap-northeast-1
+
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_region_and_sts_http_proxy
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_region = "ap-northeast-1"
+    expected_sts_http_proxy = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: expected_region, http_proxy: expected_sts_http_proxy)
+    mock(Aws::STS::Client).new(region:expected_region, http_proxy: expected_sts_http_proxy){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client,
+                                         sts_http_proxy: expected_sts_http_proxy){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      s3_region #{expected_region}
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_http_proxy #{expected_sts_http_proxy}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_sts_http_proxy
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_http_proxy = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: "us-east-1", http_proxy: expected_sts_http_proxy)
+    mock(Aws::STS::Client).new(region: "us-east-1", http_proxy: expected_sts_http_proxy){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client,
+                                         sts_http_proxy: expected_sts_http_proxy){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_http_proxy #{expected_sts_http_proxy}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_sts_endpoint_url
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_endpoint_url = 'http://example.com'
+    sts_client = Aws::STS::Client.new(region: "us-east-1", endpoint: expected_sts_endpoint_url)
+    mock(Aws::STS::Client).new(region: "us-east-1", endpoint: expected_sts_endpoint_url){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client,
+                                         sts_endpoint_url: expected_sts_endpoint_url){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_endpoint_url #{expected_sts_endpoint_url}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
+  def test_assume_role_credentials_with_sts_region
+    expected_credentials = Aws::Credentials.new("test_key", "test_secret")
+    expected_sts_region = 'ap-south-1'
+    sts_client = Aws::STS::Client.new(region: expected_sts_region)
+    mock(Aws::STS::Client).new(region: expected_sts_region){ sts_client }
+    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
+                                         role_session_name: "test_session",
+                                         client: sts_client){
+      expected_credentials
+    }
+    config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
+    config += %[
+      <assume_role_credentials>
+        role_arn test_arn
+        role_session_name test_session
+        sts_region #{expected_sts_region}
+      </assume_role_credentials>
+    ]
+    d = create_time_sliced_driver(config)
+    assert_nothing_raised { d.run {} }
+    client = d.instance.instance_variable_get(:@s3).client
+    credentials = client.config.credentials
+    assert_equal(expected_credentials, credentials)
+  end
+
   def test_web_identity_credentials
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     mock(Aws::AssumeRoleWebIdentityCredentials).new(

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.6.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-25 00:00:00.000000000 Z
+date: 2021-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -121,8 +121,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
-- ".travis.yml"
 - AUTHORS
 - ChangeLog
 - Gemfile
@@ -131,6 +131,11 @@ files:
 - Rakefile
 - VERSION
 - appveyor.yml
+- docs/credentials.md
+- docs/howto.md
+- docs/input.md
+- docs/output.md
+- docs/v0.12.md
 - fluent-plugin-s3.gemspec
 - lib/fluent/log-ext.rb
 - lib/fluent/plugin/in_s3.rb
@@ -138,6 +143,7 @@ files:
 - lib/fluent/plugin/s3_compressor_gzip_command.rb
 - lib/fluent/plugin/s3_compressor_lzma2.rb
 - lib/fluent/plugin/s3_compressor_lzo.rb
+- lib/fluent/plugin/s3_compressor_parquet.rb
 - lib/fluent/plugin/s3_extractor_gzip_command.rb
 - lib/fluent/plugin/s3_extractor_lzma2.rb
 - lib/fluent/plugin/s3_extractor_lzo.rb
@@ -162,10 +168,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector
-test_files:
-- test/test_in_s3.rb
-- test/test_out_s3.rb
+test_files: []