fluent-plugin-s3 1.5.1 → 1.7.0

data/lib/fluent/plugin/s3_compressor_parquet.rb

@@ -0,0 +1,83 @@
+require "open3"
+
+module Fluent::Plugin
+  class S3Output
+    class ParquetCompressor < Compressor
+      S3Output.register_compressor("parquet", self)
+
+      config_section :compress, multi: false do
+        desc "parquet compression codec"
+        config_param :parquet_compression_codec, :enum, list: [:uncompressed, :snappy, :gzip, :lzo, :brotli, :lz4, :zstd], default: :snappy
+        desc "parquet file page size"
+        config_param :parquet_page_size, :size, default: 8192
+        desc "parquet file row group size"
+        config_param :parquet_row_group_size, :size, default: 128 * 1024 * 1024
+        desc "record data format type"
+        config_param :record_type, :enum, list: [:avro, :csv, :jsonl, :msgpack, :tsv, :json], default: :msgpack
+        desc "schema type"
+        config_param :schema_type, :enum, list: [:avro, :bigquery], default: :avro
+        desc "path to schema file"
+        config_param :schema_file, :string
+      end
+
+      def configure(conf)
+        super
+        check_command("columnify", "-h")
+
+        if [:lzo, :brotli, :lz4].include?(@compress.parquet_compression_codec)
+          raise Fluent::ConfigError, "unsupported compression codec: #{@compress.parquet_compression_codec}"
+        end
+
+        @parquet_compression_codec = @compress.parquet_compression_codec.to_s.upcase
+        if @compress.record_type == :json
+          @record_type = :jsonl
+        else
+          @record_type = @compress.record_type
+        end
+      end
+
+      def ext
+        "parquet".freeze
+      end
+
+      def content_type
+        "application/octet-stream".freeze
+      end
+
+      def compress(chunk, tmp)
+        chunk_is_file = @buffer_type == "file"
+        path = if chunk_is_file
+                 chunk.path
+               else
+                 w = Tempfile.new("chunk-parquet-tmp")
+                 w.binmode
+                 chunk.write_to(w)
+                 w.close
+                 w.path
+               end
+        stdout, stderr, status = columnify(path, tmp.path)
+        unless status.success?
+          raise Fluent::UnrecoverableError, "failed to execute columnify command. stdout=#{stdout} stderr=#{stderr} status=#{status.inspect}"
+        end
+      ensure
+        unless chunk_is_file
+          w.close(true) rescue nil
+        end
+      end
+
+      private
+
+      def columnify(src_path, dst_path)
+        Open3.capture3("columnify",
+                       "-parquetCompressionCodec", @parquet_compression_codec,
+                       "-parquetPageSize", @compress.parquet_page_size.to_s,
+                       "-parquetRowGroupSize", @compress.parquet_row_group_size.to_s,
+                       "-recordType", @record_type.to_s,
+                       "-schemaType", @compress.schema_type.to_s,
+                       "-schemaFile", @compress.schema_file,
+                       "-output", dst_path,
+                       src_path)
+      end
+    end
+  end
+end

data/test/test_in_s3.rb

@@ -84,7 +84,7 @@ class S3InputTest < Test::Unit::TestCase
 
     def test_unknown_store_as
       config = CONFIG + "\nstore_as unknown"
-      assert_raise(Fluent::ConfigError) do
+      assert_raise(Fluent::NotFoundPluginError) do
         create_driver(config)
       end
     end
@@ -115,14 +115,19 @@ class S3InputTest < Test::Unit::TestCase
   end
 
 
-  def test_s3_endpoint_with_valid_endpoint
-    d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
-    assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
+  data('Normal endpoint' => 'riak-cs.example.com',
+       'VPCE endpoint' => 'vpce.amazonaws.com',
+       'FIPS endpoint' => 'fips.xxx.amazonaws.com',
+       'GOV endpoint' => 'gov.xxx.amazonaws.com')
+  def test_s3_endpoint_with_valid_endpoint(endpoint)
+    d = create_driver(CONFIG + "s3_endpoint #{endpoint}")
+    assert_equal endpoint, d.instance.s3_endpoint
   end
 
   data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
        'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
-       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
+       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com',
+       'Invalid VPCE' => 'vpce.xxx.amazonaws.com')
   def test_s3_endpoint_with_invalid_endpoint(endpoint)
     assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
       create_driver(CONFIG + "s3_endpoint #{endpoint}")
@@ -148,6 +153,104 @@ EOS
     }
   end
 
+  def test_sqs_with_invalid_keys_missing_secret_key
+    assert_raise(Fluent::ConfigError, "sqs/aws_key_id or sqs/aws_sec_key is missing") {
+      conf = <<"EOS"
+aws_key_id test_key_id
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+  aws_key_id sqs_test_key_id
+</sqs>
+EOS
+        create_driver(conf)
+      }
+    end
+
+  def test_sqs_with_invalid_aws_keys_missing_key_id
+    assert_raise(Fluent::ConfigError, "sqs/aws_key_id or sqs/aws_sec_key is missing") {
+      conf = <<"EOS"
+aws_key_id test_key_id
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+  aws_sec_key sqs_test_sec_key
+</sqs>
+EOS
+      create_driver(conf)
+    }
+  end
+
+  def test_sqs_with_valid_aws_keys_complete_pair
+    conf = <<"EOS"
+aws_key_id test_key_id
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+  aws_key_id sqs_test_key_id
+  aws_sec_key sqs_test_sec_key
+</sqs>
+EOS
+    d = create_driver(conf)
+    assert_equal 'sqs_test_key_id', d.instance.sqs.aws_key_id
+    assert_equal 'sqs_test_sec_key', d.instance.sqs.aws_sec_key
+  end
+
+  def test_with_invalid_aws_keys_missing_secret_key
+    assert_raise(Fluent::ConfigError, "aws_key_id or aws_sec_key is missing") {
+      conf = <<"EOS"
+aws_key_id test_key_id
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+</sqs>
+EOS
+      create_driver(conf)
+    }
+  end
+
+  def test_with_invalid_aws_keys_missing_key_id
+    assert_raise(Fluent::ConfigError, "aws_key_id or aws_sec_key is missing") {
+      conf = <<"EOS"
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+</sqs>
+EOS
+      create_driver(conf)
+    }
+  end
+
+  def test_with_valid_aws_keys_complete_pair
+    conf = <<"EOS"
+aws_key_id test_key_id
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint eu-west-1
+</sqs>
+EOS
+    d = create_driver(conf)
+    assert_equal 'test_key_id', d.instance.aws_key_id
+    assert_equal 'test_sec_key', d.instance.aws_sec_key
+  end
+
   Struct.new("StubResponse", :queue_url)
   Struct.new("StubMessage", :message_id, :receipt_handle, :body)
 

data/test/test_out_s3.rb

@@ -52,102 +52,147 @@ class S3OutputTest < Test::Unit::TestCase
     end.configure(conf)
   end
 
-  def test_configure
-    d = create_driver
-    assert_equal 'test_key_id', d.instance.aws_key_id
-    assert_equal 'test_sec_key', d.instance.aws_sec_key
-    assert_equal 'test_bucket', d.instance.s3_bucket
-    assert_equal 'log', d.instance.path
-    assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
-    assert_equal false, d.instance.force_path_style
-    assert_equal nil, d.instance.compute_checksums
-    assert_equal nil, d.instance.signature_version
-    assert_equal true, d.instance.check_bucket
-    assert_equal true, d.instance.check_object
-  end
-
-  def test_s3_endpoint_with_valid_endpoint
-    d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
-    assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
-  end
-
-  data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
-       'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
-       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
-  def test_s3_endpoint_with_invalid_endpoint(endpoint)
-    assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
-      create_driver(CONFIG + "s3_endpoint #{endpoint}")
-    }
-  end
+  sub_test_case "configure" do
+    def test_configure
+      d = create_driver
+      assert_equal 'test_key_id', d.instance.aws_key_id
+      assert_equal 'test_sec_key', d.instance.aws_sec_key
+      assert_equal 'test_bucket', d.instance.s3_bucket
+      assert_equal 'log', d.instance.path
+      assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
+      assert_equal false, d.instance.force_path_style
+      assert_equal nil, d.instance.compute_checksums
+      assert_equal nil, d.instance.signature_version
+      assert_equal true, d.instance.check_bucket
+      assert_equal true, d.instance.check_object
+    end
 
-  def test_configure_with_mime_type_json
-    conf = CONFIG.clone
-    conf << "\nstore_as json\n"
-    d = create_driver(conf)
-    assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
-  end
+    def test_s3_endpoint_with_valid_endpoint
+      d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
+      assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
+    end
 
-  def test_configure_with_mime_type_text
-    conf = CONFIG.clone
-    conf << "\nstore_as text\n"
-    d = create_driver(conf)
-    assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
-  end
+    data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
+         'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
+         'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
+    def test_s3_endpoint_with_invalid_endpoint(endpoint)
+      assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
+        create_driver(CONFIG + "s3_endpoint #{endpoint}")
+      }
+    end
 
-  def test_configure_with_mime_type_lzo
-    conf = CONFIG.clone
-    conf << "\nstore_as lzo\n"
-    d = create_driver(conf)
-    assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
-  rescue => e
-    # TODO: replace code with disable lzop command
-    assert(e.is_a?(Fluent::ConfigError))
-  end
+    def test_configure_with_mime_type_json
+      conf = CONFIG.clone
+      conf << "\nstore_as json\n"
+      d = create_driver(conf)
+      assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
+    end
 
-  def test_configure_with_path_style
-    conf = CONFIG.clone
-    conf << "\nforce_path_style true\n"
-    d = create_driver(conf)
-    assert d.instance.force_path_style
-  end
+    def test_configure_with_mime_type_text
+      conf = CONFIG.clone
+      conf << "\nstore_as text\n"
+      d = create_driver(conf)
+      assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
+    end
 
-  def test_configure_with_compute_checksums
-    conf = CONFIG.clone
-    conf << "\ncompute_checksums false\n"
-    d = create_driver(conf)
-    assert_equal false, d.instance.compute_checksums
-  end
+    def test_configure_with_mime_type_lzo
+      conf = CONFIG.clone
+      conf << "\nstore_as lzo\n"
+      d = create_driver(conf)
+      assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
+      assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
+    rescue => e
+      # TODO: replace code with disable lzop command
+      assert(e.is_a?(Fluent::ConfigError))
+    end
 
-  def test_configure_with_hex_random_length
-    conf = CONFIG.clone
-    assert_raise Fluent::ConfigError do
-      create_driver(conf + "\nhex_random_length 17\n")
+    def test_configure_with_path_style
+      conf = CONFIG.clone
+      conf << "\nforce_path_style true\n"
+      d = create_driver(conf)
+      assert d.instance.force_path_style
     end
-    assert_nothing_raised do
-      create_driver(conf + "\nhex_random_length 16\n")
+
+    def test_configure_with_compute_checksums
+      conf = CONFIG.clone
+      conf << "\ncompute_checksums false\n"
+      d = create_driver(conf)
+      assert_equal false, d.instance.compute_checksums
     end
-  end
 
-  def test_configure_with_no_check_on_s3
-    conf = CONFIG.clone
-    conf << "\ncheck_bucket false\ncheck_object false\n"
-    d = create_driver(conf)
-    assert_equal false, d.instance.check_bucket
-    assert_equal false, d.instance.check_object
-  end
+    def test_configure_with_hex_random_length
+      conf = CONFIG.clone
+      assert_raise Fluent::ConfigError do
+        create_driver(conf + "\nhex_random_length 17\n")
+      end
+      assert_nothing_raised do
+        create_driver(conf + "\nhex_random_length 16\n")
+      end
+    end
+
+    def test_configure_with_no_check_on_s3
+      conf = CONFIG.clone
+      conf << "\ncheck_bucket false\ncheck_object false\n"
+      d = create_driver(conf)
+      assert_equal false, d.instance.check_bucket
+      assert_equal false, d.instance.check_object
+    end
+
+    def test_configure_with_grant
+      conf = CONFIG.clone
+      conf << "\grant_full_control id='0123456789'\ngrant_read id='1234567890'\ngrant_read_acp id='2345678901'\ngrant_write_acp id='3456789012'\n"
+      d = create_driver(conf)
+      assert_equal "id='0123456789'", d.instance.grant_full_control
+      assert_equal "id='1234567890'", d.instance.grant_read
+      assert_equal "id='2345678901'", d.instance.grant_read_acp
+      assert_equal "id='3456789012'", d.instance.grant_write_acp
+    end
 
-  def test_configure_with_grant
-    conf = CONFIG.clone
-    conf << "\grant_full_control id='0123456789'\ngrant_read id='1234567890'\ngrant_read_acp id='2345678901'\ngrant_write_acp id='3456789012'\n"
-    d = create_driver(conf)
-    assert_equal "id='0123456789'", d.instance.grant_full_control
-    assert_equal "id='1234567890'", d.instance.grant_read
-    assert_equal "id='2345678901'", d.instance.grant_read_acp
-    assert_equal "id='3456789012'", d.instance.grant_write_acp
+    CONFIG_WITH_OBJECTKEY_DEFAULT = %[
+      s3_object_key_format "%{path}%{time_slice}_%{index}.%{file_extension}"
+      aws_key_id test_key_id
+      aws_sec_key test_sec_key
+      s3_bucket test_bucket
+      path log
+      utc
+      buffer_type memory
+      time_slice_format %Y%m%d-%H
+    ]
+
+    CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD = %[
+      s3_object_key_format "%{path}%{time_slice}_${chunk_id}.%{file_extension}"
+      aws_key_id test_key_id
+      aws_sec_key test_sec_key
+      s3_bucket test_bucket
+      path log
+      utc
+      buffer_type memory
+      time_slice_format %Y%m%d-%H
+    ]
+
+    data("non_objectkey", {"expected_warning_num" => 1, "conf" => CONFIG, "workers" => 1, "with_multi_buffers" => false})
+    data("non_objectkey-multi_buffer", {"expected_warning_num" => 2, "conf" => CONFIG, "workers" => 1, "with_multi_buffers" => true})
+    data("non_objectkey-multi_worker", {"expected_warning_num" => 2, "conf" => CONFIG, "workers" => 2, "with_multi_buffers" => false})
+    data("default_objectkey", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 1, "with_multi_buffers" => false})
+    data("default_objectkey-multi_buffer", {"expected_warning_num" => 1, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 1, "with_multi_buffers" => true})
+    data("default_objectkey-multi_worker", {"expected_warning_num" => 1, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 2, "with_multi_buffers" => false})
+    data("fixed_objectkey", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => false})
+    data("fixed_objectkey-multi_buffer", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => true})
+    data("fixed_objectkey-multi_worker", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 2, "with_multi_buffers" => false})
+    def test_configure_warning_on_parallel(data)
+      conf = data["conf"].clone
+      if data["with_multi_buffers"]
+        conf << "\n<buffer>\n@type memory\nflush_thread_count 2\n</buffer>\n"
+      end
+      assert_rr do
+        d = Fluent::Test::Driver::Output.new(Fluent::Plugin::S3Output, opts: {"workers": data["workers"]})
+        mock(d.instance.log).warn(anything).times(data["expected_warning_num"])
+        d.configure(conf)
+      end
+    end
   end
 
   def test_format
@@ -517,9 +562,9 @@ EOC
 
   def test_assume_role_credentials
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: anything){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: anything }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -540,9 +585,9 @@ EOC
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     sts_client = Aws::STS::Client.new(region: 'ap-northeast-1')
     mock(Aws::STS::Client).new(region: 'ap-northeast-1'){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -565,9 +610,9 @@ EOC
     sts_client = Aws::STS::Client.new(region: 'ap-northeast-1', credentials: expected_credentials)
     mock(Aws::Credentials).new("test_key_id", "test_sec_key") { expected_credentials }
     mock(Aws::STS::Client).new(region: 'ap-northeast-1', credentials: expected_credentials){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client } ){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE
@@ -592,10 +637,10 @@ EOC
     expected_sts_http_proxy = 'http://example.com'
     sts_client = Aws::STS::Client.new(region: expected_region, http_proxy: expected_sts_http_proxy)
     mock(Aws::STS::Client).new(region:expected_region, http_proxy: expected_sts_http_proxy){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client,
-                                         sts_http_proxy: expected_sts_http_proxy){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client,
+                                           sts_http_proxy: expected_sts_http_proxy }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -619,10 +664,10 @@ EOC
     expected_sts_http_proxy = 'http://example.com'
     sts_client = Aws::STS::Client.new(region: "us-east-1", http_proxy: expected_sts_http_proxy)
     mock(Aws::STS::Client).new(region: "us-east-1", http_proxy: expected_sts_http_proxy){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client,
-                                         sts_http_proxy: expected_sts_http_proxy){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client,
+                                           sts_http_proxy: expected_sts_http_proxy }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -645,10 +690,10 @@ EOC
     expected_sts_endpoint_url = 'http://example.com'
     sts_client = Aws::STS::Client.new(region: "us-east-1", endpoint: expected_sts_endpoint_url)
     mock(Aws::STS::Client).new(region: "us-east-1", endpoint: expected_sts_endpoint_url){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client,
-                                         sts_endpoint_url: expected_sts_endpoint_url){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client,
+                                           sts_endpoint_url: expected_sts_endpoint_url }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -671,9 +716,9 @@ EOC
     expected_sts_region = 'ap-south-1'
     sts_client = Aws::STS::Client.new(region: expected_sts_region)
     mock(Aws::STS::Client).new(region: expected_sts_region){ sts_client }
-    mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
-                                         role_session_name: "test_session",
-                                         client: sts_client){
+    mock(Aws::AssumeRoleCredentials).new({ role_arn: "test_arn",
+                                           role_session_name: "test_session",
+                                           client: sts_client }){
       expected_credentials
     }
     config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
@@ -694,10 +739,12 @@ EOC
   def test_web_identity_credentials
     expected_credentials = Aws::Credentials.new("test_key", "test_secret")
     mock(Aws::AssumeRoleWebIdentityCredentials).new(
-      role_arn: "test_arn",
-      role_session_name: "test_session",
-      web_identity_token_file: "test_file",
-      client: anything
+      {
+        role_arn: "test_arn",
+        role_session_name: "test_session",
+        web_identity_token_file: "test_file",
+        client: anything
+      }
     ){
       expected_credentials
     }
@@ -722,10 +769,12 @@ EOC
     sts_client = Aws::STS::Client.new(region: 'us-east-1')
     mock(Aws::STS::Client).new(region: 'us-east-1'){ sts_client }
     mock(Aws::AssumeRoleWebIdentityCredentials).new(
-      role_arn: "test_arn",
-      role_session_name: "test_session",
-      web_identity_token_file: "test_file",
-      client: sts_client
+      {
+        role_arn: "test_arn",
+        role_session_name: "test_session",
+        web_identity_token_file: "test_file",
+        client: sts_client
+      }
     ){
       expected_credentials
     }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-16 00:00:00.000000000 Z
+date: 2022-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -115,15 +115,32 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Amazon S3 output plugin for Fluentd event collector
 email: frsyuki@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/workflows/linux.yml"
+- ".github/workflows/stale-actions.yml"
 - ".gitignore"
-- ".travis.yml"
 - AUTHORS
 - ChangeLog
 - Gemfile
@@ -132,6 +149,11 @@ files:
 - Rakefile
 - VERSION
 - appveyor.yml
+- docs/credentials.md
+- docs/howto.md
+- docs/input.md
+- docs/output.md
+- docs/v0.12.md
 - fluent-plugin-s3.gemspec
 - lib/fluent/log-ext.rb
 - lib/fluent/plugin/in_s3.rb
@@ -139,6 +161,7 @@ files:
 - lib/fluent/plugin/s3_compressor_gzip_command.rb
 - lib/fluent/plugin/s3_compressor_lzma2.rb
 - lib/fluent/plugin/s3_compressor_lzo.rb
+- lib/fluent/plugin/s3_compressor_parquet.rb
 - lib/fluent/plugin/s3_extractor_gzip_command.rb
 - lib/fluent/plugin/s3_extractor_lzma2.rb
 - lib/fluent/plugin/s3_extractor_lzo.rb
@@ -163,10 +186,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector
-test_files:
-- test/test_in_s3.rb
-- test/test_out_s3.rb
+test_files: []