fluentd 1.16.11-x86-mingw32 → 1.17.0-x86-mingw32

data/test/plugin/test_out_file.rb

@@ -130,7 +130,7 @@ class FileOutputTest < Test::Unit::TestCase
           'path' => "#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log",
           'add_path_suffix' => 'false',
           'append' => "true",
-          'symlink_path' => "#{TMP_DIR}/${tag}/conf_test.current.log",
+          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
           'compress' => 'gzip',
           'recompress' => 'true',
         }, [
@@ -183,26 +183,6 @@ class FileOutputTest < Test::Unit::TestCase
         Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
       end
     end
-
-    test 'warning for symlink_path not including correct placeholders corresponding to chunk keys' do
-      omit "Windows doesn't support symlink" if Fluent.windows?
-      conf = config_element('match', '**', {
-          'path' => "#{TMP_DIR}/${tag}/${key1}/${key2}/conf_test.%Y%m%d.%H%M.log",
-          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
-        }, [
-          config_element('buffer', 'time,tag,key1,key2', {
-              '@type' => 'file',
-              'timekey' => '1d',
-              'path' => "#{TMP_DIR}/buf_conf_test",
-          }),
-      ])
-      assert_nothing_raised do
-        d = create_driver(conf)
-        assert do
-          d.logs.count { |log| log.include?("multiple chunks are competing for a single symlink_path") } == 2
-        end
-      end
-    end
   end
 
   sub_test_case 'fully configured output' do
@@ -334,7 +314,7 @@ class FileOutputTest < Test::Unit::TestCase
       assert_equal r5, d.formatted[4]
 
       read_gunzip = ->(path){
-        File.open(path, 'rb'){ |fio|
+        File.open(path){ |fio|
           Zlib::GzipReader.new(StringIO.new(fio.read)).read
         }
       }

data/test/plugin/test_out_forward.rb

@@ -12,8 +12,7 @@ class ForwardOutputTest < Test::Unit::TestCase
     FileUtils.rm_rf(TMP_DIR)
     FileUtils.mkdir_p(TMP_DIR)
     @d = nil
-    # forward plugin uses TCP and UDP sockets on the same port number
-    @target_port = unused_port(protocol: :all)
+    @target_port = unused_port
   end
 
   def teardown
@@ -178,42 +177,6 @@ EOL
     assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }
   end
 
-  sub_test_case 'configure compress' do
-    data('default', ['', :text])
-    data('gzip', ['compress gzip', :gzip])
-    test 'should be applied' do |(option, expected)|
-      @d = d = create_driver(config + option)
-      node = d.instance.nodes.first
-
-      assert_equal(
-        [expected, expected],
-        [d.instance.compress, node.instance_variable_get(:@compress)]
-      )
-    end
-
-    # TODO add tests that we cannot configure the different compress type between owner and buffer except for :text
-    data('gzip', ['compress gzip', :text, :gzip])
-    test 'can configure buffer compress separately when owner uses :text' do |(buffer_option, expected_owner_compress, expected_buffer_compress)|
-      @d = d = create_driver(config + %[
-         <buffer>
-           type memory
-           #{buffer_option}
-         </buffer>
-       ])
-      node = d.instance.nodes.first
-
-      assert_equal(
-        [expected_owner_compress, expected_owner_compress, expected_buffer_compress],
-        [d.instance.compress, node.instance_variable_get(:@compress), d.instance.buffer.compress],
-      )
-
-      log_message = "buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>"
-      assert do
-        d.logs.any? { |log| log.include?(log_message) }
-      end
-    end
-  end
-
   data('CA cert'     => 'tls_ca_cert_path',
        'non CA cert' => 'tls_cert_path')
   test 'configure tls_cert_path/tls_ca_cert_path' do |param|
@@ -362,6 +325,40 @@ EOL
     assert_equal 1234, d.instance.discovery_manager.services[0].port
   end
 
+  test 'compress_default_value' do
+    @d = d = create_driver
+    assert_equal :text, d.instance.compress
+
+    node = d.instance.nodes.first
+    assert_equal :text, node.instance_variable_get(:@compress)
+  end
+
+  test 'set_compress_is_gzip' do
+    @d = d = create_driver(config + %[compress gzip])
+    assert_equal :gzip, d.instance.compress
+    assert_equal :gzip, d.instance.buffer.compress
+
+    node = d.instance.nodes.first
+    assert_equal :gzip, node.instance_variable_get(:@compress)
+  end
+
+  test 'set_compress_is_gzip_in_buffer_section' do
+    mock = flexmock($log)
+    mock.should_receive(:log).with("buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>")
+
+    @d = d = create_driver(config + %[
+       <buffer>
+         type memory
+         compress gzip
+       </buffer>
+     ])
+    assert_equal :text, d.instance.compress
+    assert_equal :gzip, d.instance.buffer.compress
+
+    node = d.instance.nodes.first
+    assert_equal :text, node.instance_variable_get(:@compress)
+  end
+
   test 'phi_failure_detector disabled' do
     @d = d = create_driver(config + %[phi_failure_detector false \n phi_threshold 0])
     node = d.instance.nodes.first
@@ -613,6 +610,7 @@ EOL
 
     @d = d = create_driver(config + %[
       require_ack_response true
+      ack_response_timeout 1s
       <buffer tag>
         flush_mode immediate
         retry_type periodic
@@ -660,6 +658,7 @@ EOL
 
     @d = d = create_driver(config + %[
       require_ack_response true
+      ack_response_timeout 10s
       <buffer tag>
         flush_mode immediate
         retry_type periodic
@@ -1347,27 +1346,4 @@ EOL
       end
     end
   end
-
-  test 'establish_connection_timeout' do
-    @d = d = create_driver(%[
-      hard_timeout 1
-      <server>
-        host #{TARGET_HOST}
-        port #{@target_port}
-      </server>
-    ])
-
-    node = d.instance.nodes.first
-    mock_sock = flexmock('socket')
-    mock_sock.should_receive(:read_nonblock).with(512).and_return('').at_least.once
-
-    ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
-
-    assert_true node.available?
-    node.establish_connection(mock_sock, ri)
-    assert_false node.available?
-
-    logs = d.logs
-    assert{ logs.any?{|log| log.include?('handshake timeout after 1.0s') } }
-  end
 end

data/test/plugin/test_out_http.rb

@@ -7,6 +7,7 @@ require 'webrick/https'
 require 'net/http'
 require 'uri'
 require 'json'
+require 'aws-sdk-core'
 
 # WEBrick's ProcHandler doesn't handle PUT by default
 module WEBrick::HTTPServlet
@@ -390,6 +391,97 @@ class HTTPOutputTest < Test::Unit::TestCase
     end
   end
 
+
+  sub_test_case 'aws sigv4 auth' do
+    setup do
+      @@fake_aws_credentials = Aws::Credentials.new(
+        'fakeaccess',
+        'fakesecret',
+        'fake session token'
+      )
+    end
+
+    def server_port
+      19883
+    end
+
+    def test_aws_sigv4_sts_role_arn
+      stub(Aws::AssumeRoleCredentials).new do |credentials_provider|
+        stub(credentials_provider).credentials {
+          @@fake_aws_credentials
+        }
+        credentials_provider
+      end
+
+      d = create_driver(config + %[
+          <auth>
+            method aws_sigv4
+            aws_service someservice
+            aws_region my-region-1
+            aws_role_arn arn:aws:iam::123456789012:role/MyRole
+          </auth>
+        ])
+      d.run(default_tag: 'test.http') do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, result.data
+      assert_not_empty result.headers
+      assert_not_nil result.headers['authorization']
+      assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
+      assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
+      assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-content-sha256']
+      assert_not_empty result.headers['x-amz-content-sha256']
+      assert_not_nil result.headers['x-amz-security-token']
+      assert_not_empty result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-date']
+      assert_not_empty result.headers['x-amz-date']
+    end
+
+    def test_aws_sigv4_no_role
+      stub(Aws::CredentialProviderChain).new do |provider_chain|
+        stub(provider_chain).resolve {
+          @@fake_aws_credentials
+        }
+        provider_chain
+      end
+      d = create_driver(config + %[
+          <auth>
+            method aws_sigv4
+            aws_service someservice
+            aws_region my-region-1
+          </auth>
+        ])
+      d.run(default_tag: 'test.http') do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, result.data
+      assert_not_empty result.headers
+      assert_not_nil result.headers['authorization']
+      assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
+      assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
+      assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-content-sha256']
+      assert_not_empty result.headers['x-amz-content-sha256']
+      assert_not_nil result.headers['x-amz-security-token']
+      assert_not_empty result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-date']
+      assert_not_empty result.headers['x-amz-date']
+    end
+  end
+
   sub_test_case 'HTTPS' do
     def server_port
       19882
@@ -426,4 +518,40 @@ class HTTPOutputTest < Test::Unit::TestCase
       assert_not_empty result.headers
     end
   end
+
+  sub_test_case 'connection_reuse' do
+    def server_port
+      19883
+    end
+
+    def test_connection_recreation
+      d = create_driver(%[
+        endpoint http://127.0.0.1:#{server_port}/test
+        reuse_connections true
+      ])
+
+      d.run(default_tag: 'test.http', shutdown: false) do
+        d.feed(test_events[0])
+      end
+
+      data = @@result.data
+
+      # Restart server to simulate connection loss
+      @@http_server_thread.kill
+      @@http_server_thread.join
+      @@http_server_thread = Thread.new do
+        run_http_server
+      end
+
+      d.run(default_tag: 'test.http') do
+        d.feed(test_events[1])
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, data.concat(result.data)
+      assert_not_empty result.headers
+    end
+  end
 end

data/test/plugin/test_out_stdout.rb

@@ -95,11 +95,11 @@ class StdoutOutputTest < Test::Unit::TestCase
           d.feed(time, {'test' => 'test2'})
         end
       end
-      assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test2\"}\n", out.gsub(' => ', '=>')
+      assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test2\"}\n", out
 
       # NOTE: Float::NAN is not jsonable, but hash string can output it.
       out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
-      assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>NaN}\n", out.gsub(' => ', '=>')
+      assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>NaN}\n", out
     end
   end
 
@@ -171,7 +171,7 @@ class StdoutOutputTest < Test::Unit::TestCase
           end
         end
 
-        assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out.gsub(' => ', '=>')
+        assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
       end
     end
   end

data/test/plugin/test_out_stream.rb

@@ -54,7 +54,7 @@ class TcpOutputTest < Test::Unit::TestCase
 
   def setup
     super
-    @port = unused_port(protocol: :tcp)
+    @port = unused_port
   end
 
   def teardown

data/test/plugin/test_output.rb

@@ -1111,7 +1111,7 @@ class OutputTest < Test::Unit::TestCase
         config: config_element(
           "ROOT", "", {},
           [
-            config_element("buffer", "", {}),
+            config_element("buffer", "", {}), 
             config_element("secondary", "", {"@type" => "test", "name" => "test"}),
           ]
         ),

data/test/plugin/test_output_as_buffered.rb

@@ -1343,7 +1343,6 @@ class BufferedOutputTest < Test::Unit::TestCase
       hash = {
         'flush_interval' => 10,
         'flush_thread_count' => 1,
-        'flush_thread_interval' => 0.1,
         'flush_thread_burst_interval' => 0.1,
         'chunk_limit_size' => 1024,
       }
@@ -1446,7 +1445,7 @@ class BufferedOutputTest < Test::Unit::TestCase
 
       assert{ @i.buffer.stage.size == 3 }
 
-      # to trigger try_flush with flush_thread_interval
+      # to trigger try_flush with flush_thread_burst_interval
       Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )
       @i.enqueue_thread_wait
       Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )
@@ -1455,6 +1454,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i.enqueue_thread_wait
       Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )
       @i.enqueue_thread_wait
+      @i.flush_thread_wakeup
 
       assert{ @i.buffer.stage.size == 0 }
 

data/test/plugin/test_output_as_buffered_retries.rb

@@ -941,7 +941,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.enqueue_thread_wait
 
       @i.flush_thread_wakeup
-      waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
+      waiting(4){ Thread.pass until @i.write_count > 0 }
       waiting(4) do
         state = @i.instance_variable_get(:@output_flush_threads).first
         state.thread.status == 'sleep'
@@ -953,7 +953,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       now = @i.next_flush_time
       Timecop.freeze( now )
       @i.flush_thread_wakeup
-      waiting(4){ Thread.pass until @i.write_count > 1 && @i.num_errors > 1 }
+      waiting(4){ Thread.pass until @i.write_count > 1 }
       waiting(4) do
         state = @i.instance_variable_get(:@output_flush_threads).first
         state.thread.status == 'sleep'

data/test/plugin/test_owned_by.rb

@@ -26,7 +26,6 @@ class OwnedByMixinTest < Test::Unit::TestCase
 
       assert_equal parent.object_id, child.owner.object_id
 
-      assert child.instance_eval{ @_plugin_id_configured }
       assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
 
       assert_equal Fluent::Log::LEVEL_TRACE, child.log.level

data/test/plugin/test_parser_csv.rb

@@ -164,7 +164,7 @@ class CSVParserTest < ::Test::Unit::TestCase
       text = 'a"b,"a"""c"'
       assert_raise(CSV::MalformedCSVError) {
         normal.instance.parse(text) { |t, r| }
-      }
+      } 
       assert_nothing_raised {
         # generate broken record
         fast.instance.parse(text) { |t, r| }

data/test/plugin/test_parser_json.rb

@@ -8,37 +8,6 @@ class JsonParserTest < ::Test::Unit::TestCase
     @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
   end
 
-  sub_test_case "configure_json_parser" do
-    data("oj", [:oj, [Oj.method(:load), Oj::ParseError]])
-    data("json", [:json, [JSON.method(:load), JSON::ParserError]])
-    data("yajl", [:yajl, [Yajl.method(:load), Yajl::ParseError]])
-    def test_return_each_loader((input, expected_return))
-      result = @parser.instance.configure_json_parser(input)
-      assert_equal expected_return, result
-    end
-
-    def test_raise_exception_for_unknown_input
-      assert_raise RuntimeError do
-        @parser.instance.configure_json_parser(:unknown)
-      end
-    end
-
-    def test_fall_back_oj_to_yajl_if_oj_not_available
-      stub(Fluent::OjOptions).available? { false }
-
-      result = @parser.instance.configure_json_parser(:oj)
-
-      assert_equal [Yajl.method(:load), Yajl::ParseError], result
-      logs = @parser.logs.collect do |log|
-        log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
-      end
-      assert_equal(
-        ["[info]: Oj is not installed, and failing back to Yajl for json parser\n"],
-        logs
-      )
-    end
-  end
-
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_parse(data)
     @parser.configure('json_parser' => data)
@@ -166,4 +135,110 @@ class JsonParserTest < ::Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case "various record pattern" do
+    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
+    data("Only string without quotation", { record: "message", expected: [nil] }, keep: true)
+    data("Only number", { record: "0", expected: [nil] }, keep: true)
+    data(
+      "Array of Hash",
+      {
+        record: '[{"k1": 1}, {"k2": 2}]',
+        expected: [{"k1" => 1}, {"k2" => 2}]
+      },
+      keep: true,
+    )
+    data(
+      "Array of both Hash and invalid",
+      {
+        record: '[{"k1": 1}, "string", {"k2": 2}, 0]',
+        expected: [{"k1" => 1}, nil, {"k2" => 2}, nil]
+      },
+      keep: true,
+    )
+    data(
+      "Array of all invalid",
+      {
+        record: '["string", 0, [{"k": 0}]]',
+        expected: [nil, nil, nil]
+      },
+      keep: true,
+    )
+
+    def test_oj(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "oj")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_yajl(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "yajl")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_json(json)
+      parsed_records = []
+      @parser.configure("json_parser" => "json")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+
+  # This becomes NoMethodError if a non-Hash object is passed to convert_values.
+  # https://github.com/fluent/fluentd/issues/4100
+  sub_test_case "execute_convert_values with null_empty_string" do
+    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
+    data(
+      "Hash",
+      {
+        record: '{"k1": 1, "k2": ""}',
+        expected: [{"k1" => 1, "k2" => nil}]
+      },
+      keep: true,
+    )
+    data(
+      "Array of Hash",
+      {
+        record: '[{"k1": 1}, {"k2": ""}]',
+        expected: [{"k1" => 1}, {"k2" => nil}]
+      },
+      keep: true,
+    )
+
+    def test_oj(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "oj", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_yajl(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "yajl", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_json(json)
+      parsed_records = []
+      @parser.configure("json_parser" => "json", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
 end