fluentd 1.17.0 → 1.17.1

data/test/plugin/test_parser_csv.rb

@@ -1,200 +0,0 @@
-require_relative '../helper'
-require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser_csv'
-
-class CSVParserTest < ::Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  def create_driver(conf={})
-    Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser).configure(conf)
-  end
-
-  data('array param' => '["time","c","d"]', 'string param' => 'time,c,d')
-  def test_parse(param)
-    d = create_driver('keys' => param, 'time_key' => 'time')
-    d.instance.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
-      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
-      assert_equal({
-                     'c' => '192.168.0.1',
-                     'd' => '111',
-                   }, record)
-    }
-  end
-
-  data('array param' => '["c","d"]', 'string param' => 'c,d')
-  def test_parse_without_time(param)
-    time_at_start = Time.now.to_i
-
-    d = create_driver('keys' => param)
-    d.instance.parse("192.168.0.1,111") { |time, record|
-      assert time && time >= time_at_start, "parser puts current time without time input"
-      assert_equal({
-                     'c' => '192.168.0.1',
-                     'd' => '111',
-                   }, record)
-    }
-
-    d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
-    d.configure('keys' => param, 'estimate_current_event' => 'no')
-    d.instance.parse("192.168.0.1,111") { |time, record|
-      assert_equal({
-                     'c' => '192.168.0.1',
-                     'd' => '111',
-                   }, record)
-      assert_nil time, "parser returns nil w/o time and if configured so"
-    }
-  end
-
-  def test_parse_with_keep_time_key
-    d = create_driver(
-                     'keys'=>'time',
-                     'time_key'=>'time',
-                     'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
-                     'keep_time_key'=>'true',
-                     )
-    text = '28/Feb/2013:12:00:00 +0900'
-    d.instance.parse(text) do |time, record|
-      assert_equal text, record['time']
-    end
-  end
-
-  data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
-  def test_parse_with_null_value_pattern(param)
-    d = create_driver(
-                     'keys'=>param,
-                     'null_value_pattern'=>'^(-|null|NULL)$'
-                     )
-    d.instance.parse("-,null,NULL,,--,nuLL") do |time, record|
-      assert_nil record['a']
-      assert_nil record['b']
-      assert_nil record['c']
-      assert_nil record['d']
-      assert_equal record['e'], '--'
-      assert_equal record['f'], 'nuLL'
-    end
-  end
-
-  data('array param' => '["a","b"]', 'string param' => 'a,b')
-  def test_parse_with_null_empty_string(param)
-    d = create_driver(
-                     'keys'=>param,
-                     'null_empty_string'=>true
-                     )
-    d.instance.parse(", ") do |time, record|
-      assert_nil record['a']
-      assert_equal record['b'], ' '
-    end
-  end
-
-  data('array param' => '["a","b","c"]', 'string param' => 'a,b,c')
-  def test_parse_with_option_delimiter(param)
-    d = create_driver(
-                     'keys'=>param,
-                     'delimiter'=>' ',
-                     )
-    d.instance.parse("123 456 789") do |time, record|
-      assert_equal record['a'], '123'
-      assert_equal record['b'], '456'
-      assert_equal record['c'], '789'
-    end
-  end
-
-  sub_test_case 'parser' do
-    data('normal' => 'normal',
-         'fast' => 'fast')
-    def test_compatibility_between_normal_and_fast_parser(param)
-      d = create_driver(
-        'keys' => 'time,key1,key2,key3,key4,key5',
-        'time_key' => 'time',
-        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
-        'keep_time_key' => 'false',
-        'parser_type' => param
-      )
-
-      # non quoted
-      text = '28/Feb/2013:12:00:00 +0900,value1,value2,value3,value4,value5'
-      expected = {'key1' => 'value1', 'key2' => 'value2', 'key3' => "value3",
-                  'key4' => 'value4', 'key5' => "value5"}
-      d.instance.parse(text) do |time, record|
-        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
-        assert_equal expected, record
-      end
-
-      # quoted
-      text = '28/Feb/2013:12:00:00 +0900,"value1","val,ue2","va,lu,e3","val ue4",""'
-      expected = {'key1' => 'value1', 'key2' => 'val,ue2', 'key3' => "va,lu,e3",
-                  'key4' => 'val ue4', 'key5' => ""}
-      d.instance.parse(text) do |time, record|
-        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
-        assert_equal expected, record
-      end
-
-      # mixed
-      text = '28/Feb/2013:12:00:00 +0900,message,"mes,sage","me,ssa,ge",mess age,""'
-      expected = {'key1' => 'message', 'key2' => 'mes,sage', 'key3' => "me,ssa,ge",
-                  'key4' => 'mess age', 'key5' => ""}
-      d.instance.parse(text) do |time, record|
-        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
-        assert_equal expected, record
-      end
-
-      # escaped
-      text = '28/Feb/2013:12:00:00 +0900,"message","mes""sage","""message""",,""""""'
-      expected = {'key1' => 'message', 'key2' => 'mes"sage', 'key3' => '"message"',
-                  'key4' => nil, 'key5' => '""'}
-      d.instance.parse(text) do |time, record|
-        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: '%d/%b/%Y:%H:%M:%S %z'), time)
-        assert_equal expected, record
-      end
-    end
-
-    def test_incompatibility_between_normal_and_fast_parser
-      normal = create_driver(
-        'keys' => 'key1,key2',
-        'parser_type' => 'normal'
-      )
-      fast = create_driver(
-        'keys' => 'key1,key2',
-        'parser_type' => 'fast'
-      )
-
-      # unexpected quote position
-      text = 'a"b,"a"""c"'
-      assert_raise(CSV::MalformedCSVError) {
-        normal.instance.parse(text) { |t, r| }
-      } 
-      assert_nothing_raised {
-        # generate broken record
-        fast.instance.parse(text) { |t, r| }
-      }
-
-      # incorrect the number of column
-      text = 'a,b,c'
-      expected = {"key1" => 'a', "key2" => 'b'}
-      normal.instance.parse(text) { |t, r|
-        assert_equal expected, r
-      }
-      fast.instance.parse(text) { |t, r|
-        assert_not_equal expected, r
-      }
-
-      # And more...
-    end
-  end
-
-  # "parser_type" config shouldn't hide Fluent::Plugin::Parser#plugin_type
-  # https://github.com/fluent/fluentd/issues/3296
-  data('normal' => :normal, 'fast' => :fast)
-  def test_parser_type_method(engine)
-    d = create_driver('keys' => '["time"]','time_key' => 'time', 'parser_type' => engine.to_s)
-    assert_equal(:text_per_line, d.instance.parser_type)
-  end
-
-  data('normal' => :normal, 'fast' => :fast)
-  def test_parser_engine(engine)
-    d = create_driver('keys' => '["time"]', 'time_key' => 'time', 'parser_engine' => engine.to_s)
-    assert_equal(engine, d.instance.parser_engine)
-  end
-end

data/test/plugin/test_parser_json.rb

@@ -1,244 +0,0 @@
-require_relative '../helper'
-require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
-
-class JsonParserTest < ::Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
-  end
-
-  data('oj' => 'oj', 'yajl' => 'yajl')
-  def test_parse(data)
-    @parser.configure('json_parser' => data)
-    @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
-      assert_equal(event_time('2013-02-28 12:00:00 +0900').to_i, time)
-      assert_equal({
-                     'host'   => '192.168.0.1',
-                     'size'   => 777,
-                     'method' => 'PUT',
-                   }, record)
-    }
-  end
-
-  data('oj' => 'oj', 'yajl' => 'yajl')
-  def test_parse_with_large_float(data)
-    @parser.configure('json_parser' => data)
-    @parser.instance.parse('{"num":999999999999999999999999999999.99999}') { |time, record|
-      assert_equal(Float, record['num'].class)
-    }
-  end
-
-  data('oj' => 'oj', 'yajl' => 'yajl')
-  def test_parse_without_time(data)
-    time_at_start = Time.now.to_i
-
-    @parser.configure('json_parser' => data)
-    @parser.instance.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
-      assert time && time >= time_at_start, "parser puts current time without time input"
-      assert_equal({
-                     'host'   => '192.168.0.1',
-                     'size'   => 777,
-                     'method' => 'PUT',
-                   }, record)
-    }
-
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
-    parser.configure('json_parser' => data, 'estimate_current_event' => 'false')
-    parser.instance.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
-      assert_equal({
-                     'host'   => '192.168.0.1',
-                     'size'   => 777,
-                     'method' => 'PUT',
-                   }, record)
-      assert_nil time, "parser return nil w/o time and if specified so"
-    }
-  end
-
-  data('oj' => 'oj', 'yajl' => 'yajl')
-  def test_parse_with_colon_string(data)
-    @parser.configure('json_parser' => data)
-    @parser.instance.parse('{"time":1362020400,"log":":message"}') { |time, record|
-      assert_equal(record['log'], ':message')
-    }
-  end
-
-  data('oj' => 'oj', 'yajl' => 'yajl')
-  def test_parse_with_invalid_time(data)
-    @parser.configure('json_parser' => data)
-    assert_raise Fluent::ParserError do
-      @parser.instance.parse('{"time":[],"k":"v"}') { |time, record| }
-    end
-  end
-
-  data('oj' => 'oj', 'yajl' => 'yajl')
-  def test_parse_float_time(data)
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
-    parser.configure('json_parser' => data)
-    text = "100.1"
-    parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
-      assert_equal 100, time.sec
-      assert_equal 100_000_000, time.nsec
-    end
-  end
-
-  data('oj' => 'oj', 'yajl' => 'yajl')
-  def test_parse_with_keep_time_key(data)
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
-    format = "%d/%b/%Y:%H:%M:%S %z"
-    parser.configure(
-                     'time_format' => format,
-                     'keep_time_key' => 'true',
-                     'json_parser' => data
-                     )
-    text = "28/Feb/2013:12:00:00 +0900"
-    parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
-      assert_equal Time.strptime(text, format).to_i, time.sec
-      assert_equal text, record['time']
-    end
-  end
-
-  data('oj' => 'oj', 'yajl' => 'yajl')
-  def test_parse_with_keep_time_key_without_time_format(data)
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
-    parser.configure(
-                     'keep_time_key' => 'true',
-                     'json_parser' => data
-                     )
-    text = "100"
-    parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
-      assert_equal text.to_i, time.sec
-      assert_equal text, record['time']
-    end
-  end
-
-  def test_yajl_parse_io_with_buffer_smaller_than_input
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
-    parser.configure(
-                     'keep_time_key' => 'true',
-                     'json_parser' => 'yajl',
-                     'stream_buffer_size' => 1,
-                     )
-    text = "100"
-
-    waiting(5) do
-      rd, wr = IO.pipe
-      wr.write "{\"time\":\"#{text}\"}"
-
-      parser.instance.parse_io(rd) do |time, record|
-        assert_equal text.to_i, time.sec
-        assert_equal text, record['time']
-
-        # Once a record has been received the 'write' end of the pipe must be
-        # closed, otherwise the test will block waiting for more input.
-        wr.close
-      end
-    end
-  end
-
-  sub_test_case "various record pattern" do
-    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
-    data("Only string without quotation", { record: "message", expected: [nil] }, keep: true)
-    data("Only number", { record: "0", expected: [nil] }, keep: true)
-    data(
-      "Array of Hash",
-      {
-        record: '[{"k1": 1}, {"k2": 2}]',
-        expected: [{"k1" => 1}, {"k2" => 2}]
-      },
-      keep: true,
-    )
-    data(
-      "Array of both Hash and invalid",
-      {
-        record: '[{"k1": 1}, "string", {"k2": 2}, 0]',
-        expected: [{"k1" => 1}, nil, {"k2" => 2}, nil]
-      },
-      keep: true,
-    )
-    data(
-      "Array of all invalid",
-      {
-        record: '["string", 0, [{"k": 0}]]',
-        expected: [nil, nil, nil]
-      },
-      keep: true,
-    )
-
-    def test_oj(data)
-      parsed_records = []
-      @parser.configure("json_parser" => "oj")
-      @parser.instance.parse(data[:record]) { |time, record|
-        parsed_records.append(record)
-      }
-      assert_equal(data[:expected], parsed_records)
-    end
-
-    def test_yajl(data)
-      parsed_records = []
-      @parser.configure("json_parser" => "yajl")
-      @parser.instance.parse(data[:record]) { |time, record|
-        parsed_records.append(record)
-      }
-      assert_equal(data[:expected], parsed_records)
-    end
-
-    def test_json(json)
-      parsed_records = []
-      @parser.configure("json_parser" => "json")
-      @parser.instance.parse(data[:record]) { |time, record|
-        parsed_records.append(record)
-      }
-      assert_equal(data[:expected], parsed_records)
-    end
-  end
-
-  # This becomes NoMethodError if a non-Hash object is passed to convert_values.
-  # https://github.com/fluent/fluentd/issues/4100
-  sub_test_case "execute_convert_values with null_empty_string" do
-    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
-    data(
-      "Hash",
-      {
-        record: '{"k1": 1, "k2": ""}',
-        expected: [{"k1" => 1, "k2" => nil}]
-      },
-      keep: true,
-    )
-    data(
-      "Array of Hash",
-      {
-        record: '[{"k1": 1}, {"k2": ""}]',
-        expected: [{"k1" => 1}, {"k2" => nil}]
-      },
-      keep: true,
-    )
-
-    def test_oj(data)
-      parsed_records = []
-      @parser.configure("json_parser" => "oj", "null_empty_string" => true)
-      @parser.instance.parse(data[:record]) { |time, record|
-        parsed_records.append(record)
-      }
-      assert_equal(data[:expected], parsed_records)
-    end
-
-    def test_yajl(data)
-      parsed_records = []
-      @parser.configure("json_parser" => "yajl", "null_empty_string" => true)
-      @parser.instance.parse(data[:record]) { |time, record|
-        parsed_records.append(record)
-      }
-      assert_equal(data[:expected], parsed_records)
-    end
-
-    def test_json(json)
-      parsed_records = []
-      @parser.configure("json_parser" => "json", "null_empty_string" => true)
-      @parser.instance.parse(data[:record]) { |time, record|
-        parsed_records.append(record)
-      }
-      assert_equal(data[:expected], parsed_records)
-    end
-  end
-end

data/test/plugin/test_parser_labeled_tsv.rb

@@ -1,160 +0,0 @@
-require_relative '../helper'
-require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
-
-class LabeledTSVParserTest < ::Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  def test_config_params
-    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
-
-    assert_equal "\t", parser.instance.delimiter
-    assert_equal  ":", parser.instance.label_delimiter
-
-    parser.configure(
-                     'delimiter'       => ',',
-                     'label_delimiter' => '=',
-                     )
-
-    assert_equal ",", parser.instance.delimiter
-    assert_equal "=", parser.instance.label_delimiter
-  end
-
-  def test_parse
-    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
-    parser.configure({})
-    parser.instance.parse("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111") { |time, record|
-      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
-      assert_equal({
-                     'host'   => '192.168.0.1',
-                     'req_id' => '111',
-                   }, record)
-    }
-  end
-
-  def test_parse_with_customized_delimiter
-    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
-    parser.configure(
-                     'delimiter'       => ',',
-                     'label_delimiter' => '=',
-                     )
-    parser.instance.parse('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111') { |time, record|
-      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
-      assert_equal({
-                     'host'   => '192.168.0.1',
-                     'req_id' => '111',
-                   }, record)
-    }
-  end
-
-  def test_parse_with_customized_time_format
-    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
-    parser.configure(
-                     'time_key'    => 'mytime',
-                     'time_format' => '%d/%b/%Y:%H:%M:%S %z',
-                     )
-    parser.instance.parse("mytime:28/Feb/2013:12:00:00 +0900\thost:192.168.0.1\treq_id:111") { |time, record|
-      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
-      assert_equal({
-                     'host'   => '192.168.0.1',
-                     'req_id' => '111',
-                   }, record)
-    }
-  end
-
-  def test_parse_without_time
-    time_at_start = Time.now.to_i
-
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
-    parser.configure({})
-    parser.instance.parse("host:192.168.0.1\treq_id:111") { |time, record|
-      assert time && time >= time_at_start, "parser puts current time without time input"
-      assert_equal({
-                     'host'   => '192.168.0.1',
-                     'req_id' => '111',
-                   }, record)
-    }
-
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
-    parser.configure({'estimate_current_event' => 'no'})
-    parser.instance.parse("host:192.168.0.1\treq_id:111") { |time, record|
-      assert_equal({
-                     'host'   => '192.168.0.1',
-                     'req_id' => '111',
-                   }, record)
-      assert_nil time, "parser returns nil w/o time and if configured so"
-    }
-  end
-
-  def test_parse_with_keep_time_key
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
-    parser.configure(
-                     'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
-                     'keep_time_key'=>'true',
-                     )
-    text = '28/Feb/2013:12:00:00 +0900'
-    parser.instance.parse("time:#{text}") do |time, record|
-      assert_equal text, record['time']
-    end
-  end
-
-  def test_parse_and_reject_invalid_kv_pairs
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
-    parser.configure(
-                     'delimiter'       => ' ',
-                     'label_delimiter' => '=',
-                     )
-    text = 'A leading portion that is not LTSV    :  foo=bar baz=derp and a trailing portion'
-
-    expected = {'foo' => 'bar', 'baz' => 'derp'}
-    parser.instance.parse(text) do |time, record|
-      assert_equal expected, record
-    end
-
-  end
-
-  def test_parse_with_null_value_pattern
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
-    parser.configure(
-                     'null_value_pattern'=>'^(-|null|NULL)$'
-                     )
-    parser.instance.parse("a:-\tb:null\tc:NULL\td:\te:--\tf:nuLL") do |time, record|
-      assert_nil record['a']
-      assert_nil record['b']
-      assert_nil record['c']
-      assert_equal record['d'], ''
-      assert_equal record['e'], '--'
-      assert_equal record['f'], 'nuLL'
-    end
-  end
-
-  def test_parse_with_null_empty_string
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
-    parser.configure(
-                     'null_empty_string'=>true
-                     )
-    parser.instance.parse("a:\tb: ") do |time, record|
-      assert_nil record['a']
-      assert_equal record['b'], ' '
-    end
-  end
-
-  data("single space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
-       "multiple space" => ["k1=v1    k2=v2", { "k1" => "v1", "k2" => "v2" }],
-       "reverse" => ["k2=v2 k1=v1", { "k1" => "v1", "k2" => "v2" }],
-       "tab" => ["k2=v2\tk1=v1", { "k1" => "v1", "k2" => "v2" }],
-       "tab and space" => ["k2=v2\t k1=v1", { "k1" => "v1", "k2" => "v2" }])
-  def test_parse_with_delimiter_pattern(data)
-    text, expected = data
-    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
-    parser.configure(
-                     'delimiter_pattern' => '/\s+/',
-                     'label_delimiter' => '='
-                    )
-    parser.instance.parse(text) do |_time, record|
-      assert_equal(expected, record)
-    end
-  end
-end