fluentd 1.17.0 → 1.17.1

data/test/plugin/test_parser.rb

@@ -1,399 +0,0 @@
-require_relative '../helper'
-require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
-require 'json'
-require 'timecop'
-
-class ParserTest < ::Test::Unit::TestCase
-  class ExampleParser < Fluent::Plugin::Parser
-    def parse(data)
-      r = JSON.parse(data)
-      yield convert_values(parse_time(r), r)
-    end
-  end
-
-  def create_driver(conf={})
-    Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser).configure(conf)
-  end
-
-  def setup
-    Fluent::Test.setup
-  end
-
-  sub_test_case 'base class works as plugin' do
-    def test_init
-      i = Fluent::Plugin::Parser.new
-      assert_nil i.types
-      assert_nil i.null_value_pattern
-      assert !i.null_empty_string
-      assert i.estimate_current_event
-      assert !i.keep_time_key
-    end
-
-    def test_configure_against_string_literal
-      d = create_driver('keep_time_key true')
-      assert_true d.instance.keep_time_key
-    end
-
-    def test_parse
-      d = create_driver
-      assert_raise NotImplementedError do
-        d.instance.parse('')
-      end
-    end
-  end
-
-  sub_test_case '#string_like_null' do
-    setup do
-      @i = ExampleParser.new
-    end
-
-    test 'returns false if null_empty_string is false and null_value_regexp is nil' do
-      assert ! @i.string_like_null('a', false, nil)
-      assert ! @i.string_like_null('', false, nil)
-    end
-
-    test 'returns true if null_empty_string is true and string value is empty' do
-      assert ! @i.string_like_null('a', true, nil)
-      assert @i.string_like_null('', true, nil)
-    end
-
-    test 'returns true if null_value_regexp has regexp and it matches string value' do
-      assert ! @i.string_like_null('a', false, /null/i)
-      assert @i.string_like_null('NULL', false, /null/i)
-      assert @i.string_like_null('empty', false, /null|empty/i)
-    end
-  end
-
-  sub_test_case '#build_type_converters converters' do
-    setup do
-      @i = ExampleParser.new
-      types_config = {
-        "s" => "string",
-        "i" => "integer",
-        "f" => "float",
-        "b" => "bool",
-        "t1" => "time",
-        "t2" => "time:%Y-%m-%d %H:%M:%S.%N",
-        "t3" => "time:+0100:%Y-%m-%d %H:%M:%S.%N",
-        "t4" => "time:unixtime",
-        "t5" => "time:float",
-        "a1" => "array",
-        "a2" => "array:|",
-      }
-      @hash = {
-        'types' => JSON.dump(types_config),
-      }
-    end
-
-    test 'to do #to_s by "string" type' do
-      @i.configure(config_element('parse', '', @hash))
-      c = @i.type_converters["s"]
-      assert_equal "", c.call("")
-      assert_equal "a", c.call("a")
-      assert_equal "1", c.call(1)
-      assert_equal "1.01", c.call(1.01)
-      assert_equal "true", c.call(true)
-      assert_equal "false", c.call(false)
-    end
-
-    test 'to do #to_i by "integer" type' do
-      @i.configure(config_element('parse', '', @hash))
-      c = @i.type_converters["i"]
-      assert_equal 0, c.call("")
-      assert_equal 0, c.call("0")
-      assert_equal 0, c.call("a")
-      assert_equal(-1000, c.call("-1000"))
-      assert_equal 1, c.call(1)
-      assert_equal 1, c.call(1.01)
-      assert_equal 0, c.call(true)
-      assert_equal 0, c.call(false)
-    end
-
-    test 'to do #to_f by "float" type' do
-      @i.configure(config_element('parse', '', @hash))
-      c = @i.type_converters["f"]
-      assert_equal 0.0, c.call("")
-      assert_equal 0.0, c.call("0")
-      assert_equal 0.0, c.call("a")
-      assert_equal(-1000.0, c.call("-1000"))
-      assert_equal 1.0, c.call(1)
-      assert_equal 1.01, c.call(1.01)
-      assert_equal 0.0, c.call(true)
-      assert_equal 0.0, c.call(false)
-    end
-
-    test 'to return true/false, which returns true only for true/yes/1 (C & perl style), by "bool"' do
-      @i.configure(config_element('parse', '', @hash))
-      c = @i.type_converters["b"]
-      assert_false c.call("")
-      assert_false c.call("0")
-      assert_false c.call("a")
-      assert_true c.call("1")
-      assert_true c.call("true")
-      assert_true c.call("True")
-      assert_true c.call("YES")
-      assert_true c.call(true)
-      assert_false c.call(false)
-      assert_false c.call("1.0")
-    end
-
-    test 'to parse time string by ruby default time parser without any options' do
-      # "t1" => "time",
-      with_timezone("UTC+02") do # -0200
-        @i.configure(config_element('parse', '', @hash))
-        c = @i.type_converters["t1"]
-        assert_nil c.call("")
-        assert_equal_event_time event_time("2016-10-21 01:54:30 -0200"), c.call("2016-10-21 01:54:30")
-        assert_equal_event_time event_time("2016-10-21 03:54:30 -0200"), c.call("2016-10-21 01:54:30 -0400")
-        assert_equal_event_time event_time("2016-10-21 01:55:24 -0200"), c.call("2016-10-21T01:55:24-02:00")
-        assert_equal_event_time event_time("2016-10-21 01:55:24 -0200"), c.call("2016-10-21T03:55:24Z")
-      end
-    end
-
-    test 'to parse time string with specified time format' do
-      # "t2" => "time:%Y-%m-%d %H:%M:%S.%N",
-      with_timezone("UTC+02") do # -0200
-        @i.configure(config_element('parse', '', @hash))
-        c = @i.type_converters["t2"]
-        assert_nil c.call("")
-        assert_equal_event_time event_time("2016-10-21 01:54:30.123000000 -0200"), c.call("2016-10-21 01:54:30.123")
-        assert_equal_event_time event_time("2016-10-21 01:54:30.012345678 -0200"), c.call("2016-10-21 01:54:30.012345678")
-        assert_nil c.call("2016/10/21 015430")
-      end
-    end
-
-    test 'to parse time string with specified time format and timezone' do
-      # "t3" => "time:+0100:%Y-%m-%d %H:%M:%S.%N",
-      with_timezone("UTC+02") do # -0200
-        @i.configure(config_element('parse', '', @hash))
-        c = @i.type_converters["t3"]
-        assert_nil c.call("")
-        assert_equal_event_time event_time("2016-10-21 01:54:30.123000000 +0100"), c.call("2016-10-21 01:54:30.123")
-        assert_equal_event_time event_time("2016-10-21 01:54:30.012345678 +0100"), c.call("2016-10-21 01:54:30.012345678")
-      end
-    end
-
-    test 'to parse time string in unix timestamp' do
-      # "t4" => "time:unixtime",
-      with_timezone("UTC+02") do # -0200
-        @i.configure(config_element('parse', '', @hash))
-        c = @i.type_converters["t4"]
-        assert_equal_event_time event_time("1970-01-01 00:00:00.0 +0000"), c.call("")
-        assert_equal_event_time event_time("2016-10-21 01:54:30.0 -0200"), c.call("1477022070")
-        assert_equal_event_time event_time("2016-10-21 01:54:30.0 -0200"), c.call("1477022070.01")
-      end
-    end
-
-    test 'to parse time string in floating poing value' do
-      # "t5" => "time:float",
-      with_timezone("UTC+02") do # -0200
-        @i.configure(config_element('parse', '', @hash))
-        c = @i.type_converters["t5"]
-        assert_equal_event_time event_time("1970-01-01 00:00:00.0 +0000"), c.call("")
-        assert_equal_event_time event_time("2016-10-21 01:54:30.012 -0200"), c.call("1477022070.012")
-        assert_equal_event_time event_time("2016-10-21 01:54:30.123456789 -0200"), c.call("1477022070.123456789")
-      end
-    end
-
-    test 'to return array of string' do
-      @i.configure(config_element('parse', '', @hash))
-      c = @i.type_converters["a1"]
-      assert_equal [], c.call("")
-      assert_equal ["0"], c.call("0")
-      assert_equal ["0"], c.call(0)
-      assert_equal ["0", "1"], c.call("0,1")
-      assert_equal ["0|1", "2"], c.call("0|1,2")
-      assert_equal ["true"], c.call(true)
-    end
-
-    test 'to return array of string using specified delimiter' do
-      @i.configure(config_element('parse', '', @hash))
-      c = @i.type_converters["a2"]
-      assert_equal [], c.call("")
-      assert_equal ["0"], c.call("0")
-      assert_equal ["0"], c.call(0)
-      assert_equal ["0,1"], c.call("0,1")
-      assert_equal ["0", "1,2"], c.call("0|1,2")
-      assert_equal ["true"], c.call(true)
-    end
-  end
-
-  sub_test_case 'example parser without any configurations' do
-    setup do
-      @current_time = Time.parse("2016-10-21 14:22:01.0 +1000")
-      @current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)
-      # @current_time.to_i #=> 1477023721
-      Timecop.freeze(@current_time)
-      @i = ExampleParser.new
-      @i.configure(config_element('parse', '', {}))
-    end
-
-    teardown do
-      Timecop.return
-    end
-
-    test 'parser returns parsed JSON object, leaving empty/NULL strings, with current time' do
-      json = '{"t1":"1477023720.101","s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
-      @i.parse(json) do |time, record|
-        assert_equal_event_time @current_event_time, time
-        assert_equal "1477023720.101", record["t1"]
-        assert_equal "", record["s1"]
-        assert_equal "NULL", record["s2"]
-        assert_equal "null", record["s3"]
-        assert_equal 1, record["k1"]
-        assert_equal "13.1", record["k2"]
-        assert_equal "1", record["k3"]
-        assert_equal "yes", record["k4"]
-      end
-    end
-  end
-
-  sub_test_case 'example parser fully configured' do
-    setup do
-      @current_time = Time.parse("2016-10-21 14:22:01.0 +1000")
-      @current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)
-      # @current_time.to_i #=> 1477023721
-      Timecop.freeze(@current_time)
-      @i = ExampleParser.new
-      hash = {
-        'keep_time_key' => "no",
-        'estimate_current_event' => "yes",
-        'time_key' => "t1",
-        'time_type' => "float",
-        'null_empty_string' => 'yes',
-        'null_value_pattern' => 'NULL|null',
-        'types' => "k1:string, k2:integer, k3:float, k4:bool",
-      }
-      @i.configure(config_element('parse', '', hash))
-    end
-
-    teardown do
-      Timecop.return
-    end
-
-    test 'parser returns parsed JSON object, leaving empty/NULL strings, with current time' do
-      json = '{"t1":"1477023720.101","s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
-      @i.parse(json) do |time, record|
-        assert_equal_event_time Fluent::EventTime.new(1477023720, 101_000_000), time
-        assert !record.has_key?("t1")
-        assert{ record.has_key?("s1") && record["s1"].nil? }
-        assert{ record.has_key?("s2") && record["s2"].nil? }
-        assert{ record.has_key?("s3") && record["s3"].nil? }
-        assert_equal "1", record["k1"]
-        assert_equal 13, record["k2"]
-        assert_equal 1.0, record["k3"]
-        assert_equal true, record["k4"]
-      end
-    end
-
-    test 'parser returns current time if a field is missing specified by time_key' do
-      json = '{"s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
-      @i.parse(json) do |time, record|
-        assert_equal_event_time @current_event_time, time
-        assert !record.has_key?("t1")
-        assert{ record.has_key?("s1") && record["s1"].nil? }
-        assert{ record.has_key?("s2") && record["s2"].nil? }
-        assert{ record.has_key?("s3") && record["s3"].nil? }
-        assert_equal "1", record["k1"]
-        assert_equal 13, record["k2"]
-        assert_equal 1.0, record["k3"]
-        assert_equal true, record["k4"]
-      end
-    end
-  end
-
-  sub_test_case 'example parser configured not to estimate current time, and to keep time key' do
-    setup do
-      @current_time = Time.parse("2016-10-21 14:22:01.0 +1000")
-      @current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)
-      # @current_time.to_i #=> 1477023721
-      Timecop.freeze(@current_time)
-      @i = ExampleParser.new
-      hash = {
-        'keep_time_key' => "yes",
-        'estimate_current_event' => "no",
-        'time_key' => "t1",
-        'time_type' => "float",
-        'null_empty_string' => 'yes',
-        'null_value_pattern' => 'NULL|null',
-        'types' => "k1:string, k2:integer, k3:float, k4:bool",
-      }
-      @i.configure(config_element('parse', '', hash))
-    end
-
-    teardown do
-      Timecop.return
-    end
-
-    test 'parser returns parsed time with original field and value if the field of time exists' do
-      json = '{"t1":"1477023720.101","s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
-      @i.parse(json) do |time, record|
-        assert_equal_event_time Fluent::EventTime.new(1477023720, 101_000_000), time
-        assert_equal "1477023720.101", record["t1"]
-        assert{ record.has_key?("s1") && record["s1"].nil? }
-        assert{ record.has_key?("s2") && record["s2"].nil? }
-        assert{ record.has_key?("s3") && record["s3"].nil? }
-        assert_equal "1", record["k1"]
-        assert_equal 13, record["k2"]
-        assert_equal 1.0, record["k3"]
-        assert_equal true, record["k4"]
-      end
-    end
-
-    test 'parser returns nil as time if the field of time is missing' do
-      json = '{"s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
-      @i.parse(json) do |time, record|
-        assert_nil time
-        assert !record.has_key?("t1")
-        assert{ record.has_key?("s1") && record["s1"].nil? }
-        assert{ record.has_key?("s2") && record["s2"].nil? }
-        assert{ record.has_key?("s3") && record["s3"].nil? }
-        assert_equal "1", record["k1"]
-        assert_equal 13, record["k2"]
-        assert_equal 1.0, record["k3"]
-        assert_equal true, record["k4"]
-      end
-    end
-  end
-
-  sub_test_case 'timeout' do
-    class SleepParser < Fluent::Plugin::Parser
-      attr :test_value
-
-      def configure(conf)
-        super
-
-        @test_value = nil
-      end
-
-      def parse(data)
-        sleep 10
-        @test_value = :passed
-        yield JSON.parse(data), Fluent::EventTime.now
-      end
-    end
-
-    setup do
-      @i = SleepParser.new
-      @i.instance_variable_set(:@log, Fluent::Test::TestLogger.new)
-      @i.configure(config_element('parse', '', {'timeout' => '1.0'}))
-      @i.start
-    end
-
-    teardown do
-      @i.stop
-    end
-
-    test 'stop longer processing and return nil' do
-      waiting(10) {
-        @i.parse('{"k":"v"}') do |time, record|
-          assert_nil @i.test_value
-          assert_nil time
-          assert_nil record
-        end
-        assert_true @i.log.out.logs.first.include?('parsing timed out')
-      }
-    end
-  end
-end

data/test/plugin/test_parser_apache.rb

@@ -1,42 +0,0 @@
-require_relative '../helper'
-require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser_apache'
-
-class ApacheParserTest < ::Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-  end
-
-  def create_driver(conf = {})
-    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheParser.new).configure(conf)
-  end
-
-  data('parse' => :parse, 'call' => :call)
-  def test_call(method_name)
-    d = create_driver
-    m = d.instance.method(method_name)
-    m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
-      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
-      assert_equal({
-                     'user'    => '-',
-                     'method'  => 'GET',
-                     'code'    => '200',
-                     'size'    => '777',
-                     'host'    => '192.168.0.1',
-                     'path'    => '/'
-                   }, record)
-    }
-  end
-
-  def test_parse_with_keep_time_key
-    conf = {
-      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
-      'keep_time_key' => 'true',
-    }
-    d = create_driver(conf)
-    text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777'
-    d.instance.parse(text) do |_time, record|
-      assert_equal "28/Feb/2013:12:00:00 +0900", record['time']
-    end
-  end
-end

data/test/plugin/test_parser_apache2.rb

@@ -1,47 +0,0 @@
-require_relative '../helper'
-require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser'
-
-class Apache2ParserTest < ::Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Apache2Parser)
-    @expected = {
-      'user'    => nil,
-      'method'  => 'GET',
-      'code'    => 200,
-      'size'    => 777,
-      'host'    => '192.168.0.1',
-      'path'    => '/',
-      'referer' => nil,
-      'agent'   => 'Opera/12.0'
-    }
-    @parser.configure({})
-  end
-
-  def test_parse
-    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
-      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
-      assert_equal(@expected, record)
-    }
-    assert_equal(Fluent::Plugin::Apache2Parser::REGEXP,
-                 @parser.instance.patterns['format'])
-    assert_equal(Fluent::Plugin::Apache2Parser::TIME_FORMAT,
-                 @parser.instance.patterns['time_format'])
-  end
-
-  def test_parse_without_http_version
-    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
-      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
-      assert_equal(@expected, record)
-    }
-  end
-
-  def test_parse_with_escape_sequence
-    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /\" HTTP/1.1" 200 777 "referer \\\ \"" "user agent \\\ \""') { |_, record|
-      assert_equal('/\"', record['path'])
-      assert_equal('referer \\\ \"', record['referer'])
-      assert_equal('user agent \\\ \"', record['agent'])
-    }
-  end
-end

data/test/plugin/test_parser_apache_error.rb

@@ -1,45 +0,0 @@
-require_relative '../helper'
-require 'fluent/test/driver/parser'
-require 'fluent/plugin/parser_apache_error'
-
-class ApacheErrorParserTest < ::Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-    @expected = {
-      'level' => 'error',
-      'client' => '127.0.0.1',
-      'message' => 'client denied by server configuration'
-    }
-  end
-
-  def create_driver
-    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheErrorParser.new).configure({})
-  end
-
-  def test_parse
-    d = create_driver
-    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
-      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
-      assert_equal(@expected, record)
-    }
-  end
-
-  def test_parse_with_pid
-    d = create_driver
-    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
-      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
-      assert_equal(@expected.merge('pid' => '1000'), record)
-    }
-  end
-
-  def test_parse_without_client
-    d = create_driver
-    d.instance.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
-      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
-      assert_equal({
-                     'level' => 'notice',
-                     'message' => 'Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations'
-                   }, record)
-    }
-  end
-end