fluentd-hubspot 0.14.14.1

data/test/plugin/test_parser_apache.rb

@@ -0,0 +1,42 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser_apache'
+
+class ApacheParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = {})
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheParser.new).configure(conf)
+  end
+
+  data('parse' => :parse, 'call' => :call)
+  def test_call(method_name)
+    d = create_driver
+    m = d.instance.method(method_name)
+    m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal({
+                     'user'    => '-',
+                     'method'  => 'GET',
+                     'code'    => '200',
+                     'size'    => '777',
+                     'host'    => '192.168.0.1',
+                     'path'    => '/'
+                   }, record)
+    }
+  end
+
+  def test_parse_with_keep_time_key
+    conf = {
+      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+      'keep_time_key' => 'true',
+    }
+    d = create_driver(conf)
+    text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777'
+    d.instance.parse(text) do |_time, record|
+      assert_equal "28/Feb/2013:12:00:00 +0900", record['time']
+    end
+  end
+end

data/test/plugin/test_parser_apache2.rb

@@ -0,0 +1,46 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser'
+
+class Apache2ParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::Apache2Parser)
+    @expected = {
+      'user'    => nil,
+      'method'  => 'GET',
+      'code'    => 200,
+      'size'    => 777,
+      'host'    => '192.168.0.1',
+      'path'    => '/',
+      'referer' => nil,
+      'agent'   => 'Opera/12.0'
+    }
+  end
+
+  def test_parse
+    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal(@expected, record)
+    }
+    assert_equal(Fluent::Plugin::Apache2Parser::REGEXP,
+                 @parser.instance.patterns['format'])
+    assert_equal(Fluent::Plugin::Apache2Parser::TIME_FORMAT,
+                 @parser.instance.patterns['time_format'])
+  end
+
+  def test_parse_without_http_version
+    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal(@expected, record)
+    }
+  end
+
+  def test_parse_with_escape_sequence
+    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /\" HTTP/1.1" 200 777 "referer \\\ \"" "user agent \\\ \""') { |_, record|
+      assert_equal('/\"', record['path'])
+      assert_equal('referer \\\ \"', record['referer'])
+      assert_equal('user agent \\\ \"', record['agent'])
+    }
+  end
+end

data/test/plugin/test_parser_apache_error.rb

@@ -0,0 +1,45 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser_apache_error'
+
+class ApacheErrorParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @expected = {
+      'level' => 'error',
+      'client' => '127.0.0.1',
+      'message' => 'client denied by server configuration'
+    }
+  end
+
+  def create_driver
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheErrorParser.new).configure({})
+  end
+
+  def test_parse
+    d = create_driver
+    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
+      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
+      assert_equal(@expected, record)
+    }
+  end
+
+  def test_parse_with_pid
+    d = create_driver
+    d.instance.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
+      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
+      assert_equal(@expected.merge('pid' => '1000'), record)
+    }
+  end
+
+  def test_parse_without_client
+    d = create_driver
+    d.instance.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
+      assert_equal(event_time('Wed Oct 11 14:32:52 2000'), time)
+      assert_equal({
+                     'level' => 'notice',
+                     'message' => 'Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations'
+                   }, record)
+    }
+  end
+end

data/test/plugin/test_parser_csv.rb

@@ -0,0 +1,103 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser_csv'
+
+class CSVParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf={})
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser).configure(conf)
+  end
+
+  data('array param' => '["time","c","d"]', 'string param' => 'time,c,d')
+  def test_parse(param)
+    d = create_driver('keys' => param, 'time_key' => 'time')
+    d.instance.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
+      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
+      assert_equal({
+                     'c' => '192.168.0.1',
+                     'd' => '111',
+                   }, record)
+    }
+  end
+
+  data('array param' => '["c","d"]', 'string param' => 'c,d')
+  def test_parse_without_time(param)
+    time_at_start = Time.now.to_i
+
+    d = create_driver('keys' => param)
+    d.instance.parse("192.168.0.1,111") { |time, record|
+      assert time && time >= time_at_start, "parser puts current time without time input"
+      assert_equal({
+                     'c' => '192.168.0.1',
+                     'd' => '111',
+                   }, record)
+    }
+
+    d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
+    d.configure('keys' => param, 'estimate_current_event' => 'no')
+    d.instance.parse("192.168.0.1,111") { |time, record|
+      assert_equal({
+                     'c' => '192.168.0.1',
+                     'd' => '111',
+                   }, record)
+      assert_nil time, "parser returns nil w/o time and if configured so"
+    }
+  end
+
+  def test_parse_with_keep_time_key
+    d = create_driver(
+                     'keys'=>'time',
+                     'time_key'=>'time',
+                     'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+                     'keep_time_key'=>'true',
+                     )
+    text = '28/Feb/2013:12:00:00 +0900'
+    d.instance.parse(text) do |time, record|
+      assert_equal text, record['time']
+    end
+  end
+
+  data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
+  def test_parse_with_null_value_pattern(param)
+    d = create_driver(
+                     'keys'=>param,
+                     'null_value_pattern'=>'^(-|null|NULL)$'
+                     )
+    d.instance.parse("-,null,NULL,,--,nuLL") do |time, record|
+      assert_nil record['a']
+      assert_nil record['b']
+      assert_nil record['c']
+      assert_nil record['d']
+      assert_equal record['e'], '--'
+      assert_equal record['f'], 'nuLL'
+    end
+  end
+
+  data('array param' => '["a","b"]', 'string param' => 'a,b')
+  def test_parse_with_null_empty_string(param)
+    d = create_driver(
+                     'keys'=>param,
+                     'null_empty_string'=>true
+                     )
+    d.instance.parse(", ") do |time, record|
+      assert_nil record['a']
+      assert_equal record['b'], ' '
+    end
+  end
+
+  data('array param' => '["a","b","c"]', 'string param' => 'a,b,c')
+  def test_parse_with_option_delimiter(param)
+    d = create_driver(
+                     'keys'=>param,
+                     'delimiter'=>' ',
+                     )
+    d.instance.parse("123 456 789") do |time, record|
+      assert_equal record['a'], '123'
+      assert_equal record['b'], '456'
+      assert_equal record['c'], '789'
+    end
+  end
+end

data/test/plugin/test_parser_json.rb

@@ -0,0 +1,114 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser'
+
+class JsonParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_parse(data)
+    @parser.configure('json_parser' => data)
+    @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
+      assert_equal(event_time('2013-02-28 12:00:00 +0900').to_i, time)
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'size'   => 777,
+                     'method' => 'PUT',
+                   }, record)
+    }
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_parse_with_large_float(data)
+    @parser.configure('json_parser' => data)
+    @parser.instance.parse('{"num":999999999999999999999999999999.99999}') { |time, record|
+      assert_equal(Float, record['num'].class)
+    }
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_parse_without_time(data)
+    time_at_start = Time.now.to_i
+
+    @parser.configure('json_parser' => data)
+    @parser.instance.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
+      assert time && time >= time_at_start, "parser puts current time without time input"
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'size'   => 777,
+                     'method' => 'PUT',
+                   }, record)
+    }
+
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
+    parser.configure('json_parser' => data, 'estimate_current_event' => 'false')
+    parser.instance.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'size'   => 777,
+                     'method' => 'PUT',
+                   }, record)
+      assert_nil time, "parser return nil w/o time and if specified so"
+    }
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_parse_with_colon_string(data)
+    @parser.configure('json_parser' => data)
+    @parser.instance.parse('{"time":1362020400,"log":":message"}') { |time, record|
+      assert_equal(record['log'], ':message')
+    }
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_parse_with_invalid_time(data)
+    @parser.configure('json_parser' => data)
+    assert_raise Fluent::ParserError do
+      @parser.instance.parse('{"time":[],"k":"v"}') { |time, record| }
+    end
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_parse_float_time(data)
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
+    parser.configure('json_parser' => data)
+    text = "100.1"
+    parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
+      assert_equal 100, time.sec
+      assert_equal 100_000_000, time.nsec
+    end
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_parse_with_keep_time_key(data)
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
+    format = "%d/%b/%Y:%H:%M:%S %z"
+    parser.configure(
+                     'time_format' => format,
+                     'keep_time_key' => 'true',
+                     'json_parser' => data
+                     )
+    text = "28/Feb/2013:12:00:00 +0900"
+    parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
+      assert_equal Time.strptime(text, format).to_i, time.sec
+      assert_equal text, record['time']
+    end
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_parse_with_keep_time_key_without_time_format(data)
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
+    parser.configure(
+                     'keep_time_key' => 'true',
+                     'json_parser' => data
+                     )
+    text = "100"
+    parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
+      assert_equal text.to_i, time.sec
+      assert_equal text, record['time']
+    end
+  end
+end

data/test/plugin/test_parser_labeled_tsv.rb

@@ -0,0 +1,128 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser'
+
+class LabeledTSVParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def test_config_params
+    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
+
+    assert_equal "\t", parser.instance.delimiter
+    assert_equal  ":", parser.instance.label_delimiter
+
+    parser.configure(
+                     'delimiter'       => ',',
+                     'label_delimiter' => '=',
+                     )
+
+    assert_equal ",", parser.instance.delimiter
+    assert_equal "=", parser.instance.label_delimiter
+  end
+
+  def test_parse
+    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
+    parser.configure({})
+    parser.instance.parse("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111") { |time, record|
+      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'req_id' => '111',
+                   }, record)
+    }
+  end
+
+  def test_parse_with_customized_delimiter
+    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
+    parser.configure(
+                     'delimiter'       => ',',
+                     'label_delimiter' => '=',
+                     )
+    parser.instance.parse('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111') { |time, record|
+      assert_equal(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'req_id' => '111',
+                   }, record)
+    }
+  end
+
+  def test_parse_with_customized_time_format
+    parser = Fluent::Test::Driver::Parser.new(Fluent::TextParser::LabeledTSVParser)
+    parser.configure(
+                     'time_key'    => 'mytime',
+                     'time_format' => '%d/%b/%Y:%H:%M:%S %z',
+                     )
+    parser.instance.parse("mytime:28/Feb/2013:12:00:00 +0900\thost:192.168.0.1\treq_id:111") { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'req_id' => '111',
+                   }, record)
+    }
+  end
+
+  def test_parse_without_time
+    time_at_start = Time.now.to_i
+
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
+    parser.configure({})
+    parser.instance.parse("host:192.168.0.1\treq_id:111") { |time, record|
+      assert time && time >= time_at_start, "parser puts current time without time input"
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'req_id' => '111',
+                   }, record)
+    }
+
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
+    parser.configure({'estimate_current_event' => 'no'})
+    parser.instance.parse("host:192.168.0.1\treq_id:111") { |time, record|
+      assert_equal({
+                     'host'   => '192.168.0.1',
+                     'req_id' => '111',
+                   }, record)
+      assert_nil time, "parser returns nil w/o time and if configured so"
+    }
+  end
+
+  def test_parse_with_keep_time_key
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
+    parser.configure(
+                     'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+                     'keep_time_key'=>'true',
+                     )
+    text = '28/Feb/2013:12:00:00 +0900'
+    parser.instance.parse("time:#{text}") do |time, record|
+      assert_equal text, record['time']
+    end
+  end
+
+  def test_parse_with_null_value_pattern
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
+    parser.configure(
+                     'null_value_pattern'=>'^(-|null|NULL)$'
+                     )
+    parser.instance.parse("a:-\tb:null\tc:NULL\td:\te:--\tf:nuLL") do |time, record|
+      assert_nil record['a']
+      assert_nil record['b']
+      assert_nil record['c']
+      assert_equal record['d'], ''
+      assert_equal record['e'], '--'
+      assert_equal record['f'], 'nuLL'
+    end
+  end
+
+  def test_parse_with_null_empty_string
+    parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
+    parser.configure(
+                     'null_empty_string'=>true
+                     )
+    parser.instance.parse("a:\tb: ") do |time, record|
+      assert_nil record['a']
+      assert_equal record['b'], ' '
+    end
+  end
+end