fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/test/plugin/test_filter_parser.rb

@@ -209,10 +209,13 @@ class ParserFilterTest < Test::Unit::TestCase
 
   end
 
-  def test_filter_with_reserved_data
+  data(:keep_key_name => false,
+       :remove_key_name => true)
+  def test_filter_with_reserved_data(remove_key_name)
     d1 = create_driver(%[
       key_name data
       reserve_data yes
+      remove_key_name_field #{remove_key_name}
       <parse>
         @type regexp
         expression /^(?<x>\\d)(?<y>\\d) (?<t>.+)$/
@@ -230,6 +233,7 @@ class ParserFilterTest < Test::Unit::TestCase
     d2 = create_driver(%[
       key_name data
       reserve_data yes
+      remove_key_name_field #{remove_key_name}
       <parse>
         @type json
       </parse>
@@ -244,7 +248,11 @@ class ParserFilterTest < Test::Unit::TestCase
 
     first = filtered[0]
     assert_equal_event_time time, first[0]
-    assert_equal '{"xxx":"first","yyy":"second"}', first[1]['data']
+    if remove_key_name
+      assert_not_include first[1], 'data'
+    else
+      assert_equal '{"xxx":"first","yyy":"second"}', first[1]['data']
+    end
     assert_equal 'first', first[1]['xxx']
     assert_equal 'second', first[1]['yyy']
 
@@ -390,6 +398,29 @@ class ParserFilterTest < Test::Unit::TestCase
     assert_equal 'value"ThreeYes!', first[1]['key3']
   end
 
+  def test_filter_with_nested_record
+    d = create_driver(%[
+      key_name $.data.log
+      <parse>
+        @type csv
+        keys key1,key2,key3
+      </parse>
+    ])
+    time = @default_time.to_i
+    d.run do
+      d.feed(@tag, time, {'data' => {'log' => 'value1,"value2","value""ThreeYes!"'}, 'xxx' => 'x', 'yyy' => 'y'})
+    end
+    filtered = d.filtered
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal time, first[0]
+    assert_nil first[1]['data']
+    assert_equal 'value1', first[1]['key1']
+    assert_equal 'value2', first[1]['key2']
+    assert_equal 'value"ThreeYes!', first[1]['key3']
+  end
+
   CONFIG_HASH_VALUE_FIELD = %[
     key_name data
     hash_value_field parsed

data/test/plugin/test_filter_record_transformer.rb

@@ -44,7 +44,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       d = create_driver(config)
       d.run {
         msgs.each { |msg|
-          d.feed(@tag, @time, {'foo' => 'bar', 'message' => msg})
+          d.feed(@tag, @time, {'foo' => 'bar', 'message' => msg, 'nest' => {'k1' => 'v1', 'k2' => 'v2'}})
         }
       }
       d.filtered
@@ -68,6 +68,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         assert_equal(@tag, r['tag'])
         assert_equal(Time.at(@time).localtime.to_s, r['time'])
         assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
+        assert_equal({'k1' => 'v1', 'k2' => 'v2'}, r['nest'])
       end
     end
 
@@ -83,6 +84,14 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       end
     end
 
+    test 'remove_keys with nested key' do
+      config = CONFIG + %[remove_keys $.nest.k1]
+      filtered = filter(config)
+      filtered.each_with_index do |(_t, r), i|
+        assert_not_include(r['nest'], 'k1')
+      end
+    end
+
     test 'renew_record' do
       config = CONFIG + %[renew_record true]
       msgs = ['1', '2']
@@ -137,6 +146,18 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       end
     end
 
+    test 'keep_keys that are not present in the original record should not be included in the result record' do
+      config = %[renew_record true\nkeep_keys foo, bar, baz, message]
+      msgs = ['1', '2', nil]
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
+        assert_equal('bar', r['foo'])
+        assert_equal(msgs[i], r['message'])
+        assert_equal(false, r.has_key?('bar'))
+        assert_equal(false, r.has_key?('baz'))
+      end
+    end
+
     test 'enable_ruby' do
       config = %[
         enable_ruby yes

data/test/plugin/test_formatter_ltsv.rb

@@ -23,14 +23,17 @@ class LabeledTSVFormatterTest < ::Test::Unit::TestCase
     d = create_driver
     assert_equal "\t", d.instance.delimiter
     assert_equal  ":", d.instance.label_delimiter
+    assert_equal  true, d.instance.add_newline
 
     d = create_driver(
       'delimiter'       => ',',
       'label_delimiter' => '=',
+      'add_newline' => false,
     )
 
     assert_equal ",", d.instance.delimiter
     assert_equal "=", d.instance.label_delimiter
+    assert_equal  false, d.instance.add_newline
   end
 
   def test_format

data/test/plugin/test_formatter_tsv.rb

@@ -0,0 +1,68 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_tsv'
+
+class TSVFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::TSVFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome', 'greeting' => 'hello'}
+  end
+
+  def test_config_params
+    d = create_driver(
+      'keys' => 'message,greeting',
+    )
+    assert_equal ["message", "greeting"], d.instance.keys
+    assert_equal "\t", d.instance.delimiter
+    assert_equal true, d.instance.add_newline
+
+    d = create_driver(
+      'keys' => 'message,greeting',
+      'delimiter' => ',',
+      'add_newline' => false,
+    )
+    assert_equal ["message", "greeting"], d.instance.keys
+    assert_equal ",", d.instance.delimiter
+    assert_equal false, d.instance.add_newline
+  end
+
+  def test_format
+    d = create_driver(
+      'keys' => 'message,greeting',
+    )
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("awesome\thello\n", formatted)
+  end
+
+  def test_format_without_newline
+    d = create_driver(
+      'keys' => 'message,greeting',
+      'add_newline' => false,
+    )
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("awesome\thello", formatted)
+  end
+
+  def test_format_with_customized_delimiters
+    d = create_driver(
+      'keys' => 'message,greeting',
+      'delimiter' => ',',
+    )
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("awesome,hello\n", formatted)
+  end
+end

data/test/plugin/test_in_debug_agent.rb

@@ -25,4 +25,25 @@ class DebugAgentInputTest < Test::Unit::TestCase
       create_driver %[unix_path #{TMP_DIR}/does_not_exist/test_path]
     end
   end
+
+  def test_multi_worker_environment_with_port
+    Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
+      d = Fluent::Test::Driver::Input.new(Fluent::Plugin::DebugAgentInput)
+      d.instance.instance_eval { @_fluentd_worker_id = 2 }
+      d.configure('port 24230')
+
+      assert_true d.instance.multi_workers_ready?
+      assert_equal(24232, d.instance.instance_variable_get(:@port))
+    end
+  end
+
+  def test_multi_worker_environment_with_unix_path
+    Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
+      d = Fluent::Test::Driver::Input.new(Fluent::Plugin::DebugAgentInput)
+      d.instance.instance_eval { @_fluentd_worker_id = 2 }
+      d.configure("unix_path #{TMP_DIR}/test_path")
+
+      assert_false d.instance.multi_workers_ready?
+    end
+  end
 end

data/test/plugin/test_in_exec.rb

@@ -131,9 +131,7 @@ EOC
       d = create_driver REGEXP_CONFIG
       assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
       assert_equal "regex_tag", d.instance.tag
-      expression = <<'EXP'.chomp
-(?<time>[^\]]*) (?<message>[^ ]*)
-EXP
+      expression = /(?<time>[^\]]*) (?<message>[^ ]*)/
       assert_equal expression, d.instance.parser.expression
       assert_nil d.instance.extract_config
     end
@@ -194,8 +192,8 @@ EXP
     test 'configure_with_regexp' do
       d = create_driver REGEXP_CONFIG_COMPAT
       assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
-      assert_equal '(?<time>[^\]]*) (?<message>[^ ]*)', d.instance.parser.expression
-      assert_equal 'regex_tag', d.instance.tag
+      assert_equal(/(?<time>[^\]]*) (?<message>[^ ]*)/, d.instance.parser.expression)
+      assert_equal('regex_tag', d.instance.tag)
     end
   end
 

data/test/plugin/test_in_http.rb

@@ -137,6 +137,54 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_multi_json_with_time_field
+    d = create_driver
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    time_f = time.to_f
+
+    records = [{"a" => 1, 'time' => time_i},{"a" => 2, 'time' => time_f}]
+    events = [
+      ["tag1", time, {'a' => 1}],
+      ["tag1", time, {'a' => 2}],
+    ]
+    tag = "tag1"
+    res_codes = []
+    d.run(expect_records: 2, timeout: 5) do
+      res = post("/#{tag}", {"json" => records.to_json})
+      res_codes << res.code
+    end
+    assert_equal ["200"], res_codes
+    assert_equal events, d.events
+    assert_instance_of Fluent::EventTime, d.events[0][1]
+    assert_instance_of Fluent::EventTime, d.events[1][1]
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
+  def test_multi_json_with_nonexistent_time_key
+    d = create_driver(CONFIG + %[
+      <parse>
+        time_key missing
+      </parse>
+    ])
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    time_f = time.to_f
+
+    records = [{"a" => 1, 'time' => time_i},{"a" => 2, 'time' => time_f}]
+    tag = "tag1"
+    res_codes = []
+    d.run(expect_records: 2, timeout: 5) do
+      res = post("/#{tag}", {"json" => records.to_json})
+      res_codes << res.code
+    end
+    assert_equal ["200"], res_codes
+    assert_equal 2, d.events.size
+    assert_not_equal time_i, d.events[0][1].sec # current time is used because "missing" field doesn't exist
+    assert_not_equal time_i, d.events[1][1].sec
+  end
+
   def test_json_with_add_remote_addr
     d = create_driver(CONFIG + "add_remote_addr true")
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -322,6 +370,38 @@ class HttpInputTest < Test::Unit::TestCase
     assert include_http_header?(d.events[1][2])
   end
 
+  def test_multi_json_with_custom_parser
+    d = create_driver(CONFIG + %[
+        <parse>
+          @type json
+          keep_time_key true
+          time_key foo
+          time_format %iso8601
+        </parse>
+    ])
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_s = Time.at(time).iso8601
+
+    records = [{"foo"=>time_s,"bar"=>"test1"},{"foo"=>time_s,"bar"=>"test2"}]
+    tag = "tag1"
+    res_codes = []
+
+    d.run(expect_records: 2, timeout: 5) do
+      res = post("/#{tag}", records.to_json, {"Content-Type"=>"application/octet-stream"})
+      res_codes << res.code
+    end
+    assert_equal ["200"], res_codes
+
+    assert_equal "tag1", d.events[0][0]
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal d.events[0][2], records[0]
+
+    assert_equal "tag1", d.events[1][0]
+    assert_equal_event_time time, d.events[1][1]
+    assert_equal d.events[1][2], records[1]
+  end
+
   def test_application_json
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -524,6 +604,91 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_cors_allowed_wildcard
+    d = create_driver(CONFIG + 'cors_allow_origins ["*"]')
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    events = [
+      ["tag1", time, {"a"=>1}],
+    ]
+
+    d.run do
+      events.each do |tag, time, record|
+        headers = {"Origin" => "http://foo.com"}
+
+        res = post("/#{tag}", {"json" => record.to_json, "time" => time.to_i}, headers)
+
+        assert_equal "200", res.code
+        assert_equal "*", res["Access-Control-Allow-Origin"]
+      end
+    end
+  end
+
+  def test_cors_preflight
+    d = create_driver(CONFIG + 'cors_allow_origins ["*"]')
+
+    d.run do
+      header = {
+        "Origin" => "http://foo.com",
+        "Access-Control-Request-Method" => "POST",
+        "Access-Control-Request-Headers" => "Content-Type",
+      }
+      res = options("/cors.test", {}, header)
+
+      assert_equal "200", res.code
+      assert_equal "*", res["Access-Control-Allow-Origin"]
+      assert_equal "POST", res["Access-Control-Allow-Methods"]
+    end
+  end
+
+  def test_content_encoding_gzip
+    d = create_driver
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}],
+    ]
+    res_codes = []
+    res_headers = []
+
+    d.run do
+      events.each do |tag, time, record|
+        header = {'Content-Type'=>'application/json', 'Content-Encoding'=>'gzip'}
+        res = post("/#{tag}?time=#{time}", compress_gzip(record.to_json), header)
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
+  def test_content_encoding_deflate
+    d = create_driver
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}],
+    ]
+    res_codes = []
+    res_headers = []
+
+    d.run do
+      events.each do |tag, time, record|
+        header = {'Content-Type'=>'application/msgpack', 'Content-Encoding'=>'deflate'}
+        res = post("/#{tag}?time=#{time}", Zlib.deflate(record.to_msgpack), header)
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_cors_disallowed
     d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins
@@ -596,6 +761,12 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal $test_in_http_connection_object_ids[0], $test_in_http_connection_object_ids[1]
   end
 
+  def options(path, params, header = {})
+    http = Net::HTTP.new("127.0.0.1", PORT)
+    req = Net::HTTP::Options.new(path, header)
+    http.request(req)
+  end
+
   def post(path, params, header = {}, &block)
     http = Net::HTTP.new("127.0.0.1", PORT)
     req = Net::HTTP::Post.new(path, header)
@@ -614,6 +785,13 @@ class HttpInputTest < Test::Unit::TestCase
     http.request(req)
   end
 
+  def compress_gzip(data)
+    io = StringIO.new
+    io.binmode
+    Zlib::GzipWriter.wrap(io) { |gz| gz.write data }
+    return io.string
+  end
+
   def include_http_header?(record)
     record.keys.find { |header| header.start_with?('HTTP_') }
   end