fluentd 1.11.1-x86-mingw32 → 1.12.0-x86-mingw32

data/test/plugin/test_buffer.rb

@@ -543,6 +543,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
       assert_equal [@dm2,@dm3,m], @p.stage.keys
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 
@@ -675,6 +677,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm2,@dm3,m], @p.stage.keys
       assert_equal 1, @p.stage[m].append_count
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 

data/test/plugin/test_filter_stdout.rb

@@ -12,6 +12,11 @@ class StdoutFilterTest < Test::Unit::TestCase
     @old_tz = ENV["TZ"]
     ENV["TZ"] = "UTC"
     Timecop.freeze
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   def teardown
@@ -106,7 +111,7 @@ class StdoutFilterTest < Test::Unit::TestCase
     def test_format_json
       d = create_driver(CONFIG + config_element("", "", { "format" => "json" }))
       out = capture_log(d) { filter(d, event_time, {'test' => 'test'}) }
-      assert_equal "{\"test\":\"test\"}\n", out
+      assert_equal "{\"test\":\"test\"}#{@default_newline}", out
     end
   end
 

data/test/plugin/test_formatter_hash.rb

@@ -19,11 +19,14 @@ class HashFormatterTest < ::Test::Unit::TestCase
     {'message' => 'awesome', 'greeting' => 'hello'}
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}\n!, formatted.encode(Encoding::UTF_8))
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}#{newline}!, formatted.encode(Encoding::UTF_8))
   end
 
   def test_format_without_newline

data/test/plugin/test_formatter_json.rb

@@ -7,6 +7,11 @@ class JsonFormatterTest < ::Test::Unit::TestCase
 
   def setup
     @time = event_time
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   def create_driver(conf = "")
@@ -25,12 +30,17 @@ class JsonFormatterTest < ::Test::Unit::TestCase
     {:message => :awesome}
   end
 
-  data('oj' => 'oj', 'yajl' => 'yajl')
+  data('oj with LF' => ['oj', "lf", "\n"],
+       'oj with CRLF' => ['oj', "crlf", "\r\n"],
+       'yajl with LF' => ['yajl', "lf", "\n"],
+       'yajl with CRLF' => ['yajl', "crlf", "\r\n"]
+      )
   def test_format(data)
-    d = create_driver('json_parser' => data)
+    parser, newline_conf, newline = data
+    d = create_driver('json_parser' => parser, 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{JSON.generate(record)}\n", formatted)
+    assert_equal("#{JSON.generate(record)}#{newline}", formatted)
   end
 
   data('oj' => 'oj', 'yajl' => 'yajl')
@@ -46,6 +56,6 @@ class JsonFormatterTest < ::Test::Unit::TestCase
     d = create_driver('json_parser' => data)
     formatted = d.instance.format(tag, @time, symbolic_record)
 
-    assert_equal("#{JSON.generate(record)}\n", formatted)
+    assert_equal("#{JSON.generate(record)}#{@default_newline}", formatted)
   end
 end

data/test/plugin/test_formatter_ltsv.rb

@@ -36,11 +36,14 @@ class LabeledTSVFormatterTest < ::Test::Unit::TestCase
     assert_equal  false, d.instance.add_newline
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("message:awesome\tgreeting:hello\n", formatted)
+    assert_equal("message:awesome\tgreeting:hello#{newline}", formatted)
   end
 
   def test_format_without_newline
@@ -50,13 +53,18 @@ class LabeledTSVFormatterTest < ::Test::Unit::TestCase
     assert_equal("message:awesome\tgreeting:hello", formatted)
   end
 
-  def test_format_with_customized_delimiters
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_with_customized_delimiters(data)
+    newline_conf, newline = data
+
     d = create_driver(
       'delimiter'       => ',',
       'label_delimiter' => '=',
+      'newline' => newline_conf,
     )
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("message=awesome,greeting=hello\n", formatted)
+    assert_equal("message=awesome,greeting=hello#{newline}", formatted)
   end
 end

data/test/plugin/test_formatter_out_file.rb

@@ -5,6 +5,11 @@ require 'fluent/plugin/formatter_out_file'
 class OutFileFormatterTest < ::Test::Unit::TestCase
   def setup
     @time = event_time
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   def create_driver(conf = {})
@@ -48,48 +53,64 @@ class OutFileFormatterTest < ::Test::Unit::TestCase
       oldtz, ENV['TZ'] = ENV['TZ'], "UTC+07"
       d = create_driver(config_element('ROOT', '', {key => value}))
       tag = 'test'
-      assert_equal "#{expected}\t#{tag}\t#{Yajl.dump(record)}\n", d.instance.format(tag, time, record)
+      assert_equal "#{expected}\t#{tag}\t#{Yajl.dump(record)}#{@default_newline}", d.instance.format(tag, time, record)
     ensure
       ENV['TZ'] = oldtz
     end
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}#{newline}", formatted)
   end
 
-  def test_format_without_time
-    d = create_driver('output_time' => 'false')
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_without_time(data)
+    newline_conf, newline = data
+    d = create_driver('output_time' => 'false', 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{tag}\t#{Yajl.dump(record)}#{newline}", formatted)
   end
 
-  def test_format_without_tag
-    d = create_driver('output_tag' => 'false')
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_without_tag(data)
+    newline_conf, newline = data
+    d = create_driver('output_tag' => 'false', 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}#{newline}", formatted)
   end
 
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
   def test_format_without_time_and_tag
-    d = create_driver('output_tag' => 'false', 'output_time' => 'false')
+    newline_conf, newline = data
+    d = create_driver('output_tag' => 'false', 'output_time' => 'false', 'newline' => newline_conf)
     formatted = d.instance.format('tag', @time, record)
 
-    assert_equal("#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{Yajl.dump(record)}#{newline}", formatted)
   end
 
-  def test_format_without_time_and_tag_against_string_literal_configure
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_without_time_and_tag_against_string_literal_configure(data)
+    newline_conf, newline = data
     d = create_driver(%[
                         utc         true
                         output_tag  false
                         output_time false
+                        newline     #{newline_conf}
                       ])
     formatted = d.instance.format('tag', @time, record)
 
-    assert_equal("#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{Yajl.dump(record)}#{newline}", formatted)
   end
 end

data/test/plugin/test_formatter_single_value.rb

@@ -17,10 +17,13 @@ class SingleValueFormatterTest < ::Test::Unit::TestCase
     assert_equal "foobar", d.instance.message_key
   end
 
-  def test_format
-    d = create_driver
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver('newline' => newline_conf)
     formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})
-    assert_equal("awesome\n", formatted)
+    assert_equal("awesome#{newline}", formatted)
   end
 
   def test_format_without_newline
@@ -29,10 +32,13 @@ class SingleValueFormatterTest < ::Test::Unit::TestCase
     assert_equal("awesome", formatted)
   end
 
-  def test_format_with_message_key
-    d = create_driver('message_key' => 'foobar')
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_with_message_key(data)
+    newline_conf, newline = data
+    d = create_driver('message_key' => 'foobar', 'newline' => newline_conf)
     formatted = d.instance.format('tag', event_time, {'foobar' => 'foo'})
 
-    assert_equal("foo\n", formatted)
+    assert_equal("foo#{newline}", formatted)
   end
 end

data/test/plugin/test_formatter_tsv.rb

@@ -37,13 +37,17 @@ class TSVFormatterTest < ::Test::Unit::TestCase
     assert_equal false, d.instance.add_newline
   end
 
-  def test_format
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
     d = create_driver(
       'keys' => 'message,greeting',
+      'newline' => newline_conf
     )
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("awesome\thello\n", formatted)
+    assert_equal("awesome\thello#{newline}", formatted)
   end
 
   def test_format_without_newline
@@ -56,13 +60,17 @@ class TSVFormatterTest < ::Test::Unit::TestCase
     assert_equal("awesome\thello", formatted)
   end
 
-  def test_format_with_customized_delimiters
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_with_customized_delimiters(data)
+    newline_conf, newline = data
     d = create_driver(
       'keys' => 'message,greeting',
       'delimiter' => ',',
+      'newline' => newline_conf,
     )
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("awesome,hello\n", formatted)
+    assert_equal("awesome,hello#{newline}", formatted)
   end
 end

data/test/plugin/test_in_exec.rb

@@ -240,4 +240,22 @@ EOC
       assert_equal [tag, time, record], event
     }
   end
+
+  test 'emit error message with read_with_stderr' do
+    d = create_driver %[
+      tag test
+      command ruby #{File.join(File.dirname(SCRIPT_PATH), 'foo_bar_baz_no_existence.rb')}
+      connect_mode read_with_stderr
+      <parse>
+        @type none
+      </parse>
+    ]
+    d.run(expect_records: 1, timeout: 10)
+
+    assert{ d.events.length > 0 }
+    d.events.each do |event|
+      assert_equal 'test', event[0]
+      assert_match /LoadError/, event[2]['message']
+    end
+  end
 end

data/test/plugin/test_in_http.rb

@@ -46,6 +46,7 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal 10*1024*1024, d.instance.body_size_limit
     assert_equal 5, d.instance.keepalive_timeout
     assert_equal false, d.instance.add_http_headers
+    assert_equal false, d.instance.add_query_params
   end
 
   def test_time
@@ -907,6 +908,30 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal ["403", "403"], res_codes
   end
 
+  def test_add_query_params 
+    d = create_driver(CONFIG + "add_query_params true")
+    assert_equal true, d.instance.add_query_params
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    events = [
+      ["tag1", time, {"a"=>1, "QUERY_A"=>"b"}],
+      ["tag2", time, {"a"=>2, "QUERY_A"=>"b"}],
+    ]
+    res_codes = []
+    res_bodies = []
+
+    d.run do
+      events.each do |tag, _t, record|
+        res = post("/#{tag}?a=b", {"json"=>record.to_json, "time"=>time_i.to_s})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal [], res_bodies
+    assert_equal events, d.events
+  end
+
   $test_in_http_connection_object_ids = []
   $test_in_http_content_types = []
   $test_in_http_content_types_flag = false

data/test/plugin/{test_in_dummy.rb → test_in_sample.rb}

@@ -1,33 +1,33 @@
 require_relative '../helper'
 require 'fluent/test/driver/input'
-require 'fluent/plugin/in_dummy'
+require 'fluent/plugin/in_sample'
 require 'fileutils'
 
-class DummyTest < Test::Unit::TestCase
+class SampleTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
   def create_driver(conf)
-    Fluent::Test::Driver::Input.new(Fluent::Plugin::DummyInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SampleInput).configure(conf)
   end
 
   sub_test_case 'configure' do
     test 'required parameters' do
       assert_raise_message("'tag' parameter is required") do
-        Fluent::Plugin::DummyInput.new.configure(config_element('ROOT',''))
+        Fluent::Plugin::SampleInput.new.configure(config_element('ROOT',''))
       end
     end
 
     test 'tag' do
       d = create_driver(%[
-        tag dummy
+        tag sample
       ])
-      assert_equal "dummy", d.instance.tag
+      assert_equal "sample", d.instance.tag
     end
 
     config = %[
-      tag dummy
+      tag sample
     ]
 
     test 'auto_increment_key' do
@@ -44,30 +44,30 @@ class DummyTest < Test::Unit::TestCase
       assert_equal 10, d.instance.rate
     end
 
-    test 'dummy' do
+    test 'sample' do
       # hash is okay
-      d = create_driver(config + %[dummy {"foo":"bar"}])
-      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+      d = create_driver(config + %[sample {"foo":"bar"}])
+      assert_equal [{"foo"=>"bar"}], d.instance.sample
 
       # array of hash is okay
-      d = create_driver(config + %[dummy [{"foo":"bar"}]])
-      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+      d = create_driver(config + %[sample [{"foo":"bar"}]])
+      assert_equal [{"foo"=>"bar"}], d.instance.sample
 
       assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
-        create_driver(config + %[dummy "foo"])
+        create_driver(config + %[sample "foo"])
       end
 
       assert_raise_message(/is not a hash/) do
-        create_driver(config + %[dummy ["foo"]])
+        create_driver(config + %[sample ["foo"]])
       end
     end
   end
 
   sub_test_case "emit" do
     config = %[
-      tag dummy
+      tag sample
       rate 10
-      dummy {"foo":"bar"}
+      sample {"foo":"bar"}
     ]
 
     test 'simple' do
@@ -75,7 +75,7 @@ class DummyTest < Test::Unit::TestCase
       d.run(timeout: 0.5)
 
       d.events.each do |tag, time, record|
-        assert_equal("dummy", tag)
+        assert_equal("sample", tag)
         assert_equal({"foo"=>"bar"}, record)
         assert(time.is_a?(Fluent::EventTime))
       end
@@ -86,20 +86,20 @@ class DummyTest < Test::Unit::TestCase
       d.run(timeout: 0.5)
 
       d.events.each_with_index do |(tag, _time, record), i|
-        assert_equal("dummy", tag)
+        assert_equal("sample", tag)
         assert_equal({"foo"=>"bar", "id"=>i}, record)
       end
     end
   end
 
-  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
+  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_sample', 'store' )
   FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
 
-  sub_test_case 'when dummy plugin has storage which is not specified the path'  do
+  sub_test_case 'when sample plugin has storage which is not specified the path'  do
     config1 = {
-      'tag' => 'dummy',
+      'tag' => 'sample',
       'rate' => '0',
-      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
       'auto_increment_key' => 'id',
     }
     conf1 = config_element('ROOT', '', config1, [])
@@ -135,7 +135,7 @@ class DummyTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'when dummy plugin has storage which is specified the path'  do
+  sub_test_case 'when sample plugin has storage which is specified the path'  do
     setup do
       FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
       FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
@@ -144,9 +144,9 @@ class DummyTest < Test::Unit::TestCase
 
     config2 = {
       '@id' => 'test-02',
-      'tag' => 'dummy',
+      'tag' => 'sample',
       'rate' => '0',
-      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
       'auto_increment_key' => 'id',
     }
     conf2 = config_element('ROOT', '', config2, [