fluentd 0.14.11 → 0.14.12

data/test/plugin/test_filter_record_transformer.rb

@@ -104,6 +104,26 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       filtered.each_with_index do |(time, _record), i|
         assert_equal(times[i].to_i, time)
         assert(time.is_a?(Fluent::EventTime))
+        assert_true(_record.has_key?('message'))
+      end
+    end
+
+    test 'renew_time_key and remove_keys' do
+      config = %[
+                 renew_time_key event_time_key
+                 remove_keys event_time_key
+                 auto_typecast true
+                 <record>
+                   event_time_key ${record["message"]}
+                 </record>
+               ]
+      times = [Time.local(2, 2, 3, 4, 5, 2010, nil, nil, nil, nil), Time.local(3, 2, 3, 4, 5, 2010, nil, nil, nil, nil)]
+      msgs = times.map { |t| t.to_f.to_s }
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(time, _record), i|
+        assert_equal(times[i].to_i, time)
+        assert(time.is_a?(Fluent::EventTime))
+        assert_false(_record.has_key?('event_time_key'))
       end
     end
 

data/test/plugin/test_formatter_csv.rb

@@ -54,6 +54,15 @@ class CsvFormatterTest < ::Test::Unit::TestCase
     assert_equal("\"awesome\",\"awesome2\"\n", formatted)
   end
 
+  def test_format_without_newline
+    d = create_driver("fields" => "message,message2", "add_newline" => false)
+    formatted = d.instance.format(tag, @time, {
+                                    'message' => 'awesome',
+                                    'message2' => 'awesome2'
+                                  })
+    assert_equal("\"awesome\",\"awesome2\"", formatted)
+  end
+
   def test_format_with_customized_delimiters
     d = create_driver("fields" => "message,message2",
                       "delimiter" => "\t")

data/test/plugin/test_formatter_hash.rb

@@ -0,0 +1,35 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_hash'
+
+class HashFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::HashFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome', 'greeting' => 'hello'}
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}\n!, formatted.encode(Encoding::UTF_8))
+  end
+
+  def test_format_without_newline
+    d = create_driver('add_newline' => false)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}!, formatted.encode(Encoding::UTF_8))
+  end
+end

data/test/plugin/test_formatter_json.rb

@@ -33,6 +33,14 @@ class JsonFormatterTest < ::Test::Unit::TestCase
     assert_equal("#{JSON.generate(record)}\n", formatted)
   end
 
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_format_without_nl(data)
+    d = create_driver('json_parser' => data, 'add_newline' => false)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(JSON.generate(record), formatted)
+  end
+
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_format_with_symbolic_record(data)
     d = create_driver('json_parser' => data)

data/test/plugin/test_formatter_ltsv.rb

@@ -40,6 +40,13 @@ class LabeledTSVFormatterTest < ::Test::Unit::TestCase
     assert_equal("message:awesome\tgreeting:hello\n", formatted)
   end
 
+  def test_format_without_newline
+    d = create_driver('add_newline' => false)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("message:awesome\tgreeting:hello", formatted)
+  end
+
   def test_format_with_customized_delimiters
     d = create_driver(
       'delimiter'       => ',',

data/test/plugin/test_in_dummy.rb

@@ -112,10 +112,12 @@ class DummyTest < Test::Unit::TestCase
         d1.instance.emit(4)
       end
 
-      first_id1 = d1.events.first[2]['id']
+      events = d1.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }
+
+      first_id1 = events.first[2]['id']
       assert_equal 0, first_id1
 
-      last_id1 = d1.events.last[2]['id']
+      last_id1 = events.last[2]['id']
       assert { last_id1 > 0 }
 
       assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
@@ -125,7 +127,9 @@ class DummyTest < Test::Unit::TestCase
         d2.instance.emit(4)
       end
 
-      first_id2 = d2.events.first[2]['id']
+      events = d2.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }
+
+      first_id2 = events.first[2]['id']
       assert_equal 0, first_id2
 
       assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))

data/test/plugin/test_in_monitor_agent.rb

@@ -350,6 +350,39 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       assert_equal(expected_null_response, null_response)
     end
 
+    test "/api/plugins.json with 'with_ivars'. response contains specified instance variables of each plugin" do
+      d = create_driver("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{@port}
+  tag monitor
+")
+      d.instance.start
+      expected_test_in_response = {
+        "output_plugin"   => false,
+        "plugin_category" => "input",
+        "plugin_id"       => "test_in",
+        "retry_count"     => nil,
+        "type"            => "test_in",
+        "instance_variables" => {"id" => "test_in"}
+      }
+      expected_null_response = {
+        "buffer_queue_length" => 0,
+        "buffer_total_queued_size" => 0,
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "null",
+        "retry_count"     => 0,
+        "type"            => "null",
+        "instance_variables" => {"id" => "null", "num_errors" => 0}
+      }
+      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json?with_config=no&with_retry=no&with_ivars=id,num_errors"))
+      test_in_response = response["plugins"][0]
+      null_response = response["plugins"][5]
+      assert_equal(expected_test_in_response, test_in_response)
+      assert_equal(expected_null_response, null_response)
+    end
+
     test "/api/config" do
       d = create_driver("
   @type monitor_agent
@@ -427,11 +460,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
           "output_plugin" => true,
           "plugin_category" => "output",
           "plugin_id" => "test_out_fail_write",
-          "retry_count" => 2,
           "type" => "test_out_fail_write",
-          "retry" => {
-              "steps" => 1
-          }
       }
       output = @ra.outputs[0]
       output.start
@@ -440,12 +469,21 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       # flush few times to check steps
       2.times do
         output.force_flush
+        # output.force_flush calls #submit_flush_all, but #submit_flush_all skips to call #submit_flush_once when @retry exists.
+        # So that forced flush in retry state should be done by calling #submit_flush_once directly.
+        output.submit_flush_once
+        sleep 0.1 until output.buffer.queued?
       end
       response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json"))
       test_out_fail_write_response = response["plugins"][1]
       # remove dynamic keys
-      ["start", "next_time"].each { |key| test_out_fail_write_response["retry"].delete(key) }
+      response_retry_count = test_out_fail_write_response.delete("retry_count")
+      response_retry = test_out_fail_write_response.delete("retry")
       assert_equal(expected_test_out_fail_write_response, test_out_fail_write_response)
+      assert{ response_retry.has_key?("steps") }
+      # it's very hard to check exact retry count (because retries are called by output flush thread scheduling)
+      assert{ response_retry_count >= 1 && response_retry["steps"] >= 0 }
+      assert{ response_retry_count == response_retry["steps"] + 1 }
     end
   end
 end

data/test/plugin/test_in_tail.rb

@@ -37,6 +37,7 @@ class TailInputTest < Test::Unit::TestCase
   COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
   CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
   CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
+  CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
   SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
   PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
   MULTILINE_CONFIG = config_element(
@@ -135,8 +136,7 @@ class TailInputTest < Test::Unit::TestCase
 
       d.run(expect_emits: 1) do
         File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
-          f.puts "test3"
-          f.puts "test4"
+          f.puts "test3\ntest4"
         }
       end
 
@@ -149,6 +149,28 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal(1, d.emit_count)
     end
 
+    def test_emit_with_emit_unmatched_lines_true
+      config = config_element("", "", { "format" => "/^(?<message>test.*)/", "emit_unmatched_lines" => true })
+      File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+      d = create_driver(config)
+      d.run(expect_emits: 1) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+          f.puts "test line 1"
+          f.puts "test line 2"
+          f.puts "bad line 1"
+          f.puts "test line 3"
+        }
+      end
+
+      events = d.events
+      assert_equal(4, events.length)
+      assert_equal({"message" => "test line 1"}, events[0][2])
+      assert_equal({"message" => "test line 2"}, events[1][2])
+      assert_equal({"unmatched_line" => "bad line 1"}, events[2][2])
+      assert_equal({"message" => "test line 3"}, events[3][2])
+    end
+
     data('flat 1' => [:flat, 1, 2],
          'flat 10' => [:flat, 10, 1],
          'parse 1' => [:parse, 1, 2],
@@ -315,6 +337,20 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test6"}, events[5][2])
     end
 
+    data(flat: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
+         parse: CONFIG_OPEN_ON_EVERY_UPDATE + CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
+    def test_rotate_file_with_open_on_every_update(data)
+      config = data
+      events = sub_test_rotate_file(config, expect_records: 6)
+      assert_equal(6, events.length)
+      assert_equal({"message" => "test1"}, events[0][2])
+      assert_equal({"message" => "test2"}, events[1][2])
+      assert_equal({"message" => "test3"}, events[2][2])
+      assert_equal({"message" => "test4"}, events[3][2])
+      assert_equal({"message" => "test5"}, events[4][2])
+      assert_equal({"message" => "test6"}, events[5][2])
+    end
+
     data(flat: SINGLE_LINE_CONFIG,
          parse: PARSE_SINGLE_LINE_CONFIG)
     def test_rotate_file_with_write_old(data)
@@ -331,6 +367,8 @@ class TailInputTest < Test::Unit::TestCase
           f.puts "test6"
         }
       }
+      # This test sometimes fails and it shows a potential bug of in_tail
+      # https://github.com/fluent/fluentd/issues/1434
       assert_equal(6, events.length)
       assert_equal({"message" => "test3"}, events[0][2])
       assert_equal({"message" => "test4"}, events[1][2])
@@ -487,6 +525,30 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
   end
 
+  def test_from_encoding_utf16
+    conf = config_element(
+      "", "", {
+        "format" => "/(?<message>.*)/",
+        "read_from_head" => "true",
+        "from_encoding" => "utf-16le",
+        "encoding" => "utf-8"
+      })
+    d = create_driver(conf)
+    utf16_message = "\u306F\u308D\u30FC\u308F\u30FC\u308B\u3069\n".encode(Encoding::UTF_16LE)
+    utf8_message = utf16_message.encode(Encoding::UTF_8).strip
+
+    d.run(expect_emits: 1) do
+      File.open("#{TMP_DIR}/tail.txt", "w:utf-16le") { |f|
+        f.write utf16_message
+      }
+    end
+
+    events = d.events
+    assert_equal(utf8_message, events[0][2]['message'])
+    assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
+  end
+
+
   sub_test_case "multiline" do
     data(flat: MULTILINE_CONFIG,
          parse: PARSE_MULTILINE_CONFIG)
@@ -516,6 +578,35 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message1" => "test8"}, events[3][2])
     end
 
+    data(flat: MULTILINE_CONFIG,
+         parse: PARSE_MULTILINE_CONFIG)
+    def test_multiline_with_emit_unmatched_lines_true(data)
+      config = data + config_element("", "", { "emit_unmatched_lines" => true })
+      File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+      d = create_driver(config)
+      d.run(expect_emits: 1) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+          f.puts "f test1"
+          f.puts "s test2"
+          f.puts "f test3"
+          f.puts "f test4"
+          f.puts "s test5"
+          f.puts "s test6"
+          f.puts "f test7"
+          f.puts "s test8"
+        }
+      end
+
+      events = d.events
+      assert_equal(5, events.length)
+      assert_equal({"unmatched_line" => "f test1"}, events[0][2])
+      assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[1][2])
+      assert_equal({"message1" => "test5"}, events[2][2])
+      assert_equal({"message1" => "test6", "message2" => "test7"}, events[3][2])
+      assert_equal({"message1" => "test8"}, events[4][2])
+    end
+
     data(flat: MULTILINE_CONFIG,
          parse: PARSE_MULTILINE_CONFIG)
     def test_multiline_with_flush_interval(data)
@@ -769,7 +860,7 @@ class TailInputTest < Test::Unit::TestCase
     Timecop.freeze(2010, 1, 2, 3, 4, 5) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
-          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
+          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
             flexmock('TailWatcher') { |watcher|
               watcher.should_receive(:attach).once
               watcher.should_receive(:unwatched=).zero_or_more_times
@@ -787,7 +878,7 @@ class TailInputTest < Test::Unit::TestCase
 
     Timecop.freeze(2010, 1, 2, 3, 4, 6) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
-        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|
             watcher.should_receive(:attach).once
             watcher.should_receive(:unwatched=).zero_or_more_times
@@ -898,6 +989,8 @@ class TailInputTest < Test::Unit::TestCase
           f.puts "test4"
         }
       end
+      # This test sometimes fails and it shows a potential bug of in_tail
+      # https://github.com/fluent/fluentd/issues/1434
       events = d.events
       assert_equal(2, events.length)
       assert_equal({"message" => "test3"}, events[0][2])

data/test/plugin/test_input.rb

@@ -85,6 +85,10 @@ class InputTest < Test::Unit::TestCase
     end
   end
 
+  test 'are not available with multi workers configuration in default' do
+    assert_false @p.multi_workers_ready?
+  end
+
   test 'has router and can emit into it' do
     assert @p.has_router?
 

data/test/plugin/test_out_file.rb

@@ -22,8 +22,8 @@ class FileOutputTest < Test::Unit::TestCase
     utc
   ]
 
-  def create_driver(conf = CONFIG)
-    Fluent::Test::Driver::Output.new(Fluent::Plugin::FileOutput).configure(conf)
+  def create_driver(conf = CONFIG, opts = {})
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::FileOutput, opts: opts).configure(conf)
   end
 
   sub_test_case 'configuration' do
@@ -37,7 +37,29 @@ class FileOutputTest < Test::Unit::TestCase
       assert_equal :gzip, d.instance.instance_eval{ @compress_method }
     end
 
+    test 'using root_dir for buffer path' do
+      system_conf_opts = {'root_dir' => File.join(TMP_DIR, 'testrootdir')}
+      buf_conf = config_element('buffer', '', {'flush_interval' => '1s'})
+      conf = config_element('match', '**', {'@id' => 'myout', 'path' => 'test_path', 'append' => 'true'}, [buf_conf])
+      d = create_driver(conf, system_conf_opts)
+
+      assert_equal 'test_path', d.instance.path
+      assert d.instance.append
+
+      assert d.instance.buffer.respond_to?(:path) # file buffer
+      assert_equal 1, d.instance.buffer_config.flush_interval
+
+      assert_equal File.join(TMP_DIR, 'testrootdir', 'worker0', 'myout'), d.instance.plugin_root_dir
+
+      buffer_path_under_root_dir = File.join(TMP_DIR, 'testrootdir', 'worker0', 'myout', 'buffer', 'buffer.*.log')
+      assert_equal buffer_path_under_root_dir, d.instance.buffer.path
+    end
+
     test 'path should be writable' do
+      assert_raise(Fluent::ConfigError.new("'path' parameter is required")) do
+        create_driver ""
+      end
+
       assert_nothing_raised do
         create_driver %[path #{TMP_DIR}/test_path]
       end
@@ -718,7 +740,7 @@ class FileOutputTest < Test::Unit::TestCase
     end
 
     test 'raise error to show it is a bug when path including * specified without timekey' do
-      assert_raise "BUG: configuration error must be raised for path including '*' without timekey" do
+      assert_raise RuntimeError.new("BUG: configuration error must be raised for path including '*' without timekey") do
         @i.generate_path_template('/path/to/file.*.log', nil, false, nil)
       end
     end
@@ -808,8 +830,10 @@ class FileOutputTest < Test::Unit::TestCase
     end
 
     test 'raise error if argument path does not include index placeholder' do
-      assert_raise "BUG: index placeholder not found in path: #{@tmp}/myfile" do
-        @i.find_filepath_available("#{@tmp}/myfile")
+      assert_raise RuntimeError.new("BUG: index placeholder not found in path: #{@tmp}/myfile") do
+        @i.find_filepath_available("#{@tmp}/myfile") do |path|
+          # ...
+        end
       end
     end
 
@@ -821,14 +845,29 @@ class FileOutputTest < Test::Unit::TestCase
     )
     test 'returns filepath with _0 at first' do |data|
       expected, argument = data
-      assert_equal File.join(@tmp, expected), @i.find_filepath_available(File.join(@tmp, argument))
+      @i.find_filepath_available(File.join(@tmp, argument)) do |path|
+        assert_equal File.join(@tmp, expected), path
+      end
     end
 
     test 'returns filepath with index which does not exist yet' do
+      5.times do |i|
+        File.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
+      end
+      @i.find_filepath_available(File.join(@tmp, "exist_**.log")) do |path|
+        assert_equal File.join(@tmp, "exist_5.log"), path
+      end
+    end
+
+    test 'creates lock directory when with_lock is true to exclude operations of other worker process' do
       5.times do |i|
         File.open(File.join(@tmp, "exist_#{i}.log"), 'a')
       end
-      assert_equal File.join(@tmp, "exist_5.log"), @i.find_filepath_available(File.join(@tmp, "exist_**.log"))
+      Dir.mkdir(File.join(@tmp, "exist_5.log.lock"))
+      @i.find_filepath_available(File.join(@tmp, "exist_**.log"), with_lock: true) do |path|
+        assert Dir.exist?(File.join(@tmp, "exist_6.log.lock"))
+        assert_equal File.join(@tmp, "exist_6.log"), path
+      end
     end
   end
 end