fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

data/test/plugin/test_parser_csv.rb

@@ -37,8 +37,7 @@ class CSVParserTest < ::Test::Unit::TestCase
     }
 
     d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::CSVParser)
-    d.instance.estimate_current_event = false
-    d.configure('keys' => param)
+    d.configure('keys' => param, 'estimate_current_event' => 'no')
     d.instance.parse("192.168.0.1,111") { |time, record|
       assert_equal({
                      'c' => '192.168.0.1',

data/test/plugin/test_parser_json.rb

@@ -44,8 +44,7 @@ class JsonParserTest < ::Test::Unit::TestCase
     }
 
     parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
-    parser.instance.estimate_current_event = false
-    parser.configure('json_parser' => data)
+    parser.configure('json_parser' => data, 'estimate_current_event' => 'false')
     parser.instance.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
       assert_equal({
                      'host'   => '192.168.0.1',
@@ -78,8 +77,8 @@ class JsonParserTest < ::Test::Unit::TestCase
     parser.configure('json_parser' => data)
     text = "100.1"
     parser.instance.parse("{\"time\":\"#{text}\"}") do |time, record|
-      assert_equal Time.at(text.to_f).to_i, time.sec
-      assert_equal Time.at(text.to_f).nsec, time.nsec
+      assert_equal 100, time.sec
+      assert_equal 100_000_000, time.nsec
     end
   end
 

data/test/plugin/test_parser_labeled_tsv.rb

@@ -78,8 +78,7 @@ class LabeledTSVParserTest < ::Test::Unit::TestCase
     }
 
     parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
-    parser.instance.estimate_current_event = false
-    parser.configure({})
+    parser.configure({'estimate_current_event' => 'no'})
     parser.instance.parse("host:192.168.0.1\treq_id:111") { |time, record|
       assert_equal({
                      'host'   => '192.168.0.1',

data/test/plugin/test_parser_none.rb

@@ -43,8 +43,7 @@ class NoneParserTest < ::Test::Unit::TestCase
     }
 
     parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin.new_parser('none'))
-    parser.instance.estimate_current_event = false
-    parser.configure({})
+    parser.configure({'estimate_current_event' => 'false'})
     parser.instance.parse('log message!') { |time, record|
       assert_equal({'message' => 'log message!'}, record)
       assert_nil time, "parser returns nil w/o time if configured so"

data/test/plugin/test_parser_regexp.rb

@@ -28,7 +28,12 @@ class RegexpParserTest < ::Test::Unit::TestCase
       if initialize_conf
         Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
       else
-        Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
+        # Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
+        instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
+        instance.configure(conf)
+        d = Struct.new(:instance).new
+        d.instance = instance
+        d
       end
     end
 
@@ -188,12 +193,11 @@ class RegexpParserTest < ::Test::Unit::TestCase
       internal_test_case(d.instance)
     end
 
-    def test_parse_with_typed_and_name_separator
+    def test_parse_with_typed_by_json_hash
       conf = {
         'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
         'time_format' => "%d/%b/%Y:%H:%M:%S %z",
-        'types' => 'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer',
-        'types_label_delimiter' => '|'
+        'types' => '{"user":"string","date":"time:%d/%b/%Y:%H:%M:%S %z","flag":"bool","path":"array","code":"float","size":"integer"}',
       }
       d = create_driver(conf)
       internal_test_case(d.instance)

data/test/plugin/test_parser_tsv.rb

@@ -13,7 +13,9 @@ class TSVParserTest < ::Test::Unit::TestCase
 
   data('array param' => '["a","b"]', 'string param' => 'a,b')
   def test_config_params(param)
-    d = create_driver
+    d = create_driver(
+      'keys' => param,
+    )
 
     assert_equal "\t", d.instance.delimiter
 
@@ -51,8 +53,7 @@ class TSVParserTest < ::Test::Unit::TestCase
     }
 
     d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::TSVParser)
-    d.instance.estimate_current_event = false
-    d.configure('keys' => 'a,b', 'time_key' => 'time')
+    d.configure('keys' => 'a,b', 'time_key' => 'time', 'estimate_current_event' => 'no')
     d.instance.parse("192.168.0.1\t111") { |time, record|
       assert_equal({
                      'a' => '192.168.0.1',

data/test/plugin_helper/test_child_process.rb

@@ -2,6 +2,7 @@ require_relative '../helper'
 require 'fluent/plugin_helper/child_process'
 require 'fluent/plugin/base'
 require 'timeout'
+require 'tempfile'
 
 class ChildProcessTest < Test::Unit::TestCase
   TEST_DEADLOCK_TIMEOUT = 30
@@ -605,4 +606,187 @@ class ChildProcessTest < Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case 'on_exit_callback is specified' do
+    setup do
+      @temp = Tempfile.create("child_process_wait_with_on_exit_callback")
+      @temp_path = @temp.path
+      @temp.close
+    end
+
+    teardown do
+      File.unlink @temp_path if File.exist?(@temp_path)
+    end
+
+    test 'can return exit status for child process successfully exits using on_exit_callback' do
+      assert File.exist?(@temp_path)
+
+      block_exits = false
+      callback_called = false
+      exit_status = nil
+      args = ['-e', 'sleep ARGV[0].to_i; puts "yay"; File.unlink ARGV[1]', '1', @temp_path]
+      cb = ->(status){ exit_status = status; callback_called = true }
+
+      str = nil
+
+      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+        pid = nil
+        @d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
+          pid = @d.instance_eval{ child_process_id }
+          str = readio.read.chomp
+          block_exits = true
+        end
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) # to get exit status
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
+      end
+
+      assert callback_called
+      assert exit_status
+      assert_equal 0, exit_status.exitstatus
+      assert !File.exist?(@temp_path)
+
+      assert_equal "yay", str
+    end
+
+    test 'can return exit status with signal code for child process killed by signal using on_exit_callback' do
+      omit "SIGQUIT is unsupported on Windows" if Fluent.windows?
+
+      assert File.exist?(@temp_path)
+
+      block_exits = false
+      callback_called = false
+      exit_status = nil
+      args = ['-e', 'sleep ARGV[0].to_i; puts "yay"; File.unlink ARGV[1]', '25', @temp_path]
+      cb = ->(status){ exit_status = status; callback_called = true }
+
+      str = nil
+
+      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+        pid = nil
+        @d.child_process_execute(:st1, "ruby", arguments: args, mode: [:read], on_exit_callback: cb) do |readio|
+          pid = @d.instance_eval{ child_process_id }
+          Process.kill(:QUIT, pid)
+          Process.kill(:QUIT, pid) rescue nil # once more to kill certainly
+          str = readio.read.chomp rescue nil # empty string before EOF
+          block_exits = true
+        end
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING while @d.child_process_exist?(pid) # to get exit status
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until block_exits
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
+      end
+
+      assert callback_called
+      assert exit_status
+
+      assert_equal [nil, 3], [exit_status.exitstatus, exit_status.termsig] # SIGQUIT
+
+      assert File.exist?(@temp_path)
+      assert_equal "", str
+    end
+
+    test 'calls on_exit_callback for each process exits for interval call using on_exit_callback' do
+      read_data_list = []
+      exit_status_list = []
+
+      args = ['-e', 'puts "yay"', '1']
+      cb = ->(status){ exit_status_list << status }
+
+      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+        @d.child_process_execute(:st1, "ruby", arguments: args, immediate: true, interval: 2, mode: [:read], on_exit_callback: cb) do |readio|
+          read_data_list << readio.read.chomp
+        end
+        sleep 10
+      end
+
+      assert{ read_data_list.size >= 3 }
+      assert{ exit_status_list.size >= 3 }
+    end
+
+    test 'waits lasting child process until wait_timeout if block is not specified' do
+      assert File.exist?(@temp_path)
+
+      callback_called = false
+      exit_status = nil
+      args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '1', @temp_path]
+      cb = ->(status){ exit_status = status; callback_called = true }
+
+      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+        @d.child_process_execute(:t17, "ruby", arguments: args, on_exit_callback: cb, wait_timeout: 5)
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
+      end
+
+      assert callback_called
+      assert exit_status
+      assert_equal 0, exit_status.exitstatus
+      assert !File.exist?(@temp_path)
+    end
+
+    test 'waits lasting child process until wait_timeout after block rans if block is specified' do
+      assert File.exist?(@temp_path)
+
+      callback_called = false
+      exit_status = nil
+      args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '3', @temp_path]
+      cb = ->(status){ exit_status = status; callback_called = true }
+
+      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+        @d.child_process_execute(:t17, "ruby", arguments: args, mode: nil, on_exit_callback: cb, wait_timeout: 10) do
+          sleep 1
+        end
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
+      end
+
+      assert callback_called
+      assert exit_status
+      assert_equal 0, exit_status.exitstatus
+      assert !File.exist?(@temp_path)
+    end
+
+    test 'kills lasting child process after wait_timeout if block is not specified' do
+      assert File.exist?(@temp_path)
+
+      callback_called = false
+      exit_status = nil
+      args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '20', @temp_path]
+      cb = ->(status){ exit_status = status; callback_called = true }
+
+      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+        @d.child_process_execute(:t17, "ruby", arguments: args, on_exit_callback: cb, wait_timeout: 3)
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
+      end
+
+      assert callback_called
+      assert exit_status
+      unless Fluent.windows? # On Windows, exitstatus is always 0 and termsig is nil
+        assert_nil exit_status.exitstatus
+        assert_equal 9, exit_status.termsig # SIGKILL
+      end
+      assert File.exist?(@temp_path)
+    end
+
+    test 'kills lasting child process after block ran and wait_timeout expires if block is specified' do
+      assert File.exist?(@temp_path)
+
+      callback_called = false
+      exit_status = nil
+      args = ['-e', 'sleep ARGV[0].to_i; File.unlink ARGV[1]', '20', @temp_path]
+      cb = ->(status){ exit_status = status; callback_called = true }
+
+      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+        @d.child_process_execute(:t17, "ruby", arguments: args, mode: nil, on_exit_callback: cb, wait_timeout: 3) do
+          sleep 3
+        end
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
+      end
+
+      assert callback_called
+      assert exit_status
+      unless Fluent.windows? # On Windows, exitstatus is always 0 and termsig is nil
+        assert_nil exit_status.exitstatus
+        assert_equal 9, exit_status.termsig # SIGKILL
+      end
+      assert File.exist?(@temp_path)
+    end
+  end
 end

data/test/plugin_helper/test_compat_parameters.rb

@@ -1,6 +1,8 @@
 require_relative '../helper'
 require 'fluent/plugin_helper/compat_parameters'
-require 'fluent/plugin/base'
+require 'fluent/plugin/input'
+require 'fluent/plugin/output'
+require 'fluent/time'
 
 require 'time'
 
@@ -21,6 +23,26 @@ class CompatParameterTest < Test::Unit::TestCase
     end
   end
 
+  class DummyI0 < Fluent::Plugin::Input
+    helpers :compat_parameters, :parser, :extract
+    attr_reader :parser
+    def configure(conf)
+      compat_parameters_convert(conf, :extract, :parser)
+      super
+    end
+    def start
+      super
+      @parser = parser_create
+    end
+    def produce_events(input_data)
+      emit_events = [] # tag, time, record
+      @parser.parse(input_data) do |time, record|
+        tag = extract_tag_from_record(record) || 'dummy_tag'
+        emit_events << [tag, time, record]
+      end
+      emit_events
+    end
+  end
   class DummyO0 < Fluent::Plugin::Output
     helpers :compat_parameters
     def configure(conf)
@@ -239,4 +261,69 @@ class CompatParameterTest < Test::Unit::TestCase
       assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
     end
   end
+
+  sub_test_case 'input plugins' do
+    test 'plugin helper converts parameters into plugin configuration parameters for extract and parser' do
+      hash = {
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'tag_key' => 't2',
+        'time_key' => 't',
+        'time_format' => '%Y-%m-%d.%H:%M:%S.%N',
+        'utc' => 'yes',
+        'types' => 'A integer|B string|C bool',
+        'types_delimiter' => '|',
+        'types_label_delimiter' => ' ',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyI0.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      parser = @i.parser
+      assert{ parser.is_a? Fluent::Plugin::LabeledTSVParser }
+      assert_equal ',', parser.delimiter
+      assert_equal '%', parser.label_delimiter
+
+      events = @i.produce_events("A%1,B%x,C%true,t2%mytag,t%2016-10-20.03:50:11.987654321")
+      assert_equal 1, events.size
+
+      tag, time, record = events.first
+      assert_equal 'mytag', tag
+      assert_equal_event_time event_time("2016-10-20 03:50:11.987654321 +0000"), time
+      assert_equal 3, record.keys.size
+      assert_equal ['A','B','C'], record.keys.sort
+      assert_equal 1, record['A']
+      assert_equal 'x', record['B']
+      assert_equal true, record['C']
+    end
+
+    test 'plugin helper converts parameters into plugin configuration parameters for extract and parser, using numeric time' do
+      hash = {
+        'format' => 'ltsv',
+        'delimiter' => ',',
+        'label_delimiter' => '%',
+        'tag_key' => 't2',
+        'time_key' => 't',
+        'time_type' => 'float',
+        'localtime' => 'yes',
+      }
+      conf = config_element('ROOT', '', hash)
+      @i = DummyI0.new
+      @i.configure(conf)
+      @i.start
+      @i.after_start
+
+      parser = @i.parser
+      assert{ parser.is_a? Fluent::Plugin::LabeledTSVParser }
+      assert_equal ',', parser.delimiter
+      assert_equal '%', parser.label_delimiter
+    end
+
+    test 'plugin helper setups time extraction as unix time (integer from epoch)' do
+      # TODO:
+    end
+  end
 end

data/test/plugin_helper/test_extract.rb

@@ -50,7 +50,6 @@ class ExtractHelperTest < Test::Unit::TestCase
     assert_nil @d.instance_eval{ @_extract_time_key }
     assert_nil @d.instance_eval{ @_extract_time_parser }
 
-    time = event_time()
     record = {"key1" => "value1", "key2" => 2, "tag" => "yay", "time" => Time.now.to_i}
 
     assert_nil @d.extract_tag_from_record(record)

data/test/plugin_helper/test_formatter.rb

@@ -17,6 +17,9 @@ class FormatterHelperTest < Test::Unit::TestCase
   end
   class Dummy < Fluent::Plugin::TestBase
     helpers :formatter
+    config_section :format do
+      config_set_default :@type, 'example'
+    end
   end
 
   class Dummy2 < Fluent::Plugin::TestBase
@@ -77,12 +80,12 @@ class FormatterHelperTest < Test::Unit::TestCase
     end
   end
 
-  test 'can be configured without format sections' do
+  test 'can be configured with default type without format sections' do
     d = Dummy.new
     assert_nothing_raised do
       d.configure(config_element())
     end
-    assert_equal 0, d._formatters.size
+    assert_equal 1, d._formatters.size
   end
 
   test 'can be configured with a format section' do

data/test/plugin_helper/test_inject.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/plugin_helper/inject'
+require 'fluent/plugin/output'
 require 'fluent/event'
 require 'time'
 
@@ -15,6 +16,13 @@ class InjectHelperTest < Test::Unit::TestCase
     end
   end
 
+  class Dummy3 < Fluent::Plugin::Output
+    helpers :inject
+    def write(chunk)
+      # dummy
+    end
+  end
+
   def config_inject_section(hash = {})
     config_element('ROOT', '', {}, [config_element('inject', '', hash)])
   end
@@ -27,7 +35,9 @@ class InjectHelperTest < Test::Unit::TestCase
   teardown do
     if @d
       @d.stop unless @d.stopped?
+      @d.before_shutdown unless @d.before_shutdown?
       @d.shutdown unless @d.shutdown?
+      @d.after_shutdown unless @d.after_shutdown?
       @d.close unless @d.closed?
       @d.terminate unless @d.terminated?
     end
@@ -93,6 +103,17 @@ class InjectHelperTest < Test::Unit::TestCase
     assert_not_nil @d.instance_eval{ @_inject_time_formatter }
   end
 
+  test 'raise an error when injected hostname is used in buffer chunk key too' do
+    @d = Dummy3.new
+    conf = config_element('ROOT', '', {}, [
+      config_element('inject', '', {'hostname_key' => 'h'}),
+      config_element('buffer', 'tag,h'),
+    ])
+    assert_raise Fluent::ConfigError.new("the key specified by 'hostname_key' in <inject> cannot be used in buffering chunk key.") do
+      @d.configure(conf)
+    end
+  end
+
   sub_test_case 'using inject_values_to_record' do
     test 'injects hostname automatically detected' do
       detected_hostname = `hostname`.chomp