fluentd 0.14.8 → 0.14.9

data/test/plugin/test_filter_record_transformer.rb

@@ -250,7 +250,11 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         filtered = filter(config)
         filtered.each do |t, r|
-          assert_equal(Time.at(@time).localtime.to_s, r['message'])
+          if enable_ruby == "yes"
+            assert_equal(Time.at(@time).localtime, r['message'])
+          else
+            assert_equal(Time.at(@time).localtime.to_s, r['message'])
+          end
         end
       end
 
@@ -499,15 +503,19 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       }
     end
 
-    test 'failed to expand (enable_ruby yes)' do
+    data(auto_typecast_yes: ["yes", "unknown['bar']"],
+         auto_typecast_no: ["no", "%Q[\#{unknown['bar']}]"])
+    test 'failed to expand (enable_ruby yes)' do |(param, expected_log)|
+
       config = %[
         enable_ruby yes
+        auto_typecast #{param}
         <record>
           message ${unknown['bar']}
         </record>
       ]
       filtered = filter(config) { |d|
-        mock(d.instance.log).warn("failed to expand `%Q[\#{unknown['bar']}]`", anything)
+        mock(d.instance.log).warn("failed to expand `#{expected_log}`", anything)
       }
       filtered.each do |t, r|
         assert_nil(r['message'])

data/test/plugin/test_filter_stdout.rb

@@ -67,10 +67,9 @@ class StdoutFilterTest < Test::Unit::TestCase
 
     def test_output_type_json
       d = create_driver(CONFIG + config_element("", "", { "output_type" => "json" }))
-      etime = event_time
-      time = Time.at(etime.sec)
+      etime = event_time("2016-10-07 21:09:31.012345678 UTC")
       out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
-      assert_equal "#{time.localtime} filter.test: {\"test\":\"test\"}\n", out
+      assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\"}\n", out
 
       # NOTE: Float::NAN is not jsonable
       d = create_driver(CONFIG + config_element("", "", { "output_type" => "json" }))
@@ -80,15 +79,14 @@ class StdoutFilterTest < Test::Unit::TestCase
 
     def test_output_type_hash
       d = create_driver(CONFIG + config_element("", "", { "output_type" => "hash" }))
-      etime = event_time
-      time = Time.at(etime.sec)
+      etime = event_time("2016-10-07 21:09:31.012345678 UTC")
       out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
-      assert_equal "#{time.localtime} filter.test: {\"test\"=>\"test\"}\n", out
+      assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>\"test\"}\n", out
 
       # NOTE: Float::NAN is not jsonable, but hash string can output it.
       d = create_driver(CONFIG + config_element("", "", { "output_type" => "hash" }))
       out = capture_log(d) { filter(d, etime, {'test' => Float::NAN}) }
-      assert_equal "#{time.localtime} filter.test: {\"test\"=>NaN}\n", out
+      assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>NaN}\n", out
     end
 
     # Use include_time_key to output the message's time
@@ -99,11 +97,9 @@ class StdoutFilterTest < Test::Unit::TestCase
                                 "localtime" => false
                               })
       d = create_driver(config)
-      etime = event_time
-      time = Time.at(etime.sec)
-      message_time = event_time("2011-01-02 13:14:15 UTC")
-      out = capture_log(d) { filter(d, message_time, {'test' => 'test'}) }
-      assert_equal "#{time.localtime} filter.test: {\"test\":\"test\",\"time\":\"2011-01-02T13:14:15Z\"}\n", out
+      etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+      out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
+      assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\",\"time\":\"2016-10-07T21:09:31Z\"}\n", out
     end
 
     # out_stdout formatter itself can also be replaced
@@ -150,10 +146,9 @@ class StdoutFilterTest < Test::Unit::TestCase
         conf = config_element
         conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "json" })
         d = create_driver(conf)
-        etime = event_time
-        time = Time.at(etime.sec)
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
         out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
-        assert_equal "#{time.localtime} filter.test: {\"test\":\"test\"}\n", out
+        assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\"}\n", out
       end
 
       def test_json_nan
@@ -161,7 +156,7 @@ class StdoutFilterTest < Test::Unit::TestCase
         conf = config_element
         conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "json" })
         d = create_driver(conf)
-        etime = event_time
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
         flexmock(d.instance.router).should_receive(:emit_error_event)
         filter(d, etime, {'test' => Float::NAN})
       end
@@ -170,10 +165,9 @@ class StdoutFilterTest < Test::Unit::TestCase
         conf = config_element
         conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "hash" })
         d = create_driver(conf)
-        etime = event_time
-        time = Time.at(etime.sec)
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
         out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
-        assert_equal "#{time.localtime} filter.test: {\"test\"=>\"test\"}\n", out
+        assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>\"test\"}\n", out
       end
 
       def test_hash_nan
@@ -181,10 +175,9 @@ class StdoutFilterTest < Test::Unit::TestCase
         conf = config_element
         conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "hash" })
         d = create_driver(conf)
-        etime = event_time
-        time = Time.at(etime.sec)
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
         out = capture_log(d) { filter(d, etime, {'test' => Float::NAN}) }
-        assert_equal "#{time.localtime} filter.test: {\"test\"=>NaN}\n", out
+        assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>NaN}\n", out
       end
 
       # Use include_time_key to output the message's time
@@ -200,11 +193,9 @@ class StdoutFilterTest < Test::Unit::TestCase
                                           "localtime" => false
                                           })
         d = create_driver(conf)
-        etime = event_time
-        time = Time.at(etime.sec)
-        message_time = event_time("2011-01-02 13:14:15 UTC")
-        out = capture_log(d) { filter(d, message_time, {'test' => 'test'}) }
-        assert_equal "#{time.localtime} filter.test: {\"test\":\"test\",\"time\":\"2011-01-02T13:14:15Z\"}\n", out
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+        out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
+        assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\",\"time\":\"2016-10-07T21:09:31Z\"}\n", out
       end
     end
   end

data/test/plugin/test_in_dummy.rb

@@ -15,7 +15,7 @@ class DummyTest < Test::Unit::TestCase
   sub_test_case 'configure' do
     test 'required parameters' do
       assert_raise_message("'tag' parameter is required") do
-        create_driver('')
+        Fluent::Plugin::DummyInput.new.configure(config_element('ROOT',''))
       end
     end
 

data/test/plugin/test_in_exec.rb

@@ -1,133 +1,245 @@
 require_relative '../helper'
 require 'fluent/test/driver/input'
 require 'fluent/plugin/in_exec'
-require 'net/http'
+require 'timecop'
 
 class ExecInputTest < Test::Unit::TestCase
+  SCRIPT_PATH = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
+  TEST_TIME = "2011-01-02 13:14:15"
+  TEST_UNIX_TIME = Time.parse(TEST_TIME)
+
   def setup
     Fluent::Test.setup
-    @test_time = event_time("2011-01-02 13:14:15")
-    @script = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
+    @test_time = event_time()
   end
 
-  def create_driver(conf = tsv_config)
+  def create_driver(conf)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::ExecInput).configure(conf)
   end
 
-  def tsv_config
-    %[
-      command ruby #{@script} "2011-01-02 13:14:15" 0
+  DEFAULT_CONFIG_ONLY_WITH_KEYS = %[
+    command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
+    run_interval 1s
+    tag "my.test.data"
+    <parse>
+      keys ["k1", "k2", "k3"]
+    </parse>
+  ]
+
+  TSV_CONFIG = %[
+    command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
+    run_interval 1s
+    <parse>
+      @type tsv
+      keys time, tag, k1
+    </parse>
+    <extract>
+      tag_key tag
+      time_key time
+      time_type string
+      time_format %Y-%m-%d %H:%M:%S
+    </extract>
+  ]
+
+  JSON_CONFIG = %[
+    command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 1
+    run_interval 1s
+    <parse>
+      @type json
+    </parse>
+    <extract>
+      tag_key tag
+      time_key time
+      time_type unixtime
+    </extract>
+  ]
+
+  MSGPACK_CONFIG = %[
+    command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 2
+    run_interval 1s
+    <parse>
+      @type msgpack
+    </parse>
+    <extract>
+      tag_key tagger
+      time_key datetime
+      time_type unixtime
+    </extract>
+  ]
+
+  # here document for not de-quoting backslashes
+  REGEXP_CONFIG = %[
+    command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 3
+    run_interval 1s
+    tag regex_tag
+] + <<'EOC'
+    <parse>
+      @type regexp
+      expression "(?<time>[^\\]]*) (?<message>[^ ]*)"
+      time_key time
+      time_type string
+      time_format %Y-%m-%d %H:%M:%S
+    </parse>
+EOC
+
+  sub_test_case 'with configuration with sections' do
+    test 'configure with default tsv format without extract' do
+      d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS
+      assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }
+      assert_equal "my.test.data", d.instance.tag
+      assert_equal ["k1", "k2", "k3"], d.instance.parser.keys
+    end
+
+    test 'configure raises error if both of tag and extract.tag_key are missing' do
+      assert_raise Fluent::ConfigError.new("'tag' or 'tag_key' option is required on exec input") do
+        create_driver %[
+          command ruby -e 'puts "yay"'
+          <parse>
+            keys y1
+          </parse>
+        ]
+      end
+    end
+
+    test 'configure for tsv' do
+      d = create_driver TSV_CONFIG
+      assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }
+      assert_equal ["time", "tag", "k1"], d.instance.parser.keys
+      assert_equal "tag", d.instance.extract_config.tag_key
+      assert_equal "time", d.instance.extract_config.time_key
+      assert_equal :string, d.instance.extract_config.time_type
+      assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
+    end
+
+    test 'configure for json' do
+      d = create_driver JSON_CONFIG
+      assert{ d.instance.parser.is_a? Fluent::Plugin::JSONParser }
+      assert_equal "tag", d.instance.extract_config.tag_key
+      assert_equal "time", d.instance.extract_config.time_key
+      assert_equal :unixtime, d.instance.extract_config.time_type
+    end
+
+    test 'configure for msgpack' do
+      d = create_driver MSGPACK_CONFIG
+      assert{ d.instance.parser.is_a? Fluent::Plugin::MessagePackParser }
+      assert_equal "tagger", d.instance.extract_config.tag_key
+      assert_equal "datetime", d.instance.extract_config.time_key
+      assert_equal :unixtime, d.instance.extract_config.time_type
+    end
+
+    test 'configure for regexp' do
+      d = create_driver REGEXP_CONFIG
+      assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
+      assert_equal "regex_tag", d.instance.tag
+      expression = <<'EXP'.chomp
+(?<time>[^\]]*) (?<message>[^ ]*)
+EXP
+      assert_equal expression, d.instance.parser.expression
+      assert_nil d.instance.extract_config
+    end
+  end
+
+  TSV_CONFIG_COMPAT = %[
+      command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
       keys time,tag,k1
       time_key time
       tag_key tag
       time_format %Y-%m-%d %H:%M:%S
       run_interval 1s
-    ]
-  end
+  ]
 
-  def json_config
-    %[
-      command ruby #{@script} #{@test_time} 1
+  JSON_CONFIG_COMPAT = %[
+      command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 1
       format json
       tag_key tag
       time_key time
       run_interval 1s
-    ]
-  end
+  ]
 
-  def msgpack_config
-    %[
-      command ruby #{@script} #{@test_time} 2
+  MSGPACK_CONFIG_COMPAT = %[
+      command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 2
       format msgpack
       tag_key tagger
       time_key datetime
       run_interval 1s
-    ]
-  end
+  ]
 
-  def regexp_config
-    %[
-      command ruby #{@script} "2011-01-02 13:14:15" 3
+  REGEXP_CONFIG_COMPAT = %[
+      command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 3
       format /(?<time>[^\\\]]*) (?<message>[^ ]*)/
       tag regex_tag
       run_interval 1s
-    ]
-  end
-
-  def test_configure
-    d = create_driver
-    assert_equal 'tsv', d.instance.format
-    assert_equal ["time","tag","k1"], d.instance.keys
-    assert_equal "tag", d.instance.tag_key
-    assert_equal "time", d.instance.time_key
-    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.time_format
-  end
-
-  def test_configure_with_json
-    d = create_driver json_config
-    assert_equal 'json', d.instance.format
-    assert_equal [], d.instance.keys
-  end
-
-  def test_configure_with_msgpack
-    d = create_driver msgpack_config
-    assert_equal 'msgpack', d.instance.format
-    assert_equal [], d.instance.keys
+  ]
+
+  sub_test_case 'with traditional configuration' do
+    test 'configure' do
+      d = create_driver TSV_CONFIG_COMPAT
+      assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }
+      assert_equal ["time","tag","k1"], d.instance.parser.keys
+      assert_equal "tag", d.instance.extract_config.tag_key
+      assert_equal "time", d.instance.extract_config.time_key
+      assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
+    end
+
+    test 'configure_with_json' do
+      d = create_driver JSON_CONFIG_COMPAT
+      assert{ d.instance.parser.is_a? Fluent::Plugin::JSONParser }
+    end
+
+    test 'configure_with_msgpack' do
+      d = create_driver MSGPACK_CONFIG_COMPAT
+      assert{ d.instance.parser.is_a? Fluent::Plugin::MessagePackParser }
+    end
+
+    test 'configure_with_regexp' do
+      d = create_driver REGEXP_CONFIG_COMPAT
+      assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
+      assert_equal '(?<time>[^\]]*) (?<message>[^ ]*)', d.instance.parser.expression
+      assert_equal 'regex_tag', d.instance.tag
+    end
   end
 
-  def test_configure_with_regexp
-    d = create_driver regexp_config
-    assert_equal '/(?<time>[^\]]*) (?<message>[^ ]*)/', d.instance.format
-    assert_equal 'regex_tag', d.instance.tag
+  sub_test_case 'with default configuration' do
+    setup do
+      @current_event_time = event_time('2016-10-31 20:01:30.123 -0700')
+      Timecop.freeze(Time.at(@current_event_time))
+    end
+
+    teardown do
+      Timecop.return
+    end
+
+    test 'emits events with current timestamp if time key is not specified' do
+      d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS
+      d.run(expect_records: 2, timeout: 10)
+
+      assert{ d.events.length > 0 }
+      d.events.each do |event|
+        assert_equal ["my.test.data", @current_event_time, {"k1"=>"2011-01-02 13:14:15", "k2"=>"tag1", "k3"=>"ok"}], event
+      end
+    end
   end
 
-  # TODO: Merge following tests into one case with parameters
-
-  def test_emit
-    d = create_driver
-
-    d.run(expect_emits: 2)
-
-    assert_equal true, d.events.length > 0
-    d.events.each_with_index {|event, i|
-      assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
-      assert_equal_event_time(@test_time, event[1])
-    }
-  end
-
-  def test_emit_json
-    d = create_driver json_config
-
-    d.run(expect_emits: 2)
-
-    assert_equal true, d.events.length > 0
-    d.events.each_with_index {|event, i|
-      assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
-      assert_equal_event_time(@test_time, event[1])
-    }
-  end
-
-  def test_emit_msgpack
-    d = create_driver msgpack_config
-
-    d.run(expect_emits: 2)
-
-    assert_equal true, d.events.length > 0
-    d.events.each_with_index {|event, i|
-      assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
-      assert_equal_event_time(@test_time, event[1])
-    }
-  end
-
-  def test_emit_regexp
-    d = create_driver regexp_config
-
-    d.run(expect_emits: 2)
-
-    assert_equal true, d.events.length > 0
+  data(
+    'default' => [TSV_CONFIG,     "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'json'    => [JSON_CONFIG,    "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'msgpack' => [MSGPACK_CONFIG, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'regexp'  => [REGEXP_CONFIG,  "regex_tag", event_time("2011-01-02 13:14:15"), {"message"=>"hello"}],
+    'default_c' => [TSV_CONFIG_COMPAT,     "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'json_c'    => [JSON_CONFIG_COMPAT,    "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'msgpack_c' => [MSGPACK_CONFIG_COMPAT, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'regexp_c'  => [REGEXP_CONFIG_COMPAT,  "regex_tag", event_time("2011-01-02 13:14:15"), {"message"=>"hello"}],
+  )
+  test 'emit with formats' do |data|
+    config, tag, time, record = data
+    d = create_driver(config)
+
+    d.run(expect_emits: 2, timeout: 10)
+
+    assert{ d.events.length > 0 }
     d.events.each_with_index {|event, i|
-      assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], event
-      assert_equal_event_time(@test_time, event[1])
+      assert_equal_event_time(time, event[1])
+      assert_equal [tag, time, record], event
     }
   end
 end