fluentd 0.12.40 → 1.6.2

data/test/plugin/test_filter_stdout.rb

@@ -1,14 +1,16 @@
 require_relative '../helper'
+require 'fluent/test/driver/filter'
 require 'fluent/plugin/filter_stdout'
 require 'timecop'
 require 'flexmock/test_unit'
 
 class StdoutFilterTest < Test::Unit::TestCase
-  include Fluent
   include FlexMock::TestCase
 
   def setup
     Fluent::Test.setup
+    @old_tz = ENV["TZ"]
+    ENV["TZ"] = "UTC"
     Timecop.freeze
   end
 
@@ -16,99 +18,185 @@ class StdoutFilterTest < Test::Unit::TestCase
     super # FlexMock::TestCase requires this
     # http://flexmock.rubyforge.org/FlexMock/TestCase.html
     Timecop.return
+    ENV["TZ"] = @old_tz
   end
 
-  CONFIG = %[
-  ]
+  CONFIG = config_element('ROOT')
 
   def create_driver(conf = CONFIG)
-    Test::FilterTestDriver.new(StdoutFilter, 'filter.test').configure(conf)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::StdoutFilter).configure(conf)
   end
 
-  def emit(d, msg, time)
+  def filter(d, time, record)
     d.run {
-      d.emit(msg, time)
-    }.filtered_as_array[0][2]
+      d.feed("filter.test", time, record)
+    }
+    d.filtered_records
   end
 
   def test_through_record
     d = create_driver
-    time = Time.now
-    filtered = emit(d, {'test' => 'test'}, time)
-    assert_equal({'test' => 'test'}, filtered)
+    filtered = filter(d, event_time, {'test' => 'test'})
+    assert_equal([{'test' => 'test'}], filtered)
   end
 
-  def test_configure_default
-    d = create_driver
-    assert_equal 'json', d.instance.formatter.output_type
-  end
-
-  def test_configure_output_type
-    d = create_driver(CONFIG + "\noutput_type json")
-    assert_equal 'json', d.instance.formatter.output_type
-
-    d = create_driver(CONFIG + "\noutput_type hash")
-    assert_equal 'hash', d.instance.formatter.output_type
-
-    d = create_driver(CONFIG + "\noutput_type ltsv")
-    assert_equal 'ltsv', d.instance.formatter.output_type
-
-    assert_raise(Fluent::ConfigError) do
-      d = create_driver(CONFIG + "\noutput_type foo")
+  sub_test_case "flat style parameters" do
+    sub_test_case "configure" do
+      def test_configure_default
+        d = create_driver
+        d.run {}
+        assert_equal 'json', d.instance.formatter.output_type
+      end
+
+      data(json: "json",
+           hash: "hash",
+           ltsv: "ltsv")
+      def test_output_type(data)
+        d = create_driver(CONFIG + config_element("", "", { "output_type" => data }))
+        d.run {}
+        assert_equal data, d.instance.formatter.output_type
+      end
+
+      def test_invalid_output_type
+        assert_raise(Fluent::ConfigError) do
+          d = create_driver(CONFIG + config_element("", "", { "output_type" => "foo" }))
+          d.run {}
+        end
+      end
     end
-  end
 
-  def test_output_type_json
-    d = create_driver(CONFIG + "\noutput_type json")
-    time = Time.now
-    out = capture_log(d) { emit(d, {'test' => 'test'}, time) }
-    assert_equal "#{time.localtime} filter.test: {\"test\":\"test\"}\n", out
+    def test_output_type_json
+      d = create_driver(CONFIG + config_element("", "", { "output_type" => "json" }))
+      etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+      out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
+      assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\"}\n", out
 
-    # NOTE: Float::NAN is not jsonable
-    d = create_driver(CONFIG + "\noutput_type json")
-    flexmock(d.instance.router).should_receive(:emit_error_event)
-    emit(d, {'test' => Float::NAN}, time)
-  end
+      # NOTE: Float::NAN is not jsonable
+      d = create_driver(CONFIG + config_element("", "", { "output_type" => "json" }))
+      flexmock(d.instance.router).should_receive(:emit_error_event)
+      filter(d, etime, {'test' => Float::NAN})
+    end
 
-  def test_output_type_hash
-    d = create_driver(CONFIG + "\noutput_type hash")
-    time = Time.now
-    out = capture_log(d) { emit(d, {'test' => 'test'}, time) }
-    assert_equal "#{time.localtime} filter.test: {\"test\"=>\"test\"}\n", out
+    def test_output_type_hash
+      d = create_driver(CONFIG + config_element("", "", { "output_type" => "hash" }))
+      etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+      out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
+      assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>\"test\"}\n", out
 
-    # NOTE: Float::NAN is not jsonable, but hash string can output it.
-    d = create_driver(CONFIG + "\noutput_type hash")
-    out = capture_log(d) { emit(d, {'test' => Float::NAN}, time) }
-    assert_equal "#{time.localtime} filter.test: {\"test\"=>NaN}\n", out
-  end
+      # NOTE: Float::NAN is not jsonable, but hash string can output it.
+      d = create_driver(CONFIG + config_element("", "", { "output_type" => "hash" }))
+      out = capture_log(d) { filter(d, etime, {'test' => Float::NAN}) }
+      assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>NaN}\n", out
+    end
 
-  # Use include_time_key to output the message's time
-  def test_include_time_key
-    d = create_driver(CONFIG + "\noutput_type json\ninclude_time_key true\nutc")
-    time = Time.now
-    message_time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    out = capture_log(d) { emit(d, {'test' => 'test'}, message_time) }
-    assert_equal "#{time.localtime} filter.test: {\"test\":\"test\",\"time\":\"2011-01-02T13:14:15Z\"}\n", out
-  end
+    # Use include_time_key to output the message's time
+    def test_include_time_key
+      config = config_element("", "", {
+                                "output_type" => "json",
+                                "include_time_key" => true,
+                                "localtime" => false
+                              })
+      d = create_driver(config)
+      etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+      out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
+      assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\",\"time\":\"2016-10-07T21:09:31Z\"}\n", out
+    end
 
-  # out_stdout formatter itself can also be replaced
-  def test_format_json
-    d = create_driver(CONFIG + "\nformat json")
-    time = Time.now
-    out = capture_log(d) { emit(d, {'test' => 'test'}, time) }
-    assert_equal "{\"test\":\"test\"}\n", out
+    # out_stdout formatter itself can also be replaced
+    def test_format_json
+      d = create_driver(CONFIG + config_element("", "", { "format" => "json" }))
+      out = capture_log(d) { filter(d, event_time, {'test' => 'test'}) }
+      assert_equal "{\"test\":\"test\"}\n", out
+    end
   end
 
-  private
+  sub_test_case "with <format> sub section" do
+    sub_test_case "configure" do
+      def test_default
+        conf = config_element
+        conf.elements << config_element("format", "", { "@type" => "stdout"})
+        d = create_driver(conf)
+        d.run {}
+        assert_equal("json", d.instance.formatter.output_type)
+      end
+
+      data(json: "json",
+           hash: "hash",
+           ltsv: "ltsv")
+      def test_output_type(data)
+        conf = config_element
+        conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => data })
+        d = create_driver(conf)
+        d.run {}
+        assert_equal(data, d.instance.formatter.output_type)
+      end
+
+      def test_invalid_output_type
+        conf = config_element
+        conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "foo" })
+        assert_raise(Fluent::ConfigError) do
+          d = create_driver(conf)
+          d.run {}
+        end
+      end
+    end
 
-  # Capture the log output of the block given
-  def capture_log(d, &block)
-    tmp = d.instance.log.out
-    d.instance.log.out = StringIO.new
-    yield
-    return d.instance.log.out.string
-  ensure
-    d.instance.log.out = tmp
+    sub_test_case "output_type" do
+      def test_json
+        conf = config_element
+        conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "json" })
+        d = create_driver(conf)
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+        out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
+        assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\"}\n", out
+      end
+
+      def test_json_nan
+        # NOTE: Float::NAN is not jsonable
+        conf = config_element
+        conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "json" })
+        d = create_driver(conf)
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+        flexmock(d.instance.router).should_receive(:emit_error_event)
+        filter(d, etime, {'test' => Float::NAN})
+      end
+
+      def test_hash
+        conf = config_element
+        conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "hash" })
+        d = create_driver(conf)
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+        out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
+        assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>\"test\"}\n", out
+      end
+
+      def test_hash_nan
+        # NOTE: Float::NAN is not jsonable, but hash string can output it.
+        conf = config_element
+        conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "hash" })
+        d = create_driver(conf)
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+        out = capture_log(d) { filter(d, etime, {'test' => Float::NAN}) }
+        assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>NaN}\n", out
+      end
+
+      # Use include_time_key to output the message's time
+      def test_include_time_key
+        conf = config_element
+        conf.elements << config_element("format", "", {
+                                          "@type" => "stdout",
+                                          "output_type" => "json"
+                                        })
+        conf.elements << config_element("inject", "", {
+                                          "time_key" => "time",
+                                          "time_type" => "string",
+                                          "localtime" => false
+                                          })
+        d = create_driver(conf)
+        etime = event_time("2016-10-07 21:09:31.012345678 UTC")
+        out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
+        assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\",\"time\":\"2016-10-07T21:09:31Z\"}\n", out
+      end
+    end
   end
 end
-

data/test/plugin/test_formatter_csv.rb

@@ -0,0 +1,111 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_csv'
+
+class CsvFormatterTest < ::Test::Unit::TestCase
+
+  def setup
+    @time = event_time
+  end
+
+  CONF = %[
+    fields a,b,c
+  ]
+
+  def create_driver(conf = CONF)
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::CsvFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def test_config_params
+    d = create_driver
+    assert_equal(',', d.instance.delimiter)
+    assert_equal(true, d.instance.force_quotes)
+    assert_equal(['a', 'b', 'c'], d.instance.fields)
+  end
+
+  data('empty array' => [],
+       'array including empty string' => ['', ''])
+  def test_empty_fields(param)
+    assert_raise Fluent::ConfigError do
+      create_driver('fields' => param)
+    end
+  end
+
+  data(
+    'tab_char' => ["\t", '\t'],
+    'tab_string' => ["\t", 'TAB'],
+    'pipe' => ['|', '|'])
+  def test_config_params_with_customized_delimiters(data)
+    expected, target = data
+    d = create_driver("delimiter" => target, 'fields' => 'a,b,c')
+    assert_equal expected, d.instance.delimiter
+  end
+
+  def test_format
+    d = create_driver("fields" => "message,message2")
+    formatted = d.instance.format(tag, @time, {
+                                    'message' => 'awesome',
+                                    'message2' => 'awesome2'
+                                  })
+    assert_equal("\"awesome\",\"awesome2\"\n", formatted)
+  end
+
+  def test_format_without_newline
+    d = create_driver("fields" => "message,message2", "add_newline" => false)
+    formatted = d.instance.format(tag, @time, {
+                                    'message' => 'awesome',
+                                    'message2' => 'awesome2'
+                                  })
+    assert_equal("\"awesome\",\"awesome2\"", formatted)
+  end
+
+  def test_format_with_customized_delimiters
+    d = create_driver("fields" => "message,message2",
+                      "delimiter" => "\t")
+    formatted = d.instance.format(tag, @time, {
+                                    'message' => 'awesome',
+                                    'message2' => 'awesome2'
+                                  })
+    assert_equal("\"awesome\"\t\"awesome2\"\n", formatted)
+  end
+
+  def test_format_with_non_quote
+    d = create_driver("fields" => "message,message2",
+                      "force_quotes" => false)
+    formatted = d.instance.format(tag, @time, {
+                                    'message' => 'awesome',
+                                    'message2' => 'awesome2'
+                                  })
+    assert_equal("awesome,awesome2\n", formatted)
+  end
+
+  data(
+    'nil' => {
+      'message' => 'awesome',
+      'message2' => nil,
+      'message3' => 'awesome3'
+    },
+    'blank' => {
+      'message' => 'awesome',
+      'message2' => '',
+      'message3' => 'awesome3'
+    })
+  def test_format_with_empty_fields(data)
+    d = create_driver("fields" => "message,message2,message3")
+    formatted = d.instance.format(tag, @time, data)
+    assert_equal("\"awesome\",\"\",\"awesome3\"\n", formatted)
+  end
+
+  data(
+    'normally' => 'one,two,three',
+    'white_space' => 'one , two , three',
+    'blank' => 'one,,two,three')
+  def test_config_params_with_fields(data)
+    d = create_driver('fields' => data)
+    assert_equal %w(one two three), d.instance.fields
+  end
+end

data/test/plugin/test_formatter_hash.rb

@@ -0,0 +1,35 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_hash'
+
+class HashFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::HashFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome', 'greeting' => 'hello'}
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}\n!, formatted.encode(Encoding::UTF_8))
+  end
+
+  def test_format_without_newline
+    d = create_driver('add_newline' => false)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}!, formatted.encode(Encoding::UTF_8))
+  end
+end

data/test/plugin/test_formatter_json.rb

@@ -0,0 +1,51 @@
+require_relative '../helper'
+require 'json'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_json'
+
+class JsonFormatterTest < ::Test::Unit::TestCase
+
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::JSONFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome'}
+  end
+
+  def symbolic_record
+    {:message => :awesome}
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_format(data)
+    d = create_driver('json_parser' => data)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{JSON.generate(record)}\n", formatted)
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_format_without_nl(data)
+    d = create_driver('json_parser' => data, 'add_newline' => false)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(JSON.generate(record), formatted)
+  end
+
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_format_with_symbolic_record(data)
+    d = create_driver('json_parser' => data)
+    formatted = d.instance.format(tag, @time, symbolic_record)
+
+    assert_equal("#{JSON.generate(record)}\n", formatted)
+  end
+end

data/test/plugin/test_formatter_ltsv.rb

@@ -0,0 +1,62 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_ltsv'
+
+class LabeledTSVFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::LabeledTSVFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome', 'greeting' => 'hello'}
+  end
+
+  def test_config_params
+    d = create_driver
+    assert_equal "\t", d.instance.delimiter
+    assert_equal  ":", d.instance.label_delimiter
+    assert_equal  true, d.instance.add_newline
+
+    d = create_driver(
+      'delimiter'       => ',',
+      'label_delimiter' => '=',
+      'add_newline' => false,
+    )
+
+    assert_equal ",", d.instance.delimiter
+    assert_equal "=", d.instance.label_delimiter
+    assert_equal  false, d.instance.add_newline
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("message:awesome\tgreeting:hello\n", formatted)
+  end
+
+  def test_format_without_newline
+    d = create_driver('add_newline' => false)
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("message:awesome\tgreeting:hello", formatted)
+  end
+
+  def test_format_with_customized_delimiters
+    d = create_driver(
+      'delimiter'       => ',',
+      'label_delimiter' => '=',
+    )
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("message=awesome,greeting=hello\n", formatted)
+  end
+end

data/test/plugin/test_formatter_msgpack.rb

@@ -0,0 +1,28 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_msgpack'
+
+class MessagePackFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::MessagePackFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome'}
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(record.to_msgpack, formatted)
+  end
+end

data/test/plugin/test_formatter_out_file.rb

@@ -0,0 +1,95 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_out_file'
+
+class OutFileFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = {})
+    d = Fluent::Test::Driver::Formatter.new(Fluent::Plugin::OutFileFormatter)
+    case conf
+    when Fluent::Config::Element
+      d.configure(conf)
+    when Hash
+      d.configure({'utc' => true}.merge(conf))
+    else
+      d.configure(conf)
+    end
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome'}
+  end
+
+  data('both true' => 'true', 'both false' => 'false')
+  def test_configured_with_both_of_utc_and_localtime(value)
+    assert_raise(Fluent::ConfigError.new("both of utc and localtime are specified, use only one of them")) do
+      create_driver({'utc' => value, 'localtime' => value})
+    end
+  end
+
+  time_i = Time.parse("2016-07-26 21:08:30 -0700").to_i
+  data(
+    'configured for localtime by localtime' => ['localtime', 'true',  time_i, "2016-07-26T21:08:30-07:00"],
+    'configured for localtime by utc'       => ['utc',       'false', time_i, "2016-07-26T21:08:30-07:00"],
+    'configured for utc by localtime'       => ['localtime', 'false', time_i, "2016-07-27T04:08:30Z"],
+    'configured for utc by utc'             => ['utc',       'true',  time_i, "2016-07-27T04:08:30Z"],
+  )
+  def test_configured_with_utc_or_localtime(data)
+    key, value, time_i, expected = data
+    time = Time.at(time_i)
+    begin
+      oldtz, ENV['TZ'] = ENV['TZ'], "UTC+07"
+      d = create_driver(config_element('ROOT', '', {key => value}))
+      tag = 'test'
+      assert_equal "#{expected}\t#{tag}\t#{Yajl.dump(record)}\n", d.instance.format(tag, time, record)
+    ensure
+      ENV['TZ'] = oldtz
+    end
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_time
+    d = create_driver('output_time' => 'false')
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_tag
+    d = create_driver('output_tag' => 'false')
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_time_and_tag
+    d = create_driver('output_tag' => 'false', 'output_time' => 'false')
+    formatted = d.instance.format('tag', @time, record)
+
+    assert_equal("#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_time_and_tag_against_string_literal_configure
+    d = create_driver(%[
+                        utc         true
+                        output_tag  false
+                        output_time false
+                      ])
+    formatted = d.instance.format('tag', @time, record)
+
+    assert_equal("#{Yajl.dump(record)}\n", formatted)
+  end
+end