fluentd 0.14.3 → 0.14.4

data/test/plugin/test_in_monitor_agent.rb

@@ -0,0 +1,329 @@
+require_relative '../helper'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/engine'
+require 'fluent/config'
+require 'fluent/event_router'
+require 'fluent/supervisor'
+require 'net/http'
+require 'json'
+require_relative '../test_plugin_classes'
+
+class MonitorAgentInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = '')
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::MonitorAgentInput).configure(conf)
+  end
+
+  def configure_ra(ra, conf_str)
+    conf = Fluent::Config.parse(conf_str, "(test)", "(test_dir)", true)
+    ra.configure(conf)
+    ra
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal("0.0.0.0", d.instance.bind)
+    assert_equal(24220, d.instance.port)
+    assert_equal(nil, d.instance.tag)
+    assert_equal(60, d.instance.emit_interval)
+  end
+
+  sub_test_case "collect plugin information" do
+    setup do
+      # check @type and type in one configuration
+      conf = <<-EOC
+<source>
+  @type test_in
+  @id test_in
+</source>
+<filter>
+  @type test_filter
+  @id test_filter
+</filter>
+<match **>
+  @type relabel
+  @id test_relabel
+  @label @test
+</match>
+<label @test>
+  <match **>
+    @type test_out
+    @id test_out
+  </match>
+</label>
+<label @ERROR>
+  <match>
+    @type null
+    @id null
+  </match>
+</label>
+EOC
+      @ra = Fluent::RootAgent.new(log: $log)
+      stub(Fluent::Engine).root_agent { @ra }
+      @ra = configure_ra(@ra, conf)
+    end
+
+    test "plugin_category" do
+      d = create_driver
+      test_label = @ra.labels['@test']
+      error_label = @ra.labels['@ERROR']
+      assert_equal("input", d.instance.plugin_category(@ra.inputs.first))
+      assert_equal("filter", d.instance.plugin_category(@ra.filters.first))
+      assert_equal("output", d.instance.plugin_category(test_label.outputs.first))
+      assert_equal("output", d.instance.plugin_category(error_label.outputs.first))
+    end
+
+    test "get_monitor_info" do
+      d = create_driver
+      test_label = @ra.labels['@test']
+      error_label = @ra.labels['@ERROR']
+      input_info = {
+        "config" => {
+          "@id"   => "test_in",
+          "@type" => "test_in"
+        },
+        "output_plugin"  => false,
+        "plugin_category"=> "input",
+        "plugin_id"      => "test_in",
+        "retry_count"    => nil,
+        "type"           => "test_in"
+      }
+      filter_info = {
+        "config" => {
+          "@id"   => "test_filter",
+          "@type" => "test_filter"
+        },
+        "output_plugin"   => false,
+        "plugin_category" => "filter",
+        "plugin_id"       => "test_filter",
+        "retry_count"     => nil,
+        "type"            => "test_filter"
+      }
+      output_info = {
+        "config" => {
+          "@id"   => "test_out",
+          "@type" => "test_out"
+        },
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "test_out",
+        "retry_count"     => 0,
+        "type"            => "test_out"
+      }
+      error_label_info = {
+        "config" => {
+          "@id"=>"null",
+          "@type" => "null"
+        },
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "null",
+        "retry_count"     => 0,
+        "type"            => "null"
+      }
+      assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first))
+      assert_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first))
+      assert_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first))
+      assert_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first))
+    end
+
+    test "fluentd opts" do
+      d = create_driver
+      opts = Fluent::Supervisor.default_options
+      Fluent::Supervisor.new(opts)
+      expected_opts = {
+        "config_path" => "/etc/fluent/fluent.conf",
+        "pid_file"    => nil,
+        "plugin_dirs" => ["/etc/fluent/plugin"],
+        "log_path"    => nil
+      }
+      assert_equal(expected_opts, d.instance.fluentd_opts)
+    end
+
+    test "all_plugins" do
+      d = create_driver
+      plugins = []
+      d.instance.all_plugins.each {|plugin| plugins << plugin.class }
+      assert_equal([FluentTest::FluentTestInput,
+                    Fluent::Plugin::RelabelOutput,
+                    FluentTest::FluentTestFilter,
+                    FluentTest::FluentTestOutput,
+                    Fluent::Plugin::NullOutput], plugins)
+    end
+
+    test "emit" do
+      port = unused_port
+      d = create_driver("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{port}
+  tag monitor
+  emit_interval 1
+")
+      d.instance.start
+      d.end_if do
+        d.events.size >= 5
+      end
+      d.run
+      expect_relabel_record = {
+        "plugin_id"       => "test_relabel",
+        "plugin_category" => "output",
+        "type"            => "relabel",
+        "output_plugin"   => true,
+        "retry_count"     => 0}
+      expect_test_out_record = {
+        "plugin_id"       => "test_out",
+        "plugin_category" => "output",
+        "type"            => "test_out",
+        "output_plugin"   => true,
+        "retry_count"     => 0
+      }
+      assert_equal(expect_relabel_record, d.events[1][2])
+      assert_equal(expect_test_out_record, d.events[3][2])
+    end
+  end
+
+  def get(uri, header = {})
+    url = URI.parse(uri)
+    req = Net::HTTP::Get.new(url.path, header)
+    unless header.has_key?('Content-Type')
+      header['Content-Type'] = 'application/octet-stream'
+    end
+    res = Net::HTTP.start(url.host, url.port) {|http|
+      http.request(req)
+    }
+    res.body
+  end
+
+  sub_test_case "servlets" do
+    setup do
+      @port = unused_port
+      # check @type and type in one configuration
+      conf = <<-EOC
+<source>
+  @type test_in
+  @id test_in
+</source>
+<source>
+  @type monitor_agent
+  bind "127.0.0.1"
+  port #{@port}
+  tag monitor
+  @id monitor_agent
+</source>
+<filter>
+  @type test_filter
+  @id test_filter
+</filter>
+<match **>
+  @type relabel
+  @id test_relabel
+  @label @test
+</match>
+<label @test>
+  <match **>
+    @type test_out
+    @id test_out
+  </match>
+</label>
+<label @ERROR>
+  <match>
+    @type null
+    @id null
+  </match>
+</label>
+EOC
+      @ra = Fluent::RootAgent.new(log: $log)
+      stub(Fluent::Engine).root_agent { @ra }
+      @ra = configure_ra(@ra, conf)
+    end
+
+    test "/api/plugins" do
+      d = create_driver("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{@port}
+  tag monitor
+")
+      d.instance.start
+      expected_test_in_response = "\
+plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:"
+      expected_test_filter_response = "\
+plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:"
+
+      response = get("http://127.0.0.1:#{@port}/api/plugins")
+      test_in = response.split("\n")[0]
+      test_filter = response.split("\n")[3]
+      assert_equal(expected_test_in_response, test_in)
+      assert_equal(expected_test_filter_response, test_filter)
+    end
+
+    test "/api/plugins.json" do
+      d = create_driver("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{@port}
+  tag monitor
+")
+      d.instance.start
+      expected_test_in_response =
+        {"config" => {
+          "@id"   => "test_in",
+          "@type" => "test_in"
+        },
+        "output_plugin"   => false,
+        "plugin_category" => "input",
+        "plugin_id"       => "test_in",
+        "retry_count"     => nil,
+        "type"            => "test_in"}
+      expected_null_response =
+        {"config" => {
+          "@id"   => "null",
+          "@type" => "null"
+        },
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "null",
+        "retry_count"     => 0,
+        "type"            => "null"}
+      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json"))
+      test_in_response = response["plugins"][0]
+      null_response = response["plugins"][5]
+      assert_equal(expected_test_in_response, test_in_response)
+      assert_equal(expected_null_response, null_response)
+    end
+
+    test "/api/config" do
+      d = create_driver("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{@port}
+  tag monitor
+")
+      d.instance.start
+      expected_response_regex = /pid:\d+\tppid:\d+\tconfig_path:\/etc\/fluent\/fluent.conf\tpid_file:\tplugin_dirs:\[\"\/etc\/fluent\/plugin\"\]\tlog_path:/
+
+      assert_match(expected_response_regex,
+                   get("http://127.0.0.1:#{@port}/api/config"))
+    end
+
+    test "/api/config.json" do
+      d = create_driver("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{@port}
+  tag monitor
+")
+      d.instance.start
+      res = JSON.parse(get("http://127.0.0.1:#{@port}/api/config.json"))
+      assert_equal("/etc/fluent/fluent.conf", res["config_path"])
+      assert_nil(res["pid_file"])
+      assert_equal(["/etc/fluent/plugin"], res["plugin_dirs"])
+      assert_nil(res["log_path"])
+    end
+  end
+end

data/test/plugin/test_in_tail.rb

@@ -71,6 +71,32 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_configure_from_encoding
+    # If only specified from_encoding raise ConfigError
+    assert_raise(Fluent::ConfigError) do
+      create_driver(SINGLE_LINE_CONFIG + 'from_encoding utf-8')
+    end
+
+    # valid setting
+    d = create_driver %[
+      format /(?<message>.*)/
+      read_from_head true
+      from_encoding utf-8
+      encoding utf-8
+    ]
+    assert_equal Encoding::UTF_8, d.instance.from_encoding
+
+    # invalid from_encoding
+    assert_raise(Fluent::ConfigError) do
+      d = create_driver %[
+        format /(?<message>.*)/
+        read_from_head true
+        from_encoding no-such-encoding
+        encoding utf-8
+      ]
+    end
+  end
+
   # TODO: Should using more better approach instead of sleep wait
 
   def test_emit
@@ -403,6 +429,28 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal(encoding, emits[0][2]['message'].encoding)
   end
 
+  def test_from_encoding
+    d = create_driver %[
+      format /(?<message>.*)/
+      read_from_head true
+      from_encoding cp932
+      encoding utf-8
+    ]
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "w:cp932") {|f|
+        f.puts "\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message'])
+    assert_equal(Encoding::UTF_8, emits[0][2]['message'].encoding)
+  end
+
   # multiline mode test
 
   def test_multiline
@@ -507,6 +555,31 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_multiline_from_encoding_of_flushed_record
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+      multiline_flush_interval 2s
+      read_from_head true
+      from_encoding cp932
+      encoding utf-8
+    ]
+
+    d.run do
+      sleep 1
+      File.open("#{TMP_DIR}/tail.txt", "w:cp932") { |f|
+        f.puts "s \x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932)
+      }
+
+      sleep 4
+      emits = d.emits
+      assert_equal(1, emits.length)
+      assert_equal("\x82\xCD\x82\xEB\x81\x5B\x82\xED\x81\x5B\x82\xE9\x82\xC7".force_encoding(Encoding::CP932).encode(Encoding::UTF_8), emits[0][2]['message1'])
+      assert_equal(Encoding::UTF_8, emits[0][2]['message1'].encoding)
+    end
+  end
+
   def test_multiline_with_multiple_formats
     File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
 

data/test/plugin/test_out_forward.rb

@@ -454,6 +454,7 @@ class ForwardOutputTest < Test::Unit::TestCase
             option
           }
           @source = nil
+          @peeraddr = nil
         end
 
         if do_respond

data/test/plugin/test_output.rb

@@ -98,6 +98,17 @@ module FluentPluginOutputTest
 end
 
 class OutputTest < Test::Unit::TestCase
+    class << self
+      def startup
+        $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '../scripts'))
+        require 'fluent/plugin/out_test'
+      end
+
+      def shutdown
+        $LOAD_PATH.shift
+      end
+    end
+
   def create_output(type=:full)
     case type
     when :bare     then FluentPluginOutputTest::DummyBareOutput.new
@@ -181,6 +192,9 @@ class OutputTest < Test::Unit::TestCase
       assert !@i.started?
       @i.start
       assert @i.started?
+      assert !@i.after_started?
+      @i.after_start
+      assert @i.after_started?
       assert !@i.stopped?
       @i.stop
       assert @i.stopped?
@@ -318,6 +332,7 @@ class OutputTest < Test::Unit::TestCase
       i.register(:process){|tag, es| process_called = true }
       i.configure(config_element())
       i.start
+      i.after_start
 
       t = event_time()
       i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
@@ -333,6 +348,7 @@ class OutputTest < Test::Unit::TestCase
       i.register(:format){|tag, time, record| format_called_times += 1; '' }
       i.configure(config_element())
       i.start
+      i.after_start
 
       t = event_time()
       i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
@@ -353,6 +369,7 @@ class OutputTest < Test::Unit::TestCase
       i.configure(config_element())
       i.register(:prefer_buffered_processing){ false } # delayed decision is possible to change after (output's) configure
       i.start
+      i.after_start
 
       assert !i.prefer_buffered_processing
 
@@ -378,6 +395,7 @@ class OutputTest < Test::Unit::TestCase
       i.configure(config_element())
       i.register(:prefer_buffered_processing){ true } # delayed decision is possible to change after (output's) configure
       i.start
+      i.after_start
 
       assert i.prefer_buffered_processing
 
@@ -397,6 +415,7 @@ class OutputTest < Test::Unit::TestCase
 
       i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
       i.start
+      i.after_start
 
       t = event_time()
       i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
@@ -416,6 +435,7 @@ class OutputTest < Test::Unit::TestCase
 
       i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
       i.start
+      i.after_start
 
       t = event_time()
       i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
@@ -438,6 +458,7 @@ class OutputTest < Test::Unit::TestCase
       i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
       i.register(:prefer_delayed_commit){ false } # delayed decision is possible to change after (output's) configure
       i.start
+      i.after_start
 
       assert !i.prefer_delayed_commit
 
@@ -463,6 +484,7 @@ class OutputTest < Test::Unit::TestCase
       i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
       i.register(:prefer_delayed_commit){ true } # delayed decision is possible to change after (output's) configure
       i.start
+      i.after_start
 
       assert i.prefer_delayed_commit
 
@@ -477,6 +499,36 @@ class OutputTest < Test::Unit::TestCase
 
       i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
     end
+
+    test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
+      o = create_output(:buffered)
+      mock(o.log).warn("secondary type should be same with primary one",
+                                { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
+
+      o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
+      assert_not_nil o.instance_variable_get(:@secondary)
+    end
+
+    test "don't warn if primary type is the same as secondary type" do
+      o = Fluent::Plugin::TestOutput.new
+      mock(o.log).warn("secondary type should be same with primary one",
+                                { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+
+      o.configure(config_element('ROOT','',{'name' => "cool2"},
+                                 [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
+                                  config_element('buffer','',{'@type'=>'memory'})]
+                                ))
+      assert_not_nil o.instance_variable_get(:@secondary)
+    end
+
+    test "don't warn if primary type is different from secondary type and both don't have custom_format" do
+      o = create_output(:standard)
+      mock(o.log).warn("secondary type should be same with primary one",
+                                { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+
+      o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
+      assert_not_nil o.instance_variable_get(:@secondary)
+    end
   end
 
   sub_test_case 'sync output feature' do
@@ -501,6 +553,7 @@ class OutputTest < Test::Unit::TestCase
       @i.register(:process){|tag, es| ary << [tag, es] }
       @i.configure(config_element())
       @i.start
+      @i.after_start
 
       t = event_time()
       es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])