fluentd 1.11.3 → 1.12.0

data/test/command/test_cap_ctl.rb

@@ -0,0 +1,100 @@
+require_relative '../helper'
+
+require 'tempfile'
+require 'fluent/command/cap_ctl'
+
+class TestFluentCapCtl < Test::Unit::TestCase
+  setup do
+    omit "This environment does not handle Linux capability" unless defined?(CapNG)
+  end
+
+  sub_test_case "success" do
+    test "clear capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--clear"]).call
+      end
+      expression = /\AClear capabilities .*\n/m
+      assert_match expression, logs
+    end
+
+    test "add capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--add", "dac_override"]).call
+      end
+      expression = /\AUpdating .* done.\nAdding .*\n/m
+      assert_match expression, logs
+    end
+
+    test "drop capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--drop", "chown"]).call
+      end
+      expression = /\AUpdating .* done.\nDropping .*\n/m
+      assert_match expression, logs
+    end
+
+    test "get capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--get"]).call
+      end
+      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
+      assert_match expression, logs
+    end
+  end
+
+  sub_test_case "success with file" do
+    test "clear capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--clear-cap", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AClear capabilities .*\n/m
+      assert_match expression, logs
+    end
+
+    test "add capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--add", "dac_override", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AUpdating .* done.\nAdding .*\n/m
+      assert_match expression, logs
+    end
+
+    test "drop capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--drop", "chown", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AUpdating .* done.\nDropping .*\n/m
+      assert_match expression, logs
+    end
+
+    test "get capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--get", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
+      assert_match expression, logs
+    end
+  end
+
+  sub_test_case "invalid" do
+    test "add capability" do
+      assert_raise(ArgumentError) do
+        Fluent::CapCtl.new(["--add", "nonexitent"]).call
+      end
+    end
+
+    test "drop capability" do
+      assert_raise(ArgumentError) do
+        Fluent::CapCtl.new(["--drop", "invalid"]).call
+      end
+    end
+  end
+end

data/test/command/test_ctl.rb

@@ -0,0 +1,57 @@
+require_relative '../helper'
+
+require 'test-unit'
+require 'win32/event' if Fluent.windows?
+
+require 'fluent/command/ctl'
+
+class TestFluentdCtl < ::Test::Unit::TestCase
+  def assert_win32_event(event_name, command, pid_or_svcname)
+    command, event_suffix = data
+    event = Win32::Event.new(event_name)
+    ipc = Win32::Ipc.new(event.handle)
+    ret = Win32::Ipc::TIMEOUT
+
+    wait_thread = Thread.new do
+      ret = ipc.wait(1)
+    end
+    Fluent::Ctl.new([command, pid_or_svcname]).call
+    wait_thread.join
+    assert_equal(Win32::Ipc::SIGNALED, ret)
+  end
+
+  data("shutdown" => ["shutdown", "TERM", ""],
+       "restart" => ["restart", "HUP", "HUP"],
+       "flush" => ["flush", "USR1", "USR1"],
+       "reload" => ["reload", "USR2", "USR2"])
+  def test_commands(data)
+    command, signal, event_suffix = data
+
+    if Fluent.windows?
+      event_name = "fluentd_54321"
+      event_name << "_#{event_suffix}" unless event_suffix.empty?
+      assert_win32_event(event_name, command, "54321")
+    else
+      got_signal = false
+      Signal.trap(signal) do
+        got_signal = true
+      end
+      Fluent::Ctl.new([command, "#{$$}"]).call
+      assert_true(got_signal)
+    end
+  end
+
+  data("shutdown" => ["shutdown", ""],
+       "restart" => ["restart", "HUP"],
+       "flush" => ["flush", "USR1"],
+       "reload" => ["reload", "USR2"])
+  def test_commands_with_winsvcname(data)
+    omit "Only for Windows" unless Fluent.windows?
+
+    command, event_suffix = data
+    event_name = "testfluentdwinsvc"
+    event_name << "_#{event_suffix}" unless event_suffix.empty?
+
+    assert_win32_event(event_name, command, "testfluentdwinsvc")
+  end
+end

data/test/command/test_plugin_config_formatter.rb

@@ -56,6 +56,32 @@ class TestFluentPluginConfigFormatter < Test::Unit::TestCase
     end
   end
 
+  class FakeStorage < ::Fluent::Plugin::Storage
+    ::Fluent::Plugin.register_storage('fake', self)
+
+    def get(key)
+    end
+
+    def fetch(key, defval)
+    end
+
+    def put(key, value)
+    end
+
+    def delete(key)
+    end
+
+    def update(key, &block)
+    end
+  end
+
+  class FakeServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery
+    ::Fluent::Plugin.register_sd('fake', self)
+
+    desc "hostname"
+    config_param :hostname, :string
+  end
+
   class SimpleInput < ::Fluent::Plugin::Input
     ::Fluent::Plugin.register_input("simple", self)
     helpers :inject, :compat_parameters
@@ -88,6 +114,13 @@ class TestFluentPluginConfigFormatter < Test::Unit::TestCase
     end
   end
 
+  class SimpleServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery
+    ::Fluent::Plugin.register_sd('simple', self)
+
+    desc "servers"
+    config_param :servers, :array
+  end
+
   sub_test_case "json" do
     data(input: [FakeInput, "input"],
          output: [FakeOutput, "output"],
@@ -196,6 +229,28 @@ TEXT
       assert_equal(expected, dumped_config)
     end
 
+    data("abbrev" => "sd",
+         "normal" => "service_discovery")
+    test "service_discovery simple" do |data|
+      plugin_type = data
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", plugin_type, "simple"]).call
+      end
+      expected = <<TEXT
+* See also: [ServiceDiscovery Plugin Overview](https://docs.fluentd.org/v/1.0/servicediscovery#overview)
+
+## TestFluentPluginConfigFormatter::SimpleServiceDiscovery
+
+### servers (array) (required)
+
+servers
+
+
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+
     test "output complex" do
       dumped_config = capture_stdout do
         FluentPluginConfigFormatter.new(["--format=markdown", "output", "complex"]).call
@@ -251,7 +306,7 @@ TEXT
   sub_test_case "arguments" do
     data do
       hash = {}
-      ["input", "output", "filter", "parser", "formatter"].each do |type|
+      ["input", "output", "filter", "parser", "formatter", "storage", "service_discovery"].each do |type|
         ["txt", "json", "markdown"].each do |format|
           argv = ["--format=#{format}"]
           [
@@ -259,7 +314,7 @@ TEXT
             ["--verbose"],
             ["--compact"]
           ].each do |options|
-            hash[(argv + options).join(" ")] = argv + options + [type, "fake"]
+            hash["[#{type}] " + (argv + options).join(" ")] = argv + options + [type, "fake"]
           end
         end
       end

data/test/plugin/in_tail/test_position_file.rb

@@ -1,5 +1,6 @@
 require_relative '../../helper'
 require 'fluent/plugin/in_tail/position_file'
+require 'fluent/plugin/in_tail'
 
 require 'fileutils'
 require 'tempfile'
@@ -27,9 +28,15 @@ class IntailPositionFileTest < Test::Unit::TestCase
     f.seek(0)
   end
 
+  def follow_inodes_block
+    [true, false].each do |follow_inodes|
+      yield follow_inodes
+    end
+  end
+
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+    Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
     @file.seek(0)
     lines = @file.readlines
@@ -41,7 +48,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#try_compact' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -55,7 +62,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -64,7 +71,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change when the file is changed' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
 
       mock.proxy(pf).fetch_compacted_entries do |r|
         @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
@@ -79,11 +86,16 @@ class IntailPositionFileTest < Test::Unit::TestCase
     end
 
     test 'update seek postion of remained position entry' do
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log)
-      pf['path1']
-      pf['path2']
-      pf['path3']
-      pf.unwatch('path1')
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
+      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
+      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
+      target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)
+      pf[target_info1]
+      pf[target_info2]
+      pf[target_info3]
+
+      target_info1_2 = Fluent::Plugin::TailInput::TargetInfo.new('path1', 1234)
+      pf.unwatch(target_info1_2)
 
       pf.try_compact
 
@@ -93,8 +105,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal "path3\t0000000000000000\t0000000000000000\n", lines[1]
       assert_equal 2, lines.size
 
-      pf.unwatch('path2')
-      pf.unwatch('path3')
+      target_info2_2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', 1235)
+      target_info3_2 = Fluent::Plugin::TailInput::TargetInfo.new('path3', 1236)
+      pf.unwatch(target_info2_2)
+      pf.unwatch(target_info3_2)
       @file.seek(0)
       lines = @file.readlines
       assert_equal "path2\t#{UNWATCHED_STR}\t0000000000000000\n", lines[0]
@@ -106,7 +120,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
       @file.seek(0)
       lines = @file.readlines
@@ -120,7 +134,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, logger: $log).load
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).load
 
       @file.seek(0)
       lines = @file.readlines
@@ -131,9 +145,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#[]' do
     test 'return entry' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
-      f = pf['valid_path']
+      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)
+      f = pf[valid_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 2, f.read_pos
       assert_equal 1, f.read_inode
@@ -142,7 +157,8 @@ class IntailPositionFileTest < Test::Unit::TestCase
       lines = @file.readlines
       assert_equal 2, lines.size
 
-      f = pf['nonexist_path']
+      nonexistent_target_info = Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)
+      f = pf[nonexistent_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 0, f.read_pos
       assert_equal 0, f.read_inode
@@ -155,19 +171,19 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change other value position if other entry try to write' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
 
-      f = pf['nonexist_path']
+      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf['valid_path'].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)].update(1, 2)
 
-      f = pf['nonexist_path']
+      f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf['nonexist_path'].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)].update(1, 2)
       assert_equal 1, f.read_inode
       assert_equal 2, f.read_pos
     end
@@ -176,14 +192,18 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#unwatch' do
     test 'deletes entry by path' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, logger: $log)
-      p1 = pf['valid_path']
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
+      inode1 = Fluent::FileWrapper.stat(@file).ino
+      target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
+      p1 = pf[target_info1]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
 
-      pf.unwatch('valid_path')
+      pf.unwatch(target_info1)
       assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
 
-      p2 = pf['valid_path']
+      inode2 = Fluent::FileWrapper.stat(@file).ino
+      target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
+      p2 = pf[target_info2]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class
 
       assert_not_equal p1, p2

data/test/plugin/test_filter_stdout.rb

@@ -12,6 +12,11 @@ class StdoutFilterTest < Test::Unit::TestCase
     @old_tz = ENV["TZ"]
     ENV["TZ"] = "UTC"
     Timecop.freeze
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   def teardown
@@ -106,7 +111,7 @@ class StdoutFilterTest < Test::Unit::TestCase
     def test_format_json
       d = create_driver(CONFIG + config_element("", "", { "format" => "json" }))
       out = capture_log(d) { filter(d, event_time, {'test' => 'test'}) }
-      assert_equal "{\"test\":\"test\"}\n", out
+      assert_equal "{\"test\":\"test\"}#{@default_newline}", out
     end
   end
 

data/test/plugin/test_formatter_hash.rb

@@ -19,11 +19,14 @@ class HashFormatterTest < ::Test::Unit::TestCase
     {'message' => 'awesome', 'greeting' => 'hello'}
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}\n!, formatted.encode(Encoding::UTF_8))
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}#{newline}!, formatted.encode(Encoding::UTF_8))
   end
 
   def test_format_without_newline