fluentd 1.11.2-x64-mingw32 → 1.12.1-x64-mingw32

data/lib/fluent/system_config.rb

@@ -24,10 +24,10 @@ module Fluent
     SYSTEM_CONFIG_PARAMETERS = [
       :workers, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
-      :log_event_verbose, :ignore_repeated_log_interval,
+      :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value, :enable_msgpack_time_support
+      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
     ]
 
     config_param :workers,   :integer, default: 1
@@ -35,6 +35,7 @@ module Fluent
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
     config_param :ignore_repeated_log_interval, :time, default: nil
+    config_param :ignore_same_log_interval, :time, default: nil
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
     config_param :log_event_verbose,    :bool, default: nil
@@ -44,6 +45,7 @@ module Fluent
     config_param :process_name,    :string, default: nil
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
+    config_param :disable_shared_socket, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end

data/lib/fluent/time.rb

@@ -50,6 +50,7 @@ module Fluent
     def to_int
       @sec
     end
+    alias :to_i :to_int
 
     def to_f
       @sec + @nsec / 1_000_000_000.0

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.11.2'
+  VERSION = '1.12.1'
 
 end

data/lib/fluent/winsvc.rb

@@ -61,7 +61,6 @@ begin
     end
     
     def service_main
-    
       @pid = service_main_start(@service_name)
       while running?
         sleep 10
@@ -69,13 +68,32 @@ begin
     end
 
     def service_stop
-      ev = Win32::Event.open(@service_name)
-      ev.set
-      ev.close
+      set_event(@service_name)
       if @pid > 0
         Process.waitpid(@pid)
       end
     end
+
+    def service_paramchange
+      set_event("#{@service_name}_USR2")
+    end
+
+    def service_user_defined_control(code)
+      case code
+      when 128
+        set_event("#{@service_name}_HUP")
+      when 129
+        set_event("#{@service_name}_USR1")
+      end
+    end
+
+    private
+
+    def set_event(event_name)
+      ev = Win32::Event.open(event_name)
+      ev.set
+      ev.close
+    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

data/templates/plugin_config_formatter/param.md-table.erb

@@ -0,0 +1,10 @@
+<%-
+type = config[:type]
+required_label = config[:required] ? "required" : "optional"
+default = config[:default]
+alias_name = config[:alias]
+deprecated = config[:deprecated]
+obsoleted = config[:obsoleted]
+description = config[:desc]
+-%>
+|<%= name %>|<%= type %> (<%= required_label %>)|<%= description %><%- if type == :enum -%> (<%= config[:list].map{|x| "`#{x}`"}.join(", ") %>)<%- end -%><%- if alias_name -%><br>Alias: <%= alias_name %><%- end -%><%- if deprecated -%><br>Deprecated: <%= deprecated %><%- end -%><%- if obsoleted -%><br>Obsoleted: <%= :obsoleted %><%- end -%>|<%- if default -%>`<%= default %>`<%- end -%>|

data/test/command/test_binlog_reader.rb

@@ -82,6 +82,14 @@ class TestBaseCommand < ::Test::Unit::TestCase
 end
 
 class TestHead < TestBaseCommand
+  setup do
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
+  end
+
   sub_test_case 'initialize' do
     data(
       'file is not passed' => %w(),
@@ -138,7 +146,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 5, out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 5, out
       end
     end
 
@@ -149,7 +157,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -169,7 +177,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}\n", out
+        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -198,6 +206,14 @@ class TestHead < TestBaseCommand
 end
 
 class TestCat < TestBaseCommand
+  setup do
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
+  end
+
   sub_test_case 'initialize' do
     data(
       'file is not passed' => [],
@@ -254,7 +270,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 6, out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 6, out
       end
     end
 
@@ -265,7 +281,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -276,7 +292,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}\n", out
+        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 

data/test/command/test_cap_ctl.rb

@@ -0,0 +1,100 @@
+require_relative '../helper'
+
+require 'tempfile'
+require 'fluent/command/cap_ctl'
+
+class TestFluentCapCtl < Test::Unit::TestCase
+  setup do
+    omit "This environment does not handle Linux capability" unless defined?(CapNG)
+  end
+
+  sub_test_case "success" do
+    test "clear capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--clear"]).call
+      end
+      expression = /\AClear capabilities .*\n/m
+      assert_match expression, logs
+    end
+
+    test "add capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--add", "dac_override"]).call
+      end
+      expression = /\AUpdating .* done.\nAdding .*\n/m
+      assert_match expression, logs
+    end
+
+    test "drop capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--drop", "chown"]).call
+      end
+      expression = /\AUpdating .* done.\nDropping .*\n/m
+      assert_match expression, logs
+    end
+
+    test "get capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--get"]).call
+      end
+      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
+      assert_match expression, logs
+    end
+  end
+
+  sub_test_case "success with file" do
+    test "clear capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--clear-cap", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AClear capabilities .*\n/m
+      assert_match expression, logs
+    end
+
+    test "add capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--add", "dac_override", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AUpdating .* done.\nAdding .*\n/m
+      assert_match expression, logs
+    end
+
+    test "drop capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--drop", "chown", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AUpdating .* done.\nDropping .*\n/m
+      assert_match expression, logs
+    end
+
+    test "get capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--get", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
+      assert_match expression, logs
+    end
+  end
+
+  sub_test_case "invalid" do
+    test "add capability" do
+      assert_raise(ArgumentError) do
+        Fluent::CapCtl.new(["--add", "nonexitent"]).call
+      end
+    end
+
+    test "drop capability" do
+      assert_raise(ArgumentError) do
+        Fluent::CapCtl.new(["--drop", "invalid"]).call
+      end
+    end
+  end
+end

data/test/command/test_ctl.rb

@@ -0,0 +1,57 @@
+require_relative '../helper'
+
+require 'test-unit'
+require 'win32/event' if Fluent.windows?
+
+require 'fluent/command/ctl'
+
+class TestFluentdCtl < ::Test::Unit::TestCase
+  def assert_win32_event(event_name, command, pid_or_svcname)
+    command, event_suffix = data
+    event = Win32::Event.new(event_name)
+    ipc = Win32::Ipc.new(event.handle)
+    ret = Win32::Ipc::TIMEOUT
+
+    wait_thread = Thread.new do
+      ret = ipc.wait(1)
+    end
+    Fluent::Ctl.new([command, pid_or_svcname]).call
+    wait_thread.join
+    assert_equal(Win32::Ipc::SIGNALED, ret)
+  end
+
+  data("shutdown" => ["shutdown", "TERM", ""],
+       "restart" => ["restart", "HUP", "HUP"],
+       "flush" => ["flush", "USR1", "USR1"],
+       "reload" => ["reload", "USR2", "USR2"])
+  def test_commands(data)
+    command, signal, event_suffix = data
+
+    if Fluent.windows?
+      event_name = "fluentd_54321"
+      event_name << "_#{event_suffix}" unless event_suffix.empty?
+      assert_win32_event(event_name, command, "54321")
+    else
+      got_signal = false
+      Signal.trap(signal) do
+        got_signal = true
+      end
+      Fluent::Ctl.new([command, "#{$$}"]).call
+      assert_true(got_signal)
+    end
+  end
+
+  data("shutdown" => ["shutdown", ""],
+       "restart" => ["restart", "HUP"],
+       "flush" => ["flush", "USR1"],
+       "reload" => ["reload", "USR2"])
+  def test_commands_with_winsvcname(data)
+    omit "Only for Windows" unless Fluent.windows?
+
+    command, event_suffix = data
+    event_name = "testfluentdwinsvc"
+    event_name << "_#{event_suffix}" unless event_suffix.empty?
+
+    assert_win32_event(event_name, command, "testfluentdwinsvc")
+  end
+end

data/test/command/test_fluentd.rb

@@ -1065,4 +1065,34 @@ CONF
                          "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"])
     end
   end
+
+  sub_test_case 'sahred socket options' do
+    test 'enable shared socket by default' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path),
+                         patterns_not_match: ["shared socket for multiple workers is disabled"])
+    end
+
+    test 'disable shared socket by command line option' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+
+    test 'disable shared socket by system config' do
+      conf = <<CONF
+<system>
+  disable_shared_socket
+</system>
+CONF
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+  end
 end

data/test/command/test_plugin_config_formatter.rb

@@ -56,6 +56,32 @@ class TestFluentPluginConfigFormatter < Test::Unit::TestCase
     end
   end
 
+  class FakeStorage < ::Fluent::Plugin::Storage
+    ::Fluent::Plugin.register_storage('fake', self)
+
+    def get(key)
+    end
+
+    def fetch(key, defval)
+    end
+
+    def put(key, value)
+    end
+
+    def delete(key)
+    end
+
+    def update(key, &block)
+    end
+  end
+
+  class FakeServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery
+    ::Fluent::Plugin.register_sd('fake', self)
+
+    desc "hostname"
+    config_param :hostname, :string
+  end
+
   class SimpleInput < ::Fluent::Plugin::Input
     ::Fluent::Plugin.register_input("simple", self)
     helpers :inject, :compat_parameters
@@ -88,6 +114,13 @@ class TestFluentPluginConfigFormatter < Test::Unit::TestCase
     end
   end
 
+  class SimpleServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery
+    ::Fluent::Plugin.register_sd('simple', self)
+
+    desc "servers"
+    config_param :servers, :array
+  end
+
   sub_test_case "json" do
     data(input: [FakeInput, "input"],
          output: [FakeOutput, "output"],
@@ -196,6 +229,52 @@ TEXT
       assert_equal(expected, dumped_config)
     end
 
+    test "input simple (table)" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "--table", "input", "simple"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Input Plugin Overview](https://docs.fluentd.org/v/1.0/input#overview)
+
+## TestFluentPluginConfigFormatter::SimpleInput
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|path|string (required)|path to something||
+
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    data("abbrev" => "sd",
+         "normal" => "service_discovery")
+    test "service_discovery simple" do |data|
+      plugin_type = data
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", plugin_type, "simple"]).call
+      end
+      expected = <<TEXT
+* See also: [ServiceDiscovery Plugin Overview](https://docs.fluentd.org/v/1.0/servicediscovery#overview)
+
+## TestFluentPluginConfigFormatter::SimpleServiceDiscovery
+
+### servers (array) (required)
+
+servers
+
+
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+
     test "output complex" do
       dumped_config = capture_stdout do
         FluentPluginConfigFormatter.new(["--format=markdown", "output", "complex"]).call
@@ -243,6 +322,49 @@ Default value: `normal`.
 
 
 
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "output complex (table)" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "--table", "output", "complex"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Output Plugin Overview](https://docs.fluentd.org/v/1.0/output#overview)
+
+## TestFluentPluginConfigFormatter::ComplexOutput
+
+
+### \\<authentication\\> section (required) (single)
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|username|string (required)|username||
+|password|string (required)|password||
+
+
+### \\<parent\\> section (optional) (multiple)
+
+
+#### \\<child\\> section (optional) (multiple)
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|names|array (required)|names||
+|difficulty|enum (optional)|difficulty (`easy`, `normal`, `hard`)|`normal`|
+
+
+
 TEXT
       assert_equal(expected, dumped_config)
     end
@@ -251,7 +373,7 @@ TEXT
   sub_test_case "arguments" do
     data do
       hash = {}
-      ["input", "output", "filter", "parser", "formatter"].each do |type|
+      ["input", "output", "filter", "parser", "formatter", "storage", "service_discovery"].each do |type|
         ["txt", "json", "markdown"].each do |format|
           argv = ["--format=#{format}"]
           [
@@ -259,7 +381,7 @@ TEXT
             ["--verbose"],
             ["--compact"]
           ].each do |options|
-            hash[(argv + options).join(" ")] = argv + options + [type, "fake"]
+            hash["[#{type}] " + (argv + options).join(" ")] = argv + options + [type, "fake"]
           end
         end
       end