fluentd 1.11.5-x64-mingw32 → 1.12.4-x64-mingw32

data/lib/fluent/system_config.rb

@@ -27,7 +27,7 @@ module Fluent
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value, :enable_msgpack_time_support
+      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
     ]
 
     config_param :workers,   :integer, default: 1
@@ -45,6 +45,7 @@ module Fluent
     config_param :process_name,    :string, default: nil
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
+    config_param :disable_shared_socket, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end

data/lib/fluent/time.rb

@@ -50,6 +50,7 @@ module Fluent
     def to_int
       @sec
     end
+    alias :to_i :to_int
 
     def to_f
       @sec + @nsec / 1_000_000_000.0
@@ -131,13 +132,14 @@ module Fluent
   end
 
   module TimeMixin
-    TIME_TYPES = ['string', 'unixtime', 'float']
+    TIME_TYPES = ['string', 'unixtime', 'float', 'mixed']
 
     TIME_PARAMETERS = [
       [:time_format, :string, {default: nil}],
       [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil
       [:utc,       :bool, {default: false}], # to turn :localtime false
       [:timezone, :string, {default: nil}],
+      [:time_format_fallbacks, :array, {default: []}], # try time_format, then try fallbacks
     ]
     TIME_FULL_PARAMETERS = [
       # To avoid to define :time_type twice (in plugin_helper/inject)
@@ -169,6 +171,12 @@ module Fluent
           raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
         end
 
+        if conf.has_key?('time_type') and @time_type == :mixed
+          if @time_format.nil? and @time_format_fallbacks.empty?
+            raise Fluent::ConfigError, "time_type is :mixed but time_format and time_format_fallbacks is empty."
+          end
+        end
+
         Fluent::Timezone.validate!(@timezone) if @timezone
       end
     end
@@ -179,6 +187,7 @@ module Fluent
       end
 
       def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return MixedTimeParser.new(type, format, @localtime, timezone, @utc, force_localtime, @time_format_fallbacks) if type == :mixed
         return NumericTimeParser.new(type) if type != :string
         return TimeParser.new(format, true, nil) if force_localtime
 
@@ -451,4 +460,52 @@ module Fluent
       end
     end
   end
+
+  # MixedTimeParser is available when time_type is set to :mixed
+  #
+  # Use Case 1: primary format is specified explicitly in time_format
+  #  time_type mixed
+  #  time_format %iso8601
+  #  time_format_fallbacks unixtime
+  # Use Case 2: time_format is omitted
+  #  time_type mixed
+  #  time_format_fallbacks %iso8601, unixtime
+  #
+  class MixedTimeParser < TimeParser # to include TimeParseError
+    def initialize(type, format = nil, localtime = nil, timezone = nil, utc = nil, force_localtime = nil, fallbacks = [])
+      @parsers = []
+      fallbacks.unshift(format).each do |fallback|
+        next unless fallback
+        case fallback
+        when 'unixtime', 'float'
+          @parsers << NumericTimeParser.new(fallback, localtime, timezone)
+        else
+          if force_localtime
+            @parsers << TimeParser.new(fallback, true, nil)
+          else
+            localtime = localtime && (timezone.nil? && !utc)
+            @parsers << TimeParser.new(fallback, localtime, timezone)
+          end
+        end
+      end
+    end
+
+    def parse(value)
+      @parsers.each do |parser|
+        begin
+          Float(value) if parser.class == Fluent::NumericTimeParser
+        rescue
+          next
+        end
+        begin
+          return parser.parse(value)
+        rescue
+          # skip TimeParseError
+        end
+      end
+      fallback_class = @parsers.collect do |parser| parser.class end.join(",")
+      raise TimeParseError, "invalid time format: value = #{value}, even though fallbacks: #{fallback_class}"
+    end
+  end
+
 end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.11.5'
+  VERSION = '1.12.4'
 
 end

data/lib/fluent/winsvc.rb

@@ -61,7 +61,6 @@ begin
     end
     
     def service_main
-    
       @pid = service_main_start(@service_name)
       while running?
         sleep 10
@@ -69,13 +68,32 @@ begin
     end
 
     def service_stop
-      ev = Win32::Event.open(@service_name)
-      ev.set
-      ev.close
+      set_event(@service_name)
       if @pid > 0
         Process.waitpid(@pid)
       end
     end
+
+    def service_paramchange
+      set_event("#{@service_name}_USR2")
+    end
+
+    def service_user_defined_control(code)
+      case code
+      when 128
+        set_event("#{@service_name}_HUP")
+      when 129
+        set_event("#{@service_name}_USR1")
+      end
+    end
+
+    private
+
+    def set_event(event_name)
+      ev = Win32::Event.open(event_name)
+      ev.set
+      ev.close
+    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

data/templates/new_gem/fluent-plugin.gemspec.erb

@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.14"
-  spec.add_development_dependency "rake", "~> 12.0"
-  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_development_dependency "bundler", "~> <%= bundler_version %>"
+  spec.add_development_dependency "rake", "~> <%= rake_version %>"
+  spec.add_development_dependency "test-unit", "~> <%= test_unit_version %>"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/templates/plugin_config_formatter/param.md-table.erb

@@ -0,0 +1,10 @@
+<%-
+type = config[:type]
+required_label = config[:required] ? "required" : "optional"
+default = config[:default]
+alias_name = config[:alias]
+deprecated = config[:deprecated]
+obsoleted = config[:obsoleted]
+description = config[:desc]
+-%>
+|<%= name %>|<%= type %> (<%= required_label %>)|<%= description %><%- if type == :enum -%> (<%= config[:list].map{|x| "`#{x}`"}.join(", ") %>)<%- end -%><%- if alias_name -%><br>Alias: <%= alias_name %><%- end -%><%- if deprecated -%><br>Deprecated: <%= deprecated %><%- end -%><%- if obsoleted -%><br>Obsoleted: <%= :obsoleted %><%- end -%>|<%- if default -%>`<%= default %>`<%- end -%>|

data/test/command/test_cap_ctl.rb

@@ -0,0 +1,100 @@
+require_relative '../helper'
+
+require 'tempfile'
+require 'fluent/command/cap_ctl'
+
+class TestFluentCapCtl < Test::Unit::TestCase
+  setup do
+    omit "This environment does not handle Linux capability" unless defined?(CapNG)
+  end
+
+  sub_test_case "success" do
+    test "clear capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--clear"]).call
+      end
+      expression = /\AClear capabilities .*\n/m
+      assert_match expression, logs
+    end
+
+    test "add capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--add", "dac_override"]).call
+      end
+      expression = /\AUpdating .* done.\nAdding .*\n/m
+      assert_match expression, logs
+    end
+
+    test "drop capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--drop", "chown"]).call
+      end
+      expression = /\AUpdating .* done.\nDropping .*\n/m
+      assert_match expression, logs
+    end
+
+    test "get capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--get"]).call
+      end
+      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
+      assert_match expression, logs
+    end
+  end
+
+  sub_test_case "success with file" do
+    test "clear capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--clear-cap", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AClear capabilities .*\n/m
+      assert_match expression, logs
+    end
+
+    test "add capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--add", "dac_override", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AUpdating .* done.\nAdding .*\n/m
+      assert_match expression, logs
+    end
+
+    test "drop capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--drop", "chown", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AUpdating .* done.\nDropping .*\n/m
+      assert_match expression, logs
+    end
+
+    test "get capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--get", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
+      assert_match expression, logs
+    end
+  end
+
+  sub_test_case "invalid" do
+    test "add capability" do
+      assert_raise(ArgumentError) do
+        Fluent::CapCtl.new(["--add", "nonexitent"]).call
+      end
+    end
+
+    test "drop capability" do
+      assert_raise(ArgumentError) do
+        Fluent::CapCtl.new(["--drop", "invalid"]).call
+      end
+    end
+  end
+end

data/test/command/test_ctl.rb

@@ -0,0 +1,57 @@
+require_relative '../helper'
+
+require 'test-unit'
+require 'win32/event' if Fluent.windows?
+
+require 'fluent/command/ctl'
+
+class TestFluentdCtl < ::Test::Unit::TestCase
+  def assert_win32_event(event_name, command, pid_or_svcname)
+    command, event_suffix = data
+    event = Win32::Event.new(event_name)
+    ipc = Win32::Ipc.new(event.handle)
+    ret = Win32::Ipc::TIMEOUT
+
+    wait_thread = Thread.new do
+      ret = ipc.wait(1)
+    end
+    Fluent::Ctl.new([command, pid_or_svcname]).call
+    wait_thread.join
+    assert_equal(Win32::Ipc::SIGNALED, ret)
+  end
+
+  data("shutdown" => ["shutdown", "TERM", ""],
+       "restart" => ["restart", "HUP", "HUP"],
+       "flush" => ["flush", "USR1", "USR1"],
+       "reload" => ["reload", "USR2", "USR2"])
+  def test_commands(data)
+    command, signal, event_suffix = data
+
+    if Fluent.windows?
+      event_name = "fluentd_54321"
+      event_name << "_#{event_suffix}" unless event_suffix.empty?
+      assert_win32_event(event_name, command, "54321")
+    else
+      got_signal = false
+      Signal.trap(signal) do
+        got_signal = true
+      end
+      Fluent::Ctl.new([command, "#{$$}"]).call
+      assert_true(got_signal)
+    end
+  end
+
+  data("shutdown" => ["shutdown", ""],
+       "restart" => ["restart", "HUP"],
+       "flush" => ["flush", "USR1"],
+       "reload" => ["reload", "USR2"])
+  def test_commands_with_winsvcname(data)
+    omit "Only for Windows" unless Fluent.windows?
+
+    command, event_suffix = data
+    event_name = "testfluentdwinsvc"
+    event_name << "_#{event_suffix}" unless event_suffix.empty?
+
+    assert_win32_event(event_name, command, "testfluentdwinsvc")
+  end
+end

data/test/command/test_fluentd.rb

@@ -878,6 +878,8 @@ CONF
     end
 
     test "without RUBYOPT" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -889,6 +891,8 @@ CONF
 CONF
       conf_path = create_conf_file('rubyopt_test.conf', conf)
       assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'invalid values are set to RUBYOPT' do
@@ -912,6 +916,8 @@ CONF
 
     # https://github.com/fluent/fluentd/issues/2915
     test "ruby path contains spaces" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -940,6 +946,8 @@ CONF
         'spawn command to main:',
         '-Eascii-8bit:ascii-8bit'
       )
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
@@ -1065,4 +1073,34 @@ CONF
                          "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"])
     end
   end
+
+  sub_test_case 'sahred socket options' do
+    test 'enable shared socket by default' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path),
+                         patterns_not_match: ["shared socket for multiple workers is disabled"])
+    end
+
+    test 'disable shared socket by command line option' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+
+    test 'disable shared socket by system config' do
+      conf = <<CONF
+<system>
+  disable_shared_socket
+</system>
+CONF
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+  end
 end

data/test/command/test_plugin_config_formatter.rb

@@ -56,6 +56,32 @@ class TestFluentPluginConfigFormatter < Test::Unit::TestCase
     end
   end
 
+  class FakeStorage < ::Fluent::Plugin::Storage
+    ::Fluent::Plugin.register_storage('fake', self)
+
+    def get(key)
+    end
+
+    def fetch(key, defval)
+    end
+
+    def put(key, value)
+    end
+
+    def delete(key)
+    end
+
+    def update(key, &block)
+    end
+  end
+
+  class FakeServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery
+    ::Fluent::Plugin.register_sd('fake', self)
+
+    desc "hostname"
+    config_param :hostname, :string
+  end
+
   class SimpleInput < ::Fluent::Plugin::Input
     ::Fluent::Plugin.register_input("simple", self)
     helpers :inject, :compat_parameters
@@ -88,6 +114,13 @@ class TestFluentPluginConfigFormatter < Test::Unit::TestCase
     end
   end
 
+  class SimpleServiceDiscovery < ::Fluent::Plugin::ServiceDiscovery
+    ::Fluent::Plugin.register_sd('simple', self)
+
+    desc "servers"
+    config_param :servers, :array
+  end
+
   sub_test_case "json" do
     data(input: [FakeInput, "input"],
          output: [FakeOutput, "output"],
@@ -196,6 +229,52 @@ TEXT
       assert_equal(expected, dumped_config)
     end
 
+    test "input simple (table)" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "--table", "input", "simple"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Input Plugin Overview](https://docs.fluentd.org/v/1.0/input#overview)
+
+## TestFluentPluginConfigFormatter::SimpleInput
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|path|string (required)|path to something||
+
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    data("abbrev" => "sd",
+         "normal" => "service_discovery")
+    test "service_discovery simple" do |data|
+      plugin_type = data
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", plugin_type, "simple"]).call
+      end
+      expected = <<TEXT
+* See also: [ServiceDiscovery Plugin Overview](https://docs.fluentd.org/v/1.0/servicediscovery#overview)
+
+## TestFluentPluginConfigFormatter::SimpleServiceDiscovery
+
+### servers (array) (required)
+
+servers
+
+
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+
     test "output complex" do
       dumped_config = capture_stdout do
         FluentPluginConfigFormatter.new(["--format=markdown", "output", "complex"]).call
@@ -243,6 +322,49 @@ Default value: `normal`.
 
 
 
+TEXT
+      assert_equal(expected, dumped_config)
+    end
+
+    test "output complex (table)" do
+      dumped_config = capture_stdout do
+        FluentPluginConfigFormatter.new(["--format=markdown", "--table", "output", "complex"]).call
+      end
+      expected = <<TEXT
+## Plugin helpers
+
+* [inject](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v/1.0/plugin-helper-overview/api-plugin-helper-compat_parameters)
+
+* See also: [Output Plugin Overview](https://docs.fluentd.org/v/1.0/output#overview)
+
+## TestFluentPluginConfigFormatter::ComplexOutput
+
+
+### \\<authentication\\> section (required) (single)
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|username|string (required)|username||
+|password|string (required)|password||
+
+
+### \\<parent\\> section (optional) (multiple)
+
+
+#### \\<child\\> section (optional) (multiple)
+
+### Configuration
+
+|parameter|type|description|default|
+|---|---|---|---|
+|names|array (required)|names||
+|difficulty|enum (optional)|difficulty (`easy`, `normal`, `hard`)|`normal`|
+
+
+
 TEXT
       assert_equal(expected, dumped_config)
     end
@@ -251,7 +373,7 @@ TEXT
   sub_test_case "arguments" do
     data do
       hash = {}
-      ["input", "output", "filter", "parser", "formatter"].each do |type|
+      ["input", "output", "filter", "parser", "formatter", "storage", "service_discovery"].each do |type|
         ["txt", "json", "markdown"].each do |format|
           argv = ["--format=#{format}"]
           [
@@ -259,7 +381,7 @@ TEXT
             ["--verbose"],
             ["--compact"]
           ].each do |options|
-            hash[(argv + options).join(" ")] = argv + options + [type, "fake"]
+            hash["[#{type}] " + (argv + options).join(" ")] = argv + options + [type, "fake"]
           end
         end
       end