fluentd 1.11.4-x64-mingw32 → 1.12.3-x64-mingw32

data/lib/fluent/time.rb

@@ -50,6 +50,7 @@ module Fluent
     def to_int
       @sec
     end
+    alias :to_i :to_int
 
     def to_f
       @sec + @nsec / 1_000_000_000.0
@@ -131,13 +132,14 @@ module Fluent
   end
 
   module TimeMixin
-    TIME_TYPES = ['string', 'unixtime', 'float']
+    TIME_TYPES = ['string', 'unixtime', 'float', 'mixed']
 
     TIME_PARAMETERS = [
       [:time_format, :string, {default: nil}],
       [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil
       [:utc,       :bool, {default: false}], # to turn :localtime false
       [:timezone, :string, {default: nil}],
+      [:time_format_fallbacks, :array, {default: []}], # try time_format, then try fallbacks
     ]
     TIME_FULL_PARAMETERS = [
       # To avoid to define :time_type twice (in plugin_helper/inject)
@@ -169,6 +171,12 @@ module Fluent
           raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
         end
 
+        if conf.has_key?('time_type') and @time_type == :mixed
+          if @time_format.nil? and @time_format_fallbacks.empty?
+            raise Fluent::ConfigError, "time_type is :mixed but time_format and time_format_fallbacks is empty."
+          end
+        end
+
         Fluent::Timezone.validate!(@timezone) if @timezone
       end
     end
@@ -179,6 +187,7 @@ module Fluent
       end
 
       def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return MixedTimeParser.new(type, format, @localtime, timezone, @utc, force_localtime, @time_format_fallbacks) if type == :mixed
         return NumericTimeParser.new(type) if type != :string
         return TimeParser.new(format, true, nil) if force_localtime
 
@@ -451,4 +460,52 @@ module Fluent
       end
     end
   end
+
+  # MixedTimeParser is available when time_type is set to :mixed
+  #
+  # Use Case 1: primary format is specified explicitly in time_format
+  #  time_type mixed
+  #  time_format %iso8601
+  #  time_format_fallbacks unixtime
+  # Use Case 2: time_format is omitted
+  #  time_type mixed
+  #  time_format_fallbacks %iso8601, unixtime
+  #
+  class MixedTimeParser < TimeParser # to include TimeParseError
+    def initialize(type, format = nil, localtime = nil, timezone = nil, utc = nil, force_localtime = nil, fallbacks = [])
+      @parsers = []
+      fallbacks.unshift(format).each do |fallback|
+        next unless fallback
+        case fallback
+        when 'unixtime', 'float'
+          @parsers << NumericTimeParser.new(fallback, localtime, timezone)
+        else
+          if force_localtime
+            @parsers << TimeParser.new(fallback, true, nil)
+          else
+            localtime = localtime && (timezone.nil? && !utc)
+            @parsers << TimeParser.new(fallback, localtime, timezone)
+          end
+        end
+      end
+    end
+
+    def parse(value)
+      @parsers.each do |parser|
+        begin
+          Float(value) if parser.class == Fluent::NumericTimeParser
+        rescue
+          next
+        end
+        begin
+          return parser.parse(value)
+        rescue
+          # skip TimeParseError
+        end
+      end
+      fallback_class = @parsers.collect do |parser| parser.class end.join(",")
+      raise TimeParseError, "invalid time format: value = #{value}, even though fallbacks: #{fallback_class}"
+    end
+  end
+
 end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.11.4'
+  VERSION = '1.12.3'
 
 end

data/lib/fluent/winsvc.rb

@@ -61,7 +61,6 @@ begin
     end
     
     def service_main
-    
       @pid = service_main_start(@service_name)
       while running?
         sleep 10
@@ -69,13 +68,32 @@ begin
     end
 
     def service_stop
-      ev = Win32::Event.open(@service_name)
-      ev.set
-      ev.close
+      set_event(@service_name)
       if @pid > 0
         Process.waitpid(@pid)
       end
     end
+
+    def service_paramchange
+      set_event("#{@service_name}_USR2")
+    end
+
+    def service_user_defined_control(code)
+      case code
+      when 128
+        set_event("#{@service_name}_HUP")
+      when 129
+        set_event("#{@service_name}_USR1")
+      end
+    end
+
+    private
+
+    def set_event(event_name)
+      ev = Win32::Event.open(event_name)
+      ev.set
+      ev.close
+    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

data/templates/new_gem/fluent-plugin.gemspec.erb

@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.14"
-  spec.add_development_dependency "rake", "~> 12.0"
-  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_development_dependency "bundler", "~> <%= bundler_version %>"
+  spec.add_development_dependency "rake", "~> <%= rake_version %>"
+  spec.add_development_dependency "test-unit", "~> <%= test_unit_version %>"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
 end

data/templates/plugin_config_formatter/param.md-table.erb

@@ -0,0 +1,10 @@
+<%-
+type = config[:type]
+required_label = config[:required] ? "required" : "optional"
+default = config[:default]
+alias_name = config[:alias]
+deprecated = config[:deprecated]
+obsoleted = config[:obsoleted]
+description = config[:desc]
+-%>
+|<%= name %>|<%= type %> (<%= required_label %>)|<%= description %><%- if type == :enum -%> (<%= config[:list].map{|x| "`#{x}`"}.join(", ") %>)<%- end -%><%- if alias_name -%><br>Alias: <%= alias_name %><%- end -%><%- if deprecated -%><br>Deprecated: <%= deprecated %><%- end -%><%- if obsoleted -%><br>Obsoleted: <%= :obsoleted %><%- end -%>|<%- if default -%>`<%= default %>`<%- end -%>|

data/test/command/test_binlog_reader.rb

@@ -82,6 +82,14 @@ class TestBaseCommand < ::Test::Unit::TestCase
 end
 
 class TestHead < TestBaseCommand
+  setup do
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
+  end
+
   sub_test_case 'initialize' do
     data(
       'file is not passed' => %w(),
@@ -138,7 +146,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 5, out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 5, out
       end
     end
 
@@ -149,7 +157,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -169,7 +177,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}\n", out
+        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -198,6 +206,14 @@ class TestHead < TestBaseCommand
 end
 
 class TestCat < TestBaseCommand
+  setup do
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
+  end
+
   sub_test_case 'initialize' do
     data(
       'file is not passed' => [],
@@ -254,7 +270,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 6, out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 6, out
       end
     end
 
@@ -265,7 +281,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -276,7 +292,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}\n", out
+        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 

data/test/command/test_cap_ctl.rb

@@ -0,0 +1,100 @@
+require_relative '../helper'
+
+require 'tempfile'
+require 'fluent/command/cap_ctl'
+
+class TestFluentCapCtl < Test::Unit::TestCase
+  setup do
+    omit "This environment does not handle Linux capability" unless defined?(CapNG)
+  end
+
+  sub_test_case "success" do
+    test "clear capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--clear"]).call
+      end
+      expression = /\AClear capabilities .*\n/m
+      assert_match expression, logs
+    end
+
+    test "add capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--add", "dac_override"]).call
+      end
+      expression = /\AUpdating .* done.\nAdding .*\n/m
+      assert_match expression, logs
+    end
+
+    test "drop capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--drop", "chown"]).call
+      end
+      expression = /\AUpdating .* done.\nDropping .*\n/m
+      assert_match expression, logs
+    end
+
+    test "get capability" do
+      logs = capture_stdout do
+        Fluent::CapCtl.new(["--get"]).call
+      end
+      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
+      assert_match expression, logs
+    end
+  end
+
+  sub_test_case "success with file" do
+    test "clear capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--clear-cap", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AClear capabilities .*\n/m
+      assert_match expression, logs
+    end
+
+    test "add capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--add", "dac_override", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AUpdating .* done.\nAdding .*\n/m
+      assert_match expression, logs
+    end
+
+    test "drop capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--drop", "chown", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\AUpdating .* done.\nDropping .*\n/m
+      assert_match expression, logs
+    end
+
+    test "get capability" do
+      logs = capture_stdout do
+        Tempfile.create("fluent-cap-") do |tempfile|
+          Fluent::CapCtl.new(["--get", "-f", tempfile.path]).call
+        end
+      end
+      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
+      assert_match expression, logs
+    end
+  end
+
+  sub_test_case "invalid" do
+    test "add capability" do
+      assert_raise(ArgumentError) do
+        Fluent::CapCtl.new(["--add", "nonexitent"]).call
+      end
+    end
+
+    test "drop capability" do
+      assert_raise(ArgumentError) do
+        Fluent::CapCtl.new(["--drop", "invalid"]).call
+      end
+    end
+  end
+end

data/test/command/test_ctl.rb

@@ -0,0 +1,57 @@
+require_relative '../helper'
+
+require 'test-unit'
+require 'win32/event' if Fluent.windows?
+
+require 'fluent/command/ctl'
+
+class TestFluentdCtl < ::Test::Unit::TestCase
+  def assert_win32_event(event_name, command, pid_or_svcname)
+    command, event_suffix = data
+    event = Win32::Event.new(event_name)
+    ipc = Win32::Ipc.new(event.handle)
+    ret = Win32::Ipc::TIMEOUT
+
+    wait_thread = Thread.new do
+      ret = ipc.wait(1)
+    end
+    Fluent::Ctl.new([command, pid_or_svcname]).call
+    wait_thread.join
+    assert_equal(Win32::Ipc::SIGNALED, ret)
+  end
+
+  data("shutdown" => ["shutdown", "TERM", ""],
+       "restart" => ["restart", "HUP", "HUP"],
+       "flush" => ["flush", "USR1", "USR1"],
+       "reload" => ["reload", "USR2", "USR2"])
+  def test_commands(data)
+    command, signal, event_suffix = data
+
+    if Fluent.windows?
+      event_name = "fluentd_54321"
+      event_name << "_#{event_suffix}" unless event_suffix.empty?
+      assert_win32_event(event_name, command, "54321")
+    else
+      got_signal = false
+      Signal.trap(signal) do
+        got_signal = true
+      end
+      Fluent::Ctl.new([command, "#{$$}"]).call
+      assert_true(got_signal)
+    end
+  end
+
+  data("shutdown" => ["shutdown", ""],
+       "restart" => ["restart", "HUP"],
+       "flush" => ["flush", "USR1"],
+       "reload" => ["reload", "USR2"])
+  def test_commands_with_winsvcname(data)
+    omit "Only for Windows" unless Fluent.windows?
+
+    command, event_suffix = data
+    event_name = "testfluentdwinsvc"
+    event_name << "_#{event_suffix}" unless event_suffix.empty?
+
+    assert_win32_event(event_name, command, "testfluentdwinsvc")
+  end
+end

data/test/command/test_fluentd.rb

@@ -878,6 +878,8 @@ CONF
     end
 
     test "without RUBYOPT" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -889,6 +891,8 @@ CONF
 CONF
       conf_path = create_conf_file('rubyopt_test.conf', conf)
       assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'invalid values are set to RUBYOPT' do
@@ -912,6 +916,8 @@ CONF
 
     # https://github.com/fluent/fluentd/issues/2915
     test "ruby path contains spaces" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -940,6 +946,8 @@ CONF
         'spawn command to main:',
         '-Eascii-8bit:ascii-8bit'
       )
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
@@ -1065,4 +1073,34 @@ CONF
                          "secret xxxxxx", patterns_not_match: ["secret secret0", "secret secret1"])
     end
   end
+
+  sub_test_case 'sahred socket options' do
+    test 'enable shared socket by default' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path),
+                         patterns_not_match: ["shared socket for multiple workers is disabled"])
+    end
+
+    test 'disable shared socket by command line option' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+
+    test 'disable shared socket by system config' do
+      conf = <<CONF
+<system>
+  disable_shared_socket
+</system>
+CONF
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--disable-shared-socket"),
+                         "shared socket for multiple workers is disabled",)
+    end
+  end
 end