fluentd 1.13.3 → 1.16.5

data/lib/fluent/system_config.rb

@@ -22,15 +22,17 @@ module Fluent
     include Configurable
 
     SYSTEM_CONFIG_PARAMETERS = [
-      :workers, :root_dir, :log_level,
+      :workers, :restart_worker_interval, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
+      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
+      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit
     ]
 
     config_param :workers,   :integer, default: 1
+    config_param :restart_worker_interval, :time, default: 0
     config_param :root_dir,  :string, default: nil
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
@@ -46,6 +48,9 @@ module Fluent
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
     config_param :disable_shared_socket, :bool, default: nil
+    config_param :enable_input_metrics, :bool, default: nil
+    config_param :enable_size_metrics, :bool, default: nil
+    config_param :enable_jit, :bool, default: false
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end
@@ -57,7 +62,7 @@ module Fluent
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
       config_param :rotate_age, default: nil do |v|
         if Fluent::Log::LOG_ROTATE_AGE.include?(v)
-          v.to_sym
+          v
         else
           begin
             Integer(v)
@@ -93,6 +98,11 @@ module Fluent
       config_param :timeout, :time, default: nil
     end
 
+    config_section :metrics, multi: false do
+      config_param :@type, :string, default: "local"
+      config_param :labels, :hash, default: {}
+    end
+
     def self.create(conf, strict_config_value=false)
       systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?

data/lib/fluent/test/driver/base.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/config'
 require 'fluent/config/element'
+require 'fluent/env'
 require 'fluent/log'
 require 'fluent/clock'
 
@@ -102,11 +103,16 @@ module Fluent
 
         def instance_start
           if @instance.respond_to?(:server_wait_until_start)
-            @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
-              FileUtils.rm_f @socket_manager_path
+            if Fluent.windows?
+              @socket_manager_server = ServerEngine::SocketManager::Server.open
+              @socket_manager_path = @socket_manager_server.path
+            else
+              @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+              if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+                FileUtils.rm_f @socket_manager_path
+              end
+              @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
             end
-            @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
             ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
           end
 
@@ -173,7 +179,7 @@ module Fluent
 
           if @socket_manager_server
             @socket_manager_server.close
-            if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
               FileUtils.rm_f @socket_manager_path
             end
           end

data/lib/fluent/test/driver/filter.rb

@@ -37,6 +37,10 @@ module Fluent
           @filtered.map {|_time, record| record }
         end
 
+        def filtered_time
+          @filtered.map {|time, _record| time }
+        end
+
         def instance_hook_after_started
           super
           filter_hook = ->(time, record) { @filtered << [time, record] }

data/lib/fluent/test/startup_shutdown.rb

@@ -21,9 +21,8 @@ module Fluent
   module Test
     module StartupShutdown
       def startup
-        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-        @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
-        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+        @server = ServerEngine::SocketManager::Server.open
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
       end
 
       def shutdown
@@ -31,15 +30,14 @@ module Fluent
       end
 
       def self.setup
-        @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-        @server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
-        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+        @server = ServerEngine::SocketManager::Server.open
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
       end
 
       def self.teardown
         @server.close
-        # on Windows, socket_manager_path is a TCP port number
-        FileUtils.rm_f @socket_manager_path unless Fluent.windows?
+        # on Windows, the path is a TCP port number
+        FileUtils.rm_f @server.path unless Fluent.windows?
       end
     end
   end

data/lib/fluent/time.rb

@@ -226,19 +226,16 @@ module Fluent
 
       format_with_timezone = format && (format.include?("%z") || format.include?("%Z"))
 
-      # unixtime_in_expected_tz = unixtime_in_localtime + offset_diff
-      offset_diff = case
-                    when format_with_timezone then nil
-                    when timezone  then
-                      offset = Fluent::Timezone.utc_offset(timezone)
-                      if offset.respond_to?(:call)
-                        ->(t) { Time.now.localtime.utc_offset - offset.call(t) }
-                      else
-                        Time.now.localtime.utc_offset - offset
-                      end
-                    when localtime then 0
-                    else Time.now.localtime.utc_offset # utc
-                    end
+      utc_offset = case
+                   when format_with_timezone then
+                     nil
+                   when timezone then
+                     Fluent::Timezone.utc_offset(timezone)
+                   when localtime then
+                     nil
+                   else
+                     0 # utc
+                   end
 
       strptime = format && (Strptime.new(format) rescue nil)
 
@@ -247,16 +244,20 @@ module Fluent
                when format_with_timezone             then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }
                when format == '%iso8601'             then ->(v){ Fluent::EventTime.from_time(Time.iso8601(v)) }
                when strptime then
-                 if offset_diff.respond_to?(:call)
-                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+                 if utc_offset.nil?
+                   ->(v){ t = strptime.exec(v); Fluent::EventTime.new(t.to_i, t.nsec) }
+                 elsif utc_offset.respond_to?(:call)
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }
                  else
-                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }
                  end
-               when format   then
-                 if offset_diff.respond_to?(:call)
-                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+               when format then
+                 if utc_offset.nil?
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i, t.nsec) }
+                 elsif utc_offset.respond_to?(:call)
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }
                  else
-                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }
                  end
                else ->(v){ Fluent::EventTime.parse(v) }
                end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.13.3'
+  VERSION = '1.16.5'
 
 end

data/lib/fluent/win32api.rb

@@ -0,0 +1,38 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/env'
+
+module Fluent
+  module Win32API
+    require 'fiddle/import'
+    require 'fiddle/types'
+    extend Fiddle::Importer
+
+    if RUBY_PLATFORM.split('-')[-1] == "ucrt"
+      MSVCRT_DLL = 'ucrtbase.dll'
+    else
+      MSVCRT_DLL = 'msvcrt.dll'
+    end
+
+    dlload MSVCRT_DLL, "kernel32.dll"
+    include Fiddle::Win32Types
+
+    extern "intptr_t _get_osfhandle(int)"
+    extern "BOOL GetFileInformationByHandle(HANDLE, void *)"
+    extern "BOOL GetFileInformationByHandleEx(HANDLE, int, void *, DWORD)"
+  end if Fluent.windows?
+end

data/lib/fluent/winsvc.rb

@@ -17,14 +17,12 @@
 begin
 
   require 'optparse'
-  require 'windows/debug'
-  require 'Windows/Library'
   require 'win32/daemon'
   require 'win32/event'
+  require 'win32/registry'
+  require 'serverengine'
 
   include Win32
-  include Windows::Library
-  include Windows::Debug
 
   op = OptionParser.new
   opts = {service_name: nil}
@@ -37,16 +35,13 @@ begin
   end 
 
   def read_fluentdopt(service_name)
-    require 'win32/Registry'
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
       reg.read("fluentdopt")[1] rescue ""
     end
   end
 
   def service_main_start(service_name)
-    ruby_path = 0.chr * 260
-    GetModuleFileName.call(0, ruby_path,260)
-    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    ruby_path = ServerEngine.ruby_bin_path
     rubybin_dir = ruby_path[0, ruby_path.rindex("/")]
     opt = read_fluentdopt(service_name)
     Process.spawn("\"#{rubybin_dir}/ruby.exe\" \"#{rubybin_dir}/fluentd\" #{opt} -x #{service_name}")
@@ -84,6 +79,8 @@ begin
         set_event("#{@service_name}_HUP")
       when 129
         set_event("#{@service_name}_USR1")
+      when 130
+        set_event("#{@service_name}_CONT")
       end
     end
 

data/templates/new_gem/test/helper.rb.erb

@@ -1,4 +1,3 @@
-$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
 require "test-unit"
 require "fluent/test"
 require "fluent/test/driver/<%= type %>"

data/test/command/test_cat.rb

@@ -69,7 +69,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     def test_cat_json
       d = create_driver
       d.run(expect_records: 1) do
-        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
           stdin.puts('{"key":"value"}')
           stdin.close
         end
@@ -86,7 +86,7 @@ class TestFluentCat < ::Test::Unit::TestCase
       path = d.instance.write(@chunk)
       d = create_driver
       d.run(expect_records: 1) do
-        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
           stdin.write(File.read(path))
           stdin.close
         end
@@ -96,4 +96,33 @@ class TestFluentCat < ::Test::Unit::TestCase
                    [d.events.size, event.first, event.last])
     end
   end
+
+  sub_test_case "send specific event time" do
+    def test_without_event_time
+      event_time = Fluent::EventTime.now
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} tag") do |stdin|
+          stdin.puts('{"key":"value"}')
+          stdin.close
+        end
+      end
+      event = d.events.first
+      assert_in_delta(event_time.to_f, event[1].to_f, 3.0) # expect command to be finished in 3 seconds
+      assert_equal([1, "tag", true, @record],
+                   [d.events.size, event.first, event_time.to_f < event[1].to_f, event.last])
+    end
+
+    def test_with_event_time
+      event_time = "2021-01-02 13:14:15.0+00:00"
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --event-time '#{event_time}' tag") do |stdin|
+          stdin.puts('{"key":"value"}')
+          stdin.close
+        end
+      end
+      assert_equal([["tag", Fluent::EventTime.parse(event_time), @record]], d.events)
+    end
+  end
 end

data/test/command/test_ctl.rb

@@ -7,7 +7,6 @@ require 'fluent/command/ctl'
 
 class TestFluentdCtl < ::Test::Unit::TestCase
   def assert_win32_event(event_name, command, pid_or_svcname)
-    command, event_suffix = data
     event = Win32::Event.new(event_name)
     ipc = Win32::Ipc.new(event.handle)
     ret = Win32::Ipc::TIMEOUT
@@ -36,7 +35,7 @@ class TestFluentdCtl < ::Test::Unit::TestCase
       Signal.trap(signal) do
         got_signal = true
       end
-      Fluent::Ctl.new([command, "#{$$}"]).call
+      Fluent::Ctl.new([command, Process.pid.to_s]).call
       assert_true(got_signal)
     end
   end