fluentd 1.14.6-x86-mingw32 → 1.15.2-x86-mingw32

data/lib/fluent/system_config.rb

@@ -22,16 +22,17 @@ module Fluent
     include Configurable
 
     SYSTEM_CONFIG_PARAMETERS = [
-      :workers, :root_dir, :log_level,
+      :workers, :restart_worker_interval, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
-      :metrics, :enable_input_metrics, :enable_size_metrics
+      :metrics, :enable_input_metrics, :enable_size_metrics, :enable_jit
     ]
 
     config_param :workers,   :integer, default: 1
+    config_param :restart_worker_interval, :time, default: 0
     config_param :root_dir,  :string, default: nil
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
@@ -49,6 +50,7 @@ module Fluent
     config_param :disable_shared_socket, :bool, default: nil
     config_param :enable_input_metrics, :bool, default: nil
     config_param :enable_size_metrics, :bool, default: nil
+    config_param :enable_jit, :bool, default: false
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.14.6'
+  VERSION = '1.15.2'
 
 end

data/lib/fluent/win32api.rb

@@ -0,0 +1,38 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/env'
+
+module Fluent
+  module Win32API
+    require 'fiddle/import'
+    require 'fiddle/types'
+    extend Fiddle::Importer
+
+    if RUBY_PLATFORM.split('-')[-1] == "ucrt"
+      MSVCRT_DLL = 'ucrtbase.dll'
+    else
+      MSVCRT_DLL = 'msvcrt.dll'
+    end
+
+    dlload MSVCRT_DLL, "kernel32.dll"
+    include Fiddle::Win32Types
+
+    extern "intptr_t _get_osfhandle(int)"
+    extern "BOOL GetFileInformationByHandle(HANDLE, void *)"
+    extern "BOOL GetFileInformationByHandleEx(HANDLE, int, void *, DWORD)"
+  end if Fluent.windows?
+end

data/lib/fluent/winsvc.rb

@@ -17,14 +17,12 @@
 begin
 
   require 'optparse'
-  require 'windows/debug'
-  require 'Windows/Library'
   require 'win32/daemon'
   require 'win32/event'
+  require 'win32/registry'
+  require 'serverengine'
 
   include Win32
-  include Windows::Library
-  include Windows::Debug
 
   op = OptionParser.new
   opts = {service_name: nil}
@@ -37,16 +35,13 @@ begin
   end 
 
   def read_fluentdopt(service_name)
-    require 'win32/Registry'
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
       reg.read("fluentdopt")[1] rescue ""
     end
   end
 
   def service_main_start(service_name)
-    ruby_path = 0.chr * 260
-    GetModuleFileName.call(0, ruby_path,260)
-    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    ruby_path = ServerEngine.ruby_bin_path
     rubybin_dir = ruby_path[0, ruby_path.rindex("/")]
     opt = read_fluentdopt(service_name)
     Process.spawn("\"#{rubybin_dir}/ruby.exe\" \"#{rubybin_dir}/fluentd\" #{opt} -x #{service_name}")
@@ -84,6 +79,8 @@ begin
         set_event("#{@service_name}_HUP")
       when 129
         set_event("#{@service_name}_USR1")
+      when 130
+        set_event("#{@service_name}_CONT")
       end
     end
 

data/test/command/test_ctl.rb

@@ -7,7 +7,6 @@ require 'fluent/command/ctl'
 
 class TestFluentdCtl < ::Test::Unit::TestCase
   def assert_win32_event(event_name, command, pid_or_svcname)
-    command, event_suffix = data
     event = Win32::Event.new(event_name)
     ipc = Win32::Ipc.new(event.handle)
     ret = Win32::Ipc::TIMEOUT

data/test/command/test_fluentd.rb

@@ -589,6 +589,39 @@ CONF
       )
     end
 
+    sub_test_case "YAML config format" do
+      test 'success to start the number of workers specified in configuration' do
+        conf = <<'CONF'
+        system:
+          workers: 2
+          root_dir: "#{@root_path}"
+        config:
+          - source:
+              $type: dummy
+              $id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
+              $label: '@dummydata'
+              tag: dummy
+              dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
+
+          - label:
+              $name: '@dummydata'
+              config:
+               - match:
+                  $tag: dummy
+                  $type: "null"
+                  $id: blackhole
+CONF
+        conf_path = create_conf_file('workers1.yaml', conf)
+        assert Dir.exist?(@root_path)
+
+        assert_log_matches(
+          create_cmdline(conf_path),
+          "#0 fluentd worker is now running worker=0",
+          "#1 fluentd worker is now running worker=1"
+        )
+      end
+    end
+
     test 'success to start the number of workers specified by command line option' do
       conf = <<CONF
 <system>

data/test/config/test_system_config.rb

@@ -19,6 +19,7 @@ module Fluent::Config
       @system_config = nil
       @cl_opt = {
         wokers: nil,
+        restart_worker_interval: nil,
         root_dir: nil,
         log: FakeLoggerInitializer.new,
         log_level: Fluent::Log::LEVEL_INFO,
@@ -72,6 +73,7 @@ module Fluent::Config
       sc = Fluent::SystemConfig.new(conf)
       sc.overwrite_variables(**s.for_system_config)
       assert_equal(1, sc.workers)
+      assert_equal(0, sc.restart_worker_interval)
       assert_nil(sc.root_dir)
       assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
       assert_nil(sc.suppress_repeated_stacktrace)
@@ -82,12 +84,14 @@ module Fluent::Config
       assert_nil(sc.enable_input_metrics)
       assert_nil(sc.enable_size_metrics)
       assert_nil(sc.enable_msgpack_time_support)
+      assert(!sc.enable_jit)
       assert_equal(:text, sc.log.format)
       assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
     end
 
     data(
       'workers' => ['workers', 3],
+      'restart_worker_interval' => ['restart_worker_interval', 60],
       'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
       'log_level' => ['log_level', 'error'],
       'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
@@ -99,6 +103,7 @@ module Fluent::Config
       'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
       'enable_input_metrics' => ['enable_input_metrics', true],
       'enable_size_metrics' => ['enable_size_metrics', true],
+      'enable_jit' => ['enable_jit', true],
     )
     test "accepts parameters" do |(k, v)|
       conf = parse_text(<<-EOS)
@@ -174,7 +179,6 @@ module Fluent::Config
             </log>
           </system>
         EOS
-        s = FakeSupervisor.new
         sc = Fluent::SystemConfig.new(conf)
         assert_equal(3, sc.log.rotate_age)
       end

data/test/config/test_types.rb

@@ -190,7 +190,7 @@ class TestConfigTypes < ::Test::Unit::TestCase
     data("val" => [:val, 'val'],
          "v" => [:v, 'v'],
          "value" => [:value, 'value'])
-    test 'enum' do |(expected, val, list)|
+    test 'enum' do |(expected, val)|
       assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
     end
 

data/test/plugin/in_tail/test_io_handler.rb

@@ -21,12 +21,20 @@ class IntailIOHandlerTest < Test::Unit::TestCase
     @file.unlink rescue nil
   end
 
+  def create_target_info
+    Fluent::Plugin::TailInput::TargetInfo.new(@file.path, Fluent::FileWrapper.stat(@file.path).ino)
+  end
+
+  def create_watcher
+    Fluent::Plugin::TailInput::TailWatcher.new(create_target_info, nil, nil, nil, nil, nil, nil, nil, nil)
+  end
+
   test '#on_notify load file content and passed it to receive_lines method' do
     text = "this line is test\ntest line is test\n"
     @file.write(text)
     @file.close
 
-    watcher = 'watcher'
+    watcher = create_watcher
 
     update_pos = 0
 
@@ -61,7 +69,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
 
       update_pos = 0
 
-      watcher = 'watcher'
+      watcher = create_watcher
       stub(watcher).pe do
         pe = 'position_file'
         stub(pe).read_pos { 0 }
@@ -92,7 +100,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
 
       update_pos = 0
 
-      watcher = 'watcher'
+      watcher = create_watcher
       stub(watcher).pe do
         pe = 'position_file'
         stub(pe).read_pos { 0 }
@@ -107,6 +115,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       r.on_notify
+      assert_equal text.bytesize, update_pos
       assert_equal 8, returned_lines[0].size
     end
 
@@ -118,7 +127,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
 
       update_pos = 0
 
-      watcher = 'watcher'
+      watcher = create_watcher
       stub(watcher).pe do
         pe = 'position_file'
         stub(pe).read_pos { 0 }
@@ -133,6 +142,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       r.on_notify
+      assert_equal text.bytesize, update_pos
       assert_equal 5, returned_lines[0].size
       assert_equal 3, returned_lines[1].size
     end

data/test/plugin/in_tail/test_position_file.rb

@@ -40,10 +40,6 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    paths = {
-      "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
-      "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 2),
-    }
     Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
     @file.seek(0)
@@ -146,11 +142,6 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      invalid_path = "invalidpath100000000000000000000000000000000"
-      paths = TEST_CONTENT_PATHS.merge({
-        invalid_path => Fluent::Plugin::TailInput::TargetInfo.new(invalid_path, 0),
-        "unwatched" => Fluent::Plugin::TailInput::TargetInfo.new("unwatched", 0),
-      })
       Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       @file.seek(0)
@@ -322,58 +313,4 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 2, f.read_inode
     end
   end
-
-  sub_test_case "TargetInfo equality rules" do
-    sub_test_case "== operator" do
-      def test_equal
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1235)
-
-        assert_equal t1, t2
-      end
-
-      def test_not_equal
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
-
-        assert_not_equal t1, t2
-      end
-    end
-
-    sub_test_case "eql? method" do
-      def test_eql?
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 5321)
-
-        assert do
-          t1.eql? t2
-        end
-      end
-
-      def test_not_eql?
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test3", 1234)
-
-        assert do
-          !t1.eql? t2
-        end
-      end
-    end
-
-    sub_test_case "hash" do
-      def test_equal
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 7321)
-
-        assert_equal t1.hash, t2.hash
-      end
-
-      def test_not_equal
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
-
-        assert_not_equal t1.hash, t2.hash
-      end
-    end
-  end
 end

data/test/plugin/out_forward/test_socket_cache.rb

@@ -110,6 +110,10 @@ class SocketCacheTest < Test::Unit::TestCase
   end
 
   sub_test_case 'purge_obsolete_socks' do
+    def teardown
+      Timecop.return
+    end
+
     test 'delete key in inactive_socks' do
       c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
       sock = mock!.close { 'closed' }.subject
@@ -134,7 +138,7 @@ class SocketCacheTest < Test::Unit::TestCase
       c.checkin(sock)
 
       # wait timeout
-      Timecop.freeze(Time.parse('2016-04-13 14:20:00 +0900'))
+      Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))
       c.checkout_or('key') { sock2 }
 
       assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
@@ -145,5 +149,26 @@ class SocketCacheTest < Test::Unit::TestCase
       assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
       assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock)
     end
+
+    test 'should not purge just after checkin and purge after timeout' do
+      Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))
+
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      sock = dont_allow(mock!).close
+      stub(sock).inspect
+      c.checkout_or('key') { sock }
+
+      Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))
+      c.checkin(sock)
+
+      assert_equal(1, c.instance_variable_get(:@available_sockets).size)
+      c.purge_obsolete_socks
+      assert_equal(1, c.instance_variable_get(:@available_sockets).size)
+
+      Timecop.freeze(Time.parse('2016-04-13 14:00:22 +0900'))
+      assert_equal(1, c.instance_variable_get(:@available_sockets).size)
+      c.purge_obsolete_socks
+      assert_equal(0, c.instance_variable_get(:@available_sockets).size)
+    end
   end
 end

data/test/plugin/test_base.rb

@@ -1,4 +1,5 @@
 require_relative '../helper'
+require 'tmpdir'
 require 'fluent/plugin/base'
 
 module FluentPluginBaseTest
@@ -112,4 +113,37 @@ class BaseTest < Test::Unit::TestCase
     assert_equal 1, logger.logs.size
     assert{ logger.logs.first.include?("invalid byte sequence is replaced in ") }
   end
+
+  test 'generates worker lock path safely' do
+    Dir.mktmpdir("test-fluentd-lock-") do |lock_dir|
+      ENV['FLUENTD_LOCK_DIR'] = lock_dir
+      p = FluentPluginBaseTest::DummyPlugin.new
+      path = p.get_lock_path("Aa\\|=~/_123");
+
+      assert_equal lock_dir, File.dirname(path)
+      assert_equal "fluentd-Aa______123.lock", File.basename(path)
+    end
+  end
+
+  test 'can acquire inter-worker locking' do
+    Dir.mktmpdir("test-fluentd-lock-") do |lock_dir|
+      ENV['FLUENTD_LOCK_DIR'] = lock_dir
+      p = FluentPluginBaseTest::DummyPlugin.new
+      lock_path = p.get_lock_path("test_base")
+
+      p.acquire_worker_lock("test_base") do
+        # With LOCK_NB set, flock() returns `false` when the
+        # file is already locked.
+        File.open(lock_path, "w") do |f|
+          assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
+        end
+      end
+
+      # Lock should be release by now. In that case, flock
+      # must return 0.
+      File.open(lock_path, "w") do |f|
+        assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
+      end
+    end
+  end
 end

data/test/plugin/test_file_wrapper.rb

@@ -2,11 +2,6 @@ require_relative '../helper'
 require 'fluent/plugin/file_wrapper'
 
 class FileWrapperTest < Test::Unit::TestCase
-  require 'windows/file'
-  require 'windows/error'
-  include Windows::File
-  include Windows::Error
-
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"
 
   def setup
@@ -17,58 +12,6 @@ class FileWrapperTest < Test::Unit::TestCase
     FileUtils.rm_rf(TMP_DIR)
   end
 
-  sub_test_case 'Win32Error' do
-    test 'equal' do
-      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"),
-                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "message"))
-    end
-
-    test 'different error code' do
-      assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND),
-                       Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
-    end
-
-    test 'different error message' do
-      assert_not_equal(Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message1"),
-                       Fluent::Win32Error.new(ERROR_FILE_NOT_FOUND, "message2"))
-    end
-
-    test 'different class' do
-      assert_not_equal(Errno::EPIPE,
-                       Fluent::Win32Error.new(ERROR_SHARING_VIOLATION))
-    end
-
-    test 'ERROR_SHARING_VIOLATION message' do
-      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION).message,
-                   "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process.")
-    end
-
-    test 'ERROR_SHARING_VIOLATION with a message' do
-      assert_equal(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "cannot open the file").message,
-                   "Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process." +
-                   " - cannot open the file")
-    end
-
-    test 'to_s' do
-      assert_equal("Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt",
-                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").to_s)
-    end
-
-    test 'inspect' do
-      assert_equal("#<Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt>",
-                   Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").inspect)
-    end
-
-    data('0' => [false, 0],
-         '9999' => [false, 9999],
-         '10000' => [true, 10000],
-         '10001' => [true, 10001])
-    test 'wsaerr?' do |data|
-      expected, code = data
-      assert_equal(expected, Fluent::Win32Error.new(code).wsaerr?)
-    end
-  end
-
   sub_test_case 'WindowsFile exceptions' do
     test 'nothing raised' do
       begin
@@ -106,21 +49,5 @@ class FileWrapperTest < Test::Unit::TestCase
         file.close if file
       end
     end
-
-    test 'ERROR_SHARING_VIOLATION raised' do
-      begin
-        path = "#{TMP_DIR}/test_windows_file.txt"
-        file1 = file2 = nil
-        file1 = File.open(path, "wb")
-        win32err = Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, path)
-        assert_raise(Errno::EACCES.new(win32err.message)) do
-          file2 = Fluent::WindowsFile.new(path, 'r', FILE_SHARE_READ)
-        ensure
-          file2.close if file2
-        end
-      ensure
-        file1.close if file1
-      end
-    end
   end
 end if Fluent.windows?

data/test/plugin/test_in_object_space.rb

@@ -17,13 +17,19 @@ class ObjectSpaceInputTest < Test::Unit::TestCase
   end
 
   class FailObject
-    def self.class
-      raise "error"
-    end
   end
 
   def setup
     Fluent::Test.setup
+    # Overriding this behavior in the global scope will have an unexpected influence on other tests.
+    # So this should be overridden here and be removed in `teardown`.
+    def FailObject.class
+      raise "FailObject error for tests in ObjectSpaceInputTest."
+    end
+  end
+
+  def teardown
+    FailObject.singleton_class.remove_method(:class)
   end
 
   TESTCONFIG = %[

data/test/plugin/test_in_syslog.rb

@@ -497,7 +497,7 @@ EOS
 
   def test_send_keepalive_packet_can_not_be_enabled_for_udp
     assert_raise(Fluent::ConfigError) do
-      d = create_driver(ipv4_config + %[
+      create_driver(ipv4_config + %[
         send_keepalive_packet true
       ])
     end