fluentd 1.14.6-x64-mingw32 → 1.15.2-x64-mingw32

data/lib/fluent/config/yaml_parser.rb

@@ -0,0 +1,56 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/yaml_parser/loader'
+require 'fluent/config/yaml_parser/parser'
+require 'pathname'
+
+module Fluent
+  module Config
+    module YamlParser
+      def self.parse(path)
+        context = Kernel.binding
+
+        unless context.respond_to?(:use_nil)
+          context.define_singleton_method(:use_nil) do
+            raise Fluent::SetNil
+          end
+        end
+
+        unless context.respond_to?(:use_default)
+          context.define_singleton_method(:use_default) do
+            raise Fluent::SetDefault
+          end
+        end
+
+        unless context.respond_to?(:hostname)
+          context.define_singleton_method(:hostname) do
+            Socket.gethostname
+          end
+        end
+
+        unless context.respond_to?(:worker_id)
+          context.define_singleton_method(:worker_id) do
+            ENV['SERVERENGINE_WORKER_ID'] || ''
+          end
+        end
+
+        s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path))
+        Fluent::Config::YamlParser::Parser.new(s).build.to_element
+      end
+    end
+  end
+end

data/lib/fluent/config.rb

@@ -17,6 +17,7 @@
 require 'fluent/config/error'
 require 'fluent/config/element'
 require 'fluent/configurable'
+require 'fluent/config/yaml_parser'
 
 module Fluent
   module Config
@@ -25,7 +26,18 @@ module Fluent
     # @param additional_config [String] config which is added to last of config body
     # @param use_v1_config [Bool] config is formatted with v1 or not
     # @return [Fluent::Config]
-    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true)
+    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil)
+      if type == :guess
+        config_file_ext = File.extname(config_path)
+        if config_file_ext == '.yaml' || config_file_ext == '.yml'
+          type = :yaml
+        end
+      end
+
+      if type == :yaml || type == :yml
+        return Fluent::Config::YamlParser.parse(config_path)
+      end
+
       config_fname = File.basename(config_path)
       config_basedir = File.dirname(config_path)
       config_data = File.open(config_path, "r:#{encoding}:utf-8") do |f|
@@ -36,6 +48,7 @@ module Fluent
         end
         s
       end
+
       Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config)
     end
 

data/lib/fluent/error.rb

@@ -28,6 +28,9 @@ module Fluent
   class InvalidRootDirectory < UnrecoverableError
   end
 
+  class InvalidLockDirectory < UnrecoverableError
+  end
+
   # For internal use
   class UncatchableError < Exception
   end

data/lib/fluent/plugin/base.rb

@@ -31,6 +31,7 @@ module Fluent
       def initialize
         @log = nil
         super
+        @fluentd_lock_dir = ENV['FLUENTD_LOCK_DIR']
         @_state = State.new(false, false, false, false, false, false, false, false, false)
         @_context_router = nil
         @_fluentd_worker_id = nil
@@ -70,6 +71,24 @@ module Fluent
         true
       end
 
+      def get_lock_path(name)
+        name = name.gsub(/[^a-zA-Z0-9]/, "_")
+        File.join(@fluentd_lock_dir, "fluentd-#{name}.lock")
+      end
+
+      def acquire_worker_lock(name)
+        if @fluentd_lock_dir.nil?
+          raise InvalidLockDirectory, "can't acquire lock because FLUENTD_LOCK_DIR isn't set"
+        end
+        lock_path = get_lock_path(name)
+        File.open(lock_path, "w") do |f|
+          f.flock(File::LOCK_EX)
+          yield
+        end
+        # Update access time to prevent tmpwatch from deleting a lock file.
+        FileUtils.touch(lock_path);
+      end
+
       def string_safe_encoding(str)
         unless str.valid_encoding?
           str = str.scrub('?')

data/lib/fluent/plugin/file_wrapper.rb

@@ -14,10 +14,12 @@
 #    limitations under the License.
 #
 
+require 'fluent/win32api'
+
 module Fluent
   module FileWrapper
-    def self.open(*args)
-      io = WindowsFile.new(*args).io
+    def self.open(path, mode='r')
+      io = WindowsFile.new(path, mode).io
       if block_given?
         v = yield io
         io.close
@@ -35,126 +37,78 @@ module Fluent
     end
   end
 
-  module WindowsFileExtension
-    attr_reader :path
-
-    def stat
-      s = super
-      s.instance_variable_set :@ino, @ino
-      def s.ino; @ino; end
-      s
-    end
-  end
-
-  class Win32Error < StandardError
-    require 'windows/error'
-    include Windows::Error
-
-    attr_reader :errcode, :msg
-
-    WSABASEERR = 10000
-
-    def initialize(errcode, msg = nil)
-      @errcode = errcode
-      @msg = msg
-    end
-
-    def format_english_message(errcode)
-      buf = 0.chr * 260
-      flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ARGUMENT_ARRAY
-      english_lang_id = 1033 # The result of MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US)
-      FormatMessageA.call(flags, 0, errcode, english_lang_id, buf, buf.size, 0)
-      buf.force_encoding(Encoding.default_external).strip
-    end
-
-    def to_s
-      msg = super
-      msg << ": code: #{@errcode}, #{format_english_message(@errcode)}"
-      msg << " - #{@msg}" if @msg
-      msg
-    end
-
-    def inspect
-      "#<#{to_s}>"
-    end
-
-    def ==(other)
-      return false if other.class != Win32Error
-      @errcode == other.errcode && @msg == other.msg
-    end
-
-    def wsaerr?
-      @errcode >= WSABASEERR
-    end
-  end
-
-  # To open and get stat with setting FILE_SHARE_DELETE
   class WindowsFile
-    require 'windows/file'
-    require 'windows/error'
-    require 'windows/handle'
-    require 'windows/nio'
+    include File::Constants
 
-    include Windows::Error
-    include Windows::File
-    include Windows::Handle
-    include Windows::NIO
-
-    def initialize(path, mode='r', sharemode=FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE)
-      @path = path
-      @file_handle = INVALID_HANDLE_VALUE
-      @mode = mode
+    attr_reader :io
 
+    INVALID_HANDLE_VALUE = -1
 
-      access, creationdisposition, seektoend = case mode.delete('b')
-      when "r" ; [FILE_GENERIC_READ                     , OPEN_EXISTING, false]
-      when "r+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS  , false]
-      when "w" ; [FILE_GENERIC_WRITE                    , CREATE_ALWAYS, false]
-      when "w+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, CREATE_ALWAYS, false]
-      when "a" ; [FILE_GENERIC_WRITE                    , OPEN_ALWAYS  , true]
-      when "a+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS  , true]
-      else raise "unknown mode '#{mode}'"
-      end
-
-      @file_handle = CreateFile.call(@path, access, sharemode,
-                     0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
-      if @file_handle == INVALID_HANDLE_VALUE
-        win32err = Win32Error.new(Win32::API.last_error, path)
-        errno = ServerEngine::RbWinSock.rb_w32_map_errno(win32err.errcode)
-        if errno == Errno::EINVAL::Errno || win32err.wsaerr?
-          # maybe failed to map
-          raise win32err
-        else
-          raise SystemCallError.new(win32err.message, errno)
-        end
+    def initialize(path, mode='r')
+      @path = path
+      @io = File.open(path, mode2flags(mode))
+      @file_handle = Win32API._get_osfhandle(@io.to_i)
+      @io.instance_variable_set(:@file_index, self.ino)
+      def @io.ino
+        @file_index
       end
     end
 
     def close
-      CloseHandle.call(@file_handle)
+      @io.close
       @file_handle = INVALID_HANDLE_VALUE
     end
 
-    def io
-      fd = _open_osfhandle(@file_handle, 0)
-      raise Errno::ENOENT if fd == -1
-      io = File.for_fd(fd, @mode)
-      io.instance_variable_set :@ino, self.ino
-      io.instance_variable_set :@path, @path
-      io.extend WindowsFileExtension
-      io
-    end
-
+    # To keep backward compatibility, we continue to use GetFileInformationByHandle()
+    # to get file id.
+    # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
+    # and returned value is different with above one, former one is 64 bit while
+    # later one is 128bit.
     def ino
       by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
 
-      unless GetFileInformationByHandle.call(@file_handle, by_handle_file_information)
+      unless Win32API.GetFileInformationByHandle(@file_handle, by_handle_file_information)
         return 0
       end
 
       by_handle_file_information.unpack("I11Q1")[11] # fileindex
     end
 
+    def stat
+      raise Errno::ENOENT if delete_pending
+      s = File.stat(@path)
+      s.instance_variable_set :@ino, self.ino
+      def s.ino; @ino; end
+      s
+    end
+
+    private
+
+    def mode2flags(mode)
+      # Always inject File::Constants::SHARE_DELETE
+      # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
+      # To enable SHARE_DELETE, BINARY is also required.
+      # https://bugs.ruby-lang.org/issues/11218
+      # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
+      flags = BINARY | SHARE_DELETE
+      case mode.delete("b")
+      when "r"
+        flags |= RDONLY
+      when "r+"
+        flags |= RDWR
+      when "w"
+        flags |= WRONLY | CREAT | TRUNC
+      when "w+"
+        flags |= RDWR | CREAT | TRUNC
+      when "a"
+        flags |= WRONLY | CREAT | APPEND
+      when "a+"
+        flags |= RDWR | CREAT | APPEND
+      else
+        raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
+      end
+    end
+
     # DeletePending is a Windows-specific file state that roughly means
     # "this file is queued for deletion, so close any open handlers"
     #
@@ -167,21 +121,11 @@ module Fluent
       bufsize = 1024
       buf = '\0' * bufsize
 
-      unless GetFileInformationByHandleEx.call(@file_handle, file_standard_info, buf, bufsize)
+      unless Win32API.GetFileInformationByHandleEx(@file_handle, file_standard_info, buf, bufsize)
         return false
       end
 
       return buf.unpack("QQICC")[3] != 0
     end
-
-    private :delete_pending
-
-    def stat
-      raise Errno::ENOENT if delete_pending
-      s = File.stat(@path)
-      s.instance_variable_set :@ino, self.ino
-      def s.ino; @ino; end
-      s
-    end
   end
 end if Fluent.windows?

data/lib/fluent/plugin/in_tail/group_watch.rb

@@ -0,0 +1,204 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    module GroupWatchParams
+      include Fluent::Configurable
+
+      DEFAULT_KEY = /.*/
+      DEFAULT_LIMIT = -1
+      REGEXP_JOIN = "_"
+
+      config_section :group, param_name: :group, required: false, multi: false do
+        desc 'Regex for extracting group\'s metadata'
+        config_param :pattern,
+                     :regexp,
+                     default: /^\/var\/log\/containers\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{64})\.log$/
+
+        desc 'Period of time in which the group_line_limit is applied'
+        config_param :rate_period, :time, default: 5
+
+        config_section :rule, param_name: :rule, required: true, multi: true do
+          desc 'Key-value pairs for grouping'
+          config_param :match, :hash, value_type: :regexp, default: { namespace: [DEFAULT_KEY], podname: [DEFAULT_KEY] }
+          desc 'Maximum number of log lines allowed per group over a period of rate_period'
+          config_param :limit, :integer, default: DEFAULT_LIMIT
+        end
+      end
+    end
+
+    module GroupWatch
+      def self.included(mod)
+        mod.include GroupWatchParams
+      end
+
+      attr_reader :group_watchers, :default_group_key
+
+      def initialize
+        super
+        @group_watchers = {}
+        @group_keys = nil
+        @default_group_key = nil
+      end
+
+      def configure(conf)
+        super
+
+        unless @group.nil?
+          ## Ensuring correct time period syntax
+          @group.rule.each { |rule|
+            raise "Metadata Group Limit >= DEFAULT_LIMIT" unless rule.limit >= GroupWatchParams::DEFAULT_LIMIT
+          }
+
+          @group_keys = Regexp.compile(@group.pattern).named_captures.keys
+          @default_group_key = ([GroupWatchParams::DEFAULT_KEY] * @group_keys.length).join(GroupWatchParams::REGEXP_JOIN)
+
+          ## Ensures that "specific" rules (with larger number of `rule.match` keys)
+          ## have a higher priority against "generic" rules (with less number of `rule.match` keys).
+          ## This will be helpful when a file satisfies more than one rule.
+          @group.rule.sort_by! { |rule| -rule.match.length() }
+          construct_groupwatchers
+          @group_watchers[@default_group_key] ||= GroupWatcher.new(@group.rate_period, GroupWatchParams::DEFAULT_LIMIT)
+        end
+      end
+
+      def add_path_to_group_watcher(path)
+        return nil if @group.nil?
+        group_watcher = find_group_from_metadata(path)
+        group_watcher.add(path) unless group_watcher.include?(path)
+        group_watcher
+      end
+
+      def remove_path_from_group_watcher(path)
+        return if @group.nil?
+        group_watcher = find_group_from_metadata(path)
+        group_watcher.delete(path)
+      end
+
+      def construct_group_key(named_captures)
+        match_rule = []
+        @group_keys.each { |key|
+          match_rule.append(named_captures.fetch(key, GroupWatchParams::DEFAULT_KEY))
+        }
+        match_rule = match_rule.join(GroupWatchParams::REGEXP_JOIN)
+
+        match_rule
+      end
+
+      def construct_groupwatchers
+        @group.rule.each { |rule|
+          match_rule = construct_group_key(rule.match)
+          @group_watchers[match_rule] ||= GroupWatcher.new(@group.rate_period, rule.limit)
+        }
+      end
+
+      def find_group(metadata)
+        metadata_key = construct_group_key(metadata)
+        gw_key = @group_watchers.keys.find { |regexp| metadata_key.match?(regexp) && regexp != @default_group_key }
+        gw_key ||= @default_group_key
+
+        @group_watchers[gw_key]
+      end
+
+      def find_group_from_metadata(path)
+        begin
+          metadata = @group.pattern.match(path).named_captures
+          group_watcher = find_group(metadata)
+        rescue
+          log.warn "Cannot find group from metadata, Adding file in the default group"
+          group_watcher = @group_watchers[@default_group_key]
+        end
+
+        group_watcher
+      end
+    end
+
+    class GroupWatcher
+      attr_accessor :current_paths, :limit, :number_lines_read, :start_reading_time, :rate_period
+
+      FileCounter = Struct.new(
+        :number_lines_read,
+        :start_reading_time,
+      )
+
+      def initialize(rate_period = 60, limit = -1)
+        @current_paths = {}
+        @rate_period = rate_period
+        @limit = limit
+      end
+
+      def add(path)
+        @current_paths[path] = FileCounter.new(0, nil)
+      end
+
+      def include?(path)
+        @current_paths.key?(path)
+      end
+
+      def size
+        @current_paths.size
+      end
+
+      def delete(path)
+        @current_paths.delete(path)
+      end
+
+      def update_reading_time(path)
+        @current_paths[path].start_reading_time ||= Fluent::Clock.now
+      end
+
+      def update_lines_read(path, value)
+        @current_paths[path].number_lines_read += value
+      end
+
+      def reset_counter(path)
+        @current_paths[path].start_reading_time = nil
+        @current_paths[path].number_lines_read = 0
+      end
+
+      def time_spent_reading(path)
+        Fluent::Clock.now - @current_paths[path].start_reading_time
+      end
+
+      def limit_time_period_reached?(path)
+        time_spent_reading(path) < @rate_period
+      end
+
+      def limit_lines_reached?(path)
+        return true unless include?(path)
+        return true if @limit == 0
+
+        return false if @limit < 0
+        return false if @current_paths[path].number_lines_read < @limit / size
+
+        # update_reading_time(path)
+        if limit_time_period_reached?(path) # Exceeds limit
+          true
+        else # Does not exceed limit
+          reset_counter(path)
+          false
+        end
+      end
+
+      def to_s
+        super + " current_paths: #{@current_paths} rate_period: #{@rate_period} limit: #{@limit}"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -250,20 +250,6 @@ module Fluent::Plugin
       end
     end
 
-    TargetInfo = Struct.new(:path, :ino) do
-      def ==(other)
-        return false unless other.is_a?(TargetInfo)
-        self.path == other.path
-      end
-
-      def hash
-        self.path.hash
-      end
-
-      def eql?(other)
-        return false unless other.is_a?(TargetInfo)
-        self.path == other.path
-      end
-    end
+    TargetInfo = Struct.new(:path, :ino)
   end
 end