fluentd 1.14.4-x64-mingw32 → 1.15.0-x64-mingw32

data/lib/fluent/config/yaml_parser/parser.rb

@@ -0,0 +1,166 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/yaml_parser/section_builder'
+
+module Fluent
+  module Config
+    module YamlParser
+      class Parser
+        def initialize(config, indent: 2)
+          @base_indent = indent
+          @config = config
+        end
+
+        def build
+          s = @config['system'] && system_config_build(@config['system'])
+          c = @config['config'] && config_build(@config['config'], root: true)
+          RootBuilder.new(s, c)
+        end
+
+        private
+
+        def system_config_build(config)
+          section_build('system', config)
+        end
+
+        def config_build(config, indent: 0, root: false)
+          sb = SectionBodyBuilder.new(indent, root: root)
+          config.each do |c|
+            if (lc = c.delete('label'))
+              sb.add_section(label_build(lc, indent: indent))
+            end
+
+            if (sc = c.delete('source'))
+              sb.add_section(source_build(sc, indent: indent))
+            end
+
+            if (fc = c.delete('filter'))
+              sb.add_section(filter_build(fc, indent: indent))
+            end
+
+            if (mc = c.delete('match'))
+              sb.add_section(match_build(mc, indent: indent))
+            end
+
+            if (wc = c.delete('worker'))
+              sb.add_section(worker_build(wc, indent: indent))
+            end
+
+            included_sections_build(c, sb, indent: indent)
+          end
+
+          sb
+        end
+
+        def label_build(config, indent: 0)
+          config = config.dup
+          name = config.delete('$name')
+          c = config.delete('config')
+          SectionBuilder.new('label', config_build(c, indent: indent + @base_indent), indent, name)
+        end
+
+        def worker_build(config, indent: 0)
+          config = config.dup
+          num = config.delete('$arg')
+          c = config.delete('config')
+          SectionBuilder.new('worker', config_build(c, indent: indent + @base_indent), indent, num)
+        end
+
+        def source_build(config, indent: 0)
+          section_build('source', config, indent: indent)
+        end
+
+        def filter_build(config, indent: 0)
+          config = config.dup
+          tag = config.delete('$tag')
+          if tag.is_a?(Array)
+            section_build('filter', config, indent: indent, arg: tag&.join(','))
+          else
+            section_build('filter', config, indent: indent, arg: tag)
+          end
+        end
+
+        def match_build(config, indent: 0)
+          config = config.dup
+          tag = config.delete('$tag')
+          if tag.is_a?(Array)
+            section_build('match', config, indent: indent, arg: tag&.join(','))
+          else
+            section_build('match', config, indent: indent, arg: tag)
+          end
+        end
+
+        def included_sections_build(config, section_builder, indent: 0)
+          config.each_entry do |e|
+            k = e.keys.first
+            cc = e.delete(k)
+            case k
+            when 'label'
+              section_builder.add_section(label_build(cc, indent: indent))
+            when 'worker'
+              section_builder.add_section(worker_build(cc, indent: indent))
+            when 'source'
+              section_builder.add_section(source_build(cc, indent: indent))
+            when 'filter'
+              section_builder.add_section(filter_build(cc, indent: indent))
+            when 'match'
+              section_builder.add_section(match_build(cc, indent: indent))
+            end
+          end
+        end
+
+        def section_build(name, config, indent: 0, arg: nil)
+          sb = SectionBodyBuilder.new(indent + @base_indent)
+
+          if (v = config.delete('$type'))
+            sb.add_line('@type', v)
+          end
+
+          if (v = config.delete('$label'))
+            sb.add_line('@label', v)
+          end
+
+          if (v = config.delete('$id'))
+            sb.add_line('@id', v)
+          end
+
+          config.each do |key, val|
+            if val.is_a?(Array)
+              val.each do |v|
+                sb.add_section(section_build(key, v, indent: indent + @base_indent))
+              end
+            elsif val.is_a?(Hash)
+              harg = val.delete('$arg')
+              if harg.is_a?(Array)
+                # To prevent to generate invalid configuration for arg.
+                # "arg" should be String object and concatenated by ","
+                # when two or more objects are specified there.
+                sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg&.join(',')))
+              else
+                sb.add_section(section_build(key, val, indent: indent + @base_indent, arg: harg))
+              end
+            else
+              sb.add_line(key, val)
+            end
+          end
+
+          SectionBuilder.new(name, sb, indent, arg)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser/section_builder.rb

@@ -0,0 +1,107 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Config
+    module YamlParser
+      SectionBuilder = Struct.new(:name, :body, :indent_size, :arg) do
+        def to_s
+          indent = ' ' * indent_size
+
+          if arg && !arg.to_s.empty?
+            "#{indent}<#{name} #{arg}>\n#{body}\n#{indent}</#{name}>"
+          else
+            "#{indent}<#{name}>\n#{body}\n#{indent}</#{name}>"
+          end
+        end
+
+        def to_element
+          elem = body.to_element
+          elem.name = name
+          elem.arg = arg.to_s if arg
+          elem.v1_config = true
+          elem
+        end
+      end
+
+      class RootBuilder
+        def initialize(system, conf)
+          @system = system
+          @conf = conf
+        end
+
+        attr_reader :system, :conf
+
+        def to_element
+          Fluent::Config::Element.new('ROOT', '', {}, [@system, @conf].compact.map(&:to_element).flatten)
+        end
+
+        def to_s
+          s = StringIO.new(+'')
+          s.puts(@system.to_s) if @system
+          s.puts(@conf.to_s) if @conf
+
+          s.string
+        end
+      end
+
+      class SectionBodyBuilder
+        Row = Struct.new(:key, :value, :indent) do
+          def to_s
+            "#{indent}#{key} #{value}"
+          end
+        end
+
+        def initialize(indent, root: false)
+          @indent = ' ' * indent
+          @bodies = []
+          @root = root
+        end
+
+        def add_line(k, v)
+          @bodies << Row.new(k, v, @indent)
+        end
+
+        def add_section(section)
+          @bodies << section
+        end
+
+        def to_element
+          if @root
+            return @bodies.map(&:to_element)
+          end
+
+          not_section, section = @bodies.partition { |e| e.is_a?(Row) }
+          r = {}
+          not_section.each do |e|
+            v = e.value
+            r[e.key] = v.respond_to?(:to_element) ? v.to_element : v
+          end
+
+          if @root
+            section.map(&:to_element)
+          else
+            Fluent::Config::Element.new('', '', r, section.map(&:to_element))
+          end
+        end
+
+        def to_s
+          @bodies.map(&:to_s).join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/fluent/config/yaml_parser.rb

@@ -0,0 +1,56 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/yaml_parser/loader'
+require 'fluent/config/yaml_parser/parser'
+require 'pathname'
+
+module Fluent
+  module Config
+    module YamlParser
+      def self.parse(path)
+        context = Kernel.binding
+
+        unless context.respond_to?(:use_nil)
+          context.define_singleton_method(:use_nil) do
+            raise Fluent::SetNil
+          end
+        end
+
+        unless context.respond_to?(:use_default)
+          context.define_singleton_method(:use_default) do
+            raise Fluent::SetDefault
+          end
+        end
+
+        unless context.respond_to?(:hostname)
+          context.define_singleton_method(:hostname) do
+            Socket.gethostname
+          end
+        end
+
+        unless context.respond_to?(:worker_id)
+          context.define_singleton_method(:worker_id) do
+            ENV['SERVERENGINE_WORKER_ID'] || ''
+          end
+        end
+
+        s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path))
+        Fluent::Config::YamlParser::Parser.new(s).build.to_element
+      end
+    end
+  end
+end

data/lib/fluent/config.rb

@@ -17,6 +17,7 @@
 require 'fluent/config/error'
 require 'fluent/config/element'
 require 'fluent/configurable'
+require 'fluent/config/yaml_parser'
 
 module Fluent
   module Config
@@ -25,7 +26,18 @@ module Fluent
     # @param additional_config [String] config which is added to last of config body
     # @param use_v1_config [Bool] config is formatted with v1 or not
     # @return [Fluent::Config]
-    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true)
+    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true, type: nil)
+      if type == :guess
+        config_file_ext = File.extname(config_path)
+        if config_file_ext == '.yaml' || config_file_ext == '.yml'
+          type = :yaml
+        end
+      end
+
+      if type == :yaml || type == :yml
+        return Fluent::Config::YamlParser.parse(config_path)
+      end
+
       config_fname = File.basename(config_path)
       config_basedir = File.dirname(config_path)
       config_data = File.open(config_path, "r:#{encoding}:utf-8") do |f|
@@ -36,6 +48,7 @@ module Fluent
         end
         s
       end
+
       Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config)
     end
 

data/lib/fluent/event_router.rb

@@ -116,6 +116,8 @@ module Fluent
       if callback = find_callback
         callback.call(es)
       end
+    rescue Pipeline::OutputError => e
+      @emit_error_handler.handle_emits_error(tag, e.processed_es, e.internal_error)
     rescue => e
       @emit_error_handler.handle_emits_error(tag, es, e)
     end
@@ -161,6 +163,17 @@ module Fluent
     private
 
     class Pipeline
+
+      class OutputError < StandardError
+        attr_reader :internal_error
+        attr_reader :processed_es
+
+        def initialize(internal_error, processed_es)
+          @internal_error = internal_error
+          @processed_es = processed_es
+        end
+      end
+
       def initialize
         @filters = []
         @output = nil
@@ -178,7 +191,12 @@ module Fluent
 
       def emit_events(tag, es)
         processed = @optimizer.filter_stream(tag, es)
-        @output.emit_events(tag, processed)
+
+        begin
+          @output.emit_events(tag, processed)
+        rescue => e
+          raise OutputError.new(e, processed)
+        end
       end
 
       class FilterOptimizer

data/lib/fluent/plugin/bare_output.rb

@@ -70,7 +70,7 @@ module Fluent
         super
 
         @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
-        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_count", help_text: "Number of count emits")
         @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
         @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
         @enable_size_metrics = !!system_config.enable_size_metrics

data/lib/fluent/plugin/base.rb

@@ -72,8 +72,8 @@ module Fluent
 
       def string_safe_encoding(str)
         unless str.valid_encoding?
-          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
           str = str.scrub('?')
+          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
         end
         yield str
       end

data/lib/fluent/plugin/file_wrapper.rb

@@ -16,8 +16,8 @@
 
 module Fluent
   module FileWrapper
-    def self.open(*args)
-      io = WindowsFile.new(*args).io
+    def self.open(path, mode='r')
+      io = WindowsFile.new(path, mode).io
       if block_given?
         v = yield io
         io.close
@@ -35,116 +35,36 @@ module Fluent
     end
   end
 
-  module WindowsFileExtension
-    attr_reader :path
-
-    def stat
-      s = super
-      s.instance_variable_set :@ino, @ino
-      def s.ino; @ino; end
-      s
-    end
-  end
-
-  class Win32Error < StandardError
-    require 'windows/error'
-    include Windows::Error
-
-    attr_reader :errcode, :msg
-
-    WSABASEERR = 10000
-
-    def initialize(errcode, msg = nil)
-      @errcode = errcode
-      @msg = msg
-    end
-
-    def format_english_message(errcode)
-      buf = 0.chr * 260
-      flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ARGUMENT_ARRAY
-      english_lang_id = 1033 # The result of MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US)
-      FormatMessageA.call(flags, 0, errcode, english_lang_id, buf, buf.size, 0)
-      buf.force_encoding(Encoding.default_external).strip
-    end
-
-    def to_s
-      msg = super
-      msg << ": code: #{@errcode}, #{format_english_message(@errcode)}"
-      msg << " - #{@msg}" if @msg
-      msg
-    end
-
-    def inspect
-      "#<#{to_s}>"
-    end
-
-    def ==(other)
-      return false if other.class != Win32Error
-      @errcode == other.errcode && @msg == other.msg
-    end
-
-    def wsaerr?
-      @errcode >= WSABASEERR
-    end
-  end
-
-  # To open and get stat with setting FILE_SHARE_DELETE
   class WindowsFile
     require 'windows/file'
-    require 'windows/error'
     require 'windows/handle'
-    require 'windows/nio'
 
-    include Windows::Error
+    include File::Constants
     include Windows::File
     include Windows::Handle
-    include Windows::NIO
-
-    def initialize(path, mode='r', sharemode=FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE)
-      @path = path
-      @file_handle = INVALID_HANDLE_VALUE
-      @mode = mode
 
+    attr_reader :io
 
-      access, creationdisposition, seektoend = case mode.delete('b')
-      when "r" ; [FILE_GENERIC_READ                     , OPEN_EXISTING, false]
-      when "r+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS  , false]
-      when "w" ; [FILE_GENERIC_WRITE                    , CREATE_ALWAYS, false]
-      when "w+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, CREATE_ALWAYS, false]
-      when "a" ; [FILE_GENERIC_WRITE                    , OPEN_ALWAYS  , true]
-      when "a+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS  , true]
-      else raise "unknown mode '#{mode}'"
-      end
-
-      @file_handle = CreateFile.call(@path, access, sharemode,
-                     0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
-      if @file_handle == INVALID_HANDLE_VALUE
-        win32err = Win32Error.new(Win32::API.last_error, path)
-        errno = ServerEngine::RbWinSock.rb_w32_map_errno(win32err.errcode)
-        if errno == Errno::EINVAL::Errno || win32err.wsaerr?
-          # maybe failed to map
-          raise win32err
-        else
-          raise SystemCallError.new(win32err.message, errno)
-        end
+    def initialize(path, mode='r')
+      @path = path
+      @io = File.open(path, mode2flags(mode))
+      @file_handle = _get_osfhandle(@io.to_i)
+      @io.instance_variable_set(:@file_index, self.ino)
+      def @io.ino
+        @file_index
       end
     end
 
     def close
-      CloseHandle.call(@file_handle)
+      @io.close
       @file_handle = INVALID_HANDLE_VALUE
     end
 
-    def io
-      fd = _open_osfhandle(@file_handle, 0)
-      raise Errno::ENOENT if fd == -1
-      io = File.for_fd(fd, @mode)
-      io.instance_variable_set :@ino, self.ino
-      io.instance_variable_set :@path, @path
-      io.extend WindowsFileExtension
-      io
-    end
-
+    # To keep backward compatibility, we continue to use GetFileInformationByHandle()
+    # to get file id.
+    # Note that Ruby's File.stat uses GetFileInformationByHandleEx() with FileIdInfo
+    # and returned value is different with above one, former one is 64 bit while
+    # later one is 128bit.
     def ino
       by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
 
@@ -155,6 +75,41 @@ module Fluent
       by_handle_file_information.unpack("I11Q1")[11] # fileindex
     end
 
+    def stat
+      raise Errno::ENOENT if delete_pending
+      s = File.stat(@path)
+      s.instance_variable_set :@ino, self.ino
+      def s.ino; @ino; end
+      s
+    end
+
+    private
+
+    def mode2flags(mode)
+      # Always inject File::Constants::SHARE_DELETE
+      # https://github.com/fluent/fluentd/pull/3585#issuecomment-1101502617
+      # To enable SHARE_DELETE, BINARY is also required.
+      # https://bugs.ruby-lang.org/issues/11218
+      # https://github.com/ruby/ruby/blob/d6684f063bc53e3cab025bd39526eca3b480b5e7/win32/win32.c#L6332-L6345
+      flags = BINARY | SHARE_DELETE
+      case mode.delete("b")
+      when "r"
+        flags |= RDONLY
+      when "r+"
+        flags |= RDWR
+      when "w"
+        flags |= WRONLY | CREAT | TRUNC
+      when "w+"
+        flags |= RDWR | CREAT | TRUNC
+      when "a"
+        flags |= WRONLY | CREAT | APPEND
+      when "a+"
+        flags |= RDWR | CREAT | APPEND
+      else
+        raise Errno::EINVAL.new("Unsupported mode by Fluent::FileWrapper: #{mode}")
+      end
+    end
+
     # DeletePending is a Windows-specific file state that roughly means
     # "this file is queued for deletion, so close any open handlers"
     #
@@ -173,15 +128,5 @@ module Fluent
 
       return buf.unpack("QQICC")[3] != 0
     end
-
-    private :delete_pending
-
-    def stat
-      raise Errno::ENOENT if delete_pending
-      s = File.stat(@path)
-      s.instance_variable_set :@ino, self.ino
-      def s.ino; @ino; end
-      s
-    end
   end
 end if Fluent.windows?

data/lib/fluent/plugin/in_forward.rb

@@ -40,7 +40,7 @@ module Fluent::Plugin
     config_param :backlog, :integer, default: nil
     # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
     desc 'The timeout time used to set linger option.'
-    config_param :linger_timeout, :integer, default: 0
+    config_param :linger_timeout, :integer, default: nil, deprecated: "use transport directive"
     # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
     config_param :blocking_timeout, :time, default: 0.5
     desc 'Try to resolve hostname from IP addresses or not.'

data/lib/fluent/plugin/in_http.rb

@@ -314,8 +314,16 @@ module Fluent::Plugin
         @parser_json.parse(js) do |_time, record|
           return nil, record
         end
+      elsif ndjson = params['ndjson']
+        events = []
+        ndjson.split(/\r?\n/).each do |js|
+          @parser_json.parse(js) do |_time, record|
+            events.push(record)
+          end
+        end
+        return nil, events
       else
-        raise "'json' or 'msgpack' parameter is required"
+        raise "'json', 'ndjson' or 'msgpack' parameter is required"
       end
     end
 
@@ -567,6 +575,8 @@ module Fluent::Plugin
           params['json'] = @body
         elsif @content_type =~ /^application\/msgpack/
           params['msgpack'] = @body
+        elsif @content_type =~ /^application\/x-ndjson/
+          params['ndjson'] = @body
         end
         path_info = uri.path