fluentd 1.10.1 → 1.11.1

data/lib/fluent/plugin/parser_multiline.rb

@@ -104,7 +104,7 @@ module Fluent
       end
 
       def firstline?(text)
-        @firstline_regex.match(text)
+        @firstline_regex.match?(text)
       end
 
       private

data/lib/fluent/plugin/parser_syslog.rb

@@ -63,16 +63,22 @@ module Fluent
       def initialize
         super
         @mutex = Mutex.new
-        @space_count = nil
+        @regexp = nil
+        @regexp3164 = nil
+        @regexp5424 = nil
+        @regexp_parser = nil
+        @time_parser_rfc3164 = nil
+        @time_parser_rfc5424 = nil
+        @space_count_rfc3164 = nil
         @space_count_rfc5424 = nil
+        @skip_space_count_rfc3164 = false
+        @skip_space_count_rfc5424 = false
+        @time_parser_rfc5424_without_subseconds = nil
       end
 
       def configure(conf)
         super
 
-        @time_parser_rfc3164 = @time_parser_rfc5424 = nil
-        @time_parser_rfc5424_without_subseconds = nil
-        @support_rfc5424_without_subseconds = false
         @regexp_parser = @parser_type == :regexp
         @regexp = case @message_format
                   when :rfc3164
@@ -85,27 +91,49 @@ module Fluent
                         alias_method :parse, :parse_rfc3164
                       end
                     end
+                    setup_time_parser_3164(@time_format)
                     RFC3164_WITHOUT_TIME_AND_PRI_REGEXP
                   when :rfc5424
-                    class << self
-                      alias_method :parse, :parse_rfc5424_regex
+                    if @regexp_parser
+                      class << self
+                        alias_method :parse, :parse_rfc5424_regex
+                      end
+                    else
+                      class << self
+                        alias_method :parse, :parse_rfc5424
+                      end
                     end
                     @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
-                    @support_rfc5424_without_subseconds = true
+                    setup_time_parser_5424(@time_format)
                     RFC5424_WITHOUT_TIME_AND_PRI_REGEXP
                   when :auto
                     class << self
                       alias_method :parse, :parse_auto
                     end
-                    @time_parser_rfc3164 = time_parser_create(format: @time_format)
-                    @time_parser_rfc5424 = time_parser_create(format: @rfc5424_time_format)
+                    setup_time_parser_3164(@time_format)
+                    setup_time_parser_5424(@rfc5424_time_format)
                     nil
                   end
 
-        @space_count = @time_format.squeeze(' ').count(' ') + 1
-        @space_count_rfc5424 = @rfc5424_time_format.squeeze(' ').count(' ') + 1
-        @time_parser = time_parser_create
+        if @regexp_parser
+          @regexp3164 = RFC3164_WITHOUT_TIME_AND_PRI_REGEXP
+          @regexp5424 = RFC5424_WITHOUT_TIME_AND_PRI_REGEXP
+        end
+      end
+
+      def setup_time_parser_3164(time_fmt)
+        @time_parser_rfc3164 = time_parser_create(format: time_fmt)
+        if ['%b %d %H:%M:%S', '%b %d %H:%M:%S.%N'].include?(time_fmt)
+          @skip_space_count_rfc3164 = true
+        end
+        @space_count_rfc3164 = time_fmt.squeeze(' ').count(' ') + 1
+      end
+
+      def setup_time_parser_5424(time_fmt)
+        @time_parser_rfc5424 = time_parser_create(format: time_fmt)
         @time_parser_rfc5424_without_subseconds = time_parser_create(format: "%Y-%m-%dT%H:%M:%S%z")
+        @skip_space_count_rfc5424 = time_fmt.count(' ').zero?
+        @space_count_rfc5424 = time_fmt.squeeze(' ').count(' ') + 1
       end
 
       # this method is for tests
@@ -118,14 +146,13 @@ module Fluent
       end
 
       def parse_auto(text, &block)
-        if REGEXP_DETECT_RFC5424.match(text)
-          @regexp = RFC5424_WITHOUT_TIME_AND_PRI_REGEXP
-          @time_parser = @time_parser_rfc5424
-          @support_rfc5424_without_subseconds = true
-          parse_rfc5424_regex(text, &block)
+        if REGEXP_DETECT_RFC5424.match?(text)
+          if @regexp_parser
+            parse_rfc5424_regex(text, &block)
+          else
+            parse_rfc5424(text, &block)
+          end
         else
-          @regexp = RFC3164_WITHOUT_TIME_AND_PRI_REGEXP
-          @time_parser = @time_parser_rfc3164
           if @regexp_parser
             parse_rfc3164_regex(text, &block)
           else
@@ -134,6 +161,8 @@ module Fluent
         end
       end
 
+      SPLIT_CHAR = ' '.freeze
+
       def parse_rfc3164_regex(text, &block)
         idx = 0
         record = {}
@@ -151,22 +180,22 @@ module Fluent
 
         i = idx - 1
         sq = false
-        @space_count.times do
-          while text[i + 1] == ' '.freeze
+        @space_count_rfc3164.times do
+          while text[i + 1] == SPLIT_CHAR
             sq = true
             i += 1
           end
 
-          i = text.index(' '.freeze, i + 1)
+          i = text.index(SPLIT_CHAR, i + 1)
         end
 
-        time_str = sq ? text.slice(idx, i - idx).squeeze(' ') : text.slice(idx, i - idx)
-        time = @mutex.synchronize { @time_parser.parse(time_str) }
+        time_str = sq ? text.slice(idx, i - idx).squeeze(SPLIT_CHAR) : text.slice(idx, i - idx)
+        time = @mutex.synchronize { @time_parser_rfc3164.parse(time_str) }
         if @keep_time_key
           record['time'] = time_str
         end
 
-        parse_plain(time, text, i + 1, record, RFC3164_CAPTURES, &block)
+        parse_plain(@regexp3164, time, text, i + 1, record, RFC3164_CAPTURES, &block)
       end
 
       def parse_rfc5424_regex(text, &block)
@@ -186,40 +215,36 @@ module Fluent
         i = idx - 1
         sq = false
         @space_count_rfc5424.times {
-          while text[i + 1] == ' '.freeze
+          while text[i + 1] == SPLIT_CHAR
             sq = true
             i += 1
           end
 
-          i = text.index(' '.freeze, i + 1)
+          i = text.index(SPLIT_CHAR, i + 1)
         }
 
-        time_str = sq ? text.slice(idx, i - idx).squeeze(' '.freeze) : text.slice(idx, i - idx)
+        time_str = sq ? text.slice(idx, i - idx).squeeze(SPLIT_CHAR) : text.slice(idx, i - idx)
         time = @mutex.synchronize do
           begin
-            @time_parser.parse(time_str)
+            @time_parser_rfc5424.parse(time_str)
           rescue Fluent::TimeParser::TimeParseError => e
-            if @support_rfc5424_without_subseconds
-              log.trace(e)
-              @time_parser_rfc5424_without_subseconds.parse(time_str)
-            else
-              raise
-            end
+            log.trace(e)
+            @time_parser_rfc5424_without_subseconds.parse(time_str)
           end
         end
 
         if @keep_time_key
           record['time'] = time_str
         end
-        parse_plain(time, text, i + 1, record, RFC5424_CAPTURES, &block)
+        parse_plain(@regexp5424, time, text, i + 1, record, RFC5424_CAPTURES, &block)
       end
 
       # @param time [EventTime]
       # @param idx [Integer] note: this argument is needed to avoid string creation
       # @param record [Hash]
       # @param capture_list [Array] for performance
-      def parse_plain(time, text, idx, record, capture_list, &block)
-        m = @regexp.match(text, idx)
+      def parse_plain(re, time, text, idx, record, capture_list, &block)
+        m = re.match(text, idx)
         if m.nil?
           yield nil, nil
           return
@@ -244,8 +269,6 @@ module Fluent
         yield time, record
       end
 
-      SPLIT_CHAR = ' '.freeze
-
       def parse_rfc3164(text, &block)
         pri = nil
         cursor = 0
@@ -264,20 +287,35 @@ module Fluent
           end
         end
 
-        # header part
-        time_size = 15 # skip Mmm dd hh:mm:ss
-        time_end = text[cursor + time_size]
-        if time_end == SPLIT_CHAR
-          time_str = text.slice(cursor, time_size)
-          cursor += 16 # time + ' '
-        elsif time_end == '.'.freeze
-          # support subsecond time
-          i = text.index(SPLIT_CHAR, time_size)
-          time_str = text.slice(cursor, i - cursor)
-          cursor = i + 1
+        if @skip_space_count_rfc3164
+          # header part
+          time_size = 15 # skip Mmm dd hh:mm:ss
+          time_end = text[cursor + time_size]
+          if time_end == SPLIT_CHAR
+            time_str = text.slice(cursor, time_size)
+            cursor += 16 # time + ' '
+          elsif time_end == '.'.freeze
+            # support subsecond time
+            i = text.index(SPLIT_CHAR, time_size)
+            time_str = text.slice(cursor, i - cursor)
+            cursor = i + 1
+          else
+            yield nil, nil
+            return
+          end
         else
-          yield nil, nil
-          return
+          i = cursor - 1
+          sq = false
+          @space_count_rfc3164.times do
+            while text[i + 1] == SPLIT_CHAR
+              sq = true
+              i += 1
+            end
+            i = text.index(SPLIT_CHAR, i + 1)
+          end
+
+          time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)
+          cursor = i + 1
         end
 
         i = text.index(SPLIT_CHAR, cursor)
@@ -325,7 +363,130 @@ module Fluent
         msg.chomp!
         record['message'] = msg
 
-        time = @time_parser.parse(time_str)
+        time = @time_parser_rfc3164.parse(time_str)
+        record['time'] = time_str if @keep_time_key
+
+        yield time, record
+      end
+
+      NILVALUE = '-'.freeze
+
+      def parse_rfc5424(text, &block)
+        pri = nil
+        cursor = 0
+        if @with_priority
+          if text.start_with?('<'.freeze)
+            i = text.index('>'.freeze, 1)
+            if i < 2
+              yield nil, nil
+              return
+            end
+            pri = text.slice(1, i - 1).to_i
+            i = text.index(SPLIT_CHAR, i)
+            cursor = i + 1
+          else
+            yield nil, nil
+            return
+          end
+        end
+
+        # timestamp part
+        if @skip_space_count_rfc5424
+          i = text.index(SPLIT_CHAR, cursor)
+          time_str = text.slice(cursor, i - cursor)
+          cursor = i + 1
+        else
+          i = cursor - 1
+          sq = false
+          @space_count_rfc5424.times do
+            while text[i + 1] == SPLIT_CHAR
+              sq = true
+              i += 1
+            end
+            i = text.index(SPLIT_CHAR, i + 1)
+          end
+
+          time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)
+          cursor = i + 1
+        end
+
+        # Repeat same code for the performance
+
+        # host part
+        i = text.index(SPLIT_CHAR, cursor)
+        unless i
+          yield nil, nil
+          return
+        end
+        slice_size = i - cursor
+        host = text.slice(cursor, slice_size)
+        cursor += slice_size + 1
+
+        # ident part
+        i = text.index(SPLIT_CHAR, cursor)
+        unless i
+          yield nil, nil
+          return
+        end
+        slice_size = i - cursor
+        ident = text.slice(cursor, slice_size)
+        cursor += slice_size + 1
+
+        # pid part
+        i = text.index(SPLIT_CHAR, cursor)
+        unless i
+          yield nil, nil
+          return
+        end
+        slice_size = i - cursor
+        pid = text.slice(cursor, slice_size)
+        cursor += slice_size + 1
+
+        # msgid part
+        i = text.index(SPLIT_CHAR, cursor)
+        unless i
+          yield nil, nil
+          return
+        end
+        slice_size = i - cursor
+        msgid = text.slice(cursor, slice_size)
+        cursor += slice_size + 1
+
+        record = {'host' => host, 'ident' => ident, 'pid' => pid, 'msgid' => msgid}
+        record['pri'] = pri if pri
+
+        # extradata part
+        ed_start = text[cursor]
+        if ed_start == NILVALUE
+          record['extradata'] = NILVALUE
+          cursor += 1
+        else
+          start = cursor
+          i = text.index('] '.freeze, cursor)
+          extradata = if i
+                        diff = i + 1 - start # calculate ']' position
+                        cursor += diff
+                        text.slice(start, diff)
+                      else  # No message part case
+                        cursor = text.bytesize
+                        text.slice(start, cursor)
+                      end
+          extradata.tr!("\\".freeze, ''.freeze)
+          record['extradata'] = extradata
+        end
+
+        # message part
+        if cursor != text.bytesize
+          msg = text.slice(cursor + 1, text.bytesize)
+          msg.chomp!
+          record['message'] = msg
+        end
+
+        time = begin
+                 @time_parser_rfc5424.parse(time_str)
+               rescue Fluent::TimeParser::TimeParseError => e
+                 @time_parser_rfc5424_without_subseconds.parse(time_str)
+               end
         record['time'] = time_str if @keep_time_key
 
         yield time, record

data/lib/fluent/plugin_helper/child_process.rb

@@ -259,6 +259,9 @@ module Fluent
 
         if !mode.include?(:stderr) && !mode.include?(:read_with_stderr)
           spawn_opts[:err] = IO::NULL if stderr == :discard
+          if !mode.include?(:read) && !mode.include?(:read_with_stderr)
+            spawn_opts[:out] = IO::NULL
+          end
           writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)
         elsif mode.include?(:read_with_stderr)
           writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
@@ -275,8 +278,6 @@ module Fluent
         if mode.include?(:read) || mode.include?(:read_with_stderr)
           readio.set_encoding(external_encoding, internal_encoding, **encoding_options)
           readio_in_use = true
-        else
-          readio.reopen(IO::NULL) if readio
         end
         if mode.include?(:stderr)
           stderrio.set_encoding(external_encoding, internal_encoding, **encoding_options)

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -41,9 +41,11 @@ module Fluent
             else
               mcall = method(:call_dig)
               mdelete = method(:delete_nest)
+              mset = method(:set_nest)
               singleton_class.module_eval do
                 define_method(:call, mcall)
                 define_method(:delete, mdelete)
+                define_method(:set, mset)
               end
             end
           end
@@ -75,6 +77,18 @@ module Fluent
           nil
         end
 
+        def set(r, v)
+          r[@keys] = v
+        end
+
+        # set_nest doesn't create intermediate object. If key doesn't exist, no effect.
+        # See also: https://bugs.ruby-lang.org/issues/11747
+        def set_nest(r, v)
+          r.dig(*@dig_keys)&.[]=(@last_key, v)
+        rescue
+          nil
+        end
+
         def self.parse_parameter(param)
           if param.start_with?('$.')
             parse_dot_notation(param)

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -43,6 +43,13 @@ module Fluent
         super
       end
 
+      %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
+        define_method(mth) do
+          @discovery_manager&.__send__(mth)
+          super()
+        end
+      end
+
       private
 
       # @param title [Symbol] the thread name. this value should be unique.