fluentd 1.9.3 → 1.11.1

data/lib/fluent/plugin/out_copy.rb

@@ -61,7 +61,7 @@ module Fluent::Plugin
           output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
         rescue => e
           if @ignore_errors[i]
-            log.error "ignore emit error", error: e
+            log.error "ignore emit error in #{output.plugin_id}", error: e
           else
             raise e
           end

data/lib/fluent/plugin/out_file.rb

@@ -155,7 +155,7 @@ module Fluent::Plugin
         dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
         dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
 
-        test_chunk1 = chunk_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
+        test_chunk1 = chunk_for_test(dummy_tag, Fluent::EventTime.now, dummy_record)
         test_path = extract_placeholders(@path_template, test_chunk1)
         unless ::Fluent::FileUtil.writable_p?(test_path)
           raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"

data/lib/fluent/plugin/out_forward.rb

@@ -324,7 +324,7 @@ module Fluent::Plugin
         end
       end
 
-      if @keepalive && @keepalive_timeout
+      if @keepalive
         timer_execute(:out_forward_keep_alived_socket_watcher, @keep_alive_watcher_interval, &method(:on_purge_obsolete_socks))
       end
     end
@@ -382,8 +382,11 @@ module Fluent::Plugin
           cert_logical_store_name: @tls_cert_logical_store_name,
           cert_use_enterprise_store: @tls_cert_use_enterprise_store,
 
-          # Enabling SO_LINGER causes data loss on Windows
-          # https://github.com/fluent/fluentd/issues/1968
+          # Enabling SO_LINGER causes tcp port exhaustion on Windows.
+          # This is because dynamic ports are only 16384 (from 49152 to 65535) and
+          # expiring SO_LINGER enabled ports should wait 4 minutes
+          # where set by TcpTimeDelay. Its default value is 4 minutes.
+          # So, we should disable SO_LINGER on Windows to prevent flood of waiting ports.
           linger_timeout: Fluent.windows? ? nil : @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
@@ -510,7 +513,7 @@ module Fluent::Plugin
 
     class Node
       extend Forwardable
-      def_delegators :@server, :discovery_id, :host, :port, :name, :weight, :standby, :username, :password, :shared_key
+      def_delegators :@server, :discovery_id, :host, :port, :name, :weight, :standby
 
       # @param connection_manager [Fluent::Plugin::ForwardOutput::ConnectionManager]
       # @param ack_handler [Fluent::Plugin::ForwardOutput::AckHandler]
@@ -542,8 +545,8 @@ module Fluent::Plugin
           log: @log,
           hostname: sender.security && sender.security.self_hostname,
           shared_key: server.shared_key || (sender.security && sender.security.shared_key) || '',
-          password: server.password,
-          username: server.username,
+          password: server.password || '',
+          username: server.username || '',
         )
 
         @unpacker = Fluent::MessagePackFactory.msgpack_unpacker
@@ -580,12 +583,7 @@ module Fluent::Plugin
 
       def verify_connection
         connect do |sock, ri|
-          if ri.state != :established
-            establish_connection(sock, ri)
-            if ri.state != :established
-              raise "Failed to establish connection to #{@host}:#{@port}"
-            end
-          end
+          ensure_established_connection(sock, ri)
         end
       end
 
@@ -654,14 +652,7 @@ module Fluent::Plugin
       def send_data(tag, chunk)
         ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self)
         connect(nil, ack: ack) do |sock, ri|
-          if ri.state != :established
-            establish_connection(sock, ri)
-
-            if ri.state != :established
-              raise ConnectionClosedError, "failed to establish connection with node #{@name}"
-            end
-          end
-
+          ensure_established_connection(sock, ri)
           send_data_actual(sock, tag, chunk)
         end
 
@@ -686,7 +677,9 @@ module Fluent::Plugin
 
         case @sender.heartbeat_type
         when :transport
-          connect(dest_addr) do |_ri, _sock|
+          connect(dest_addr) do |sock, ri|
+            ensure_established_connection(sock, ri)
+
             ## don't send any data to not cause a compatibility problem
             # sock.write FORWARD_TCP_HEARTBEAT_DATA
 
@@ -716,7 +709,7 @@ module Fluent::Plugin
           @resolved_host ||= resolve_dns!
 
         else
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           rh = @resolved_host
           if !rh || now - @resolved_time >= @sender.expire_dns_cache
             rh = @resolved_host = resolve_dns!
@@ -778,6 +771,16 @@ module Fluent::Plugin
 
       private
 
+      def ensure_established_connection(sock, request_info)
+        if request_info.state != :established
+          establish_connection(sock, request_info)
+
+          if request_info.state != :established
+            raise ConnectionClosedError, "failed to establish connection with node #{@name}"
+          end
+        end
+      end
+
       def connect(host = nil, ack: false, &block)
         @connection_manager.connect(host: host || resolved_host, port: port, hostname: @hostname, ack: ack, &block)
       end

data/lib/fluent/plugin/out_forward/handshake_protocol.rb

@@ -105,6 +105,10 @@ module Fluent::Plugin
           .hexdigest
         ping = ['PING', @hostname, @shared_key_salt, shared_key_hexdigest]
         if !ri.auth.empty?
+          if @username.nil? || @password.nil?
+            raise PingpongError, "username and password are required"
+          end
+
           password_hexdigest = Digest::SHA512.new.update(ri.auth).update(@username).update(@password).hexdigest
           ping.push(@username, password_hexdigest)
         else

data/lib/fluent/plugin/out_forward/load_balancer.rb

@@ -56,7 +56,7 @@ module Fluent::Plugin
       end
 
       def rebuild_weight_array(nodes)
-        standby_nodes, regular_nodes = nodes.partition {|n|
+        standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
           n.standby?
         }
 

data/lib/fluent/plugin/out_http.rb

@@ -37,6 +37,8 @@ module Fluent::Plugin
     config_param :proxy, :string, default: ENV['HTTP_PROXY'] || ENV['http_proxy']
     desc 'Content-Type for HTTP request'
     config_param :content_type, :string, default: nil
+    desc 'JSON array data format for HTTP request body'
+    config_param :json_array, :bool, default: false
     desc 'Additional headers for HTTP request'
     config_param :headers, :hash, default: nil
 
@@ -100,6 +102,13 @@ module Fluent::Plugin
       @proxy_uri = URI.parse(@proxy) if @proxy
       @formatter = formatter_create
       @content_type = setup_content_type unless @content_type
+
+      if @json_array
+        if @formatter_configs.first[:@type] != "json"
+          raise Fluent::ConfigError, "json_array option could be used with json formatter only"
+        end
+        define_singleton_method(:format, method(:format_json_array))
+      end
     end
 
     def multi_workers_ready?
@@ -114,6 +123,10 @@ module Fluent::Plugin
       @formatter.format(tag, time, record)
     end
 
+    def format_json_array(tag, time, record)
+      @formatter.format(tag, time, record) << ","
+    end
+
     def write(chunk)
       uri = parse_endpoint(chunk)
       req = create_request(chunk, uri)
@@ -128,7 +141,7 @@ module Fluent::Plugin
     def setup_content_type
       case @formatter_configs.first[:@type]
       when 'json'
-        'application/x-ndjson'
+        @json_array ? 'application/json' : 'application/x-ndjson'
       when 'csv'
         'text/csv'
       when 'tsv', 'ltsv'
@@ -202,7 +215,7 @@ module Fluent::Plugin
         req.basic_auth(@auth.username, @auth.password)
       end
       set_headers(req)
-      req.body = chunk.read
+      req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read
       req
     end
 

data/lib/fluent/plugin/parser_multiline.rb

@@ -104,7 +104,7 @@ module Fluent
       end
 
       def firstline?(text)
-        @firstline_regex.match(text)
+        @firstline_regex.match?(text)
       end
 
       private

data/lib/fluent/plugin/parser_syslog.rb

@@ -23,6 +23,7 @@ module Fluent
     class SyslogParser < Parser
       Plugin.register_parser('syslog', self)
 
+      # TODO: Remove them since these regexps are no longer needed. but keep them for compatibility for now
       # From existence TextParser pattern
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
@@ -36,7 +37,16 @@ module Fluent
       REGEXP_RFC5424_WITH_PRI = Regexp.new(<<~'EOS'.chomp % REGEXP_RFC5424, Regexp::MULTILINE)
         \A<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} %s\z
       EOS
-      REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
+
+      REGEXP_DETECT_RFC5424 = /^\<[0-9]{1,3}\>[1-9]\d{0,2}/
+
+      RFC3164_WITHOUT_TIME_AND_PRI_REGEXP = /(?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      RFC3164_CAPTURES = RFC3164_WITHOUT_TIME_AND_PRI_REGEXP.names.freeze
+      RFC3164_PRI_REGEXP = /^<(?<pri>[0-9]{1,3})>/
+
+      RFC5424_WITHOUT_TIME_AND_PRI_REGEXP = /(?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|(?:\[.*?(?<!\\)\])+))(?: (?<message>.+))?\z/m
+      RFC5424_CAPTURES = RFC5424_WITHOUT_TIME_AND_PRI_REGEXP.names.freeze
+      RFC5424_PRI_REGEXP = /^<(?<pri>\d{1,3})>\d\d{0,2}\s/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
       desc 'If the incoming logs have priority prefix, e.g. <9>, set true'
@@ -53,46 +63,80 @@ module Fluent
       def initialize
         super
         @mutex = Mutex.new
+        @regexp = nil
+        @regexp3164 = nil
+        @regexp5424 = nil
+        @regexp_parser = nil
+        @time_parser_rfc3164 = nil
+        @time_parser_rfc5424 = nil
+        @space_count_rfc3164 = nil
+        @space_count_rfc5424 = nil
+        @skip_space_count_rfc3164 = false
+        @skip_space_count_rfc5424 = false
+        @time_parser_rfc5424_without_subseconds = nil
       end
 
       def configure(conf)
         super
 
-        @time_parser_rfc3164 = @time_parser_rfc5424 = nil
-        @time_parser_rfc5424_without_subseconds = nil
-        @support_rfc5424_without_subseconds = false
         @regexp_parser = @parser_type == :regexp
         @regexp = case @message_format
                   when :rfc3164
                     if @regexp_parser
                       class << self
-                        alias_method :parse, :parse_plain
+                        alias_method :parse, :parse_rfc3164_regex
                       end
                     else
                       class << self
                         alias_method :parse, :parse_rfc3164
                       end
                     end
-                    @with_priority ? REGEXP_WITH_PRI : REGEXP
+                    setup_time_parser_3164(@time_format)
+                    RFC3164_WITHOUT_TIME_AND_PRI_REGEXP
                   when :rfc5424
-                    class << self
-                      alias_method :parse, :parse_plain
+                    if @regexp_parser
+                      class << self
+                        alias_method :parse, :parse_rfc5424_regex
+                      end
+                    else
+                      class << self
+                        alias_method :parse, :parse_rfc5424
+                      end
                     end
                     @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
-                    @support_rfc5424_without_subseconds = true
-                    @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424_NO_PRI
+                    setup_time_parser_5424(@time_format)
+                    RFC5424_WITHOUT_TIME_AND_PRI_REGEXP
                   when :auto
                     class << self
                       alias_method :parse, :parse_auto
                     end
-                    @time_parser_rfc3164 = time_parser_create(format: @time_format)
-                    @time_parser_rfc5424 = time_parser_create(format: @rfc5424_time_format)
+                    setup_time_parser_3164(@time_format)
+                    setup_time_parser_5424(@rfc5424_time_format)
                     nil
                   end
-        @time_parser = time_parser_create
+
+        if @regexp_parser
+          @regexp3164 = RFC3164_WITHOUT_TIME_AND_PRI_REGEXP
+          @regexp5424 = RFC5424_WITHOUT_TIME_AND_PRI_REGEXP
+        end
+      end
+
+      def setup_time_parser_3164(time_fmt)
+        @time_parser_rfc3164 = time_parser_create(format: time_fmt)
+        if ['%b %d %H:%M:%S', '%b %d %H:%M:%S.%N'].include?(time_fmt)
+          @skip_space_count_rfc3164 = true
+        end
+        @space_count_rfc3164 = time_fmt.squeeze(' ').count(' ') + 1
+      end
+
+      def setup_time_parser_5424(time_fmt)
+        @time_parser_rfc5424 = time_parser_create(format: time_fmt)
         @time_parser_rfc5424_without_subseconds = time_parser_create(format: "%Y-%m-%dT%H:%M:%S%z")
+        @skip_space_count_rfc5424 = time_fmt.count(' ').zero?
+        @space_count_rfc5424 = time_fmt.squeeze(' ').count(' ') + 1
       end
 
+      # this method is for tests
       def patterns
         {'format' => @regexp, 'time_format' => @time_format}
       end
@@ -102,52 +146,113 @@ module Fluent
       end
 
       def parse_auto(text, &block)
-        if REGEXP_DETECT_RFC5424.match(text)
-          @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424_NO_PRI
-          @time_parser = @time_parser_rfc5424
-          @support_rfc5424_without_subseconds = true
-          parse_plain(text, &block)
+        if REGEXP_DETECT_RFC5424.match?(text)
+          if @regexp_parser
+            parse_rfc5424_regex(text, &block)
+          else
+            parse_rfc5424(text, &block)
+          end
         else
-          @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
-          @time_parser = @time_parser_rfc3164
           if @regexp_parser
-            parse_plain(text, &block)
+            parse_rfc3164_regex(text, &block)
           else
             parse_rfc3164(text, &block)
           end
         end
       end
 
-      def parse_plain(text, &block)
-        m = @regexp.match(text)
-        unless m
-          yield nil, nil
-          return
+      SPLIT_CHAR = ' '.freeze
+
+      def parse_rfc3164_regex(text, &block)
+        idx = 0
+        record = {}
+
+        if @with_priority
+          if RFC3164_PRI_REGEXP.match?(text)
+            v = text.index('>')
+            record['pri'] = text[1..v].to_i # trim `<` and ``>
+            idx = v + 1
+          else
+            yield(nil, nil)
+            return
+          end
+        end
+
+        i = idx - 1
+        sq = false
+        @space_count_rfc3164.times do
+          while text[i + 1] == SPLIT_CHAR
+            sq = true
+            i += 1
+          end
+
+          i = text.index(SPLIT_CHAR, i + 1)
         end
 
-        time = nil
+        time_str = sq ? text.slice(idx, i - idx).squeeze(SPLIT_CHAR) : text.slice(idx, i - idx)
+        time = @mutex.synchronize { @time_parser_rfc3164.parse(time_str) }
+        if @keep_time_key
+          record['time'] = time_str
+        end
+
+        parse_plain(@regexp3164, time, text, i + 1, record, RFC3164_CAPTURES, &block)
+      end
+
+      def parse_rfc5424_regex(text, &block)
+        idx = 0
         record = {}
 
-        m.names.each { |name|
-          if value = m[name]
+        if @with_priority
+          if (m = RFC5424_PRI_REGEXP.match(text))
+            record['pri'] = m['pri'].to_i
+            idx = m.end(0)
+          else
+            yield(nil, nil)
+            return
+          end
+        end
+
+        i = idx - 1
+        sq = false
+        @space_count_rfc5424.times {
+          while text[i + 1] == SPLIT_CHAR
+            sq = true
+            i += 1
+          end
+
+          i = text.index(SPLIT_CHAR, i + 1)
+        }
+
+        time_str = sq ? text.slice(idx, i - idx).squeeze(SPLIT_CHAR) : text.slice(idx, i - idx)
+        time = @mutex.synchronize do
+          begin
+            @time_parser_rfc5424.parse(time_str)
+          rescue Fluent::TimeParser::TimeParseError => e
+            log.trace(e)
+            @time_parser_rfc5424_without_subseconds.parse(time_str)
+          end
+        end
+
+        if @keep_time_key
+          record['time'] = time_str
+        end
+        parse_plain(@regexp5424, time, text, i + 1, record, RFC5424_CAPTURES, &block)
+      end
+
+      # @param time [EventTime]
+      # @param idx [Integer] note: this argument is needed to avoid string creation
+      # @param record [Hash]
+      # @param capture_list [Array] for performance
+      def parse_plain(re, time, text, idx, record, capture_list, &block)
+        m = re.match(text, idx)
+        if m.nil?
+          yield nil, nil
+          return
+        end
+
+        capture_list.each { |name|
+          if value = (m[name] rescue nil)
             case name
-            when "pri"
-              record['pri'] = value.to_i
-            when "time"
-              time = @mutex.synchronize do
-                time_str = value.squeeze(' ')
-                begin
-                  @time_parser.parse(time_str)
-                rescue Fluent::TimeParser::TimeParseError => e
-                  if @support_rfc5424_without_subseconds
-                    log.trace(e)
-                    @time_parser_rfc5424_without_subseconds.parse(time_str)
-                  else
-                    raise
-                  end
-                end
-              end
-              record[name] = value if @keep_time_key
             when "message"
               value.chomp!
               record[name] = value
@@ -164,8 +269,6 @@ module Fluent
         yield time, record
       end
 
-      SPLIT_CHAR = ' '.freeze
-
       def parse_rfc3164(text, &block)
         pri = nil
         cursor = 0
@@ -184,20 +287,35 @@ module Fluent
           end
         end
 
-        # header part
-        time_size = 15 # skip Mmm dd hh:mm:ss
-        time_end = text[cursor + time_size]
-        if time_end == SPLIT_CHAR
-          time_str = text.slice(cursor, time_size)
-          cursor += 16 # time + ' '
-        elsif time_end == '.'.freeze
-          # support subsecond time
-          i = text.index(SPLIT_CHAR, time_size)
-          time_str = text.slice(cursor, i - cursor)
-          cursor = i + 1
+        if @skip_space_count_rfc3164
+          # header part
+          time_size = 15 # skip Mmm dd hh:mm:ss
+          time_end = text[cursor + time_size]
+          if time_end == SPLIT_CHAR
+            time_str = text.slice(cursor, time_size)
+            cursor += 16 # time + ' '
+          elsif time_end == '.'.freeze
+            # support subsecond time
+            i = text.index(SPLIT_CHAR, time_size)
+            time_str = text.slice(cursor, i - cursor)
+            cursor = i + 1
+          else
+            yield nil, nil
+            return
+          end
         else
-          yield nil, nil
-          return
+          i = cursor - 1
+          sq = false
+          @space_count_rfc3164.times do
+            while text[i + 1] == SPLIT_CHAR
+              sq = true
+              i += 1
+            end
+            i = text.index(SPLIT_CHAR, i + 1)
+          end
+
+          time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)
+          cursor = i + 1
         end
 
         i = text.index(SPLIT_CHAR, cursor)
@@ -245,7 +363,130 @@ module Fluent
         msg.chomp!
         record['message'] = msg
 
-        time = @time_parser.parse(time_str)
+        time = @time_parser_rfc3164.parse(time_str)
+        record['time'] = time_str if @keep_time_key
+
+        yield time, record
+      end
+
+      NILVALUE = '-'.freeze
+
+      def parse_rfc5424(text, &block)
+        pri = nil
+        cursor = 0
+        if @with_priority
+          if text.start_with?('<'.freeze)
+            i = text.index('>'.freeze, 1)
+            if i < 2
+              yield nil, nil
+              return
+            end
+            pri = text.slice(1, i - 1).to_i
+            i = text.index(SPLIT_CHAR, i)
+            cursor = i + 1
+          else
+            yield nil, nil
+            return
+          end
+        end
+
+        # timestamp part
+        if @skip_space_count_rfc5424
+          i = text.index(SPLIT_CHAR, cursor)
+          time_str = text.slice(cursor, i - cursor)
+          cursor = i + 1
+        else
+          i = cursor - 1
+          sq = false
+          @space_count_rfc5424.times do
+            while text[i + 1] == SPLIT_CHAR
+              sq = true
+              i += 1
+            end
+            i = text.index(SPLIT_CHAR, i + 1)
+          end
+
+          time_str = sq ? text.slice(idx, i - cursor).squeeze(SPLIT_CHAR) : text.slice(cursor, i - cursor)
+          cursor = i + 1
+        end
+
+        # Repeat same code for the performance
+
+        # host part
+        i = text.index(SPLIT_CHAR, cursor)
+        unless i
+          yield nil, nil
+          return
+        end
+        slice_size = i - cursor
+        host = text.slice(cursor, slice_size)
+        cursor += slice_size + 1
+
+        # ident part
+        i = text.index(SPLIT_CHAR, cursor)
+        unless i
+          yield nil, nil
+          return
+        end
+        slice_size = i - cursor
+        ident = text.slice(cursor, slice_size)
+        cursor += slice_size + 1
+
+        # pid part
+        i = text.index(SPLIT_CHAR, cursor)
+        unless i
+          yield nil, nil
+          return
+        end
+        slice_size = i - cursor
+        pid = text.slice(cursor, slice_size)
+        cursor += slice_size + 1
+
+        # msgid part
+        i = text.index(SPLIT_CHAR, cursor)
+        unless i
+          yield nil, nil
+          return
+        end
+        slice_size = i - cursor
+        msgid = text.slice(cursor, slice_size)
+        cursor += slice_size + 1
+
+        record = {'host' => host, 'ident' => ident, 'pid' => pid, 'msgid' => msgid}
+        record['pri'] = pri if pri
+
+        # extradata part
+        ed_start = text[cursor]
+        if ed_start == NILVALUE
+          record['extradata'] = NILVALUE
+          cursor += 1
+        else
+          start = cursor
+          i = text.index('] '.freeze, cursor)
+          extradata = if i
+                        diff = i + 1 - start # calculate ']' position
+                        cursor += diff
+                        text.slice(start, diff)
+                      else  # No message part case
+                        cursor = text.bytesize
+                        text.slice(start, cursor)
+                      end
+          extradata.tr!("\\".freeze, ''.freeze)
+          record['extradata'] = extradata
+        end
+
+        # message part
+        if cursor != text.bytesize
+          msg = text.slice(cursor + 1, text.bytesize)
+          msg.chomp!
+          record['message'] = msg
+        end
+
+        time = begin
+                 @time_parser_rfc5424.parse(time_str)
+               rescue Fluent::TimeParser::TimeParseError => e
+                 @time_parser_rfc5424_without_subseconds.parse(time_str)
+               end
         record['time'] = time_str if @keep_time_key
 
         yield time, record