fluentd 1.19.2 → 1.19.4

data/lib/fluent/plugin/extractor.rb

@@ -0,0 +1,132 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'stringio'
+require 'zlib'
+require 'zstd-ruby'
+require 'fluent/error'
+
+module Fluent
+  module Plugin
+    module Extractor
+      class SizeLimitError < UnrecoverableError; end
+
+      BYTES_TO_READ = 64 * 1024
+      INFLATE_BYTES_TO_READ = 1024
+      # Zstd::StreamReader#read takes a number of compressed bytes, unlike
+      # Zlib::GzipReader#read which takes decompressed bytes
+      ZSTD_BYTES_TO_READ = 1024
+
+      def self.decompress_gzip(compressed_data, limit:)
+        io = StringIO.new(compressed_data)
+        out = ''
+        loop do
+          reader = Zlib::GzipReader.new(io)
+          while (chunk = reader.read(BYTES_TO_READ))
+            out << chunk
+            if out.bytesize > limit
+              raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+            end
+          end
+
+          unused = reader.unused
+          reader.finish
+          unless unused.nil?
+            adjust = unused.length
+            io.pos -= adjust
+          end
+          break if io.eof?
+        end
+        out
+      end
+
+      def self.decompress_zstd(compressed_data, limit:)
+        io = StringIO.new(compressed_data)
+        reader = Zstd::StreamReader.new(io)
+        out = ''
+        loop do
+          out << reader.read(ZSTD_BYTES_TO_READ)
+          if out.bytesize > limit
+            raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+          end
+
+          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
+          break if io.eof?
+        end
+        out
+      end
+
+      def self.decompress_deflate(compressed_data, limit:)
+        io = StringIO.new(compressed_data)
+        out = ''
+        begin
+          zstream = Zlib::Inflate.new
+          while (chunk = io.read(INFLATE_BYTES_TO_READ))
+            out << zstream.inflate(chunk)
+            if out.bytesize > limit
+              raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+            end
+          end
+          out << zstream.finish
+          if out.bytesize > limit
+            raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+          end
+        ensure
+          zstream&.close
+        end
+        out
+      end
+
+      def self.io_decompress_gzip(input, output, limit:)
+        size = 0
+        loop do
+          reader = Zlib::GzipReader.new(input)
+          while (chunk = reader.read(BYTES_TO_READ))
+            size += chunk.bytesize
+            if size > limit
+              raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+            end
+            output.write(chunk)
+          end
+          unused = reader.unused
+          reader.finish
+          unless unused.nil?
+            adjust = unused.length
+            input.pos -= adjust
+          end
+          break if input.eof?
+        end
+        output
+      end
+
+      def self.io_decompress_zstd(input, output, limit:)
+        reader = Zstd::StreamReader.new(input)
+        size = 0
+        loop do
+          chunk = reader.read(ZSTD_BYTES_TO_READ)
+          size += chunk.bytesize
+          if size > limit
+            raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+          end
+          output.write(chunk)
+          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
+          break if input.eof?
+        end
+        output
+      end
+    end
+  end
+end

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -18,6 +18,7 @@ require 'socket'
 require 'json'
 require 'ostruct'
 
+require 'fluent/env'
 require 'fluent/plugin/filter'
 require 'fluent/config/error'
 require 'fluent/event'
@@ -116,7 +117,7 @@ module Fluent::Plugin
 
     def parse_value(value_str)
       if value_str.start_with?('{', '[')
-        JSON.parse(value_str)
+        JSON.parse(value_str, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
       else
         value_str
       end

data/lib/fluent/plugin/in_debug_agent.rb

@@ -26,7 +26,7 @@ module Fluent::Plugin
       super
     end
 
-    config_param :bind, :string, default: '0.0.0.0'
+    config_param :bind, :string, default: '127.0.0.1'
     config_param :port, :integer, default: 24230
     config_param :unix_path, :string, default: nil
     #config_param :unix_mode  # TODO

data/lib/fluent/plugin/in_forward.rb

@@ -54,6 +54,8 @@ module Fluent::Plugin
     config_param :chunk_size_warn_limit, :size, default: nil
     desc 'Received chunk is dropped if it is larger than this value.'
     config_param :chunk_size_limit, :size, default: nil
+    desc 'The size limit of the decompressed element.'
+    config_param :decompression_size_limit, :size, default: 256*1024*1024
     desc 'Skip an event if incoming event is invalid.'
     config_param :skip_invalid_event, :bool, default: true
 
@@ -311,7 +313,7 @@ module Fluent::Plugin
         size = option['size'] || 0
 
         if option['compressed'] && option['compressed'] != 'text'
-          es = Fluent::CompressedMessagePackEventStream.new(entries, nil, size.to_i, compress: option['compressed'].to_sym)
+          es = Fluent::CompressedMessagePackEventStream.new(entries, nil, size.to_i, compress: option['compressed'].to_sym, decompression_size_limit: @decompression_size_limit)
         else
           es = Fluent::MessagePackEventStream.new(entries, nil, size.to_i)
         end

data/lib/fluent/plugin/in_http.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/plugin/extractor'
 require 'fluent/plugin/input'
 require 'fluent/plugin/parser'
 require 'fluent/event'
@@ -23,6 +24,8 @@ require 'webrick/httputils'
 require 'uri'
 require 'socket'
 require 'json'
+require 'ipaddr'
+require 'openssl'
 
 module Fluent::Plugin
   class InHttpParser < Parser
@@ -64,6 +67,8 @@ module Fluent::Plugin
     config_param :bind, :string, default: '0.0.0.0'
     desc 'The size limit of the POSTed element. Default is 32MB.'
     config_param :body_size_limit, :size, default: 32*1024*1024  # TODO default
+    desc 'The size limit of the decompressed element.'
+    config_param :decompression_size_limit, :size, default: 256*1024*1024  # TODO default
     desc 'The timeout limit for keeping the connection alive.'
     config_param :keepalive_timeout, :time, default: 10   # TODO default
     config_param :backlog, :integer, default: nil
@@ -87,6 +92,22 @@ module Fluent::Plugin
     desc "Add prefix to incoming tag"
     config_param :add_tag_prefix, :string, default: nil
 
+    config_section :auth, required: false, multi: false do
+      desc 'The method for HTTP authentication'
+      config_param :method, :enum, list: [:basic], default: :basic
+      desc 'The username for basic authentication'
+      config_param :username, :string
+      desc 'The password for basic authentication'
+      config_param :password, :string, secret: true
+    end
+
+    config_section :security, required: false, multi: false do
+      config_section :client, param_name: :clients, required: false, multi: true do
+        desc 'Address or network of the client, IPv4 or IPv6. e.g. 192.168.1.10, 192.168.0.0/16, fd00::/8'
+        config_param :network, :string
+      end
+    end
+
     config_section :parse do
       config_set_default :@type, 'in_http'
     end
@@ -97,6 +118,7 @@ module Fluent::Plugin
       super
 
       @km = nil
+      @nodes = []
       @format_name = nil
       @parser_time_key = nil
 
@@ -124,6 +146,8 @@ module Fluent::Plugin
 
       raise Fluent::ConfigError, "'add_tag_prefix' parameter must not be empty" if @add_tag_prefix && @add_tag_prefix.empty?
 
+      configure_security if @security
+
       m = if @parser_configs.first['@type'] == 'in_http'
             @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
             @parser_msgpack.time_key = nil
@@ -257,11 +281,33 @@ module Fluent::Plugin
 
     private
 
+    def configure_security
+      @nodes = @security.clients.map do |client|
+        begin
+          IPAddr.new(client.network)
+        rescue ArgumentError
+          raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
+        end
+      end
+    end
+
+    def allowed_client?(address)
+      return true if @nodes.empty?
+      @nodes.any? { |node| node.include?(address) rescue false }
+    end
+
     def on_server_connect(conn)
+      unless allowed_client?(conn.remote_addr)
+        log.warn "client address does not match any allowed network", address: conn.remote_addr
+        conn.data {|_data| }
+        conn.close
+        return
+      end
+
       handler = Handler.new(conn, @km, method(:on_request),
-                            @body_size_limit, @format_name, log,
+                            @body_size_limit, @decompression_size_limit, @format_name, log,
                             @cors_allow_origins, @cors_allow_credentials,
-                            @add_query_params)
+                            @add_query_params, @auth)
 
       conn.on(:data) do |data|
         handler.on_read(data)
@@ -343,12 +389,13 @@ module Fluent::Plugin
     class Handler
       attr_reader :content_type
 
-      def initialize(io, km, callback, body_size_limit, format_name, log,
-                     cors_allow_origins, cors_allow_credentials, add_query_params)
+      def initialize(io, km, callback, body_size_limit, decompression_size_limit, format_name, log,
+                     cors_allow_origins, cors_allow_credentials, add_query_params, auth = nil)
         @io = io
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
+        @decompression_size_limit = decompression_size_limit
         @next_close = false
         @format_name = format_name
         @log = log
@@ -356,6 +403,8 @@ module Fluent::Plugin
         @cors_allow_credentials = cors_allow_credentials
         @idle = 0
         @add_query_params = add_query_params
+        @auth = auth
+        @authorization = nil
         @km.add(self)
 
         @remote_port, @remote_addr = io.remote_port, io.remote_addr
@@ -395,6 +444,7 @@ module Fluent::Plugin
         @env = {}
         @content_type = ""
         @content_encoding = ""
+        @authorization = nil
         headers.each_pair {|k,v|
           @env["HTTP_#{k.tr('-','_').upcase}"] = v
           case k
@@ -422,8 +472,22 @@ module Fluent::Plugin
             @access_control_request_method = v
           when /\AAccess-Control-Request-Headers\Z/i
             @access_control_request_headers = v
+          when /\AAuthorization\Z/i
+            @authorization = v.is_a?(Array) ? v.first : v
           end
         }
+
+        if @auth
+          # Never let the credential reach a record built by add_http_headers.
+          @env.delete("HTTP_AUTHORIZATION")
+
+          unless preflight_request? || authenticate
+            @log.warn "authentication failed", address: @remote_addr
+            send_response_and_close(RES_401_STATUS, {'WWW-Authenticate' => AUTH_CHALLENGE}, "")
+            return
+          end
+        end
+
         if expect
           if expect == '100-continue'.freeze
             if !size || size < @body_size_limit
@@ -438,6 +502,8 @@ module Fluent::Plugin
       end
 
       def on_body(chunk)
+        return if closing?
+
         if @body.bytesize + chunk.bytesize > @body_size_limit
           unless closing?
             send_response_and_close("413 Request Entity Too Large", {}, "Too large")
@@ -448,6 +514,8 @@ module Fluent::Plugin
       end
 
       RES_200_STATUS = "200 OK".freeze
+      RES_401_STATUS = "401 Unauthorized".freeze
+      AUTH_CHALLENGE = 'Basic realm="fluentd"'.freeze
       RES_403_STATUS = "403 Forbidden".freeze
 
       # Azure App Service sends GET requests for health checking purpose.
@@ -518,9 +586,9 @@ module Fluent::Plugin
         # For now, we only support 'gzip' and 'deflate'.
         begin
           if @content_encoding == 'gzip'.freeze
-            @body = Zlib::GzipReader.new(StringIO.new(@body)).read
+            @body = Extractor.decompress_gzip(@body, limit: @decompression_size_limit)
           elsif @content_encoding == 'deflate'.freeze
-            @body = Zlib::Inflate.inflate(@body)
+            @body = Extractor.decompress_deflate(@body, limit: @decompression_size_limit)
           end
         rescue
           @log.warn 'fails to decode payload', error: $!.to_s
@@ -647,6 +715,23 @@ module Fluent::Plugin
       def parse_query(query)
         query.nil? ? {} : Hash[URI.decode_www_form(query, Encoding::ASCII_8BIT)]
       end
+
+      # Browsers never attach Authorization to a CORS preflight.
+      def preflight_request?
+        @parser.http_method == 'OPTIONS'.freeze
+      end
+
+      def authenticate
+        scheme, credentials = @authorization.to_s.split(' ', 2)
+        return false unless scheme == 'Basic'.freeze
+
+        username, password = credentials.to_s.unpack1('m').to_s.split(':', 2)
+        username_matched = OpenSSL.secure_compare(username.to_s, @auth.username)
+        password_matched = OpenSSL.secure_compare(password.to_s, @auth.password)
+        username_matched && password_matched
+      rescue ArgumentError
+        false
+      end
     end
   end
 end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -16,6 +16,7 @@
 
 require 'json'
 
+require 'fluent/env'
 require 'fluent/config/types'
 require 'fluent/plugin/input'
 require 'fluent/plugin/output'
@@ -37,9 +38,11 @@ module Fluent::Plugin
     desc 'Determine the rate to emit internal metrics as events.'
     config_param :emit_interval, :time, default: 60
     desc 'Determine whether to include the config information.'
-    config_param :include_config, :bool, default: true
+    config_param :include_config, :bool, default: false
     desc 'Determine whether to include the retry information.'
-    config_param :include_retry, :bool, default: true
+    config_param :include_retry, :bool, default: false
+    desc 'Determine whether to include the debug information.'
+    config_param :include_debug_info, :bool, default: false
 
     class APIHandler
       def initialize(agent)
@@ -99,7 +102,7 @@ module Fluent::Plugin
       end
 
       def render_ltsv(obj, code: 200)
-        normalized = JSON.parse(obj.to_json)
+        normalized = JSON.parse(obj.to_json, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
         text = ''
         normalized.each do |hash|
           row = []
@@ -151,28 +154,23 @@ module Fluent::Plugin
         # parse ?=query string
         qs.merge!(req.query || {})
 
-        # if ?debug=1 is set, set :with_debug_info for get_monitor_info
-        # and :pretty_json for render_json_error
-        opts = { query: qs }
-        if qs['debug'.freeze].first
-          opts[:with_debug_info] = true
-          opts[:pretty_json] = true
-        end
-
-        if ivars = qs['with_ivars'.freeze].first
-          opts[:ivars] = ivars.split(',')
-        end
+        opts = {
+          query: qs,
+          with_config: @agent.include_config,
+          with_retry: @agent.include_retry
+        }
 
-        if with_config = qs['with_config'.freeze].first
-          opts[:with_config] = Fluent::Config.bool_value(with_config)
-        else
-          opts[:with_config] = @agent.include_config
-        end
+        if @agent.include_debug_info
+          # if ?debug=1 is set, set :with_debug_info for get_monitor_info
+          # and :pretty_json for render_json_error
+          if qs['debug'.freeze].first
+            opts[:with_debug_info] = true
+            opts[:pretty_json] = true
+          end
 
-        if with_retry = qs['with_retry'.freeze].first
-          opts[:with_retry] = Fluent::Config.bool_value(with_retry)
-        else
-          opts[:with_retry] = @agent.include_retry
+          if ivars = qs['with_ivars'.freeze].first
+            opts[:ivars] = ivars.split(',')
+          end
         end
 
         opts

data/lib/fluent/plugin/in_sample.rb

@@ -16,6 +16,7 @@
 
 require 'json'
 
+require 'fluent/env'
 require 'fluent/plugin/input'
 require 'fluent/config/error'
 
@@ -44,7 +45,7 @@ module Fluent::Plugin
     desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
     config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
       begin
-        parsed = JSON.parse(val)
+        parsed = JSON.parse(val, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
       rescue JSON::ParserError => ex
         # Fluent::ConfigParseError, "got incomplete JSON" will be raised
         # at literal_parser.rb with --use-v1-config, but I had to

data/lib/fluent/plugin/in_syslog.rb

@@ -98,8 +98,8 @@ module Fluent::Plugin
     desc 'The field name of the facility.'
     config_param :facility_key, :string, default: nil
 
-    desc "The max bytes of message"
-    config_param :message_length_limit, :size, default: 2048
+    desc "The max bytes of message. The messages exceeding this limit are dropped (UDP, TCP/TLS with traditional frame_type) or the connection is closed (TCP/TLS with octet_count frame_type)."
+    config_param :message_length_limit, :size, default: 8192
 
     config_param :blocking_timeout, :time, default: 0.5
 
@@ -190,29 +190,95 @@ module Fluent::Plugin
         resolve_name: @resolve_hostname,
         send_keepalive_packet: @send_keepalive_packet
       ) do |conn|
+        discard_till_next_delimiter = false
         conn.data do |data|
           buffer = conn.buffer
           buffer << data
           pos = 0
           if octet_count_frame
+            # Once an octet-counted stream contains an untrustworthy frame header, the stream
+            # can't be resynchronized. So close the connection instead of skipping bytes.
+            close_connection = false
             while idx = buffer.index(delimiter, pos)
-              num = Integer(buffer[pos..idx])
+              begin
+                num = Integer(buffer[pos..idx])
+              rescue ArgumentError
+                log.info "The message length is not a number, closing connection"
+                log.on_debug { log.debug "The message length is not a number:", head: buffer[pos, 32].dump }
+                close_connection = true
+                break
+              end
+
+              if num < 0
+                log.info "The message length is negative, closing connection:", size: num
+                close_connection = true
+                break
+              end
+
+              if num > @message_length_limit
+                log.info "The message length exceeds 'message_length_limit', closing connection:", limit: @message_length_limit, size: num
+                close_connection = true
+                break
+              end
+
               msg = buffer[idx + delimiter_size, num]
               if msg.size != num
+                # The frame is not fully received yet.
                 break
               end
 
               pos = idx + delimiter_size + num
               message_handler(msg, conn)
             end
+            buffer.slice!(0, pos) if pos > 0
+
+            # If the remaining buffer has no delimiter, no message length can be parsed from it.
+            # Since a valid frame header is much shorter than the limit, such a buffer never
+            # becomes a valid frame. (Note that a partially received valid frame always has a
+            # delimiter after its length, so this doesn't drop it even if it slightly exceeds
+            # the limit with its length header.)
+            if !close_connection && !buffer.index(delimiter) && buffer.bytesize > @message_length_limit
+              log.info "The buffer size exceeds 'message_length_limit', closing connection:", limit: @message_length_limit, size: buffer.bytesize
+              log.on_debug { log.debug "The buffer size exceeds 'message_length_limit':", head: buffer[...32] }
+              close_connection = true
+            end
+
+            if close_connection
+              buffer.clear
+              conn.close
+              next
+            end
           else
             while idx = buffer.index(delimiter, pos)
               msg = buffer[pos...idx]
               pos = idx + delimiter_size
+
+              if discard_till_next_delimiter
+                discard_till_next_delimiter = false
+                next
+              end
+
+              if msg.bytesize > @message_length_limit
+                log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize
+                log.on_debug { log.debug "The received data is larger than 'message_length_limit':", head: msg[...32] }
+                next
+              end
+
               message_handler(msg, conn)
             end
+            buffer.slice!(0, pos) if pos > 0
+
+            # If the buffer size exceeds the limit here, it means that the next message will
+            # definitely exceed the limit. So we should clear the buffer here. Otherwise, it will
+            # keep storing useless data until the next delimiter comes.
+            if buffer.bytesize > @message_length_limit
+              log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buffer.bytesize
+              log.on_debug { log.debug "The buffer size exceeds 'message_length_limit':", head: buffer[...32] }
+              buffer.clear
+              # We should discard the subsequent data until the next delimiter comes.
+              discard_till_next_delimiter = true
+            end
           end
-          buffer.slice!(0, pos) if pos > 0
         end
       end
     end

data/lib/fluent/plugin/out_file.rb

@@ -117,7 +117,17 @@ module Fluent::Plugin
       configured_time_slice_format = conf['time_slice_format']
 
       if conf.elements(name: 'buffer').empty?
+        # no <buffer> section, default time chunk key and timekey (1d) will be used.
+        log.warn "default timekey interval (1d) will be used because of missing <buffer> section. To change the output frequency, please modify the timekey value"
+
         conf.add_element('buffer', 'time')
+      else
+        unless conf.elements(name: 'buffer').first.has_key?('timekey')
+          if conf.elements(name: 'buffer').first.arg != "[]"
+            # with <buffer> section (except <buffer []>), and no timekey
+            log.warn "default timekey interval (1d) will be used. To change the output frequency, please modify the timekey value"
+          end
+        end
       end
       buffer_conf = conf.elements(name: 'buffer').first
       # Fluent::PluginId#configure is not called yet, so we can't use #plugin_root_dir here.

data/lib/fluent/plugin/out_forward/connection_manager.rb

@@ -78,6 +78,24 @@ module Fluent::Plugin
         end
       end
 
+      # Drop a socket instead of returning it to the keepalive reuse pool.
+      #
+      # Used when a chunk's ack timed out or came back with a mismatched chunk
+      # id: such a connection may still carry an in-flight/stale ack for a
+      # previous chunk. Reusing it would let that stale ack be read against a
+      # chunk sent on the reused socket, permanently offsetting every following
+      # ack and flooding the log with 'ack in response and chunk id in sent data
+      # are different' warnings (see #4057). Revoking removes it from the pool so
+      # the next chunk uses a fresh connection with a clean ack stream.
+      def discard(sock)
+        if @socket_cache
+          @socket_cache.revoke(sock)
+        else
+          sock.close_write rescue nil
+          sock.close rescue nil
+        end
+      end
+
       private
 
       def connect_keepalive(host:, port:, hostname:, ack: nil)