fluentd 1.19.3 → 1.20.0

data/lib/fluent/plugin/in_exec.rb

@@ -15,7 +15,7 @@
 #
 
 require 'fluent/plugin/input'
-require 'yajl'
+require 'json'
 
 module Fluent::Plugin
   class ExecInput < Fluent::Plugin::Input
@@ -43,6 +43,8 @@ module Fluent::Plugin
     config_param :tag, :string, default: nil
     desc 'The interval time between periodic program runs.'
     config_param :run_interval, :time, default: nil
+    desc 'Command (program) execution timeout.'
+    config_param :command_timeout, :time, default: nil
     desc 'The default block size to read if parser requires partial read.'
     config_param :read_block_size, :size, default: 10240 # 10k
     desc 'The encoding to receive the result of the command, especially for non-ascii characters.'
@@ -86,9 +88,9 @@ module Fluent::Plugin
       options[:external_encoding] = @encoding if @encoding
 
       if @run_interval
-        child_process_execute(:exec_input, @command, interval: @run_interval, **options, &method(:run))
+        child_process_execute(:exec_input, @command, interval: @run_interval, wait_timeout: @command_timeout, **options, &method(:run))
       else
-        child_process_execute(:exec_input, @command, immediate: true, **options, &method(:run))
+        child_process_execute(:exec_input, @command, immediate: true, wait_timeout: @command_timeout, **options, &method(:run))
       end
     end
 
@@ -115,7 +117,7 @@ module Fluent::Plugin
       time ||= extract_time_from_record(record) || Fluent::EventTime.now
       router.emit(tag, time, record)
     rescue => e
-      log.error "exec failed to emit", tag: tag, record: Yajl.dump(record), error: e
+      log.error "exec failed to emit", tag: tag, record: JSON.generate(record), error: e
       router.emit_error_event(tag, time, record, e) if tag && time && record
     end
   end

data/lib/fluent/plugin/in_forward.rb

@@ -15,9 +15,10 @@
 #
 
 
+require 'fluent/env'
 require 'fluent/plugin/input'
 require 'fluent/msgpack_factory'
-require 'yajl'
+require 'json'
 require 'digest'
 require 'securerandom'
 
@@ -31,6 +32,12 @@ module Fluent::Plugin
     helpers :server
 
     LISTEN_PORT = 24224
+    OPTION_ACK = 'ack'.freeze
+    OPTION_CHUNK = 'chunk'.freeze
+    OPTION_COMPRESSED = 'compressed'.freeze
+    OPTION_FLUENT_SIGNAL = 'fluent_signal'.freeze
+    OPTION_SIZE = 'size'.freeze
+    OPTION_TEXT = 'text'.freeze
 
     desc 'The port to listen to.'
     config_param :port, :integer, default: LISTEN_PORT
@@ -250,13 +257,15 @@ module Fluent::Plugin
         unless feeder
           first = data[0]
           if first == '{' || first == '[' # json
-            parser = Yajl::Parser.new
-            parser.on_parse_complete = ->(obj){
-              block.call(obj, bytes, serializer)
-              bytes = 0
-            }
+            parser = JSON::ResumableParser.new(**Fluent::DEFAULT_JSON_PARSE_OPTIONS)
             serializer = :to_json.to_proc
-            feeder = ->(d){ parser << d }
+            feeder = ->(d){
+              parser << d
+              while parser.parse
+                block.call(parser.value, bytes, serializer)
+                bytes = 0
+              end
+            }
           else # msgpack
             parser = Fluent::MessagePackFactory.msgpack_unpacker
             serializer = :to_msgpack.to_proc
@@ -275,8 +284,8 @@ module Fluent::Plugin
     end
 
     def response(option)
-      if option && option['chunk']
-        return { 'ack' => option['chunk'] }
+      if option && option[OPTION_CHUNK]
+        return { OPTION_ACK => option[OPTION_CHUNK] }
       end
       nil
     end
@@ -310,14 +319,19 @@ module Fluent::Plugin
       when String
         # PackedForward
         option = msg[2] || {}
-        size = option['size'] || 0
+        size = option[OPTION_SIZE] || 0
+        compressed = option[OPTION_COMPRESSED]
 
-        if option['compressed'] && option['compressed'] != 'text'
-          es = Fluent::CompressedMessagePackEventStream.new(entries, nil, size.to_i, compress: option['compressed'].to_sym, decompression_size_limit: @decompression_size_limit)
+        if compressed && compressed != OPTION_TEXT
+          es = Fluent::CompressedMessagePackEventStream.new(entries, nil, size.to_i, compress: compressed.to_sym, decompression_size_limit: @decompression_size_limit)
         else
           es = Fluent::MessagePackEventStream.new(entries, nil, size.to_i)
         end
-        es = check_and_skip_invalid_event(tag, es, conn.remote_host) if @skip_invalid_event
+        if @skip_invalid_event
+          es = normalize_event_stream(tag, es, conn.remote_host)
+        elsif option[OPTION_FLUENT_SIGNAL] == 0
+          es = Fluent::MetadataTimeEventStream.new(es)
+        end
         if @enable_field_injection
           es = add_source_info(es, conn)
         end
@@ -325,19 +339,7 @@ module Fluent::Plugin
 
       when Array
         # Forward
-        es = if @skip_invalid_event
-               check_and_skip_invalid_event(tag, entries, conn.remote_host)
-             else
-               es = Fluent::MultiEventStream.new
-               entries.each { |e|
-                 record = e[1]
-                 next if record.nil?
-                 time = e[0]
-                 time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
-                 es.add(time, record)
-               }
-               es
-             end
+        es = normalize_event_stream(tag, entries, conn.remote_host)
         if @enable_field_injection
           es = add_source_info(es, conn)
         end
@@ -346,7 +348,7 @@ module Fluent::Plugin
 
       else
         # Message
-        time = msg[1]
+        time = extract_event_time(msg[1])
         record = msg[2]
         if @skip_invalid_event && invalid_event?(tag, time, record)
           log.warn "got invalid event and drop it:", host: conn.remote_host, tag: tag, time: time, record: record
@@ -367,21 +369,29 @@ module Fluent::Plugin
     end
 
     def invalid_event?(tag, time, record)
+      time = extract_event_time(time)
       !((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))
     end
 
-    def check_and_skip_invalid_event(tag, es, remote_host)
+    def normalize_event_stream(tag, es, remote_host)
       new_es = Fluent::MultiEventStream.new
       es.each { |time, record|
-        if invalid_event?(tag, time, record)
+        time = extract_event_time(time)
+        if @skip_invalid_event && invalid_event?(tag, time, record)
           log.warn "skip invalid event:", host: remote_host, tag: tag, time: time, record: record
           next
         end
+        next if record.nil?
+        time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
         new_es.add(time, record)
       }
       new_es
     end
 
+    def extract_event_time(time)
+      time.is_a?(Array) ? time[0] : time
+    end
+
     def add_source_info(es, conn)
       new_es = Fluent::MultiEventStream.new
       if @source_address_key && @source_hostname_key

data/lib/fluent/plugin/in_http.rb

@@ -24,6 +24,8 @@ require 'webrick/httputils'
 require 'uri'
 require 'socket'
 require 'json'
+require 'ipaddr'
+require 'openssl'
 
 module Fluent::Plugin
   class InHttpParser < Parser
@@ -90,6 +92,22 @@ module Fluent::Plugin
     desc "Add prefix to incoming tag"
     config_param :add_tag_prefix, :string, default: nil
 
+    config_section :auth, required: false, multi: false do
+      desc 'The method for HTTP authentication'
+      config_param :method, :enum, list: [:basic], default: :basic
+      desc 'The username for basic authentication'
+      config_param :username, :string
+      desc 'The password for basic authentication'
+      config_param :password, :string, secret: true
+    end
+
+    config_section :security, required: false, multi: false do
+      config_section :client, param_name: :clients, required: false, multi: true do
+        desc 'Address or network of the client, IPv4 or IPv6. e.g. 192.168.1.10, 192.168.0.0/16, fd00::/8'
+        config_param :network, :string
+      end
+    end
+
     config_section :parse do
       config_set_default :@type, 'in_http'
     end
@@ -100,6 +118,7 @@ module Fluent::Plugin
       super
 
       @km = nil
+      @nodes = []
       @format_name = nil
       @parser_time_key = nil
 
@@ -127,6 +146,8 @@ module Fluent::Plugin
 
       raise Fluent::ConfigError, "'add_tag_prefix' parameter must not be empty" if @add_tag_prefix && @add_tag_prefix.empty?
 
+      configure_security if @security
+
       m = if @parser_configs.first['@type'] == 'in_http'
             @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
             @parser_msgpack.time_key = nil
@@ -260,11 +281,33 @@ module Fluent::Plugin
 
     private
 
+    def configure_security
+      @nodes = @security.clients.map do |client|
+        begin
+          IPAddr.new(client.network)
+        rescue ArgumentError
+          raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
+        end
+      end
+    end
+
+    def allowed_client?(address)
+      return true if @nodes.empty?
+      @nodes.any? { |node| node.include?(address) rescue false }
+    end
+
     def on_server_connect(conn)
+      unless allowed_client?(conn.remote_addr)
+        log.warn "client address does not match any allowed network", address: conn.remote_addr
+        conn.data {|_data| }
+        conn.close
+        return
+      end
+
       handler = Handler.new(conn, @km, method(:on_request),
                             @body_size_limit, @decompression_size_limit, @format_name, log,
                             @cors_allow_origins, @cors_allow_credentials,
-                            @add_query_params)
+                            @add_query_params, @auth)
 
       conn.on(:data) do |data|
         handler.on_read(data)
@@ -347,7 +390,7 @@ module Fluent::Plugin
       attr_reader :content_type
 
       def initialize(io, km, callback, body_size_limit, decompression_size_limit, format_name, log,
-                     cors_allow_origins, cors_allow_credentials, add_query_params)
+                     cors_allow_origins, cors_allow_credentials, add_query_params, auth = nil)
         @io = io
         @km = km
         @callback = callback
@@ -360,6 +403,8 @@ module Fluent::Plugin
         @cors_allow_credentials = cors_allow_credentials
         @idle = 0
         @add_query_params = add_query_params
+        @auth = auth
+        @authorization = nil
         @km.add(self)
 
         @remote_port, @remote_addr = io.remote_port, io.remote_addr
@@ -399,6 +444,7 @@ module Fluent::Plugin
         @env = {}
         @content_type = ""
         @content_encoding = ""
+        @authorization = nil
         headers.each_pair {|k,v|
           @env["HTTP_#{k.tr('-','_').upcase}"] = v
           case k
@@ -426,8 +472,22 @@ module Fluent::Plugin
             @access_control_request_method = v
           when /\AAccess-Control-Request-Headers\Z/i
             @access_control_request_headers = v
+          when /\AAuthorization\Z/i
+            @authorization = v.is_a?(Array) ? v.first : v
           end
         }
+
+        if @auth
+          # Never let the credential reach a record built by add_http_headers.
+          @env.delete("HTTP_AUTHORIZATION")
+
+          unless preflight_request? || authenticate
+            @log.warn "authentication failed", address: @remote_addr
+            send_response_and_close(RES_401_STATUS, {'WWW-Authenticate' => AUTH_CHALLENGE}, "")
+            return
+          end
+        end
+
         if expect
           if expect == '100-continue'.freeze
             if !size || size < @body_size_limit
@@ -442,6 +502,8 @@ module Fluent::Plugin
       end
 
       def on_body(chunk)
+        return if closing?
+
         if @body.bytesize + chunk.bytesize > @body_size_limit
           unless closing?
             send_response_and_close("413 Request Entity Too Large", {}, "Too large")
@@ -452,6 +514,8 @@ module Fluent::Plugin
       end
 
       RES_200_STATUS = "200 OK".freeze
+      RES_401_STATUS = "401 Unauthorized".freeze
+      AUTH_CHALLENGE = 'Basic realm="fluentd"'.freeze
       RES_403_STATUS = "403 Forbidden".freeze
 
       # Azure App Service sends GET requests for health checking purpose.
@@ -651,6 +715,23 @@ module Fluent::Plugin
       def parse_query(query)
         query.nil? ? {} : Hash[URI.decode_www_form(query, Encoding::ASCII_8BIT)]
       end
+
+      # Browsers never attach Authorization to a CORS preflight.
+      def preflight_request?
+        @parser.http_method == 'OPTIONS'.freeze
+      end
+
+      def authenticate
+        scheme, credentials = @authorization.to_s.split(' ', 2)
+        return false unless scheme == 'Basic'.freeze
+
+        username, password = credentials.to_s.unpack1('m').to_s.split(':', 2)
+        username_matched = OpenSSL.secure_compare(username.to_s, @auth.username)
+        password_matched = OpenSSL.secure_compare(password.to_s, @auth.password)
+        username_matched && password_matched
+      rescue ArgumentError
+        false
+      end
     end
   end
 end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -16,6 +16,7 @@
 
 require 'json'
 
+require 'fluent/env'
 require 'fluent/config/types'
 require 'fluent/plugin/input'
 require 'fluent/plugin/output'
@@ -101,7 +102,7 @@ module Fluent::Plugin
       end
 
       def render_ltsv(obj, code: 200)
-        normalized = JSON.parse(obj.to_json)
+        normalized = JSON.parse(obj.to_json, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
         text = ''
         normalized.each do |hash|
           row = []

data/lib/fluent/plugin/in_object_space.rb

@@ -14,7 +14,7 @@
 #    limitations under the License.
 #
 
-require 'yajl'
+require 'json'
 
 require 'fluent/plugin/input'
 
@@ -86,7 +86,7 @@ module Fluent::Plugin
 
       router.emit(@tag, now, record)
     rescue => e
-      log.error "object space failed to emit", error: e, tag: @tag, record: Yajl.dump(record)
+      log.error "object space failed to emit", error: e, tag: @tag, record: JSON.generate(record)
       log.error_backtrace
     end
   end

data/lib/fluent/plugin/in_sample.rb

@@ -16,6 +16,7 @@
 
 require 'json'
 
+require 'fluent/env'
 require 'fluent/plugin/input'
 require 'fluent/config/error'
 
@@ -44,7 +45,7 @@ module Fluent::Plugin
     desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
     config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
       begin
-        parsed = JSON.parse(val)
+        parsed = JSON.parse(val, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
       rescue JSON::ParserError => ex
         # Fluent::ConfigParseError, "got incomplete JSON" will be raised
         # at literal_parser.rb with --use-v1-config, but I had to

data/lib/fluent/plugin/in_syslog.rb

@@ -18,7 +18,7 @@ require 'fluent/plugin/input'
 require 'fluent/config/error'
 require 'fluent/plugin/parser'
 
-require 'yajl'
+require 'json'
 
 module Fluent::Plugin
   class SyslogInput < Input
@@ -98,8 +98,8 @@ module Fluent::Plugin
     desc 'The field name of the facility.'
     config_param :facility_key, :string, default: nil
 
-    desc "The max bytes of message"
-    config_param :message_length_limit, :size, default: 2048
+    desc "The max bytes of message. The messages exceeding this limit are dropped (UDP, TCP/TLS with traditional frame_type) or the connection is closed (TCP/TLS with octet_count frame_type)."
+    config_param :message_length_limit, :size, default: 8192
 
     config_param :blocking_timeout, :time, default: 0.5
 
@@ -190,29 +190,95 @@ module Fluent::Plugin
         resolve_name: @resolve_hostname,
         send_keepalive_packet: @send_keepalive_packet
       ) do |conn|
+        discard_till_next_delimiter = false
         conn.data do |data|
           buffer = conn.buffer
           buffer << data
           pos = 0
           if octet_count_frame
+            # Once an octet-counted stream contains an untrustworthy frame header, the stream
+            # can't be resynchronized. So close the connection instead of skipping bytes.
+            close_connection = false
             while idx = buffer.index(delimiter, pos)
-              num = Integer(buffer[pos..idx])
+              begin
+                num = Integer(buffer[pos..idx])
+              rescue ArgumentError
+                log.info "The message length is not a number, closing connection"
+                log.on_debug { log.debug "The message length is not a number:", head: buffer[pos, 32].dump }
+                close_connection = true
+                break
+              end
+
+              if num < 0
+                log.info "The message length is negative, closing connection:", size: num
+                close_connection = true
+                break
+              end
+
+              if num > @message_length_limit
+                log.info "The message length exceeds 'message_length_limit', closing connection:", limit: @message_length_limit, size: num
+                close_connection = true
+                break
+              end
+
               msg = buffer[idx + delimiter_size, num]
               if msg.size != num
+                # The frame is not fully received yet.
                 break
               end
 
               pos = idx + delimiter_size + num
               message_handler(msg, conn)
             end
+            buffer.slice!(0, pos) if pos > 0
+
+            # If the remaining buffer has no delimiter, no message length can be parsed from it.
+            # Since a valid frame header is much shorter than the limit, such a buffer never
+            # becomes a valid frame. (Note that a partially received valid frame always has a
+            # delimiter after its length, so this doesn't drop it even if it slightly exceeds
+            # the limit with its length header.)
+            if !close_connection && !buffer.index(delimiter) && buffer.bytesize > @message_length_limit
+              log.info "The buffer size exceeds 'message_length_limit', closing connection:", limit: @message_length_limit, size: buffer.bytesize
+              log.on_debug { log.debug "The buffer size exceeds 'message_length_limit':", head: buffer[...32] }
+              close_connection = true
+            end
+
+            if close_connection
+              buffer.clear
+              conn.close
+              next
+            end
           else
             while idx = buffer.index(delimiter, pos)
               msg = buffer[pos...idx]
               pos = idx + delimiter_size
+
+              if discard_till_next_delimiter
+                discard_till_next_delimiter = false
+                next
+              end
+
+              if msg.bytesize > @message_length_limit
+                log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize
+                log.on_debug { log.debug "The received data is larger than 'message_length_limit':", head: msg[...32] }
+                next
+              end
+
               message_handler(msg, conn)
             end
+            buffer.slice!(0, pos) if pos > 0
+
+            # If the buffer size exceeds the limit here, it means that the next message will
+            # definitely exceed the limit. So we should clear the buffer here. Otherwise, it will
+            # keep storing useless data until the next delimiter comes.
+            if buffer.bytesize > @message_length_limit
+              log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buffer.bytesize
+              log.on_debug { log.debug "The buffer size exceeds 'message_length_limit':", head: buffer[...32] }
+              buffer.clear
+              # We should discard the subsequent data until the next delimiter comes.
+              discard_till_next_delimiter = true
+            end
           end
-          buffer.slice!(0, pos) if pos > 0
         end
       end
     end
@@ -274,7 +340,7 @@ module Fluent::Plugin
     def emit(tag, time, record)
       router.emit(tag, time, record)
     rescue => e
-      log.error "syslog failed to emit", error: e, tag: tag, record: Yajl.dump(record)
+      log.error "syslog failed to emit", error: e, tag: tag, record: JSON.generate(record)
     end
   end
 end

data/lib/fluent/plugin/in_tail/compatibility.rb

@@ -0,0 +1,111 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'fluent/clock'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    module Compatibility
+      # Keep these methods as compatibility entry points for plugins which
+      # inherit TailInput and override line processing methods. TailInput is a
+      # built-in plugin, not a public inheritance API, and these methods may be
+      # removed in a future release.
+      def flush_buffer(tw, buf)
+        @line_feeder.flush_buffer(tw, buf)
+      end
+
+      # Feeds lines through the per-file feed, or through LineFeeder for watchers
+      # created by plugins using the deprecated LineBufferTimerFlusher.
+      #
+      # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
+      def receive_lines(lines, tail_watcher)
+        return async_receive_lines(lines, tail_watcher) if @worker_pool
+
+        file_feed = tail_watcher.file_feed
+        return file_feed.feed_lines(lines, tail_watcher) if file_feed.respond_to?(:feed_lines)
+
+        @line_feeder.feed_lines(lines, tail_watcher)
+      end
+
+      def convert_line_to_event(line, es, tail_watcher)
+        @line_feeder.convert_line_to_event(line, es, tail_watcher)
+      end
+
+      def parse_singleline(lines, tail_watcher)
+        @line_feeder.parse_singleline(lines, tail_watcher)
+      end
+
+      # TailInput's default #parse_multilines path starts the flush timer before
+      # parsing. An override that skips super must call
+      # tail_watcher.line_buffer_timer_flusher.reset_timer when needed.
+      def parse_multilines(lines, tail_watcher)
+        @line_feeder.parse_multilines(lines, tail_watcher)
+      end
+    end
+
+    class TailWatcher
+      # The object which keeps the line buffer of the multiline mode of the file:
+      # the FileFeed built by LineFeeder#new_file_feed, or the deprecated
+      # LineBufferTimerFlusher which a plugin overriding TailInput#setup_watcher
+      # passes to keep working without changes.
+      def line_buffer_timer_flusher
+        @file_feed
+      end
+
+      # Kept for compatibility with the plugins which build it by themselves and
+      # pass it to TailWatcher in the overridden TailInput#setup_watcher.
+      # LineFeeder#feed_lines feeds the lines of a watcher which keeps it, and
+      # LineFeeder::FileFeed built by LineFeeder#new_file_feed replaces it.
+      class LineBufferTimerFlusher
+        attr_accessor :line_buffer
+
+        def initialize(log, flush_interval, &flush_method)
+          @log = log
+          @flush_interval = flush_interval
+          @flush_method = flush_method
+          @start = nil
+          @line_buffer = nil
+        end
+
+        def on_notify(tw)
+          unless @start && @flush_method
+            return
+          end
+
+          if Fluent::Clock.now - @start >= @flush_interval
+            @flush_method.call(tw, @line_buffer) if @line_buffer
+            @line_buffer = nil
+            @start = nil
+          end
+        end
+
+        def close(tw)
+          return unless @line_buffer
+
+          @flush_method.call(tw, @line_buffer)
+          @line_buffer = nil
+        end
+
+        def reset_timer
+          return unless @flush_interval
+
+          @start = Fluent::Clock.now
+        end
+      end
+    end
+  end
+end