fluentd 1.18.0 → 1.19.0

data/lib/fluent/plugin/compressable.rb

@@ -16,29 +16,35 @@
 
 require 'stringio'
 require 'zlib'
+require 'zstd-ruby'
 
 module Fluent
   module Plugin
     module Compressable
-      def compress(data, **kwargs)
+      def compress(data, type: :gzip, **kwargs)
         output_io = kwargs[:output_io]
         io = output_io || StringIO.new
-        Zlib::GzipWriter.wrap(io) do |gz|
-          gz.write data
+        if type == :gzip
+          writer = Zlib::GzipWriter.new(io)
+        elsif type == :zstd
+          writer = Zstd::StreamWriter.new(io)
+        else
+          raise ArgumentError, "Unknown compression type: #{type}"
         end
-
+        writer.write(data)
+        writer.finish
         output_io || io.string
       end
 
       # compressed_data is String like `compress(data1) + compress(data2) + ... + compress(dataN)`
       # https://www.ruby-forum.com/topic/971591#979503
-      def decompress(compressed_data = nil, output_io: nil, input_io: nil)
+      def decompress(compressed_data = nil, output_io: nil, input_io: nil, type: :gzip)
         case
         when input_io && output_io
-          io_decompress(input_io, output_io)
+          io_decompress(input_io, output_io, type)
         when input_io
           output_io = StringIO.new
-          io = io_decompress(input_io, output_io)
+          io = io_decompress(input_io, output_io, type)
           io.string
         when compressed_data.nil? || compressed_data.empty?
           # check compressed_data(String) is 0 length
@@ -46,51 +52,91 @@ module Fluent
         when output_io
           # execute after checking compressed_data is empty or not
           io = StringIO.new(compressed_data)
-          io_decompress(io, output_io)
+          io_decompress(io, output_io, type)
         else
-          string_decompress(compressed_data)
+          string_decompress(compressed_data, type)
         end
       end
 
       private
 
-      def string_decompress(compressed_data)
+      def string_decompress_gzip(compressed_data)
         io = StringIO.new(compressed_data)
-
         out = ''
         loop do
-          gz = Zlib::GzipReader.new(io)
-          out << gz.read
-          unused = gz.unused
-          gz.finish
-
+          reader = Zlib::GzipReader.new(io)
+          out << reader.read
+          unused = reader.unused
+          reader.finish
           unless unused.nil?
             adjust = unused.length
             io.pos -= adjust
           end
           break if io.eof?
         end
+        out
+      end
 
+      def string_decompress_zstd(compressed_data)
+        io = StringIO.new(compressed_data)
+        reader = Zstd::StreamReader.new(io)
+        out = ''
+        loop do
+          # Zstd::StreamReader needs to specify the size of the buffer
+          out << reader.read(1024)
+          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
+          break if io.eof?
+        end
         out
       end
 
-      def io_decompress(input, output)
+      def string_decompress(compressed_data, type = :gzip)
+        if type == :gzip
+          string_decompress_gzip(compressed_data)
+        elsif type == :zstd
+          string_decompress_zstd(compressed_data)
+        else
+          raise ArgumentError, "Unknown compression type: #{type}"
+        end
+      end
+
+      def io_decompress_gzip(input, output)
         loop do
-          gz = Zlib::GzipReader.new(input)
-          v = gz.read
+          reader = Zlib::GzipReader.new(input)
+          v = reader.read
           output.write(v)
-          unused = gz.unused
-          gz.finish
-
+          unused = reader.unused
+          reader.finish
           unless unused.nil?
             adjust = unused.length
             input.pos -= adjust
           end
           break if input.eof?
         end
+        output
+      end
 
+      def io_decompress_zstd(input, output)
+        reader = Zstd::StreamReader.new(input)
+        loop do
+          # Zstd::StreamReader needs to specify the size of the buffer
+          v = reader.read(1024)
+          output.write(v)
+          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
+          break if input.eof?
+        end
         output
       end
+
+      def io_decompress(input, output, type = :gzip)
+        if type == :gzip
+          io_decompress_gzip(input, output)
+        elsif type == :zstd
+          io_decompress_zstd(input, output)
+        else
+          raise ArgumentError, "Unknown compression type: #{type}"
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/filter.rb

@@ -41,14 +41,6 @@ module Fluent
         @enable_size_metrics = false
       end
 
-      def emit_records
-        @emit_records_metrics.get
-      end
-
-      def emit_size
-        @emit_size_metrics.get
-      end
-
       def configure(conf)
         super
 

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -207,7 +207,7 @@ module Fluent::Plugin
             value.each do |k, v|
               placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
             end
-          else # string, interger, float, and others?
+          else # string, integer, float, and others?
             placeholders.store("${#{key}}", value)
           end
         end

data/lib/fluent/plugin/formatter_csv.rb

@@ -35,6 +35,22 @@ module Fluent
       config_param :fields, :array, value_type: :string
       config_param :add_newline, :bool, default: true
 
+      def csv_cacheable?
+        !!owner
+      end
+
+      def csv_thread_key
+        csv_cacheable? ? "#{owner.plugin_id}_csv_formatter_#{@usage}_csv" : nil
+      end
+
+      def csv_for_thread
+        if csv_cacheable?
+          Thread.current[csv_thread_key] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
+        else
+          CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
+        end
+      end
+
       def configure(conf)
         super
 
@@ -51,12 +67,10 @@ module Fluent
 
         @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes, headers: @fields,
                           row_sep: @add_newline ? :auto : "".force_encoding(Encoding::ASCII_8BIT)}
-        # Cache CSV object per thread to avoid internal state sharing
-        @cache = {}
       end
 
       def format(tag, time, record)
-        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
+        csv = csv_for_thread
         line = (csv << record).string.dup
         # Need manual cleanup because CSV writer doesn't provide such method.
         csv.rewind
@@ -65,7 +79,7 @@ module Fluent
       end
 
       def format_with_nested_fields(tag, time, record)
-        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
+        csv = csv_for_thread
         values = @accessors.map { |a| a.call(record) }
         line = (csv << values).string.dup
         # Need manual cleanup because CSV writer doesn't provide such method.

data/lib/fluent/plugin/formatter_json.rb

@@ -34,11 +34,11 @@ module Fluent
           if Fluent::OjOptions.available?
             @dump_proc = Oj.method(:dump)
           else
-            log.info "Oj isn't installed, fallback to Yajl as json parser"
-            @dump_proc = Yajl.method(:dump)
+            log.info "Oj isn't installed, fallback to JSON as json parser"
+            @dump_proc = JSON.method(:generate)
           end
         else
-          @dump_proc = Yajl.method(:dump)
+          @dump_proc = JSON.method(:generate)
         end
 
         # format json is used on various highload environment, so re-define method to skip if check
@@ -48,7 +48,10 @@ module Fluent
       end
 
       def format(tag, time, record)
-        "#{@dump_proc.call(record)}#{@newline}"
+        json_str = @dump_proc.call(record)
+        "#{json_str}#{@newline}"
+      ensure
+        json_str&.clear
       end
 
       def format_without_nl(tag, time, record)

data/lib/fluent/plugin/formatter_out_file.rb

@@ -16,7 +16,7 @@
 
 require 'fluent/plugin/formatter'
 require 'fluent/time'
-require 'yajl'
+require 'json'
 
 module Fluent
   module Plugin
@@ -43,10 +43,13 @@ module Fluent
       end
 
       def format(tag, time, record)
+        json_str = JSON.generate(record)
         header = ''
         header << "#{@timef.format(time)}#{@delimiter}" if @output_time
         header << "#{tag}#{@delimiter}" if @output_tag
-        "#{header}#{Yajl.dump(record)}#{@newline}"
+        "#{header}#{json_str}#{@newline}"
+      ensure
+        json_str&.clear
       end
     end
   end

data/lib/fluent/plugin/in_forward.rb

@@ -55,7 +55,7 @@ module Fluent::Plugin
     desc 'Received chunk is dropped if it is larger than this value.'
     config_param :chunk_size_limit, :size, default: nil
     desc 'Skip an event if incoming event is invalid.'
-    config_param :skip_invalid_event, :bool, default: false
+    config_param :skip_invalid_event, :bool, default: true
 
     desc "The field name of the client's source address."
     config_param :source_address_key, :string, default: nil
@@ -307,10 +307,14 @@ module Fluent::Plugin
       case entries
       when String
         # PackedForward
-        option = msg[2]
-        size = (option && option['size']) || 0
-        es_class = (option && option['compressed'] == 'gzip') ? Fluent::CompressedMessagePackEventStream : Fluent::MessagePackEventStream
-        es = es_class.new(entries, nil, size.to_i)
+        option = msg[2] || {}
+        size = option['size'] || 0
+
+        if option['compressed'] && option['compressed'] != 'text'
+          es = Fluent::CompressedMessagePackEventStream.new(entries, nil, size.to_i, compress: option['compressed'].to_sym)
+        else
+          es = Fluent::MessagePackEventStream.new(entries, nil, size.to_i)
+        end
         es = check_and_skip_invalid_event(tag, es, conn.remote_host) if @skip_invalid_event
         if @enable_field_injection
           es = add_source_info(es, conn)

data/lib/fluent/plugin/in_http.rb

@@ -504,8 +504,9 @@ module Fluent::Plugin
         # ==========
         # For every incoming request, we check if we have some CORS
         # restrictions and allow listed origins through @cors_allow_origins.
+        # If origin is empty, it's likely a server-to-server request and considered safe.
         unless @cors_allow_origins.nil?
-          unless @cors_allow_origins.include?('*') || include_cors_allow_origin
+          unless @cors_allow_origins.include?('*') || include_cors_allow_origin || @origin.nil?
             send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
             return
           end
@@ -530,12 +531,12 @@ module Fluent::Plugin
         @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
 
         uri = URI.parse(@parser.request_url)
-        params = WEBrick::HTTPUtils.parse_query(uri.query)
+        params = parse_query(uri.query)
 
         if @format_name != 'default'
           params[EVENT_RECORD_PARAMETER] = @body
         elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
-          params.update WEBrick::HTTPUtils.parse_query(@body)
+          params.update parse_query(@body)
         elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
           boundary = WEBrick::HTTPUtils.dequote($1)
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
@@ -552,7 +553,7 @@ module Fluent::Plugin
 
         if (@add_query_params)
 
-          query_params = WEBrick::HTTPUtils.parse_query(uri.query)
+          query_params = parse_query(uri.query)
 
           query_params.each_pair {|k,v|
             params["QUERY_#{k.tr('-','_').upcase}"] = v
@@ -642,6 +643,10 @@ module Fluent::Plugin
 
         !r.nil?
       end
+
+      def parse_query(query)
+        query.nil? ? {} : Hash[URI.decode_www_form(query, Encoding::ASCII_8BIT)]
+      end
     end
   end
 end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -15,8 +15,6 @@
 #
 
 require 'json'
-require 'webrick'
-require 'cgi'
 
 require 'fluent/config/types'
 require 'fluent/plugin/input'
@@ -151,9 +149,7 @@ module Fluent::Plugin
       def build_option(req)
         qs = Hash.new { |_, _| [] }
         # parse ?=query string
-        if req.query_string
-          qs.merge!(CGI.parse(req.query_string))
-        end
+        qs.merge!(req.query || {})
 
         # if ?debug=1 is set, set :with_debug_info for get_monitor_info
         # and :pretty_json for render_json_error
@@ -208,7 +204,7 @@ module Fluent::Plugin
     def start
       super
 
-      log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}"
+      log.debug { "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}" }
       api_handler = APIHandler.new(self)
       http_server_create_http_server(:in_monitor_http_server_helper, addr: @bind, port: @port, logger: log, default_app: NotFoundJson) do |serv|
         serv.get('/api/plugins') { |req| api_handler.plugins_ltsv(req) }
@@ -218,7 +214,7 @@ module Fluent::Plugin
       end
 
       if @tag
-        log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
+        log.debug { "tag parameter is specified. Emit plugins info to '#{@tag}'" }
 
         opts = {with_config: false, with_retry: false}
         timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
@@ -254,7 +250,7 @@ module Fluent::Plugin
       array.concat Fluent::Engine.root_agent.filters
 
       Fluent::Engine.root_agent.labels.each { |name, l|
-        # TODO: Add label name to outputs / filters for identifing plugins
+        # TODO: Add label name to outputs / filters for identifying plugins
         array.concat l.outputs
         array.concat l.filters
       }

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -91,7 +91,7 @@ module Fluent::Plugin
         @map = map
       end
 
-      # This method is similer to #compact but it tries to get less lock to avoid a lock contention
+      # This method is similar to #compact but it tries to get less lock to avoid a lock contention
       def try_compact
         last_modified = nil
         size = nil