fluentd 1.17.1-x86-mingw32 → 1.19.0-x86-mingw32

data/lib/fluent/plugin/buffer/chunk.rb

@@ -59,8 +59,11 @@ module Fluent
           @size = 0
           @created_at = Fluent::Clock.real_now
           @modified_at = Fluent::Clock.real_now
-
-          extend Decompressable if compress == :gzip
+          if compress == :gzip
+            extend GzipDecompressable
+          elsif compress == :zstd
+            extend ZstdDecompressable
+          end
         end
 
         attr_reader :unique_id, :metadata, :state
@@ -85,10 +88,17 @@ module Fluent
 
         # data is array of formatted record string
         def append(data, **kwargs)
-          raise ArgumentError, '`compress: gzip` can be used for Compressable module' if kwargs[:compress] == :gzip
-          adding = ''.b
-          data.each do |d|
-            adding << d.b
+          raise ArgumentError, "`compress: #{kwargs[:compress]}` can be used for Compressable module" if kwargs[:compress] == :gzip || kwargs[:compress] == :zstd
+          begin
+            adding = data.join.force_encoding(Encoding::ASCII_8BIT)
+          rescue
+            # Fallback
+            # Array#join throws an exception if data contains strings with a different encoding.
+            # Although such cases may be rare, it should be considered as a safety precaution.
+            adding = ''.force_encoding(Encoding::ASCII_8BIT)
+            data.each do |d|
+              adding << d.b
+            end
           end
           concat(adding, data.size)
         end
@@ -165,23 +175,23 @@ module Fluent
         end
 
         def read(**kwargs)
-          raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
+          raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
           raise NotImplementedError, "Implement this method in child class"
         end
 
         def open(**kwargs, &block)
-          raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
+          raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
           raise NotImplementedError, "Implement this method in child class"
         end
 
         def write_to(io, **kwargs)
-          raise ArgumentError, '`compressed: gzip` can be used for Compressable module' if kwargs[:compressed] == :gzip
+          raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
           open do |i|
             IO.copy_stream(i, io)
           end
         end
 
-        module Decompressable
+        module GzipDecompressable
           include Fluent::Plugin::Compressable
 
           def append(data, **kwargs)
@@ -234,6 +244,60 @@ module Fluent
             end
           end
         end
+
+        module ZstdDecompressable
+          include Fluent::Plugin::Compressable
+
+          def append(data, **kwargs)
+            if kwargs[:compress] == :zstd
+              io = StringIO.new
+              stream = Zstd::StreamWriter.new(io)
+              data.each do |d|
+                stream.write(d)
+              end
+              stream.finish
+              concat(io.string, data.size)
+            else
+              super
+            end
+          end
+
+          def open(**kwargs, &block)
+            if kwargs[:compressed] == :zstd
+              super
+            else
+              super(**kwargs) do |chunk_io|
+                output_io = if chunk_io.is_a?(StringIO)
+                              StringIO.new
+                            else
+                              Tempfile.new('decompressed-data')
+                            end
+                output_io.binmode if output_io.is_a?(Tempfile)
+                decompress(input_io: chunk_io, output_io: output_io, type: :zstd)
+                output_io.seek(0, IO::SEEK_SET)
+                yield output_io
+              end
+            end
+          end
+
+          def read(**kwargs)
+            if kwargs[:compressed] == :zstd
+              super
+            else
+              decompress(super,type: :zstd)
+            end
+          end
+
+          def write_to(io, **kwargs)
+            open(compressed: :zstd) do |chunk_io|
+              if kwargs[:compressed] == :zstd
+                IO.copy_stream(chunk_io, io)
+              else
+                decompress(input_io: chunk_io, output_io: io, type: :zstd)
+              end
+            end
+          end
+        end
       end
     end
   end

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -37,7 +37,7 @@ module Fluent
         # path_prefix: path prefix string, ended with '.'
         # path_suffix: path suffix string, like '.log' (or any other user specified)
 
-        attr_reader :path, :permission
+        attr_reader :path, :meta_path, :permission
 
         def initialize(metadata, path, mode, perm: nil, compress: :text)
           super(metadata, compress: compress)
@@ -219,13 +219,17 @@ module Fluent
             # old type of restore
             data = Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
           end
+          raise FileChunkError, "invalid meta data" if data.nil? || !data.is_a?(Hash)
+          raise FileChunkError, "invalid unique_id" unless data[:id]
+          raise FileChunkError, "invalid created_at" unless data[:c].to_i > 0
+          raise FileChunkError, "invalid modified_at" unless data[:m].to_i > 0
 
           now = Fluent::Clock.real_now
 
-          @unique_id = data[:id] || self.class.unique_id_from_path(@path) || @unique_id
+          @unique_id = data[:id]
           @size = data[:s] || 0
-          @created_at = data.fetch(:c, now.to_i)
-          @modified_at = data.fetch(:m, now.to_i)
+          @created_at = data[:c]
+          @modified_at = data[:m]
 
           @metadata.timekey = data[:timekey]
           @metadata.tag = data[:tag]
@@ -285,7 +289,7 @@ module Fluent
             @chunk.binmode
           rescue => e
             # Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
-            # If other cases are possible, we will change erorr handling with proper classes.
+            # If other cases are possible, we will change error handling with proper classes.
             raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
           end
           begin

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -243,11 +243,11 @@ module Fluent
         def encode_key(metadata)
           k = @key ? metadata.variables[@key] : metadata.tag
           k ||= ''
-          URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
+          URI::RFC2396_PARSER.escape(k, ESCAPE_REGEXP)
         end
 
         def decode_key(key)
-          URI::DEFAULT_PARSER.unescape(key)
+          URI::RFC2396_PARSER.unescape(key)
         end
 
         def create_new_chunk(path, metadata, perm)
@@ -259,7 +259,7 @@ module Fluent
             @chunk.binmode
           rescue => e
             # Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
-            # If other cases are possible, we will change erorr handling with proper classes.
+            # If other cases are possible, we will change error handling with proper classes.
             raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
           end
 

data/lib/fluent/plugin/buffer/memory_chunk.rb

@@ -68,13 +68,13 @@ module Fluent
 
         def purge
           super
-          @chunk = ''.force_encoding("ASCII-8BIT")
+          @chunk.clear
           @chunk_bytes = @size = @adding_bytes = @adding_size = 0
           true
         end
 
         def read(**kwargs)
-          @chunk
+          @chunk.dup
         end
 
         def open(**kwargs, &block)

data/lib/fluent/plugin/buffer.rb

@@ -64,7 +64,7 @@ module Fluent
       config_param :queued_chunks_limit_size, :integer, default: nil
 
       desc 'Compress buffered data.'
-      config_param :compress, :enum, list: [:text, :gzip], default: :text
+      config_param :compress, :enum, list: [:text, :gzip, :zstd], default: :text
 
       desc 'If true, chunks are thrown away when unrecoverable error happens'
       config_param :disable_chunk_backup, :bool, default: false
@@ -196,6 +196,8 @@ module Fluent
         @mutex = Mutex.new
       end
 
+      # The metrics_create method defines getter methods named stage_byte_size and queue_byte_size.
+      # For compatibility, stage_size, stage_size=, queue_size, and queue_size= are still available.
       def stage_size
         @stage_size_metrics.get
       end
@@ -385,7 +387,7 @@ module Fluent
           end
 
           errors = []
-          # Buffer plugin estimates there's no serious error cause: will commit for all chunks eigher way
+          # Buffer plugin estimates there's no serious error cause: will commit for all chunks either way
           operated_chunks.each do |chunk|
             begin
               chunk.commit
@@ -523,7 +525,7 @@ module Fluent
           chunks = @stage.values
           chunks.concat(@queue)
           @timekeys = chunks.each_with_object({}) do |chunk, keys|
-            if chunk.metadata && chunk.metadata.timekey
+            if chunk.metadata&.timekey
               t = chunk.metadata.timekey
               keys[t] = keys.fetch(t, 0) + 1
             end
@@ -623,6 +625,7 @@ module Fluent
           until @queue.empty?
             begin
               q = @queue.shift
+              evacuate_chunk(q)
               log.trace("purging a chunk in queue"){ {id: dump_unique_id_hex(chunk.unique_id), bytesize: chunk.bytesize, size: chunk.size} }
               q.purge
             rescue => e
@@ -634,6 +637,25 @@ module Fluent
         end
       end
 
+      def evacuate_chunk(chunk)
+        # Overwrite this on demand.
+        #
+        # Note: Difference from the `backup` feature.
+        #       The `backup` feature is for unrecoverable errors, mainly for bad chunks.
+        #       On the other hand, this feature is for normal chunks.
+        #       The main motivation for this feature is to enable recovery by evacuating buffer files
+        #       when the retry limit is reached due to external factors such as network issues.
+        #
+        # Note: Difference from the `secondary` feature.
+        #       The `secondary` feature is not suitable for recovery.
+        #       It can be difficult to recover files made by `out_secondary_file` because the metadata
+        #       is lost.
+        #       For file buffers, the easiest way for recovery is to evacuate the chunk files as is.
+        #       Once the issue is recovered, we can put back the chunk files, and restart Fluentd to
+        #       load them.
+        #       This feature enables it.
+      end
+
       def chunk_size_over?(chunk)
         chunk.bytesize > @chunk_limit_size || (@chunk_limit_records && chunk.size > @chunk_limit_records)
       end
@@ -923,8 +945,6 @@ module Fluent
           return
         end
 
-        safe_owner_id = owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
-        backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
         backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_owner_id, "#{unique_id}.log")
         backup_dir = File.dirname(backup_file)
 
@@ -938,11 +958,19 @@ module Fluent
       def optimistic_queued?(metadata = nil)
         if metadata
           n = @queued_num[metadata]
-          n && n.nonzero?
+          n&.nonzero?
         else
           !@queue.empty?
         end
       end
+
+      def safe_owner_id
+        owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
+      end
+
+      def backup_base_dir
+        system_config.root_dir || DEFAULT_BACKUP_DIR
+      end
     end
   end
 end

data/lib/fluent/plugin/compressable.rb

@@ -16,29 +16,35 @@
 
 require 'stringio'
 require 'zlib'
+require 'zstd-ruby'
 
 module Fluent
   module Plugin
     module Compressable
-      def compress(data, **kwargs)
+      def compress(data, type: :gzip, **kwargs)
         output_io = kwargs[:output_io]
         io = output_io || StringIO.new
-        Zlib::GzipWriter.wrap(io) do |gz|
-          gz.write data
+        if type == :gzip
+          writer = Zlib::GzipWriter.new(io)
+        elsif type == :zstd
+          writer = Zstd::StreamWriter.new(io)
+        else
+          raise ArgumentError, "Unknown compression type: #{type}"
         end
-
+        writer.write(data)
+        writer.finish
         output_io || io.string
       end
 
       # compressed_data is String like `compress(data1) + compress(data2) + ... + compress(dataN)`
       # https://www.ruby-forum.com/topic/971591#979503
-      def decompress(compressed_data = nil, output_io: nil, input_io: nil)
+      def decompress(compressed_data = nil, output_io: nil, input_io: nil, type: :gzip)
         case
         when input_io && output_io
-          io_decompress(input_io, output_io)
+          io_decompress(input_io, output_io, type)
         when input_io
           output_io = StringIO.new
-          io = io_decompress(input_io, output_io)
+          io = io_decompress(input_io, output_io, type)
           io.string
         when compressed_data.nil? || compressed_data.empty?
           # check compressed_data(String) is 0 length
@@ -46,51 +52,91 @@ module Fluent
         when output_io
           # execute after checking compressed_data is empty or not
           io = StringIO.new(compressed_data)
-          io_decompress(io, output_io)
+          io_decompress(io, output_io, type)
         else
-          string_decompress(compressed_data)
+          string_decompress(compressed_data, type)
         end
       end
 
       private
 
-      def string_decompress(compressed_data)
+      def string_decompress_gzip(compressed_data)
         io = StringIO.new(compressed_data)
-
         out = ''
         loop do
-          gz = Zlib::GzipReader.new(io)
-          out << gz.read
-          unused = gz.unused
-          gz.finish
-
+          reader = Zlib::GzipReader.new(io)
+          out << reader.read
+          unused = reader.unused
+          reader.finish
           unless unused.nil?
             adjust = unused.length
             io.pos -= adjust
           end
           break if io.eof?
         end
+        out
+      end
 
+      def string_decompress_zstd(compressed_data)
+        io = StringIO.new(compressed_data)
+        reader = Zstd::StreamReader.new(io)
+        out = ''
+        loop do
+          # Zstd::StreamReader needs to specify the size of the buffer
+          out << reader.read(1024)
+          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
+          break if io.eof?
+        end
         out
       end
 
-      def io_decompress(input, output)
+      def string_decompress(compressed_data, type = :gzip)
+        if type == :gzip
+          string_decompress_gzip(compressed_data)
+        elsif type == :zstd
+          string_decompress_zstd(compressed_data)
+        else
+          raise ArgumentError, "Unknown compression type: #{type}"
+        end
+      end
+
+      def io_decompress_gzip(input, output)
         loop do
-          gz = Zlib::GzipReader.new(input)
-          v = gz.read
+          reader = Zlib::GzipReader.new(input)
+          v = reader.read
           output.write(v)
-          unused = gz.unused
-          gz.finish
-
+          unused = reader.unused
+          reader.finish
           unless unused.nil?
             adjust = unused.length
             input.pos -= adjust
           end
           break if input.eof?
         end
+        output
+      end
 
+      def io_decompress_zstd(input, output)
+        reader = Zstd::StreamReader.new(input)
+        loop do
+          # Zstd::StreamReader needs to specify the size of the buffer
+          v = reader.read(1024)
+          output.write(v)
+          # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
+          break if input.eof?
+        end
         output
       end
+
+      def io_decompress(input, output, type = :gzip)
+        if type == :gzip
+          io_decompress_gzip(input, output)
+        elsif type == :zstd
+          io_decompress_zstd(input, output)
+        else
+          raise ArgumentError, "Unknown compression type: #{type}"
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/filter.rb

@@ -41,14 +41,6 @@ module Fluent
         @enable_size_metrics = false
       end
 
-      def emit_records
-        @emit_records_metrics.get
-      end
-
-      def emit_size
-        @emit_size_metrics.get
-      end
-
       def configure(conf)
         super
 

data/lib/fluent/plugin/filter_parser.rb

@@ -54,29 +54,32 @@ module Fluent::Plugin
       @parser = parser_create
     end
 
-    FAILED_RESULT = [nil, nil].freeze # reduce allocation cost
     REPLACE_CHAR = '?'.freeze
 
-    def filter_with_time(tag, time, record)
-      raw_value = @accessor.call(record)
-      if raw_value.nil?
-        if @emit_invalid_record_to_error
-          router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
-        end
-        if @reserve_data
-          return time, handle_parsed(tag, record, time, {})
-        else
-          return FAILED_RESULT
+    def filter_stream(tag, es)
+      new_es = Fluent::MultiEventStream.new
+      es.each do |time, record|
+        begin
+          raw_value = @accessor.call(record)
+          if raw_value.nil?
+            new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data
+            raise ArgumentError, "#{@key_name} does not exist"
+          else
+            filter_one_record(tag, time, record, raw_value) do |result_time, result_record|
+              new_es.add(result_time, result_record)
+            end
+          end
+        rescue => e
+          router.emit_error_event(tag, time, record, e) if @emit_invalid_record_to_error
         end
       end
-      begin
-        # Note: https://github.com/fluent/fluentd/issues/4100
-        # If the parser returns multiple records from one raw_value,
-        # this returns only the first one record.
-        # This should be fixed in the future version.
-        result_time = nil
-        result_record = nil
+      new_es
+    end
+
+    private
 
+    def filter_one_record(tag, time, record, raw_value)
+      begin
         @parser.parse(raw_value) do |t, values|
           if values
             t = if @reserve_time
@@ -85,38 +88,17 @@ module Fluent::Plugin
                   t.nil? ? time : t
                 end
             @accessor.delete(record) if @remove_key_name_field
-            r = handle_parsed(tag, record, t, values)
-
-            if result_record.nil?
-              result_time = t
-              result_record = r
-            else
-              if @emit_invalid_record_to_error
-                router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
-                  "Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
-                ))
-              end
-            end
           else
-            if @emit_invalid_record_to_error
-              router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
-            end
-
+            router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'")) if @emit_invalid_record_to_error
             next unless @reserve_data
-            next unless result_record.nil?
-
-            result_time = time
-            result_record = handle_parsed(tag, record, time, {})
+            t = time
+            values = {}
           end
+          yield(t, handle_parsed(tag, record, t, values))
         end
 
-        return result_time, result_record
       rescue Fluent::Plugin::Parser::ParserError => e
-        if @emit_invalid_record_to_error
-          raise e
-        else
-          return FAILED_RESULT
-        end
+        raise e
       rescue ArgumentError => e
         raise unless @replace_invalid_sequence
         raise unless e.message.index("invalid byte sequence in") == 0
@@ -124,16 +106,10 @@ module Fluent::Plugin
         raw_value = raw_value.scrub(REPLACE_CHAR)
         retry
       rescue => e
-        if @emit_invalid_record_to_error
-          raise Fluent::Plugin::Parser::ParserError, "parse failed #{e.message}"
-        else
-          return FAILED_RESULT
-        end
+        raise Fluent::Plugin::Parser::ParserError, "parse failed #{e.message}"
       end
     end
 
-    private
-
     def handle_parsed(tag, record, t, values)
       if values && @inject_key_prefix
         values = Hash[values.map { |k, v| [@inject_key_prefix + k, v] }]

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -207,7 +207,7 @@ module Fluent::Plugin
             value.each do |k, v|
               placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
             end
-          else # string, interger, float, and others?
+          else # string, integer, float, and others?
             placeholders.store("${#{key}}", value)
           end
         end

data/lib/fluent/plugin/formatter_csv.rb

@@ -35,6 +35,22 @@ module Fluent
       config_param :fields, :array, value_type: :string
       config_param :add_newline, :bool, default: true
 
+      def csv_cacheable?
+        !!owner
+      end
+
+      def csv_thread_key
+        csv_cacheable? ? "#{owner.plugin_id}_csv_formatter_#{@usage}_csv" : nil
+      end
+
+      def csv_for_thread
+        if csv_cacheable?
+          Thread.current[csv_thread_key] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
+        else
+          CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts)
+        end
+      end
+
       def configure(conf)
         super
 
@@ -51,12 +67,10 @@ module Fluent
 
         @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes, headers: @fields,
                           row_sep: @add_newline ? :auto : "".force_encoding(Encoding::ASCII_8BIT)}
-        # Cache CSV object per thread to avoid internal state sharing
-        @cache = {}
       end
 
       def format(tag, time, record)
-        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
+        csv = csv_for_thread
         line = (csv << record).string.dup
         # Need manual cleanup because CSV writer doesn't provide such method.
         csv.rewind
@@ -65,7 +79,7 @@ module Fluent
       end
 
       def format_with_nested_fields(tag, time, record)
-        csv = (@cache[Thread.current] ||= CSV.new("".force_encoding(Encoding::ASCII_8BIT), **@generate_opts))
+        csv = csv_for_thread
         values = @accessors.map { |a| a.call(record) }
         line = (csv << values).string.dup
         # Need manual cleanup because CSV writer doesn't provide such method.

data/lib/fluent/plugin/formatter_json.rb

@@ -34,11 +34,11 @@ module Fluent
           if Fluent::OjOptions.available?
             @dump_proc = Oj.method(:dump)
           else
-            log.info "Oj isn't installed, fallback to Yajl as json parser"
-            @dump_proc = Yajl.method(:dump)
+            log.info "Oj isn't installed, fallback to JSON as json parser"
+            @dump_proc = JSON.method(:generate)
           end
         else
-          @dump_proc = Yajl.method(:dump)
+          @dump_proc = JSON.method(:generate)
         end
 
         # format json is used on various highload environment, so re-define method to skip if check
@@ -48,7 +48,10 @@ module Fluent
       end
 
       def format(tag, time, record)
-        "#{@dump_proc.call(record)}#{@newline}"
+        json_str = @dump_proc.call(record)
+        "#{json_str}#{@newline}"
+      ensure
+        json_str&.clear
       end
 
       def format_without_nl(tag, time, record)