fluentd 1.19.3 → 1.20.0

data/lib/fluent/log.rb

@@ -221,7 +221,7 @@ module Fluent
           if wid = get_worker_id(type)
             r['worker_id'] = wid
           end
-          Yajl.dump(r)
+          JSON.generate(r)
         }
       end
 
@@ -554,7 +554,7 @@ module Fluent
           Thread.current[:last_repeated_stacktrace] = CachedLog.new(backtrace, time) if @suppress_repeated_stacktrace
         end
 
-        puts Yajl.dump(r)
+        puts JSON.generate(r)
       end
 
       nil

data/lib/fluent/plugin/base.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'zlib'
 require 'fluent/plugin'
 require 'fluent/configurable'
 require 'fluent/system_config'
@@ -69,9 +70,13 @@ module Fluent
         true
       end
 
+      LOCK_FILE_BUCKETS = 65536
+
       def get_lock_path(name)
-        name = name.gsub(/[^a-zA-Z0-9]/, "_")
-        File.join(@fluentd_lock_dir, "fluentd-#{name}.lock")
+        # The mapping from a name to a bucket MUST be identical across worker processes.
+        # Ruby's String#hash is randomly seeded per process and MUST NOT be used here.
+        bucket = Zlib.crc32(name.to_s) % LOCK_FILE_BUCKETS
+        File.join(@fluentd_lock_dir, "fluentd-bucket-#{bucket}.lock")
       end
 
       def acquire_worker_lock(name)

data/lib/fluent/plugin/buf_file.rb

@@ -163,7 +163,7 @@ module Fluent
           end
 
           begin
-            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
+            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress, decompression_size_limit: @decompression_size_limit) # file chunk resumes contents of metadata
           rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
             exist_broken_file = true
             handle_broken_files(path, mode, e)
@@ -174,7 +174,7 @@ module Fluent
           when :staged
             # unstaged chunk created at Buffer#write_step_by_step is identified as the staged chunk here because FileChunk#assume_chunk_state checks only the file name.
             # https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L663
-            # This case can happen when fluentd process is killed by signal or other reasons between creating unstaged chunks and changing them to staged mode in Buffer#write
+            # This case can happen when fluentd process is killed by signal or other reasons between creating unstaged chunks and enqueueing them (which renames them) in Buffer#write
             # these chunks(unstaged chunks) has shared the same metadata
             # So perform enqueue step again https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L364
             if chunk_size_full?(chunk) || stage.key?(chunk.metadata)
@@ -234,7 +234,7 @@ module Fluent
           raise ArgumentError, "The chunk must be FileChunk, but it was #{chunk.class}."
         end
 
-        backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)
+        backup_dir = evacuate_dir
         FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
 
         FileUtils.copy([chunk.path, chunk.meta_path], backup_dir)

data/lib/fluent/plugin/buf_file_single.rb

@@ -193,7 +193,11 @@ module Fluent
 
           case chunk.state
           when :staged
-            stage[chunk.metadata] = chunk
+            if chunk_size_full?(chunk) || stage.key?(chunk.metadata)
+              queue << chunk.enqueued!
+            else
+              stage[chunk.metadata] = chunk
+            end
           when :queued
             queue << chunk
           end
@@ -246,7 +250,7 @@ module Fluent
           raise ArgumentError, "The chunk must be FileSingleChunk, but it was #{chunk.class}."
         end
 
-        backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)
+        backup_dir = evacuate_dir
         FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
 
         FileUtils.copy(chunk.path, backup_dir)

data/lib/fluent/plugin/buffer/chunk.rb

@@ -220,9 +220,13 @@ module Fluent
                               Tempfile.new('decompressed-data')
                             end
                 output_io.binmode if output_io.is_a?(Tempfile)
-                decompress(input_io: chunk_io, output_io: output_io)
-                output_io.seek(0, IO::SEEK_SET)
-                yield output_io
+                begin
+                  decompress(input_io: chunk_io, output_io: output_io)
+                  output_io.seek(0, IO::SEEK_SET)
+                  yield output_io
+                ensure
+                  output_io.close! if output_io.is_a?(Tempfile)
+                end
               end
             end
           end
@@ -274,9 +278,13 @@ module Fluent
                               Tempfile.new('decompressed-data')
                             end
                 output_io.binmode if output_io.is_a?(Tempfile)
-                decompress(input_io: chunk_io, output_io: output_io, type: :zstd)
-                output_io.seek(0, IO::SEEK_SET)
-                yield output_io
+                begin
+                  decompress(input_io: chunk_io, output_io: output_io, type: :zstd)
+                  output_io.seek(0, IO::SEEK_SET)
+                  yield output_io
+                ensure
+                  output_io.close! if output_io.is_a?(Tempfile)
+                end
               end
             end
           end

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -100,8 +100,14 @@ module Fluent
         end
 
         def enqueued!
-          return unless self.staged?
+          return self unless self.writable?
 
+          super
+          rename_to_queued_path
+          self
+        end
+
+        def rename_to_queued_path
           new_chunk_path = self.class.generate_queued_chunk_path(@path, @unique_id)
           new_meta_path = new_chunk_path + '.meta'
 
@@ -139,9 +145,8 @@ module Fluent
 
           @path = new_chunk_path
           @meta_path = new_meta_path
-
-          super
         end
+        private :rename_to_queued_path
 
         def close
           super
@@ -168,9 +173,11 @@ module Fluent
 
         def open(**kwargs, &block)
           @chunk.seek(0, IO::SEEK_SET)
-          val = yield @chunk
-          @chunk.seek(0, IO::SEEK_END) if self.staged?
-          val
+          begin
+            yield @chunk
+          ensure
+            @chunk.seek(0, IO::SEEK_END) if self.staged?
+          end
         end
 
         def self.assume_chunk_state(path)
@@ -264,19 +271,27 @@ module Fluent
 
         def file_rename(file, old_path, new_path, callback=nil)
           pos = file.pos
+          setup = ->(f) {
+            f.set_encoding(Encoding::ASCII_8BIT)
+            f.sync = true
+            f.binmode
+            f.pos = pos
+            callback.call(f) if callback
+          }
           if Fluent.windows?
             file.close
-            File.rename(old_path, new_path)
-            file = File.open(new_path, 'rb', @permission)
+            renamed = false
+            begin
+              File.rename(old_path, new_path)
+              renamed = true
+            ensure
+              setup.call(File.open(renamed ? new_path : old_path, 'rb', @permission))
+            end
           else
             File.rename(old_path, new_path)
             file.reopen(new_path, 'rb')
+            setup.call(file)
           end
-          file.set_encoding(Encoding::ASCII_8BIT)
-          file.sync = true
-          file.binmode
-          file.pos = pos
-          callback.call(file) if callback
         end
 
         def create_new_chunk(path, perm)

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -93,8 +93,14 @@ module Fluent
         end
 
         def enqueued!
-          return unless self.staged?
+          return self unless self.writable?
 
+          super
+          rename_to_queued_path
+          self
+        end
+
+        def rename_to_queued_path
           new_chunk_path = self.class.generate_queued_chunk_path(@path, @unique_id)
 
           begin
@@ -114,9 +120,8 @@ module Fluent
           end
 
           @path = new_chunk_path
-
-          super
         end
+        private :rename_to_queued_path
 
         def close
           super
@@ -141,9 +146,11 @@ module Fluent
 
         def open(**kwargs, &block)
           @chunk.seek(0, IO::SEEK_SET)
-          val = yield @chunk
-          @chunk.seek(0, IO::SEEK_END) if self.staged?
-          val
+          begin
+            yield @chunk
+          ensure
+            @chunk.seek(0, IO::SEEK_END) if self.staged?
+          end
         end
 
         def self.assume_chunk_state(path)
@@ -223,19 +230,27 @@ module Fluent
 
         def file_rename(file, old_path, new_path, callback = nil)
           pos = file.pos
+          setup = ->(f) {
+            f.set_encoding(Encoding::ASCII_8BIT)
+            f.sync = true
+            f.binmode
+            f.pos = pos
+            callback.call(f) if callback
+          }
           if Fluent.windows?
             file.close
-            File.rename(old_path, new_path)
-            file = File.open(new_path, 'rb', @permission)
+            renamed = false
+            begin
+              File.rename(old_path, new_path)
+              renamed = true
+            ensure
+              setup.call(File.open(renamed ? new_path : old_path, 'rb', @permission))
+            end
           else
             File.rename(old_path, new_path)
             file.reopen(new_path, 'rb')
+            setup.call(file)
           end
-          file.set_encoding(Encoding::ASCII_8BIT)
-          file.sync = true
-          file.binmode
-          file.pos = pos
-          callback.call(file) if callback
         end
 
         ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n

data/lib/fluent/plugin/buffer.rb

@@ -72,6 +72,9 @@ module Fluent
       desc 'If true, chunks are thrown away when unrecoverable error happens'
       config_param :disable_chunk_backup, :bool, default: false
 
+      desc 'The approximate size limit of the evacuated chunk files in the evacuation directory. When the retry limit is reached, queued chunks that fit within the remaining capacity are evacuated, and the others are purged without being evacuated. The total size may exceed this limit due to chunk metadata or concurrent evacuation by multiple workers. Set to 0 to disable evacuation. nil (the default) means no limit.'
+      config_param :evacuate_limit_size, :size, default: nil
+
       Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
         def initialize(timekey, tag, variables)
           super(timekey, tag, variables, 0)
@@ -227,6 +230,20 @@ module Fluent
         unless @queue_limit_length.nil?
           @total_limit_size = @chunk_limit_size * @queue_limit_length
         end
+        if @evacuate_limit_size
+          raise Fluent::ConfigError, "evacuate_limit_size must be 0 or larger" if @evacuate_limit_size < 0
+          if !persistent?
+            log.warn "evacuate_limit_size is ignored because this buffer does not evacuate chunks"
+            @evacuate_limit_size = nil
+          elsif @evacuate_limit_size > 0
+            unless owner.plugin_id_configured?
+              log.warn "evacuate_limit_size should be used with @id, because the evacuation directory without @id can change when the configuration is changed or reloaded"
+            end
+            if @evacuate_limit_size < @chunk_limit_size
+              log.warn "evacuate_limit_size is smaller than chunk_limit_size, so full chunks may not be evacuated", evacuate_limit_size: @evacuate_limit_size, chunk_limit_size: @chunk_limit_size
+            end
+          end
+        end
         @stage_length_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "stage_length",
                                                help_text: 'Length of stage buffers', prefer_gauge: true)
         @stage_length_metrics.set(0)
@@ -383,6 +400,12 @@ module Fluent
             if enqueue || first_chunk.unstaged? || chunk_size_full?(first_chunk)
               chunks_to_enqueue << first_chunk
             end
+            if (bytesize = staged_bytesizes_by_chunk[first_chunk])
+              # Account while the chunk lock is still held so enqueue_chunk
+              # cannot remove the chunk and subtract its bytes first.
+              @stage_size_metrics.add(bytesize)
+              log.on_trace { log.trace { "chunk #{first_chunk.path} size_added: #{bytesize} new_size: #{first_chunk.bytesize}" } }
+            end
             first_chunk.mon_exit
           rescue
             operated_chunks.unshift(first_chunk)
@@ -397,6 +420,10 @@ module Fluent
               if enqueue || chunk.unstaged? || chunk_size_full?(chunk)
                 chunks_to_enqueue << chunk
               end
+              if (bytesize = staged_bytesizes_by_chunk[chunk])
+                @stage_size_metrics.add(bytesize)
+                log.on_trace { log.trace { "chunk #{chunk.path} size_added: #{bytesize} new_size: #{chunk.bytesize}" } }
+              end
               chunk.mon_exit
             rescue => e
               chunk.rollback
@@ -405,44 +432,46 @@ module Fluent
             end
           end
 
-          # All locks about chunks are released.
-
-          #
-          # Now update the stage, stage_size with proper locking
-          # FIX FOR stage_size miscomputation - https://github.com/fluent/fluentd/issues/2712
-          #
-          staged_bytesizes_by_chunk.each do |chunk, bytesize|
-            chunk.synchronize do
-              synchronize { @stage_size_metrics.add(bytesize) }
-              log.on_trace { log.trace { "chunk #{chunk.path} size_added: #{bytesize} new_size: #{chunk.bytesize}" } }
-            end
-          end
+          # All locks about chunks are released. Stage-size accounting for the
+          # staged chunks happened inside each chunk monitor above, so a
+          # concurrent enqueue_chunk cannot observe a transiently negative
+          # stage_size (issue #2712 race).
 
+          enqueue_errors = []
           chunks_to_enqueue.each do |c|
-            if c.staged? && (enqueue || chunk_size_full?(c))
-              m = c.metadata
-              enqueue_chunk(m)
-              if unstaged_chunks[m] && !unstaged_chunks[m].empty?
-                u = unstaged_chunks[m].pop
-                u.synchronize do
-                  if u.unstaged? && !chunk_size_full?(u)
-                    # `u.metadata.seq` and `m.seq` can be different but Buffer#enqueue_chunk expect them to be the same value
-                    u.metadata.seq = 0
-                    synchronize {
-                      @stage[m] = u.staged!
-                      @stage_size_metrics.add(u.bytesize)
-                    }
+            begin
+              if c.staged? && (enqueue || chunk_size_full?(c))
+                m = c.metadata
+                enqueue_chunk(m)
+                if unstaged_chunks[m] && !unstaged_chunks[m].empty?
+                  u = unstaged_chunks[m].pop
+                  u.synchronize do
+                    if u.unstaged? && !chunk_size_full?(u)
+                      # `u.metadata.seq` and `m.seq` can be different but Buffer#enqueue_chunk expect them to be the same value
+                      u.metadata.seq = 0
+                      synchronize {
+                        @stage[m] = u.staged!
+                        @stage_size_metrics.add(u.bytesize)
+                      }
+                    end
                   end
                 end
+              elsif c.unstaged?
+                enqueue_unstaged_chunk(c)
+              else
+                # previously staged chunk is already enqueued, closed or purged.
+                # no problem.
               end
-            elsif c.unstaged?
-              enqueue_unstaged_chunk(c)
-            else
-              # previously staged chunk is already enqueued, closed or purged.
-              # no problem.
+            rescue => e
+              enqueue_errors << e
             end
           end
 
+          enqueue_errors.each do |e|
+            log.warn "error occurs in enqueueing a chunk", error: e
+            log.warn_backtrace e.backtrace
+          end
+
           operated_chunks.clear if errors.empty?
 
           if errors.size > 0
@@ -492,6 +521,9 @@ module Fluent
 
         chunk.synchronize do
           synchronize do
+            bytesize = chunk.bytesize
+            @stage_size_metrics.sub(bytesize)
+            @queue_size_metrics.add(bytesize)
             if chunk.empty?
               chunk.close
             else
@@ -500,9 +532,6 @@ module Fluent
               @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
               chunk.enqueued!
             end
-            bytesize = chunk.bytesize
-            @stage_size_metrics.sub(bytesize)
-            @queue_size_metrics.add(bytesize)
           end
         end
         nil
@@ -517,9 +546,9 @@ module Fluent
             metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num
             @queue << chunk
             @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
+            @queue_size_metrics.add(chunk.bytesize)
             chunk.enqueued!
           end
-          @queue_size_metrics.add(chunk.bytesize)
         end
       end
 
@@ -601,13 +630,19 @@ module Fluent
           metadata = chunk.metadata
           log.on_trace { log.trace "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }
 
+          bytesize = chunk.bytesize
           begin
-            bytesize = chunk.bytesize
             chunk.purge
-            @queue_size_metrics.sub(bytesize)
           rescue => e
             log.error "failed to purge buffer chunk", chunk_id: dump_unique_id_hex(chunk_id), error_class: e.class, error: e
             log.error_backtrace
+          ensure
+            # Always release the queued byte counter, even when purge raises
+            # (e.g. unlink/close failure on the buffer path). Otherwise
+            # @queue_size is never decremented for a dequeued chunk that is
+            # gone from both @queue and @dequeued, so the leak ratchets toward
+            # total_limit_size and storable? becomes permanently false (#5468).
+            @queue_size_metrics.sub(bytesize)
           end
 
           @dequeued_num[chunk.metadata] -= 1
@@ -624,12 +659,27 @@ module Fluent
       def clear_queue!
         log.on_trace { log.trace "clearing queue", instance: self.object_id }
 
+        evacuate_budget = nil
+        skipped_num = 0
+        skipped_bytesize = 0
+
         synchronize do
           until @queue.empty?
             begin
               q = @queue.shift
-              evacuate_chunk(q)
-              log.trace("purging a chunk in queue"){ {id: dump_unique_id_hex(chunk.unique_id), bytesize: chunk.bytesize, size: chunk.size} }
+              if @evacuate_limit_size.nil?
+                evacuate_chunk(q)
+              elsif @evacuate_limit_size > 0
+                evacuate_budget ||= remaining_evacuate_size
+                if q.bytesize <= evacuate_budget
+                  evacuate_chunk(q)
+                  evacuate_budget -= q.bytesize
+                else
+                  skipped_num += 1
+                  skipped_bytesize += q.bytesize
+                end
+              end
+              log.trace("purging a chunk in queue"){ {id: dump_unique_id_hex(q.unique_id), bytesize: q.bytesize, size: q.size} }
               q.purge
             rescue => e
               log.error "unexpected error while clearing buffer queue", error_class: e.class, error: e
@@ -638,6 +688,10 @@ module Fluent
           end
           @queue_size_metrics.set(0)
         end
+
+        if skipped_num > 0
+          log.warn "some chunks are purged without being evacuated to stay within evacuate_limit_size.", evacuate_limit_size: @evacuate_limit_size, chunks: skipped_num, bytesize: skipped_bytesize
+        end
       end
 
       def evacuate_chunk(chunk)
@@ -850,7 +904,12 @@ module Fluent
                     # As already processed content is kept after rollback, then unstaged chunk should be queued.
                     # After that, re-process current split again.
                     # New chunk should be allocated, to do it, modify @stage and so on.
-                    synchronize { @stage.delete(modified_metadata) }
+                    synchronize do
+                      @stage.delete(modified_metadata)
+                      # Subtract this chunk's already-counted staged bytes here;
+                      # otherwise @stage_size leaks upward over the buffer's lifetime.
+                      @stage_size_metrics.sub(original_bytesize) if chunk.staged?
+                    end
                     staged_chunk_used = false
                     chunk.unstaged!
                     break
@@ -912,8 +971,15 @@ module Fluent
       ]
 
       def statistics
-        stage_size, queue_size = @stage_size_metrics.get, @queue_size_metrics.get
-        buffer_space = 1.0 - ((stage_size + queue_size * 1.0) / @total_limit_size)
+        # Export-only clamp: internal gauges may go transiently negative during
+        # the deferred stage_size add vs enqueue_chunk sub race (#5303, #2712).
+        # Clamping the gauge store itself would turn that into a permanent
+        # over-count and break Buffer#storable? -- keep raw gauge semantics.
+        stage_size = [@stage_size_metrics.get, 0].max
+        queue_size = [@queue_size_metrics.get, 0].max
+        denom = @total_limit_size.to_f
+        # denom > 0 already excludes 0/0 NaN; stage/queue are floored above.
+        buffer_space = denom > 0.0 ? (1.0 - (stage_size + queue_size).to_f / denom).clamp(0.0, 1.0) : 0.0
         @stage_length_metrics.set(@stage.size)
         @queue_length_metrics.set(@queue.size)
         @available_buffer_space_ratios_metrics.set(buffer_space * 100)
@@ -974,6 +1040,24 @@ module Fluent
       def backup_base_dir
         system_config.root_dir || DEFAULT_BACKUP_DIR
       end
+
+      def evacuate_dir
+        File.join(backup_base_dir, 'buffer', safe_owner_id)
+      end
+
+      def evacuated_bytesize
+        dir = evacuate_dir
+        Dir.children(dir, encoding: dir.encoding).sum { |name| File.size?(File.join(dir, name)).to_i }
+      rescue Errno::ENOENT
+        0
+      end
+
+      def remaining_evacuate_size
+        @evacuate_limit_size - evacuated_bytesize
+      rescue => e
+        log.warn "failed to get the size of the evacuated chunk files. queued chunks are purged without being evacuated.", error: e
+        0
+      end
     end
   end
 end

data/lib/fluent/plugin/compressable.rb

@@ -75,9 +75,9 @@ module Fluent
 
       def io_decompress(input, output, type = :gzip)
         if type == :gzip
-          Extractor.io_decompress_gzip(input, output)
+          Extractor.io_decompress_gzip(input, output, limit: @decompression_size_limit || DEFAULT_DECOMPRESSION_SIZE_LIMIT)
         elsif type == :zstd
-          Extractor.io_decompress_zstd(input, output)
+          Extractor.io_decompress_zstd(input, output, limit: @decompression_size_limit || DEFAULT_DECOMPRESSION_SIZE_LIMIT)
         else
           raise ArgumentError, "Unknown compression type: #{type}"
         end

data/lib/fluent/plugin/extractor.rb

@@ -26,6 +26,9 @@ module Fluent
 
       BYTES_TO_READ = 64 * 1024
       INFLATE_BYTES_TO_READ = 1024
+      # Zstd::StreamReader#read takes a number of compressed bytes, unlike
+      # Zlib::GzipReader#read which takes decompressed bytes
+      ZSTD_BYTES_TO_READ = 1024
 
       def self.decompress_gzip(compressed_data, limit:)
         io = StringIO.new(compressed_data)
@@ -55,8 +58,7 @@ module Fluent
         reader = Zstd::StreamReader.new(io)
         out = ''
         loop do
-          # Zstd::StreamReader needs to specify the size of the buffer
-          out << reader.read(BYTES_TO_READ)
+          out << reader.read(ZSTD_BYTES_TO_READ)
           if out.bytesize > limit
             raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
           end
@@ -88,10 +90,15 @@ module Fluent
         out
       end
 
-      def self.io_decompress_gzip(input, output)
+      def self.io_decompress_gzip(input, output, limit:)
+        size = 0
         loop do
           reader = Zlib::GzipReader.new(input)
           while (chunk = reader.read(BYTES_TO_READ))
+            size += chunk.bytesize
+            if size > limit
+              raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+            end
             output.write(chunk)
           end
           unused = reader.unused
@@ -105,11 +112,15 @@ module Fluent
         output
       end
 
-      def self.io_decompress_zstd(input, output)
+      def self.io_decompress_zstd(input, output, limit:)
         reader = Zstd::StreamReader.new(input)
+        size = 0
         loop do
-          # Zstd::StreamReader needs to specify the size of the buffer
-          chunk = reader.read(BYTES_TO_READ)
+          chunk = reader.read(ZSTD_BYTES_TO_READ)
+          size += chunk.bytesize
+          if size > limit
+            raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+          end
           output.write(chunk)
           # Zstd::StreamReader doesn't provide unused data, so we have to manually adjust the position
           break if input.eof?

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -18,6 +18,7 @@ require 'socket'
 require 'json'
 require 'ostruct'
 
+require 'fluent/env'
 require 'fluent/plugin/filter'
 require 'fluent/config/error'
 require 'fluent/event'
@@ -116,7 +117,7 @@ module Fluent::Plugin
 
     def parse_value(value_str)
       if value_str.start_with?('{', '[')
-        JSON.parse(value_str)
+        JSON.parse(value_str, **Fluent::DEFAULT_JSON_PARSE_OPTIONS)
       else
         value_str
       end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -44,10 +44,9 @@ module Fluent
 
       def format(tag, time, record)
         json_str = JSON.generate(record)
-        header = ''
-        header << "#{@timef.format(time)}#{@delimiter}" if @output_time
-        header << "#{tag}#{@delimiter}" if @output_tag
-        "#{header}#{json_str}#{@newline}"
+        time_str = @output_time ? "#{@timef.format(time)}#{@delimiter}" : ''
+        tag_str = @output_tag ? "#{tag}#{@delimiter}" : ''
+        "#{time_str}#{tag_str}#{json_str}#{@newline}"
       ensure
         json_str&.clear
       end