fluentd 1.19.4 → 1.20.0

data/lib/fluent/plugin/buffer.rb

@@ -72,6 +72,9 @@ module Fluent
       desc 'If true, chunks are thrown away when unrecoverable error happens'
       config_param :disable_chunk_backup, :bool, default: false
 
+      desc 'The approximate size limit of the evacuated chunk files in the evacuation directory. When the retry limit is reached, queued chunks that fit within the remaining capacity are evacuated, and the others are purged without being evacuated. The total size may exceed this limit due to chunk metadata or concurrent evacuation by multiple workers. Set to 0 to disable evacuation. nil (the default) means no limit.'
+      config_param :evacuate_limit_size, :size, default: nil
+
       Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
         def initialize(timekey, tag, variables)
           super(timekey, tag, variables, 0)
@@ -227,6 +230,20 @@ module Fluent
         unless @queue_limit_length.nil?
           @total_limit_size = @chunk_limit_size * @queue_limit_length
         end
+        if @evacuate_limit_size
+          raise Fluent::ConfigError, "evacuate_limit_size must be 0 or larger" if @evacuate_limit_size < 0
+          if !persistent?
+            log.warn "evacuate_limit_size is ignored because this buffer does not evacuate chunks"
+            @evacuate_limit_size = nil
+          elsif @evacuate_limit_size > 0
+            unless owner.plugin_id_configured?
+              log.warn "evacuate_limit_size should be used with @id, because the evacuation directory without @id can change when the configuration is changed or reloaded"
+            end
+            if @evacuate_limit_size < @chunk_limit_size
+              log.warn "evacuate_limit_size is smaller than chunk_limit_size, so full chunks may not be evacuated", evacuate_limit_size: @evacuate_limit_size, chunk_limit_size: @chunk_limit_size
+            end
+          end
+        end
         @stage_length_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "stage_length",
                                                help_text: 'Length of stage buffers', prefer_gauge: true)
         @stage_length_metrics.set(0)
@@ -383,6 +400,12 @@ module Fluent
             if enqueue || first_chunk.unstaged? || chunk_size_full?(first_chunk)
               chunks_to_enqueue << first_chunk
             end
+            if (bytesize = staged_bytesizes_by_chunk[first_chunk])
+              # Account while the chunk lock is still held so enqueue_chunk
+              # cannot remove the chunk and subtract its bytes first.
+              @stage_size_metrics.add(bytesize)
+              log.on_trace { log.trace { "chunk #{first_chunk.path} size_added: #{bytesize} new_size: #{first_chunk.bytesize}" } }
+            end
             first_chunk.mon_exit
           rescue
             operated_chunks.unshift(first_chunk)
@@ -397,6 +420,10 @@ module Fluent
               if enqueue || chunk.unstaged? || chunk_size_full?(chunk)
                 chunks_to_enqueue << chunk
               end
+              if (bytesize = staged_bytesizes_by_chunk[chunk])
+                @stage_size_metrics.add(bytesize)
+                log.on_trace { log.trace { "chunk #{chunk.path} size_added: #{bytesize} new_size: #{chunk.bytesize}" } }
+              end
               chunk.mon_exit
             rescue => e
               chunk.rollback
@@ -405,44 +432,46 @@ module Fluent
             end
           end
 
-          # All locks about chunks are released.
-
-          #
-          # Now update the stage, stage_size with proper locking
-          # FIX FOR stage_size miscomputation - https://github.com/fluent/fluentd/issues/2712
-          #
-          staged_bytesizes_by_chunk.each do |chunk, bytesize|
-            chunk.synchronize do
-              synchronize { @stage_size_metrics.add(bytesize) }
-              log.on_trace { log.trace { "chunk #{chunk.path} size_added: #{bytesize} new_size: #{chunk.bytesize}" } }
-            end
-          end
+          # All locks about chunks are released. Stage-size accounting for the
+          # staged chunks happened inside each chunk monitor above, so a
+          # concurrent enqueue_chunk cannot observe a transiently negative
+          # stage_size (issue #2712 race).
 
+          enqueue_errors = []
           chunks_to_enqueue.each do |c|
-            if c.staged? && (enqueue || chunk_size_full?(c))
-              m = c.metadata
-              enqueue_chunk(m)
-              if unstaged_chunks[m] && !unstaged_chunks[m].empty?
-                u = unstaged_chunks[m].pop
-                u.synchronize do
-                  if u.unstaged? && !chunk_size_full?(u)
-                    # `u.metadata.seq` and `m.seq` can be different but Buffer#enqueue_chunk expect them to be the same value
-                    u.metadata.seq = 0
-                    synchronize {
-                      @stage[m] = u.staged!
-                      @stage_size_metrics.add(u.bytesize)
-                    }
+            begin
+              if c.staged? && (enqueue || chunk_size_full?(c))
+                m = c.metadata
+                enqueue_chunk(m)
+                if unstaged_chunks[m] && !unstaged_chunks[m].empty?
+                  u = unstaged_chunks[m].pop
+                  u.synchronize do
+                    if u.unstaged? && !chunk_size_full?(u)
+                      # `u.metadata.seq` and `m.seq` can be different but Buffer#enqueue_chunk expect them to be the same value
+                      u.metadata.seq = 0
+                      synchronize {
+                        @stage[m] = u.staged!
+                        @stage_size_metrics.add(u.bytesize)
+                      }
+                    end
                   end
                 end
+              elsif c.unstaged?
+                enqueue_unstaged_chunk(c)
+              else
+                # previously staged chunk is already enqueued, closed or purged.
+                # no problem.
               end
-            elsif c.unstaged?
-              enqueue_unstaged_chunk(c)
-            else
-              # previously staged chunk is already enqueued, closed or purged.
-              # no problem.
+            rescue => e
+              enqueue_errors << e
             end
           end
 
+          enqueue_errors.each do |e|
+            log.warn "error occurs in enqueueing a chunk", error: e
+            log.warn_backtrace e.backtrace
+          end
+
           operated_chunks.clear if errors.empty?
 
           if errors.size > 0
@@ -492,6 +521,9 @@ module Fluent
 
         chunk.synchronize do
           synchronize do
+            bytesize = chunk.bytesize
+            @stage_size_metrics.sub(bytesize)
+            @queue_size_metrics.add(bytesize)
             if chunk.empty?
               chunk.close
             else
@@ -500,9 +532,6 @@ module Fluent
               @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
               chunk.enqueued!
             end
-            bytesize = chunk.bytesize
-            @stage_size_metrics.sub(bytesize)
-            @queue_size_metrics.add(bytesize)
           end
         end
         nil
@@ -517,9 +546,9 @@ module Fluent
             metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num
             @queue << chunk
             @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
+            @queue_size_metrics.add(chunk.bytesize)
             chunk.enqueued!
           end
-          @queue_size_metrics.add(chunk.bytesize)
         end
       end
 
@@ -630,12 +659,27 @@ module Fluent
       def clear_queue!
         log.on_trace { log.trace "clearing queue", instance: self.object_id }
 
+        evacuate_budget = nil
+        skipped_num = 0
+        skipped_bytesize = 0
+
         synchronize do
           until @queue.empty?
             begin
               q = @queue.shift
-              evacuate_chunk(q)
-              log.trace("purging a chunk in queue"){ {id: dump_unique_id_hex(chunk.unique_id), bytesize: chunk.bytesize, size: chunk.size} }
+              if @evacuate_limit_size.nil?
+                evacuate_chunk(q)
+              elsif @evacuate_limit_size > 0
+                evacuate_budget ||= remaining_evacuate_size
+                if q.bytesize <= evacuate_budget
+                  evacuate_chunk(q)
+                  evacuate_budget -= q.bytesize
+                else
+                  skipped_num += 1
+                  skipped_bytesize += q.bytesize
+                end
+              end
+              log.trace("purging a chunk in queue"){ {id: dump_unique_id_hex(q.unique_id), bytesize: q.bytesize, size: q.size} }
               q.purge
             rescue => e
               log.error "unexpected error while clearing buffer queue", error_class: e.class, error: e
@@ -644,6 +688,10 @@ module Fluent
           end
           @queue_size_metrics.set(0)
         end
+
+        if skipped_num > 0
+          log.warn "some chunks are purged without being evacuated to stay within evacuate_limit_size.", evacuate_limit_size: @evacuate_limit_size, chunks: skipped_num, bytesize: skipped_bytesize
+        end
       end
 
       def evacuate_chunk(chunk)
@@ -992,6 +1040,24 @@ module Fluent
       def backup_base_dir
         system_config.root_dir || DEFAULT_BACKUP_DIR
       end
+
+      def evacuate_dir
+        File.join(backup_base_dir, 'buffer', safe_owner_id)
+      end
+
+      def evacuated_bytesize
+        dir = evacuate_dir
+        Dir.children(dir, encoding: dir.encoding).sum { |name| File.size?(File.join(dir, name)).to_i }
+      rescue Errno::ENOENT
+        0
+      end
+
+      def remaining_evacuate_size
+        @evacuate_limit_size - evacuated_bytesize
+      rescue => e
+        log.warn "failed to get the size of the evacuated chunk files. queued chunks are purged without being evacuated.", error: e
+        0
+      end
     end
   end
 end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -44,10 +44,9 @@ module Fluent
 
       def format(tag, time, record)
         json_str = JSON.generate(record)
-        header = ''
-        header << "#{@timef.format(time)}#{@delimiter}" if @output_time
-        header << "#{tag}#{@delimiter}" if @output_tag
-        "#{header}#{json_str}#{@newline}"
+        time_str = @output_time ? "#{@timef.format(time)}#{@delimiter}" : ''
+        tag_str = @output_tag ? "#{tag}#{@delimiter}" : ''
+        "#{time_str}#{tag_str}#{json_str}#{@newline}"
       ensure
         json_str&.clear
       end

data/lib/fluent/plugin/in_exec.rb

@@ -15,7 +15,7 @@
 #
 
 require 'fluent/plugin/input'
-require 'yajl'
+require 'json'
 
 module Fluent::Plugin
   class ExecInput < Fluent::Plugin::Input
@@ -43,6 +43,8 @@ module Fluent::Plugin
     config_param :tag, :string, default: nil
     desc 'The interval time between periodic program runs.'
     config_param :run_interval, :time, default: nil
+    desc 'Command (program) execution timeout.'
+    config_param :command_timeout, :time, default: nil
     desc 'The default block size to read if parser requires partial read.'
     config_param :read_block_size, :size, default: 10240 # 10k
     desc 'The encoding to receive the result of the command, especially for non-ascii characters.'
@@ -86,9 +88,9 @@ module Fluent::Plugin
       options[:external_encoding] = @encoding if @encoding
 
       if @run_interval
-        child_process_execute(:exec_input, @command, interval: @run_interval, **options, &method(:run))
+        child_process_execute(:exec_input, @command, interval: @run_interval, wait_timeout: @command_timeout, **options, &method(:run))
       else
-        child_process_execute(:exec_input, @command, immediate: true, **options, &method(:run))
+        child_process_execute(:exec_input, @command, immediate: true, wait_timeout: @command_timeout, **options, &method(:run))
       end
     end
 
@@ -115,7 +117,7 @@ module Fluent::Plugin
       time ||= extract_time_from_record(record) || Fluent::EventTime.now
       router.emit(tag, time, record)
     rescue => e
-      log.error "exec failed to emit", tag: tag, record: Yajl.dump(record), error: e
+      log.error "exec failed to emit", tag: tag, record: JSON.generate(record), error: e
       router.emit_error_event(tag, time, record, e) if tag && time && record
     end
   end

data/lib/fluent/plugin/in_forward.rb

@@ -15,9 +15,10 @@
 #
 
 
+require 'fluent/env'
 require 'fluent/plugin/input'
 require 'fluent/msgpack_factory'
-require 'yajl'
+require 'json'
 require 'digest'
 require 'securerandom'
 
@@ -31,6 +32,12 @@ module Fluent::Plugin
     helpers :server
 
     LISTEN_PORT = 24224
+    OPTION_ACK = 'ack'.freeze
+    OPTION_CHUNK = 'chunk'.freeze
+    OPTION_COMPRESSED = 'compressed'.freeze
+    OPTION_FLUENT_SIGNAL = 'fluent_signal'.freeze
+    OPTION_SIZE = 'size'.freeze
+    OPTION_TEXT = 'text'.freeze
 
     desc 'The port to listen to.'
     config_param :port, :integer, default: LISTEN_PORT
@@ -250,13 +257,15 @@ module Fluent::Plugin
         unless feeder
           first = data[0]
           if first == '{' || first == '[' # json
-            parser = Yajl::Parser.new
-            parser.on_parse_complete = ->(obj){
-              block.call(obj, bytes, serializer)
-              bytes = 0
-            }
+            parser = JSON::ResumableParser.new(**Fluent::DEFAULT_JSON_PARSE_OPTIONS)
             serializer = :to_json.to_proc
-            feeder = ->(d){ parser << d }
+            feeder = ->(d){
+              parser << d
+              while parser.parse
+                block.call(parser.value, bytes, serializer)
+                bytes = 0
+              end
+            }
           else # msgpack
             parser = Fluent::MessagePackFactory.msgpack_unpacker
             serializer = :to_msgpack.to_proc
@@ -275,8 +284,8 @@ module Fluent::Plugin
     end
 
     def response(option)
-      if option && option['chunk']
-        return { 'ack' => option['chunk'] }
+      if option && option[OPTION_CHUNK]
+        return { OPTION_ACK => option[OPTION_CHUNK] }
       end
       nil
     end
@@ -310,14 +319,19 @@ module Fluent::Plugin
       when String
         # PackedForward
         option = msg[2] || {}
-        size = option['size'] || 0
+        size = option[OPTION_SIZE] || 0
+        compressed = option[OPTION_COMPRESSED]
 
-        if option['compressed'] && option['compressed'] != 'text'
-          es = Fluent::CompressedMessagePackEventStream.new(entries, nil, size.to_i, compress: option['compressed'].to_sym, decompression_size_limit: @decompression_size_limit)
+        if compressed && compressed != OPTION_TEXT
+          es = Fluent::CompressedMessagePackEventStream.new(entries, nil, size.to_i, compress: compressed.to_sym, decompression_size_limit: @decompression_size_limit)
         else
           es = Fluent::MessagePackEventStream.new(entries, nil, size.to_i)
         end
-        es = check_and_skip_invalid_event(tag, es, conn.remote_host) if @skip_invalid_event
+        if @skip_invalid_event
+          es = normalize_event_stream(tag, es, conn.remote_host)
+        elsif option[OPTION_FLUENT_SIGNAL] == 0
+          es = Fluent::MetadataTimeEventStream.new(es)
+        end
         if @enable_field_injection
           es = add_source_info(es, conn)
         end
@@ -325,19 +339,7 @@ module Fluent::Plugin
 
       when Array
         # Forward
-        es = if @skip_invalid_event
-               check_and_skip_invalid_event(tag, entries, conn.remote_host)
-             else
-               es = Fluent::MultiEventStream.new
-               entries.each { |e|
-                 record = e[1]
-                 next if record.nil?
-                 time = e[0]
-                 time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
-                 es.add(time, record)
-               }
-               es
-             end
+        es = normalize_event_stream(tag, entries, conn.remote_host)
         if @enable_field_injection
           es = add_source_info(es, conn)
         end
@@ -346,7 +348,7 @@ module Fluent::Plugin
 
       else
         # Message
-        time = msg[1]
+        time = extract_event_time(msg[1])
         record = msg[2]
         if @skip_invalid_event && invalid_event?(tag, time, record)
           log.warn "got invalid event and drop it:", host: conn.remote_host, tag: tag, time: time, record: record
@@ -367,21 +369,29 @@ module Fluent::Plugin
     end
 
     def invalid_event?(tag, time, record)
+      time = extract_event_time(time)
       !((time.is_a?(Integer) || time.is_a?(::Fluent::EventTime)) && record.is_a?(Hash) && tag.is_a?(String))
     end
 
-    def check_and_skip_invalid_event(tag, es, remote_host)
+    def normalize_event_stream(tag, es, remote_host)
       new_es = Fluent::MultiEventStream.new
       es.each { |time, record|
-        if invalid_event?(tag, time, record)
+        time = extract_event_time(time)
+        if @skip_invalid_event && invalid_event?(tag, time, record)
           log.warn "skip invalid event:", host: remote_host, tag: tag, time: time, record: record
           next
         end
+        next if record.nil?
+        time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
         new_es.add(time, record)
       }
       new_es
     end
 
+    def extract_event_time(time)
+      time.is_a?(Array) ? time[0] : time
+    end
+
     def add_source_info(es, conn)
       new_es = Fluent::MultiEventStream.new
       if @source_address_key && @source_hostname_key

data/lib/fluent/plugin/in_object_space.rb

@@ -14,7 +14,7 @@
 #    limitations under the License.
 #
 
-require 'yajl'
+require 'json'
 
 require 'fluent/plugin/input'
 
@@ -86,7 +86,7 @@ module Fluent::Plugin
 
       router.emit(@tag, now, record)
     rescue => e
-      log.error "object space failed to emit", error: e, tag: @tag, record: Yajl.dump(record)
+      log.error "object space failed to emit", error: e, tag: @tag, record: JSON.generate(record)
       log.error_backtrace
     end
   end

data/lib/fluent/plugin/in_syslog.rb

@@ -18,7 +18,7 @@ require 'fluent/plugin/input'
 require 'fluent/config/error'
 require 'fluent/plugin/parser'
 
-require 'yajl'
+require 'json'
 
 module Fluent::Plugin
   class SyslogInput < Input
@@ -340,7 +340,7 @@ module Fluent::Plugin
     def emit(tag, time, record)
       router.emit(tag, time, record)
     rescue => e
-      log.error "syslog failed to emit", error: e, tag: tag, record: Yajl.dump(record)
+      log.error "syslog failed to emit", error: e, tag: tag, record: JSON.generate(record)
     end
   end
 end

data/lib/fluent/plugin/in_tail/compatibility.rb

@@ -0,0 +1,111 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'fluent/clock'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    module Compatibility
+      # Keep these methods as compatibility entry points for plugins which
+      # inherit TailInput and override line processing methods. TailInput is a
+      # built-in plugin, not a public inheritance API, and these methods may be
+      # removed in a future release.
+      def flush_buffer(tw, buf)
+        @line_feeder.flush_buffer(tw, buf)
+      end
+
+      # Feeds lines through the per-file feed, or through LineFeeder for watchers
+      # created by plugins using the deprecated LineBufferTimerFlusher.
+      #
+      # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
+      def receive_lines(lines, tail_watcher)
+        return async_receive_lines(lines, tail_watcher) if @worker_pool
+
+        file_feed = tail_watcher.file_feed
+        return file_feed.feed_lines(lines, tail_watcher) if file_feed.respond_to?(:feed_lines)
+
+        @line_feeder.feed_lines(lines, tail_watcher)
+      end
+
+      def convert_line_to_event(line, es, tail_watcher)
+        @line_feeder.convert_line_to_event(line, es, tail_watcher)
+      end
+
+      def parse_singleline(lines, tail_watcher)
+        @line_feeder.parse_singleline(lines, tail_watcher)
+      end
+
+      # TailInput's default #parse_multilines path starts the flush timer before
+      # parsing. An override that skips super must call
+      # tail_watcher.line_buffer_timer_flusher.reset_timer when needed.
+      def parse_multilines(lines, tail_watcher)
+        @line_feeder.parse_multilines(lines, tail_watcher)
+      end
+    end
+
+    class TailWatcher
+      # The object which keeps the line buffer of the multiline mode of the file:
+      # the FileFeed built by LineFeeder#new_file_feed, or the deprecated
+      # LineBufferTimerFlusher which a plugin overriding TailInput#setup_watcher
+      # passes to keep working without changes.
+      def line_buffer_timer_flusher
+        @file_feed
+      end
+
+      # Kept for compatibility with the plugins which build it by themselves and
+      # pass it to TailWatcher in the overridden TailInput#setup_watcher.
+      # LineFeeder#feed_lines feeds the lines of a watcher which keeps it, and
+      # LineFeeder::FileFeed built by LineFeeder#new_file_feed replaces it.
+      class LineBufferTimerFlusher
+        attr_accessor :line_buffer
+
+        def initialize(log, flush_interval, &flush_method)
+          @log = log
+          @flush_interval = flush_interval
+          @flush_method = flush_method
+          @start = nil
+          @line_buffer = nil
+        end
+
+        def on_notify(tw)
+          unless @start && @flush_method
+            return
+          end
+
+          if Fluent::Clock.now - @start >= @flush_interval
+            @flush_method.call(tw, @line_buffer) if @line_buffer
+            @line_buffer = nil
+            @start = nil
+          end
+        end
+
+        def close(tw)
+          return unless @line_buffer
+
+          @flush_method.call(tw, @line_buffer)
+          @line_buffer = nil
+        end
+
+        def reset_timer
+          return unless @flush_interval
+
+          @start = Fluent::Clock.now
+        end
+      end
+    end
+  end
+end