fluentd 1.13.1-x64-mingw32 → 1.14.1-x64-mingw32

data/lib/fluent/oj_options.rb

@@ -0,0 +1,62 @@
+require 'fluent/config/types'
+
+module Fluent
+  class OjOptions
+    OPTIONS = {
+      'bigdecimal_load': :symbol,
+      'max_nesting': :integer,
+      'mode': :symbol,
+      'use_to_json': :bool
+    }
+
+    ALLOWED_VALUES = {
+      'bigdecimal_load': %i[bigdecimal float auto],
+      'mode': %i[strict null compat json rails object custom]
+    }
+
+    DEFAULTS = {
+      'bigdecimal_load': :float,
+      'mode': :compat,
+      'use_to_json': true
+    }
+
+    @@available = false
+
+    def self.available?
+      @@available
+    end
+
+    def self.load_env
+      options = self.get_options
+      begin
+        require 'oj'
+        Oj.default_options = options
+        @@available = true
+      rescue LoadError
+        @@available = false
+      end
+      options
+    end
+
+    private
+
+    def self.get_options
+      options = {}
+      DEFAULTS.each { |key, value| options[key] = value }
+
+      OPTIONS.each do |key, type|
+        env_value = ENV["FLUENT_OJ_OPTION_#{key.upcase}"]
+        next if env_value.nil?
+
+        cast_value = Fluent::Config.reformatted_value(OPTIONS[key], env_value, { strict: true })
+        next if cast_value.nil?
+
+        next if ALLOWED_VALUES[key] && !ALLOWED_VALUES[key].include?(cast_value)
+
+        options[key.to_sym] = cast_value
+      end
+
+      options
+    end
+  end
+end

data/lib/fluent/plugin/bare_output.rb

@@ -23,37 +23,78 @@ require 'fluent/plugin_helper'
 module Fluent
   module Plugin
     class BareOutput < Base
+      include PluginHelper::Mixin # for metrics
+
       # DO NOT USE THIS plugin for normal output plugin. Use Output instead.
       # This output plugin base class is only for meta-output plugins
       # which cannot be implemented on MultiOutput.
       # E.g,: forest, config-expander
 
+      helpers_internal :metrics
+
       include PluginId
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      attr_reader :num_errors, :emit_count, :emit_records
-
       def process(tag, es)
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
       end
 
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
       def initialize
         super
         @counter_mutex = Mutex.new
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+      end
+
+      def configure(conf)
+        super
+
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'bare_output' => stats }
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end

data/lib/fluent/plugin/buffer.rb

@@ -16,6 +16,8 @@
 
 require 'fluent/plugin/base'
 require 'fluent/plugin/owned_by_mixin'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
 require 'fluent/unique_id'
 require 'fluent/ext_monitor_require'
 
@@ -24,7 +26,9 @@ module Fluent
     class Buffer < Base
       include OwnedByMixin
       include UniqueId::Mixin
+      include PluginId
       include MonitorMixin
+      include PluginHelper::Mixin # for metrics
 
       class BufferError < StandardError; end
       class BufferOverflowError < BufferError; end
@@ -39,6 +43,8 @@ module Fluent
 
       configured_in :buffer
 
+      helpers_internal :metrics
+
       # TODO: system total buffer limit size in bytes by SystemConfig
 
       config_param :chunk_limit_size, :size, default: DEFAULT_CHUNK_LIMIT_SIZE
@@ -153,8 +159,11 @@ module Fluent
         end
       end
 
+      # for metrics
+      attr_reader :stage_size_metrics, :stage_length_metrics, :queue_size_metrics, :queue_length_metrics
+      attr_reader :available_buffer_space_ratios_metrics, :total_queued_size_metrics
+      attr_reader :newest_timekey_metrics, :oldest_timekey_metrics
       # for tests
-      attr_accessor :stage_size, :queue_size
       attr_reader :stage, :queue, :dequeued, :queued_num
 
       def initialize
@@ -171,12 +180,35 @@ module Fluent
         @queued_num = {} # metadata => int (number of queued chunks)
         @dequeued_num = {} # metadata => int (number of dequeued chunks)
 
-        @stage_size = @queue_size = 0
+        @stage_length_metrics = nil
+        @stage_size_metrics = nil
+        @queue_length_metrics = nil
+        @queue_size_metrics = nil
+        @available_buffer_space_ratios_metrics = nil
+        @total_queued_size_metrics = nil
+        @newest_timekey_metrics = nil
+        @oldest_timekey_metrics = nil
         @timekeys = Hash.new(0)
         @enable_update_timekeys = false
         @mutex = Mutex.new
       end
 
+      def stage_size
+        @stage_size_metrics.get
+      end
+
+      def stage_size=(value)
+        @stage_size_metrics.set(value)
+      end
+
+      def queue_size
+        @queue_size_metrics.get
+      end
+
+      def queue_size=(value)
+        @queue_size_metrics.set(value)
+      end
+
       def persistent?
         false
       end
@@ -187,6 +219,28 @@ module Fluent
         unless @queue_limit_length.nil?
           @total_limit_size = @chunk_limit_size * @queue_limit_length
         end
+        @stage_length_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "stage_length",
+                                               help_text: 'Length of stage buffers', prefer_gauge: true)
+        @stage_length_metrics.set(0)
+        @stage_size_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "stage_byte_size",
+                                             help_text: 'Total size of stage buffers', prefer_gauge: true)
+        @stage_size_metrics.set(0) # Ensure zero.
+        @queue_length_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "queue_length",
+                                               help_text: 'Length of queue buffers', prefer_gauge: true)
+        @queue_length_metrics.set(0)
+        @queue_size_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "queue_byte_size",
+                                             help_text: 'Total size of queue buffers', prefer_gauge: true)
+        @queue_size_metrics.set(0) # Ensure zero.
+        @available_buffer_space_ratios_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "available_buffer_space_ratios",
+                                                                help_text: 'Ratio of available space in buffer', prefer_gauge: true)
+        @available_buffer_space_ratios_metrics.set(100) # Default is 100%.
+        @total_queued_size_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "total_queued_size",
+                                                    help_text: 'Total size of stage and queue buffers', prefer_gauge: true)
+        @total_queued_size_metrics.set(0)
+        @newest_timekey_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "newest_timekey",
+                                                 help_text: 'Newest timekey in buffer', prefer_gauge: true)
+        @oldest_timekey_metrics = metrics_create(namespace: "fluentd", subsystem: "buffer", name: "oldest_timekey",
+                                                 help_text: 'Oldest timekey in buffer', prefer_gauge: true)
       end
 
       def enable_update_timekeys
@@ -198,15 +252,15 @@ module Fluent
 
         @stage, @queue = resume
         @stage.each_pair do |metadata, chunk|
-          @stage_size += chunk.bytesize
+          @stage_size_metrics.add(chunk.bytesize)
         end
         @queue.each do |chunk|
           @queued_num[chunk.metadata] ||= 0
           @queued_num[chunk.metadata] += 1
-          @queue_size += chunk.bytesize
+          @queue_size_metrics.add(chunk.bytesize)
         end
         update_timekeys
-        log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
+        log.debug "buffer started", instance: self.object_id, stage_size: @stage_size_metrics.get, queue_size: @queue_size_metrics.get
       end
 
       def close
@@ -228,17 +282,19 @@ module Fluent
       def terminate
         super
         @dequeued = @stage = @queue = @queued_num = nil
-        @stage_size = @queue_size = 0
+        @stage_length_metrics = @stage_size_metrics = @queue_length_metrics = @queue_size_metrics = nil
+        @available_buffer_space_ratios_metrics = @total_queued_size_metrics = nil
+        @newest_timekey_metrics = @oldest_timekey_metrics = nil
         @timekeys.clear
       end
 
       def storable?
-        @total_limit_size > @stage_size + @queue_size
+        @total_limit_size > @stage_size_metrics.get + @queue_size_metrics.get
       end
 
       ## TODO: for back pressure feature
       # def used?(ratio)
-      #   @total_limit_size * ratio > @stage_size + @queue_size
+      #   @total_limit_size * ratio > @stage_size_metrics.get + @queue_size_metrics.get
       # end
 
       def resume
@@ -344,7 +400,7 @@ module Fluent
           #
           staged_bytesizes_by_chunk.each do |chunk, bytesize|
             chunk.synchronize do
-              synchronize { @stage_size += bytesize }
+              synchronize { @stage_size_metrics.add(bytesize) }
               log.on_trace { log.trace { "chunk #{chunk.path} size_added: #{bytesize} new_size: #{chunk.bytesize}" } }
             end
           end
@@ -361,7 +417,7 @@ module Fluent
                     u.metadata.seq = 0
                     synchronize {
                       @stage[m] = u.staged!
-                      @stage_size += u.bytesize
+                      @stage_size_metrics.add(u.bytesize)
                     }
                   end
                 end
@@ -428,8 +484,8 @@ module Fluent
               chunk.enqueued!
             end
             bytesize = chunk.bytesize
-            @stage_size -= bytesize
-            @queue_size += bytesize
+            @stage_size_metrics.sub(bytesize)
+            @queue_size_metrics.add(bytesize)
           end
         end
         nil
@@ -446,7 +502,7 @@ module Fluent
             @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
             chunk.enqueued!
           end
-          @queue_size += chunk.bytesize
+          @queue_size_metrics.add(chunk.bytesize)
         end
       end
 
@@ -531,7 +587,7 @@ module Fluent
           begin
             bytesize = chunk.bytesize
             chunk.purge
-            @queue_size -= bytesize
+            @queue_size_metrics.sub(bytesize)
           rescue => e
             log.error "failed to purge buffer chunk", chunk_id: dump_unique_id_hex(chunk_id), error_class: e.class, error: e
             log.error_backtrace
@@ -562,7 +618,7 @@ module Fluent
               log.error_backtrace
             end
           end
-          @queue_size = 0
+          @queue_size_metrics.set(0)
         end
       end
 
@@ -765,23 +821,29 @@ module Fluent
       ]
 
       def statistics
-        stage_size, queue_size = @stage_size, @queue_size
+        stage_size, queue_size = @stage_size_metrics.get, @queue_size_metrics.get
         buffer_space = 1.0 - ((stage_size + queue_size * 1.0) / @total_limit_size)
+        @stage_length_metrics.set(@stage.size)
+        @queue_length_metrics.set(@queue.size)
+        @available_buffer_space_ratios_metrics.set(buffer_space * 100)
+        @total_queued_size_metrics.set(stage_size + queue_size)
         stats = {
-          'stage_length' => @stage.size,
+          'stage_length' => @stage_length_metrics.get,
           'stage_byte_size' => stage_size,
-          'queue_length' => @queue.size,
+          'queue_length' => @queue_length_metrics.get,
           'queue_byte_size' => queue_size,
-          'available_buffer_space_ratios' => (buffer_space * 100).round(1),
-          'total_queued_size' => stage_size + queue_size,
+          'available_buffer_space_ratios' => @available_buffer_space_ratios_metrics.get.round(1),
+          'total_queued_size' => @total_queued_size_metrics.get,
         }
 
         tkeys = timekeys
         if (m = tkeys.min)
-          stats['oldest_timekey'] = m
+          @oldest_timekey_metrics.set(m)
+          stats['oldest_timekey'] = @oldest_timekey_metrics.get
         end
         if (m = tkeys.max)
-          stats['newest_timekey'] = m
+          @newest_timekey_metrics.set(m)
+          stats['newest_timekey'] = @newest_timekey_metrics.get
         end
 
         { 'buffer' => stats }

data/lib/fluent/plugin/file_wrapper.rb

@@ -155,7 +155,29 @@ module Fluent
       by_handle_file_information.unpack("I11Q1")[11] # fileindex
     end
 
+    # DeletePending is a Windows-specific file state that roughly means
+    # "this file is queued for deletion, so close any open handlers"
+    #
+    # This flag can be retrieved via GetFileInformationByHandleEx().
+    #
+    # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getfileinformationbyhandleex
+    #
+    def delete_pending
+      file_standard_info = 0x01
+      bufsize = 1024
+      buf = '\0' * bufsize
+
+      unless GetFileInformationByHandleEx.call(@file_handle, file_standard_info, buf, bufsize)
+        return false
+      end
+
+      return buf.unpack("QQICC")[3] != 0
+    end
+
+    private :delete_pending
+
     def stat
+      raise Errno::ENOENT if delete_pending
       s = File.stat(@path)
       s.instance_variable_set :@ino, self.ino
       def s.ino; @ino; end

data/lib/fluent/plugin/filter.rb

@@ -28,13 +28,47 @@ module Fluent
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      helpers_internal :event_emitter
+      helpers_internal :event_emitter, :metrics
 
       attr_reader :has_filter_with_time
 
       def initialize
         super
         @has_filter_with_time = has_filter_with_time?
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        @counter_mutex = Mutex.new
+        @enable_size_metrics = false
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def configure(conf)
+        super
+
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "filter", name: "emit_records", help_text: "Number of count emit records")
+        @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "filter", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'emit_records' => @emit_records_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'filter' => stats }
+      end
+
+      def measure_metrics(es)
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
       end
 
       def filter(tag, time, record)

data/lib/fluent/plugin/formatter.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'fluent/env'
 require 'fluent/plugin/base'
 require 'fluent/plugin/owned_by_mixin'
 require 'fluent/time'

data/lib/fluent/plugin/formatter_json.rb

@@ -15,7 +15,7 @@
 #
 
 require 'fluent/plugin/formatter'
-require 'fluent/env'
+require 'fluent/oj_options'
 
 module Fluent
   module Plugin
@@ -30,12 +30,14 @@ module Fluent
       def configure(conf)
         super
 
-        begin
-          raise LoadError unless @json_parser == 'oj'
-          require 'oj'
-          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
-          @dump_proc = Oj.method(:dump)
-        rescue LoadError
+        if @json_parser == 'oj'
+          if Fluent::OjOptions.available?
+            @dump_proc = Oj.method(:dump)
+          else
+            log.info "Oj isn't installed, fallback to Yajl as json parser"
+            @dump_proc = Yajl.method(:dump)
+          end
+        else
           @dump_proc = Yajl.method(:dump)
         end
 

data/lib/fluent/plugin/in_http.rb

@@ -74,6 +74,8 @@ module Fluent::Plugin
     config_param :blocking_timeout, :time, default: 0.5
     desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
     config_param :cors_allow_origins, :array, default: nil
+    desc 'Tells browsers whether to expose the response to frontend when the credentials mode is "include".'
+    config_param :cors_allow_credentials, :bool, default: false
     desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, default: false
     desc 'Respond status code with 204.'
@@ -112,6 +114,12 @@ module Fluent::Plugin
 
       super
 
+      if @cors_allow_credentials
+        if @cors_allow_origins.nil? || @cors_allow_origins.include?('*')
+          raise Fluent::ConfigError, "Cannot enable cors_allow_credentials without specific origins"
+        end
+      end
+
       m = if @parser_configs.first['@type'] == 'in_http'
             @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
             @parser_msgpack.time_key = nil
@@ -279,7 +287,10 @@ module Fluent::Plugin
     private
 
     def on_server_connect(conn)
-      handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins, @add_query_params)
+      handler = Handler.new(conn, @km, method(:on_request),
+                            @body_size_limit, @format_name, log,
+                            @cors_allow_origins, @cors_allow_credentials,
+                            @add_query_params)
 
       conn.on(:data) do |data|
         handler.on_read(data)
@@ -356,7 +367,8 @@ module Fluent::Plugin
     class Handler
       attr_reader :content_type
 
-      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins, add_query_params)
+      def initialize(io, km, callback, body_size_limit, format_name, log,
+                     cors_allow_origins, cors_allow_credentials, add_query_params)
         @io = io
         @km = km
         @callback = callback
@@ -365,6 +377,7 @@ module Fluent::Plugin
         @format_name = format_name
         @log = log
         @cors_allow_origins = cors_allow_origins
+        @cors_allow_credentials = cors_allow_credentials
         @idle = 0
         @add_query_params = add_query_params
         @km.add(self)
@@ -491,6 +504,9 @@ module Fluent::Plugin
           send_response_and_close(RES_200_STATUS, header, "")
         elsif include_cors_allow_origin
           header["Access-Control-Allow-Origin"] = @origin
+          if @cors_allow_credentials
+            header["Access-Control-Allow-Credentials"] = true
+          end
           send_response_and_close(RES_200_STATUS, header, "")
         else
           send_response_and_close(RES_403_STATUS, {}, "")
@@ -576,6 +592,9 @@ module Fluent::Plugin
             header['Access-Control-Allow-Origin'] = '*'
           elsif include_cors_allow_origin
             header['Access-Control-Allow-Origin'] = @origin
+            if @cors_allow_credentials
+              header["Access-Control-Allow-Credentials"] = true
+            end
           end
         end
 

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -238,7 +238,7 @@ module Fluent::Plugin
       'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
       'buffer_timekeys' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.timekeys },
       'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },
-      'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
+      'retry_count' => ->(){ respond_to?(:num_errors) ? num_errors : nil },
     }
 
     def all_plugins
@@ -335,7 +335,9 @@ module Fluent::Plugin
       }
 
       if pe.respond_to?(:statistics)
-        obj.merge!(pe.statistics['output'] || {})
+        obj.merge!(pe.statistics.dig('output') || {})
+        obj.merge!(pe.statistics.dig('filter') || {})
+        obj.merge!(pe.statistics.dig('input') || {})
       end
 
       obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)

data/lib/fluent/plugin/in_syslog.rb

@@ -89,6 +89,8 @@ module Fluent::Plugin
     config_param :source_hostname_key, :string, default: nil
     desc 'Try to resolve hostname from IP addresses or not.'
     config_param :resolve_hostname, :bool, default: nil
+    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
+    config_param :send_keepalive_packet, :bool, default: false
     desc 'The field name of source address of sender.'
     config_param :source_address_key, :string, default: nil
     desc 'The field name of the severity.'
@@ -143,6 +145,11 @@ module Fluent::Plugin
       end
 
       @_event_loop_run_timeout = @blocking_timeout
+
+      protocol = @protocol_type || @transport_config.protocol
+      if @send_keepalive_packet && protocol == :udp
+        raise Fluent::ConfigError, "send_keepalive_packet is available for tcp/tls"
+      end
     end
 
     def multi_workers_ready?
@@ -173,7 +180,12 @@ module Fluent::Plugin
 
       delimiter = octet_count_frame ? " " : @delimiter
       delimiter_size = delimiter.size
-      server_create_connection(tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
+      server_create_connection(
+        tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port,
+        bind: @bind,
+        resolve_name: @resolve_hostname,
+        send_keepalive_packet: @send_keepalive_packet
+      ) do |conn|
         conn.data do |data|
           buffer = conn.buffer
           buffer << data