fluent-plugin-time-cutoff 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5eddac80384bc759095b8ead85af572e50aafb3d0820b8f87b46b8fbc7aa6361
-  data.tar.gz: 2df3ffbc38531125d8b893eec4a144ffda0348505efa46a9d9a29412a5da1505
+  metadata.gz: a52993553c1cdc40bb6800791120e5248662a1f84546bf13b686c90c16f5c35d
+  data.tar.gz: c58051186b9056ee4fe0b6aa25850e11d1013de01038cca983323e525e551bdd
 SHA512:
-  metadata.gz: 3bb580a34a68cd2859431f64cb034046b7ffaee4680241617d9f279ed0eb2140b55a1b610dccbf7b35558c1e1b92e3f39b42e9f5db35eab5334cbe1e6dd64918
-  data.tar.gz: ff4f97ac6abec3a87807cee266cbf1597dfeb8dae566ff742ba797aef4946238ec89ecdc89374be259dfaea1ae4730e476cdb0dfc46369c2b7023b63abe173f8
+  metadata.gz: 7b2c8c998860001c4142850878506fa29978fd996049019babdc7331dc74b6889c90a1caa55344075c4d399e883cdf384e9e7f2f3f9b8f3c7ba07c38c181a45c
+  data.tar.gz: 1be548ff25ac8d3e9db3314a3c01a36404c7c77cae535f1857366deb87ce2b62199913b84ffd3ffbf88c2de86a421c0cb5f4f936b722f8835d13cbda2064982b

data/README.md

@@ -104,20 +104,22 @@ $ bundle
  
   old_cutoff 24h
   old_action pass
-  old_log true
+  old_log full
 
   new_cutoff 24h
   new_action pass
-  new_log true
+  new_log full
 
   source_time_key source_time
   source_time_format iso8601
 
   bad_time_action replace_timestamp
-  bad_time_log true
+  bad_time_log full
   bad_time_key source_time_raw
 
   time_key_conflict skip
+
+  metrics false
 </filter>
 
 ```
@@ -138,11 +140,15 @@ Available values: pass, replace_timestamp, drop
 
 Default value: `pass`.
 
-### old_log (bool) (optional)
+### old_log (enum) (optional)
+
+Log the "old" messages to Fluentd's own log. Defaults to "full".
+
+Available values: none (or false/no), metadata, full (or true/yes)
 
-Log the "old" messages to Fluentd's own log. Defaults to "true".
+See [Logging](#logging) for what each mode logs.
 
-Default value: `true`.
+Default value: `full`.
 
 ### new_cutoff (time) (optional)
 
@@ -160,11 +166,15 @@ Available values: pass, replace_timestamp, drop
 
 Default value: `pass`.
 
-### new_log (bool) (optional)
+### new_log (enum) (optional)
+
+Log the "new" messages to Fluentd's own log. Defaults to "full".
+
+Available values: none (or false/no), metadata, full (or true/yes)
 
-Log the "new" messages to Fluentd's own log. Defaults to "true".
+See [Logging](#logging) for what each mode logs.
 
-Default value: `true`.
+Default value: `full`.
 
 ### source_time_key (string) (optional)
 
@@ -186,17 +196,21 @@ Default value: `iso8601`.
 
 The action that will be performed to all messages with an unparseable time. Defaults to replacing it with current time.
 
-Event times that aren't a `Fluent::EventTime` are parsed before the age check. Integers, floats, numeric strings and ISO 8601 datetime strings are accepted. ISO 8601 strings may use a space instead of `T` and may have a space before the timezone (e.g. `2025-04-08 14:43:14 +0300` or `2025-04-08 11:43:14 UTC`, as produced by Ruby's `Time#to_s`). Strings without a timezone are treated as local time. Any other time is deemed unparseable and is handled according to this action (and logged, unless `bad_time_log` is disabled).
+Event times that aren't a `Fluent::EventTime` are parsed before the age check. Integers, floats, numeric strings and ISO 8601 datetime strings are accepted. ISO 8601 strings may use a space instead of `T` and may have a space before the timezone (e.g. `2025-04-08 14:43:14 +0300` or `2025-04-08 11:43:14 UTC`, as produced by Ruby's `Time#to_s`). Strings without a timezone are treated as local time. Any other time is deemed unparseable and is handled according to this action (and logged, unless `bad_time_log` is `none`).
 
 Available values: replace_timestamp, drop
 
 Default value: `replace_timestamp`.
 
-### bad_time_log (bool) (optional)
+### bad_time_log (enum) (optional)
 
-Log the messages with an unparseable time to Fluentd's own log. Defaults to "true".
+Log the messages with an unparseable time to Fluentd's own log. Defaults to "full".
 
-Default value: `true`.
+Available values: none (or false/no), metadata, full (or true/yes)
+
+See [Logging](#logging) for what each mode logs.
+
+Default value: `full`.
 
 ### bad_time_key (string) (optional)
 
@@ -215,13 +229,78 @@ The action that will be performed when the record already has source_time_key or
 This happens e.g. when a record passes through more than one `time_cutoff` filter, in which case the existing value is the true original time. The event time itself is replaced with current time in any case.
 
 * `skip`: keep the existing value.
-* `warn`: keep the existing value and log a warning to Fluentd's own log.
+* `warn`: keep the existing value and log a warning to Fluentd's own log. The warning is logged even if the record's log mode is `none`, but includes the record only in `full` mode.
 * `overwrite`: replace the existing value.
 
 Available values: skip, warn, overwrite
 
 Default value: `skip`.
 
+### metrics (bool) (optional)
+
+Count caught records in a Prometheus counter (needs the prometheus-client gem). Defaults to "false".
+
+See [Metrics](#metrics) for details.
+
+Default value: `false`.
+
+## Logging
+
+Each caught record is logged as a warning to Fluentd's own log, according to the `old_log`, `new_log` or `bad_time_log` mode:
+
+* `none` (or `false`/`no`): nothing is logged.
+* `metadata`: only the tag, the event time and its age (or the raw unparseable time) are logged, but not the record itself. Use it for records that may hold sensitive data, e.g. audit logs.
+* `full` (or `true`/`yes`): the whole record is logged as well.
+
+```
+# metadata
+[warn]: [time_cutoff] Record caught [old, drop]: "vault.audit: 2025-04-07T14:43:24+03:00 (age 1d 0h 0m 0s)".
+[warn]: [time_cutoff] Record caught [new, pass]: "vault.audit: 2025-04-08T16:43:29+03:00 (age -2h 0m 5s)".
+[warn]: [time_cutoff] Record caught [bad_time, drop]: "vault.audit: "not-a-time"".
+
+# full
+[warn]: [time_cutoff] Record caught [old, drop]: "vault.audit: 2025-04-07T14:43:24+03:00 {"message":"hello"}".
+[warn]: [time_cutoff] Record caught [bad_time, drop]: "vault.audit: "not-a-time" {"message":"hello"}".
+```
+
+## Metrics
+
+With `metrics true`, caught records are counted in a Prometheus counter:
+
+```
+fluentd_time_cutoff_records_total{plugin_id, worker_id, hostname, tag, reason, action}
+```
+
+* `plugin_id`: the filter's `@id`. Give each `time_cutoff` filter a stable `@id` to tell them apart.
+* `worker_id`, `hostname`: the Fluentd worker and host, so that series from multiple workers (e.g. on fluent-plugin-prometheus's `/aggregated_metrics` endpoint) and hosts don't collide.
+* `tag`: the record's tag.
+* `reason`: why the record was caught: `old`, `new` or `bad_time`.
+* `action`: what was done to it: `pass`, `replace_timestamp` or `drop`.
+
+Records inside the time window aren't counted. Records are counted regardless of their log mode, so the counter can replace per-record logging.
+
+The counter is registered in the default registry of the [prometheus-client](https://github.com/prometheus/client_ruby) gem (version 2.1 or newer), which is what [fluent-plugin-prometheus](https://github.com/fluent/fluent-plugin-prometheus) serves, so adding its `prometheus` input is enough to expose it:
+
+``` xml
+<source>
+  @type prometheus
+</source>
+
+<filter **>
+  @type time_cutoff
+  @id cutoff_main
+  metrics true
+</filter>
+```
+
+The prometheus-client gem is an optional dependency of this plugin. If it's not installed, enabling `metrics` only logs a warning on startup. Errors while updating the counter are logged once and never affect how records are processed.
+
+Example query, records dropped per second for being too old, by tag:
+
+```
+sum by (tag) (rate(fluentd_time_cutoff_records_total{reason="old", action="drop"}[5m]))
+```
+
 ## Copyright
 
 * Copyright(c) 2025-2026 Qrator Labs and Serge Tkatchouk

data/lib/fluent/plugin/filter_time_cutoff.rb

@@ -2,6 +2,7 @@
 
 require 'fluent/plugin/filter'
 require 'json'
+require 'socket'
 require 'time'
 
 module Fluent
@@ -50,6 +51,29 @@ module Fluent
         :overwrite
       ].freeze
 
+      # Available log modes for caught records.
+      LOG_MODES = [
+        # Do not log:
+        :none,
+        # Log tag, time and age (or the raw unparseable time) only:
+        :metadata,
+        # Log the whole record:
+        :full,
+        # Aliases for :none and :full, kept for compatibility with the
+        # boolean *_log options of earlier versions:
+        :false,
+        :no,
+        :true,
+        :yes
+      ].freeze
+
+      # Log modes the boolean aliases stand for.
+      LOG_MODE_ALIASES = { false: :none, no: :none, true: :full, yes: :full }.freeze
+
+      # Name and label names of the Prometheus counter for caught records.
+      METRIC_NAME = :fluentd_time_cutoff_records_total
+      METRIC_LABELS = %i[plugin_id worker_id hostname tag reason action].freeze
+
       # strftime() format string for :iso8601 time format.
       TIME_ISO8601 = '%FT%T%:z'
 
@@ -72,26 +96,28 @@ module Fluent
       config_param :old_cutoff, :time, default: 86_400 # 24 hours
       desc 'The action that will be performed to all "old" messages. Defaults to passthrough.'
       config_param :old_action, :enum, list: CUTOFF_ACTIONS, default: :pass
-      desc 'Log the "old" messages to Fluentd\'s own log. Defaults to "true".'
-      config_param :old_log, :bool, default: true
+      desc 'Log the "old" messages to Fluentd\'s own log: none (false/no), metadata or full (true/yes). Defaults to "full".'
+      config_param :old_log, :enum, list: LOG_MODES, default: :full
       desc 'Records newer than this amount of time are deemed "new". Defaults to 24 hours.'
       config_param :new_cutoff, :time, default: 86_400 # 24 hours
       desc 'The action that will be performed to all "new" messages. Defaults to passthrough.'
       config_param :new_action, :enum, list: CUTOFF_ACTIONS, default: :pass
-      desc 'Log the "new" messages to Fluentd\'s own log. Defaults to "true".'
-      config_param :new_log, :bool, default: true
+      desc 'Log the "new" messages to Fluentd\'s own log: none (false/no), metadata or full (true/yes). Defaults to "full".'
+      config_param :new_log, :enum, list: LOG_MODES, default: :full
       desc 'The key that will hold the original time (for :replace_timestamp action). Defaults to "source_time".'
       config_param :source_time_key, :string, default: 'source_time'
       desc 'Time format for the original timestamp field. Defaults to ISO8601-formatted string.'
       config_param :source_time_format, :enum, list: TIME_FORMATS, default: :iso8601
       desc 'The action that will be performed to all messages with an unparseable time. Defaults to replacing it with current time.'
       config_param :bad_time_action, :enum, list: BAD_TIME_ACTIONS, default: :replace_timestamp
-      desc 'Log the messages with an unparseable time to Fluentd\'s own log. Defaults to "true".'
-      config_param :bad_time_log, :bool, default: true
+      desc 'Log the messages with an unparseable time to Fluentd\'s own log: none (false/no), metadata or full (true/yes). Defaults to "full".'
+      config_param :bad_time_log, :enum, list: LOG_MODES, default: :full
       desc 'The key that will hold the original unparseable time (for :replace_timestamp action). Defaults to "source_time_raw".'
       config_param :bad_time_key, :string, default: 'source_time_raw'
       desc 'The action that will be performed when the record already has source_time_key or bad_time_key. Defaults to keeping the existing value.'
       config_param :time_key_conflict, :enum, list: TIME_KEY_CONFLICT_ACTIONS, default: :skip
+      desc 'Count caught records in a Prometheus counter (needs the prometheus-client gem). Defaults to "false".'
+      config_param :metrics, :bool, default: false
 
       def configure(conf)
         super
@@ -107,9 +133,16 @@ module Fluent
         %i[source_time_key bad_time_key].each do |param|
           raise Fluent::ConfigError, "#{param} must not be empty" if instance_variable_get("@#{param}").empty?
         end
-        return unless @source_time_key == @bad_time_key
+        if @source_time_key == @bad_time_key
+          raise Fluent::ConfigError, "source_time_key and bad_time_key must differ, both are \"#{@source_time_key}\""
+        end
 
-        raise Fluent::ConfigError, "source_time_key and bad_time_key must differ, both are \"#{@source_time_key}\""
+        %i[old_log new_log bad_time_log].each do |param|
+          mode = instance_variable_get("@#{param}")
+          instance_variable_set("@#{param}", LOG_MODE_ALIASES.fetch(mode, mode))
+        end
+
+        setup_metrics
       end
 
       def filter_with_time(tag, time, record)
@@ -139,6 +172,67 @@ module Fluent
 
       private
 
+      # Set up the Prometheus counter for caught records, if enabled. The
+      # counter is shared by all time_cutoff filters in the process.
+      def setup_metrics
+        return unless @metrics
+
+        unless load_prometheus_client
+          log.warn 'Metrics are enabled, but the prometheus-client gem is not installed, so caught records will not be counted.'
+          return
+        end
+
+        registry = prometheus_registry
+        @records_counter = registry.get(METRIC_NAME) || registry.counter(
+          METRIC_NAME,
+          docstring: 'Number of records caught by the time_cutoff filter.',
+          labels: METRIC_LABELS
+        )
+        @metric_labels = { plugin_id: plugin_id, worker_id: worker_id.to_s, hostname: Socket.gethostname }
+      end
+
+      # Load the prometheus-client gem, which is an optional dependency.
+      # @return [Boolean] whether the gem could be loaded
+      def load_prometheus_client
+        require 'prometheus/client'
+        true
+      rescue LoadError
+        false
+      end
+
+      # The Prometheus registry to register the counter in. This is the one
+      # fluent-plugin-prometheus serves on its /metrics endpoints.
+      def prometheus_registry
+        ::Prometheus::Client.registry
+      end
+
+      # ID of the Fluentd worker this plugin runs in.
+      # NOTE: Plugin::Base#fluentd_worker_id isn't available in the oldest
+      # Fluentd releases declared in the gemspec, so fall back to the env var
+      # it reads.
+      # @return [Integer] worker ID
+      def worker_id
+        return fluentd_worker_id if respond_to?(:fluentd_worker_id, true)
+
+        (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+      end
+
+      # Count a caught record. Errors are logged once and never affect how
+      # the record is processed.
+      # @param tag [String] log/stream's tag name
+      # @param reason [Symbol] why the record was caught (:old, :new or :bad_time)
+      # @param action [Symbol] determined action for this record
+      def count_record(tag, reason, action)
+        return unless @records_counter
+
+        @records_counter.increment(labels: @metric_labels.merge(tag: tag, reason: reason.to_s, action: action.to_s))
+      rescue StandardError => e
+        return if @metrics_error_logged
+
+        @metrics_error_logged = true
+        log.warn "Failed to update metrics, further errors will not be logged: #{e.class}: #{e.message}"
+      end
+
       # Current UNIX time, used as the reference point for the age check.
       # Kept as a separate method so that tests can freeze the clock.
       # @return [Integer] current time in seconds
@@ -176,10 +270,11 @@ module Fluent
       # @param tag [String] log/stream's tag name
       # @param time [Fluent::EventTime] the original event time
       # @param record [Hash] the original record contents
+      # @param log_mode [Symbol] log mode for this record
       # @param @source_time_key [String] name of the field to put the old timestamp to.
       # @return [Array] new event time and a modified record.
-      def do_replace_timestamp(tag, time, record)
-        [Fluent::EventTime.now, put_time_key(tag, record, @source_time_key, format_time(time))]
+      def do_replace_timestamp(tag, time, record, log_mode)
+        [Fluent::EventTime.now, put_time_key(tag, record, @source_time_key, format_time(time), log_mode)]
       end
 
       # Put the original time into the given field of a copy of the record,
@@ -189,10 +284,11 @@ module Fluent
       # @param record [Hash] the original record contents
       # @param key [String] name of the field to put the time to
       # @param value the time value to put into the field
+      # @param log_mode [Symbol] log mode for this record
       # @return [Hash] the modified copy of the record, or the original record if kept as is
-      def put_time_key(tag, record, key, value)
+      def put_time_key(tag, record, key, value, log_mode) # rubocop:disable Metrics/ParameterLists
         if record.key?(key) && @time_key_conflict != :overwrite
-          log_time_key_conflict(tag, record, key, value) if @time_key_conflict == :warn
+          log_time_key_conflict(tag, record, key, value, log_mode) if @time_key_conflict == :warn
           return record
         end
 
@@ -201,44 +297,57 @@ module Fluent
         new_record
       end
 
-      # Log a time key conflict to Fluentd's log.
+      # Log a time key conflict to Fluentd's log. It is logged even if the
+      # record's log mode is :none, since it was asked for explicitly, but the
+      # record itself is only included in :full mode.
       # @param tag [String] log/stream's tag name
       # @param record [Hash] the original record contents
       # @param key [String] name of the conflicting field
       # @param value the time value that was not put into the field
-      def log_time_key_conflict(tag, record, key, value)
-        log_line = format(
-          'Record already has the "%<key>s" key, keeping its existing value instead of %<val>s: "%<tag>s: %<msg>s".',
-          key: key, val: value.inspect, tag: tag, msg: dump_record(record)
+      # @param log_mode [Symbol] log mode for this record
+      def log_time_key_conflict(tag, record, key, value, log_mode) # rubocop:disable Metrics/ParameterLists
+        subject = log_mode == :full ? "#{tag}: #{dump_record(record)}" : tag
+        log.warn format(
+          'Record already has the "%<key>s" key, keeping its existing value instead of %<val>s: "%<subject>s".',
+          key: key, val: value.inspect, subject: subject
         )
-        log.warn log_line
       end
 
       # Log the caught event to Fluentd's log.
       # @param tag [String] log/stream's tag name
-      # @param time [Fluent::EventTime] the original event time
+      # @param time [Fluent::EventTime] the original event time, or the raw
+      #   time for :bad_time records
       # @param record [Hash] the original record contents
       # @param action [Symbol] determined action for this record
-      # @param age [Symbol] detemined age for this record (:old or :new)
-      def log_record(tag, time, record, action, age)
-        log_line = format(
-          'Record caught [%<age>s, %<act>s]: "%<tag>s: %<time>s %<msg>s".',
-          age: age, act: action, tag: tag, time: iso8601(time), msg: dump_record(record)
+      # @param reason [Symbol] why the record was caught (:old, :new or :bad_time)
+      # @param log_mode [Symbol] :none, :metadata (no record contents) or :full
+      def log_record(tag, time, record, action, reason, log_mode) # rubocop:disable Metrics/ParameterLists
+        return if log_mode == :none
+
+        details = reason == :bad_time ? time.inspect : iso8601(time)
+        if log_mode == :full
+          details = "#{details} #{dump_record(record)}"
+        elsif reason != :bad_time
+          details = "#{details} (age #{format_age(current_time - time.to_i)})"
+        end
+        log.warn format(
+          'Record caught [%<reason>s, %<act>s]: "%<tag>s: %<details>s".',
+          reason: reason, act: action, tag: tag, details: details
         )
-        log.warn log_line
       end
 
-      # Log the caught event with an unparseable time to Fluentd's log.
-      # @param tag [String] log/stream's tag name
-      # @param time the original (unparseable) event time
-      # @param record [Hash] the original record contents
-      # @param action [Symbol] determined action for this record
-      def log_bad_time_record(tag, time, record, action)
-        log_line = format(
-          'Record caught [unparseable, %<act>s]: "%<tag>s: %<time>s %<msg>s".',
-          act: action, tag: tag, time: time.inspect, msg: dump_record(record)
-        )
-        log.warn log_line
+      # Format an age in seconds as a compact human-readable string, e.g.
+      # "1d 2h 3m 4s", "5m 0s" or "-2h 0m 5s" for times in the future.
+      # @param seconds [Integer] the age in seconds
+      # @return [String] the formatted age
+      def format_age(seconds)
+        rest = seconds.abs
+        parts = [[86_400, 'd'], [3600, 'h'], [60, 'm'], [1, 's']].map do |size, unit|
+          value, rest = rest.divmod(size)
+          "#{value}#{unit}"
+        end
+        parts.shift while parts.size > 1 && parts.first.start_with?('0')
+        "#{'-' if seconds.negative?}#{parts.join(' ')}"
       end
 
       # Serialize the record for logging. Falls back to #inspect when the
@@ -304,17 +413,18 @@ module Fluent
       # @param tag [String] log/stream's tag name
       # @param time [Fluent::EventTime] the original event time
       # @param record [Hash] the original record contents
-      # @param do_log [Boolean] should we log this record or not
+      # @param log_mode [Symbol] log mode for this record
       # @param action [Symbol] determined action for this record
-      # @param age [Symbol] detemined age for this record (:old or :new)
-      def process_record(tag, time, record, do_log, action, age) # rubocop:disable Metrics/ParameterLists
-        log_record(tag, time, record, action, age) if do_log
+      # @param reason [Symbol] why the record was caught (:old or :new)
+      def process_record(tag, time, record, log_mode, action, reason) # rubocop:disable Metrics/ParameterLists
+        count_record(tag, reason, action)
+        log_record(tag, time, record, action, reason, log_mode)
 
         case action
         when :pass
           [time, record]
         when :replace_timestamp
-          do_replace_timestamp(tag, time, record)
+          do_replace_timestamp(tag, time, record, log_mode)
         when :drop
           nil
         end
@@ -326,7 +436,8 @@ module Fluent
       # @param record [Hash] the original record contents
       # @return [Array,nil] current time and the record, or nil if dropped.
       def process_bad_time_record(tag, time, record)
-        log_bad_time_record(tag, time, record, @bad_time_action) if @bad_time_log
+        count_record(tag, :bad_time, @bad_time_action)
+        log_record(tag, time, record, @bad_time_action, :bad_time, @bad_time_log)
 
         case @bad_time_action
         when :replace_timestamp
@@ -347,7 +458,7 @@ module Fluent
       # @return [Array] new event time and a modified record.
       def do_replace_bad_timestamp(tag, time, record)
         raw_time = time.is_a?(String) ? time : time.inspect
-        [Fluent::EventTime.now, put_time_key(tag, record, @bad_time_key, raw_time)]
+        [Fluent::EventTime.now, put_time_key(tag, record, @bad_time_key, raw_time, @bad_time_log)]
       end
 
       # Process the "old" record according to the configuration

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-time-cutoff
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Qrator Labs
@@ -30,6 +30,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement