fluent-plugin-time-cutoff 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77b479a1cb54f5eb1a189032c5c0acfde4a99f0f813afb800ec84d39a4e09e2e
-  data.tar.gz: 3d27e8cee579842f6705467ddbd8577f47aa72db134f592a82676c01ef758166
+  metadata.gz: a52993553c1cdc40bb6800791120e5248662a1f84546bf13b686c90c16f5c35d
+  data.tar.gz: c58051186b9056ee4fe0b6aa25850e11d1013de01038cca983323e525e551bdd
 SHA512:
-  metadata.gz: 12ea75e58112114c23637be4f19083324040a64bd1528f9628da85077316c52b285aaafd54162a50421da9eff4df4e02d08754399a20cde5f87f374ec357a258
-  data.tar.gz: c4731fb66d9b66fc6778a5a614e5534d2f600f61c15ab26adaa75687fdcede90ebccf9714a97b1be69d27761ecf412d2f66917834e9e46ecbb211d365a09d381
+  metadata.gz: 7b2c8c998860001c4142850878506fa29978fd996049019babdc7331dc74b6889c90a1caa55344075c4d399e883cdf384e9e7f2f3f9b8f3c7ba07c38c181a45c
+  data.tar.gz: 1be548ff25ac8d3e9db3314a3c01a36404c7c77cae535f1857366deb87ce2b62199913b84ffd3ffbf88c2de86a421c0cb5f4f936b722f8835d13cbda2064982b

data/README.md

@@ -104,14 +104,22 @@ $ bundle
  
   old_cutoff 24h
   old_action pass
-  old_log true
+  old_log full
 
   new_cutoff 24h
   new_action pass
-  new_log true
+  new_log full
 
   source_time_key source_time
   source_time_format iso8601
+
+  bad_time_action replace_timestamp
+  bad_time_log full
+  bad_time_key source_time_raw
+
+  time_key_conflict skip
+
+  metrics false
 </filter>
 
 ```
@@ -120,6 +128,8 @@ $ bundle
 
 Records older than this amount of time are deemed "old". Defaults to 24 hours.
 
+Must not be negative.
+
 Default value: `86400`.
 
 ### old_action (enum) (optional)
@@ -130,16 +140,22 @@ Available values: pass, replace_timestamp, drop
 
 Default value: `pass`.
 
-### old_log (bool) (optional)
+### old_log (enum) (optional)
 
-Log the "old" messages to Fluentd's own log. Defaults to "true".
+Log the "old" messages to Fluentd's own log. Defaults to "full".
 
-Default value: `true`.
+Available values: none (or false/no), metadata, full (or true/yes)
+
+See [Logging](#logging) for what each mode logs.
+
+Default value: `full`.
 
 ### new_cutoff (time) (optional)
 
 Records newer than this amount of time are deemed "new". Defaults to 24 hours.
 
+Must not be negative.
+
 Default value: `86400`.
 
 ### new_action (enum) (optional)
@@ -150,16 +166,22 @@ Available values: pass, replace_timestamp, drop
 
 Default value: `pass`.
 
-### new_log (bool) (optional)
+### new_log (enum) (optional)
+
+Log the "new" messages to Fluentd's own log. Defaults to "full".
 
-Log the "new" messages to Fluentd's own log. Defaults to "true".
+Available values: none (or false/no), metadata, full (or true/yes)
 
-Default value: `true`.
+See [Logging](#logging) for what each mode logs.
+
+Default value: `full`.
 
 ### source_time_key (string) (optional)
 
 The key that will hold the original time (for :replace_timestamp action). Defaults to "source_time".
 
+Must not be empty or equal to `bad_time_key`.
+
 Default value: `source_time`.
 
 ### source_time_format (enum) (optional)
@@ -170,7 +192,116 @@ Available values: epoch, epoch_float, iso8601
 
 Default value: `iso8601`.
 
+### bad_time_action (enum) (optional)
+
+The action that will be performed to all messages with an unparseable time. Defaults to replacing it with current time.
+
+Event times that aren't a `Fluent::EventTime` are parsed before the age check. Integers, floats, numeric strings and ISO 8601 datetime strings are accepted. ISO 8601 strings may use a space instead of `T` and may have a space before the timezone (e.g. `2025-04-08 14:43:14 +0300` or `2025-04-08 11:43:14 UTC`, as produced by Ruby's `Time#to_s`). Strings without a timezone are treated as local time. Any other time is deemed unparseable and is handled according to this action (and logged, unless `bad_time_log` is `none`).
+
+Available values: replace_timestamp, drop
+
+Default value: `replace_timestamp`.
+
+### bad_time_log (enum) (optional)
+
+Log the messages with an unparseable time to Fluentd's own log. Defaults to "full".
+
+Available values: none (or false/no), metadata, full (or true/yes)
+
+See [Logging](#logging) for what each mode logs.
+
+Default value: `full`.
+
+### bad_time_key (string) (optional)
+
+The key that will hold the original unparseable time (for :replace_timestamp action). Defaults to "source_time_raw".
+
+Must not be empty or equal to `source_time_key`.
+
+The original time is stored as is if it's a string, or as its Ruby `#inspect` representation otherwise (e.g. `nil`), so this field always holds a string. It is kept separate from `source_time_key`, so that `source_time_key` always holds a value in `source_time_format` and doesn't cause type conflicts in downstream storage.
+
+Default value: `source_time_raw`.
+
+### time_key_conflict (enum) (optional)
+
+The action that will be performed when the record already has source_time_key or bad_time_key. Defaults to keeping the existing value.
+
+This happens e.g. when a record passes through more than one `time_cutoff` filter, in which case the existing value is the true original time. The event time itself is replaced with current time in any case.
+
+* `skip`: keep the existing value.
+* `warn`: keep the existing value and log a warning to Fluentd's own log. The warning is logged even if the record's log mode is `none`, but includes the record only in `full` mode.
+* `overwrite`: replace the existing value.
+
+Available values: skip, warn, overwrite
+
+Default value: `skip`.
+
+### metrics (bool) (optional)
+
+Count caught records in a Prometheus counter (needs the prometheus-client gem). Defaults to "false".
+
+See [Metrics](#metrics) for details.
+
+Default value: `false`.
+
+## Logging
+
+Each caught record is logged as a warning to Fluentd's own log, according to the `old_log`, `new_log` or `bad_time_log` mode:
+
+* `none` (or `false`/`no`): nothing is logged.
+* `metadata`: only the tag, the event time and its age (or the raw unparseable time) are logged, but not the record itself. Use it for records that may hold sensitive data, e.g. audit logs.
+* `full` (or `true`/`yes`): the whole record is logged as well.
+
+```
+# metadata
+[warn]: [time_cutoff] Record caught [old, drop]: "vault.audit: 2025-04-07T14:43:24+03:00 (age 1d 0h 0m 0s)".
+[warn]: [time_cutoff] Record caught [new, pass]: "vault.audit: 2025-04-08T16:43:29+03:00 (age -2h 0m 5s)".
+[warn]: [time_cutoff] Record caught [bad_time, drop]: "vault.audit: "not-a-time"".
+
+# full
+[warn]: [time_cutoff] Record caught [old, drop]: "vault.audit: 2025-04-07T14:43:24+03:00 {"message":"hello"}".
+[warn]: [time_cutoff] Record caught [bad_time, drop]: "vault.audit: "not-a-time" {"message":"hello"}".
+```
+
+## Metrics
+
+With `metrics true`, caught records are counted in a Prometheus counter:
+
+```
+fluentd_time_cutoff_records_total{plugin_id, worker_id, hostname, tag, reason, action}
+```
+
+* `plugin_id`: the filter's `@id`. Give each `time_cutoff` filter a stable `@id` to tell them apart.
+* `worker_id`, `hostname`: the Fluentd worker and host, so that series from multiple workers (e.g. on fluent-plugin-prometheus's `/aggregated_metrics` endpoint) and hosts don't collide.
+* `tag`: the record's tag.
+* `reason`: why the record was caught: `old`, `new` or `bad_time`.
+* `action`: what was done to it: `pass`, `replace_timestamp` or `drop`.
+
+Records inside the time window aren't counted. Records are counted regardless of their log mode, so the counter can replace per-record logging.
+
+The counter is registered in the default registry of the [prometheus-client](https://github.com/prometheus/client_ruby) gem (version 2.1 or newer), which is what [fluent-plugin-prometheus](https://github.com/fluent/fluent-plugin-prometheus) serves, so adding its `prometheus` input is enough to expose it:
+
+``` xml
+<source>
+  @type prometheus
+</source>
+
+<filter **>
+  @type time_cutoff
+  @id cutoff_main
+  metrics true
+</filter>
+```
+
+The prometheus-client gem is an optional dependency of this plugin. If it's not installed, enabling `metrics` only logs a warning on startup. Errors while updating the counter are logged once and never affect how records are processed.
+
+Example query, records dropped per second for being too old, by tag:
+
+```
+sum by (tag) (rate(fluentd_time_cutoff_records_total{reason="old", action="drop"}[5m]))
+```
+
 ## Copyright
 
-* Copyright(c) 2025 Qrator Labs and Serge Tkatchouk
+* Copyright(c) 2025-2026 Qrator Labs and Serge Tkatchouk
 * License: MIT

data/lib/fluent/plugin/filter_time_cutoff.rb

@@ -2,6 +2,7 @@
 
 require 'fluent/plugin/filter'
 require 'json'
+require 'socket'
 require 'time'
 
 module Fluent
@@ -31,36 +32,136 @@ module Fluent
         :iso8601
       ].freeze
 
+      # Available actions for records with an unparseable time.
+      BAD_TIME_ACTIONS = [
+        # Replace time with "now" and pass:
+        :replace_timestamp,
+        # Drop record as invalid:
+        :drop
+      ].freeze
+
+      # Available actions for records that already have the key the original
+      # time should be put into (for :replace_timestamp action).
+      TIME_KEY_CONFLICT_ACTIONS = [
+        # Keep the existing value:
+        :skip,
+        # Keep the existing value and log a warning:
+        :warn,
+        # Replace the existing value:
+        :overwrite
+      ].freeze
+
+      # Available log modes for caught records.
+      LOG_MODES = [
+        # Do not log:
+        :none,
+        # Log tag, time and age (or the raw unparseable time) only:
+        :metadata,
+        # Log the whole record:
+        :full,
+        # Aliases for :none and :full, kept for compatibility with the
+        # boolean *_log options of earlier versions:
+        :false,
+        :no,
+        :true,
+        :yes
+      ].freeze
+
+      # Log modes the boolean aliases stand for.
+      LOG_MODE_ALIASES = { false: :none, no: :none, true: :full, yes: :full }.freeze
+
+      # Name and label names of the Prometheus counter for caught records.
+      METRIC_NAME = :fluentd_time_cutoff_records_total
+      METRIC_LABELS = %i[plugin_id worker_id hostname tag reason action].freeze
+
       # strftime() format string for :iso8601 time format.
       TIME_ISO8601 = '%FT%T%:z'
 
+      # Matches a "relaxed" ISO 8601 datetime string, which uses a space
+      # instead of "T" between the date and the time and/or has a space
+      # before the timezone (e.g. Ruby's Time#to_s: "2026-10-10 03:57:04 +0300"
+      # or "2026-10-10 00:57:04 UTC"), so that it can be rebuilt as a strict one.
+      ISO8601_RELAXED = /
+        \A
+        (?<date>\d{4}-\d{2}-\d{2})
+        [T\ ]
+        (?<time>\d{2}:\d{2}:\d{2}(?:\.\d+)?)
+        (?:\ ?(?<zone>Z|UTC|[+-]\d{2}(?::?\d{2})?))?
+        \z
+      /x.freeze
+
       Fluent::Plugin.register_filter('time_cutoff', self)
 
       desc 'Records older than this amount of time are deemed "old". Defaults to 24 hours.'
       config_param :old_cutoff, :time, default: 86_400 # 24 hours
       desc 'The action that will be performed to all "old" messages. Defaults to passthrough.'
       config_param :old_action, :enum, list: CUTOFF_ACTIONS, default: :pass
-      desc 'Log the "old" messages to Fluentd\'s own log. Defaults to "true".'
-      config_param :old_log, :bool, default: true
+      desc 'Log the "old" messages to Fluentd\'s own log: none (false/no), metadata or full (true/yes). Defaults to "full".'
+      config_param :old_log, :enum, list: LOG_MODES, default: :full
       desc 'Records newer than this amount of time are deemed "new". Defaults to 24 hours.'
       config_param :new_cutoff, :time, default: 86_400 # 24 hours
       desc 'The action that will be performed to all "new" messages. Defaults to passthrough.'
       config_param :new_action, :enum, list: CUTOFF_ACTIONS, default: :pass
-      desc 'Log the "new" messages to Fluentd\'s own log. Defaults to "true".'
-      config_param :new_log, :bool, default: true
+      desc 'Log the "new" messages to Fluentd\'s own log: none (false/no), metadata or full (true/yes). Defaults to "full".'
+      config_param :new_log, :enum, list: LOG_MODES, default: :full
       desc 'The key that will hold the original time (for :replace_timestamp action). Defaults to "source_time".'
       config_param :source_time_key, :string, default: 'source_time'
       desc 'Time format for the original timestamp field. Defaults to ISO8601-formatted string.'
       config_param :source_time_format, :enum, list: TIME_FORMATS, default: :iso8601
+      desc 'The action that will be performed to all messages with an unparseable time. Defaults to replacing it with current time.'
+      config_param :bad_time_action, :enum, list: BAD_TIME_ACTIONS, default: :replace_timestamp
+      desc 'Log the messages with an unparseable time to Fluentd\'s own log: none (false/no), metadata or full (true/yes). Defaults to "full".'
+      config_param :bad_time_log, :enum, list: LOG_MODES, default: :full
+      desc 'The key that will hold the original unparseable time (for :replace_timestamp action). Defaults to "source_time_raw".'
+      config_param :bad_time_key, :string, default: 'source_time_raw'
+      desc 'The action that will be performed when the record already has source_time_key or bad_time_key. Defaults to keeping the existing value.'
+      config_param :time_key_conflict, :enum, list: TIME_KEY_CONFLICT_ACTIONS, default: :skip
+      desc 'Count caught records in a Prometheus counter (needs the prometheus-client gem). Defaults to "false".'
+      config_param :metrics, :bool, default: false
+
+      def configure(conf)
+        super
+
+        %i[old_cutoff new_cutoff].each do |param|
+          # NOTE: Fluentd's :time type drops the sign of values with a unit
+          # (e.g. "-1h" is parsed as 3600), so the raw value is checked too.
+          raw = conf[param.to_s].to_s.strip
+          next unless raw.start_with?('-') || instance_variable_get("@#{param}").negative?
+
+          raise Fluent::ConfigError, "#{param} must not be negative, got \"#{raw}\""
+        end
+        %i[source_time_key bad_time_key].each do |param|
+          raise Fluent::ConfigError, "#{param} must not be empty" if instance_variable_get("@#{param}").empty?
+        end
+        if @source_time_key == @bad_time_key
+          raise Fluent::ConfigError, "source_time_key and bad_time_key must differ, both are \"#{@source_time_key}\""
+        end
+
+        %i[old_log new_log bad_time_log].each do |param|
+          mode = instance_variable_get("@#{param}")
+          instance_variable_set("@#{param}", LOG_MODE_ALIASES.fetch(mode, mode))
+        end
+
+        setup_metrics
+      end
 
       def filter_with_time(tag, time, record)
+        ## Safeguard against rogue integer/float/string time. This has to happen
+        ## before classification, so that the age check uses the parsed value.
+        unless time.is_a?(Fluent::EventTime)
+          normalized = normalize_time(time)
+          return process_bad_time_record(tag, time, record) if normalized.nil?
+
+          time = normalized
+        end
+
         event_time = time.to_i
-        current_time = Time.now.to_i
+        now = current_time
 
-        if event_time < current_time - @old_cutoff
+        if event_time < now - @old_cutoff
           # Too old
           process_old_record(tag, time, record)
-        elsif event_time > current_time + @new_cutoff
+        elsif event_time > now + @new_cutoff
           # Too new
           process_new_record(tag, time, record)
         else
@@ -71,6 +172,85 @@ module Fluent
 
       private
 
+      # Set up the Prometheus counter for caught records, if enabled. The
+      # counter is shared by all time_cutoff filters in the process.
+      def setup_metrics
+        return unless @metrics
+
+        unless load_prometheus_client
+          log.warn 'Metrics are enabled, but the prometheus-client gem is not installed, so caught records will not be counted.'
+          return
+        end
+
+        registry = prometheus_registry
+        @records_counter = registry.get(METRIC_NAME) || registry.counter(
+          METRIC_NAME,
+          docstring: 'Number of records caught by the time_cutoff filter.',
+          labels: METRIC_LABELS
+        )
+        @metric_labels = { plugin_id: plugin_id, worker_id: worker_id.to_s, hostname: Socket.gethostname }
+      end
+
+      # Load the prometheus-client gem, which is an optional dependency.
+      # @return [Boolean] whether the gem could be loaded
+      def load_prometheus_client
+        require 'prometheus/client'
+        true
+      rescue LoadError
+        false
+      end
+
+      # The Prometheus registry to register the counter in. This is the one
+      # fluent-plugin-prometheus serves on its /metrics endpoints.
+      def prometheus_registry
+        ::Prometheus::Client.registry
+      end
+
+      # ID of the Fluentd worker this plugin runs in.
+      # NOTE: Plugin::Base#fluentd_worker_id isn't available in the oldest
+      # Fluentd releases declared in the gemspec, so fall back to the env var
+      # it reads.
+      # @return [Integer] worker ID
+      def worker_id
+        return fluentd_worker_id if respond_to?(:fluentd_worker_id, true)
+
+        (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+      end
+
+      # Count a caught record. Errors are logged once and never affect how
+      # the record is processed.
+      # @param tag [String] log/stream's tag name
+      # @param reason [Symbol] why the record was caught (:old, :new or :bad_time)
+      # @param action [Symbol] determined action for this record
+      def count_record(tag, reason, action)
+        return unless @records_counter
+
+        @records_counter.increment(labels: @metric_labels.merge(tag: tag, reason: reason.to_s, action: action.to_s))
+      rescue StandardError => e
+        return if @metrics_error_logged
+
+        @metrics_error_logged = true
+        log.warn "Failed to update metrics, further errors will not be logged: #{e.class}: #{e.message}"
+      end
+
+      # Current UNIX time, used as the reference point for the age check.
+      # Kept as a separate method so that tests can freeze the clock.
+      # @return [Integer] current time in seconds
+      def current_time
+        Time.now.to_i
+      end
+
+      # Format the event time as an ISO 8601 datetime string (in local time).
+      # NOTE: Time.at(int) is used instead of EventTime#to_time, which was
+      # only added to Fluentd in v1.8.0, and instead of Time.at(EventTime),
+      # which relies on Ruby coercing EventTime via #to_r/#to_int. EventTime#to_i
+      # is available in all Fluentd releases declared in the gemspec.
+      # @param time [Fluent::EventTime] the event time
+      # @return [String] ISO 8601-formatted datetime string
+      def iso8601(time)
+        Time.at(time.to_i).strftime(TIME_ISO8601)
+      end
+
       # Format the event time to a given type/format.
       # @param time [Fluent::EventTime] the original event time
       # @param @source_time_format [Symbol] the target format
@@ -82,45 +262,109 @@ module Fluent
         when :epoch_float
           time.to_f
         when :iso8601
-          # NOTE: Time.at(int) is used instead of EventTime#to_time, which was
-          # only added to Fluentd in v1.8.0. This keeps us compatible with the
-          # older Fluentd releases declared in the gemspec.
-          Time.at(time.to_i).strftime(TIME_ISO8601)
+          iso8601(time)
         end
       end
 
       # Replace event time and put the old time into a specified field.
+      # @param tag [String] log/stream's tag name
       # @param time [Fluent::EventTime] the original event time
       # @param record [Hash] the original record contents
+      # @param log_mode [Symbol] log mode for this record
       # @param @source_time_key [String] name of the field to put the old timestamp to.
       # @return [Array] new event time and a modified record.
-      def do_replace_timestamp(time, record)
-        new_time = Fluent::EventTime.now
+      def do_replace_timestamp(tag, time, record, log_mode)
+        [Fluent::EventTime.now, put_time_key(tag, record, @source_time_key, format_time(time), log_mode)]
+      end
+
+      # Put the original time into the given field of a copy of the record,
+      # unless the record already has that field and @time_key_conflict says
+      # to keep it. The original record is never modified.
+      # @param tag [String] log/stream's tag name
+      # @param record [Hash] the original record contents
+      # @param key [String] name of the field to put the time to
+      # @param value the time value to put into the field
+      # @param log_mode [Symbol] log mode for this record
+      # @return [Hash] the modified copy of the record, or the original record if kept as is
+      def put_time_key(tag, record, key, value, log_mode) # rubocop:disable Metrics/ParameterLists
+        if record.key?(key) && @time_key_conflict != :overwrite
+          log_time_key_conflict(tag, record, key, value, log_mode) if @time_key_conflict == :warn
+          return record
+        end
+
         new_record = record.dup
-        old_time = format_time(time)
-        new_record[@source_time_key] = old_time
+        new_record[key] = value
+        new_record
+      end
 
-        [new_time, new_record]
+      # Log a time key conflict to Fluentd's log. It is logged even if the
+      # record's log mode is :none, since it was asked for explicitly, but the
+      # record itself is only included in :full mode.
+      # @param tag [String] log/stream's tag name
+      # @param record [Hash] the original record contents
+      # @param key [String] name of the conflicting field
+      # @param value the time value that was not put into the field
+      # @param log_mode [Symbol] log mode for this record
+      def log_time_key_conflict(tag, record, key, value, log_mode) # rubocop:disable Metrics/ParameterLists
+        subject = log_mode == :full ? "#{tag}: #{dump_record(record)}" : tag
+        log.warn format(
+          'Record already has the "%<key>s" key, keeping its existing value instead of %<val>s: "%<subject>s".',
+          key: key, val: value.inspect, subject: subject
+        )
       end
 
       # Log the caught event to Fluentd's log.
       # @param tag [String] log/stream's tag name
-      # @param time [Fluent::EventTime] the original event time
+      # @param time [Fluent::EventTime] the original event time, or the raw
+      #   time for :bad_time records
       # @param record [Hash] the original record contents
       # @param action [Symbol] determined action for this record
-      # @param age [Symbol] detemined age for this record (:old or :new)
-      def log_record(tag, time, record, action, age)
-        fmt_time = Time.at(time).strftime(TIME_ISO8601)
-        log_line = format(
-          'Record caught [%<age>s, %<act>s]: "%<tag>s: %<time>s %<msg>s".',
-          age: age, act: action, tag: tag, time: fmt_time, msg: record.to_json
+      # @param reason [Symbol] why the record was caught (:old, :new or :bad_time)
+      # @param log_mode [Symbol] :none, :metadata (no record contents) or :full
+      def log_record(tag, time, record, action, reason, log_mode) # rubocop:disable Metrics/ParameterLists
+        return if log_mode == :none
+
+        details = reason == :bad_time ? time.inspect : iso8601(time)
+        if log_mode == :full
+          details = "#{details} #{dump_record(record)}"
+        elsif reason != :bad_time
+          details = "#{details} (age #{format_age(current_time - time.to_i)})"
+        end
+        log.warn format(
+          'Record caught [%<reason>s, %<act>s]: "%<tag>s: %<details>s".',
+          reason: reason, act: action, tag: tag, details: details
         )
-        log.warn log_line
       end
 
-      # Normalize time to Fluent::EventTime if it's plain int or float.
-      # @param time [Integer|Float] incoming event time
-      # @return [Fluent::EventTime] event time normalized to Fluentd's internal format
+      # Format an age in seconds as a compact human-readable string, e.g.
+      # "1d 2h 3m 4s", "5m 0s" or "-2h 0m 5s" for times in the future.
+      # @param seconds [Integer] the age in seconds
+      # @return [String] the formatted age
+      def format_age(seconds)
+        rest = seconds.abs
+        parts = [[86_400, 'd'], [3600, 'h'], [60, 'm'], [1, 's']].map do |size, unit|
+          value, rest = rest.divmod(size)
+          "#{value}#{unit}"
+        end
+        parts.shift while parts.size > 1 && parts.first.start_with?('0')
+        "#{'-' if seconds.negative?}#{parts.join(' ')}"
+      end
+
+      # Serialize the record for logging. Falls back to #inspect when the
+      # record can't be represented as JSON (e.g. invalid UTF-8 or NaN), so
+      # that a logging failure never affects how the record is processed.
+      # @param record [Hash] the original record contents
+      # @return [String] the record as a JSON string, or its #inspect output
+      def dump_record(record)
+        record.to_json
+      rescue JSON::GeneratorError, EncodingError
+        record.inspect
+      end
+
+      # Normalize time to Fluent::EventTime if it's plain int or float, a
+      # numeric string or an ISO 8601 datetime string.
+      # @param time [Integer|Float|String] incoming event time
+      # @return [Fluent::EventTime,nil] event time normalized to Fluentd's internal format, or nil if unparseable
       def normalize_time(time)
         case time
         when Integer
@@ -129,50 +373,94 @@ module Fluent
           time_int, time_frac = time.divmod(1)
           Fluent::EventTime.new(time_int, (time_frac * 10**9).to_i)
         else
-          log.warn "Unknown time format given (#{time.class}): \"#{time.inspect}\"."
-          numeric = coerce_to_number(time)
-          numeric ? normalize_time(numeric) : Fluent::EventTime.now
+          if (numeric = coerce_to_number(time))
+            normalize_time(numeric)
+          elsif (parsed = parse_iso8601(time))
+            Fluent::EventTime.new(parsed.to_i, parsed.nsec)
+          end
         end
       end
 
       # Try to coerce an arbitrary value into a number, or return nil.
-      # Uses begin/rescue rather than the Integer()/Float() "exception:"
-      # keyword (Ruby 2.6+) so the plugin keeps working on older Rubies.
+      # Integer() is called with an explicit base 10 so that zero-padded
+      # strings (e.g. "01700000000") aren't parsed as octal.
       # @param value the value to coerce
       # @return [Integer,Float,nil] the parsed number, or nil if not numeric
       def coerce_to_number(value)
-        Integer(value)
-      rescue ArgumentError, TypeError
-        begin
-          Float(value)
-        rescue ArgumentError, TypeError
-          nil
+        Integer(value, 10, exception: false) || Float(value, exception: false)
+      end
+
+      # Try to parse a strict ISO 8601 datetime string, or return nil.
+      # Time.iso8601 is used instead of the lenient Time.parse, which would
+      # happily turn partial or ambiguous strings into a wrong timestamp.
+      # "Relaxed" strings (see ISO8601_RELAXED) are rebuilt as strict ones
+      # before parsing. Strings without a UTC offset are treated as local time.
+      # @param value the value to parse
+      # @return [Time,nil] the parsed time, or nil if not an ISO 8601 string
+      def parse_iso8601(value)
+        return nil unless value.is_a?(String)
+
+        if (match = ISO8601_RELAXED.match(value))
+          zone = match[:zone] == 'UTC' ? 'Z' : match[:zone]
+          value = "#{match[:date]}T#{match[:time]}#{zone}"
         end
+        Time.iso8601(value)
+      rescue ArgumentError
+        nil
       end
 
       # Process the record according to its determined "age"
       # @param tag [String] log/stream's tag name
       # @param time [Fluent::EventTime] the original event time
       # @param record [Hash] the original record contents
-      # @param do_log [Boolean] should we log this record or not
+      # @param log_mode [Symbol] log mode for this record
       # @param action [Symbol] determined action for this record
-      # @param age [Symbol] detemined age for this record (:old or :new)
-      def process_record(tag, time, record, do_log, action, age) # rubocop:disable Metrics/ParameterLists
-        ## Safeguard against rogue integer time.
-        time = normalize_time(time) unless time.is_a?(Fluent::EventTime)
-
-        log_record(tag, time, record, action, age) if do_log
+      # @param reason [Symbol] why the record was caught (:old or :new)
+      def process_record(tag, time, record, log_mode, action, reason) # rubocop:disable Metrics/ParameterLists
+        count_record(tag, reason, action)
+        log_record(tag, time, record, action, reason, log_mode)
 
         case action
         when :pass
           [time, record]
         when :replace_timestamp
-          do_replace_timestamp(time, record)
+          do_replace_timestamp(tag, time, record, log_mode)
+        when :drop
+          nil
+        end
+      end
+
+      # Process the record with an unparseable time according to the configuration
+      # @param tag [String] log/stream's tag name
+      # @param time the original (unparseable) event time
+      # @param record [Hash] the original record contents
+      # @return [Array,nil] current time and the record, or nil if dropped.
+      def process_bad_time_record(tag, time, record)
+        count_record(tag, :bad_time, @bad_time_action)
+        log_record(tag, time, record, @bad_time_action, :bad_time, @bad_time_log)
+
+        case @bad_time_action
+        when :replace_timestamp
+          do_replace_bad_timestamp(tag, time, record)
         when :drop
           nil
         end
       end
 
+      # Replace unparseable event time and put the raw original time into a
+      # separate field. The raw time is always stored as a string, so that the
+      # field keeps a stable type for downstream storage. It is not put into
+      # @source_time_key, which holds a value in @source_time_format instead.
+      # @param tag [String] log/stream's tag name
+      # @param time the original (unparseable) event time
+      # @param record [Hash] the original record contents
+      # @param @bad_time_key [String] name of the field to put the raw time to.
+      # @return [Array] new event time and a modified record.
+      def do_replace_bad_timestamp(tag, time, record)
+        raw_time = time.is_a?(String) ? time : time.inspect
+        [Fluent::EventTime.now, put_time_key(tag, record, @bad_time_key, raw_time, @bad_time_log)]
+      end
+
       # Process the "old" record according to the configuration
       # @param tag [String] log/stream's tag name
       # @param time [Fluent::EventTime] the original event time

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-time-cutoff
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Qrator Labs
@@ -31,33 +31,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 2.6.3
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 2.6.3
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.1.0
+        version: 13.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.1.0
+        version: 13.2.0
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -81,11 +87,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".rubocop.yml"
 - LICENSE.txt
 - README.md
-- Rakefile
-- fluent-plugin-time-cutoff.gemspec
 - lib/fluent/plugin/filter_time_cutoff.rb
 homepage: https://github.com/QratorLabs/fluent-plugin-time-cutoff
 licenses:
@@ -98,14 +101,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.9
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Fluentd time-based filter plugin.
 test_files: []

data/.rubocop.yml

@@ -1,2 +0,0 @@
-AllCops:
-  TargetRubyVersion: 2.3

data/Rakefile

@@ -1,13 +0,0 @@
-require 'bundler'
-Bundler::GemHelper.install_tasks
-
-require 'rake/testtask'
-
-Rake::TestTask.new(:test) do |t|
-  t.libs.push('lib', 'test')
-  t.test_files = FileList['test/**/test_*.rb']
-  t.verbose = true
-  t.warning = true
-end
-
-task default: [:test]

data/fluent-plugin-time-cutoff.gemspec

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-lib = File.expand_path('../lib', __dir__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-Gem::Specification.new do |spec|
-  spec.name    = 'fluent-plugin-time-cutoff'
-  spec.version = '0.2.0'
-  spec.authors = ['Qrator Labs', 'Serge Tkatchouk']
-  spec.email   = ['devops@qrator.net', 'st@qrator.net']
-
-  spec.summary       = 'Fluentd time-based filter plugin.'
-  spec.description   = 'A plugin that lets Fluentd to prune/rewrite messages '\
-                       'that have a timestamp that is too old or too new.'
-  spec.homepage      = 'https://github.com/QratorLabs/fluent-plugin-time-cutoff'
-  spec.license       = 'MIT'
-
-  spec.files         = Dir.chdir(__dir__) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      (File.expand_path(f) == __FILE__) ||
-        f.start_with?(*%w[test/ testbed/ spec/ features/ .git .circleci appveyor Gemfile])
-    end
-  end
-
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ['lib']
-
-  spec.platform = Gem::Platform::RUBY
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.3')
-
-  spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
-
-  spec.add_development_dependency 'bundler', '~> 2.6.3'
-  spec.add_development_dependency 'rake', '~> 13.1.0'
-  spec.add_development_dependency 'test-unit', '~> 3.6.1'
-end