fluent-plugin-time-cutoff 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77b479a1cb54f5eb1a189032c5c0acfde4a99f0f813afb800ec84d39a4e09e2e
-  data.tar.gz: 3d27e8cee579842f6705467ddbd8577f47aa72db134f592a82676c01ef758166
+  metadata.gz: 5eddac80384bc759095b8ead85af572e50aafb3d0820b8f87b46b8fbc7aa6361
+  data.tar.gz: 2df3ffbc38531125d8b893eec4a144ffda0348505efa46a9d9a29412a5da1505
 SHA512:
-  metadata.gz: 12ea75e58112114c23637be4f19083324040a64bd1528f9628da85077316c52b285aaafd54162a50421da9eff4df4e02d08754399a20cde5f87f374ec357a258
-  data.tar.gz: c4731fb66d9b66fc6778a5a614e5534d2f600f61c15ab26adaa75687fdcede90ebccf9714a97b1be69d27761ecf412d2f66917834e9e46ecbb211d365a09d381
+  metadata.gz: 3bb580a34a68cd2859431f64cb034046b7ffaee4680241617d9f279ed0eb2140b55a1b610dccbf7b35558c1e1b92e3f39b42e9f5db35eab5334cbe1e6dd64918
+  data.tar.gz: ff4f97ac6abec3a87807cee266cbf1597dfeb8dae566ff742ba797aef4946238ec89ecdc89374be259dfaea1ae4730e476cdb0dfc46369c2b7023b63abe173f8

data/README.md

@@ -112,6 +112,12 @@ $ bundle
 
   source_time_key source_time
   source_time_format iso8601
+
+  bad_time_action replace_timestamp
+  bad_time_log true
+  bad_time_key source_time_raw
+
+  time_key_conflict skip
 </filter>
 
 ```
@@ -120,6 +126,8 @@ $ bundle
 
 Records older than this amount of time are deemed "old". Defaults to 24 hours.
 
+Must not be negative.
+
 Default value: `86400`.
 
 ### old_action (enum) (optional)
@@ -140,6 +148,8 @@ Default value: `true`.
 
 Records newer than this amount of time are deemed "new". Defaults to 24 hours.
 
+Must not be negative.
+
 Default value: `86400`.
 
 ### new_action (enum) (optional)
@@ -160,6 +170,8 @@ Default value: `true`.
 
 The key that will hold the original time (for :replace_timestamp action). Defaults to "source_time".
 
+Must not be empty or equal to `bad_time_key`.
+
 Default value: `source_time`.
 
 ### source_time_format (enum) (optional)
@@ -170,7 +182,47 @@ Available values: epoch, epoch_float, iso8601
 
 Default value: `iso8601`.
 
+### bad_time_action (enum) (optional)
+
+The action that will be performed to all messages with an unparseable time. Defaults to replacing it with current time.
+
+Event times that aren't a `Fluent::EventTime` are parsed before the age check. Integers, floats, numeric strings and ISO 8601 datetime strings are accepted. ISO 8601 strings may use a space instead of `T` and may have a space before the timezone (e.g. `2025-04-08 14:43:14 +0300` or `2025-04-08 11:43:14 UTC`, as produced by Ruby's `Time#to_s`). Strings without a timezone are treated as local time. Any other time is deemed unparseable and is handled according to this action (and logged, unless `bad_time_log` is disabled).
+
+Available values: replace_timestamp, drop
+
+Default value: `replace_timestamp`.
+
+### bad_time_log (bool) (optional)
+
+Log the messages with an unparseable time to Fluentd's own log. Defaults to "true".
+
+Default value: `true`.
+
+### bad_time_key (string) (optional)
+
+The key that will hold the original unparseable time (for :replace_timestamp action). Defaults to "source_time_raw".
+
+Must not be empty or equal to `source_time_key`.
+
+The original time is stored as is if it's a string, or as its Ruby `#inspect` representation otherwise (e.g. `nil`), so this field always holds a string. It is kept separate from `source_time_key`, so that `source_time_key` always holds a value in `source_time_format` and doesn't cause type conflicts in downstream storage.
+
+Default value: `source_time_raw`.
+
+### time_key_conflict (enum) (optional)
+
+The action that will be performed when the record already has source_time_key or bad_time_key. Defaults to keeping the existing value.
+
+This happens e.g. when a record passes through more than one `time_cutoff` filter, in which case the existing value is the true original time. The event time itself is replaced with current time in any case.
+
+* `skip`: keep the existing value.
+* `warn`: keep the existing value and log a warning to Fluentd's own log.
+* `overwrite`: replace the existing value.
+
+Available values: skip, warn, overwrite
+
+Default value: `skip`.
+
 ## Copyright
 
-* Copyright(c) 2025 Qrator Labs and Serge Tkatchouk
+* Copyright(c) 2025-2026 Qrator Labs and Serge Tkatchouk
 * License: MIT

data/lib/fluent/plugin/filter_time_cutoff.rb

@@ -31,9 +31,41 @@ module Fluent
         :iso8601
       ].freeze
 
+      # Available actions for records with an unparseable time.
+      BAD_TIME_ACTIONS = [
+        # Replace time with "now" and pass:
+        :replace_timestamp,
+        # Drop record as invalid:
+        :drop
+      ].freeze
+
+      # Available actions for records that already have the key the original
+      # time should be put into (for :replace_timestamp action).
+      TIME_KEY_CONFLICT_ACTIONS = [
+        # Keep the existing value:
+        :skip,
+        # Keep the existing value and log a warning:
+        :warn,
+        # Replace the existing value:
+        :overwrite
+      ].freeze
+
       # strftime() format string for :iso8601 time format.
       TIME_ISO8601 = '%FT%T%:z'
 
+      # Matches a "relaxed" ISO 8601 datetime string, which uses a space
+      # instead of "T" between the date and the time and/or has a space
+      # before the timezone (e.g. Ruby's Time#to_s: "2026-10-10 03:57:04 +0300"
+      # or "2026-10-10 00:57:04 UTC"), so that it can be rebuilt as a strict one.
+      ISO8601_RELAXED = /
+        \A
+        (?<date>\d{4}-\d{2}-\d{2})
+        [T\ ]
+        (?<time>\d{2}:\d{2}:\d{2}(?:\.\d+)?)
+        (?:\ ?(?<zone>Z|UTC|[+-]\d{2}(?::?\d{2})?))?
+        \z
+      /x.freeze
+
       Fluent::Plugin.register_filter('time_cutoff', self)
 
       desc 'Records older than this amount of time are deemed "old". Defaults to 24 hours.'
@@ -52,15 +84,51 @@ module Fluent
       config_param :source_time_key, :string, default: 'source_time'
       desc 'Time format for the original timestamp field. Defaults to ISO8601-formatted string.'
       config_param :source_time_format, :enum, list: TIME_FORMATS, default: :iso8601
+      desc 'The action that will be performed to all messages with an unparseable time. Defaults to replacing it with current time.'
+      config_param :bad_time_action, :enum, list: BAD_TIME_ACTIONS, default: :replace_timestamp
+      desc 'Log the messages with an unparseable time to Fluentd\'s own log. Defaults to "true".'
+      config_param :bad_time_log, :bool, default: true
+      desc 'The key that will hold the original unparseable time (for :replace_timestamp action). Defaults to "source_time_raw".'
+      config_param :bad_time_key, :string, default: 'source_time_raw'
+      desc 'The action that will be performed when the record already has source_time_key or bad_time_key. Defaults to keeping the existing value.'
+      config_param :time_key_conflict, :enum, list: TIME_KEY_CONFLICT_ACTIONS, default: :skip
+
+      def configure(conf)
+        super
+
+        %i[old_cutoff new_cutoff].each do |param|
+          # NOTE: Fluentd's :time type drops the sign of values with a unit
+          # (e.g. "-1h" is parsed as 3600), so the raw value is checked too.
+          raw = conf[param.to_s].to_s.strip
+          next unless raw.start_with?('-') || instance_variable_get("@#{param}").negative?
+
+          raise Fluent::ConfigError, "#{param} must not be negative, got \"#{raw}\""
+        end
+        %i[source_time_key bad_time_key].each do |param|
+          raise Fluent::ConfigError, "#{param} must not be empty" if instance_variable_get("@#{param}").empty?
+        end
+        return unless @source_time_key == @bad_time_key
+
+        raise Fluent::ConfigError, "source_time_key and bad_time_key must differ, both are \"#{@source_time_key}\""
+      end
 
       def filter_with_time(tag, time, record)
+        ## Safeguard against rogue integer/float/string time. This has to happen
+        ## before classification, so that the age check uses the parsed value.
+        unless time.is_a?(Fluent::EventTime)
+          normalized = normalize_time(time)
+          return process_bad_time_record(tag, time, record) if normalized.nil?
+
+          time = normalized
+        end
+
         event_time = time.to_i
-        current_time = Time.now.to_i
+        now = current_time
 
-        if event_time < current_time - @old_cutoff
+        if event_time < now - @old_cutoff
           # Too old
           process_old_record(tag, time, record)
-        elsif event_time > current_time + @new_cutoff
+        elsif event_time > now + @new_cutoff
           # Too new
           process_new_record(tag, time, record)
         else
@@ -71,6 +139,24 @@ module Fluent
 
       private
 
+      # Current UNIX time, used as the reference point for the age check.
+      # Kept as a separate method so that tests can freeze the clock.
+      # @return [Integer] current time in seconds
+      def current_time
+        Time.now.to_i
+      end
+
+      # Format the event time as an ISO 8601 datetime string (in local time).
+      # NOTE: Time.at(int) is used instead of EventTime#to_time, which was
+      # only added to Fluentd in v1.8.0, and instead of Time.at(EventTime),
+      # which relies on Ruby coercing EventTime via #to_r/#to_int. EventTime#to_i
+      # is available in all Fluentd releases declared in the gemspec.
+      # @param time [Fluent::EventTime] the event time
+      # @return [String] ISO 8601-formatted datetime string
+      def iso8601(time)
+        Time.at(time.to_i).strftime(TIME_ISO8601)
+      end
+
       # Format the event time to a given type/format.
       # @param time [Fluent::EventTime] the original event time
       # @param @source_time_format [Symbol] the target format
@@ -82,25 +168,50 @@ module Fluent
         when :epoch_float
           time.to_f
         when :iso8601
-          # NOTE: Time.at(int) is used instead of EventTime#to_time, which was
-          # only added to Fluentd in v1.8.0. This keeps us compatible with the
-          # older Fluentd releases declared in the gemspec.
-          Time.at(time.to_i).strftime(TIME_ISO8601)
+          iso8601(time)
         end
       end
 
       # Replace event time and put the old time into a specified field.
+      # @param tag [String] log/stream's tag name
       # @param time [Fluent::EventTime] the original event time
       # @param record [Hash] the original record contents
       # @param @source_time_key [String] name of the field to put the old timestamp to.
       # @return [Array] new event time and a modified record.
-      def do_replace_timestamp(time, record)
-        new_time = Fluent::EventTime.now
+      def do_replace_timestamp(tag, time, record)
+        [Fluent::EventTime.now, put_time_key(tag, record, @source_time_key, format_time(time))]
+      end
+
+      # Put the original time into the given field of a copy of the record,
+      # unless the record already has that field and @time_key_conflict says
+      # to keep it. The original record is never modified.
+      # @param tag [String] log/stream's tag name
+      # @param record [Hash] the original record contents
+      # @param key [String] name of the field to put the time to
+      # @param value the time value to put into the field
+      # @return [Hash] the modified copy of the record, or the original record if kept as is
+      def put_time_key(tag, record, key, value)
+        if record.key?(key) && @time_key_conflict != :overwrite
+          log_time_key_conflict(tag, record, key, value) if @time_key_conflict == :warn
+          return record
+        end
+
         new_record = record.dup
-        old_time = format_time(time)
-        new_record[@source_time_key] = old_time
+        new_record[key] = value
+        new_record
+      end
 
-        [new_time, new_record]
+      # Log a time key conflict to Fluentd's log.
+      # @param tag [String] log/stream's tag name
+      # @param record [Hash] the original record contents
+      # @param key [String] name of the conflicting field
+      # @param value the time value that was not put into the field
+      def log_time_key_conflict(tag, record, key, value)
+        log_line = format(
+          'Record already has the "%<key>s" key, keeping its existing value instead of %<val>s: "%<tag>s: %<msg>s".',
+          key: key, val: value.inspect, tag: tag, msg: dump_record(record)
+        )
+        log.warn log_line
       end
 
       # Log the caught event to Fluentd's log.
@@ -110,17 +221,41 @@ module Fluent
       # @param action [Symbol] determined action for this record
       # @param age [Symbol] detemined age for this record (:old or :new)
       def log_record(tag, time, record, action, age)
-        fmt_time = Time.at(time).strftime(TIME_ISO8601)
         log_line = format(
           'Record caught [%<age>s, %<act>s]: "%<tag>s: %<time>s %<msg>s".',
-          age: age, act: action, tag: tag, time: fmt_time, msg: record.to_json
+          age: age, act: action, tag: tag, time: iso8601(time), msg: dump_record(record)
         )
         log.warn log_line
       end
 
-      # Normalize time to Fluent::EventTime if it's plain int or float.
-      # @param time [Integer|Float] incoming event time
-      # @return [Fluent::EventTime] event time normalized to Fluentd's internal format
+      # Log the caught event with an unparseable time to Fluentd's log.
+      # @param tag [String] log/stream's tag name
+      # @param time the original (unparseable) event time
+      # @param record [Hash] the original record contents
+      # @param action [Symbol] determined action for this record
+      def log_bad_time_record(tag, time, record, action)
+        log_line = format(
+          'Record caught [unparseable, %<act>s]: "%<tag>s: %<time>s %<msg>s".',
+          act: action, tag: tag, time: time.inspect, msg: dump_record(record)
+        )
+        log.warn log_line
+      end
+
+      # Serialize the record for logging. Falls back to #inspect when the
+      # record can't be represented as JSON (e.g. invalid UTF-8 or NaN), so
+      # that a logging failure never affects how the record is processed.
+      # @param record [Hash] the original record contents
+      # @return [String] the record as a JSON string, or its #inspect output
+      def dump_record(record)
+        record.to_json
+      rescue JSON::GeneratorError, EncodingError
+        record.inspect
+      end
+
+      # Normalize time to Fluent::EventTime if it's plain int or float, a
+      # numeric string or an ISO 8601 datetime string.
+      # @param time [Integer|Float|String] incoming event time
+      # @return [Fluent::EventTime,nil] event time normalized to Fluentd's internal format, or nil if unparseable
       def normalize_time(time)
         case time
         when Integer
@@ -129,25 +264,40 @@ module Fluent
           time_int, time_frac = time.divmod(1)
           Fluent::EventTime.new(time_int, (time_frac * 10**9).to_i)
         else
-          log.warn "Unknown time format given (#{time.class}): \"#{time.inspect}\"."
-          numeric = coerce_to_number(time)
-          numeric ? normalize_time(numeric) : Fluent::EventTime.now
+          if (numeric = coerce_to_number(time))
+            normalize_time(numeric)
+          elsif (parsed = parse_iso8601(time))
+            Fluent::EventTime.new(parsed.to_i, parsed.nsec)
+          end
         end
       end
 
       # Try to coerce an arbitrary value into a number, or return nil.
-      # Uses begin/rescue rather than the Integer()/Float() "exception:"
-      # keyword (Ruby 2.6+) so the plugin keeps working on older Rubies.
+      # Integer() is called with an explicit base 10 so that zero-padded
+      # strings (e.g. "01700000000") aren't parsed as octal.
       # @param value the value to coerce
       # @return [Integer,Float,nil] the parsed number, or nil if not numeric
       def coerce_to_number(value)
-        Integer(value)
-      rescue ArgumentError, TypeError
-        begin
-          Float(value)
-        rescue ArgumentError, TypeError
-          nil
+        Integer(value, 10, exception: false) || Float(value, exception: false)
+      end
+
+      # Try to parse a strict ISO 8601 datetime string, or return nil.
+      # Time.iso8601 is used instead of the lenient Time.parse, which would
+      # happily turn partial or ambiguous strings into a wrong timestamp.
+      # "Relaxed" strings (see ISO8601_RELAXED) are rebuilt as strict ones
+      # before parsing. Strings without a UTC offset are treated as local time.
+      # @param value the value to parse
+      # @return [Time,nil] the parsed time, or nil if not an ISO 8601 string
+      def parse_iso8601(value)
+        return nil unless value.is_a?(String)
+
+        if (match = ISO8601_RELAXED.match(value))
+          zone = match[:zone] == 'UTC' ? 'Z' : match[:zone]
+          value = "#{match[:date]}T#{match[:time]}#{zone}"
         end
+        Time.iso8601(value)
+      rescue ArgumentError
+        nil
       end
 
       # Process the record according to its determined "age"
@@ -158,21 +308,48 @@ module Fluent
       # @param action [Symbol] determined action for this record
       # @param age [Symbol] detemined age for this record (:old or :new)
       def process_record(tag, time, record, do_log, action, age) # rubocop:disable Metrics/ParameterLists
-        ## Safeguard against rogue integer time.
-        time = normalize_time(time) unless time.is_a?(Fluent::EventTime)
-
         log_record(tag, time, record, action, age) if do_log
 
         case action
         when :pass
           [time, record]
         when :replace_timestamp
-          do_replace_timestamp(time, record)
+          do_replace_timestamp(tag, time, record)
+        when :drop
+          nil
+        end
+      end
+
+      # Process the record with an unparseable time according to the configuration
+      # @param tag [String] log/stream's tag name
+      # @param time the original (unparseable) event time
+      # @param record [Hash] the original record contents
+      # @return [Array,nil] current time and the record, or nil if dropped.
+      def process_bad_time_record(tag, time, record)
+        log_bad_time_record(tag, time, record, @bad_time_action) if @bad_time_log
+
+        case @bad_time_action
+        when :replace_timestamp
+          do_replace_bad_timestamp(tag, time, record)
         when :drop
           nil
         end
       end
 
+      # Replace unparseable event time and put the raw original time into a
+      # separate field. The raw time is always stored as a string, so that the
+      # field keeps a stable type for downstream storage. It is not put into
+      # @source_time_key, which holds a value in @source_time_format instead.
+      # @param tag [String] log/stream's tag name
+      # @param time the original (unparseable) event time
+      # @param record [Hash] the original record contents
+      # @param @bad_time_key [String] name of the field to put the raw time to.
+      # @return [Array] new event time and a modified record.
+      def do_replace_bad_timestamp(tag, time, record)
+        raw_time = time.is_a?(String) ? time : time.inspect
+        [Fluent::EventTime.now, put_time_key(tag, record, @bad_time_key, raw_time)]
+      end
+
       # Process the "old" record according to the configuration
       # @param tag [String] log/stream's tag name
       # @param time [Fluent::EventTime] the original event time

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-time-cutoff
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Qrator Labs
@@ -30,34 +30,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.6.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.6.3
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.1.0
+        version: 13.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.1.0
+        version: 13.2.0
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -81,11 +67,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".rubocop.yml"
 - LICENSE.txt
 - README.md
-- Rakefile
-- fluent-plugin-time-cutoff.gemspec
 - lib/fluent/plugin/filter_time_cutoff.rb
 homepage: https://github.com/QratorLabs/fluent-plugin-time-cutoff
 licenses:
@@ -98,14 +81,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.9
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Fluentd time-based filter plugin.
 test_files: []

data/.rubocop.yml

@@ -1,2 +0,0 @@
-AllCops:
-  TargetRubyVersion: 2.3

data/Rakefile

@@ -1,13 +0,0 @@
-require 'bundler'
-Bundler::GemHelper.install_tasks
-
-require 'rake/testtask'
-
-Rake::TestTask.new(:test) do |t|
-  t.libs.push('lib', 'test')
-  t.test_files = FileList['test/**/test_*.rb']
-  t.verbose = true
-  t.warning = true
-end
-
-task default: [:test]

data/fluent-plugin-time-cutoff.gemspec

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-lib = File.expand_path('../lib', __dir__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-Gem::Specification.new do |spec|
-  spec.name    = 'fluent-plugin-time-cutoff'
-  spec.version = '0.2.0'
-  spec.authors = ['Qrator Labs', 'Serge Tkatchouk']
-  spec.email   = ['devops@qrator.net', 'st@qrator.net']
-
-  spec.summary       = 'Fluentd time-based filter plugin.'
-  spec.description   = 'A plugin that lets Fluentd to prune/rewrite messages '\
-                       'that have a timestamp that is too old or too new.'
-  spec.homepage      = 'https://github.com/QratorLabs/fluent-plugin-time-cutoff'
-  spec.license       = 'MIT'
-
-  spec.files         = Dir.chdir(__dir__) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      (File.expand_path(f) == __FILE__) ||
-        f.start_with?(*%w[test/ testbed/ spec/ features/ .git .circleci appveyor Gemfile])
-    end
-  end
-
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ['lib']
-
-  spec.platform = Gem::Platform::RUBY
-  spec.required_ruby_version = Gem::Requirement.new('>= 2.3')
-
-  spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
-
-  spec.add_development_dependency 'bundler', '~> 2.6.3'
-  spec.add_development_dependency 'rake', '~> 13.1.0'
-  spec.add_development_dependency 'test-unit', '~> 3.6.1'
-end