fluent-plugin-time-cutoff 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8e038b330cfb51044909264e92646dfef5dbd368f5cec0db66952d816375c34
-  data.tar.gz: 2e16bc7505e8bb78c8d83ae0eb5425bab1dc7f5b6081d00f253a2534713f9e6d
+  metadata.gz: 3ced312074114226376fedaf33671d11804667825d8a50430c1ac54137aac535
+  data.tar.gz: c11dc8c0510a8adec9475bac1382a003255bdb8e3af7de6f8aba46defa60ff16
 SHA512:
-  metadata.gz: 12af52a9288c7468e33e5a7c4f617b2ac55cad85092ea701ade19305a16dfc49d86452093a179eec8adeeaf01d3445339e57b24b62ad4bcfb2706b1e0d73154d
-  data.tar.gz: fe9b813a02d46ede91152758c64a6b0ee2d809c530e37540c7dec4ce738c130381c9425030c80cbd2de6df8dffbc360ba4be0e2ef58c6a450047cbd0b963637d
+  metadata.gz: dd2ea5a6907c0b27e95580c0081030de242dcd3cb85ef4bbd79a99918c369ee53680187a866c78e12016ae47d1d534c2b4461a5b6bd522e3f0b2ef691a3c1fb2
+  data.tar.gz: 6d39a4673c35dac85aff1e96d4575e87c03f807973acdaa86dca01b6e729f587fbeeec8939a4683e7491ebf0d5cef86fbb0e38f65b70aa90d259db5347188b98

data/README.md

@@ -2,7 +2,76 @@
 
 [Fluentd](https://fluentd.org/) filter plugin to cut off log records with a timestamp outside of given window.
 
-TODO: write description for you plugin.
+## Examples
+
+### Example 1: Delete all logs older than 12 hours, pass through records from the future
+
+**Configuration**
+
+``` xml
+<filter **>
+  @type time_cutoff
+  @id time_cutoff
+ 
+  old_cutoff 12h
+  old_action drop
+
+  new_action pass
+  new_log false
+</filter>
+```
+
+**Input**
+
+``` bash
+# Local time is 2025-04-08T14:43:14+03:00
+fluent-cat --event-time '2025-04-08T02:42:00+03:00' test.cutoff <<< '{"message": "hello"}'
+fluent-cat --event-time '2025-04-08T12:42:00+03:00' test.cutoff <<< '{"message": "hello"}'
+fluent-cat --event-time '2025-04-09T14:42:00+03:00' test.cutoff <<< '{"message": "hello"}'
+```
+
+**Output**
+
+```
+2025-04-08 14:43:24 +0300 [warn]: [time_cutoff] Record caught [old, drop]: "test.cutoff: 2025-04-08T02:42:00+03:00 {"message":"hello"}".
+2025-04-08 12:42:00.000000000 +0300 test.cutoff: {"message":"hello"}
+2025-04-09 14:42:00.000000000 +0300 test.cutoff: {"message":"hello"}
+```
+
+### Example 2: Delete all logs older than 24 hours (omit log), rewrite timestamps for all records newer than 3 hours:
+
+**Configuration**
+
+``` xml
+<filter **>
+  @type time_cutoff
+  @id time_cutoff
+ 
+  old_cutoff 24h
+  old_action drop
+  old_log false
+
+  new_cutoff 3h
+  new_action replace_timestamp
+</filter>
+```
+
+**Input**
+
+``` bash
+# Local time is 2025-04-08T14:48:42+03:00
+fluent-cat --event-time '2025-04-07T14:48:00+03:00' test.cutoff <<< '{"message": "hello"}'
+fluent-cat --event-time '2025-04-08T14:48:00+03:00' test.cutoff <<< '{"message": "hello"}'
+fluent-cat --event-time '2025-04-08T18:48:00+03:00' test.cutoff <<< '{"message": "hello"}'
+```
+
+**Output**
+
+```
+2025-04-08 14:48:00.000000000 +0300 test.cutoff: {"message":"hello"}
+2025-04-08 14:49:13 +0300 [warn]: [time_cutoff] Record caught [new, replace_timestamp]: "test.cutoff: 2025-04-08T18:48:00+03:00 {"message":"hello"}".
+2025-04-08 14:49:13.090783000 +0300 test.cutoff: {"message":"hello","source_time":"2025-04-08T18:48:00+03:00"}
+```
 
 ## Installation
 
@@ -28,13 +97,78 @@ $ bundle
 
 ## Configuration
 
-You can generate configuration template:
+``` xml
+<filter **>
+  @type time_cutoff
+  @id time_cutoff
+ 
+  old_cutoff 24h
+  old_action pass
+  old_log true
+
+  new_cutoff 24h
+  new_action pass
+  new_log true
+
+  source_time_key source_time
+  source_time_format iso8601
+</filter>
 
 ```
-$ fluent-plugin-config-format filter time-cutoff
-```
 
-You can copy and paste generated documents here.
+### old_cutoff (time) (optional)
+
+Records older than this amount of time are deemed "old". Defaults to 24 hours.
+
+Default value: `86400`.
+
+### old_action (enum) (optional)
+
+The action that will be performed to all "old" messages. Defaults to passthrough.
+
+Available values: pass, replace_timestamp, drop
+
+Default value: `pass`.
+
+### old_log (bool) (optional)
+
+Log the "old" messages to Fluentd's own log. Defaults to "true".
+
+Default value: `true`.
+
+### new_cutoff (time) (optional)
+
+Records newer than this amount of time are deemed "new". Defaults to 24 hours.
+
+Default value: `86400`.
+
+### new_action (enum) (optional)
+
+The action that will be performed to all "new" messages. Defaults to passthrough.
+
+Available values: pass, replace_timestamp, drop
+
+Default value: `pass`.
+
+### new_log (bool) (optional)
+
+Log the "new" messages to Fluentd's own log. Defaults to "true".
+
+Default value: `true`.
+
+### source_time_key (string) (optional)
+
+The key that will hold the original time (for :replace_timestamp action). Defaults to "source_time".
+
+Default value: `source_time`.
+
+### source_time_format (enum) (optional)
+
+Time format for the original timestamp field. Defaults to ISO8601-formatted string.
+
+Available values: epoch, epoch_float, iso8601
+
+Default value: `iso8601`.
 
 ## Copyright
 

data/fluent-plugin-time-cutoff.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-time-cutoff'
-  spec.version = '0.1.1'
+  spec.version = '0.1.3'
   spec.authors = ['Qrator Labs', 'Serge Tkatchouk']
   spec.email   = ['devops@qrator.net', 'st@qrator.net']
 

data/lib/fluent/plugin/filter_time_cutoff.rb

@@ -6,7 +6,12 @@ require 'time'
 
 module Fluent
   module Plugin
+    # A filter plugin that allows Fluentd to modify or drop messages that are
+    # newer or older than a set threshold. By default this filter will not
+    # modify or drop any messages, but it will log all messages that are 24
+    # hours older or newer than current time.
     class TimeCutoffFilter < Fluent::Plugin::Filter
+      # Available actions for messages.
       CUTOFF_ACTIONS = [
         # Pass record as is:
         :pass,
@@ -16,6 +21,7 @@ module Fluent
         :drop
       ].freeze
 
+      # Available time formats for :replace_timestamp action.
       TIME_FORMATS = [
         # UNIX time (integer)
         :epoch,
@@ -25,25 +31,26 @@ module Fluent
         :iso8601
       ].freeze
 
+      # strftime() format string for :iso8601 time format.
       TIME_ISO8601 = '%FT%T%:z'
 
       Fluent::Plugin.register_filter('time_cutoff', self)
-      ## Records older than this are deemed "old":
+
+      desc 'Records older than this amount of time are deemed "old". Defaults to 24 hours.'
       config_param :old_cutoff, :time, default: 86_400 # 24 hours
-      ## What to do with "old" records:
+      desc 'The action that will be performed to all "old" messages. Defaults to passthrough.'
       config_param :old_action, :enum, list: CUTOFF_ACTIONS, default: :pass
-      ## Print the "old" records to Fluentd's log:
+      desc 'Log the "old" messages to Fluentd\'s own log. Defaults to "true".'
       config_param :old_log, :bool, default: true
-      ## Records newer than this are deemed "new":
+      desc 'Records newer than this amount of time are deemed "new". Defaults to 24 hours.'
       config_param :new_cutoff, :time, default: 86_400 # 24 hours
-      ## What to do with "new" records:
+      desc 'The action that will be performed to all "new" messages. Defaults to passthrough.'
       config_param :new_action, :enum, list: CUTOFF_ACTIONS, default: :pass
-      ## Print the "new" records to Fluentd's log:
+      desc 'Log the "new" messages to Fluentd\'s own log. Defaults to "true".'
       config_param :new_log, :bool, default: true
-      ## Name of the key that will hold original timestamp
-      #  (for action = :replace_timestamp):
+      desc 'The key that will hold the original time (for :replace_timestamp action). Defaults to "source_time".'
       config_param :source_time_key, :string, default: 'source_time'
-      ## How to format the original time (for action = :replace_timestamp):
+      desc 'Time format for the original timestamp field. Defaults to ISO8601-formatted string.'
       config_param :source_time_format, :enum, list: TIME_FORMATS, default: :iso8601
 
       def filter_with_time(tag, time, record)
@@ -64,6 +71,10 @@ module Fluent
 
       private
 
+      # Format the event time to a given type/format.
+      # @param time [Fluent::EventTime] the original event time
+      # @param @source_time_format [Symbol] the target format
+      # @return [Integer,Float,String] formatted time as an integer, float (with microseconds), or an ISO8601-formatted datetime string.
       def format_time(time)
         case @source_time_format
         when :epoch
@@ -75,6 +86,11 @@ module Fluent
         end
       end
 
+      # Replace event time and put the old time into a specified field.
+      # @param time [Fluent::EventTime] the original event time
+      # @param record [Hash] the original record contents
+      # @param @source_time_key [String] name of the field to put the old timestamp to.
+      # @return [Array] new event time and a modified record.
       def do_replace_timestamp(time, record)
         new_time = Fluent::EventTime.now
         new_record = record.dup
@@ -84,6 +100,12 @@ module Fluent
         [new_time, new_record]
       end
 
+      # Log the caught event to Fluentd's log.
+      # @param tag [String] log/stream's tag name
+      # @param time [Fluent::EventTime] the original event time
+      # @param record [Hash] the original record contents
+      # @param action [Symbol] determined action for this record
+      # @param age [Symbol] detemined age for this record (:old or :new)
       def log_record(tag, time, record, action, age)
         fmt_time = Time.at(time).strftime(TIME_ISO8601)
         log_line = format(
@@ -93,7 +115,33 @@ module Fluent
         log.warn log_line
       end
 
+      # Normalize time to Fluent::EventTime if it's plain int or float.
+      # @param time [Integer|Float] incoming event time
+      # @return [Fluent::EventTime] event time normalized to Fluentd's internal format
+      def normalize_time(time)
+        case time.class
+        when Integer
+          Fluent::EventTime.new(time)
+        when Float
+          time_int, time_frac = time.divmod(1)
+          Fluent::EventTime.new(time_int, (time_frac * 10**9).to_i)
+        else
+          log.warn "Unknown time format given: \"#{time.inspect}\"."
+          Fluent::EventTime.new(time.to_i) || Fluent::EventTime.now
+        end
+      end
+
+      # Process the record according to its determined "age"
+      # @param tag [String] log/stream's tag name
+      # @param time [Fluent::EventTime] the original event time
+      # @param record [Hash] the original record contents
+      # @param do_log [Boolean] should we log this record or not
+      # @param action [Symbol] determined action for this record
+      # @param age [Symbol] detemined age for this record (:old or :new)
       def process_record(tag, time, record, do_log, action, age) # rubocop:disable Metrics/ParameterLists
+        ## Safeguard against rogue integer time.
+        time = normalize_time(time) unless time.is_a?(Fluent::EventTime)
+
         log_record(tag, time, record, action, age) if do_log
 
         case action
@@ -106,10 +154,18 @@ module Fluent
         end
       end
 
+      # Process the "old" record according to the configuration
+      # @param tag [String] log/stream's tag name
+      # @param time [Fluent::EventTime] the original event time
+      # @param record [Hash] the original record contents
       def process_old_record(tag, time, record)
         process_record(tag, time, record, @old_log, @old_action, :old)
       end
 
+      # Process the "new" record according to the configuration
+      # @param tag [String] log/stream's tag name
+      # @param time [Fluent::EventTime] the original event time
+      # @param record [Hash] the original record contents
       def process_new_record(tag, time, record)
         process_record(tag, time, record, @new_log, @new_action, :new)
       end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-time-cutoff
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Qrator Labs
 - Serge Tkatchouk
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-04-08 00:00:00.000000000 Z
+date: 2025-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -82,7 +83,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".rubocop.yml"
-- ".ruby-version"
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -92,6 +92,7 @@ homepage: https://github.com/QratorLabs/fluent-plugin-time-cutoff
 licenses:
 - MIT
 metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -106,7 +107,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.3
+rubygems_version: 3.3.27
+signing_key:
 specification_version: 4
 summary: Fluentd time-based filter plugin.
 test_files: []

data/.ruby-version

@@ -1 +0,0 @@
-3.3.7