fluent-plugin-time-cutoff 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b8e038b330cfb51044909264e92646dfef5dbd368f5cec0db66952d816375c34
+  data.tar.gz: 2e16bc7505e8bb78c8d83ae0eb5425bab1dc7f5b6081d00f253a2534713f9e6d
+SHA512:
+  metadata.gz: 12af52a9288c7468e33e5a7c4f617b2ac55cad85092ea701ade19305a16dfc49d86452093a179eec8adeeaf01d3445339e57b24b62ad4bcfb2706b1e0d73154d
+  data.tar.gz: fe9b813a02d46ede91152758c64a6b0ee2d809c530e37540c7dec4ce738c130381c9425030c80cbd2de6df8dffbc360ba4be0e2ef58c6a450047cbd0b963637d

data/.rubocop.yml

@@ -0,0 +1,2 @@
+AllCops:
+  TargetRubyVersion: 2.3

data/.ruby-version

@@ -0,0 +1 @@
+3.3.7

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Qrator Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# fluent-plugin-time-cutoff
+
+[Fluentd](https://fluentd.org/) filter plugin to cut off log records with a timestamp outside of given window.
+
+TODO: write description for you plugin.
+
+## Installation
+
+### RubyGems
+
+```
+$ gem install fluent-plugin-time-cutoff
+```
+
+### Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "fluent-plugin-time-cutoff"
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+## Configuration
+
+You can generate configuration template:
+
+```
+$ fluent-plugin-config-format filter time-cutoff
+```
+
+You can copy and paste generated documents here.
+
+## Copyright
+
+* Copyright(c) 2025 Qrator Labs and Serge Tkatchouk
+* License: MIT

data/Rakefile

@@ -0,0 +1,13 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push('lib', 'test')
+  t.test_files = FileList['test/**/test_*.rb']
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/fluent-plugin-time-cutoff.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('../lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = 'fluent-plugin-time-cutoff'
+  spec.version = '0.1.1'
+  spec.authors = ['Qrator Labs', 'Serge Tkatchouk']
+  spec.email   = ['devops@qrator.net', 'st@qrator.net']
+
+  spec.summary       = 'Fluentd time-based filter plugin.'
+  spec.description   = 'A plugin that lets Fluentd to prune/rewrite messages '\
+                       'that have a timestamp that is too old or too new.'
+  spec.homepage      = 'https://github.com/QratorLabs/fluent-plugin-time-cutoff'
+  spec.license       = 'MIT'
+
+  spec.files         = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[test/ testbed/ spec/ features/ .git .circleci appveyor Gemfile])
+    end
+  end
+
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.platform = Gem::Platform::RUBY
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.3')
+
+  spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
+
+  spec.add_development_dependency 'bundler', '~> 2.6.3'
+  spec.add_development_dependency 'rake', '~> 13.1.0'
+  spec.add_development_dependency 'test-unit', '~> 3.6.1'
+end

data/lib/fluent/plugin/filter_time_cutoff.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/filter'
+require 'json'
+require 'time'
+
+module Fluent
+  module Plugin
+    class TimeCutoffFilter < Fluent::Plugin::Filter
+      CUTOFF_ACTIONS = [
+        # Pass record as is:
+        :pass,
+        # Replace time with "now" and pass:
+        :replace_timestamp,
+        # Drop record as invalid:
+        :drop
+      ].freeze
+
+      TIME_FORMATS = [
+        # UNIX time (integer)
+        :epoch,
+        # UNIX time (float)
+        :epoch_float,
+        # ISO 8601 datetime (string)
+        :iso8601
+      ].freeze
+
+      TIME_ISO8601 = '%FT%T%:z'
+
+      Fluent::Plugin.register_filter('time_cutoff', self)
+      ## Records older than this are deemed "old":
+      config_param :old_cutoff, :time, default: 86_400 # 24 hours
+      ## What to do with "old" records:
+      config_param :old_action, :enum, list: CUTOFF_ACTIONS, default: :pass
+      ## Print the "old" records to Fluentd's log:
+      config_param :old_log, :bool, default: true
+      ## Records newer than this are deemed "new":
+      config_param :new_cutoff, :time, default: 86_400 # 24 hours
+      ## What to do with "new" records:
+      config_param :new_action, :enum, list: CUTOFF_ACTIONS, default: :pass
+      ## Print the "new" records to Fluentd's log:
+      config_param :new_log, :bool, default: true
+      ## Name of the key that will hold original timestamp
+      #  (for action = :replace_timestamp):
+      config_param :source_time_key, :string, default: 'source_time'
+      ## How to format the original time (for action = :replace_timestamp):
+      config_param :source_time_format, :enum, list: TIME_FORMATS, default: :iso8601
+
+      def filter_with_time(tag, time, record)
+        event_time = time.to_i
+        current_time = Time.now.to_i
+
+        if event_time < current_time - @old_cutoff
+          # Too old
+          process_old_record(tag, time, record)
+        elsif event_time > current_time + @new_cutoff
+          # Too new
+          process_new_record(tag, time, record)
+        else
+          # OK
+          [time, record]
+        end
+      end
+
+      private
+
+      def format_time(time)
+        case @source_time_format
+        when :epoch
+          time.to_i
+        when :epoch_float
+          time.to_f
+        when :iso8601
+          time.to_time.strftime(TIME_ISO8601)
+        end
+      end
+
+      def do_replace_timestamp(time, record)
+        new_time = Fluent::EventTime.now
+        new_record = record.dup
+        old_time = format_time(time)
+        new_record[@source_time_key] = old_time
+
+        [new_time, new_record]
+      end
+
+      def log_record(tag, time, record, action, age)
+        fmt_time = Time.at(time).strftime(TIME_ISO8601)
+        log_line = format(
+          'Record caught [%<age>s, %<act>s]: "%<tag>s: %<time>s %<msg>s".',
+          age: age, act: action, tag: tag, time: fmt_time, msg: record.to_json
+        )
+        log.warn log_line
+      end
+
+      def process_record(tag, time, record, do_log, action, age) # rubocop:disable Metrics/ParameterLists
+        log_record(tag, time, record, action, age) if do_log
+
+        case action
+        when :pass
+          [time, record]
+        when :replace_timestamp
+          do_replace_timestamp(time, record)
+        when :drop
+          nil
+        end
+      end
+
+      def process_old_record(tag, time, record)
+        process_record(tag, time, record, @old_log, @old_action, :old)
+      end
+
+      def process_new_record(tag, time, record)
+        process_record(tag, time, record, @new_log, @new_action, :new)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-time-cutoff
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Qrator Labs
+- Serge Tkatchouk
+bindir: bin
+cert_chain: []
+date: 2025-04-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.1
+description: A plugin that lets Fluentd to prune/rewrite messages that have a timestamp
+  that is too old or too new.
+email:
+- devops@qrator.net
+- st@qrator.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- ".ruby-version"
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-time-cutoff.gemspec
+- lib/fluent/plugin/filter_time_cutoff.rb
+homepage: https://github.com/QratorLabs/fluent-plugin-time-cutoff
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.3
+specification_version: 4
+summary: Fluentd time-based filter plugin.
+test_files: []