fluentd-json-size-limit 0.1.4 → 0.2.0

data/lib/fluent/plugin/json_size_limit/record_processor.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/json_size_limit/processing_guard"
+require "fluent/plugin/json_size_limit/record_measurer"
+require "fluent/plugin/json_size_limit/record_reducer"
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      class RecordProcessor
+        Result = Struct.new(:status, :original_json_bytes, :json_bytes)
+
+        def initialize(max_json_bytes:, safety_options:, clock: ProcessingGuard::MONOTONIC_TIME)
+          @max_json_bytes = max_json_bytes
+          @safety_options = safety_options
+          @clock = clock
+        end
+
+        def call(record)
+          guard = ProcessingGuard.new(**@safety_options, clock: @clock)
+          original_json_bytes = RecordMeasurer.new(guard).measure(record)
+
+          if original_json_bytes <= @max_json_bytes
+            return Result.new(
+              status: :within_limit,
+              original_json_bytes: original_json_bytes,
+              json_bytes: original_json_bytes
+            )
+          end
+
+          # Node and input counters are phase-local, while the absolute
+          # deadline remains shared by measurement and reduction.
+          guard.reset_phase!(track_input: false)
+          reduced_json_bytes = RecordReducer.new(
+            record,
+            original_json_bytes: original_json_bytes,
+            max_json_bytes: @max_json_bytes,
+            guard: guard,
+            validated_record: true
+          ).reduce_to_limit
+          guard.checkpoint!
+
+          Result.new(
+            status: (reduced_json_bytes <= @max_json_bytes) ? :reduced : :overflow,
+            original_json_bytes: original_json_bytes,
+            json_bytes: reduced_json_bytes
+          )
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit/record_reducer.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/json_size_limit/json_string"
+require "fluent/plugin/json_size_limit/processing_guard"
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      # Reduces string values proportionally while preserving record shape,
+      # aliases, key names, encodings, and non-string values.
+      class RecordReducer
+        HASH_EACH_PAIR = Hash.instance_method(:each_pair)
+        ARRAY_EACH_WITH_INDEX = Array.instance_method(:each_with_index)
+        HASH_SET = Hash.instance_method(:[]=)
+        ARRAY_SET = Array.instance_method(:[]=)
+
+        class StringLocation < JsonString
+          attr_reader :container, :key, :original_value
+          attr_accessor :occurrences, :replacement_value, :replacement_content_bytes
+
+          def initialize(container, key, value, encoding_validated: false)
+            super(value, validate_encoding: !encoding_validated)
+            @container = container
+            @key = key
+            @original_value = value
+            @occurrences = 1
+            @replacement_value = value
+            @replacement_content_bytes = content_bytes
+          end
+
+          def truncate_to(content_budget)
+            @replacement_value = truncate(content_budget)
+            @replacement_content_bytes = truncated_content_bytes
+          end
+        end
+
+        def initialize(record, original_json_bytes:, max_json_bytes:, guard: nil, validated_record: false)
+          @record = record
+          @original_json_bytes = original_json_bytes
+          @max_json_bytes = max_json_bytes
+          @guard = guard || ProcessingGuard.new(
+            timeout: 5,
+            max_nodes: 1_000_000,
+            max_input_bytes: Float::INFINITY,
+            max_nesting: 100
+          )
+          @validated_record = validated_record
+        end
+
+        def reduce_to_limit
+          locations, original_content_bytes = collect_string_locations
+          return @original_json_bytes if locations.empty?
+
+          immutable_json_bytes = @original_json_bytes - original_content_bytes
+          content_budget = @max_json_bytes - immutable_json_bytes
+          @guard.reset_phase!(track_input: false)
+          replacements, current_content_bytes = build_replacements(locations, content_budget, original_content_bytes)
+
+          @guard.reset_phase!(track_input: false)
+          apply_replacements(replacements)
+          immutable_json_bytes + current_content_bytes
+        end
+
+        private
+
+        # JSON serializes aliases once per occurrence. The common tree path
+        # avoids a per-field index; alias accounting is activated only when a
+        # shared Hash or Array identity is observed.
+        def collect_string_locations
+          collect_tree_string_locations || begin
+            @guard.reset_phase!(track_input: false)
+            collect_aliased_string_locations
+          end
+        end
+
+        def collect_tree_string_locations
+          locations = []
+          seen_containers = {}.compare_by_identity
+          total_content_bytes = 0
+          containers_to_scan = [[@record, 0]]
+          @scan_operations = 0
+
+          until containers_to_scan.empty?
+            container, depth = containers_to_scan.pop
+            scan_visit!(depth)
+            return nil if seen_containers.key?(container)
+
+            seen_containers[container] = true
+
+            each_container_value(container) do |key, value|
+              if value.instance_of?(String) && !value.empty?
+                location = StringLocation.new(container, key, value, encoding_validated: @validated_record)
+                locations << location
+                total_content_bytes += location.content_bytes
+                scan_visit!(depth + 1)
+              elsif value.instance_of?(Hash) || value.instance_of?(Array)
+                containers_to_scan << [value, depth + 1]
+              else
+                scan_visit!(depth + 1)
+              end
+            end
+          end
+
+          [locations, total_content_bytes]
+        end
+
+        def collect_aliased_string_locations
+          locations = []
+          locations_by_container = {}.compare_by_identity
+          total_content_bytes = 0
+          containers_to_scan = [[@record, 0]]
+          @scan_operations = 0
+
+          until containers_to_scan.empty?
+            container, depth = containers_to_scan.pop
+            scan_visit!(depth)
+            location_indexes = locations_by_container[container] ||= {}
+
+            each_container_value(container) do |key, value|
+              if value.instance_of?(String) && !value.empty?
+                location = location_indexes[key]
+
+                if location
+                  location.occurrences += 1
+                else
+                  location = StringLocation.new(container, key, value, encoding_validated: @validated_record)
+                  location_indexes[key] = location
+                  locations << location
+                end
+
+                total_content_bytes += location.content_bytes
+                scan_visit!(depth + 1)
+              elsif value.instance_of?(Hash) || value.instance_of?(Array)
+                containers_to_scan << [value, depth + 1]
+              else
+                scan_visit!(depth + 1)
+              end
+            end
+          end
+
+          [locations, total_content_bytes]
+        end
+
+        def each_container_value(container, &block)
+          if container.instance_of?(Hash)
+            HASH_EACH_PAIR.bind_call(container, &block)
+          else
+            ARRAY_EACH_WITH_INDEX.bind_call(container) { |value, index| block.call(index, value) }
+          end
+        end
+
+        def scan_visit!(depth)
+          unless @validated_record
+            @guard.visit!(depth)
+            return
+          end
+
+          @scan_operations += 1
+          @guard.checkpoint! if (@scan_operations & 255).zero?
+        end
+
+        def build_replacements(locations, content_budget, original_content_bytes)
+          remaining_budget = [content_budget, 0].max
+          remaining_original = original_content_bytes
+          current_content_bytes = 0
+
+          locations.each_with_index do |location, index|
+            @guard.checkpoint! if (index & 255).zero?
+            weighted_original = location.content_bytes * location.occurrences
+            weighted_target = remaining_budget.positive? ? remaining_budget * weighted_original / remaining_original : 0
+            target_per_occurrence = weighted_target / location.occurrences
+
+            location.truncate_to(target_per_occurrence)
+            @guard.checkpoint! if location.content_bytes >= 64 * 1024
+            weighted_replacement = location.replacement_content_bytes * location.occurrences
+
+            current_content_bytes += weighted_replacement
+            remaining_budget -= weighted_replacement
+            remaining_original -= weighted_original
+          end
+
+          @guard.reset_phase!(track_input: false)
+          current_content_bytes += distribute_remaining_budget(locations, remaining_budget)
+          @guard.reset_phase!(track_input: false)
+          replacements = build_replacement_list(locations)
+
+          [replacements, current_content_bytes]
+        end
+
+        def distribute_remaining_budget(locations, remaining_budget)
+          added_content_bytes = 0
+
+          locations.each_with_index do |location, index|
+            @guard.checkpoint! if (index & 255).zero?
+            break unless remaining_budget.positive?
+            next if location.replacement_content_bytes >= location.content_bytes
+
+            additional_per_occurrence = remaining_budget / location.occurrences
+            next unless additional_per_occurrence.positive?
+
+            previous_replacement_bytes = location.replacement_content_bytes
+            location.truncate_to(previous_replacement_bytes + additional_per_occurrence)
+            @guard.checkpoint! if location.content_bytes >= 64 * 1024
+            replacement_bytes = location.replacement_content_bytes
+            weighted_addition = (replacement_bytes - previous_replacement_bytes) * location.occurrences
+            remaining_budget -= weighted_addition
+            added_content_bytes += weighted_addition
+          end
+
+          added_content_bytes
+        end
+
+        def build_replacement_list(locations)
+          # A flat tuple buffer avoids allocating one small Array per changed
+          # field on records with many strings.
+          replacements = []
+          index = 0
+
+          locations.each do |location|
+            @guard.checkpoint! if (index & 255).zero?
+            unless location.replacement_value.equal?(location.original_value)
+              replacements << location.container << location.key << location.original_value << location.replacement_value
+            end
+            index += 1
+          end
+
+          replacements
+        end
+
+        def apply_replacements(replacements)
+          applied = 0
+
+          while applied < replacements.length
+            @guard.checkpoint! if (applied & 1023).zero?
+            set_container_value(replacements[applied], replacements[applied + 1], replacements[applied + 3])
+            applied += 4
+          end
+        rescue
+          rollback_replacements(replacements, applied)
+          raise
+        end
+
+        def rollback_replacements(replacements, applied)
+          index = 0
+          while index < applied
+            set_container_value(replacements[index], replacements[index + 1], replacements[index + 2])
+            index += 4
+          end
+        end
+
+        def set_container_value(container, key, value)
+          method = container.instance_of?(Hash) ? HASH_SET : ARRAY_SET
+          method.bind_call(container, key, value)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit/string_encoding.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/json_size_limit/errors"
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      module StringEncoding
+        module_function
+
+        def validate!(value)
+          unless value.valid_encoding?
+            raise InvalidStringEncodingError, value.encoding.name
+          end
+
+          # Binary bytes have no character boundaries, so truncating them could
+          # silently change payload semantics. ASCII-only binary strings are safe.
+          if value.encoding == Encoding::ASCII_8BIT && !value.ascii_only?
+            raise InvalidStringEncodingError, value.encoding.name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit/telemetry.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/json_size_limit/errors"
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      class Telemetry
+        DEFINITIONS = {
+          oversized_records: "Number of records exceeding max_size",
+          reduced_records: "Number of oversized records reduced to max_size",
+          unreducible_records: "Number of records whose immutable JSON exceeds max_size",
+          error_records: "Number of records that failed measurement or reduction",
+          dropped_records: "Number of records dropped by configured actions",
+          processing_limit_records: "Number of records stopped by automatic safety limits",
+          unsupported_records: "Number of records containing unsupported Ruby value classes",
+          removed_bytes: "Total JSON bytes removed from oversized records"
+        }.freeze
+
+        def initialize(&metric_factory)
+          raise ArgumentError, "metric factory block is required" unless metric_factory
+
+          @metrics = DEFINITIONS.to_h do |name, help_text|
+            metric = metric_factory.call(name.to_s, help_text)
+            validate_metric!(name, metric)
+            [name, metric]
+          end
+        end
+
+        def increment(name, amount = 1)
+          unless amount.is_a?(Integer) && amount.positive?
+            raise ArgumentError, "metric increment must be a positive integer"
+          end
+
+          metric = @metrics.fetch(name)
+          (amount == 1) ? metric.inc : metric.add(amount)
+        end
+
+        def record_error(error)
+          increment(:error_records)
+          increment(:processing_limit_records) if error.is_a?(ProcessingLimitError) || error.is_a?(CyclicRecordError)
+          increment(:unsupported_records) if error.is_a?(UnsupportedRecordError)
+        end
+
+        def statistics
+          @metrics.transform_keys(&:to_s).transform_values(&:get)
+        end
+
+        private
+
+        def validate_metric!(name, metric)
+          missing_methods = %i[inc add get].reject { |method_name| metric.respond_to?(method_name) }
+          return if missing_methods.empty?
+
+          raise ArgumentError, "metric #{name} must respond to #{missing_methods.map { |name| "##{name}" }.join(", ")}"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      VERSION = "0.2.0"
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Internal component bundle. Fluentd registration remains in the stable
+# filter_jsonsizelimit discovery entry point to avoid a circular require.
+require "fluent/plugin/json_size_limit/version"
+require "fluent/plugin/json_size_limit/errors"
+require "fluent/plugin/json_size_limit/string_encoding"
+require "fluent/plugin/json_size_limit/json_string"
+require "fluent/plugin/json_size_limit/rate_limited_logger"
+require "fluent/plugin/json_size_limit/processing_guard"
+require "fluent/plugin/json_size_limit/record_measurer"
+require "fluent/plugin/json_size_limit/record_reducer"
+require "fluent/plugin/json_size_limit/record_processor"
+require "fluent/plugin/json_size_limit/filter_configuration"
+require "fluent/plugin/json_size_limit/telemetry"

metadata

@@ -1,64 +1,141 @@
 --- !ruby/object:Gem::Specification
 name: fluentd-json-size-limit
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - CreatorIQ
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-09 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.6'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 2.4.22
+        version: '5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: benchmark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.4'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 2.4.22
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '13.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '14'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.56'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 13.0.6
+        version: '2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 13.0.6
+        version: '1.56'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.6'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 3.5.7
+        version: '4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.6'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 3.5.7
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 1.16.11
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -68,35 +145,63 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 1.16.11
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description: This plugin reduces the size of JSON records if they exceed a specified
-  limit.
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.19.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.19.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+description: Reduces oversized Fluentd JSON records by safely and proportionally truncating
+  nested string values.
 email:
-- subcription-rubygems@creatoriq.com
+- subscription-rubygems@creatoriq.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".gitlab-ci.yml"
-- Gemfile
-- Gemfile.lock
+- CHANGELOG.md
 - LICENSE
+- NOTICE.md
 - README.md
-- Rakefile
 - fluentd-json-size-limit.gemspec
 - lib/fluent/plugin/filter_jsonsizelimit.rb
-- test/helper.rb
-- test/plugin/test_fluentd-json-size-limit.rb
-homepage: ''
+- lib/fluent/plugin/json_size_limit.rb
+- lib/fluent/plugin/json_size_limit/errors.rb
+- lib/fluent/plugin/json_size_limit/filter.rb
+- lib/fluent/plugin/json_size_limit/filter_configuration.rb
+- lib/fluent/plugin/json_size_limit/json_string.rb
+- lib/fluent/plugin/json_size_limit/processing_guard.rb
+- lib/fluent/plugin/json_size_limit/rate_limited_logger.rb
+- lib/fluent/plugin/json_size_limit/record_measurer.rb
+- lib/fluent/plugin/json_size_limit/record_processor.rb
+- lib/fluent/plugin/json_size_limit/record_reducer.rb
+- lib/fluent/plugin/json_size_limit/string_encoding.rb
+- lib/fluent/plugin/json_size_limit/telemetry.rb
+- lib/fluent/plugin/json_size_limit/version.rb
+homepage: https://creatoriq.com
 licenses:
-- Copyright 2017 - 2024 SocialEdge, Inc., dba CreatorIQ.
+- Copyright 2017 - 2026 SocialEdge, Inc., dba CreatorIQ.
 - Nonstandard
-metadata: {}
-post_install_message:
+metadata:
+  allowed_push_host: https://rubygems.org
 rdoc_options: []
 require_paths:
 - lib
@@ -104,17 +209,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.3'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
-signing_key:
+rubygems_version: 4.0.16
 specification_version: 4
-summary: Fluentd plugin to limit JSON size
-test_files:
-- test/helper.rb
-- test/plugin/test_fluentd-json-size-limit.rb
+summary: Fluentd filter that enforces a maximum JSON record size
+test_files: []

data/.gitignore

@@ -1,2 +0,0 @@
-*.gem
-vendor