fluentd-json-size-limit 0.1.4 → 0.2.0

data/lib/fluent/plugin/json_size_limit/filter.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/filter"
+require "fluent/plugin/json_size_limit"
+
+module Fluent
+  module Plugin
+    class JsonSizeLimitFilter < Filter
+      include JsonSizeLimit::FilterConfiguration
+
+      Fluent::Plugin.register_filter("jsonsizelimit", self)
+
+      # Compatibility constant for callers that referenced the earlier class.
+      UnreducibleRecordError = JsonSizeLimit::UnreducibleRecordError
+
+      def configure(conf)
+        super
+        validate_json_size_limit_configuration!
+
+        @telemetry = JsonSizeLimit::Telemetry.new { |name, help_text| create_metric(name, help_text) }
+        @record_processor = JsonSizeLimit::RecordProcessor.new(
+          max_json_bytes: @max_size,
+          safety_options: processing_safety_options
+        )
+        @operational_log = JsonSizeLimit::RateLimitedLogger.new(log, @warning_interval)
+
+        log.on_debug do
+          log.debug(
+            "Configured JsonSizeLimitFilter",
+            max_size: @max_size,
+            enabled: @enabled,
+            overflow_action: @overflow_action,
+            error_action: @error_action,
+            processing_timeout: @processing_timeout,
+            max_record_nodes: @max_record_nodes,
+            max_input_size: @max_input_size,
+            max_nesting: @max_nesting
+          )
+        end
+      end
+
+      # The processor has no mutable cross-record state. Fluentd automatically
+      # invokes the same implementation from every configured worker/thread.
+      def multi_workers_ready?
+        true
+      end
+
+      def filter(_tag, _time, record)
+        return record unless @enabled
+
+        result = @record_processor.call(record)
+
+        case result.status
+        when :within_limit
+          record
+        when :reduced
+          handle_reduced(record, result)
+        when :overflow
+          handle_overflow(record, result)
+        else
+          raise JsonSizeLimit::UnexpectedProcessorStatusError, result.status
+        end
+      rescue UnreducibleRecordError
+        raise
+      rescue => error
+        handle_error(record, error)
+      end
+
+      def statistics
+        super.merge("json_size_limit" => @telemetry&.statistics || {})
+      end
+
+      def shutdown
+        @operational_log&.flush
+      ensure
+        super
+      end
+
+      private
+
+      def create_metric(name, help_text)
+        metrics_create(
+          namespace: "fluentd",
+          subsystem: "json_size_limit",
+          name: name,
+          help_text: help_text
+        )
+      end
+
+      def handle_reduced(record, result)
+        # RecordProcessor applies replacements in place so Fluentd retains the
+        # original record identity and any aliases owned by upstream plugins.
+        @telemetry.increment(:oversized_records)
+        @telemetry.increment(:reduced_records)
+        @telemetry.increment(:removed_bytes, result.original_json_bytes - result.json_bytes)
+        log.on_debug do
+          log.debug(
+            "Reduced oversized JSON record",
+            original_json_bytes: result.original_json_bytes,
+            reduced_json_bytes: result.json_bytes,
+            max_json_bytes: @max_size
+          )
+        end
+        record
+      end
+
+      def handle_overflow(record, result)
+        @telemetry.increment(:oversized_records)
+        @telemetry.increment(:unreducible_records)
+        fields = {
+          action: @overflow_action,
+          original_json_bytes: result.original_json_bytes,
+          reduced_json_bytes: result.json_bytes,
+          max_json_bytes: @max_size
+        }
+
+        case @overflow_action
+        when :pass
+          @operational_log.emit(:warn, :overflow, "Record remains above max_size after string reduction", fields)
+          record
+        when :drop
+          @telemetry.increment(:dropped_records)
+          @operational_log.emit(:warn, :overflow, "Dropping record that remains above max_size", fields)
+          nil
+        when :raise
+          raise UnreducibleRecordError,
+            "Record remains above max_size after string reduction: " \
+              "original=#{result.original_json_bytes} bytes, reduced=#{result.json_bytes} bytes, max=#{@max_size} bytes"
+        end
+      end
+
+      def handle_error(record, error)
+        @telemetry.record_error(error)
+        fields = {
+          action: @error_action,
+          error_class: error.class.name
+        }
+        fields[:safety_limit] = error.limit if error.is_a?(JsonSizeLimit::ProcessingLimitError)
+
+        case @error_action
+        when :pass
+          @operational_log.emit(:error, :error, "Failed to measure or reduce record JSON", fields)
+          record
+        when :drop
+          @telemetry.increment(:dropped_records)
+          @operational_log.emit(:error, :error, "Dropping record after JSON measurement or reduction failure", fields)
+          nil
+        when :raise
+          raise error
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit/filter_configuration.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      module FilterConfiguration
+        DEFAULT_MAX_INPUT_SIZE = 64 * 1024 * 1024
+
+        def self.included(base)
+          base.class_eval do
+            config_param :max_size, :size, default: 250 * 1024
+            config_param :enabled, :bool, default: true
+            config_param :overflow_action, :enum, list: %i[pass drop raise], default: :pass
+            config_param :error_action, :enum, list: %i[pass drop raise], default: :pass
+            config_param :warning_interval, :time, default: 60
+
+            # Automatic safety budgets. Existing configurations need no new
+            # parameters; operators may raise them for exceptional workloads.
+            config_param :processing_timeout, :time, default: 5
+            config_param :max_record_nodes, :integer, default: 1_000_000
+            config_param :max_input_size, :size, default: DEFAULT_MAX_INPUT_SIZE
+            config_param :max_nesting, :integer, default: 100
+          end
+        end
+
+        private
+
+        def validate_json_size_limit_configuration!
+          raise Fluent::ConfigError, "max_size must be greater than zero" unless @max_size.positive?
+          raise Fluent::ConfigError, "warning_interval must not be negative" if @warning_interval.negative?
+          raise Fluent::ConfigError, "processing_timeout must be greater than zero" unless @processing_timeout.positive?
+          raise Fluent::ConfigError, "max_record_nodes must be greater than zero" unless @max_record_nodes.positive?
+          raise Fluent::ConfigError, "max_input_size must be greater than zero" unless @max_input_size.positive?
+          raise Fluent::ConfigError, "max_nesting must be greater than zero" unless @max_nesting.positive?
+        end
+
+        def processing_safety_options
+          {
+            timeout: @processing_timeout,
+            max_nodes: @max_record_nodes,
+            max_input_bytes: [@max_input_size, @max_size].max,
+            max_nesting: @max_nesting
+          }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit/json_string.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "json"
+require "fluent/plugin/json_size_limit/string_encoding"
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      # Computes and truncates the bytes between a JSON string's quotes. Plain
+      # UTF-8 takes a zero-serialization path; escaped UTF-8 is generated once
+      # and decoded at a safe JSON token boundary.
+      class JsonString
+        ESCAPED_CHARACTERS = "\"\\\\\x00-\x1f"
+        ESCAPE_REQUIRED = /["\\\x00-\x1f]/
+        COUNT_SCAN_THRESHOLD = 1024
+        UTF_8 = Encoding::UTF_8
+
+        attr_reader :content_bytes, :truncated_content_bytes, :value
+
+        def self.content_bytes(value)
+          new(value).content_bytes
+        end
+
+        def self.truncate(value, allowed_content_bytes)
+          json_string = new(value)
+          truncated_value = json_string.truncate(allowed_content_bytes)
+          [truncated_value, json_string.truncated_content_bytes]
+        end
+
+        def initialize(value, validate_encoding: true)
+          @value = value
+          StringEncoding.validate!(value) if validate_encoding
+
+          if raw_json_content?
+            @strategy = :raw
+            @content_bytes = value.bytesize
+          else
+            @generated = JSON.generate(value)
+            @content_bytes = @generated.bytesize - 2
+            @strategy = directly_decodable? ? :generated : :fallback
+          end
+        end
+
+        def truncate(allowed_content_bytes)
+          if allowed_content_bytes <= 0
+            @truncated_content_bytes = 0
+            return empty_value
+          end
+
+          if allowed_content_bytes >= @content_bytes
+            @truncated_content_bytes = @content_bytes
+            return @value
+          end
+
+          case @strategy
+          when :raw
+            truncate_raw(allowed_content_bytes)
+          when :generated
+            truncate_generated(allowed_content_bytes)
+          when :fallback
+            truncate_with_binary_search(allowed_content_bytes)
+          end
+        end
+
+        private
+
+        def directly_decodable?
+          return false unless @value.instance_of?(String)
+
+          @value.ascii_only? || (@value.encoding == UTF_8 && @value.valid_encoding?)
+        end
+
+        def raw_json_content?
+          directly_decodable? && !escape_required?
+        end
+
+        def escape_required?
+          if @value.bytesize < COUNT_SCAN_THRESHOLD
+            ESCAPE_REQUIRED.match?(@value)
+          else
+            !@value.count(ESCAPED_CHARACTERS).zero?
+          end
+        end
+
+        def truncate_raw(allowed_content_bytes)
+          byte_limit = allowed_content_bytes
+
+          if @value.encoding == UTF_8
+            byte_limit -= 1 while byte_limit.positive? && continuation_byte?(@value.getbyte(byte_limit))
+          end
+
+          prefix = @value.byteslice(0, byte_limit)
+          @truncated_content_bytes = prefix.bytesize
+          prefix
+        end
+
+        # A cut can only split a UTF-8 sequence (at most 4 bytes) or a JSON
+        # escape (at most 6 bytes). Retrying from the desired byte budget finds
+        # the maximal valid prefix without serializing source prefixes again.
+        def truncate_generated(allowed_content_bytes)
+          content_bytes = allowed_content_bytes
+
+          loop do
+            candidate = @generated.byteslice(0, content_bytes + 1)
+            candidate << '"'
+
+            begin
+              prefix = JSON.parse(candidate)
+              unless prefix.valid_encoding?
+                content_bytes -= 1
+                next
+              end
+
+              prefix.force_encoding(@value.encoding) if @value.ascii_only? && @value.encoding != UTF_8
+              @truncated_content_bytes = content_bytes
+              return prefix
+            rescue JSON::ParserError
+              content_bytes -= 1
+            end
+          end
+        end
+
+        def truncate_with_binary_search(allowed_content_bytes)
+          low = 0
+          high = @value.bytesize
+          best_prefix = empty_value
+          best_json_bytes = 0
+
+          while low <= high
+            midpoint = (low + high) / 2
+            prefix = valid_encoding_prefix(midpoint)
+            prefix_json_bytes = JSON.generate(prefix).bytesize - 2
+
+            if prefix_json_bytes <= allowed_content_bytes
+              best_prefix = prefix
+              best_json_bytes = prefix_json_bytes
+              low = midpoint + 1
+            else
+              high = midpoint - 1
+            end
+          end
+
+          @truncated_content_bytes = best_json_bytes
+          best_prefix
+        end
+
+        def valid_encoding_prefix(byte_limit)
+          return @value if byte_limit >= @value.bytesize
+
+          prefix = @value.byteslice(0, byte_limit)
+          while byte_limit.positive? && !prefix.valid_encoding?
+            byte_limit -= 1
+            prefix = @value.byteslice(0, byte_limit)
+          end
+
+          prefix || empty_value
+        end
+
+        def continuation_byte?(byte)
+          byte && byte >= 0x80 && byte < 0xC0
+        end
+
+        def empty_value
+          @value.byteslice(0, 0) || String.new(encoding: @value.encoding)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit/processing_guard.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/json_size_limit/errors"
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      # Cooperative per-record limits. Clock checks are sampled to keep the hot
+      # path cheap, while explicit checkpoints surround potentially large work.
+      class ProcessingGuard
+        MONOTONIC_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+        CHECK_INTERVAL = 256
+
+        def initialize(timeout:, max_nodes:, max_input_bytes:, max_nesting:, clock: MONOTONIC_TIME,
+          check_interval: CHECK_INTERVAL)
+          validate_options!(timeout, max_nodes, max_input_bytes, max_nesting, clock, check_interval)
+
+          @clock = clock
+          @deadline = clock.call + timeout
+          @max_nodes = max_nodes
+          @max_input_bytes = max_input_bytes
+          @max_nesting = max_nesting
+          @check_interval = check_interval
+          reset_phase!
+        end
+
+        def visit!(depth, input_bytes = 0)
+          raise ProcessingLimitError, :max_nesting if depth > @max_nesting
+
+          @nodes += 1
+          raise ProcessingLimitError, :max_record_nodes if @nodes > @max_nodes
+
+          account_input_bytes!(input_bytes)
+          checkpoint! if (@nodes % @check_interval).zero?
+        end
+
+        def account_input_bytes!(bytes)
+          return unless @track_input
+
+          @input_bytes += bytes
+          raise ProcessingLimitError, :max_input_size if @input_bytes > @max_input_bytes
+        end
+
+        def enforce_input_bytes!(bytes)
+          raise ProcessingLimitError, :max_input_size if bytes > @max_input_bytes
+        end
+
+        def checkpoint!
+          raise ProcessingLimitError, :processing_timeout if @clock.call > @deadline
+        end
+
+        def reset_phase!(track_input: true)
+          @nodes = 0
+          @input_bytes = 0
+          @track_input = track_input
+        end
+
+        private
+
+        def validate_options!(timeout, max_nodes, max_input_bytes, max_nesting, clock, check_interval)
+          raise ArgumentError, "timeout must be a positive number" unless timeout.is_a?(Numeric) && timeout.positive?
+          raise ArgumentError, "max_nodes must be a positive integer" unless max_nodes.is_a?(Integer) && max_nodes.positive?
+          unless max_input_bytes.is_a?(Numeric) && max_input_bytes.positive?
+            raise ArgumentError, "max_input_bytes must be a positive number"
+          end
+          unless max_nesting.is_a?(Integer) && max_nesting.positive?
+            raise ArgumentError, "max_nesting must be a positive integer"
+          end
+          unless check_interval.is_a?(Integer) && check_interval.positive?
+            raise ArgumentError, "check_interval must be a positive integer"
+          end
+          raise ArgumentError, "clock must respond to call" unless clock.respond_to?(:call)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit/rate_limited_logger.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      # Serializes rate-limit state across Fluentd threads without putting a
+      # lock on the normal record-processing path.
+      class RateLimitedLogger
+        MONOTONIC_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+
+        def initialize(logger, interval, clock: MONOTONIC_TIME)
+          raise ArgumentError, "interval must not be negative" unless interval.is_a?(Numeric) && !interval.negative?
+          raise ArgumentError, "clock must respond to call" unless clock.respond_to?(:call)
+
+          @logger = logger
+          @interval = interval
+          @clock = clock
+          @mutex = Mutex.new
+          @states = {}
+        end
+
+        def emit(level, category, message, fields = {})
+          return write(level, message, fields) if @interval.zero?
+
+          now = @clock.call
+          suppressed = @mutex.synchronize do
+            state = (@states[category] ||= {last_at: nil, suppressed: 0})
+
+            if state[:last_at].nil? || now - state[:last_at] >= @interval
+              suppressed_count = state[:suppressed]
+              state[:last_at] = now
+              state[:suppressed] = 0
+              suppressed_count
+            else
+              state[:suppressed] += 1
+              nil
+            end
+          end
+          return unless suppressed
+
+          fields = fields.merge(suppressed_events: suppressed) if suppressed.positive?
+          write(level, message, fields)
+        end
+
+        def flush
+          summaries = @mutex.synchronize do
+            @states.filter_map do |category, state|
+              next unless state[:suppressed].positive?
+
+              summary = [category, state[:suppressed]]
+              state[:suppressed] = 0
+              summary
+            end
+          end
+
+          summaries.each do |category, suppressed|
+            write(
+              :info,
+              "Suppressed repeated jsonsizelimit events before shutdown",
+              {category: category, suppressed_events: suppressed}
+            )
+          end
+        end
+
+        private
+
+        # Observability must never become a data-path or shutdown failure.
+        def write(level, message, fields)
+          @logger.public_send(level, message, fields)
+        rescue
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/json_size_limit/record_measurer.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "json"
+require "fluent/plugin/json_size_limit/errors"
+require "fluent/plugin/json_size_limit/string_encoding"
+
+module Fluent
+  module Plugin
+    module JsonSizeLimit
+      # Performs a bounded, side-effect-free preflight before using the native
+      # JSON generator exactly once. The preflight prevents arbitrary Ruby
+      # serializers or traversal overrides from entering the hot path.
+      class RecordMeasurer
+        EXIT_CONTAINER = Object.new.freeze
+        HASH_EACH_PAIR = Hash.instance_method(:each_pair)
+        HASH_LENGTH = Hash.instance_method(:length)
+        ARRAY_EACH = Array.instance_method(:each)
+        ARRAY_LENGTH = Array.instance_method(:length)
+        STRING_BYTESIZE = String.instance_method(:bytesize)
+        SINGLETON_METHODS = Object.instance_method(:singleton_methods)
+
+        def initialize(guard)
+          @guard = guard
+        end
+
+        def measure(record)
+          validate_record(record)
+          @guard.checkpoint!
+
+          json_bytes = JSON.generate(record).bytesize
+          @guard.enforce_input_bytes!(json_bytes)
+          @guard.checkpoint!
+          json_bytes
+        end
+
+        private
+
+        def validate_record(record)
+          active_containers = {}.compare_by_identity
+          stack = [record, 0]
+
+          until stack.empty?
+            frame_data = stack.pop
+            value = stack.pop
+
+            if value.equal?(EXIT_CONTAINER)
+              # Exit frames store the container identity in the same flat stack
+              # slot that regular frames use for their numeric depth.
+              active_containers.delete(frame_data)
+            else
+              validate_value(value, frame_data, stack, active_containers)
+            end
+          end
+        end
+
+        def validate_value(value, depth, stack, active_containers)
+          case value
+          when String
+            reject_subclass!(value, String)
+            StringEncoding.validate!(value)
+            @guard.visit!(depth, STRING_BYTESIZE.bind_call(value))
+          when Hash
+            reject_subclass!(value, Hash)
+            validate_hash(value, depth, stack, active_containers)
+          when Array
+            reject_subclass!(value, Array)
+            validate_array(value, depth, stack, active_containers)
+          when Integer, Float, Symbol
+            reject_subclass!(value, value.class)
+            @guard.visit!(depth)
+          when true, false, nil
+            @guard.visit!(depth)
+          else
+            raise UnsupportedRecordError, value.class.name
+          end
+        end
+
+        def validate_hash(hash, depth, stack, active_containers)
+          reject_singleton_methods!(hash)
+          enter_container!(hash, active_containers)
+          length = HASH_LENGTH.bind_call(hash)
+          @guard.visit!(depth, 2 + length + [length - 1, 0].max)
+          stack << EXIT_CONTAINER << hash
+
+          HASH_EACH_PAIR.bind_call(hash) do |key, value|
+            validate_key(key)
+            stack << value << (depth + 1)
+          end
+        end
+
+        def validate_array(array, depth, stack, active_containers)
+          reject_singleton_methods!(array)
+          enter_container!(array, active_containers)
+          length = ARRAY_LENGTH.bind_call(array)
+          @guard.visit!(depth, 2 + [length - 1, 0].max)
+          stack << EXIT_CONTAINER << array
+          ARRAY_EACH.bind_call(array) { |value| stack << value << (depth + 1) }
+        end
+
+        def validate_key(key)
+          case key
+          when String
+            reject_subclass!(key, String)
+            StringEncoding.validate!(key)
+            @guard.account_input_bytes!(STRING_BYTESIZE.bind_call(key))
+          when Symbol, Integer, Float, true, false, nil
+            reject_subclass!(key, key.class) unless key.nil?
+          else
+            raise UnsupportedRecordError, key.class.name
+          end
+        end
+
+        def enter_container!(container, active_containers)
+          raise CyclicRecordError if active_containers.key?(container)
+
+          active_containers[container] = true
+        end
+
+        def reject_subclass!(value, expected_class)
+          return if value.instance_of?(expected_class)
+
+          raise UnsupportedRecordError, value.class.name
+        end
+
+        def reject_singleton_methods!(value)
+          return if SINGLETON_METHODS.bind_call(value, false).empty?
+
+          raise UnsupportedRecordError, value.class.name
+        end
+      end
+    end
+  end
+end