activesupport-json_logging 1.2.1 → 1.2.2

data/lib/json_logging/sanitizer.rb

@@ -1,3 +1,5 @@
+require_relative "structured_hash_sanitizer"
+
 module JsonLogging
   module Sanitizer
     # Control characters that should be escaped or removed from log messages
@@ -18,6 +20,10 @@ module JsonLogging
     # Common sensitive key patterns (case insensitive) - fallback when Rails ParameterFilter not available
     SENSITIVE_KEY_PATTERNS = /\b(password|passwd|pwd|secret|token|api_key|apikey|access_token|auth_token|private_key|credential)\b/i
 
+    @parameter_filter = nil
+    @parameter_filter_config = nil
+    @parameter_filter_requires_full_tree_walk = false
+
     module_function
 
     # Get Rails ParameterFilter if available, nil otherwise
@@ -28,14 +34,49 @@ module JsonLogging
       filter_params = Rails.application.config.filter_parameters
       return nil if filter_params.empty?
 
-      ActiveSupport::ParameterFilter.new(filter_params)
+      if @parameter_filter && @parameter_filter_config == filter_params
+        return @parameter_filter
+      end
+
+      @parameter_filter_config = filter_params
+      @parameter_filter_requires_full_tree_walk = parameter_filter_requires_full_tree_walk?(filter_params)
+      @parameter_filter = ActiveSupport::ParameterFilter.new(filter_params)
     rescue
       nil
     end
 
+    def rails_parameter_filter_requires_full_tree_walk?
+      rails_parameter_filter
+      @parameter_filter_requires_full_tree_walk
+    end
+
+    def parameter_filter_requires_full_tree_walk?(filter_params)
+      filter_params.any? do |filter|
+        case filter
+        when Proc
+          true
+        when Regexp
+          filter.to_s.include?("\\.")
+        else
+          filter.to_s.include?(".")
+        end
+      end
+    end
+
+    def reset_rails_parameter_filter_cache!
+      @parameter_filter = nil
+      @parameter_filter_config = nil
+      @parameter_filter_requires_full_tree_walk = false
+    end
+
+    def prepare_tags(tags)
+      tags.flatten.compact.map(&:to_s).reject(&:empty?).map { |tag| sanitize_string(tag) }
+    end
+
     # Sanitize a string by removing/escaping control characters and truncating
     def sanitize_string(str)
       return str unless str.is_a?(String)
+      return str if str.length <= MAX_STRING_LENGTH && !str.match?(CONTROL_CHARS)
 
       # Remove or replace control characters
       sanitized = str.gsub(CONTROL_CHARS, "")
@@ -58,15 +99,33 @@ module JsonLogging
       # Prevent excessive nesting
       return {"error" => "max_depth_exceeded"} if depth > MAX_DEPTH
 
-      # Limit hash size first
-      limited_hash = if hash.size > MAX_CONTEXT_SIZE
+      limited_hash = limited_hash_for_sanitization(hash)
+      fast_path_result = fast_path_sanitized_hash(limited_hash, depth: depth)
+      return fast_path_result if fast_path_result
+
+      sanitize_hash_with_filtering(limited_hash, depth: depth)
+    rescue
+      {"sanitization_error" => true}
+    end
+
+    def limited_hash_for_sanitization(hash)
+      if hash.size > MAX_CONTEXT_SIZE
         truncated = hash.first(MAX_CONTEXT_SIZE).to_h
         truncated["_truncated"] = true
         truncated
       else
         hash
       end
+    end
 
+    def fast_path_sanitized_hash(hash, depth: 0)
+      return sanitize_primitive_hash(hash) if primitive_log_hash?(hash)
+      return StructuredHash.sanitize(hash, depth: depth) if StructuredHash.structured_log_hash?(hash)
+
+      nil
+    end
+
+    def sanitize_hash_with_filtering(limited_hash, depth: 0)
       # Use Rails ParameterFilter if available (handles encrypted attributes automatically)
       filter = rails_parameter_filter
       if filter
@@ -78,25 +137,23 @@ module JsonLogging
 
         # Then sanitize values (strings, control chars, etc.) preserving filtered structure
         filtered.each_with_object({}) do |(key, value), result|
-          result[key] = sanitize_value(value, depth: depth + 1)
+          result[key.to_s] = sanitize_value(value, depth: depth + 1)
         end
 
       else
         # Fallback: use pattern matching for sensitive keys
         limited_hash.each_with_object({}) do |(key, value), result|
-          key_str = key.to_s
+          key_string = key.to_s
 
           # Skip sensitive keys
-          if SENSITIVE_KEY_PATTERNS.match?(key_str)
-            result[key_str.gsub(/(?<!^)(?=[A-Z])/, "_").downcase + "_filtered"] = "[FILTERED]"
+          if SENSITIVE_KEY_PATTERNS.match?(key_string)
+            result[sensitive_filtered_key_name(key_string)] = "[FILTERED]"
             next
           end
 
-          result[key] = sanitize_value(value, depth: depth + 1)
+          result[key_string] = sanitize_value(value, depth: depth + 1)
         end
       end
-    rescue
-      {"sanitization_error" => true}
     end
 
     # Sanitize a value (handles strings, hashes, arrays, etc.)
@@ -154,5 +211,49 @@ module JsonLogging
     def sensitive_key?(key)
       SENSITIVE_KEY_PATTERNS.match?(key.to_s)
     end
+
+    def primitive_log_hash?(hash)
+      hash.all? { |_key, value| primitive_log_value?(value) }
+    end
+
+    def primitive_log_value?(value)
+      case value
+      when String
+        value.length <= MAX_STRING_LENGTH && !value.match?(CONTROL_CHARS)
+      when Numeric, TrueClass, FalseClass, NilClass
+        true
+      else
+        false
+      end
+    end
+
+    def sanitize_primitive_hash(hash)
+      filter = rails_parameter_filter
+      if filter
+        stringified = stringify_primitive_hash(hash)
+        filter.filter(stringified.dup)
+      else
+        hash.each_with_object({}) do |(key, value), result|
+          key_string = key.to_s
+
+          if SENSITIVE_KEY_PATTERNS.match?(key_string)
+            result[sensitive_filtered_key_name(key_string)] = "[FILTERED]"
+            next
+          end
+
+          result[key_string] = value
+        end
+      end
+    end
+
+    def stringify_primitive_hash(hash)
+      hash.each_with_object({}) do |(key, value), result|
+        result[key.to_s] = value
+      end
+    end
+
+    def sensitive_filtered_key_name(key_string)
+      key_string.gsub(/(?<!^)(?=[A-Z])/, "_").downcase + "_filtered"
+    end
   end
 end

data/lib/json_logging/severity.rb

@@ -0,0 +1,18 @@
+module JsonLogging
+  module Severity
+    NAMES = {
+      ::Logger::DEBUG => "DEBUG",
+      ::Logger::INFO => "INFO",
+      ::Logger::WARN => "WARN",
+      ::Logger::ERROR => "ERROR",
+      ::Logger::FATAL => "FATAL",
+      ::Logger::UNKNOWN => "UNKNOWN"
+    }.freeze
+
+    module_function
+
+    def name_for(severity)
+      NAMES[severity] || severity.to_s
+    end
+  end
+end

data/lib/json_logging/structured_hash_json_encoder.rb

@@ -0,0 +1,111 @@
+require_relative "structured_hash_sanitizer"
+
+module JsonLogging
+  module StructuredHashJsonEncoder
+    LEAF_THRESHOLD = 40
+
+    module_function
+
+    def eligible?(hash)
+      return false if Sanitizer.primitive_log_hash?(hash)
+      return false unless Sanitizer::StructuredHash.structured_log_hash?(hash)
+      return false unless large_structured_hash?(hash)
+
+      filter = Sanitizer.rails_parameter_filter
+      return true unless filter
+
+      !Sanitizer.rails_parameter_filter_requires_full_tree_walk?
+    end
+
+    def large_structured_hash?(hash, leaf_threshold: LEAF_THRESHOLD)
+      leaf_count = 0
+      walker = lambda do |value|
+        case value
+        when Hash
+          value.each_value { |entry| walker.call(entry) }
+        when Array
+          value.each { |entry| walker.call(entry) }
+        else
+          leaf_count += 1
+          throw(:enough, true) if leaf_count > leaf_threshold
+        end
+      end
+
+      catch(:enough) { walker.call(hash) }
+      leaf_count > leaf_threshold
+    end
+
+    def try_encode_line(hash, severity:, timestamp:, field_overrides: {})
+      tree = prepared_tree(hash, field_overrides: field_overrides)
+      return nil unless tree
+
+      append_json_line(
+        tree,
+        severity: severity,
+        timestamp: timestamp,
+        field_overrides: field_overrides
+      )
+    end
+
+    def encode_line(hash, severity:, timestamp:, field_overrides: {})
+      tree = encode_tree(hash, field_overrides: field_overrides)
+      append_json_line(
+        tree,
+        severity: severity,
+        timestamp: timestamp,
+        field_overrides: field_overrides
+      )
+    end
+
+    def encode_tree(hash, field_overrides: {})
+      limited_hash = Sanitizer.limited_hash_for_sanitization(hash)
+      filter = Sanitizer.rails_parameter_filter
+      omit_keys = override_keys(field_overrides)
+      jsonable = Sanitizer::StructuredHash.jsonable_tree(limited_hash, omit_keys: omit_keys)
+      filtered_jsonable_tree(jsonable, filter: filter)
+    end
+
+    def prepared_tree(hash, field_overrides: {})
+      return nil if Sanitizer.primitive_log_hash?(hash)
+
+      filter = Sanitizer.rails_parameter_filter
+      return nil if filter && Sanitizer.rails_parameter_filter_requires_full_tree_walk?
+      return nil unless Sanitizer::StructuredHash.structured_log_hash?(hash)
+
+      limited_hash = Sanitizer.limited_hash_for_sanitization(hash)
+      omit_keys = override_keys(field_overrides)
+      jsonable = Sanitizer::StructuredHash.jsonable_tree(
+        limited_hash,
+        omit_keys: omit_keys,
+        count_leaves: true
+      )
+      return nil if jsonable.leaf_count <= LEAF_THRESHOLD
+
+      filtered_jsonable_tree(jsonable, filter: filter)
+    end
+
+    def filtered_jsonable_tree(jsonable, filter:)
+      return jsonable.tree unless filter
+      return jsonable.tree if Sanitizer.rails_parameter_filter_requires_full_tree_walk?
+
+      tree = if jsonable.owned
+        jsonable.tree
+      else
+        Sanitizer::StructuredHash.stringify_keys_copy(jsonable.tree)
+      end
+
+      filter.filter(tree)
+    end
+
+    def append_json_line(tree, severity:, timestamp:, field_overrides:)
+      body = JSON.generate(tree)
+      extras = field_overrides.merge("severity" => severity, "timestamp" => timestamp)
+      extra_json = extras.map { |key, value| "#{JSON.generate(key.to_s)}:#{JSON.generate(value)}" }.join(",")
+      "#{body.chop},#{extra_json}}\n"
+    end
+
+    def override_keys(field_overrides)
+      field_overrides.keys.map(&:to_s)
+    end
+  end
+end

data/lib/json_logging/structured_hash_sanitizer.rb

@@ -0,0 +1,206 @@
+module JsonLogging
+  module Sanitizer
+    module StructuredHash
+      JsonableResult = Struct.new(:tree, :owned, :leaf_count, keyword_init: true)
+
+      module_function
+
+      def structured_log_hash?(hash, seen = nil)
+        seen ||= Set.new.compare_by_identity
+        return false unless seen.add?(hash)
+
+        result = hash.all? { |_key, value| structured_log_value?(value, seen) }
+        seen.delete(hash)
+        result
+      end
+
+      def structured_log_value?(value, seen = nil)
+        case value
+        when String, Numeric, TrueClass, FalseClass, NilClass
+          Sanitizer.primitive_log_value?(value)
+        when Hash
+          structured_log_hash?(value, seen)
+        when Array
+          structured_log_array?(value, seen)
+        else
+          false
+        end
+      end
+
+      def structured_log_array?(array, seen = nil)
+        array.all? { |value| structured_log_value?(value, seen) }
+      end
+
+      def sanitize(hash, depth: 0)
+        return {"error" => "max_depth_exceeded"} if depth > Sanitizer::MAX_DEPTH
+
+        filter = Sanitizer.rails_parameter_filter
+        if filter
+          filter.filter(stringify(hash))
+        else
+          sanitize_without_filter(hash, depth: depth)
+        end
+      end
+
+      def sanitize_without_filter(hash, depth: 0)
+        jsonable = jsonable_tree(hash, depth: depth)
+        if jsonable.owned
+          stringify(jsonable.tree)
+        else
+          jsonable.tree.dup
+        end
+      end
+
+      def jsonable_tree(hash, depth: 0, seen: nil, parent_key: nil, omit_keys: nil, count_leaves: false, leaf_counter: nil)
+        seen ||= Set.new.compare_by_identity
+        leaf_counter = [0] if count_leaves && leaf_counter.nil?
+        if depth > Sanitizer::MAX_DEPTH
+          return JsonableResult.new(tree: {"error" => "max_depth_exceeded"}, owned: true, leaf_count: leaf_counter&.[](0))
+        end
+        unless seen.add?(hash)
+          return JsonableResult.new(tree: {"error" => "cyclic_reference"}, owned: true, leaf_count: leaf_counter&.[](0))
+        end
+
+        entries = []
+        owned = false
+
+        hash.each do |key, value|
+          entry = process_jsonable_entry(
+            key,
+            value,
+            depth: depth,
+            seen: seen,
+            parent_key: parent_key,
+            omit_keys: omit_keys,
+            leaf_counter: leaf_counter
+          )
+          next unless entry
+
+          key_string, encoded_value, pair_owned = entry
+          owned ||= pair_owned
+          entries << [key_string, encoded_value, pair_owned]
+        end
+
+        seen.delete(hash)
+        leaf_count = leaf_counter&.[](0)
+        unless owned
+          tree = omit_root_keys(hash, omit_keys, depth: depth)
+          return JsonableResult.new(tree: tree, owned: false, leaf_count: leaf_count)
+        end
+
+        result = entries.each_with_object({}) do |(key_string, encoded_value, _pair_owned), object|
+          object[key_string] = encoded_value
+        end
+        JsonableResult.new(tree: result, owned: true, leaf_count: leaf_count)
+      end
+
+      def jsonable_value(value, depth:, seen:, parent_key:, leaf_counter: nil)
+        case value
+        when String
+          leaf_counter[0] += 1 if leaf_counter
+          sanitized = Sanitizer.sanitize_string(value)
+          JsonableResult.new(tree: sanitized, owned: sanitized != value)
+        when Hash
+          jsonable_tree(value, depth: depth + 1, seen: seen, parent_key: parent_key, leaf_counter: leaf_counter)
+        when Array
+          jsonable_array(value, depth: depth, seen: seen, parent_key: parent_key, leaf_counter: leaf_counter)
+        else
+          leaf_counter[0] += 1 if leaf_counter
+          JsonableResult.new(tree: value, owned: false)
+        end
+      end
+
+      def jsonable_array(array, depth:, seen:, parent_key:, leaf_counter: nil)
+        owned = false
+        result = nil
+
+        array.each_with_index do |entry, index|
+          child = jsonable_value(entry, depth: depth, seen: seen, parent_key: parent_key, leaf_counter: leaf_counter)
+          next unless child.owned
+
+          owned = true
+          result ||= array.dup
+          result[index] = child.tree
+        end
+
+        JsonableResult.new(tree: owned ? result : array, owned: owned)
+      end
+
+      def process_jsonable_entry(key, value, depth:, seen:, parent_key:, omit_keys:, leaf_counter:)
+        key_string = key.to_s
+        if root_key_omitted?(depth, key_string, omit_keys)
+          count_leaves_in_value(value, leaf_counter, depth: depth, seen: seen) if leaf_counter
+          return
+        end
+
+        if Sanitizer::SENSITIVE_KEY_PATTERNS.match?(key_string)
+          return [Sanitizer.sensitive_filtered_key_name(key_string), "[FILTERED]", true]
+        end
+
+        nested_parent = parent_key ? "#{parent_key}.#{key_string}" : key_string
+        child = jsonable_value(
+          value,
+          depth: depth,
+          seen: seen,
+          parent_key: nested_parent,
+          leaf_counter: leaf_counter
+        )
+        encoded_value = child.owned ? child.tree : value
+        [key_string, encoded_value, child.owned]
+      end
+
+      def root_key_omitted?(depth, key_string, omit_keys)
+        depth.zero? && omit_keys&.include?(key_string)
+      end
+
+      def omit_root_keys(hash, omit_keys, depth:)
+        return hash unless depth.zero? && omit_keys&.any?
+
+        hash.each_with_object({}) do |(key, value), object|
+          object[key] = value unless omit_keys.include?(key.to_s)
+        end
+      end
+
+      def count_leaves_in_value(value, leaf_counter, depth:, seen:)
+        case value
+        when Hash
+          return unless seen.add?(value)
+
+          value.each_value do |entry|
+            count_leaves_in_value(entry, leaf_counter, depth: depth + 1, seen: seen)
+          end
+          seen.delete(value)
+        when Array
+          value.each { |entry| count_leaves_in_value(entry, leaf_counter, depth: depth, seen: seen) }
+        else
+          leaf_counter[0] += 1
+        end
+      end
+
+      def stringify(hash)
+        hash.each_with_object({}) do |(key, value), result|
+          result[key.to_s] = stringify_value(value)
+        end
+      end
+
+      def stringify_value(value)
+        case value
+        when Hash
+          stringify(value)
+        when Array
+          value.map { |entry| stringify_value(entry) }
+        when String
+          Sanitizer.sanitize_string(value)
+        else
+          value
+        end
+      end
+
+      def stringify_keys_copy(hash)
+        return hash if hash.keys.all? { |key| key.is_a?(String) }
+
+        hash.transform_keys(&:to_s)
+      end
+    end
+  end
+end

data/lib/json_logging/version.rb

@@ -1,3 +1,3 @@
 module JsonLogging
-  VERSION = "1.2.1"
+  VERSION = "1.2.2"
 end

data/lib/json_logging.rb

@@ -7,6 +7,7 @@ require_relative "json_logging/helpers"
 require_relative "json_logging/sanitizer"
 require_relative "json_logging/message_parser"
 require_relative "json_logging/payload_builder"
+require_relative "json_logging/line_encoder"
 require_relative "json_logging/formatter"
 require_relative "json_logging/formatter_with_tags"
 require_relative "json_logging/json_logger_extension"
@@ -14,13 +15,28 @@ require_relative "json_logging/json_logger"
 
 module JsonLogging
   THREAD_CONTEXT_KEY = :__json_logging_context
+  SANITIZED_CONTEXT_CACHE_KEY = :__json_logging_sanitized_context
+  @additional_context_warned = Set.new
+  @additional_context_transformer = nil
+
+  def self.context_storage
+    if defined?(ActiveSupport::IsolatedExecutionState)
+      ActiveSupport::IsolatedExecutionState
+    else
+      Thread.current
+    end
+  end
+  private_class_method :context_storage
 
   def self.with_context(extra_context)
-    original = Thread.current[THREAD_CONTEXT_KEY]
-    Thread.current[THREAD_CONTEXT_KEY] = (original || {}).merge(safe_hash(extra_context))
+    storage = context_storage
+    original = storage[THREAD_CONTEXT_KEY]
+    storage[THREAD_CONTEXT_KEY] = (original || {}).merge(safe_hash(extra_context))
+    invalidate_sanitized_context_cache(storage)
     yield
   ensure
-    Thread.current[THREAD_CONTEXT_KEY] = original
+    storage[THREAD_CONTEXT_KEY] = original
+    invalidate_sanitized_context_cache(storage)
   end
 
   # Returns the current thread-local context when called without arguments,
@@ -42,8 +58,9 @@ module JsonLogging
     end
 
     begin
-      base_context = (Thread.current[THREAD_CONTEXT_KEY] || {}).dup
-    rescue
+      base_context = (context_storage[THREAD_CONTEXT_KEY] || {}).dup
+    rescue => e
+      warn_additional_context_once(:dup, "thread context dup failed", e)
       base_context = {}
     end
 
@@ -51,7 +68,8 @@ module JsonLogging
     if transformer.is_a?(Proc)
       begin
         transformer.call(base_context)
-      rescue
+      rescue => e
+        warn_additional_context_once(:transformer, "additional_context transformer failed", e)
         base_context
       end
     else
@@ -61,6 +79,26 @@ module JsonLogging
 
   def self.additional_context=(proc_or_block)
     @additional_context_transformer = proc_or_block
+    invalidate_sanitized_context_cache(context_storage)
+  end
+
+  def self.additional_context_for_payload
+    if @additional_context_transformer.is_a?(Proc)
+      return sanitized_context_from(compact_context(additional_context))
+    end
+
+    storage = context_storage
+    cached = storage[SANITIZED_CONTEXT_CACHE_KEY]
+    return cached if cached
+
+    raw_context = storage[THREAD_CONTEXT_KEY]
+    if raw_context.nil? || (raw_context.is_a?(Hash) && raw_context.empty?)
+      return storage[SANITIZED_CONTEXT_CACHE_KEY] = {}.freeze
+    end
+
+    sanitized = sanitized_context_from(compact_context(raw_context))
+    storage[SANITIZED_CONTEXT_CACHE_KEY] = sanitized.freeze
+    sanitized
   end
 
   def self.safe_hash(obj)
@@ -69,6 +107,34 @@ module JsonLogging
     {}
   end
 
+  def self.warn_additional_context_once(key, message, error)
+    return if @additional_context_warned.include?(key)
+
+    @additional_context_warned.add(key)
+    warn "[activesupport-json_logging] #{message} (#{error.class}: #{error.message})"
+  end
+  private_class_method :warn_additional_context_once
+
+  def self.compact_context(context)
+    return {} unless context.is_a?(Hash)
+    return {} if context.empty?
+
+    context.compact
+  end
+  private_class_method :compact_context
+
+  def self.sanitized_context_from(compacted_context)
+    return {} if compacted_context.empty?
+
+    Sanitizer.sanitize_hash(compacted_context)
+  end
+  private_class_method :sanitized_context_from
+
+  def self.invalidate_sanitized_context_cache(storage)
+    storage[SANITIZED_CONTEXT_CACHE_KEY] = nil
+  end
+  private_class_method :invalidate_sanitized_context_cache
+
   # Returns an `ActiveSupport::Logger` that has already been wrapped with JSON logging concern.
   #
   # @param *args Arguments passed to ActiveSupport::Logger.new
@@ -97,6 +163,17 @@ module JsonLogging
   #   logger.tagged("BCX") { logger.info("Stuff") }  # Logs with tags
   #   logger.tagged("BCX").info("Stuff")  # Logs with tags (non-block form)
   def self.new(logger)
+    logger = prepare_logger_clone(logger)
+    logger.extend(JsonLoggerExtension)
+    formatter_with_tags = FormatterWithTags.new(logger)
+    logger.instance_variable_set(:@formatter_with_tags, formatter_with_tags)
+    # Custom formatters are not supported: JSON output requires FormatterWithTags.
+    logger.formatter = formatter_with_tags
+
+    logger
+  end
+
+  def self.prepare_logger_clone(logger)
     logger = logger.clone
     if logger.formatter
       logger.formatter = logger.formatter.clone
@@ -114,11 +191,7 @@ module JsonLogging
       end
     end
 
-    logger.extend(JsonLoggerExtension)
-    formatter_with_tags = FormatterWithTags.new(logger)
-    logger.instance_variable_set(:@formatter_with_tags, formatter_with_tags)
-    logger.formatter = formatter_with_tags
-
     logger
   end
+  private_class_method :prepare_logger_clone
 end

data/sig/json_logging.rbs

@@ -0,0 +1,3 @@
+module JsonLogging
+  VERSION: String
+end