activesupport-json_logging 1.2.0 → 1.2.2

data/lib/json_logging/line_encoder.rb

@@ -0,0 +1,245 @@
+require "active_support/core_ext/hash/keys"
+
+require_relative "message_parser"
+require_relative "severity"
+require_relative "payload_builder"
+require_relative "structured_hash_json_encoder"
+
+module JsonLogging
+  module LineEncoder
+    module_function
+
+    def build_line(msg:, severity:, timestamp:, tags:, additional_context:, additional_context_sanitized: false, sanitize_tags: true)
+      if simple_string_line?(msg, tags, additional_context)
+        return simple_string_json_line(
+          message: msg,
+          severity: Severity.name_for(severity),
+          timestamp: timestamp
+        )
+      end
+
+      if tagged_simple_string_line?(msg, tags, additional_context)
+        return tagged_simple_string_json_line(
+          message: msg,
+          severity: Severity.name_for(severity),
+          timestamp: timestamp,
+          tags: tags,
+          sanitize_tags: sanitize_tags
+        )
+      end
+
+      if contextual_simple_string_line?(msg, tags, additional_context, additional_context_sanitized: additional_context_sanitized)
+        return contextual_simple_string_json_line(
+          message: msg,
+          severity: Severity.name_for(severity),
+          timestamp: timestamp,
+          additional_context: additional_context
+        )
+      end
+
+      if standalone_hash_line?(msg, tags, additional_context)
+        return standalone_hash_json_line(
+          message: msg,
+          severity: Severity.name_for(severity),
+          timestamp: timestamp
+        )
+      end
+
+      if tagged_hash_line?(msg, tags, additional_context)
+        return tagged_hash_json_line(
+          message: msg,
+          severity: Severity.name_for(severity),
+          timestamp: timestamp,
+          tags: tags,
+          sanitize_tags: sanitize_tags
+        )
+      end
+
+      if contextual_hash_line?(msg, tags, additional_context, additional_context_sanitized: additional_context_sanitized)
+        return contextual_hash_json_line(
+          message: msg,
+          severity: Severity.name_for(severity),
+          timestamp: timestamp,
+          additional_context: additional_context
+        )
+      end
+
+      sev = Severity.name_for(severity)
+      payload = PayloadBuilder.build_base_payload(msg, severity: sev, timestamp: timestamp)
+      unless tags.empty? && PayloadBuilder.empty_additional_context?(additional_context)
+        payload = PayloadBuilder.merge_context(
+          payload,
+          additional_context: additional_context,
+          tags: tags,
+          additional_context_sanitized: additional_context_sanitized,
+          sanitize_tags: sanitize_tags
+        )
+      end
+      to_json_line(payload)
+    end
+
+    def to_json_line(payload_hash)
+      compacted = payload_hash.compact
+      json_payload = string_keyed_structure?(compacted) ? compacted : compacted.deep_stringify_keys
+      "#{json_payload.to_json}\n"
+    end
+
+    def string_keyed_structure?(object)
+      case object
+      when Hash
+        object.keys.all?(String) && object.values.all? { |value| string_keyed_structure?(value) }
+      when Array
+        object.all? { |value| string_keyed_structure?(value) }
+      else
+        true
+      end
+    end
+
+    def simple_string_line?(message, tags, additional_context)
+      message.is_a?(String) &&
+        !MessageParser.json_string?(message) &&
+        tags.respond_to?(:empty?) && tags.empty? &&
+        additional_context.respond_to?(:empty?) && additional_context.empty?
+    end
+
+    def simple_string_json_line(message:, severity:, timestamp:)
+      sanitized_message = Sanitizer.sanitize_string(message)
+      "#{JSON.generate("message" => sanitized_message, "severity" => severity, "timestamp" => timestamp)}\n"
+    end
+
+    def tagged_simple_string_line?(message, tags, additional_context)
+      message.is_a?(String) &&
+        !MessageParser.json_string?(message) &&
+        tags.respond_to?(:empty?) && !tags.empty? &&
+        additional_context.respond_to?(:empty?) && additional_context.empty?
+    end
+
+    def tagged_simple_string_json_line(message:, severity:, timestamp:, tags:, sanitize_tags:)
+      sanitized_message = Sanitizer.sanitize_string(message)
+      prepared_tags = sanitize_tags ? Sanitizer.prepare_tags(tags) : tags
+      "#{JSON.generate("message" => sanitized_message, "severity" => severity, "timestamp" => timestamp, "tags" => prepared_tags.uniq)}\n"
+    end
+
+    def contextual_simple_string_line?(message, tags, additional_context, additional_context_sanitized:)
+      message.is_a?(String) &&
+        !MessageParser.json_string?(message) &&
+        tags.respond_to?(:empty?) && tags.empty? &&
+        !PayloadBuilder.empty_additional_context?(additional_context) &&
+        additional_context_sanitized
+    end
+
+    def contextual_simple_string_json_line(message:, severity:, timestamp:, additional_context:)
+      sanitized_message = Sanitizer.sanitize_string(message)
+      payload = {
+        "message" => sanitized_message,
+        "severity" => severity,
+        "timestamp" => timestamp
+      }
+      context = PayloadBuilder.context_payload_portion(additional_context)
+      payload["context"] = context unless context.empty?
+      "#{JSON.generate(payload)}\n"
+    end
+
+    def standalone_hash_line?(message, tags, additional_context)
+      hash_message?(message) &&
+        tags.respond_to?(:empty?) && tags.empty? &&
+        additional_context.respond_to?(:empty?) && additional_context.empty?
+    end
+
+    def hash_message?(message)
+      message.is_a?(Hash) || (message.respond_to?(:to_hash) && !message.is_a?(String))
+    end
+
+    def standalone_hash_json_line(message:, severity:, timestamp:)
+      hash = message.is_a?(Hash) ? message : message.to_hash
+      line = StructuredHashJsonEncoder.try_encode_line(hash, severity: severity, timestamp: timestamp)
+      return line if line
+
+      payload = Sanitizer.sanitize_hash(hash)
+      payload["severity"] = severity
+      payload["timestamp"] = timestamp
+      "#{JSON.generate(payload)}\n"
+    end
+
+    def tagged_hash_line?(message, tags, additional_context)
+      hash_message?(message) &&
+        tags.respond_to?(:empty?) && !tags.empty? &&
+        additional_context.respond_to?(:empty?) && additional_context.empty?
+    end
+
+    def tagged_hash_json_line(message:, severity:, timestamp:, tags:, sanitize_tags:)
+      hash = message.is_a?(Hash) ? message : message.to_hash
+      prepared_tags = sanitize_tags ? Sanitizer.prepare_tags(tags) : tags
+      merged_tags = (Array(hash[:tags] || hash["tags"]) + prepared_tags).uniq
+      field_overrides = {"tags" => merged_tags}
+
+      line = StructuredHashJsonEncoder.try_encode_line(
+        hash,
+        severity: severity,
+        timestamp: timestamp,
+        field_overrides: field_overrides
+      )
+      return line if line
+
+      payload = Sanitizer.sanitize_hash(hash)
+      payload["severity"] = severity
+      payload["timestamp"] = timestamp
+      payload["tags"] = merged_tags
+      "#{JSON.generate(payload)}\n"
+    end
+
+    def contextual_hash_line?(message, tags, additional_context, additional_context_sanitized:)
+      hash_message?(message) &&
+        tags.respond_to?(:empty?) && tags.empty? &&
+        !PayloadBuilder.empty_additional_context?(additional_context) &&
+        additional_context_sanitized
+    end
+
+    def contextual_hash_json_line(message:, severity:, timestamp:, additional_context:)
+      hash = message.is_a?(Hash) ? message : message.to_hash
+      context = PayloadBuilder.context_payload_portion(additional_context)
+      merged_context = {}
+      unless context.empty?
+        existing_context = hash[:context] || hash["context"]
+        merged_context = existing_context.is_a?(Hash) ? existing_context.merge(context) : context
+      end
+      field_overrides = merged_context.empty? ? {} : {"context" => merged_context}
+
+      line = StructuredHashJsonEncoder.try_encode_line(
+        hash,
+        severity: severity,
+        timestamp: timestamp,
+        field_overrides: field_overrides
+      )
+      return line if line
+
+      payload = Sanitizer.sanitize_hash(hash)
+      payload["severity"] = severity
+      payload["timestamp"] = timestamp
+      unless merged_context.empty?
+        existing_context = payload["context"].is_a?(Hash) ? payload["context"] : {}
+        payload["context"] = existing_context.merge(merged_context)
+      end
+      "#{JSON.generate(payload)}\n"
+    end
+
+    def build_sanitized_hash_payload(message:, severity:, timestamp:)
+      hash = message.is_a?(Hash) ? message : message.to_hash
+      payload = Sanitizer.sanitize_hash(hash)
+      payload["severity"] = severity
+      payload["timestamp"] = timestamp
+      payload
+    end
+
+    def deep_stringify_structure(obj)
+      case obj
+      when Hash
+        obj.deep_stringify_keys
+      when Array
+        obj.map { |v| deep_stringify_structure(v) }
+      else
+        obj
+      end
+    end
+  end
+end

data/lib/json_logging/payload_builder.rb

@@ -1,5 +1,7 @@
 module JsonLogging
   module PayloadBuilder
+    SYSTEM_CONTROLLED_KEYS = [:tags, "tags", :severity, "severity", :timestamp, "timestamp", :message, "message", :context, "context"].freeze
+
     module_function
 
     def build_base_payload(msg, severity: nil, timestamp: nil)
@@ -9,46 +11,80 @@ module JsonLogging
       if parsed.is_a?(Hash)
         payload.merge!(parsed)
       else
-        payload[:message] = parsed
+        payload["message"] = parsed
       end
 
-      payload[:severity] = severity if severity
-      payload[:timestamp] = timestamp if timestamp
+      payload["severity"] = severity if severity
+      payload["timestamp"] = timestamp if timestamp
 
       payload
     end
 
-    def merge_context(payload, additional_context:, tags: [])
-      existing_context = payload[:context].is_a?(Hash) ? payload[:context] : {}
+    def merge_context(payload, additional_context:, tags: [], additional_context_sanitized: false, sanitize_tags: true)
+      return payload if merge_context_skippable?(payload, additional_context: additional_context, tags: tags)
+      return merge_tags_only(payload, tags: tags, sanitize_tags: sanitize_tags) if tags_only_merge?(payload, additional_context: additional_context, tags: tags)
+
+      existing_context = payload["context"].is_a?(Hash) ? payload["context"] : {}
 
-      # Sanitize additional context before merging (filters sensitive keys, limits size, sanitizes strings only)
-      # Only sanitize if it's a hash - preserve other types
-      sanitized_context = if additional_context.is_a?(Hash) && !additional_context.empty?
+      sanitized_context = if additional_context_sanitized
+        additional_context.is_a?(Hash) ? additional_context : {}
+      elsif additional_context.is_a?(Hash) && !additional_context.empty?
         Sanitizer.sanitize_hash(additional_context)
       else
         additional_context || {}
       end
 
-      # Filter out system-controlled keys from user context
-      # These keys should never be set by user context as they're controlled by the logger
-      system_controlled_keys = [:tags, "tags", :severity, "severity", :timestamp, "timestamp", :message, "message", :context, "context"]
-      user_context_filtered = sanitized_context.except(*system_controlled_keys)
+      user_context_filtered = sanitized_context.except(*SYSTEM_CONTROLLED_KEYS)
 
-      # Also prevent overriding any existing payload keys (additional safety)
-      deduped_additional = user_context_filtered.reject { |k, _| payload.key?(k) }
+      deduped_additional = user_context_filtered.reject { |key, _| payload.key?(key) || payload.key?(key.to_s) }
       merged_context = existing_context.merge(deduped_additional)
 
-      # Put tags at root level, separate from context
-      # Merge with existing tags from payload (e.g., when logging a hash with tags: [...] at root)
       unless tags.empty?
-        existing_tags = Array(payload[:tags] || payload["tags"])
-        # Sanitize tag strings (remove control chars, truncate)
-        sanitized_tags = tags.map { |tag| Sanitizer.sanitize_string(tag.to_s) }
-        payload[:tags] = (existing_tags + sanitized_tags).uniq
+        existing_tags = Array(payload["tags"])
+        prepared_tags = sanitize_tags ? tags.map { |tag| Sanitizer.sanitize_string(tag.to_s) } : tags
+        payload["tags"] = (existing_tags + prepared_tags).uniq
       end
 
-      payload[:context] = merged_context unless merged_context.empty?
+      unless merged_context.empty?
+        payload["context"] = merged_context.transform_keys(&:to_s)
+      end
+      payload
+    end
+
+    def empty_additional_context?(additional_context)
+      additional_context.nil? || (additional_context.respond_to?(:empty?) && additional_context.empty?)
+    end
+
+    def merge_context_skippable?(payload, additional_context:, tags:)
+      tags.empty? &&
+        empty_additional_context?(additional_context) &&
+        payload_context_empty?(payload)
+    end
+
+    def payload_context_empty?(payload)
+      existing_context = payload["context"]
+      existing_context.nil? || (existing_context.respond_to?(:empty?) && existing_context.empty?)
+    end
+
+    def tags_only_merge?(payload, additional_context:, tags:)
+      !tags.empty? &&
+        empty_additional_context?(additional_context) &&
+        payload_context_empty?(payload)
+    end
+
+    def merge_tags_only(payload, tags:, sanitize_tags:)
+      existing_tags = Array(payload["tags"])
+      prepared_tags = sanitize_tags ? tags.map { |tag| Sanitizer.sanitize_string(tag.to_s) } : tags
+      payload["tags"] = (existing_tags + prepared_tags).uniq
       payload
     end
+
+    def context_payload_portion(additional_context)
+      return {} unless additional_context.is_a?(Hash)
+
+      additional_context.reject do |key, _|
+        SYSTEM_CONTROLLED_KEYS.include?(key) || SYSTEM_CONTROLLED_KEYS.include?(key.to_s)
+      end
+    end
   end
 end

data/lib/json_logging/sanitizer.rb

@@ -1,3 +1,5 @@
+require_relative "structured_hash_sanitizer"
+
 module JsonLogging
   module Sanitizer
     # Control characters that should be escaped or removed from log messages
@@ -18,6 +20,10 @@ module JsonLogging
     # Common sensitive key patterns (case insensitive) - fallback when Rails ParameterFilter not available
     SENSITIVE_KEY_PATTERNS = /\b(password|passwd|pwd|secret|token|api_key|apikey|access_token|auth_token|private_key|credential)\b/i
 
+    @parameter_filter = nil
+    @parameter_filter_config = nil
+    @parameter_filter_requires_full_tree_walk = false
+
     module_function
 
     # Get Rails ParameterFilter if available, nil otherwise
@@ -28,14 +34,49 @@ module JsonLogging
       filter_params = Rails.application.config.filter_parameters
       return nil if filter_params.empty?
 
-      ActiveSupport::ParameterFilter.new(filter_params)
+      if @parameter_filter && @parameter_filter_config == filter_params
+        return @parameter_filter
+      end
+
+      @parameter_filter_config = filter_params
+      @parameter_filter_requires_full_tree_walk = parameter_filter_requires_full_tree_walk?(filter_params)
+      @parameter_filter = ActiveSupport::ParameterFilter.new(filter_params)
     rescue
       nil
     end
 
+    def rails_parameter_filter_requires_full_tree_walk?
+      rails_parameter_filter
+      @parameter_filter_requires_full_tree_walk
+    end
+
+    def parameter_filter_requires_full_tree_walk?(filter_params)
+      filter_params.any? do |filter|
+        case filter
+        when Proc
+          true
+        when Regexp
+          filter.to_s.include?("\\.")
+        else
+          filter.to_s.include?(".")
+        end
+      end
+    end
+
+    def reset_rails_parameter_filter_cache!
+      @parameter_filter = nil
+      @parameter_filter_config = nil
+      @parameter_filter_requires_full_tree_walk = false
+    end
+
+    def prepare_tags(tags)
+      tags.flatten.compact.map(&:to_s).reject(&:empty?).map { |tag| sanitize_string(tag) }
+    end
+
     # Sanitize a string by removing/escaping control characters and truncating
     def sanitize_string(str)
       return str unless str.is_a?(String)
+      return str if str.length <= MAX_STRING_LENGTH && !str.match?(CONTROL_CHARS)
 
       # Remove or replace control characters
       sanitized = str.gsub(CONTROL_CHARS, "")
@@ -58,15 +99,33 @@ module JsonLogging
       # Prevent excessive nesting
       return {"error" => "max_depth_exceeded"} if depth > MAX_DEPTH
 
-      # Limit hash size first
-      limited_hash = if hash.size > MAX_CONTEXT_SIZE
+      limited_hash = limited_hash_for_sanitization(hash)
+      fast_path_result = fast_path_sanitized_hash(limited_hash, depth: depth)
+      return fast_path_result if fast_path_result
+
+      sanitize_hash_with_filtering(limited_hash, depth: depth)
+    rescue
+      {"sanitization_error" => true}
+    end
+
+    def limited_hash_for_sanitization(hash)
+      if hash.size > MAX_CONTEXT_SIZE
         truncated = hash.first(MAX_CONTEXT_SIZE).to_h
         truncated["_truncated"] = true
         truncated
       else
         hash
       end
+    end
 
+    def fast_path_sanitized_hash(hash, depth: 0)
+      return sanitize_primitive_hash(hash) if primitive_log_hash?(hash)
+      return StructuredHash.sanitize(hash, depth: depth) if StructuredHash.structured_log_hash?(hash)
+
+      nil
+    end
+
+    def sanitize_hash_with_filtering(limited_hash, depth: 0)
       # Use Rails ParameterFilter if available (handles encrypted attributes automatically)
       filter = rails_parameter_filter
       if filter
@@ -78,25 +137,23 @@ module JsonLogging
 
         # Then sanitize values (strings, control chars, etc.) preserving filtered structure
         filtered.each_with_object({}) do |(key, value), result|
-          result[key] = sanitize_value(value, depth: depth + 1)
+          result[key.to_s] = sanitize_value(value, depth: depth + 1)
         end
 
       else
         # Fallback: use pattern matching for sensitive keys
         limited_hash.each_with_object({}) do |(key, value), result|
-          key_str = key.to_s
+          key_string = key.to_s
 
           # Skip sensitive keys
-          if SENSITIVE_KEY_PATTERNS.match?(key_str)
-            result[key_str.gsub(/(?<!^)(?=[A-Z])/, "_").downcase + "_filtered"] = "[FILTERED]"
+          if SENSITIVE_KEY_PATTERNS.match?(key_string)
+            result[sensitive_filtered_key_name(key_string)] = "[FILTERED]"
             next
           end
 
-          result[key] = sanitize_value(value, depth: depth + 1)
+          result[key_string] = sanitize_value(value, depth: depth + 1)
         end
       end
-    rescue
-      {"sanitization_error" => true}
     end
 
     # Sanitize a value (handles strings, hashes, arrays, etc.)
@@ -154,5 +211,49 @@ module JsonLogging
     def sensitive_key?(key)
       SENSITIVE_KEY_PATTERNS.match?(key.to_s)
     end
+
+    def primitive_log_hash?(hash)
+      hash.all? { |_key, value| primitive_log_value?(value) }
+    end
+
+    def primitive_log_value?(value)
+      case value
+      when String
+        value.length <= MAX_STRING_LENGTH && !value.match?(CONTROL_CHARS)
+      when Numeric, TrueClass, FalseClass, NilClass
+        true
+      else
+        false
+      end
+    end
+
+    def sanitize_primitive_hash(hash)
+      filter = rails_parameter_filter
+      if filter
+        stringified = stringify_primitive_hash(hash)
+        filter.filter(stringified.dup)
+      else
+        hash.each_with_object({}) do |(key, value), result|
+          key_string = key.to_s
+
+          if SENSITIVE_KEY_PATTERNS.match?(key_string)
+            result[sensitive_filtered_key_name(key_string)] = "[FILTERED]"
+            next
+          end
+
+          result[key_string] = value
+        end
+      end
+    end
+
+    def stringify_primitive_hash(hash)
+      hash.each_with_object({}) do |(key, value), result|
+        result[key.to_s] = value
+      end
+    end
+
+    def sensitive_filtered_key_name(key_string)
+      key_string.gsub(/(?<!^)(?=[A-Z])/, "_").downcase + "_filtered"
+    end
   end
 end

data/lib/json_logging/severity.rb

@@ -0,0 +1,18 @@
+module JsonLogging
+  module Severity
+    NAMES = {
+      ::Logger::DEBUG => "DEBUG",
+      ::Logger::INFO => "INFO",
+      ::Logger::WARN => "WARN",
+      ::Logger::ERROR => "ERROR",
+      ::Logger::FATAL => "FATAL",
+      ::Logger::UNKNOWN => "UNKNOWN"
+    }.freeze
+
+    module_function
+
+    def name_for(severity)
+      NAMES[severity] || severity.to_s
+    end
+  end
+end

data/lib/json_logging/structured_hash_json_encoder.rb

@@ -0,0 +1,111 @@
+require_relative "structured_hash_sanitizer"
+
+module JsonLogging
+  module StructuredHashJsonEncoder
+    LEAF_THRESHOLD = 40
+
+    module_function
+
+    def eligible?(hash)
+      return false if Sanitizer.primitive_log_hash?(hash)
+      return false unless Sanitizer::StructuredHash.structured_log_hash?(hash)
+      return false unless large_structured_hash?(hash)
+
+      filter = Sanitizer.rails_parameter_filter
+      return true unless filter
+
+      !Sanitizer.rails_parameter_filter_requires_full_tree_walk?
+    end
+
+    def large_structured_hash?(hash, leaf_threshold: LEAF_THRESHOLD)
+      leaf_count = 0
+      walker = lambda do |value|
+        case value
+        when Hash
+          value.each_value { |entry| walker.call(entry) }
+        when Array
+          value.each { |entry| walker.call(entry) }
+        else
+          leaf_count += 1
+          throw(:enough, true) if leaf_count > leaf_threshold
+        end
+      end
+
+      catch(:enough) { walker.call(hash) }
+      leaf_count > leaf_threshold
+    end
+
+    def try_encode_line(hash, severity:, timestamp:, field_overrides: {})
+      tree = prepared_tree(hash, field_overrides: field_overrides)
+      return nil unless tree
+
+      append_json_line(
+        tree,
+        severity: severity,
+        timestamp: timestamp,
+        field_overrides: field_overrides
+      )
+    end
+
+    def encode_line(hash, severity:, timestamp:, field_overrides: {})
+      tree = encode_tree(hash, field_overrides: field_overrides)
+      append_json_line(
+        tree,
+        severity: severity,
+        timestamp: timestamp,
+        field_overrides: field_overrides
+      )
+    end
+
+    def encode_tree(hash, field_overrides: {})
+      limited_hash = Sanitizer.limited_hash_for_sanitization(hash)
+      filter = Sanitizer.rails_parameter_filter
+      omit_keys = override_keys(field_overrides)
+      jsonable = Sanitizer::StructuredHash.jsonable_tree(limited_hash, omit_keys: omit_keys)
+      filtered_jsonable_tree(jsonable, filter: filter)
+    end
+
+    def prepared_tree(hash, field_overrides: {})
+      return nil if Sanitizer.primitive_log_hash?(hash)
+
+      filter = Sanitizer.rails_parameter_filter
+      return nil if filter && Sanitizer.rails_parameter_filter_requires_full_tree_walk?
+      return nil unless Sanitizer::StructuredHash.structured_log_hash?(hash)
+
+      limited_hash = Sanitizer.limited_hash_for_sanitization(hash)
+      omit_keys = override_keys(field_overrides)
+      jsonable = Sanitizer::StructuredHash.jsonable_tree(
+        limited_hash,
+        omit_keys: omit_keys,
+        count_leaves: true
+      )
+      return nil if jsonable.leaf_count <= LEAF_THRESHOLD
+
+      filtered_jsonable_tree(jsonable, filter: filter)
+    end
+
+    def filtered_jsonable_tree(jsonable, filter:)
+      return jsonable.tree unless filter
+      return jsonable.tree if Sanitizer.rails_parameter_filter_requires_full_tree_walk?
+
+      tree = if jsonable.owned
+        jsonable.tree
+      else
+        Sanitizer::StructuredHash.stringify_keys_copy(jsonable.tree)
+      end
+
+      filter.filter(tree)
+    end
+
+    def append_json_line(tree, severity:, timestamp:, field_overrides:)
+      body = JSON.generate(tree)
+      extras = field_overrides.merge("severity" => severity, "timestamp" => timestamp)
+      extra_json = extras.map { |key, value| "#{JSON.generate(key.to_s)}:#{JSON.generate(value)}" }.join(",")
+      "#{body.chop},#{extra_json}}\n"
+    end
+
+    def override_keys(field_overrides)
+      field_overrides.keys.map(&:to_s)
+    end
+  end
+end