logstruct 0.1.12 → 0.2.0

data/lib/log_struct/log.rb

@@ -8,6 +8,7 @@ require_relative "enums/level"
 require_relative "enums/log_field"
 require_relative "log/interfaces/public_common_fields"
 require_relative "shared/serialize_common_public"
+require_relative "params_errors"
 
 # Dynamically require all top-level log structs under log/*
 # Nested per-event files are required by their parent files.
@@ -23,7 +24,7 @@ module LogStruct
     sig do
       params(
         source: Source,
-        ex: StandardError,
+        ex: Exception,
         additional_data: T::Hash[T.any(String, Symbol), T.untyped],
         timestamp: Time
       ).returns(LogStruct::Log::Error)
@@ -32,7 +33,8 @@ module LogStruct
       LogStruct::Log::Error.new(
         source: source,
         error_class: ex.class,
-        message: ex.message,
+        # Without the input a params parser quoted, for a request's params error that the app logs itself
+        message: ParamsErrors.messages(ex).first,
         backtrace: ex.backtrace,
         additional_data: additional_data,
         timestamp: timestamp

data/lib/log_struct/logged_errors.rb

@@ -0,0 +1,25 @@
+# typed: strict
+# frozen_string_literal: true
+
+module LogStruct
+  # Exceptions LogStruct has already logged as a structured line, so an integration that sees the same
+  # exception again doesn't log it twice. GoodJob passes a failed job's exception to on_thread_error after
+  # the ActiveJob integration has logged it, for example. Exceptions are held weakly, so a mark doesn't keep
+  # one alive.
+  module LoggedErrors
+    extend T::Sig
+
+    @logged = T.let(ObjectSpace::WeakMap.new, ObjectSpace::WeakMap)
+    @mutex = T.let(Mutex.new, Mutex)
+
+    sig { params(error: Exception).void }
+    def self.mark(error)
+      @mutex.synchronize { @logged[error] = true }
+    end
+
+    sig { params(error: Exception).returns(T::Boolean) }
+    def self.logged?(error)
+      @mutex.synchronize { @logged.key?(error) }
+    end
+  end
+end

data/lib/log_struct/monkey_patches/active_support/tagged_logging/formatter.rb

@@ -31,14 +31,13 @@ module ActiveSupport
         # Preserve original Rails behavior when LogStruct is disabled
         return super unless ::LogStruct.enabled?
 
-        # Get current tags
-        tags = T.unsafe(self).respond_to?(:current_tags) ? current_tags : (Thread.current[:activesupport_tagged_logging_tags] || [])
+        tags = current_tags
 
         # Add tags to data as hash entry (not text prefix)
         data_with_tags = if data.is_a?(Hash)
           tags.present? ? data.merge(tags: tags) : data
         elsif data.is_a?(::LogStruct::Log::Interfaces::CommonFields) || (data.is_a?(T::Struct) && data.respond_to?(:serialize))
-          hash = T.unsafe(data).serialize
+          hash = data.serialize
           tags.present? ? hash.merge(tags: tags) : hash
         else
           base = {LogStruct::LogField::Message.serialize => data.to_s}

data/lib/log_struct/param_filters.rb

@@ -1,8 +1,9 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "digest"
+require "json"
 require_relative "hash_utils"
+require_relative "string_scrubber"
 require_relative "config_struct/filters"
 require_relative "enums/source"
 
@@ -13,19 +14,47 @@ module LogStruct
     class << self
       extend T::Sig
 
-      # Check if a key should be filtered based on our defined sensitive keys
-      sig { params(key: T.untyped, value: T.untyped).returns(T::Boolean) }
-      def should_filter_key?(key, value = nil)
+      # Check if a key should be filtered based on our defined sensitive keys.
+      # logstruct_field: the key is one of LogStruct's own fields at the top of a line.
+      # path: the keys from the line's top-level field down to this one (the key alone if not given).
+      # root: the top-level field's value, or the whole line for a top-level key.
+      sig do
+        params(
+          key: T.untyped,
+          value: T.untyped,
+          logstruct_field: T::Boolean,
+          path: T.nilable(T::Array[String]),
+          root: T.untyped
+        ).returns(T::Boolean)
+      end
+      def should_filter_key?(key, value = nil, logstruct_field: false, path: nil, root: nil)
         filters = LogStruct.config.filters
         normalized_key = key.to_s
-        normalized_symbol = normalized_key.downcase.to_sym
 
-        return true if filters.filter_keys.include?(normalized_symbol)
+        words = KeyWords.normalize(normalized_key).to_sym
+        # By its words too, so :api_key filters apiKey and X-Api-Key. Keys in filter_keys_with_hashes are
+        # filtered too, with a hash of their value.
+        [filters.filter_keys, filters.filter_keys_with_hashes].each do |keys|
+          return true if keys.include?(normalized_key.downcase.to_sym) || keys.include?(words)
+        end
+
+        matched = filters.filter_matchers.any? do |matcher|
+          next false if logstruct_field && !matcher.matches_logstruct_fields
 
-        filters.filter_matchers.any? do |matcher|
           matcher.matches?(normalized_key, value)
         rescue => e
-          handle_filter_matcher_error(e, matcher, normalized_key)
+          handle_filter_matcher_error(e, matcher.label, normalized_key)
+          false
+        end
+        return true if matched
+        return false if filters.path_matchers.empty?
+
+        path ||= [normalized_key]
+        root = {} if root.nil?
+        filters.path_matchers.any? do |matcher|
+          matcher.matches?(path, value, root)
+        rescue => e
+          handle_filter_matcher_error(e, matcher.label, normalized_key)
           false
         end
       end
@@ -33,7 +62,8 @@ module LogStruct
       # Check if a key should be hashed rather than completely filtered
       sig { params(key: T.untyped).returns(T::Boolean) }
       def should_include_string_hash?(key)
-        LogStruct.config.filters.filter_keys_with_hashes.include?(key.to_s.downcase.to_sym)
+        keys = LogStruct.config.filters.filter_keys_with_hashes
+        keys.include?(key.to_s.downcase.to_sym) || keys.include?(KeyWords.normalize(key.to_s).to_sym)
       end
 
       # Convert a value to a filtered summary hash (e.g. { _filtered: { class: "String", ... }})
@@ -51,56 +81,39 @@ module LogStruct
         end
       end
 
-      # Summarize a String for logging, including details and an SHA256 hash (if configured)
+      # Summarize a filtered String: its class, and an HMAC of it for keys in filter_keys_with_hashes.
+      # No length, which would say how long a password or token is.
       sig { params(string: String, include_hash: T::Boolean).returns(T::Hash[Symbol, T.untyped]) }
       def summarize_string(string, include_hash)
         filtered_string = {
           _class: String
         }
-        if include_hash
-          filtered_string[:_hash] = HashUtils.hash_value(string)
-        else
-          filtered_string[:_bytes] = string.bytesize
-        end
+        filtered_string[:_hash] = StringScrubber.value_hash(string) if include_hash
 
         filtered_string
       end
 
-      # Summarize a Hash for logging, including details about the size and keys
+      # Summarize a filtered Hash: its first 10 keys and how many there are. No size, which would say
+      # how long the secrets in it are.
       sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
       def summarize_hash(hash)
         return {_class: "Hash", _empty: true} if hash.empty?
 
-        # Don't include byte size if hash contains any filtered keys
-        has_sensitive_keys = T.let(false, T::Boolean)
-        normalized_keys = []
-
-        hash.each do |key, value|
-          has_sensitive_keys ||= should_filter_key?(key, value)
-          normalized_keys << normalize_summary_key(key)
-        end
-
-        summary = {
+        {
           _class: Hash,
           _keys_count: hash.keys.size,
-          _keys: normalized_keys.take(10)
+          _keys: hash.keys.take(10).map { |key| normalize_summary_key(StringScrubber.scrub_name(key)) }
         }
-
-        # Only add byte size if no sensitive keys are present
-        summary[:_bytes] = hash.to_json.bytesize unless has_sensitive_keys
-
-        summary
       end
 
-      # Summarize an Array for logging, including details about the size and items
+      # Summarize a filtered Array: how many items it has
       sig { params(array: T::Array[T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
       def summarize_array(array)
         return {_class: "Array", _empty: true} if array.empty?
 
         {
           _class: Array,
-          _count: array.size,
-          _bytes: array.to_json.bytesize
+          _count: array.size
         }
       end
 
@@ -119,10 +132,10 @@ module LogStruct
         key.to_s
       end
 
-      sig { params(error: StandardError, matcher: ConfigStruct::FilterMatcher, key: String).void }
-      def handle_filter_matcher_error(error, matcher, key)
+      sig { params(error: StandardError, label: String, key: String).void }
+      def handle_filter_matcher_error(error, label, key)
         context = {
-          matcher: matcher.label,
+          matcher: label,
           key: key
         }
         LogStruct.handle_exception(error, source: Source::Internal, context: context)

data/lib/log_struct/params_errors.rb

@@ -0,0 +1,78 @@
+# typed: strict
+# frozen_string_literal: true
+
+module LogStruct
+  # Errors from parsing a request's params: Rails' ParseError (and its ParamErrors), and Rack's. Their messages, and
+  # those of errors raised because of them (Rails' "Invalid request parameters: ..."), quote the request's input: a
+  # JSON parser's excerpt of the body, Rack's "invalid %-encoding (<value>)". Nothing can filter a body that didn't
+  # parse, so LogStruct's lines leave the input out.
+  module ParamsErrors
+    extend T::Sig
+
+    # Matched by name, so this works without ActionDispatch or Rack loaded
+    NAMES = T.let(
+      [
+        "ActionDispatch::Http::Parameters::ParseError",
+        "Rack::BadRequest",
+        "Rack::QueryParser::InvalidParameterError",
+        "Rack::QueryParser::ParameterTypeError",
+        "Rack::QueryParser::ParamsTooDeepError"
+      ].freeze,
+      T::Array[String]
+    )
+    # Rails' message for a ParseError
+    PARSE_ERROR_MESSAGE = "Error occurred while parsing request parameters"
+    PARAMS_ERROR_MESSAGE = "Invalid request parameters"
+    CAUSE_MESSAGE_LEFT_OUT = "(left out: it quotes the request's input)"
+
+    # The message and the cause's message to log for an error, without the input a params parser quoted
+    sig { params(error: Exception).returns([String, T.nilable(String)]) }
+    def self.messages(error)
+      cause = error.cause
+      if params_error?(error)
+        [parse_error?(error) ? PARSE_ERROR_MESSAGE : PARAMS_ERROR_MESSAGE, nil]
+      elsif cause && params_error?(cause)
+        message = error.message.sub(cause.message, "").sub(/[\s:]+\z/, "")
+        [message.empty? ? PARAMS_ERROR_MESSAGE : message, nil]
+      else
+        [error.message, cause&.message]
+      end
+    end
+
+    # For text that writes an error's whole chain (its causes too): each message in it that quotes a params parser's
+    # input, with what to write instead. Longest first, since an error's message can contain its cause's. An error
+    # raised without a message (Rack's and Rails' depth limits) has its class name as its message, which quotes
+    # nothing and is also the text's label for its class, so it stays.
+    sig { params(error: Exception).returns(T::Array[[String, String]]) }
+    def self.replacements(error)
+      pairs = T.let([], T::Array[[String, String]])
+      seen = T.let({}.compare_by_identity, T::Hash[Exception, TrueClass])
+      current = T.let(error, T.nilable(Exception))
+      while current && !seen.key?(current)
+        seen[current] = true
+        message, cause_message = messages(current)
+        pairs << [current.message, message] if message != current.message && !default_message?(current)
+        cause = current.cause
+        pairs << [cause.message, CAUSE_MESSAGE_LEFT_OUT] if cause && cause_message.nil? && !default_message?(cause)
+        current = cause
+      end
+      pairs.reject { |quoted, _| quoted.empty? }.uniq.sort_by { |quoted, _| -quoted.length }
+    end
+
+    sig { params(error: Exception).returns(T::Boolean) }
+    def self.default_message?(error)
+      error.message == error.class.name
+    end
+
+    sig { params(error: Exception).returns(T::Boolean) }
+    def self.parse_error?(error)
+      !!defined?(::ActionDispatch::Http::Parameters::ParseError) &&
+        error.instance_of?(::ActionDispatch::Http::Parameters::ParseError)
+    end
+
+    sig { params(error: Exception).returns(T::Boolean) }
+    def self.params_error?(error)
+      error.class.ancestors.any? { |ancestor| NAMES.include?(ancestor.name) }
+    end
+  end
+end

data/lib/log_struct/rails_boot_banner_silencer.rb

@@ -15,6 +15,9 @@ module LogStruct
       @installed = true
 
       return unless ARGV.include?("server")
+      # Rails' own banner stays when LogStruct is disabled
+      return unless LogStruct.enabled?
+
       patch!
     end
 
@@ -51,6 +54,8 @@ module LogStruct
 
     module ServerCommandSilencer
       extend T::Sig
+      extend T::Helpers
+      requires_ancestor { Object }
 
       sig { params(args: T.untyped, block: T.nilable(T.proc.returns(T.untyped))).returns(T.untyped) }
       def perform(*args, &block)
@@ -105,9 +110,9 @@ module LogStruct
 
       sig { returns(String) }
       def lookup_executable
-        return "rails" unless T.unsafe(self).respond_to?(:executable, true)
+        return "rails" unless respond_to?(:executable, true)
 
-        T.cast(T.unsafe(self).send(:executable), String)
+        T.cast(send(:executable), String)
       rescue
         "rails"
       end

data/lib/log_struct/railtie.rb

@@ -10,31 +10,49 @@ require_relative "integrations"
 module LogStruct
   # Railtie to integrate with Rails
   class Railtie < ::Rails::Railtie
-    # Configure early, right after logger initialization
+    # Replace the logger as soon as Rails creates it, so boot-time logs and anything that keeps
+    # a reference to Rails.logger use LogStruct's logger
     initializer "logstruct.configure_logger", after: :initialize_logger do |app|
       next unless LogStruct.enabled?
 
       # Use SemanticLogger for powerful logging features
       LogStruct::SemanticLogger::Setup.configure_semantic_logger(app)
+
+      # Initializers that configure Sidekiq log with its logger while config/initializers run
+      Integrations::Sidekiq.configure_logger if LogStruct.config.integrations.enable_sidekiq
     end
 
-    # Setup all integrations after logger setup is complete
-    initializer "logstruct.setup", before: :build_middleware_stack do |app|
+    # Set up integrations after config/initializers, so that the settings in
+    # config/initializers/logstruct.rb and filter_parameters added by other initializers apply
+    initializer "logstruct.setup", after: :load_config_initializers, before: :build_middleware_stack do |app|
+      # An initializer may have changed `enabled` or `enabled_environments` after the logger was
+      # (or wasn't) replaced
+      LogStruct.apply_initializer_settings!
+      if LogStruct.enabled?
+        if LogStruct::SemanticLogger::Setup.configured?
+          # The formatter, colors, noisy logger filter and log file settings in config/initializers apply too
+          LogStruct::SemanticLogger::Setup.reconfigure_appenders(app)
+        else
+          LogStruct::SemanticLogger::Setup.configure_semantic_logger(app)
+        end
+      elsif LogStruct::SemanticLogger::Setup.configured?
+        LogStruct::SemanticLogger::Setup.restore_rails_logger(app)
+      end
+      unless LogStruct.enabled? && LogStruct.config.integrations.enable_sidekiq
+        Integrations::Sidekiq.restore_logger
+      end
+      Integrations::ActionMailer.setup_fallback_error_handlers(LogStruct.config)
+
       next unless LogStruct.enabled?
 
       # Merge Rails filter parameters into our filters
       LogStruct.merge_rails_filter_parameters!
 
-      # Set up non-middleware integrations first
       Integrations.setup_integrations(stage: :non_middleware)
-
-      # Note: Host allowances are managed by the test app itself.
     end
 
-    # Setup middleware integrations during Rails configuration (before middleware stack is built)
-    # Must be done in the Railtie class body, not in an initializer
-    initializer "logstruct.configure_middleware", before: :build_middleware_stack do |app|
-      # This runs before middleware stack is frozen, so we can configure it
+    # Add middleware before the middleware stack is built
+    initializer "logstruct.configure_middleware", after: "logstruct.setup", before: :build_middleware_stack do |app|
       next unless LogStruct.enabled?
 
       Integrations.setup_integrations(stage: :middleware)
@@ -42,6 +60,7 @@ module LogStruct
 
     # Emit Puma lifecycle logs when running `rails server`
     initializer "logstruct.puma_lifecycle", after: "logstruct.configure_logger" do
+      next unless LogStruct.enabled?
       is_server = ::LogStruct.server_mode?
       next unless is_server
       begin
@@ -53,10 +72,6 @@ module LogStruct
           process_id: Process.pid,
           listening_addresses: port ? ["tcp://127.0.0.1:#{port}"] : nil
         )
-        begin
-          warn("[logstruct] puma lifecycle init")
-        rescue
-        end
         LogStruct.info(started)
 
         at_exit do

data/lib/log_struct/semantic_logger/color_formatter.rb

@@ -6,46 +6,12 @@ require_relative "formatter"
 
 module LogStruct
   module SemanticLogger
-    # Development-Optimized Colorized JSON Formatter
-    #
-    # This formatter extends SemanticLogger's Color formatter to provide beautiful,
-    # readable JSON output in development environments. It significantly improves
-    # the developer experience when working with structured logs.
-    #
-    # ## Benefits of Colorized Output:
-    #
-    # ### Readability
-    # - **Syntax highlighting**: JSON keys, values, and data types are color-coded
-    # - **Visual hierarchy**: Different colors help identify structure at a glance
-    # - **Error spotting**: Quickly identify malformed data or unexpected values
-    # - **Context separation**: Log entries are visually distinct from each other
-    #
-    # ### Performance in Development
-    # - **Faster debugging**: Quickly scan logs without reading every character
-    # - **Pattern recognition**: Colors help identify common log patterns
-    # - **Reduced cognitive load**: Less mental effort required to parse log output
-    # - **Improved workflow**: Spend less time reading logs, more time coding
-    #
-    # ### Customization
-    # - **Configurable colors**: Customize colors for keys, strings, numbers, etc.
-    # - **Environment-aware**: Automatically disabled in production/CI environments
-    # - **Fallback support**: Gracefully falls back to standard formatting if needed
-    #
-    # ## Color Mapping:
-    # - **Keys**: Yellow - Easy to spot field names
-    # - **Strings**: Green - Clear indication of text values
-    # - **Numbers**: Blue - Numeric values stand out
-    # - **Booleans**: Magenta - true/false values are distinctive
-    # - **Null**: Red - Missing values are immediately visible
-    # - **Logger names**: Cyan - Source identification
-    #
-    # ## Integration with SemanticLogger:
-    # This formatter preserves all SemanticLogger benefits (performance, threading,
-    # reliability) while adding visual enhancements. It processes LogStruct types,
-    # hashes, and plain messages with appropriate colorization.
-    #
-    # The formatter is automatically enabled in development when `enable_color_output`
-    # is true (default), providing zero-configuration enhanced logging experience.
+    # Writes lines as pretty-printed JSON in color, for reading in a terminal: keys yellow, strings green, numbers
+    # blue, booleans magenta, nil red and logger names cyan (config.color_map changes them). LogStruct uses it in
+    # development when LogStruct is enabled there, prefer_json_in_development is false and enable_color_output is
+    # true (its default). LogStruct entries and Hashes are the JSON line, pretty-printed. Plain lines keep
+    # SemanticLogger's layout, with their message, exception and tags scrubbed and their named tags filtered as a
+    # JSON line's ctx is.
     class ColorFormatter < ::SemanticLogger::Formatters::Color
       extend T::Sig
 
@@ -53,19 +19,17 @@ module LogStruct
       def initialize(color_map: nil, **args)
         super(**args)
         @logstruct_formatter = T.let(LogStruct::Formatter.new, LogStruct::Formatter)
+        @json_formatter = T.let(LogStruct::SemanticLogger::Formatter.new, LogStruct::SemanticLogger::Formatter)
 
         # Set up custom color mapping
-        @custom_colors = T.let(color_map || default_color_map, T::Hash[Symbol, Symbol])
+        @custom_colors = T.let(default_color_map.merge(color_map || {}), T::Hash[Symbol, Symbol])
       end
 
       sig { override.params(log: ::SemanticLogger::Log, logger: T.untyped).returns(String) }
       def call(log, logger)
         # Handle LogStruct types specially with colorization
-        if log.payload.is_a?(LogStruct::Log::Interfaces::CommonFields)
-          formatted = format_logstruct_payload(log)
-          formatted if formatted
-        elsif log.payload.is_a?(Hash) || log.payload.is_a?(T::Struct)
-          formatted = format_logstruct_payload(log)
+        if log.payload.is_a?(Hash) || log.payload.is_a?(T::Struct)
+          formatted = format_logstruct_payload(log, logger)
           formatted if formatted
         else
           # For plain messages, use SemanticLogger's default colorization
@@ -73,8 +37,51 @@ module LogStruct
         end
       end
 
+      # The parts of a plain line that hold what was logged, scrubbed like the strings in a JSON line
+      sig { returns(T.untyped) }
+      def message
+        scrub_text(super)
+      end
+
+      sig { returns(T.untyped) }
+      def payload
+        scrub_text(super)
+      end
+
+      # SemanticLogger's text for the exception and its causes, without the input a params parser quoted
+      sig { returns(T.untyped) }
+      def exception
+        text = super
+        error = log.exception
+        if text.is_a?(String) && error.is_a?(Exception)
+          ParamsErrors.replacements(error).each { |quoted, logged| text = text.gsub(quoted, logged) }
+        end
+        scrub_text(text)
+      end
+
+      sig { returns(T.untyped) }
+      def tags
+        scrub_text(super)
+      end
+
+      # Filtered and scrubbed as they are in a JSON line's ctx: a key filtering hides there is hidden here too
+      sig { returns(T.nilable(String)) }
+      def named_tags
+        named_tags = log.named_tags
+        return if named_tags.nil? || named_tags.empty?
+
+        filtered = @logstruct_formatter.process_values({ctx: named_tags.to_h.transform_keys(&:to_sym)})[:ctx]
+        list = filtered.map { |name, value| "#{color}#{name}: #{value}#{color_map.clear}" }
+        "{#{list.join(", ")}}"
+      end
+
       private
 
+      sig { params(text: T.untyped).returns(T.untyped) }
+      def scrub_text(text)
+        text.is_a?(String) ? StringScrubber.scrub(text) : text
+      end
+
       sig { returns(LogStruct::Formatter) }
       attr_reader :logstruct_formatter
 
@@ -91,11 +98,9 @@ module LogStruct
         }
       end
 
-      # Simple JSON colorizer that adds ANSI codes
+      # The line as pretty-printed JSON, with ANSI colors for its keys, strings, numbers, booleans and nulls
       sig { params(data: T::Hash[String, T.untyped]).returns(String) }
       def colorize_json(data)
-        # For now, just return a simple colorized version of the JSON
-        # This is much simpler than the full recursive approach
         json_str = JSON.pretty_generate(data)
 
         # Apply basic colorization with regex
@@ -106,9 +111,10 @@ module LogStruct
           .gsub(": null", ": " + colorize_text("null", :nil))
       end
 
-      sig { params(log: ::SemanticLogger::Log).returns(T.nilable(String)) }
-      def format_logstruct_payload(log)
-        logstruct_json = @logstruct_formatter.call(log.level, log.time, log.name, log.payload)
+      # The line LogStruct::SemanticLogger::Formatter writes (message, tags and ctx included), in color
+      sig { params(log: ::SemanticLogger::Log, logger: T.untyped).returns(T.nilable(String)) }
+      def format_logstruct_payload(log, logger)
+        logstruct_json = @json_formatter.call(log, logger)
 
         parsed_data = T.let(JSON.parse(logstruct_json), T::Hash[String, T.untyped])
         colorized_json = colorize_json(parsed_data)
@@ -119,7 +125,8 @@ module LogStruct
           process: log.process_info,
           name: format_name(log.name))
 
-        "#{prefix}#{colorized_json}\n"
+        # SemanticLogger's appenders add the newline
+        "#{prefix}#{colorized_json}"
       rescue JSON::ParserError
         nil
       end