logstruct 0.1.11 → 0.2.0

data/lib/log_struct/semantic_logger/setup.rb

@@ -5,78 +5,41 @@ require "semantic_logger"
 require_relative "formatter"
 require_relative "color_formatter"
 require_relative "logger"
+require_relative "reopen_after_fork"
 
 module LogStruct
-  # SemanticLogger Integration
-  #
-  # LogStruct uses SemanticLogger as its core logging engine, providing significant
-  # performance and functionality benefits over Rails' default logger:
-  #
-  # ## Performance Benefits
-  # - **Asynchronous logging**: Logs are written in a background thread, eliminating
-  #   I/O blocking in your main application threads
-  # - **High throughput**: Can handle 100,000+ log entries per second
-  # - **Memory efficient**: Structured data processing with minimal allocations
-  # - **Zero-copy serialization**: Direct JSON generation without intermediate objects
-  #
-  # ## Reliability Benefits
-  # - **Thread-safe**: All operations are thread-safe by design
-  # - **Graceful degradation**: Continues logging even if appenders fail
-  # - **Error isolation**: Logging errors don't crash your application
-  # - **Buffered writes**: Reduces disk I/O with intelligent batching
-  #
-  # ## Feature Benefits
-  # - **Multiple appenders**: Log to files, STDOUT, databases, cloud services simultaneously
-  # - **Structured metadata**: Rich context including process ID, thread ID, tags, and more
-  # - **Log filtering**: Runtime filtering by logger name, level, or custom rules
-  # - **Formatters**: Pluggable output formatting (JSON, colorized, custom)
-  # - **Metrics integration**: Built-in performance metrics and timing data
-  #
-  # ## Development Experience
-  # - **Colorized output**: Beautiful, readable logs in development with ANSI colors
-  # - **Tagged logging**: Hierarchical context tracking (requests, jobs, etc.)
-  # - **Debugging tools**: Detailed timing and memory usage information
-  # - **Hot reloading**: Configuration changes without application restart
-  #
-  # ## Production Benefits
-  # - **Log rotation**: Automatic file rotation with size/time-based policies
-  # - **Compression**: Automatic log compression to save disk space
-  # - **Cloud integration**: Direct integration with CloudWatch, Splunk, etc.
-  # - **Alerting**: Built-in support for error alerting and monitoring
-  #
-  # ## LogStruct Specific Enhancements
-  # - **Type safety**: Full Sorbet type annotations for compile-time error detection
-  # - **Structured data**: Native support for LogStruct's typed log structures
-  # - **Filtering integration**: Seamless integration with LogStruct's data filters
-  # - **Error handling**: Enhanced error reporting with full stack traces and context
-  #
-  # SemanticLogger is a production-grade logging framework used by companies processing
-  # millions of requests per day. It provides the performance and reliability needed
-  # for high-traffic Rails applications while maintaining an elegant developer experience.
+  # LogStruct writes its lines through SemanticLogger. It replaces Rails.logger and the framework loggers with a
+  # SemanticLogger logger whose formatter writes LogStruct's JSON lines. SemanticLogger formats and writes each line on
+  # a background thread, so a request or job doesn't wait for the write, and it can write to several appenders
+  # (STDOUT, a file, a log service) at once.
   module SemanticLogger
-    # Handles setup and configuration of SemanticLogger for Rails applications
-    #
-    # This module provides the core integration between LogStruct and SemanticLogger,
-    # configuring appenders, formatters, and logger replacement to provide optimal
-    # logging performance while maintaining full compatibility with Rails conventions.
+    # Sets up SemanticLogger for a Rails app: its level, appenders and formatters, and the loggers it replaces
     module Setup
       extend T::Sig
 
-      # Configures SemanticLogger as the primary logging engine for the Rails application
-      #
-      # This method replaces Rails' default logger with SemanticLogger, providing:
-      # - **10-100x performance improvement** for high-volume logging
-      # - **Non-blocking I/O** through background thread processing
-      # - **Enhanced reliability** with graceful error handling
-      # - **Multiple output destinations** (files, STDOUT, cloud services)
-      # - **Structured metadata** including process/thread IDs and timing
-      #
-      # The configuration automatically:
-      # - Determines optimal log levels based on environment
-      # - Sets up appropriate appenders (console, file, etc.)
-      # - Enables colorized output in development
-      # - Replaces Rails.logger and component loggers
-      # - Preserves full Rails.logger API compatibility
+      # Load hooks for the framework components whose loggers LogStruct replaces
+      COMPONENT_LOAD_HOOKS = T.let(
+        %i[active_record action_controller action_mailer active_job action_view action_cable].freeze,
+        T::Array[Symbol]
+      )
+
+      # The logger LogStruct installed, and the loggers it replaced
+      @logger = T.let(nil, T.nilable(LogStruct::SemanticLogger::Logger))
+      @original_rails_logger = T.let(nil, T.untyped)
+      @original_log_subscriber_logger = T.let(nil, T.untyped)
+      @original_app_logger = T.let(nil, T.untyped)
+      # The appenders add_appenders added. LogStruct replaces only these, and keeps any an app adds.
+      @appenders = T.let([], T::Array[T.untyped])
+
+      sig { returns(T::Boolean) }
+      def self.configured?
+        !@logger.nil?
+      end
+
+      # Configures SemanticLogger and replaces Rails' loggers with LogStruct's:
+      # - the level from config.log_level (info in production, debug elsewhere)
+      # - the appenders for the environment (see add_appenders)
+      # - Rails.logger and the framework loggers (ActiveRecord, ActionController, ActiveJob, ...)
       #
       # @param app [Rails::Application] The Rails application instance
       sig { params(app: T.untyped).void }
@@ -89,16 +52,34 @@ module LogStruct
         log_level = determine_log_level(app)
         ::SemanticLogger.default_level = log_level
 
-        # Clear existing appenders
-        ::SemanticLogger.clear_appenders!
-
         # Add appropriate appenders based on environment
+        remove_appenders
         add_appenders(app)
+        ReopenAfterFork.install!
 
         # Replace Rails.logger with SemanticLogger
         replace_rails_logger(app)
       end
 
+      # Adds the appenders again, with the settings config/initializers changed after the logger was set up
+      # (prefer_json_in_development, enable_color_output, color_map, filter_noisy_loggers, local_environments).
+      # Appenders the app added in config/initializers stay.
+      sig { params(app: T.untyped).void }
+      def self.reconfigure_appenders(app)
+        remove_appenders
+        add_appenders(app)
+      end
+
+      # Removes the appenders add_appenders added, after writing the lines already queued for them
+      sig { void }
+      def self.remove_appenders
+        return if @appenders.empty?
+
+        ::SemanticLogger.flush
+        @appenders.each { |appender| ::SemanticLogger.remove_appender(appender) }
+        @appenders = []
+      end
+
       sig { params(app: T.untyped).returns(Symbol) }
       def self.determine_log_level(app)
         level = if app.config.log_level
@@ -121,7 +102,13 @@ module LogStruct
         # Determine output destination
         io = determine_output(app)
 
-        if Rails.env.development?
+        # Ruby buffers STDOUT when it isn't a terminal, so in a container lines waited until the buffer
+        # filled or the process exited. (GoodJob's executable only syncs STDOUT when it writes its own
+        # text logs there.)
+        io.sync = true if io.equal?($stdout)
+
+        appenders = []
+        appenders << if Rails.env.development?
           if config.prefer_json_in_development
             # Default to production-style JSON in development when enabled
             ::SemanticLogger.add_appender(
@@ -154,14 +141,19 @@ module LogStruct
           )
         end
 
-        # Add file appender if Rails has a log path configured (normal Rails behavior)
-        if app.config.paths["log"].first
-          ::SemanticLogger.add_appender(
+        # Log to Rails' log file too in development and test (config.local_environments), unless
+        # RAILS_LOG_TO_STDOUT says not to. Production and other servers log to STDOUT only: nothing
+        # rotates the file, and in a container it only fills the disk.
+        local = config.local_environments.include?(Rails.env.to_sym)
+        if local && app.config.paths["log"].first && ENV["RAILS_LOG_TO_STDOUT"].blank?
+          appenders << ::SemanticLogger.add_appender(
             file_name: app.config.paths["log"].first,
             formatter: LogStruct::SemanticLogger::Formatter.new,
             filter: determine_filter
           )
         end
+        # add_appender returns nil when it ignores a second STDOUT appender
+        @appenders = appenders.compact
       end
 
       sig { params(app: T.untyped).returns(T.untyped) }
@@ -178,64 +170,75 @@ module LogStruct
         end
       end
 
-      sig { returns(T.nilable(Regexp)) }
+      NOISY_LOGGERS = /\A(ActionView|ActionController::RoutingError|ActiveRecord::SchemaMigration)/
+
+      # With filter_noisy_loggers on, an appender filter that leaves out NOISY_LOGGERS' lines. SemanticLogger
+      # writes a line when a Proc filter returns true (a Regexp filter would keep only the names it matches).
+      sig { returns(T.nilable(T.proc.params(log: ::SemanticLogger::Log).returns(T::Boolean))) }
       def self.determine_filter
-        # Filter out noisy loggers if configured
-        config = LogStruct.config
-        return nil unless config.filter_noisy_loggers
+        return nil unless LogStruct.config.filter_noisy_loggers
 
-        # Common noisy loggers to filter
-        /\A(ActionView|ActionController::RoutingError|ActiveRecord::SchemaMigration)/
+        ->(log) { !NOISY_LOGGERS.match?(log.name.to_s) }
       end
 
-      # Replaces Rails.logger and all component loggers with LogStruct's SemanticLogger
-      #
-      # This method provides seamless integration by replacing the default Rails logger
-      # throughout the entire Rails stack, ensuring all logging flows through the
-      # high-performance SemanticLogger system.
-      #
-      # ## Benefits of Complete Logger Replacement:
-      # - **Consistent performance**: All Rails components benefit from SemanticLogger speed
-      # - **Unified formatting**: All logs use the same structured JSON format
-      # - **Centralized configuration**: Single point of control for all logging
-      # - **Complete compatibility**: Maintains all Rails.logger API contracts
-      #
-      # ## Components Updated:
-      # - Rails.logger (framework core)
-      # - ActiveRecord::Base.logger (database queries)
-      # - ActionController::Base.logger (request processing)
-      # - ActionMailer::Base.logger (email delivery)
-      # - ActiveJob::Base.logger (background jobs)
-      # - ActionView::Base.logger (template rendering)
-      # - ActionCable.server.config.logger (WebSocket connections)
-      #
-      # After replacement, all Rails logging maintains API compatibility while gaining
-      # SemanticLogger's performance, reliability, and feature benefits.
+      # Replaces Rails.logger, ActiveSupport::LogSubscriber.logger, config.logger and the framework
+      # loggers (ActiveRecord, ActionController, ActionMailer, ActiveJob, ActionView, ActionCable)
+      # with LogStruct's SemanticLogger logger, so every Rails line is a LogStruct JSON line
       #
       # @param app [Rails::Application] The Rails application instance
       sig { params(app: T.untyped).void }
       def self.replace_rails_logger(app)
         # Create new SemanticLogger instance
         logger = LogStruct::SemanticLogger::Logger.new("Rails")
+        @logger = logger
+        @original_rails_logger = Rails.logger
+        @original_log_subscriber_logger = ActiveSupport::LogSubscriber.logger
+        @original_app_logger = app.config.logger
 
         # Replace Rails.logger
         Rails.logger = logger
 
-        # Also replace various component loggers
-        ActiveRecord::Base.logger = logger if defined?(ActiveRecord::Base)
-        ActionController::Base.logger = logger if defined?(ActionController::Base)
-        if defined?(ActionMailer::Base)
-          ActionMailer::Base.logger = logger
-          # Ensure ActionMailer.logger is also set (it might be accessed directly)
-          T.unsafe(::ActionMailer).logger = logger if T.unsafe(::ActionMailer).respond_to?(:logger=)
+        # Rails assigns ActiveSupport::LogSubscriber.logger once during boot
+        # (in the active_support.set_log_subscriber_logger initializer), so log
+        # subscribers such as Lograge keep writing to the original boot logger
+        # unless this reference is updated along with Rails.logger.
+        ActiveSupport::LogSubscriber.logger = logger
+
+        # Also replace the component loggers. Use load hooks rather than referencing the base
+        # classes: loading them this early would apply their on_load hooks before the app's
+        # initializers run, e.g. ActiveRecord would copy an empty filter_parameters into
+        # filter_attributes.
+        COMPONENT_LOAD_HOOKS.each do |hook|
+          ActiveSupport.on_load(hook) { self.logger = logger }
         end
-        ActiveJob::Base.logger = logger if defined?(ActiveJob::Base)
-        ActionView::Base.logger = logger if defined?(ActionView::Base)
-        ActionCable.server.config.logger = logger if defined?(ActionCable)
 
         # Store reference in app config
         app.config.logger = logger
       end
+
+      # Puts back the loggers that replace_rails_logger replaced. Used when an initializer
+      # disables LogStruct after the logger was replaced at boot.
+      sig { params(app: T.untyped).void }
+      def self.restore_rails_logger(app)
+        logger = @logger
+        return unless logger
+
+        original_logger = @original_rails_logger
+        Rails.logger = original_logger
+        ActiveSupport::LogSubscriber.logger = @original_log_subscriber_logger
+        app.config.logger = @original_app_logger
+        remove_appenders
+
+        # These hooks run after the ones that installed LogStruct's logger, including for
+        # components that are already loaded
+        COMPONENT_LOAD_HOOKS.each do |hook|
+          ActiveSupport.on_load(hook) do
+            self.logger = original_logger if self.logger.equal?(logger)
+          end
+        end
+
+        @logger = nil
+      end
     end
   end
 end

data/lib/log_struct/shared/merge_additional_data_fields.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require_relative "interfaces/additional_data_field"
+require_relative "../enums/source"
 
 module LogStruct
   module Log
@@ -13,12 +14,24 @@ module LogStruct
         requires_ancestor { T::Struct }
         requires_ancestor { Interfaces::AdditionalDataField }
 
+        # Reserved keys that cannot be overwritten by additional_data.
+        # These are the core log structure fields that must not be modified.
+        RESERVED_KEYS = T.let(%i[src evt lvl ts].freeze, T::Array[Symbol])
+
         sig { params(hash: T::Hash[Symbol, T.untyped]).void }
         def merge_additional_data_fields(hash)
           ad = additional_data
           return unless ad
           ad.each do |key, value|
-            hash[key.to_sym] = value
+            sym_key = key.to_sym
+            if RESERVED_KEYS.include?(sym_key)
+              LogStruct.handle_exception(
+                ArgumentError.new("additional_data attempted to overwrite reserved key: #{sym_key}"),
+                source: Source::Internal
+              )
+              next
+            end
+            hash[sym_key] = value
           end
         end
       end

data/lib/log_struct/shared/serialize_common.rb

@@ -25,13 +25,8 @@ module LogStruct
           field_hash.each do |log_field, value|
             next if value.nil?
             key = log_field.serialize
-
-            # Limit backtrace to first 5 lines
-            if key == :backtrace && value.is_a?(Array)
-              value = value.first(5)
-            end
-
-            out[key] = value.is_a?(::Time) ? value.iso8601 : value
+            # (The formatter cuts a backtrace to config.limits.max_backtrace_lines)
+            out[key] = value.is_a?(::Time) ? value.iso8601(3) : value
           end
 
           # Merge any additional_data at top level if available

data/lib/log_struct/string_scrubber.rb

@@ -1,64 +1,120 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "digest"
+require "base64"
+require "ipaddr"
+require "uri"
+require_relative "hash_utils"
+require_relative "param_filters"
 
 module LogStruct
   # StringScrubber is inspired by logstop by @ankane: https://github.com/ankane/logstop
   # Enhancements:
   # - Shows which type of data was filtered
-  # - Includes an SHA256 hash with filtered emails for request tracing
+  # - Includes a hash of each filtered email (HashUtils) for request tracing
   # - Uses configuration options from LogStruct.config
   module StringScrubber
     class << self
       extend T::Sig
 
-      # Also supports URL-encoded URLs like https%3A%2F%2Fuser%3Asecret%40example.com
+      # The password in a URL's user info (postgres://user:password@host, redis://:password@host).
+      # The user ends at its first colon; the password runs to the last @ before the host.
+      URL_PASSWORD_REGEX = %r{(//[^\s/:@]*:)[^\s/]+(@)}
+      # The same in a URL-encoded URL, like https%3A%2F%2Fuser%3Asecret%40example.com
       # cspell:ignore Fuser Asecret
-      URL_PASSWORD_REGEX = /((?:\/\/|%2F%2F)[^\s\/]+(?::|%3A))[^\s\/]+(@|%40)/
+      ENCODED_URL_PASSWORD_REGEX = %r{(%2F%2F(?:(?!%3A|%2F|%40)[^\s/&])*%3A)(?:(?!%2F)[^\s/&])+(%40)}i
       URL_PASSWORD_REPLACEMENT = '\1[PASSWORD]\2'
+      # A URL's user info that runs past what the scrubbers saw of a long string, where the cut can separate the
+      # password from the @ that closes it. Everything after the user's colon goes: the password can contain an
+      # @ of its own (p@ss).
+      CUT_URL_PASSWORD_REGEX = %r{(//[^\s/:@]*:)[^\s/"'<>]*\z}
+      CUT_ENCODED_URL_PASSWORD_REGEX = %r{(%2F%2F(?:(?!%3A|%2F|%40)[^\s/&])*%3A)(?:(?!%2F)[^\s/&"'<>])*\z}i
+      CUT_URL_PASSWORD_REPLACEMENT = '\1[PASSWORD]'
+      # Basic credentials that run past what the scrubbers saw: a cut token doesn't decode, so it can't be checked
+      CUT_BASIC_CREDENTIALS_REGEX = %r{\b(Basic\s+)[A-Za-z0-9+/]+=*\z}i
 
-      EMAIL_REGEX = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i
+      # A URL's query string, for the sensitive params in it
+      QUERY_STRING_REGEX = /\?([^\s#"'<>]+)/
+      # OAuth authorization codes, which no key pattern would filter
+      SENSITIVE_QUERY_PARAMS = T.let(["code"].freeze, T::Array[String])
+      QUERY_PARAM_REPLACEMENT = "[FILTERED]"
 
-      CREDIT_CARD_REGEX_SHORT = /\b[3456]\d{15}\b/
-      CREDIT_CARD_REGEX_DELIMITERS = /\b[3456]\d{3}[\s-]\d{4}[\s-]\d{4}[\s-]\d{4}\b/
+      # Letters and digits of any script (an accented name, a domain in another alphabet)
+      EMAIL_REGEX = /(?<![\p{L}\p{N}._%+-])[\p{L}\p{N}._%+-]+@[\p{L}\p{N}.-]+\.\p{L}{2,}(?![\p{L}\p{N}])/
+      # A whole String that is an email address
+      WHOLE_EMAIL_REGEX = /\A#{EMAIL_REGEX}\z/
+
+      # The credentials after Bearer or Basic, as in an Authorization header. A Bearer token has at least 16
+      # characters, and Basic credentials must decode to user:password, so "Bearer token" and "Basic plan" stay.
+      AUTHORIZATION_CREDENTIALS_REGEX = %r{\b(Bearer|Basic)(\s+)([A-Za-z0-9\-._~+/]+=*)}i
+      AUTHORIZATION_CREDENTIALS_REPLACEMENT = "[FILTERED]"
+
+      # 13 to 19 digits, written together, in fours (4111 1111 1111 1111, with a 19-digit card's last
+      # three) or in American Express and Diners Club groups (3782 822463 10005). Numbers that fail
+      # the Luhn check are left alone, so most IDs and timestamps are too.
+      CREDIT_CARD_REGEX = /
+        (?<![A-Za-z0-9.-])
+        (?:
+          [2-6]\d{12,18}
+          | [2-6]\d{3}(?<sep>[\ -])\d{4}\k<sep>\d{4}\k<sep>\d{4}(?:\k<sep>\d{3})?
+          | 3\d{3}(?<sep2>[\ -])\d{6}\k<sep2>\d{4,5}
+        )
+        (?![A-Za-z0-9]|-\d)
+      /x
       CREDIT_CARD_REPLACEMENT = "[CREDIT_CARD]"
 
-      PHONE_REGEX = /\b\d{3}[\s-]\d{3}[\s-]\d{4}\b/
+      # US numbers: 555-123-4567, 555 123 4567, 555.123.4567 and (555) 123-4567, with or without +1 or 1
+      PHONE_REGEX = /(?<![\w+])(?:\+?1[\s.-]?)?(?:\(\d{3}\)[\s.-]?|\d{3}[\s.-])\d{3}[\s.-]\d{4}(?!\w)/
       PHONE_REPLACEMENT = "[PHONE]"
 
-      SSN_REGEX = /\b\d{3}[\s-]\d{2}[\s-]\d{4}\b/
+      # 123-45-6789 and 123 45 6789, but not inside a longer number (9-123-45-6789, 123-45-6789-0)
+      SSN_REGEX = /(?<!\w|\d-)\d{3}[\s-]\d{2}[\s-]\d{4}(?!\w|-\d)/
       SSN_REPLACEMENT = "[SSN]"
 
       IP_REGEX = /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/
+      # Text that could be an IPv6 address (2001:db8::1, ::ffff:192.168.1.1, fe80::1%en0 with its zone). IPAddr
+      # decides whether it is one, and it must have a digit, so Ruby constants (Face::Bead) and times (12:30:45)
+      # stay.
+      IPV6_CANDIDATE_REGEX = /(?<![\w:.])(?=[\h:.]*\d)(?:\h{0,4}:){2,7}(?:\d{1,3}(?:\.\d{1,3}){3}|\h{1,4})?(?:%\w+)?(?![\w:])/
       IP_REPLACEMENT = "[IP]"
 
-      MAC_REGEX = /\b[0-9a-f]{2}(:[0-9a-f]{2}){5}\b/i
+      # Six pairs of hex digits, separated by colons or by dashes
+      MAC_REGEX = /\b[0-9a-f]{2}([:-])[0-9a-f]{2}(?:\1[0-9a-f]{2}){4}\b/i
       MAC_REPLACEMENT = "[MAC]"
 
+      # Hash keys and Symbols like these are never changed by the scrubbers, so they're skipped
+      IDENTIFIER_REGEX = /\A[A-Za-z_][A-Za-z0-9_]*\z/
+
       # Scrub sensitive information from a string
       sig { params(string: String).returns(String) }
       def scrub(string)
         return string if string.empty?
 
-        string = string.to_s.dup
+        # The scrubbers match letters of any script, which needs valid UTF-8
+        string = valid_utf8(string.to_s).dup
         config = LogStruct.config.filters
 
+        scrub_authorization_credentials!(string) if config.authorization_credentials
+
+        # Sensitive params in URLs' query strings
+        scrub_query_params!(string) if config.url_query_params && string.include?("?")
+
         # Passwords in URLs
-        string.gsub!(URL_PASSWORD_REGEX, URL_PASSWORD_REPLACEMENT) if config.url_passwords
+        if config.url_passwords
+          string.gsub!(URL_PASSWORD_REGEX, URL_PASSWORD_REPLACEMENT)
+          string.gsub!(ENCODED_URL_PASSWORD_REGEX, URL_PASSWORD_REPLACEMENT) if string.include?("%")
+        end
 
         # Emails
         if config.email_addresses
-          string.gsub!(EMAIL_REGEX) do |email|
-            email_hash = HashUtils.hash_value(email)
-            "[EMAIL:#{email_hash}]"
-          end
+          string.gsub!(EMAIL_REGEX) { |email| "[EMAIL:#{value_hash(email)}]" }
         end
 
         # Credit card numbers
         if config.credit_card_numbers
-          string.gsub!(CREDIT_CARD_REGEX_SHORT, CREDIT_CARD_REPLACEMENT)
-          string.gsub!(CREDIT_CARD_REGEX_DELIMITERS, CREDIT_CARD_REPLACEMENT)
+          string.gsub!(CREDIT_CARD_REGEX) do |number|
+            luhn_valid?(number.delete(" -")) ? CREDIT_CARD_REPLACEMENT : number
+          end
         end
 
         # Phone numbers
@@ -67,8 +123,11 @@ module LogStruct
         # SSNs
         string.gsub!(SSN_REGEX, SSN_REPLACEMENT) if config.ssns
 
-        # IPs
-        string.gsub!(IP_REGEX, IP_REPLACEMENT) if config.ip_addresses
+        # IPs: IPv6 first, which can end in an IPv4 address
+        if config.ip_addresses
+          scrub_ipv6_addresses!(string) if string.include?(":")
+          string.gsub!(IP_REGEX, IP_REPLACEMENT)
+        end
 
         # MAC addresses
         string.gsub!(MAC_REGEX, MAC_REPLACEMENT) if config.mac_addresses
@@ -79,6 +138,156 @@ module LogStruct
 
         string
       end
+
+      # The end of the part of a long string the scrubbers saw (Formatter#scrub_limited_string). They saw only the
+      # start of a value that runs past it, so a URL password separated from its closing @ (which
+      # URL_PASSWORD_REGEX needs) and Basic credentials cut short (which no longer decode) are replaced, however
+      # long they were.
+      sig { params(string: String).returns(String) }
+      def scrub_cut_end(string)
+        filters = LogStruct.config.filters
+        if filters.url_passwords
+          string = string.sub(CUT_URL_PASSWORD_REGEX, CUT_URL_PASSWORD_REPLACEMENT)
+            .sub(CUT_ENCODED_URL_PASSWORD_REGEX, CUT_URL_PASSWORD_REPLACEMENT)
+        end
+        if filters.authorization_credentials
+          string = string.sub(CUT_BASIC_CREDENTIALS_REGEX, "\\1#{AUTHORIZATION_CREDENTIALS_REPLACEMENT}")
+        end
+        string
+      end
+
+      # The hash of a value (HashUtils.hash_value). Email addresses are hashed in lowercase, so an address has
+      # one hash however it's written, and the same hash under a filtered key as in a scrubbed string.
+      sig { params(value: String).returns(String) }
+      def value_hash(value)
+        HashUtils.hash_value(WHOLE_EMAIL_REGEX.match?(value) ? value.downcase : value)
+      end
+
+      # A String as valid UTF-8: binary and other encodings are converted, and bytes that aren't valid
+      # become U+FFFD
+      sig { params(string: String).returns(String) }
+      def valid_utf8(string)
+        return string if string.ascii_only? || (string.encoding == Encoding::UTF_8 && string.valid_encoding?)
+
+        utf8 = if string.encoding == Encoding::BINARY || string.encoding == Encoding::US_ASCII
+          string.dup.force_encoding(Encoding::UTF_8)
+        else
+          begin
+            string.encode(Encoding::UTF_8)
+          rescue EncodingError
+            string.dup.force_encoding(Encoding::UTF_8)
+          end
+        end
+        utf8.valid_encoding? ? utf8 : utf8.scrub
+      end
+
+      # A Hash key or a Symbol, scrubbed like a String unless it's a plain identifier (user_id,
+      # createdAt). Returns it unchanged, or the scrubbed String when scrubbing changed it.
+      sig { params(name: T.untyped).returns(T.untyped) }
+      def scrub_name(name)
+        return name unless name.is_a?(String) || name.is_a?(Symbol)
+
+        string = name.to_s
+        return name if IDENTIFIER_REGEX.match?(string)
+
+        scrubbed = scrub(string)
+        (scrubbed == string) ? name : scrubbed
+      end
+
+      # The URL without its query string and fragment (for Referer headers)
+      sig { params(url: T.nilable(String)).returns(T.nilable(String)) }
+      def without_query(url)
+        return url if url.nil?
+
+        url.split(/[?#]/, 2).first || ""
+      end
+
+      private
+
+      sig { params(string: String).void }
+      def scrub_ipv6_addresses!(string)
+        string.gsub!(IPV6_CANDIDATE_REGEX) do |candidate|
+          # The zone (an interface's name) is replaced with its address
+          ipv6_address?(candidate.split("%", 2).first.to_s) ? IP_REPLACEMENT : candidate
+        end
+      end
+
+      sig { params(candidate: String).returns(T::Boolean) }
+      def ipv6_address?(candidate)
+        ::IPAddr.new(candidate).ipv6?
+      rescue ::IPAddr::Error
+        false
+      end
+
+      # Bearer tokens and Basic credentials (Authorization: Bearer eyJ... becomes Bearer [FILTERED])
+      sig { params(string: String).void }
+      def scrub_authorization_credentials!(string)
+        return unless string.match?(/bearer|basic/i)
+
+        string.gsub!(AUTHORIZATION_CREDENTIALS_REGEX) do |match|
+          scheme = Regexp.last_match(1).to_s
+          credentials = Regexp.last_match(3).to_s
+          if authorization_credentials?(scheme, credentials)
+            "#{scheme}#{Regexp.last_match(2)}#{AUTHORIZATION_CREDENTIALS_REPLACEMENT}"
+          else
+            match
+          end
+        end
+      end
+
+      sig { params(scheme: String, credentials: String).returns(T::Boolean) }
+      def authorization_credentials?(scheme, credentials)
+        return credentials.length >= 16 if scheme.casecmp?("bearer")
+
+        credentials.length >= 8 && Base64.strict_decode64(credentials).include?(":")
+      rescue ArgumentError
+        false
+      end
+
+      # Replaces the values of query params whose names would be filtered as keys
+      # (/reset?token=abc&page=2 becomes /reset?token=[FILTERED]&page=2)
+      sig { params(string: String).void }
+      def scrub_query_params!(string)
+        string.gsub!(QUERY_STRING_REGEX) do
+          query = Regexp.last_match(1).to_s
+          params = query.split("&", -1).map do |param|
+            name, value = param.split("=", 2)
+            next param if value.nil? || value.empty? || value == QUERY_PARAM_REPLACEMENT
+
+            sensitive_query_param?(name.to_s, value) ? "#{name}=#{QUERY_PARAM_REPLACEMENT}" : param
+          end
+          "?#{params.join("&")}"
+        end
+      end
+
+      # user[password] is checked as password, written plainly or URL-encoded
+      sig { params(name: String, value: String).returns(T::Boolean) }
+      def sensitive_query_param?(name, value)
+        decoded = begin
+          URI.decode_www_form_component(name)
+        rescue ArgumentError
+          name
+        end
+        key = decoded[/\[([^\[\]]*)\]\z/, 1] || decoded
+        return false if key.empty?
+
+        SENSITIVE_QUERY_PARAMS.include?(key.downcase) || ParamFilters.should_filter_key?(key, value)
+      end
+
+      # The Luhn checksum that every card number passes
+      sig { params(digits: String).returns(T::Boolean) }
+      def luhn_valid?(digits)
+        sum = 0
+        digits.reverse.each_char.with_index do |char, index|
+          digit = char.to_i
+          if index.odd?
+            digit *= 2
+            digit -= 9 if digit > 9
+          end
+          sum += digit
+        end
+        (sum % 10).zero?
+      end
     end
   end
 end

data/lib/log_struct/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module LogStruct
-  VERSION = "0.1.11"
+  VERSION = "0.2.0"
 end