logstruct 0.1.12 → 0.2.1

data/lib/log_struct/config_struct/limits.rb

@@ -0,0 +1,29 @@
+# typed: strict
+# frozen_string_literal: true
+
+module LogStruct
+  module ConfigStruct
+    # How much of each logged value LogStruct writes. The limits keep a huge value (a request body,
+    # a large Hash) from slowing down the thread that writes the logs.
+    class Limits < T::Struct
+      include Sorbet::SerializeSymbolKeys
+
+      # The most keys written from a Hash. The rest are counted in _more_keys.
+      # Default: 100
+      prop :max_hash_keys, Integer, default: 100
+
+      # The most items written from an Array, followed by "... and N more items"
+      # Default: 10
+      prop :max_array_items, Integer, default: 10
+
+      # The most lines written from an Array that looks like a backtrace, such as an error line's
+      # backtrace. The rest are counted in a last "... and N more lines" item.
+      # Default: 20
+      prop :max_backtrace_lines, Integer, default: 20
+
+      # Strings longer than this many bytes are cut, and say how long they were
+      # Default: 32768 (32 KB)
+      prop :max_string_bytes, Integer, default: 32_768
+    end
+  end
+end

data/lib/log_struct/configuration.rb

@@ -5,6 +5,7 @@ require_relative "handlers"
 require_relative "config_struct/error_handling_modes"
 require_relative "config_struct/integrations"
 require_relative "config_struct/filters"
+require_relative "config_struct/limits"
 
 module LogStruct
   # Core configuration class that provides a type-safe API
@@ -35,6 +36,7 @@ module LogStruct
 
     const :integrations, ConfigStruct::Integrations, factory: -> { ConfigStruct::Integrations.new }
     const :filters, ConfigStruct::Filters, factory: -> { ConfigStruct::Filters.new }
+    const :limits, ConfigStruct::Limits, factory: -> { ConfigStruct::Limits.new }
 
     # Custom log scrubbing handler for any additional string scrubbing
     # Default: nil

data/lib/log_struct/enums/event.rb

@@ -16,6 +16,9 @@ module LogStruct
       Schedule = new(:schedule)
       Start = new(:start)
       Finish = new(:finish)
+      Retry = new(:retry)
+      RetryStopped = new(:retry_stopped)
+      Discard = new(:discard)
 
       # File storage events (ActiveStorage, Shrine, CarrierWave, etc.)
       Upload = new(:upload)
@@ -23,7 +26,6 @@ module LogStruct
       Delete = new(:delete)
       Metadata = new(:metadata)
       Exist = new(:exist)
-      Stream = new(:stream)
       Url = new(:url)
 
       # Data generation events

data/lib/log_struct/enums/log_field.rb

@@ -91,6 +91,11 @@ module LogStruct
       Priority = new(:priority)
       CronKey = new(:cron_key)
       ErrorMessage = new(:error_message)
+      # What the integration was doing when the error happened, e.g. Sidekiq's "Job raised exception"
+      ErrorContext = new(:error_context)
+      # The cause of an error the app didn't handle in a request (Exception#cause)
+      CauseClass = new(:cause_class)
+      CauseMessage = new(:cause_message)
       Result = new(:result)
       EnqueueCaller = new(:enqueue_caller)
 

data/lib/log_struct/formatter.rb

@@ -3,10 +3,12 @@
 
 require "logger"
 require "active_support/core_ext/object/blank"
+require "active_support/hash_with_indifferent_access"
 require "json"
 require "globalid"
 require_relative "enums/source"
 require_relative "enums/event"
+require_relative "enums/log_field"
 require_relative "string_scrubber"
 require_relative "log"
 require_relative "param_filters"
@@ -49,43 +51,60 @@ module LogStruct
       StringScrubber.scrub(string)
     end
 
-    sig { params(arg: T.untyped, recursion_depth: Integer).returns(T.untyped) }
-    def process_values(arg, recursion_depth: 0)
-      # Prevent infinite recursion in case any args have circular references
-      # or are too deeply nested. Just return args.
-      return arg if recursion_depth > 20
+    MAX_RECURSION_DEPTH = 20
+
+    # The mask Rails' parameter filter puts in place of filtered values
+    RAILS_FILTERED_MASK = "[FILTERED]"
+
+    # LogStruct's own fields, which some filters don't apply to at the top of a line
+    LOG_FIELD_KEYS = T.let(LogField.values.map(&:serialize).to_set.freeze, T::Set[T.untyped])
+
+    # path and root are for filters that match a key by its path (see ParamFilters.should_filter_key?).
+    # They're only tracked when such filters exist: path is nil otherwise.
+    sig do
+      params(
+        arg: T.untyped,
+        recursion_depth: Integer,
+        ancestors: T::Hash[T.untyped, T::Boolean],
+        path: T.nilable(T::Array[String]),
+        root: T.untyped
+      ).returns(T.untyped)
+    end
+    def process_values(arg, recursion_depth: 0, ancestors: {}.compare_by_identity, path: nil, root: nil)
+      if recursion_depth == 0 && path.nil? && !LogStruct.config.filters.path_matchers.empty?
+        path = []
+      end
+
+      # Circular references and deeply nested values stop here. Values below this depth
+      # would skip filtering and scrubbing, so they're summarized instead of logged.
+      return summarize_unprocessed_value(arg) if recursion_depth > MAX_RECURSION_DEPTH || ancestors.key?(arg)
 
       case arg
-      when Hash
-        result = {}
-
-        # Process each key-value pair
-        arg.each do |key, value|
-          # Check if this key should be filtered at any depth
-          result[key] = if ParamFilters.should_filter_key?(key, value)
-            # Filter the value
-            {_filtered: ParamFilters.summarize_json_attribute(key, value)}
+      when Hash, Array
+        ancestors[arg] = true
+        begin
+          if arg.is_a?(Hash)
+            process_hash(arg, recursion_depth: recursion_depth, ancestors: ancestors, path: path, root: root)
           else
-            # Process the value normally
-            process_values(value, recursion_depth: recursion_depth + 1)
+            process_array(arg, recursion_depth: recursion_depth, ancestors: ancestors, path: path, root: root)
           end
+        ensure
+          ancestors.delete(arg)
         end
-
-        result
-      when Array
-        process_array(arg, recursion_depth: recursion_depth)
       when GlobalID::Identification
+        # Scrubbed like any String: a record's ID can be an email address
         begin
-          arg.to_global_id
+          scrub_string(arg.to_global_id.to_s)
         rescue
           begin
-            case arg
-            when ActiveRecord::Base
-              "#{arg.class}(##{arg.id})"
+            object = T.cast(arg, Object)
+            # Active Record is optional
+            if defined?(::ActiveRecord::Base) && object.is_a?(::ActiveRecord::Base)
+              scrub_string("#{object.class}(##{object.id})")
             else
               # For non-ActiveRecord objects that failed to_global_id, try to get a string representation
               # If this also fails, we want to catch it and return the error placeholder
-              String(T.cast(arg, Object))
+              scrub_string(String(object))
             end
           rescue => e
             LogStruct.handle_exception(e, source: Source::Internal)
@@ -95,14 +114,24 @@ module LogStruct
       when Source, Event
         arg.serialize
       when String
-        scrub_string(arg)
+        scrub_limited_string(valid_utf8(arg))
       when Time
         arg.iso8601(3)
-      else
-        # Any other type (e.g. Symbol, Integer, Float, Boolean etc.)
+      when Symbol
+        StringScrubber.scrub_name(arg)
+      when Float
+        # JSON has no NaN or Infinity
+        arg.finite? ? arg : arg.to_s
+      when Numeric, TrueClass, FalseClass, NilClass
         arg
+      else
+        # Structs, ActionController::Parameters and other objects: filter and scrub what
+        # to_json would write
+        process_values(arg.as_json, recursion_depth: recursion_depth + 1, ancestors: ancestors, path: path, root: root)
       end
-    rescue => e
+    rescue SystemStackError, StandardError => e
+      # A SystemStackError comes from an object whose as_json refers back to itself
+      e = RuntimeError.new("#{e.class}: #{e.message}") unless e.is_a?(StandardError)
       # Report error through LogStruct's framework
       context = {
         processor_method: "process_values",
@@ -110,7 +139,141 @@ module LogStruct
         recursion_depth: recursion_depth
       }
       LogStruct.handle_exception(e, source: Source::Internal, context: context)
-      arg
+      summarize_unprocessed_value(arg)
+    end
+
+    sig do
+      params(
+        hash: T::Hash[T.untyped, T.untyped],
+        recursion_depth: Integer,
+        ancestors: T::Hash[T.untyped, T::Boolean],
+        path: T.nilable(T::Array[String]),
+        root: T.untyped
+      ).returns(T::Hash[T.untyped, T.untyped])
+    end
+    def process_hash(hash, recursion_depth:, ancestors:, path: nil, root: nil)
+      result = {}
+
+      # Path matchers' roots are built only if a matcher reads them (most don't)
+      line_root = (path && recursion_depth == 0) ? ConfigStruct::LazyRoot.new(hash) : nil
+
+      max_keys = LogStruct.config.limits.max_hash_keys
+      hash.each_with_index do |(key, value), index|
+        if index >= max_keys
+          result[:_more_keys] = hash.size - max_keys
+          break
+        end
+
+        key = valid_utf8_key(key)
+        logstruct_field = recursion_depth == 0 && LOG_FIELD_KEYS.include?(key)
+        key_path = path && [*path, key.to_s]
+        # A top-level key's root is the line; the keys below it get the top-level field's value
+        key_root = (recursion_depth == 0) ? line_root : root
+        children_root = (path && recursion_depth == 0) ? ConfigStruct::LazyRoot.new(value) : root
+        # Check if this key should be filtered at any depth
+        filtered = ParamFilters.should_filter_key?(key, value, logstruct_field: logstruct_field, path: key_path, root: key_root)
+        # Keys are scrubbed too: an email address can be a key
+        result[StringScrubber.scrub_name(key)] = if !filtered
+          process_values(value, recursion_depth: recursion_depth + 1, ancestors: ancestors, path: key_path, root: children_root)
+        elsif value == RAILS_FILTERED_MASK
+          # Rails' parameter filter has already replaced the value
+          value
+        else
+          {_filtered: ParamFilters.summarize_json_attribute(key, value)}
+        end
+      end
+
+      result
+    end
+
+    # The string as valid UTF-8. JSON can't hold other bytes: Puma passes headers as binary
+    # strings, and a byte like \xFF in a User-Agent would stop the whole line from being written.
+    # Bytes that aren't valid UTF-8 become U+FFFD.
+    sig { params(string: String).returns(String) }
+    def valid_utf8(string)
+      StringScrubber.valid_utf8(string)
+    end
+
+    # A String or Symbol key as valid UTF-8 (symbolize_keys makes Symbols of binary String keys)
+    sig { params(key: T.untyped).returns(T.untyped) }
+    def valid_utf8_key(key)
+      case key
+      when String
+        valid_utf8(key)
+      when Symbol
+        name = key.name
+        fixed = valid_utf8(name)
+        fixed.equal?(name) ? key : fixed.to_sym
+      else
+        key
+      end
+    end
+
+    # The scrubbers see a little more than a cut string keeps, so data at the cut is scrubbed whole
+    SCRUB_MARGIN_BYTES = 256
+
+    # A string scrubbed, and cut to limits.max_string_bytes. The scrubbers see its first max_string_bytes plus
+    # SCRUB_MARGIN_BYTES: a value running past that was only partly seen, so scrub_cut_end replaces it if it can
+    # be a credential.
+    sig { params(string: String).returns(String) }
+    def scrub_limited_string(string)
+      max_bytes = LogStruct.config.limits.max_string_bytes
+      return scrub_string(string) if string.bytesize <= max_bytes
+
+      seen_bytes = max_bytes + SCRUB_MARGIN_BYTES
+      scrubbed = scrub_string(string.byteslice(0, seen_bytes).to_s.scrub(""))
+      if string.bytesize > seen_bytes
+        scrubbed = StringScrubber.scrub_cut_end(scrubbed)
+      elsif scrubbed.bytesize <= max_bytes
+        # The scrubbers saw all of it, and their replacements made it short enough
+        return scrubbed
+      end
+      kept = scrubbed.byteslice(0, max_bytes).to_s.scrub("")
+      "#{kept}... (cut from #{string.bytesize} bytes)"
+    end
+
+    # Like deep_symbolize_keys, but stops at circular references and at the depth where
+    # process_values stops, leaving those values for process_values to summarize
+    sig { params(value: T.untyped, recursion_depth: Integer, ancestors: T::Hash[T.untyped, T::Boolean]).returns(T.untyped) }
+    def symbolize_keys(value, recursion_depth: 0, ancestors: {}.compare_by_identity)
+      return value if recursion_depth > MAX_RECURSION_DEPTH || ancestors.key?(value)
+
+      case value
+      when Hash
+        ancestors[value] = true
+        result = value.each_with_object({}) do |(key, nested_value), symbolized|
+          symbol_key = key.respond_to?(:to_sym) ? key.to_sym : key
+          symbolized[symbol_key] = symbolize_keys(nested_value, recursion_depth: recursion_depth + 1, ancestors: ancestors)
+        end
+        ancestors.delete(value)
+        result
+      when Array
+        ancestors[value] = true
+        result = value.map { |item| symbolize_keys(item, recursion_depth: recursion_depth + 1, ancestors: ancestors) }
+        ancestors.delete(value)
+        result
+      else
+        value
+      end
+    end
+
+    # Describes a value without filtering, scrubbing or serializing it, so that values which
+    # couldn't be processed are never logged raw. Hashes and arrays are not serialized because
+    # they may contain circular references.
+    sig { params(arg: T.untyped).returns(T.untyped) }
+    def summarize_unprocessed_value(arg)
+      case arg
+      when Hash
+        {_filtered: {_class: arg.class.name, _keys_count: arg.size}}
+      when Array
+        {_filtered: {_class: arg.class.name, _count: arg.size}}
+      when String
+        {_filtered: {_class: arg.class.name}}
+      when Numeric, Symbol, TrueClass, FalseClass, NilClass
+        arg
+      else
+        {_filtered: {_class: arg.class.name}}
+      end
     end
 
     sig { params(log_value: T.untyped, time: Time).returns(T::Hash[Symbol, T.untyped]) }
@@ -122,11 +285,15 @@ module LogStruct
 
       when T::Struct
         # Default T::Struct.serialize methods returns a hash with string keys, so convert them to symbols
-        log_value.serialize.deep_symbolize_keys
+        symbolize_keys(log_value.serialize)
 
       when Hash
         # Use hash as is and convert string keys to symbols
-        log_value.dup.deep_symbolize_keys
+        symbolize_keys(log_value)
+
+      when Exception
+        # logger.error(exception): its class, message and backtrace
+        Log.from_exception(Source::App, log_value, {}, time).serialize
 
       else
         # Create a Plain log with the message as a string and serialize it with symbol keys
@@ -174,9 +341,22 @@ module LogStruct
       end
     end
 
-    # Serializes Log (or string) into JSON
-    sig { params(severity: T.any(String, Symbol, Integer), time: Time, progname: T.nilable(String), log_value: T.untyped).returns(String) }
-    def call(severity, time, progname, log_value)
+    # Serializes Log (or string) into JSON.
+    # request_id: the request's ID (default: the one RequestContext stored for this thread).
+    # context: fields for the line's ctx, like the running job's ID and class. The line's own ctx keys win.
+    # tags: the logger's tags (logger.tagged("billing")), written as tags unless the line has its own.
+    sig do
+      params(
+        severity: T.any(String, Symbol, Integer),
+        time: Time,
+        progname: T.nilable(String),
+        log_value: T.untyped,
+        request_id: T.nilable(String),
+        context: T.nilable(T::Hash[T.untyped, T.untyped]),
+        tags: T.nilable(T::Array[T.untyped])
+      ).returns(String)
+    end
+    def call(severity, time, progname, log_value, request_id: nil, context: nil, tags: nil)
       level_enum = Level.from_severity(severity)
 
       data = log_value_to_hash(log_value, time: time)
@@ -189,13 +369,77 @@ module LogStruct
       data[:evt] ||= Event::Log
       data[:ts] ||= time.iso8601(3)
       data[:lvl] = level_enum # Set level from severity parameter
-      data[:prog] = progname if progname.present?
-      request_id = Thread.current[:logstruct_request_id]
+      add_metadata(data, progname: progname, request_id: request_id || Thread.current[:logstruct_request_id], context: context, tags: tags)
+
+      begin
+        generate_json(data)
+      rescue SystemStackError, StandardError => e
+        unwritable_line_json(data, e)
+      end
+    end
+
+    # A line that can't be written as JSON is replaced by one that says so, instead of being lost
+    sig { params(data: T::Hash[T.untyped, T.untyped], error: Exception).returns(String) }
+    def unwritable_line_json(data, error)
+      level = data[:lvl]
+      line = {
+        src: enum_or_string(data[:src]) || Source::Internal.serialize,
+        evt: enum_or_string(data[:evt]) || Event::Error.serialize,
+        ts: data[:ts].is_a?(String) ? data[:ts] : Time.now.iso8601(3),
+        lvl: level.is_a?(Level) ? level.serialize : Level::Error.serialize,
+        msg: "LogStruct couldn't write this line as JSON",
+        error_class: error.class.name
+      }
+      unless Thread.current[:logstruct_reporting_unwritable_line]
+        begin
+          Thread.current[:logstruct_reporting_unwritable_line] = true
+          reported = error.is_a?(StandardError) ? error : RuntimeError.new("#{error.class}: #{error.message}")
+          LogStruct.handle_exception(reported, source: Source::Internal, context: {src: line[:src], evt: line[:evt]})
+        ensure
+          Thread.current[:logstruct_reporting_unwritable_line] = nil
+        end
+      end
+      "#{::JSON.generate(line)}\n"
+    end
+
+    # The program name, request ID and job context, filtered and scrubbed like the rest of the line:
+    # an X-Request-Id header can carry anything
+    sig do
+      params(
+        data: T::Hash[T.untyped, T.untyped],
+        progname: T.nilable(String),
+        request_id: T.untyped,
+        context: T.nilable(T::Hash[T.untyped, T.untyped]),
+        tags: T.nilable(T::Array[T.untyped])
+      ).void
+    end
+    def add_metadata(data, progname:, request_id:, context:, tags:)
+      metadata = {}
+      metadata[:prog] = progname if progname.present?
+      metadata[:tags] = tags if tags.is_a?(Array) && !tags.empty? && !data.key?(:tags)
       if request_id.is_a?(String) && !request_id.empty? && !data.key?(:req_id)
-        data[:req_id] = request_id
+        metadata[:req_id] = request_id
       end
+      metadata[:ctx] = symbolize_keys(context) if context.present?
+      return if metadata.empty?
+
+      processed = process_values(metadata)
+      job_context = processed.delete(:ctx)
+      data.merge!(processed)
+      return if job_context.nil?
+
+      line_context = data[:ctx]
+      # A logged Hash's own ctx keeps its keys and gains the job's
+      data[:ctx] = line_context.is_a?(Hash) ? job_context.merge(line_context) : (line_context || job_context)
+    end
 
-      generate_json(data)
+    sig { params(value: T.untyped).returns(T.nilable(String)) }
+    def enum_or_string(value)
+      if value.is_a?(Source) || value.is_a?(Event)
+        value.serialize.to_s
+      elsif value.is_a?(String)
+        value
+      end
     end
 
     # Output as JSON with a newline. We mock this method in tests so we can
@@ -205,26 +449,30 @@ module LogStruct
       "#{data.to_json}\n"
     end
 
-    sig { params(array: T::Array[T.untyped], recursion_depth: Integer).returns(T::Array[T.untyped]) }
-    def process_array(array, recursion_depth:)
+    # An array's items keep the array's path, as in Rails (users.name filters each user's name)
+    sig do
+      params(
+        array: T::Array[T.untyped],
+        recursion_depth: Integer,
+        ancestors: T::Hash[T.untyped, T::Boolean],
+        path: T.nilable(T::Array[String]),
+        root: T.untyped
+      ).returns(T::Array[T.untyped])
+    end
+    def process_array(array, recursion_depth:, ancestors:, path: nil, root: nil)
       return [] if array.empty?
 
-      if looks_like_backtrace_array?(array)
-        array.map { |value| process_values(value, recursion_depth: recursion_depth + 1) }
-      else
-        processed = []
-        array.each_with_index do |value, index|
-          break if index >= 10
-
-          processed << process_values(value, recursion_depth: recursion_depth + 1)
-        end
-
-        if array.size > 10
-          processed << "... and #{array.size - 10} more items"
-        end
-
-        processed
+      limits = LogStruct.config.limits
+      backtrace = looks_like_backtrace_array?(array)
+      max_items = backtrace ? limits.max_backtrace_lines : limits.max_array_items
+      processed = array.first(max_items).map do |value|
+        process_values(value, recursion_depth: recursion_depth + 1, ancestors: ancestors, path: path, root: root)
       end
+      if array.size > max_items
+        processed << "... and #{array.size - max_items} more #{backtrace ? "lines" : "items"}"
+      end
+
+      processed
     end
 
     # Check if an array looks like a backtrace (array of strings with file:line pattern)

data/lib/log_struct/hash_utils.rb

@@ -1,20 +1,49 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "digest"
+require "openssl"
 
 module LogStruct
   # Utility module for hashing sensitive data
   module HashUtils
+    # The key when there's neither a configured hash_salt nor a Rails app with a secret_key_base.
+    # It's public, so these hashes can be matched to a list of candidate values.
+    FALLBACK_KEY = "l0g5t0p"
+
     class << self
       extend T::Sig
 
-      # Create a hash of a string value for tracing while preserving privacy
+      # An HMAC-SHA256 of the value, so lines about the same email can be traced without logging it
       sig { params(value: String).returns(String) }
       def hash_value(value)
-        salt = LogStruct.config.filters.hash_salt
         length = LogStruct.config.filters.hash_length
-        Digest::SHA256.hexdigest("#{salt}#{value}")[0...length] || "error"
+        OpenSSL::HMAC.hexdigest("SHA256", hash_key, value)[0...length] || "error"
+      end
+
+      # The configured hash_salt, or a key derived from the app's secret_key_base
+      sig { returns(String) }
+      def hash_key
+        LogStruct.config.filters.hash_salt || derived_key || FALLBACK_KEY
+      end
+
+      # Forget the derived key (for tests that change the Rails app)
+      sig { void }
+      def reset!
+        @derived_key = T.let(nil, T.nilable(String))
+      end
+
+      private
+
+      sig { returns(T.nilable(String)) }
+      def derived_key
+        @derived_key ||= T.let(begin
+          application = ::Rails.application if defined?(::Rails) && ::Rails.respond_to?(:application)
+          # Rails derives keys like this one for cookies and messages; nil until the app has its secret
+          application&.key_generator&.generate_key("LogStruct hashed values", 32)
+        rescue
+          nil
+        end,
+          T.nilable(String))
       end
     end
   end