standard_audit 0.6.0 → 0.8.0

data/lib/standard_audit/configuration.rb

@@ -5,7 +5,9 @@ module StandardAudit
                   :current_actor_resolver, :current_request_id_resolver,
                   :current_ip_address_resolver, :current_user_agent_resolver,
                   :current_session_id_resolver,
-                  :sensitive_keys, :metadata_builder,
+                  :sensitive_keys, :sensitive_key_patterns,
+                  :sensitive_key_exceptions, :filter_nested_metadata,
+                  :metadata_builder, :before_checksum_hooks,
                   :anonymizable_metadata_keys, :retention_days
 
     def initialize
@@ -43,7 +45,37 @@ module StandardAudit
         private_key certificate_chain
         ssn credit_card authorization
       ]
+      # Regexps matched against every metadata key name, in addition to the
+      # exact-match `sensitive_keys` list. This is the supported way to catch a
+      # family of keys: `/secret/i` redacts `client_secret`, `webhook_secret`,
+      # and `secret` alike.
+      #
+      # There is deliberately NO substring *mode* for `sensitive_keys` — see
+      # the note in MetadataFilter. Patterns are opt-in and per-app, which is
+      # the only safe shape for a rule applied to append-only rows.
+      @sensitive_key_patterns = []
+
+      # Key names (String/Symbol, exact) or Regexps that are never redacted,
+      # even when `sensitive_keys` or `sensitive_key_patterns` matches. Lets an
+      # app adopt a broad pattern while keeping the handful of real audit keys
+      # it would otherwise swallow, e.g.
+      # `sensitive_key_patterns = [/token/i]` with
+      # `sensitive_key_exceptions = %i[input_tokens output_tokens]`.
+      @sensitive_key_exceptions = []
+
+      # When true, redaction descends into nested Hashes (and Hashes inside
+      # Arrays), so `metadata: { stripe: { client_secret: … } }` is caught.
+      # Defaults to false: it changes what gets written, and audit rows are
+      # append-only. Reserved keys are never descended into.
+      @filter_nested_metadata = false
+
       @metadata_builder = nil
+
+      # Callables (or Symbols naming an AuditLog instance method) run on
+      # `before_create` AFTER the UUID is assigned and BEFORE the checksum is
+      # computed. See Configuration#before_checksum.
+      @before_checksum_hooks = []
+
       @anonymizable_metadata_keys = %i[email name ip_address]
 
       # Retention defaults from ENV so it can be set per-environment without a
@@ -63,6 +95,37 @@ module StandardAudit
       days&.positive? ? days : nil
     end
 
+    # Registers a hook to run between `assign_uuid` and `compute_checksum` on
+    # every audit write that instantiates a model.
+    #
+    #   config.before_checksum { |log| log.scope = derive_scope(log) }
+    #   config.before_checksum :backfill_organization_scope
+    #
+    # A hook may set a `CHECKSUM_FIELDS` member (`scope_gid`, `metadata`, …) and
+    # the row will still verify, because the checksum is computed afterwards.
+    # That is the whole point: before this existed, hosts had to register their
+    # own `before_create ..., prepend: true` to beat the gem's checksum
+    # callback, which is fragile ordering knowledge no host should need.
+    #
+    # Hooks accumulate and run in registration order. Each is rescued
+    # individually — a failing hook logs and is skipped; it never fails the
+    # audit write.
+    #
+    # A Symbol/String is sent to the AuditLog instance (use this for methods
+    # supplied by a concern mixed into the model). A callable is passed the
+    # instance.
+    #
+    # NOTE: batched writes (`StandardAudit.batch { … }` → `insert_all!`) never
+    # instantiate a model, so hooks do not run there. A batched writer that
+    # needs a derived column has to set it on the buffered attrs.
+    def before_checksum(hook = nil, &block)
+      hook ||= block
+      raise ArgumentError, "before_checksum needs a callable, a Symbol, or a block" if hook.nil?
+
+      @before_checksum_hooks << hook
+      hook
+    end
+
     def subscribe_to(pattern)
       @subscriptions << pattern
     end

data/lib/standard_audit/metadata_filter.rb

@@ -0,0 +1,155 @@
+module StandardAudit
+  # The single implementation of sensitive-key redaction for audit metadata.
+  #
+  # There used to be two: one in `StandardAudit.record` and one in
+  # `Subscriber#extract_metadata`. They had already diverged — the subscriber
+  # copy did not subtract `RESERVED_METADATA_KEYS`, so an app that (reasonably)
+  # added `:_tags` to `sensitive_keys` got the reserved key preserved on the
+  # `record` path and stripped on the `ActiveSupport::Notifications` path. Two
+  # copies of a security filter drifting apart is exactly the failure mode
+  # worth designing out, so both paths now call here.
+  #
+  # The divergence is resolved in favour of `record`'s behaviour: reserved keys
+  # are subtracted from the sensitive set and can never be filtered.
+  #
+  # == Matching
+  #
+  # A key is redacted when it matches `config.sensitive_keys` **exactly** (by
+  # name, string/symbol insensitive) or matches one of
+  # `config.sensitive_key_patterns` (Regexps, always applied), unless it is
+  # listed in `config.sensitive_key_exceptions` or is a reserved key.
+  #
+  # == Why there is no substring mode
+  #
+  # Matching `sensitive_keys` by substring instead of exactly is the obvious
+  # "fix" for `client_secret` not matching `:secret`, and it is a trap. Against
+  # the current default key list it would strip real audit content across the
+  # estate:
+  #
+  #   :token   => input_tokens, output_tokens (live LLM cost accounting),
+  #              token_digest (rendered in a staff audit UI)
+  #   :password => password_reset_sent_at, onepassword
+  #   :authorization => authorization_endpoint
+  #
+  # Audit rows are append-only, so a bad default cannot be undone — the content
+  # is simply never written. `sensitive_key_patterns` is the supported tool
+  # instead: `/secret/i` solves the motivating `client_secret` case exactly,
+  # opt-in and per-app, and `rake standard_audit:sensitive_keys:dry_run` turns
+  # "is this rule safe for my data?" into a command rather than a guess.
+  class MetadataFilter
+    # Raised when metadata is neither nil nor hash-like. Deliberately a raise
+    # rather than a pass-through: this filter **fails closed**. An
+    # `ActionController::Parameters` is not a `Hash`, and letting an unrecognised
+    # object through unfiltered would write raw params — passwords included —
+    # into an append-only row. Pre-0.7.0 the same input blew up with
+    # `NoMethodError` on `#reject`, so raising preserves the outcome while
+    # naming the cause.
+    class UnfilterableMetadataError < ArgumentError; end
+
+    class << self
+      def call(metadata, config: StandardAudit.config)
+        new(config: config).call(metadata)
+      end
+    end
+
+    def initialize(config: StandardAudit.config)
+      @config = config
+    end
+
+    # Filters anything hash-like, not just `Hash` — `ActionController::Parameters`
+    # and other `each_pair`-able wrappers are filtered rather than waved
+    # through. `nil` passes (there is nothing to leak); anything else raises.
+    #
+    # Descends into nested Hashes (and Hashes inside Arrays) only when
+    # `config.filter_nested_metadata` is true. Reserved keys are preserved and
+    # their values are left entirely alone, **at every depth** — `_tags` and
+    # `_source` are gem-owned, not host payload, and immunity that held only at
+    # the top level would be a confusing half-guarantee.
+    def call(metadata)
+      return nil if metadata.nil?
+
+      unless hash_like?(metadata)
+        raise UnfilterableMetadataError,
+          "audit metadata must be nil or hash-like (got #{metadata.class}); " \
+          "refusing to write it unfiltered"
+      end
+
+      filtered = metadata.reject { |key, _| filter?(key) }
+      return filtered unless @config.filter_nested_metadata
+
+      filter_values(filtered)
+    end
+
+    # True when `key` would be redacted from metadata. Public so hosts (and the
+    # dry-run tooling) can ask the question without writing a row.
+    def filter?(key)
+      key = key.to_s
+
+      # `_tags` and `_source` are owned by EventSubscriber and are never
+      # stripped, even if a consumer lists them in `sensitive_keys`.
+      return false if StandardAudit::RESERVED_METADATA_KEYS.include?(key)
+      return false if exception?(key)
+
+      sensitive_keys.include?(key) || sensitive_key_patterns.any? { |pattern| pattern.match?(key) }
+    end
+
+    private
+
+    def hash_like?(value)
+      value.respond_to?(:each_pair) && value.respond_to?(:reject)
+    end
+
+    # Rewrites each value in place on the already-rejected copy. In place
+    # because `transform_values` is not guaranteed across every hash-like
+    # wrapper, whereas `[]=` is.
+    def filter_values(hash)
+      hash.each_pair do |key, value|
+        # Reserved subtree: immune at every depth, never descended into.
+        next if StandardAudit::RESERVED_METADATA_KEYS.include?(key.to_s)
+
+        replacement = filter_descendant(value)
+        hash[key] = replacement unless replacement.equal?(value)
+      end
+
+      hash
+    end
+
+    def filter_descendant(value)
+      if hash_like?(value)
+        filter_values(value.reject { |key, _| filter?(key) })
+      elsif value.is_a?(Array)
+        value.map { |element| filter_descendant(element) }
+      else
+        value
+      end
+    end
+
+    def sensitive_keys
+      @sensitive_keys ||= @config.sensitive_keys.map(&:to_s) - StandardAudit::RESERVED_METADATA_KEYS
+    end
+
+    # A String entry is read as a Regexp source, so the rake task (and a
+    # `SENSITIVE_KEY_PATTERNS` env var) can pass one through without eval.
+    def sensitive_key_patterns
+      @sensitive_key_patterns ||= Array(@config.sensitive_key_patterns).map do |pattern|
+        pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern.to_s)
+      end
+    end
+
+    def exception?(key)
+      exact_exceptions.include?(key) || pattern_exceptions.any? { |pattern| pattern.match?(key) }
+    end
+
+    def exceptions
+      @exceptions ||= Array(@config.sensitive_key_exceptions).partition { |entry| entry.is_a?(Regexp) }
+    end
+
+    def pattern_exceptions
+      exceptions.first
+    end
+
+    def exact_exceptions
+      @exact_exceptions ||= exceptions.last.map(&:to_s)
+    end
+  end
+end

data/lib/standard_audit/reference_preloading.rb

@@ -0,0 +1,298 @@
+require "cgi"
+require "active_support/concern"
+
+module StandardAudit
+  # Batch resolution of the GlobalID-backed `actor` / `target` / `scope`
+  # references on a page of audit rows, plus the per-row memo the readers
+  # consult.
+  #
+  # Why this exists: `AuditLog#actor` resolves `actor_gid` through
+  # `GlobalID::Locator.locate`, which is one query per row. Rendering N rows
+  # that each read `actor` and `target` issues O(N) lookups — an N+1 that
+  # Prosopite fails in every consuming app. Before this concern existed, apps
+  # worked around it by defining their own `preloaded_actor=` writer (or
+  # reaching into `@preloaded_actor` with `instance_variable_set`) and
+  # hand-rolling a per-type whitelist loader.
+  #
+  # == Memo semantics
+  #
+  # The memo is a Hash consulted with `key?`, not a pair of ivars checked with
+  # `defined?`. That distinction matters: a reference that was preloaded but
+  # whose record has since been deleted memoizes `nil`, and reads back as
+  # `nil` *without* falling through to a query. "Preloaded, and the answer is
+  # nothing" is therefore distinct from "not preloaded".
+  #
+  # `actor=` / `target=` / `scope=` populate the memo too, since the writer
+  # already holds the record. `reload` clears it.
+  #
+  # == Batched writes
+  #
+  # `StandardAudit.batch { ... }` flushes through `insert_all!` and never
+  # instantiates an AuditLog, so nothing in this concern runs on that path —
+  # neither the memo nor the writers. It is a no-op for batched writes by
+  # construction, not by accident.
+  module ReferencePreloading
+    extend ActiveSupport::Concern
+
+    # The GlobalID-backed reference columns. `scope` is preloadable but not in
+    # the default `refs:` set, since most read surfaces render actor/target
+    # only.
+    REFERENCES = %i[actor target scope].freeze
+
+    DEFAULT_REFS = %i[actor target].freeze
+
+    class_methods do
+      # Resolves the given references for a whole collection of audit logs in a
+      # fixed number of queries (one per distinct stored `*_type`), then
+      # memoizes the result on each row.
+      #
+      #   AuditLog.preload_references(
+      #     logs,
+      #     refs: %i[actor target],
+      #     only: [Account, Profile, Order],
+      #     includes: { "Profile" => [:account], "Order" => [:user] }
+      #   )
+      #
+      # [refs:]     Which references to resolve. Defaults to `%i[actor target]`.
+      # [only:]     Optional whitelist of permitted classes. See the note below —
+      #             this is matched against the *stored type string*, so it is
+      #             a stricter gate than `GlobalID::Locator`'s own `:only`.
+      # [includes:] Either a uniform Active Record `includes` spec applied to
+      #             every type, or a per-type Hash keyed by class name String
+      #             or Class constant (e.g. `{ "Profile" => [:account] }`).
+      #             A Hash counts as per-type only when *every* key is a String
+      #             or a Module; `{ account: :identifiers }` is therefore read
+      #             as a uniform nested-includes spec, as you would expect.
+      #
+      # == Why `only:` is matched on the stored string
+      #
+      # `GlobalID::Locator`'s `:only` option is evaluated as
+      # `gid.model_class <= klass`, which means it *constantizes the stored
+      # type string before deciding whether it was allowed*. That is the wrong
+      # order for audit rows, whose type strings are historical: a class that
+      # has since been renamed or removed raises `NameError` rather than being
+      # filtered out. So this method filters on the stored `*_type` string
+      # first, by class name, and only then hands the surviving gids to
+      # `locate_many` (still passing `only:` as a second gate).
+      #
+      # The consequence is that `only:` does **not** expand to subclasses or
+      # to modules the way GlobalID's does — list every concrete class whose
+      # name you expect to see in `actor_type` / `target_type`. Deny-by-default
+      # is the safe direction for a whitelist.
+      #
+      # A reference whose type is not in `only:` memoizes `nil`, so it reads
+      # back as nil without a query rather than silently re-N+1ing.
+      #
+      # Returns the logs as an Array.
+      def preload_references(logs, refs: DEFAULT_REFS, only: nil, includes: nil)
+        logs = logs.to_a
+        return logs if logs.empty?
+
+        Array(refs).each do |ref|
+          ref = ref.to_sym
+          unless REFERENCES.include?(ref)
+            raise ArgumentError, "unknown audit reference #{ref.inspect} (expected one of #{REFERENCES.inspect})"
+          end
+
+          preload_one_reference(logs, ref, only: only, includes: includes)
+        end
+
+        logs
+      end
+
+      # Extracts the model id from a GlobalID string without resolving it:
+      # "gid://some-app/Account/123" => "123".
+      #
+      # Deliberately *not* `GlobalID.parse(gid)&.model_id`. Audit gids are
+      # historical and the rows are append-only, so a gid written by another
+      # app — or before this app was renamed — is a real, permanent row whose
+      # id must still be readable. Parsing it back through the locator
+      # machinery resolves against the *current* `GlobalID.app`, which is not
+      # what these rows are keyed on.
+      #
+      # This mirrors `URI::GID#set_model_components` exactly, minus the app
+      # check and the validations: drop any `?params` query, take everything
+      # after the model-name segment, split composite ids on "/", and
+      # `CGI.unescape` each part (`URI::GID.build` `CGI.escape`s them). Returns
+      # a String for a single-column primary key and an Array of Strings for a
+      # composite one, matching `GlobalID#model_id`.
+      def reference_model_id(gid)
+        return nil if gid.blank?
+
+        # "gid:" , "" , app , ModelName , <id segment(s)>
+        segment = gid.to_s.split("?", 2).first.to_s.split("/", 5)[4]
+        return nil if segment.blank?
+
+        parts = segment.split("/").reject(&:blank?).map { |part| CGI.unescape(part) }
+        return nil if parts.empty?
+
+        parts.length == 1 ? parts.first : parts
+      end
+
+      private
+
+      def preload_one_reference(logs, ref, only:, includes:)
+        gid_attr = :"#{ref}_gid"
+        type_attr = :"#{ref}_type"
+        allowed_names = only && Array(only).map { |klass| klass.is_a?(Module) ? klass.name : klass.to_s }
+
+        index = {}
+        unresolvable_types = []
+
+        logs.group_by { |log| log.public_send(type_attr) }.each do |type, type_logs|
+          # A blank `*_type` with a populated `*_gid` happens on historical and
+          # partially-backfilled rows. The per-row reader can still resolve
+          # those (the gid carries the model name), so mark the group
+          # unresolvable rather than memoizing nil — batching must never make a
+          # resolvable reference read back as nothing.
+          if type.blank?
+            unresolvable_types << type
+            next
+          end
+
+          # Not whitelisted: intentionally left out of the index so it memoizes
+          # nil (no query, now or later).
+          next if allowed_names && !allowed_names.include?(type)
+
+          gids = type_logs.filter_map { |log| log.public_send(gid_attr).presence }.uniq
+          next if gids.empty?
+
+          begin
+            locate_reference_records(gids, only: only, includes: includes_for(includes, type)).each do |record|
+              index[[type, normalized_record_key(record.id)]] = record
+            end
+          rescue NameError, ActiveRecord::StatementInvalid => e
+            # A historical type string that no longer constantizes, or a
+            # relation the current schema can't satisfy. Leave these rows
+            # *unmemoized* so the per-row reader behaves exactly as it did
+            # before preloading was attempted — memoizing nil here would
+            # silently rewrite behaviour on a bad `includes:`.
+            unresolvable_types << type
+            Rails.logger.warn("[StandardAudit] Could not preload #{ref} for type #{type.inspect}: #{e.class}: #{e.message}")
+          end
+        end
+
+        logs.each do |log|
+          type = log.public_send(type_attr)
+          next if unresolvable_types.include?(type)
+
+          log.write_preloaded_reference(
+            ref,
+            index[[type, reference_model_id(log.public_send(gid_attr))]]
+          )
+        end
+      end
+
+      # `ignore_missing: true` makes this a `where(id: ids)` instead of
+      # `find(ids)`, so a deleted record is simply absent from the result
+      # rather than raising — which is what lets a deleted reference memoize
+      # nil.
+      def locate_reference_records(gids, only:, includes:)
+        options = { ignore_missing: true }
+        options[:only] = only if only
+        options[:includes] = includes if includes
+
+        GlobalID::Locator.locate_many(gids, options)
+      end
+
+      # Mirrors how `GlobalID::Locator::BaseLocator#locate_many` keys its own
+      # result index, so composite primary keys line up with
+      # `reference_model_id`.
+      def normalized_record_key(id)
+        id.is_a?(Array) ? id.map(&:to_s) : id.to_s
+      end
+
+      def includes_for(includes, type)
+        return nil if includes.nil?
+        return includes unless per_type_includes?(includes)
+
+        entry = includes.find { |key, _| (key.is_a?(Module) ? key.name : key.to_s) == type }
+        entry&.last
+      end
+
+      # A Hash counts as a per-type mapping only when every key names a class.
+      # Symbol keys mean the caller passed an Active Record includes spec such
+      # as `{ account: :identifiers }`, which must be applied uniformly.
+      def per_type_includes?(includes)
+        includes.is_a?(Hash) &&
+          includes.any? &&
+          includes.keys.all? { |key| key.is_a?(String) || key.is_a?(Module) }
+      end
+    end
+
+    # Writers. `preloaded_actor = record_or_nil` is the supported way to hand a
+    # separately-resolved record to a log; `nil` means "resolved to nothing",
+    # not "clear the memo".
+    REFERENCES.each do |ref|
+      define_method(:"preloaded_#{ref}=") do |record|
+        write_preloaded_reference(ref, record)
+      end
+
+      define_method(:"#{ref}_preloaded?") do
+        reference_preloaded?(ref)
+      end
+
+      # `actor_model_id` / `target_model_id` / `scope_model_id` — the trailing
+      # gid segment, without needing the record.
+      define_method(:"#{ref}_model_id") do
+        self.class.reference_model_id(public_send(:"#{ref}_gid"))
+      end
+    end
+
+    # The memo itself. Public because `preload_references` writes through it
+    # from the class side; treat it as gem-internal — `preloaded_actor=` and
+    # `actor_preloaded?` are the supported surface.
+    def preloaded_references
+      @preloaded_references ||= {}
+    end
+
+    def reference_preloaded?(ref)
+      preloaded_references.key?(ref.to_sym)
+    end
+
+    # See `preloaded_references` — gem-internal, but public so the class-level
+    # preloader can reach it.
+    def write_preloaded_reference(ref, record)
+      preloaded_references[ref.to_sym] = record
+    end
+
+    # Drops the memo so the next read re-resolves from the (possibly changed)
+    # gid columns.
+    def reload(...)
+      @preloaded_references = nil
+      super
+    end
+
+    private
+
+    # The single-row fallback: identical to the pre-0.7.0 reader behaviour.
+    def locate_reference(ref)
+      gid = public_send(:"#{ref}_gid")
+      return nil if gid.blank?
+
+      GlobalID::Locator.locate(gid)
+    rescue ActiveRecord::RecordNotFound
+      nil
+    end
+
+    def read_reference(ref)
+      ref = ref.to_sym
+      return preloaded_references[ref] if preloaded_references.key?(ref)
+
+      locate_reference(ref)
+    end
+
+    def assign_reference(ref, record)
+      if record.nil?
+        public_send(:"#{ref}_gid=", nil)
+        public_send(:"#{ref}_type=", nil)
+      else
+        public_send(:"#{ref}_gid=", record.to_global_id.to_s)
+        public_send(:"#{ref}_type=", record.class.name)
+      end
+
+      write_preloaded_reference(ref, record)
+      record
+    end
+  end
+end

data/lib/standard_audit/rspec.rb

@@ -8,9 +8,23 @@ require "standard_audit"
 # - Resets the Configuration via `StandardAudit.reset_configuration!` so
 #   that mutations to `StandardAudit.config` (subscriptions, sensitive
 #   keys, async flag, custom resolvers, etc.) do not bleed across specs.
-#   Consumers that customise configuration must re-call
-#   `StandardAudit.configure { |c| ... }` from a `before` hook in their
-#   own suite if they need a non-default baseline.
+#
+# IMPORTANT — declare your configuration as a baseline. The reset restores
+# gem defaults, and the config object holds *behaviour*, not just data:
+# `before_checksum_hooks` live there. A suite that installs this plugin
+# without a baseline silently loses its write-time hooks after the first
+# example, and the specs that would have caught it pass vacuously.
+#
+# So in `config/initializers/standard_audit.rb`, use:
+#
+#   StandardAudit.configure(baseline: true) do |config|
+#     config.subscribe_to "myapp.**"
+#     config.before_checksum :backfill_scope
+#   end
+#
+# `reset_configuration!` replays that block onto the fresh Configuration,
+# so every example starts from your app's real configuration. There is
+# nothing to re-apply by hand in a `before` hook.
 #
 # The memoized `Subscriber` and `EventSubscriber` instances are *not*
 # torn down here — they are wired up at engine boot via initializers and