standard_audit 0.6.0 → 0.8.0

data/lib/standard_audit/sensitive_keys_dry_run.rb

@@ -0,0 +1,177 @@
+require "set"
+
+module StandardAudit
+  # Answers "is this redaction rule safe for MY data?" against the rows an app
+  # already has, before the rule is switched on.
+  #
+  # This matters more here than in most gems: audit rows are append-only, so a
+  # rule that swallows real audit content cannot be undone after the fact — the
+  # content is simply never written from then on. Reading the historical rows
+  # first turns the judgement call into a command.
+  #
+  # Keys are extracted **in Ruby**, not with `jsonb_object_keys`, so this works
+  # against SQLite (the dummy app), MySQL, and Postgres alike. The install
+  # template's migration uses jsonb + GIN, but the gem itself stays
+  # backend-neutral.
+  #
+  #   StandardAudit::SensitiveKeysDryRun.call(sensitive_key_patterns: [/secret/i])
+  #   # => #<Report rows_scanned=1204 stripped={"client_secret"=>18, ...} ...>
+  #
+  class SensitiveKeysDryRun
+    # [rows_scanned]        how many audit rows were read
+    # [stripped]            key path => row count, for keys the candidate rule
+    #                       would redact
+    # [kept]                key path => row count, for keys it would keep
+    # [nested_unfiltered]   key path => row count, for *nested* keys the rule
+    #                       matches but which are NOT redacted because
+    #                       `filter_nested_metadata` is off. This is the
+    #                       exposure `filter_nested_metadata` exists to close;
+    #                       empty when nested filtering is enabled.
+    Report = Struct.new(:rows_scanned, :stripped, :kept, :nested_unfiltered, :nested, keyword_init: true) do
+      def any_stripped?
+        stripped.any?
+      end
+
+      def to_s
+        lines = []
+        lines << "Rows scanned: #{rows_scanned}"
+        lines << "Nested filtering: #{nested ? 'on' : 'off'}"
+        lines << ""
+
+        lines << section("WOULD BE STRIPPED", stripped, "Nothing would be stripped.")
+        lines << ""
+        lines << section("KEPT", kept, "No metadata keys found.")
+
+        if nested_unfiltered.any?
+          lines << ""
+          lines << section(
+            "MATCHES BUT NOT STRIPPED (nested; set config.filter_nested_metadata = true to redact)",
+            nested_unfiltered,
+            nil
+          )
+        end
+
+        lines.join("\n")
+      end
+
+      private
+
+      def section(title, counts, empty_message)
+        out = [title, "=" * title.length]
+
+        if counts.empty?
+          out << (empty_message || "None.")
+        else
+          width = counts.keys.map(&:length).max
+          counts.sort_by { |key, count| [-count, key] }.each do |key, count|
+            out << format("  %-#{width}s  %d row(s)", key, count)
+          end
+        end
+
+        out.join("\n")
+      end
+    end
+
+    class << self
+      # Every keyword defaults to the app's live configuration, so calling it
+      # with no arguments reports what the *current* config does.
+      def call(sensitive_keys: nil, sensitive_key_patterns: nil, sensitive_key_exceptions: nil,
+               nested: nil, relation: nil, batch_size: 1000)
+        new(
+          sensitive_keys: sensitive_keys,
+          sensitive_key_patterns: sensitive_key_patterns,
+          sensitive_key_exceptions: sensitive_key_exceptions,
+          nested: nested
+        ).call(relation: relation, batch_size: batch_size)
+      end
+    end
+
+    def initialize(sensitive_keys: nil, sensitive_key_patterns: nil, sensitive_key_exceptions: nil, nested: nil)
+      live = StandardAudit.config
+
+      candidate = Configuration.new
+      candidate.sensitive_keys = sensitive_keys || live.sensitive_keys
+      candidate.sensitive_key_patterns = sensitive_key_patterns || live.sensitive_key_patterns
+      candidate.sensitive_key_exceptions = sensitive_key_exceptions || live.sensitive_key_exceptions
+
+      @nested = nested.nil? ? live.filter_nested_metadata : nested
+      @filter = MetadataFilter.new(config: candidate)
+    end
+
+    def call(relation: nil, batch_size: 1000)
+      relation ||= StandardAudit::AuditLog.all
+
+      rows = 0
+      stripped = Hash.new(0)
+      kept = Hash.new(0)
+      nested_unfiltered = Hash.new(0)
+
+      relation.select(:id, :metadata).find_each(batch_size: batch_size) do |log|
+        rows += 1
+        metadata = log.metadata
+        next unless metadata.respond_to?(:each_pair)
+
+        # Collect distinct paths per row first, then increment once each. A row
+        # whose `charges[]` array holds two matching hashes is ONE affected row,
+        # not two — the counts are documented as row counts.
+        row = { stripped: Set.new, kept: Set.new, nested_unfiltered: Set.new }
+        walk(metadata, nil, row, top_level: true)
+
+        row[:stripped].each { |path| stripped[path] += 1 }
+        row[:kept].each { |path| kept[path] += 1 }
+        row[:nested_unfiltered].each { |path| nested_unfiltered[path] += 1 }
+      end
+
+      Report.new(
+        rows_scanned: rows,
+        stripped: stripped,
+        kept: kept,
+        nested_unfiltered: nested_unfiltered,
+        nested: @nested
+      )
+    end
+
+    private
+
+    def walk(hash, prefix, row, top_level:)
+      hash.each_pair do |key, value|
+        name = key.to_s
+        path = prefix ? "#{prefix}.#{name}" : name
+
+        # Reserved keys are immune at every depth, and their subtree is
+        # gem-owned — never descended into. Mirrors MetadataFilter exactly, so
+        # the dry run cannot misrepresent the rule it is modelling.
+        if StandardAudit::RESERVED_METADATA_KEYS.include?(name)
+          row[:kept] << path
+          next
+        end
+
+        matches = @filter.filter?(name)
+
+        if matches && (top_level || @nested)
+          row[:stripped] << path
+          next
+        end
+
+        # A nested match that survives because nested filtering is off. Reported
+        # separately rather than silently counted as "kept" — this is exactly
+        # the leak `filter_nested_metadata` closes.
+        if matches
+          row[:nested_unfiltered] << path
+        else
+          row[:kept] << path
+        end
+
+        descend(value, path, row)
+      end
+    end
+
+    def descend(value, path, row)
+      if value.respond_to?(:each_pair)
+        walk(value, path, row, top_level: false)
+      elsif value.is_a?(Array)
+        value.each { |element| descend(element, "#{path}[]", row) }
+      end
+    end
+  end
+end

data/lib/standard_audit/subscriber.rb

@@ -79,9 +79,11 @@ module StandardAudit
         raw_metadata = config.metadata_builder.call(raw_metadata)
       end
 
-      # Filter sensitive keys
-      sensitive = config.sensitive_keys.map(&:to_s)
-      raw_metadata.reject { |k, _| sensitive.include?(k.to_s) }
+      # Redaction lives in MetadataFilter, shared with StandardAudit.record.
+      # This path previously carried its own copy that did *not* subtract
+      # RESERVED_METADATA_KEYS, so `_tags`/`_source` were strippable here and
+      # not there.
+      StandardAudit::MetadataFilter.call(raw_metadata, config: config)
     end
   end
 end

data/lib/standard_audit/version.rb

@@ -1,3 +1,3 @@
 module StandardAudit
-  VERSION = "0.6.0"
+  VERSION = "0.8.0"
 end

data/lib/standard_audit.rb

@@ -1,8 +1,11 @@
 require "standard_audit/version"
 require "standard_audit/engine"
 require "standard_audit/configuration"
+require "standard_audit/metadata_filter"
+require "standard_audit/sensitive_keys_dry_run"
 require "standard_audit/subscriber"
 require "standard_audit/event_subscriber"
+require "standard_audit/reference_preloading"
 require "standard_audit/auditable"
 require "standard_audit/audit_scope"
 require "standard_audit/checks/retention"
@@ -13,8 +16,30 @@ module StandardAudit
   RESERVED_METADATA_KEYS = %w[_tags _source].freeze
 
   class << self
-    def configure
-      yield(config) if block_given?
+    # Applies configuration to the single mutable Configuration instance.
+    #
+    # `baseline: true` also *remembers* the block, so `reset_configuration!`
+    # replays it. That matters because the config object holds behaviour, not
+    # just data — `before_checksum_hooks` in particular. Without a baseline, a
+    # suite that installs the rspec plugin (which calls `reset_configuration!`
+    # before every example) silently loses write-time hooks after the first
+    # example, and the specs that would notice pass vacuously.
+    #
+    # Idiomatic host usage, in config/initializers/standard_audit.rb:
+    #
+    #   StandardAudit.configure(baseline: true) do |config|
+    #     config.subscribe_to "myapp.**"
+    #     config.before_checksum :backfill_scope
+    #   end
+    #
+    # Only the most recent `baseline: true` block is remembered; call it once,
+    # from the initializer.
+    def configure(baseline: false, &block)
+      return config unless block
+
+      @baseline_configuration = block if baseline
+      block.call(config)
+      config
     end
 
     def config
@@ -26,10 +51,9 @@ module StandardAudit
 
       actor ||= config.current_actor_resolver.call
 
-      # Filter sensitive keys. `_tags` and `_source` are reserved internal
-      # metadata keys owned by EventSubscriber and are never stripped.
-      sensitive = config.sensitive_keys.map(&:to_s) - RESERVED_METADATA_KEYS
-      filtered_metadata = metadata.reject { |k, _| sensitive.include?(k.to_s) }
+      # Redaction lives in MetadataFilter, shared with Subscriber, so the two
+      # write paths cannot drift apart.
+      filtered_metadata = MetadataFilter.call(metadata, config: config)
 
       attrs = {
         event_type: event_type,
@@ -101,8 +125,24 @@ module StandardAudit
       @event_subscriber ||= EventSubscriber.new
     end
 
-    def reset_configuration!
+    # Drops the memoized Configuration. Any block registered with
+    # `configure(baseline: true)` is replayed onto the fresh instance, so a
+    # per-example reset restores the app's real configuration rather than the
+    # gem defaults.
+    def reset_configuration!(replay_baseline: true)
       @configuration = nil
+      @baseline_configuration&.call(config) if replay_baseline
+      config
+    end
+
+    # Forgets the `configure(baseline: true)` block. Mainly for the gem's own
+    # specs and for a host that needs a genuinely pristine configuration.
+    def clear_baseline_configuration!
+      @baseline_configuration = nil
+    end
+
+    def baseline_configured?
+      !@baseline_configuration.nil?
     end
 
     private
@@ -113,10 +153,8 @@ module StandardAudit
 
     def flush_batch(buffer)
       now = Time.current
-      previous_checksum = StandardAudit::AuditLog
-        .order(created_at: :desc, id: :desc)
-        .limit(1)
-        .pick(:checksum)
+      records_parent = StandardAudit::AuditLog.chain_parent_column?
+      previous_checksum = StandardAudit::AuditLog.chain_tip_checksum
 
       # Generate sorted UUIDs to ensure batch ordering matches id ordering.
       # UUIDv7 within the same millisecond can have non-monotonic lower bits;
@@ -135,6 +173,7 @@ module StandardAudit
           row.stringify_keys,
           previous_checksum: previous_checksum
         )
+        row[:previous_checksum] = previous_checksum if records_parent
         row[:checksum] = checksum
         previous_checksum = checksum
         row

data/lib/tasks/standard_audit_tasks.rake

@@ -63,22 +63,81 @@ namespace :standard_audit do
     puts "============================="
     puts "Records verified: #{result[:verified]}"
     puts "Chain valid: #{result[:valid]}"
+    puts "Forked links recovered: #{result[:recovered]}"
 
     if result[:failures].any?
-      puts "\nTampered records detected: #{result[:failures].size}"
+      puts "\nUnverifiable records detected: #{result[:failures].size}"
       result[:failures].each do |failure|
-        puts "  #{failure[:id]} (#{failure[:event_type]}) at #{failure[:created_at]}"
+        puts "  #{failure[:id]} (#{failure[:event_type]}) at #{failure[:created_at]} — #{failure[:reason]}"
       end
       abort "Chain verification failed"
     end
   end
 
+  desc "Record the parent digest each existing row was signed against (never re-signs)"
+  task relink_checksums: :environment do
+    unless StandardAudit::AuditLog.chain_parent_column?
+      abort "audit_logs has no previous_checksum column — run `rails generate standard_audit:add_previous_checksum` first"
+    end
+
+    result = StandardAudit::AuditLog.relink_checksums!
+
+    puts "Relinked: #{result[:relinked]}"
+    puts "Left as-is (already linked, or a segment root): #{result[:skipped]}"
+    puts "Unresolved: #{result[:unresolved]}"
+    puts ""
+    puts "No checksum was rewritten. A parent is recorded only when it reproduces"
+    puts "the digest the row has held since it was written; unresolved rows keep"
+    puts "failing verification, which is the point."
+  end
+
   desc "Backfill checksums for records that don't have them"
   task backfill_checksums: :environment do
     count = StandardAudit::AuditLog.backfill_checksums!
     puts "Backfilled checksums for #{count} audit log records"
   end
 
+  namespace :sensitive_keys do
+    desc "Report which historical metadata keys a redaction rule would strip (read-only)"
+    task :dry_run, [:pattern] => :environment do |_t, args|
+      # Audit rows are append-only, so a redaction rule that swallows real
+      # content cannot be undone. This reads the rows you already have and
+      # reports, per key, what the rule would have stripped.
+      #
+      #   rake standard_audit:sensitive_keys:dry_run
+      #   rake "standard_audit:sensitive_keys:dry_run[secret]"
+      #   NESTED=1 rake "standard_audit:sensitive_keys:dry_run[secret|token]"
+      #
+      # The argument is a Regexp source, matched case-insensitively, and is
+      # applied *in addition to* the app's configured sensitive_keys.
+      # NESTED=1/0 overrides config.filter_nested_metadata for the run.
+      pattern = args[:pattern].presence || ENV["PATTERN"].presence
+      patterns = StandardAudit.config.sensitive_key_patterns.dup
+      patterns << Regexp.new(pattern, Regexp::IGNORECASE) if pattern
+
+      nested =
+        case ENV["NESTED"]
+        when nil, "" then nil
+        when "0", "false" then false
+        else true
+        end
+
+      report = StandardAudit::SensitiveKeysDryRun.call(
+        sensitive_key_patterns: patterns,
+        nested: nested,
+        batch_size: (ENV["BATCH_SIZE"] || 1000).to_i
+      )
+
+      puts "StandardAudit sensitive-key dry run"
+      puts "==================================="
+      puts "Candidate pattern: #{pattern ? Regexp.new(pattern, Regexp::IGNORECASE).inspect : '(none — reporting current config)'}"
+      puts ""
+      puts report
+      puts ""
+      puts "Nothing was written. Rows are append-only; a rule you enable applies only to future writes."
+    end
+  end
+
   desc "Export audit logs for a specific actor (GDPR right to access)"
   task :export_actor, [:actor_gid, :output] => :environment do |_t, args|
     raise "actor_gid is required" unless args[:actor_gid].present?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: standard_audit
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Jaryl Sim
@@ -99,6 +99,8 @@ files:
 - config/routes.rb
 - lib/generators/standard_audit/add_checksums/add_checksums_generator.rb
 - lib/generators/standard_audit/add_checksums/templates/add_checksum_to_audit_logs.rb.erb
+- lib/generators/standard_audit/add_previous_checksum/add_previous_checksum_generator.rb
+- lib/generators/standard_audit/add_previous_checksum/templates/add_previous_checksum_to_audit_logs.rb.erb
 - lib/generators/standard_audit/install/install_generator.rb
 - lib/generators/standard_audit/install/templates/create_audit_logs.rb.erb
 - lib/generators/standard_audit/install/templates/initializer.rb.erb
@@ -109,7 +111,10 @@ files:
 - lib/standard_audit/configuration.rb
 - lib/standard_audit/engine.rb
 - lib/standard_audit/event_subscriber.rb
+- lib/standard_audit/metadata_filter.rb
+- lib/standard_audit/reference_preloading.rb
 - lib/standard_audit/rspec.rb
+- lib/standard_audit/sensitive_keys_dry_run.rb
 - lib/standard_audit/subscriber.rb
 - lib/standard_audit/version.rb
 - lib/tasks/standard_audit_tasks.rake