pumice 0.7.1

data/lib/pumice/sanitizer.rb

@@ -0,0 +1,336 @@
+# frozen_string_literal: true
+
+module Pumice
+  class UndefinedAttributeError < StandardError; end
+  class VerificationError < StandardError; end
+
+  class Sanitizer
+    extend Pumice::DSL
+    include Pumice::Helpers
+
+    class << self
+      def inherited(subclass)
+        super
+        Pumice.register(subclass)
+      end
+
+      # Non-destructive sanitization - returns values without persisting
+      # sanitize(record)              → returns hash of all sanitized values
+      # sanitize(record, :attr)       → returns single sanitized value
+      def sanitize(record, attr_name = nil, raw_value: nil)
+        with_seed_for(record) do
+          instance = new(record)
+          attr_name ? instance.scrub(attr_name, raw_value) : instance.scrub_all
+        end
+      end
+
+      # Destructive scrubbing - persists to database
+      # scrub!(record)             → persists all scrubbed values
+      # scrub!(record, :attr)      → persists single scrubbed value
+      def scrub!(record, attr_name = nil)
+        result = sanitize(record, attr_name)
+        persist(record, attr_name, result)
+        result
+      end
+
+      # Batch operation - sanitize all records of this model
+      # If a bulk operation (truncate!, delete_all, destroy_all) is defined,
+      # it runs instead of record-by-record sanitization.
+      # If a prune operation is defined, matching records are deleted first,
+      # then remaining records are scrubbed one-by-one.
+      def scrub_all!
+        validate_coverage! if Pumice.strict? && !bulk_operation
+
+        logger.initialize_stats
+        logger.log_start(name)
+
+        if Pumice.dry_run? && !bulk_operation && scrubbed.any?
+          logger.log_progress("Columns: #{scrubbed_columns.join(', ')}")
+        end
+
+        count = if bulk_operation
+                  run_bulk_operation
+                else
+                  pruned = prune_operation ? run_prune : 0
+                  scrubbed_count = run_record_sanitization
+                  pruned + scrubbed_count
+                end
+
+        run_verification unless Pumice.dry_run?
+
+        logger.log_complete(name, count)
+      rescue NameError => e
+        logger.log_progress("Skipping #{name} (model not found)")
+      end
+
+      private
+
+      def run_bulk_operation
+        op = bulk_operation
+
+        if Pumice.dry_run?
+          count = if op[:scope]
+                    model_class.instance_exec(&op[:scope]).count
+                  else
+                    model_class.count
+                  end
+          logger.log_progress("[DRY RUN] Would #{op[:type]} #{count} records")
+          return count
+        end
+
+        case op[:type]
+        when :truncate
+          run_truncate
+        when :delete
+          run_delete(op[:scope])
+        when :destroy
+          run_destroy(op[:scope])
+        end
+      end
+
+      def run_truncate
+        table = model_class.table_name
+        count = model_class.count
+        ActiveRecord::Base.connection.truncate(table)
+        logger.log_progress("Truncated #{table}")
+        count
+      end
+
+      def run_delete(scope_block)
+        scope = scope_block ? model_class.instance_exec(&scope_block) : model_class.all
+        count = scope.delete_all
+        logger.log_progress("Deleted #{count} records")
+        count
+      end
+
+      def run_destroy(scope_block)
+        scope = scope_block ? model_class.instance_exec(&scope_block) : model_class.all
+        count = scope.destroy_all.count
+        logger.log_progress("Destroyed #{count} records")
+        count
+      end
+
+      def run_prune
+        scope_block = prune_operation[:scope]
+        scope = model_class.instance_exec(&scope_block)
+
+        if Pumice.dry_run?
+          count = scope.count
+          logger.log_progress("[DRY RUN] Would prune #{count} records")
+          return count
+        end
+
+        count = scope.delete_all
+        logger.log_progress("Pruned #{count} records")
+        count
+      end
+
+      def run_record_sanitization
+        total = model_class.count
+        progress = Pumice::Progress.new(title: model_class.name, total: total)
+        count = 0
+        model_class.find_each do |record|
+          scrub!(record)
+          run_record_verification(record) unless Pumice.dry_run?
+          count += 1
+          progress.increment
+        rescue => e
+          logger.log_error(name, e)
+          raise unless Pumice.config.continue_on_error
+        end
+        progress.finish
+        count
+      end
+
+      def run_record_verification(record)
+        return unless record_verification
+
+        block = record_verification[:block]
+        message = record_verification[:message]
+
+        # Reload record to get persisted values
+        record.reload
+
+        result = block.call(record)
+
+        unless result
+          error_message = message || "Record verification failed for #{name} (ID: #{record.id})"
+          logger.log_progress("VERIFICATION FAILED: #{error_message}")
+          raise VerificationError, error_message
+        end
+      end
+
+      def run_verification
+        return unless verification
+
+        if verification[:block]
+          execute_verification(verification[:block], verification[:message])
+        elsif verification[:use_default]
+          execute_default_verification(verification[:message])
+        end
+      end
+
+      def execute_verification(block, message)
+        result = model_class.instance_exec(&block)
+
+        unless result
+          error_message = message || "Verification failed for #{name}"
+          logger.log_progress("VERIFICATION FAILED: #{error_message}")
+          raise VerificationError, error_message
+        end
+
+        logger.log_progress("Verification passed")
+      end
+
+      def execute_default_verification(message)
+        unless bulk_operation
+          raise ArgumentError,
+            "#{name}: verify_all without a block requires a bulk operation (truncate!, delete_all, destroy_all)"
+        end
+
+        default_block = Pumice.config.default_verification.call(model_class, bulk_operation)
+
+        # For scoped operations, the default policy returns the scope block.
+        # We execute it and check .none? to verify records are gone.
+        scope_or_result = model_class.instance_exec(&default_block)
+
+        # If the result is an ActiveRecord relation, check .none?
+        # Otherwise treat it as a boolean result
+        result = if scope_or_result.respond_to?(:none?)
+                   scope_or_result.none?
+                 else
+                   scope_or_result
+                 end
+
+        unless result
+          error_message = message || "Verification failed for #{name}"
+          logger.log_progress("VERIFICATION FAILED: #{error_message}")
+          raise VerificationError, error_message
+        end
+
+        logger.log_progress("Verification passed")
+      end
+
+      def persist(record, attr_name, result)
+        if attr_name
+          persist_attribute(record, attr_name, result)
+        else
+          persist_record(record, result)
+        end
+      end
+
+      def persist_record(record, data)
+        if Pumice.dry_run?
+          details = if Pumice.verbose?
+                      changes = data.map { |attr, new_val|
+                        "#{attr} (#{record.read_attribute(attr).inspect} → #{new_val.inspect})"
+                      }.join(', ')
+                      "ID #{record.id}: #{changes}"
+                    else
+                      "ID #{record.id} — #{data.keys.join(', ')}"
+                    end
+          logger.log_record(:would_sanitize, details)
+        else
+          record.update_columns(data)
+          logger.log_record(:sanitized, "ID #{record.id}")
+        end
+      end
+
+      def persist_attribute(record, attr_name, value)
+        if Pumice.dry_run?
+          original = record.read_attribute(attr_name)
+          logger.log_record(:would_sanitize, "ID #{record.id}.#{attr_name} (#{original.inspect} → #{value.inspect})")
+        else
+          record.update_column(attr_name, value)
+          logger.log_record(:sanitized, "ID #{record.id}.#{attr_name}")
+        end
+      end
+
+      # Seeds Faker per-record for deterministic output. Thread-safe because
+      # Faker 3.x stores Config.random in Thread.current (gemspec requires >= 3.0).
+      def with_seed_for(record)
+        previous = Faker::Config.random
+        Faker::Config.random = Random.new(record&.id || record.object_id)
+        yield
+      ensure
+        Faker::Config.random = previous
+      end
+
+      def validate_coverage!
+        return if undefined_columns.empty?
+
+        raise UndefinedAttributeError,
+          "#{name} is missing definitions for: #{undefined_columns.join(', ')}. " \
+          "Add scrub(:column) { value } for each, or set Pumice.configure { |c| c.strict = false }"
+      end
+
+      def logger
+        Pumice::Logger
+      end
+    end
+
+    attr_reader :record
+
+    def initialize(record)
+      @record = record
+    end
+
+    def scrub(attr_name, raw_value = nil)
+      raw_value ||= record.send(attr_name)
+      block = self.class.scrubbed[attr_name.to_sym]
+      return raw_value unless block
+
+      instance_exec(raw_value, &block)
+    end
+
+    def scrub_all
+      self.class.scrubbed.keys.each_with_object({}) do |attr_name, hash|
+        hash[attr_name] = scrub(attr_name)
+      end
+    end
+
+    # Read an original database value, bypassing scrubbing.
+    #
+    #   scrub(:email) { "#{raw(:first_name)}.#{raw(:last_name)}@example.test" }
+    def raw(attr_name)
+      record.public_send(attr_name)
+    end
+
+    # Provides a clean DSL for referencing attributes within scrub blocks:
+    # - Bare attribute names return scrubbed values: `name` → scrub(:name)
+    # - raw_* methods return original database values: `raw_name` → raw(:name)
+    def method_missing(method_name, *args, &block)
+      if raw_attribute_method?(method_name)
+        return raw(extract_raw_attribute_name(method_name))
+      end
+
+      if self.class.scrubbed_column?(method_name)
+        return scrub(method_name)
+      end
+
+      if record.respond_to?(method_name)
+        return record.public_send(method_name, *args, &block)
+      end
+
+      super
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      raw_attribute_method?(method_name) ||
+        self.class.scrubbed_column?(method_name) ||
+        record.respond_to?(method_name, include_private) ||
+        super
+    end
+
+    private
+
+    def raw_attribute_method?(method_name)
+      method_name.to_s.start_with?('raw_') &&
+        record.respond_to?(extract_raw_attribute_name(method_name))
+    end
+
+    def extract_raw_attribute_name(method_name)
+      method_name.to_s.delete_prefix('raw_').to_sym
+    end
+  end
+end

data/lib/pumice/soft_scrubbing/policy.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Pumice
+  module SoftScrubbing
+    # Policy determines when soft scrubbing applies to a record.
+    #
+    # TRANSITIONAL: This module currently uses a binary on/off policy check.
+    # Future versions will support:
+    #   - Per-attribute policies (SSN vs email have different rules)
+    #   - Role-graduated scrubbing (admin/manager/user see different levels)
+    #   - Viewer context passed to scrub blocks for conditional masking
+    #
+    # See lib/pumice/README.md for the roadmap.
+    module Policy
+      extend self
+
+      THREAD_KEY = :pumice_soft_scrub_context
+      CONTEXT_SET_KEY = :pumice_soft_scrub_context_set
+
+      def context=(context)
+        Thread.current[THREAD_KEY] = context
+        Thread.current[CONTEXT_SET_KEY] = true
+      end
+
+      def current(record = nil)
+        resolve(Thread.current[THREAD_KEY], record)
+      end
+
+      # Returns true if context has been explicitly set for this request/thread.
+      # Used to distinguish "no logged-in user" from "not in a request context".
+      def context_set?
+        Thread.current[CONTEXT_SET_KEY] == true
+      end
+
+      def with_context(context)
+        previous = Thread.current[THREAD_KEY]
+        previous_set = Thread.current[CONTEXT_SET_KEY]
+        self.context = context
+        yield
+      ensure
+        Thread.current[THREAD_KEY] = previous
+        Thread.current[CONTEXT_SET_KEY] = previous_set
+      end
+
+      # Temporarily disable soft scrubbing for a block.
+      # Used during authentication/session management to skip policy checks.
+      def without_context
+        previous = Thread.current[THREAD_KEY]
+        previous_set = Thread.current[CONTEXT_SET_KEY]
+        Thread.current[THREAD_KEY] = nil
+        Thread.current[CONTEXT_SET_KEY] = nil
+        yield
+      ensure
+        Thread.current[THREAD_KEY] = previous
+        Thread.current[CONTEXT_SET_KEY] = previous_set
+      end
+
+      def enabled_for?(record)
+        return false unless Pumice.soft_scrubbing?
+        return false unless context_set?  # Skip during boot/initialization
+
+        viewer = current(record)
+        Pumice.config.soft_scrubbing[:policy].call(record, viewer)
+      end
+
+      def reset!
+        Thread.current[THREAD_KEY] = nil
+        Thread.current[CONTEXT_SET_KEY] = nil
+      end
+
+      private
+
+      def resolve(raw_context, record)
+        config_context = Pumice.soft_scrubbing? ? Pumice.config.soft_scrubbing[:context] : nil
+        value = raw_context.nil? ? config_context : raw_context
+
+        case value
+        when Proc
+          value.arity.zero? ? value.call : value.call(record)
+        when Symbol, String
+          resolve_symbol(value.to_sym, record)
+        when nil
+          nil
+        else
+          value
+        end
+      end
+
+      def resolve_symbol(method_name, record)
+        if record&.respond_to?(method_name)
+          record.public_send(method_name)
+        elsif Pumice.respond_to?(method_name)
+          Pumice.public_send(method_name)
+        elsif defined?(Current) && Current.respond_to?(method_name)
+          Current.public_send(method_name)
+        elsif Thread.current.key?(method_name)
+          Thread.current[method_name]
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/pumice/soft_scrubbing.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Pumice
+  module SoftScrubbing
+    extend ActiveSupport::Concern
+
+    RECURSION_GUARD_KEY = :pumice_soft_scrub_in_progress
+
+    # System attributes that should never be scrubbed (needed for Rails internals)
+    SYSTEM_ATTRIBUTES = %w[id created_at updated_at].freeze
+
+    module AttributeInterceptor
+      def _read_attribute(attr_name)
+        # Prevent infinite recursion - if we're already inside the interceptor, bail out
+        return super if Thread.current[Pumice::SoftScrubbing::RECURSION_GUARD_KEY]
+
+        # Quick check: skip if soft_scrubbing not configured
+        return super unless Pumice.soft_scrubbing?
+
+        # Skip system attributes needed for Rails/Devise internals (session serialization, etc.)
+        return super if Pumice::SoftScrubbing::SYSTEM_ATTRIBUTES.include?(attr_name.to_s)
+
+        begin
+          Thread.current[Pumice::SoftScrubbing::RECURSION_GUARD_KEY] = true
+
+          unless Pumice.soft_scrubbing_enabled_for?(self)
+            return super
+          end
+
+          sanitizer = Pumice.sanitizer_for(self.class)
+          return super unless sanitizer.scrubbed_column?(attr_name)
+
+          soft_scrubbed_value(attr_name, sanitizer)
+        ensure
+          Thread.current[Pumice::SoftScrubbing::RECURSION_GUARD_KEY] = false
+        end
+      end
+
+      def reload(*)
+        @_soft_scrubbed_cache = nil
+        super
+      end
+
+      def write_attribute(attr_name, value)
+        @_soft_scrubbed_cache&.delete(attr_name.to_s)
+        super
+      end
+
+      private
+
+      def soft_scrubbed_value(attr_name, sanitizer)
+        @_soft_scrubbed_cache ||= {}
+        @_soft_scrubbed_cache[attr_name] ||= begin
+          raw_value = @attributes.fetch_value(attr_name.to_s)
+          sanitizer.sanitize(self, attr_name, raw_value: raw_value)
+        end
+      end
+    end
+
+    # Call this once at boot to enable the feature
+    def self.init!
+      return if @initialized
+
+      ActiveRecord::Base.prepend(AttributeInterceptor)
+      @initialized = true
+
+      # Eager-load sanitizers using Rails' reloader
+      Rails.application.reloader.to_prepare do
+        Pumice::SoftScrubbing.eager_load_sanitizers!
+      end
+
+      Rails.logger.info("[Pumice] Soft scrubbing initialized")
+    end
+
+    def self.initialized?
+      @initialized == true
+    end
+
+    # For debugging: force re-initialization (use in console only)
+    def self.reinit!
+      @initialized = false
+      init!
+    end
+
+    def self.eager_load_sanitizers!
+      sanitizer_paths = Rails.root.join('app/sanitizers')
+      return unless sanitizer_paths.exist?
+
+      Dir[sanitizer_paths.join('**/*.rb')].sort.each do |file|
+        relative_path = Pathname.new(file).relative_path_from(sanitizer_paths)
+        const_name = relative_path.to_s.delete_suffix('.rb').camelize
+
+        begin
+          const_name.constantize
+        rescue NameError => e
+          Rails.logger.warn("[Pumice] Could not load sanitizer #{const_name}: #{e.message}")
+        end
+      end
+    end
+  end
+end

data/lib/pumice/validator.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Pumice
+  class Validator
+    Result = Struct.new(:errors, :checks, keyword_init: true) do
+      def passed?
+        errors.empty?
+      end
+    end
+
+    Check = Struct.new(:name, :count, :passed, keyword_init: true)
+
+    def initialize(email_domains: nil)
+      @email_domains = Array(email_domains || Pumice.config.sensitive_email_domains)
+    end
+
+    def run
+      errors = []
+      checks = []
+
+      # Check for real email domains
+      email_check = check_real_emails
+      errors.concat(email_check[:errors])
+      checks << email_check[:check]
+
+      # Check for test emails
+      checks << check_test_emails
+
+      # Check for cleared tokens
+      token_checks = check_cleared_tokens
+      errors.concat(token_checks[:errors])
+      checks.concat(token_checks[:checks])
+
+      # Check for cleared external IDs
+      external_checks = check_external_ids
+      errors.concat(external_checks[:errors])
+      checks.concat(external_checks[:checks])
+
+      Result.new(errors: errors, checks: checks)
+    end
+
+    private
+
+    def email_model
+      @email_model ||= Pumice.config.sensitive_email_model.constantize
+    rescue NameError
+      raise NameError,
+        "Pumice validator: model '#{Pumice.config.sensitive_email_model}' not found. " \
+        "Set config.sensitive_email_model to your app's user model (e.g. 'Account', 'Member')."
+    end
+
+    def email_column
+      @email_column ||= Pumice.config.sensitive_email_column
+    end
+
+    def check_real_emails
+      errors = []
+
+      @email_domains.each do |domain|
+        count = email_model.where("#{email_column} LIKE ?", "%@#{domain}").count
+        errors << "Found #{count} emails with real domain #{domain}" if count > 0
+      end
+
+      {
+        errors: errors,
+        check: Check.new(name: 'real_email_domains', count: errors.size, passed: errors.empty?)
+      }
+    end
+
+    def check_test_emails
+      count = email_model.where("#{email_column} LIKE ?", "%@example.test").count
+      Check.new(name: 'test_emails', count: count, passed: count > 0)
+    end
+
+    def check_cleared_tokens
+      errors = []
+      checks = []
+
+      token_columns = Pumice.config.sensitive_token_columns
+
+      token_columns.each do |column|
+        next unless email_model.column_names.include?(column.to_s)
+
+        count = email_model.where.not(column => nil).count
+        if count > 0
+          errors << "Found #{count} users with #{column}"
+        end
+        checks << Check.new(name: column.to_s, count: count, passed: count == 0)
+      end
+
+      { errors: errors, checks: checks }
+    end
+
+    def check_external_ids
+      errors = []
+      checks = []
+
+      external_id_columns = Pumice.config.sensitive_external_id_columns
+
+      external_id_columns.each do |column|
+        next unless email_model.column_names.include?(column.to_s)
+
+        count = email_model.where.not(column => nil).count
+        if count > 0
+          errors << "Found #{count} users with #{column} (should be cleared)"
+        end
+        checks << Check.new(name: column.to_s, count: count, passed: count == 0)
+      end
+
+      { errors: errors, checks: checks }
+    end
+  end
+end

data/lib/pumice/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Pumice
+  VERSION = "0.7.1"
+end

data/lib/pumice.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative 'pumice/version'
+
+module Pumice; end
+
+require_relative 'pumice/configuration'
+require_relative 'pumice/output'
+require_relative 'pumice/progress'
+require_relative 'pumice/helpers'
+require_relative 'pumice/logger'
+require_relative 'pumice/dsl'
+require_relative 'pumice/sanitizer'
+require_relative 'pumice/empty_sanitizer'
+require_relative 'pumice/soft_scrubbing'
+require_relative 'pumice/analyzer'
+require_relative 'pumice/validator'
+require_relative 'pumice/runner'
+require_relative 'pumice/dump_generator'
+require_relative 'pumice/pruner'
+require_relative 'pumice/pruning/analyzer'
+require_relative 'pumice/safe_scrubber'
+require_relative 'pumice/railtie' if defined?(Rails::Railtie)