pumice 0.7.1

data/lib/pumice/analyzer.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Pumice
+  class Analyzer
+    def initialize(limit: 20, schema: 'public', tables: nil)
+      @limit = limit
+      @schema = schema
+      @tables = Array(tables || Pumice.config.sensitive_tables).map(&:to_s)
+    end
+
+    def table_sizes
+      @sizes ||= fetch_table_sizes
+    end
+
+    def total_bytes
+      @total ||= table_sizes.sum { |s| s.bytes.to_i }
+    end
+
+    def row_counts
+      @row_counts ||= fetch_row_counts
+    end
+
+    private
+
+    TableSize = Struct.new(:name, :size, :bytes, keyword_init: true)
+    RowCount  = Struct.new(:table, :count, keyword_init: true)
+
+    def fetch_table_sizes
+      conn = ActiveRecord::Base.connection
+      quoted_schema = conn.quote(@schema)
+
+      sql = <<-SQL
+        SELECT
+          tablename,
+          pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS size,
+          pg_total_relation_size(schemaname||'.'||tablename) AS bytes
+        FROM pg_tables
+        WHERE schemaname = #{quoted_schema}
+        ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC
+        LIMIT #{@limit.to_i};
+      SQL
+
+      conn.execute(sql).map do |row|
+        TableSize.new(
+          name: row['tablename'],
+          size: row['size'],
+          bytes: row['bytes'].to_i
+        )
+      end
+    end
+
+    def fetch_row_counts
+      @tables.filter_map do |table|
+        count = fetch_row_count(table)
+        RowCount.new(table: table, count: count) if count
+      end
+    end
+
+    def fetch_row_count(table_name)
+      conn = ActiveRecord::Base.connection
+      quoted_table = conn.quote_table_name(table_name)
+      conn.execute("SELECT COUNT(*) FROM #{quoted_table}").first['count'].to_i
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/pumice/configuration.rb

@@ -0,0 +1,330 @@
+# frozen_string_literal: true
+
+require_relative 'soft_scrubbing/policy'
+
+module Pumice
+  class Configuration
+    attr_accessor :verbose, :strict, :continue_on_error,
+                  :allow_keep_undefined_columns, :sensitive_tables, :sensitive_email_domains,
+                  :sensitive_email_model, :sensitive_email_column, :default_verification,
+                  # Validator configuration
+                  :sensitive_token_columns,      # Token columns to verify are cleared (e.g., Devise tokens)
+                  :sensitive_external_id_columns, # External ID columns to verify are cleared
+                  # Safe scrub configuration
+                  :source_database_url,   # Source database (read-only, never modified)
+                  :target_database_url,   # Target database (scrubbed copy)
+                  :export_path,           # Optional: path to export the scrubbed dump
+                  :export_format,         # Export format: :custom (pg_dump -Fc) or :plain (SQL)
+                  :require_readonly_source, # Enforce read-only source credentials (default: false, warns only)
+                  # Pruning configuration
+                  :pruning               # Delete old records before sanitization (see pruning_config)
+
+    attr_writer :soft_scrubbing
+
+    # Default verification policy for bulk operations.
+    # Used when `verify_all` is called without a block.
+    # Receives (model_class, bulk_operation) and returns a verification proc.
+    # The bulk_operation hash contains :type (:truncate, :delete, :destroy) and :scope (optional block).
+    DEFAULT_VERIFICATION_POLICY = lambda do |_model_class, bulk_operation|
+      case bulk_operation[:type]
+      when :truncate
+        -> { count.zero? }
+      when :delete, :destroy
+        if bulk_operation[:scope]
+          # Re-run the scope and check .none?
+          bulk_operation[:scope]
+        else
+          -> { count.zero? }
+        end
+      end
+    end
+
+    def initialize
+      @verbose = false
+      @strict = true
+      @continue_on_error = false
+      @soft_scrubbing = false  # Disabled by default; set to hash to enable
+      @allow_keep_undefined_columns = true
+      @sensitive_tables = []
+      @sensitive_email_domains = []
+      @sensitive_email_model = 'User'
+      @sensitive_email_column = 'email'
+      @default_verification = DEFAULT_VERIFICATION_POLICY
+      # Validator defaults (Devise-compatible)
+      @sensitive_token_columns = %w[reset_password_token confirmation_token]
+      @sensitive_external_id_columns = []
+      # Safe scrub defaults
+      @source_database_url = nil
+      @target_database_url = nil
+      @export_path = nil
+      @export_format = :custom
+      @require_readonly_source = false  # Warn by default, set true to enforce
+      # Pruning defaults
+      @pruning = false  # Disabled by default; set to hash to enable
+    end
+
+    # Returns true if soft_scrubbing config is set
+    def soft_scrubbing_configured?
+      @soft_scrubbing.is_a?(Hash)
+    end
+
+    # Returns normalized soft_scrubbing configuration
+    # Overrides attr_writer getter to return normalized config
+    def soft_scrubbing
+      return nil unless soft_scrubbing_configured?
+
+      {
+        context: @soft_scrubbing.fetch(:context, nil),
+        policy: resolve_soft_scrubbing_policy
+      }
+    end
+
+    private
+
+    # Resolves the policy from if:/unless: options
+    # if: and unless: are mutually exclusive (if: takes precedence)
+    def resolve_soft_scrubbing_policy
+      if_condition = @soft_scrubbing[:if]
+      unless_condition = @soft_scrubbing[:unless]
+
+      if if_condition
+        if_condition
+      elsif unless_condition
+        # Invert the unless condition
+        ->(record, viewer) { !unless_condition.call(record, viewer) }
+      else
+        # Default: always scrub
+        ->(_record, _viewer) { true }
+      end
+    end
+
+    public
+
+    # Returns true if pruning config is set (not checking ENV)
+    def pruning_configured?
+      @pruning.is_a?(Hash)
+    end
+
+    # Returns normalized pruning configuration
+    # Overrides attr_accessor getter to return normalized config
+    def pruning
+      return nil unless Pumice.pruning_enabled?
+
+      validate_pruning_config!
+
+      {
+        older_than: @pruning[:older_than],
+        newer_than: @pruning[:newer_than],
+        column: @pruning.fetch(:column, :created_at).to_sym,
+        only: @pruning.fetch(:only, []).map(&:to_s),
+        except: @pruning.fetch(:except, []).map(&:to_s),
+        analyzer: normalize_analyzer_config(@pruning.fetch(:analyzer, {}))
+      }
+    end
+
+    private
+
+    def validate_pruning_config!
+      has_older = @pruning.key?(:older_than)
+      has_newer = @pruning.key?(:newer_than)
+
+      if has_older && has_newer
+        raise ArgumentError,
+          "Pruning config cannot specify both older_than and newer_than. " \
+          "Use one or the other."
+      end
+
+      unless has_older || has_newer
+        raise ArgumentError,
+          "Pruning config requires either older_than: or newer_than: to specify " \
+          "which records to prune."
+      end
+    end
+
+    def normalize_analyzer_config(analyzer_config)
+      {
+        table_patterns: Array(analyzer_config.fetch(:table_patterns, [])).map(&:to_s),
+        min_table_size: analyzer_config.fetch(:min_table_size, 10_000_000),  # 10 MB
+        min_row_count: analyzer_config.fetch(:min_row_count, 1000)
+      }
+    end
+
+    def database_url_from_rails_config
+      config_hash = ActiveRecord::Base.connection_db_config.configuration_hash
+
+      # Staging/production: config already has a url key
+      return config_hash[:url] if config_hash[:url].present?
+
+      # Development/test: build from components
+      build_database_url(config_hash)
+    end
+
+    def build_database_url(config_hash)
+      return nil unless config_hash[:adapter] == 'postgresql'
+
+      host     = config_hash[:host] || 'localhost'
+      port     = config_hash[:port] || 5432
+      database = config_hash[:database]
+      username = config_hash[:username]
+      password = config_hash[:password]
+
+      return nil if database.blank?
+
+      userinfo = if username.present? && password.present?
+                   "#{URI.encode_www_form_component(username)}:#{URI.encode_www_form_component(password)}@"
+                 elsif username.present?
+                   "#{URI.encode_www_form_component(username)}@"
+                 else
+                   ''
+                 end
+
+      "postgresql://#{userinfo}#{host}:#{port}/#{database}"
+    end
+
+    public
+
+    # Resolves source_database_url, handling the :auto sentinel.
+    # Returns a concrete URL string or nil.
+    def resolved_source_database_url
+      case @source_database_url
+      when :auto then database_url_from_rails_config
+      when String then @source_database_url
+      end
+    end
+
+    def sensitive_tables=(value)
+      @sensitive_tables = normalize_collection(value)
+    end
+
+    def sensitive_email_domains=(value)
+      @sensitive_email_domains = normalize_collection(value)
+    end
+
+    def add_sensitive_tables(value)
+      @sensitive_tables |= normalize_collection(value)
+    end
+
+    def add_sensitive_email_domains(value)
+      @sensitive_email_domains |= normalize_collection(value)
+    end
+
+    private
+
+    def normalize_collection(value)
+      Array(value).flatten.compact.map(&:to_s)
+    end
+  end
+
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  def self.configure
+    yield(config)
+    Pumice::SoftScrubbing.init! if config.soft_scrubbing_configured?
+  end
+
+  def self.dry_run?
+    ENV['DRY_RUN'] == 'true'
+  end
+
+  def self.verbose?
+    config.verbose
+  end
+
+  def self.strict?
+    config.strict
+  end
+
+  def self.soft_scrubbing?
+    config.soft_scrubbing_configured?
+  end
+
+  def self.allow_keep_undefined_columns?
+    config.allow_keep_undefined_columns
+  end
+
+  def self.soft_scrubbing_context=(context)
+    SoftScrubbing::Policy.context = context
+  end
+
+  def self.soft_scrubbing_context
+    SoftScrubbing::Policy.current
+  end
+
+  def self.with_soft_scrubbing_context(context, &block)
+    SoftScrubbing::Policy.with_context(context, &block)
+  end
+
+  def self.soft_scrubbing_enabled_for?(record)
+    SoftScrubbing::Policy.enabled_for?(record)
+  end
+
+  # Returns true if pruning is configured and not disabled by ENV
+  # Set PRUNE=false to disable pruning without changing config
+  def self.pruning_enabled?
+    return false if ENV['PRUNE'] == 'false'
+
+    config.pruning_configured?
+  end
+
+  # Returns true if the given table should be pruned
+  def self.prune_table?(table_name)
+    return false unless pruning_enabled?
+
+    pruning_config = config.pruning
+    table = table_name.to_s
+
+    if pruning_config[:only].present?
+      pruning_config[:only].include?(table)
+    elsif pruning_config[:except].present?
+      !pruning_config[:except].include?(table)
+    else
+      true # Prune all tables if no filter specified
+    end
+  end
+
+  def self.sanitizer_for(model_class)
+    @sanitizer_map ||= {}
+    @sanitizer_map[model_class] ||= sanitizers.find do |s|
+      s.model_class == model_class
+    rescue StandardError
+      nil
+    end
+
+    @sanitizer_map[model_class] || Pumice::EmptySanitizer
+  end
+
+  def self.lint!
+    issues = []
+
+    sanitizers.each do |sanitizer|
+      issues.concat(sanitizer.lint!)
+    end
+
+    if issues.any?
+      puts "\n🔍 Pumice Lint Errors:\n"
+      issues.each { |issue| puts "  ❌ #{issue}" }
+      puts "\n"
+    else
+      puts "\n✅ Pumice: All sanitizers have complete coverage\n"
+    end
+
+    issues
+  end
+
+  def self.sanitizers
+    @sanitizers ||= []
+  end
+
+  def self.register(sanitizer)
+    sanitizers << sanitizer unless sanitizers.include?(sanitizer)
+  end
+
+  def self.reset!
+    @sanitizers = []
+    @sanitizer_map = {}
+    @config = nil
+    SoftScrubbing::Policy.reset!
+  end
+end

data/lib/pumice/dsl.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+module Pumice
+  # DSL methods for defining sanitizer behavior.
+  # Extended by Sanitizer subclasses to provide scrub/keep declarations.
+  module DSL
+    PROTECTED_COLUMNS = %w[id created_at updated_at].freeze
+
+    # Explicitly declare the model this sanitizer handles.
+    # Uses pluralized form (like has_many):
+    #   sanitizes :users                       # infers User
+    #   sanitizes :admin_users, class_name: 'Admin::User'  # namespaced model
+    #   sanitizes :users, class_name: User     # explicit constant
+    def sanitizes(model_name, class_name: model_name.to_s.classify)
+      @model_class = if class_name.is_a?(String)
+                       class_name.constantize
+                     else
+                       class_name
+                     end
+    end
+
+    def model_class
+      @model_class ||= infer_model_class
+    end
+
+    # Override the auto-derived friendly name for rake tasks
+    # Example: friendly_name 'legacy_users' in LegacyUserDataSanitizer
+    def friendly_name(name = nil)
+      if name
+        @friendly_name = name.to_s
+      else
+        @friendly_name || infer_friendly_name
+      end
+    end
+
+    # Define a scrubbing rule for a column
+    def scrub(name, &block)
+      @scrubbed ||= {}
+      @scrubbed[name] = block
+    end
+
+    # Mark columns as safe to keep unchanged (not PII)
+    def keep(*names)
+      @kept ||= []
+      @kept.concat(names.map(&:to_sym))
+    end
+
+    # UNSAFE: Keep all columns not explicitly declared via scrub or keep.
+    # Bypasses PII review - use only for development/testing.
+    # Disable with: Pumice.configure { |c| c.allow_keep_undefined_columns = false }
+    def keep_undefined_columns!
+      unless Pumice.allow_keep_undefined_columns?
+        raise "keep_undefined_columns! is disabled. " \
+              "This method bypasses PII review and should not be used in production."
+      end
+
+      @kept ||= []
+      @kept.concat(undefined_columns.map(&:to_sym))
+    end
+
+    # Prune Operation
+    # Removes matching records BEFORE record-by-record scrubbing.
+    # Use when you want to delete old/irrelevant records AND scrub the survivors.
+    #
+    # Unlike bulk operations (truncate!, delete_all, destroy_all) which are terminal,
+    # prune is a pre-step: it deletes matching records, then scrubbing continues
+    # on the remaining records.
+    #
+    # Examples:
+    #   prune { where(created_at: ..1.year.ago) }  # Delete old, scrub the rest
+    #   prune { where(status: 'archived') }        # Delete archived, scrub active
+    def prune(&scope)
+      raise ArgumentError, 'prune requires a block' unless scope
+
+      @prune_operation = { scope: scope }
+    end
+
+    # Convenience: prune records older than the given age.
+    # Accepts a duration (1.year), DateTime, or date string ("2024-01-01").
+    #
+    # Examples:
+    #   prune_older_than 1.year
+    #   prune_older_than 90.days
+    #   prune_older_than DateTime.new(2024, 1, 1)
+    #   prune_older_than "2024-01-01"
+    def prune_older_than(age, column: :created_at)
+      cutoff = resolve_prune_cutoff(age)
+      prune { where(column => ...cutoff) }
+    end
+
+    # Convenience: prune records newer than the given age.
+    # Accepts a duration (1.year), DateTime, or date string ("2024-01-01").
+    #
+    # Examples:
+    #   prune_newer_than 1.year
+    #   prune_newer_than 30.days
+    #   prune_newer_than "2025-06-01"
+    def prune_newer_than(age, column: :created_at)
+      cutoff = resolve_prune_cutoff(age)
+      prune { where(column => cutoff..) }
+    end
+
+    def prune_operation
+      @prune_operation
+    end
+
+    # Bulk Operations (Terminal)
+    # These replace record-by-record sanitization with fast bulk SQL operations.
+    # No scrubbing runs after a bulk operation.
+    # Use for audit logs, sessions, and other high-volume tables.
+
+    # TRUNCATE TABLE - fastest, resets auto-increment, no conditions
+    # Examples:
+    #   truncate!
+    #   truncate!(verify: true)  # verifies count.zero? after truncation
+    def truncate!(verify: false)
+      @bulk_operation = { type: :truncate }
+      self.verify_all if verify
+    end
+
+    # DELETE with optional scope - fast, no callbacks/associations
+    # Examples:
+    #   delete_all                                    # deletes all records
+    #   delete_all { where(item_type: 'User') }       # deletes matching records
+    #   delete_all(verify: true) { where(...) }       # verifies scope.none? after deletion
+    def delete_all(verify: false, &scope)
+      @bulk_operation = { type: :delete, scope: scope }
+      self.verify_all if verify
+    end
+
+    # DESTROY with optional scope - runs callbacks, handles associations
+    # Examples:
+    #   destroy_all                                   # destroys all records
+    #   destroy_all { where(attachable_id: nil) }     # destroys orphaned records
+    #   destroy_all(verify: true) { where(...) }      # verifies scope.none? after destruction
+    def destroy_all(verify: false, &scope)
+      @bulk_operation = { type: :destroy, scope: scope }
+      self.verify_all if verify
+    end
+
+    def bulk_operation
+      @bulk_operation
+    end
+
+    # Verification
+    # Define post-sanitization checks to confirm the operation succeeded.
+
+    # Verify after all records are processed (bulk or record-by-record)
+    # Block executes in model scope and should return truthy for success.
+    # Examples:
+    #   verify_all                                          # uses default for bulk ops
+    #   verify_all { where(item_type: SENSITIVE_TYPES).none? }
+    #   verify_all "No sensitive data should remain" do
+    #     where(pii_column: true).count.zero?
+    #   end
+    def verify_all(message = nil, &block)
+      @verification = if block
+                        { message: message, block: block }
+                      else
+                        { message: message, use_default: true }
+                      end
+    end
+
+    # Verify each record after sanitization (record-by-record only)
+    # Block receives the sanitized record and should return truthy for success.
+    # Examples:
+    #   verify_each { |record| !record.email.include?('@gmail.com') }
+    #   verify_each "Record should not contain real email" do |record|
+    #     record.email.end_with?('@example.com')
+    #   end
+    def verify_each(message = nil, &block)
+      raise ArgumentError, 'verify_each requires a block' unless block
+
+      @record_verification = { message: message, block: block }
+    end
+
+    def verification
+      @verification
+    end
+
+    def record_verification
+      @record_verification
+    end
+
+    def scrubbed
+      @scrubbed ||= {}
+    end
+
+    def kept
+      @kept ||= []
+    end
+
+    def scrubbed_columns
+      scrubbed.keys.map(&:to_s)
+    end
+
+    def scrubbed_column?(name)
+      scrubbed_columns.include?(name.to_s)
+    end
+
+    def kept_columns
+      kept.map(&:to_s)
+    end
+
+    def defined_columns
+      scrubbed_columns + kept_columns
+    end
+
+    def undefined_columns
+      model_class.column_names - defined_columns - PROTECTED_COLUMNS
+    end
+
+    def stale_columns
+      defined_columns - model_class.column_names
+    end
+
+    def lint!
+      issues = []
+
+      # Bulk operations are terminal — no scrubbing happens, so column coverage is irrelevant
+      if undefined_columns.any? && !bulk_operation
+        issues << "#{name} (#{model_class.name}) has undefined columns: #{undefined_columns.join(', ')}"
+      end
+
+      if stale_columns.any?
+        issues << "#{name} (#{model_class.name}) has stale columns (removed from model): #{stale_columns.join(', ')}"
+      end
+
+      if bulk_operation && (scrubbed.any? || kept.any?)
+        ignored = (scrubbed_columns + kept_columns).join(', ')
+        issues << "#{name} uses a terminal bulk operation (#{bulk_operation[:type]}) but also declares scrub/keep columns (#{ignored}). These will be ignored."
+      end
+
+      issues
+    rescue NameError, RuntimeError => e
+      ["#{name} references a model that doesn't exist: #{e.message}"]
+    end
+
+    private
+
+    def infer_model_class
+      # e.g. UserSanitizer -> User, StudentSanitizer -> Student
+      model_name = name.delete_suffix('Sanitizer')
+      model_name.constantize
+    rescue NameError
+      raise "Could not infer model for #{name}. Use `sanitizes :model_names` to specify explicitly."
+    end
+
+    def infer_friendly_name
+      name.delete_suffix('Sanitizer').underscore.pluralize
+    end
+
+    def resolve_prune_cutoff(age)
+      case age
+      when ActiveSupport::Duration
+        age.ago
+      when DateTime, Time, Date
+        age
+      when String
+        DateTime.parse(age)
+      else
+        raise ArgumentError,
+          "prune cutoff must be a Duration (1.year), DateTime, or date string, got #{age.class}"
+      end
+    end
+  end
+end