lethe 0.1.0

data/lib/lethe/configuration.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Lethe
+  class Configuration
+    # Tables Rails and lethe keep for themselves. Active Storage's are left
+    # out because attachments are found through has_one_attached and
+    # has_many_attached, and declared with destroy_attachments.
+    BUILTIN_SKIP_TABLES = %w[
+      ar_internal_metadata schema_migrations lethe_erasures
+      active_storage_attachments active_storage_blobs active_storage_variant_records
+    ].freeze
+
+    # Where the definitions live, relative to the application root.
+    attr_accessor :definition_path
+
+    # Where lethe:export writes, without extension: .md and .json are added.
+    attr_accessor :export_path
+
+    # Tables to leave out of the search, besides Rails' own.
+    attr_accessor :skip_tables
+
+    # How erased attachments' files go, after the transaction commits:
+    # :later through Active Storage's purge job, or :now.
+    attr_reader :purge
+
+    def initialize
+      @definition_path = "config/lethe.rb"
+      @export_path = "docs/lethe"
+      @skip_tables = []
+      @purge = :later
+      @guess_columns = Hash.new { |hash, key| hash[key] = [] }
+    end
+
+    # More column names to treat as pointing at a subject, per subject:
+    #   config.guess_columns[User] = %w[follower_id following_id]
+    def guess_columns
+      GuessColumns.new(@guess_columns)
+    end
+
+    def purge=(value)
+      raise ArgumentError, "lethe: purge must be :later or :now" unless %i[later now].include?(value)
+
+      @purge = value
+    end
+
+    def guess_columns_for(subject_name)
+      @guess_columns.fetch(subject_name, []).map(&:to_s)
+    end
+
+    def skipped_table?(table)
+      BUILTIN_SKIP_TABLES.include?(table) || skip_tables.map(&:to_s).include?(table)
+    end
+
+    # Lets guess_columns be keyed by the class or its name.
+    class GuessColumns
+      def initialize(store)
+        @store = store
+      end
+
+      def [](subject)
+        @store[key(subject)]
+      end
+
+      def []=(subject, columns)
+        @store[key(subject)] = Array(columns).map(&:to_s)
+      end
+
+      private
+
+      def key(subject)
+        subject.is_a?(Module) ? subject.name : subject.to_s
+      end
+    end
+  end
+end

data/lib/lethe/definition.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Lethe
+  # The policies declared for one subject, such as User.
+  class Definition
+    attr_reader :subject_name, :policies, :hooks
+    attr_accessor :subject_policy
+
+    def initialize(subject_name)
+      @subject_name = subject_name
+      @policies = []
+      @hooks = []
+      @subject_policy = nil
+    end
+
+    def hooks_for(moment)
+      hooks.select { |hook| hook.moment == moment }
+    end
+
+    # The methods available inside Lethe.define.
+    class Builder
+      def initialize(definition)
+        @definition = definition
+      end
+
+      # Delete the rows. cascade: true deletes them through Active Record,
+      # so that dependent: associations, callbacks and attachments go too.
+      def destroy(*targets, reason: nil, where: nil, cascade: false)
+        add(:destroy, targets, location: caller_location, reason: reason, where: where, cascade: cascade)
+      end
+
+      # Keep the rows but overwrite columns: scrub :posts, set: { body: "" }
+      # With for:, lethe:sweep deletes the rows once that time has passed:
+      # scrub :orders, set: { phone: nil }, for: 7.years. cascade: true makes
+      # that deletion go through Active Record, as with destroy.
+      def scrub(*targets, set:, reason: nil, where: nil, cascade: false, **options)
+        add(:scrub, targets, location: caller_location, reason: reason, where: where, set: set,
+          retention: retention(options), cascade: cascade)
+      end
+
+      # Leave the rows as they are, for a reason: keep :orders, for: 7.years, reason: "..."
+      def keep(*targets, reason: nil, where: nil, cascade: false, **options)
+        add(:keep, targets, location: caller_location, reason: reason, where: where,
+          retention: retention(options), cascade: cascade)
+      end
+
+      # The column does not point at the subject after all (a wrong guess).
+      def ignore(*targets, reason: nil)
+        add(:ignore, targets, location: caller_location, reason: reason)
+      end
+
+      # Delete Active Storage attachments of the subject.
+      def destroy_attachments(*names, reason: nil)
+        raise ArgumentError, "lethe: destroy_attachments needs at least one name" if names.empty?
+
+        location = caller_location
+        names.each do |name|
+          @definition.policies << Policy.new(
+            action: :destroy_attachment, target: Target.attachment(name), reason: reason,
+            retention: nil, where: nil, set: nil, cascade: false, location: location
+          )
+        end
+      end
+
+      # How the subject's own row goes: subject soft_delete: :deleted_at, scrub: { email: nil }
+      def subject(destroy: false, soft_delete: nil, scrub: {})
+        @definition.subject_policy = SubjectPolicy.new(
+          destroy: destroy, soft_delete: soft_delete, scrub: scrub, location: caller_location
+        )
+      end
+
+      # Runs inside the erasure's transaction, before anything is deleted.
+      def before_erase(&block)
+        hook(:before, block)
+      end
+
+      # Runs inside the erasure's transaction, after everything is deleted.
+      # Anything that cannot be rolled back, such as a call to another
+      # service, belongs in a job enqueued from here.
+      def after_erase(&block)
+        hook(:after, block)
+      end
+
+      private
+
+      # for: is a Ruby keyword, so it arrives in the options hash.
+      def retention(options)
+        unknown = options.keys - [:for]
+        raise ArgumentError, "lethe: unknown keyword#{"s" if unknown.size > 1}: #{unknown.join(", ")}" if unknown.any?
+
+        options[:for]
+      end
+
+      def add(action, targets, location:, reason: nil, where: nil, set: nil, retention: nil, cascade: false)
+        raise ArgumentError, "lethe: #{action} needs at least one target" if targets.empty?
+
+        targets.each do |value|
+          @definition.policies << Policy.new(
+            action: action, target: Target.parse(value), reason: reason,
+            retention: retention, where: where, set: set, cascade: cascade ? true : false, location: location
+          )
+        end
+      end
+
+      def hook(moment, block)
+        raise ArgumentError, "lethe: #{moment}_erase needs a block" unless block
+
+        location = caller_locations(2, 1).first
+        @definition.hooks << Hook.new(moment: moment, block: block, location: "#{location.path}:#{location.lineno}")
+      end
+
+      def caller_location
+        location = caller_locations(2, 1).first
+        "#{location.path}:#{location.lineno}"
+      end
+    end
+  end
+end

data/lib/lethe/discovery.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Lethe
+  # Finds everything that points at a subject model, from the schema and the
+  # models' associations. Reads table and column definitions only, never rows.
+  class Discovery
+    def initialize(subject, models:, connection:, configuration:)
+      @subject = subject
+      @models = models
+      @connection = connection
+      @configuration = configuration
+    end
+
+    # Returns the references, one per table.column or attachment, sorted by key.
+    def call
+      found = {}
+      [foreign_keys, own_associations, other_associations, attachments, column_names].each do |references|
+        references.each do |reference|
+          found[reference.key] = found.key?(reference.key) ? found[reference.key].merge(reference) : reference
+        end
+      end
+      found.values.sort_by(&:key)
+    end
+
+    def tables
+      @tables ||= @connection.tables.reject { |table| @configuration.skipped_table?(table) }.sort
+    end
+
+    private
+
+    def subject_table
+      @subject.table_name
+    end
+
+    # Foreign key constraints that point at the subject's table.
+    def foreign_keys
+      tables.flat_map do |table|
+        @connection.foreign_keys(table)
+          .select { |key| key.to_table == subject_table }
+          .map { |key| Reference.column(table, key.column, :foreign_key) }
+      end
+    end
+
+    # The subject's has_many, has_one and has_and_belongs_to_many.
+    def own_associations
+      @subject.reflect_on_all_associations.filter_map do |reflection|
+        # Through associations read other tables; has_one_attached's own
+        # associations are reported as attachments instead.
+        next if reflection.through_reflection? || reflection.class_name == "ActiveStorage::Attachment"
+
+        case reflection.macro
+        when :has_many, :has_one
+          table = table_of(reflection)
+          next unless table
+
+          via = reflection.options[:as] ? :polymorphic : :association
+          Reference.column(table, reflection.foreign_key, via)
+        when :has_and_belongs_to_many
+          Reference.column(reflection.join_table, reflection.foreign_key, :join_table)
+        end
+      end
+    end
+
+    # belongs_to in other models that points at the subject. A polymorphic
+    # belongs_to might point at anything, so it is only a guess.
+    def other_associations
+      @models.flat_map do |model|
+        next [] if model.abstract_class? || !table_exists?(model)
+
+        model.reflect_on_all_associations(:belongs_to).filter_map do |reflection|
+          if reflection.polymorphic?
+            Reference.column(model.table_name, reflection.foreign_key, :polymorphic_guess)
+          elsif table_of(reflection) == subject_table
+            Reference.column(model.table_name, reflection.foreign_key, :association)
+          end
+        end
+      end
+    end
+
+    def attachments
+      return [] unless @subject.respond_to?(:reflect_on_all_attachments)
+
+      @subject.reflect_on_all_attachments.map { |reflection| Reference.attachment(reflection.name) }
+    end
+
+    # Columns named like user_id or *_user_id, and the ones configured.
+    def column_names
+      # An STI subclass is stored in its base class's table, and other tables
+      # point at it as user_id, not admin_id.
+      singular = @subject.base_class.model_name.singular
+      extra = (@configuration.guess_columns_for(@subject.name) + @configuration.guess_columns_for(@subject.base_class.name)).uniq
+      tables.flat_map do |table|
+        @connection.columns(table).filter_map do |column|
+          name = column.name
+          next if table == subject_table && name == @subject.primary_key
+
+          if name == "#{singular}_id" || name.end_with?("_#{singular}_id") || extra.include?(name)
+            Reference.column(table, name, :column_name)
+          end
+        end
+      end
+    end
+
+    # The table an association reads, or nil when its class cannot be loaded.
+    def table_of(reflection)
+      reflection.klass.table_name
+    rescue NameError, ArgumentError
+      nil
+    end
+
+    def table_exists?(model)
+      tables.include?(model.table_name)
+    rescue ActiveRecord::ActiveRecordError
+      false
+    end
+  end
+end

data/lib/lethe/enums.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Lethe
+  # lethe reaches rows through plain table classes, which know nothing of the
+  # application's enums: { status: "answered" } would become status = NULL
+  # and match nothing. These turn enum names into the values stored, using
+  # the application's model for the table.
+  module Enums
+    # The hash with enum names replaced by their stored values. Values that
+    # are not enum names, such as nil or a stored value, stay as they are.
+    def self.cast(model, hash)
+      return hash unless model && hash.is_a?(Hash)
+
+      enums = model.defined_enums
+      hash.to_h do |column, value|
+        mapping = enums[column.to_s]
+        [column, mapping ? cast_value(mapping, value) : value]
+      end
+    end
+
+    # The names in hash that are neither a name nor a stored value of the
+    # column's enum, as [column, value] pairs.
+    def self.unknown(model, hash)
+      return [] unless model && hash.is_a?(Hash)
+
+      hash.flat_map do |column, value|
+        mapping = model.defined_enums[column.to_s]
+        next [] unless mapping
+
+        Array(value).reject { |item| item.nil? || item.is_a?(Proc) || known?(mapping, item) }.map { |item| [column.to_s, item] }
+      end
+    end
+
+    def self.cast_value(mapping, value)
+      return value.map { |item| cast_value(mapping, item) } if value.is_a?(Array)
+      return mapping[value.to_s] if (value.is_a?(String) || value.is_a?(Symbol)) && mapping.key?(value.to_s)
+
+      value
+    end
+
+    def self.known?(mapping, item)
+      mapping.key?(item.to_s) || mapping.value?(item)
+    end
+    private_class_method :cast_value, :known?
+  end
+end

data/lib/lethe/eraser.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Lethe
+  # Carries out a Plan in one transaction. Attachments are detached inside
+  # it; their files are purged only after it commits, since files cannot be
+  # rolled back.
+  class Eraser
+    Erasure = Data.define(:subject, :id, :counts)
+
+    def initialize(plan, configuration:, ledger:)
+      @plan = plan
+      @record = plan.record
+      @definition = plan.definition
+      @configuration = configuration
+      @ledger = ledger
+    end
+
+    def call
+      blobs = []
+      counts = {}
+      ActiveRecord::Base.transaction do
+        @definition.hooks_for(:before).each { |hook| hook.block.call(@record) }
+        # { "posts.user_id" => { "destroy" => 1, "scrub" => 4 } }: a table can have a policy per condition.
+        @plan.each do |step|
+          rows = perform(step, blobs)
+          (counts[step.key] ||= Hash.new(0))[step.kind.to_s] += rows
+        end
+        @definition.hooks_for(:after).each { |hook| hook.block.call(@record) }
+        Ledger.record(subject_type, @record.id, "erase", counts) if @ledger
+      end
+
+      erasure = Erasure.new(subject: subject_type, id: @record.id, counts: counts)
+      ActiveRecord.after_all_transactions_commit do
+        failed = purge(blobs)
+        ActiveSupport::Notifications.instrument(
+          "erase.lethe", subject: erasure.subject, id: erasure.id, counts: counts, purge_failures: failed
+        )
+      end
+      erasure
+    end
+
+    private
+
+    # The definition's subject, so that an STI subclass with a definition of
+    # its own is swept by its own retention.
+    def subject_type
+      @definition.subject_name
+    end
+
+    def perform(step, blobs)
+      case step.kind
+      when :attachment then detach(step.policy.target.name, blobs)
+      when :destroy then destroy(step)
+      when :scrub then scrub(step)
+      when :keep then 0
+      when :subject then erase_subject(step)
+      end
+    end
+
+    def destroy(step)
+      if step.policy.cascade
+        step.location.model.unscoped.where(step.conditions).destroy_all.size
+      else
+        rows(step).where(step.conditions).delete_all
+      end
+    end
+
+    def scrub(step)
+      set = step.policy.set
+      unless step.policy.computed_set?
+        return rows(step).where(step.conditions).update_all(Enums.cast(step.location.model, set))
+      end
+
+      count = 0
+      rows(step).where(step.conditions).find_each do |row|
+        row.update_columns(Enums.cast(step.location.model, evaluate(set, row)))
+        count += 1
+      end
+      count
+    end
+
+    def erase_subject(step)
+      policy = step.policy
+      relation = rows(step).where(step.conditions)
+      return relation.delete_all if policy.destroy
+
+      # The first erasure's time stays, so that retention counts from it.
+      relation.where(policy.soft_delete => nil).update_all(policy.soft_delete => Time.current)
+      relation.update_all(Enums.cast(@record.class, evaluate(policy.scrub, @record))) unless policy.scrub.empty?
+      relation.count
+    end
+
+    def detach(name, blobs)
+      attached = @record.public_send(name)
+      attachments = attached.respond_to?(:attachments) ? attached.attachments.to_a : [attached.attachment].compact
+      blobs.concat(attachments.map(&:blob))
+      attached.detach if attachments.any?
+      attachments.size
+    end
+
+    # The erasure has committed by now, so a file that cannot be purged must
+    # not make it look as if it failed. Returns the blob ids that could not be
+    # purged, for the notification; their blobs are already detached.
+    def purge(blobs)
+      blobs.filter_map do |blob|
+        (@configuration.purge == :now) ? blob.purge : blob.purge_later
+        nil
+      rescue => error
+        ActiveRecord::Base.logger&.warn("lethe: could not purge blob #{blob.id}: #{error.class}: #{error.message}")
+        blob.id
+      end
+    end
+
+    def evaluate(set, row)
+      set.to_h do |column, value|
+        next [column, value] unless value.is_a?(Proc)
+
+        [column, value.arity.zero? ? value.call : value.call(row)]
+      end
+    end
+
+    def rows(step)
+      Rows.for(step.location.table)
+    end
+  end
+end

data/lib/lethe/exporter.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+
+module Lethe
+  # Writes what happens to each kind of data when a subject leaves, as a
+  # Markdown table for people and as JSON for tools. Only covered references
+  # appear; ignored ones do not point at the subject.
+  class Exporter
+    def initialize(report, definitions:, models:, root:, definition_path:, locale: I18n.default_locale)
+      @report = report
+      @definitions = definitions
+      @models = models
+      @root = Pathname(root)
+      @definition_path = definition_path
+      @locale = locale
+    end
+
+    def markdown
+      sections = subjects.map { |subject| markdown_section(subject) }
+      "<!-- #{t("lethe.export.generated", path: relative(@definition_path))} -->\n\n#{sections.join("\n")}"
+    end
+
+    def json
+      ::JSON.pretty_generate(
+        "version" => 1,
+        "subjects" => subjects.map { |subject| json_subject(subject) }
+      ) + "\n"
+    end
+
+    private
+
+    def subjects
+      @definitions.map(&:subject_name).select { |name| @report.subjects.include?(name) }.sort
+    end
+
+    def rows(subject)
+      @report.results
+        .select { |result| result.subject == subject && result.status == :covered && result.reference }
+        .sort_by(&:key)
+        .flat_map { |result| result.policies.reject(&:ignore?).map { |policy| [result.reference, policy] } }
+    end
+
+    def markdown_section(subject)
+      lines = [
+        "# #{t("lethe.export.title", subject: subject_label(subject))}",
+        "",
+        "| #{t("lethe.export.data")} | #{t("lethe.export.action")} | #{t("lethe.export.retention")} | #{t("lethe.export.reason")} |",
+        "| --- | --- | --- | --- |"
+      ]
+      definition = definition_for(subject)
+      if definition.subject_policy
+        lines << row([t("lethe.export.subject_row", subject: subject_label(subject)), subject_action(definition.subject_policy), "", ""])
+      end
+      rows(subject).each do |reference, policy|
+        lines << row([label(subject, reference), action(policy), retention(policy.retention), policy.reason.to_s])
+      end
+      lines.join("\n") + "\n"
+    end
+
+    def json_subject(subject)
+      definition = definition_for(subject)
+      subject_policy = definition.subject_policy
+      {
+        "subject" => subject,
+        "subject_row" => subject_policy && {
+          "destroy" => subject_policy.destroy ? true : false,
+          "soft_delete" => subject_policy.soft_delete&.to_s,
+          "scrub" => subject_policy.scrub.keys.map(&:to_s)
+        }.compact,
+        "data" => rows(subject).map do |reference, policy|
+          {
+            "key" => reference.key,
+            "table" => reference.table,
+            "column" => reference.column,
+            "attachment" => reference.attachment,
+            "label" => label(subject, reference),
+            "action" => policy.action.to_s,
+            "where" => policy.where&.transform_keys(&:to_s),
+            "set" => policy.set&.keys&.map(&:to_s),
+            "retention_seconds" => policy.retention&.to_i,
+            "retention" => policy.retention && retention(policy.retention),
+            "reason" => policy.reason
+          }.compact
+        end
+      }.compact
+    end
+
+    def definition_for(subject)
+      @definitions.find { |definition| definition.subject_name == subject }
+    end
+
+    def subject_model(subject)
+      subject.safe_constantize
+    end
+
+    def subject_label(subject)
+      model = subject_model(subject)
+      model ? model.model_name.human(locale: @locale) : subject
+    end
+
+    def label(subject, reference)
+      if reference.attachment
+        model = subject_model(subject)
+        name = model ? model.human_attribute_name(reference.attachment, locale: @locale) : reference.attachment
+        return t("lethe.export.attachment", name: name)
+      end
+
+      model = @models.find { |candidate| candidate.table_name == reference.table && candidate.base_class == candidate }
+      base = model ? model.model_name.human(locale: @locale) : reference.table
+      conventional = subject_model(subject)&.model_name&.singular&.then { |singular| "#{singular}_id" }
+      (reference.column == conventional) ? base : t("lethe.export.column_suffix", label: base, column: reference.column)
+    end
+
+    def action(policy)
+      text =
+        case policy.action
+        when :scrub then t("lethe.export.scrub", columns: list(policy.set.keys))
+        else t("lethe.export.#{policy.action}")
+        end
+      return text unless policy.where
+
+      conditions = policy.where.map { |column, value| "#{column} = #{Array(value).join(" / ")}" }
+      t("lethe.export.where", action: text, conditions: conditions.join(t("lethe.export.list_separator")))
+    end
+
+    def subject_action(policy)
+      return t("lethe.export.destroy") if policy.destroy
+
+      scrubbed = policy.scrub.keys
+      if scrubbed.empty?
+        t("lethe.export.soft_delete", column: policy.soft_delete)
+      else
+        t("lethe.export.soft_delete_scrub", column: policy.soft_delete, columns: list(scrubbed))
+      end
+    end
+
+    def retention(duration)
+      return "" unless duration
+
+      duration.parts.map { |unit, count| t("lethe.export.parts.#{unit}", count: count) }.join(" ")
+    end
+
+    def list(names)
+      names.join(t("lethe.export.list_separator"))
+    end
+
+    def row(cells)
+      "| #{cells.map { |cell| cell.to_s.gsub("|", "\\|").gsub(/\s*\n\s*/, " ") }.join(" | ")} |"
+    end
+
+    def relative(path)
+      Pathname(path).relative_path_from(@root).to_s
+    rescue ArgumentError
+      path.to_s
+    end
+
+    def t(key, **values)
+      I18n.t(key, **values, locale: @locale)
+    end
+  end
+end