lethe 0.1.0

data/lib/lethe/reference.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Lethe
+  # Something that points at a subject: a table's column, or an attachment.
+  #
+  # vias says how it was found. Foreign keys, associations and attachments
+  # are certain; a column name alone, or a polymorphic belongs_to the subject
+  # does not answer with an `as:` association, is only a guess.
+  Reference = Data.define(:table, :column, :attachment, :vias) do
+    def self.column(table, column, via)
+      new(table: table.to_s, column: column.to_s, attachment: nil, vias: [via])
+    end
+
+    def self.attachment(name)
+      new(table: nil, column: nil, attachment: name.to_s, vias: [:attachment])
+    end
+
+    def key
+      attachment ? "attachment:#{attachment}" : "#{table}.#{column}"
+    end
+
+    def certain?
+      vias.any? { |via| self.class::CERTAIN_VIAS.include?(via) }
+    end
+
+    def merge(other)
+      with(vias: self.class.order(vias + other.vias))
+    end
+
+    # Clues in a fixed order, the most certain first, so output does not
+    # depend on which clue found a reference first.
+    def self.order(vias)
+      vias.uniq.sort_by { |via| self::VIA_ORDER.index(via) || self::VIA_ORDER.size }
+    end
+
+    def to_s
+      attachment ? "attachment :#{attachment}" : key
+    end
+  end
+
+  Reference::CERTAIN_VIAS = %i[foreign_key association join_table polymorphic attachment declared].freeze
+  Reference::VIA_ORDER = (Reference::CERTAIN_VIAS + %i[polymorphic_guess column_name]).freeze
+end

data/lib/lethe/report.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Lethe
+  class Report
+    attr_reader :results
+
+    def initialize(results, strict: false)
+      @results = results
+      @strict = strict
+    end
+
+    def strict?
+      @strict
+    end
+
+    def problems
+      results.select { |result| result.problem?(strict: @strict) }
+    end
+
+    def clean?
+      problems.empty?
+    end
+
+    def exit_status
+      clean? ? 0 : 1
+    end
+
+    def count(status)
+      results.count { |result| result.status == status }
+    end
+
+    def subjects
+      results.map(&:subject).uniq
+    end
+  end
+end

data/lib/lethe/result.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Lethe
+  # What the check concluded about one reference or one declaration.
+  #
+  #   status    :covered  a policy applies
+  #             :ignored  declared not to point at the subject
+  #             :missing  certainly points at the subject, and nothing is declared
+  #             :guess    might point at the subject, and nothing is declared
+  #             :invalid  a declaration is wrong or names something that is gone
+  #   subject   the subject class name
+  #   key       "table.column", "attachment:name" or "subject" for the subject's own row
+  #   reference the Reference, when there is one
+  #   policies  the policies that apply
+  #   message   an I18n key and its values, for :invalid
+  #   location  "path:line" of the declaration, when there is one
+  Result = Data.define(:status, :subject, :key, :reference, :policies, :message, :location) do
+    def problem?(strict: false)
+      status == :missing || status == :invalid || (strict && status == :guess)
+    end
+  end
+end

data/lib/lethe/rows.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Lethe
+  # Plain Active Record classes for reaching a table's rows without the
+  # application's model: no default scopes, callbacks or single table
+  # inheritance get in the way of deleting or overwriting exactly the rows
+  # a policy names. One class per table, made once.
+  module Rows
+    @classes = {}
+    @mutex = Mutex.new
+
+    # Forgets the classes, for when the schema changes in the same process.
+    def self.reset!
+      @mutex.synchronize { @classes.clear }
+    end
+
+    def self.for(table)
+      @mutex.synchronize do
+        @classes[table] ||= Class.new(ActiveRecord::Base) do
+          self.table_name = table
+          self.inheritance_column = nil
+
+          def self.name
+            "Lethe::Rows"
+          end
+        end
+      end
+    end
+  end
+
+  # The application's models, leaving out the classes lethe makes.
+  def self.application_models
+    ActiveRecord::Base.descendants.reject do |model|
+      model.abstract_class? || model.name.nil? || model.name.start_with?("Lethe::")
+    end
+  end
+end

data/lib/lethe/rules.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Lethe
+  # The rules a declared policy must follow once its target is known to
+  # exist. Each rule returns the problems it finds, as :invalid Results.
+  # They exist so that a definition that would erase the wrong rows, or
+  # fail halfway, never passes the check.
+  class Rules
+    RULES = %i[
+      reason_given reason_is_text
+      scrub_sets_columns where_names_columns
+      retention_only_where_rows_stay cascade_on_kept_rows_needs_retention retention_is_duration
+      enum_values_exist polymorphic_type_given cascade_has_model computed_values_have_primary_key
+    ].freeze
+
+    # schema answers column_names(table), model_for_table(table) and
+    # primary_key(table); invalid builds the :invalid Result for a policy.
+    def initialize(subject, schema, &invalid)
+      @subject = subject
+      @schema = schema
+      @invalid = invalid
+    end
+
+    def call(policy, key)
+      table = key.start_with?("attachment:") ? nil : key.split(".", 2).first
+      RULES.flat_map { |rule| Array(send(rule, policy, key, table)) }
+    end
+
+    private
+
+    # keep and ignore are decisions to leave data in place; the export and
+    # anyone auditing it need to know why.
+    def reason_given(policy, _key, _table)
+      return unless %i[keep ignore].include?(policy.action) && blank?(policy.reason)
+
+      problem(policy, "reason_required", action: policy.action)
+    end
+
+    def reason_is_text(policy, _key, _table)
+      problem(policy, "reason_not_string") unless policy.reason.nil? || policy.reason.is_a?(String)
+    end
+
+    def scrub_sets_columns(policy, _key, table)
+      return unless policy.action == :scrub
+      return problem(policy, "set_required") if !policy.set.is_a?(Hash) || policy.set.empty?
+
+      unknown_columns(policy, table, policy.set.keys, "set")
+    end
+
+    def where_names_columns(policy, _key, table)
+      return if policy.where.nil?
+      return problem(policy, "where_not_hash") if !policy.where.is_a?(Hash) || policy.where.empty?
+
+      unknown_columns(policy, table, policy.where.keys, "where")
+    end
+
+    # Only rows that stay can stay for a while.
+    def retention_only_where_rows_stay(policy, _key, _table)
+      return unless policy.retention && !staying?(policy)
+
+      problem(policy, "retention_not_allowed", action: policy.action)
+    end
+
+    def retention_is_duration(policy, _key, _table)
+      return if policy.retention.nil? || policy.retention.is_a?(ActiveSupport::Duration)
+
+      problem(policy, "retention_not_duration")
+    end
+
+    # On keep and scrub, cascade only changes how lethe:sweep deletes.
+    def cascade_on_kept_rows_needs_retention(policy, _key, _table)
+      return unless policy.cascade && staying?(policy) && policy.retention.nil?
+
+      problem(policy, "cascade_without_retention", action: policy.action)
+    end
+
+    # Rows are reached without the application's model, so an enum name the
+    # enum does not have would silently match nothing.
+    def enum_values_exist(policy, _key, table)
+      return unless table
+
+      model = @schema.model_for_table(table)
+      {"where" => policy.where, "set" => (policy.action == :scrub) ? policy.set : nil}.flat_map do |option, hash|
+        Enums.unknown(model, hash).map do |column, value|
+          problem(policy, "unknown_enum_value", option: option, table: table, column: column, value: value.inspect)
+        end
+      end
+    end
+
+    # A "table.column" for a polymorphic *_id must say which type it means,
+    # or erasing User 5 would also erase what belongs to Post 5.
+    def polymorphic_type_given(policy, key, table)
+      return unless table && policy.target.kind == :column
+
+      column = key.split(".", 2).last
+      return unless column.end_with?("_id")
+
+      type_column = "#{column.delete_suffix("_id")}_type"
+      return unless @schema.column_names(table).include?(type_column)
+      return if policy.where.is_a?(Hash) && policy.where.keys.map(&:to_s).include?(type_column)
+
+      problem(policy, "polymorphic_type_required", key: key, type_column: type_column, subject_type: @subject.polymorphic_name)
+    end
+
+    def cascade_has_model(policy, _key, table)
+      problem(policy, "cascade_needs_model", table: table) if table && policy.cascade && !@schema.model_for_table(table)
+    end
+
+    # A Proc in set: is evaluated row by row, which needs a key to update by.
+    def computed_values_have_primary_key(policy, _key, table)
+      return unless table && policy.action == :scrub && policy.computed_set?
+
+      problem(policy, "computed_set_needs_primary_key", table: table) unless @schema.primary_key(table)
+    end
+
+    def unknown_columns(policy, table, names, option)
+      return [] unless table
+
+      (names.map(&:to_s) - @schema.column_names(table)).map do |column|
+        problem(policy, "unknown_column", option: option, table: table, column: column)
+      end
+    end
+
+    def staying?(policy)
+      %i[keep scrub].include?(policy.action)
+    end
+
+    def problem(policy, message_key, **values)
+      @invalid.call(policy, message_key, **values)
+    end
+
+    def blank?(value)
+      value.nil? || (value.is_a?(String) && value.strip.empty?)
+    end
+  end
+end

data/lib/lethe/runner.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Lethe
+  # Runs lethe:check and lethe:export: reads the options from the environment,
+  # prints, and returns the exit status (0 clean, 1 problems, 2 cannot check).
+  class Runner
+    def initialize(root:, env: ENV, out: $stdout, err: $stderr, models: nil)
+      @root = File.expand_path(root)
+      @env = env
+      @out = out
+      @err = err
+      @models = models
+    end
+
+    def check
+      report = run_check or return 2
+      formatter = Formatters.for(@env["FORMAT"]).new(report, root: @root, locale: locale, definition_path: definition_path)
+      @out.print formatter.render
+      report.exit_status
+    rescue ArgumentError => error
+      @err.puts error.message
+      2
+    end
+
+    def export
+      report = run_check or return 2
+      unless report.clean?
+        @err.print Formatters::Text.new(report, root: @root, locale: locale, definition_path: definition_path).render
+        return 1
+      end
+
+      exporter = Exporter.new(
+        report, definitions: Lethe.definitions.values, models: models, root: @root,
+        definition_path: definition_path, locale: locale
+      )
+      files = {"#{export_path}.md" => exporter.markdown, "#{export_path}.json" => exporter.json}
+      truthy?(@env["CHECK"]) ? compare(files) : write(files)
+    end
+
+    def sweep
+      return 2 unless valid_locale?
+
+      Lethe.load_definitions(definition_path)
+      swept = Lethe.sweep!(models: models)
+      lines = swept.flat_map { |subject, counts| counts.map { |key, count| "lethe: swept #{count} from #{subject} #{key}" } }
+      @out.puts(lines.empty? ? "lethe: nothing to sweep" : lines)
+      0
+    rescue Error => error
+      @err.puts error.message
+      2
+    rescue ScriptError, StandardError => error
+      @err.puts I18n.t("lethe.errors.load_failed", path: relative(definition_path), error: "#{error.class}: #{error.message}", locale: locale)
+      2
+    end
+
+    private
+
+    def run_check
+      return nil unless valid_locale?
+
+      Lethe.load_definitions(definition_path)
+      Lethe.check(models: models, strict: truthy?(@env["STRICT"]))
+    rescue DefinitionError => error
+      @err.puts error.message
+      nil
+    rescue ScriptError, StandardError => error
+      @err.puts I18n.t("lethe.errors.load_failed", path: relative(definition_path), error: "#{error.class}: #{error.message}", locale: locale)
+      nil
+    end
+
+    def compare(files)
+      stale = files.reject { |path, content| File.file?(path) && File.read(path) == content }
+      if stale.empty?
+        @out.puts "lethe: #{files.keys.map { |path| relative(path) }.join(", ")} up to date"
+        0
+      else
+        @err.puts "lethe: #{stale.keys.map { |path| relative(path) }.join(", ")} out of date; run bin/rails lethe:export"
+        1
+      end
+    end
+
+    def write(files)
+      files.each do |path, content|
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, content)
+        @out.puts "lethe: wrote #{relative(path)}"
+      end
+      0
+    end
+
+    def models
+      @models || Lethe.application_models
+    end
+
+    def definition_path
+      File.expand_path(Lethe.configuration.definition_path, @root)
+    end
+
+    def export_path
+      File.expand_path(Lethe.configuration.export_path, @root)
+    end
+
+    def locale
+      (@env["LOCALE"].nil? || @env["LOCALE"].empty?) ? I18n.default_locale : @env["LOCALE"].to_sym
+    end
+
+    def valid_locale?
+      return true if I18n.available_locales.include?(locale)
+
+      @err.puts "lethe: LOCALE=#{locale} is not one of #{I18n.available_locales.join(", ")}"
+      false
+    end
+
+    def truthy?(value)
+      %w[1 true yes].include?(value.to_s.downcase)
+    end
+
+    def relative(path)
+      Pathname(path).relative_path_from(Pathname(@root)).to_s
+    rescue ArgumentError
+      path.to_s
+    end
+  end
+end

data/lib/lethe/sweeper.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Lethe
+  # Deletes kept rows whose time is up: `keep ..., for: 2.years` (or
+  # `scrub ..., for:`) keeps a subject's rows for two years after it was
+  # first erased, as recorded in lethe_erasures, and this deletes them afterwards.
+  class Sweeper
+    def initialize(definitions, models:, now: Time.current)
+      @definitions = definitions
+      @models = models
+      @now = now
+    end
+
+    # Returns { "User" => { "inquiries.user_id" => 3 } } for what it deleted.
+    def call
+      @definitions.each_with_object({}) do |definition, swept|
+        policies = definition.policies.select { |policy| %i[keep scrub].include?(policy.action) && policy.retention }
+        next if policies.empty?
+
+        subject = definition.subject_name.constantize
+        erased = Ledger.erased(definition.subject_name)
+        locator = Locator.new(subject, models: @models)
+        policies.each do |policy|
+          ids = erased.select { |_, at| at <= @now - policy.retention }.keys
+          count = sweep(locator, policy, ids, subject)
+          (swept[subject.name] ||= {})[policy.target.to_s] = count if count.positive?
+        end
+      end
+    end
+
+    private
+
+    def sweep(locator, policy, ids, subject)
+      location = locator.locate(policy)
+      rows = locator.rows(location)
+      ids.each_slice(500).sum do |slice|
+        present = rows.where(locator.conditions(location, policy, slice)).distinct.pluck(location.column)
+        present.sum do |id|
+          ActiveRecord::Base.transaction do
+            relation = policy.cascade ? location.model.unscoped : rows
+            relation = relation.where(locator.conditions(location, policy, id))
+            count = policy.cascade ? relation.destroy_all.size : relation.delete_all
+            Ledger.record(subject.name, id, "sweep", {policy.target.to_s => {"sweep" => count}})
+            count
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lethe/target.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Lethe
+  # What a policy points at, as written in the definitions:
+  # an association of the subject (:ratings), a "table.column" string,
+  # or an attachment of the subject.
+  Target = Data.define(:kind, :name, :table, :column) do
+    def self.parse(value)
+      case value
+      when Symbol
+        new(kind: :association, name: value, table: nil, column: nil)
+      when String
+        table, column, rest = value.split(".")
+        unless rest.nil? && present?(table) && present?(column)
+          raise ArgumentError, "lethe: #{value.inspect} is not \"table.column\"; use a symbol for an association"
+        end
+        new(kind: :column, name: value, table: table, column: column)
+      else
+        raise ArgumentError, "lethe: #{value.inspect} must be an association name (a Symbol) or \"table.column\""
+      end
+    end
+
+    def self.attachment(value)
+      raise ArgumentError, "lethe: attachment #{value.inspect} must be a Symbol" unless value.is_a?(Symbol)
+
+      new(kind: :attachment, name: value, table: nil, column: nil)
+    end
+
+    def self.present?(part)
+      !part.nil? && !part.empty?
+    end
+    private_class_method :present?
+
+    def to_s
+      case kind
+      when :association then ":#{name}"
+      when :attachment then "attachment :#{name}"
+      else name
+      end
+    end
+  end
+end

data/lib/lethe/tasks/lethe.rake

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# Exit only on failure, so that the tasks can be chained:
+#   bin/rails lethe:check lethe:export
+namespace :lethe do
+  desc "Check that everything pointing at a subject has a policy (FORMAT=text|github|json STRICT=1 LOCALE=en|ja)"
+  task check: :environment do
+    Rails.application.eager_load!
+    status = Lethe::Runner.new(root: Rails.root).check
+    exit status unless status.zero?
+  end
+
+  desc "Write what happens to each kind of data when a subject leaves (CHECK=1 to compare instead, LOCALE=en|ja)"
+  task export: :environment do
+    Rails.application.eager_load!
+    status = Lethe::Runner.new(root: Rails.root).export
+    exit status unless status.zero?
+  end
+
+  desc "Delete kept rows whose `for:` has passed since their subject was erased"
+  task sweep: :environment do
+    Rails.application.eager_load!
+    status = Lethe::Runner.new(root: Rails.root).sweep
+    exit status unless status.zero?
+  end
+end

data/lib/lethe/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Lethe
+  VERSION = "0.1.0"
+end