prune_ar 0.1.0

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'prune_ar'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/prune_ar.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'active_record'
+require 'prune_ar/version'
+require 'prune_ar/pruner'
+
+# Namespace for all prune_ar code
+module PruneAr
+  # deletion_criteria
+  # => The core pruning criteria that you want to execute (will be executed up front)
+  # => {
+  # =>   Account => ['accounts.id NOT IN (1, 2)']
+  # =>   User => ["users.internal = 'f'", "users.active = 'f'"]
+  # => }
+  #
+  # full_delete_models
+  # => Models for which you want to purge all records
+  # => [Model1, Model2]
+  #
+  # pre_queries_to_run
+  # => Arbitrary SQL statements to execute before pruning
+  # => [ 'UPDATE users SET invited_by_id = NULL WHERE invited_by_id IS NOT NULL' ]
+  #
+  # conjunctive_deletion_criteria
+  # => Pruning criteria you want executed in conjunction with each iteration of pruning
+  # => of orphaned records (one case where this is useful if pruning entities which
+  # => don't have a belongs_to chain to the entities we pruned but instead are associated
+  # => via join tables)
+  # => {
+  # =>   Image => ['NOT EXISTS (SELECT 1 FROM imagings WHERE imagings.image_id = images.id)']
+  # => }
+  #
+  # perform_sanity_check
+  # => Determines whether `PruneAr` sanity checks it's own pruning by setting (& subsequently
+  # => removing) foreign key constraints for all belongs_to relations. This is to prove that
+  # => we maintained referential integrity.
+  def self.prune_all_models(
+    deletion_criteria: {},
+    full_delete_models: [],
+    pre_queries_to_run: [],
+    conjunctive_deletion_criteria: {},
+    perform_sanity_check: true,
+    logger: Logger.new(STDOUT).tap { |l| l.level = Logger::WARN }
+  )
+    Pruner.new(
+      models: all_models,
+      deletion_criteria: deletion_criteria,
+      full_delete_models: full_delete_models,
+      pre_queries_to_run: pre_queries_to_run,
+      conjunctive_deletion_criteria: conjunctive_deletion_criteria,
+      perform_sanity_check: perform_sanity_check,
+      logger: logger
+    ).prune
+  end
+
+  def self.all_models
+    ActiveRecord::Base
+      .descendants
+      .reject { |c| ['ApplicationRecord'].any? { |start| c.name.start_with?(start) } }
+      .uniq(&:table_name)
+  end
+end

data/lib/prune_ar/belongs_to_association.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module PruneAr
+  # Represents a ActiveRecord belongs_to association
+  class BelongsToAssociation
+    attr_reader :source_model,
+                :destination_model,
+                :foreign_key_column,
+                :association_primary_key_column,
+                :foreign_type_column
+
+    def initialize(
+      source_model:,
+      destination_model:,
+      foreign_key_column:,
+      association_primary_key_column: 'id',
+      foreign_type_column: nil, # Indicates that relation is polymorphic
+      **_extra # Ignore extra
+    )
+      @source_model = source_model
+      @destination_model = destination_model
+      @foreign_key_column = foreign_key_column
+      @association_primary_key_column = association_primary_key_column
+      @foreign_type_column = foreign_type_column
+    end
+
+    def polymorphic?
+      !foreign_type_column.nil?
+    end
+
+    def source_table
+      source_model.table_name
+    end
+
+    def destination_table
+      destination_model.table_name
+    end
+
+    def destination_model_name
+      destination_model.name
+    end
+
+    def ==(other) # rubocop:disable Metrics/AbcSize
+      source_model == other.source_model &&
+        destination_model == other.destination_model &&
+        foreign_key_column == other.foreign_key_column &&
+        association_primary_key_column == other.association_primary_key_column &&
+        foreign_type_column == other.foreign_type_column
+    end
+  end
+end

data/lib/prune_ar/belongs_to_association_gatherer.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'active_record'
+require 'prune_ar/belongs_to_association'
+
+module PruneAr
+  # Given ActiveRecord models, produces BelongsToAssociation objects
+  class BelongsToAssociationGatherer
+    attr_reader :models, :connection, :logger
+
+    def initialize(
+      models,
+      connection: ActiveRecord::Base.connection,
+      logger: Logger.new(STDOUT).tap { |l| l.level = Logger::WARN }
+    )
+      @models = models
+      @connection = connection
+      @logger = logger
+    end
+
+    def associations
+      @associations ||= gather_belongs_to_for_models
+    end
+
+    private
+
+    def gather_belongs_to_for_models
+      models.flat_map(&method(:gather_belongs_to_for_model))
+    end
+
+    def gather_belongs_to_for_model(source_model)
+      source_model.reflect_on_all_associations.flat_map do |assoc|
+        next [] if assoc.macro != :belongs_to
+
+        build_belongs_to_associations(source_model, assoc)
+      end
+    end
+
+    def build_belongs_to_associations(source_model, assoc)
+      destination_models = gather_belongs_to_destination_models(source_model, assoc)
+      curried = method(:build_belongs_to_association).curry.call(source_model, assoc)
+      destination_models.flat_map(&curried)
+    end
+
+    def validate_belongs_to_association(foreign_key_column, source_model, destination_model)
+      unless destination_model.column_names.include?('id')
+        logger.warn("bad association? Column #{destination_model.table_name}.id doesn't exist")
+        return false
+      end
+
+      unless source_model.column_names.include?(foreign_key_column)
+        logger.warn("bad association? Column #{source_model.table_name}.#{foreign_key_column}"\
+                    " doesn't exist")
+        return false
+      end
+
+      true
+    end
+
+    def build_belongs_to_association(source_model, assoc, destination_model)
+      foreign_key_column = get_foreign_key_column(assoc, destination_model)
+      unless validate_belongs_to_association(foreign_key_column, source_model, destination_model)
+        return []
+      end
+
+      [
+        BelongsToAssociation.new(
+          {
+            source_model: source_model,
+            destination_model: destination_model,
+            foreign_key_column: foreign_key_column
+          }.merge(assoc.polymorphic? ? { foreign_type_column: assoc.foreign_type.to_s } : {})
+        )
+      ]
+    end
+
+    def gather_belongs_to_destination_models(source_model, assoc)
+      begin
+        return [assoc.klass] unless assoc.polymorphic?
+      rescue StandardError => e
+        logger.error("error encountered loading association class: #{e}")
+        return []
+      end
+
+      foreign_types = read_foreign_types(source_model, assoc)
+      foreign_types.flat_map(&method(:model_string_to_class))
+    end
+
+    def read_foreign_types(source_model, assoc)
+      foreign_type_column = assoc.foreign_type.to_s
+
+      sql = <<~SQL
+        SELECT DISTINCT #{foreign_type_column}
+        FROM #{source_model.table_name}
+        WHERE #{foreign_type_column} IS NOT NULL
+      SQL
+
+      sql = sql.gsub(/\s+/, ' ').strip
+
+      begin
+        foreign_types = connection.exec_query(sql).map { |t| t[foreign_type_column] }
+      rescue StandardError => e
+        logger.error("error encountered reading foreign types for #{source_model}: #{e}")
+        return []
+      end
+
+      foreign_types
+    end
+
+    def get_foreign_key_column(assoc, destination_model)
+      foreign_key = assoc.foreign_key.to_s
+
+      # Rails strangeness on HABTM. `assoc.foreign_key` shows up as `left_side_id` for one
+      # => field in the join table.
+      if foreign_key == 'left_side_id'
+        foreign_key = destination_model.table_name.gsub(/s\z/, '') + '_id'
+      end
+
+      foreign_key
+    end
+
+    def model_string_to_class(type_string)
+      [type_string.constantize]
+    rescue StandardError => e
+      logger.error("error encountered constantizing #{type_string}: #{e}")
+      []
+    end
+  end
+end

data/lib/prune_ar/deleter_by_criteria.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'active_record'
+
+module PruneAr
+  # Core of this gem. Prunes records based on parameters given.
+  class DeleterByCriteria
+    attr_reader :criteria, :connection, :logger
+
+    # criteria is of form [['users', "name = 'andrew'"], ['comments', 'id NOT IN (1, 2, 3)']]
+    def initialize(
+        criteria,
+        connection: ActiveRecord::Base.connection,
+        logger: Logger.new(STDOUT).tap { |l| l.level = Logger::WARN }
+      )
+      @criteria = criteria
+      @connection = connection
+      @logger = logger
+    end
+
+    def delete
+      i = 0
+      loop do
+        logger.info("deletion loop iteration #{i}")
+        i += 1
+
+        return unless anything_to_delete?
+
+        criteria.each do |table, selection|
+          delete_selection(table, selection)
+        end
+      end
+    end
+
+    private
+
+    def anything_to_delete?
+      criteria.any? do |table, selection|
+        count = count_to_be_deleted(table, selection)
+        count.positive?.tap do |positive|
+          if positive
+            logger.info("found something to delete (#{count} records to delete from #{table}"\
+                        " where #{selection})")
+          end
+        end
+      end
+    end
+
+    def count_to_be_deleted(table, selection)
+      results = connection.exec_query("SELECT COUNT(*) as count FROM #{table} WHERE #{selection};")
+      results.entries.first['count'].tap do |count|
+        logger.debug("found #{count} records to delete from #{table} where #{selection}")
+      end
+    end
+
+    def delete_selection(table, selection)
+      sql = "DELETE FROM #{table} WHERE #{selection};"
+      logger.debug("deleting all records from #{table} where #{selection}")
+      connection.exec_query(sql)
+    end
+  end
+end

data/lib/prune_ar/foreign_key_handler.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'active_record'
+
+module PruneAr
+  # Core of this gem. Prunes records based on parameters given.
+  class ForeignKeyHandler
+    attr_reader :connection, :logger, :original_foreign_keys, :foreign_key_supported
+
+    def initialize(
+        models:,
+        connection: ActiveRecord::Base.connection,
+        logger: Logger.new(STDOUT).tap { |l| l.level = Logger::WARN }
+      )
+      @connection = connection
+      @logger = logger
+      @foreign_key_supported = connection.supports_foreign_keys?
+      @original_foreign_keys = snapshot_foreign_keys(models)
+    end
+
+    def drop(foreign_keys)
+      return unless foreign_key_supported
+
+      foreign_keys.each do |fk|
+        logger.debug("dropping #{fk.name} from #{fk.from_table} (#{fk.column})")
+        connection.remove_foreign_key(fk.from_table, name: fk.name)
+      end
+    end
+
+    def create(foreign_keys)
+      return unless foreign_key_supported
+
+      foreign_keys.each do |fk|
+        logger.debug("creating #{fk.name} on #{fk.from_table} (#{fk.column})")
+        connection.add_foreign_key(fk.from_table, fk.to_table, fk.options)
+      end
+    end
+
+    def create_from_belongs_to_associations(associations)
+      return [] unless foreign_key_supported
+
+      associations.map do |assoc|
+        constraint_name = generate_belongs_to_foreign_key_name(assoc)
+        create_from_belongs_to_association(constraint_name, assoc)
+      end
+    end
+
+    private
+
+    def snapshot_foreign_keys(models)
+      return [] unless foreign_key_supported
+
+      models.flat_map do |model|
+        connection.foreign_keys(model.table_name)
+      end
+    end
+
+    def create_from_belongs_to_association(name, assoc)
+      fk = ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new(
+        assoc.source_table,
+        assoc.destination_table,
+        name: name,
+        column: assoc.foreign_key_column,
+        primary_key: assoc.association_primary_key_column,
+        on_delete: :restrict,
+        on_update: :restrict,
+        validate: true
+      )
+
+      logger.debug("creating #{name} on #{fk.from_table} (#{fk.column})")
+      connection.add_foreign_key(fk.from_table, fk.to_table, fk.options)
+      fk
+    end
+
+    # Limited to 64 characters
+    def generate_belongs_to_foreign_key_name(assoc)
+      source = assoc.source_table[0..7]
+      column = assoc.foreign_key_column[0..7]
+      destination = assoc.destination_table[0..7]
+      "fk_#{source}_#{column}_#{destination}_#{SecureRandom.hex}"
+    end
+  end
+end

data/lib/prune_ar/orphaned_selection_builder.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module PruneAr
+  # Builds SQL selection to select orphaned records based on a BelongsToAssociation
+  class OrphanedSelectionBuilder
+    def initialize
+      @cache = {}
+    end
+
+    def orphaned_selection(assoc)
+      @cache[assoc] ||= self.class.build_orphaned_selection(assoc)
+    end
+
+    def self.build_orphaned_selection(assoc)
+      if assoc.polymorphic?
+        build_orphaned_selection_polymorphic(assoc)
+      else
+        build_orphaned_selection_simple(assoc)
+      end
+    end
+
+    def self.build_orphaned_selection_simple(assoc)
+      src = assoc.source_table
+      dst = assoc.destination_table
+      sql = <<~SQL
+        #{src}.#{assoc.foreign_key_column} IS NOT NULL
+        AND NOT EXISTS (
+          SELECT 1
+          FROM #{dst} dst
+          WHERE dst.#{assoc.association_primary_key_column} = #{src}.#{assoc.foreign_key_column}
+        )
+      SQL
+
+      sql.gsub(/\s+/, ' ').strip
+    end
+
+    def self.build_orphaned_selection_polymorphic(assoc)
+      src = assoc.source_table
+      dst = assoc.destination_table
+      sql = <<~SQL
+        #{src}.#{assoc.foreign_type_column} = '#{assoc.destination_model_name}'
+        AND #{src}.#{assoc.foreign_key_column} IS NOT NULL
+        AND NOT EXISTS (
+          SELECT 1
+          FROM #{dst} dst
+          WHERE dst.#{assoc.association_primary_key_column} = #{src}.#{assoc.foreign_key_column}
+        )
+      SQL
+
+      sql.gsub(/\s+/, ' ').strip
+    end
+  end
+end