db-purger 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b756bb20c0bcea96d050a3772cb2606676891d485de6db0bf9058253f0092c3
-  data.tar.gz: 9dfe62f982ecd99ef96dadd535a7496d1802d0392083b8f63f9166f354bb9368
+  metadata.gz: 45ab4b4aa9e6e8c3778061c5deb8a8eb37fecf153bb1bae9cf298f2c83433f22
+  data.tar.gz: bbb9aec2aaa9dcd8e7c0609186795bf17e8fce64ffa951ce6dd1877981b7dbcc
 SHA512:
-  metadata.gz: 2a365e0a0c25b33eb79060930289c2820823e9e25fefbf4edcdf13ccfad6e55595e3d380d5da97a76476262051b9ee36095cdf51e30ebf3248bcf43dfd8f0fa2
-  data.tar.gz: 9171105e396d405cacde9debad9b6923d1d318c24ebbf269a9f872acbf6e087a0ea90506e9d24c0ab99407e1874fcd9d46e74afb73a736203b1f1da178791970
+  metadata.gz: 41d99640be3f800277142f9ef9a9ec0f0945785282551c757f92ba92de36dfad331b735533713bb98385023378727ca5365cf995cff84580dfe725cbbf10240d
+  data.tar.gz: bfe3512b7f289cec1b4ac7f466918c4aaa6a5c12f9deb61e60e825153dd3e1badbbcbb1eb578e80dd453504f9b7b08f70a996e4d9d10b7d1e7a654245d12ec00

data/ARCHITECTURE.md

@@ -31,12 +31,13 @@ db-purger is small (~900 lines) and splits cleanly into three layers: **describe
 | `lib/db-purger.rb` | Autoloads everything; holds the global `DBPurger.config`. |
 | `config.rb` | Global options: `explain?`, `explain_file`, `datetime_format`. |
 | `table.rb` | Value object for one table in the plan: name, match field, options, and a lazily created nested `Plan`. |
-| `plan.rb` | A node in the plan tree: one optional `base_table` plus lists of parent, child, search and ignored tables. `#purge!` is the run entry point; `#root_tables` are the tables it starts from. |
+| `plan.rb` | A node in the plan tree: one optional `base_table` plus lists of parent, child, nullify, search and ignored tables. `#purge!` is the run entry point; `#root_tables` are the tables it starts from. |
 | `plan_builder.rb` | The DSL. `instance_eval`s a plan file or block against a `Plan`; nested blocks get a new builder bound to that table's nested plan. |
 | `plan_validator.rb` | `ActiveModel::Validations` over plan vs. schema: missing tables, unknown tables, unknown columns. |
 | `executor.rb` | Convenience façade: loads a plan file, applies config options, `verify!`, `purge!`. |
 | `purge_table.rb` | Purges one table by `field = value(s)` in primary-key batches. Recurses into nested tables. |
 | `purge_table_scanner.rb` | Purges a `purge_table_search` table: full `find_in_batches` scan filtered through the user's `search_proc`. |
+| `nullify_table.rb` | Sets a nullable column to `NULL` on rows pointing at a batch of ids about to be deleted (`ON DELETE SET NULL` at purge time). |
 | `purge_table_helper.rb` | Shared behaviour for both purgers: nested-table recursion, delete vs. soft delete vs. explain, transactions. |
 | `metrics.rb` / `metric_subscriber.rb` | Aggregate timing and row counts per table from the notification events. |
 | `dynamic_plan_builder.rb` | Generates plan-file source (`build` for a base table, `build_for` for several roots); a bootstrap tool, not used at purge time. |
@@ -54,6 +55,7 @@ Plan (root)
     └── nested Plan
         ├── parent_tables: [company_tags(:company_id)]
         ├── child_tables:  [employments(:company_id) ─▶ nested Plan ..., websites(:id, fk: website_id) ...]
+        ├── nullify_tables: [companies(:acquired_by_company_id)]
         ├── search_tables: [users(:id)]
         └── ignore_tables
 ```
@@ -87,6 +89,8 @@ each_batch = loop:
     break if batch empty
 
 purge_children(batch) =
+    for each nullify_table:                        # optional rows pointing at us
+      UPDATE nullify SET field = NULL WHERE field IN (batch.pks) [AND conditions]
     for each child_table without foreign_key:      # rows pointing at us
       PurgeTable(child, child.field, batch.pks).purge!    (recursive)
 
@@ -118,6 +122,9 @@ Key properties:
 
 - **Depth-first, children first.** A row is only deleted after everything referencing it, so FK
   constraints hold without `ON DELETE CASCADE`.
+- **Unlink before delete.** A `nullify_table` is updated for each batch before that batch's children and rows
+  are deleted, so a self-referential foreign key (or a reference from a row that must survive) never blocks the
+  delete, whatever the chain depth or id order across batches.
 - **Two passes when there are parent tables.** A parent table can reference this table (e.g.
   `company_tags.company_id → companies.id`) *and* be referenced by one of its children, so it is purged
   between the child pass and the delete pass. Tables without parent tables keep the single pass.
@@ -146,7 +153,7 @@ All three go through ActiveRecord's `*_all` methods: no model callbacks or valid
 ## Instrumentation
 
 Every unit of work is wrapped in `ActiveSupport::Notifications.instrument` under the `db_purger` namespace
-(`purge`, `next_batch`, `delete_records`, `search_filter`). The purgers never talk to `Metrics` directly;
+(`purge`, `next_batch`, `delete_records`, `nullify_records`, `search_filter`). The purgers never talk to `Metrics` directly;
 `MetricSubscriber` (an `ActiveSupport::Subscriber`) translates events into `Metrics` counters. This keeps the
 purge code free of reporting concerns and lets callers attach their own subscribers (StatsD, logs, progress
 bars) without changes to the library.

data/README.md

@@ -152,6 +152,7 @@ an error (there is no enclosing batch to take ids from). Existing `base_table` p
 | `child_table(table, field, opts = {}, &block)` | Rows whose `field` matches the **primary key** of the enclosing table's current batch. Purged before that batch is deleted. |
 | `child_table(table, :id, foreign_key: :col, &block)` | Inverted relationship: the *enclosing* table holds `col` pointing at this table's `id`. Deleted in the same transaction, right after the enclosing batch. |
 | `parent_table(table, field, opts = {}, &block)` | Rows whose `field` matches the original **purge value**. At the top level of a plan without a `base_table`, each one is a root, purged in declaration order. Purged after the enclosing table's child tables but before the enclosing table's own rows, so it may both reference the base (`company_tags.company_id → companies.id`) and be referenced by a child table. Use for sibling tables that share the same key (e.g. `company_id`). |
+| `nullify_table(table, field, conditions: nil)` | Rows whose `field` matches the **primary key** of the enclosing table's current batch get `field = NULL` instead of being deleted, before anything in that batch is deleted (`ON DELETE SET NULL` at purge time). Use for optional references that must not take the referencing row down with them: a self-referential `parent_id`/"copied from" column, or a link from another tenant's row. Only `conditions:` is supported. |
 | `purge_table_search(table, field, opts = {}) { \|batch\| ... }` | Scans the whole table in batches; the block receives each batch and returns the records to purge. For orphans that can't be reached by a key. |
 | `ignore_table(name_or_regexp)` | Exclude a table from validation. |
 
@@ -224,7 +225,9 @@ Treat the output as a first draft: it cannot infer polymorphic (`as:`), soft-del
 - every field, `foreign_key` and `mark_deleted_field` named in the plan is a real column
 - the plan has a `base_table` or at least one top-level `parent_table`, no top-level `child_table` is left
   unreachable, and every `batch_size` is positive
-- tables without a primary key have no nested child or parent tables (there would be no ids to propagate)
+- tables without a primary key have no nested child, parent or nullify tables (there would be no ids to propagate)
+- every `nullify_table` field is a nullable column, and no `nullify_table` is left at the top level without a
+  `base_table`
 
 Run it in CI against your schema so a new table can't ship without a purge decision.
 
@@ -262,6 +265,7 @@ DBPurger::MetricSubscriber.metrics.as_json
 # => { took: 12.4, started_at: ..., finished_at: ...,
 #      purge_stats:  { employments: { duration:, num_purges:, num_records: } },
 #      delete_stats: { employments: { duration:, num_delete_queries:, num_deleted:, num_expected_to_delete: } },
+#      nullify_stats: { cadences: { duration:, num_nullify_queries:, num_nullified: } },
 #      lookup_stats: { ... }, filter_stats: { ... } }
 ```
 
@@ -273,6 +277,7 @@ Metrics are reset at the start of each `Plan#purge!`. To feed your own telemetry
 | `purge.db_purger` | `table_name`, `purge_field`, `deleted` |
 | `next_batch.db_purger` | `table_name`, `start_id`, `num_records` |
 | `delete_records.db_purger` | `table_name`, `num_records`, `records_deleted`, `deleted` |
+| `nullify_records.db_purger` | `table_name`, `nullify_field`, `num_records`, `records_nullified` |
 | `search_filter.db_purger` | `table_name`, `num_records`, `num_records_selected` |
 
 ## Caveats
@@ -303,7 +308,7 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for how the pieces fit together.
 ## Releasing
 
 1. Bump `s.version` in `db-purger.gemspec` and merge to `master`.
-2. Tag and push: `git tag v0.6.0 && git push origin v0.6.0`
+2. Tag and push: `git tag v0.7.0 && git push origin v0.7.0`
 
 `.github/workflows/release.yml` re-runs CI, checks the tag matches the gemspec version, publishes to RubyGems
 via trusted publishing (no API key), and creates a GitHub release with the `.gem` attached.

data/lib/db-purger/metric_subscriber.rb

@@ -36,6 +36,14 @@ module DBPurger
       )
     end
 
+    def nullify_records(event)
+      self.class.metrics.update_nullify_records_stats(
+        event.payload[:table_name],
+        event.duration,
+        event.payload[:records_nullified] || 0
+      )
+    end
+
     def next_batch(event)
       self.class.metrics.update_lookup_stats(
         event.payload[:table_name],

data/lib/db-purger/metrics.rb

@@ -7,6 +7,7 @@ module DBPurger
                 :finished_at,
                 :purge_stats,
                 :delete_stats,
+                :nullify_stats,
                 :lookup_stats,
                 :filter_stats
 
@@ -18,6 +19,7 @@ module DBPurger
       @started_at = Time.now
       @purge_stats = {}
       @delete_stats = {}
+      @nullify_stats = {}
       @lookup_stats = {}
       @filter_stats = {}
       @finished_at = nil
@@ -48,6 +50,14 @@ module DBPurger
       stats
     end
 
+    def update_nullify_records_stats(table_name, duration, num_nullified)
+      stats = (@nullify_stats[table_name] ||= Hash.new(0))
+      stats[:duration] += duration
+      stats[:num_nullify_queries] += 1
+      stats[:num_nullified] += num_nullified
+      stats
+    end
+
     def update_lookup_stats(table_name, duration, records_found)
       stats = (@lookup_stats[table_name] ||= Hash.new(0))
       stats[:duration] += duration
@@ -72,6 +82,7 @@ module DBPurger
         finished_at: @finished_at,
         purge_stats: @purge_stats,
         delete_stats: @delete_stats,
+        nullify_stats: @nullify_stats,
         lookup_stats: @lookup_stats,
         filter_stats: @filter_stats
       }

data/lib/db-purger/nullify_table.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::NullifyTable clears a nullable column that points at a batch of rows about to be deleted,
+  # the purge-time equivalent of ON DELETE SET NULL. It updates the referencing rows instead of deleting them.
+  class NullifyTable
+    include PurgeTableHelper
+
+    def initialize(database, table, purge_values)
+      @database = database
+      @table = table
+      @purge_values = purge_values
+    end
+
+    def nullify!
+      ActiveSupport::Notifications.instrument('nullify_records.db_purger',
+                                              table_name: @table.name,
+                                              nullify_field: @table.field,
+                                              num_records: @purge_values.size) do |payload|
+        payload[:records_nullified] = ::DBPurger.config.explain? ? explain_nullify : nullify_records
+      end
+    end
+
+    private
+
+    def nullify_records
+      scope.update_all(@table.field => nil)
+    end
+
+    def scope
+      scope = model.where(@table.field => @purge_values)
+      scope = scope.where(@table.conditions) if @table.conditions
+      scope
+    end
+
+    def explain_nullify
+      ::DBPurger.config.explain_file.puts("#{explain_update_sql(scope, field_quoted, 'NULL')};")
+      scope.count
+    end
+
+    def field_quoted
+      model.connection.quote_column_name(@table.field)
+    end
+  end
+end

data/lib/db-purger/plan.rb

@@ -8,18 +8,21 @@ module DBPurger
     attr_reader :parent_tables,
                 :child_tables,
                 :ignore_tables,
-                :search_tables
+                :search_tables,
+                :nullify_tables
 
     def initialize
       @parent_tables = []
       @child_tables = []
       @ignore_tables = []
       @search_tables = []
+      @nullify_tables = []
     end
 
     def purge!(database, purge_value)
       raise('plan has no base_table or top-level parent_table') if root_tables.empty?
       raise('top-level child_tables require a base_table') unless @base_table || @child_tables.empty?
+      raise('top-level nullify_tables require a base_table') unless @base_table || @nullify_tables.empty?
 
       MetricSubscriber.reset!
       num_deleted = purge_root_tables(database, purge_value)
@@ -37,7 +40,8 @@ module DBPurger
       all_tables = @base_table ? [@base_table] + @base_table.tables : []
       all_tables += @parent_tables + @parent_tables.map(&:tables) +
                     @child_tables + @child_tables.map(&:tables) +
-                    @search_tables + @search_tables.map(&:tables)
+                    @search_tables + @search_tables.map(&:tables) +
+                    @nullify_tables
       all_tables.flatten!
       all_tables.compact!
       all_tables
@@ -56,7 +60,8 @@ module DBPurger
       @base_table.nil? &&
         @parent_tables.empty? &&
         @child_tables.empty? &&
-        @search_tables.empty?
+        @search_tables.empty? &&
+        @nullify_tables.empty?
     end
 
     def ignore_table?(table_name)

data/lib/db-purger/plan_builder.rb

@@ -3,6 +3,8 @@
 module DBPurger
   # DBPurger::PlanBuilder is used to build the relationships between tables in a convenient way
   class PlanBuilder
+    NULLIFY_TABLE_OPTIONS = %i[conditions].freeze
+
     def initialize(plan)
       @plan = plan
     end
@@ -31,6 +33,22 @@ module DBPurger
       table
     end
 
+    # rows whose field matches the enclosing batch's primary keys get field set to NULL instead of being deleted
+    def nullify_table(table_name, field, options = {})
+      unsupported_options = options.keys - NULLIFY_TABLE_OPTIONS
+      unless unsupported_options.empty?
+        raise(ArgumentError, "nullify_table does not support #{unsupported_options.map(&:inspect).join(', ')}")
+      end
+
+      table = create_table(table_name, field, options)
+      if @plan.base_table
+        @plan.base_table.nested_plan.nullify_tables << table
+      else
+        @plan.nullify_tables << table
+      end
+      table
+    end
+
     def ignore_table(table_name)
       @plan.ignore_tables << table_name
     end

data/lib/db-purger/plan_validator.rb

@@ -11,6 +11,7 @@ module DBPurger
     validate :validate_no_missing_tables
     validate :validate_no_unknown_tables
     validate :validate_tables
+    validate :validate_nullify_tables
 
     def initialize(database, plan)
       @database = database
@@ -36,6 +37,8 @@ module DBPurger
       elsif !@plan.child_tables.empty?
         # without a base_table there are no ids to propagate; declared before one, they are never reached
         errors.add(:base_table, 'must be declared before top-level child_tables')
+      elsif !@plan.nullify_tables.empty?
+        errors.add(:base_table, 'must be declared before top-level nullify_tables')
       end
     end
 
@@ -51,6 +54,27 @@ module DBPurger
       @plan.tables.each { |table| validate_table_definition(table) }
     end
 
+    # a NOT NULL column can't be unlinked, so the purge would fail at runtime instead of here
+    def validate_nullify_tables
+      nullify_tables.each do |table|
+        next unless (model = find_model_for_table(table)) # reported by validate_tables
+        next unless not_nullable_column?(model, table.field)
+
+        errors.add(:table, "#{table.name}.#{table.field} (nullify_table) is not nullable")
+      end
+    end
+
+    # a missing column is reported by validate_tables
+    def not_nullable_column?(model, field)
+      column = model.columns_hash[field.to_s]
+      column && !column.null
+    end
+
+    def nullify_tables
+      @plan.nullify_tables +
+        @plan.tables.select(&:nested_tables?).flat_map { |table| table.nested_plan.nullify_tables }
+    end
+
     def validate_table_definition(table)
       unless (model = find_model_for_table(table))
         errors.add(:table, "#{table.name} has no model")

data/lib/db-purger/purge_table_helper.rb

@@ -10,9 +10,19 @@ module DBPurger
     private
 
     def purge_nested_tables(batch)
+      nullify_tables(batch) unless @table.nested_plan.nullify_tables.empty?
       purge_child_tables(batch) unless @table.nested_plan.child_tables.empty?
     end
 
+    # unlink rows that point at this batch before anything in it is deleted
+    def nullify_tables(batch)
+      ids = batch_values(batch, model.primary_key)
+
+      @table.nested_plan.nullify_tables.each do |table|
+        NullifyTable.new(@database, table, ids).nullify!
+      end
+    end
+
     def purge_child_tables(batch)
       ids = batch_values(batch, model.primary_key)
 
@@ -69,7 +79,7 @@ module DBPurger
     def explain(scope)
       sql =
         if @table.mark_deleted_field
-          explain_update_sql(scope)
+          explain_update_sql(scope, mark_deleted_field_quoted, mark_deleted_value_quoted)
         else
           scope.to_sql.sub(/SELECT .*?FROM/, 'DELETE FROM')
         end
@@ -77,10 +87,10 @@ module DBPurger
       scope.count
     end
 
-    def explain_update_sql(scope)
+    def explain_update_sql(scope, field_quoted, value_quoted)
       sql = scope.to_sql.dup
       sql.sub!(/SELECT .*?FROM/, 'UPDATE')
-      sql.sub!('WHERE', "SET #{mark_deleted_field_quoted} = #{mark_deleted_value_quoted} WHERE")
+      sql.sub!('WHERE', "SET #{field_quoted} = #{value_quoted} WHERE")
       sql
     end
 

data/lib/db-purger/table.rb

@@ -38,7 +38,8 @@ module DBPurger
 
     # nested tables that depend on this table's rows (search tables scan independently)
     def nested_key_tables?
-      nested_tables? && !(@nested_plan.child_tables.empty? && @nested_plan.parent_tables.empty?)
+      nested_tables? &&
+        !(@nested_plan.child_tables.empty? && @nested_plan.parent_tables.empty? && @nested_plan.nullify_tables.empty?)
     end
 
     def tables

data/lib/db-purger.rb

@@ -8,6 +8,7 @@ module DBPurger
   autoload :Executor, 'db-purger/executor'
   autoload :Metrics, 'db-purger/metrics'
   autoload :MetricSubscriber, 'db-purger/metric_subscriber'
+  autoload :NullifyTable, 'db-purger/nullify_table'
   autoload :PurgeTable, 'db-purger/purge_table'
   autoload :PurgeTableHelper, 'db-purger/purge_table_helper'
   autoload :PurgeTableScanner, 'db-purger/purge_table_scanner'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: db-purger
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Doug Youch
@@ -63,6 +63,7 @@ files:
 - lib/db-purger/executor.rb
 - lib/db-purger/metric_subscriber.rb
 - lib/db-purger/metrics.rb
+- lib/db-purger/nullify_table.rb
 - lib/db-purger/plan.rb
 - lib/db-purger/plan_builder.rb
 - lib/db-purger/plan_validator.rb