db-purger 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5b7144cdb2cda2433339283a8ca98c35ace1015c30fce5642a4c064234f07e72
+  data.tar.gz: 8b199fe73457afb3dabf37a2e6746cc839d42514a0ff81b99e0bd0ab62a1d2b5
+SHA512:
+  metadata.gz: 3c56fb34ca9276ad3e3e258825f703631f80ab6a1711a8e703feb4fb2b2d5676672c221f2f86ca8c310712928eb23705d86cb7fbdca8e01e56e2e0b653e859a3
+  data.tar.gz: 7283b99020d4164eadba078b9560921339c4e87d467bad5b0870991f6ed00f42be464db212949e6f9e083cc3d8d79e8bedada03bb3450bf36c2d57cfd5776d64

data/lib/db-purger/config.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::Config keeps track of global config options for the purge process
+  class Config
+    attr_writer :explain,
+                :explain_file
+
+    def explain?
+      @explain == true
+    end
+
+    def explain_file
+      (@explain_file || $stdout)
+    end
+  end
+end

data/lib/db-purger/metric_subscriber.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::MetricSubscriber tracks the progress of the Plan#purge process
+  class MetricSubscriber < ActiveSupport::Subscriber
+    def self.metrics
+      @metrics ||= Metrics.new
+    end
+
+    def self.reset!
+      metrics.reset!
+    end
+
+    def purge(event)
+      self.class.metrics.update_purge_stats(
+        event.payload[:table_name],
+        event.duration,
+        event.payload[:deleted]
+      )
+    end
+
+    def delete_records(event)
+      self.class.metrics.update_delete_records_stats(
+        event.payload[:table_name],
+        event.duration,
+        event.payload[:records_deleted],
+        event.payload[:num_records]
+      )
+    end
+
+    def next_batch(event)
+      self.class.metrics.update_lookup_stats(
+        event.payload[:table_name],
+        event.duration,
+        event.payload[:num_records]
+      )
+    end
+
+    def search_filter(event)
+      self.class.metrics.update_search_filter_stats(
+        event.payload[:table_name],
+        event.duration,
+        event.payload[:num_records],
+        event.payload[:num_records_selected]
+      )
+    end
+  end
+end

data/lib/db-purger/metrics.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::Metrics keeps track of each part of the purge process
+  class Metrics
+    attr_reader :started_at,
+                :finished_at,
+                :purge_stats,
+                :delete_stats,
+                :lookup_stats,
+                :filter_stats
+
+    def initialize
+      reset!
+    end
+
+    def reset!
+      @started_at = Time.now
+      @purge_stats = {}
+      @delete_stats = {}
+      @lookup_stats = {}
+      @filter_stats = {}
+      @finished_at = nil
+    end
+
+    def finished!
+      @finished_at = Time.now
+    end
+
+    def elapsed_time_in_seconds
+      (@finished_at || Time.now) - @started_at
+    end
+
+    def update_purge_stats(table_name, duration, num_records)
+      stats = (@purge_stats[table_name] ||= Hash.new(0))
+      stats[:duration] += duration
+      stats[:num_purges] += 1
+      stats[:num_records] += num_records
+      stats
+    end
+
+    def update_delete_records_stats(table_name, duration, num_deleted, num_expected_to_delete = nil)
+      stats = (@delete_stats[table_name] ||= Hash.new(0))
+      stats[:duration] += duration
+      stats[:num_delete_queries] += 1
+      stats[:num_deleted] += num_deleted
+      stats[:num_expected_to_delete] += num_expected_to_delete if num_expected_to_delete
+      stats
+    end
+
+    def update_lookup_stats(table_name, duration, records_found)
+      stats = (@lookup_stats[table_name] ||= Hash.new(0))
+      stats[:duration] += duration
+      stats[:num_lookups] += 1
+      stats[:records_found] += records_found
+      stats
+    end
+
+    def update_search_filter_stats(table_name, duration, records_found, records_selected)
+      stats = (@filter_stats[table_name] ||= Hash.new(0))
+      stats[:duration] += duration
+      stats[:num_lookups] += 1
+      stats[:records_found] += records_found
+      stats[:records_selected] += records_selected
+      stats
+    end
+  end
+end

data/lib/db-purger/plan.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::Plan is used to describe the relationship between tables
+  class Plan
+    attr_accessor :base_table
+
+    attr_reader :parent_tables,
+                :child_tables,
+                :ignore_tables,
+                :search_tables
+
+    def initialize
+      @parent_tables = []
+      @child_tables = []
+      @ignore_tables = []
+      @search_tables = []
+    end
+
+    def purge!(database, purge_value)
+      MetricSubscriber.reset!
+      PurgeTable.new(database, @base_table, @base_table.field, purge_value).purge!
+    end
+
+    def tables
+      all_tables = @base_table ? [@base_table] + @base_table.tables : []
+      all_tables += @parent_tables + @parent_tables.map(&:tables) +
+                    @child_tables + @child_tables.map(&:tables) +
+                    @search_tables + @search_tables.map(&:tables)
+      all_tables.flatten!
+      all_tables.compact!
+      all_tables
+    end
+
+    def table_names
+      tables.map(&:name)
+    end
+
+    def empty?
+      @base_table.nil? &&
+        @parent_tables.empty? &&
+        @child_tables.empty? &&
+        @search_tables.empty?
+    end
+
+    def ignore_table?(table_name)
+      @ignore_tables.any? do |ignore_table_name|
+        if ignore_table_name.is_a?(Regexp)
+          ignore_table_name.match(table_name)
+        else
+          ignore_table_name.to_s == table_name
+        end
+      end
+    end
+  end
+end

data/lib/db-purger/plan_builder.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::PlanBuilder is used to build the relationships between tables in a convenient way
+  class PlanBuilder
+    def initialize(plan)
+      @plan = plan
+    end
+
+    def base_table(table_name, field, options = {}, &block)
+      @plan.base_table = create_table(table_name, field, options, &block)
+    end
+
+    def parent_table(table_name, field, options = {}, &block)
+      table = create_table(table_name, field, options, &block)
+      if @plan.base_table
+        @plan.base_table.nested_plan.parent_tables << table
+      else
+        @plan.parent_tables << table
+      end
+      table
+    end
+
+    def child_table(table_name, field, options = {}, &block)
+      table = create_table(table_name, field, options, &block)
+      if @plan.base_table
+        @plan.base_table.nested_plan.child_tables << table
+      else
+        @plan.child_tables << table
+      end
+      table
+    end
+
+    def ignore_table(table_name)
+      @plan.ignore_tables << table_name
+    end
+
+    def purge_table_search(table_name, field, options = {}, &block)
+      table = create_table(table_name, field, options)
+      table.search_proc = block || raise('no block given for search_proc')
+      if @plan.base_table
+        @plan.base_table.nested_plan.search_tables << table
+      else
+        @plan.search_tables << table
+      end
+      table
+    end
+
+    def self.build(&block)
+      plan = Plan.new
+      helper = new(plan)
+      helper.instance_eval(&block)
+      plan
+    end
+
+    def build_nested_plan(table, &block)
+      helper = self.class.new(table.nested_plan)
+      helper.instance_eval(&block)
+    end
+
+    private
+
+    def create_table(table_name, field, options, &block)
+      table = Table.new(table_name, field)
+      table.foreign_key = options[:foreign_key]
+      table.batch_size = options[:batch_size]
+      table.conditions = options[:conditions]
+      build_nested_plan(table, &block) if block
+      table
+    end
+  end
+end

data/lib/db-purger/plan_validator.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::PlanValidator is used to ensure that all tables in the database are part of the purge plan.
+  #  It verifies that all table definitions are correct as well, checking that child/parent fields are part
+  #  of the table.
+  class PlanValidator
+    include ActiveModel::Validations
+
+    validate :validate_no_missing_tables
+    validate :validate_no_unknown_tables
+    validate :validate_tables
+
+    def initialize(database, plan)
+      @database = database
+      @plan = plan
+    end
+
+    # tables that are part of the database but not part of the plan
+    def missing_tables
+      database_table_names - @plan.table_names.map(&:to_s)
+    end
+
+    # tables that are part of the plan put not part of the database
+    def unknown_tables
+      @plan.table_names.map(&:to_s) - database_table_names
+    end
+
+    private
+
+    def validate_no_missing_tables
+      errors.add(:missing_tables, missing_tables.sort.join(',')) unless missing_tables.empty?
+    end
+
+    def validate_no_unknown_tables
+      errors.add(:unknown_tables, unknown_tables.sort.join(',')) unless unknown_tables.empty?
+    end
+
+    def validate_tables
+      @plan.tables.each { |table| validate_table_definition(table) }
+    end
+
+    def validate_table_definition(table)
+      unless (model = find_model_for_table(table))
+        errors.add(:table, "#{table.name} has no model")
+        return
+      end
+
+      table.fields.each do |field|
+        unless model.column_names.include?(field.to_s)
+          errors.add(:table, "#{table.name}.#{field} is missing in the database")
+        end
+      end
+    end
+
+    def find_model_for_table(table)
+      @database.models.detect { |model| model.table_name == table.name.to_s }
+    end
+
+    def filter_table_names(table_names)
+      table_names.reject { |table_name| @plan.ignore_table?(table_name) }
+    end
+
+    def database_table_names
+      @database_table_names ||= filter_table_names(@database.models.map(&:table_name))
+    end
+  end
+end

data/lib/db-purger/purge_table.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::PurgeTable is used to delete table from tables in batches if possible
+  class PurgeTable
+    include PurgeTableHelper
+
+    def initialize(database, table, purge_field, purge_value)
+      @database = database
+      @table = table
+      @purge_field = purge_field
+      @purge_value = purge_value
+      @num_deleted = 0
+    end
+
+    def model
+      @model ||= @database.models.detect { |m| m.table_name == @table.name.to_s }
+    end
+
+    def purge!
+      ActiveSupport::Notifications.instrument('purge.db_purger',
+                                              table_name: @table.name,
+                                              purge_field: @purge_field) do |payload|
+        if model.primary_key
+          purge_in_batches!
+        else
+          purge_all!
+        end
+        purge_search_tables
+        payload[:deleted] = @num_deleted
+      end
+      @num_deleted
+    end
+
+    private
+
+    def purge_all!
+      scope = model.where(@purge_field => @purge_value)
+      scope = scope.where(@table.conditions) if @table.conditions
+      delete_records_with_instrumentation(scope)
+    end
+
+    def purge_in_batches!
+      start_id = nil
+      until (batch = next_batch(start_id)).empty?
+        start_id = batch.last.send(model.primary_key)
+        purge_nested_tables(batch) if @table.nested_tables?
+        delete_records(batch)
+      end
+    end
+
+    def next_batch(start_id)
+      ActiveSupport::Notifications.instrument('next_batch.db_purger',
+                                              table_name: @table.name,
+                                              start_id: start_id) do |payload|
+        records = batch_scope(start_id).to_a
+        payload[:num_records] = records.size
+        records
+      end
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def batch_scope(start_id)
+      scope = model
+              .select([model.primary_key] + @table.foreign_keys)
+              .where(@purge_field => @purge_value)
+              .order(model.primary_key)
+              .limit(@table.batch_size)
+      scope = scope.where(@table.conditions) if @table.conditions
+      scope = scope.where("#{model.primary_key} > #{model.connection.quote(start_id)}") if start_id
+      scope
+    end
+    # rubocop:enable Metrics/AbcSize
+  end
+end

data/lib/db-purger/purge_table_helper.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::PurgeTableHelper keeps track of common code between purgers
+  module PurgeTableHelper
+    def model
+      @model ||= @database.models.detect { |m| m.table_name == @table.name.to_s }
+    end
+
+    private
+
+    def purge_nested_tables(batch)
+      purge_child_tables(batch) unless @table.nested_plan.child_tables.empty?
+      purge_parent_tables
+    end
+
+    def purge_child_tables(batch)
+      ids = batch_values(batch, model.primary_key)
+
+      @table.nested_plan.child_tables.each do |table|
+        next if table.foreign_key
+
+        PurgeTable.new(@database, table, table.field, ids).purge!
+      end
+    end
+
+    def purge_parent_tables
+      @table.nested_plan.parent_tables.each do |table|
+        PurgeTable.new(@database, table, table.field, @purge_value).purge!
+      end
+    end
+
+    def purge_foreign_tables(batch)
+      @table.nested_plan.child_tables.each do |table|
+        next unless table.foreign_key
+        next if (purge_values = batch_values(batch, table.foreign_key)).empty?
+
+        PurgeTable.new(@database, table, table.field, purge_values).purge!
+      end
+    end
+
+    def batch_values(batch, field)
+      batch.map { |record| record.send(field) }.compact
+    end
+
+    def foreign_tables?
+      @table.nested_plan.child_tables.any?(&:foreign_key)
+    end
+
+    def delete_records_with_instrumentation(scope, num_records = nil)
+      @num_deleted ||= 0
+      ActiveSupport::Notifications.instrument('delete_records.db_purger',
+                                              table_name: @table.name,
+                                              num_records: num_records) do |payload|
+        records_deleted =
+          if ::DBPurger.config.explain?
+            delete_sql = scope.to_sql.sub(/SELECT .*?FROM/, 'DELETE FROM')
+            ::DBPurger.config.explain_file.puts(delete_sql)
+            scope.count
+          else
+            scope.delete_all
+          end
+        @num_deleted += records_deleted
+        payload[:records_deleted] = records_deleted
+        payload[:deleted] = @num_deleted
+      end
+    end
+
+    def delete_records(batch)
+      if foreign_tables?
+        delete_records_and_foreign_tables(batch)
+      else
+        delete_records_by_primary_key(batch)
+      end
+    end
+
+    def delete_records_and_foreign_tables(batch)
+      model.transaction do
+        delete_records_by_primary_key(batch)
+        purge_foreign_tables(batch)
+      end
+    end
+
+    def delete_records_by_primary_key(batch)
+      ids = batch_values(batch, model.primary_key)
+      scope = model.where(model.primary_key => ids)
+      delete_records_with_instrumentation(scope, ids.size)
+    end
+
+    def purge_search_tables
+      @table.nested_plan.search_tables.each do |table|
+        PurgeTableScanner.new(@database, table).purge!
+      end
+    end
+  end
+end

data/lib/db-purger/purge_table_scanner.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::PurgeTableScanner scans the entire table in batches use the search_proc to determine what ids to delete
+  class PurgeTableScanner
+    include PurgeTableHelper
+
+    def initialize(database, table)
+      @database = database
+      @table = table
+      @num_deleted = 0
+    end
+
+    def model
+      @model ||= @database.models.detect { |m| m.table_name == @table.name.to_s }
+    end
+
+    def purge!
+      ActiveSupport::Notifications.instrument('purge.db_purger',
+                                              table_name: @table.name) do |payload|
+        purge_in_batches!
+        purge_search_tables
+        payload[:deleted] = @num_deleted
+      end
+      @num_deleted
+    end
+
+    private
+
+    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/MethodLength
+    def purge_in_batches!
+      scope = model
+      scope = scope.where(@table.conditions) if @table.conditions
+
+      instrumentation_name = 'next_batch.db_purger'
+      start_instrumentation(instrumentation_name)
+
+      scope.find_in_batches(batch_size: @table.batch_size) do |batch|
+        finish_instrumentation(
+          instrumentation_name,
+          table_name: @table.name,
+          num_records: batch.size
+        )
+
+        batch = ActiveSupport::Notifications.instrument('search_filter.db_purger',
+                                                        table_name: @table.name,
+                                                        num_records: batch.size) do |payload|
+          records_selected = @table.search_proc.call(batch)
+          payload[:num_records_selected] = records_selected.size
+          records_selected
+        end
+
+        if batch.empty?
+          start_instrumentation(instrumentation_name)
+          next
+        end
+
+        purge_nested_tables(batch) if @table.nested_tables?
+        delete_records(batch)
+
+        start_instrumentation(instrumentation_name)
+      end
+
+      finish_instrumentation(
+        instrumentation_name,
+        table_name: @table.name,
+        num_records: 0,
+        num_selected: 0
+      )
+    end
+    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/MethodLength
+
+    def start_instrumentation(name)
+      @instrumenter = ActiveSupport::Notifications.instrumenter
+      @instrumenter.start(name, {})
+    end
+
+    def finish_instrumentation(name, payload)
+      @instrumenter.finish(name, payload)
+    end
+  end
+end

data/lib/db-purger/table.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module DBPurger
+  # DBPurger::Table is an entity to keep track of basic table data
+  class Table
+    DEFAULT_BATCH_SIZE = 10_000
+
+    attr_accessor :foreign_key,
+                  :batch_size,
+                  :conditions,
+                  :search_proc
+
+    attr_reader :name,
+                :field
+
+    def initialize(name, field)
+      @name = name
+      @field = field
+      @batch_size = DEFAULT_BATCH_SIZE
+    end
+
+    def nested_plan(&block)
+      @nested_plan ||= Plan.new
+      PlanBuilder.new(@nested_plan).build_nested_plan(self, &block) if block
+      @nested_plan
+    end
+
+    def nested_tables?
+      @nested_plan != nil
+    end
+
+    def tables
+      @nested_plan ? @nested_plan.tables : []
+    end
+
+    def foreign_keys
+      @nested_plan ? @nested_plan.tables.map(&:foreign_key).compact : []
+    end
+
+    def fields
+      [@field] + foreign_keys
+    end
+  end
+end

data/lib/db-purger.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# DBPurger is a tool to delete data from tables based on a initial purge value
+module DBPurger
+  autoload :Config, 'db-purger/config'
+  autoload :Metrics, 'db-purger/metrics'
+  autoload :MetricSubscriber, 'db-purger/metric_subscriber'
+  autoload :PurgeTable, 'db-purger/purge_table'
+  autoload :PurgeTableHelper, 'db-purger/purge_table_helper'
+  autoload :PurgeTableScanner, 'db-purger/purge_table_scanner'
+  autoload :Plan, 'db-purger/plan'
+  autoload :PlanBuilder, 'db-purger/plan_builder'
+  autoload :PlanValidator, 'db-purger/plan_validator'
+  autoload :Table, 'db-purger/table'
+
+  def self.config
+    @config ||= Config.new
+  end
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: db-purger
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Doug Youch
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-07-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dynamic-active-model
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: inheritance-helper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Purge database tables by top level id in batches
+email: dougyouch@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/db-purger.rb
+- lib/db-purger/config.rb
+- lib/db-purger/metric_subscriber.rb
+- lib/db-purger/metrics.rb
+- lib/db-purger/plan.rb
+- lib/db-purger/plan_builder.rb
+- lib/db-purger/plan_validator.rb
+- lib/db-purger/purge_table.rb
+- lib/db-purger/purge_table_helper.rb
+- lib/db-purger/purge_table_scanner.rb
+- lib/db-purger/table.rb
+homepage: https://github.com/dougyouch/db-purger
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: DB Purger by top level id
+test_files: []