prune_ar 0.1.0

data/lib/prune_ar/pruner.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'active_record'
+require 'prune_ar/belongs_to_association_gatherer'
+require 'prune_ar/orphaned_selection_builder'
+require 'prune_ar/deleter_by_criteria'
+require 'prune_ar/foreign_key_handler'
+
+module PruneAr
+  # Core of this gem. Prunes records based on parameters given.
+  class Pruner
+    attr_reader :associations,
+                :deletion_criteria,
+                :full_delete_models,
+                :pre_queries_to_run,
+                :conjunctive_deletion_criteria,
+                :logger,
+                :foreign_key_handler,
+                :perform_sanity_check
+
+    def initialize(
+      models:,
+      deletion_criteria: {},
+      full_delete_models: [],
+      pre_queries_to_run: [],
+      conjunctive_deletion_criteria: {},
+      perform_sanity_check: true,
+      logger: Logger.new(STDOUT).tap { |l| l.level = Logger::WARN }
+    )
+      @associations = BelongsToAssociationGatherer.new(models, connection: connection).associations
+      @full_delete_models = full_delete_models
+      @pre_queries_to_run = pre_queries_to_run
+      @deletion_criteria = deletion_criteria
+      @conjunctive_deletion_criteria = conjunctive_deletion_criteria
+      @logger = logger
+      @perform_sanity_check = perform_sanity_check
+      @foreign_key_handler = ForeignKeyHandler.new(
+        models: models,
+        connection: connection,
+        logger: logger
+      )
+    end
+
+    def prune
+      # Can't wrap in transaction if DDL is not supported in transactions. We use ALTER TABLE
+      # => statements (which are DDL)
+      return prune_core unless connection.supports_ddl_transactions?
+
+      # This transaction is helpful when developing/working on this code. If a SQL statement errors,
+      # => it leaves the database untouched.
+      connection.transaction do
+        prune_core
+      end
+    end
+
+    private
+
+    def prune_core
+      drop_original_foreign_key_constraints
+
+      # Run any pre-queries we were told to run
+      pre_queries
+
+      # Delete using deletion_criteria before we sanitize anything
+      pre_delete
+
+      # Truncate tables we were told to wipe completely
+      full_delete
+
+      # Main deletion (conjunctive_deletion_criteria + orphaned records)
+      main_delete
+
+      # Now that there are no violations, create foreign key constraints on all :belongs_to
+      # => this is a sanity check that we eliminated violations
+      # => this ignores polymorphic relations since FKs cannot be set on polymorphic :belongs_to
+      # => these foreign key constraints are dropped right after since they're only for sanity
+      foreign_key_sanity_check if perform_sanity_check
+
+      recreate_original_foreign_key_constraints
+    end
+
+    def drop_original_foreign_key_constraints
+      logger.info('dropping existing foreign key constraints')
+      foreign_key_handler.drop(foreign_key_handler.original_foreign_keys)
+    end
+
+    def recreate_original_foreign_key_constraints
+      logger.info('recreating original foreign key constraints')
+      foreign_key_handler.create(foreign_key_handler.original_foreign_keys)
+    end
+
+    def pre_queries
+      logger.info('running pre_queries_to_run')
+      pre_queries_to_run.each do |sql|
+        logger.debug("running pre-query #{sql}")
+        connection.exec_query(sql)
+      end
+    end
+
+    def pre_delete
+      logger.info('deleting via deletion_criteria')
+      DeleterByCriteria.new(
+        flatten_deletion_criteria(deletion_criteria),
+        connection: connection,
+        logger: logger
+      ).delete
+    end
+
+    def full_delete
+      logger.info('truncating full_delete_models')
+      full_delete_models.each do |model|
+        logger.debug("truncating #{model}")
+        connection.exec_query("TRUNCATE #{model.table_name};")
+      end
+    end
+
+    def main_delete
+      logger.info('deleting via conjunctive_deletion_criteria & pruning orphaned records')
+      association_deletion_criteria = flat_associations_deletion_criteria(associations)
+      flat_conjunctive_criteria = flatten_deletion_criteria(conjunctive_deletion_criteria)
+      DeleterByCriteria.new(
+        (association_deletion_criteria + flat_conjunctive_criteria).sort,
+        connection: connection,
+        logger: logger
+      ).delete
+    end
+
+    def foreign_key_sanity_check
+      logger.info('sanity checking via foreign key constraints')
+      created_foreign_keys = foreign_key_handler.create_from_belongs_to_associations(
+        associations.reject(&:polymorphic?)
+      )
+
+      foreign_key_handler.drop(created_foreign_keys)
+    end
+
+    def flat_associations_deletion_criteria(associations)
+      associations.map do |assoc|
+        [assoc.source_table, orphaned_selection_builder.orphaned_selection(assoc)]
+      end
+    end
+
+    def flatten_deletion_criteria(criteria)
+      criteria.flat_map do |model, selections|
+        selections.map { |s| [model.table_name, s] }
+      end
+    end
+
+    def connection
+      @connection ||= ActiveRecord::Base.connection
+    end
+
+    def orphaned_selection_builder
+      @orphaned_selection_builder ||= OrphanedSelectionBuilder.new
+    end
+  end
+end

data/lib/prune_ar/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module PruneAr
+  VERSION = '0.1.0'
+end

data/prune_ar.gemspec

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'prune_ar/version'
+
+Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
+  spec.name          = 'prune_ar'
+  spec.version       = PruneAr::VERSION
+  spec.authors       = ['Anirban Mukhopadhyay']
+  spec.email         = ['amukhopadhyay@contently.com']
+
+  spec.summary       = 'Prunes database tables using ActiveRecord belongs_to relations.'
+  spec.description   = %w[
+    Given an initial set of records to delete prune_ar deletes all other records (accessible via
+    ActiveRecord) that are now orphaned due to a belongs_to relation which is now non-existent.
+    This allows you to safely delete records that you want to delete without creating orphaned
+    records in another table (& without violating foreign key constraints if you use them). This
+    can be used to prune a production database (given deletion criteria for top level parent-less
+    independent entities) for use in a development environment without compromising customer data.
+  ].join(' ')
+  spec.homepage = 'https://github.com/contently/prune_ar'
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+  else
+    raise 'RubyGems 2.0 or newer is required to protect against ' \
+      'public gem pushes.'
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.files.reject! { |f| f.start_with?('.') }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activerecord'
+
+  spec.add_development_dependency 'bundler', '~> 1.17'
+  spec.add_development_dependency 'database_cleaner', '~> 1.7'
+  spec.add_development_dependency 'mysql2', '~> 0.5'
+  spec.add_development_dependency 'pg', '~>1.1'
+  spec.add_development_dependency 'pry', '~> 0.12'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.8'
+  spec.add_development_dependency 'rspec_junit_formatter', '~> 0.4'
+  spec.add_development_dependency 'rubocop', '~> 0.61'
+  spec.add_development_dependency 'rubocop-junit_formatter', '~> 0.2'
+  spec.add_development_dependency 'simplecov', '~> 0.12'
+  spec.add_development_dependency 'sqlite3', '~> 1.3'
+end

metadata

@@ -0,0 +1,250 @@
+--- !ruby/object:Gem::Specification
+name: prune_ar
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Anirban Mukhopadhyay
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-12-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.61'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.61'
+- !ruby/object:Gem::Dependency
+  name: rubocop-junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: Given an initial set of records to delete prune_ar deletes all other
+  records (accessible via ActiveRecord) that are now orphaned due to a belongs_to
+  relation which is now non-existent. This allows you to safely delete records that
+  you want to delete without creating orphaned records in another table (& without
+  violating foreign key constraints if you use them). This can be used to prune a
+  production database (given deletion criteria for top level parent-less independent
+  entities) for use in a development environment without compromising customer data.
+email:
+- amukhopadhyay@contently.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTING.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/prune_ar.rb
+- lib/prune_ar/belongs_to_association.rb
+- lib/prune_ar/belongs_to_association_gatherer.rb
+- lib/prune_ar/deleter_by_criteria.rb
+- lib/prune_ar/foreign_key_handler.rb
+- lib/prune_ar/orphaned_selection_builder.rb
+- lib/prune_ar/pruner.rb
+- lib/prune_ar/version.rb
+- prune_ar.gemspec
+homepage: https://github.com/contently/prune_ar
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Prunes database tables using ActiveRecord belongs_to relations.
+test_files: []