yeet_dba 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32599ee97b8b627e2a093054e31eed282e2141e93569723b98b40357781d1399
-  data.tar.gz: 2ee79b2e47bee2c889ccee622be08de60f84d5f8daff92585b1cf52e5880e98c
+  metadata.gz: 8d312f90603469c805f7d69d49ae5ed3fe8f825ef64c0a324f74a49557934804
+  data.tar.gz: faf652821f509fee8db20df69f426ac5e80af5e4be0d8bea4a578ce2d4130839
 SHA512:
-  metadata.gz: 8d36e9b4683ea1a9f09238db879e29383ec53fa0e6c28dc4e722738831264833d361a30f2cc6647ad84904e695da4033d7fe0ef5c2a7abc86c59e9a73b30c1f2
-  data.tar.gz: 22fb42465caa8ec621aa472924eeb27c5f14e78a5abe9a858a6f48064d19af3651a42f5d65bd7170608bc8b4078cb0baa970204592cb04942d5993925ab84240
+  metadata.gz: 94aa0ebb9b45927eacb6ec29ce75678ea83fbbb26b48503d72cf77e9581e3d8e8d321f1b360ada978bd54aabd54d3cb94023241e61fd6968908f86def82e0b8c
+  data.tar.gz: fb1e94beb1c6a44d9debac764069cbdfceecf490c3f5d5fc24396a0f4c5529b5b065cf8575a698db0c9e11193df01a00065070ba7ae01180ebbb6ec31f08644d

data/.rubocop

@@ -0,0 +1,159 @@
+AllCops:
+  DisplayCopNames: true
+  TargetRubyVersion: 2.5
+  TargetRailsVersion: 5.1
+
+Rails:
+  Enabled: true
+
+Style/FormatStringToken:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Metrics/LineLength:
+  Max: 100
+
+Metrics/AbcSize:
+  Max: 40
+
+Rails/SkipsModelValidations:
+  Enabled: true
+
+Style/GuardClause:
+  Enabled: false
+
+Style/AsciiComments:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 25
+
+Metrics/CyclomaticComplexity:
+  Max: 7
+
+Metrics/PerceivedComplexity:
+  Max: 8
+
+# Relaxed.Ruby.Style
+
+Style/Alias:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylealias
+
+Metrics/BlockLength:
+  Enabled: true
+
+Style/BeginBlock:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylebeginblock
+
+Style/BlockDelimiters:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleblockdelimiters
+
+Style/Documentation:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styledocumentation
+
+Layout/DotPosition:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styledotposition
+
+Style/DoubleNegation:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styledoublenegation
+
+Style/EndBlock:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleendblock
+
+Style/FormatString:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleformatstring
+
+Style/IfUnlessModifier:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleifunlessmodifier
+
+Style/Lambda:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylelambda
+
+Style/ModuleFunction:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylemodulefunction
+
+Style/MultilineBlockChain:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylemultilineblockchain
+
+Style/NegatedIf:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylenegatedif
+
+Style/NegatedWhile:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylenegatedwhile
+
+Style/ParallelAssignment:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleparallelassignment
+
+Style/PercentLiteralDelimiters:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylepercentliteraldelimiters
+
+Style/PerlBackrefs:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#styleperlbackrefs
+
+Style/Semicolon:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylesemicolon
+
+Style/SignalException:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylesignalexception
+
+Style/SingleLineBlockParams:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylesinglelineblockparams
+
+Style/SingleLineMethods:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylesinglelinemethods
+
+Layout/SpaceBeforeBlockBraces:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylespacebeforeblockbraces
+
+Layout/SpaceInsideParens:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylespaceinsideparens
+
+Style/SpecialGlobalVars:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylespecialglobalvars
+
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+
+Style/TrailingCommaInHashLiteral:
+  Enabled: false
+
+Style/WhileUntilModifier:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#stylewhileuntilmodifier
+
+Style/RegexpLiteral:
+  Enabled: false
+
+Lint/AmbiguousRegexpLiteral:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#lintambiguousregexpliteral
+
+Lint/AssignmentInCondition:
+  Enabled: false
+  StyleGuide: http://relaxed.ruby.style/#lintassignmentincondition

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+# 0.1.2
+
+## Major changes
+
+Added rake task to find invalid data
+Added rake task to nullify and delete invalid data
+
+## Minor changes
+
+Add rubocop
+Add rspec tests
+
+# 0.1.1
+
+Patch bug with skipping invalid columns that have orphaned data
+
 # 0.1.0
 
 Initial release

data/Gemfile

@@ -1,6 +1,15 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in yeet_dba.gemspec
 gemspec
+
+gem 'actionpack'
+gem 'activerecord'
+gem 'railties'
+gem 'rspec-rails'
+gem 'sqlite3'
+gem 'test-unit'
+
+gem 'pry', require: true

data/Gemfile.lock

@@ -0,0 +1,118 @@
+PATH
+  remote: .
+  specs:
+    yeet_dba (0.1.2)
+      actionpack (>= 3.0, < 6.0)
+      activerecord (>= 3.0, < 6.0)
+      railties (>= 3.0, < 6.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (5.2.3)
+      actionview (= 5.2.3)
+      activesupport (= 5.2.3)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.2.3)
+      activesupport (= 5.2.3)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activemodel (5.2.3)
+      activesupport (= 5.2.3)
+    activerecord (5.2.3)
+      activemodel (= 5.2.3)
+      activesupport (= 5.2.3)
+      arel (>= 9.0)
+    activesupport (5.2.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (9.0.0)
+    builder (3.2.3)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.5)
+    crass (1.0.4)
+    diff-lcs (1.3)
+    erubi (1.8.0)
+    i18n (1.6.0)
+      concurrent-ruby (~> 1.0)
+    loofah (2.2.3)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    method_source (0.9.2)
+    mini_portile2 (2.4.0)
+    minitest (5.11.3)
+    nokogiri (1.10.2)
+      mini_portile2 (~> 2.4.0)
+    power_assert (1.1.4)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rack (2.0.6)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
+    railties (5.2.3)
+      actionpack (= 5.2.3)
+      activesupport (= 5.2.3)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.19.0, < 2.0)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-rails (3.8.2)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    sqlite3 (1.4.0)
+    test-unit (3.3.1)
+      power_assert
+    thor (0.20.3)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  actionpack
+  activerecord
+  bundler (~> 1.17)
+  pry
+  railties
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rspec-rails
+  sqlite3
+  test-unit
+  yeet_dba!
+
+BUNDLED WITH
+   1.17.3

data/README.md

@@ -1,12 +1,26 @@
 ![Foreign Key by Ary Prasetyo from the Noun Project](./yeet_dba.png)
 
 # yeet_dba - find missing foreign key constraints
+[![Gem Version](https://badge.fury.io/rb/yeet_dba.svg)](https://badge.fury.io/rb/yeet_dba) <a href="https://codeclimate.com/github/KevinColemanInc/yeet_dba/maintainability"><img src="https://api.codeclimate.com/v1/badges/a0baa6373d4be7f0d630/maintainability" /></a>[![Build Status](https://travis-ci.com/KevinColemanInc/yeet_dba.svg?branch=master)](https://travis-ci.com/KevinColemanInc/yeet_dba)
 
 yeet_dba scans your rails tables for missing foreign key constraints. If there are no dangling records, it will create a migration to add the foreign key constraints on all the table it is safe.
 
 If you have dangling migrations, check the generator logs to see where you have invalid orphaned rows. Orphaned row meaning a row with an id that doesn't exist in the associated table.
 
-but [why should I use foreign keys?](https://softwareengineering.stackexchange.com/questions/375704/why-should-i-use-foreign-keys-in-database)
+### But why should I use foreign keys?
+
+You can save yourself an N+1 call by checking if the id has a value instead of loading up the object.
+
+```ruby
+user.company.id # bad - N+1
+user.company_id # good
+```
+
+But this doesn't work if you don't nullify the `company_id` when the company is deleted. Foreign key constraints prevent you from deleting a record without cleaning out the associated tables.
+
+### But what is the difference between yeet_db and [lol_dba](https://github.com/plentz/lol_dba)?
+
+lol_dba will only add indexes for RoR models. yeet_dba looks at every table (including join tables) to add foreign key constraints, which also add indexes.
 
 ## Installation
 
@@ -22,6 +36,8 @@ And then execute:
 
 ## Usage
 
+### Foriegn keys migration
+
 This probably should run against the production database so you can know if there are dangling records. If there are records with a value, but not the corresponding table does not have an id, then the migration will fail.
 
 ```
@@ -30,27 +46,97 @@ $ RAILS_ENV=production rails g yeet_dba:foreign_key_migration
 
 This will create a new migration with for every foreign_key that can safely be added without running into orphaned data errors. We also warn you if active_record models that are missing association declarations (`has_many`, `belongs_to`, etc.)`
 
-`WARNING - cannot find association for alternative_housings . supplier_id | suppliers`
+`WARNING - cannot find an association for alternative_housings . supplier_id | suppliers`
 
-We also warn if we have tables that don't have existing models attached to them. This can be safe to ignore, because join tables on many to many relations don't need models, but ideally everything should have an AR model backing it.
+We also warn if we have tables that don't have existing models attached to them. This can be safe to ignore because join tables on many to many relations don't need models, but ideally, everything should have an AR model backing it.
 
-`WARNING - cannot find model for alternative_housings . supplier_id | suppliers`
+`WARNING - cannot find a model for alternative_housings . supplier_id | suppliers`
 
 Finnally, if there is a table that we think should have a foreign key constraint, but there are dangling values we warn you against that too.
 
 `WARNING - orphaned rows alternative_housings . supplier_id | suppliers`
 
-## Requirements
+### Invalid rows
+
+If a row has an id, but there doesn't exist an id the expected associated table, then the row has bad data and should either be fixed by nulling the orphaned row or assigning it to an existing row.
+
+This rake task will scan every column for orphaned rows.
+
+```
+$ RAILS_ENV=production rake yeet_dba:find_invalid_columns
+```
+
+Sample output:
+
+```
+---RESULTS---
+
+🚨Houston, we have a problem 🚨. We found 1 invalid column.
+
+-> notifications.primary_image_id
+Invalid rows:   83
+Foreign table:  active_storage_attachments
+
+This query should return no results:
+SELECT "notifications".* FROM "notifications" left join active_storage_attachments as association_table on association_table.id = notifications.primary_image_id WHERE "notifications"."primary_image_id" IS NOT NULL AND (association_table.id is null)
+
+```
+
+### Fix invalid rows
+
+If a row has an id, but there doesn't exist an id the expected associated table, then the row has bad data and should either be fixed by nulling the orphaned row or assigning it to an existing row.
+
+This rake task will scan every column for orphaned rows.
+
+```
+$ RAILS_ENV=production rake yeet_dba:fix_invalid_columns
+```
+
+Sample output:
+
+```
+---RESULTS---
+
+🚨Houston, we have a problem 🚨. We found 1 invalid column.
+
+-> notifications.primary_image_id
+Invalid rows:   83
+Foreign table:  active_storage_attachments
 
-Rails 5.2 (but it may work with 5.0+)
+This query should return no results:
+SELECT "notifications".* FROM "notifications" left join active_storage_attachments as association_table on association_table.id = notifications.primary_image_id WHERE "notifications"."primary_image_id" IS NOT NULL AND (association_table.id is null)
+
+```
+
+### Add missing foriegn keys as a rake task
+
+You might want to add foreign keys outside of your regular deployment flow in case there are failures and deployment would be blocked by bad data. This would be especially obnoxious for MySql users since you can't rollback migrations.
+
+```
+$ RAILS_ENV=production rake yeet_dba:add_foreign_keys
+```
+
+Sample output
+
+```
+ERROR - users . profile_id failed to add key
+```
+
+This rake task is idempotent (safe to run as many times as you need).
+
+## Compatibility
+
+- Rails 5.2 (but it may work with 5.0+)
+- Ruby 2.4+
 
 ## Road map to v1
 
-- [ ] rspec tests
-- [ ] add rake task identify all dangling records
-- [ ] add rake task to automatically nullify or destroy dangling records
-- [ ] run as a rake task
+- [x] rspec tests
+- [x] add rake task identify all dangling records
+- [x] add rake task to automatically nullify or destroy dangling records
+- [x] run adding foreign keys as rake task instead of generating a migration
 - [ ] support "soft delete" gems
+- [ ] Use rails associations to find columns that should be "not null" to [improve performance](https://stackoverflow.com/questions/1017239/how-do-null-values-affect-performance-in-a-database-search)
 
 
 ## Development
@@ -72,4 +158,12 @@ The gem is available as open source under the terms of the [MIT License](https:/
 Everyone interacting in the YeetDb project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/kevincolemaninc/yeet_dba/blob/master/CODE_OF_CONDUCT.md).
 
 ## Logo design attribute
-Foreign Key by Ary Prasetyo from the Noun Project
+Foreign Key by Ary Prasetyo from the Noun Project
+
+## Thanks
+
+[AvoVietnam - Chat with Vietnamese](https://www.avovietnam.com)
+
+## Author
+
+Kevin Coleman, [https://kcoleman.me/](https://kcoleman.me)

data/Rakefile

@@ -1,6 +1,6 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "yeet_dbaa"
+require 'bundler/setup'
+require 'yeet_dba'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +10,5 @@ require "yeet_dbaa"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start(__FILE__)

data/lib/generators/yeet_dba/foreign_key_migration_generator.rb

@@ -4,11 +4,11 @@ module YeetDba
   # Custom scaffolding generator
   class ForeignKeyMigrationGenerator < Rails::Generators::Base
     include Rails::Generators::Migration
-    source_root File.expand_path('../templates', __FILE__)
-    desc "Generates migration for adding foreign key constraints."   
+    source_root File.expand_path('templates', __dir__)
+    desc 'Generates migration for adding foreign key constraints.'
 
     def copy_migration_and_spec_files
-      migration_template "add_foreign_keys_yeet_dba.rb",
+      migration_template 'add_foreign_keys_yeet_dba.rb',
                          migration_file,
                          migration_version: migration_version
     end
@@ -16,15 +16,15 @@ module YeetDba
     private
 
     def migration_file
-      File.join("db/migrate", "add_foreign_keys_yeet_dba.rb")
+      File.join('db/migrate', 'add_foreign_keys_yeet_dba.rb')
     end
 
     def migration_version
       "[#{Rails::VERSION::MAJOR}.#{Rails::VERSION::MINOR}]"
     end
 
-    def self.next_migration_number(path)
-      Time.now.utc.strftime("%Y%m%d%H%M%S%L")
+    def self.next_migration_number(_path)
+      Time.now.utc.strftime('%Y%m%d%H%M%S%L')
     end
   end
-end
+end

data/lib/yeet_dba/Rakefile

@@ -0,0 +1,5 @@
+# lib/Rakefile
+require 'yeet_dba'
+
+path = File.expand_path(__dir__)
+Dir.glob("#{path}/tasks/**/*.rake").each { |f| import f }

data/{yeet_dba/ar_column.rb → lib/yeet_dba/column.rb}

@@ -1,19 +1,19 @@
 module YeetDba
-  class ArColumn
-    attr_accessor :column_name, :table_name, :tables
+  class Column
+    attr_accessor :db_column, :table_name, :tables
 
-    def initialize(column_name:, table_name:, tables:)
-      @column_name = column_name
+    def initialize(db_column:, table_name:, tables:)
+      @db_column = db_column
       @table_name = table_name
       @tables = tables
     end
 
     def is_association?
-      column_name.name =~ /_id\z/
+      db_column.name =~ /_id\z/
     end
 
     def association_klass
-      model && model.reflections[association_name]&.klass
+      association&.klass
     end
 
     def association_table_name
@@ -21,7 +21,7 @@ module YeetDba
     end
 
     def association_name
-      column_name.name.gsub(/_id\z/, '')
+      db_column.name.gsub(/_id\z/, '')
     end
 
     def model
@@ -37,7 +37,7 @@ module YeetDba
     end
 
     def foreign_key_exists?
-      ActiveRecord::Migration.foreign_key_exists?(table_name, column: column_name.name)
+      ActiveRecord::Migration.foreign_key_exists?(table_name, column: db_column.name)
     end
 
     def guessed_table_name

data/lib/yeet_dba/missing_foreign_keys.rb

@@ -0,0 +1,27 @@
+module YeetDba
+  class MissingForeignKeys
+    def self.foreign_keys
+      eager_load!
+      tables.map do |table_name|
+        Table.new(table_name: table_name,
+                  tables: tables).missing_keys
+      end.flatten
+    end
+
+    def self.invalid_columns
+      eager_load!
+      tables.map do |table_name|
+        Table.new(table_name: table_name,
+                  tables: tables).invalid_columns
+      end.flatten
+    end
+
+    def self.eager_load!
+      Rails.application.eager_load! if defined?(Rails) && !Rails.env.test?
+    end
+
+    def self.tables
+      ActiveRecord::Base.connection.tables
+    end
+  end
+end

data/{yeet_dba → lib/yeet_dba/models}/foreign_key.rb

@@ -10,4 +10,4 @@ module YeetDba
       @column = column
     end
   end
-end
+end

data/lib/yeet_dba/models/invalid_column.rb

@@ -0,0 +1,20 @@
+module YeetDba
+  class InvalidColumn
+    attr_accessor :table_name,
+                  :column,
+                  :verify_data
+
+    def initialize(table_name:, column:, verify_data:)
+      @table_name = table_name
+      @column = column
+      @verify_data = verify_data
+    end
+
+    delegate :association_table_name, :db_column, :association, to: :column
+    delegate :orphaned_rows_count, :query, to: :verify_data
+
+    def to_s
+      "#{table_name} . #{db_column.name} has #{orphaned_rows_count} invalid rows with foreign table #{association_table_name}"
+    end
+  end
+end

data/lib/yeet_dba/railtie.rb

@@ -0,0 +1,12 @@
+require 'rails'
+
+module YeetDba
+  class Railtie < Rails::Railtie
+    railtie_name :yeet_dba
+
+    rake_tasks do
+      path = File.expand_path(__dir__)
+      Dir.glob("#{path}/tasks/**/*.rake").each { |f| load f }
+    end
+  end
+end

data/lib/yeet_dba/table.rb

@@ -0,0 +1,67 @@
+module YeetDba
+  class Table
+    attr_accessor :table_name, :tables
+
+    def initialize(table_name:, tables:)
+      @table_name = table_name
+      @tables = tables
+    end
+
+    def invalid_columns
+      missing_keys_array = []
+      columns.each do |db_column|
+        column = Column.new(db_column: db_column, table_name: table_name, tables: tables)
+        next unless column.is_association?
+        next if column.polymorphic_association?
+        next if column.foreign_key_exists?
+        next if column.association_table_name.blank?
+        verify_data = VerifyData.new(column: column)
+        next unless verify_data.orphaned_rows?
+
+        invalid_column = InvalidColumn.new(table_name: table_name,
+                                           column: column,
+                                           verify_data: verify_data)
+        missing_keys_array.push(invalid_column)
+
+      end
+      missing_keys_array
+    end
+
+    def missing_keys
+      missing_keys_array = []
+      columns.each do |db_column|
+        column = Column.new(db_column: db_column, table_name: table_name, tables: tables)
+        next unless column.is_association?
+
+        unless column.model
+          puts "WARNING - cannot find a model for #{table_name} . #{db_column.name} | #{column&.association_table_name}"
+        end
+
+        unless column.association
+          puts "WARNING - cannot find an association for #{table_name} . #{db_column.name} | #{column&.association_table_name}"
+        end
+
+        next if column.polymorphic_association?
+        next if column.foreign_key_exists?
+        next if column.association_table_name.blank?
+
+        if VerifyData.new(column: column).orphaned_rows?
+          puts "YeetDba - orphaned rows. Skipping #{table_name} . #{db_column.name} | #{column&.association_table_name}"
+          next
+        end
+
+        foreign_key = ForeignKey.new(table_a: table_name,
+                                     table_b: column&.association_table_name,
+                                     column: db_column.name)
+        missing_keys_array.push(foreign_key)
+      end
+      missing_keys_array
+    end
+
+    private
+
+    def columns
+      ActiveRecord::Base.connection.columns(table_name)
+    end
+  end
+end

data/lib/yeet_dba/tasks/add_foreign_keys.rake

@@ -0,0 +1,20 @@
+namespace :yeet_dba do
+  desc 'Add foreign keys in a rake migration'
+  task add_foreign_keys: :environment do
+    foreign_keys = YeetDba::MissingForeignKeys.foreign_keys
+
+    puts "Trying to add #{foreign_keys.length}"
+    puts
+    foreign_keys.each do |foreign_key|
+      
+      begin
+        ActiveRecord::Migration.add_foreign_key(foreign_key.table_a,
+                                                foreign_key.table_b,
+                                                column: foreign_key.column)
+
+      rescue ActiveRecord::InvalidForeignKey
+        puts "ERROR - #{foreign_key.table_a} . #{foreign_key.column} failed to add key"
+      end
+    end
+  end
+end

data/lib/yeet_dba/tasks/bad_data/find_orphaned_rows.rake

@@ -0,0 +1,55 @@
+namespace :yeet_dba do
+  desc 'Show all of the tables.columns with bad data'
+  task find_invalid_columns: :environment do
+    columns = YeetDba::MissingForeignKeys.invalid_columns
+    puts
+    puts '---RESULTS---'
+    puts
+    if columns.empty?
+      puts 'All good here. 👍'
+    else
+      puts "🚨Houston, we have a problem 🚨. We found #{columns.length} invalid column#{columns.length == 1 ? '' : 's'}."
+      puts
+      columns.each do |invalid_column|
+        puts "-> #{invalid_column.table}.#{invalid_column.column}"
+        puts "Invalid rows:   #{invalid_column.orphaned_rows_count}"
+        puts "Foreign table:  #{invalid_column.association_table_name}"
+        puts
+        puts 'This query should return no results:'
+        puts invalid_column.query
+        puts
+      end
+    end
+  end
+
+  desc 'Set all of the rows to null if there is bad data'
+  task nullify_or_destroy_invalid_rows: :environment do
+    columns = YeetDba::MissingForeignKeys.invalid_columns
+    next puts "Your data looks good!" if columns.empty?
+
+    columns.each do |column|
+      puts column.to_s
+    end
+    puts
+    puts "WARNING - THIS MAY CAUSE PERM DATA LOSS"
+    puts
+    puts "I am going to give you 8s to change your mind"
+    sleep 8
+    puts "ok, here we go..."
+    sleep 1
+
+    columns.each do |invalid_column|
+      required = invalid_column.column.association.options&.key?(:optional) ? !invalid_column.column.association.options[:optional] : invalid_column.column.model.belongs_to_required_by_default
+      nullable = invalid_column.column.db_column.null
+      if required
+        # delete
+        invalid_column.verify_data.orphaned_rows.destroy_all
+      elsif nullable
+        # null it out
+        invalid_column.verify_data.orphaned_rows.update_all(invalid_column.db_column.name => nil)
+      else
+        puts "WARNING - #{invalid_column.table_name} . #{invalid_column.db_column.name} is not nullable"
+      end
+    end
+  end
+end

data/lib/yeet_dba/verify_data.rb

@@ -0,0 +1,37 @@
+require 'pry'
+
+module YeetDba
+  class VerifyData
+    attr_accessor :column
+
+    def initialize(column:)
+      @column = column
+    end
+
+    def orphaned_rows?
+      orphaned_rows.first
+    end
+
+    def orphaned_rows_count
+      orphaned_rows.count
+    end
+
+    def query
+      orphaned_rows.to_sql
+    end
+
+    def orphaned_rows
+      association = column.association
+
+      column_name = column.db_column.name
+      table_name = column.table_name
+      association_table = column.association_table_name
+      model = column.model
+
+      # Check to see there could be rows with bad data
+      model.joins("left join #{association_table} as association_table on association_table.id = #{table_name}.#{column_name}")
+           .where.not(column_name => nil)
+           .where('association_table.id is null')
+    end
+  end
+end

data/lib/yeet_dba/version.rb

@@ -0,0 +1,3 @@
+module YeetDba
+  VERSION = '0.1.2'.freeze
+end

data/lib/yeet_dba.rb

@@ -1,9 +1,11 @@
-require "yeet_dba/version"
-require "yeet_dba/ar_column"
-require "yeet_dba/ar_table"
-require "yeet_dba/foreign_key"
-require "yeet_dba/missing_foreign_keys"
-require "yeet_dba/verify_data"
+require 'yeet_dba/version'
+require 'yeet_dba/table'
+require 'yeet_dba/missing_foreign_keys'
+require 'yeet_dba/verify_data'
+require 'yeet_dba/railtie' if defined?(Rails)
+require 'yeet_dba/column'
+require 'yeet_dba/models/foreign_key'
+require 'yeet_dba/models/invalid_column'
 
 module YeetDba
   class Error < StandardError; end

data/yeet_dba.gemspec

@@ -1,40 +1,45 @@
-
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "./yeet_dba/version"
+require 'yeet_dba/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "yeet_dba"
+  spec.name          = 'yeet_dba'
   spec.version       = YeetDba::VERSION
-  spec.authors       = ["Kevin Coleman"]
-  spec.email         = ["kevin.coleman@sparkstart.io"]
+  spec.platform      = Gem::Platform::RUBY
+  spec.authors       = ['Kevin Coleman']
+  spec.email         = ['kevin.coleman@sparkstart.io']
 
-  spec.summary       = %q{Generates foreign key constraint migrations for rails databases}
-  spec.description   = %q{This scan every ActiveRecord model looking for relationships ('has_many', 'belongs_to', etc.) and adds foreign key constraints.}
+  spec.summary       = 'Generates foreign key constraint migrations for rails databases'
+  spec.description   = "This scan every ActiveRecord model looking for relationships ('has_many', 'belongs_to', etc.) and adds foreign key constraints."
   spec.homepage      = 'http://rubygems.org/gems/yeet_dba'
-  spec.license       = "MIT"
+  spec.license       = 'MIT'
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.
   if spec.respond_to?(:metadata)
-    spec.metadata["homepage_uri"] = spec.homepage
-    spec.metadata["source_code_uri"] = "https://github.com/kevincolemaninc/yeet_dba"
-    spec.metadata["changelog_uri"] = "https://github.com/kevincolemaninc/yeet_dba/master/CHANGELOG.md"
+    spec.metadata['homepage_uri'] = spec.homepage
+    spec.metadata['source_code_uri'] = 'https://github.com/kevincolemaninc/yeet_dba'
+    spec.metadata['changelog_uri'] = 'https://github.com/kevincolemaninc/yeet_dba/master/CHANGELOG.md'
   else
-    raise "RubyGems 2.0 or newer is required to protect against " \
-      "public gem pushes."
+    raise 'RubyGems 2.0 or newer is required to protect against ' \
+      'public gem pushes.'
   end
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.17'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
 
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.required_ruby_version = '>= 2.4.0'
+  spec.add_dependency 'actionpack', '>= 3.0', '< 6.0'
+  spec.add_dependency 'activerecord', '>= 3.0', '< 6.0'
+  spec.add_dependency 'railties', '>= 3.0', '< 6.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yeet_dba
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Kevin Coleman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-29 00:00:00.000000000 Z
+date: 2019-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,66 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: actionpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
 description: This scan every ActiveRecord model looking for relationships ('has_many',
   'belongs_to', etc.) and adds foreign key constraints.
 email:
@@ -63,28 +123,35 @@ files:
 - ".DS_Store"
 - ".gitignore"
 - ".rspec"
+- ".rubocop"
 - ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
 - lib/.DS_Store
+- lib/generators/.DS_Store
 - lib/generators/yeet_dba/foreign_key_migration_generator.rb
-- lib/generators/yeet_dba/generator_helpers.rb
 - lib/generators/yeet_dba/templates/add_foreign_keys_yeet_dba.rb
 - lib/yeet_dba.rb
+- lib/yeet_dba/Rakefile
+- lib/yeet_dba/column.rb
+- lib/yeet_dba/missing_foreign_keys.rb
+- lib/yeet_dba/models/foreign_key.rb
+- lib/yeet_dba/models/invalid_column.rb
+- lib/yeet_dba/railtie.rb
+- lib/yeet_dba/table.rb
+- lib/yeet_dba/tasks/add_foreign_keys.rake
+- lib/yeet_dba/tasks/bad_data/find_orphaned_rows.rake
+- lib/yeet_dba/verify_data.rb
+- lib/yeet_dba/version.rb
 - yeet_dba.gemspec
 - yeet_dba.png
-- yeet_dba/ar_column.rb
-- yeet_dba/ar_table.rb
-- yeet_dba/foreign_key.rb
-- yeet_dba/missing_foreign_keys.rb
-- yeet_dba/verify_data.rb
-- yeet_dba/version.rb
 homepage: http://rubygems.org/gems/yeet_dba
 licenses:
 - MIT
@@ -100,7 +167,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/generators/yeet_dba/generator_helpers.rb

@@ -1,5 +0,0 @@
-module YeetDba
-  module GeneratorHelpers
-
-  end
-end

data/yeet_dba/ar_table.rb

@@ -1,47 +0,0 @@
-module YeetDba
-  class ArTable
-    attr_accessor :table_name, :tables
-
-    def initialize(table_name:, tables:)
-      @table_name = table_name
-      @tables = tables
-    end
-
-    def missing_keys
-      missing_keys_array = []
-      columns.each do |column_name|
-        column = ArColumn.new(column_name: column_name, table_name: table_name, tables: tables)
-        next unless column.is_association?
-
-        unless column.model
-          puts "YeetDba - cannot find model for #{table_name} . #{column_name.name} | #{column&.association_table_name}"
-        end
-
-        unless column.association
-          puts "YeetDba - cannot find association for #{table_name} . #{column_name.name} | #{column&.association_table_name}"
-        end
-
-        next if column.polymorphic_association?
-        next if column.foreign_key_exists?
-        next if column.association_table_name.blank?
-
-        if VerifyData.new(column: column).orphaned_rows?
-          puts "YeetDba - orphaned rows. Skipping #{table_name} . #{column_name.name} | #{column&.association_table_name}"
-          next
-        end
-
-        foreign_key = ForeignKey.new(table_a: table_name,
-                                     table_b: column&.association_table_name,
-                                     column: column_name.name)
-        missing_keys_array.push(foreign_key)
-      end
-      missing_keys_array
-    end
-
-    private
-
-    def columns
-      ActiveRecord::Base.connection.columns(table_name)
-    end
-  end
-end

data/yeet_dba/missing_foreign_keys.rb

@@ -1,11 +0,0 @@
-module YeetDba
-  class MissingForeignKeys
-    def self.foreign_keys
-      Rails.application.eager_load!
-      tables = ActiveRecord::Base.connection.tables
-      tables.map do |table_name|
-        ArTable.new(table_name: table_name, tables: tables).missing_keys
-      end.flatten
-    end
-  end
-end

data/yeet_dba/verify_data.rb

@@ -1,35 +0,0 @@
-module YeetDba
-  class VerifyData
-  attr_accessor :column
-
-  def initialize(column:)
-    @column = column
-  end
-
-  def orphaned_rows?
-    orphaned_rows.first
-  end
-
-  private
-
-  def orphaned_rows
-    association = column.association
-    binding.pry if column.column_name.is_a?(String)
-
-    column_name = column.column_name.name
-    table_name = column.table_name
-    association_table = column.association_table_name
-    model = column.model
-
-    # Check to see there could be rows with bad data 
-    bad_ids = model.joins("left join #{association_table} as association_table on association_table.id = #{table_name}.#{column_name}")
-          .where.not(column_name => nil)
-          .where('association_table.id is null')
-          .pluck(:id)
-
-    # select the rows with the invalid ids
-    # AR doesn't support joins with update, so the bad_ids are selected in a different query (above ^)
-    model.where(id: bad_ids)
-  end
-end
-end

data/yeet_dba/version.rb

@@ -1,3 +0,0 @@
-module YeetDba
-  VERSION = "0.1.1"
-end