active_sanity 0.1.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.circleci/config.yml +18 -0
- data/.gitignore +3 -0
- data/.rubocop.yml +33 -0
- data/CHANGELOG.md +24 -0
- data/Gemfile +4 -1
- data/README.md +26 -10
- data/Rakefile +12 -0
- data/active_sanity.gemspec +15 -15
- data/commits.csv +0 -0
- data/features/check_sanity.feature +10 -3
- data/features/check_sanity_with_db_storage.feature +13 -13
- data/features/step_definitions/rails_app.rb +41 -42
- data/features/support/env.rb +13 -10
- data/lib/active_sanity.rb +0 -1
- data/lib/active_sanity/checker.rb +73 -18
- data/lib/active_sanity/invalid_record.rb +3 -2
- data/lib/active_sanity/railtie.rb +0 -1
- data/lib/active_sanity/tasks.rb +2 -3
- data/lib/active_sanity/version.rb +1 -1
- data/lib/generators/active_sanity/active_sanity_generator.rb +14 -4
- data/lib/generators/active_sanity/templates/create_invalid_records.rb.erb +13 -0
- data/test/rails_template.rb +37 -18
- metadata +91 -98
- data/lib/generators/active_sanity/templates/create_invalid_records.rb +0 -15
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: '0558af4db68be1d5ad7994453fee0a5abdfd3a79fd0e01417fa25afa7dc6ad98'
|
4
|
+
data.tar.gz: 7422349844512998be28691efc24d2211554a819f395cc39b0540d4edeedf398
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4f825526ec0ee8c0a52ad40beef09c86cb5f64f7a8e61f2ed4beb01e1f22e56a97a77dfc4b7cf3368e147181f33b3829ad631bf9631d6b16fab12805ae11e493
|
7
|
+
data.tar.gz: bee1b677fd834b29b3b8edc1c573372b6de0a3db0b8edf2cf86a88c0ecdc7b89fbb0d576c17525c04fea54de13f8671e009c9f16472b846b1b6b6aa3e403fa1d
|
@@ -0,0 +1,18 @@
|
|
1
|
+
version: 2.1
|
2
|
+
orbs:
|
3
|
+
ruby: circleci/ruby@0.1.2
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
docker:
|
8
|
+
- image: circleci/ruby:2.6.3-stretch-node
|
9
|
+
executor: ruby/default
|
10
|
+
steps:
|
11
|
+
- checkout
|
12
|
+
- run:
|
13
|
+
name: Which bundler?
|
14
|
+
command: bundle -v
|
15
|
+
- ruby/bundle-install
|
16
|
+
- run:
|
17
|
+
name: cucumber
|
18
|
+
command: RAILS_ENV=test bundle exec cucumber
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
AllCops:
|
2
|
+
Include:
|
3
|
+
- 'Gemfile'
|
4
|
+
- 'Rakefile'
|
5
|
+
- '*.gemspec'
|
6
|
+
- 'lib/**/*'
|
7
|
+
- 'test/**/*'
|
8
|
+
Exclude:
|
9
|
+
- 'features/**/*'
|
10
|
+
- 'test/rails_app/**/*'
|
11
|
+
RunRailsCops: false
|
12
|
+
Metrics/LineLength:
|
13
|
+
Enabled: false
|
14
|
+
Style/AlignHash:
|
15
|
+
EnforcedHashRocketStyle: table
|
16
|
+
EnforcedColonStyle: table
|
17
|
+
Style/Documentation:
|
18
|
+
Enabled: false
|
19
|
+
Style/RegexpLiteral:
|
20
|
+
Enabled: false
|
21
|
+
Style/CollectionMethods:
|
22
|
+
# Mapping from undesired method to desired_method
|
23
|
+
# e.g. to use `detect` over `find`:
|
24
|
+
#
|
25
|
+
# CollectionMethods:
|
26
|
+
# PreferredMethods:
|
27
|
+
# find: detect
|
28
|
+
PreferredMethods:
|
29
|
+
map: 'collect'
|
30
|
+
map!: 'collect!'
|
31
|
+
reduce: 'inject'
|
32
|
+
detect: 'find'
|
33
|
+
select: 'find_all'
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
## 0.3.0
|
2
|
+
|
3
|
+
* Add Rails ~4 compatibility.
|
4
|
+
* Does not rerun validations for the 'InvalidRecord' model on the second pass.
|
5
|
+
* batch size for fetched records is now configurable by setting ActiveSanity::Checker.batch_size after the gem has been loaded ie in 'config\application.rb' (or 'config\environments\test.rb') inside an 'after_initialize' block
|
6
|
+
|
7
|
+
## 0.2.0
|
8
|
+
|
9
|
+
* Add Rails ~3.1 compatibility. Pull Request [#6][] by [@skateinmars][].
|
10
|
+
|
11
|
+
## 0.1.1
|
12
|
+
|
13
|
+
* Fix bug where records stored with STI would be duplicated in the
|
14
|
+
database. Issue [#1][]. [@vraravam][]
|
15
|
+
|
16
|
+
## 0.1.0
|
17
|
+
|
18
|
+
Initial release.
|
19
|
+
|
20
|
+
<!--- The following link definition list is generated by PimpMyChangelog --->
|
21
|
+
[#1]: https://github.com/versapay/active_sanity/issues/1
|
22
|
+
[#6]: https://github.com/versapay/active_sanity/issues/6
|
23
|
+
[@skateinmars]: https://github.com/skateinmars
|
24
|
+
[@vraravam]: https://github.com/vraravam
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,8 +1,17 @@
|
|
1
1
|
# Active Sanity
|
2
2
|
|
3
|
+
[](https://circleci.com/gh/pcreux/active_sanity)
|
4
|
+
|
3
5
|
Perform a sanity check on your database through active record
|
4
6
|
validation.
|
5
7
|
|
8
|
+
## Requirements
|
9
|
+
|
10
|
+
* ActiveSanity ~0.5 Rails ~5.0 or ~6.0
|
11
|
+
* ActiveSanity ~0.3 and ~0.4 requires Rails ~4.0
|
12
|
+
* ActiveSanity ~0.2 requires Rails ~3.1
|
13
|
+
* ActiveSanity ~0.1 requires Rails ~3.0
|
14
|
+
|
6
15
|
## Install
|
7
16
|
|
8
17
|
Add the following line to your Gemfile
|
@@ -11,17 +20,17 @@ Add the following line to your Gemfile
|
|
11
20
|
|
12
21
|
If you wish to store invalid records in your database run:
|
13
22
|
|
14
|
-
$ rails generate active_sanity
|
15
|
-
$
|
23
|
+
$ bin/rails generate active_sanity
|
24
|
+
$ bin/rails db:migrate
|
16
25
|
|
17
26
|
## Usage
|
18
27
|
|
19
28
|
Just run:
|
20
29
|
|
21
|
-
|
30
|
+
bin/rails db:check_sanity
|
22
31
|
|
23
32
|
ActiveSanity will iterate over every records of all your models to check
|
24
|
-
|
33
|
+
whether they're valid or not. It will save invalid records in the table
|
25
34
|
invalid_records if it exists and output all invalid records.
|
26
35
|
|
27
36
|
The output might look like the following:
|
@@ -31,13 +40,20 @@ The output might look like the following:
|
|
31
40
|
Flight | 123 | { "arrival_time" => ["can't be nil"], "departure_time" => ["is invalid"] }
|
32
41
|
Flight | 323 | { "arrival_time" => ["can't be nil"] }
|
33
42
|
|
34
|
-
|
43
|
+
By default, the number of records fetched from the database for validation is set to 500. If this causes any issues in your domain/codebase, you can configure it this way in `config\application.rb` (or `config\environments\test.rb`):
|
44
|
+
|
45
|
+
class Application < Rails::Application
|
46
|
+
config.after_initialize do
|
47
|
+
ActiveSanity::Checker.batch_size = 439
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
If you want to ignore certain models from being verified, you can create a file named `active_sanity.ignore.yml` at the root of your project with the following structure
|
35
52
|
|
36
|
-
|
37
|
-
|
38
|
-
|
53
|
+
models:
|
54
|
+
- '<name of class to ignore>'
|
55
|
+
- '<name of class to ignore>'
|
39
56
|
|
40
|
-
See https://rails.lighthouseapp.com/projects/8994-ruby-on-rails/tickets/6646-orderedhash-serialization-does-not-work-when-storing-arrays
|
41
57
|
|
42
58
|
## Contribute & Dev environment
|
43
59
|
|
@@ -47,7 +63,7 @@ This gem is quite simple so I experiment using features only. To run the
|
|
47
63
|
acceptance test suite, just run:
|
48
64
|
|
49
65
|
bundle install
|
50
|
-
cucumber features
|
66
|
+
RAILS_ENV=test bundle exec cucumber features
|
51
67
|
|
52
68
|
Using features only was kinda handsome until I had to deal with two
|
53
69
|
different database schema (with / without the table invalid_records) in
|
data/Rakefile
CHANGED
@@ -1,2 +1,14 @@
|
|
1
1
|
require 'bundler'
|
2
2
|
Bundler::GemHelper.install_tasks
|
3
|
+
|
4
|
+
task default: :features
|
5
|
+
|
6
|
+
desc 'Run features'
|
7
|
+
task :features do
|
8
|
+
fail 'Failed!' unless system('export RAILS_ENV=test && bundle exec cucumber features')
|
9
|
+
end
|
10
|
+
|
11
|
+
desc 'Clean test rails app'
|
12
|
+
task :clean do
|
13
|
+
puts 'test/rails_app deleted successfully' if system('rm -r test/rails_app')
|
14
|
+
end
|
data/active_sanity.gemspec
CHANGED
@@ -1,26 +1,26 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
|
-
|
3
|
-
require
|
2
|
+
$LOAD_PATH.push File.expand_path('../lib', __FILE__)
|
3
|
+
require 'active_sanity/version'
|
4
4
|
|
5
5
|
Gem::Specification.new do |s|
|
6
|
-
s.name =
|
6
|
+
s.name = 'active_sanity'
|
7
7
|
s.version = ActiveSanity::VERSION
|
8
8
|
s.platform = Gem::Platform::RUBY
|
9
|
-
s.authors = [
|
10
|
-
s.email = [
|
11
|
-
s.
|
12
|
-
s.
|
13
|
-
s.
|
14
|
-
s.homepage = "https://github.com/versapay/active_sanity"
|
9
|
+
s.authors = ['Philippe Creux']
|
10
|
+
s.email = ['pcreux@gmail.com']
|
11
|
+
s.summary = 'Checks Sanity of Active Record records'
|
12
|
+
s.description = 'Performs a Sanity Check of your database by logging all invalid Active Records'
|
13
|
+
s.homepage = 'https://github.com/pcreux/active_sanity'
|
15
14
|
|
16
|
-
s.add_dependency
|
15
|
+
s.add_dependency 'rails', '>=5.0'
|
17
16
|
|
18
|
-
s.add_development_dependency
|
19
|
-
s.add_development_dependency
|
20
|
-
s.add_development_dependency
|
17
|
+
s.add_development_dependency 'rails', '~>6.0'
|
18
|
+
s.add_development_dependency 'rspec', '~>3.1'
|
19
|
+
s.add_development_dependency 'cucumber', '~>1.3'
|
20
|
+
s.add_development_dependency 'sqlite3', '~>1.3'
|
21
21
|
|
22
22
|
s.files = `git ls-files`.split("\n")
|
23
23
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
24
|
-
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
25
|
-
s.require_paths = [
|
24
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
25
|
+
s.require_paths = ['lib']
|
26
26
|
end
|
data/commits.csv
ADDED
File without changes
|
@@ -9,7 +9,7 @@ Feature: Check sanity
|
|
9
9
|
|
10
10
|
Scenario: Check sanity on empty database
|
11
11
|
When I run "rake db:check_sanity"
|
12
|
-
Then I should see "Checking the following models:
|
12
|
+
Then I should see "Checking the following models: Category, Post, User"
|
13
13
|
Then I should not see any invalid records
|
14
14
|
|
15
15
|
Scenario: Check sanity on database with valid records
|
@@ -19,8 +19,15 @@ Feature: Check sanity
|
|
19
19
|
|
20
20
|
Scenario: Check sanity on database with invalid records
|
21
21
|
Given the database contains a few valid records
|
22
|
-
And the first
|
22
|
+
And the first author's username is empty and the first post category_id is nil
|
23
23
|
When I run "rake db:check_sanity"
|
24
24
|
Then I should see the following invalid records:
|
25
25
|
| User | 1 | {:username=>["can't be blank", "is too short (minimum is 3 characters)"]} |
|
26
|
-
| Post | 1 | {:category=>["can't be blank"]} |
|
26
|
+
| Post | 1 | {:category=>["must exist", "can't be blank"]} |
|
27
|
+
|
28
|
+
Scenario: Check sanity on database with invalid records with ignored classes
|
29
|
+
Given the database contains a few valid records
|
30
|
+
And the first author's username is empty and the first post category_id is nil
|
31
|
+
And the User class is ignored
|
32
|
+
When I run "rake db:check_sanity"
|
33
|
+
Then I should see "Checking the following models: Category, Post"
|
@@ -9,7 +9,7 @@ Feature: Check sanity with db storage
|
|
9
9
|
|
10
10
|
Scenario: Check sanity on empty database
|
11
11
|
When I run "rake db:check_sanity"
|
12
|
-
Then I should see "Checking the following models:
|
12
|
+
Then I should see "Checking the following models: Category, Post, User"
|
13
13
|
Then the table "invalid_records" should be empty
|
14
14
|
|
15
15
|
Scenario: Check sanity on database with valid records
|
@@ -19,37 +19,37 @@ Feature: Check sanity with db storage
|
|
19
19
|
|
20
20
|
Scenario: Check sanity on database with invalid records
|
21
21
|
Given the database contains a few valid records
|
22
|
-
And the first
|
22
|
+
And the first author's username is empty and the first post category_id is nil
|
23
23
|
When I run "rake db:check_sanity"
|
24
24
|
Then the table "invalid_records" should contain:
|
25
|
-
| User | 1 | {:username=>["is too short (minimum is 3 characters)"]} |
|
26
|
-
| Post | 1 | {:category=>["can't be blank"]} |
|
25
|
+
| User | 1 | {:username=>["can't be blank", "is too short (minimum is 3 characters)"]} |
|
26
|
+
| Post | 1 | {:category=>["must exist", "can't be blank"]} |
|
27
27
|
|
28
28
|
Scenario: Check sanity on database with invalid records now valid
|
29
29
|
Given the database contains a few valid records
|
30
|
-
And the first
|
30
|
+
And the first author's username is empty and the first post category_id is nil
|
31
31
|
When I run "rake db:check_sanity"
|
32
32
|
Then the table "invalid_records" should contain:
|
33
|
-
| User | 1 | {:username=>["is too short (minimum is 3 characters)"]} |
|
34
|
-
| Post | 1 | {:category=>["can't be blank"]} |
|
33
|
+
| User | 1 | {:username=>["can't be blank", "is too short (minimum is 3 characters)"]} |
|
34
|
+
| Post | 1 | {:category=>["must exist", "can't be blank"]} |
|
35
35
|
|
36
|
-
Given the first
|
36
|
+
Given the first author's username is "Greg"
|
37
37
|
When I run "rake db:check_sanity"
|
38
38
|
Then the table "invalid_records" should contain:
|
39
|
-
| Post | 1 | {:category=>["can't be blank"]} |
|
39
|
+
| Post | 1 | {:category=>["must exist", "can't be blank"]} |
|
40
40
|
Then the table "invalid_records" should not contain errors for "User" "1"
|
41
41
|
|
42
42
|
Scenario: Check sanity on database with invalid records that were invalid for different reasons earlier
|
43
43
|
Given the database contains a few valid records
|
44
|
-
And the first
|
44
|
+
And the first author's username is empty and the first post category_id is nil
|
45
45
|
When I run "rake db:check_sanity"
|
46
46
|
Then the table "invalid_records" should contain:
|
47
|
-
| User | 1 | {:username=>["is too short (minimum is 3 characters)"]} |
|
48
|
-
| Post | 1 | {:category=>["can't be blank"]} |
|
47
|
+
| User | 1 | {:username=>["can't be blank", "is too short (minimum is 3 characters)"]} |
|
48
|
+
| Post | 1 | {:category=>["must exist", "can't be blank"]} |
|
49
49
|
|
50
50
|
Given the first post category is set
|
51
51
|
And the first post title is empty
|
52
52
|
When I run "rake db:check_sanity"
|
53
53
|
Then the table "invalid_records" should contain:
|
54
|
-
| User | 1 | {:username=>["is too short (minimum is 3 characters)"]} |
|
54
|
+
| User | 1 | {:username=>["can't be blank", "is too short (minimum is 3 characters)"]} |
|
55
55
|
| Post | 1 | {:title=>["can't be blank"]} |
|
@@ -1,50 +1,66 @@
|
|
1
|
-
|
2
|
-
return if File.directory?("test/rails_app")
|
1
|
+
SystemCommandFailed = Class.new(StandardError)
|
3
2
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
3
|
+
def system!(cmd)
|
4
|
+
system(cmd) || raise(SystemCommandFailed, cmd)
|
5
|
+
end
|
6
|
+
|
7
|
+
def setup_rails_app
|
8
|
+
return if File.directory?('test/rails_app')
|
9
|
+
|
10
|
+
system! 'bundle exec rails new test/rails_app --skip-sprockets --skip-spring --skip-javascript --skip-turbolinks'
|
11
|
+
system! 'cd ./test/rails_app && bundle'
|
12
|
+
system! 'cd ./test/rails_app && bin/rails app:template LOCATION=../rails_template.rb'
|
13
|
+
system!' cd ./test/rails_app && RAILS_ENV=test bin/rails db:migrate'
|
14
|
+
rescue SystemCommandFailed => e
|
15
|
+
system!('rm -fr test/rails_app')
|
16
|
+
raise e
|
8
17
|
end
|
9
18
|
|
10
19
|
Given /^I have a rails app using 'active_sanity'$/ do
|
11
|
-
Dir[
|
12
|
-
|
20
|
+
Dir['./test/rails_app/db/migrate/*create_invalid_records.rb'].each do |migration|
|
21
|
+
fail unless system("rm #{migration}")
|
13
22
|
end
|
14
23
|
|
15
24
|
setup_rails_app
|
16
25
|
|
17
26
|
require './test/rails_app/config/environment'
|
27
|
+
|
28
|
+
# Reset connection
|
29
|
+
ActiveRecord::Base.connection.reconnect!
|
18
30
|
end
|
19
31
|
|
20
32
|
Given /^I have a rails app using 'active_sanity' with db storage$/ do
|
21
33
|
setup_rails_app
|
22
34
|
|
23
|
-
|
35
|
+
fail unless system('cd ./test/rails_app && bundle exec rails generate active_sanity && RAILS_ENV=test bundle exec rake db:migrate')
|
24
36
|
|
25
37
|
require './test/rails_app/config/environment'
|
26
38
|
|
27
39
|
# Reset connection
|
28
40
|
ActiveRecord::Base.connection.reconnect!
|
29
|
-
InvalidRecord.table_exists
|
41
|
+
InvalidRecord.table_exists?.should be true # Looks up if table exists.
|
30
42
|
end
|
31
43
|
|
32
44
|
Given /^the database contains a few valid records$/ do
|
33
|
-
|
34
|
-
|
35
|
-
Category.create!(:
|
36
|
-
Post.create!(:
|
37
|
-
:
|
38
|
-
:
|
45
|
+
Author.create!(first_name: 'Greg', last_name: 'Bell', username: 'gregbell')
|
46
|
+
Publisher.create!(first_name: 'Sam', last_name: 'Vincent', username: 'samvincent')
|
47
|
+
Category.create!(name: 'Uncategorized')
|
48
|
+
Post.create!(author: Author.first!, category: Category.first!,
|
49
|
+
title: 'How ActiveAdmin changed the world', body: 'Lot of love.',
|
50
|
+
published_at: 4.years.from_now)
|
39
51
|
end
|
40
52
|
|
41
|
-
Given /^the first
|
42
|
-
|
53
|
+
Given /^the first author's username is empty and the first post category_id is nil$/ do
|
54
|
+
Author.first.update_attribute(:username, '')
|
43
55
|
Post.first.update_attribute(:category_id, nil)
|
44
56
|
end
|
45
57
|
|
46
|
-
Given /^the
|
47
|
-
|
58
|
+
Given /^the User class is ignored$/ do
|
59
|
+
system("echo 'models: User' > active_sanity.ignore.yml")
|
60
|
+
end
|
61
|
+
|
62
|
+
Given /^the first author's username is "([^"]*)"$/ do |username|
|
63
|
+
Author.first.update_attribute('username', username)
|
48
64
|
end
|
49
65
|
|
50
66
|
Given /^the first post category is set$/ do
|
@@ -55,13 +71,11 @@ Given /^the first post title is empty$/ do
|
|
55
71
|
Post.first.update_attribute('title', '')
|
56
72
|
end
|
57
73
|
|
58
|
-
|
59
74
|
When /^I run "([^"]*)"$/ do |command|
|
60
|
-
puts @output = `cd ./test/rails_app && #{command}; echo "RETURN:$?"`
|
61
|
-
|
75
|
+
puts @output = `cd ./test/rails_app && RAILS_ENV=test bundle exec #{command} --trace; echo "RETURN:$?"`
|
76
|
+
fail unless @output['RETURN:0']
|
62
77
|
end
|
63
78
|
|
64
|
-
|
65
79
|
Then /^I should see the following invalid records:$/ do |table|
|
66
80
|
table.raw.each do |model, id, errors|
|
67
81
|
@output.should =~ /#{model}\s+\|\s+#{id}\s+\|\s+#{Regexp.escape errors}/
|
@@ -82,30 +96,15 @@ end
|
|
82
96
|
|
83
97
|
Then /^the table "([^"]*)" should contain:$/ do |_, table|
|
84
98
|
table.raw.each do |model, id, errors|
|
85
|
-
invalid_record = InvalidRecord.where(:
|
99
|
+
invalid_record = InvalidRecord.where(record_type: model, record_id: id).first
|
86
100
|
invalid_record.should be_an_instance_of(InvalidRecord)
|
87
101
|
errors = eval(errors)
|
88
102
|
errors.each do |k, v|
|
89
|
-
|
90
|
-
invalid_record.validation_errors[k].should == v
|
91
|
-
rescue RSpec::Expectations::ExpectationNotMetError => e
|
92
|
-
# A bug in serialization of ordered hash get rid of the array of
|
93
|
-
# errors. The following is stored in the db:
|
94
|
-
# --- !omap
|
95
|
-
# - :username: can't be blank
|
96
|
-
# - :username: is too short (minimum is 3 characters)
|
97
|
-
#
|
98
|
-
# https://rails.lighthouseapp.com/projects/8994-ruby-on-rails/tickets/6646-orderedhash-serialization-does-not-work-when-storing-arrays
|
99
|
-
#
|
100
|
-
# You actually get the last error on an attribute only
|
101
|
-
#
|
102
|
-
invalid_record.validation_errors[k].should == v.last
|
103
|
-
end
|
103
|
+
invalid_record.validation_errors[k].should == v
|
104
104
|
end
|
105
105
|
end
|
106
106
|
end
|
107
107
|
|
108
108
|
Then /^the table "([^"]*)" should not contain errors for "([^"]*)" "([^"]*)"$/ do |_, model, id|
|
109
|
-
|
109
|
+
InvalidRecord.where(record_type: model, record_id: id).first.should be_nil
|
110
110
|
end
|
111
|
-
|
data/features/support/env.rb
CHANGED
@@ -1,20 +1,23 @@
|
|
1
1
|
ENV['BUNDLE_GEMFILE'] = File.expand_path('../../../Gemfile', __FILE__)
|
2
2
|
|
3
3
|
require 'rubygems'
|
4
|
-
require
|
4
|
+
require 'bundler'
|
5
5
|
Bundler.setup
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
|
7
|
+
if File.directory?('test/rails_app')
|
8
|
+
Dir.chdir('test/rails_app') do
|
9
|
+
fail unless system('rm -f db/migrate/*create_invalid_records.rb && RAILS_ENV=test bin/rails db:drop db:create db:migrate')
|
10
|
+
end
|
11
|
+
end
|
10
12
|
|
11
13
|
After do
|
12
14
|
# Reset DB!
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
15
|
+
tables = %w(categories invalid_records posts users)
|
16
|
+
conn = ActiveRecord::Base.connection
|
17
|
+
tables.each do |table|
|
18
|
+
if conn.table_exists?(table)
|
19
|
+
conn.execute("DELETE FROM '#{table}'")
|
20
|
+
conn.execute("DELETE FROM sqlite_sequence WHERE name='#{table}'")
|
21
|
+
end
|
19
22
|
end
|
20
23
|
end
|
data/lib/active_sanity.rb
CHANGED
@@ -1,44 +1,91 @@
|
|
1
1
|
module ActiveSanity
|
2
2
|
class Checker
|
3
|
+
class << self
|
4
|
+
attr_accessor :batch_size
|
5
|
+
end
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
Checker.batch_size ||= 500
|
9
|
+
end
|
10
|
+
|
3
11
|
def self.check!
|
4
12
|
new.check!
|
5
13
|
end
|
6
14
|
|
7
15
|
def check!
|
8
|
-
puts
|
16
|
+
puts 'Sanity Check'
|
9
17
|
puts "Checking the following models: #{models.join(', ')}"
|
10
18
|
|
19
|
+
# TODO: Wouldnt this list already be checked by the next all records call if those records do exist?
|
20
|
+
# This will validate and destroy the records that either dont exist currently, or are now valid. But the ones are continue to be invalid - these will
|
21
|
+
# have been run through the validation process twice
|
11
22
|
check_previously_invalid_records
|
12
23
|
check_all_records
|
13
24
|
end
|
14
25
|
|
26
|
+
# @return [Array] of [ActiveRecord::Base] direct descendants
|
15
27
|
def models
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
28
|
+
return @models if @models
|
29
|
+
|
30
|
+
load_all_models
|
31
|
+
|
32
|
+
@models ||= direct_active_record_base_descendants
|
33
|
+
@models -= [InvalidRecord]
|
34
|
+
if File.exist?('active_sanity.ignore.yml')
|
35
|
+
yaml_contents = YAML.load_file('active_sanity.ignore.yml')
|
36
|
+
model_names_to_ignore = Array(yaml_contents['models'] || []).uniq
|
37
|
+
@models = @models.reject { |model| model_names_to_ignore.include?(model.name) }
|
20
38
|
end
|
21
|
-
|
22
|
-
@models ||= ActiveRecord::Base.subclasses
|
39
|
+
@models
|
23
40
|
end
|
24
41
|
|
25
42
|
protected
|
26
43
|
|
44
|
+
# Require all files under /app/models.
|
45
|
+
# All models under /lib are required when the rails app loads.
|
46
|
+
def load_all_models
|
47
|
+
Dir["#{Rails.root}/app/models/**/*.rb"].each { |file_path| require file_path rescue nil }
|
48
|
+
end
|
49
|
+
|
50
|
+
# @return [Array] of direct ActiveRecord::Base descendants.
|
51
|
+
# Example:
|
52
|
+
# The following tree:
|
53
|
+
# ActiveRecord::Base
|
54
|
+
# |
|
55
|
+
# |- User
|
56
|
+
# |- Account
|
57
|
+
# | |
|
58
|
+
# | |- PersonalAccount
|
59
|
+
# | |- BusinessAccount
|
60
|
+
#
|
61
|
+
# Should return: [Account, User]
|
62
|
+
def direct_active_record_base_descendants
|
63
|
+
parent_class = defined?(ApplicationRecord) ? ApplicationRecord : ActiveRecord::Base
|
64
|
+
parent_class.descendants.select(&:descends_from_active_record?).sort_by(&:name)
|
65
|
+
end
|
66
|
+
|
67
|
+
# Remove records that are now valid from the list of invalid records.
|
27
68
|
def check_previously_invalid_records
|
28
69
|
return unless InvalidRecord.table_exists?
|
29
70
|
|
30
|
-
InvalidRecord.find_each do |invalid_record|
|
31
|
-
|
71
|
+
InvalidRecord.find_each(batch_size: Checker.batch_size) do |invalid_record|
|
72
|
+
begin
|
73
|
+
invalid_record.destroy if invalid_record.record.valid?
|
74
|
+
rescue
|
75
|
+
# Record does not exists.
|
76
|
+
invalid_record.delete
|
77
|
+
end
|
32
78
|
end
|
33
79
|
end
|
34
80
|
|
81
|
+
# Go over every single record. When the record is not valid
|
82
|
+
# log it to STDOUT and into the invalid_records table if it exists.
|
35
83
|
def check_all_records
|
36
84
|
models.each do |model|
|
37
85
|
begin
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
end
|
86
|
+
# TODO: Can we filter based on those records that are already present in the 'invalid_records' table - especially since they have been re-verified in the method before?
|
87
|
+
model.find_each(batch_size: Checker.batch_size) do |record|
|
88
|
+
invalid_record!(record) unless record.valid?
|
42
89
|
end
|
43
90
|
rescue => e
|
44
91
|
# Rescue from exceptions (table does not exists,
|
@@ -54,22 +101,30 @@ module ActiveSanity
|
|
54
101
|
store_invalid_record(record)
|
55
102
|
end
|
56
103
|
|
104
|
+
# Say that the record is invalid. Example:
|
105
|
+
#
|
106
|
+
# Account | 10 | :name => "Can't be blank"
|
57
107
|
def log_invalid_record(record)
|
58
|
-
puts record
|
108
|
+
puts "#{type_of(record)} | #{record.id} | #{pretty_errors(record)}"
|
59
109
|
end
|
60
|
-
|
110
|
+
|
111
|
+
# Store invalid record in InvalidRecord table if it exists
|
61
112
|
def store_invalid_record(record)
|
62
113
|
return unless InvalidRecord.table_exists?
|
63
114
|
|
64
|
-
invalid_record = InvalidRecord.where(
|
115
|
+
invalid_record = InvalidRecord.where(record: record).first
|
65
116
|
invalid_record ||= InvalidRecord.new
|
66
117
|
invalid_record.record = record
|
67
|
-
invalid_record.validation_errors = record.errors
|
118
|
+
invalid_record.validation_errors = record.errors.messages
|
68
119
|
invalid_record.save!
|
69
120
|
end
|
70
121
|
|
122
|
+
def type_of(record)
|
123
|
+
record.class.base_class
|
124
|
+
end
|
125
|
+
|
71
126
|
def pretty_errors(record)
|
72
|
-
record.errors.inspect.sub(/^#<OrderedHash
|
127
|
+
record.errors.messages.inspect.sub(/^#<OrderedHash (.*)>$/, '\1')
|
73
128
|
end
|
74
129
|
end
|
75
130
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
class InvalidRecord < ActiveRecord::Base
|
2
|
-
belongs_to :record, :
|
2
|
+
belongs_to :record, polymorphic: true
|
3
3
|
serialize :validation_errors
|
4
4
|
|
5
|
-
|
5
|
+
validates :record, :validation_errors, presence: true
|
6
|
+
validates :record_id, presence: true, uniqueness: { scope: :record_type }
|
6
7
|
end
|
data/lib/active_sanity/tasks.rb
CHANGED
@@ -6,21 +6,31 @@ module ActiveSanity
|
|
6
6
|
include Rails::Generators::Migration
|
7
7
|
|
8
8
|
def self.source_root
|
9
|
-
|
9
|
+
@source_root ||= File.join(File.dirname(__FILE__), 'templates')
|
10
10
|
end
|
11
11
|
|
12
12
|
def self.next_migration_number(dirname) #:nodoc:
|
13
13
|
next_migration_number = current_migration_number(dirname) + 1
|
14
14
|
if ActiveRecord::Base.timestamped_migrations
|
15
|
-
[Time.now.utc.strftime(
|
15
|
+
[Time.now.utc.strftime('%Y%m%d%H%M%S'), '%.14d' % next_migration_number].max
|
16
16
|
else
|
17
|
-
|
17
|
+
'%.3d' % next_migration_number
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
21
21
|
def create_migration_file
|
22
|
-
|
22
|
+
migration_template 'create_invalid_records.rb.erb', 'db/migrate/create_invalid_records.rb'
|
23
23
|
end
|
24
|
+
|
25
|
+
# Used by migration file
|
26
|
+
def rails_version
|
27
|
+
if Rails.version >= "5"
|
28
|
+
"[#{Rails.version.gsub(/\.\d+$/, '')}]" # 5.0
|
29
|
+
else
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
24
34
|
end
|
25
35
|
end
|
26
36
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
class CreateInvalidRecords < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
|
+
def up
|
3
|
+
create_table :invalid_records do |t|
|
4
|
+
t.references :record, polymorphic: true, null: false
|
5
|
+
t.text :validation_errors
|
6
|
+
t.timestamps
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
def down
|
11
|
+
drop_table :invalid_records
|
12
|
+
end
|
13
|
+
end
|
data/test/rails_template.rb
CHANGED
@@ -1,37 +1,56 @@
|
|
1
|
-
|
2
1
|
# Generate some test models
|
3
|
-
|
2
|
+
|
3
|
+
BASE_MODEL_CLASS = Rails.version >= "5." ? "ApplicationRecord" : "ActiveRecord::Base"
|
4
|
+
|
5
|
+
# Post
|
6
|
+
generate :model, 'post title:string body:text published_at:datetime author_id:integer category_id:integer'
|
4
7
|
post_code = <<-CODE
|
5
8
|
belongs_to :author, :class_name => 'User'
|
6
9
|
belongs_to :category
|
7
10
|
accepts_nested_attributes_for :author
|
8
11
|
|
9
|
-
|
10
|
-
CODE
|
11
|
-
inject_into_file 'app/models/post.rb', post_code, :after => "class Post < ActiveRecord::Base\n"
|
12
|
-
|
13
|
-
generate :model, "user first_name:string last_name:string username:string"
|
14
|
-
user_code = <<-CODE
|
15
|
-
has_many :posts, :foreign_key => 'author_id'
|
16
|
-
|
17
|
-
validates_presence_of :first_name, :last_name, :username
|
18
|
-
validates_length_of :username, :minimum => 3
|
12
|
+
validates :author, :category, :title, :published_at, presence: true
|
19
13
|
CODE
|
20
|
-
inject_into_file 'app/models/
|
14
|
+
inject_into_file 'app/models/post.rb', post_code, after: "class Post < #{BASE_MODEL_CLASS}\n"
|
21
15
|
|
16
|
+
# Category
|
22
17
|
generate :model, 'category name:string description:text'
|
23
18
|
category_code = <<-CODE
|
24
19
|
has_many :posts
|
25
20
|
|
26
|
-
|
21
|
+
validates :name, presence: true
|
27
22
|
CODE
|
28
|
-
inject_into_file 'app/models/category.rb', category_code, :
|
23
|
+
inject_into_file 'app/models/category.rb', category_code, after: "class Category < #{BASE_MODEL_CLASS}\n"
|
29
24
|
|
30
|
-
|
25
|
+
# User
|
26
|
+
generate :model, 'user first_name:string last_name:string username:string type:string'
|
27
|
+
user_code = <<-CODE
|
28
|
+
has_many :posts, :foreign_key => 'author_id'
|
29
|
+
|
30
|
+
validates :first_name, :last_name, :username, presence: true
|
31
|
+
validates :username, length: { minimum: 3 }
|
32
|
+
CODE
|
33
|
+
inject_into_file 'app/models/user.rb', user_code, after: "class User < #{BASE_MODEL_CLASS}\n"
|
34
|
+
|
35
|
+
# Author < User
|
36
|
+
create_file 'app/models/author.rb', 'class Author < User; end'
|
37
|
+
# Publisher < User
|
38
|
+
create_file 'app/models/publisher.rb', 'class Publisher < User; end'
|
39
|
+
|
40
|
+
# NotAModel
|
41
|
+
create_file 'app/models/not_a_model.rb', 'class NotAModel; end'
|
31
42
|
|
32
43
|
# Add active_sanity
|
33
44
|
append_file 'Gemfile', "gem 'active_sanity', :path => '../../'"
|
34
45
|
|
35
|
-
|
36
|
-
|
46
|
+
# Configure for custom batch_size
|
47
|
+
custom_batch_size_code = <<-CODE
|
48
|
+
config.after_initialize do
|
49
|
+
ActiveSanity::Checker.batch_size = 439
|
50
|
+
end
|
51
|
+
CODE
|
52
|
+
|
53
|
+
inject_into_file 'config/application.rb', custom_batch_size_code, after: " < Rails::Application\n"
|
37
54
|
|
55
|
+
run 'bundle'
|
56
|
+
rake 'db:drop db:create db:migrate'
|
metadata
CHANGED
@@ -1,97 +1,102 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_sanity
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
prerelease: false
|
6
|
-
segments:
|
7
|
-
- 0
|
8
|
-
- 1
|
9
|
-
- 0
|
10
|
-
version: 0.1.0
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.5.0
|
11
5
|
platform: ruby
|
12
|
-
authors:
|
13
|
-
- VersaPay
|
6
|
+
authors:
|
14
7
|
- Philippe Creux
|
15
8
|
autorequire:
|
16
9
|
bindir: bin
|
17
10
|
cert_chain: []
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
dependencies:
|
22
|
-
- !ruby/object:Gem::Dependency
|
11
|
+
date: 2020-06-24 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
23
14
|
name: rails
|
24
|
-
|
25
|
-
|
26
|
-
none: false
|
27
|
-
requirements:
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
28
17
|
- - ">="
|
29
|
-
- !ruby/object:Gem::Version
|
30
|
-
|
31
|
-
segments:
|
32
|
-
- 3
|
33
|
-
- 0
|
34
|
-
- 0
|
35
|
-
version: 3.0.0
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '5.0'
|
36
20
|
type: :runtime
|
37
|
-
version_requirements: *id001
|
38
|
-
- !ruby/object:Gem::Dependency
|
39
|
-
name: rspec
|
40
21
|
prerelease: false
|
41
|
-
|
42
|
-
|
43
|
-
requirements:
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
44
24
|
- - ">="
|
45
|
-
- !ruby/object:Gem::Version
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '5.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rails
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '6.0'
|
50
34
|
type: :development
|
51
|
-
version_requirements: *id002
|
52
|
-
- !ruby/object:Gem::Dependency
|
53
|
-
name: cucumber
|
54
35
|
prerelease: false
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '6.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.1'
|
64
48
|
type: :development
|
65
|
-
version_requirements: *id003
|
66
|
-
- !ruby/object:Gem::Dependency
|
67
|
-
name: sqlite3
|
68
49
|
prerelease: false
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.1'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: cucumber
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.3'
|
78
62
|
type: :development
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
-
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.3'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: sqlite3
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.3'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.3'
|
83
|
+
description: Performs a Sanity Check of your database by logging all invalid Active
|
84
|
+
Records
|
85
|
+
email:
|
86
|
+
- pcreux@gmail.com
|
83
87
|
executables: []
|
84
|
-
|
85
88
|
extensions: []
|
86
|
-
|
87
89
|
extra_rdoc_files: []
|
88
|
-
|
89
|
-
|
90
|
-
- .gitignore
|
90
|
+
files:
|
91
|
+
- ".circleci/config.yml"
|
92
|
+
- ".gitignore"
|
93
|
+
- ".rubocop.yml"
|
94
|
+
- CHANGELOG.md
|
91
95
|
- Gemfile
|
92
96
|
- README.md
|
93
97
|
- Rakefile
|
94
98
|
- active_sanity.gemspec
|
99
|
+
- commits.csv
|
95
100
|
- features/check_sanity.feature
|
96
101
|
- features/check_sanity_with_db_storage.feature
|
97
102
|
- features/step_definitions/rails_app.rb
|
@@ -103,43 +108,31 @@ files:
|
|
103
108
|
- lib/active_sanity/tasks.rb
|
104
109
|
- lib/active_sanity/version.rb
|
105
110
|
- lib/generators/active_sanity/active_sanity_generator.rb
|
106
|
-
- lib/generators/active_sanity/templates/create_invalid_records.rb
|
111
|
+
- lib/generators/active_sanity/templates/create_invalid_records.rb.erb
|
107
112
|
- test/rails_template.rb
|
108
|
-
|
109
|
-
homepage: https://github.com/versapay/active_sanity
|
113
|
+
homepage: https://github.com/pcreux/active_sanity
|
110
114
|
licenses: []
|
111
|
-
|
115
|
+
metadata: {}
|
112
116
|
post_install_message:
|
113
117
|
rdoc_options: []
|
114
|
-
|
115
|
-
require_paths:
|
118
|
+
require_paths:
|
116
119
|
- lib
|
117
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
118
|
-
|
119
|
-
requirements:
|
120
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
120
122
|
- - ">="
|
121
|
-
- !ruby/object:Gem::Version
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
version: "0"
|
126
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
127
|
-
none: false
|
128
|
-
requirements:
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
126
|
+
requirements:
|
129
127
|
- - ">="
|
130
|
-
- !ruby/object:Gem::Version
|
131
|
-
|
132
|
-
segments:
|
133
|
-
- 0
|
134
|
-
version: "0"
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: '0'
|
135
130
|
requirements: []
|
136
|
-
|
137
|
-
rubyforge_project:
|
138
|
-
rubygems_version: 1.3.7
|
131
|
+
rubygems_version: 3.0.3
|
139
132
|
signing_key:
|
140
|
-
specification_version:
|
133
|
+
specification_version: 4
|
141
134
|
summary: Checks Sanity of Active Record records
|
142
|
-
test_files:
|
135
|
+
test_files:
|
143
136
|
- features/check_sanity.feature
|
144
137
|
- features/check_sanity_with_db_storage.feature
|
145
138
|
- features/step_definitions/rails_app.rb
|
@@ -1,15 +0,0 @@
|
|
1
|
-
class CreateInvalidRecords < ActiveRecord::Migration
|
2
|
-
def self.up
|
3
|
-
create_table :invalid_records do |t|
|
4
|
-
t.references :record, :polymorphic => true, :null => false
|
5
|
-
t.text :validation_errors
|
6
|
-
t.timestamps
|
7
|
-
end
|
8
|
-
add_index :invalid_records, [:record_type, :record_id]
|
9
|
-
end
|
10
|
-
|
11
|
-
def self.down
|
12
|
-
drop_table :invalid_records
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|