db_validator 0.1.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/config/initializers/db_validator.rb +1 -1
- data/lib/db_validator/cli.rb +162 -0
- data/lib/db_validator/configuration.rb +2 -2
- data/lib/db_validator/formatters/json_formatter.rb +44 -0
- data/lib/db_validator/reporter.rb +63 -13
- data/lib/db_validator/validator.rb +95 -33
- data/lib/db_validator/version.rb +1 -1
- data/lib/db_validator.rb +1 -1
- data/lib/generators/db_validator/templates/initializer.rb +2 -5
- data/lib/tasks/db_validator_tasks.rake +84 -8
- data/readme.md +62 -19
- metadata +50 -14
- data/lib/db_validator/fixer.rb +0 -35
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 527983d42b2851e593a40202407ff01d8b82cb13c97741239307d21a12416567
|
4
|
+
data.tar.gz: 389a234732560ec7f0fcf1ba9c0bb995d9552a842719d75f20569b9d7f86818e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e4cec830738a0de711477620603757c2fade101075d64a9cbf824ecef806a994c7e0da655add3f495e2ac828a15a3e2b7db0143e32718a6aebbfa3fa3911fc70
|
7
|
+
data.tar.gz: e3a6d02dd6ce60e17e0ecdf5a255d0fe9886fc7533e0ad8c7334c0f3d0c03970c0aa4b01bdd90fa8342c3cb4225816b3a3658908e2e23e2ccfae2aa8fd1fc286
|
@@ -0,0 +1,162 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "tty-prompt"
|
4
|
+
require "tty-box"
|
5
|
+
require "tty-spinner"
|
6
|
+
require "optparse"
|
7
|
+
require "logger"
|
8
|
+
|
9
|
+
module DbValidator
|
10
|
+
class CLI
|
11
|
+
def initialize
|
12
|
+
@prompt = TTY::Prompt.new
|
13
|
+
@options = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def start
|
17
|
+
if ARGV.empty?
|
18
|
+
interactive_mode
|
19
|
+
else
|
20
|
+
parse_command_line_args
|
21
|
+
validate_with_options
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def display_progress(message)
|
26
|
+
spinner = TTY::Spinner.new("[:spinner] #{message}", format: :dots)
|
27
|
+
spinner.auto_spin
|
28
|
+
yield if block_given?
|
29
|
+
spinner.success
|
30
|
+
end
|
31
|
+
|
32
|
+
def select_models(available_models)
|
33
|
+
system "clear"
|
34
|
+
display_header
|
35
|
+
|
36
|
+
if available_models.empty?
|
37
|
+
@prompt.error("No models found in the application.")
|
38
|
+
exit
|
39
|
+
end
|
40
|
+
|
41
|
+
choices = available_models.map { |model| { name: model, value: model } }
|
42
|
+
choices.unshift({ name: "All Models", value: "all" })
|
43
|
+
|
44
|
+
@prompt.say("\n")
|
45
|
+
selected = @prompt.multi_select(
|
46
|
+
"Select models to validate:",
|
47
|
+
choices,
|
48
|
+
per_page: 10,
|
49
|
+
echo: false,
|
50
|
+
show_help: :always,
|
51
|
+
filter: true,
|
52
|
+
cycle: true
|
53
|
+
)
|
54
|
+
|
55
|
+
if selected.include?("all")
|
56
|
+
available_models
|
57
|
+
else
|
58
|
+
selected
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def parse_command_line_args # rubocop:disable Metrics/AbcSize
|
63
|
+
args = ARGV.join(" ").split(/\s+/)
|
64
|
+
args.each do |arg|
|
65
|
+
key, value = arg.split("=")
|
66
|
+
case key
|
67
|
+
when "models"
|
68
|
+
@options[:only_models] = value.split(",").map(&:strip).map(&:classify)
|
69
|
+
when "limit"
|
70
|
+
@options[:limit] = value.to_i
|
71
|
+
when "format"
|
72
|
+
@options[:report_format] = value.to_sym
|
73
|
+
when "show_records"
|
74
|
+
@options[:show_records] = value.to_sym
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def validate_with_options
|
80
|
+
load_rails
|
81
|
+
configure_validator(@options[:only_models], @options)
|
82
|
+
validator = DbValidator::Validator.new
|
83
|
+
report = validator.validate_all
|
84
|
+
Rails.logger.debug { "\n#{report}" }
|
85
|
+
end
|
86
|
+
|
87
|
+
def interactive_mode
|
88
|
+
load_rails
|
89
|
+
display_header
|
90
|
+
|
91
|
+
display_progress("Loading models") do
|
92
|
+
Rails.application.eager_load!
|
93
|
+
end
|
94
|
+
|
95
|
+
available_models = ActiveRecord::Base.descendants
|
96
|
+
.reject(&:abstract_class?)
|
97
|
+
.select(&:table_exists?)
|
98
|
+
.map(&:name)
|
99
|
+
.sort
|
100
|
+
|
101
|
+
if available_models.empty?
|
102
|
+
Rails.logger.debug "No models found in the application."
|
103
|
+
exit 1
|
104
|
+
end
|
105
|
+
|
106
|
+
selected_models = select_models(available_models)
|
107
|
+
options = configure_options
|
108
|
+
|
109
|
+
configure_validator(selected_models, options)
|
110
|
+
validator = DbValidator::Validator.new
|
111
|
+
report = validator.validate_all
|
112
|
+
Rails.logger.debug { "\n#{report}" }
|
113
|
+
end
|
114
|
+
|
115
|
+
def load_rails
|
116
|
+
require File.expand_path("config/environment", Dir.pwd)
|
117
|
+
rescue LoadError
|
118
|
+
Rails.logger.debug "Error: Rails application not found. Please run this command from your Rails application root."
|
119
|
+
exit 1
|
120
|
+
end
|
121
|
+
|
122
|
+
def configure_validator(models = nil, options = {})
|
123
|
+
config = DbValidator.configuration
|
124
|
+
config.only_models = models if models
|
125
|
+
config.limit = options[:limit] if options[:limit]
|
126
|
+
config.batch_size = options[:batch_size] if options[:batch_size]
|
127
|
+
config.report_format = options[:format] if options[:format]
|
128
|
+
config.show_records = options[:show_records] if options[:show_records]
|
129
|
+
end
|
130
|
+
|
131
|
+
def configure_options
|
132
|
+
options = {}
|
133
|
+
|
134
|
+
@prompt.say("\n")
|
135
|
+
limit_input = @prompt.ask("Enter record limit (leave blank for no limit):") do |q|
|
136
|
+
q.validate(/^\d*$/, "Please enter a valid number")
|
137
|
+
q.convert(:int, nil)
|
138
|
+
end
|
139
|
+
options[:limit] = limit_input if limit_input.present?
|
140
|
+
|
141
|
+
options[:format] = @prompt.select("Select report format:", %w[text json], default: "text")
|
142
|
+
|
143
|
+
options
|
144
|
+
end
|
145
|
+
|
146
|
+
def display_header
|
147
|
+
title = TTY::Box.frame(
|
148
|
+
"DB Validator",
|
149
|
+
"Interactive Model Validation",
|
150
|
+
padding: 1,
|
151
|
+
align: :center,
|
152
|
+
border: :thick,
|
153
|
+
style: {
|
154
|
+
border: {
|
155
|
+
fg: :cyan
|
156
|
+
}
|
157
|
+
}
|
158
|
+
)
|
159
|
+
Rails.logger.debug title
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module DbValidator
|
4
4
|
class Configuration
|
5
|
-
attr_accessor :only_models, :ignored_models, :ignored_attributes, :batch_size, :report_format, :
|
5
|
+
attr_accessor :only_models, :ignored_models, :ignored_attributes, :batch_size, :report_format, :limit, :show_records
|
6
6
|
|
7
7
|
def initialize
|
8
8
|
@only_models = []
|
@@ -10,8 +10,8 @@ module DbValidator
|
|
10
10
|
@ignored_attributes = {}
|
11
11
|
@batch_size = 1000
|
12
12
|
@report_format = :text
|
13
|
-
@auto_fix = false
|
14
13
|
@limit = nil
|
14
|
+
@show_records = true
|
15
15
|
end
|
16
16
|
|
17
17
|
def only_models=(models)
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
require "fileutils"
|
5
|
+
|
6
|
+
module DbValidator
|
7
|
+
module Formatters
|
8
|
+
class JsonFormatter
|
9
|
+
def initialize(invalid_records)
|
10
|
+
@invalid_records = invalid_records
|
11
|
+
end
|
12
|
+
|
13
|
+
def format
|
14
|
+
formatted_data = @invalid_records.group_by { |r| r[:model] }.transform_values do |records|
|
15
|
+
{
|
16
|
+
error_count: records.length,
|
17
|
+
records: records.map { |r| format_record(r) }
|
18
|
+
}
|
19
|
+
end
|
20
|
+
|
21
|
+
save_to_file(formatted_data)
|
22
|
+
formatted_data.to_json
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def format_record(record)
|
28
|
+
{
|
29
|
+
id: record[:id],
|
30
|
+
errors: record[:errors]
|
31
|
+
}
|
32
|
+
end
|
33
|
+
|
34
|
+
def save_to_file(data)
|
35
|
+
FileUtils.mkdir_p("db_validator_reports")
|
36
|
+
timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
|
37
|
+
filename = "db_validator_reports/validation_report_#{timestamp}.json"
|
38
|
+
|
39
|
+
File.write(filename, JSON.pretty_generate(data))
|
40
|
+
Rails.logger.info "JSON report saved to #{filename}"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -1,5 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "tty-box"
|
4
|
+
require "tty-spinner"
|
5
|
+
require "db_validator/formatters/json_formatter"
|
6
|
+
|
3
7
|
module DbValidator
|
4
8
|
class Reporter
|
5
9
|
def initialize
|
@@ -10,11 +14,11 @@ module DbValidator
|
|
10
14
|
enhanced_errors = record.errors.map do |error|
|
11
15
|
field_value = record.send(error.attribute)
|
12
16
|
message = error.message
|
13
|
-
|
17
|
+
|
14
18
|
if error.options[:in].present?
|
15
19
|
"#{error.attribute} #{message} (allowed values: #{error.options[:in].join(', ')}, actual value: #{field_value.inspect})"
|
16
20
|
else
|
17
|
-
"#{error.attribute} #{message} (actual value: #{field_value
|
21
|
+
"#{error.attribute} #{message} (actual value: #{format_value(field_value)})"
|
18
22
|
end
|
19
23
|
end
|
20
24
|
|
@@ -28,7 +32,7 @@ module DbValidator
|
|
28
32
|
def generate_report
|
29
33
|
case DbValidator.configuration.report_format
|
30
34
|
when :json
|
31
|
-
|
35
|
+
Formatters::JsonFormatter.new(@invalid_records).format
|
32
36
|
else
|
33
37
|
generate_text_report
|
34
38
|
end
|
@@ -36,35 +40,81 @@ module DbValidator
|
|
36
40
|
|
37
41
|
private
|
38
42
|
|
43
|
+
def format_value(value)
|
44
|
+
case value
|
45
|
+
when true, false, Symbol
|
46
|
+
value.to_s
|
47
|
+
when String
|
48
|
+
"\"#{value}\""
|
49
|
+
when nil
|
50
|
+
"nil"
|
51
|
+
else
|
52
|
+
value
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
39
56
|
def generate_text_report
|
40
57
|
report = StringIO.new
|
41
|
-
|
42
|
-
|
58
|
+
|
59
|
+
title_box = TTY::Box.frame(
|
60
|
+
width: 50,
|
61
|
+
align: :center,
|
62
|
+
padding: [1, 2],
|
63
|
+
title: { top_left: "DbValidator" },
|
64
|
+
style: {
|
65
|
+
fg: :cyan,
|
66
|
+
border: {
|
67
|
+
fg: :cyan
|
68
|
+
}
|
69
|
+
}
|
70
|
+
) do
|
71
|
+
"Database Validation Report"
|
72
|
+
end
|
73
|
+
|
74
|
+
report.puts title_box
|
43
75
|
report.puts
|
44
76
|
|
45
77
|
if @invalid_records.empty?
|
46
78
|
report.puts "No invalid records found."
|
47
79
|
else
|
48
|
-
|
80
|
+
is_plural = @invalid_records.count > 1
|
81
|
+
report.puts "Found #{@invalid_records.count} invalid #{is_plural ? 'records' : 'record'} across #{@invalid_records.group_by do |r|
|
82
|
+
r[:model]
|
83
|
+
end.keys.count} #{is_plural ? 'models' : 'model'}"
|
49
84
|
report.puts
|
50
85
|
|
51
86
|
@invalid_records.group_by { |r| r[:model] }.each do |model, records|
|
52
|
-
report.puts "#{model}: #{records.count} invalid records"
|
87
|
+
report.puts "#{model}: #{records.count} invalid #{records.count == 1 ? 'record' : 'records'}"
|
88
|
+
|
89
|
+
next if DbValidator.configuration.show_records == false
|
90
|
+
|
91
|
+
report.puts
|
92
|
+
|
53
93
|
records.each do |record|
|
54
|
-
|
94
|
+
record_obj = record[:model].constantize.find_by(id: record[:id])
|
95
|
+
|
96
|
+
info = ["ID: #{record[:id]}"]
|
97
|
+
if record_obj.respond_to?(:created_at)
|
98
|
+
info << "Created: #{record_obj.created_at.strftime('%Y-%m-%d %H:%M:%S')}"
|
99
|
+
end
|
100
|
+
if record_obj.respond_to?(:updated_at)
|
101
|
+
info << "Updated: #{record_obj.updated_at.strftime('%Y-%m-%d %H:%M:%S')}"
|
102
|
+
end
|
103
|
+
info << "Name: #{record_obj.name}" if record_obj.respond_to?(:name)
|
104
|
+
info << "Title: #{record_obj.title}" if record_obj.respond_to?(:title)
|
105
|
+
|
106
|
+
report.puts " #{info.join(' | ')}"
|
55
107
|
record[:errors].each do |error|
|
56
|
-
report.puts "
|
108
|
+
report.puts " ⚠️ #{error}"
|
57
109
|
end
|
110
|
+
report.puts
|
58
111
|
end
|
112
|
+
|
59
113
|
report.puts
|
60
114
|
end
|
61
115
|
end
|
62
116
|
|
63
117
|
report.string
|
64
118
|
end
|
65
|
-
|
66
|
-
def generate_json_report
|
67
|
-
@invalid_records.to_json
|
68
|
-
end
|
69
119
|
end
|
70
120
|
end
|
@@ -1,27 +1,62 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "ruby-progressbar"
|
4
|
-
require "colorize"
|
5
4
|
|
6
5
|
module DbValidator
|
7
6
|
class Validator
|
7
|
+
attr_reader :reporter
|
8
|
+
|
8
9
|
def initialize(options = {})
|
9
|
-
|
10
|
+
configure_from_options(options)
|
10
11
|
@reporter = Reporter.new
|
11
|
-
@fixer = Fixer.new if DbValidator.configuration.auto_fix
|
12
12
|
end
|
13
13
|
|
14
14
|
def validate_all
|
15
|
-
|
15
|
+
models = get_models_to_validate
|
16
|
+
invalid_count = 0
|
16
17
|
|
17
|
-
models
|
18
|
+
models.each do |model|
|
19
|
+
model_count = validate_model(model)
|
20
|
+
invalid_count += model_count if model_count
|
21
|
+
end
|
22
|
+
|
23
|
+
if invalid_count.zero?
|
24
|
+
Rails.logger.debug "\nValidation passed! All records are valid."
|
25
|
+
else
|
26
|
+
total_records = models.sum(&:count)
|
27
|
+
is_plural = invalid_count > 1
|
28
|
+
Rails.logger.debug do
|
29
|
+
"\nFound #{invalid_count} invalid #{is_plural ? 'records' : 'record'} out of #{total_records} total #{is_plural ? 'records' : 'record'}."
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
@reporter.generate_report
|
34
|
+
end
|
35
|
+
|
36
|
+
def validate_test_model(model_name)
|
37
|
+
model = model_name.constantize
|
38
|
+
scope = model.all
|
39
|
+
scope = scope.limit(DbValidator.configuration.limit) if DbValidator.configuration.limit
|
40
|
+
|
41
|
+
total_count = scope.count
|
42
|
+
progress_bar = create_progress_bar("Testing #{model.name}", total_count)
|
43
|
+
invalid_count = 0
|
18
44
|
|
19
|
-
|
20
|
-
|
45
|
+
begin
|
46
|
+
scope.find_each(batch_size: DbValidator.configuration.batch_size) do |record|
|
47
|
+
invalid_count += 1 unless validate_record(record)
|
48
|
+
progress_bar.increment
|
49
|
+
end
|
50
|
+
rescue StandardError => e
|
51
|
+
Rails.logger.debug { "Error validating #{model.name}: #{e.message}" }
|
52
|
+
end
|
21
53
|
|
22
|
-
|
23
|
-
Rails.logger.debug "
|
24
|
-
|
54
|
+
if invalid_count.zero?
|
55
|
+
Rails.logger.debug "\nValidation rule passed! All records would be valid."
|
56
|
+
else
|
57
|
+
Rails.logger.debug do
|
58
|
+
"\nFound #{invalid_count} records that would become invalid out of #{total_count} total records."
|
59
|
+
end
|
25
60
|
end
|
26
61
|
|
27
62
|
@reporter.generate_report
|
@@ -29,8 +64,17 @@ module DbValidator
|
|
29
64
|
|
30
65
|
private
|
31
66
|
|
67
|
+
def configure_from_options(options)
|
68
|
+
return unless options.is_a?(Hash)
|
69
|
+
|
70
|
+
DbValidator.configuration.only_models = Array(options[:only_models]) if options[:only_models]
|
71
|
+
DbValidator.configuration.limit = options[:limit] if options[:limit]
|
72
|
+
DbValidator.configuration.batch_size = options[:batch_size] if options[:batch_size]
|
73
|
+
DbValidator.configuration.report_format = options[:report_format] if options[:report_format]
|
74
|
+
DbValidator.configuration.show_records = options[:show_records] if options[:show_records]
|
75
|
+
end
|
76
|
+
|
32
77
|
def find_all_models
|
33
|
-
# Include all classes inheriting from ActiveRecord::Base
|
34
78
|
ObjectSpace.each_object(Class).select do |klass|
|
35
79
|
klass < ActiveRecord::Base
|
36
80
|
end
|
@@ -42,45 +86,63 @@ module DbValidator
|
|
42
86
|
|
43
87
|
config = DbValidator.configuration
|
44
88
|
model_name = model.name.downcase
|
45
|
-
return config.only_models.include?(model_name) if config.only_models.any?
|
46
89
|
|
47
|
-
config.
|
90
|
+
if config.only_models.any?
|
91
|
+
return config.only_models.map(&:downcase).include?(model_name) ||
|
92
|
+
config.only_models.map(&:downcase).include?(model_name.singularize) ||
|
93
|
+
config.only_models.map(&:downcase).include?(model_name.pluralize)
|
94
|
+
end
|
95
|
+
|
96
|
+
config.ignored_models.map(&:downcase).exclude?(model_name)
|
48
97
|
end
|
49
98
|
|
50
99
|
def validate_model(model)
|
51
|
-
|
52
|
-
|
100
|
+
config = DbValidator.configuration
|
101
|
+
batch_size = config.batch_size || 100
|
102
|
+
limit = config.limit
|
53
103
|
|
54
|
-
|
55
|
-
|
56
|
-
return
|
57
|
-
end
|
104
|
+
scope = model.all
|
105
|
+
scope = scope.limit(limit) if limit
|
58
106
|
|
59
|
-
|
107
|
+
total_count = scope.count
|
108
|
+
return 0 if total_count.zero?
|
60
109
|
|
61
|
-
|
62
|
-
|
110
|
+
progress_bar = create_progress_bar(model.name, total_count)
|
111
|
+
invalid_count = 0
|
63
112
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
Rails.logger.debug "Validated #{processed_records}/#{total_records} records for model #{model.name}".colorize(:green)
|
113
|
+
begin
|
114
|
+
scope.find_in_batches(batch_size: batch_size) do |batch|
|
115
|
+
batch.each do |record|
|
116
|
+
invalid_count += 1 unless validate_record(record)
|
117
|
+
progress_bar.increment
|
70
118
|
end
|
71
119
|
end
|
72
120
|
rescue StandardError => e
|
73
|
-
Rails.logger.debug "Error validating #{model.name}: #{e.message}"
|
121
|
+
Rails.logger.debug { "Error validating #{model.name}: #{e.message}" }
|
74
122
|
end
|
75
|
-
|
76
|
-
|
123
|
+
|
124
|
+
invalid_count
|
125
|
+
end
|
126
|
+
|
127
|
+
def create_progress_bar(model_name, total)
|
128
|
+
ProgressBar.create(
|
129
|
+
title: "Validating #{model_name}",
|
130
|
+
total: total,
|
131
|
+
format: "%t: |%B| %p%% %e",
|
132
|
+
output: $stderr
|
133
|
+
)
|
77
134
|
end
|
78
135
|
|
79
136
|
def validate_record(record)
|
80
|
-
return if record.valid?
|
137
|
+
return true if record.valid?
|
81
138
|
|
82
139
|
@reporter.add_invalid_record(record)
|
83
|
-
|
140
|
+
false
|
141
|
+
end
|
142
|
+
|
143
|
+
def get_models_to_validate
|
144
|
+
models = find_all_models
|
145
|
+
models.select { |model| should_validate_model?(model) }
|
84
146
|
end
|
85
147
|
end
|
86
148
|
end
|
data/lib/db_validator/version.rb
CHANGED
data/lib/db_validator.rb
CHANGED
@@ -13,12 +13,9 @@ DbValidator.configure do |config|
|
|
13
13
|
# "Post" => ["cached_votes"]
|
14
14
|
# }
|
15
15
|
|
16
|
-
# Set the batch size for processing records (default: 1000)
|
17
|
-
# config.batch_size = 1000
|
18
|
-
|
19
16
|
# Set the report format (:text or :json)
|
20
17
|
# config.report_format = :text
|
21
18
|
|
22
|
-
#
|
23
|
-
# config.
|
19
|
+
# Show detailed record information in reports
|
20
|
+
# config.show_records = true
|
24
21
|
end
|
@@ -1,19 +1,95 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
namespace :db_validator do
|
4
|
-
desc "Validate
|
4
|
+
desc "Validate records in the database"
|
5
5
|
task validate: :environment do
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
cli = DbValidator::CLI.new
|
7
|
+
|
8
|
+
has_any_args = ENV["models"].present? || ENV["limit"].present? || ENV["format"].present? || ENV["show_records"].present?
|
9
|
+
|
10
|
+
if has_any_args
|
11
|
+
if ENV["models"].present?
|
12
|
+
models = ENV["models"].split(",").map(&:strip).map(&:classify)
|
13
|
+
DbValidator.configuration.only_models = models
|
14
|
+
end
|
15
|
+
|
16
|
+
DbValidator.configuration.limit = ENV["limit"].to_i if ENV["limit"].present?
|
17
|
+
DbValidator.configuration.report_format = ENV["format"].to_sym if ENV["format"].present?
|
18
|
+
DbValidator.configuration.show_records = ENV["show_records"] != "false" if ENV["show_records"].present?
|
19
|
+
else
|
20
|
+
cli.display_progress("Loading models") do
|
21
|
+
Rails.application.eager_load!
|
22
|
+
end
|
10
23
|
|
11
|
-
|
24
|
+
available_models = ActiveRecord::Base.descendants
|
25
|
+
.reject(&:abstract_class?)
|
26
|
+
.select(&:table_exists?)
|
27
|
+
.map(&:name)
|
28
|
+
.sort
|
12
29
|
|
13
|
-
|
30
|
+
selected_models = cli.select_models(available_models)
|
31
|
+
options = cli.configure_options
|
32
|
+
|
33
|
+
DbValidator.configuration.only_models = selected_models
|
34
|
+
DbValidator.configuration.limit = options[:limit] if options[:limit].present?
|
35
|
+
DbValidator.configuration.batch_size = options[:batch_size] if options[:batch_size].present?
|
36
|
+
DbValidator.configuration.report_format = options[:format].to_sym if options[:format].present?
|
37
|
+
DbValidator.configuration.show_records = options[:show_records] if options[:show_records].present?
|
38
|
+
end
|
14
39
|
|
15
40
|
validator = DbValidator::Validator.new
|
16
41
|
report = validator.validate_all
|
17
|
-
puts report
|
42
|
+
puts "\n#{report}"
|
43
|
+
end
|
44
|
+
|
45
|
+
desc "Test validation rules on existing records"
|
46
|
+
task test: :environment do
|
47
|
+
unless ENV["model"] && ENV["rule"]
|
48
|
+
puts "Usage: rake db_validator:test model=user rule='validates :field, presence: true' [show_records=false] [limit=1000] [format=json]"
|
49
|
+
exit 1
|
50
|
+
end
|
51
|
+
|
52
|
+
model_name = ENV.fetch("model").classify
|
53
|
+
validation_rule = ENV.fetch("rule", nil)
|
54
|
+
|
55
|
+
# Configure options
|
56
|
+
DbValidator.configuration.show_records = ENV["show_records"] != "false" if ENV["show_records"].present?
|
57
|
+
DbValidator.configuration.limit = ENV["limit"].to_i if ENV["limit"].present?
|
58
|
+
DbValidator.configuration.report_format = ENV["format"].to_sym if ENV["format"].present?
|
59
|
+
|
60
|
+
begin
|
61
|
+
base_model = model_name.constantize
|
62
|
+
# Extract attribute name from validation rule
|
63
|
+
attribute_match = validation_rule.match(/validates\s+:(\w+)/)
|
64
|
+
if attribute_match
|
65
|
+
attribute_name = attribute_match[1]
|
66
|
+
unless base_model.column_names.include?(attribute_name) || base_model.method_defined?(attribute_name)
|
67
|
+
puts "\n❌ Error: Attribute '#{attribute_name}' does not exist for model '#{model_name}'"
|
68
|
+
puts "Available columns: #{base_model.column_names.join(', ')}"
|
69
|
+
exit 1
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
# Create temporary subclass with new validation
|
74
|
+
temp_model = Class.new(base_model) do
|
75
|
+
self.table_name = base_model.table_name
|
76
|
+
class_eval(validation_rule)
|
77
|
+
end
|
78
|
+
|
79
|
+
Object.const_set("Temporary#{model_name}", temp_model)
|
80
|
+
|
81
|
+
validator = DbValidator::Validator.new
|
82
|
+
report = validator.validate_test_model("Temporary#{model_name}")
|
83
|
+
puts "\n#{report}"
|
84
|
+
rescue NameError
|
85
|
+
puts "\n❌ Error: Model '#{model_name}' not found"
|
86
|
+
exit 1
|
87
|
+
rescue SyntaxError => e
|
88
|
+
puts "\n❌ Error: Invalid validation rule syntax"
|
89
|
+
puts e.message
|
90
|
+
exit 1
|
91
|
+
ensure
|
92
|
+
Object.send(:remove_const, "Temporary#{model_name}") if Object.const_defined?("Temporary#{model_name}")
|
93
|
+
end
|
18
94
|
end
|
19
95
|
end
|
data/readme.md
CHANGED
@@ -1,6 +1,9 @@
|
|
1
|
+
[![Gem Version](https://badge.fury.io/rb/db_validator.svg?icon=si%3Arubygems)](https://badge.fury.io/rb/db_validator)
|
2
|
+
[![RSpec Tests](https://github.com/krzysztoff1/db-validator/actions/workflows/rspec.yml/badge.svg)](https://github.com/krzysztoff1/db-validator/actions/workflows/rspec.yml)
|
3
|
+
|
1
4
|
# DbValidator
|
2
5
|
|
3
|
-
DbValidator helps identify invalid records in your Rails application that don't meet model validation requirements.
|
6
|
+
DbValidator helps identify invalid records in your Rails application that don't meet model validation requirements. It finds records that became invalid after validation rule changes, and validates imported or manually edited data. You can use it to audit records before deploying new validations and catch any data that bypassed validation checks.
|
4
7
|
|
5
8
|
## Installation
|
6
9
|
|
@@ -16,24 +19,22 @@ Then execute:
|
|
16
19
|
$ bundle install
|
17
20
|
```
|
18
21
|
|
19
|
-
Or install it yourself:
|
20
|
-
|
21
|
-
```bash
|
22
|
-
$ gem install db_validator
|
23
|
-
```
|
24
|
-
|
25
22
|
## Usage
|
26
23
|
|
27
24
|
### Rake Task
|
28
25
|
|
29
26
|
The simplest way to run validation is using the provided rake task:
|
30
27
|
|
31
|
-
#### Validate
|
28
|
+
#### Validate models in interactive mode
|
29
|
+
|
30
|
+
<img width="798" alt="Screenshot 2024-11-07 at 21 50 57" src="https://github.com/user-attachments/assets/33fbdb8b-b8ec-4284-9313-c1eeaf2eab2d">
|
32
31
|
|
33
32
|
```bash
|
34
33
|
$ rake db_validator:validate
|
35
34
|
```
|
36
35
|
|
36
|
+
This will start an interactive mode where you can select which models to validate and adjust other options.
|
37
|
+
|
37
38
|
#### Validate specific models
|
38
39
|
|
39
40
|
```bash
|
@@ -52,6 +53,39 @@ $ rake db_validator:validate limit=1000
|
|
52
53
|
$ rake db_validator:validate format=json
|
53
54
|
```
|
54
55
|
|
56
|
+
### Test Mode
|
57
|
+
|
58
|
+
You can test new validation rules before applying them to your models:
|
59
|
+
|
60
|
+
```bash
|
61
|
+
$ rake db_validator:test model=User rule='validates :name, presence: true'
|
62
|
+
```
|
63
|
+
|
64
|
+
#### Testing Email Format Validation
|
65
|
+
|
66
|
+
Here's an example of testing email format validation rules:
|
67
|
+
|
68
|
+
```bash
|
69
|
+
# Testing invalid email format (without @)
|
70
|
+
$ rake db_validator:test model=User rule='validates :email, format: { without: /@/, message: "must contain @" }'
|
71
|
+
|
72
|
+
Found 100 records that would become invalid out of 100 total records.
|
73
|
+
|
74
|
+
# Testing valid email format (with @)
|
75
|
+
$ rake db_validator:test model=User rule='validates :email, format: { with: /@/, message: "must contain @" }'
|
76
|
+
|
77
|
+
No invalid records found.
|
78
|
+
```
|
79
|
+
|
80
|
+
#### Error Handling
|
81
|
+
|
82
|
+
Trying to test a validation rule for a non-existent attribute will return an error:
|
83
|
+
|
84
|
+
```
|
85
|
+
❌ Error: Attribute 'i_dont_exist' does not exist for model 'User'
|
86
|
+
Available columns: id, email, created_at, updated_at, name
|
87
|
+
```
|
88
|
+
|
55
89
|
### Ruby Code
|
56
90
|
|
57
91
|
You can also run validation from your Ruby code:
|
@@ -95,17 +129,26 @@ ID: 5
|
|
95
129
|
|
96
130
|
### JSON Format
|
97
131
|
|
132
|
+
The JSON report is saved to a file in the `db_validator_reports` directory.
|
133
|
+
|
98
134
|
```json
|
99
|
-
|
100
|
-
{
|
101
|
-
"
|
102
|
-
"
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
135
|
+
{
|
136
|
+
"User": {
|
137
|
+
"error_count": 2,
|
138
|
+
"records": [
|
139
|
+
{
|
140
|
+
"id": 1,
|
141
|
+
"errors": [
|
142
|
+
"email is invalid (actual value: \"invalid-email\")"
|
143
|
+
]
|
144
|
+
},
|
145
|
+
{
|
146
|
+
"id": 2,
|
147
|
+
"errors": [
|
148
|
+
"name can't be blank (actual value: \"\")"
|
149
|
+
]
|
150
|
+
}
|
151
|
+
]
|
109
152
|
}
|
110
|
-
|
153
|
+
}
|
111
154
|
```
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: db_validator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Krzysztof Duda
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-11-
|
11
|
+
date: 2024-11-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -67,35 +67,66 @@ dependencies:
|
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '1.4'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
70
|
+
name: ruby-progressbar
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: '1.11'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
82
|
+
version: '1.11'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
84
|
+
name: tty-box
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
89
|
+
version: 0.7.0
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
97
|
-
|
98
|
-
|
96
|
+
version: 0.7.0
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: tty-prompt
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 0.23.1
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 0.23.1
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: tty-spinner
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 0.9.3
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 0.9.3
|
125
|
+
description: DbValidator helps identify invalid records in your Rails application
|
126
|
+
that don't meet model validation requirements. It finds records that became invalid
|
127
|
+
after validation rule changes, and validates imported or manually edited data. You
|
128
|
+
can use it to audit records before deploying new validations and catch any data
|
129
|
+
that bypassed validation checks.
|
99
130
|
email:
|
100
131
|
- duda_krzysztof@outlook.com
|
101
132
|
executables: []
|
@@ -104,8 +135,9 @@ extra_rdoc_files: []
|
|
104
135
|
files:
|
105
136
|
- config/initializers/db_validator.rb
|
106
137
|
- lib/db_validator.rb
|
138
|
+
- lib/db_validator/cli.rb
|
107
139
|
- lib/db_validator/configuration.rb
|
108
|
-
- lib/db_validator/
|
140
|
+
- lib/db_validator/formatters/json_formatter.rb
|
109
141
|
- lib/db_validator/railtie.rb
|
110
142
|
- lib/db_validator/reporter.rb
|
111
143
|
- lib/db_validator/validator.rb
|
@@ -114,10 +146,13 @@ files:
|
|
114
146
|
- lib/generators/db_validator/templates/initializer.rb
|
115
147
|
- lib/tasks/db_validator_tasks.rake
|
116
148
|
- readme.md
|
117
|
-
homepage: https://github.com/
|
149
|
+
homepage: https://github.com/krzysztoff1/db-validator
|
118
150
|
licenses:
|
119
151
|
- MIT
|
120
152
|
metadata:
|
153
|
+
source_code_uri: https://github.com/krzysztoff1/db-validator/
|
154
|
+
documentation_uri: https://github.com/krzysztoff1/db-validator/blob/main/README.md
|
155
|
+
changelog_uri: https://github.com/krzysztoff1/db-validator/blob/main/changelog.md
|
121
156
|
rubygems_mfa_required: 'true'
|
122
157
|
post_install_message:
|
123
158
|
rdoc_options: []
|
@@ -134,8 +169,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
134
169
|
- !ruby/object:Gem::Version
|
135
170
|
version: '0'
|
136
171
|
requirements: []
|
137
|
-
rubygems_version: 3.5.
|
172
|
+
rubygems_version: 3.5.9
|
138
173
|
signing_key:
|
139
174
|
specification_version: 4
|
140
|
-
summary:
|
175
|
+
summary: DbValidator helps identify invalid records in your Rails application that
|
176
|
+
don't meet model validation requirements
|
141
177
|
test_files: []
|
data/lib/db_validator/fixer.rb
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module DbValidator
|
4
|
-
class Fixer
|
5
|
-
def initialize
|
6
|
-
@fixed_records = 0
|
7
|
-
@failed_fixes = 0
|
8
|
-
end
|
9
|
-
|
10
|
-
def fix_record(record)
|
11
|
-
return false unless record.invalid?
|
12
|
-
|
13
|
-
success = attempt_fix(record)
|
14
|
-
if success
|
15
|
-
@fixed_records += 1
|
16
|
-
else
|
17
|
-
@failed_fixes += 1
|
18
|
-
end
|
19
|
-
success
|
20
|
-
end
|
21
|
-
|
22
|
-
def statistics
|
23
|
-
{
|
24
|
-
fixed_records: @fixed_records,
|
25
|
-
failed_fixes: @failed_fixes
|
26
|
-
}
|
27
|
-
end
|
28
|
-
|
29
|
-
private
|
30
|
-
|
31
|
-
def attempt_fix(record)
|
32
|
-
record.save
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|