db_validator 0.2.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/db_validator/cli.rb +95 -24
- data/lib/db_validator/config_updater.rb +40 -0
- data/lib/db_validator/configuration.rb +2 -1
- data/lib/db_validator/formatters/json_formatter.rb +44 -0
- data/lib/db_validator/formatters/message_formatter.rb +59 -0
- data/lib/db_validator/reporter.rb +84 -59
- data/lib/db_validator/test_task.rb +59 -0
- data/lib/db_validator/validate_task.rb +60 -0
- data/lib/db_validator/validator.rb +93 -33
- data/lib/db_validator/version.rb +1 -1
- data/lib/db_validator.rb +3 -0
- data/lib/generators/db_validator/templates/initializer.rb +3 -3
- data/lib/tasks/db_validator_tasks.rake +15 -31
- data/readme.md +49 -14
- metadata +17 -23
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 130796e31968cc4d325f42092eb82f8e21a67c56fdfc751116f354f6998c93f5
|
4
|
+
data.tar.gz: e4ec35da042e41699b25e607f642d9208564bf49f88f99be191029756f1e7f96
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6cdeaf577ff2b53836dca41616b8c90fb0bdce77d0e8e8d456fcb1fc3889ae02373b72cf70df4829c77d22f3ef0217f211c02a39b779d3a06fdfd2aaba66c085
|
7
|
+
data.tar.gz: b1abf3df297599475a5876aef7fed5a4e6c304f5175ed686c003ee75214c08f87755a5c5fa7825a8171d37af495723e0d357c8e56511ca358190f998f4498dad
|
data/lib/db_validator/cli.rb
CHANGED
@@ -3,11 +3,30 @@
|
|
3
3
|
require "tty-prompt"
|
4
4
|
require "tty-box"
|
5
5
|
require "tty-spinner"
|
6
|
+
require "optparse"
|
7
|
+
require "logger"
|
6
8
|
|
7
9
|
module DbValidator
|
8
10
|
class CLI
|
9
11
|
def initialize
|
10
12
|
@prompt = TTY::Prompt.new
|
13
|
+
@options = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def start
|
17
|
+
if ARGV.empty?
|
18
|
+
interactive_mode
|
19
|
+
else
|
20
|
+
parse_command_line_args
|
21
|
+
validate_with_options
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def display_progress(message)
|
26
|
+
spinner = TTY::Spinner.new("[:spinner] #{message}", format: :dots)
|
27
|
+
spinner.auto_spin
|
28
|
+
yield if block_given?
|
29
|
+
spinner.success
|
11
30
|
end
|
12
31
|
|
13
32
|
def select_models(available_models)
|
@@ -40,37 +59,89 @@ module DbValidator
|
|
40
59
|
end
|
41
60
|
end
|
42
61
|
|
43
|
-
def
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
62
|
+
def parse_command_line_args # rubocop:disable Metrics/AbcSize
|
63
|
+
args = ARGV.join(" ").split(/\s+/)
|
64
|
+
args.each do |arg|
|
65
|
+
key, value = arg.split("=")
|
66
|
+
case key
|
67
|
+
when "models"
|
68
|
+
@options[:only_models] = value.split(",").map(&:strip).map(&:classify)
|
69
|
+
when "limit"
|
70
|
+
@options[:limit] = value.to_i
|
71
|
+
when "format"
|
72
|
+
@options[:report_format] = value.to_sym
|
73
|
+
when "show_records"
|
74
|
+
@options[:show_records] = value.to_sym
|
51
75
|
end
|
52
|
-
|
76
|
+
end
|
77
|
+
end
|
53
78
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
79
|
+
def validate_with_options
|
80
|
+
load_rails
|
81
|
+
configure_validator(@options[:only_models], @options)
|
82
|
+
validator = DbValidator::Validator.new
|
83
|
+
report = validator.validate_all
|
84
|
+
Rails.logger.debug { "\n#{report}" }
|
85
|
+
end
|
59
86
|
|
60
|
-
|
87
|
+
def interactive_mode
|
88
|
+
load_rails
|
89
|
+
display_header
|
90
|
+
|
91
|
+
display_progress("Loading models") do
|
92
|
+
Rails.application.eager_load!
|
61
93
|
end
|
62
94
|
|
63
|
-
|
95
|
+
available_models = ActiveRecord::Base.descendants
|
96
|
+
.reject(&:abstract_class?)
|
97
|
+
.select(&:table_exists?)
|
98
|
+
.map(&:name)
|
99
|
+
.sort
|
100
|
+
|
101
|
+
if available_models.empty?
|
102
|
+
Rails.logger.debug "No models found in the application."
|
103
|
+
raise "No models found in the application. Please run this command from your Rails application root."
|
104
|
+
end
|
105
|
+
|
106
|
+
selected_models = select_models(available_models)
|
107
|
+
options = configure_options
|
108
|
+
|
109
|
+
configure_validator(selected_models, options)
|
110
|
+
validator = DbValidator::Validator.new
|
111
|
+
report = validator.validate_all
|
112
|
+
Rails.logger.debug { "\n#{report}" }
|
64
113
|
end
|
65
114
|
|
66
|
-
def
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
115
|
+
def load_rails
|
116
|
+
require File.expand_path("config/environment", Dir.pwd)
|
117
|
+
rescue LoadError
|
118
|
+
Rails.logger.debug "Error: Rails application not found. Please run this command from your Rails application root."
|
119
|
+
raise "Rails application not found. Please run this command from your Rails application root."
|
71
120
|
end
|
72
121
|
|
73
|
-
|
122
|
+
def configure_validator(models = nil, options = {})
|
123
|
+
config = DbValidator.configuration
|
124
|
+
config.only_models = models if models
|
125
|
+
config.limit = options[:limit] if options[:limit]
|
126
|
+
config.batch_size = options[:batch_size] if options[:batch_size]
|
127
|
+
config.report_format = options[:format] if options[:format]
|
128
|
+
config.show_records = options[:show_records] if options[:show_records]
|
129
|
+
end
|
130
|
+
|
131
|
+
def configure_options
|
132
|
+
options = {}
|
133
|
+
|
134
|
+
@prompt.say("\n")
|
135
|
+
limit_input = @prompt.ask("Enter record limit (leave blank for no limit):") do |q|
|
136
|
+
q.validate(/^\d*$/, "Please enter a valid number")
|
137
|
+
q.convert(:int, nil)
|
138
|
+
end
|
139
|
+
options[:limit] = limit_input if limit_input.present?
|
140
|
+
|
141
|
+
options[:format] = @prompt.select("Select report format:", %w[text json], default: "text")
|
142
|
+
|
143
|
+
options
|
144
|
+
end
|
74
145
|
|
75
146
|
def display_header
|
76
147
|
title = TTY::Box.frame(
|
@@ -85,7 +156,7 @@ module DbValidator
|
|
85
156
|
}
|
86
157
|
}
|
87
158
|
)
|
88
|
-
|
159
|
+
Rails.logger.debug title
|
89
160
|
end
|
90
161
|
end
|
91
|
-
end
|
162
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DbValidator
|
4
|
+
class ConfigUpdater
|
5
|
+
def self.update_from_env
|
6
|
+
new.update_from_env
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.update_from_options(options)
|
10
|
+
new.update_from_options(options)
|
11
|
+
end
|
12
|
+
|
13
|
+
def update_from_env
|
14
|
+
update_config(
|
15
|
+
limit: ENV["limit"]&.to_i,
|
16
|
+
report_format: ENV["format"]&.to_sym,
|
17
|
+
show_records: ENV["show_records"] != "false"
|
18
|
+
)
|
19
|
+
end
|
20
|
+
|
21
|
+
def update_from_options(options)
|
22
|
+
update_config(
|
23
|
+
limit: options[:limit],
|
24
|
+
batch_size: options[:batch_size],
|
25
|
+
report_format: options[:format]&.to_sym,
|
26
|
+
show_records: options[:show_records]
|
27
|
+
)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def update_config(settings)
|
33
|
+
settings.each do |key, value|
|
34
|
+
next unless value
|
35
|
+
|
36
|
+
DbValidator.configuration.public_send("#{key}=", value)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module DbValidator
|
4
4
|
class Configuration
|
5
|
-
attr_accessor :only_models, :ignored_models, :ignored_attributes, :batch_size, :report_format, :limit
|
5
|
+
attr_accessor :only_models, :ignored_models, :ignored_attributes, :batch_size, :report_format, :limit, :show_records
|
6
6
|
|
7
7
|
def initialize
|
8
8
|
@only_models = []
|
@@ -11,6 +11,7 @@ module DbValidator
|
|
11
11
|
@batch_size = 1000
|
12
12
|
@report_format = :text
|
13
13
|
@limit = nil
|
14
|
+
@show_records = true
|
14
15
|
end
|
15
16
|
|
16
17
|
def only_models=(models)
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
require "fileutils"
|
5
|
+
|
6
|
+
module DbValidator
|
7
|
+
module Formatters
|
8
|
+
class JsonFormatter
|
9
|
+
def initialize(invalid_records)
|
10
|
+
@invalid_records = invalid_records
|
11
|
+
end
|
12
|
+
|
13
|
+
def format
|
14
|
+
formatted_data = @invalid_records.group_by { |r| r[:model] }.transform_values do |records|
|
15
|
+
{
|
16
|
+
error_count: records.length,
|
17
|
+
records: records.map { |r| format_record(r) }
|
18
|
+
}
|
19
|
+
end
|
20
|
+
|
21
|
+
save_to_file(formatted_data)
|
22
|
+
formatted_data.to_json
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def format_record(record)
|
28
|
+
{
|
29
|
+
id: record[:id],
|
30
|
+
errors: record[:errors]
|
31
|
+
}
|
32
|
+
end
|
33
|
+
|
34
|
+
def save_to_file(data)
|
35
|
+
FileUtils.mkdir_p("db_validator_reports")
|
36
|
+
timestamp = Time.zone.now.strftime("%Y%m%d_%H%M%S")
|
37
|
+
filename = "db_validator_reports/validation_report_#{timestamp}.json"
|
38
|
+
|
39
|
+
File.write(filename, JSON.pretty_generate(data))
|
40
|
+
Rails.logger.info "JSON report saved to #{filename}"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DbValidator
|
4
|
+
module Formatters
|
5
|
+
class MessageFormatter
|
6
|
+
def initialize(record)
|
7
|
+
@record = record
|
8
|
+
end
|
9
|
+
|
10
|
+
def format_error_message(error, field_value, message)
|
11
|
+
return enum_validation_message(error, field_value, message) if error.options[:in].present?
|
12
|
+
return enum_field_message(error, field_value, message) if enum_field?(error)
|
13
|
+
|
14
|
+
basic_validation_message(error, field_value, message)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
attr_reader :record
|
20
|
+
|
21
|
+
def enum_validation_message(error, field_value, message)
|
22
|
+
allowed = error.options[:in].join(", ")
|
23
|
+
error_message = "#{error.attribute} #{message}"
|
24
|
+
details = " (allowed values: #{allowed}, actual value: #{field_value.inspect})"
|
25
|
+
|
26
|
+
"#{error_message} #{details}"
|
27
|
+
end
|
28
|
+
|
29
|
+
def enum_field_message(error, field_value, message)
|
30
|
+
enum_values = record.class.defined_enums[error.attribute.to_s].keys
|
31
|
+
error_message = "#{error.attribute} #{message}"
|
32
|
+
details = " (allowed values: #{enum_values.join(', ')}, actual value: #{field_value.inspect})"
|
33
|
+
|
34
|
+
"#{error_message} #{details}"
|
35
|
+
end
|
36
|
+
|
37
|
+
def enum_field?(error)
|
38
|
+
record.class.defined_enums[error.attribute.to_s].present?
|
39
|
+
end
|
40
|
+
|
41
|
+
def basic_validation_message(error, field_value, message)
|
42
|
+
"#{error.attribute} #{message} (actual value: #{format_value(field_value)})"
|
43
|
+
end
|
44
|
+
|
45
|
+
def format_value(value)
|
46
|
+
case value
|
47
|
+
when true, false, Symbol
|
48
|
+
value.to_s
|
49
|
+
when String
|
50
|
+
"\"#{value}\""
|
51
|
+
when nil
|
52
|
+
"nil"
|
53
|
+
else
|
54
|
+
value
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
require "tty-box"
|
4
4
|
require "tty-spinner"
|
5
|
+
require "db_validator/formatters/json_formatter"
|
6
|
+
require "db_validator/formatters/message_formatter"
|
5
7
|
|
6
8
|
module DbValidator
|
7
9
|
class Reporter
|
@@ -10,15 +12,11 @@ module DbValidator
|
|
10
12
|
end
|
11
13
|
|
12
14
|
def add_invalid_record(record)
|
15
|
+
formatter = Formatters::MessageFormatter.new(record)
|
13
16
|
enhanced_errors = record.errors.map do |error|
|
14
17
|
field_value = record.send(error.attribute)
|
15
18
|
message = error.message
|
16
|
-
|
17
|
-
if error.options[:in].present?
|
18
|
-
"#{error.attribute} #{message} (allowed values: #{error.options[:in].join(', ')}, actual value: #{field_value.inspect})"
|
19
|
-
else
|
20
|
-
"#{error.attribute} #{message} (actual value: #{format_value(field_value)})"
|
21
|
-
end
|
19
|
+
formatter.format_error_message(error, field_value, message)
|
22
20
|
end
|
23
21
|
|
24
22
|
@invalid_records << {
|
@@ -28,10 +26,15 @@ module DbValidator
|
|
28
26
|
}
|
29
27
|
end
|
30
28
|
|
29
|
+
def generate_report_message(error, field_value, message)
|
30
|
+
formatter = Formatters::MessageFormatter.new(record)
|
31
|
+
formatter.format_error_message(error, field_value, message)
|
32
|
+
end
|
33
|
+
|
31
34
|
def generate_report
|
32
35
|
case DbValidator.configuration.report_format
|
33
36
|
when :json
|
34
|
-
|
37
|
+
Formatters::JsonFormatter.new(@invalid_records).format
|
35
38
|
else
|
36
39
|
generate_text_report
|
37
40
|
end
|
@@ -39,23 +42,81 @@ module DbValidator
|
|
39
42
|
|
40
43
|
private
|
41
44
|
|
42
|
-
def
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
"
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
45
|
+
def generate_text_report
|
46
|
+
print_title
|
47
|
+
|
48
|
+
report = StringIO.new
|
49
|
+
|
50
|
+
if @invalid_records.empty?
|
51
|
+
report.puts "No invalid records found."
|
52
|
+
return report.string
|
53
|
+
end
|
54
|
+
|
55
|
+
report.puts print_summary
|
56
|
+
report.puts
|
57
|
+
|
58
|
+
@invalid_records.group_by { |r| r[:model] }.each do |model, records|
|
59
|
+
report.puts generate_model_report(model, records)
|
53
60
|
end
|
61
|
+
|
62
|
+
report.string
|
54
63
|
end
|
55
64
|
|
56
|
-
def
|
65
|
+
def print_summary
|
66
|
+
report = StringIO.new
|
67
|
+
is_plural = @invalid_records.count > 1
|
68
|
+
record_word = is_plural ? "records" : "record"
|
69
|
+
model_word = is_plural ? "models" : "model"
|
70
|
+
|
71
|
+
report.puts "Found #{@invalid_records.count} invalid #{record_word} across #{@invalid_records.group_by do |r|
|
72
|
+
r[:model]
|
73
|
+
end.keys.count} #{model_word}"
|
74
|
+
|
75
|
+
report.string
|
76
|
+
end
|
77
|
+
|
78
|
+
def generate_model_report(model, records)
|
79
|
+
report = StringIO.new
|
80
|
+
report.puts
|
81
|
+
report.puts "#{model}: #{records.count} invalid #{records.count == 1 ? 'record' : 'records'}"
|
82
|
+
report.puts
|
83
|
+
|
84
|
+
records.each_with_index do |record, index|
|
85
|
+
report.puts generate_record_report(record, index)
|
86
|
+
end
|
87
|
+
|
88
|
+
report.string
|
89
|
+
end
|
90
|
+
|
91
|
+
def generate_record_report(record, index) # rubocop:disable Metrics/AbcSize
|
57
92
|
report = StringIO.new
|
58
|
-
|
93
|
+
|
94
|
+
record_obj = record[:model].constantize.find_by(id: record[:id])
|
95
|
+
info = []
|
96
|
+
info << "Record ##{index + 1}"
|
97
|
+
info << "ID: #{record[:id]}"
|
98
|
+
|
99
|
+
# Add timestamps if available
|
100
|
+
if record_obj.respond_to?(:created_at)
|
101
|
+
info << "Created: #{record_obj.created_at.strftime('%b %d, %Y at %I:%M %p')}"
|
102
|
+
end
|
103
|
+
if record_obj.respond_to?(:updated_at)
|
104
|
+
info << "Updated: #{record_obj.updated_at.strftime('%b %d, %Y at %I:%M %p')}"
|
105
|
+
end
|
106
|
+
|
107
|
+
# Add identifying fields if available
|
108
|
+
info << "Name: #{record_obj.name}" if record_obj.respond_to?(:name) && record_obj.name.present?
|
109
|
+
info << "Title: #{record_obj.title}" if record_obj.respond_to?(:title) && record_obj.title.present?
|
110
|
+
|
111
|
+
report.puts " #{info.join(', ')}"
|
112
|
+
record[:errors].each do |error|
|
113
|
+
report.puts " \e[31m- #{error}\e[0m"
|
114
|
+
end
|
115
|
+
|
116
|
+
report.string
|
117
|
+
end
|
118
|
+
|
119
|
+
def print_title
|
59
120
|
title_box = TTY::Box.frame(
|
60
121
|
width: 50,
|
61
122
|
align: :center,
|
@@ -70,46 +131,10 @@ module DbValidator
|
|
70
131
|
) do
|
71
132
|
"Database Validation Report"
|
72
133
|
end
|
73
|
-
|
74
|
-
report.puts title_box
|
75
|
-
report.puts
|
76
|
-
|
77
|
-
if @invalid_records.empty?
|
78
|
-
report.puts "No invalid records found.".colorize(:green)
|
79
|
-
else
|
80
|
-
report.puts "Found #{@invalid_records.count} invalid records across #{@invalid_records.group_by { |r| r[:model] }.keys.count} models".colorize(:yellow)
|
81
|
-
report.puts
|
82
|
-
|
83
|
-
@invalid_records.group_by { |r| r[:model] }.each do |model, records|
|
84
|
-
report.puts "#{model}: #{records.count} invalid records".colorize(:red)
|
85
|
-
report.puts
|
86
|
-
|
87
|
-
records.each do |record|
|
88
|
-
record_obj = record[:model].constantize.find_by(id: record[:id])
|
89
|
-
next unless record_obj
|
90
|
-
|
91
|
-
info = ["ID: #{record[:id]}"]
|
92
|
-
info << "Created: #{record_obj.created_at.strftime('%Y-%m-%d %H:%M:%S')}" if record_obj.respond_to?(:created_at)
|
93
|
-
info << "Updated: #{record_obj.updated_at.strftime('%Y-%m-%d %H:%M:%S')}" if record_obj.respond_to?(:updated_at)
|
94
|
-
info << "Name: #{record_obj.name}" if record_obj.respond_to?(:name)
|
95
|
-
info << "Title: #{record_obj.title}" if record_obj.respond_to?(:title)
|
96
|
-
|
97
|
-
report.puts " #{info.join(' | ')}".colorize(:white)
|
98
|
-
record[:errors].each do |error|
|
99
|
-
report.puts " ⚠️ #{error}".colorize(:white)
|
100
|
-
end
|
101
|
-
report.puts
|
102
|
-
end
|
103
|
-
|
104
|
-
report.puts
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
report.string
|
109
|
-
end
|
110
134
|
|
111
|
-
|
112
|
-
|
135
|
+
puts
|
136
|
+
puts title_box
|
137
|
+
puts
|
113
138
|
end
|
114
139
|
end
|
115
140
|
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DbValidator
|
4
|
+
class TestTask
|
5
|
+
def initialize(model_name, validation_rule)
|
6
|
+
@model_name = model_name
|
7
|
+
@validation_rule = validation_rule
|
8
|
+
end
|
9
|
+
|
10
|
+
def execute
|
11
|
+
validate_and_test_model
|
12
|
+
rescue NameError
|
13
|
+
puts "Model '#{@model_name}' not found"
|
14
|
+
raise "Model '#{@model_name}' not found"
|
15
|
+
rescue SyntaxError
|
16
|
+
puts "Invalid validation rule syntax"
|
17
|
+
raise "Invalid validation rule syntax"
|
18
|
+
ensure
|
19
|
+
cleanup_temporary_model
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def validate_and_test_model
|
25
|
+
base_model = @model_name.constantize
|
26
|
+
validate_attribute(base_model)
|
27
|
+
|
28
|
+
temp_model = create_temporary_model(base_model)
|
29
|
+
Object.const_set("Temporary#{@model_name}", temp_model)
|
30
|
+
|
31
|
+
validator = DbValidator::Validator.new
|
32
|
+
report = validator.validate_test_model("Temporary#{@model_name}")
|
33
|
+
puts report
|
34
|
+
end
|
35
|
+
|
36
|
+
def validate_attribute(base_model)
|
37
|
+
attribute_match = @validation_rule.match(/validates\s+:(\w+)/)
|
38
|
+
return unless attribute_match
|
39
|
+
|
40
|
+
attribute_name = attribute_match[1]
|
41
|
+
return if base_model.column_names.include?(attribute_name) || base_model.method_defined?(attribute_name)
|
42
|
+
|
43
|
+
puts "Attribute '#{attribute_name}' does not exist for model '#{@model_name}'"
|
44
|
+
raise "Attribute '#{attribute_name}' does not exist for model '#{@model_name}'"
|
45
|
+
end
|
46
|
+
|
47
|
+
def create_temporary_model(base_model)
|
48
|
+
Class.new(base_model) do
|
49
|
+
self.table_name = base_model.table_name
|
50
|
+
class_eval(@validation_rule)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def cleanup_temporary_model
|
55
|
+
temp_const_name = "Temporary#{@model_name}"
|
56
|
+
Object.send(:remove_const, temp_const_name) if Object.const_defined?(temp_const_name)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DbValidator
|
4
|
+
class ValidateTask
|
5
|
+
def initialize(cli = DbValidator::CLI.new)
|
6
|
+
@cli = cli
|
7
|
+
end
|
8
|
+
|
9
|
+
def execute
|
10
|
+
configure_from_env_or_cli
|
11
|
+
run_validation
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
def configure_from_env_or_cli
|
17
|
+
if env_args_present?
|
18
|
+
configure_from_env
|
19
|
+
else
|
20
|
+
configure_from_cli
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def env_args_present?
|
25
|
+
ENV["models"].present? || ENV["limit"].present? ||
|
26
|
+
ENV["format"].present? || ENV["show_records"].present?
|
27
|
+
end
|
28
|
+
|
29
|
+
def configure_from_env
|
30
|
+
if ENV["models"].present?
|
31
|
+
models = ENV["models"].split(",").map(&:strip).map(&:classify)
|
32
|
+
DbValidator.configuration.only_models = models
|
33
|
+
end
|
34
|
+
|
35
|
+
ConfigUpdater.update_from_env
|
36
|
+
end
|
37
|
+
|
38
|
+
def configure_from_cli
|
39
|
+
@cli.display_progress("Loading models") { Rails.application.eager_load! }
|
40
|
+
|
41
|
+
available_models = ActiveRecord::Base.descendants
|
42
|
+
.reject(&:abstract_class?)
|
43
|
+
.select(&:table_exists?)
|
44
|
+
.map(&:name)
|
45
|
+
.sort
|
46
|
+
|
47
|
+
selected_models = @cli.select_models(available_models)
|
48
|
+
options = @cli.configure_options
|
49
|
+
|
50
|
+
DbValidator.configuration.only_models = selected_models
|
51
|
+
ConfigUpdater.update_from_options(options)
|
52
|
+
end
|
53
|
+
|
54
|
+
def run_validation
|
55
|
+
validator = DbValidator::Validator.new
|
56
|
+
report = validator.validate_all
|
57
|
+
puts report
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -1,33 +1,68 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "ruby-progressbar"
|
4
|
-
require "colorize"
|
5
4
|
|
6
5
|
module DbValidator
|
7
6
|
class Validator
|
8
7
|
attr_reader :reporter
|
9
8
|
|
10
9
|
def initialize(options = {})
|
11
|
-
@options = options
|
12
|
-
@reporter = Reporter.new
|
13
10
|
configure_from_options(options)
|
11
|
+
@reporter = Reporter.new
|
14
12
|
end
|
15
13
|
|
16
14
|
def validate_all
|
17
|
-
|
15
|
+
models = models_to_validate
|
16
|
+
invalid_count = 0
|
18
17
|
|
19
|
-
models
|
20
|
-
|
21
|
-
|
18
|
+
models.each do |model|
|
19
|
+
model_count = validate_model(model)
|
20
|
+
invalid_count += model_count if model_count
|
21
|
+
end
|
22
|
+
|
23
|
+
if invalid_count.zero?
|
24
|
+
Rails.logger.debug "\nValidation passed! All records are valid."
|
25
|
+
else
|
26
|
+
total_records = models.sum(&:count)
|
27
|
+
Rails.logger.debug get_summary(total_records, invalid_count)
|
28
|
+
end
|
29
|
+
|
30
|
+
@reporter.generate_report
|
31
|
+
end
|
32
|
+
|
33
|
+
def get_summary(records_count, invalid_count)
|
34
|
+
is_plural = invalid_count > 1
|
35
|
+
records_word = is_plural ? "records" : "record"
|
36
|
+
first_part = "\nFound #{invalid_count} invalid #{records_word} out of #{records_count} total #{records_word}."
|
37
|
+
second_part = "\nValidation failed! Some records are invalid." if invalid_count.positive?
|
22
38
|
|
23
|
-
|
24
|
-
|
25
|
-
|
39
|
+
"#{first_part} #{second_part}"
|
40
|
+
end
|
41
|
+
|
42
|
+
def validate_test_model(model_name)
|
43
|
+
model = model_name.constantize
|
44
|
+
scope = model.all
|
45
|
+
scope = scope.limit(DbValidator.configuration.limit) if DbValidator.configuration.limit
|
46
|
+
|
47
|
+
total_count = scope.count
|
48
|
+
progress_bar = create_progress_bar("Testing #{model.name}", total_count)
|
49
|
+
invalid_count = 0
|
50
|
+
|
51
|
+
begin
|
52
|
+
scope.find_each(batch_size: DbValidator.configuration.batch_size) do |record|
|
53
|
+
invalid_count += 1 unless validate_record(record)
|
54
|
+
progress_bar.increment
|
55
|
+
end
|
56
|
+
rescue StandardError => e
|
57
|
+
Rails.logger.debug { "Error validating #{model.name}: #{e.message}" }
|
26
58
|
end
|
27
59
|
|
28
|
-
|
29
|
-
Rails.logger.debug "
|
30
|
-
|
60
|
+
if invalid_count.zero?
|
61
|
+
Rails.logger.debug "\nValidation rule passed! All records would be valid."
|
62
|
+
else
|
63
|
+
Rails.logger.debug do
|
64
|
+
"\nFound #{invalid_count} records that would become invalid out of #{total_count} total records."
|
65
|
+
end
|
31
66
|
end
|
32
67
|
|
33
68
|
@reporter.generate_report
|
@@ -38,13 +73,11 @@ module DbValidator
|
|
38
73
|
def configure_from_options(options)
|
39
74
|
return unless options.is_a?(Hash)
|
40
75
|
|
41
|
-
if options[:only_models]
|
42
|
-
DbValidator.configuration.only_models = Array(options[:only_models])
|
43
|
-
end
|
44
|
-
|
76
|
+
DbValidator.configuration.only_models = Array(options[:only_models]) if options[:only_models]
|
45
77
|
DbValidator.configuration.limit = options[:limit] if options[:limit]
|
46
78
|
DbValidator.configuration.batch_size = options[:batch_size] if options[:batch_size]
|
47
79
|
DbValidator.configuration.report_format = options[:report_format] if options[:report_format]
|
80
|
+
DbValidator.configuration.show_records = options[:show_records] if options[:show_records]
|
48
81
|
end
|
49
82
|
|
50
83
|
def find_all_models
|
@@ -59,37 +92,57 @@ module DbValidator
|
|
59
92
|
|
60
93
|
config = DbValidator.configuration
|
61
94
|
model_name = model.name.downcase
|
62
|
-
|
95
|
+
|
63
96
|
if config.only_models.any?
|
64
|
-
return config.only_models.map(&:downcase).include?(model_name)
|
97
|
+
return config.only_models.map(&:downcase).include?(model_name) ||
|
98
|
+
config.only_models.map(&:downcase).include?(model_name.singularize) ||
|
99
|
+
config.only_models.map(&:downcase).include?(model_name.pluralize)
|
65
100
|
end
|
66
101
|
|
67
|
-
|
102
|
+
config.ignored_models.map(&:downcase).exclude?(model_name)
|
68
103
|
end
|
69
104
|
|
70
105
|
def validate_model(model)
|
71
|
-
|
72
|
-
|
73
|
-
|
106
|
+
scope = build_scope(model)
|
107
|
+
total_count = scope.count
|
108
|
+
return 0 if total_count.zero?
|
74
109
|
|
75
|
-
scope
|
76
|
-
|
110
|
+
process_records(scope, model, total_count)
|
111
|
+
end
|
77
112
|
|
78
|
-
|
79
|
-
|
113
|
+
def build_scope(model)
|
114
|
+
scope = model.all
|
115
|
+
scope = scope.limit(DbValidator.configuration.limit) if DbValidator.configuration.limit
|
116
|
+
scope
|
117
|
+
end
|
80
118
|
|
119
|
+
def process_records(scope, model, total_count)
|
81
120
|
progress_bar = create_progress_bar(model.name, total_count)
|
121
|
+
process_batches(scope, progress_bar, model)
|
122
|
+
end
|
123
|
+
|
124
|
+
def process_batches(scope, progress_bar, model)
|
125
|
+
invalid_count = 0
|
126
|
+
batch_size = DbValidator.configuration.batch_size || 100
|
82
127
|
|
83
128
|
begin
|
84
129
|
scope.find_in_batches(batch_size: batch_size) do |batch|
|
85
|
-
batch
|
86
|
-
validate_record(record)
|
87
|
-
progress_bar.increment
|
88
|
-
end
|
130
|
+
invalid_count += process_batch(batch, progress_bar)
|
89
131
|
end
|
90
132
|
rescue StandardError => e
|
91
|
-
Rails.logger.debug "Error validating #{model.name}: #{e.message}"
|
133
|
+
Rails.logger.debug { "Error validating #{model.name}: #{e.message}" }
|
134
|
+
end
|
135
|
+
|
136
|
+
invalid_count
|
137
|
+
end
|
138
|
+
|
139
|
+
def process_batch(batch, progress_bar)
|
140
|
+
invalid_count = 0
|
141
|
+
batch.each do |record|
|
142
|
+
invalid_count += 1 unless validate_record(record)
|
143
|
+
progress_bar.increment
|
92
144
|
end
|
145
|
+
invalid_count
|
93
146
|
end
|
94
147
|
|
95
148
|
def create_progress_bar(model_name, total)
|
@@ -102,8 +155,15 @@ module DbValidator
|
|
102
155
|
end
|
103
156
|
|
104
157
|
def validate_record(record)
|
105
|
-
return if record.valid?
|
158
|
+
return true if record.valid?
|
159
|
+
|
106
160
|
@reporter.add_invalid_record(record)
|
161
|
+
false
|
162
|
+
end
|
163
|
+
|
164
|
+
def models_to_validate
|
165
|
+
models = find_all_models
|
166
|
+
models.select { |model| should_validate_model?(model) }
|
107
167
|
end
|
108
168
|
end
|
109
169
|
end
|
data/lib/db_validator/version.rb
CHANGED
data/lib/db_validator.rb
CHANGED
@@ -6,6 +6,9 @@ require "db_validator/configuration"
|
|
6
6
|
require "db_validator/validator"
|
7
7
|
require "db_validator/reporter"
|
8
8
|
require "db_validator/cli"
|
9
|
+
require "db_validator/config_updater"
|
10
|
+
require "db_validator/test_task"
|
11
|
+
require "db_validator/validate_task"
|
9
12
|
|
10
13
|
module DbValidator
|
11
14
|
class Error < StandardError; end
|
@@ -13,9 +13,9 @@ DbValidator.configure do |config|
|
|
13
13
|
# "Post" => ["cached_votes"]
|
14
14
|
# }
|
15
15
|
|
16
|
-
# Set the batch size for processing records (default: 1000)
|
17
|
-
# config.batch_size = 100
|
18
|
-
|
19
16
|
# Set the report format (:text or :json)
|
20
17
|
# config.report_format = :text
|
18
|
+
|
19
|
+
# Show detailed record information in reports
|
20
|
+
# config.show_records = true
|
21
21
|
end
|
@@ -1,41 +1,25 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
namespace :db_validator do
|
4
|
-
desc "Validate
|
4
|
+
desc "Validate records in the database"
|
5
5
|
task validate: :environment do
|
6
|
-
|
7
|
-
|
8
|
-
has_any_args = ENV["models"] || ENV["limit"] || ENV["batch_size"] || ENV["format"]
|
9
|
-
|
10
|
-
if has_any_args
|
11
|
-
models = ENV["models"].split(",").map(&:strip).map(&:downcase).map(&:singularize)
|
12
|
-
|
13
|
-
DbValidator.configuration.only_models = models
|
14
|
-
DbValidator.configuration.limit = ENV["limit"].to_i if ENV["limit"].present?
|
15
|
-
DbValidator.configuration.batch_size = ENV["batch_size"].to_i if ENV["batch_size"].present?
|
16
|
-
DbValidator.configuration.report_format = ENV["format"].to_sym if ENV["format"].present?
|
17
|
-
else
|
18
|
-
cli.display_progress("Loading models") do
|
19
|
-
Rails.application.eager_load!
|
20
|
-
end
|
6
|
+
DbValidator::ValidateTask.new.execute
|
7
|
+
end
|
21
8
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
9
|
+
desc "Test validation rules on existing records"
|
10
|
+
task test: :environment do
|
11
|
+
unless ENV["model"] && ENV["rule"]
|
12
|
+
puts "Usage: rake db_validator:test model=user rule='validates :field, presence: true' [show_records=false] [limit=1000] [format=json]"
|
13
|
+
raise "No models found in the application. Please run this command from your Rails application root."
|
14
|
+
end
|
27
15
|
|
28
|
-
|
29
|
-
|
16
|
+
model_name = ENV.fetch("model").classify
|
17
|
+
validation_rule = ENV.fetch("rule", nil)
|
30
18
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
DbValidator.configuration.report_format = options[:format].to_sym if options[:format].present?
|
35
|
-
end
|
19
|
+
DbValidator.configuration.show_records = ENV["show_records"] != "false" if ENV["show_records"].present?
|
20
|
+
DbValidator.configuration.limit = ENV["limit"].to_i if ENV["limit"].present?
|
21
|
+
DbValidator.configuration.report_format = ENV["format"].to_sym if ENV["format"].present?
|
36
22
|
|
37
|
-
|
38
|
-
report = validator.validate_all
|
39
|
-
puts "\n#{report}"
|
23
|
+
DbValidator::TestTask.new(model_name, validation_rule).execute
|
40
24
|
end
|
41
25
|
end
|
data/readme.md
CHANGED
@@ -26,6 +26,7 @@ $ bundle install
|
|
26
26
|
The simplest way to run validation is using the provided rake task:
|
27
27
|
|
28
28
|
#### Validate models in interactive mode
|
29
|
+
|
29
30
|
<img width="798" alt="Screenshot 2024-11-07 at 21 50 57" src="https://github.com/user-attachments/assets/33fbdb8b-b8ec-4284-9313-c1eeaf2eab2d">
|
30
31
|
|
31
32
|
```bash
|
@@ -52,12 +53,37 @@ $ rake db_validator:validate limit=1000
|
|
52
53
|
$ rake db_validator:validate format=json
|
53
54
|
```
|
54
55
|
|
55
|
-
###
|
56
|
+
### Test Mode
|
56
57
|
|
57
|
-
|
58
|
+
You can test new validation rules before applying them to your models:
|
58
59
|
|
59
60
|
```bash
|
60
|
-
$ rake db_validator:
|
61
|
+
$ rake db_validator:test model=User rule='validates :name, presence: true'
|
62
|
+
```
|
63
|
+
|
64
|
+
#### Testing Email Format Validation
|
65
|
+
|
66
|
+
Here's an example of testing email format validation rules:
|
67
|
+
|
68
|
+
```bash
|
69
|
+
# Testing invalid email format (without @)
|
70
|
+
$ rake db_validator:test model=User rule='validates :email, format: { without: /@/, message: "must contain @" }'
|
71
|
+
|
72
|
+
Found 100 records that would become invalid out of 100 total records.
|
73
|
+
|
74
|
+
# Testing valid email format (with @)
|
75
|
+
$ rake db_validator:test model=User rule='validates :email, format: { with: /@/, message: "must contain @" }'
|
76
|
+
|
77
|
+
No invalid records found.
|
78
|
+
```
|
79
|
+
|
80
|
+
#### Error Handling
|
81
|
+
|
82
|
+
Trying to test a validation rule for a non-existent attribute will return an error:
|
83
|
+
|
84
|
+
```
|
85
|
+
❌ Error: Attribute 'i_dont_exist' does not exist for model 'User'
|
86
|
+
Available columns: id, email, created_at, updated_at, name
|
61
87
|
```
|
62
88
|
|
63
89
|
### Ruby Code
|
@@ -103,17 +129,26 @@ ID: 5
|
|
103
129
|
|
104
130
|
### JSON Format
|
105
131
|
|
132
|
+
The JSON report is saved to a file in the `db_validator_reports` directory.
|
133
|
+
|
106
134
|
```json
|
107
|
-
|
108
|
-
{
|
109
|
-
"
|
110
|
-
"
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
135
|
+
{
|
136
|
+
"User": {
|
137
|
+
"error_count": 2,
|
138
|
+
"records": [
|
139
|
+
{
|
140
|
+
"id": 1,
|
141
|
+
"errors": [
|
142
|
+
"email is invalid (actual value: \"invalid-email\")"
|
143
|
+
]
|
144
|
+
},
|
145
|
+
{
|
146
|
+
"id": 2,
|
147
|
+
"errors": [
|
148
|
+
"name can't be blank (actual value: \"\")"
|
149
|
+
]
|
150
|
+
}
|
151
|
+
]
|
117
152
|
}
|
118
|
-
|
153
|
+
}
|
119
154
|
```
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: db_validator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Krzysztof Duda
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-12-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -66,20 +66,6 @@ dependencies:
|
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '1.4'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: colorize
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - "~>"
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: 0.8.1
|
76
|
-
type: :runtime
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - "~>"
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: 0.8.1
|
83
69
|
- !ruby/object:Gem::Dependency
|
84
70
|
name: ruby-progressbar
|
85
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -95,33 +81,33 @@ dependencies:
|
|
95
81
|
- !ruby/object:Gem::Version
|
96
82
|
version: '1.11'
|
97
83
|
- !ruby/object:Gem::Dependency
|
98
|
-
name: tty-
|
84
|
+
name: tty-box
|
99
85
|
requirement: !ruby/object:Gem::Requirement
|
100
86
|
requirements:
|
101
87
|
- - "~>"
|
102
88
|
- !ruby/object:Gem::Version
|
103
|
-
version: 0.
|
89
|
+
version: 0.7.0
|
104
90
|
type: :runtime
|
105
91
|
prerelease: false
|
106
92
|
version_requirements: !ruby/object:Gem::Requirement
|
107
93
|
requirements:
|
108
94
|
- - "~>"
|
109
95
|
- !ruby/object:Gem::Version
|
110
|
-
version: 0.
|
96
|
+
version: 0.7.0
|
111
97
|
- !ruby/object:Gem::Dependency
|
112
|
-
name: tty-
|
98
|
+
name: tty-prompt
|
113
99
|
requirement: !ruby/object:Gem::Requirement
|
114
100
|
requirements:
|
115
101
|
- - "~>"
|
116
102
|
- !ruby/object:Gem::Version
|
117
|
-
version: 0.
|
103
|
+
version: 0.23.1
|
118
104
|
type: :runtime
|
119
105
|
prerelease: false
|
120
106
|
version_requirements: !ruby/object:Gem::Requirement
|
121
107
|
requirements:
|
122
108
|
- - "~>"
|
123
109
|
- !ruby/object:Gem::Version
|
124
|
-
version: 0.
|
110
|
+
version: 0.23.1
|
125
111
|
- !ruby/object:Gem::Dependency
|
126
112
|
name: tty-spinner
|
127
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -150,9 +136,14 @@ files:
|
|
150
136
|
- config/initializers/db_validator.rb
|
151
137
|
- lib/db_validator.rb
|
152
138
|
- lib/db_validator/cli.rb
|
139
|
+
- lib/db_validator/config_updater.rb
|
153
140
|
- lib/db_validator/configuration.rb
|
141
|
+
- lib/db_validator/formatters/json_formatter.rb
|
142
|
+
- lib/db_validator/formatters/message_formatter.rb
|
154
143
|
- lib/db_validator/railtie.rb
|
155
144
|
- lib/db_validator/reporter.rb
|
145
|
+
- lib/db_validator/test_task.rb
|
146
|
+
- lib/db_validator/validate_task.rb
|
156
147
|
- lib/db_validator/validator.rb
|
157
148
|
- lib/db_validator/version.rb
|
158
149
|
- lib/generators/db_validator/install_generator.rb
|
@@ -163,6 +154,9 @@ homepage: https://github.com/krzysztoff1/db-validator
|
|
163
154
|
licenses:
|
164
155
|
- MIT
|
165
156
|
metadata:
|
157
|
+
source_code_uri: https://github.com/krzysztoff1/db-validator/
|
158
|
+
documentation_uri: https://github.com/krzysztoff1/db-validator/blob/main/README.md
|
159
|
+
changelog_uri: https://github.com/krzysztoff1/db-validator/blob/main/changelog.md
|
166
160
|
rubygems_mfa_required: 'true'
|
167
161
|
post_install_message:
|
168
162
|
rdoc_options: []
|
@@ -179,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
179
173
|
- !ruby/object:Gem::Version
|
180
174
|
version: '0'
|
181
175
|
requirements: []
|
182
|
-
rubygems_version: 3.5.
|
176
|
+
rubygems_version: 3.5.9
|
183
177
|
signing_key:
|
184
178
|
specification_version: 4
|
185
179
|
summary: DbValidator helps identify invalid records in your Rails application that
|