db_validator 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d5b1e93e6a494c74779345e004eb7f6050fa5fd4e15cb27846d48fa8390a78ef
+  data.tar.gz: 9ab575840ad1cb722db62b40162b527e4793305ed4255b38eb1166ba5e4786fe
+SHA512:
+  metadata.gz: 20cc83d66461dff7faed7ec1eee28b4c5459152d723403c15b23718554e5ae208e965acde7b3ee4bb2ba030e19bdd86c1e4e63bd0072e7e2bfbfdd25048647e9
+  data.tar.gz: b8e9a4175498cf0f96eec2ed624a9c1be35c6092206bfd7117ecad929f59760236d361cbf524cce37fb603feaafa4ed981f5cbf0308cb0b79a2b2adb2c39a5d5

data/config/initializers/db_validator.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+DbValidator.configure do |config|
+  config.only_models = %w[Documents Users]
+  config.batch_size = 500
+  config.limit = 500
+  config.model_limits = {
+    "documents" => 500,
+    "users" => 1000
+  }
+  config.report_format = :json
+end

data/lib/db_validator/configuration.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module DbValidator
+  class Configuration
+    attr_accessor :only_models, :ignored_models, :ignored_attributes, :batch_size, :report_format, :auto_fix, :limit
+
+    def initialize
+      @only_models = []
+      @ignored_models = []
+      @ignored_attributes = {}
+      @batch_size = 1000
+      @report_format = :text
+      @auto_fix = false
+      @limit = nil
+    end
+
+    def only_models=(models)
+      @only_models = models.map(&:downcase)
+    end
+
+    def ignored_models
+      @ignored_models ||= []
+    end
+
+    def ignored_models=(models)
+      @ignored_models = models.map(&:downcase)
+    end
+  end
+
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+  end
+end

data/lib/db_validator/fixer.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module DbValidator
+  class Fixer
+    def initialize
+      @fixed_records = 0
+      @failed_fixes = 0
+    end
+
+    def fix_record(record)
+      return false unless record.invalid?
+
+      success = attempt_fix(record)
+      if success
+        @fixed_records += 1
+      else
+        @failed_fixes += 1
+      end
+      success
+    end
+
+    def statistics
+      {
+        fixed_records: @fixed_records,
+        failed_fixes: @failed_fixes
+      }
+    end
+
+    private
+
+    def attempt_fix(record)
+      record.save
+    end
+  end
+end

data/lib/db_validator/railtie.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DbValidator
+  class Railtie < Rails::Railtie
+    railtie_name :db_validator
+
+    rake_tasks do
+      load "tasks/db_validator_tasks.rake"
+    end
+
+    generators do
+      require "generators/db_validator/install_generator"
+    end
+  end
+end

data/lib/db_validator/reporter.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module DbValidator
+  class Reporter
+    def initialize
+      @invalid_records = []
+    end
+
+    def add_invalid_record(record)
+      enhanced_errors = record.errors.map do |error|
+        field_value = record.send(error.attribute)
+        message = error.message
+        
+        if error.options[:in].present?
+          "#{error.attribute} #{message} (allowed values: #{error.options[:in].join(', ')}, actual value: #{field_value.inspect})"
+        else
+          "#{error.attribute} #{message} (actual value: #{field_value.inspect})"
+        end
+      end
+
+      @invalid_records << {
+        model: record.class.name,
+        id: record.id,
+        errors: enhanced_errors
+      }
+    end
+
+    def generate_report
+      case DbValidator.configuration.report_format
+      when :json
+        generate_json_report
+      else
+        generate_text_report
+      end
+    end
+
+    private
+
+    def generate_text_report
+      report = StringIO.new
+      report.puts "DbValidator Report"
+      report.puts "=================="
+      report.puts
+
+      if @invalid_records.empty?
+        report.puts "No invalid records found."
+      else
+        report.puts "Found invalid records:"
+        report.puts
+
+        @invalid_records.group_by { |r| r[:model] }.each do |model, records|
+          report.puts "#{model}: #{records.count} invalid records"
+          records.each do |record|
+            report.puts "  ID: #{record[:id]}"
+            record[:errors].each do |error|
+              report.puts "    - #{error}"
+            end
+          end
+          report.puts
+        end
+      end
+
+      report.string
+    end
+
+    def generate_json_report
+      @invalid_records.to_json
+    end
+  end
+end

data/lib/db_validator/validator.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "ruby-progressbar"
+require "colorize"
+
+module DbValidator
+  class Validator
+    def initialize(options = {})
+      @options = options
+      @reporter = Reporter.new
+      @fixer = Fixer.new if DbValidator.configuration.auto_fix
+    end
+
+    def validate_all
+      Rails.application.eager_load! if defined?(Rails)
+
+      models = find_all_models
+
+      models_to_validate = models.select { |model| should_validate_model?(model) }
+      total_models = models_to_validate.size
+
+      models_to_validate.each_with_index do |model, index|
+        Rails.logger.debug "Validating model #{index + 1}/#{total_models}: #{model.name}".colorize(:cyan)
+        validate_model(model)
+      end
+
+      @reporter.generate_report
+    end
+
+    private
+
+    def find_all_models
+      # Include all classes inheriting from ActiveRecord::Base
+      ObjectSpace.each_object(Class).select do |klass|
+        klass < ActiveRecord::Base
+      end
+    end
+
+    def should_validate_model?(model)
+      return false if model.abstract_class?
+      return false unless model.table_exists?
+
+      config = DbValidator.configuration
+      model_name = model.name.downcase
+      return config.only_models.include?(model_name) if config.only_models.any?
+
+      config.ignored_models.exclude?(model_name)
+    end
+
+    def validate_model(model)
+      limit = DbValidator.configuration.limit
+      total_records = limit || model.count
+
+      if total_records.zero?
+        Rails.logger.debug { "No records to validate for model #{model.name}." }
+        return
+      end
+
+      processed_records = 0
+
+      query = model.all
+      query = query.limit(limit) if limit
+
+      query.find_in_batches(batch_size: DbValidator.configuration.batch_size) do |batch|
+        batch.each do |record|
+          validate_record(record)
+          processed_records += 1
+          if (processed_records % 100).zero? || processed_records == total_records
+            Rails.logger.debug "Validated #{processed_records}/#{total_records} records for model #{model.name}".colorize(:green)
+          end
+        end
+      rescue StandardError => e
+        Rails.logger.debug "Error validating #{model.name}: #{e.message}".colorize(:red)
+      end
+    rescue ActiveRecord::StatementInvalid => e
+      Rails.logger.debug { "Skipping validation for #{model.name}: #{e.message}" }
+    end
+
+    def validate_record(record)
+      return if record.valid?
+
+      @reporter.add_invalid_record(record)
+      @fixer&.attempt_fix(record)
+    end
+  end
+end

data/lib/db_validator/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DbValidator
+  VERSION = "0.1.0"
+end

data/lib/db_validator.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "rails"
+require "db_validator/version"
+require "db_validator/configuration"
+require "db_validator/validator"
+require "db_validator/reporter"
+require "db_validator/fixer"
+
+module DbValidator
+  class Error < StandardError; end
+
+  class << self
+    def validate(options = {})
+      validator = Validator.new(options)
+      validator.validate_all
+    end
+  end
+end
+
+require "db_validator/railtie" if defined?(Rails)

data/lib/generators/db_validator/install_generator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/base"
+
+module DbValidator
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      def create_initializer
+        template "initializer.rb", "config/initializers/db_validator.rb"
+      end
+    end
+  end
+end

data/lib/generators/db_validator/templates/initializer.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+DbValidator.configure do |config|
+  # Specify specific models to validate
+  config.only_models = %w[User Post]
+
+  # Ignore specific models from validation
+  # config.ignored_models = ["AdminUser"]
+
+  # Ignore specific attributes for specific models
+  # config.ignored_attributes = {
+  #   "User" => ["encrypted_password", "reset_password_token"],
+  #   "Post" => ["cached_votes"]
+  # }
+
+  # Set the batch size for processing records (default: 1000)
+  # config.batch_size = 1000
+
+  # Set the report format (:text or :json)
+  # config.report_format = :text
+
+  # Enable automatic fixing of simple validation errors
+  # config.auto_fix = false
+end

data/lib/tasks/db_validator_tasks.rake

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+namespace :db_validator do
+  desc "Validate all records in the database"
+  task validate: :environment do
+    if ENV["models"]
+      models = ENV["models"].split(",").map(&:strip).map(&:downcase)
+      DbValidator.configuration.only_models = models
+    end
+
+    DbValidator.configuration.limit = ENV["limit"].to_i if ENV["limit"]
+
+    DbValidator.configuration.report_format = ENV["format"].to_sym if ENV["format"]
+
+    validator = DbValidator::Validator.new
+    report = validator.validate_all
+    puts report
+  end
+end

data/readme.md

@@ -0,0 +1,111 @@
+# DbValidator
+
+DbValidator helps identify invalid records in your Rails application that don't meet model validation requirements.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'db_validator'
+```
+
+Then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself:
+
+```bash
+$ gem install db_validator
+```
+
+## Usage
+
+### Rake Task
+
+The simplest way to run validation is using the provided rake task:
+
+#### Validate all models
+
+```bash
+$ rake db_validator:validate
+```
+
+#### Validate specific models
+
+```bash
+$ rake db_validator:validate models=user,post
+```
+
+#### Limit the number of records to validate
+
+```bash
+$ rake db_validator:validate limit=1000
+```
+
+#### Generate JSON report
+
+```bash
+$ rake db_validator:validate format=json
+```
+
+### Ruby Code
+
+You can also run validation from your Ruby code:
+
+#### Validate all models
+
+```ruby
+report = DbValidator.validate
+```
+
+#### Validate with options
+
+```ruby
+report = DbValidator.validate(
+  only_models: ['User', 'Post'],
+  limit: 1000,
+  report_format: :json
+)
+```
+
+## Report Format
+
+### Text Format (Default)
+
+```
+DbValidator Report
+==================
+Found invalid records:
+
+User: 2 invalid records
+ID: 1
+  email is invalid (actual value: "invalid-email")
+ID: 2
+  name can't be blank (actual value: "")
+
+Post: 1 invalid record
+ID: 5
+  title can't be blank (actual value: "")
+  category is not included in the list (allowed values: news, blog, actual value: "invalid")
+```
+
+### JSON Format
+
+```json
+[
+  {
+    "model": "User",
+    "id": 1,
+    "errors": ["email is invalid (actual value: \"invalid-email\")"]
+  },
+  {
+    "model": "User",
+    "id": 2,
+    "errors": ["name can't be blank (actual value: \"\")"]
+  }
+]
+```

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: db_validator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Krzysztof Duda
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-11-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner-active_record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+description: A comprehensive solution for validating existing database records in
+  Rails applications
+email:
+- duda_krzysztof@outlook.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- config/initializers/db_validator.rb
+- lib/db_validator.rb
+- lib/db_validator/configuration.rb
+- lib/db_validator/fixer.rb
+- lib/db_validator/railtie.rb
+- lib/db_validator/reporter.rb
+- lib/db_validator/validator.rb
+- lib/db_validator/version.rb
+- lib/generators/db_validator/install_generator.rb
+- lib/generators/db_validator/templates/initializer.rb
+- lib/tasks/db_validator_tasks.rake
+- readme.md
+homepage: https://github.com/yourusername/db_validator
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.16
+signing_key:
+specification_version: 4
+summary: Database-wide validation for Rails applications
+test_files: []