validb 1.2.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5029c1e329cd8ca5e201dbf789a1fae902d8e0cd
-  data.tar.gz: d6da89be0e3a798c594a90c2c8e4d4dbb635fa00
+  metadata.gz: 36c876f755ed39d03ffa2089430e120485a33ef3
+  data.tar.gz: aee472a56c2a3a13d83da6cd6affecfddb4754c7
 SHA512:
-  metadata.gz: 9b00aa393b7fcec846bfa37f82d5610070cb6d6a50116ab7e949f411d5c4fc9a9ef924b56ea7cd01bf5397dc8c461caa7dd82d4a99a42485f16bc01c8a717d28
-  data.tar.gz: b5f36565dc2ebce1dfac46f7289200b9f31ff83701cd3142ee68942e85e9cb303224b4e4e20d2915cd8e471478772bdde3231cf4e2a00cb9f2ff041c81250c23
+  metadata.gz: 091a64b809e1652481cd32b847f34643246ae1772b9657e34bb24e2b7e17b2fcc9e908f7b1c8438c6809633deda8f66f08671e8cb12c60274831e04d64c4341a
+  data.tar.gz: e0fecdeef959242ec3f02bdb3ebdabca9c86aabaf632ea558505055a6a61a7376ec5d89e0833de55ffb0820a13f27e3d06adaee21cdb94e08b20192ca5b889f6

data/.gitignore

@@ -43,3 +43,4 @@ tramp
 .rbx
 b/
 vendor/*
+validb.log

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.0.0, released 2013-04-07
+* Distribute validation jobs using sidekiq
+* Remove console logger
+
 ## 1.2.1, released 2013-04-05
 * Revert printing 'x' on validation failure
 

data/README.md

@@ -10,22 +10,15 @@
 Check your database for invalid models.
 
 ## Installation
-    gem install validb
-
-### Rails 3 In Gemfile
+Add this line to your Gemfile
 
     gem 'validb'
 
-## Usage
-
-### All models, default logger
-    rake validb:validate 
+And then:
 
-### Specific models
-    rake validb:validate models="Model,Model"
+    bundle
 
-### Specific logger
-    rake validb:validate logger="Validb::Logger::Console"
+## Usage
 
 ### Generate config file in config/validb.json
     rake validb:generate_config
@@ -43,5 +36,19 @@ Check your database for invalid models.
       "batch_size": 1000
     }
 
+### Launch sidekiq
+    sidekiq -L log/sidekiq.log
+
+### Validate all model records
+    rake validb:validate 
+
+### Validate specific model records
+    rake validb:validate models="Blog,Comment"
+
+## Uses
+[sidekiq](http://mperham.github.io/sidekiq)
+
+[sidekiq_status](https://github.com/cryo28/sidekiq_status)
+
 ## License
 Released under the MIT License

data/lib/tasks/validb.rake

@@ -2,23 +2,9 @@ namespace :validb do
   desc "Check DB for invalid records"
   task :validate => :environment do
     models = ENV["models"] || ""
-    logger_class = ENV["logger"] || "Validb::Logger::FileSystem"
     filename = ENV["config"] || default_configuration_file
 
-    # force all models to load so we can find them
-    Rails.application.eager_load!
-    ActiveRecord::Base.logger.level = 1
-
-    params = Validb::Configuration.new(filename).params
-    finder = Validb::Finder.new(params, models)
-
-    logger = logger_class.constantize.new
-    logger.prepare
-
-    started_at = Time.now
-    checker = Validb::Checker.new(params, logger)
-    checker.check(finder.models)
-    puts "\nFinished in #{Time.now-started_at} seconds"
+    Validb::Tasker.new(models, filename).run
   end
 
   desc "Generate config/validb.json"

data/lib/validb.rb

@@ -13,9 +13,11 @@ module Validb
   autoload :Batcher, 'validb/batcher'
   autoload :RecordValidator, 'validb/record_validator'
   autoload :Parameters, 'validb/parameters'
+  autoload :FileSystemLoggerWorker, 'validb/file_system_logger_worker'
+  autoload :QueueMonitor, 'validb/queue_monitor'
+  autoload :Counter, 'validb/counter'
+  autoload :Tasker, 'validb/tasker'
   module Logger
-    autoload :Base, 'validb/logger/base'
     autoload :FileSystem, 'validb/logger/file_system'
-    autoload :Console, 'validb/logger/console'
   end
 end

data/lib/validb/batcher.rb

@@ -1,14 +1,11 @@
 module Validb
   class Batcher
+    include SidekiqStatus::Worker
 
-    def initialize(logger)
-      @record_validator = RecordValidator.new(logger)
-    end
-
-    def validate(record_batch)
-      $stdout.print "."
-      record_batch.each do |record|
-        @record_validator.validate(record)
+    def perform(model_name, model_ids)
+      model_name.constantize.find(model_ids).each do |record|
+        record_validator = Validb::RecordValidator.new
+        record_validator.validate(record)
       end
     end
   end

data/lib/validb/checker.rb

@@ -1,13 +1,10 @@
 module Validb
   class Checker
+    include SidekiqStatus::Worker
 
-    def initialize(params, logger)
-      @model_validator = Validb::ModelValidator.new(params, logger)
-    end
-
-    def check(models)
-      models.each do |model|
-        @model_validator.validate(model)
+    def perform(model_names, batch_size)
+      model_names.each do |model_name|
+        Validb::ModelValidator.perform_async(model_name, batch_size)
       end
     end
   end

data/lib/validb/counter.rb

@@ -0,0 +1,14 @@
+module Validb
+  class Counter
+    def initialize(model_names)
+      @models = model_names.map(&:constantize)
+    end
+
+    def count
+      $stdout.puts("\nChecking models:")
+      @models.each do |model|
+        $stdout.puts("#{model}(#{model.table_name}) (#{model.count} records)")
+      end
+    end
+  end
+end

data/lib/validb/file_system_logger_worker.rb

@@ -0,0 +1,10 @@
+module Validb
+  class FileSystemLoggerWorker
+    include SidekiqStatus::Worker
+
+    def perform(message)
+      logger = Validb::Logger::FileSystem.new
+      logger.out(message)
+    end
+  end
+end

data/lib/validb/finder.rb

@@ -7,7 +7,7 @@ module Validb
     end
 
     def models
-      @model_filterer.filter(selected_models)
+      @model_filterer.filter(selected_models).map(&:name)
     end
 
     private

data/lib/validb/logger/file_system.rb

@@ -1,14 +1,15 @@
 module Validb
   module Logger
-    class FileSystem < Base
+    class FileSystem
       def prepare
         File.delete(filename) if File.exists?(filename)
+        File.open(filename, "w") {}
         $stdout.puts("Writing output to #{filename}")
       end
 
-      def out(record_hash)
+      def out(message)
         File.open(filename, "ab+") do |file|
-          file.puts("#{record_hash[:model]}:#{record_hash[:id]} - #{record_hash[:error_messages]}")
+          file.puts(message)
         end
       end
 

data/lib/validb/model_validator.rb

@@ -1,15 +1,11 @@
 module Validb
   class ModelValidator
+    include SidekiqStatus::Worker
 
-    def initialize(params, logger)
-      @params = params
-      @batcher = Validb::Batcher.new(logger)
-    end
-
-    def validate(model)
-      $stdout.print "\nChecking #{model}(#{model.table_name}) (#{model.count} records)"
-      model.find_in_batches(batch_size: @params.batch_size) do |record_batch|
-        @batcher.validate(record_batch)
+    def perform(model_name, batch_size)
+      model_name.constantize.select(:id).find_in_batches(batch_size: batch_size) do |record_batch|
+        model_ids = record_batch.map(&:id)
+        Validb::Batcher.perform_async(model_name, model_ids)
       end
     end
   end

data/lib/validb/queue_monitor.rb

@@ -0,0 +1,30 @@
+module Validb
+  class QueueMonitor
+
+    def monitor
+      saved_time = Time.now
+      $stdout.print "\nValidating database records"
+
+      wait_for_completion
+
+      $stdout.puts "\nComplete in #{Time.now - saved_time} seconds"
+    end
+
+    private
+
+    def wait_for_completion
+      while in_process_jobs?
+        sleep 1
+        $stdout.print "."
+      end
+    end
+
+    def in_process_jobs?
+      statuses.detect { |job| job.status != "complete" }
+    end
+
+    def statuses
+      SidekiqStatus::Container.statuses
+    end
+  end
+end

data/lib/validb/railtie.rb

@@ -1,3 +1,5 @@
+require 'sidekiq'
+require 'sidekiq_status'
 require 'validb'
 require 'rails'
 

data/lib/validb/record_validator.rb

@@ -1,19 +1,10 @@
 module Validb
   class RecordValidator
 
-    def initialize(logger)
-      @logger = logger
-    end
-
     def validate(record)
       if !record.valid?
-        @logger.out(
-          {
-            model: record.class.name,
-            id: record.id,
-            error_messages: record.errors.full_messages.join(',')
-          }
-        )
+        message = "FAIL: #{record.class.name}:#{record.id} - #{record.errors.full_messages.join(',')}"
+        Validb::FileSystemLoggerWorker.perform_async(message)
       end
     end
   end

data/lib/validb/tasker.rb

@@ -0,0 +1,46 @@
+module Validb
+  class Tasker
+
+    def initialize(model_names, filename)
+      @model_names = model_names
+      @filename = filename
+    end
+
+    def run
+      load_models_and_prepare_logger
+      output_model_table_record_count
+      validate_database_records
+    end
+
+    private
+
+    def params
+      @params ||= Validb::Configuration.new(@filename).params
+    end
+
+    def finder
+      @finder ||= Validb::Finder.new(params, @model_names)
+    end
+
+    def load_models_and_prepare_logger
+      # force all models to load so we can find them
+      Rails.application.eager_load!
+      ActiveRecord::Base.logger.level = 1
+      Validb::Logger::FileSystem.new.prepare
+    end
+
+    def output_model_table_record_count
+      Validb::Counter.new(finder.models).count
+    end
+
+    def validate_database_records
+      Validb::Checker.perform_async(finder.models, params.batch_size)
+      monitor_job_progress
+    end
+
+    def monitor_job_progress
+      queue_monitor = Validb::QueueMonitor.new
+      queue_monitor.monitor
+    end
+  end
+end

data/lib/validb/version.rb

@@ -1,3 +1,3 @@
 module Validb
-  VERSION = "1.2.1"
+  VERSION = "2.0.0"
 end

data/spec/spec_helper.rb

@@ -1,10 +1,17 @@
 require File.expand_path(File.dirname(__FILE__) + '/../lib/validb')
 
 require "pry"
+require "sidekiq"
+require 'sidekiq/testing'
+require 'sidekiq_status'
 
 RSpec.configure do |config|
   config.treat_symbols_as_metadata_keys_with_true_values = true
   config.run_all_when_everything_filtered = true
+
+  config.after(:all) do
+    SidekiqStatus::Container.delete
+  end
 end
 
 RSpec::Matchers::OperatorMatcher.register(ActiveRecord::Relation, '=~', RSpec::Matchers::BuiltIn::MatchArray)

data/spec/validb/batcher_spec.rb

@@ -1,22 +1,16 @@
 require 'spec_helper'
 
 describe Validb::Batcher do
-  describe "#initialize" do
-    it "creates a batcher" do
-      logger = double('logger')
-      Validb::RecordValidator.should_receive(:new).with(logger)
-      Validb::Batcher.new(logger)
-    end
-  end
-
-  describe "#validate" do
-    it "validates the batch of records" do
-      record_batch = [Blog.new(title: "Title")]
-      logger = double('logger')
-      batcher = Validb::Batcher.new(logger)
+  describe "#perform" do
+    it "sends an invalid record logging event" do
+      record = Blog.new
+      record.save(validate: false)
+      batcher = Validb::Batcher.new
+      jid = Validb::Batcher.perform_async("Blog", [record.id])
 
-      $stdout.should_receive(:print).with(".")
-      batcher.validate(record_batch)
+      expect {
+        batcher.perform(jid)
+      }.to change(Validb::FileSystemLoggerWorker.jobs, :size).by(1)
     end
   end
 end

data/spec/validb/checker_spec.rb

@@ -1,27 +1,14 @@
 require 'spec_helper'
 
 describe Validb::Checker do
-  describe "#initialize" do
-    it "creates a checker" do
-      logger = double('logger')
-      params = double('params')
-      Validb::ModelValidator.should_receive(:new).with(params, logger)
-      Validb::Checker.new(params, logger)
-    end
-  end
-
-  describe "#check" do
-    it "validates the passed in models records" do
-      logger = double('logger')
-      params = double('params')
-      model_validator = double('model_validator')
-      Validb::ModelValidator.should_receive(:new).with(params, logger).and_return(model_validator)
-      checker = Validb::Checker.new(params, logger)
-
-      model_validator.should_receive(:validate).with(Blog)
-      model_validator.should_receive(:validate).with(Post)
-
-      checker.check([Blog, Post])
+  describe "#perform" do
+    it "generates validb model validator jobs" do
+      checker = Validb::Checker.new
+      jid = Validb::Batcher.perform_async(["Blog", "Post"], 100)
+
+      expect {
+        checker.perform(jid)
+      }.to change(Validb::ModelValidator.jobs, :size).by(2)
     end
   end
 end

data/spec/validb/counter_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe Validb::Counter do
+  describe "#count" do
+    it "prints the model names, tables and record counts" do
+      counter = Validb::Counter.new(["Blog"])
+      $stdout.should_receive(:puts).with("\nChecking models:")
+      $stdout.should_receive(:puts).with("Blog(blogs) (1 records)")
+
+      counter.count
+    end
+  end
+end

data/spec/validb/file_system_logger_worker_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Validb::FileSystemLoggerWorker do
+  describe "#perform" do
+    it "sends the message to the File System logger" do
+      Validb::Logger::FileSystem.any_instance.should_receive(:out).with("message")
+
+      logger = Validb::FileSystemLoggerWorker.new
+      jid = Validb::Batcher.perform_async("message")
+
+      logger.perform(jid)
+    end
+  end
+end

data/spec/validb/finder_spec.rb

@@ -17,7 +17,7 @@ describe Validb::Finder do
       it "returns all the models with tables" do
         params = Validb::Configuration.new("").params
         finder = Validb::Finder.new(params, "")
-        finder.models.should == [Author, Blog, Comment, Post]
+        finder.models.should == ["Author", "Blog", "Comment", "Post"]
       end
     end
 
@@ -25,7 +25,7 @@ describe Validb::Finder do
       it "returns all the selected models with tables" do
         params = Validb::Configuration.new("").params
         finder = Validb::Finder.new(params, "Post, Blog")
-        finder.models.should == [Blog, Post]
+        finder.models.should == ["Blog", "Post"]
       end
     end
 
@@ -33,7 +33,7 @@ describe Validb::Finder do
       it "returns all the valid models with tables" do
         params = Validb::Configuration.new("").params
         finder = Validb::Finder.new(params, "Post, Blog, Fake")
-        finder.models.should == [Blog, Post]
+        finder.models.should == ["Blog", "Post"]
       end
     end
   end

data/spec/validb/logger/file_system_spec.rb

@@ -30,12 +30,12 @@ describe Validb::Logger::FileSystem do
   describe "#out" do
     it "writes the hash output to a file and prints an X" do
       file = double('file')
+      message = "Blog:1 - Error message"
       File.should_receive(:open).with("validb.log", "ab+").and_yield(file)
-      file.should_receive(:puts).with("Blog:1 - Error message")
+      file.should_receive(:puts).with(message)
 
-      hash = { model: Blog, id: 1, error_messages: "Error message" }
       file_logger = Validb::Logger::FileSystem.new
-      file_logger.out(hash)
+      file_logger.out(message)
     end
   end
 end

data/spec/validb/model_validator_spec.rb

@@ -1,25 +1,30 @@
 require 'spec_helper'
 
 describe Validb::ModelValidator do
-  describe "#initialize" do
-    it "creates a model validator" do
-      logger = double('logger')
-      params = double('params')
-      Validb::Batcher.should_receive(:new).with(logger)
-      Validb::ModelValidator.new(params, logger)
+  describe "#perform" do
+    context "with available records" do
+      it "creates a batcher job" do
+        record = Blog.new(title: "title")
+        record.save
+        model_validator = Validb::ModelValidator.new
+        jid = Validb::ModelValidator.perform_async("Blog", 100)
+
+        expect {
+          model_validator.perform(jid)
+        }.to change(Validb::Batcher.jobs, :size).by(1)
+      end
     end
-  end
 
-  describe "#validate" do
-    it "validates the records of the model" do
-      model = Blog
-      logger = double('logger')
-      params = double('params', batch_size: 200)
-      model_validator = Validb::ModelValidator.new(params, logger)
+    context "with no available records" do
+      it "does not create a batcher job" do
+        Blog.delete_all
+        model_validator = Validb::ModelValidator.new
+        jid = Validb::ModelValidator.perform_async("Blog", 100)
 
-      $stdout.should_receive(:print).with("\nChecking Blog(blogs) (0 records)")
-      model.should_receive(:find_in_batches).with(batch_size: 200)
-      model_validator.validate(model)
+        expect {
+          model_validator.perform(jid)
+        }.not_to change(Validb::Batcher.jobs, :size)
+      end
     end
   end
 end

data/spec/validb/queue_monitor_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Validb::QueueMonitor do
+
+  describe "#monitor" do
+    it "prints progress information" do
+      $stdout.should_receive(:print).with("\nValidating database records")
+      $stdout.should_receive(:puts)
+
+      queue_monitor = Validb::QueueMonitor.new
+      queue_monitor.monitor
+    end
+  end
+end

data/spec/validb/record_validator_spec.rb

@@ -4,29 +4,24 @@ describe Validb::RecordValidator do
   describe "#validate" do
     context "with a valid record" do
       it "validates the passed in record" do
-        logger = double(:logger)
-        record_validator = Validb::RecordValidator.new(logger)
         record = Blog.new(title: "title")
+        record_validator = Validb::RecordValidator.new
 
-        logger.should_not_receive(:out)
-        record_validator.validate(record)
+        expect {
+          record_validator.validate(record)
+        }.not_to change(Validb::FileSystemLoggerWorker.jobs, :size)
       end
     end
 
     context "with an invalid record" do
       it "outputs the record information" do
-        logger = double(:logger)
-        record_validator = Validb::RecordValidator.new(logger)
+        record_validator = Validb::RecordValidator.new
         record = Blog.new
         record.save(validate: false)
 
-        hash = {
-          model: "Blog",
-          id: record.id,
-          error_messages: "Title can't be blank"
-        }
-        logger.should_receive(:out).with(hash)
-        record_validator.validate(record)
+        expect {
+          record_validator.validate(record)
+        }.to change(Validb::FileSystemLoggerWorker.jobs, :size).by(1)
       end
     end
   end

data/spec/validb/tasker_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe Validb::Tasker do
+  describe "#run" do
+    it "runs the database validation" do
+      filename = File.expand_path(File.dirname(__FILE__) + '/../config/validb.json')
+      tasker = Validb::Tasker.new("", filename)
+
+      tasker.should_receive(:load_models_and_prepare_logger)
+      Validb::Counter.any_instance.should_receive(:count)
+      Validb::QueueMonitor.any_instance.should_receive(:monitor)
+
+      expect {
+        tasker.run
+      }.to change(Validb::Checker.jobs, :size).by(1)
+    end
+  end
+end

data/spec/validb/version_spec.rb

@@ -2,6 +2,6 @@ require 'spec_helper'
 
 describe Validb do
   it "returns the version" do
-    Validb::VERSION.should == "1.2.1"
+    Validb::VERSION.should == "2.0.0"
   end
 end

data/validb.gemspec

@@ -20,4 +20,6 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   s.add_dependency 'rails', '>=3'
+  s.add_dependency 'sidekiq'
+  s.add_dependency 'sidekiq_status'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: validb
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Joey Geiger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-05 00:00:00.000000000 Z
+date: 2013-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -24,6 +24,34 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq_status
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Check the contents of your database by validating the models.
 email: jgeiger@gmail.com
 executables: []
@@ -41,27 +69,32 @@ files:
 - lib/validb/batcher.rb
 - lib/validb/checker.rb
 - lib/validb/configuration.rb
+- lib/validb/counter.rb
+- lib/validb/file_system_logger_worker.rb
 - lib/validb/finder.rb
-- lib/validb/logger/base.rb
-- lib/validb/logger/console.rb
 - lib/validb/logger/file_system.rb
 - lib/validb/model_filterer.rb
 - lib/validb/model_validator.rb
 - lib/validb/parameters.rb
+- lib/validb/queue_monitor.rb
 - lib/validb/railtie.rb
 - lib/validb/record_validator.rb
+- lib/validb/tasker.rb
 - lib/validb/version.rb
 - spec/config/validb.json
 - spec/spec_helper.rb
 - spec/validb/batcher_spec.rb
 - spec/validb/checker_spec.rb
 - spec/validb/configuration_spec.rb
+- spec/validb/counter_spec.rb
+- spec/validb/file_system_logger_worker_spec.rb
 - spec/validb/finder_spec.rb
-- spec/validb/logger/console_spec.rb
 - spec/validb/logger/file_system_spec.rb
 - spec/validb/model_filterer_spec.rb
 - spec/validb/model_validator_spec.rb
+- spec/validb/queue_monitor_spec.rb
 - spec/validb/record_validator_spec.rb
+- spec/validb/tasker_spec.rb
 - spec/validb/version_spec.rb
 - validb.gemspec
 homepage: http://github.com/jgeiger/validb

data/lib/validb/logger/base.rb

@@ -1,11 +0,0 @@
-module Validb
-  module Logger
-    class Base
-      def prepare
-      end
-
-      def out(record_hash)
-      end
-    end
-  end
-end

data/lib/validb/logger/console.rb

@@ -1,9 +0,0 @@
-module Validb
-  module Logger
-    class Console < Base
-      def out(record_hash)
-        $stdout.puts "\n#{record_hash[:model]}:#{record_hash[:id]} - #{record_hash[:error_messages]}"
-      end
-    end
-  end
-end

data/spec/validb/logger/console_spec.rb

@@ -1,13 +0,0 @@
-require 'spec_helper'
-
-describe Validb::Logger::Console do
-  describe "#out" do
-    it "prints the hash output" do
-      hash = { model: Blog, id: 1, error_messages: "Error message" }
-
-      console_logger = Validb::Logger::Console.new
-      $stdout.should_receive(:puts).with("\nBlog:1 - Error message")
-      console_logger.out(hash)
-    end
-  end
-end