broken_record 0.0.6 → 0.0.7

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NGU5YWE0ODA2MWVkY2M4MTRkYjc3NTZkN2U3ZDdlYjc2OGRlNmM0YQ==
-  data.tar.gz: !binary |-
-    OWM4MTEwYmZjMmI3YmJhMjIzNzk3ZDMwYzg3YzE3ZDM2MGE3NTI0Zg==
+SHA1:
+  metadata.gz: 0b5338e93e1f7a715a2f8bb0416a4a945d9ad297
+  data.tar.gz: 71a2a195aa089ee10b57675f6b9bd08a58fcffaf
 SHA512:
-  metadata.gz: !binary |-
-    Njg3OWNjMmE0NDIxMDc0YWUzNzFlMjBjNDIwNzhkMDdiODBhZmYzOGFlMmNh
-    OGY3Yjk1Mzc4YjQyOWFlYTc0MDMzZjUwYzNlYzcyODA4ZGVhMTkyYTQzNGE1
-    ODUxZTczNDg5ZTA1YzczM2NlY2NiNDhlOGM1Y2QwNjgzODgwNWU=
-  data.tar.gz: !binary |-
-    ZGYyYmI0ZDlkNjZmMzMxYmZkNGQ0YWUyZTBkMzNiMDAzYjliOWIwNTMzNDY0
-    NTY4YmUwNmE2MmQ4ZjY1ZGIzNDI5MjgxYzBlYzdhZjllOGZlYzk5MWUzYWI0
-    YmUzNjg4NWQ1Nzk4MGY3OTliMzJmYTA1NDY1MjIyMzY2ZjljOTc=
+  metadata.gz: fc9009d880eb5ad5cbae3f63d3f0b8e8dc2a8924b6a5d0a044a9b8bd9142118fa96a91f48b2b82eee1a8c44a09ec1e667c138fa9f2f5f9d67bd7628c0802eb84
+  data.tar.gz: 159098ba5d8b0fc69cadba33b3daf4c80da6fc34e8a62d456134bda83d33eed0247910054fea40c1e178ea0b2d8952a2008634169fd83eb9e6925c241596a853

data/CHANGELOG.md

@@ -17,4 +17,13 @@
 
 ## v0.0.6
 
-* Allow classes_to_skip and default_scope keys to be strings as well as classes
+* Allow classes_to_skip and default_scope keys to be strings as well as classes
+
+## v0.0.7
+
+* Make BrokenRecord work with colorize >= 0.5.8
+* Allow BrokenRecord::Scanner to be used programmatically (i.e. from rails console)
+* Remove assumption that all models are in app/models/**/*.rb
+* Show backtrace when exception occurs during scan
+* Change parallelization strategy to better distribute load across cores
+* Add an after_fork hook

data/README.md

@@ -6,43 +6,68 @@ Provides a rake task for scanning your ActiveRecord models and detecting validat
 
 Add this line to your application's Gemfile:
 
-    gem 'broken_record'
+```ruby
+gem 'broken_record'
+```
 
 And then execute:
 
-    $ bundle
+```bash
+bundle
+```
 
 Or install it yourself as:
 
-    $ gem install broken_record
+```bash
+gem install broken_record
+```
 
 ## Usage
 
 To scan all records of all models in your project:
 
-    rake broken_record:scan
+```bash
+rake broken_record:scan
+```
 
-If you want to scan all records of a specific model (e.g. the User model)
+If you want to scan all records of a specific model (e.g. the User model):
 
-    rake broken_record:scan[User]
+```bash
+rake broken_record:scan[User]
+```
+
+You can also specify a list of models to scan:
+
+```bash
+rake broken_record:scan[Product,User]
+```
 
 ## Configuration
 
-BrokenRecord provides a configure method with two options.  Here's an example:
+BrokenRecord provides a configure method with multiple options.  Here's an example:
 
-    BrokenRecord.configure do |config|
-        # Skip the Foo and Bar models when scanning.
-        config.classes_to_skip = [Foo, Bar]
+```ruby
+BrokenRecord.configure do |config|
+    # Skip the Foo and Bar models when scanning.
+    config.classes_to_skip = [Foo, Bar]
 
-        # Set a scope for which models should be validated
-        config.default_scopes = { Foo => proc { with_bars } }
+    # Set a scope for which models should be validated
+    config.default_scopes = { Foo => proc { with_bars } }
+
+    # The follow block will be called before scanning your records.
+    # This is useful for skipping validations you want to ignore.
+    config.before_scan do
+        User.skip_callback :validate, :before, :user_must_be_active
+    end
 
-        # BrokenRecord will call the block provided in before_scan before scanning
-        # your records.  This is useful for skipping validations you want to ignore.
-        config.before_scan do
-            User.skip_callback :validate, :before, :user_must_be_active
-        end
+	# BrokenRecord uses the parallelize gem to distribute work across
+	# multiple cores. The following block will be called every time
+	# the process is forked (useful for re-establishing connections).
+	config.after_fork do
+      Rails.cache.reconnect if Rails.cache.respond_to? :reconnect
     end
+end
+```
 
 ## Contributing
 

data/broken_record.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.3'
 
-  spec.add_runtime_dependency 'rake', '~> 10'
-  spec.add_runtime_dependency 'parallel', '~> 0'
-  spec.add_runtime_dependency 'colorize', '0.5.8'
+  spec.add_runtime_dependency 'rake', '~> 10.1', '>= 10.1.10'
+  spec.add_runtime_dependency 'parallel', '~> 1.2', '>= 1.2.3'
+  spec.add_runtime_dependency 'colorize', '~> 0.5', '>= 0.5.8'
 end

data/lib/broken_record/config.rb

@@ -1,12 +1,17 @@
 module BrokenRecord
   module Config
     extend self
-    attr_accessor :classes_to_skip, :before_scan_callbacks, :default_scopes
+    attr_accessor :classes_to_skip, :before_scan_callbacks, :after_fork_callbacks, :default_scopes
     self.before_scan_callbacks = []
+    self.after_fork_callbacks = []
     self.default_scopes = {}
 
     def before_scan(&block)
       self.before_scan_callbacks << block
     end
+
+    def after_fork(&block)
+      self.after_fork_callbacks << block
+    end
   end
 end

data/lib/broken_record/job.rb

@@ -0,0 +1,66 @@
+require 'broken_record/job_result'
+
+module BrokenRecord
+  class Job
+    JOBS_PER_PROCESSOR = 1
+
+    attr_accessor :klass, :index
+
+    def self.jobs_per_class
+      JOBS_PER_PROCESSOR * Parallel.processor_count
+    end
+
+    def self.build_jobs(classes)
+      jobs = []
+      classes.each do |klass|
+        jobs_per_class.times do |index|
+          jobs << Job.new(:klass => klass, :index => index)
+        end
+      end
+      jobs
+    end
+
+    def initialize(options)
+      options.each { |k, v| send("#{k}=", v) }
+    end
+
+    def perform
+      result = BrokenRecord::JobResult.new(self)
+      result.start_timer
+
+      records.each do |r|
+        begin
+          if !r.valid?
+            message = "    Invalid record in #{klass} id=#{r.id}."
+            r.errors.each { |attr,msg| message <<  "\n        #{attr} - #{msg}" }
+            result.add_error message
+          end
+        rescue Exception => e
+          message = "    Exception for record in #{klass} id=#{r.id} - #{e}.\n"
+          message << e.backtrace.map { |line| "        #{line}"}.join("\n")
+          result.add_error message
+        end
+      end
+
+      result.stop_timer
+      result
+    end
+
+    private
+
+    def records
+      default_scope = BrokenRecord::Config.default_scopes[klass] || BrokenRecord::Config.default_scopes[klass.to_s]
+
+      model_scope = if default_scope
+        klass.instance_exec &default_scope
+      else
+        klass.unscoped
+      end
+
+      records_per_group = model_scope.count / self.class.jobs_per_class
+      scope = model_scope.offset(records_per_group * index)
+
+      index == Parallel.processor_count - 1 ? scope.load : scope.first(records_per_group)
+    end
+  end
+end

data/lib/broken_record/job_result.rb

@@ -0,0 +1,22 @@
+module BrokenRecord
+  class JobResult
+    attr_reader :start_time, :end_time, :errors, :job
+
+    def initialize(job)
+      @job = job
+      @errors = []
+    end
+
+    def start_timer
+      @start_time = Time.now
+    end
+
+    def stop_timer
+      @end_time = Time.now
+    end
+
+    def add_error(error)
+      @errors << "#{error.red}\n"
+    end
+  end
+end

data/lib/broken_record/result_aggregator.rb

@@ -0,0 +1,50 @@
+module BrokenRecord
+  class ResultAggregator
+    def initialize
+      @total_errors = 0
+      @aggregated_results = {}
+    end
+
+    def add_result(result)
+      @aggregated_results[result.job.klass] ||= []
+      @aggregated_results[result.job.klass] << result
+
+      if klass_done?(result.job.klass)
+        report_results result.job.klass
+      end
+    end
+
+    def report_final_results
+      if @total_errors == 0
+        puts "\nAll models validated successfully.".green
+      else
+        puts "\n#{@total_errors} errors were found while running validations.".red
+        exit 1
+      end
+    end
+
+    private
+
+    def klass_done?(klass)
+      @aggregated_results[klass].count == Job.jobs_per_class
+    end
+
+    def report_results(klass)
+      all_errors = @aggregated_results[klass].map(&:errors).flatten
+      start_time = @aggregated_results[klass].map(&:start_time).min
+      end_time = @aggregated_results[klass].map(&:end_time).max
+      duration = (end_time - start_time).round(3)
+
+      @total_errors += all_errors.count
+
+      print "Validating model #{klass}... ".ljust(70)
+      if all_errors.empty?
+        print '[PASS]'.green
+      else
+        print '[FAIL]'.red
+      end
+      print "  (#{duration}s)\n"
+      print all_errors.join if all_errors.any?
+    end
+  end
+end

data/lib/broken_record/scanner.rb

@@ -1,78 +1,51 @@
-require "broken_record/logger"
+require 'broken_record/job'
+require 'broken_record/result_aggregator'
 require 'parallel'
 
 module BrokenRecord
   class Scanner
-    def run(model_name = nil)
-      models = models_to_validate(model_name)
+    def run(class_names)
+      ResultAggregator.new.tap do |aggregator|
+        classes = classes_to_validate(class_names)
 
-      BrokenRecord::Config.before_scan_callbacks.each { |callback| callback.call }
+        BrokenRecord::Config.before_scan_callbacks.each { |callback| callback.call }
 
-      results = BrokenRecord::Logger.parallel do |lock|
-        Parallel.map(models) do |model|
-          result = validate_model(model)
-          BrokenRecord::Logger.report_output(result, lock)
-          result
+        jobs = BrokenRecord::Job.build_jobs(classes)
+
+        callback = proc do |_, _, result|
+          aggregator.add_result result if result.is_a? BrokenRecord::JobResult
         end
-      end
 
-      BrokenRecord::Logger.report_results(results)
+        Parallel.each(jobs, :finish => callback) do |job|
+          ActiveRecord::Base.connection.reconnect!
+          BrokenRecord::Config.after_fork_callbacks.each { |callback| callback.call }
+          job.perform
+        end
+      end
     end
 
     private
 
-    def models_to_validate(model_name)
-      if model_name
-        [ model_name.constantize ]
-      else
+    def classes_to_validate(class_names)
+      if class_names.empty?
         load_all_active_record_classes
+      else
+        class_names.map(&:strip).map(&:constantize)
       end
     end
 
     def load_all_active_record_classes
-      Dir.glob(Rails.root.to_s + '/app/models/**/*.rb').each { |file| require file }
+      Rails.application.eager_load!
       objects = Set.new
       # Classes to skip may either be constants or strings.  Convert all to strings for easier lookup
       classes_to_skip = BrokenRecord::Config.classes_to_skip.map(&:to_s)
-      ObjectSpace.each_object(Class) do |klass|
-        if ActiveRecord::Base > klass
-          # Use base_class so we don't try to validate abstract classes and so we don't validate
-          # STI classes multiple times.  See active_record/inheritance.rb for more details.
-          objects.add klass.base_class unless classes_to_skip.include?(klass.to_s)
-        end
+      ActiveRecord::Base.descendants.each do |klass|
+        # Use base_class so we don't try to validate abstract classes and so we don't validate
+        # STI classes multiple times.  See active_record/inheritance.rb for more details.
+        objects.add klass.base_class unless classes_to_skip.include?(klass.to_s)
       end
 
       objects.sort_by(&:name)
     end
-
-    def validate_model(model)
-      ActiveRecord::Base.connection.reconnect!
-
-      BrokenRecord::Logger.log(model) do |logger|
-        begin
-          default_scope = BrokenRecord::Config.default_scopes[model] || BrokenRecord::Config.default_scopes[model.to_s]
-
-          if default_scope
-            model_scope = model.instance_exec &default_scope
-          else
-            model_scope = model.unscoped
-          end
-
-          model_scope.find_each do |r|
-            begin
-              if !r.valid?
-                message = "    Invalid record in #{model} id=#{r.id}."
-                r.errors.each { |attr,msg| message <<  "\n        #{attr} - #{msg}" }
-                logger.log_error message
-              end
-            rescue Exception => msg
-              logger.log_error "    Exception for record in #{model} id=#{r.id} - #{msg}."
-            end
-          end
-        rescue Exception => msg
-          logger.log_error "    Error querying model #{model} - #{msg}."
-        end
-      end
-    end
   end
-end
+end

data/lib/broken_record/tasks.rb

@@ -1,9 +1,12 @@
 require 'rake'
 
 namespace :broken_record do
-  desc 'Scans all models for validation errors'
-  task :scan, [:model_name] => :environment do |t, args|
+  desc 'Scans models for validation errors'
+  task :scan, [:class_name] => :environment do |t, args|
     scanner = BrokenRecord::Scanner.new
-    scanner.run(args[:model_name])
+    class_names = args[:class_name] ? [args[:class_name]] : []
+    class_names += args.extras
+    aggregator = scanner.run(class_names)
+    aggregator.report_final_results
   end
-end
+end

data/lib/broken_record/version.rb

@@ -1,3 +1,3 @@
 module BrokenRecord
-  VERSION = "0.0.6"
+  VERSION = '0.0.7'
 end

metadata

@@ -1,69 +1,87 @@
 --- !ruby/object:Gem::Specification
 name: broken_record
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Nicholas Gervasi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-30 00:00:00.000000000 Z
+date: 2015-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '10.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 10.1.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.1'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10'
+        version: 10.1.10
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.3
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.8
 description: Detects ActiveRecord models that are not valid.
@@ -73,7 +91,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -82,8 +100,10 @@ files:
 - broken_record.gemspec
 - lib/broken_record.rb
 - lib/broken_record/config.rb
-- lib/broken_record/logger.rb
+- lib/broken_record/job.rb
+- lib/broken_record/job_result.rb
 - lib/broken_record/railtie.rb
+- lib/broken_record/result_aggregator.rb
 - lib/broken_record/scanner.rb
 - lib/broken_record/tasks.rb
 - lib/broken_record/version.rb
@@ -97,17 +117,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: Provides a rake task for scanning your ActiveRecord models and detecting

data/lib/broken_record/logger.rb

@@ -1,85 +0,0 @@
-require 'colorize'
-require 'tempfile'
-
-module BrokenRecord
-  class Logger
-
-    # Static Methods
-
-    def self.report_output(result, lock = nil)
-      lock.flock File::LOCK_EX if lock
-      $stdout.print result[:stdout]
-      $stdout.flush
-    ensure
-      lock.flock File::LOCK_UN if lock
-    end
-
-    def self.report_results(test_results)
-      total_errors = 0
-      test_results.each { |result| total_errors += result[:error_count] }
-      if total_errors == 0
-        puts "\nAll models validated successfully.".green
-      else
-        puts "\n#{total_errors} errors were found while running validations.".red
-        exit 1
-      end
-    end
-
-    def self.parallel
-      Tempfile.open 'broken_record_lock' do |lock|
-        yield lock
-      end
-    end
-
-    def self.log(model, &block)
-      logger = new
-      logger.start_log
-      logger.log_header "Validating model #{model}... ".ljust(70)
-
-      yield(logger)
-
-      logger.log_result
-      logger.result
-    end
-
-    # Instance Methods
-
-    def initialize
-      @header = ''
-      @errors = []
-
-      @stdout = ''
-    end
-
-    def log_header(header_message)
-      @header = header_message
-    end
-
-    def start_log
-      @start_time = Time.now
-    end
-
-    def log_error(message)
-      @errors << "#{message.red}\n"
-    end
-
-    def log_result
-      @stdout << @header
-
-      if @errors.empty?
-        @stdout << '[PASS]'.green
-      else
-        @stdout << '[FAIL]'.red
-      end
-
-      duration = (Time.now - @start_time).round(3)
-      @stdout << "  (#{duration}s)\n"
-
-      @stdout << @errors.join
-    end
-
-    def result
-      { stdout: @stdout, error_count: @errors.count}
-    end
-  end
-end