data_janitor 0.3.7 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 28258e2ec98f8f7db00397740aca503713a4ffba
-  data.tar.gz: 58f562e76d73c73214d49c0c2616d33e80b9b975
+  metadata.gz: 8354b3129d9d3776972692c4173df69c43df65c7
+  data.tar.gz: dac7277b118a995afe2b6adb021e3e558d0cb8dc
 SHA512:
-  metadata.gz: 3be58cb1964db6279c0227cb8efc99c1dd0bcf008d860ea00bb7c71bbc8b18665a78dcb7a28cee88f3ec0507ed8bb0bde0c567015b1eb690694bdbc58e48057e
-  data.tar.gz: b04f75842fc3bde77e987d264b4ac3946caad5f28b33bf0c414bf774f5e7dbbc03171a7399cc59f7bd5b7b2e4759c24527eb9e0dda2066f927a0efd003b33616
+  metadata.gz: 0a12f79726392bb25ef7953ab47772da7fb5a900cf1aef7c0ccea56b745de6ab6e8c06a4676fd722bc1d57a13472647749d380d86b0a4709dd21bdd71ed589fb
+  data.tar.gz: 36574fa58940279106682e87c1a04662c811c5a5da2294e86f43843607292f03d07aec8e8aaa5ad0c3a8e59e22c301fdc7d004ce4dcb0630371066c34cf250bd

data/README.md

@@ -81,6 +81,24 @@ rake data_janitor:cleanse[SomeModel]
 
 This will apply all the fixes that do not require semantic analysis of the data (e.g. replace `nil` values with `""` for strings)
 
+Data Janitor has the experimental ability to perform some built-in type checks, only a small part of which is implemented and currently tends to be noisy when on. It currently defaults to off. Each audit command takes an option string that defaults to 'no-type-checks' and can be set other colon-separated values to turn off specific checks. The values are:
+type-checks (or any other string not below)
+no-type-checks
+no-boolean
+no-decimal
+no-float
+no-integer
+no-string
+no-text
+no-array
+
+For example:
+```
+rake data_janitor:audit_model[SomeModel,no-string:no-boolean]
+rake data_janitor:audit[tmp/out.json,false,false,no-string:no-boolean]
+```
+
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/westfield/data_janitor.

data/lib/data_janitor.rb

@@ -1,5 +1,6 @@
+require "rails"
 require "data_janitor/version"
-require 'rails'
+require "data_janitor/data_janitor"
 require "data_janitor/universal_validator"
 require "data_janitor/audit_validatable"
 

data/lib/data_janitor/data_janitor.rb

@@ -0,0 +1,80 @@
+module DataJanitor
+  def self.audit(output_file, verbose, unscoped, options)
+    output = {}
+    all_models.each do |ar_model|
+      begin
+        audit_model ar_model, output, verbose, unscoped, options
+      rescue ActiveRecord::StatementInvalid # used to catch HABTM and schema migration. Only care about real Models
+        puts "skipping #{ar_model}"
+      end
+    end
+
+    File.write(output_file, output.to_json)
+
+    puts "Wrote results to #{output_file}"
+  end
+
+  def self.audit_model(model, output = {}, verbose = false, unscoped = false, options = 'no-type-check')
+    total = 0
+    failed = 0
+    puts "Validating: #{model.name}"
+    output[model.name] = {}
+
+    model = model.unscoped if unscoped
+    model.include(DataJanitor::UniversalValidator)
+    model.validate do |record|
+      record.validate_field_values options
+    end
+
+    model.find_each do |rec|
+      if rec.invalid?(:dj_audit)
+        rec.errors.to_h.each_pair do |attribute, error_message|
+          output[model.name][attribute] ||= {}
+          output[model.name][attribute][error_message] ||= []
+          output[model.name][attribute][error_message] << rec.id
+        end
+
+        failed += 1
+      end
+
+      total += 1
+    end
+
+    puts output.to_json if verbose
+    puts "Completed #{total} records with #{failed} failures"
+  end
+
+  def self.cleanse
+    all_models.each do |ar_model|
+      cleanse_model! ar_model
+    end
+  end
+
+  def self.cleanse_model(model)
+    cleanse_model! model.constantize
+  end
+
+  private
+
+  def self.all_models
+    # Needs this executed before here: Rails.application.eager_load!
+    ActiveRecord::Base.descendants
+  end
+
+  def self.cleanse_model!(model)
+    string_columns = model.columns.select{|c| (c.type == :string || c.type == :text) && c.array == false}
+    boolean_columns = model.columns.select{|c| c.type == :boolean && c.array == false}
+    array_columns = model.columns.select{|c| c.array == true}
+
+    clean_nils_from! model, string_columns, ""
+    clean_nils_from! model, boolean_columns, false
+    clean_nils_from! model, array_columns, []
+  end
+
+  def self.clean_nils_from!(model, columns, default)
+    columns.each do |column|
+      count = model.where(column.name => nil).update_all(column.name => default)
+      puts "Fixed #{count} #{model} records where #{column.name} was nil" if count > 0
+    end
+  end
+end

data/lib/data_janitor/universal_validator.rb

@@ -4,21 +4,26 @@ module DataJanitor
     # TODO: Run standard validators instead of home-brewed
     # validate :validate_field_values
     # ACCEPTABLE_BOOLEAN_VALUES = %w(t true y yes on 1 f false n no off 0) # this list was taken from Postgres spec. TRUE FALSE, that are also there, are not listed because they are DB-native literals and have no representation in Ruby code
-    def validate_field_values
+    def validate_field_values(options)
+      optional = options.split(':').map(&:strip)
+      return if optional.include? 'no-type-check'
+
       # selected_attributes = self.changed? ? self.changed_attributes : self.attributes
       selected_attributes = self.attributes
 
       selected_attributes.each do |field_name, field_val|
         column = self.column_for_attribute field_name
-        report_error = lambda {|msg| errors[column.name] << msg}
+        report_error = lambda { |msg| errors[column.name] << msg }
 
         if column.array
+          next if optional.include? 'no-array'
           report_error.call "cannot be nil" if field_val.nil?
           next
         end
 
         case column.type
         when :boolean
+          next if optional.include? 'no-boolean'
           report_error.call "cannot be nil" if field_val.nil?
           # report_error.call("must be a valid boolean") unless ACCEPTABLE_BOOLEAN_VALUES.include? field_val
         when :date
@@ -28,15 +33,19 @@ module DataJanitor
         when :datetime
           # Time.iso8601(field_val) rescue report_error.call("must be a datetime in ISO-8601")
         when :decimal
+          next if optional.include? 'no-decimal'
           report_error.call "cannot be nil" if field_val.nil?
           # TODO: run numericality test
         when :float
+          next if optional.include? 'no-float'
           report_error.call "cannot be nil" if field_val.nil?
           # TODO: run numericality test
         when :integer
+          next if optional.include? 'no-integer'
           report_error.call "cannot be nil" if field_val.nil?
           # TODO: run numericality test
         when :string, :text
+          next if optional.include?('no-string') || optional.include?('no-text')
           if field_val.nil?
             # Almost never does an app need to distinguish between nil and empty string, yet nil needs special handling in all cases
             report_error.call "cannot be nil. Use an empty string instead if that's what you wanted."

data/lib/data_janitor/version.rb

@@ -1,3 +1,3 @@
 module DataJanitor
-  VERSION = "0.3.7"
+  VERSION = "0.4.0"
 end

data/lib/tasks/data_janitor.rake

@@ -1,30 +1,26 @@
 namespace :data_janitor do
   desc 'Summarize invalid database records'
-  task :audit, [:output_file, :verbose, :unscoped] => [:environment] do |_t, args|
+  task :audit, [:output_file, :verbose, :unscoped, :options] => [:environment] do |_t, args|
     args.with_defaults(
       output_file: Rails.root.join('tmp', 'data_janitor_results.json'),
-      verbose: false,
-      unscoped: false
+      verbose: 'false',
+      unscoped: 'false',
+      options: 'no-type-check'
     )
+    verbose = args[:verbose] == 'true'
+    unscoped = args[:unscoped] == 'true'
 
-    output = {}
-    all_models.each do |ar_model|
-      begin
-        audit ar_model, output, args[:verbose], args[:unscoped]
-      rescue ActiveRecord::StatementInvalid # used to catch HABTM and schema migration. Only care about real Models
-        puts "skipping #{ar_model}"
-      end
-    end
-
-    File.write(args[:output_file], output.to_json)
-
-    puts "Wrote results to #{args[:output_file]}"
+    Rails.application.eager_load!
+    DataJanitor::audit(args[:output_file], verbose, unscoped, args[:options])
   end
 
   desc 'Audit one model for data issues'
-  task :audit_model, [:model] => [:environment] do |_t, args|
+  task :audit_model, [:model, :options] => [:environment] do |_t, args|
+    args.with_defaults(
+      options: 'no-type-check'
+    )
     Rails.application.eager_load!
-    audit args[:model].constantize, {}, true
+    DataJanitor::audit_model args[:model].constantize, {}, true, false, args[:options]
   end
 
   # For each model, apply trivial data corrections (those that do not require looking at data semantics).
@@ -34,67 +30,14 @@ namespace :data_janitor do
   # - replace all null arrays with []
   desc 'Apply common and safe data corrections'
   task cleanse: :environment do
-    all_models.each do |ar_model|
-      cleanse_model! ar_model
-    end
+    Rails.application.eager_load!
+    DataJanitor::clense
   end
 
   desc 'Apply fixes to one model only'
   task :cleanse_model, [:model] => [:environment] do |_t, args|
     Rails.application.eager_load!
-
-    cleanse_model! args[:model].constantize
-  end
-
-  private
-
-  def all_models
-    Rails.application.eager_load!
-    ActiveRecord::Base.descendants
+    DataJanitor::clense_model args[:model].constantize
   end
 
-  def audit(model, output = {}, verbose = false, unscoped = false)
-    total = 0
-    failed = 0
-    puts "Validating: #{model.name}"
-    output[model.to_s] = {}
-    model = model.unscoped if unscoped
-
-    model.include(DataJanitor::UniversalValidator)
-    model.validate :validate_field_values
-
-    model.find_each do |rec|
-      if rec.invalid?(:dj_audit)
-        rec.errors.to_h.each_pair do |attribute, error_message|
-          output[model.to_s][attribute] ||= {}
-          output[model.to_s][attribute][error_message] ||= []
-          output[model.to_s][attribute][error_message] << rec.id
-        end
-
-        failed += 1
-      end
-
-      total += 1
-    end
-
-    puts output.to_json if verbose
-    puts "Completed #{total} records with #{failed} failures"
-  end
-
-  def cleanse_model!(model)
-    string_columns = model.columns.select{|c| (c.type == :string || c.type == :text) && c.array == false}
-    boolean_columns = model.columns.select{|c| c.type == :boolean && c.array == false}
-    array_columns = model.columns.select{|c| c.array == true}
-
-    clean_nils_from! model, string_columns, ""
-    clean_nils_from! model, boolean_columns, false
-    clean_nils_from! model, array_columns, []
-  end
-
-  def clean_nils_from!(model, columns, default)
-    columns.each do |column|
-      count = model.where(column.name => nil).update_all(column.name => default)
-      puts "Fixed #{count} #{model} records where #{column.name} was nil" if count > 0
-    end
-  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_janitor
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.4.0
 platform: ruby
 authors:
 - Louis Tran
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-31 00:00:00.000000000 Z
+date: 2017-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -95,6 +95,7 @@ files:
 - data_janitor.gemspec
 - lib/data_janitor.rb
 - lib/data_janitor/audit_validatable.rb
+- lib/data_janitor/data_janitor.rb
 - lib/data_janitor/universal_validator.rb
 - lib/data_janitor/version.rb
 - lib/tasks/data_janitor.rake
@@ -118,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Rake task to check validity of column types and values.