acts_as_scrubbable 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: a11d2fbbda7f3984de9911cba7b449d3f6c6720b17430f6511cf46407cd8a5c9
-  data.tar.gz: 69a93697794fc1c7abd9b2acd2f9c068639b04f03391b5cc028492ebdebf44a1
+SHA1:
+  metadata.gz: ead6fbeb0cee99c7046ad13c00423805d1099550
+  data.tar.gz: e31b55515b4cef3df1d78e482ec37745fdcf518c
 SHA512:
-  metadata.gz: 4b40343ddb3c0b7f764fd48167d3ce3c0dd2cc91f2221652f515849effd1b6da67040fe376e2543240a0efd391aa91eb0347d9e2780b184bee500ac5cad2a319
-  data.tar.gz: 18c59e0bb668fe98bbf584297d56fb41b14d2bef2e3f4e9e052b330e012a61b5eb9988ab493233db953959776390e529218bff2e0bc8203033712a48054cad7f
+  metadata.gz: 7c56b941e491ed7e7d76e8d1abaeaddebbea42e6d1d6b023f122698f0c5d6483b6959a0dbb45007b5ef0443401d12fb0b37d7c975282675a7acefdf332603783
+  data.tar.gz: 6a13305d96f630c4f37494d92ce64a83dbba5d19e0a6528460bb34616d80ba13b172803e3d112fbf5d1a6db113fdc145605f47c1808395a07f811f2814f09660

data/lib/acts_as_scrubbable.rb

@@ -37,7 +37,7 @@ module ActsAsScrubbable
       :middle_name       => -> { Faker::Name.name },
       :name              => -> { Faker::Name.name },
       :email             => -> { Faker::Internet.email },
-      :name_title        => -> { Faker::Name.title },
+      :name_title        => -> { defined? Faker::Job ? Faker::Job.title : Faker::Name.title },
       :company_name      => -> { Faker::Company.name },
       :street_address    => -> { Faker::Address.street_address },
       :secondary_address => -> { Faker::Address.secondary_address },

data/lib/acts_as_scrubbable/parallel_table_scrubber.rb

@@ -0,0 +1,84 @@
+require "parallel"
+
+module ActsAsScrubbable
+  class ParallelTableScrubber
+    def initialize(ar_class)
+      @ar_class = ar_class
+    end
+
+    def scrub(num_batches:)
+      # Removing any find or initialize callbacks from model
+      ar_class.reset_callbacks(:initialize)
+      ar_class.reset_callbacks(:find)
+
+      queries = parallel_queries(ar_class: ar_class, num_batches: num_batches)
+      scrubbed_count = Parallel.map(queries) { |query|
+        scrubbed_count = 0
+        ActiveRecord::Base.connection_pool.with_connection do
+          relation = ar_class
+          relation = relation.send(:scrubbable_scope) if ar_class.respond_to?(:scrubbable_scope)
+          relation.where(query).find_in_batches(batch_size: 1000) do |batch|
+            ActiveRecord::Base.transaction do
+              batch.each do |obj|
+                obj.scrub!
+                scrubbed_count += 1
+              end
+            end
+          end
+        end
+        scrubbed_count
+      }.reduce(:+)
+    end
+
+    private
+
+    attr_reader :ar_class
+
+    # create even ID ranges for the table
+    def parallel_queries(ar_class:, num_batches:)
+      raise "Model is missing id column" if ar_class.columns.none? { |column| column.name == "id" }
+
+      if ar_class.respond_to?(:scrubbable_scope)
+        num_records = ar_class.send(:scrubbable_scope).count
+      else
+        num_records = ar_class.count
+      end
+      return [] if num_records == 0 # no records to import
+
+      record_window_size, modulus = num_records.divmod(num_batches)
+      if record_window_size < 1
+        record_window_size = 1
+        modulus = 0
+      end
+
+      start_id = next_id(ar_class: ar_class, offset: 0)
+      queries = num_batches.times.each_with_object([]) do |_, queries|
+        next unless start_id
+
+        end_id = next_id(ar_class: ar_class, id: start_id, offset: record_window_size-1)
+        if modulus > 0
+          end_id = next_id(ar_class: ar_class, id: end_id)
+          modulus -= 1
+        end
+        queries << {id: start_id..end_id} if end_id
+        start_id = next_id(ar_class: ar_class, id: end_id || start_id)
+      end
+
+      # just in case new records are added since we started, extend the end ID
+      queries[-1] = ["#{ar_class.quoted_table_name}.id >= ?", queries[-1][:id].begin] if queries.any?
+
+      queries
+    end
+
+    def next_id(ar_class:, id: nil, offset: 1)
+      if ar_class.respond_to?(:scrubbable_scope)
+        collection = ar_class.send(:scrubbable_scope)
+      else
+        collection = ar_class.all
+      end
+      collection.reorder(:id)
+      collection = collection.where("#{ar_class.quoted_table_name}.id >= :id", id: id) if id
+      collection.offset(offset).limit(1).pluck(:id).first
+    end
+  end
+end

data/lib/acts_as_scrubbable/tasks.rb

@@ -3,64 +3,55 @@ require 'rake'
 
 namespace :scrub do
 
-  desc "scrub all"
+  desc "scrub all scrubbable tables"
   task all: :environment do
-
     require 'highline/import'
     require 'term/ansicolor'
     require 'logger'
     require 'parallel'
 
-
     include Term::ANSIColor
 
-    @logger = Logger.new($stdout)
-    @logger.formatter = proc do |severity, datetime, progname, msg|
+    logger = Logger.new($stdout)
+    logger.formatter = proc do |severity, datetime, progname, msg|
        "#{datetime}: [#{severity}] - #{msg}\n"
     end
 
     db_host = ActiveRecord::Base.connection_config[:host]
     db_name = ActiveRecord::Base.connection_config[:database]
 
-    @logger.warn "Please verify the information below to continue".red
-    @logger.warn "Host: ".red + " #{db_host}".white
-    @logger.warn "Database: ".red + "#{db_name}".white
+    logger.warn "Please verify the information below to continue".red
+    logger.warn "Host: ".red + " #{db_host}".white
+    logger.warn "Database: ".red + "#{db_name}".white
 
     unless ENV["SKIP_CONFIRM"] == "true"
-
       answer = ask("Type '#{db_host}' to continue. \n".red + '-> '.white)
       unless answer == db_host
-        @logger.error "exiting ...".red
+        logger.error "exiting ...".red
         exit
       end
     end
 
-    @logger.warn "Scrubbing classes".red
+    logger.warn "Scrubbing classes".red
 
     Rails.application.eager_load! # make sure all the classes are loaded
 
-    @total_scrubbed = 0
-
     ar_classes = ActiveRecord::Base.descendants.select{|d| d.scrubbable? }.sort_by{|d| d.to_s }
 
-
-    # if the ENV variable is set
-
-    unless ENV["SCRUB_CLASSES"].blank?
+    if ENV["SCRUB_CLASSES"].present?
       class_list = ENV["SCRUB_CLASSES"].split(",")
       class_list = class_list.map {|_class_str| _class_str.constantize }
       ar_classes = ar_classes & class_list
     end
 
-    @logger.info "Srubbable Classes: #{ar_classes.join(', ')}".white
+    logger.info "Scrubbable Classes: #{ar_classes.join(', ')}".white
 
     Parallel.each(ar_classes) do |ar_class|
-
       # Removing any find or initialize callbacks from model
       ar_class.reset_callbacks(:initialize)
       ar_class.reset_callbacks(:find)
 
-      @logger.info "Scrubbing #{ar_class} ...".green
+      logger.info "Scrubbing #{ar_class} ...".green
 
       scrubbed_count = 0
 
@@ -81,16 +72,59 @@ namespace :scrub do
         end
       end
 
-      @logger.info "#{scrubbed_count} #{ar_class} objects scrubbed".blue
+      logger.info "#{scrubbed_count} #{ar_class} objects scrubbed".blue
     end
     ActiveRecord::Base.connection.verify!
 
     if ENV["SKIP_AFTERHOOK"].blank?
-      @logger.info "Running after hook".red
+      logger.info "Running after hook".red
       ActsAsScrubbable.execute_after_hook
     end
 
-    @logger.info "Scrub Complete!".white
+    logger.info "Scrub Complete!".white
+  end
+
+  desc "Scrub one table"
+  task :model, [:ar_class] => :environment do |_, args|
+    require 'highline/import'
+    require 'term/ansicolor'
+    require 'logger'
+    require 'acts_as_scrubbable/parallel_table_scrubber'
+
+    include Term::ANSIColor
+
+    logger = Logger.new($stdout)
+    logger.formatter = proc do |severity, datetime, progname, msg|
+       "#{datetime}: [#{severity}] - #{msg}\n"
+    end
+
+    db_host = ActiveRecord::Base.connection_config[:host]
+    db_name = ActiveRecord::Base.connection_config[:database]
+
+    logger.warn "Please verify the information below to continue".red
+    logger.warn "Host: ".red + " #{db_host}".white
+    logger.warn "Database: ".red + "#{db_name}".white
+
+    unless ENV["SKIP_CONFIRM"] == "true"
+      answer = ask("Type '#{db_host}' to continue. \n".red + '-> '.white)
+      unless answer == db_host
+        logger.error "exiting ...".red
+        exit
+      end
+    end
+
+    Rails.application.eager_load! # make sure all the classes are loaded
+
+    ar_class = args[:ar_class].constantize
+    logger.info "Scrubbing #{ar_class} ...".green
+
+    num_batches = Integer(ENV.fetch("SCRUB_BATCHES", "256"))
+    scrubbed_count = ActsAsScrubbable::ParallelTableScrubber.new(ar_class).scrub(num_batches: num_batches)
+
+    logger.info "#{scrubbed_count} #{ar_class} objects scrubbed".blue
+    ActiveRecord::Base.connection.verify!
+
+    logger.info "Scrub Complete!".white
   end
 end
 

data/lib/acts_as_scrubbable/version.rb

@@ -1,3 +1,3 @@
 module ActsAsScrubbable
-  VERSION = '1.0.2'
+  VERSION = '1.1.0'
 end

data/spec/db/schema.rb

@@ -2,8 +2,23 @@ ActiveRecord::Schema.define(version: 20150421224501) do
 
   create_table "scrubbable_models", force: true do |t|
     t.string   "first_name"
+    t.string   "last_name"
+    t.string   "middle_name"
+    t.string   "name"
+    t.string   "email"
+    t.string   "title"
+    t.string   "company_name"
     t.string   "address1"
+    t.string   "address2"
+    t.string   "zip_code"
+    t.string   "state"
+    t.string   "state_short"
+    t.string   "city"
     t.string   "lat"
+    t.string   "lon"
+    t.string   "username"
+    t.boolean  "active"
+    t.string   "school"
   end
 
 end

data/spec/lib/acts_as_scrubbable/scrub_spec.rb

@@ -5,9 +5,35 @@ RSpec.describe ActsAsScrubbable::Scrub do
   describe '.scrub' do
 
     # update_columns cannot be run on a new record
-    subject{ ScrubbableModel.new }
+    subject { ScrubbableModel.new }
     before(:each) { subject.save }
 
+    it 'scrubs all columns' do
+      subject.attributes = {
+        first_name: "Ted",
+        last_name: "Lowe",
+        middle_name: "Cassidy",
+        name: "Miss Vincenzo Smitham",
+        email: "trentdibbert@wiza.com",
+        title: "Internal Consultant",
+        company_name: "Greenfelder, Collier and Lesch",
+        address1: "86780 Watsica Flats",
+        address2: "Apt. 913",
+        zip_code: "49227",
+        state: "Ohio",
+        state_short: "OH",
+        city: "Port Hildegard",
+        lat: -79.5855309778974,
+        lon: 13.517352691513906,
+        username: "oscar.hermann",
+        active: false,
+        school: "Eastern Lebsack",
+      }
+      expect {
+        subject.scrub!
+      }.not_to raise_error
+    end
+
     it 'changes the first_name attribute when scrub is run' do
       subject.first_name = "Ted"
       allow(Faker::Name).to receive(:first_name).and_return("John")

data/spec/support/database.rb

@@ -15,7 +15,24 @@ end
 class NonScrubbableModel < ActiveRecord::Base; end
 
 class ScrubbableModel < ActiveRecord::Base
-  acts_as_scrubbable :first_name, :address1 => :street_address, :lat => :latitude
+  acts_as_scrubbable :first_name,
+    :last_name,
+    :middle_name,
+    :name,
+    :email,
+    :company_name,
+    :zip_code,
+    :state,
+    :city,
+    :username,
+    :school,
+    :title => :name_title,
+    :address1 => :street_address,
+    :address2 => :secondary_address,
+    :state_short => :state_abbr,
+    :lat => :latitude,
+    :lon => :longitude,
+    :active => :boolean
   attr_accessor :scrubbing_begun, :scrubbing_finished
   set_callback :scrub, :before do
     self.scrubbing_begun = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: acts_as_scrubbable
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Samer Masry
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-22 00:00:00.000000000 Z
+date: 2019-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -226,6 +226,7 @@ files:
 - README.md
 - acts_as_scrubbable.gemspec
 - lib/acts_as_scrubbable.rb
+- lib/acts_as_scrubbable/parallel_table_scrubber.rb
 - lib/acts_as_scrubbable/scrub.rb
 - lib/acts_as_scrubbable/scrubbable.rb
 - lib/acts_as_scrubbable/tasks.rb
@@ -256,7 +257,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Scrubbing data made easy