acts_as_scrubbable 1.0.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2a4b66ce45eefc1baa779af4eecf03005b196a6b
-  data.tar.gz: 5f4d1febcb268c6891a02d1ddf5d48adf1dc8b22
+SHA256:
+  metadata.gz: bb73960ded68e0d1a181ea4bf86300ae30dccb3e29330e63ddc8cae835662750
+  data.tar.gz: 534cdd721d48284f9d714093f307108f0c12ac13ef9636bd28b08745f984fcc9
 SHA512:
-  metadata.gz: bcbae2c5dddaa9a6ec84a40c9bbf91ee283a422ac441af330c23eb2f216806cbde92ef47eb50435f99fd74416e62e4944e68491aefd6621ed5a15a0bcabef462
-  data.tar.gz: 67a3505a44a30b0946c0e6dd2b8235433bc7c7459571aba934325d9157f097c0991ea7c94f8824a14ff5aa8e7218cbcea27c9bca307884b843bc54f9de86a430
+  metadata.gz: 8685741230ee6f515b9b5738dd5b32772b66f513f991f19fe95d0db2dfcb6414f3424d1a28218e4f2b5a285bb6a0b739ce6446e2bfd569023caf1dc432ec9924
+  data.tar.gz: 4883b89adccb21c37572a3e4aa98fba58b19765d356748875921ab877c6bcc4a201b9c798f38777d8ac99cd46c0b6c4d27aac2595657d03c6199fdcc012fe69b

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+
+rvm:
+  - 2.4.3
+
+script:
+  - bundle exec rspec

data/Gemfile

@@ -1,5 +1,3 @@
-ruby '2.3.1'
-
 source 'https://rubygems.org'
 
 gemspec

data/acts_as_scrubbable.gemspec

@@ -13,9 +13,9 @@ Gem::Specification.new do |s|
   s.license     = "MIT"
   s.required_ruby_version = '~> 2.0'
 
-  s.add_runtime_dependency 'activesupport'    , '>= 4.1', '< 5.1'
-  s.add_runtime_dependency 'activerecord'     , '>= 4.1', '< 5.1'
-  s.add_runtime_dependency 'railties'         , '>= 4.1', '< 5.1'
+  s.add_runtime_dependency 'activesupport'    , '>= 4.1', '< 6'
+  s.add_runtime_dependency 'activerecord'     , '>= 4.1', '< 6'
+  s.add_runtime_dependency 'railties'         , '>= 4.1', '< 6'
   s.add_runtime_dependency 'faker'            , '>= 1.4'
   s.add_runtime_dependency 'highline'         , '>= 1.7'
   s.add_runtime_dependency 'term-ansicolor'   , '>= 1.3'

data/lib/acts_as_scrubbable.rb

@@ -37,7 +37,7 @@ module ActsAsScrubbable
       :middle_name       => -> { Faker::Name.name },
       :name              => -> { Faker::Name.name },
       :email             => -> { Faker::Internet.email },
-      :name_title        => -> { Faker::Name.title },
+      :name_title        => -> { defined? Faker::Job ? Faker::Job.title : Faker::Name.title },
       :company_name      => -> { Faker::Company.name },
       :street_address    => -> { Faker::Address.street_address },
       :secondary_address => -> { Faker::Address.secondary_address },

data/lib/acts_as_scrubbable/parallel_table_scrubber.rb

@@ -0,0 +1,84 @@
+require "parallel"
+
+module ActsAsScrubbable
+  class ParallelTableScrubber
+    def initialize(ar_class)
+      @ar_class = ar_class
+    end
+
+    def scrub(num_batches:)
+      # Removing any find or initialize callbacks from model
+      ar_class.reset_callbacks(:initialize)
+      ar_class.reset_callbacks(:find)
+
+      queries = parallel_queries(ar_class: ar_class, num_batches: num_batches)
+      scrubbed_count = Parallel.map(queries) { |query|
+        scrubbed_count = 0
+        ActiveRecord::Base.connection_pool.with_connection do
+          relation = ar_class
+          relation = relation.send(:scrubbable_scope) if ar_class.respond_to?(:scrubbable_scope)
+          relation.where(query).find_in_batches(batch_size: 1000) do |batch|
+            ActiveRecord::Base.transaction do
+              batch.each do |obj|
+                obj.scrub!
+                scrubbed_count += 1
+              end
+            end
+          end
+        end
+        scrubbed_count
+      }.reduce(:+)
+    end
+
+    private
+
+    attr_reader :ar_class
+
+    # create even ID ranges for the table
+    def parallel_queries(ar_class:, num_batches:)
+      raise "Model is missing id column" if ar_class.columns.none? { |column| column.name == "id" }
+
+      if ar_class.respond_to?(:scrubbable_scope)
+        num_records = ar_class.send(:scrubbable_scope).count
+      else
+        num_records = ar_class.count
+      end
+      return [] if num_records == 0 # no records to import
+
+      record_window_size, modulus = num_records.divmod(num_batches)
+      if record_window_size < 1
+        record_window_size = 1
+        modulus = 0
+      end
+
+      start_id = next_id(ar_class: ar_class, offset: 0)
+      queries = num_batches.times.each_with_object([]) do |_, queries|
+        next unless start_id
+
+        end_id = next_id(ar_class: ar_class, id: start_id, offset: record_window_size-1)
+        if modulus > 0
+          end_id = next_id(ar_class: ar_class, id: end_id)
+          modulus -= 1
+        end
+        queries << {id: start_id..end_id} if end_id
+        start_id = next_id(ar_class: ar_class, id: end_id || start_id)
+      end
+
+      # just in case new records are added since we started, extend the end ID
+      queries[-1] = ["#{ar_class.quoted_table_name}.id >= ?", queries[-1][:id].begin] if queries.any?
+
+      queries
+    end
+
+    def next_id(ar_class:, id: nil, offset: 1)
+      if ar_class.respond_to?(:scrubbable_scope)
+        collection = ar_class.send(:scrubbable_scope)
+      else
+        collection = ar_class.all
+      end
+      collection = collection.reorder(:id)
+      collection = collection.where("#{ar_class.quoted_table_name}.id >= :id", id: id) if id
+      collection.offset(offset).limit(1).pluck(:id).first
+    end
+  end
+end

data/lib/acts_as_scrubbable/tasks.rb

@@ -3,64 +3,55 @@ require 'rake'
 
 namespace :scrub do
 
-  desc "scrub all"
+  desc "scrub all scrubbable tables"
   task all: :environment do
-
     require 'highline/import'
     require 'term/ansicolor'
     require 'logger'
     require 'parallel'
 
-
     include Term::ANSIColor
 
-    @logger = Logger.new($stdout)
-    @logger.formatter = proc do |severity, datetime, progname, msg|
+    logger = Logger.new($stdout)
+    logger.formatter = proc do |severity, datetime, progname, msg|
        "#{datetime}: [#{severity}] - #{msg}\n"
     end
 
     db_host = ActiveRecord::Base.connection_config[:host]
     db_name = ActiveRecord::Base.connection_config[:database]
 
-    @logger.warn "Please verify the information below to continue".red
-    @logger.warn "Host: ".red + " #{db_host}".white
-    @logger.warn "Database: ".red + "#{db_name}".white
+    logger.warn "Please verify the information below to continue".red
+    logger.warn "Host: ".red + " #{db_host}".white
+    logger.warn "Database: ".red + "#{db_name}".white
 
     unless ENV["SKIP_CONFIRM"] == "true"
-
       answer = ask("Type '#{db_host}' to continue. \n".red + '-> '.white)
       unless answer == db_host
-        @logger.error "exiting ...".red
+        logger.error "exiting ...".red
         exit
       end
     end
 
-    @logger.warn "Scrubbing classes".red
+    logger.warn "Scrubbing classes".red
 
     Rails.application.eager_load! # make sure all the classes are loaded
 
-    @total_scrubbed = 0
-
     ar_classes = ActiveRecord::Base.descendants.select{|d| d.scrubbable? }.sort_by{|d| d.to_s }
 
-
-    # if the ENV variable is set
-
-    unless ENV["SCRUB_CLASSES"].blank?
+    if ENV["SCRUB_CLASSES"].present?
       class_list = ENV["SCRUB_CLASSES"].split(",")
       class_list = class_list.map {|_class_str| _class_str.constantize }
       ar_classes = ar_classes & class_list
     end
 
-    @logger.info "Srubbable Classes: #{ar_classes.join(', ')}".white
+    logger.info "Scrubbable Classes: #{ar_classes.join(', ')}".white
 
     Parallel.each(ar_classes) do |ar_class|
-
       # Removing any find or initialize callbacks from model
       ar_class.reset_callbacks(:initialize)
       ar_class.reset_callbacks(:find)
 
-      @logger.info "Scrubbing #{ar_class} ...".green
+      logger.info "Scrubbing #{ar_class} ...".green
 
       scrubbed_count = 0
 
@@ -81,16 +72,59 @@ namespace :scrub do
         end
       end
 
-      @logger.info "#{scrubbed_count} #{ar_class} objects scrubbed".blue
+      logger.info "#{scrubbed_count} #{ar_class} objects scrubbed".blue
     end
     ActiveRecord::Base.connection.verify!
 
     if ENV["SKIP_AFTERHOOK"].blank?
-      @logger.info "Running after hook".red
+      logger.info "Running after hook".red
       ActsAsScrubbable.execute_after_hook
     end
 
-    @logger.info "Scrub Complete!".white
+    logger.info "Scrub Complete!".white
+  end
+
+  desc "Scrub one table"
+  task :model, [:ar_class] => :environment do |_, args|
+    require 'highline/import'
+    require 'term/ansicolor'
+    require 'logger'
+    require 'acts_as_scrubbable/parallel_table_scrubber'
+
+    include Term::ANSIColor
+
+    logger = Logger.new($stdout)
+    logger.formatter = proc do |severity, datetime, progname, msg|
+       "#{datetime}: [#{severity}] - #{msg}\n"
+    end
+
+    db_host = ActiveRecord::Base.connection_config[:host]
+    db_name = ActiveRecord::Base.connection_config[:database]
+
+    logger.warn "Please verify the information below to continue".red
+    logger.warn "Host: ".red + " #{db_host}".white
+    logger.warn "Database: ".red + "#{db_name}".white
+
+    unless ENV["SKIP_CONFIRM"] == "true"
+      answer = ask("Type '#{db_host}' to continue. \n".red + '-> '.white)
+      unless answer == db_host
+        logger.error "exiting ...".red
+        exit
+      end
+    end
+
+    Rails.application.eager_load! # make sure all the classes are loaded
+
+    ar_class = args[:ar_class].constantize
+    logger.info "Scrubbing #{ar_class} ...".green
+
+    num_batches = Integer(ENV.fetch("SCRUB_BATCHES", "256"))
+    scrubbed_count = ActsAsScrubbable::ParallelTableScrubber.new(ar_class).scrub(num_batches: num_batches)
+
+    logger.info "#{scrubbed_count} #{ar_class} objects scrubbed".blue
+    ActiveRecord::Base.connection.verify!
+
+    logger.info "Scrub Complete!".white
   end
 end
 

data/lib/acts_as_scrubbable/version.rb

@@ -1,3 +1,3 @@
 module ActsAsScrubbable
-  VERSION = '1.0.0'
+  VERSION = '1.1.1'
 end

data/spec/db/schema.rb

@@ -2,8 +2,23 @@ ActiveRecord::Schema.define(version: 20150421224501) do
 
   create_table "scrubbable_models", force: true do |t|
     t.string   "first_name"
+    t.string   "last_name"
+    t.string   "middle_name"
+    t.string   "name"
+    t.string   "email"
+    t.string   "title"
+    t.string   "company_name"
     t.string   "address1"
+    t.string   "address2"
+    t.string   "zip_code"
+    t.string   "state"
+    t.string   "state_short"
+    t.string   "city"
     t.string   "lat"
+    t.string   "lon"
+    t.string   "username"
+    t.boolean  "active"
+    t.string   "school"
   end
 
 end

data/spec/lib/acts_as_scrubbable/scrub_spec.rb

@@ -5,9 +5,35 @@ RSpec.describe ActsAsScrubbable::Scrub do
   describe '.scrub' do
 
     # update_columns cannot be run on a new record
-    subject{ ScrubbableModel.new }
+    subject { ScrubbableModel.new }
     before(:each) { subject.save }
 
+    it 'scrubs all columns' do
+      subject.attributes = {
+        first_name: "Ted",
+        last_name: "Lowe",
+        middle_name: "Cassidy",
+        name: "Miss Vincenzo Smitham",
+        email: "trentdibbert@wiza.com",
+        title: "Internal Consultant",
+        company_name: "Greenfelder, Collier and Lesch",
+        address1: "86780 Watsica Flats",
+        address2: "Apt. 913",
+        zip_code: "49227",
+        state: "Ohio",
+        state_short: "OH",
+        city: "Port Hildegard",
+        lat: -79.5855309778974,
+        lon: 13.517352691513906,
+        username: "oscar.hermann",
+        active: false,
+        school: "Eastern Lebsack",
+      }
+      expect {
+        subject.scrub!
+      }.not_to raise_error
+    end
+
     it 'changes the first_name attribute when scrub is run' do
       subject.first_name = "Ted"
       allow(Faker::Name).to receive(:first_name).and_return("John")

data/spec/support/database.rb

@@ -1,7 +1,7 @@
 require 'nulldb/rails'
 require 'nulldb_rspec'
 
-ActiveRecord::Base.configurations = {"test" => {adapter: :nulldb}}
+ActiveRecord::Base.configurations.merge!("test" => {adapter: 'nulldb'})
 
 NullDB.configure do |c|
   c.project_root = './spec'
@@ -15,7 +15,24 @@ end
 class NonScrubbableModel < ActiveRecord::Base; end
 
 class ScrubbableModel < ActiveRecord::Base
-  acts_as_scrubbable :first_name, :address1 => :street_address, :lat => :latitude
+  acts_as_scrubbable :first_name,
+    :last_name,
+    :middle_name,
+    :name,
+    :email,
+    :company_name,
+    :zip_code,
+    :state,
+    :city,
+    :username,
+    :school,
+    :title => :name_title,
+    :address1 => :street_address,
+    :address2 => :secondary_address,
+    :state_short => :state_abbr,
+    :lat => :latitude,
+    :lon => :longitude,
+    :active => :boolean
   attr_accessor :scrubbing_begun, :scrubbing_finished
   set_callback :scrub, :before do
     self.scrubbing_begun = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: acts_as_scrubbable
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Samer Masry
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-20 00:00:00.000000000 Z
+date: 2021-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -19,7 +19,7 @@ dependencies:
         version: '4.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: '4.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -39,7 +39,7 @@ dependencies:
         version: '4.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -49,7 +49,7 @@ dependencies:
         version: '4.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -59,7 +59,7 @@ dependencies:
         version: '4.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -69,7 +69,7 @@ dependencies:
         version: '4.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: faker
   requirement: !ruby/object:Gem::Requirement
@@ -220,11 +220,13 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".travis.yml"
 - Gemfile
 - Guardfile
 - README.md
 - acts_as_scrubbable.gemspec
 - lib/acts_as_scrubbable.rb
+- lib/acts_as_scrubbable/parallel_table_scrubber.rb
 - lib/acts_as_scrubbable/scrub.rb
 - lib/acts_as_scrubbable/scrubbable.rb
 - lib/acts_as_scrubbable/tasks.rb
@@ -239,7 +241,7 @@ homepage: https://github.com/smasry/acts_as_scrubbable
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -254,9 +256,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Scrubbing data made easy
 test_files: