acts_as_scrubbable 1.1.1 → 1.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/README.md +19 -0
- data/lib/acts_as_scrubbable/ar_class_processor.rb +36 -0
- data/lib/acts_as_scrubbable/base_processor.rb +28 -0
- data/lib/acts_as_scrubbable/import_processor.rb +24 -0
- data/lib/acts_as_scrubbable/parallel_table_scrubber.rb +14 -26
- data/lib/acts_as_scrubbable/scrub.rb +2 -2
- data/lib/acts_as_scrubbable/task_runner.rb +66 -0
- data/lib/acts_as_scrubbable/tasks.rb +12 -117
- data/lib/acts_as_scrubbable/update_processor.rb +18 -0
- data/lib/acts_as_scrubbable/version.rb +1 -1
- data/lib/acts_as_scrubbable.rb +42 -25
- data/spec/lib/acts_as_scrubbable/ar_class_processor_spec.rb +56 -0
- data/spec/lib/acts_as_scrubbable/import_processor_spec.rb +25 -0
- data/spec/lib/acts_as_scrubbable/scrub_spec.rb +8 -8
- data/spec/lib/acts_as_scrubbable/update_processor_spec.rb +19 -0
- data/spec/support/database.rb +4 -0
- metadata +13 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 98b03e06478fe5c46d1be95047d60b28b91f3e21699743940e18ae0ee6ac8075
|
4
|
+
data.tar.gz: 28e6b08a628f43b121b4419b1571466f4810c9ad619849ce8f1b7bdc7700a658
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 74246e4b9ea4209288e2709cf86a0b35e8d5f4abc1f7548c6a5cac4d628fbfc11c1d5a69c61d2de357748e4b6157a107ad7bc34551c3734f9d1630f62b0986e2
|
7
|
+
data.tar.gz: 4c5407c1f0689663ff28562880784b89442f1bce5d0cdb26e9ae0c089d4a0435ac516b692399dd8eb4cf15a115c771545bad2367aa75a8b0e9b727e6dcc43fe6
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -97,3 +97,22 @@ ActsAsScrubbable.configure do |c|
|
|
97
97
|
end
|
98
98
|
end
|
99
99
|
```
|
100
|
+
|
101
|
+
### UPDATE VS UPSERT
|
102
|
+
|
103
|
+
By default, the scrubbing proces will run independent UPDATE statements for each scrubbed model. This can be time
|
104
|
+
consuming if you are scrubbing a large number of records. As an alternative, some
|
105
|
+
databases support doing bulk database updates using an upsert using the INSERT command. `activerecord-import`
|
106
|
+
gives us easy support for this and as such it is a requirement to using
|
107
|
+
this feature. Some details about the implementation can be found here. https://github.com/zdennis/activerecord-import#duplicate-key-update
|
108
|
+
|
109
|
+
Note that we only support the MySQL implementation at this time.
|
110
|
+
|
111
|
+
To use UPSERT over UPDATE, it can be enabled by specifying the environment variable `USE_UPSERT='true'` or through configuration.
|
112
|
+
|
113
|
+
```ruby
|
114
|
+
ActsAsScrubbable.configure do |c|
|
115
|
+
c.use_upsert = true
|
116
|
+
end
|
117
|
+
```
|
118
|
+
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'acts_as_scrubbable/import_processor'
|
2
|
+
require 'acts_as_scrubbable/update_processor'
|
3
|
+
require 'term/ansicolor'
|
4
|
+
|
5
|
+
module ActsAsScrubbable
|
6
|
+
class ArClassProcessor
|
7
|
+
|
8
|
+
attr_reader :ar_class, :query_processor
|
9
|
+
|
10
|
+
def initialize(ar_class)
|
11
|
+
@ar_class = ar_class
|
12
|
+
|
13
|
+
if ActsAsScrubbable.use_upsert
|
14
|
+
ActsAsScrubbable.logger.info Term::ANSIColor.white("Using Upsert")
|
15
|
+
@query_processor = ImportProcessor.new(ar_class)
|
16
|
+
else
|
17
|
+
ActsAsScrubbable.logger.info Term::ANSIColor.white("Using Update")
|
18
|
+
@query_processor = UpdateProcessor.new(ar_class)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def process(num_of_batches)
|
23
|
+
ActsAsScrubbable.logger.info Term::ANSIColor.green("Scrubbing #{ar_class} ...")
|
24
|
+
|
25
|
+
num_of_batches = Integer(ENV.fetch("SCRUB_BATCHES", "256")) if num_of_batches.nil?
|
26
|
+
scrubbed_count = ActsAsScrubbable::ParallelTableScrubber.new(ar_class, num_of_batches).each_query do |query|
|
27
|
+
query_processor.scrub_query(query)
|
28
|
+
end
|
29
|
+
|
30
|
+
ActsAsScrubbable.logger.info Term::ANSIColor.blue("#{scrubbed_count} #{ar_class} objects scrubbed")
|
31
|
+
ActiveRecord::Base.connection.verify!
|
32
|
+
|
33
|
+
ActsAsScrubbable.logger.info Term::ANSIColor.white("Scrub Complete!")
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module ActsAsScrubbable
|
2
|
+
module BaseProcessor
|
3
|
+
attr_reader :ar_class
|
4
|
+
private :ar_class
|
5
|
+
|
6
|
+
def initialize(ar_class)
|
7
|
+
@ar_class = ar_class
|
8
|
+
end
|
9
|
+
|
10
|
+
def scrub_query(query = nil)
|
11
|
+
scrubbed_count = 0
|
12
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
13
|
+
if ar_class.respond_to?(:scrubbable_scope)
|
14
|
+
relation = ar_class.send(:scrubbable_scope)
|
15
|
+
else
|
16
|
+
relation = ar_class.all
|
17
|
+
end
|
18
|
+
|
19
|
+
relation.where(query).find_in_batches(batch_size: 1000) do |batch|
|
20
|
+
ActiveRecord::Base.transaction do
|
21
|
+
scrubbed_count += handle_batch(batch)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
scrubbed_count
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'acts_as_scrubbable/base_processor'
|
2
|
+
|
3
|
+
module ActsAsScrubbable
|
4
|
+
class ImportProcessor
|
5
|
+
include BaseProcessor
|
6
|
+
|
7
|
+
private
|
8
|
+
def handle_batch(batch)
|
9
|
+
scrubbed_count = 0
|
10
|
+
batch.each do |obj|
|
11
|
+
_updates = obj.scrubbed_values
|
12
|
+
obj.assign_attributes(_updates)
|
13
|
+
scrubbed_count += 1
|
14
|
+
end
|
15
|
+
ar_class.import(
|
16
|
+
batch,
|
17
|
+
on_duplicate_key_update: ar_class.scrubbable_fields.keys.map { |x| "`#{x}` = values(`#{x}`)" }.join(" , "),
|
18
|
+
validate: false,
|
19
|
+
timestamps: false
|
20
|
+
)
|
21
|
+
scrubbed_count
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -2,40 +2,28 @@ require "parallel"
|
|
2
2
|
|
3
3
|
module ActsAsScrubbable
|
4
4
|
class ParallelTableScrubber
|
5
|
-
|
5
|
+
attr_reader :ar_class, :num_of_batches
|
6
|
+
private :ar_class, :num_of_batches
|
7
|
+
|
8
|
+
def initialize(ar_class, num_of_batches)
|
6
9
|
@ar_class = ar_class
|
10
|
+
@num_of_batches = num_of_batches
|
7
11
|
end
|
8
12
|
|
9
|
-
def
|
13
|
+
def each_query
|
10
14
|
# Removing any find or initialize callbacks from model
|
11
15
|
ar_class.reset_callbacks(:initialize)
|
12
16
|
ar_class.reset_callbacks(:find)
|
13
17
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
ActiveRecord::Base.connection_pool.with_connection do
|
18
|
-
relation = ar_class
|
19
|
-
relation = relation.send(:scrubbable_scope) if ar_class.respond_to?(:scrubbable_scope)
|
20
|
-
relation.where(query).find_in_batches(batch_size: 1000) do |batch|
|
21
|
-
ActiveRecord::Base.transaction do
|
22
|
-
batch.each do |obj|
|
23
|
-
obj.scrub!
|
24
|
-
scrubbed_count += 1
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
scrubbed_count
|
30
|
-
}.reduce(:+)
|
18
|
+
Parallel.map(parallel_queries) { |query|
|
19
|
+
yield(query)
|
20
|
+
}.reduce(:+) # returns the aggregated scrub count
|
31
21
|
end
|
32
22
|
|
33
23
|
private
|
34
24
|
|
35
|
-
attr_reader :ar_class
|
36
|
-
|
37
25
|
# create even ID ranges for the table
|
38
|
-
def parallel_queries
|
26
|
+
def parallel_queries
|
39
27
|
raise "Model is missing id column" if ar_class.columns.none? { |column| column.name == "id" }
|
40
28
|
|
41
29
|
if ar_class.respond_to?(:scrubbable_scope)
|
@@ -45,22 +33,22 @@ module ActsAsScrubbable
|
|
45
33
|
end
|
46
34
|
return [] if num_records == 0 # no records to import
|
47
35
|
|
48
|
-
record_window_size, modulus = num_records.divmod(
|
36
|
+
record_window_size, modulus = num_records.divmod(num_of_batches)
|
49
37
|
if record_window_size < 1
|
50
38
|
record_window_size = 1
|
51
39
|
modulus = 0
|
52
40
|
end
|
53
41
|
|
54
42
|
start_id = next_id(ar_class: ar_class, offset: 0)
|
55
|
-
queries =
|
43
|
+
queries = num_of_batches.times.each_with_object([]) do |_, queries|
|
56
44
|
next unless start_id
|
57
45
|
|
58
|
-
end_id = next_id(ar_class: ar_class, id: start_id, offset: record_window_size-1)
|
46
|
+
end_id = next_id(ar_class: ar_class, id: start_id, offset: record_window_size - 1)
|
59
47
|
if modulus > 0
|
60
48
|
end_id = next_id(ar_class: ar_class, id: end_id)
|
61
49
|
modulus -= 1
|
62
50
|
end
|
63
|
-
queries << {id: start_id..end_id} if end_id
|
51
|
+
queries << { id: start_id..end_id } if end_id
|
64
52
|
start_id = next_id(ar_class: ar_class, id: end_id || start_id)
|
65
53
|
end
|
66
54
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
module ActsAsScrubbable
|
2
2
|
module Scrub
|
3
3
|
|
4
|
-
def
|
4
|
+
def scrubbed_values
|
5
5
|
return unless self.class.scrubbable?
|
6
6
|
|
7
7
|
run_callbacks(:scrub) do
|
@@ -20,7 +20,7 @@ module ActsAsScrubbable
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
|
23
|
+
_updates
|
24
24
|
end
|
25
25
|
end
|
26
26
|
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'acts_as_scrubbable/parallel_table_scrubber'
|
2
|
+
require 'highline/import'
|
3
|
+
require 'acts_as_scrubbable/ar_class_processor'
|
4
|
+
require 'term/ansicolor'
|
5
|
+
|
6
|
+
module ActsAsScrubbable
|
7
|
+
class TaskRunner
|
8
|
+
attr_reader :ar_classes
|
9
|
+
private :ar_classes
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@ar_classes = []
|
13
|
+
end
|
14
|
+
|
15
|
+
def prompt_db_configuration
|
16
|
+
db_host = ActiveRecord::Base.connection_config[:host]
|
17
|
+
db_name = ActiveRecord::Base.connection_config[:database]
|
18
|
+
|
19
|
+
ActsAsScrubbable.logger.warn Term::ANSIColor.red("Please verify the information below to continue")
|
20
|
+
ActsAsScrubbable.logger.warn Term::ANSIColor.red("Host: ") + Term::ANSIColor.white(" #{db_host}")
|
21
|
+
ActsAsScrubbable.logger.warn Term::ANSIColor.red("Database: ") + Term::ANSIColor.white("#{db_name}")
|
22
|
+
end
|
23
|
+
|
24
|
+
def confirmed_configuration?
|
25
|
+
db_host = ActiveRecord::Base.connection_config[:host]
|
26
|
+
|
27
|
+
unless ENV["SKIP_CONFIRM"] == "true"
|
28
|
+
answer = ask("Type '#{db_host}' to continue. \n".red + '-> '.white)
|
29
|
+
unless answer == db_host
|
30
|
+
ActsAsScrubbable.logger.error Term::ANSIColor.red("exiting ...")
|
31
|
+
return false
|
32
|
+
end
|
33
|
+
end
|
34
|
+
true
|
35
|
+
end
|
36
|
+
|
37
|
+
def extract_ar_classes
|
38
|
+
Rails.application.eager_load! # make sure all the classes are loaded
|
39
|
+
@ar_classes = ActiveRecord::Base.descendants.select { |d| d.scrubbable? }.sort_by { |d| d.to_s }
|
40
|
+
|
41
|
+
if ENV["SCRUB_CLASSES"].present?
|
42
|
+
class_list = ENV["SCRUB_CLASSES"].split(",")
|
43
|
+
class_list = class_list.map { |_class_str| _class_str.constantize }
|
44
|
+
@ar_classes = ar_classes & class_list
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def set_ar_class(ar_class)
|
49
|
+
ar_classes << ar_class
|
50
|
+
end
|
51
|
+
|
52
|
+
def scrub(num_of_batches: nil)
|
53
|
+
Parallel.each(ar_classes) do |ar_class|
|
54
|
+
ActsAsScrubbable::ArClassProcessor.new(ar_class).process(num_of_batches)
|
55
|
+
end
|
56
|
+
ActiveRecord::Base.connection.verify!
|
57
|
+
end
|
58
|
+
|
59
|
+
def after_hooks
|
60
|
+
if ENV["SKIP_AFTERHOOK"].blank?
|
61
|
+
ActsAsScrubbable.logger.info Term::ANSIColor.red("Running after hook")
|
62
|
+
ActsAsScrubbable.execute_after_hook
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -1,130 +1,25 @@
|
|
1
|
-
|
2
1
|
require 'rake'
|
2
|
+
require 'acts_as_scrubbable/task_runner'
|
3
3
|
|
4
4
|
namespace :scrub do
|
5
5
|
|
6
6
|
desc "scrub all scrubbable tables"
|
7
7
|
task all: :environment do
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
logger = Logger.new($stdout)
|
16
|
-
logger.formatter = proc do |severity, datetime, progname, msg|
|
17
|
-
"#{datetime}: [#{severity}] - #{msg}\n"
|
18
|
-
end
|
19
|
-
|
20
|
-
db_host = ActiveRecord::Base.connection_config[:host]
|
21
|
-
db_name = ActiveRecord::Base.connection_config[:database]
|
22
|
-
|
23
|
-
logger.warn "Please verify the information below to continue".red
|
24
|
-
logger.warn "Host: ".red + " #{db_host}".white
|
25
|
-
logger.warn "Database: ".red + "#{db_name}".white
|
26
|
-
|
27
|
-
unless ENV["SKIP_CONFIRM"] == "true"
|
28
|
-
answer = ask("Type '#{db_host}' to continue. \n".red + '-> '.white)
|
29
|
-
unless answer == db_host
|
30
|
-
logger.error "exiting ...".red
|
31
|
-
exit
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
logger.warn "Scrubbing classes".red
|
36
|
-
|
37
|
-
Rails.application.eager_load! # make sure all the classes are loaded
|
38
|
-
|
39
|
-
ar_classes = ActiveRecord::Base.descendants.select{|d| d.scrubbable? }.sort_by{|d| d.to_s }
|
40
|
-
|
41
|
-
if ENV["SCRUB_CLASSES"].present?
|
42
|
-
class_list = ENV["SCRUB_CLASSES"].split(",")
|
43
|
-
class_list = class_list.map {|_class_str| _class_str.constantize }
|
44
|
-
ar_classes = ar_classes & class_list
|
45
|
-
end
|
46
|
-
|
47
|
-
logger.info "Scrubbable Classes: #{ar_classes.join(', ')}".white
|
48
|
-
|
49
|
-
Parallel.each(ar_classes) do |ar_class|
|
50
|
-
# Removing any find or initialize callbacks from model
|
51
|
-
ar_class.reset_callbacks(:initialize)
|
52
|
-
ar_class.reset_callbacks(:find)
|
53
|
-
|
54
|
-
logger.info "Scrubbing #{ar_class} ...".green
|
55
|
-
|
56
|
-
scrubbed_count = 0
|
57
|
-
|
58
|
-
ActiveRecord::Base.connection_pool.with_connection do
|
59
|
-
if ar_class.respond_to?(:scrubbable_scope)
|
60
|
-
relation = ar_class.send(:scrubbable_scope)
|
61
|
-
else
|
62
|
-
relation = ar_class.all
|
63
|
-
end
|
64
|
-
|
65
|
-
relation.find_in_batches(batch_size: 1000) do |batch|
|
66
|
-
ActiveRecord::Base.transaction do
|
67
|
-
batch.each do |obj|
|
68
|
-
obj.scrub!
|
69
|
-
scrubbed_count += 1
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
logger.info "#{scrubbed_count} #{ar_class} objects scrubbed".blue
|
76
|
-
end
|
77
|
-
ActiveRecord::Base.connection.verify!
|
78
|
-
|
79
|
-
if ENV["SKIP_AFTERHOOK"].blank?
|
80
|
-
logger.info "Running after hook".red
|
81
|
-
ActsAsScrubbable.execute_after_hook
|
82
|
-
end
|
83
|
-
|
84
|
-
logger.info "Scrub Complete!".white
|
8
|
+
task_runner = ActsAsScrubbable::TaskRunner.new
|
9
|
+
task_runner.prompt_db_configuration
|
10
|
+
exit unless task_runner.confirmed_configuration?
|
11
|
+
task_runner.extract_ar_classes
|
12
|
+
task_runner.scrub(num_of_batches: 1)
|
13
|
+
task_runner.after_hooks
|
85
14
|
end
|
86
15
|
|
87
16
|
desc "Scrub one table"
|
88
17
|
task :model, [:ar_class] => :environment do |_, args|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
include Term::ANSIColor
|
95
|
-
|
96
|
-
logger = Logger.new($stdout)
|
97
|
-
logger.formatter = proc do |severity, datetime, progname, msg|
|
98
|
-
"#{datetime}: [#{severity}] - #{msg}\n"
|
99
|
-
end
|
100
|
-
|
101
|
-
db_host = ActiveRecord::Base.connection_config[:host]
|
102
|
-
db_name = ActiveRecord::Base.connection_config[:database]
|
103
|
-
|
104
|
-
logger.warn "Please verify the information below to continue".red
|
105
|
-
logger.warn "Host: ".red + " #{db_host}".white
|
106
|
-
logger.warn "Database: ".red + "#{db_name}".white
|
107
|
-
|
108
|
-
unless ENV["SKIP_CONFIRM"] == "true"
|
109
|
-
answer = ask("Type '#{db_host}' to continue. \n".red + '-> '.white)
|
110
|
-
unless answer == db_host
|
111
|
-
logger.error "exiting ...".red
|
112
|
-
exit
|
113
|
-
end
|
114
|
-
end
|
115
|
-
|
116
|
-
Rails.application.eager_load! # make sure all the classes are loaded
|
117
|
-
|
118
|
-
ar_class = args[:ar_class].constantize
|
119
|
-
logger.info "Scrubbing #{ar_class} ...".green
|
120
|
-
|
121
|
-
num_batches = Integer(ENV.fetch("SCRUB_BATCHES", "256"))
|
122
|
-
scrubbed_count = ActsAsScrubbable::ParallelTableScrubber.new(ar_class).scrub(num_batches: num_batches)
|
123
|
-
|
124
|
-
logger.info "#{scrubbed_count} #{ar_class} objects scrubbed".blue
|
125
|
-
ActiveRecord::Base.connection.verify!
|
126
|
-
|
127
|
-
logger.info "Scrub Complete!".white
|
18
|
+
task_runner = ActsAsScrubbable::TaskRunner.new
|
19
|
+
task_runner.prompt_db_configuration
|
20
|
+
exit unless task_runner.confirmed_configuration?
|
21
|
+
task_runner.set_ar_class(args[:ar_class].constantize)
|
22
|
+
task_runner.scrub
|
128
23
|
end
|
129
24
|
end
|
130
25
|
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'acts_as_scrubbable/base_processor'
|
2
|
+
|
3
|
+
module ActsAsScrubbable
|
4
|
+
class UpdateProcessor
|
5
|
+
include BaseProcessor
|
6
|
+
|
7
|
+
private
|
8
|
+
def handle_batch(batch)
|
9
|
+
scrubbed_count = 0
|
10
|
+
batch.each do |obj|
|
11
|
+
_updates = obj.scrubbed_values
|
12
|
+
obj.update_columns(_updates) unless _updates.empty?
|
13
|
+
scrubbed_count += 1
|
14
|
+
end
|
15
|
+
scrubbed_count
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
data/lib/acts_as_scrubbable.rb
CHANGED
@@ -2,59 +2,76 @@ require 'active_record'
|
|
2
2
|
require 'active_record/version'
|
3
3
|
require 'active_support/core_ext/module'
|
4
4
|
require 'acts_as_scrubbable/tasks'
|
5
|
-
|
5
|
+
require 'term/ansicolor'
|
6
|
+
require 'logger'
|
6
7
|
|
7
8
|
module ActsAsScrubbable
|
9
|
+
extend self
|
8
10
|
extend ActiveSupport::Autoload
|
11
|
+
include Term::ANSIColor
|
9
12
|
|
10
13
|
autoload :Scrubbable
|
11
14
|
autoload :Scrub
|
12
15
|
autoload :VERSION
|
13
16
|
|
17
|
+
attr_accessor :use_upsert
|
18
|
+
|
19
|
+
class << self
|
20
|
+
def configure(&block)
|
21
|
+
self.use_upsert = ENV["USE_UPSERT"] == "true"
|
14
22
|
|
15
|
-
|
16
|
-
|
23
|
+
yield self
|
24
|
+
end
|
17
25
|
end
|
18
26
|
|
19
|
-
def
|
27
|
+
def after_hook(&block)
|
20
28
|
@after_hook = block
|
21
29
|
end
|
22
30
|
|
23
|
-
def
|
31
|
+
def execute_after_hook
|
24
32
|
@after_hook.call if @after_hook
|
25
33
|
end
|
26
34
|
|
27
|
-
def
|
35
|
+
def logger
|
36
|
+
@logger ||= begin
|
37
|
+
loggger = Logger.new($stdout)
|
38
|
+
loggger.formatter = proc do |severity, datetime, progname, msg|
|
39
|
+
"#{datetime}: [#{severity}] - #{msg}\n"
|
40
|
+
end
|
41
|
+
loggger
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def add(key, value)
|
28
46
|
ActsAsScrubbable.scrub_map[key] = value
|
29
47
|
end
|
30
48
|
|
31
|
-
def
|
49
|
+
def scrub_map
|
32
50
|
require 'faker'
|
33
51
|
|
34
52
|
@_scrub_map ||= {
|
35
|
-
:first_name
|
36
|
-
:last_name
|
37
|
-
:middle_name
|
38
|
-
:name
|
39
|
-
:email
|
40
|
-
:name_title
|
41
|
-
:company_name
|
42
|
-
:street_address
|
53
|
+
:first_name => -> { Faker::Name.first_name },
|
54
|
+
:last_name => -> { Faker::Name.last_name },
|
55
|
+
:middle_name => -> { Faker::Name.name },
|
56
|
+
:name => -> { Faker::Name.name },
|
57
|
+
:email => -> { Faker::Internet.email },
|
58
|
+
:name_title => -> { defined? Faker::Job ? Faker::Job.title : Faker::Name.title },
|
59
|
+
:company_name => -> { Faker::Company.name },
|
60
|
+
:street_address => -> { Faker::Address.street_address },
|
43
61
|
:secondary_address => -> { Faker::Address.secondary_address },
|
44
|
-
:zip_code
|
45
|
-
:state_abbr
|
46
|
-
:state
|
47
|
-
:city
|
48
|
-
:latitude
|
49
|
-
:longitude
|
50
|
-
:username
|
51
|
-
:boolean
|
52
|
-
:school
|
62
|
+
:zip_code => -> { Faker::Address.zip_code },
|
63
|
+
:state_abbr => -> { Faker::Address.state_abbr },
|
64
|
+
:state => -> { Faker::Address.state },
|
65
|
+
:city => -> { Faker::Address.city },
|
66
|
+
:latitude => -> { Faker::Address.latitude },
|
67
|
+
:longitude => -> { Faker::Address.longitude },
|
68
|
+
:username => -> { Faker::Internet.user_name },
|
69
|
+
:boolean => -> { [true, false].sample },
|
70
|
+
:school => -> { Faker::University.name }
|
53
71
|
}
|
54
72
|
end
|
55
73
|
end
|
56
74
|
|
57
|
-
|
58
75
|
ActiveSupport.on_load(:active_record) do
|
59
76
|
extend ActsAsScrubbable::Scrubbable
|
60
77
|
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe ActsAsScrubbable::ArClassProcessor do
|
4
|
+
let(:ar_class) { ScrubbableModel }
|
5
|
+
|
6
|
+
describe "#initialize" do
|
7
|
+
subject { described_class.new(ar_class) }
|
8
|
+
|
9
|
+
context "with upsert enabled" do
|
10
|
+
before do
|
11
|
+
allow(ActsAsScrubbable).to receive(:use_upsert).and_return(true)
|
12
|
+
end
|
13
|
+
|
14
|
+
it "includes the ImportProcessor module" do
|
15
|
+
expect(subject.query_processor).to be_kind_of(ActsAsScrubbable::ImportProcessor)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
context "without upsert enabled" do
|
20
|
+
it "includes the UpdateProcessor module" do
|
21
|
+
expect(subject.query_processor).to be_kind_of(ActsAsScrubbable::UpdateProcessor)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "#process" do
|
27
|
+
let(:num_of_batches) { nil }
|
28
|
+
let(:query) { nil }
|
29
|
+
let(:parallel_table_scrubber_mock) { instance_double("ParallelTableScrubber") }
|
30
|
+
let(:update_processor_mock) { instance_double("UpdateProcessor", scrub_query: nil) }
|
31
|
+
subject { described_class.new(ar_class) }
|
32
|
+
|
33
|
+
before do
|
34
|
+
allow(ActsAsScrubbable::ParallelTableScrubber).to receive(:new).and_return(parallel_table_scrubber_mock)
|
35
|
+
allow(ActsAsScrubbable::UpdateProcessor).to receive(:new).and_return(update_processor_mock)
|
36
|
+
allow(parallel_table_scrubber_mock).to receive(:each_query).and_yield(query)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "calls the expected helper classes with the expected batch size" do
|
40
|
+
expect(ActiveRecord::Base.connection).to receive(:verify!)
|
41
|
+
expect(update_processor_mock).to receive(:scrub_query).with(query)
|
42
|
+
subject.process(num_of_batches)
|
43
|
+
expect(ActsAsScrubbable::ParallelTableScrubber).to have_received(:new).with(ar_class, 256)
|
44
|
+
end
|
45
|
+
|
46
|
+
context "with an inputted batch size" do
|
47
|
+
let(:num_of_batches) { 10 }
|
48
|
+
|
49
|
+
it "calls ParallelTableScrubber with the passed batch size" do
|
50
|
+
subject.process(num_of_batches)
|
51
|
+
expect(ActsAsScrubbable::ParallelTableScrubber).to have_received(:new).with(ar_class, num_of_batches)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe ActsAsScrubbable::ImportProcessor do
|
4
|
+
let(:ar_class) { ScrubbableModel }
|
5
|
+
let(:model) { ar_class.new }
|
6
|
+
subject { described_class.new(ar_class) }
|
7
|
+
|
8
|
+
before do
|
9
|
+
ar_class.extend(ImportSupport)
|
10
|
+
end
|
11
|
+
|
12
|
+
describe "#handle_batch" do
|
13
|
+
it "calls import with the correct parameters" do
|
14
|
+
expect(model).to receive(:scrubbed_values).and_call_original
|
15
|
+
expect(ar_class).to receive(:import).with(
|
16
|
+
[model],
|
17
|
+
on_duplicate_key_update: "`first_name` = values(`first_name`) , `last_name` = values(`last_name`) , `middle_name` = values(`middle_name`) , `name` = values(`name`) , `email` = values(`email`) , `company_name` = values(`company_name`) , `zip_code` = values(`zip_code`) , `state` = values(`state`) , `city` = values(`city`) , `username` = values(`username`) , `school` = values(`school`) , `title` = values(`title`) , `address1` = values(`address1`) , `address2` = values(`address2`) , `state_short` = values(`state_short`) , `lat` = values(`lat`) , `lon` = values(`lon`) , `active` = values(`active`)",
|
18
|
+
validate: false,
|
19
|
+
timestamps: false
|
20
|
+
)
|
21
|
+
|
22
|
+
expect(subject.send(:handle_batch, [model])).to eq 1
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -30,35 +30,35 @@ RSpec.describe ActsAsScrubbable::Scrub do
|
|
30
30
|
school: "Eastern Lebsack",
|
31
31
|
}
|
32
32
|
expect {
|
33
|
-
subject.
|
33
|
+
subject.scrubbed_values
|
34
34
|
}.not_to raise_error
|
35
35
|
end
|
36
36
|
|
37
37
|
it 'changes the first_name attribute when scrub is run' do
|
38
38
|
subject.first_name = "Ted"
|
39
39
|
allow(Faker::Name).to receive(:first_name).and_return("John")
|
40
|
-
subject.
|
41
|
-
expect(
|
40
|
+
_updates = subject.scrubbed_values
|
41
|
+
expect(_updates[:first_name]).to eq "John"
|
42
42
|
end
|
43
43
|
|
44
44
|
it 'calls street address on faker and updates address1' do
|
45
45
|
subject.address1 = "123 abc"
|
46
46
|
subject.save
|
47
47
|
allow(Faker::Address).to receive(:street_address).and_return("1 Embarcadero")
|
48
|
-
subject.
|
49
|
-
expect(
|
48
|
+
_updates = subject.scrubbed_values
|
49
|
+
expect(_updates[:address1]).to eq "1 Embarcadero"
|
50
50
|
end
|
51
51
|
|
52
52
|
it "doesn't update the field if it's blank" do
|
53
53
|
subject.address1 = nil
|
54
54
|
subject.save
|
55
55
|
allow(Faker::Address).to receive(:street_address).and_return("1 Embarcadero")
|
56
|
-
subject.
|
57
|
-
expect(
|
56
|
+
_updates = subject.scrubbed_values
|
57
|
+
expect(_updates[:address1]).to be_nil
|
58
58
|
end
|
59
59
|
|
60
60
|
it 'runs scrub callbacks' do
|
61
|
-
subject.
|
61
|
+
subject.scrubbed_values
|
62
62
|
expect(subject.scrubbing_begun).to be(true)
|
63
63
|
expect(subject.scrubbing_finished).to be(true)
|
64
64
|
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe ActsAsScrubbable::UpdateProcessor do
|
4
|
+
let(:ar_class) { ScrubbableModel }
|
5
|
+
let(:model) { ar_class.create(
|
6
|
+
first_name: "Ted",
|
7
|
+
last_name: "Lowe",
|
8
|
+
) }
|
9
|
+
subject { described_class.new(ar_class) }
|
10
|
+
|
11
|
+
describe "#handle_batch" do
|
12
|
+
it "calls update with the updated attributes" do
|
13
|
+
expect(model).to receive(:scrubbed_values).and_call_original
|
14
|
+
expect(model).to receive(:update_columns).and_call_original
|
15
|
+
|
16
|
+
expect(subject.send(:handle_batch, [model])).to eq 1
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/spec/support/database.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: acts_as_scrubbable
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Samer Masry
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-08-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -226,15 +226,23 @@ files:
|
|
226
226
|
- README.md
|
227
227
|
- acts_as_scrubbable.gemspec
|
228
228
|
- lib/acts_as_scrubbable.rb
|
229
|
+
- lib/acts_as_scrubbable/ar_class_processor.rb
|
230
|
+
- lib/acts_as_scrubbable/base_processor.rb
|
231
|
+
- lib/acts_as_scrubbable/import_processor.rb
|
229
232
|
- lib/acts_as_scrubbable/parallel_table_scrubber.rb
|
230
233
|
- lib/acts_as_scrubbable/scrub.rb
|
231
234
|
- lib/acts_as_scrubbable/scrubbable.rb
|
235
|
+
- lib/acts_as_scrubbable/task_runner.rb
|
232
236
|
- lib/acts_as_scrubbable/tasks.rb
|
237
|
+
- lib/acts_as_scrubbable/update_processor.rb
|
233
238
|
- lib/acts_as_scrubbable/version.rb
|
234
239
|
- spec/db/database.yml
|
235
240
|
- spec/db/schema.rb
|
241
|
+
- spec/lib/acts_as_scrubbable/ar_class_processor_spec.rb
|
242
|
+
- spec/lib/acts_as_scrubbable/import_processor_spec.rb
|
236
243
|
- spec/lib/acts_as_scrubbable/scrub_spec.rb
|
237
244
|
- spec/lib/acts_as_scrubbable/scrubbable_spec.rb
|
245
|
+
- spec/lib/acts_as_scrubbable/update_processor_spec.rb
|
238
246
|
- spec/spec_helper.rb
|
239
247
|
- spec/support/database.rb
|
240
248
|
homepage: https://github.com/smasry/acts_as_scrubbable
|
@@ -263,7 +271,10 @@ summary: Scrubbing data made easy
|
|
263
271
|
test_files:
|
264
272
|
- spec/db/database.yml
|
265
273
|
- spec/db/schema.rb
|
274
|
+
- spec/lib/acts_as_scrubbable/ar_class_processor_spec.rb
|
275
|
+
- spec/lib/acts_as_scrubbable/import_processor_spec.rb
|
266
276
|
- spec/lib/acts_as_scrubbable/scrub_spec.rb
|
267
277
|
- spec/lib/acts_as_scrubbable/scrubbable_spec.rb
|
278
|
+
- spec/lib/acts_as_scrubbable/update_processor_spec.rb
|
268
279
|
- spec/spec_helper.rb
|
269
280
|
- spec/support/database.rb
|