bulk_dependency_eraser 4.0.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bulk_dependency_eraser/base.rb +10 -10
- data/lib/bulk_dependency_eraser/builder.rb +150 -53
- data/lib/bulk_dependency_eraser/deleter.rb +49 -14
- data/lib/bulk_dependency_eraser/manager.rb +6 -2
- data/lib/bulk_dependency_eraser/nullifier.rb +15 -5
- data/lib/bulk_dependency_eraser/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 10daaabe21acb05d187dcf4c76fae13bd3e3b3aee727be9c6ebd0a315d581b54
|
4
|
+
data.tar.gz: d07c7a1303090c9e9291c19c8b6ec9974f2e9feee050e5962f63522ffd23c623
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2152f55aab950c36636bce7e67507ef6a5a27ee7b553a60727796550dc7e2bb489e1bc364beb47e8bce4adf1e0a778b5d5d7011631a77c8734035f85d77f1ded
|
7
|
+
data.tar.gz: 5889cd52acbcde3642db18e60f5958c189e4ecfc8066b635b33f5dfe5eb55810141244828be8043b1c19b403e773fde73dac302de6eabcf9147008e8f6191d85
|
@@ -128,17 +128,17 @@ module BulkDependencyEraser
|
|
128
128
|
def custom_scope_for_query(query)
|
129
129
|
klass = query.klass
|
130
130
|
if opts_c.proc_scopes_per_class_name.key?(klass.name)
|
131
|
-
opts_c.proc_scopes_per_class_name[klass.name].call(query)
|
132
|
-
|
133
|
-
# See if non-class-mapped proc returns a value
|
134
|
-
non_class_name_mapped_query = opts_c.proc_scopes.call(query)
|
135
|
-
if !non_class_name_mapped_query.nil?
|
136
|
-
return non_class_name_mapped_query
|
137
|
-
else
|
138
|
-
# No custom wrapper, return non-effect default
|
139
|
-
return self.class::DEFAULT_KLASS_MAPPED_SCOPE_WRAPPER.call(query)
|
140
|
-
end
|
131
|
+
result = opts_c.proc_scopes_per_class_name[klass.name].call(query)
|
132
|
+
return result unless result.nil?
|
141
133
|
end
|
134
|
+
|
135
|
+
if opts_c.proc_scopes
|
136
|
+
result = opts_c.proc_scopes.call(query)
|
137
|
+
return result unless result.nil?
|
138
|
+
end
|
139
|
+
|
140
|
+
# Could just return query, but don't wnat to break backwards compatiblity
|
141
|
+
return self.class::DEFAULT_KLASS_MAPPED_SCOPE_WRAPPER.call(query)
|
142
142
|
end
|
143
143
|
|
144
144
|
# Create options container
|
@@ -42,20 +42,26 @@ module BulkDependencyEraser
|
|
42
42
|
disable_read_batching: nil,
|
43
43
|
# Applied to all queries. Useful for taking advantage of specific indexes
|
44
44
|
# - not indexed by klass name. Proc would handle the logic for that.
|
45
|
-
# -
|
45
|
+
# - 4th, and lowest, priority of scopes
|
46
46
|
# - accepts rails query as parameter
|
47
47
|
# - return nil if no applicable scope.
|
48
48
|
proc_scopes: self::DEFAULT_SCOPE_WRAPPER,
|
49
49
|
# Applied to all queries. Useful for taking advantage of specific indexes
|
50
|
-
# -
|
50
|
+
# - 3rd highest priority of scopes
|
51
51
|
proc_scopes_per_class_name: {},
|
52
|
+
# Applied to all queries. Useful for taking advantage of specific indexes
|
53
|
+
# - 2nd highest priority of scopes
|
54
|
+
reading_proc_scopes: self::DEFAULT_SCOPE_WRAPPER,
|
52
55
|
# Applied to reading queries
|
53
56
|
# - 1st priority of scopes
|
54
57
|
reading_proc_scopes_per_class_name: {},
|
58
|
+
# Using PG system column CTID can result in a 10x deletion speed increase
|
59
|
+
# - is used in combination with the primary key column to ensure CTID hasn't changed.
|
60
|
+
use_pg_system_column_ctid: false,
|
55
61
|
}.freeze
|
56
62
|
|
57
63
|
# write access so that these can be edited in-place by end-users who might need to manually adjust deletion order.
|
58
|
-
attr_accessor :deletion_list, :nullification_list
|
64
|
+
attr_accessor :deletion_list, :nullification_list, :deletion_list_with_additional_identifiers
|
59
65
|
attr_reader :ignore_table_deletion_list, :ignore_table_nullification_list
|
60
66
|
attr_reader :query_schema_parser
|
61
67
|
attr_reader :current_klass_name
|
@@ -65,6 +71,15 @@ module BulkDependencyEraser
|
|
65
71
|
def initialize query:, opts: {}
|
66
72
|
@query = query
|
67
73
|
@deletion_list = {}
|
74
|
+
# CTID column values are stored here
|
75
|
+
# - key: <klass_name (with or without suffixes)>
|
76
|
+
# - value: Hash
|
77
|
+
# - key: Individual record's ID
|
78
|
+
# - value: Hash
|
79
|
+
# - key: Column Name
|
80
|
+
# - value: plucked column value
|
81
|
+
# - these additional identifiers will be used when deleting the klasses
|
82
|
+
@deletion_list_with_additional_identifiers = {}
|
68
83
|
@nullification_list = {}
|
69
84
|
|
70
85
|
# For any ignored table results, they will be stored here
|
@@ -95,9 +110,13 @@ module BulkDependencyEraser
|
|
95
110
|
# - prior approach was to use table_name.classify, but we can't trust that approach.
|
96
111
|
opts_c.ignore_tables.each do |table_name|
|
97
112
|
table_names_to_parsed_klass_names.dig(table_name)&.each do |klass_name|
|
98
|
-
|
99
|
-
|
100
|
-
|
113
|
+
klass = klass_name.constantize
|
114
|
+
# Delete klasses from the deletion/nullification lists if they are from ignored tables.
|
115
|
+
# - Handles klass name indexes, with and without suffices.
|
116
|
+
deletion_index_keys(klass).each do |klass_index|
|
117
|
+
ignore_table_deletion_list[klass_index] = deletion_list.delete(klass_index) if deletion_list.key?(klass_index)
|
118
|
+
ignore_table_nullification_list[klass_index] = nullification_list.delete(klass_index) if nullification_list.key?(klass_index)
|
119
|
+
end
|
101
120
|
end
|
102
121
|
end
|
103
122
|
|
@@ -130,34 +149,58 @@ module BulkDependencyEraser
|
|
130
149
|
|
131
150
|
def custom_scope_for_query(query)
|
132
151
|
klass = query.klass
|
152
|
+
|
133
153
|
if opts_c.reading_proc_scopes_per_class_name.key?(klass.name)
|
134
|
-
opts_c.reading_proc_scopes_per_class_name[klass.name].call(query)
|
135
|
-
|
136
|
-
|
154
|
+
result = opts_c.reading_proc_scopes_per_class_name[klass.name].call(query)
|
155
|
+
return result unless result.nil?
|
156
|
+
end
|
157
|
+
|
158
|
+
if opts_c.reading_proc_scopes
|
159
|
+
result = opts_c.reading_proc_scopes.call(query)
|
160
|
+
return result unless result.nil?
|
137
161
|
end
|
162
|
+
|
163
|
+
super(query)
|
138
164
|
end
|
139
165
|
|
140
|
-
|
166
|
+
# @param [ActiveRecord::Relation] query
|
167
|
+
# @param [Symbol | Array<Symbol>] column - primary_key must be the first or only element in the column.
|
168
|
+
# @return [
|
169
|
+
# [Array<String | Int>] array of column values,
|
170
|
+
# [Hash] hash of column/value pairings - additional identifiers.
|
171
|
+
# ]
|
172
|
+
def pluck_from_query(query, column = :id, skip_ctid_check: false)
|
173
|
+
if column.is_a?(Array)
|
174
|
+
columns = column
|
175
|
+
else
|
176
|
+
columns = [column]
|
177
|
+
end
|
178
|
+
|
179
|
+
# Only pluck CTID if we're plucking column :id (the only primary key supported)
|
180
|
+
if columns == [:id] && opts_c.use_pg_system_column_ctid && skip_ctid_check == false
|
181
|
+
columns << :ctid
|
182
|
+
end
|
183
|
+
|
141
184
|
set_current_klass_name(query)
|
142
185
|
# ordering shouldn't matter in these queries, and would slow it down
|
143
186
|
# - we're ignoring default_scope ordering, but assoc-defined ordering would still take effect
|
144
187
|
query = query.reorder('')
|
145
188
|
query = custom_scope_for_query(query)
|
146
189
|
|
147
|
-
|
190
|
+
query_results = []
|
148
191
|
read_from_db do
|
149
192
|
# If the query has a limit, then we don't want to clobber with batching.
|
150
193
|
if batching_disabled? || !query.where({}).limit_value.nil?
|
151
194
|
# query without batching
|
152
|
-
|
195
|
+
query_results = query.pluck(*columns)
|
153
196
|
elsif opts_c.disable_batch_ordering || opts_c.disable_batch_ordering_for_klasses.include?(current_klass_name)
|
154
197
|
# query with orderless batching
|
155
198
|
offset = 0
|
156
199
|
loop do
|
157
|
-
|
158
|
-
|
200
|
+
new_query_results = query.offset(offset).limit(batch_size).pluck(*columns)
|
201
|
+
query_results += new_query_results
|
159
202
|
|
160
|
-
break if
|
203
|
+
break if new_query_results.size < batch_size
|
161
204
|
|
162
205
|
# Move to the next batch
|
163
206
|
offset += batch_size
|
@@ -165,12 +208,35 @@ module BulkDependencyEraser
|
|
165
208
|
else
|
166
209
|
# query with ordered batching
|
167
210
|
query.in_batches(of: batch_size) do |subset_query|
|
168
|
-
|
211
|
+
query_results += subset_query.pluck(*columns)
|
169
212
|
end
|
170
213
|
end
|
171
214
|
end
|
172
215
|
|
173
|
-
|
216
|
+
# Early exit if we only plucked one column.
|
217
|
+
# - query_results would just be an array of IDs
|
218
|
+
return [query_results, nil] if columns.count == 1
|
219
|
+
|
220
|
+
transposed_results = query_results.transpose
|
221
|
+
|
222
|
+
query_primary_keys = transposed_results.shift || [] # could be an empty id set
|
223
|
+
columns.shift # shift of the primary key column
|
224
|
+
|
225
|
+
query_additional_identifiers = {}
|
226
|
+
results_per_additional_column = {}
|
227
|
+
columns.each do |column|
|
228
|
+
column_results = transposed_results.shift
|
229
|
+
results_per_additional_column[column] = column_results
|
230
|
+
end
|
231
|
+
|
232
|
+
query_primary_keys.each_with_index do |primary_key, plucked_index|
|
233
|
+
query_additional_identifiers[primary_key] ||= {}
|
234
|
+
results_per_additional_column.each do |column, column_results|
|
235
|
+
query_additional_identifiers[primary_key][column] = column_results[plucked_index]
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
return query_primary_keys, query_additional_identifiers
|
174
240
|
end
|
175
241
|
|
176
242
|
def batch_size
|
@@ -181,6 +247,32 @@ module BulkDependencyEraser
|
|
181
247
|
opts_c.disable_read_batching.nil? ? opts_c.disable_batching : opts_c.disable_read_batching
|
182
248
|
end
|
183
249
|
|
250
|
+
# No need for recursion here. Nullifying does not effect nested dependencies
|
251
|
+
def nullification_query_parser(query, nullfication_klass, klass_foreign_key, klass_foreign_type = nil)
|
252
|
+
# We're not destroying these assocs (just nullifying foreign_key columns) so we don't need to parse their dependencies.
|
253
|
+
# Setting skip_ctid_check: true because we currently don't support CTID with nullifications yet
|
254
|
+
assoc_ids, additional_identifiers_by_id = pluck_from_query(query, skip_ctid_check: true)
|
255
|
+
klass_name = nullfication_klass.name
|
256
|
+
|
257
|
+
# No assoc_ids, no need to add it to the nullification list
|
258
|
+
return if assoc_ids.none?
|
259
|
+
|
260
|
+
klass_foreign_key = klass_foreign_key.to_s
|
261
|
+
nullification_list[klass_name] ||= {}
|
262
|
+
nullification_list[klass_name][klass_foreign_key] ||= []
|
263
|
+
nullification_list[klass_name][klass_foreign_key] += assoc_ids
|
264
|
+
nullification_list[klass_name][klass_foreign_key].uniq!
|
265
|
+
|
266
|
+
|
267
|
+
# Also nullify the 'type' field, if the association is polymorphic
|
268
|
+
if klass_foreign_type
|
269
|
+
klass_foreign_type = klass_foreign_type.to_s
|
270
|
+
nullification_list[klass_name][klass_foreign_type] ||= []
|
271
|
+
nullification_list[klass_name][klass_foreign_type] += assoc_ids
|
272
|
+
nullification_list[klass_name][klass_foreign_type].uniq!
|
273
|
+
end
|
274
|
+
end
|
275
|
+
|
184
276
|
def deletion_query_parser query, association_parent = nil
|
185
277
|
# necessary for "ActiveRecord::Reflection::ThroughReflection" use-case
|
186
278
|
# force_through_destroy_chains = options[:force_destroy_chain] || {}
|
@@ -225,24 +317,28 @@ module BulkDependencyEraser
|
|
225
317
|
end
|
226
318
|
|
227
319
|
# Pluck IDs of the current query
|
228
|
-
query_ids = pluck_from_query(query)
|
320
|
+
query_ids, additional_identifiers_by_id = pluck_from_query(query)
|
229
321
|
|
230
322
|
klass_index = initialize_deletion_list_for_klass(klass)
|
231
323
|
|
232
324
|
# prevent infinite recursion here.
|
233
325
|
# - Remove any IDs that have been processed before
|
234
|
-
query_ids = remove_already_deletion_processed_ids(klass, query_ids)
|
326
|
+
query_ids, additional_identifiers_by_id = remove_already_deletion_processed_ids(klass, query_ids, additional_identifiers_by_id:)
|
235
327
|
|
236
328
|
# If ids are nil, let's find that error
|
237
329
|
if query_ids.none? #|| query_ids.nil?
|
238
330
|
# quick cleanup, if turns out was an empty class
|
239
|
-
|
331
|
+
if deletion_list[klass_index].none?
|
332
|
+
deletion_list.delete(klass_index)
|
333
|
+
deletion_list_with_additional_identifiers.delete(klass_index)
|
334
|
+
end
|
240
335
|
return
|
241
336
|
end
|
242
337
|
|
243
338
|
# Use-case: We have more IDs to process
|
244
339
|
# - can now safely add to the list, since we've prevented infinite recursion
|
245
340
|
deletion_list[klass_index] += query_ids
|
341
|
+
deletion_list_with_additional_identifiers[klass_index].merge!(additional_identifiers_by_id)
|
246
342
|
|
247
343
|
# ignore associations that aren't a dependent destroyable type
|
248
344
|
destroy_associations = query.reflect_on_all_associations.select do |reflection|
|
@@ -378,25 +474,25 @@ module BulkDependencyEraser
|
|
378
474
|
end
|
379
475
|
|
380
476
|
# Look for manually specified keys in the assocation first
|
381
|
-
specified_primary_key = reflection.options[:primary_key]&.
|
382
|
-
specified_foreign_key = reflection.options[:foreign_key]&.
|
477
|
+
specified_primary_key = reflection.options[:primary_key]&.to_sym
|
478
|
+
specified_foreign_key = reflection.options[:foreign_key]&.to_sym
|
383
479
|
# For polymorphic_associations
|
384
480
|
specified_foreign_type = nil
|
385
481
|
|
386
482
|
# handle foreign_key edge cases
|
387
483
|
if specified_foreign_key.nil?
|
388
484
|
if reflection.options[:as]
|
389
|
-
specified_foreign_type = "#{reflection.options[:as]}_type"
|
390
|
-
specified_foreign_key = "#{reflection.options[:as]}_id"
|
485
|
+
specified_foreign_type = "#{reflection.options[:as]}_type".to_sym
|
486
|
+
specified_foreign_key = "#{reflection.options[:as]}_id".to_sym
|
391
487
|
# Only filtering by type here, the extra work for a poly assoc. We filter by IDs later
|
392
|
-
assoc_query = assoc_query.where({ specified_foreign_type
|
488
|
+
assoc_query = assoc_query.where({ specified_foreign_type => parent_class.name })
|
393
489
|
else
|
394
|
-
specified_foreign_key = parent_class.table_name.singularize
|
490
|
+
specified_foreign_key = "#{parent_class.table_name.singularize}_id".to_sym
|
395
491
|
end
|
396
492
|
end
|
397
493
|
|
398
494
|
# Check to see if foreign_key exists in association class's table
|
399
|
-
unless assoc_klass.column_names.include?(specified_foreign_key)
|
495
|
+
unless assoc_klass.column_names.include?(specified_foreign_key.to_s)
|
400
496
|
report_error(
|
401
497
|
"
|
402
498
|
For #{parent_class.name}'s assoc '#{assoc_klass.name}': Could not determine the assoc's foreign key.
|
@@ -409,11 +505,11 @@ module BulkDependencyEraser
|
|
409
505
|
# Build association query, based on parent class's primary key and the assoc's foreign key
|
410
506
|
# - handle primary key edge cases
|
411
507
|
# - The associations might not be using the primary_key of the klass table, but we can support that here.
|
412
|
-
if specified_primary_key && specified_primary_key
|
413
|
-
alt_primary_ids = pluck_from_query(query, specified_primary_key)
|
414
|
-
assoc_query = assoc_query.where(specified_foreign_key
|
508
|
+
if specified_primary_key && specified_primary_key != :id
|
509
|
+
alt_primary_ids, _ = pluck_from_query(query, specified_primary_key)
|
510
|
+
assoc_query = assoc_query.where(specified_foreign_key => alt_primary_ids)
|
415
511
|
else
|
416
|
-
assoc_query = assoc_query.where(specified_foreign_key
|
512
|
+
assoc_query = assoc_query.where(specified_foreign_key => query_ids)
|
417
513
|
end
|
418
514
|
|
419
515
|
# remove any ordering or limits imposed on the association queries from the association definitions
|
@@ -427,25 +523,7 @@ module BulkDependencyEraser
|
|
427
523
|
deletion_query_parser(assoc_query, parent_class)
|
428
524
|
end
|
429
525
|
elsif type == :nullify
|
430
|
-
|
431
|
-
# - we're not destroying these assocs (just nullifying foreign_key columns) so we don't need to parse their dependencies.
|
432
|
-
assoc_ids = pluck_from_query(assoc_query)
|
433
|
-
|
434
|
-
# No assoc_ids, no need to add it to the nullification list
|
435
|
-
return if assoc_ids.none?
|
436
|
-
|
437
|
-
nullification_list[assoc_klass_name] ||= {}
|
438
|
-
nullification_list[assoc_klass_name][specified_foreign_key] ||= []
|
439
|
-
nullification_list[assoc_klass_name][specified_foreign_key] += assoc_ids
|
440
|
-
nullification_list[assoc_klass_name][specified_foreign_key].uniq!
|
441
|
-
|
442
|
-
|
443
|
-
# Also nullify the 'type' field, if the association is polymorphic
|
444
|
-
if specified_foreign_type
|
445
|
-
nullification_list[assoc_klass_name][specified_foreign_type] ||= []
|
446
|
-
nullification_list[assoc_klass_name][specified_foreign_type] += assoc_ids
|
447
|
-
nullification_list[assoc_klass_name][specified_foreign_type].uniq!
|
448
|
-
end
|
526
|
+
nullification_query_parser(assoc_query, assoc_klass, specified_foreign_key, specified_foreign_type)
|
449
527
|
else
|
450
528
|
raise "invalid parsing type: #{type}"
|
451
529
|
end
|
@@ -531,7 +609,7 @@ module BulkDependencyEraser
|
|
531
609
|
return
|
532
610
|
end
|
533
611
|
|
534
|
-
foreign_keys = pluck_from_query(query, specified_foreign_key)
|
612
|
+
foreign_keys, _ = pluck_from_query(query, specified_foreign_key, skip_ctid_check: true)
|
535
613
|
assoc_query = assoc_query.where(
|
536
614
|
specified_primary_key.to_sym => foreign_keys
|
537
615
|
)
|
@@ -690,8 +768,14 @@ module BulkDependencyEraser
|
|
690
768
|
opts_c.db_read_wrapper.call(block)
|
691
769
|
end
|
692
770
|
|
693
|
-
|
771
|
+
# @return [
|
772
|
+
# [Array] array of IDs that haven't already been added to the deletion list
|
773
|
+
# [Hash | Nil] the additional deletion query identifiers, minus the ones already processed
|
774
|
+
# - will be nil if we're not plucking additional columns.
|
775
|
+
# ]
|
776
|
+
def remove_already_deletion_processed_ids(klass, new_ids, additional_identifiers_by_id: nil)
|
694
777
|
already_processed_ids = []
|
778
|
+
additional_identifiers_by_id ||= {}
|
695
779
|
|
696
780
|
if is_a_circular_dependency_klass?(klass)
|
697
781
|
klass_keys = find_circular_dependency_deletion_keys(klass)
|
@@ -702,7 +786,10 @@ module BulkDependencyEraser
|
|
702
786
|
already_processed_ids = deletion_list[klass.name]
|
703
787
|
end
|
704
788
|
|
705
|
-
|
789
|
+
return [
|
790
|
+
(new_ids - already_processed_ids),
|
791
|
+
additional_identifiers_by_id.except(*already_processed_ids)
|
792
|
+
]
|
706
793
|
end
|
707
794
|
|
708
795
|
# Initializes deletion_list index
|
@@ -714,9 +801,11 @@ module BulkDependencyEraser
|
|
714
801
|
raise "circular_index already existed for klass: #{klass.name}" if deletion_list.key?(klass_index)
|
715
802
|
|
716
803
|
deletion_list[klass_index] = []
|
804
|
+
deletion_list_with_additional_identifiers[klass_index] = {}
|
717
805
|
else
|
718
806
|
# Not a circular dependency, define as normal
|
719
807
|
deletion_list[klass_index] ||= []
|
808
|
+
deletion_list_with_additional_identifiers[klass_index] ||= {}
|
720
809
|
end
|
721
810
|
|
722
811
|
klass_index
|
@@ -734,6 +823,14 @@ module BulkDependencyEraser
|
|
734
823
|
deletion_list.keys.select { |key| key.match?(regex) }
|
735
824
|
end
|
736
825
|
|
826
|
+
# returns any current index keys, with or without suffix, in the deletion_list for the given klass param
|
827
|
+
def deletion_index_keys(klass)
|
828
|
+
klass_keys = []
|
829
|
+
klass_keys << klass.name if deletion_list.keys.include?(klass.name)
|
830
|
+
klass_keys += find_circular_dependency_deletion_keys(klass)
|
831
|
+
return klass_keys
|
832
|
+
end
|
833
|
+
|
737
834
|
# If circular dependency, append a index suffix to the deletion hash key
|
738
835
|
# - they will be deleted in highest index to lowest index order.
|
739
836
|
def deletion_index_key(klass, increment_circular_index: false)
|
@@ -41,20 +41,24 @@ module BulkDependencyEraser
|
|
41
41
|
disable_delete_batching: nil,
|
42
42
|
# Applied to all queries. Useful for taking advantage of specific indexes
|
43
43
|
# - not indexed by klass name. Proc would handle the logic for that.
|
44
|
-
# -
|
44
|
+
# - 4th, and lowest, priority of scopes
|
45
45
|
# - accepts rails query as parameter
|
46
46
|
# - return nil if no applicable scope.
|
47
47
|
proc_scopes: self::DEFAULT_SCOPE_WRAPPER,
|
48
48
|
# Applied to all queries. Useful for taking advantage of specific indexes
|
49
|
-
# -
|
49
|
+
# - 3rd highest priority of scopes
|
50
50
|
proc_scopes_per_class_name: {},
|
51
|
+
# Applied to all queries. Useful for taking advantage of specific indexes
|
52
|
+
# - 2nd highest priority of scopes
|
53
|
+
deletion_proc_scopes: self::DEFAULT_SCOPE_WRAPPER,
|
51
54
|
# Applied to deletion queries
|
52
55
|
# - 1st priority of scopes
|
53
56
|
deletion_proc_scopes_per_class_name: {},
|
54
57
|
}.freeze
|
55
58
|
|
56
|
-
def initialize class_names_and_ids: {}, opts: {}
|
59
|
+
def initialize class_names_and_ids: {}, additional_identifiers_by_id: {}, opts: {}
|
57
60
|
@class_names_and_ids = class_names_and_ids
|
61
|
+
@additional_identifiers_by_id = additional_identifiers_by_id
|
58
62
|
super(opts:)
|
59
63
|
end
|
60
64
|
|
@@ -68,19 +72,23 @@ module BulkDependencyEraser
|
|
68
72
|
begin
|
69
73
|
class_names_and_ids.keys.reverse.each do |class_name|
|
70
74
|
current_class_name = class_name
|
75
|
+
additional_identifiers = additional_identifiers_by_id[class_name]
|
76
|
+
raise "invalid state! #{class_name} not found in 'additional_identifiers_by_id' list" if additional_identifiers.nil?
|
77
|
+
|
78
|
+
# Last in, First out
|
71
79
|
ids = class_names_and_ids[class_name].reverse
|
72
80
|
klass = constantize(class_name)
|
73
81
|
|
74
82
|
if opts_c.enable_invalid_foreign_key_detection
|
75
83
|
# delete with referential integrity
|
76
|
-
delete_by_klass_and_ids(klass, ids)
|
84
|
+
delete_by_klass_and_ids(klass, ids, additional_identifiers:)
|
77
85
|
else
|
78
86
|
# delete without referential integrity
|
79
87
|
# Disable any ActiveRecord::InvalidForeignKey raised errors.
|
80
88
|
# - src: https://stackoverflow.com/questions/41005849/rails-migrations-temporarily-ignore-foreign-key-constraint
|
81
89
|
# https://apidock.com/rails/ActiveRecord/ConnectionAdapters/AbstractAdapter/disable_referential_integrity
|
82
90
|
ActiveRecord::Base.connection.disable_referential_integrity do
|
83
|
-
delete_by_klass_and_ids(klass, ids)
|
91
|
+
delete_by_klass_and_ids(klass, ids, additional_identifiers:)
|
84
92
|
end
|
85
93
|
end
|
86
94
|
end
|
@@ -94,15 +102,22 @@ module BulkDependencyEraser
|
|
94
102
|
|
95
103
|
protected
|
96
104
|
|
97
|
-
attr_reader :class_names_and_ids
|
105
|
+
attr_reader :class_names_and_ids, :additional_identifiers_by_id
|
98
106
|
|
99
107
|
def custom_scope_for_query(query)
|
100
108
|
klass = query.klass
|
109
|
+
|
101
110
|
if opts_c.deletion_proc_scopes_per_class_name.key?(klass.name)
|
102
|
-
opts_c.deletion_proc_scopes_per_class_name[klass.name].call(query)
|
103
|
-
|
104
|
-
|
111
|
+
result = opts_c.deletion_proc_scopes_per_class_name[klass.name].call(query)
|
112
|
+
return result unless result.nil?
|
113
|
+
end
|
114
|
+
|
115
|
+
if opts_c.deletion_proc_scopes
|
116
|
+
result = opts_c.deletion_proc_scopes.call(query)
|
117
|
+
return result unless result.nil?
|
105
118
|
end
|
119
|
+
|
120
|
+
return super(query)
|
106
121
|
end
|
107
122
|
|
108
123
|
def batch_size
|
@@ -113,29 +128,49 @@ module BulkDependencyEraser
|
|
113
128
|
opts_c.disable_delete_batching.nil? ? opts_c.disable_batching : opts_c.disable_delete_batching
|
114
129
|
end
|
115
130
|
|
116
|
-
def delete_by_klass_and_ids
|
131
|
+
def delete_by_klass_and_ids(klass, ids, additional_identifiers:)
|
117
132
|
puts "Deleting #{klass.name}'s IDs: #{ids}" if opts_c.verbose
|
118
133
|
query = klass.unscoped
|
119
134
|
query = custom_scope_for_query(query)
|
120
135
|
|
121
136
|
if batching_disabled?
|
122
137
|
puts "Deleting without batching" if opts_c.verbose
|
138
|
+
# Get column-names/keys of any additional identifer columns
|
139
|
+
detected_additional_identifier_columns = additional_identifiers.values.flat_map(&:keys).uniq
|
123
140
|
delete_in_db do
|
124
|
-
|
141
|
+
deletion_query = query.where(id: ids)
|
142
|
+
# Apply any additional query identifiers (i.e. :ctid column)
|
143
|
+
detected_additional_identifier_columns.each do |column|
|
144
|
+
deletion_query = deletion_query.where(column => additional_identifiers.values.pluck(column))
|
145
|
+
end
|
146
|
+
|
147
|
+
# Perform Deletion
|
148
|
+
deletion_result = deletion_query.delete_all
|
125
149
|
# Returning the following data in the event that the gem-implementer wants to insert their own db_delete_wrapper proc
|
126
150
|
# and have access to these objects in their proc.
|
127
151
|
# - query can give them access to the klass and table_name
|
128
|
-
[deletion_result, query, ids]
|
152
|
+
[deletion_result, query, ids, additional_identifiers]
|
129
153
|
end
|
130
154
|
else
|
131
155
|
puts "Deleting with batching" if opts_c.verbose
|
132
156
|
ids.each_slice(batch_size) do |ids_subset|
|
157
|
+
additional_identifiers_subset = additional_identifiers.slice(*ids_subset)
|
158
|
+
# Get column-names/keys of any additional identifer columns
|
159
|
+
detected_additional_identifier_columns = additional_identifiers_subset.values.flat_map(&:keys).uniq
|
133
160
|
delete_in_db do
|
134
|
-
|
161
|
+
deletion_query = query.where(id: ids_subset)
|
162
|
+
# Apply any additional query identifiers (i.e. :ctid column)
|
163
|
+
detected_additional_identifier_columns.each do |column|
|
164
|
+
deletion_query = deletion_query.where(column => additional_identifiers_subset.values.pluck(column))
|
165
|
+
end
|
166
|
+
|
167
|
+
# Perform Deletion
|
168
|
+
deletion_result = deletion_query.delete_all
|
169
|
+
|
135
170
|
# Returning the following data in the event that the gem-implementer wants to insert their own db_delete_wrapper proc
|
136
171
|
# and have access to these objects in their proc.
|
137
172
|
# - query can give them access to the klass and table_name
|
138
|
-
[deletion_result, query, ids_subset]
|
173
|
+
[deletion_result, query, ids_subset, additional_identifiers_subset]
|
139
174
|
end
|
140
175
|
end
|
141
176
|
end
|
@@ -7,7 +7,7 @@ module BulkDependencyEraser
|
|
7
7
|
verbose: false,
|
8
8
|
}.freeze
|
9
9
|
|
10
|
-
delegate :nullification_list, :deletion_list, to: :dependency_builder
|
10
|
+
delegate :nullification_list, :deletion_list, :deletion_list_with_additional_identifiers, to: :dependency_builder
|
11
11
|
delegate :ignore_table_deletion_list, :ignore_table_nullification_list, to: :dependency_builder
|
12
12
|
delegate :circular_dependency_klasses, :flat_dependencies_per_klass, to: :dependency_builder
|
13
13
|
|
@@ -49,7 +49,11 @@ module BulkDependencyEraser
|
|
49
49
|
end
|
50
50
|
|
51
51
|
def delete!
|
52
|
-
@deleter = BulkDependencyEraser::Deleter.new(
|
52
|
+
@deleter = BulkDependencyEraser::Deleter.new(
|
53
|
+
class_names_and_ids: deletion_list,
|
54
|
+
additional_identifiers_by_id: deletion_list_with_additional_identifiers,
|
55
|
+
opts:
|
56
|
+
)
|
53
57
|
deleter_execution = deleter.execute
|
54
58
|
unless deleter_execution
|
55
59
|
puts "Deleter execution FAILED" if opts_c.verbose
|
@@ -43,13 +43,16 @@ module BulkDependencyEraser
|
|
43
43
|
disable_nullify_batching: nil,
|
44
44
|
# Applied to all queries. Useful for taking advantage of specific indexes
|
45
45
|
# - not indexed by klass name. Proc would handle the logic for that.
|
46
|
-
# -
|
46
|
+
# - 4th, and lowest, priority of scopes
|
47
47
|
# - accepts rails query as parameter
|
48
48
|
# - return nil if no applicable scope.
|
49
49
|
proc_scopes: self::DEFAULT_SCOPE_WRAPPER,
|
50
50
|
# Applied to all queries. Useful for taking advantage of specific indexes
|
51
|
-
# -
|
51
|
+
# - 3rd highest priority of scopes
|
52
52
|
proc_scopes_per_class_name: {},
|
53
|
+
# Applied to all queries. Useful for taking advantage of specific indexes
|
54
|
+
# - 2nd highest priority of scopes
|
55
|
+
nullification_proc_scopes: self::DEFAULT_SCOPE_WRAPPER,
|
53
56
|
# Applied to nullification queries
|
54
57
|
# - 1st priority of scopes
|
55
58
|
nullification_proc_scopes_per_class_name: {},
|
@@ -164,11 +167,18 @@ module BulkDependencyEraser
|
|
164
167
|
|
165
168
|
def custom_scope_for_query(query)
|
166
169
|
klass = query.klass
|
170
|
+
|
167
171
|
if opts_c.nullification_proc_scopes_per_class_name.key?(klass.name)
|
168
|
-
opts_c.nullification_proc_scopes_per_class_name[klass.name].call(query)
|
169
|
-
|
170
|
-
|
172
|
+
result = opts_c.nullification_proc_scopes_per_class_name[klass.name].call(query)
|
173
|
+
return result unless result.nil?
|
174
|
+
end
|
175
|
+
|
176
|
+
if opts_c.nullification_proc_scopes
|
177
|
+
result = opts_c.nullification_proc_scopes.call(query)
|
178
|
+
return result unless result.nil?
|
171
179
|
end
|
180
|
+
|
181
|
+
return super(query)
|
172
182
|
end
|
173
183
|
|
174
184
|
def batch_size
|