bulk_dependency_eraser 2.2.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/bulk_dependency_eraser/base.rb +2 -0
- data/lib/bulk_dependency_eraser/builder.rb +54 -27
- data/lib/bulk_dependency_eraser/deleter.rb +24 -15
- data/lib/bulk_dependency_eraser/errors/base_error.rb +5 -0
- data/lib/bulk_dependency_eraser/errors/builder_error.rb +13 -0
- data/lib/bulk_dependency_eraser/errors/deleter_error.rb +13 -0
- data/lib/bulk_dependency_eraser/errors/nullifier_error.rb +14 -0
- data/lib/bulk_dependency_eraser/nullifier.rb +16 -2
- data/lib/bulk_dependency_eraser/version.rb +1 -1
- data/lib/bulk_dependency_eraser.rb +5 -1
- metadata +5 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e215e610f3c90e10cb386499bea48adc2a12a6cc7ce2a35466de12d5dc2dbb1d
|
4
|
+
data.tar.gz: 3c56acb8837e54c3387a6c2754dcab92e88402084f99198ab62c0ee1ea46d45b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c1a1b05c9fd919c87a4947409296d65129f82817722d2de883b5f09e873b0851364348face62dc96a2251bb2aa8c8cba678798e0083681c7fa94f3ad0e1b6df6
|
7
|
+
data.tar.gz: 321cda6cc39a9ef3af66023aca8dbd385d46357ead943908d659880f968f2c7e1d5c9c8035d7f8c1c022b21e505aa25fc07be6a93296c73e73c67e0079061ff3
|
@@ -123,6 +123,8 @@ module BulkDependencyEraser
|
|
123
123
|
end
|
124
124
|
end
|
125
125
|
|
126
|
+
# We're supporting custom query scopes on by klass name.
|
127
|
+
# - apply them here
|
126
128
|
def custom_scope_for_query(query)
|
127
129
|
klass = query.klass
|
128
130
|
if opts_c.proc_scopes_per_class_name.key?(klass.name)
|
@@ -1,8 +1,22 @@
|
|
1
1
|
module BulkDependencyEraser
|
2
2
|
class Builder < Base
|
3
|
+
DEFAULT_DB_BUILD_ALL_WRAPPER = ->(builder, block) do
|
4
|
+
begin
|
5
|
+
block.call
|
6
|
+
rescue StandardError => e
|
7
|
+
builder.report_error(
|
8
|
+
<<~STRING.strip
|
9
|
+
Issue attempting to build deletion query for '#{e.building_klass_name}'
|
10
|
+
=> #{e.original_error_klass.name}: #{e.message}
|
11
|
+
STRING
|
12
|
+
)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
3
16
|
DEFAULT_OPTS = {
|
4
17
|
force_destroy_restricted: false,
|
5
18
|
verbose: false,
|
19
|
+
db_build_all_wrapper: self::DEFAULT_DB_BUILD_ALL_WRAPPER,
|
6
20
|
# Some associations scopes take parameters.
|
7
21
|
# - We would have to instantiate if we wanted to apply that scope filter.
|
8
22
|
instantiate_if_assoc_scope_with_arity: false,
|
@@ -14,6 +28,12 @@ module BulkDependencyEraser
|
|
14
28
|
ignore_tables_and_dependencies: [],
|
15
29
|
ignore_klass_names_and_dependencies: [],
|
16
30
|
disable_batching: false,
|
31
|
+
# Disable model ordering during build batching
|
32
|
+
# - Some tables can be too big to order, and just need to trust the DB order
|
33
|
+
# - Not 100% guaranteed and could leave orphaned records behind.
|
34
|
+
disable_batch_ordering: false,
|
35
|
+
# Same as 'disable_batch_ordering', but only for select classes
|
36
|
+
disable_batch_ordering_for_klasses: [],
|
17
37
|
# a general batching size
|
18
38
|
batch_size: 10_000,
|
19
39
|
# A specific batching size for this class, overrides the batch_size
|
@@ -38,6 +58,7 @@ module BulkDependencyEraser
|
|
38
58
|
attr_accessor :deletion_list, :nullification_list
|
39
59
|
attr_reader :ignore_table_deletion_list, :ignore_table_nullification_list
|
40
60
|
attr_reader :query_schema_parser
|
61
|
+
attr_reader :current_klass_name
|
41
62
|
|
42
63
|
delegate :circular_dependency_klasses, :flat_dependencies_per_klass, to: :query_schema_parser
|
43
64
|
|
@@ -57,6 +78,8 @@ module BulkDependencyEraser
|
|
57
78
|
@ignore_table_name_and_dependencies = opts_c.ignore_tables_and_dependencies.collect { |table_name| table_name }
|
58
79
|
@ignore_klass_name_and_dependencies = opts_c.ignore_klass_names_and_dependencies.collect { |klass_name| klass_name }
|
59
80
|
@query_schema_parser = BulkDependencyEraser::QuerySchemaParser.new(query:, opts:)
|
81
|
+
# Moving pointer, points to the current class that is being queries
|
82
|
+
@current_klass_name = query.is_a?(ActiveRecord::Relation) ? query.klass.name : query.name
|
60
83
|
end
|
61
84
|
|
62
85
|
def execute
|
@@ -82,33 +105,20 @@ module BulkDependencyEraser
|
|
82
105
|
end
|
83
106
|
|
84
107
|
def build
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
108
|
+
build_all_in_db do
|
109
|
+
begin
|
110
|
+
if opts_c.verbose
|
111
|
+
puts "Starting build for #{@query.is_a?(ActiveRecord::Relation) ? @query.klass.name : @query.name}"
|
112
|
+
end
|
89
113
|
|
90
|
-
|
114
|
+
deletion_query_parser(@query)
|
91
115
|
|
92
|
-
|
116
|
+
uniqify_errors!
|
93
117
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
klass = @query.klass
|
98
|
-
else
|
99
|
-
# current_query is a normal rails class
|
100
|
-
klass = @query
|
118
|
+
return errors.none?
|
119
|
+
rescue StandardError => e
|
120
|
+
raise BulkDependencyEraser::Errors::BuilderError.new(e.class, e.message, building_klass_name: current_klass_name)
|
101
121
|
end
|
102
|
-
report_error(
|
103
|
-
"
|
104
|
-
Error Encountered in 'execute' for '#{klass.name}':
|
105
|
-
#{e.class.name}
|
106
|
-
#{e.message}
|
107
|
-
"
|
108
|
-
)
|
109
|
-
raise e
|
110
|
-
|
111
|
-
return false
|
112
122
|
end
|
113
123
|
end
|
114
124
|
|
@@ -127,7 +137,8 @@ module BulkDependencyEraser
|
|
127
137
|
end
|
128
138
|
end
|
129
139
|
|
130
|
-
def pluck_from_query
|
140
|
+
def pluck_from_query(query, column = :id)
|
141
|
+
set_current_klass_name(query)
|
131
142
|
# ordering shouldn't matter in these queries, and would slow it down
|
132
143
|
# - we're ignoring default_scope ordering, but assoc-defined ordering would still take effect
|
133
144
|
query = query.reorder('')
|
@@ -139,8 +150,8 @@ module BulkDependencyEraser
|
|
139
150
|
if batching_disabled? || !query.where({}).limit_value.nil?
|
140
151
|
# query without batching
|
141
152
|
query_ids = query.pluck(column)
|
142
|
-
|
143
|
-
# query with batching
|
153
|
+
elsif opts_c.disable_batch_ordering || opts_c.disable_batch_ordering_for_klasses.include?(current_klass_name)
|
154
|
+
# query with orderless batching
|
144
155
|
offset = 0
|
145
156
|
loop do
|
146
157
|
new_query_ids = query.offset(offset).limit(batch_size).pluck(column)
|
@@ -151,6 +162,11 @@ module BulkDependencyEraser
|
|
151
162
|
# Move to the next batch
|
152
163
|
offset += batch_size
|
153
164
|
end
|
165
|
+
else
|
166
|
+
# query with ordered batching
|
167
|
+
query.in_batches(of: batch_size) do |subset_query|
|
168
|
+
query_ids += subset_query.pluck(column)
|
169
|
+
end
|
154
170
|
end
|
155
171
|
end
|
156
172
|
|
@@ -293,6 +309,7 @@ module BulkDependencyEraser
|
|
293
309
|
is_polymorphic = reflection.options[:polymorphic]
|
294
310
|
unless is_polymorphic
|
295
311
|
klass = reflection.klass
|
312
|
+
set_current_klass_name(reflection.klass)
|
296
313
|
|
297
314
|
if ignore_table_name_and_dependencies.include?(klass.table_name)
|
298
315
|
# Not parsing, table and dependencies ignorable
|
@@ -445,7 +462,6 @@ module BulkDependencyEraser
|
|
445
462
|
assoc_klass = reflection.klass
|
446
463
|
assoc_klass_name = assoc_klass.name
|
447
464
|
|
448
|
-
|
449
465
|
# specified_primary_key = reflection.options[:primary_key]&.to_s
|
450
466
|
# specified_foreign_key = reflection.options[:foreign_key]&.to_s
|
451
467
|
|
@@ -739,5 +755,16 @@ module BulkDependencyEraser
|
|
739
755
|
|
740
756
|
return klass_index
|
741
757
|
end
|
758
|
+
|
759
|
+
def build_all_in_db(&block)
|
760
|
+
puts "Building all from DB..." if opts_c.verbose
|
761
|
+
opts_c.db_build_all_wrapper.call(self, block)
|
762
|
+
puts "Building all from DB complete." if opts_c.verbose
|
763
|
+
end
|
764
|
+
|
765
|
+
def set_current_klass_name(query_or_klass)
|
766
|
+
klass = query_or_klass.is_a?(ActiveRecord::Relation) ? query_or_klass.klass : query_or_klass
|
767
|
+
@current_klass_name = klass.name
|
768
|
+
end
|
742
769
|
end
|
743
770
|
end
|
@@ -3,8 +3,13 @@ module BulkDependencyEraser
|
|
3
3
|
DEFAULT_DB_DELETE_ALL_WRAPPER = ->(deleter, block) do
|
4
4
|
begin
|
5
5
|
block.call
|
6
|
-
rescue
|
7
|
-
report_error(
|
6
|
+
rescue BulkDependencyEraser::Errors::DeleterError => e
|
7
|
+
deleter.report_error(
|
8
|
+
<<~STRING.strip
|
9
|
+
Issue attempting to delete klass '#{e.deleting_klass_name}'
|
10
|
+
=> #{e.original_error_klass.name}: #{e.message}
|
11
|
+
STRING
|
12
|
+
)
|
8
13
|
end
|
9
14
|
end
|
10
15
|
|
@@ -60,23 +65,27 @@ module BulkDependencyEraser
|
|
60
65
|
|
61
66
|
current_class_name = 'N/A'
|
62
67
|
delete_all_in_db do
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
68
|
+
begin
|
69
|
+
class_names_and_ids.keys.reverse.each do |class_name|
|
70
|
+
current_class_name = class_name
|
71
|
+
ids = class_names_and_ids[class_name].reverse
|
72
|
+
klass = constantize(class_name)
|
67
73
|
|
68
|
-
|
69
|
-
|
70
|
-
delete_by_klass_and_ids(klass, ids)
|
71
|
-
else
|
72
|
-
# delete without referential integrity
|
73
|
-
# Disable any ActiveRecord::InvalidForeignKey raised errors.
|
74
|
-
# - src: https://stackoverflow.com/questions/41005849/rails-migrations-temporarily-ignore-foreign-key-constraint
|
75
|
-
# https://apidock.com/rails/ActiveRecord/ConnectionAdapters/AbstractAdapter/disable_referential_integrity
|
76
|
-
ActiveRecord::Base.connection.disable_referential_integrity do
|
74
|
+
if opts_c.enable_invalid_foreign_key_detection
|
75
|
+
# delete with referential integrity
|
77
76
|
delete_by_klass_and_ids(klass, ids)
|
77
|
+
else
|
78
|
+
# delete without referential integrity
|
79
|
+
# Disable any ActiveRecord::InvalidForeignKey raised errors.
|
80
|
+
# - src: https://stackoverflow.com/questions/41005849/rails-migrations-temporarily-ignore-foreign-key-constraint
|
81
|
+
# https://apidock.com/rails/ActiveRecord/ConnectionAdapters/AbstractAdapter/disable_referential_integrity
|
82
|
+
ActiveRecord::Base.connection.disable_referential_integrity do
|
83
|
+
delete_by_klass_and_ids(klass, ids)
|
84
|
+
end
|
78
85
|
end
|
79
86
|
end
|
87
|
+
rescue StandardError => e
|
88
|
+
raise BulkDependencyEraser::Errors::DeleterError.new(e.class, e.message, deleting_klass_name: current_class_name)
|
80
89
|
end
|
81
90
|
end
|
82
91
|
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module BulkDependencyEraser
|
2
|
+
module Errors
|
3
|
+
class BuilderError < BaseError
|
4
|
+
attr_reader :original_error_klass, :building_klass_name
|
5
|
+
|
6
|
+
def initialize(original_error_klass, message, building_klass_name:)
|
7
|
+
@original_error_klass = original_error_klass
|
8
|
+
@building_klass_name = building_klass_name
|
9
|
+
super(message)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module BulkDependencyEraser
|
2
|
+
module Errors
|
3
|
+
class DeleterError < BaseError
|
4
|
+
attr_reader :original_error_klass, :deleting_klass_name
|
5
|
+
|
6
|
+
def initialize(original_error_klass, message, deleting_klass_name:)
|
7
|
+
@original_error_klass = original_error_klass
|
8
|
+
@deleting_klass_name = deleting_klass_name
|
9
|
+
super(message)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module BulkDependencyEraser
|
2
|
+
module Errors
|
3
|
+
class NullifierError < BaseError
|
4
|
+
attr_reader :original_error_klass, :nullifying_klass_name, :nullifying_columns
|
5
|
+
|
6
|
+
def initialize(original_error_klass, message, nullifying_klass_name:, nullifying_columns:)
|
7
|
+
@original_error_klass = original_error_klass
|
8
|
+
@nullifying_klass_name = nullifying_klass_name
|
9
|
+
@nullifying_columns = nullifying_columns
|
10
|
+
super(message)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -3,8 +3,13 @@ module BulkDependencyEraser
|
|
3
3
|
DEFAULT_DB_NULLIFY_ALL_WRAPPER = ->(nullifier, block) do
|
4
4
|
begin
|
5
5
|
block.call
|
6
|
-
rescue
|
7
|
-
nullifier.report_error(
|
6
|
+
rescue BulkDependencyEraser::Errors::NullifierError => e
|
7
|
+
nullifier.report_error(
|
8
|
+
<<~STRING.strip
|
9
|
+
Issue attempting to nullify klass '#{e.nullifying_klass_name}' on column(s) '#{e.nullifying_columns}'
|
10
|
+
=> #{e.original_error_klass.name}: #{e.message}
|
11
|
+
STRING
|
12
|
+
)
|
8
13
|
end
|
9
14
|
end
|
10
15
|
|
@@ -114,6 +119,7 @@ module BulkDependencyEraser
|
|
114
119
|
current_class_name = 'N/A'
|
115
120
|
current_column = 'N/A'
|
116
121
|
nullify_all_in_db do
|
122
|
+
begin
|
117
123
|
# column_and_ids should have already been reversed in builder
|
118
124
|
class_names_columns_and_ids.keys.reverse.each do |class_name|
|
119
125
|
current_class_name = class_name
|
@@ -139,6 +145,14 @@ module BulkDependencyEraser
|
|
139
145
|
end
|
140
146
|
end
|
141
147
|
end
|
148
|
+
rescue StandardError => e
|
149
|
+
raise BulkDependencyEraser::Errors::NullifierError.new(
|
150
|
+
e.class,
|
151
|
+
e.message,
|
152
|
+
nullifying_klass_name: current_class_name,
|
153
|
+
nullifying_columns: current_column.to_s # could be an array, string, or symbol
|
154
|
+
)
|
155
|
+
end
|
142
156
|
end
|
143
157
|
|
144
158
|
return errors.none?
|
@@ -6,4 +6,8 @@ require_relative 'bulk_dependency_eraser/nullifier'
|
|
6
6
|
require_relative 'bulk_dependency_eraser/query_schema_parser'
|
7
7
|
require_relative 'bulk_dependency_eraser/manager'
|
8
8
|
require_relative 'bulk_dependency_eraser/utils'
|
9
|
-
require_relative 'bulk_dependency_eraser/version'
|
9
|
+
require_relative 'bulk_dependency_eraser/version'
|
10
|
+
require_relative 'bulk_dependency_eraser/errors/base_error'
|
11
|
+
require_relative 'bulk_dependency_eraser/errors/builder_error'
|
12
|
+
require_relative 'bulk_dependency_eraser/errors/deleter_error'
|
13
|
+
require_relative 'bulk_dependency_eraser/errors/nullifier_error'
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulk_dependency_eraser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 4.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- benjamin.dana.software.dev@gmail.com
|
@@ -146,6 +146,10 @@ files:
|
|
146
146
|
- lib/bulk_dependency_eraser/base.rb
|
147
147
|
- lib/bulk_dependency_eraser/builder.rb
|
148
148
|
- lib/bulk_dependency_eraser/deleter.rb
|
149
|
+
- lib/bulk_dependency_eraser/errors/base_error.rb
|
150
|
+
- lib/bulk_dependency_eraser/errors/builder_error.rb
|
151
|
+
- lib/bulk_dependency_eraser/errors/deleter_error.rb
|
152
|
+
- lib/bulk_dependency_eraser/errors/nullifier_error.rb
|
149
153
|
- lib/bulk_dependency_eraser/full_schema_parser.rb
|
150
154
|
- lib/bulk_dependency_eraser/manager.rb
|
151
155
|
- lib/bulk_dependency_eraser/nullifier.rb
|