bulk_dependency_eraser 1.0.5 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cbea15215c816eca6790c1d7c3cbf4f4f68534fc3ff18cc38e8c26dc7e94513d
4
- data.tar.gz: 532a711c1290ea4fea4200acc75d9916eb18a2451207b89d1163372d7372028d
3
+ metadata.gz: 31e61c4af0756f5f3cb2f75ee9d3247e238fd83386fd8bdeace5bd60ebf17655
4
+ data.tar.gz: c3edbcc3cdfa4d56d398616780ba81ae254cf9224f9371fa60dbfbbc25b2b343
5
5
  SHA512:
6
- metadata.gz: e5639d23e7fbad812f11db05ad4b4dc3e88cd8823b3623ec38c892ed144ca6002b2e6bf56a826cca62aa2e87312fc676ac0668d1d6301a66496f91a62c7896fe
7
- data.tar.gz: 20f81c4ddae6e875de07849088493f21095a567ccf2e1e39bfe21bbe1486dbfaa76ad06bc53f247724ca0f5e79a7f7af699e480345c2b5a800f617c06bb83844
6
+ metadata.gz: '09c8931ca0720cf4a40639df42d1376745d32915167f2b78e43400280761249458e582bb872b8a4b5106d6c580ad82c88a26f2270a0227e457d045fe9490d388'
7
+ data.tar.gz: fa6224f8af00e73b0114666728547c4e64fdd2976c4026411d621ef411ffbc1bf484275e93076f8942d1ae99834600da794efa9590eab4827e1c55bcd1978912
@@ -13,7 +13,10 @@ module BulkDependencyEraser
13
13
  # Won't parse any table in this list
14
14
  ignore_tables_and_dependencies: [],
15
15
  ignore_klass_names_and_dependencies: [],
16
- batching_size_limit: 500,
16
+ # a general batching size
17
+ batch_size: 10_000,
18
+ # A specific batching size for this class, overrides the batch_size
19
+ read_batch_size: nil,
17
20
  }.freeze
18
21
 
19
22
  DEFAULT_DB_WRAPPER = ->(block) do
@@ -48,7 +51,8 @@ module BulkDependencyEraser
48
51
  'ActiveRecord::Reflection::HasAndBelongsToManyReflection'
49
52
  ].freeze
50
53
 
51
- attr_reader :deletion_list, :nullification_list
54
+ # write access so that these can be edited in-place by end-users who might need to manually adjust deletion order.
55
+ attr_accessor :deletion_list, :nullification_list
52
56
  attr_reader :ignore_table_deletion_list, :ignore_table_nullification_list
53
57
 
54
58
  def initialize query:, opts: {}
@@ -119,6 +123,10 @@ module BulkDependencyEraser
119
123
  attr_reader :table_names_to_parsed_klass_names
120
124
  attr_reader :ignore_table_name_and_dependencies, :ignore_klass_name_and_dependencies
121
125
 
126
+ def batch_size
127
+ opts_c.read_batch_size || opts_c.batch_size
128
+ end
129
+
122
130
  def deletion_query_parser query, association_parent = nil
123
131
  # necessary for "ActiveRecord::Reflection::ThroughReflection" use-case
124
132
  # force_through_destroy_chains = options[:force_destroy_chain] || {}
@@ -164,9 +172,12 @@ module BulkDependencyEraser
164
172
  return
165
173
  end
166
174
 
167
- # Pluck IDs of the current query
168
- query_ids = read_from_db do
169
- query.pluck(:id)
175
+ # Pluck IDs of the current query (batchified)
176
+ query_ids = []
177
+ query.in_batches(of: batch_size) do |batch|
178
+ read_from_db do
179
+ query_ids += batch.pluck(:id)
180
+ end
170
181
  end
171
182
 
172
183
  deletion_list[klass_name] ||= []
@@ -339,8 +350,11 @@ module BulkDependencyEraser
339
350
  # - handle primary key edge cases
340
351
  # - The associations might not be using the primary_key of the klass table, but we can support that here.
341
352
  if specified_primary_key && specified_primary_key&.to_s != 'id'
342
- alt_primary_ids = read_from_db do
343
- query.pluck(specified_primary_key)
353
+ alt_primary_ids = []
354
+ query.in_batches(of: batch_size) do |batch|
355
+ read_from_db do
356
+ alt_primary_ids += query.pluck(specified_primary_key)
357
+ end
344
358
  end
345
359
  assoc_query = assoc_query.where(specified_foreign_key.to_sym => alt_primary_ids)
346
360
  else
@@ -363,8 +377,11 @@ module BulkDependencyEraser
363
377
  elsif type == :nullify
364
378
  # No need for recursion here.
365
379
  # - we're not destroying these assocs (just nullifying foreign_key columns) so we don't need to parse their dependencies.
366
- assoc_ids = read_from_db do
367
- assoc_query.pluck(:id)
380
+ assoc_ids = []
381
+ assoc_query.in_batches(of: batch_size) do |batch|
382
+ read_from_db do
383
+ assoc_ids += batch.pluck(:id)
384
+ end
368
385
  end
369
386
 
370
387
  # No assoc_ids, no need to add it to the nullification list
@@ -408,7 +425,7 @@ module BulkDependencyEraser
408
425
  # assoc_query = assoc_klass.unscoped
409
426
  # query.in_batches
410
427
 
411
- assoc_klass.in_batches(of: opts_c.batching_size_limit) do |batch|
428
+ assoc_klass.in_batches(of: batch_size) do |batch|
412
429
  batch.each do |record|
413
430
  record.send(association_name)
414
431
  end
@@ -471,21 +488,23 @@ module BulkDependencyEraser
471
488
  return
472
489
  end
473
490
 
474
- assoc_query = read_from_db do
475
- assoc_query.where(
476
- specified_primary_key.to_sym => query.pluck(specified_foreign_key)
477
- )
478
- end
491
+ query.in_batches(of: batch_size) do |batch|
492
+ assoc_batch_query = read_from_db do
493
+ assoc_query.where(
494
+ specified_primary_key.to_sym => batch.pluck(specified_foreign_key)
495
+ )
496
+ end
479
497
 
480
- if type == :delete
481
- # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
482
- deletion_query_parser(assoc_query, parent_class)
483
- elsif type == :restricted
484
- if traverse_restricted_dependency?(parent_class, reflection, assoc_query)
485
- deletion_query_parser(assoc_query, parent_class)
498
+ if type == :delete
499
+ # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
500
+ deletion_query_parser(assoc_batch_query, parent_class)
501
+ elsif type == :restricted
502
+ if traverse_restricted_dependency?(parent_class, reflection, assoc_batch_query)
503
+ deletion_query_parser(assoc_batch_query, parent_class)
504
+ end
505
+ else
506
+ raise "invalid parsing type: #{type}"
486
507
  end
487
- else
488
- raise "invalid parsing type: #{type}"
489
508
  end
490
509
  end
491
510
 
@@ -537,28 +556,30 @@ module BulkDependencyEraser
537
556
  return
538
557
  end
539
558
 
540
- foreign_ids_by_type = read_from_db do
541
- query.pluck(specified_foreign_key, specified_foreign_type).each_with_object({}) do |(id, type), hash|
542
- hash.key?(type) ? hash[type] << id : hash[type] = [id]
559
+ query.in_batches(of: batch_size) do |batch|
560
+ foreign_ids_by_type = read_from_db do
561
+ batch.pluck(specified_foreign_key, specified_foreign_type).each_with_object({}) do |(id, type), hash|
562
+ hash.key?(type) ? hash[type] << id : hash[type] = [id]
563
+ end
543
564
  end
544
- end
545
565
 
546
- if type == :delete
547
- # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
548
- foreign_ids_by_type.each do |type, ids|
549
- assoc_klass = type.constantize
550
- deletion_query_parser(assoc_klass.where(id: ids), assoc_klass)
551
- end
552
- elsif type == :restricted
553
- if traverse_restricted_dependency_for_belongs_to_poly?(parent_class, reflection, foreign_ids_by_type)
566
+ if type == :delete
554
567
  # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
555
568
  foreign_ids_by_type.each do |type, ids|
556
569
  assoc_klass = type.constantize
557
570
  deletion_query_parser(assoc_klass.where(id: ids), assoc_klass)
558
571
  end
572
+ elsif type == :restricted
573
+ if traverse_restricted_dependency_for_belongs_to_poly?(parent_class, reflection, foreign_ids_by_type)
574
+ # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
575
+ foreign_ids_by_type.each do |type, ids|
576
+ assoc_klass = type.constantize
577
+ deletion_query_parser(assoc_klass.where(id: ids), assoc_klass)
578
+ end
579
+ end
580
+ else
581
+ raise "invalid parsing type: #{type}"
559
582
  end
560
- else
561
- raise "invalid parsing type: #{type}"
562
583
  end
563
584
  end
564
585
 
@@ -4,7 +4,11 @@ module BulkDependencyEraser
4
4
  verbose: false,
5
5
  db_delete_wrapper: self::DEFAULT_DB_WRAPPER,
6
6
  # Set to true if you want 'ActiveRecord::InvalidForeignKey' errors raised during deletions
7
- enable_invalid_foreign_key_detection: false
7
+ enable_invalid_foreign_key_detection: false,
8
+ # a general batching size
9
+ batch_size: 300,
10
+ # A specific batching size for this class, overrides the batch_size
11
+ delete_batch_size: nil,
8
12
  }.freeze
9
13
 
10
14
  DEFAULT_DB_WRAPPER = ->(block) do
@@ -55,15 +59,21 @@ module BulkDependencyEraser
55
59
 
56
60
  protected
57
61
 
62
+ attr_reader :class_names_and_ids
63
+
64
+ def batch_size
65
+ opts_c.delete_batch_size || opts_c.batch_size
66
+ end
67
+
58
68
  def delete_by_klass_and_ids klass, ids
59
69
  puts "Deleting #{klass.name}'s IDs: #{ids}" if opts_c.verbose
60
- delete_in_db do
61
- klass.unscoped.where(id: ids).delete_all
70
+ ids.each_slice(batch_size) do |ids_subset|
71
+ delete_in_db do
72
+ klass.unscoped.where(id: ids_subset).delete_all
73
+ end
62
74
  end
63
75
  end
64
76
 
65
- attr_reader :class_names_and_ids
66
-
67
77
  def delete_in_db(&block)
68
78
  puts "Deleting from DB..." if opts_c.verbose
69
79
  opts_c.db_delete_wrapper.call(block)
@@ -6,7 +6,11 @@ module BulkDependencyEraser
6
6
  # Set to true if you want 'ActiveRecord::InvalidForeignKey' errors raised during nullifications
7
7
  # - I can't think of a use-case where a nullification would generate an invalid key error
8
8
  # - Not hurting anything to leave it in, but might remove it in the future.
9
- enable_invalid_foreign_key_detection: false
9
+ enable_invalid_foreign_key_detection: false,
10
+ # a general batching size
11
+ batch_size: 300,
12
+ # A specific batching size for this class, overrides the batch_size
13
+ nullify_batch_size: nil,
10
14
  }.freeze
11
15
 
12
16
  DEFAULT_DB_WRAPPER = ->(block) do
@@ -115,6 +119,10 @@ module BulkDependencyEraser
115
119
 
116
120
  attr_reader :class_names_columns_and_ids
117
121
 
122
+ def batch_size
123
+ opts_c.nullify_batch_size || opts_c.batch_size
124
+ end
125
+
118
126
  def nullify_by_klass_column_and_ids klass, columns, ids
119
127
  nullify_columns = {}
120
128
 
@@ -127,8 +135,10 @@ module BulkDependencyEraser
127
135
  nullify_columns[columns] = nil
128
136
  end
129
137
 
130
- nullify_in_db do
131
- klass.unscoped.where(id: ids).update_all(nullify_columns)
138
+ ids.each_slice(batch_size) do |ids_subset|
139
+ nullify_in_db do
140
+ klass.unscoped.where(id: ids_subset).update_all(nullify_columns)
141
+ end
132
142
  end
133
143
  end
134
144
 
@@ -0,0 +1,3 @@
1
+ module BulkDependencyEraser
2
+ VERSION = "1.2.0".freeze
3
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bulk_dependency_eraser
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.5
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - benjamin.dana.software.dev@gmail.com
@@ -148,6 +148,7 @@ files:
148
148
  - lib/bulk_dependency_eraser/deleter.rb
149
149
  - lib/bulk_dependency_eraser/manager.rb
150
150
  - lib/bulk_dependency_eraser/nullifier.rb
151
+ - lib/bulk_dependency_eraser/version.rb
151
152
  homepage: https://github.com/danabr75/bulk_dependency_eraser
152
153
  licenses:
153
154
  - LGPL-3.0-only