bulk_dependency_eraser 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc49b9b507510ed728ba6434832c29c44d76fe144ee4b738a7f5ade224caf0d2
-  data.tar.gz: 3437d5c44ca6ae2aaaf6d5e283da42d7267e5df2f378ef7abf7a72b63662cc5e
+  metadata.gz: 31e61c4af0756f5f3cb2f75ee9d3247e238fd83386fd8bdeace5bd60ebf17655
+  data.tar.gz: c3edbcc3cdfa4d56d398616780ba81ae254cf9224f9371fa60dbfbbc25b2b343
 SHA512:
-  metadata.gz: 8f15ccf36462a1fa821843dd5b05144e078be9a624620b297c02b0bb9e4252b997c0db56ae7546962163559a4f141eeaa2c3be571e1edafb5faa41cb1c83c563
-  data.tar.gz: 6a72116ef8842358a7de07f461cae3a3fbb2b6bfce2a0b725dd373534480a84fd69e81c18838178bea8d65788039671244c76df2d1669bc470687354929c130d
+  metadata.gz: '09c8931ca0720cf4a40639df42d1376745d32915167f2b78e43400280761249458e582bb872b8a4b5106d6c580ad82c88a26f2270a0227e457d045fe9490d388'
+  data.tar.gz: fa6224f8af00e73b0114666728547c4e64fdd2976c4026411d621ef411ffbc1bf484275e93076f8942d1ae99834600da794efa9590eab4827e1c55bcd1978912

data/lib/bulk_dependency_eraser/builder.rb

@@ -13,7 +13,10 @@ module BulkDependencyEraser
       # Won't parse any table in this list
       ignore_tables_and_dependencies: [],
       ignore_klass_names_and_dependencies: [],
-      batching_size_limit: 500,
+      # a general batching size
+      batch_size: 10_000,
+      # A specific batching size for this class, overrides the batch_size
+      read_batch_size: nil,
     }.freeze
 
     DEFAULT_DB_WRAPPER = ->(block) do
@@ -120,6 +123,10 @@ module BulkDependencyEraser
     attr_reader :table_names_to_parsed_klass_names
     attr_reader :ignore_table_name_and_dependencies, :ignore_klass_name_and_dependencies
 
+    def batch_size
+      opts_c.read_batch_size || opts_c.batch_size
+    end
+
     def deletion_query_parser query, association_parent = nil
       # necessary for "ActiveRecord::Reflection::ThroughReflection" use-case
       # force_through_destroy_chains = options[:force_destroy_chain] || {}
@@ -165,9 +172,12 @@ module BulkDependencyEraser
         return
       end
 
-      # Pluck IDs of the current query
-      query_ids = read_from_db do
-        query.pluck(:id)
+      # Pluck IDs of the current query (batchified)
+      query_ids = []
+      query.in_batches(of: batch_size) do |batch|
+        read_from_db do
+          query_ids += batch.pluck(:id)
+        end
       end
 
       deletion_list[klass_name] ||= []
@@ -340,8 +350,11 @@ module BulkDependencyEraser
       # - handle primary key edge cases
       # - The associations might not be using the primary_key of the klass table, but we can support that here.
       if specified_primary_key && specified_primary_key&.to_s != 'id'
-        alt_primary_ids = read_from_db do
-          query.pluck(specified_primary_key)
+        alt_primary_ids = []
+        query.in_batches(of: batch_size) do |batch|
+          read_from_db do
+            alt_primary_ids += query.pluck(specified_primary_key)
+          end
         end
         assoc_query = assoc_query.where(specified_foreign_key.to_sym => alt_primary_ids)
       else
@@ -364,8 +377,11 @@ module BulkDependencyEraser
       elsif type == :nullify
         # No need for recursion here.
         # - we're not destroying these assocs (just nullifying foreign_key columns) so we don't need to parse their dependencies.
-        assoc_ids = read_from_db do
-          assoc_query.pluck(:id)
+        assoc_ids = []
+        assoc_query.in_batches(of: batch_size) do |batch|
+          read_from_db do
+            assoc_ids += batch.pluck(:id)
+          end
         end
 
         # No assoc_ids, no need to add it to the nullification list
@@ -409,7 +425,7 @@ module BulkDependencyEraser
       # assoc_query = assoc_klass.unscoped
       # query.in_batches
 
-      assoc_klass.in_batches(of: opts_c.batching_size_limit) do |batch|
+      assoc_klass.in_batches(of: batch_size) do |batch|
         batch.each do |record|
           record.send(association_name)
         end
@@ -472,21 +488,23 @@ module BulkDependencyEraser
         return
       end
 
-      assoc_query = read_from_db do
-        assoc_query.where(
-          specified_primary_key.to_sym => query.pluck(specified_foreign_key)
-        )
-      end
+      query.in_batches(of: batch_size) do |batch|
+        assoc_batch_query = read_from_db do
+          assoc_query.where(
+            specified_primary_key.to_sym => batch.pluck(specified_foreign_key)
+          )
+        end
 
-      if type == :delete
-        # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
-        deletion_query_parser(assoc_query, parent_class)
-      elsif type == :restricted
-        if traverse_restricted_dependency?(parent_class, reflection, assoc_query)
-          deletion_query_parser(assoc_query, parent_class)
+        if type == :delete
+          # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
+          deletion_query_parser(assoc_batch_query, parent_class)
+        elsif type == :restricted
+          if traverse_restricted_dependency?(parent_class, reflection, assoc_batch_query)
+            deletion_query_parser(assoc_batch_query, parent_class)
+          end
+        else
+          raise "invalid parsing type: #{type}"
         end
-      else
-        raise "invalid parsing type: #{type}"
       end
     end
 
@@ -538,28 +556,30 @@ module BulkDependencyEraser
         return
       end
 
-      foreign_ids_by_type = read_from_db do
-        query.pluck(specified_foreign_key, specified_foreign_type).each_with_object({}) do |(id, type), hash|
-          hash.key?(type) ? hash[type] << id : hash[type] = [id]
+      query.in_batches(of: batch_size) do |batch|
+        foreign_ids_by_type = read_from_db do
+          batch.pluck(specified_foreign_key, specified_foreign_type).each_with_object({}) do |(id, type), hash|
+            hash.key?(type) ? hash[type] << id : hash[type] = [id]
+          end
         end
-      end
 
-      if type == :delete
-        # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
-        foreign_ids_by_type.each do |type, ids|
-          assoc_klass = type.constantize
-          deletion_query_parser(assoc_klass.where(id: ids), assoc_klass)
-        end
-      elsif type == :restricted
-        if traverse_restricted_dependency_for_belongs_to_poly?(parent_class, reflection, foreign_ids_by_type)
+        if type == :delete
           # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
           foreign_ids_by_type.each do |type, ids|
             assoc_klass = type.constantize
             deletion_query_parser(assoc_klass.where(id: ids), assoc_klass)
           end
+        elsif type == :restricted
+          if traverse_restricted_dependency_for_belongs_to_poly?(parent_class, reflection, foreign_ids_by_type)
+            # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
+            foreign_ids_by_type.each do |type, ids|
+              assoc_klass = type.constantize
+              deletion_query_parser(assoc_klass.where(id: ids), assoc_klass)
+            end
+          end
+        else
+          raise "invalid parsing type: #{type}"
         end
-      else
-        raise "invalid parsing type: #{type}"
       end
     end
 

data/lib/bulk_dependency_eraser/deleter.rb

@@ -4,7 +4,11 @@ module BulkDependencyEraser
       verbose: false,
       db_delete_wrapper: self::DEFAULT_DB_WRAPPER,
       # Set to true if you want 'ActiveRecord::InvalidForeignKey' errors raised during deletions
-      enable_invalid_foreign_key_detection: false
+      enable_invalid_foreign_key_detection: false,
+      # a general batching size
+      batch_size: 300,
+      # A specific batching size for this class, overrides the batch_size
+      delete_batch_size: nil,
     }.freeze
 
     DEFAULT_DB_WRAPPER = ->(block) do
@@ -55,15 +59,21 @@ module BulkDependencyEraser
 
     protected
 
+    attr_reader :class_names_and_ids
+
+    def batch_size
+      opts_c.delete_batch_size || opts_c.batch_size
+    end
+
     def delete_by_klass_and_ids klass, ids
       puts "Deleting #{klass.name}'s IDs: #{ids}" if opts_c.verbose
-      delete_in_db do
-        klass.unscoped.where(id: ids).delete_all
+      ids.each_slice(batch_size) do |ids_subset|
+        delete_in_db do
+          klass.unscoped.where(id: ids_subset).delete_all
+        end
       end
     end
 
-    attr_reader :class_names_and_ids
-
     def delete_in_db(&block)
       puts "Deleting from DB..." if opts_c.verbose
       opts_c.db_delete_wrapper.call(block)

data/lib/bulk_dependency_eraser/nullifier.rb

@@ -6,7 +6,11 @@ module BulkDependencyEraser
       # Set to true if you want 'ActiveRecord::InvalidForeignKey' errors raised during nullifications
       # - I can't think of a use-case where a nullification would generate an invalid key error
       # - Not hurting anything to leave it in, but might remove it in the future.
-      enable_invalid_foreign_key_detection: false
+      enable_invalid_foreign_key_detection: false,
+      # a general batching size
+      batch_size: 300,
+      # A specific batching size for this class, overrides the batch_size
+      nullify_batch_size: nil,
     }.freeze
 
     DEFAULT_DB_WRAPPER = ->(block) do
@@ -115,6 +119,10 @@ module BulkDependencyEraser
 
     attr_reader :class_names_columns_and_ids
 
+    def batch_size
+      opts_c.nullify_batch_size || opts_c.batch_size
+    end
+
     def nullify_by_klass_column_and_ids klass, columns, ids
       nullify_columns = {}
 
@@ -127,8 +135,10 @@ module BulkDependencyEraser
         nullify_columns[columns] = nil
       end
 
-      nullify_in_db do
-        klass.unscoped.where(id: ids).update_all(nullify_columns)
+      ids.each_slice(batch_size) do |ids_subset|
+        nullify_in_db do
+          klass.unscoped.where(id: ids_subset).update_all(nullify_columns)
+        end
       end
     end
 

data/lib/bulk_dependency_eraser/version.rb

@@ -1,3 +1,3 @@
 module BulkDependencyEraser
-  VERSION = "1.1.0".freeze
+  VERSION = "1.2.0".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bulk_dependency_eraser
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - benjamin.dana.software.dev@gmail.com