bulk_dependency_eraser 1.2.2 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80c6e41e13092670b6f67fc6cdb8d117c0222b0d2ce7c0043e1f12dcf01c1273
-  data.tar.gz: 9b93602042b1863caa6a1eac272547d76559e7ba62e97e29831c5d171050eafe
+  metadata.gz: 60f1df6c3a60ce0e7d3048cdcb263986148ca7989464f90859455afea67eae78
+  data.tar.gz: 5069d8a435cc8ac13a2b4e873318c1c0a4ada041a5ed292b8df58212af3bc887
 SHA512:
-  metadata.gz: 26e795e7ecf8727eacf36267502454478922430dc86424f6ef0483bb195a09dc1c52c47067c0a87d70f9235173715a975422548ebd50da7fb1cf70ccbc7f0721
-  data.tar.gz: e6722b7ca1fd1c58f1b4a9d619af965eddb440c159bc2b9fd6efd436c2e927fd6e58679bb25f71438d24987aea8726ceb69a5ee081654e6cb8fbf4336d9c7dce
+  metadata.gz: 590c7dc30ca2d04140332d0c97128dcddbfb9a95e07d8a413006faa32011d3343fa5363e1a5a43f3678ae2b96f7ca9e44cdc23775f69a6971d576f000e4254c3
+  data.tar.gz: 6bff0c448928f611a0c8ce7f684df5b636d66bf0570c793ced8204eb759a1e0ad020e3947caa562610a5410f967b8ee4ef2bdc191aa37abd2e5ec47d497d11c6

data/lib/bulk_dependency_eraser/builder.rb

@@ -13,10 +13,13 @@ module BulkDependencyEraser
       # Won't parse any table in this list
       ignore_tables_and_dependencies: [],
       ignore_klass_names_and_dependencies: [],
+      disable_batching: false,
       # a general batching size
       batch_size: 10_000,
       # A specific batching size for this class, overrides the batch_size
       read_batch_size: nil,
+      # A specific read batching disable option
+      disable_read_batching: nil,
     }.freeze
 
     DEFAULT_DB_WRAPPER = ->(block) do
@@ -85,6 +88,27 @@ module BulkDependencyEraser
         end
       end
 
+      # Reverse order IDs
+      # - if a class self-referenced itself the children will be deleted/nullified before parent
+      deletion_list.keys.each do |class_name|
+        deletion_list[class_name] = deletion_list.delete(class_name).reverse
+
+        # TODO: Hard to test if not sorted
+        deletion_list[class_name] = deletion_list[class_name].sort if Rails.env.test?
+      end
+
+      nullification_list.keys.each do |class_name|
+        columns_and_ids = nullification_list.delete(class_name)
+        # we don't need to reverse the column keys
+        columns_and_ids.keys.each do |column_name|
+          columns_and_ids[column_name] = columns_and_ids.delete(column_name).reverse
+
+          # TODO: Hard to test if not sorted
+          columns_and_ids[column_name] = columns_and_ids.delete(column_name).sort if Rails.env.test?
+        end
+        nullification_list[class_name] = columns_and_ids
+      end
+
       return build_result
     end
 
@@ -127,8 +151,37 @@ module BulkDependencyEraser
     attr_reader :table_names_to_parsed_klass_names
     attr_reader :ignore_table_name_and_dependencies, :ignore_klass_name_and_dependencies
 
+    def pluck_from_query query, column = :id
+      query_ids = []
+      read_from_db do
+        # If the query has a limit, then we don't want to clobber with batching.
+        if batching_disabled? || !query.where({}).limit_value.nil?
+          # query without batching
+          query_ids = query.pluck(column)
+        else
+          # query with batching
+          offset = 0
+          loop do
+            new_query_ids = query.offset(offset).limit(batch_size).pluck(column)
+            query_ids += new_query_ids
+
+            break if new_query_ids.size < batch_size
+
+            # Move to the next batch
+            offset += batch_size
+          end
+        end
+      end
+
+      return query_ids
+    end
+
     def batch_size
-      opts_c.read_batch_size || opts_c.batch_size
+      opts_c.read_batch_size.nil? ? opts_c.batch_size : opts_c.read_batch_size
+    end
+
+    def batching_disabled?
+      opts_c.disable_read_batching.nil? ? opts_c.disable_batching : opts_c.disable_read_batching
     end
 
     def deletion_query_parser query, association_parent = nil
@@ -176,13 +229,8 @@ module BulkDependencyEraser
         return
       end
 
-      # Pluck IDs of the current query (batchified)
-      query_ids = []
-      query.in_batches(of: batch_size) do |batch|
-        read_from_db do
-          query_ids += batch.pluck(:id)
-        end
-      end
+      # Pluck IDs of the current query
+      query_ids = pluck_from_query(query)
 
       deletion_list[klass_name] ||= []
 
@@ -203,7 +251,7 @@ module BulkDependencyEraser
 
       # Hard to test if not sorted
       # - if we had more advanced rspec matches, we could do away with this.
-      deletion_list[klass_name].sort! if Rails.env.test?
+      # deletion_list[klass_name].sort! if Rails.env.test?
 
       # ignore associations that aren't a dependent destroyable type
       destroy_associations = query.reflect_on_all_associations.select do |reflection|
@@ -370,12 +418,7 @@ module BulkDependencyEraser
       # - handle primary key edge cases
       # - The associations might not be using the primary_key of the klass table, but we can support that here.
       if specified_primary_key && specified_primary_key&.to_s != 'id'
-        alt_primary_ids = []
-        query.in_batches(of: batch_size) do |batch|
-          read_from_db do
-            alt_primary_ids += query.pluck(specified_primary_key)
-          end
-        end
+        alt_primary_ids = pluck_from_query(query, specified_primary_key)
         assoc_query = assoc_query.where(specified_foreign_key.to_sym => alt_primary_ids)
       else
         assoc_query = assoc_query.where(specified_foreign_key.to_sym => query_ids)
@@ -397,12 +440,7 @@ module BulkDependencyEraser
       elsif type == :nullify
         # No need for recursion here.
         # - we're not destroying these assocs (just nullifying foreign_key columns) so we don't need to parse their dependencies.
-        assoc_ids = []
-        assoc_query.in_batches(of: batch_size) do |batch|
-          read_from_db do
-            assoc_ids += batch.pluck(:id)
-          end
-        end
+        assoc_ids = pluck_from_query(assoc_query)
 
         # No assoc_ids, no need to add it to the nullification list
         return if assoc_ids.none?
@@ -412,7 +450,7 @@ module BulkDependencyEraser
         nullification_list[assoc_klass_name][specified_foreign_key] += assoc_ids
         nullification_list[assoc_klass_name][specified_foreign_key].uniq!
 
-        nullification_list[assoc_klass_name][specified_foreign_key].sort! if Rails.env.test?
+        # nullification_list[assoc_klass_name][specified_foreign_key].sort! if Rails.env.test?
 
         # Also nullify the 'type' field, if the association is polymorphic
         if specified_foreign_type
@@ -420,7 +458,7 @@ module BulkDependencyEraser
           nullification_list[assoc_klass_name][specified_foreign_type] += assoc_ids
           nullification_list[assoc_klass_name][specified_foreign_type].uniq!
 
-          nullification_list[assoc_klass_name][specified_foreign_type].sort! if Rails.env.test?
+          # nullification_list[assoc_klass_name][specified_foreign_type].sort! if Rails.env.test?
         end
       else
         raise "invalid parsing type: #{type}"
@@ -508,23 +546,20 @@ module BulkDependencyEraser
         return
       end
 
-      query.in_batches(of: batch_size) do |batch|
-        assoc_batch_query = read_from_db do
-          assoc_query.where(
-            specified_primary_key.to_sym => batch.pluck(specified_foreign_key)
-          )
-        end
+      foreign_keys = pluck_from_query(query, specified_foreign_key)
+      assoc_query = assoc_query.where(
+        specified_primary_key.to_sym => foreign_keys
+      )
 
-        if type == :delete
-          # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
-          deletion_query_parser(assoc_batch_query, parent_class)
-        elsif type == :restricted
-          if traverse_restricted_dependency?(parent_class, reflection, assoc_batch_query)
-            deletion_query_parser(assoc_batch_query, parent_class)
-          end
-        else
-          raise "invalid parsing type: #{type}"
+      if type == :delete
+        # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
+        deletion_query_parser(assoc_query, parent_class)
+      elsif type == :restricted
+        if traverse_restricted_dependency?(parent_class, reflection, assoc_query)
+          deletion_query_parser(assoc_query, parent_class)
         end
+      else
+        raise "invalid parsing type: #{type}"
       end
     end
 
@@ -539,6 +574,8 @@ module BulkDependencyEraser
     # In this case, it's like a `belongs_to :polymorphicable, polymorphic: true, dependent: :destroy` use-case
     # - it's unusual, but valid use-case
     def association_parser_belongs_to_polymorphic(parent_class, query, query_ids, association_name, type)
+      # raise "Unsupported use-case: #{parent_class.name} -> belongs_to :polymorphicable, polymorphic: true, dependent: :destroy"
+
       reflection = parent_class.reflect_on_association(association_name)
       reflection_type = reflection.class.name
 
@@ -576,30 +613,30 @@ module BulkDependencyEraser
         return
       end
 
-      query.in_batches(of: batch_size) do |batch|
-        foreign_ids_by_type = read_from_db do
-          batch.pluck(specified_foreign_key, specified_foreign_type).each_with_object({}) do |(id, type), hash|
-            hash.key?(type) ? hash[type] << id : hash[type] = [id]
-          end
+      # Not sure how to limit/offset batch this right now.
+      # - it's a rare use-case, let's just leave this as a TODO:
+      foreign_ids_by_type = read_from_db do
+        query.pluck(specified_foreign_key, specified_foreign_type).each_with_object({}) do |(id, type), hash|
+          hash.key?(type) ? hash[type] << id : hash[type] = [id]
         end
+      end
 
-        if type == :delete
+      if type == :delete
+        # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
+        foreign_ids_by_type.each do |type, ids|
+          assoc_klass = type.constantize
+          deletion_query_parser(assoc_klass.where(id: ids), assoc_klass)
+        end
+      elsif type == :restricted
+        if traverse_restricted_dependency_for_belongs_to_poly?(parent_class, reflection, foreign_ids_by_type)
           # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
           foreign_ids_by_type.each do |type, ids|
             assoc_klass = type.constantize
             deletion_query_parser(assoc_klass.where(id: ids), assoc_klass)
           end
-        elsif type == :restricted
-          if traverse_restricted_dependency_for_belongs_to_poly?(parent_class, reflection, foreign_ids_by_type)
-            # Recursively call 'deletion_query_parser' on association query, to delete any if the assoc's dependencies
-            foreign_ids_by_type.each do |type, ids|
-              assoc_klass = type.constantize
-              deletion_query_parser(assoc_klass.where(id: ids), assoc_klass)
-            end
-          end
-        else
-          raise "invalid parsing type: #{type}"
         end
+      else
+        raise "invalid parsing type: #{type}"
       end
     end
 

data/lib/bulk_dependency_eraser/deleter.rb

@@ -5,10 +5,13 @@ module BulkDependencyEraser
       db_delete_wrapper: self::DEFAULT_DB_WRAPPER,
       # Set to true if you want 'ActiveRecord::InvalidForeignKey' errors raised during deletions
       enable_invalid_foreign_key_detection: false,
+      disable_batching: false,
       # a general batching size
       batch_size: 300,
       # A specific batching size for this class, overrides the batch_size
       delete_batch_size: nil,
+      # A specific batching size for this class, overrides the batch_size
+      disable_delete_batching: nil,
     }.freeze
 
     DEFAULT_DB_WRAPPER = ->(block) do
@@ -30,6 +33,7 @@ module BulkDependencyEraser
       ActiveRecord::Base.transaction do
         current_class_name = 'N/A'
         begin
+          # IDs should have already been reversed in builder
           class_names_and_ids.keys.reverse.each do |class_name|
             current_class_name = class_name
             ids = class_names_and_ids[class_name]
@@ -65,11 +69,23 @@ module BulkDependencyEraser
       opts_c.delete_batch_size || opts_c.batch_size
     end
 
+    def batching_disabled?
+      opts_c.disable_delete_batching.nil? ? opts_c.disable_batching : opts_c.disable_delete_batching
+    end
+
     def delete_by_klass_and_ids klass, ids
       puts "Deleting #{klass.name}'s IDs: #{ids}" if opts_c.verbose
-      ids.each_slice(batch_size) do |ids_subset|
+      if batching_disabled?
+        puts "Deleting without batching" if opts_c.verbose
         delete_in_db do
-          klass.unscoped.where(id: ids_subset).delete_all
+          klass.unscoped.where(id: ids).delete_all
+        end
+      else
+        puts "Deleting with batching" if opts_c.verbose
+        ids.each_slice(batch_size) do |ids_subset|
+          delete_in_db do
+            klass.unscoped.where(id: ids_subset).delete_all
+          end
         end
       end
     end
@@ -78,6 +94,5 @@ module BulkDependencyEraser
       puts "Deleting from DB..." if opts_c.verbose
       opts_c.db_delete_wrapper.call(block)
     end
-
   end
 end

data/lib/bulk_dependency_eraser/nullifier.rb

@@ -7,10 +7,13 @@ module BulkDependencyEraser
       # - I can't think of a use-case where a nullification would generate an invalid key error
       # - Not hurting anything to leave it in, but might remove it in the future.
       enable_invalid_foreign_key_detection: false,
+      disable_batching: false,
       # a general batching size
       batch_size: 300,
       # A specific batching size for this class, overrides the batch_size
       nullify_batch_size: nil,
+      # A specific batching size for this class, overrides the batch_size
+      disable_nullify_batching: nil,
     }.freeze
 
     DEFAULT_DB_WRAPPER = ->(block) do
@@ -84,11 +87,12 @@ module BulkDependencyEraser
         current_class_name = 'N/A'
         current_column = 'N/A'
         begin
+          # column_and_ids should have already been reversed in builder
           class_names_columns_and_ids.keys.reverse.each do |class_name|
             current_class_name = class_name
             klass = class_name.constantize
-
             columns_and_ids = class_names_columns_and_ids[class_name]
+
             columns_and_ids.each do |column, ids|
               current_column = column
 
@@ -123,6 +127,10 @@ module BulkDependencyEraser
       opts_c.nullify_batch_size || opts_c.batch_size
     end
 
+    def batching_disabled?
+      opts_c.disable_nullify_batching.nil? ? opts_c.disable_batching : opts_c.disable_nullify_batching
+    end
+
     def nullify_by_klass_column_and_ids klass, columns, ids
       nullify_columns = {}
 
@@ -135,9 +143,15 @@ module BulkDependencyEraser
         nullify_columns[columns] = nil
       end
 
-      ids.each_slice(batch_size) do |ids_subset|
+      if batching_disabled?
         nullify_in_db do
-          klass.unscoped.where(id: ids_subset).update_all(nullify_columns)
+          klass.unscoped.where(id: ids).update_all(nullify_columns)
+        end
+      else
+        ids.each_slice(batch_size) do |ids_subset|
+          nullify_in_db do
+            klass.unscoped.where(id: ids_subset).update_all(nullify_columns)
+          end
         end
       end
     end

data/lib/bulk_dependency_eraser/version.rb

@@ -1,3 +1,3 @@
 module BulkDependencyEraser
-  VERSION = "1.2.2".freeze
+  VERSION = "1.3.1".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bulk_dependency_eraser
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.3.1
 platform: ruby
 authors:
 - benjamin.dana.software.dev@gmail.com