bulk_dependency_eraser 2.2.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73ac2c6209f874175a8a1208c184a3fdaa73f2020a411f2f41bd069668b63fbc
-  data.tar.gz: 86df357905f75f34337d30446cfc6b34b9aa55fc1947d17a008329379f2beae7
+  metadata.gz: e215e610f3c90e10cb386499bea48adc2a12a6cc7ce2a35466de12d5dc2dbb1d
+  data.tar.gz: 3c56acb8837e54c3387a6c2754dcab92e88402084f99198ab62c0ee1ea46d45b
 SHA512:
-  metadata.gz: 69fd26f7fcbd04f158adf4bc038181818136f17d878bad544bb888b1d0f547d557153204876a91b222154f7154d0f38b143f4db08f5081242c107488832fa682
-  data.tar.gz: b31893d532b1be190f5288ddfb495008615a3c54f2e5a9fb12344a733c4322345911597822c835277f9f4632657aeefe4ee7a77fe7eecea16cf64151dd63a797
+  metadata.gz: c1a1b05c9fd919c87a4947409296d65129f82817722d2de883b5f09e873b0851364348face62dc96a2251bb2aa8c8cba678798e0083681c7fa94f3ad0e1b6df6
+  data.tar.gz: 321cda6cc39a9ef3af66023aca8dbd385d46357ead943908d659880f968f2c7e1d5c9c8035d7f8c1c022b21e505aa25fc07be6a93296c73e73c67e0079061ff3

data/lib/bulk_dependency_eraser/base.rb

@@ -123,6 +123,8 @@ module BulkDependencyEraser
       end
     end
 
+    # We're supporting custom query scopes on by klass name.
+    # - apply them here
     def custom_scope_for_query(query)
       klass = query.klass
       if opts_c.proc_scopes_per_class_name.key?(klass.name)

data/lib/bulk_dependency_eraser/builder.rb

@@ -1,8 +1,22 @@
 module BulkDependencyEraser
   class Builder < Base
+    DEFAULT_DB_BUILD_ALL_WRAPPER = ->(builder, block) do
+      begin
+        block.call
+      rescue StandardError => e
+        builder.report_error(
+          <<~STRING.strip
+          Issue attempting to build deletion query for '#{e.building_klass_name}'
+            => #{e.original_error_klass.name}: #{e.message}
+          STRING
+        )
+      end
+    end
+
     DEFAULT_OPTS = {
       force_destroy_restricted: false,
       verbose: false,
+      db_build_all_wrapper: self::DEFAULT_DB_BUILD_ALL_WRAPPER,
       # Some associations scopes take parameters.
       # - We would have to instantiate if we wanted to apply that scope filter.
       instantiate_if_assoc_scope_with_arity: false,
@@ -14,6 +28,12 @@ module BulkDependencyEraser
       ignore_tables_and_dependencies: [],
       ignore_klass_names_and_dependencies: [],
       disable_batching: false,
+      # Disable model ordering during build batching
+      # - Some tables can be too big to order, and just need to trust the DB order
+      # - Not 100% guaranteed and could leave orphaned records behind.
+      disable_batch_ordering: false,
+      # Same as 'disable_batch_ordering', but only for select classes
+      disable_batch_ordering_for_klasses: [],
       # a general batching size
       batch_size: 10_000,
       # A specific batching size for this class, overrides the batch_size
@@ -38,6 +58,7 @@ module BulkDependencyEraser
     attr_accessor :deletion_list, :nullification_list
     attr_reader :ignore_table_deletion_list, :ignore_table_nullification_list
     attr_reader :query_schema_parser
+    attr_reader :current_klass_name
 
     delegate :circular_dependency_klasses, :flat_dependencies_per_klass, to: :query_schema_parser
 
@@ -57,6 +78,8 @@ module BulkDependencyEraser
       @ignore_table_name_and_dependencies = opts_c.ignore_tables_and_dependencies.collect { |table_name| table_name }
       @ignore_klass_name_and_dependencies = opts_c.ignore_klass_names_and_dependencies.collect { |klass_name| klass_name }
       @query_schema_parser = BulkDependencyEraser::QuerySchemaParser.new(query:, opts:)
+      # Moving pointer, points to the current class that is being queries
+      @current_klass_name = query.is_a?(ActiveRecord::Relation) ? query.klass.name : query.name
     end
 
     def execute
@@ -82,33 +105,20 @@ module BulkDependencyEraser
     end
 
     def build
-      begin
-        if opts_c.verbose
-          puts "Starting build for #{@query.is_a?(ActiveRecord::Relation) ? @query.klass.name : @query.name}"
-        end
+      build_all_in_db do
+        begin
+          if opts_c.verbose
+            puts "Starting build for #{@query.is_a?(ActiveRecord::Relation) ? @query.klass.name : @query.name}"
+          end
 
-        deletion_query_parser(@query)
+          deletion_query_parser(@query)
 
-        uniqify_errors!
+          uniqify_errors!
 
-        return errors.none?
-      rescue StandardError => e
-        if @query.is_a?(ActiveRecord::Relation)
-          klass      = @query.klass
-        else
-          # current_query is a normal rails class
-          klass      = @query
+          return errors.none?
+        rescue StandardError => e
+          raise BulkDependencyEraser::Errors::BuilderError.new(e.class, e.message, building_klass_name: current_klass_name)
         end
-        report_error(
-          "
-            Error Encountered in 'execute' for '#{klass.name}':
-              #{e.class.name}
-              #{e.message}
-          "
-        )
-        raise e
-
-        return false
       end
     end
 
@@ -127,7 +137,8 @@ module BulkDependencyEraser
       end
     end
 
-    def pluck_from_query query, column = :id
+    def pluck_from_query(query, column = :id)
+      set_current_klass_name(query)
       # ordering shouldn't matter in these queries, and would slow it down
       # - we're ignoring default_scope ordering, but assoc-defined ordering would still take effect
       query = query.reorder('')
@@ -139,8 +150,8 @@ module BulkDependencyEraser
         if batching_disabled? || !query.where({}).limit_value.nil?
           # query without batching
           query_ids = query.pluck(column)
-        else
-          # query with batching
+        elsif opts_c.disable_batch_ordering || opts_c.disable_batch_ordering_for_klasses.include?(current_klass_name)
+          # query with orderless batching
           offset = 0
           loop do
             new_query_ids = query.offset(offset).limit(batch_size).pluck(column)
@@ -151,6 +162,11 @@ module BulkDependencyEraser
             # Move to the next batch
             offset += batch_size
           end
+        else
+          # query with ordered batching
+          query.in_batches(of: batch_size) do |subset_query|
+            query_ids += subset_query.pluck(column)
+          end
         end
       end
 
@@ -293,6 +309,7 @@ module BulkDependencyEraser
       is_polymorphic = reflection.options[:polymorphic]
       unless is_polymorphic
         klass = reflection.klass
+        set_current_klass_name(reflection.klass)
 
         if ignore_table_name_and_dependencies.include?(klass.table_name)
           # Not parsing, table and dependencies ignorable
@@ -445,7 +462,6 @@ module BulkDependencyEraser
       assoc_klass = reflection.klass
       assoc_klass_name = assoc_klass.name
 
-
       # specified_primary_key = reflection.options[:primary_key]&.to_s
       # specified_foreign_key = reflection.options[:foreign_key]&.to_s
 
@@ -739,5 +755,16 @@ module BulkDependencyEraser
 
       return klass_index
     end
+
+    def build_all_in_db(&block)
+      puts "Building all from DB..." if opts_c.verbose
+      opts_c.db_build_all_wrapper.call(self, block)
+      puts "Building all from DB complete." if opts_c.verbose
+    end
+
+    def set_current_klass_name(query_or_klass)
+      klass = query_or_klass.is_a?(ActiveRecord::Relation) ? query_or_klass.klass : query_or_klass
+      @current_klass_name = klass.name
+    end
   end
 end

data/lib/bulk_dependency_eraser/deleter.rb

@@ -3,8 +3,13 @@ module BulkDependencyEraser
     DEFAULT_DB_DELETE_ALL_WRAPPER = ->(deleter, block) do
       begin
         block.call
-      rescue StandardError => e
-        report_error("Issue attempting to delete '#{current_class_name}': #{e.class.name} - #{e.message}")
+      rescue BulkDependencyEraser::Errors::DeleterError => e
+        deleter.report_error(
+          <<~STRING.strip
+          Issue attempting to delete klass '#{e.deleting_klass_name}'
+            => #{e.original_error_klass.name}: #{e.message}
+          STRING
+        )
       end
     end
 
@@ -60,23 +65,27 @@ module BulkDependencyEraser
 
       current_class_name = 'N/A'
       delete_all_in_db do
-        class_names_and_ids.keys.reverse.each do |class_name|
-          current_class_name = class_name
-          ids = class_names_and_ids[class_name].reverse
-          klass = constantize(class_name)
+        begin
+          class_names_and_ids.keys.reverse.each do |class_name|
+            current_class_name = class_name
+            ids = class_names_and_ids[class_name].reverse
+            klass = constantize(class_name)
 
-          if opts_c.enable_invalid_foreign_key_detection
-            # delete with referential integrity
-            delete_by_klass_and_ids(klass, ids)
-          else
-            # delete without referential integrity
-            # Disable any ActiveRecord::InvalidForeignKey raised errors.
-            # - src: https://stackoverflow.com/questions/41005849/rails-migrations-temporarily-ignore-foreign-key-constraint
-            #        https://apidock.com/rails/ActiveRecord/ConnectionAdapters/AbstractAdapter/disable_referential_integrity
-            ActiveRecord::Base.connection.disable_referential_integrity do
+            if opts_c.enable_invalid_foreign_key_detection
+              # delete with referential integrity
               delete_by_klass_and_ids(klass, ids)
+            else
+              # delete without referential integrity
+              # Disable any ActiveRecord::InvalidForeignKey raised errors.
+              # - src: https://stackoverflow.com/questions/41005849/rails-migrations-temporarily-ignore-foreign-key-constraint
+              #        https://apidock.com/rails/ActiveRecord/ConnectionAdapters/AbstractAdapter/disable_referential_integrity
+              ActiveRecord::Base.connection.disable_referential_integrity do
+                delete_by_klass_and_ids(klass, ids)
+              end
             end
           end
+        rescue StandardError => e
+          raise BulkDependencyEraser::Errors::DeleterError.new(e.class, e.message, deleting_klass_name: current_class_name)
         end
       end
 

data/lib/bulk_dependency_eraser/errors/base_error.rb

@@ -0,0 +1,5 @@
+module BulkDependencyEraser
+  module Errors
+    class BaseError < StandardError; end
+  end
+end

data/lib/bulk_dependency_eraser/errors/builder_error.rb

@@ -0,0 +1,13 @@
+module BulkDependencyEraser
+  module Errors
+    class BuilderError < BaseError
+      attr_reader :original_error_klass, :building_klass_name
+
+      def initialize(original_error_klass, message, building_klass_name:)
+        @original_error_klass = original_error_klass
+        @building_klass_name = building_klass_name
+        super(message)
+      end
+    end
+  end
+end

data/lib/bulk_dependency_eraser/errors/deleter_error.rb

@@ -0,0 +1,13 @@
+module BulkDependencyEraser
+  module Errors
+    class DeleterError < BaseError
+      attr_reader :original_error_klass, :deleting_klass_name
+
+      def initialize(original_error_klass, message, deleting_klass_name:)
+        @original_error_klass = original_error_klass
+        @deleting_klass_name = deleting_klass_name
+        super(message)
+      end
+    end
+  end
+end

data/lib/bulk_dependency_eraser/errors/nullifier_error.rb

@@ -0,0 +1,14 @@
+module BulkDependencyEraser
+  module Errors
+    class NullifierError < BaseError
+      attr_reader :original_error_klass, :nullifying_klass_name, :nullifying_columns
+
+      def initialize(original_error_klass, message, nullifying_klass_name:, nullifying_columns:)
+        @original_error_klass = original_error_klass
+        @nullifying_klass_name = nullifying_klass_name
+        @nullifying_columns = nullifying_columns
+        super(message)
+      end
+    end
+  end
+end

data/lib/bulk_dependency_eraser/nullifier.rb

@@ -3,8 +3,13 @@ module BulkDependencyEraser
     DEFAULT_DB_NULLIFY_ALL_WRAPPER = ->(nullifier, block) do
       begin
         block.call
-      rescue StandardError => e
-        nullifier.report_error("Issue attempting to nullify: #{e.class.name} - #{e.message}")
+      rescue BulkDependencyEraser::Errors::NullifierError => e
+        nullifier.report_error(
+          <<~STRING.strip
+          Issue attempting to nullify klass '#{e.nullifying_klass_name}' on column(s) '#{e.nullifying_columns}'
+            => #{e.original_error_klass.name}: #{e.message}
+          STRING
+        )
       end
     end
 
@@ -114,6 +119,7 @@ module BulkDependencyEraser
       current_class_name = 'N/A'
       current_column = 'N/A'
       nullify_all_in_db do
+        begin
         # column_and_ids should have already been reversed in builder
         class_names_columns_and_ids.keys.reverse.each do |class_name|
           current_class_name = class_name
@@ -139,6 +145,14 @@ module BulkDependencyEraser
             end
           end
         end
+        rescue StandardError => e
+          raise BulkDependencyEraser::Errors::NullifierError.new(
+            e.class,
+            e.message,
+            nullifying_klass_name: current_class_name,
+            nullifying_columns: current_column.to_s # could be an array, string, or symbol
+          )
+        end
       end
 
       return errors.none?

data/lib/bulk_dependency_eraser/version.rb

@@ -1,3 +1,3 @@
 module BulkDependencyEraser
-  VERSION = "2.2.0".freeze
+  VERSION = "4.0.0".freeze
 end

data/lib/bulk_dependency_eraser.rb

@@ -6,4 +6,8 @@ require_relative 'bulk_dependency_eraser/nullifier'
 require_relative 'bulk_dependency_eraser/query_schema_parser'
 require_relative 'bulk_dependency_eraser/manager'
 require_relative 'bulk_dependency_eraser/utils'
-require_relative 'bulk_dependency_eraser/version'
+require_relative 'bulk_dependency_eraser/version'
+require_relative 'bulk_dependency_eraser/errors/base_error'
+require_relative 'bulk_dependency_eraser/errors/builder_error'
+require_relative 'bulk_dependency_eraser/errors/deleter_error'
+require_relative 'bulk_dependency_eraser/errors/nullifier_error'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bulk_dependency_eraser
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 4.0.0
 platform: ruby
 authors:
 - benjamin.dana.software.dev@gmail.com
@@ -146,6 +146,10 @@ files:
 - lib/bulk_dependency_eraser/base.rb
 - lib/bulk_dependency_eraser/builder.rb
 - lib/bulk_dependency_eraser/deleter.rb
+- lib/bulk_dependency_eraser/errors/base_error.rb
+- lib/bulk_dependency_eraser/errors/builder_error.rb
+- lib/bulk_dependency_eraser/errors/deleter_error.rb
+- lib/bulk_dependency_eraser/errors/nullifier_error.rb
 - lib/bulk_dependency_eraser/full_schema_parser.rb
 - lib/bulk_dependency_eraser/manager.rb
 - lib/bulk_dependency_eraser/nullifier.rb