bulk_dependency_eraser 4.3.0 → 4.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6cc809556aa24a68e132b8b18c5107452d98a7659594c8ae60807d4a41192941
-  data.tar.gz: e21b344baed301a3b8732d9b8b6d517fa38353de1169eb3842f752d1aec31be9
+  metadata.gz: 886ff4e4e736393ca417bfca554cc0f0cac8dd3d2c78233e8cfdb9c007d4b0f1
+  data.tar.gz: d81c392e904f999442287d77349b52659dd14621da208ce5f942ff8f4529e5a3
 SHA512:
-  metadata.gz: 3715bf4c61448253a260e6b968c68bd4f27350f2fe960787c58c60023cade1a62af66ad611c1bf50d7c9c261c9d5f847d9fe28ebb3f31975a4dbe0122de5e56e
-  data.tar.gz: 38dc0a80a042241fbb5964f8da4c0228e4f978cbaba9ebb328b6bd25d78133519c5573b453120cabbc899f8e8b1a041511579e9c5f946a5b65b49a2350761e92
+  metadata.gz: e0d7a0e10751d57d39dfc99ed6639170af4484f228b9778b8a7187c2168004738c1faf538c5417cf0037862a48635937723677367226816291696f20fad0eaae
+  data.tar.gz: edbefd2f654115b2566ad2b1c5b38fb6ca2970a61f0c7371e9254896c6a34c6c81044020a5fc4ac32d6bf4199c96858e059c1ed2c16578ceac3ac0383ec09901

data/lib/bulk_dependency_eraser/builder.rb

@@ -58,6 +58,7 @@ module BulkDependencyEraser
       # Using PG system column CTID can result in a 10x deletion speed increase
       # - is used in combination with the primary key column to ensure CTID hasn't changed.
       use_pg_system_column_ctid: false,
+      delete_ctids_by_partions: false,
     }.freeze
 
     # write access so that these can be edited in-place by end-users who might need to manually adjust deletion order.
@@ -177,8 +178,14 @@ module BulkDependencyEraser
       end
 
       # Only pluck CTID if we're plucking column :id (the only primary key supported)
-      if columns == [:id] && opts_c.use_pg_system_column_ctid && skip_ctid_check == false
-        columns << :ctid
+      if columns == [:id] && skip_ctid_check == false
+        columns << :ctid if opts_c.use_pg_system_column_ctid
+        columns << 'tableoid::regclass' if opts_c.delete_ctids_by_partions # to support partioned tables
+      end
+
+      # Gem test ENV only! Swap out SQL column value with SQLite3 mocked, compatible value
+      if ENV['TEST_GEM_ENV'] == 'bulk_dependency_eraser' && columns.include?('tableoid::regclass')
+        columns[columns.index('tableoid::regclass')] = 'tableoid'
       end
 
       set_current_klass_name(query)
@@ -222,6 +229,11 @@ module BulkDependencyEraser
       # - query_results would just be an array of IDs
       return [query_results, nil] if columns.count == 1
 
+      # Gem test ENV only! Swap out SQL column value with SQLite3 mocked, compatible value
+      if ENV['TEST_GEM_ENV'] == 'bulk_dependency_eraser' && columns.include?('tableoid')
+        columns[columns.index('tableoid')] = 'tableoid::regclass'
+      end
+
       transposed_results = query_results.transpose
 
       query_primary_keys = transposed_results.shift || [] # could be an empty id set

data/lib/bulk_dependency_eraser/deleter.rb

@@ -55,6 +55,10 @@ module BulkDependencyEraser
       # - 1st priority of scopes
       deletion_proc_scopes_per_class_name: {},
       use_ctid_over_primary_key: false,
+      # Using PG system column CTID can result in a 10x deletion speed increase
+      # - is used in combination with the primary key column to ensure CTID hasn't changed.
+      use_pg_system_column_ctid: false,
+      delete_ctids_by_partions: false,
     }.freeze
 
     def initialize class_names_and_ids: {}, additional_identifiers_by_id: {}, opts: {}
@@ -131,63 +135,117 @@ module BulkDependencyEraser
     end
 
     # @param additional_identifiers [Hash] can be nil, if chosen to be so by implementers
+    # - Key: <ID> (primary_key)
+    # - Val: Hash of column/value pairings
     def delete_by_klass_and_ids(klass, ids, additional_identifiers:)
       puts "Deleting #{klass.name}'s IDs: #{ids}" if opts_c.verbose
       query = klass.unscoped
       query = custom_scope_for_query(query)
 
+      # Get column-names/keys of any additional identifer columns
+      detected_additional_identifier_columns = additional_identifiers&.values&.flat_map(&:keys)&.uniq || []
+
+      if opts_c.use_pg_system_column_ctid && opts_c.delete_ctids_by_partions && (detected_additional_identifier_columns & ['ctid', 'tableoid::regclass']).size == 2
+        deletable_partioned_tables_by_ctids = additional_identifiers.values.pluck('tableoid::regclass').uniq
+      end
+
       if batching_disabled?
         puts "Deleting without batching" if opts_c.verbose
-        # Get column-names/keys of any additional identifer columns
-        detected_additional_identifier_columns = additional_identifiers&.values&.flat_map(&:keys)&.uniq || []
-        delete_in_db do
-          deletion_query = query
-          if opts_c.use_ctid_over_primary_key && detected_additional_identifier_columns.include?('ctid')
-            # Do nothing, query will be deleted via ctid
-          else
-            deletion_query = deletion_query.where(id: ids)
-          end
-
-          # Apply any additional query identifiers (i.e. :ctid column)
-          detected_additional_identifier_columns.each do |column|
-            deletion_query = deletion_query.where(column => additional_identifiers.values.pluck(column))
+        # Delete by partioned CTIDs
+        if deletable_partioned_tables_by_ctids
+          puts "Deleting via CTID and table partitions" if opts_c.verbose
+          deletable_partioned_tables_by_ctids.each do |table_partition|
+            identifiers_by_partition = additional_identifiers.filter { |id, identifiers| identifiers['tableoid::regclass'] == table_partition }
+            ctids = identifiers_by_partition.values.pluck('ctid')
+            ids = identifiers_by_partition.values.pluck('id')
+            deletion_by_partitioned_ctids(query, ids, ctids, table_partition, identifiers_by_partition)
           end
-
-          # Perform Deletion
-          deletion_result = deletion_query.delete_all
-          # Returning the following data in the event that the gem-implementer wants to insert their own db_delete_wrapper proc
-          # and have access to these objects in their proc.
-            # - query can give them access to the klass and table_name
-          [deletion_result, query, ids, additional_identifiers]
-        end
-      else
-        puts "Deleting with batching" if opts_c.verbose
-        ids.each_slice(batch_size) do |ids_subset|
-          additional_identifiers_subset = additional_identifiers&.slice(*ids_subset)
-          # Get column-names/keys of any additional identifer columns
-          detected_additional_identifier_columns = additional_identifiers_subset&.values&.flat_map(&:keys)&.uniq || []
+        else
           delete_in_db do
             deletion_query = query
             if opts_c.use_ctid_over_primary_key && detected_additional_identifier_columns.include?('ctid')
               # Do nothing, query will be deleted via ctid
             else
-              deletion_query = deletion_query.where(id: ids_subset)
+              deletion_query = deletion_query.where(id: ids)
             end
 
             # Apply any additional query identifiers (i.e. :ctid column)
             detected_additional_identifier_columns.each do |column|
-              deletion_query = deletion_query.where(column => additional_identifiers_subset.values.pluck(column))
+              deletion_query = deletion_query.where(column => additional_identifiers.values.pluck(column))
             end
 
             # Perform Deletion
             deletion_result = deletion_query.delete_all
-
             # Returning the following data in the event that the gem-implementer wants to insert their own db_delete_wrapper proc
             # and have access to these objects in their proc.
-            # - query can give them access to the klass and table_name
-            [deletion_result, query, ids_subset, additional_identifiers_subset]
+              # - query can give them access to the klass and table_name
+            [deletion_result, query, ids, additional_identifiers]
           end
         end
+      else
+        puts "Deleting with batching" if opts_c.verbose
+        if deletable_partioned_tables_by_ctids
+          puts "Deleting via CTID and table partitions" if opts_c.verbose
+          deletable_partioned_tables_by_ctids.each do |table_partition|
+            identifiers_by_partition = additional_identifiers.filter { |id, identifiers| identifiers['tableoid::regclass'] == table_partition }
+            ids = identifiers_by_partition.keys
+            ids.each_slice(batch_size) do |ids_subset|
+              additional_identifiers_subset = identifiers_by_partition.slice(*ids_subset)
+              ctids = additional_identifiers_subset.values.pluck('ctid')
+              deletion_by_partitioned_ctids(query, ids_subset, ctids, table_partition, identifiers_by_partition)
+            end
+          end
+        else
+          ids.each_slice(batch_size) do |ids_subset|
+            additional_identifiers_subset = additional_identifiers&.slice(*ids_subset)
+            # Get column-names/keys of any additional identifer columns
+            detected_additional_identifier_columns = additional_identifiers_subset&.values&.flat_map(&:keys)&.uniq || []
+            delete_in_db do
+              deletion_query = query
+              if opts_c.use_ctid_over_primary_key && detected_additional_identifier_columns.include?('ctid')
+                # Do nothing, query will be deleted via ctid
+              else
+                deletion_query = deletion_query.where(id: ids_subset)
+              end
+
+              # Apply any additional query identifiers (i.e. :ctid column)
+              detected_additional_identifier_columns.each do |column|
+                deletion_query = deletion_query.where(column => additional_identifiers_subset.values.pluck(column))
+              end
+
+              # Perform Deletion
+              deletion_result = deletion_query.delete_all
+
+              # Returning the following data in the event that the gem-implementer wants to insert their own db_delete_wrapper proc
+              # and have access to these objects in their proc.
+              # - query can give them access to the klass and table_name
+              [deletion_result, query, ids_subset, additional_identifiers_subset]
+            end
+          end
+        end
+      end
+    end
+
+    def deletion_by_partitioned_ctids query, ids, ctids, partition_table_name, additional_identifiers
+      delete_in_db do
+        where_clause_sql = <<~SQL
+          ctid IN (#{ctids.map { |c| "'#{c}'" }.join(', ')})
+        SQL
+
+        deletion_sql = <<~SQL
+          DELETE FROM #{partition_table_name}
+          WHERE #{where_clause_sql};
+        SQL
+
+        # Due to the way we've mocked/aliased the partition table in this gem test, we need to also add the partition name as a where clause.
+        if ENV['TEST_GEM_ENV'] == 'bulk_dependency_eraser'
+          deletion_sql.sub!(/;\s*$/, "") # remove terminating semicolon
+          deletion_sql << " AND tableoid = '#{partition_table_name}';"
+        end
+
+        deleted_count = query.klass.connection.delete(deletion_sql)
+        selector_query = query.where(Arel.sql(where_clause_sql))
+        [deleted_count, selector_query, ids, additional_identifiers]
       end
     end
 

data/lib/bulk_dependency_eraser/version.rb

@@ -1,3 +1,3 @@
 module BulkDependencyEraser
-  VERSION = "4.3.0".freeze
+  VERSION = "4.4.1".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bulk_dependency_eraser
 version: !ruby/object:Gem::Version
-  version: 4.3.0
+  version: 4.4.1
 platform: ruby
 authors:
 - benjamin.dana.software.dev@gmail.com