bulk_dependency_eraser 4.3.0 → 4.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6cc809556aa24a68e132b8b18c5107452d98a7659594c8ae60807d4a41192941
-  data.tar.gz: e21b344baed301a3b8732d9b8b6d517fa38353de1169eb3842f752d1aec31be9
+  metadata.gz: dcd62dc450607a490d23f589f457c2e2ad30b6ba4096b775266372d2f3952445
+  data.tar.gz: 8e430d8e74cf2d401b75d335a7fc3da82672aa4fedbc0446024bf94c1f632116
 SHA512:
-  metadata.gz: 3715bf4c61448253a260e6b968c68bd4f27350f2fe960787c58c60023cade1a62af66ad611c1bf50d7c9c261c9d5f847d9fe28ebb3f31975a4dbe0122de5e56e
-  data.tar.gz: 38dc0a80a042241fbb5964f8da4c0228e4f978cbaba9ebb328b6bd25d78133519c5573b453120cabbc899f8e8b1a041511579e9c5f946a5b65b49a2350761e92
+  metadata.gz: f9added303e9a0982bd9cc026ef03937e0c095421fe89ba7edbdf24c7af27af61cce2f1e3c84b8f56d79fbf441ab5a4d72362fe9053b536af0286cb917c1abbc
+  data.tar.gz: 794831e885f3659586f5ffd075e93aee67d15e919bc2915023e733460e8739d4e5d7f86079f23feaf1cecb0a4a9ffaa6be5ccf885376203464fa7641952413c7

data/lib/bulk_dependency_eraser/builder.rb

@@ -58,6 +58,7 @@ module BulkDependencyEraser
       # Using PG system column CTID can result in a 10x deletion speed increase
       # - is used in combination with the primary key column to ensure CTID hasn't changed.
       use_pg_system_column_ctid: false,
+      delete_ctids_by_partions: false,
     }.freeze
 
     # write access so that these can be edited in-place by end-users who might need to manually adjust deletion order.
@@ -177,8 +178,14 @@ module BulkDependencyEraser
       end
 
       # Only pluck CTID if we're plucking column :id (the only primary key supported)
-      if columns == [:id] && opts_c.use_pg_system_column_ctid && skip_ctid_check == false
-        columns << :ctid
+      if columns == [:id] && skip_ctid_check == false
+        columns << :ctid if opts_c.use_pg_system_column_ctid
+        columns << 'tableoid::regclass' if opts_c.delete_ctids_by_partions # to support partioned tables
+      end
+
+      # Gem test ENV only! Swap out SQL column value with SQLite3 mocked, compatible value
+      if ENV['TEST_GEM_ENV'] == 'bulk_dependency_eraser' && columns.include?('tableoid::regclass')
+        columns[columns.index('tableoid::regclass')] = 'tableoid'
       end
 
       set_current_klass_name(query)
@@ -222,6 +229,11 @@ module BulkDependencyEraser
       # - query_results would just be an array of IDs
       return [query_results, nil] if columns.count == 1
 
+      # Gem test ENV only! Swap out SQL column value with SQLite3 mocked, compatible value
+      if ENV['TEST_GEM_ENV'] == 'bulk_dependency_eraser' && columns.include?('tableoid')
+        columns[columns.index('tableoid')] = 'tableoid::regclass'
+      end
+
       transposed_results = query_results.transpose
 
       query_primary_keys = transposed_results.shift || [] # could be an empty id set

data/lib/bulk_dependency_eraser/deleter.rb

@@ -55,6 +55,10 @@ module BulkDependencyEraser
       # - 1st priority of scopes
       deletion_proc_scopes_per_class_name: {},
       use_ctid_over_primary_key: false,
+      # Using PG system column CTID can result in a 10x deletion speed increase
+      # - is used in combination with the primary key column to ensure CTID hasn't changed.
+      use_pg_system_column_ctid: false,
+      delete_ctids_by_partions: false,
     }.freeze
 
     def initialize class_names_and_ids: {}, additional_identifiers_by_id: {}, opts: {}
@@ -131,61 +135,117 @@ module BulkDependencyEraser
     end
 
     # @param additional_identifiers [Hash] can be nil, if chosen to be so by implementers
+    # - Key: <ID> (primary_key)
+    # - Val: Hash of column/value pairings
     def delete_by_klass_and_ids(klass, ids, additional_identifiers:)
       puts "Deleting #{klass.name}'s IDs: #{ids}" if opts_c.verbose
       query = klass.unscoped
       query = custom_scope_for_query(query)
 
+      # Get column-names/keys of any additional identifer columns
+      detected_additional_identifier_columns = additional_identifiers&.values&.flat_map(&:keys)&.uniq || []
+
+      if opts_c.use_pg_system_column_ctid && opts_c.delete_ctids_by_partions && (detected_additional_identifier_columns & ['ctid', 'tableoid::regclass']).size == 2
+        partioned_tables = additional_identifiers.values.pluck('tableoid::regclass').uniq
+      end
+
       if batching_disabled?
         puts "Deleting without batching" if opts_c.verbose
-        # Get column-names/keys of any additional identifer columns
-        detected_additional_identifier_columns = additional_identifiers&.values&.flat_map(&:keys)&.uniq || []
-        delete_in_db do
-          deletion_query = query
-          if opts_c.use_ctid_over_primary_key && detected_additional_identifier_columns.include?('ctid')
-            # Do nothing, query will be deleted via ctid
-          else
-            deletion_query = deletion_query.where(id: ids)
-          end
-
-          # Apply any additional query identifiers (i.e. :ctid column)
-          detected_additional_identifier_columns.each do |column|
-            deletion_query = deletion_query.where(column => additional_identifiers.values.pluck(column))
+        # Delete by partioned CTIDs
+        if opts_c.use_pg_system_column_ctid && opts_c.delete_ctids_by_partions && (detected_additional_identifier_columns & ['ctid', 'tableoid::regclass']).size == 2
+          puts "Deleting via CTID and table partitions" if opts_c.verbose
+          partioned_tables.each do |table_partition|
+            identifiers_by_partition = additional_identifiers.filter { |id, identifiers| identifiers['tableoid::regclass'] == table_partition }
+            ctids = identifiers_by_partition.values.pluck('ctid')
+            delete_in_db do
+              sql = <<~SQL
+                DELETE FROM #{table_partition}
+                WHERE ctid IN (#{ctids.map { |c| "'#{c}'" }.join(', ')});
+              SQL
+              # Due to the way we've mocked/aliased the partition table in this gem test, we need to also add the partition name as a where clause.
+              if ENV['TEST_GEM_ENV'] == 'bulk_dependency_eraser'
+                sql.sub!(/;\s*$/, "") # remove terminating semicolon
+                sql << " AND tableoid = '#{table_partition}';"
+              end
+              deleted_count = klass.connection.delete(sql)
+              [deleted_count, sql, identifiers_by_partition.keys, identifiers_by_partition]
+            end
           end
-
-          # Perform Deletion
-          deletion_result = deletion_query.delete_all
-          # Returning the following data in the event that the gem-implementer wants to insert their own db_delete_wrapper proc
-          # and have access to these objects in their proc.
-            # - query can give them access to the klass and table_name
-          [deletion_result, query, ids, additional_identifiers]
-        end
-      else
-        puts "Deleting with batching" if opts_c.verbose
-        ids.each_slice(batch_size) do |ids_subset|
-          additional_identifiers_subset = additional_identifiers&.slice(*ids_subset)
-          # Get column-names/keys of any additional identifer columns
-          detected_additional_identifier_columns = additional_identifiers_subset&.values&.flat_map(&:keys)&.uniq || []
+        else
           delete_in_db do
             deletion_query = query
             if opts_c.use_ctid_over_primary_key && detected_additional_identifier_columns.include?('ctid')
               # Do nothing, query will be deleted via ctid
             else
-              deletion_query = deletion_query.where(id: ids_subset)
+              deletion_query = deletion_query.where(id: ids)
             end
 
             # Apply any additional query identifiers (i.e. :ctid column)
             detected_additional_identifier_columns.each do |column|
-              deletion_query = deletion_query.where(column => additional_identifiers_subset.values.pluck(column))
+              deletion_query = deletion_query.where(column => additional_identifiers.values.pluck(column))
             end
 
             # Perform Deletion
             deletion_result = deletion_query.delete_all
-
             # Returning the following data in the event that the gem-implementer wants to insert their own db_delete_wrapper proc
             # and have access to these objects in their proc.
-            # - query can give them access to the klass and table_name
-            [deletion_result, query, ids_subset, additional_identifiers_subset]
+              # - query can give them access to the klass and table_name
+            [deletion_result, query, ids, additional_identifiers]
+          end
+        end
+      else
+        puts "Deleting with batching" if opts_c.verbose
+        if opts_c.use_pg_system_column_ctid && opts_c.delete_ctids_by_partions && (detected_additional_identifier_columns & ['ctid', 'tableoid::regclass']).size == 2
+          puts "Deleting via CTID and table partitions" if opts_c.verbose
+          partioned_tables.each do |table_partition|
+            identifiers_by_partition = additional_identifiers.filter { |id, identifiers| identifiers['tableoid::regclass'] == table_partition }
+            ids = identifiers_by_partition.keys
+            ids.each_slice(batch_size) do |ids_subset|
+              additional_identifiers_subset = identifiers_by_partition.slice(*ids_subset)
+              ctids = additional_identifiers_subset.values.pluck('ctid')
+              delete_in_db do
+                sql = <<~SQL
+                  DELETE FROM #{table_partition}
+                  WHERE ctid IN (#{ctids.map { |c| "'#{c}'" }.join(', ')});
+                SQL
+
+                # Due to the way we've mocked/aliased the partition table in this gem test, we need to also add the partition name as a where clause.
+                if ENV['TEST_GEM_ENV'] == 'bulk_dependency_eraser'
+                  sql.sub!(/;\s*$/, "") # remove terminating semicolon
+                  sql << " AND tableoid = '#{table_partition}';"
+                end
+
+                deleted_count = klass.connection.delete(sql)
+                [deleted_count, sql, ids_subset, additional_identifiers_subset]
+              end
+            end
+          end
+        else
+          ids.each_slice(batch_size) do |ids_subset|
+            additional_identifiers_subset = additional_identifiers&.slice(*ids_subset)
+            # Get column-names/keys of any additional identifer columns
+            detected_additional_identifier_columns = additional_identifiers_subset&.values&.flat_map(&:keys)&.uniq || []
+            delete_in_db do
+              deletion_query = query
+              if opts_c.use_ctid_over_primary_key && detected_additional_identifier_columns.include?('ctid')
+                # Do nothing, query will be deleted via ctid
+              else
+                deletion_query = deletion_query.where(id: ids_subset)
+              end
+
+              # Apply any additional query identifiers (i.e. :ctid column)
+              detected_additional_identifier_columns.each do |column|
+                deletion_query = deletion_query.where(column => additional_identifiers_subset.values.pluck(column))
+              end
+
+              # Perform Deletion
+              deletion_result = deletion_query.delete_all
+
+              # Returning the following data in the event that the gem-implementer wants to insert their own db_delete_wrapper proc
+              # and have access to these objects in their proc.
+              # - query can give them access to the klass and table_name
+              [deletion_result, query, ids_subset, additional_identifiers_subset]
+            end
           end
         end
       end

data/lib/bulk_dependency_eraser/version.rb

@@ -1,3 +1,3 @@
 module BulkDependencyEraser
-  VERSION = "4.3.0".freeze
+  VERSION = "4.4.0".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bulk_dependency_eraser
 version: !ruby/object:Gem::Version
-  version: 4.3.0
+  version: 4.4.0
 platform: ruby
 authors:
 - benjamin.dana.software.dev@gmail.com