pg_online_schema_change 0.7.3 → 0.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70f01145ab63bcb06e4aea9e79b8dc5ac1bf5cc37749964589d2dc940214d8c4
-  data.tar.gz: 52f2b7e5fe1b793f1f8a414972d282a41661eb0045b65d095b65b4c9ab2d6a2a
+  metadata.gz: 973cb542595ffacbaa942636be7b366e018a2141a75c1fd73e38e65e263ddd39
+  data.tar.gz: abe5178f45a6c8a00c0b64a5cee543b7fc22e126d7a5744adefada83fa6ab844
 SHA512:
-  metadata.gz: d4d0c36d71d4aa8020063c63bba2358ffd2d088ad9da21ce3419a8916acbd74c80dfefd5a134423e708ef6b72f444dcbf0ebfd6ba4ff26956d562c258a94fbcc
-  data.tar.gz: c73369d1a69a0824474d5a930411b2f2eee0e7ac61986fb7b8df47d84e010da2cb35fec6e054e5e8986e6877f1992f256b420404a41f1cdd85620f8f1b9a3bb1
+  metadata.gz: 718ae2f4b7c1f0319f9c36d5c5b6f24441ac94e108fb281d357cec5790ee68ba4fdd1237aae4d414e284ccbf8228796027b9e2d6ffb19730bfe8260acb8120ba
+  data.tar.gz: 030dbca6e2f7b2a4f117a6b639f7c1454e7df5514adf4c81f3957d0a2310ce62042ea16d9707f6b7f7590c4f2b04f4276de33100bfc7a1611a1df388a21f8e5e

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2022-09-24 14:22:44 UTC using RuboCop version 1.23.0.
+# on 2023-03-12 14:28:52 UTC using RuboCop version 1.23.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -9,13 +9,18 @@
 # Offense count: 2
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 266
+  Max: 285
 
 # Offense count: 2
 # Configuration parameters: IgnoredMethods.
 Metrics/CyclomaticComplexity:
   Max: 15
 
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ModuleLength:
+  Max: 112
+
 # Offense count: 2
 # Configuration parameters: IgnoredMethods.
 Metrics/PerceivedComplexity:
@@ -26,14 +31,14 @@ Packaging/GemspecGit:
   Exclude:
     - 'pg_online_schema_change.gemspec'
 
-# Offense count: 68
+# Offense count: 69
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
   Max: 55
 
-# Offense count: 24
+# Offense count: 25
 RSpec/MultipleExpectations:
-  Max: 13
+  Max: 14
 
 # Offense count: 6
 # Configuration parameters: AllowedMethods.

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 
+
+## [0.7.4] - 2022-09-24
+* off-by-one: Don't skip PK sequence value by one https://github.com/shayonj/pg-osc/pull/74
+
+## [0.7.3] - 2022-09-24
+* Update primary key sequence on shadow table https://github.com/shayonj/pg-osc/pull/72
+  - Thanks to @brycethornton for the report
+* Only refresh primary key when a sequence is attached https://github.com/shayonj/pg-osc/pull/73
+
 ## [0.7.2] - 2022-09-17
+**NOTE: Skip to 0.7.3. 0.7.2 release missed the change.**
 * Update primary key sequence on shadow table https://github.com/shayonj/pg-osc/pull/72
   - Thanks to @brycethornton for the report
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_online_schema_change (0.7.3)
+    pg_online_schema_change (0.7.5)
       ougai (~> 2.0.0)
       pg (~> 1.3.2)
       pg_query (~> 2.1.3)
@@ -13,37 +13,37 @@ GEM
     ast (2.4.2)
     coderay (1.1.3)
     diff-lcs (1.5.0)
-    google-protobuf (3.19.4)
+    google-protobuf (3.21.7)
     method_source (1.0.0)
-    oj (3.13.11)
+    oj (3.13.21)
     ougai (2.0.0)
       oj (~> 3.10)
-    parallel (1.21.0)
-    parser (3.0.3.2)
+    parallel (1.22.1)
+    parser (3.1.2.1)
       ast (~> 2.4.1)
-    pg (1.3.4)
-    pg_query (2.1.3)
+    pg (1.3.5)
+    pg_query (2.1.4)
       google-protobuf (>= 3.19.2)
     pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rainbow (3.0.0)
+    rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.2.0)
+    regexp_parser (2.5.0)
     rexml (3.2.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.3)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.1)
     rubocop (1.23.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
@@ -53,8 +53,8 @@ GEM
       rubocop-ast (>= 1.12.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.15.1)
-      parser (>= 3.0.1.1)
+    rubocop-ast (1.21.0)
+      parser (>= 3.1.1.0)
     rubocop-packaging (0.5.1)
       rubocop (>= 0.89, < 2.0)
     rubocop-performance (1.12.0)
@@ -66,7 +66,7 @@ GEM
       rubocop (~> 1.19)
     ruby-progressbar (1.11.0)
     thor (1.2.1)
-    unicode-display_width (2.1.0)
+    unicode-display_width (2.3.0)
 
 PLATFORMS
   arm64-darwin-20
@@ -85,4 +85,4 @@ DEPENDENCIES
   rubocop-rspec (~> 2.7.0)
 
 BUNDLED WITH
-   2.3.3
+   2.3.24

data/docker-compose.yml

@@ -6,4 +6,4 @@ services:
       - "5432:5432"
     environment:
       POSTGRES_USER: jamesbond
-      POSTGRES_DB: postgres
+      POSTGRES_DB: postgres

data/lib/pg_online_schema_change/client.rb

@@ -4,7 +4,7 @@ require "pg"
 
 module PgOnlineSchemaChange
   class Client
-    attr_accessor :alter_statement, :schema, :dbname, :host, :username, :port, :password, :connection, :table, :drop,
+    attr_accessor :alter_statement, :schema, :dbname, :host, :username, :port, :password, :connection, :table, :table_name, :drop,
                   :kill_backends, :wait_time_for_lock, :copy_statement, :pull_batch_count, :delta_count
 
     def initialize(options)
@@ -33,6 +33,7 @@ module PgOnlineSchemaChange
       )
 
       @table = Query.table(@alter_statement)
+      @table_name = Query.table_name(@alter_statement, @table)
 
       PgOnlineSchemaChange.logger.debug("Connection established")
     end

data/lib/pg_online_schema_change/helper.rb

@@ -6,7 +6,7 @@ module PgOnlineSchemaChange
       result = Store.get(:primary_key)
       return result if result
 
-      Store.set(:primary_key, Query.primary_key_for(client, client.table))
+      Store.set(:primary_key, Query.primary_key_for(client, client.table_name))
     end
 
     def logger

data/lib/pg_online_schema_change/orchestrate.rb

@@ -30,16 +30,17 @@ module PgOnlineSchemaChange
         # Set this early on to ensure their creation and cleanup (unexpected)
         # happens at all times. IOW, the calls from Store.get always return
         # the same value.
-        Store.set(:old_primary_table, "pgosc_op_table_#{client.table}")
-        Store.set(:audit_table, "pgosc_at_#{client.table}_#{pgosc_identifier}")
+        Store.set(:old_primary_table, "pgosc_op_table_#{client.table.downcase}")
+        Store.set(:audit_table, "pgosc_at_#{client.table.downcase}_#{pgosc_identifier}")
         Store.set(:operation_type_column, "operation_type_#{pgosc_identifier}")
         Store.set(:trigger_time_column, "trigger_time_#{pgosc_identifier}")
         Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
         Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
-        Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
+        Store.set(:shadow_table, "pgosc_st_#{client.table.downcase}_#{pgosc_identifier}")
 
-        Store.set(:referential_foreign_key_statements, Query.referential_foreign_keys_to_refresh(client, client.table))
-        Store.set(:self_foreign_key_statements, Query.self_foreign_keys_to_refresh(client, client.table))
+        Store.set(:referential_foreign_key_statements, Query.referential_foreign_keys_to_refresh(client, client.table_name))
+        Store.set(:self_foreign_key_statements, Query.self_foreign_keys_to_refresh(client, client.table_name))
+        Store.set(:trigger_statements, Query.get_triggers_for(client, client.table_name))
       end
 
       def run!(options)
@@ -49,6 +50,7 @@ module PgOnlineSchemaChange
         raise Error, "Parent table has no primary key, exiting..." if primary_key.nil?
 
         setup_audit_table!
+
         setup_trigger!
         setup_shadow_table! # re-uses transaction with serializable
         disable_vacuum! # re-uses transaction with serializable
@@ -98,7 +100,7 @@ module PgOnlineSchemaChange
         logger.info("Setting up audit table", { audit_table: audit_table })
 
         sql = <<~SQL
-          CREATE TABLE #{audit_table} (#{audit_table_pk} SERIAL PRIMARY KEY, #{operation_type_column} text, #{trigger_time_column} timestamp, LIKE #{client.table});
+          CREATE TABLE #{audit_table} (#{audit_table_pk} SERIAL PRIMARY KEY, #{operation_type_column} text, #{trigger_time_column} timestamp, LIKE #{client.table_name});
         SQL
 
         Query.run(client.connection, sql)
@@ -108,14 +110,14 @@ module PgOnlineSchemaChange
         # acquire access exclusive lock to ensure audit triggers
         # are setup fine. This also calls kill_backends (if opted in via flag)
         # so any competing backends will be killed to setup the trigger
-        opened = Query.open_lock_exclusive(client, client.table)
+        opened = Query.open_lock_exclusive(client, client.table_name)
 
         raise AccessExclusiveLockNotAcquired unless opened
 
         logger.info("Setting up triggers")
 
         sql = <<~SQL
-          DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
+          DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table_name};
 
           CREATE OR REPLACE FUNCTION primary_to_audit_table_trigger()
           RETURNS TRIGGER AS
@@ -135,7 +137,7 @@ module PgOnlineSchemaChange
           $$ LANGUAGE PLPGSQL SECURITY DEFINER;
 
           CREATE TRIGGER primary_to_audit_table_trigger
-          AFTER INSERT OR UPDATE OR DELETE ON #{client.table}
+          AFTER INSERT OR UPDATE OR DELETE ON #{client.table_name}
           FOR EACH ROW EXECUTE PROCEDURE primary_to_audit_table_trigger();
         SQL
 
@@ -156,16 +158,16 @@ module PgOnlineSchemaChange
         Query.run(client.connection, "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", true)
         logger.info("Setting up shadow table", { shadow_table: shadow_table })
 
-        Query.run(client.connection, "SELECT create_table_all('#{client.table}', '#{shadow_table}');", true)
+        Query.run(client.connection, "SELECT create_table_all('#{client.table_name}', '#{shadow_table}');", true)
 
         # update serials
-        Query.run(client.connection, "SELECT fix_serial_sequence('#{client.table}', '#{shadow_table}');", true)
+        Query.run(client.connection, "SELECT fix_serial_sequence('#{client.table_name}', '#{shadow_table}');", true)
       end
 
       def disable_vacuum!
         # re-uses transaction with serializable
         # Disabling vacuum to avoid any issues during the process
-        result = Query.storage_parameters_for(client, client.table, true) || ""
+        result = Query.storage_parameters_for(client, client.table_name, true) || ""
         Store.set(:primary_table_storage_parameters, result)
 
         logger.debug("Disabling vacuum on shadow and audit table",
@@ -186,7 +188,7 @@ module PgOnlineSchemaChange
         # re-uses transaction with serializable
         statement = Query.alter_statement_for(client, shadow_table)
         logger.info("Running alter statement on shadow table",
-                    { shadow_table: shadow_table, parent_table: client.table })
+                    { shadow_table: shadow_table, parent_table: client.table_name })
         Query.run(client.connection, statement, true)
 
         Store.set(:dropped_columns_list, Query.dropped_columns(client))
@@ -198,10 +200,10 @@ module PgOnlineSchemaChange
         # Begin the process to copy data into copy table
         # depending on the size of the table, this can be a time
         # taking operation.
-        logger.info("Clearing contents of audit table before copy..", { shadow_table: shadow_table, parent_table: client.table })
+        logger.info("Clearing contents of audit table before copy..", { shadow_table: shadow_table, parent_table: client.table_name })
         Query.run(client.connection, "DELETE FROM #{audit_table}", true)
 
-        logger.info("Copying contents..", { shadow_table: shadow_table, parent_table: client.table })
+        logger.info("Copying contents..", { shadow_table: shadow_table, parent_table: client.table_name })
         if client.copy_statement
           query = format(client.copy_statement, shadow_table: shadow_table)
           return Query.run(client.connection, query, true)
@@ -224,12 +226,12 @@ module PgOnlineSchemaChange
       def swap!
         logger.info("Performing swap!")
 
-        storage_params_reset = primary_table_storage_parameters.empty? ? "" : "ALTER TABLE #{client.table} SET (#{primary_table_storage_parameters});"
+        storage_params_reset = primary_table_storage_parameters.empty? ? "" : "ALTER TABLE #{client.table_name} SET (#{primary_table_storage_parameters});"
 
         # From here on, all statements are carried out in a single
         # transaction with access exclusive lock
 
-        opened = Query.open_lock_exclusive(client, client.table)
+        opened = Query.open_lock_exclusive(client, client.table_name)
 
         raise AccessExclusiveLockNotAcquired unless opened
 
@@ -238,16 +240,17 @@ module PgOnlineSchemaChange
         rows = Replay.rows_to_play(opened)
         Replay.play!(rows, opened)
 
-        query_for_primary_key_refresh = Query.query_for_primary_key_refresh(shadow_table, primary_key, client.table, opened)
+        query_for_primary_key_refresh = Query.query_for_primary_key_refresh(shadow_table, primary_key, client.table_name, opened)
 
         sql = <<~SQL
           #{query_for_primary_key_refresh};
-          ALTER TABLE #{client.table} RENAME to #{old_primary_table};
-          ALTER TABLE #{shadow_table} RENAME to #{client.table};
+          ALTER TABLE #{client.table_name} RENAME to #{old_primary_table};
+          ALTER TABLE #{shadow_table} RENAME to #{client.table_name};
           #{referential_foreign_key_statements}
           #{self_foreign_key_statements}
+          #{trigger_statements}
           #{storage_params_reset}
-          DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
+          DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table_name};
         SQL
 
         Query.run(client.connection, sql, opened)
@@ -259,13 +262,13 @@ module PgOnlineSchemaChange
       def run_analyze!
         logger.info("Performing ANALYZE!")
 
-        Query.run(client.connection, "ANALYZE VERBOSE #{client.table};")
+        Query.run(client.connection, "ANALYZE VERBOSE #{client.table_name};")
       end
 
       def validate_constraints!
         logger.info("Validating constraints!")
 
-        validate_statements = Query.get_foreign_keys_to_validate(client, client.table)
+        validate_statements = Query.get_foreign_keys_to_validate(client, client.table_name)
 
         Query.run(client.connection, validate_statements)
       end
@@ -276,7 +279,7 @@ module PgOnlineSchemaChange
         shadow_table_drop = shadow_table ? "DROP TABLE IF EXISTS #{shadow_table}" : ""
 
         sql = <<~SQL
-          DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
+          DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table_name};
           #{audit_table_drop};
           #{shadow_table_drop};
           #{primary_drop}

data/lib/pg_online_schema_change/query.rb

@@ -42,6 +42,15 @@ module PgOnlineSchemaChange
         PgQuery.parse(query).tables[0] || from_rename_statement
       end
 
+      def table_name(query, table)
+        table_name = "\"#{table}\""
+        if table =~ /[A-Z]/ && (query.include? table_name) && table[0] != '"'
+          table_name
+        else
+          table
+        end
+      end
+
       def run(connection, query, reuse_trasaction = false, &block)
         connection.cancel if [PG::PQTRANS_INERROR, PG::PQTRANS_UNKNOWN].include?(connection.transaction_status)
 
@@ -65,7 +74,7 @@ module PgOnlineSchemaChange
       def table_columns(client, table = nil, reuse_trasaction = false)
         sql = <<~SQL
           SELECT attname as column_name, format_type(atttypid, atttypmod) as type, attnum as column_position FROM   pg_attribute
-          WHERE  attrelid = \'#{table || client.table}\'::regclass AND attnum > 0 AND NOT attisdropped
+          WHERE  attrelid = \'#{table || client.table_name}\'::regclass AND attnum > 0 AND NOT attisdropped
           ORDER  BY attnum;
         SQL
         mapped_columns = []
@@ -108,6 +117,20 @@ module PgOnlineSchemaChange
         indexes
       end
 
+      def get_triggers_for(client, table)
+        query = <<~SQL
+          SELECT pg_get_triggerdef(oid) as tdef FROM pg_trigger
+          WHERE  tgrelid = \'#{client.schema}.#{table}\'::regclass AND tgisinternal = FALSE;
+        SQL
+
+        triggers = []
+        run(client.connection, query) do |result|
+          triggers = result.map { |row| "#{row["tdef"]};" }
+        end
+
+        triggers.join(";")
+      end
+
       def get_all_constraints_for(client)
         query = <<~SQL
           SELECT  conrelid::regclass AS table_on,
@@ -255,7 +278,7 @@ module PgOnlineSchemaChange
 
         query = <<~SQL
           SET lock_timeout = '#{client.wait_time_for_lock}s';
-          LOCK TABLE #{client.table} IN ACCESS EXCLUSIVE MODE;
+          LOCK TABLE #{client.table_name} IN ACCESS EXCLUSIVE MODE;
         SQL
         run(client.connection, query, true)
 
@@ -289,7 +312,7 @@ module PgOnlineSchemaChange
       end
 
       def copy_data_statement(client, shadow_table, reuse_trasaction = false)
-        select_columns = table_columns(client, client.table, reuse_trasaction).map do |entry|
+        select_columns = table_columns(client, client.table_name, reuse_trasaction).map do |entry|
           entry["column_name_regular"]
         end
 
@@ -316,7 +339,7 @@ module PgOnlineSchemaChange
         <<~SQL
           INSERT INTO #{shadow_table}(#{insert_into_columns.join(", ")})
           SELECT #{select_columns.join(", ")}
-          FROM ONLY #{client.table}
+          FROM ONLY #{client.table_name}
         SQL
       end
 
@@ -338,7 +361,7 @@ module PgOnlineSchemaChange
         return "" if sequence_name.nil?
 
         <<~SQL
-          SELECT setval((select pg_get_serial_sequence(\'#{shadow_table}\', \'#{primary_key}\')), (SELECT max(#{primary_key}) FROM #{table})+1);
+          SELECT setval((select pg_get_serial_sequence(\'#{shadow_table}\', \'#{primary_key}\')), (SELECT max(#{primary_key}) FROM #{table}));
         SQL
       end
     end

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.7.3"
+  VERSION = "0.7.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.5
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-24 00:00:00.000000000 Z
+date: 2023-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai