pg_online_schema_change 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6580f86cf1fc33883f10391990817fb8c885b03eb4ecd1cd3095d46fc2a6bff5
-  data.tar.gz: 28b702f57765278ed7cdf9f7590c2bbf957d2debe8ace180bacd2a3f2fad98e3
+  metadata.gz: 3d69a5ecadf6f0c3b3c27bb2e16e163cfbc6526c4c83b5cbdf4ebef73be62cf9
+  data.tar.gz: 5a8ac6b5491692d4680f40637550392ebe5e8a206e7e3b1729967a379231ee8a
 SHA512:
-  metadata.gz: b761947d35092a7479589ef2b1bc4f79290e5172d83e1f24bd23af37e4ecdfaf7e26a0d0b91dafed3ad68b6e3012dfcd9ff8f54c861f1270037e9582e1f9db7b
-  data.tar.gz: a4f8f44660858b0e3fb5d74847e8f4e46e46b5d5237cbea669dcb9880dd3cb6deae9c5b598120169769e57cdb07e67f72cfc0edd6d72ef8c46d8bc9f5284ded6
+  metadata.gz: 5b8960dae33e3cf5bace38cb45cacaa555d16433b15703ffda2f2338b5076e716c23271d5f08d77f033cfbe9c6f2ea450e764a8ce934bb3ec9bfd9327cf89735
+  data.tar.gz: f003a55abb25ad2bcecf5a4e3b2358bfac4b3c0c55c401f7500a8d92e2b5bd30ebb4891d934cec52a1449e65a4fd9d177a9f87a2493d72b3f949ef47493943e8

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## [0.2.0] - 2022-02-17
+
+- Use ISOLATION LEVEL SERIALIZABLE ([#42](https://github.com/shayonj/pg-online-schema-change/pull/42)) (props to @jfrost)
+
 ## [0.1.0] - 2022-02-16
 
 Initial release

data/README.md

@@ -1,9 +1,10 @@
 # pg-online-schema-change / pg-osc
 [![CircleCI](https://circleci.com/gh/shayonj/pg-online-schema-change/tree/main.svg?style=shield)](https://circleci.com/gh/shayonj/pg-online-schema-change/tree/main)
+[![Gem Version](https://badge.fury.io/rb/pg_online_schema_change.svg)](https://badge.fury.io/rb/pg_online_schema_change)
 
 pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads. 
 
-`pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it copies the contents from a primary table to a shadow table, performs the schema change on the shadow table and swaps the table names in the end while preserving all changes to the primary table using triggers (via audit table).
+`pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it creates a shadow table that looks structurally the same as the primary table, performs the schema change on the shadow table, copies contents from the primary table to the shadow table and swaps the table names in the end while preserving all changes to the primary table using triggers (via audit table).
 
 `pg-osc` is inspired by the design and workings of tools like `pg_repack` and `pt-online-schema-change` (MySQL). Read more below on [how does it work](#how-does-it-work), [prominent features](#prominent-features), the [caveats](#caveats) and [examples](#examples)
 
@@ -33,6 +34,8 @@ Or install it yourself as:
 ## Usage
 
 ```
+pg-online-schema-change help perform
+
 Usage:
   pg-online-schema-change perform -a, --alter-statement=ALTER_STATEMENT -d, --dbname=DBNAME -h, --host=HOST -p, --port=N -s, --schema=SCHEMA -u, --username=USERNAME -w, --password=PASSWORD
 

data/lib/pg_online_schema_change/orchestrate.rb

@@ -37,10 +37,10 @@ module PgOnlineSchemaChange
 
         setup_audit_table!
         setup_trigger!
-        setup_shadow_table!
-        disable_vacuum!
-        run_alter_statement!
-        copy_data!
+        setup_shadow_table! # re-uses transaction with serializable
+        disable_vacuum! # re-uses transaction with serializable
+        run_alter_statement! # re-uses transaction with serializable
+        copy_data! # re-uses transaction with serializable
         run_analyze!
         replay_and_swap!
         run_analyze!
@@ -132,17 +132,27 @@ module PgOnlineSchemaChange
       end
 
       def setup_shadow_table!
+        # re-uses transaction with serializable
+        # This ensures that all queries from here till copy_data run with serializable.
+        # This is to to ensure that once the trigger is added to the primay table
+        # and contents being copied into the shadow, after a delete all on audit table,
+        # any replaying of rows that happen next from audit table do not contain
+        # any duplicates. We are ensuring there are no race conditions between
+        # adding the trigger, till the copy ends, since they all happen in the
+        # same serializable transaction.
+        Query.run(client.connection, "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", true)
         logger.info("Setting up shadow table", { shadow_table: shadow_table })
 
-        Query.run(client.connection, "SELECT create_table_all('#{client.table}', '#{shadow_table}');")
+        Query.run(client.connection, "SELECT create_table_all('#{client.table}', '#{shadow_table}');", true)
 
         # update serials
-        Query.run(client.connection, "SELECT fix_serial_sequence('#{client.table}', '#{shadow_table}');")
+        Query.run(client.connection, "SELECT fix_serial_sequence('#{client.table}', '#{shadow_table}');", true)
       end
 
-      # Disabling vacuum to avoid any issues during the process
       def disable_vacuum!
-        result = Query.storage_parameters_for(client, client.table) || ""
+        # re-uses transaction with serializable
+        # Disabling vacuum to avoid any issues during the process
+        result = Query.storage_parameters_for(client, client.table, true) || ""
         primary_table_storage_parameters = Store.set(:primary_table_storage_parameters, result)
 
         logger.debug("Disabling vacuum on shadow and audit table",
@@ -156,32 +166,39 @@ module PgOnlineSchemaChange
             autovacuum_enabled = false, toast.autovacuum_enabled = false
           );
         SQL
-        Query.run(client.connection, sql)
+        Query.run(client.connection, sql, true)
       end
 
       def run_alter_statement!
+        # re-uses transaction with serializable
         statement = Query.alter_statement_for(client, shadow_table)
         logger.info("Running alter statement on shadow table",
                     { shadow_table: shadow_table, parent_table: client.table })
-        Query.run(client.connection, statement)
+        Query.run(client.connection, statement, true)
 
         Store.set(:dropped_columns_list, Query.dropped_columns(client))
         Store.set(:renamed_columns_list, Query.renamed_columns(client))
       end
 
-      # Begin the process to copy data into copy table
-      # depending on the size of the table, this can be a time
-      # taking operation.
       def copy_data!
-        logger.info("Copying contents..", { shadow_table: shadow_table, parent_table: client.table })
+        # re-uses transaction with serializable
+        # Begin the process to copy data into copy table
+        # depending on the size of the table, this can be a time
+        # taking operation.
+        logger.info("Clearing contents of audit table before copy..",
+                    { shadow_table: shadow_table, parent_table: client.table })
+        Query.run(client.connection, "DELETE FROM #{audit_table}", true)
 
+        logger.info("Copying contents..", { shadow_table: shadow_table, parent_table: client.table })
         if client.copy_statement
           query = format(client.copy_statement, shadow_table: shadow_table)
-          return Query.run(client.connection, query)
+          return Query.run(client.connection, query, true)
         end
 
         sql = Query.copy_data_statement(client, shadow_table)
-        Query.run(client.connection, sql)
+        Query.run(client.connection, sql, true)
+      ensure
+        Query.run(client.connection, "COMMIT;") # commit the serializable transaction
       end
 
       def replay_and_swap!

data/lib/pg_online_schema_change/query.rb

@@ -60,7 +60,7 @@ module PgOnlineSchemaChange
         result
       end
 
-      def table_columns(client, table = nil)
+      def table_columns(client, table = nil, reuse_trasaction = false)
         sql = <<~SQL
           SELECT attname as column_name, format_type(atttypid, atttypmod) as type, attnum as column_position FROM   pg_attribute
           WHERE  attrelid = \'#{table || client.table}\'::regclass AND attnum > 0 AND NOT attisdropped
@@ -68,7 +68,7 @@ module PgOnlineSchemaChange
         SQL
         mapped_columns = []
 
-        run(client.connection, sql) do |result|
+        run(client.connection, sql, reuse_trasaction) do |result|
           mapped_columns = result.map do |row|
             row["column_name_regular"] = row["column_name"]
             row["column_name"] = client.connection.quote_ident(row["column_name"])
@@ -210,13 +210,13 @@ module PgOnlineSchemaChange
         columns.first
       end
 
-      def storage_parameters_for(client, table)
+      def storage_parameters_for(client, table, reuse_trasaction = false)
         query = <<~SQL
           SELECT array_to_string(reloptions, ',') as params FROM pg_class WHERE relname=\'#{table}\';
         SQL
 
         columns = []
-        run(client.connection, query) do |result|
+        run(client.connection, query, reuse_trasaction) do |result|
           columns = result.map { |row| row["params"] }
         end
 
@@ -266,8 +266,8 @@ module PgOnlineSchemaChange
         run(client.connection, query, true)
       end
 
-      def copy_data_statement(client, shadow_table)
-        select_columns = table_columns(client).map do |entry|
+      def copy_data_statement(client, shadow_table, reuse_trasaction = false)
+        select_columns = table_columns(client, client.table, reuse_trasaction).map do |entry|
           entry["column_name_regular"]
         end
 

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-16 00:00:00.000000000 Z
+date: 2022-02-17 00:00:00.000000000 Z
 dependencies: []
 description: pg-online-schema-change (pg-osc) is a tool for making schema changes
   (any ALTER statements) in Postgres tables with minimal locks, thus helping achieve