pg_online_schema_change 0.9.6 → 0.9.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 839fe414842291d9118a4115a56e06f1e0308ab3b7a740cafd9994314ded2d76
-  data.tar.gz: 22e9639093bf386f8a1e554e15e8d83df5e3c6de340c372f181fa3b280aee98d
+  metadata.gz: e1838409c592e1497790a6e02f883b857851450181854baf660b9b7ee3664c6e
+  data.tar.gz: 4f7e58eeafb2413ec774988b3190391033e819a444a25da3b1ff0fcbf6bf64e4
 SHA512:
-  metadata.gz: a160d19b3f1cb6655f70e0481b3293110deaee04d7a017ff9fe0c2b4d6100fd7d475520092db1b3bb5e068df5cb70243a00397a5cb1c2c259f2fe5b776b7e405
-  data.tar.gz: 9e4543fde48359fdc12046ff019420aafe51732f5cd1332285b3762fac3dcb198acc7be4f5fc6f6f90b910532758eb4a5752a8c9835d1f394774e6f77eda3898
+  metadata.gz: c769f18ab1815fa543246fd18bb3157add074894f15371abb88e1af61e9381eade6850e4fba3a7d6d3701711594ed10a9018c7fe1c45a5ffdaf501891bf7e548
+  data.tar.gz: 84667ac77dab1d97478d51d9635dfcacd38e23ffe59ef6171863ca52d69294af19672bd89dba49a6d325b1c64c4e76a5dc8c57bf3dfee1120e2bd4bec1ca9e21

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## [0.9.6] - 2023-11-04
+
+- Fix and add links to caveats section in #130
+- Refresh views across all schemas post swap in #134
+
 ## [0.9.5] - 2023-10-15
 
 - Validate one constraint at a time in #124

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_online_schema_change (0.9.6)
+    pg_online_schema_change (0.9.7)
       ougai (~> 2.0.0)
       pg (>= 1.3.2, < 1.6.0)
       pg_query (>= 2.1.3, < 4.3.0)
@@ -11,9 +11,11 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
+    bigdecimal (3.1.5)
     coderay (1.1.3)
     diff-lcs (1.5.0)
-    google-protobuf (3.25.0-arm64-darwin)
+    google-protobuf (3.25.2-arm64-darwin)
+    google-protobuf (3.25.2-x86_64-linux)
     haml (6.1.1)
       temple (>= 0.8.2)
       thor
@@ -21,7 +23,8 @@ GEM
     json (2.6.3)
     language_server-protocol (3.17.0.3)
     method_source (1.0.0)
-    oj (3.16.1)
+    oj (3.16.3)
+      bigdecimal (>= 3.0)
     ougai (2.0.0)
       oj (~> 3.10)
     parallel (1.23.0)

data/README.md

@@ -107,9 +107,9 @@ print the version
 - `pg-osc` acquires minimal locks throughout the process (read more below on the [caveats](#few-things-to-keep-in-mind)).
 - Copies over indexes and Foreign keys.
 - Optionally drop or retain old tables in the end.
+- Reduce bloat (since pg-osc creates a new table and drops the old one post swap).
 - Tune how slow or fast should replays be from the audit/log table ([Replaying larger workloads](#replaying-larger-workloads)).
 - Backfill old/new columns as data is copied from primary table to shadow table, and then perform the swap. [Example](#backfill-data)
-- **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-osc/issues/14)
 
 ## Load test
 

data/lib/pg_online_schema_change/client.rb

@@ -67,5 +67,9 @@ module PgOnlineSchemaChange
 
       @copy_statement = File.binread(file_path)
     end
+
+    def checkout_connection
+      PG.connect(dbname: dbname, host: host, user: username, password: password, port: port)
+    end
   end
 end

data/lib/pg_online_schema_change/orchestrate.rb

@@ -5,6 +5,7 @@ require "securerandom"
 module PgOnlineSchemaChange
   class Orchestrate
     SWAP_STATEMENT_TIMEOUT = "5s"
+    TRACK_PROGRESS_INTERVAL = 60 # seconds
 
     extend Helper
 
@@ -58,6 +59,10 @@ module PgOnlineSchemaChange
 
         raise Error, "Parent table has no primary key, exiting..." if primary_key.nil?
 
+        logger.info("Performing some house keeping....")
+        run_analyze!
+        run_vacuum!
+
         setup_audit_table!
 
         setup_trigger!
@@ -159,6 +164,12 @@ module PgOnlineSchemaChange
       end
 
       def setup_shadow_table!
+        logger.info("Setting up shadow table", { shadow_table: shadow_table })
+        Query.run(
+          client.connection,
+          "SELECT create_table_all('#{client.table_name}', '#{shadow_table}');",
+        )
+
         # re-uses transaction with serializable
         # This ensures that all queries from here till copy_data run with serializable.
         # This is to to ensure that once the trigger is added to the primay table
@@ -168,13 +179,6 @@ module PgOnlineSchemaChange
         # adding the trigger, till the copy ends, since they all happen in the
         # same serializable transaction.
         Query.run(client.connection, "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", true)
-        logger.info("Setting up shadow table", { shadow_table: shadow_table })
-
-        Query.run(
-          client.connection,
-          "SELECT create_table_all('#{client.table_name}', '#{shadow_table}');",
-          true,
-        )
 
         # update serials
         Query.run(
@@ -208,19 +212,44 @@ module PgOnlineSchemaChange
         )
         Query.run(client.connection, "DELETE FROM #{audit_table}", true)
 
-        logger.info(
-          "Copying contents..",
-          { shadow_table: shadow_table, parent_table: client.table_name },
-        )
         if client.copy_statement
           query = format(client.copy_statement, shadow_table: shadow_table)
           return Query.run(client.connection, query, true)
         end
 
+        logger.info(
+          "Copying contents..",
+          { shadow_table: shadow_table, parent_table: client.table_name },
+        )
+
+        @copy_finished = false
+        log_progress
+
         sql = Query.copy_data_statement(client, shadow_table, true)
         Query.run(client.connection, sql, true)
       ensure
         Query.run(client.connection, "COMMIT;") # commit the serializable transaction
+        @copy_finished = true
+      end
+
+      def log_progress
+        new_connection = client.checkout_connection
+        source_table_size = Query.get_table_size(new_connection, client.schema, client.table_name)
+
+        Thread.new do
+          loop do
+            break if @copy_finished
+
+            shadow_table_size = Query.get_table_size(new_connection, client.schema, shadow_table)
+            progress = (shadow_table_size.to_f / source_table_size) * 100
+            logger.info("Estimated copy progress: #{progress.round(2)}% complete")
+
+            break if @copy_finished || progress >= 100
+            sleep(TRACK_PROGRESS_INTERVAL) unless ENV["CI"]
+          rescue StandardError => e
+            logger.info("Reporting progress failed: #{e.message}")
+          end
+        end
       end
 
       def replay_and_swap!
@@ -272,7 +301,13 @@ module PgOnlineSchemaChange
       def run_analyze!
         logger.info("Performing ANALYZE!")
 
-        Query.run(client.connection, "ANALYZE VERBOSE #{client.table_name};")
+        client.connection.async_exec("ANALYZE VERBOSE #{client.schema}.#{client.table_name};")
+      end
+
+      def run_vacuum!
+        logger.info("Performing VACUUM!")
+
+        client.connection.async_exec("VACUUM VERBOSE #{client.schema}.#{client.table_name};")
       end
 
       def validate_constraints!

data/lib/pg_online_schema_change/query.rb

@@ -428,6 +428,15 @@ module PgOnlineSchemaChange
           SELECT setval((select pg_get_serial_sequence('#{shadow_table}', '#{primary_key}')), (SELECT max(#{primary_key}) FROM #{table}));
         SQL
       end
+
+      def get_table_size(connection, schema, table_name)
+        size_query = "SELECT pg_table_size('#{schema}.#{table_name}');"
+        result = run(connection, size_query).first
+        result["pg_table_size"].to_i
+      rescue StandardError => e
+        logger.error("Error getting table size: #{e.message}")
+        0
+      end
     end
   end
 end

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.9.6"
+  VERSION = "0.9.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 0.9.7
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-04 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -311,7 +311,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="