pg_online_schema_change 0.9.6 → 0.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 839fe414842291d9118a4115a56e06f1e0308ab3b7a740cafd9994314ded2d76
-  data.tar.gz: 22e9639093bf386f8a1e554e15e8d83df5e3c6de340c372f181fa3b280aee98d
+  metadata.gz: aed221e19c7fe40cf5a34844625c78b02c56fe13aff7f65183a1cff717185ad7
+  data.tar.gz: 81d3a564f43ebf08a7cac37fddd9a035d253e6dcc8c1c5d178c3c247e6357bc0
 SHA512:
-  metadata.gz: a160d19b3f1cb6655f70e0481b3293110deaee04d7a017ff9fe0c2b4d6100fd7d475520092db1b3bb5e068df5cb70243a00397a5cb1c2c259f2fe5b776b7e405
-  data.tar.gz: 9e4543fde48359fdc12046ff019420aafe51732f5cd1332285b3762fac3dcb198acc7be4f5fc6f6f90b910532758eb4a5752a8c9835d1f394774e6f77eda3898
+  metadata.gz: 5a4cfb3e17d007803cb84cf0686cadcfd0e7f7734ad20f6f5fa8b71a5198a24569a390ef3173f7bb518e345c7c4931c10812c053535d1df20ba774382c239f80
+  data.tar.gz: f82f723f5cac41a77c6adec6ddaa83c0377b172b3b283c05f1b81e7538c8427e182cdbac1db53e9358b39842685403b3a6e85cdb5a0371b6bffeef608e67a690

data/.rubocop.yml

@@ -215,6 +215,9 @@ RSpec/DescribeClass:
 RSpec/DescribedClass:
   Enabled: false
 
+Rspec/FilePath:
+  Enabled: false
+
 RSpec/SpecFilePathFormat:
   Enabled: false
 

data/.ruby-version

@@ -1 +1 @@
-3.1.3
+3.3.0

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## [0.9.7] - 2024-01-15
+
+- Introduce the ability to show estimated progress of copy - #146
+
+## [0.9.6] - 2023-11-04
+
+- Fix and add links to caveats section in #130
+- Refresh views across all schemas post swap in #134
+
 ## [0.9.5] - 2023-10-15
 
 - Validate one constraint at a time in #124

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_online_schema_change (0.9.6)
+    pg_online_schema_change (0.9.8)
       ougai (~> 2.0.0)
       pg (>= 1.3.2, < 1.6.0)
       pg_query (>= 2.1.3, < 4.3.0)
@@ -10,22 +10,25 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
+    abbrev (0.1.2)
     ast (2.4.2)
+    bigdecimal (3.1.5)
     coderay (1.1.3)
     diff-lcs (1.5.0)
-    google-protobuf (3.25.0-arm64-darwin)
-    haml (6.1.1)
+    google-protobuf (3.25.2)
+    haml (6.3.0)
       temple (>= 0.8.2)
       thor
       tilt
-    json (2.6.3)
+    json (2.7.1)
     language_server-protocol (3.17.0.3)
     method_source (1.0.0)
-    oj (3.16.1)
+    oj (3.16.3)
+      bigdecimal (>= 3.0)
     ougai (2.0.0)
       oj (~> 3.10)
-    parallel (1.23.0)
-    parser (3.2.2.4)
+    parallel (1.24.0)
+    parser (3.3.0.4)
       ast (~> 2.4.1)
       racc
     pg (1.5.4)
@@ -35,11 +38,12 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    racc (1.7.1)
+    racc (1.7.3)
     rainbow (3.1.1)
     rake (13.1.0)
-    rbs (3.1.0)
-    regexp_parser (2.8.2)
+    rbs (3.4.1)
+      abbrev
+    regexp_parser (2.9.0)
     rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -50,36 +54,36 @@ GEM
     rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.5)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-support (3.12.0)
-    rubocop (1.57.2)
+    rspec-support (3.12.1)
+    rubocop (1.60.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.4)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
     rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
-    rubocop-capybara (2.18.0)
+    rubocop-capybara (2.20.0)
+      rubocop (~> 1.41)
+    rubocop-factory_bot (2.25.1)
       rubocop (~> 1.41)
-    rubocop-factory_bot (2.23.1)
-      rubocop (~> 1.33)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.19.1)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
+    rubocop-performance (1.20.2)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.24.0)
-      rubocop (~> 1.33)
+    rubocop-rspec (2.26.1)
+      rubocop (~> 1.40)
       rubocop-capybara (~> 2.17)
       rubocop-factory_bot (~> 2.22)
     ruby-progressbar (1.13.0)
@@ -93,9 +97,9 @@ GEM
       prettier_print
       rbs
       syntax_tree (>= 2.0.1)
-    temple (0.10.0)
+    temple (0.10.3)
     thor (1.3.0)
-    tilt (2.1.0)
+    tilt (2.3.0)
     unicode-display_width (2.5.0)
 
 PLATFORMS

data/README.md

@@ -107,9 +107,9 @@ print the version
 - `pg-osc` acquires minimal locks throughout the process (read more below on the [caveats](#few-things-to-keep-in-mind)).
 - Copies over indexes and Foreign keys.
 - Optionally drop or retain old tables in the end.
+- Reduce bloat (since pg-osc creates a new table and drops the old one post swap).
 - Tune how slow or fast should replays be from the audit/log table ([Replaying larger workloads](#replaying-larger-workloads)).
 - Backfill old/new columns as data is copied from primary table to shadow table, and then perform the swap. [Example](#backfill-data)
-- **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-osc/issues/14)
 
 ## Load test
 

data/lib/pg_online_schema_change/client.rb

@@ -67,5 +67,9 @@ module PgOnlineSchemaChange
 
       @copy_statement = File.binread(file_path)
     end
+
+    def checkout_connection
+      PG.connect(dbname: dbname, host: host, user: username, password: password, port: port)
+    end
   end
 end

data/lib/pg_online_schema_change/orchestrate.rb

@@ -5,6 +5,7 @@ require "securerandom"
 module PgOnlineSchemaChange
   class Orchestrate
     SWAP_STATEMENT_TIMEOUT = "5s"
+    TRACK_PROGRESS_INTERVAL = 60 # seconds
 
     extend Helper
 
@@ -58,6 +59,10 @@ module PgOnlineSchemaChange
 
         raise Error, "Parent table has no primary key, exiting..." if primary_key.nil?
 
+        logger.info("Performing some house keeping....")
+        run_analyze!
+        run_vacuum!
+
         setup_audit_table!
 
         setup_trigger!
@@ -159,6 +164,12 @@ module PgOnlineSchemaChange
       end
 
       def setup_shadow_table!
+        logger.info("Setting up shadow table", { shadow_table: shadow_table })
+        Query.run(
+          client.connection,
+          "SELECT create_table_all('#{client.table_name}', '#{shadow_table}');",
+        )
+
         # re-uses transaction with serializable
         # This ensures that all queries from here till copy_data run with serializable.
         # This is to to ensure that once the trigger is added to the primay table
@@ -168,13 +179,6 @@ module PgOnlineSchemaChange
         # adding the trigger, till the copy ends, since they all happen in the
         # same serializable transaction.
         Query.run(client.connection, "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", true)
-        logger.info("Setting up shadow table", { shadow_table: shadow_table })
-
-        Query.run(
-          client.connection,
-          "SELECT create_table_all('#{client.table_name}', '#{shadow_table}');",
-          true,
-        )
 
         # update serials
         Query.run(
@@ -208,19 +212,44 @@ module PgOnlineSchemaChange
         )
         Query.run(client.connection, "DELETE FROM #{audit_table}", true)
 
-        logger.info(
-          "Copying contents..",
-          { shadow_table: shadow_table, parent_table: client.table_name },
-        )
         if client.copy_statement
           query = format(client.copy_statement, shadow_table: shadow_table)
           return Query.run(client.connection, query, true)
         end
 
+        logger.info(
+          "Copying contents..",
+          { shadow_table: shadow_table, parent_table: client.table_name },
+        )
+
+        @copy_finished = false
+        log_progress
+
         sql = Query.copy_data_statement(client, shadow_table, true)
         Query.run(client.connection, sql, true)
       ensure
         Query.run(client.connection, "COMMIT;") # commit the serializable transaction
+        @copy_finished = true
+      end
+
+      def log_progress
+        new_connection = client.checkout_connection
+        source_table_size = Query.get_table_size(new_connection, client.schema, client.table_name)
+
+        Thread.new do
+          loop do
+            break if @copy_finished
+
+            shadow_table_size = Query.get_table_size(new_connection, client.schema, shadow_table)
+            progress = (shadow_table_size.to_f / source_table_size) * 100
+            logger.info("Estimated copy progress: #{progress.round(2)}% complete")
+
+            break if @copy_finished || progress >= 100
+            sleep(TRACK_PROGRESS_INTERVAL) unless ENV["CI"]
+          rescue StandardError => e
+            logger.info("Reporting progress failed: #{e.message}")
+          end
+        end
       end
 
       def replay_and_swap!
@@ -272,7 +301,13 @@ module PgOnlineSchemaChange
       def run_analyze!
         logger.info("Performing ANALYZE!")
 
-        Query.run(client.connection, "ANALYZE VERBOSE #{client.table_name};")
+        client.connection.async_exec("ANALYZE VERBOSE #{client.schema}.#{client.table_name};")
+      end
+
+      def run_vacuum!
+        logger.info("Performing VACUUM!")
+
+        client.connection.async_exec("VACUUM VERBOSE #{client.schema}.#{client.table_name};")
       end
 
       def validate_constraints!

data/lib/pg_online_schema_change/query.rb

@@ -66,7 +66,7 @@ module PgOnlineSchemaChange
         end
       end
 
-      def run(connection, query, reuse_trasaction = false, &block)
+      def run(connection, query, reuse_trasaction = false, &block) # rubocop:disable Style/ArgumentsForwarding
         if [PG::PQTRANS_INERROR, PG::PQTRANS_UNKNOWN].include?(connection.transaction_status)
           connection.cancel
         end
@@ -75,7 +75,7 @@ module PgOnlineSchemaChange
 
         connection.async_exec("BEGIN;")
 
-        result = connection.async_exec(query, &block)
+        result = connection.async_exec(query, &block) # rubocop:disable Style/ArgumentsForwarding
       rescue Exception # rubocop:disable Lint/RescueException
         connection.cancel if connection.transaction_status != PG::PQTRANS_IDLE
         connection.block
@@ -428,6 +428,15 @@ module PgOnlineSchemaChange
           SELECT setval((select pg_get_serial_sequence('#{shadow_table}', '#{primary_key}')), (SELECT max(#{primary_key}) FROM #{table}));
         SQL
       end
+
+      def get_table_size(connection, schema, table_name)
+        size_query = "SELECT pg_table_size('#{schema}.#{table_name}');"
+        result = run(connection, size_query).first
+        result["pg_table_size"].to_i
+      rescue StandardError => e
+        logger.error("Error getting table size: #{e.message}")
+        0
+      end
     end
   end
 end

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.9.6"
+  VERSION = "0.9.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 0.9.8
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-04 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -311,14 +311,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: Easy CLI tool for making non-blocking zero downtime schema changes in PostgreSQL