pg_online_schema_change 0.9.5 → 0.9.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4b544005274743fc18f3147204997861e3ac55e7c619a6859849049eff42324
-  data.tar.gz: a4123844149ed79c7e8781588ca055f49954afa245a7a63c50886e076be9562e
+  metadata.gz: e1838409c592e1497790a6e02f883b857851450181854baf660b9b7ee3664c6e
+  data.tar.gz: 4f7e58eeafb2413ec774988b3190391033e819a444a25da3b1ff0fcbf6bf64e4
 SHA512:
-  metadata.gz: d5b08b1f26c9d3f3c19d819cf65d0787bbbd6221845ce44bcf2aaf13a0389f8824139d3034507cb8118751b2e09713ec4116322e8b86b8988276d6ab8f4a2390
-  data.tar.gz: e5e589f58797e28fbe15fd355367a61c2673f6d90ccadb67c9b43427ea0a5b86afba9da4ec5199dd4656397f4c7a5e8b27a3805876915d7eddb8366a19892a4b
+  metadata.gz: c769f18ab1815fa543246fd18bb3157add074894f15371abb88e1af61e9381eade6850e4fba3a7d6d3701711594ed10a9018c7fe1c45a5ffdaf501891bf7e548
+  data.tar.gz: 84667ac77dab1d97478d51d9635dfcacd38e23ffe59ef6171863ca52d69294af19672bd89dba49a6d325b1c64c4e76a5dc8c57bf3dfee1120e2bd4bec1ca9e21

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## [0.9.6] - 2023-11-04
+
+- Fix and add links to caveats section in #130
+- Refresh views across all schemas post swap in #134
+
+## [0.9.5] - 2023-10-15
+
+- Validate one constraint at a time in #124
+- Introduce --skip-foreign-key-validation in #125
+
 ## [0.9.4] - 2023-09-17
 
 - Resolving gem push and sync glitch in 0.9.3
@@ -13,6 +23,7 @@
 
 - Dependency updates
 - Create shadow and audit with auatovacuum default turned off. Should avoid lock queues when disabling vacuum on audit table. #97
+
 ## [0.9.1] - 2023-06-24
 
 - Dependency updates and refresh docker release process with multi-platform build

data/Gemfile.lock

@@ -1,21 +1,21 @@
 PATH
   remote: .
   specs:
-    pg_online_schema_change (0.9.5)
+    pg_online_schema_change (0.9.7)
       ougai (~> 2.0.0)
       pg (>= 1.3.2, < 1.6.0)
       pg_query (>= 2.1.3, < 4.3.0)
-      thor (~> 1.2.1)
+      thor (>= 1.2.1, < 1.4.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    base64 (0.1.1)
+    bigdecimal (3.1.5)
     coderay (1.1.3)
     diff-lcs (1.5.0)
-    google-protobuf (3.24.4-arm64-darwin)
-    google-protobuf (3.24.4-x86_64-linux)
+    google-protobuf (3.25.2-arm64-darwin)
+    google-protobuf (3.25.2-x86_64-linux)
     haml (6.1.1)
       temple (>= 0.8.2)
       thor
@@ -23,11 +23,12 @@ GEM
     json (2.6.3)
     language_server-protocol (3.17.0.3)
     method_source (1.0.0)
-    oj (3.16.1)
+    oj (3.16.3)
+      bigdecimal (>= 3.0)
     ougai (2.0.0)
       oj (~> 3.10)
     parallel (1.23.0)
-    parser (3.2.2.3)
+    parser (3.2.2.4)
       ast (~> 2.4.1)
       racc
     pg (1.5.4)
@@ -39,9 +40,9 @@ GEM
       method_source (~> 1.0)
     racc (1.7.1)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.1.0)
     rbs (3.1.0)
-    regexp_parser (2.8.1)
+    regexp_parser (2.8.2)
     rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -56,19 +57,18 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.56.4)
-      base64 (~> 0.1.1)
+    rubocop (1.57.2)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.3)
+      parser (>= 3.2.2.4)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
       rubocop-ast (>= 1.28.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.29.0)
+    rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
     rubocop-capybara (2.18.0)
       rubocop (~> 1.41)
@@ -97,7 +97,7 @@ GEM
       rbs
       syntax_tree (>= 2.0.1)
     temple (0.10.0)
-    thor (1.2.2)
+    thor (1.3.0)
     tilt (2.1.0)
     unicode-display_width (2.5.0)
 

data/README.md

@@ -9,10 +9,11 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
 
 `pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it creates a shadow table that looks structurally the same as the primary table, performs the schema change on the shadow table, copies contents from the primary table to the shadow table and swaps the table names in the end while preserving all changes to the primary table using triggers (via audit table).
 
-`pg-osc` is inspired by the design and workings of tools like `pg_repack` and `pt-online-schema-change` (MySQL). Read more below on [how does it work](#how-does-it-work), [prominent features](#prominent-features), the [caveats](#caveats) and [examples](#examples)
+`pg-osc` is inspired by the design and workings of tools like `pg_repack` and `pt-online-schema-change` (MySQL). Read more below on [how does it work](#how-does-it-work), [prominent features](#prominent-features), the [caveats](#few-things-to-keep-in-mind) and [examples](#examples)
 
 ## Table of Contents
 
+- [Table of Contents](#table-of-contents)
 - [Installation](#installation)
 - [Requirements](#requirements)
 - [Usage](#usage)
@@ -22,15 +23,15 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
   - [Renaming a column](#renaming-a-column)
   - [Multiple ALTER statements](#multiple-alter-statements)
   - [Kill other backends after 5s](#kill-other-backends-after-5s)
+  - [Replaying larger workloads](#replaying-larger-workloads)
   - [Backfill data](#backfill-data)
   - [Running using Docker](#running-using-docker)
-- [Caveats](#caveats)
+- [Few things to keep in mind](#few-things-to-keep-in-mind)
 - [How does it work](#how-does-it-work)
 - [Development](#development)
+  - [Local testing](#local-testing)
 - [Releasing](#releasing)
 - [Contributing](#contributing)
-- [License](#license)
-- [Code of Conduct](#code-of-conduct)
 
 ## Installation
 
@@ -103,12 +104,12 @@ print the version
 ## Prominent features
 
 - `pg-osc` supports when a column is being added, dropped or renamed with no data loss.
-- `pg-osc` acquires minimal locks throughout the process (read more below on the caveats).
+- `pg-osc` acquires minimal locks throughout the process (read more below on the [caveats](#few-things-to-keep-in-mind)).
 - Copies over indexes and Foreign keys.
 - Optionally drop or retain old tables in the end.
+- Reduce bloat (since pg-osc creates a new table and drops the old one post swap).
 - Tune how slow or fast should replays be from the audit/log table ([Replaying larger workloads](#replaying-larger-workloads)).
 - Backfill old/new columns as data is copied from primary table to shadow table, and then perform the swap. [Example](#backfill-data)
-- **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-osc/issues/14)
 
 ## Load test
 
@@ -216,7 +217,7 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
     --drop
 ```
 
-## Caveats
+## Few things to keep in mind
 
 - Partitioned tables are not supported as of yet. Pull requests and ideas welcome.
 - A primary key should exist on the table; without it, `pg-osc` will raise an exception
@@ -229,8 +230,6 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
 - Due to the nature of duplicating a table, there needs to be enough space on the disk to support the operation.
 - Index, constraints and sequence names will be altered and lose their original naming.
   - Can be fixed in future releases. Feel free to open a feature req.
-- Triggers are not carried over.
-  - Can be fixed in future releases. Feel free to open a feature req.
 - Foreign keys are dropped & re-added to referencing tables with a `NOT VALID`. A follow on `VALIDATE CONSTRAINT` is run.
   - Ensures that integrity is maintained and re-introducing FKs doesn't acquire additional locks, hence the `NOT VALID`.
 
@@ -297,11 +296,3 @@ bundle exec bin/pg-online-schema-change perform -a 'ALTER TABLE pgbench_accounts
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-osc.
-
-## License
-
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
-
-## Code of Conduct
-
-Everyone interacting in the PgOnlineSchemaChange project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/shayonj/pg-osc/blob/main/CODE_OF_CONDUCT.md).

data/lib/pg_online_schema_change/client.rb

@@ -67,5 +67,9 @@ module PgOnlineSchemaChange
 
       @copy_statement = File.binread(file_path)
     end
+
+    def checkout_connection
+      PG.connect(dbname: dbname, host: host, user: username, password: password, port: port)
+    end
   end
 end

data/lib/pg_online_schema_change/orchestrate.rb

@@ -5,6 +5,7 @@ require "securerandom"
 module PgOnlineSchemaChange
   class Orchestrate
     SWAP_STATEMENT_TIMEOUT = "5s"
+    TRACK_PROGRESS_INTERVAL = 60 # seconds
 
     extend Helper
 
@@ -58,6 +59,10 @@ module PgOnlineSchemaChange
 
         raise Error, "Parent table has no primary key, exiting..." if primary_key.nil?
 
+        logger.info("Performing some house keeping....")
+        run_analyze!
+        run_vacuum!
+
         setup_audit_table!
 
         setup_trigger!
@@ -159,6 +164,12 @@ module PgOnlineSchemaChange
       end
 
       def setup_shadow_table!
+        logger.info("Setting up shadow table", { shadow_table: shadow_table })
+        Query.run(
+          client.connection,
+          "SELECT create_table_all('#{client.table_name}', '#{shadow_table}');",
+        )
+
         # re-uses transaction with serializable
         # This ensures that all queries from here till copy_data run with serializable.
         # This is to to ensure that once the trigger is added to the primay table
@@ -168,13 +179,6 @@ module PgOnlineSchemaChange
         # adding the trigger, till the copy ends, since they all happen in the
         # same serializable transaction.
         Query.run(client.connection, "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", true)
-        logger.info("Setting up shadow table", { shadow_table: shadow_table })
-
-        Query.run(
-          client.connection,
-          "SELECT create_table_all('#{client.table_name}', '#{shadow_table}');",
-          true,
-        )
 
         # update serials
         Query.run(
@@ -208,19 +212,44 @@ module PgOnlineSchemaChange
         )
         Query.run(client.connection, "DELETE FROM #{audit_table}", true)
 
-        logger.info(
-          "Copying contents..",
-          { shadow_table: shadow_table, parent_table: client.table_name },
-        )
         if client.copy_statement
           query = format(client.copy_statement, shadow_table: shadow_table)
           return Query.run(client.connection, query, true)
         end
 
+        logger.info(
+          "Copying contents..",
+          { shadow_table: shadow_table, parent_table: client.table_name },
+        )
+
+        @copy_finished = false
+        log_progress
+
         sql = Query.copy_data_statement(client, shadow_table, true)
         Query.run(client.connection, sql, true)
       ensure
         Query.run(client.connection, "COMMIT;") # commit the serializable transaction
+        @copy_finished = true
+      end
+
+      def log_progress
+        new_connection = client.checkout_connection
+        source_table_size = Query.get_table_size(new_connection, client.schema, client.table_name)
+
+        Thread.new do
+          loop do
+            break if @copy_finished
+
+            shadow_table_size = Query.get_table_size(new_connection, client.schema, shadow_table)
+            progress = (shadow_table_size.to_f / source_table_size) * 100
+            logger.info("Estimated copy progress: #{progress.round(2)}% complete")
+
+            break if @copy_finished || progress >= 100
+            sleep(TRACK_PROGRESS_INTERVAL) unless ENV["CI"]
+          rescue StandardError => e
+            logger.info("Reporting progress failed: #{e.message}")
+          end
+        end
       end
 
       def replay_and_swap!
@@ -272,7 +301,13 @@ module PgOnlineSchemaChange
       def run_analyze!
         logger.info("Performing ANALYZE!")
 
-        Query.run(client.connection, "ANALYZE VERBOSE #{client.table_name};")
+        client.connection.async_exec("ANALYZE VERBOSE #{client.schema}.#{client.table_name};")
+      end
+
+      def run_vacuum!
+        logger.info("Performing VACUUM!")
+
+        client.connection.async_exec("VACUUM VERBOSE #{client.schema}.#{client.table_name};")
       end
 
       def validate_constraints!

data/lib/pg_online_schema_change/query.rb

@@ -311,20 +311,25 @@ module PgOnlineSchemaChange
 
       def view_definitions_for(client, table)
         query = <<~SQL
-          SELECT DISTINCT dependent_view.relname as view_name, pg_get_viewdef(format('%I.%I', '#{client.schema}', dependent_view.relname)::regclass) as view_definition
-          FROM pg_depend
-          JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid
-          JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid
-          JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid
-          JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace
-          WHERE
-          source_ns.nspname = '#{client.schema}'
-          AND source_table.relname = '#{table}'
+          SELECT DISTINCT
+            dependent_view.relname AS view_name,
+            pg_get_viewdef(dependent_view.oid) AS view_definition,
+            view_ns.nspname AS schema_name
+          FROM pg_class AS source_table
+          JOIN pg_depend ON pg_depend.refobjid = source_table.oid
+          JOIN pg_rewrite ON pg_rewrite.oid = pg_depend.objid
+          JOIN pg_class AS dependent_view ON dependent_view.oid = pg_rewrite.ev_class
+          JOIN pg_namespace AS view_ns ON dependent_view.relnamespace = view_ns.oid
+          AND dependent_view.relkind = 'v'
+          AND source_table.relname = '#{table}';
         SQL
 
         definitions = []
         run(client.connection, query) do |result|
-          definitions = result.map { |row| { row["view_name"] => row["view_definition"].strip } }
+          definitions =
+            result.map do |row|
+              { "#{row["schema_name"]}.#{row["view_name"]}" => row["view_definition"].strip }
+            end
         end
 
         definitions
@@ -423,6 +428,15 @@ module PgOnlineSchemaChange
           SELECT setval((select pg_get_serial_sequence('#{shadow_table}', '#{primary_key}')), (SELECT max(#{primary_key}) FROM #{table}));
         SQL
       end
+
+      def get_table_size(connection, schema, table_name)
+        size_query = "SELECT pg_table_size('#{schema}.#{table_name}');"
+        result = run(connection, size_query).first
+        result["pg_table_size"].to_i
+      rescue StandardError => e
+        logger.error("Error getting table size: #{e.message}")
+        0
+      end
     end
   end
 end

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.9.5"
+  VERSION = "0.9.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 0.9.7
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-15 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -68,16 +68,22 @@ dependencies:
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: prettier_print
   requirement: !ruby/object:Gem::Requirement
@@ -305,7 +311,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="