pg_online_schema_change 0.9.4 → 0.9.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01d9f8568b13763a56446a47e4d1c9f8d04b4d29cfd5eec3f6bdafd4355ce163
-  data.tar.gz: c14d2ce856b0023484864559a8d39de9f05510b7123125df12028be5d4a280ea
+  metadata.gz: 839fe414842291d9118a4115a56e06f1e0308ab3b7a740cafd9994314ded2d76
+  data.tar.gz: 22e9639093bf386f8a1e554e15e8d83df5e3c6de340c372f181fa3b280aee98d
 SHA512:
-  metadata.gz: bcf50ee7f529ce49c922ad5ff514233992ef5b3dcbe58fea8766efcb5ee0cce24798448e5d825a9b24a5f97a8e8864cd423dbb81898ebf3ab165a8e905745c58
-  data.tar.gz: 045a92a13bebe1e4ebdc8a09411b5c7536f12df800a1a66f4ad4709fdb1ec31a57ef98db136643042d87bda13945d46a52c228b3e9650e3a0a63efb4f2e1c186
+  metadata.gz: a160d19b3f1cb6655f70e0481b3293110deaee04d7a017ff9fe0c2b4d6100fd7d475520092db1b3bb5e068df5cb70243a00397a5cb1c2c259f2fe5b776b7e405
+  data.tar.gz: 9e4543fde48359fdc12046ff019420aafe51732f5cd1332285b3762fac3dcb198acc7be4f5fc6f6f90b910532758eb4a5752a8c9835d1f394774e6f77eda3898

data/CHANGELOG.md

@@ -1,7 +1,24 @@
+## [0.9.5] - 2023-10-15
+
+- Validate one constraint at a time in #124
+- Introduce --skip-foreign-key-validation in #125
+
+## [0.9.4] - 2023-09-17
+
+- Resolving gem push and sync glitch in 0.9.3
+
+## [0.9.3] - 2023-09-17
+
+- Dependency updates
+- Adding support for showing the gem version with -v or --version by @brycethornton #101
+- Fix for INSERT's failing for long table names by @ahilmer #116
+- Get view definition of a view from dedicated schema by @shayonj #117
+
 ## [0.9.2] - 2023-07-03
 
 - Dependency updates
 - Create shadow and audit with auatovacuum default turned off. Should avoid lock queues when disabling vacuum on audit table. #97
+
 ## [0.9.1] - 2023-06-24
 
 - Dependency updates and refresh docker release process with multi-platform build

data/Gemfile.lock

@@ -1,21 +1,19 @@
 PATH
   remote: .
   specs:
-    pg_online_schema_change (0.9.4)
+    pg_online_schema_change (0.9.6)
       ougai (~> 2.0.0)
       pg (>= 1.3.2, < 1.6.0)
       pg_query (>= 2.1.3, < 4.3.0)
-      thor (~> 1.2.1)
+      thor (>= 1.2.1, < 1.4.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    base64 (0.1.1)
     coderay (1.1.3)
     diff-lcs (1.5.0)
-    google-protobuf (3.24.3-arm64-darwin)
-    google-protobuf (3.24.3-x86_64-linux)
+    google-protobuf (3.25.0-arm64-darwin)
     haml (6.1.1)
       temple (>= 0.8.2)
       thor
@@ -27,7 +25,7 @@ GEM
     ougai (2.0.0)
       oj (~> 3.10)
     parallel (1.23.0)
-    parser (3.2.2.3)
+    parser (3.2.2.4)
       ast (~> 2.4.1)
       racc
     pg (1.5.4)
@@ -39,9 +37,9 @@ GEM
       method_source (~> 1.0)
     racc (1.7.1)
     rainbow (3.1.1)
-    rake (13.0.6)
+    rake (13.1.0)
     rbs (3.1.0)
-    regexp_parser (2.8.1)
+    regexp_parser (2.8.2)
     rexml (3.2.6)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -56,19 +54,18 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.56.3)
-      base64 (~> 0.1.1)
+    rubocop (1.57.2)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.3)
+      parser (>= 3.2.2.4)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
       rubocop-ast (>= 1.28.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.29.0)
+    rubocop-ast (1.30.0)
       parser (>= 3.2.1.0)
     rubocop-capybara (2.18.0)
       rubocop (~> 1.41)
@@ -76,7 +73,7 @@ GEM
       rubocop (~> 1.33)
     rubocop-packaging (0.5.2)
       rubocop (>= 1.33, < 2.0)
-    rubocop-performance (1.19.0)
+    rubocop-performance (1.19.1)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     rubocop-rake (0.6.0)
@@ -86,7 +83,7 @@ GEM
       rubocop-capybara (~> 2.17)
       rubocop-factory_bot (~> 2.22)
     ruby-progressbar (1.13.0)
-    syntax_tree (6.1.1)
+    syntax_tree (6.2.0)
       prettier_print (>= 1.2.0)
     syntax_tree-haml (4.0.3)
       haml (>= 5.2)
@@ -97,9 +94,9 @@ GEM
       rbs
       syntax_tree (>= 2.0.1)
     temple (0.10.0)
-    thor (1.2.2)
+    thor (1.3.0)
     tilt (2.1.0)
-    unicode-display_width (2.4.2)
+    unicode-display_width (2.5.0)
 
 PLATFORMS
   arm64-darwin-20

data/README.md

@@ -9,12 +9,11 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
 
 `pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it creates a shadow table that looks structurally the same as the primary table, performs the schema change on the shadow table, copies contents from the primary table to the shadow table and swaps the table names in the end while preserving all changes to the primary table using triggers (via audit table).
 
-`pg-osc` is inspired by the design and workings of tools like `pg_repack` and `pt-online-schema-change` (MySQL). Read more below on [how does it work](#how-does-it-work), [prominent features](#prominent-features), the [caveats](#caveats) and [examples](#examples)
-
-⚠️ Proceed with caution when using this on production like workloads. Best to try on similar setup or staging like environment first. Read on below for some examples and caveats.
+`pg-osc` is inspired by the design and workings of tools like `pg_repack` and `pt-online-schema-change` (MySQL). Read more below on [how does it work](#how-does-it-work), [prominent features](#prominent-features), the [caveats](#few-things-to-keep-in-mind) and [examples](#examples)
 
 ## Table of Contents
 
+- [Table of Contents](#table-of-contents)
 - [Installation](#installation)
 - [Requirements](#requirements)
 - [Usage](#usage)
@@ -24,15 +23,15 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
   - [Renaming a column](#renaming-a-column)
   - [Multiple ALTER statements](#multiple-alter-statements)
   - [Kill other backends after 5s](#kill-other-backends-after-5s)
+  - [Replaying larger workloads](#replaying-larger-workloads)
   - [Backfill data](#backfill-data)
   - [Running using Docker](#running-using-docker)
-- [Caveats](#caveats)
+- [Few things to keep in mind](#few-things-to-keep-in-mind)
 - [How does it work](#how-does-it-work)
 - [Development](#development)
+  - [Local testing](#local-testing)
 - [Releasing](#releasing)
 - [Contributing](#contributing)
-- [License](#license)
-- [Code of Conduct](#code-of-conduct)
 
 ## Installation
 
@@ -70,28 +69,29 @@ https://hub.docker.com/r/shayonj/pg-osc
 pg-online-schema-change help perform
 
 Usage:
-  pg-online-schema-change perform -a, --alter-statement=ALTER_STATEMENT -d, --dbname=DBNAME -h, --host=HOST -p, --port=N -s, --schema=SCHEMA -u, --username=USERNAME -w, --password=PASSWORD
+  pg-online-schema-change perform -a, --alter-statement=ALTER_STATEMENT -d, --dbname=DBNAME -h, --host=HOST -p, --port=N -s, --schema=SCHEMA -u, --username=USERNAME
 
 Options:
-  -a, --alter-statement=ALTER_STATEMENT        # The ALTER statement to perform the schema change
-  -s, --schema=SCHEMA                          # The schema in which the table is
-                                               # Default: public
-  -d, --dbname=DBNAME                          # Name of the database
-  -h, --host=HOST                              # Server host where the Database is located
-  -u, --username=USERNAME                      # Username for the Database
-  -p, --port=N                                 # Port for the Database
-                                               # Default: 5432
-  -w, --password=PASSWORD                      # DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead.
-  -v, [--verbose], [--no-verbose]              # Emit logs in debug mode
-  -f, [--drop], [--no-drop]                    # Drop the original table in the end after the swap
-  -k, [--kill-backends], [--no-kill-backends]  # Kill other competing queries/backends when trying to acquire lock for the shadow table creation and swap. It will wait for --wait-time-for-lock duration before killing backends and try upto 3 times.
-  -w, [--wait-time-for-lock=N]                 # Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock.
-                                               # Default: 10
-  -c, [--copy-statement=COPY_STATEMENT]        # Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README.
-  -b, [--pull-batch-count=N]                   # Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count.
-                                               # Default: 1000
-  -e, [--delta-count=N]                        # Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count.
-                                               # Default: 20
+  -a, --alter-statement=ALTER_STATEMENT                                    # The ALTER statement to perform the schema change
+  -s, --schema=SCHEMA                                                      # The schema in which the table is
+                                                                           # Default: public
+  -d, --dbname=DBNAME                                                      # Name of the database
+  -h, --host=HOST                                                          # Server host where the Database is located
+  -u, --username=USERNAME                                                  # Username for the Database
+  -p, --port=N                                                             # Port for the Database
+                                                                           # Default: 5432
+  -w, [--password=PASSWORD]                                                # DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead.
+  -v, [--verbose], [--no-verbose]                                          # Emit logs in debug mode
+  -f, [--drop], [--no-drop]                                                # Drop the original table in the end after the swap
+  -k, [--kill-backends], [--no-kill-backends]                              # Kill other competing queries/backends when trying to acquire lock for the shadow table creation and swap. It will wait for --wait-time-for-lock duration before killing backends and try upto 3 times.
+  -w, [--wait-time-for-lock=N]                                             # Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock.
+                                                                           # Default: 10
+  -c, [--copy-statement=COPY_STATEMENT]                                    # Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README.
+  -b, [--pull-batch-count=N]                                               # Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count.
+                                                                           # Default: 1000
+  -e, [--delta-count=N]                                                    # Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count.
+                                                                           # Default: 20
+  -o, [--skip-foreign-key-validation], [--no-skip-foreign-key-validation]  # Skip foreign key validation after swap. You shouldn't need this unless you have a very specific use case, like manually validating foreign key constraints after swap.
 ```
 
 ```
@@ -104,7 +104,7 @@ print the version
 ## Prominent features
 
 - `pg-osc` supports when a column is being added, dropped or renamed with no data loss.
-- `pg-osc` acquires minimal locks throughout the process (read more below on the caveats).
+- `pg-osc` acquires minimal locks throughout the process (read more below on the [caveats](#few-things-to-keep-in-mind)).
 - Copies over indexes and Foreign keys.
 - Optionally drop or retain old tables in the end.
 - Tune how slow or fast should replays be from the audit/log table ([Replaying larger workloads](#replaying-larger-workloads)).
@@ -217,7 +217,7 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
     --drop
 ```
 
-## Caveats
+## Few things to keep in mind
 
 - Partitioned tables are not supported as of yet. Pull requests and ideas welcome.
 - A primary key should exist on the table; without it, `pg-osc` will raise an exception
@@ -230,8 +230,6 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
 - Due to the nature of duplicating a table, there needs to be enough space on the disk to support the operation.
 - Index, constraints and sequence names will be altered and lose their original naming.
   - Can be fixed in future releases. Feel free to open a feature req.
-- Triggers are not carried over.
-  - Can be fixed in future releases. Feel free to open a feature req.
 - Foreign keys are dropped & re-added to referencing tables with a `NOT VALID`. A follow on `VALIDATE CONSTRAINT` is run.
   - Ensures that integrity is maintained and re-introducing FKs doesn't acquire additional locks, hence the `NOT VALID`.
 
@@ -298,11 +296,3 @@ bundle exec bin/pg-online-schema-change perform -a 'ALTER TABLE pgbench_accounts
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-osc.
-
-## License
-
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
-
-## Code of Conduct
-
-Everyone interacting in the PgOnlineSchemaChange project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/shayonj/pg-osc/blob/main/CODE_OF_CONDUCT.md).

data/lib/pg_online_schema_change/cli.rb

@@ -89,6 +89,13 @@ module PgOnlineSchemaChange
                   default: DELTA_COUNT,
                   desc:
                     "Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count."
+    method_option :skip_foreign_key_validation,
+                  aliases: "-o",
+                  type: :boolean,
+                  required: false,
+                  default: false,
+                  desc:
+                    "Skip foreign key validation after swap. You shouldn't need this unless you have a very specific use case, like manually validating foreign key constraints after swap."
 
     def perform
       client_options = Struct.new(*options.keys.map(&:to_sym)).new(*options.values)

data/lib/pg_online_schema_change/client.rb

@@ -19,7 +19,8 @@ module PgOnlineSchemaChange
                   :wait_time_for_lock,
                   :copy_statement,
                   :pull_batch_count,
-                  :delta_count
+                  :delta_count,
+                  :skip_foreign_key_validation
 
     def initialize(options)
       @alter_statement = options.alter_statement
@@ -34,6 +35,7 @@ module PgOnlineSchemaChange
       @wait_time_for_lock = options.wait_time_for_lock
       @pull_batch_count = options.pull_batch_count
       @delta_count = options.delta_count
+      @skip_foreign_key_validation = options.skip_foreign_key_validation
 
       handle_copy_statement(options.copy_statement)
       handle_validations
@@ -54,7 +56,7 @@ module PgOnlineSchemaChange
 
       return if Query.same_table?(@alter_statement)
 
-      raise Error("All statements should belong to the same table: #{@alter_statement}")
+      raise Error, "All statements should belong to the same table: #{@alter_statement}"
     end
 
     def handle_copy_statement(statement)

data/lib/pg_online_schema_change/orchestrate.rb

@@ -36,7 +36,10 @@ module PgOnlineSchemaChange
         Store.set(:trigger_time_column, "trigger_time_#{pgosc_identifier}")
         Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
         Store.set(:shadow_table, "pgosc_st_#{client.table.downcase}_#{pgosc_identifier}")
-        Store.set(:primary_table_storage_parameters, Query.storage_parameters_for(client, client.table_name, true) || "")
+        Store.set(
+          :primary_table_storage_parameters,
+          Query.storage_parameters_for(client, client.table_name, true) || "",
+        )
 
         Store.set(
           :referential_foreign_key_statements,
@@ -109,7 +112,10 @@ module PgOnlineSchemaChange
 
         Query.run(client.connection, sql)
 
-        Store.set(:audit_table_pk_sequence, Query.get_sequence_name(client, audit_table, audit_table_pk))
+        Store.set(
+          :audit_table_pk_sequence,
+          Query.get_sequence_name(client, audit_table, audit_table_pk),
+        )
       end
 
       def setup_trigger!
@@ -270,11 +276,14 @@ module PgOnlineSchemaChange
       end
 
       def validate_constraints!
-        logger.info("Validating constraints!")
-
-        validate_statements = Query.get_foreign_keys_to_validate(client, client.table_name)
+        return if client.skip_foreign_key_validation
 
-        Query.run(client.connection, validate_statements)
+        Query
+          .get_foreign_keys_to_validate(client, client.table_name)
+          .each do |statement|
+            logger.info("Validating constraints!", statement: statement)
+            Query.run(client.connection, statement)
+          end
       end
 
       def replace_views!

data/lib/pg_online_schema_change/query.rb

@@ -140,10 +140,9 @@ module PgOnlineSchemaChange
           SELECT pg_get_serial_sequence('#{table}', '#{column}');
         SQL
 
-        run(client.connection, query) { |result| result.map { |row|
-          row["pg_get_serial_sequence"]
-        }
-        }.first
+        run(client.connection, query) do |result|
+          result.map { |row| row["pg_get_serial_sequence"] }
+        end.first
       end
 
       def get_triggers_for(client, table)
@@ -238,11 +237,9 @@ module PgOnlineSchemaChange
         self_foreign_keys =
           constraints.select { |row| row["table_on"] == table && row["constraint_type"] == "f" }
 
-        [referential_foreign_keys, self_foreign_keys].flatten
-          .map do |row|
-            "ALTER TABLE #{row["table_on"]} VALIDATE CONSTRAINT #{row["constraint_name"]};"
-          end
-          .join
+        [referential_foreign_keys, self_foreign_keys].flatten.map do |row|
+          "ALTER TABLE #{row["table_on"]} VALIDATE CONSTRAINT #{row["constraint_name"]};"
+        end
       end
 
       def dropped_columns(client)
@@ -314,20 +311,25 @@ module PgOnlineSchemaChange
 
       def view_definitions_for(client, table)
         query = <<~SQL
-          SELECT DISTINCT dependent_view.relname as view_name, pg_get_viewdef(format('%I.%I', '#{client.schema}', dependent_view.relname)::regclass) as view_definition
-          FROM pg_depend
-          JOIN pg_rewrite ON pg_depend.objid = pg_rewrite.oid
-          JOIN pg_class as dependent_view ON pg_rewrite.ev_class = dependent_view.oid
-          JOIN pg_class as source_table ON pg_depend.refobjid = source_table.oid
-          JOIN pg_namespace source_ns ON source_ns.oid = source_table.relnamespace
-          WHERE
-          source_ns.nspname = '#{client.schema}'
-          AND source_table.relname = '#{table}'
+          SELECT DISTINCT
+            dependent_view.relname AS view_name,
+            pg_get_viewdef(dependent_view.oid) AS view_definition,
+            view_ns.nspname AS schema_name
+          FROM pg_class AS source_table
+          JOIN pg_depend ON pg_depend.refobjid = source_table.oid
+          JOIN pg_rewrite ON pg_rewrite.oid = pg_depend.objid
+          JOIN pg_class AS dependent_view ON dependent_view.oid = pg_rewrite.ev_class
+          JOIN pg_namespace AS view_ns ON dependent_view.relnamespace = view_ns.oid
+          AND dependent_view.relkind = 'v'
+          AND source_table.relname = '#{table}';
         SQL
 
         definitions = []
         run(client.connection, query) do |result|
-          definitions = result.map { |row| {row["view_name"] => row["view_definition"].strip} }
+          definitions =
+            result.map do |row|
+              { "#{row["schema_name"]}.#{row["view_name"]}" => row["view_definition"].strip }
+            end
         end
 
         definitions

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.9.4"
+  VERSION = "0.9.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.9.6
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-17 00:00:00.000000000 Z
+date: 2023-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -68,16 +68,22 @@ dependencies:
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: prettier_print
   requirement: !ruby/object:Gem::Requirement