pg_online_schema_change 0.4.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ef1069f7d159544838ec27cf518f98eb31609a0a2e1878ebd40246b10a6b534
-  data.tar.gz: 8e83cbac78164fb10870537bf4867020eae7cf06f4cd91930a2c8846fbc76c7c
+  metadata.gz: 273557a58c9492e628277e6eed7b655233c47bb51b12a61ca3eaf3377557205a
+  data.tar.gz: 6ec45511249401c420cc3c61652a4ad18225917b644574fb58329819fb422ab1
 SHA512:
-  metadata.gz: 2657c94b3b07730aa3bb282f34cd1ee95f0549d3649b39a61be3ce1b6d2ba5832a1914b6a9788778c912464cbd9773b5e967ed4e6c896127d60e62a0daa76c99
-  data.tar.gz: 4788523d691c25045b7f3c539f9743d880fabcbb01887ae34ec55a6c8a829804a9f4d43f619e79e0c44b1c9fff6ea88361fb5ac1111bc1f491aa17e6e43336d0
+  metadata.gz: 29f07b0af4c6a78de6945b6f43e271518f9c5953388e99dcc7582c4db865881ac5e5b8d0f1dff9a1215216ea534ad1e393fb268816e3799832911688b7abc14e
+  data.tar.gz: 0362f9c666c58acdb2fb63f1cd0208c719eb788555b218a57e2aee7bb9d98bbd9a7ff4d1e0ee0a56deec8e9679def1ae116fef919366e86ae30b2444d3379e6a

data/.rspec

@@ -1,4 +1,5 @@
 --format documentation
 --color
 --require spec_helper
---fail-fast
+--fail-fast
+--exclude-pattern spec/lib/smoke_spec.rb

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2022-02-21 22:46:44 UTC using RuboCop version 1.23.0.
+# on 2022-03-13 19:35:49 UTC using RuboCop version 1.23.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -9,7 +9,7 @@
 # Offense count: 2
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 233
+  Max: 250
 
 # Offense count: 2
 # Configuration parameters: IgnoredMethods.
@@ -26,14 +26,14 @@ Packaging/GemspecGit:
   Exclude:
     - 'pg_online_schema_change.gemspec'
 
-# Offense count: 62
+# Offense count: 67
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
   Max: 55
 
-# Offense count: 38
+# Offense count: 24
 RSpec/MultipleExpectations:
-  Max: 14
+  Max: 13
 
 # Offense count: 6
 # Configuration parameters: AllowedMethods.

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+## [0.6.0] - 2022-02-26
+* Delete items by audit table PK when replaying by @shayonj @jfrost in https://github.com/shayonj/pg-osc/pull/60
+  - Fixes a race condition issue: https://github.com/shayonj/pg-osc/issues/58
+
+## [0.5.0] - 2022-02-26
+* Share some preliminary load test figures in https://github.com/shayonj/pg-osc/pull/54
+* Reuse existing transaction open for reading table columns in https://github.com/shayonj/pg-osc/pull/53
+* Start to deprecate --password with PGPASSWORD in https://github.com/shayonj/pg-osc/pull/56
+* Introduce configurable PULL_BATCH_COUNT and DELTA_COUNT in https://github.com/shayonj/pg-osc/pull/57
+
+## [0.4.0] - 2022-02-22
+* Lint sourcecode, setup Rubocop proper and Lint in CI by @shayonj in https://github.com/shayonj/pg-osc/pull/46
+* Uniquely identify operation_type column by @shayonj in https://github.com/shayonj/pg-osc/pull/50
+* Introduce primary key on audit table for ordered reads by @shayonj in https://github.com/shayonj/pg-osc/pull/49
+  - This addresses an edge case with replay.
+* Uniquely identify trigger_time column by @shayonj in https://github.com/shayonj/pg-osc/pull/51
+* Abstract assertions into a helper function by @shayonj in https://github.com/shayonj/pg-osc/pull/52
+
 ## [0.3.0] - 2022-02-21
 
 - Explicitly call dependencies and bump dependencies by @shayonj https://github.com/shayonj/pg-osc/pull/44

data/CODE_OF_CONDUCT.md

@@ -39,7 +39,7 @@ This Code of Conduct applies within all community spaces, and also applies when
 
 ## Enforcement
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at shayon@loom.com. All complaints will be reviewed and investigated promptly and fairly.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at shayonj@gmail.com. All complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the reporter of any incident.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_online_schema_change (0.3.0)
+    pg_online_schema_change (0.6.0)
       ougai (~> 2.0.0)
       pg (~> 1.3.2)
       pg_query (~> 2.1.3)
@@ -22,7 +22,7 @@ GEM
     parallel (1.21.0)
     parser (3.0.3.2)
       ast (~> 2.4.1)
-    pg (1.3.2)
+    pg (1.3.3)
     pg_query (2.1.3)
       google-protobuf (>= 3.19.2)
     pry (0.14.1)

data/README.md

@@ -1,6 +1,8 @@
 # pg-osc
 
-[![CircleCI](https://circleci.com/gh/shayonj/pg-osc/tree/main.svg?style=shield)](https://circleci.com/gh/shayonj/pg-osc/tree/main)
+[![CI](https://github.com/shayonj/pg-osc/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/ci.yaml)
+[![Smoke Test PG 9.6](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-9-6.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-9-6.yaml)
+[![Smoke Test PG 13.6](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml)
 [![Gem Version](https://badge.fury.io/rb/pg_online_schema_change.svg)](https://badge.fury.io/rb/pg_online_schema_change)
 
 pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads. 
@@ -16,8 +18,8 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
 - [Installation](#installation)
 - [Requirements](#requirements)
 - [Usage](#usage)
-- [How does it work](#how-does-it-work)
 - [Prominent features](#prominent-features)
+- [Load test](#load-test)
 - [Examples](#examples)
   * [Renaming a column](#renaming-a-column)
   * [Multiple ALTER statements](#multiple-alter-statements)
@@ -25,6 +27,7 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
   * [Backfill data](#backfill-data)
   * [Running using Docker](#running-using-docker)
 - [Caveats](#caveats)
+- [How does it work](#how-does-it-work)
 - [Development](#development)
 - [Releasing](#releasing)
 - [Contributing](#contributing)
@@ -75,13 +78,17 @@ Options:
   -u, --username=USERNAME                      # Username for the Database
   -p, --port=N                                 # Port for the Database
                                                # Default: 5432
-  -w, --password=PASSWORD                      # Password for the Database
+  -w, --password=PASSWORD                      # DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead.
   -v, [--verbose], [--no-verbose]              # Emit logs in debug mode
   -f, [--drop], [--no-drop]                    # Drop the original table in the end after the swap
   -k, [--kill-backends], [--no-kill-backends]  # Kill other competing queries/backends when trying to acquire lock for the shadow table creation and swap. It will wait for --wait-time-for-lock duration before killing backends and try upto 3 times.
   -w, [--wait-time-for-lock=N]                 # Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock.
                                                # Default: 10
-  -c, [--copy-statement=COPY_STATEMENT]        # Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pg-osc copies data from the primary to the shadow table. More examples in README.
+  -c, [--copy-statement=COPY_STATEMENT]        # Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README.
+  -b, [--pull-batch-count=N]                   # Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count.
+                                               # Default: 1000
+  -e, [--delta-count=N]                        # Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count.
+                                               # Default: 20
 ```
 
 ```
@@ -90,57 +97,39 @@ Usage:
 
 print the version
 ```
-## How does it work
-
-- **Primary table**: A table against which a potential schema change is to be run
-- **Shadow table**: A copy of an existing primary table
-- **Audit table**: A table to store any updates/inserts/delete on a primary table
-
-![how-it-works](diagrams/how-it-works.png)
-
-
-1. Create an audit table to record changes made to the parent table.
-2. Acquire a brief `ACCESS EXCLUSIVE` lock to add a trigger on the parent table (for inserts, updates, deletes) to the audit table.
-3. Create a new shadow table and run ALTER/migration on the shadow table. 
-4. Copy all rows from the old table.
-5. Build indexes on the new table.
-6. Replay all changes accumulated in the audit table against the shadow table.
-   - Delete rows in the audit table as they are replayed.
-7. Once the delta (remaining rows) is ~20 rows, acquire an `ACCESS EXCLUSIVE` lock against the parent table within a transaction and:
-   - swap table names (shadow table <> parent table).
-   - update references in other tables (FKs) by dropping and re-creating the FKs with a `NOT VALID`.
-8. Runs `ANALYZE` on the new table.
-9. Validates all FKs that were added with `NOT VALID`.
-10. Drop parent (now old) table (OPTIONAL).
-
 ## Prominent features
 - `pg-osc` supports when a column is being added, dropped or renamed with no data loss. 
 - `pg-osc` acquires minimal locks throughout the process (read more below on the caveats).
 - Copies over indexes and Foreign keys.
 - Optionally drop or retain old tables in the end.
+- Tune how slow or fast should replays be from the audit/log table ([Replaying larger workloads](#replaying-larger-workloads)).
 - Backfill old/new columns as data is copied from primary table to shadow table, and then perform the swap. [Example](#backfill-data)
 - **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-osc/issues/14)
 
+## Load test
+
+[More about the preliminary load test figures here](docs/load-test.md)
+
 ## Examples
 
 ### Renaming a column
 ```
+export PGPASSWORD=""
 pg-online-schema-change perform \
   --alter-statement 'ALTER TABLE books RENAME COLUMN email TO new_email' \
   --dbname "postgres" \
   --host "localhost" \
   --username "jamesbond" \
-  --password "" \
 ```
 
 ### Multiple ALTER statements
 ```
+export PGPASSWORD=""
 pg-online-schema-change perform \
   --alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE; ALTER TABLE books RENAME COLUMN email TO new_email;' \
   --dbname "postgres" \
   --host "localhost" \
   --username "jamesbond" \
-  --password "" \
   --drop
 ```
 
@@ -148,13 +137,30 @@ pg-online-schema-change perform \
 If the operation is being performed on a busy table, you can use `pg-osc`'s `kill-backend` functionality to kill other backends that may be competing with the `pg-osc` operation to acquire a lock for a brief while. The `ACCESS EXCLUSIVE` lock acquired by `pg-osc` is only held for a brief while and released after. You can tune how long `pg-osc` should wait before killing other backends (or if at all `pg-osc` should kill backends in the first place).
 
 ```
+export PGPASSWORD=""
 pg-online-schema-change perform \
   --alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
   --dbname "postgres" \
   --host "localhost" \
   --username "jamesbond" \
-  --password "" \
-  --wait-time-for-lock=5 \
+  --wait-time-for-lock 5 \
+  --kill-backends \
+  --drop
+```
+
+### Replaying larger workloads
+If you have a table with high write volume, the default replay iteration may not suffice. That is - you may see that `pg-osc` is replaying 1000 rows (`pull-batch-count`) in one go from the audit table. `pg-osc` also waits until the remaining row count (`delta-count`) in audit table is 20 before making the swap. You can tune these values to be higher for faster catch up on these kind of workloads.
+
+```
+export PGPASSWORD=""
+pg-online-schema-change perform \
+  --alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
+  --dbname "postgres" \
+  --host "localhost" \
+  --username "jamesbond" \
+  --pull-batch-count 2000
+  --delta-count 500
+  --wait-time-for-lock 5 \
   --kill-backends \
   --drop
 ```
@@ -183,7 +189,6 @@ pg-online-schema-change perform \
   --dbname "postgres" \
   --host "localhost" \
   --username "jamesbond" \
-  --password "" \
   --copy-statement "/src/query.sql" \
   --drop
 ```
@@ -197,7 +202,6 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
     --dbname "postgres" \
     --host "localhost" \
     --username "jamesbond" \
-    --password "" \
     --drop
 ```
 ## Caveats
@@ -215,6 +219,29 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
   - Can be fixed in future releases. Feel free to open a feature req.
 - Foreign keys are dropped & re-added to referencing tables with a `NOT VALID`. A follow on `VALIDATE CONSTRAINT` is run.
  	- Ensures that integrity is maintained and re-introducing FKs doesn't acquire additional locks, hence the `NOT VALID`.
+## How does it work
+
+- **Primary table**: A table against which a potential schema change is to be run
+- **Shadow table**: A copy of an existing primary table
+- **Audit table**: A table to store any updates/inserts/delete on a primary table
+
+![how-it-works](docs/how-it-works.png)
+
+
+1. Create an audit table to record changes made to the parent table.
+2. Acquire a brief `ACCESS EXCLUSIVE` lock to add a trigger on the parent table (for inserts, updates, deletes) to the audit table.
+3. Create a new shadow table and run ALTER/migration on the shadow table. 
+4. Copy all rows from the old table.
+5. Build indexes on the new table.
+6. Replay all changes accumulated in the audit table against the shadow table.
+   - Delete rows in the audit table as they are replayed.
+7. Once the delta (remaining rows) is ~20 rows, acquire an `ACCESS EXCLUSIVE` lock against the parent table within a transaction and:
+   - swap table names (shadow table <> parent table).
+   - update references in other tables (FKs) by dropping and re-creating the FKs with a `NOT VALID`.
+8. Runs `ANALYZE` on the new table.
+9. Validates all FKs that were added with `NOT VALID`.
+10. Drop parent (now old) table (OPTIONAL).
+
 ## Development
 
 - Install ruby 3.0

data/docs/load-test.md

@@ -0,0 +1,138 @@
+# Preliminary Load Test
+
+## pg-osc: No downtime schema changes with 7K+ writes/s & 12k+ reads/s
+
+This is a very basic load test performed with `pgbench` against a single instance PostgreSQL DB running on DigitialOcean with the following configuration:
+
+- **128GB RAM**
+- **32vCPU**
+- **695GB Disk**
+- Trasanction based connection pool with **500 pool limit**
+
+Total time taken to run schema change: **<3mins**
+
+## Simulating load with pgbench
+
+**Initialize**
+```
+pgbench -p $PORT --initialize -s 20 -F 20 --foreign-keys --host $HOST -U $USERNAME  -d $DB 
+```
+
+This creates bunch of pgbench tables. The table being used with `pg-osc` is `pgbench_accounts` which has FKs and also references by other tables with FKS, containing 2M rows.
+
+**Begin**
+```
+pgbench -p $PORT -j 72 -c 288 -T 500 -r --host $DB_HOST -U $USERNAME -d $DB  
+```
+
+## Running pg-osc
+
+Simple `ALTER` statement for experimentation purposes.
+
+```sql
+ALTER TABLE pgbench_accounts ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;
+```
+
+**Execution**
+
+```bash
+bundle exec bin/pg-online-schema-change perform \ 
+-a 'ALTER TABLE pgbench_accounts ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
+-d "pool" \
+-p 25061
+-h "..." \
+-u "..." \
+--pull-batch-count 2000 \
+--delta-count 200
+```
+
+## Outcome
+
+**pgbench results**
+
+```
+number of transactions actually processed: 1060382
+latency average = 144.874 ms
+tps = 1767.057392 (including connections establishing)
+tps = 1777.971823 (excluding connections establishing)
+statement latencies in milliseconds:
+         0.479  \set aid random(1, 100000 * :scale)
+         0.409  \set bid random(1, 1 * :scale)
+         0.247  \set tid random(1, 10 * :scale)
+         0.208  \set delta random(-5000, 5000)
+         3.136  BEGIN;
+         4.243  UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;
+         4.488  SELECT abalance FROM pgbench_accounts WHERE aid = :aid;
+        71.017  UPDATE pgbench_tellers SET tbalance = tbalance + :delta WHERE tid = :tid;
+        46.689  UPDATE pgbench_branches SET bbalance = bbalance + :delta WHERE bid = :bid;
+         4.035  INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);
+         4.166  END;
+```
+
+**Metrics**
+![load-test](load-test-1.png)
+
+**New table structure**
+
+Added `purchased` column.
+
+```
+defaultdb=> \d+ pgbench_accounts;
+                                  Table "public.pgbench_accounts"
+  Column   |     Type      | Collation | Nullable | Default | Storage  | Stats target | Description 
+-----------+---------------+-----------+----------+---------+----------+--------------+-------------
+ aid       | integer       |           | not null |         | plain    |              | 
+ bid       | integer       |           |          |         | plain    |              | 
+ abalance  | integer       |           |          |         | plain    |              | 
+ filler    | character(84) |           |          |         | extended |              | 
+ purchased | boolean       |           |          | false   | plain    |              | 
+Indexes:
+    "pgosc_st_pgbench_accounts_815029_pkey" PRIMARY KEY, btree (aid)
+Foreign-key constraints:
+    "pgbench_accounts_bid_fkey" FOREIGN KEY (bid) REFERENCES pgbench_branches(bid)
+Referenced by:
+    TABLE "pgbench_history" CONSTRAINT "pgbench_history_aid_fkey" FOREIGN KEY (aid) REFERENCES pgbench_accounts(aid)
+Options: autovacuum_enabled=false, fillfactor=20
+```
+
+**Logs**
+
+<details>
+<summary>Logs from pg-osc</summary>
+
+```json
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:01.147-05:00","v":0,"msg":"Setting up audit table","audit_table":"pgosc_at_pgbench_accounts_714a8b","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:01.660-05:00","v":0,"msg":"Setting up triggers","version":"0.4.0"}
+NOTICE:  trigger "primary_to_audit_table_trigger" for relation "pgbench_accounts" does not exist, skipping
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:01.814-05:00","v":0,"msg":"Setting up shadow table","shadow_table":"pgosc_st_pgbench_accounts_714a8b","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:02.169-05:00","v":0,"msg":"Running alter statement on shadow table","shadow_table":"pgosc_st_pgbench_accounts_714a8b","parent_table":"pgbench_accounts","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:02.204-05:00","v":0,"msg":"Clearing contents of audit table before copy..","shadow_table":"pgosc_st_pgbench_accounts_714a8b","parent_table":"pgbench_accounts","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:02.240-05:00","v":0,"msg":"Copying contents..","shadow_table":"pgosc_st_pgbench_accounts_714a8b","parent_table":"pgbench_accounts","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:20.481-05:00","v":0,"msg":"Performing ANALYZE!","version":"0.4.0"}
+INFO:  analyzing "public.pgbench_accounts"
+INFO:  "pgbench_accounts": scanned 30000 of 166667 pages, containing 360000 live rows and 200 dead rows; 30000 rows in sample, 2000004 estimated total rows
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:21.078-05:00","v":0,"msg":"Replaying rows, count: 2000","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:21.580-05:00","v":0,"msg":"Replaying rows, count: 2000","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:22.022-05:00","v":0,"msg":"Replaying rows, count: 2000","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:22.490-05:00","v":0,"msg":"Replaying rows, count: 2000","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:22.866-05:00","v":0,"msg":"Replaying rows, count: 661","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:23.212-05:00","v":0,"msg":"Replaying rows, count: 533","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:23.512-05:00","v":0,"msg":"Replaying rows, count: 468","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:23.809-05:00","v":0,"msg":"Remaining rows below delta count, proceeding towards swap","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:23.809-05:00","v":0,"msg":"Performing swap!","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:24.259-05:00","v":0,"msg":"Replaying rows, count: 449","version":"0.4.0"}
+NOTICE:  trigger "primary_to_audit_table_trigger" for relation "pgbench_accounts" does not exist, skipping
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:24.650-05:00","v":0,"msg":"Performing ANALYZE!","version":"0.4.0"}
+INFO:  analyzing "public.pgbench_accounts"
+INFO:  "pgbench_accounts": scanned 30000 of 32935 pages, containing 1821834 live rows and 6056 dead rows; 30000 rows in sample, 2000070 estimated total rows
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:24.941-05:00","v":0,"msg":"Validating constraints!","version":"0.4.0"}
+NOTICE:  table "pgosc_st_pgbench_accounts_714a8b" does not exist, skipping
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:26.159-05:00","v":0,"msg":"All tasks successfully completed","version":"0.4.0"}
+```
+
+</details>
+
+
+## Conclusion
+
+By tweaking `--pull-batch-count` to `2000` (replay 2k rows at once) and `--delta-count` to `200` (time to swap when remaining rows is <200), `pg-osc` was able to perform the schema change with no impact within very quick time. Depending on the database size and load on the table, you can further tune them to achieve desired impact. At some point this is going to plateau - I can imagine the replay factor not working quite well for say 100k commits/s workloads. So, YMMV.

data/lib/pg_online_schema_change/cli.rb

@@ -3,8 +3,10 @@
 require "thor"
 
 module PgOnlineSchemaChange
+  PULL_BATCH_COUNT = 1000
+  DELTA_COUNT = 20
   class CLI < Thor
-    desc "perform", "Perform the set of operations to safely apply the schema change with minimal locks"
+    desc "perform", "Safely apply schema changes with minimal locks"
     method_option :alter_statement, aliases: "-a", type: :string, required: true,
                                     desc: "The ALTER statement to perform the schema change"
     method_option :schema, aliases: "-s", type: :string, required: true, default: "public",
@@ -13,7 +15,7 @@ module PgOnlineSchemaChange
     method_option :host, aliases: "-h", type: :string, required: true, desc: "Server host where the Database is located"
     method_option :username, aliases: "-u", type: :string, required: true, desc: "Username for the Database"
     method_option :port, aliases: "-p", type: :numeric, required: true, default: 5432, desc: "Port for the Database"
-    method_option :password, aliases: "-w", type: :string, required: true, desc: "Password for the Database"
+    method_option :password, aliases: "-w", type: :string, required: false, default: "", desc: "DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
     method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Emit logs in debug mode"
     method_option :drop, aliases: "-f", type: :boolean, default: false,
                          desc: "Drop the original table in the end after the swap"
@@ -23,11 +25,19 @@ module PgOnlineSchemaChange
                                        desc: "Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock."
     method_option :copy_statement, aliases: "-c", type: :string, required: false, default: "",
                                    desc: "Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README."
+    method_option :pull_batch_count, aliases: "-b", type: :numeric, required: false, default: PULL_BATCH_COUNT,
+                                     desc: "Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count."
+    method_option :delta_count, aliases: "-e", type: :numeric, required: false, default: DELTA_COUNT,
+                                desc: "Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count."
 
     def perform
       client_options = Struct.new(*options.keys.map(&:to_sym)).new(*options.values)
-
       PgOnlineSchemaChange.logger(verbose: client_options.verbose)
+
+      PgOnlineSchemaChange.logger.warn("DEPRECATED: -w is deprecated. Please pass PGPASSWORD environment variable instead.") if client_options.password
+
+      client_options.password = ENV["PGPASSWORD"] || client_options.password
+
       PgOnlineSchemaChange::Orchestrate.run!(client_options)
     end
 

data/lib/pg_online_schema_change/client.rb

@@ -5,7 +5,7 @@ require "pg"
 module PgOnlineSchemaChange
   class Client
     attr_accessor :alter_statement, :schema, :dbname, :host, :username, :port, :password, :connection, :table, :drop,
-                  :kill_backends, :wait_time_for_lock, :copy_statement
+                  :kill_backends, :wait_time_for_lock, :copy_statement, :pull_batch_count, :delta_count
 
     def initialize(options)
       @alter_statement = options.alter_statement
@@ -18,6 +18,8 @@ module PgOnlineSchemaChange
       @drop = options.drop
       @kill_backends = options.kill_backends
       @wait_time_for_lock = options.wait_time_for_lock
+      @pull_batch_count = options.pull_batch_count
+      @delta_count = options.delta_count
 
       handle_copy_statement(options.copy_statement)
       handle_validations

data/lib/pg_online_schema_change/functions.rb

@@ -47,18 +47,6 @@ FUNC_CREATE_TABLE_ALL = <<~SQL
     EXECUTE format(
           'CREATE TABLE %s (LIKE %s including all)',
           newsource_table, source_table);
-      for rec in
-          SELECT oid, conname
-          FROM pg_constraint
-          WHERE contype = 'f'
-          AND conrelid = source_table::regclass
-      LOOP
-      EXECUTE format(
-              'ALTER TABLE %s add constraint %s %s',
-              newsource_table,
-              rec.conname,
-              pg_get_constraintdef(rec.oid));
-      END LOOP;
     END
   $$;
 SQL

data/lib/pg_online_schema_change/orchestrate.rb

@@ -37,6 +37,9 @@ module PgOnlineSchemaChange
         Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
         Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
         Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
+
+        Store.set(:referential_foreign_key_statements, Query.referential_foreign_keys_to_refresh(client, client.table))
+        Store.set(:self_foreign_key_statements, Query.self_foreign_keys_to_refresh(client, client.table))
       end
 
       def run!(options)
@@ -204,7 +207,7 @@ module PgOnlineSchemaChange
           return Query.run(client.connection, query, true)
         end
 
-        sql = Query.copy_data_statement(client, shadow_table)
+        sql = Query.copy_data_statement(client, shadow_table, true)
         Query.run(client.connection, sql, true)
       ensure
         Query.run(client.connection, "COMMIT;") # commit the serializable transaction
@@ -221,7 +224,6 @@ module PgOnlineSchemaChange
       def swap!
         logger.info("Performing swap!")
 
-        foreign_key_statements = Query.get_foreign_keys_to_refresh(client, client.table)
         storage_params_reset = primary_table_storage_parameters.empty? ? "" : "ALTER TABLE #{client.table} SET (#{primary_table_storage_parameters});"
 
         # From here on, all statements are carried out in a single
@@ -239,12 +241,13 @@ module PgOnlineSchemaChange
         sql = <<~SQL
           ALTER TABLE #{client.table} RENAME to #{old_primary_table};
           ALTER TABLE #{shadow_table} RENAME to #{client.table};
-          #{foreign_key_statements}
+          #{referential_foreign_key_statements}
+          #{self_foreign_key_statements}
           #{storage_params_reset}
           DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
         SQL
 
-        Query.run(client.connection, sql)
+        Query.run(client.connection, sql, opened)
       ensure
         Query.run(client.connection, "COMMIT;")
         Query.run(client.connection, "SET statement_timeout = 0;")
@@ -270,6 +273,7 @@ module PgOnlineSchemaChange
         shadow_table_drop = shadow_table ? "DROP TABLE IF EXISTS #{shadow_table}" : ""
 
         sql = <<~SQL
+          DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
           #{audit_table_drop};
           #{shadow_table_drop};
           #{primary_drop}

data/lib/pg_online_schema_change/query.rb

@@ -140,7 +140,7 @@ module PgOnlineSchemaChange
         end
       end
 
-      def get_foreign_keys_to_refresh(client, table)
+      def referential_foreign_keys_to_refresh(client, table)
         references = get_all_constraints_for(client).select do |row|
           row["table_from"] == table && row["constraint_type"] == "f"
         end
@@ -158,12 +158,32 @@ module PgOnlineSchemaChange
         end.join
       end
 
-      def get_foreign_keys_to_validate(client, table)
+      def self_foreign_keys_to_refresh(client, table)
         references = get_all_constraints_for(client).select do |row|
-          row["table_from"] == table && row["constraint_type"] == "f"
+          row["table_on"] == table && row["constraint_type"] == "f"
         end
 
         references.map do |row|
+          add_statement = if row["definition"].end_with?("NOT VALID")
+                            "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
+                          else
+                            "ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
+                          end
+          add_statement
+        end.join
+      end
+
+      def get_foreign_keys_to_validate(client, table)
+        constraints = get_all_constraints_for(client)
+        referential_foreign_keys = constraints.select do |row|
+          row["table_from"] == table && row["constraint_type"] == "f"
+        end
+
+        self_foreign_keys = constraints.select do |row|
+          row["table_on"] == table && row["constraint_type"] == "f"
+        end
+
+        [referential_foreign_keys, self_foreign_keys].flatten.map do |row|
           "ALTER TABLE #{row["table_on"]} VALIDATE CONSTRAINT #{row["constraint_name"]};"
         end.join
       end

data/lib/pg_online_schema_change/replay.rb

@@ -7,9 +7,6 @@ module PgOnlineSchemaChange
     extend Helper
 
     class << self
-      PULL_BATCH_COUNT = 1000
-      DELTA_COUNT = 20
-
       # This, picks PULL_BATCH_COUNT rows by primary key from audit_table,
       # replays it on the shadow_table. Once the batch is done,
       # it them deletes those PULL_BATCH_COUNT rows from audit_table. Then, pull another batch,
@@ -20,7 +17,7 @@ module PgOnlineSchemaChange
         loop do
           rows = rows_to_play
 
-          raise CountBelowDelta if rows.count <= DELTA_COUNT
+          raise CountBelowDelta if rows.count <= client.delta_count
 
           play!(rows)
         end
@@ -28,7 +25,7 @@ module PgOnlineSchemaChange
 
       def rows_to_play(reuse_trasaction = false)
         select_query = <<~SQL
-          SELECT * FROM #{audit_table} ORDER BY #{audit_table_pk} LIMIT #{PULL_BATCH_COUNT};
+          SELECT * FROM #{audit_table} ORDER BY #{audit_table_pk} LIMIT #{client.pull_batch_count};
         SQL
 
         rows = []
@@ -90,7 +87,7 @@ module PgOnlineSchemaChange
             SQL
             to_be_replayed << sql
 
-            to_be_deleted_rows << "'#{row[primary_key]}'"
+            to_be_deleted_rows << "'#{row[audit_table_pk]}'"
           when "UPDATE"
             set_values = new_row.map do |column, value|
               "#{column} = '#{value}'"
@@ -103,14 +100,14 @@ module PgOnlineSchemaChange
             SQL
             to_be_replayed << sql
 
-            to_be_deleted_rows << "'#{row[primary_key]}'"
+            to_be_deleted_rows << "'#{row[audit_table_pk]}'"
           when "DELETE"
             sql = <<~SQL
               DELETE FROM #{shadow_table} WHERE #{primary_key}=\'#{row[primary_key]}\';
             SQL
             to_be_replayed << sql
 
-            to_be_deleted_rows << "'#{row[primary_key]}'"
+            to_be_deleted_rows << "'#{row[audit_table_pk]}'"
           end
         end
 
@@ -120,7 +117,7 @@ module PgOnlineSchemaChange
         return unless to_be_deleted_rows.count >= 1
 
         delete_query = <<~SQL
-          DELETE FROM #{audit_table} WHERE #{primary_key} IN (#{to_be_deleted_rows.join(",")})
+          DELETE FROM #{audit_table} WHERE #{audit_table_pk} IN (#{to_be_deleted_rows.join(",")})
         SQL
         Query.run(client.connection, delete_query, reuse_trasaction)
       end

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.4.0"
+  VERSION = "0.7.0"
 end

data/lib/pg_online_schema_change.rb

@@ -6,12 +6,12 @@ require "ougai"
 require "pg_online_schema_change/version"
 require "pg_online_schema_change/helper"
 require "pg_online_schema_change/functions"
-require "pg_online_schema_change/cli"
 require "pg_online_schema_change/client"
 require "pg_online_schema_change/query"
 require "pg_online_schema_change/store"
 require "pg_online_schema_change/replay"
 require "pg_online_schema_change/orchestrate"
+require "pg_online_schema_change/cli"
 
 module PgOnlineSchemaChange
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-22 00:00:00.000000000 Z
+date: 2022-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -204,9 +204,11 @@ files:
 - bin/console
 - bin/pg-online-schema-change
 - bin/setup
-- diagrams/how-it-works.excalidraw
-- diagrams/how-it-works.png
 - docker-compose.yml
+- docs/how-it-works.excalidraw
+- docs/how-it-works.png
+- docs/load-test-1.png
+- docs/load-test.md
 - lib/pg_online_schema_change.rb
 - lib/pg_online_schema_change/cli.rb
 - lib/pg_online_schema_change/client.rb