pg_online_schema_change 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ef1069f7d159544838ec27cf518f98eb31609a0a2e1878ebd40246b10a6b534
-  data.tar.gz: 8e83cbac78164fb10870537bf4867020eae7cf06f4cd91930a2c8846fbc76c7c
+  metadata.gz: b14af6aa2b98ab8f1b2aab5ae0a8555e017f92531ce1214e50f2ac62ff354224
+  data.tar.gz: 66ba81f4e90f4dc612d863d043838fec8676e468924eeb5c39d21094d002e48a
 SHA512:
-  metadata.gz: 2657c94b3b07730aa3bb282f34cd1ee95f0549d3649b39a61be3ce1b6d2ba5832a1914b6a9788778c912464cbd9773b5e967ed4e6c896127d60e62a0daa76c99
-  data.tar.gz: 4788523d691c25045b7f3c539f9743d880fabcbb01887ae34ec55a6c8a829804a9f4d43f619e79e0c44b1c9fff6ea88361fb5ac1111bc1f491aa17e6e43336d0
+  metadata.gz: a272a3749f8f053a528d859cc35b97b5fde7d6353a3c0c710500e35d33df6b4bc8526b18e855fd154b742eef8189b661e8e710a6b25221df6e47d55a47381a2e
+  data.tar.gz: c0dd7f41e204840b54b768df1f1e8486f79038bede199021ef13684c4b1369e3de43be6fbfd8f03b7ca652251c59a8a580ecefb235e00e54858d77f88a7334de

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## [0.4.0] - 2022-02-22
+* Lint sourcecode, setup Rubocop proper and Lint in CI by @shayonj in https://github.com/shayonj/pg-osc/pull/46
+* Uniquely identify operation_type column by @shayonj in https://github.com/shayonj/pg-osc/pull/50
+* Introduce primary key on audit table for ordered reads by @shayonj in https://github.com/shayonj/pg-osc/pull/49
+  - This addresses an edge case with replay.
+* Uniquely identify trigger_time column by @shayonj in https://github.com/shayonj/pg-osc/pull/51
+* Abstract assertions into a helper function by @shayonj in https://github.com/shayonj/pg-osc/pull/52
+
 ## [0.3.0] - 2022-02-21
 
 - Explicitly call dependencies and bump dependencies by @shayonj https://github.com/shayonj/pg-osc/pull/44

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pg_online_schema_change (0.3.0)
+    pg_online_schema_change (0.4.0)
       ougai (~> 2.0.0)
       pg (~> 1.3.2)
       pg_query (~> 2.1.3)
@@ -14,7 +14,6 @@ GEM
     coderay (1.1.3)
     diff-lcs (1.5.0)
     google-protobuf (3.19.4)
-    google-protobuf (3.19.4-x86_64-linux)
     method_source (1.0.0)
     oj (3.13.11)
     ougai (2.0.0)

data/README.md

@@ -16,8 +16,8 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
 - [Installation](#installation)
 - [Requirements](#requirements)
 - [Usage](#usage)
-- [How does it work](#how-does-it-work)
 - [Prominent features](#prominent-features)
+- [Load test](#load-test)
 - [Examples](#examples)
   * [Renaming a column](#renaming-a-column)
   * [Multiple ALTER statements](#multiple-alter-statements)
@@ -25,6 +25,7 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
   * [Backfill data](#backfill-data)
   * [Running using Docker](#running-using-docker)
 - [Caveats](#caveats)
+- [How does it work](#how-does-it-work)
 - [Development](#development)
 - [Releasing](#releasing)
 - [Contributing](#contributing)
@@ -75,13 +76,17 @@ Options:
   -u, --username=USERNAME                      # Username for the Database
   -p, --port=N                                 # Port for the Database
                                                # Default: 5432
-  -w, --password=PASSWORD                      # Password for the Database
+  -w, --password=PASSWORD                      # DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead.
   -v, [--verbose], [--no-verbose]              # Emit logs in debug mode
   -f, [--drop], [--no-drop]                    # Drop the original table in the end after the swap
   -k, [--kill-backends], [--no-kill-backends]  # Kill other competing queries/backends when trying to acquire lock for the shadow table creation and swap. It will wait for --wait-time-for-lock duration before killing backends and try upto 3 times.
   -w, [--wait-time-for-lock=N]                 # Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock.
                                                # Default: 10
-  -c, [--copy-statement=COPY_STATEMENT]        # Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pg-osc copies data from the primary to the shadow table. More examples in README.
+  -c, [--copy-statement=COPY_STATEMENT]        # Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README.
+  -b, [--pull-batch-count=N]                   # Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count.
+                                               # Default: 1000
+  -e, [--delta-count=N]                        # Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count.
+                                               # Default: 20
 ```
 
 ```
@@ -90,29 +95,6 @@ Usage:
 
 print the version
 ```
-## How does it work
-
-- **Primary table**: A table against which a potential schema change is to be run
-- **Shadow table**: A copy of an existing primary table
-- **Audit table**: A table to store any updates/inserts/delete on a primary table
-
-![how-it-works](diagrams/how-it-works.png)
-
-
-1. Create an audit table to record changes made to the parent table.
-2. Acquire a brief `ACCESS EXCLUSIVE` lock to add a trigger on the parent table (for inserts, updates, deletes) to the audit table.
-3. Create a new shadow table and run ALTER/migration on the shadow table. 
-4. Copy all rows from the old table.
-5. Build indexes on the new table.
-6. Replay all changes accumulated in the audit table against the shadow table.
-   - Delete rows in the audit table as they are replayed.
-7. Once the delta (remaining rows) is ~20 rows, acquire an `ACCESS EXCLUSIVE` lock against the parent table within a transaction and:
-   - swap table names (shadow table <> parent table).
-   - update references in other tables (FKs) by dropping and re-creating the FKs with a `NOT VALID`.
-8. Runs `ANALYZE` on the new table.
-9. Validates all FKs that were added with `NOT VALID`.
-10. Drop parent (now old) table (OPTIONAL).
-
 ## Prominent features
 - `pg-osc` supports when a column is being added, dropped or renamed with no data loss. 
 - `pg-osc` acquires minimal locks throughout the process (read more below on the caveats).
@@ -121,26 +103,30 @@ print the version
 - Backfill old/new columns as data is copied from primary table to shadow table, and then perform the swap. [Example](#backfill-data)
 - **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-osc/issues/14)
 
+## Load test
+
+[More about the preliminary load test figures here](docs/load-test.md)
+
 ## Examples
 
 ### Renaming a column
 ```
+export PGPASSWORD=""
 pg-online-schema-change perform \
   --alter-statement 'ALTER TABLE books RENAME COLUMN email TO new_email' \
   --dbname "postgres" \
   --host "localhost" \
   --username "jamesbond" \
-  --password "" \
 ```
 
 ### Multiple ALTER statements
 ```
+export PGPASSWORD=""
 pg-online-schema-change perform \
   --alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE; ALTER TABLE books RENAME COLUMN email TO new_email;' \
   --dbname "postgres" \
   --host "localhost" \
   --username "jamesbond" \
-  --password "" \
   --drop
 ```
 
@@ -148,13 +134,30 @@ pg-online-schema-change perform \
 If the operation is being performed on a busy table, you can use `pg-osc`'s `kill-backend` functionality to kill other backends that may be competing with the `pg-osc` operation to acquire a lock for a brief while. The `ACCESS EXCLUSIVE` lock acquired by `pg-osc` is only held for a brief while and released after. You can tune how long `pg-osc` should wait before killing other backends (or if at all `pg-osc` should kill backends in the first place).
 
 ```
+export PGPASSWORD=""
 pg-online-schema-change perform \
   --alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
   --dbname "postgres" \
   --host "localhost" \
   --username "jamesbond" \
-  --password "" \
-  --wait-time-for-lock=5 \
+  --wait-time-for-lock 5 \
+  --kill-backends \
+  --drop
+```
+
+### Replaying larger workloads
+If you have a table with high write volume, the default replay iteration may not suffice. That is - you may see that `pg-osc` is replaying 1000 rows (`pull-batch-count`) in one go from the audit table. `pg-osc` also waits until the remaining row count (`delta-count`) in audit table is 20 before making the swap. You can tune these values to be higher for faster catch up on these kind of workloads.
+
+```
+export PGPASSWORD=""
+pg-online-schema-change perform \
+  --alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
+  --dbname "postgres" \
+  --host "localhost" \
+  --username "jamesbond" \
+  --pull-batch-count 2000
+  --delta-count 500
+  --wait-time-for-lock 5 \
   --kill-backends \
   --drop
 ```
@@ -183,7 +186,6 @@ pg-online-schema-change perform \
   --dbname "postgres" \
   --host "localhost" \
   --username "jamesbond" \
-  --password "" \
   --copy-statement "/src/query.sql" \
   --drop
 ```
@@ -197,7 +199,6 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
     --dbname "postgres" \
     --host "localhost" \
     --username "jamesbond" \
-    --password "" \
     --drop
 ```
 ## Caveats
@@ -215,6 +216,29 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
   - Can be fixed in future releases. Feel free to open a feature req.
 - Foreign keys are dropped & re-added to referencing tables with a `NOT VALID`. A follow on `VALIDATE CONSTRAINT` is run.
  	- Ensures that integrity is maintained and re-introducing FKs doesn't acquire additional locks, hence the `NOT VALID`.
+## How does it work
+
+- **Primary table**: A table against which a potential schema change is to be run
+- **Shadow table**: A copy of an existing primary table
+- **Audit table**: A table to store any updates/inserts/delete on a primary table
+
+![how-it-works](docs/how-it-works.png)
+
+
+1. Create an audit table to record changes made to the parent table.
+2. Acquire a brief `ACCESS EXCLUSIVE` lock to add a trigger on the parent table (for inserts, updates, deletes) to the audit table.
+3. Create a new shadow table and run ALTER/migration on the shadow table. 
+4. Copy all rows from the old table.
+5. Build indexes on the new table.
+6. Replay all changes accumulated in the audit table against the shadow table.
+   - Delete rows in the audit table as they are replayed.
+7. Once the delta (remaining rows) is ~20 rows, acquire an `ACCESS EXCLUSIVE` lock against the parent table within a transaction and:
+   - swap table names (shadow table <> parent table).
+   - update references in other tables (FKs) by dropping and re-creating the FKs with a `NOT VALID`.
+8. Runs `ANALYZE` on the new table.
+9. Validates all FKs that were added with `NOT VALID`.
+10. Drop parent (now old) table (OPTIONAL).
+
 ## Development
 
 - Install ruby 3.0

data/docs/load-test.md

@@ -0,0 +1,138 @@
+# Preliminary Load Test
+
+## pg-osc: No downtime schema changes with 7K+ writes/s & 12k+ reads/s
+
+This is a very basic load test performed with `pgbench` against a single instance PostgreSQL DB running on DigitialOcean with the following configuration:
+
+- **128GB RAM**
+- **32vCPU**
+- **695GB Disk**
+- Trasanction based connection pool with **500 pool limit**
+
+Total time taken to run schema change: **<3mins**
+
+## Simulating load with pgbench
+
+**Initialize**
+```
+pgbench -p $PORT --initialize -s 20 -F 20 --foreign-keys --host $HOST -U $USERNAME  -d $DB 
+```
+
+This creates bunch of pgbench tables. The table being used with `pg-osc` is `pgbench_accounts` which has FKs and also references by other tables with FKS, containing 2M rows.
+
+**Begin**
+```
+pgbench -p $PORT -j 72 -c 288 -T 500 -r --host $DB_HOST -U $USERNAME -d $DB  
+```
+
+## Running pg-osc
+
+Simple `ALTER` statement for experimentation purposes.
+
+```sql
+ALTER TABLE pgbench_accounts ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;
+```
+
+**Execution**
+
+```bash
+bundle exec bin/pg-online-schema-change perform \ 
+-a 'ALTER TABLE pgbench_accounts ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
+-d "pool" \
+-p 25061
+-h "..." \
+-u "..." \
+--pull-batch-count 2000 \
+--delta-count 200
+```
+
+## Outcome
+
+**pgbench results**
+
+```
+number of transactions actually processed: 1060382
+latency average = 144.874 ms
+tps = 1767.057392 (including connections establishing)
+tps = 1777.971823 (excluding connections establishing)
+statement latencies in milliseconds:
+         0.479  \set aid random(1, 100000 * :scale)
+         0.409  \set bid random(1, 1 * :scale)
+         0.247  \set tid random(1, 10 * :scale)
+         0.208  \set delta random(-5000, 5000)
+         3.136  BEGIN;
+         4.243  UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;
+         4.488  SELECT abalance FROM pgbench_accounts WHERE aid = :aid;
+        71.017  UPDATE pgbench_tellers SET tbalance = tbalance + :delta WHERE tid = :tid;
+        46.689  UPDATE pgbench_branches SET bbalance = bbalance + :delta WHERE bid = :bid;
+         4.035  INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);
+         4.166  END;
+```
+
+**Metrics**
+![load-test](load-test-1.png)
+
+**New table structure**
+
+Added `purchased` column.
+
+```
+defaultdb=> \d+ pgbench_accounts;
+                                  Table "public.pgbench_accounts"
+  Column   |     Type      | Collation | Nullable | Default | Storage  | Stats target | Description 
+-----------+---------------+-----------+----------+---------+----------+--------------+-------------
+ aid       | integer       |           | not null |         | plain    |              | 
+ bid       | integer       |           |          |         | plain    |              | 
+ abalance  | integer       |           |          |         | plain    |              | 
+ filler    | character(84) |           |          |         | extended |              | 
+ purchased | boolean       |           |          | false   | plain    |              | 
+Indexes:
+    "pgosc_st_pgbench_accounts_815029_pkey" PRIMARY KEY, btree (aid)
+Foreign-key constraints:
+    "pgbench_accounts_bid_fkey" FOREIGN KEY (bid) REFERENCES pgbench_branches(bid)
+Referenced by:
+    TABLE "pgbench_history" CONSTRAINT "pgbench_history_aid_fkey" FOREIGN KEY (aid) REFERENCES pgbench_accounts(aid)
+Options: autovacuum_enabled=false, fillfactor=20
+```
+
+**Logs**
+
+<details>
+<summary>Logs from pg-osc</summary>
+
+```json
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:01.147-05:00","v":0,"msg":"Setting up audit table","audit_table":"pgosc_at_pgbench_accounts_714a8b","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:01.660-05:00","v":0,"msg":"Setting up triggers","version":"0.4.0"}
+NOTICE:  trigger "primary_to_audit_table_trigger" for relation "pgbench_accounts" does not exist, skipping
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:01.814-05:00","v":0,"msg":"Setting up shadow table","shadow_table":"pgosc_st_pgbench_accounts_714a8b","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:02.169-05:00","v":0,"msg":"Running alter statement on shadow table","shadow_table":"pgosc_st_pgbench_accounts_714a8b","parent_table":"pgbench_accounts","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:02.204-05:00","v":0,"msg":"Clearing contents of audit table before copy..","shadow_table":"pgosc_st_pgbench_accounts_714a8b","parent_table":"pgbench_accounts","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:02.240-05:00","v":0,"msg":"Copying contents..","shadow_table":"pgosc_st_pgbench_accounts_714a8b","parent_table":"pgbench_accounts","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:20.481-05:00","v":0,"msg":"Performing ANALYZE!","version":"0.4.0"}
+INFO:  analyzing "public.pgbench_accounts"
+INFO:  "pgbench_accounts": scanned 30000 of 166667 pages, containing 360000 live rows and 200 dead rows; 30000 rows in sample, 2000004 estimated total rows
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:21.078-05:00","v":0,"msg":"Replaying rows, count: 2000","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:21.580-05:00","v":0,"msg":"Replaying rows, count: 2000","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:22.022-05:00","v":0,"msg":"Replaying rows, count: 2000","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:22.490-05:00","v":0,"msg":"Replaying rows, count: 2000","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:22.866-05:00","v":0,"msg":"Replaying rows, count: 661","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:23.212-05:00","v":0,"msg":"Replaying rows, count: 533","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:23.512-05:00","v":0,"msg":"Replaying rows, count: 468","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:23.809-05:00","v":0,"msg":"Remaining rows below delta count, proceeding towards swap","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:23.809-05:00","v":0,"msg":"Performing swap!","version":"0.4.0"}
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:24.259-05:00","v":0,"msg":"Replaying rows, count: 449","version":"0.4.0"}
+NOTICE:  trigger "primary_to_audit_table_trigger" for relation "pgbench_accounts" does not exist, skipping
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:24.650-05:00","v":0,"msg":"Performing ANALYZE!","version":"0.4.0"}
+INFO:  analyzing "public.pgbench_accounts"
+INFO:  "pgbench_accounts": scanned 30000 of 32935 pages, containing 1821834 live rows and 6056 dead rows; 30000 rows in sample, 2000070 estimated total rows
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:24.941-05:00","v":0,"msg":"Validating constraints!","version":"0.4.0"}
+NOTICE:  table "pgosc_st_pgbench_accounts_714a8b" does not exist, skipping
+{"name":"pg-online-schema-change","hostname":"MacBook-Pro.local","pid":13263,"level":30,"time":"2022-02-25T17:22:26.159-05:00","v":0,"msg":"All tasks successfully completed","version":"0.4.0"}
+```
+
+</details>
+
+
+## Conclusion
+
+By tweaking `--pull-batch-count` to `2000` (replay 2k rows at once) and `--delta-count` to `200` (time to swap when remaining rows is <200), `pg-osc` was able to perform the schema change with no impact within very quick time. Depending on the database size and load on the table, you can further tune them to achieve desired impact. At some point this is going to plateau - I can imagine the replay factor not working quite well for say 100k commits/s workloads. So, YMMV.

data/lib/pg_online_schema_change/cli.rb

@@ -3,8 +3,10 @@
 require "thor"
 
 module PgOnlineSchemaChange
+  PULL_BATCH_COUNT = 1000
+  DELTA_COUNT = 20
   class CLI < Thor
-    desc "perform", "Perform the set of operations to safely apply the schema change with minimal locks"
+    desc "perform", "Safely apply schema changes with minimal locks"
     method_option :alter_statement, aliases: "-a", type: :string, required: true,
                                     desc: "The ALTER statement to perform the schema change"
     method_option :schema, aliases: "-s", type: :string, required: true, default: "public",
@@ -13,7 +15,7 @@ module PgOnlineSchemaChange
     method_option :host, aliases: "-h", type: :string, required: true, desc: "Server host where the Database is located"
     method_option :username, aliases: "-u", type: :string, required: true, desc: "Username for the Database"
     method_option :port, aliases: "-p", type: :numeric, required: true, default: 5432, desc: "Port for the Database"
-    method_option :password, aliases: "-w", type: :string, required: true, desc: "Password for the Database"
+    method_option :password, aliases: "-w", type: :string, required: true, desc: "DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
     method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Emit logs in debug mode"
     method_option :drop, aliases: "-f", type: :boolean, default: false,
                          desc: "Drop the original table in the end after the swap"
@@ -23,11 +25,19 @@ module PgOnlineSchemaChange
                                        desc: "Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock."
     method_option :copy_statement, aliases: "-c", type: :string, required: false, default: "",
                                    desc: "Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README."
+    method_option :pull_batch_count, aliases: "-b", type: :numeric, required: false, default: PULL_BATCH_COUNT,
+                                     desc: "Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count."
+    method_option :delta_count, aliases: "-e", type: :numeric, required: false, default: DELTA_COUNT,
+                                desc: "Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count."
 
     def perform
       client_options = Struct.new(*options.keys.map(&:to_sym)).new(*options.values)
-
       PgOnlineSchemaChange.logger(verbose: client_options.verbose)
+
+      PgOnlineSchemaChange.logger.warn("DEPRECATED: -w is deprecated. Please pass PGPASSWORD environment variable instead.") if client_options.password
+
+      client_options.password = ENV["PGPASSWORD"] || client_options.password
+
       PgOnlineSchemaChange::Orchestrate.run!(client_options)
     end
 

data/lib/pg_online_schema_change/client.rb

@@ -5,7 +5,7 @@ require "pg"
 module PgOnlineSchemaChange
   class Client
     attr_accessor :alter_statement, :schema, :dbname, :host, :username, :port, :password, :connection, :table, :drop,
-                  :kill_backends, :wait_time_for_lock, :copy_statement
+                  :kill_backends, :wait_time_for_lock, :copy_statement, :pull_batch_count, :delta_count
 
     def initialize(options)
       @alter_statement = options.alter_statement
@@ -18,6 +18,8 @@ module PgOnlineSchemaChange
       @drop = options.drop
       @kill_backends = options.kill_backends
       @wait_time_for_lock = options.wait_time_for_lock
+      @pull_batch_count = options.pull_batch_count
+      @delta_count = options.delta_count
 
       handle_copy_statement(options.copy_statement)
       handle_validations

data/lib/pg_online_schema_change/orchestrate.rb

@@ -204,7 +204,7 @@ module PgOnlineSchemaChange
           return Query.run(client.connection, query, true)
         end
 
-        sql = Query.copy_data_statement(client, shadow_table)
+        sql = Query.copy_data_statement(client, shadow_table, true)
         Query.run(client.connection, sql, true)
       ensure
         Query.run(client.connection, "COMMIT;") # commit the serializable transaction

data/lib/pg_online_schema_change/replay.rb

@@ -7,9 +7,6 @@ module PgOnlineSchemaChange
     extend Helper
 
     class << self
-      PULL_BATCH_COUNT = 1000
-      DELTA_COUNT = 20
-
       # This, picks PULL_BATCH_COUNT rows by primary key from audit_table,
       # replays it on the shadow_table. Once the batch is done,
       # it them deletes those PULL_BATCH_COUNT rows from audit_table. Then, pull another batch,
@@ -20,7 +17,7 @@ module PgOnlineSchemaChange
         loop do
           rows = rows_to_play
 
-          raise CountBelowDelta if rows.count <= DELTA_COUNT
+          raise CountBelowDelta if rows.count <= client.delta_count
 
           play!(rows)
         end
@@ -28,7 +25,7 @@ module PgOnlineSchemaChange
 
       def rows_to_play(reuse_trasaction = false)
         select_query = <<~SQL
-          SELECT * FROM #{audit_table} ORDER BY #{audit_table_pk} LIMIT #{PULL_BATCH_COUNT};
+          SELECT * FROM #{audit_table} ORDER BY #{audit_table_pk} LIMIT #{client.pull_batch_count};
         SQL
 
         rows = []

data/lib/pg_online_schema_change/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgOnlineSchemaChange
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/pg_online_schema_change.rb

@@ -6,12 +6,12 @@ require "ougai"
 require "pg_online_schema_change/version"
 require "pg_online_schema_change/helper"
 require "pg_online_schema_change/functions"
-require "pg_online_schema_change/cli"
 require "pg_online_schema_change/client"
 require "pg_online_schema_change/query"
 require "pg_online_schema_change/store"
 require "pg_online_schema_change/replay"
 require "pg_online_schema_change/orchestrate"
+require "pg_online_schema_change/cli"
 
 module PgOnlineSchemaChange
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pg_online_schema_change
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Shayon Mukherjee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-22 00:00:00.000000000 Z
+date: 2022-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ougai
@@ -204,9 +204,11 @@ files:
 - bin/console
 - bin/pg-online-schema-change
 - bin/setup
-- diagrams/how-it-works.excalidraw
-- diagrams/how-it-works.png
 - docker-compose.yml
+- docs/how-it-works.excalidraw
+- docs/how-it-works.png
+- docs/load-test-1.png
+- docs/load-test.md
 - lib/pg_online_schema_change.rb
 - lib/pg_online_schema_change/cli.rb
 - lib/pg_online_schema_change/client.rb