pg_online_schema_change 0.7.5 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- pg_online_schema_change (0.7.5)
4
+ pg_online_schema_change (0.7.6)
5
5
  ougai (~> 2.0.0)
6
6
  pg (~> 1.3.2)
7
7
  pg_query (~> 2.1.3)
@@ -13,76 +13,107 @@ GEM
13
13
  ast (2.4.2)
14
14
  coderay (1.1.3)
15
15
  diff-lcs (1.5.0)
16
- google-protobuf (3.21.7)
16
+ google-protobuf (3.23.0-arm64-darwin)
17
+ haml (6.1.1)
18
+ temple (>= 0.8.2)
19
+ thor
20
+ tilt
21
+ json (2.6.3)
17
22
  method_source (1.0.0)
18
- oj (3.13.21)
23
+ oj (3.14.3)
19
24
  ougai (2.0.0)
20
25
  oj (~> 3.10)
21
- parallel (1.22.1)
22
- parser (3.1.2.1)
26
+ parallel (1.23.0)
27
+ parser (3.2.2.1)
23
28
  ast (~> 2.4.1)
24
29
  pg (1.3.5)
25
30
  pg_query (2.1.4)
26
31
  google-protobuf (>= 3.19.2)
27
- pry (0.14.1)
32
+ prettier_print (1.2.1)
33
+ pry (0.14.2)
28
34
  coderay (~> 1.1)
29
35
  method_source (~> 1.0)
30
36
  rainbow (3.1.1)
31
37
  rake (13.0.6)
32
- regexp_parser (2.5.0)
38
+ rbs (3.1.0)
39
+ regexp_parser (2.8.0)
33
40
  rexml (3.2.5)
34
- rspec (3.11.0)
35
- rspec-core (~> 3.11.0)
36
- rspec-expectations (~> 3.11.0)
37
- rspec-mocks (~> 3.11.0)
38
- rspec-core (3.11.0)
39
- rspec-support (~> 3.11.0)
40
- rspec-expectations (3.11.1)
41
+ rspec (3.12.0)
42
+ rspec-core (~> 3.12.0)
43
+ rspec-expectations (~> 3.12.0)
44
+ rspec-mocks (~> 3.12.0)
45
+ rspec-core (3.12.2)
46
+ rspec-support (~> 3.12.0)
47
+ rspec-expectations (3.12.3)
41
48
  diff-lcs (>= 1.2.0, < 2.0)
42
- rspec-support (~> 3.11.0)
43
- rspec-mocks (3.11.1)
49
+ rspec-support (~> 3.12.0)
50
+ rspec-mocks (3.12.5)
44
51
  diff-lcs (>= 1.2.0, < 2.0)
45
- rspec-support (~> 3.11.0)
46
- rspec-support (3.11.1)
47
- rubocop (1.23.0)
52
+ rspec-support (~> 3.12.0)
53
+ rspec-support (3.12.0)
54
+ rubocop (1.51.0)
55
+ json (~> 2.3)
48
56
  parallel (~> 1.10)
49
- parser (>= 3.0.0.0)
57
+ parser (>= 3.2.0.0)
50
58
  rainbow (>= 2.2.2, < 4.0)
51
59
  regexp_parser (>= 1.8, < 3.0)
52
- rexml
53
- rubocop-ast (>= 1.12.0, < 2.0)
60
+ rexml (>= 3.2.5, < 4.0)
61
+ rubocop-ast (>= 1.28.0, < 2.0)
54
62
  ruby-progressbar (~> 1.7)
55
- unicode-display_width (>= 1.4.0, < 3.0)
56
- rubocop-ast (1.21.0)
57
- parser (>= 3.1.1.0)
58
- rubocop-packaging (0.5.1)
59
- rubocop (>= 0.89, < 2.0)
60
- rubocop-performance (1.12.0)
63
+ unicode-display_width (>= 2.4.0, < 3.0)
64
+ rubocop-ast (1.28.1)
65
+ parser (>= 3.2.1.0)
66
+ rubocop-capybara (2.18.0)
67
+ rubocop (~> 1.41)
68
+ rubocop-factory_bot (2.22.0)
69
+ rubocop (~> 1.33)
70
+ rubocop-packaging (0.5.2)
71
+ rubocop (>= 1.33, < 2.0)
72
+ rubocop-performance (1.17.1)
61
73
  rubocop (>= 1.7.0, < 2.0)
62
74
  rubocop-ast (>= 0.4.0)
63
75
  rubocop-rake (0.6.0)
64
76
  rubocop (~> 1.0)
65
- rubocop-rspec (2.7.0)
66
- rubocop (~> 1.19)
67
- ruby-progressbar (1.11.0)
68
- thor (1.2.1)
69
- unicode-display_width (2.3.0)
77
+ rubocop-rspec (2.22.0)
78
+ rubocop (~> 1.33)
79
+ rubocop-capybara (~> 2.17)
80
+ rubocop-factory_bot (~> 2.22)
81
+ ruby-progressbar (1.13.0)
82
+ syntax_tree (6.1.1)
83
+ prettier_print (>= 1.2.0)
84
+ syntax_tree-haml (4.0.3)
85
+ haml (>= 5.2)
86
+ prettier_print (>= 1.2.1)
87
+ syntax_tree (>= 6.0.0)
88
+ syntax_tree-rbs (1.0.0)
89
+ prettier_print
90
+ rbs
91
+ syntax_tree (>= 2.0.1)
92
+ temple (0.10.0)
93
+ thor (1.2.2)
94
+ tilt (2.1.0)
95
+ unicode-display_width (2.4.2)
70
96
 
71
97
  PLATFORMS
72
98
  arm64-darwin-20
73
99
  arm64-darwin-21
100
+ arm64-darwin-22
74
101
  x86_64-linux
75
102
 
76
103
  DEPENDENCIES
77
104
  pg_online_schema_change!
78
- pry (~> 0.14.1)
79
- rake (~> 13.0)
80
- rspec (~> 3.0)
81
- rubocop (~> 1.23.0)
82
- rubocop-packaging (~> 0.5.1)
83
- rubocop-performance (~> 1.12.0)
84
- rubocop-rake (~> 0.6.0)
85
- rubocop-rspec (~> 2.7.0)
105
+ prettier_print
106
+ pry
107
+ rake
108
+ rspec
109
+ rubocop
110
+ rubocop-packaging
111
+ rubocop-performance
112
+ rubocop-rake
113
+ rubocop-rspec
114
+ syntax_tree
115
+ syntax_tree-haml
116
+ syntax_tree-rbs
86
117
 
87
118
  BUNDLED WITH
88
119
  2.3.24
data/README.md CHANGED
@@ -5,7 +5,7 @@
5
5
  [![Smoke Test PG 13.6](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml)
6
6
  [![Gem Version](https://badge.fury.io/rb/pg_online_schema_change.svg)](https://badge.fury.io/rb/pg_online_schema_change)
7
7
 
8
- pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads.
8
+ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads.
9
9
 
10
10
  `pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it creates a shadow table that looks structurally the same as the primary table, performs the schema change on the shadow table, copies contents from the primary table to the shadow table and swaps the table names in the end while preserving all changes to the primary table using triggers (via audit table).
11
11
 
@@ -21,11 +21,11 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
21
21
  - [Prominent features](#prominent-features)
22
22
  - [Load test](#load-test)
23
23
  - [Examples](#examples)
24
- * [Renaming a column](#renaming-a-column)
25
- * [Multiple ALTER statements](#multiple-alter-statements)
26
- * [Kill other backends after 5s](#kill-other-backends-after-5s)
27
- * [Backfill data](#backfill-data)
28
- * [Running using Docker](#running-using-docker)
24
+ - [Renaming a column](#renaming-a-column)
25
+ - [Multiple ALTER statements](#multiple-alter-statements)
26
+ - [Kill other backends after 5s](#kill-other-backends-after-5s)
27
+ - [Backfill data](#backfill-data)
28
+ - [Running using Docker](#running-using-docker)
29
29
  - [Caveats](#caveats)
30
30
  - [How does it work](#how-does-it-work)
31
31
  - [Development](#development)
@@ -33,12 +33,13 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
33
33
  - [Contributing](#contributing)
34
34
  - [License](#license)
35
35
  - [Code of Conduct](#code-of-conduct)
36
+
36
37
  ## Installation
37
38
 
38
39
  Add this line to your application's Gemfile:
39
40
 
40
41
  ```ruby
41
- gem 'pg_online_schema_change'
42
+ gem "pg_online_schema_change"
42
43
  ```
43
44
 
44
45
  And then execute:
@@ -56,7 +57,9 @@ Or via Docker:
56
57
  docker pull shayonj/pg-osc:latest
57
58
 
58
59
  https://hub.docker.com/r/shayonj/pg-osc
60
+
59
61
  ## Requirements
62
+
60
63
  - PostgreSQL 9.6 and later
61
64
  - Ruby 2.6 and later
62
65
  - Database user should have permissions for `TRIGGER` and/or a `SUPERUSER`
@@ -97,8 +100,10 @@ Usage:
97
100
 
98
101
  print the version
99
102
  ```
103
+
100
104
  ## Prominent features
101
- - `pg-osc` supports when a column is being added, dropped or renamed with no data loss.
105
+
106
+ - `pg-osc` supports when a column is being added, dropped or renamed with no data loss.
102
107
  - `pg-osc` acquires minimal locks throughout the process (read more below on the caveats).
103
108
  - Copies over indexes and Foreign keys.
104
109
  - Optionally drop or retain old tables in the end.
@@ -113,6 +118,7 @@ print the version
113
118
  ## Examples
114
119
 
115
120
  ### Renaming a column
121
+
116
122
  ```
117
123
  export PGPASSWORD=""
118
124
  pg-online-schema-change perform \
@@ -123,6 +129,7 @@ pg-online-schema-change perform \
123
129
  ```
124
130
 
125
131
  ### Multiple ALTER statements
132
+
126
133
  ```
127
134
  export PGPASSWORD=""
128
135
  pg-online-schema-change perform \
@@ -134,6 +141,7 @@ pg-online-schema-change perform \
134
141
  ```
135
142
 
136
143
  ### Kill other backends after 5s
144
+
137
145
  If the operation is being performed on a busy table, you can use `pg-osc`'s `kill-backend` functionality to kill other backends that may be competing with the `pg-osc` operation to acquire a lock for a brief while. The `ACCESS EXCLUSIVE` lock acquired by `pg-osc` is only held for a brief while and released after. You can tune how long `pg-osc` should wait before killing other backends (or if at all `pg-osc` should kill backends in the first place).
138
146
 
139
147
  ```
@@ -149,6 +157,7 @@ pg-online-schema-change perform \
149
157
  ```
150
158
 
151
159
  ### Replaying larger workloads
160
+
152
161
  If you have a table with high write volume, the default replay iteration may not suffice. That is - you may see that `pg-osc` is replaying 1000 rows (`pull-batch-count`) in one go from the audit table. `pg-osc` also waits until the remaining row count (`delta-count`) in audit table is 20 before making the swap. You can tune these values to be higher for faster catch up on these kind of workloads.
153
162
 
154
163
  ```
@@ -164,10 +173,13 @@ pg-online-schema-change perform \
164
173
  --kill-backends \
165
174
  --drop
166
175
  ```
176
+
167
177
  ### Backfill data
178
+
168
179
  When inserting data into the shadow table, instead of just copying all columns and rows from the primary table, you can pass in a custom sql file to perform the copy and do any additional work. For instance - backfilling certain columns. By providing the `copy-statement`, `pg-osc` will instead play the query to perform the copy operation.
169
180
 
170
181
  **IMPORTANT NOTES:**
182
+
171
183
  - It is possible to violate a constraint accidentally or not copy data, **so proceed with caution**.
172
184
  - You must use OUTER JOINs when joining in the custom SQL, or you will **lose rows** which do not match the joined table.
173
185
  - The `ALTER` statement can change the table's structure, **so proceed with caution**.
@@ -204,22 +216,25 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
204
216
  --username "jamesbond" \
205
217
  --drop
206
218
  ```
219
+
207
220
  ## Caveats
221
+
208
222
  - Partitioned tables are not supported as of yet. Pull requests and ideas welcome.
209
223
  - A primary key should exist on the table; without it, `pg-osc` will raise an exception
210
- - This is because - currently there is no other way to uniquely identify rows during replay.
224
+ - This is because - currently there is no other way to uniquely identify rows during replay.
211
225
  - `pg-osc` will acquire `ACCESS EXCLUSIVE` lock on the parent table twice during the operation.
212
- - First, when setting up the triggers and the shadow table.
213
- - Next, when performing the swap and updating FK references.
214
- - Note: If `kill-backends` is passed, it will attempt to terminate any competing operations during both times.
226
+ - First, when setting up the triggers and the shadow table.
227
+ - Next, when performing the swap and updating FK references.
228
+ - Note: If `kill-backends` is passed, it will attempt to terminate any competing operations during both times.
215
229
  - By design, `pg-osc` doesn't kill any other DDLs being performed. It's best to not run any DDLs against the parent table during the operation.
216
230
  - Due to the nature of duplicating a table, there needs to be enough space on the disk to support the operation.
217
231
  - Index, constraints and sequence names will be altered and lose their original naming.
218
- - Can be fixed in future releases. Feel free to open a feature req.
219
- - Triggers are not carried over.
232
+ - Can be fixed in future releases. Feel free to open a feature req.
233
+ - Triggers are not carried over.
220
234
  - Can be fixed in future releases. Feel free to open a feature req.
221
235
  - Foreign keys are dropped & re-added to referencing tables with a `NOT VALID`. A follow on `VALIDATE CONSTRAINT` is run.
222
- - Ensures that integrity is maintained and re-introducing FKs doesn't acquire additional locks, hence the `NOT VALID`.
236
+ - Ensures that integrity is maintained and re-introducing FKs doesn't acquire additional locks, hence the `NOT VALID`.
237
+
223
238
  ## How does it work
224
239
 
225
240
  - **Primary table**: A table against which a potential schema change is to be run
@@ -228,10 +243,9 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
228
243
 
229
244
  ![how-it-works](docs/how-it-works.png)
230
245
 
231
-
232
246
  1. Create an audit table to record changes made to the parent table.
233
247
  2. Acquire a brief `ACCESS EXCLUSIVE` lock to add a trigger on the parent table (for inserts, updates, deletes) to the audit table.
234
- 3. Create a new shadow table and run ALTER/migration on the shadow table.
248
+ 3. Create a new shadow table and run ALTER/migration on the shadow table.
235
249
  4. Copy all rows from the old table.
236
250
  5. Build indexes on the new table.
237
251
  6. Replay all changes accumulated in the audit table against the shadow table.
@@ -245,22 +259,24 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
245
259
 
246
260
  ## Development
247
261
 
248
- - Install ruby 3.0
262
+ - Install ruby 3.1.3
263
+
249
264
  ```
250
265
  \curl -sSL https://get.rvm.io | bash
251
266
 
252
- rvm install 3.0.0
267
+ rvm install 3.1.3
253
268
 
254
- rvm use 3.0.0
269
+ rvm use 3.1.3
255
270
  ```
256
271
 
257
272
  - Spin up postgres via Docker Compose - `docker compose up`
258
- - `bundle exec rspec` to run the tests.
273
+ - `bundle exec rspec` to run the tests.
259
274
  - You can also run `bin/console` for an interactive prompt that will allow you to experiment.
260
275
 
261
- To install this gem onto your local machine, run `bundle exec rake install`.
276
+ To install this gem onto your local machine, run `bundle exec rake install`.
277
+
278
+ ### Local testing
262
279
 
263
- ### Local testing
264
280
  ```
265
281
  docker compose up
266
282
 
@@ -280,7 +296,7 @@ bundle exec bin/pg-online-schema-change perform -a 'ALTER TABLE pgbench_accounts
280
296
 
281
297
  ## Contributing
282
298
 
283
- Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-osc.
299
+ Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-osc.
284
300
 
285
301
  ## License
286
302
 
data/Rakefile CHANGED
@@ -9,4 +9,4 @@ require "rubocop/rake_task"
9
9
 
10
10
  RuboCop::RakeTask.new
11
11
 
12
- task default: %i[spec rubocop]
12
+ task default: [:spec, :rubocop]
data/docker-compose.yml CHANGED
@@ -1,4 +1,4 @@
1
- version: '2'
1
+ version: "2"
2
2
  services:
3
3
  postgres:
4
4
  image: postgres:9.6.2-alpine
data/docs/load-test.md CHANGED
@@ -14,15 +14,17 @@ Total time taken to run schema change: **<3mins**
14
14
  ## Simulating load with pgbench
15
15
 
16
16
  **Initialize**
17
+
17
18
  ```
18
- pgbench -p $PORT --initialize -s 20 -F 20 --foreign-keys --host $HOST -U $USERNAME -d $DB
19
+ pgbench -p $PORT --initialize -s 20 -F 20 --foreign-keys --host $HOST -U $USERNAME -d $DB
19
20
  ```
20
21
 
21
22
  This creates bunch of pgbench tables. The table being used with `pg-osc` is `pgbench_accounts` which has FKs and also references by other tables with FKS, containing 2M rows.
22
23
 
23
24
  **Begin**
25
+
24
26
  ```
25
- pgbench -p $PORT -j 72 -c 288 -T 500 -r --host $DB_HOST -U $USERNAME -d $DB
27
+ pgbench -p $PORT -j 72 -c 288 -T 500 -r --host $DB_HOST -U $USERNAME -d $DB
26
28
  ```
27
29
 
28
30
  ## Running pg-osc
@@ -36,7 +38,7 @@ ALTER TABLE pgbench_accounts ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;
36
38
  **Execution**
37
39
 
38
40
  ```bash
39
- bundle exec bin/pg-online-schema-change perform \
41
+ bundle exec bin/pg-online-schema-change perform \
40
42
  -a 'ALTER TABLE pgbench_accounts ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
41
43
  -d "pool" \
42
44
  -p 25061
@@ -79,13 +81,13 @@ Added `purchased` column.
79
81
  ```
80
82
  defaultdb=> \d+ pgbench_accounts;
81
83
  Table "public.pgbench_accounts"
82
- Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
84
+ Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
83
85
  -----------+---------------+-----------+----------+---------+----------+--------------+-------------
84
- aid | integer | | not null | | plain | |
85
- bid | integer | | | | plain | |
86
- abalance | integer | | | | plain | |
87
- filler | character(84) | | | | extended | |
88
- purchased | boolean | | | false | plain | |
86
+ aid | integer | | not null | | plain | |
87
+ bid | integer | | | | plain | |
88
+ abalance | integer | | | | plain | |
89
+ filler | character(84) | | | | extended | |
90
+ purchased | boolean | | | false | plain | |
89
91
  Indexes:
90
92
  "pgosc_st_pgbench_accounts_815029_pkey" PRIMARY KEY, btree (aid)
91
93
  Foreign-key constraints:
@@ -132,7 +134,6 @@ NOTICE: table "pgosc_st_pgbench_accounts_714a8b" does not exist, skipping
132
134
 
133
135
  </details>
134
136
 
135
-
136
137
  ## Conclusion
137
138
 
138
139
  By tweaking `--pull-batch-count` to `2000` (replay 2k rows at once) and `--delta-count` to `200` (time to swap when remaining rows is <200), `pg-osc` was able to perform the schema change with no impact within very quick time. Depending on the database size and load on the table, you can further tune them to achieve desired impact. At some point this is going to plateau - I can imagine the replay factor not working quite well for say 100k commits/s workloads. So, YMMV.
@@ -7,34 +7,98 @@ module PgOnlineSchemaChange
7
7
  DELTA_COUNT = 20
8
8
  class CLI < Thor
9
9
  desc "perform", "Safely apply schema changes with minimal locks"
10
- method_option :alter_statement, aliases: "-a", type: :string, required: true,
11
- desc: "The ALTER statement to perform the schema change"
12
- method_option :schema, aliases: "-s", type: :string, required: true, default: "public",
13
- desc: "The schema in which the table is"
14
- method_option :dbname, aliases: "-d", type: :string, required: true, desc: "Name of the database"
15
- method_option :host, aliases: "-h", type: :string, required: true, desc: "Server host where the Database is located"
16
- method_option :username, aliases: "-u", type: :string, required: true, desc: "Username for the Database"
17
- method_option :port, aliases: "-p", type: :numeric, required: true, default: 5432, desc: "Port for the Database"
18
- method_option :password, aliases: "-w", type: :string, required: false, default: "", desc: "DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
19
- method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Emit logs in debug mode"
20
- method_option :drop, aliases: "-f", type: :boolean, default: false,
21
- desc: "Drop the original table in the end after the swap"
22
- method_option :kill_backends, aliases: "-k", type: :boolean, default: false,
23
- desc: "Kill other competing queries/backends when trying to acquire lock for the shadow table creation and swap. It will wait for --wait-time-for-lock duration before killing backends and try upto 3 times."
24
- method_option :wait_time_for_lock, aliases: "-w", type: :numeric, default: 10,
25
- desc: "Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock."
26
- method_option :copy_statement, aliases: "-c", type: :string, required: false, default: "",
27
- desc: "Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README."
28
- method_option :pull_batch_count, aliases: "-b", type: :numeric, required: false, default: PULL_BATCH_COUNT,
29
- desc: "Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count."
30
- method_option :delta_count, aliases: "-e", type: :numeric, required: false, default: DELTA_COUNT,
31
- desc: "Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count."
10
+ method_option :alter_statement,
11
+ aliases: "-a",
12
+ type: :string,
13
+ required: true,
14
+ desc: "The ALTER statement to perform the schema change"
15
+ method_option :schema,
16
+ aliases: "-s",
17
+ type: :string,
18
+ required: true,
19
+ default: "public",
20
+ desc: "The schema in which the table is"
21
+ method_option :dbname,
22
+ aliases: "-d",
23
+ type: :string,
24
+ required: true,
25
+ desc: "Name of the database"
26
+ method_option :host,
27
+ aliases: "-h",
28
+ type: :string,
29
+ required: true,
30
+ desc: "Server host where the Database is located"
31
+ method_option :username,
32
+ aliases: "-u",
33
+ type: :string,
34
+ required: true,
35
+ desc: "Username for the Database"
36
+ method_option :port,
37
+ aliases: "-p",
38
+ type: :numeric,
39
+ required: true,
40
+ default: 5432,
41
+ desc: "Port for the Database"
42
+ method_option :password,
43
+ aliases: "-w",
44
+ type: :string,
45
+ required: false,
46
+ default: "",
47
+ desc:
48
+ "DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
49
+ method_option :verbose,
50
+ aliases: "-v",
51
+ type: :boolean,
52
+ default: false,
53
+ desc: "Emit logs in debug mode"
54
+ method_option :drop,
55
+ aliases: "-f",
56
+ type: :boolean,
57
+ default: false,
58
+ desc: "Drop the original table in the end after the swap"
59
+ method_option :kill_backends,
60
+ aliases: "-k",
61
+ type: :boolean,
62
+ default: false,
63
+ desc:
64
+ "Kill other competing queries/backends when trying to acquire lock for the shadow table creation and swap. It will wait for --wait-time-for-lock duration before killing backends and try upto 3 times."
65
+ method_option :wait_time_for_lock,
66
+ aliases: "-w",
67
+ type: :numeric,
68
+ default: 10,
69
+ desc:
70
+ "Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock."
71
+ method_option :copy_statement,
72
+ aliases: "-c",
73
+ type: :string,
74
+ required: false,
75
+ default: "",
76
+ desc:
77
+ "Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README."
78
+ method_option :pull_batch_count,
79
+ aliases: "-b",
80
+ type: :numeric,
81
+ required: false,
82
+ default: PULL_BATCH_COUNT,
83
+ desc:
84
+ "Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count."
85
+ method_option :delta_count,
86
+ aliases: "-e",
87
+ type: :numeric,
88
+ required: false,
89
+ default: DELTA_COUNT,
90
+ desc:
91
+ "Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count."
32
92
 
33
93
  def perform
34
94
  client_options = Struct.new(*options.keys.map(&:to_sym)).new(*options.values)
35
95
  PgOnlineSchemaChange.logger(verbose: client_options.verbose)
36
96
 
37
- PgOnlineSchemaChange.logger.warn("DEPRECATED: -w is deprecated. Please pass PGPASSWORD environment variable instead.") if client_options.password
97
+ if client_options.password
98
+ PgOnlineSchemaChange.logger.warn(
99
+ "DEPRECATED: -w is deprecated. Please pass PGPASSWORD environment variable instead.",
100
+ )
101
+ end
38
102
 
39
103
  client_options.password = ENV["PGPASSWORD"] || client_options.password
40
104
 
@@ -4,8 +4,22 @@ require "pg"
4
4
 
5
5
  module PgOnlineSchemaChange
6
6
  class Client
7
- attr_accessor :alter_statement, :schema, :dbname, :host, :username, :port, :password, :connection, :table, :table_name, :drop,
8
- :kill_backends, :wait_time_for_lock, :copy_statement, :pull_batch_count, :delta_count
7
+ attr_accessor :alter_statement,
8
+ :schema,
9
+ :dbname,
10
+ :host,
11
+ :username,
12
+ :port,
13
+ :password,
14
+ :connection,
15
+ :table,
16
+ :table_name,
17
+ :drop,
18
+ :kill_backends,
19
+ :wait_time_for_lock,
20
+ :copy_statement,
21
+ :pull_batch_count,
22
+ :delta_count
9
23
 
10
24
  def initialize(options)
11
25
  @alter_statement = options.alter_statement
@@ -24,13 +38,8 @@ module PgOnlineSchemaChange
24
38
  handle_copy_statement(options.copy_statement)
25
39
  handle_validations
26
40
 
27
- @connection = PG.connect(
28
- dbname: @dbname,
29
- host: @host,
30
- user: @username,
31
- password: @password,
32
- port: @port,
33
- )
41
+ @connection =
42
+ PG.connect(dbname: @dbname, host: @host, user: @username, password: @password, port: @port)
34
43
 
35
44
  @table = Query.table(@alter_statement)
36
45
  @table_name = Query.table_name(@alter_statement, @table)
@@ -39,11 +48,13 @@ module PgOnlineSchemaChange
39
48
  end
40
49
 
41
50
  def handle_validations
42
- raise Error, "Not a valid ALTER statement: #{@alter_statement}" unless Query.alter_statement?(@alter_statement)
51
+ unless Query.alter_statement?(@alter_statement)
52
+ raise Error, "Not a valid ALTER statement: #{@alter_statement}"
53
+ end
43
54
 
44
55
  return if Query.same_table?(@alter_statement)
45
56
 
46
- raise Error "All statements should belong to the same table: #{@alter_statement}"
57
+ raise Error("All statements should belong to the same table: #{@alter_statement}")
47
58
  end
48
59
 
49
60
  def handle_copy_statement(statement)
@@ -52,7 +63,7 @@ module PgOnlineSchemaChange
52
63
  file_path = File.expand_path(statement)
53
64
  raise Error, "File not found: #{file_path}" unless File.file?(file_path)
54
65
 
55
- @copy_statement = File.open(file_path, "rb", &:read)
66
+ @copy_statement = File.binread(file_path)
56
67
  end
57
68
  end
58
69
  end