pg_online_schema_change 0.7.5 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.prettierignore +4 -0
- data/.prettierrc +13 -0
- data/.rubocop.yml +204 -75
- data/.rubocop_todo.yml +11 -30
- data/.ruby-version +1 -1
- data/CHANGELOG.md +35 -21
- data/CODE_OF_CONDUCT.md +11 -11
- data/Gemfile.lock +72 -41
- data/README.md +40 -24
- data/Rakefile +1 -1
- data/docker-compose.yml +1 -1
- data/docs/load-test.md +11 -10
- data/lib/pg_online_schema_change/cli.rb +87 -23
- data/lib/pg_online_schema_change/client.rb +23 -12
- data/lib/pg_online_schema_change/orchestrate.rb +59 -24
- data/lib/pg_online_schema_change/query.rb +128 -98
- data/lib/pg_online_schema_change/replay.rb +10 -20
- data/lib/pg_online_schema_change/version.rb +1 -1
- data/lib/pg_online_schema_change.rb +15 -9
- data/package.json +13 -0
- data/yarn.lock +15 -0
- metadata +96 -36
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
pg_online_schema_change (0.7.
|
|
4
|
+
pg_online_schema_change (0.7.6)
|
|
5
5
|
ougai (~> 2.0.0)
|
|
6
6
|
pg (~> 1.3.2)
|
|
7
7
|
pg_query (~> 2.1.3)
|
|
@@ -13,76 +13,107 @@ GEM
|
|
|
13
13
|
ast (2.4.2)
|
|
14
14
|
coderay (1.1.3)
|
|
15
15
|
diff-lcs (1.5.0)
|
|
16
|
-
google-protobuf (3.
|
|
16
|
+
google-protobuf (3.23.0-arm64-darwin)
|
|
17
|
+
haml (6.1.1)
|
|
18
|
+
temple (>= 0.8.2)
|
|
19
|
+
thor
|
|
20
|
+
tilt
|
|
21
|
+
json (2.6.3)
|
|
17
22
|
method_source (1.0.0)
|
|
18
|
-
oj (3.
|
|
23
|
+
oj (3.14.3)
|
|
19
24
|
ougai (2.0.0)
|
|
20
25
|
oj (~> 3.10)
|
|
21
|
-
parallel (1.
|
|
22
|
-
parser (3.
|
|
26
|
+
parallel (1.23.0)
|
|
27
|
+
parser (3.2.2.1)
|
|
23
28
|
ast (~> 2.4.1)
|
|
24
29
|
pg (1.3.5)
|
|
25
30
|
pg_query (2.1.4)
|
|
26
31
|
google-protobuf (>= 3.19.2)
|
|
27
|
-
|
|
32
|
+
prettier_print (1.2.1)
|
|
33
|
+
pry (0.14.2)
|
|
28
34
|
coderay (~> 1.1)
|
|
29
35
|
method_source (~> 1.0)
|
|
30
36
|
rainbow (3.1.1)
|
|
31
37
|
rake (13.0.6)
|
|
32
|
-
|
|
38
|
+
rbs (3.1.0)
|
|
39
|
+
regexp_parser (2.8.0)
|
|
33
40
|
rexml (3.2.5)
|
|
34
|
-
rspec (3.
|
|
35
|
-
rspec-core (~> 3.
|
|
36
|
-
rspec-expectations (~> 3.
|
|
37
|
-
rspec-mocks (~> 3.
|
|
38
|
-
rspec-core (3.
|
|
39
|
-
rspec-support (~> 3.
|
|
40
|
-
rspec-expectations (3.
|
|
41
|
+
rspec (3.12.0)
|
|
42
|
+
rspec-core (~> 3.12.0)
|
|
43
|
+
rspec-expectations (~> 3.12.0)
|
|
44
|
+
rspec-mocks (~> 3.12.0)
|
|
45
|
+
rspec-core (3.12.2)
|
|
46
|
+
rspec-support (~> 3.12.0)
|
|
47
|
+
rspec-expectations (3.12.3)
|
|
41
48
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
42
|
-
rspec-support (~> 3.
|
|
43
|
-
rspec-mocks (3.
|
|
49
|
+
rspec-support (~> 3.12.0)
|
|
50
|
+
rspec-mocks (3.12.5)
|
|
44
51
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
45
|
-
rspec-support (~> 3.
|
|
46
|
-
rspec-support (3.
|
|
47
|
-
rubocop (1.
|
|
52
|
+
rspec-support (~> 3.12.0)
|
|
53
|
+
rspec-support (3.12.0)
|
|
54
|
+
rubocop (1.51.0)
|
|
55
|
+
json (~> 2.3)
|
|
48
56
|
parallel (~> 1.10)
|
|
49
|
-
parser (>= 3.
|
|
57
|
+
parser (>= 3.2.0.0)
|
|
50
58
|
rainbow (>= 2.2.2, < 4.0)
|
|
51
59
|
regexp_parser (>= 1.8, < 3.0)
|
|
52
|
-
rexml
|
|
53
|
-
rubocop-ast (>= 1.
|
|
60
|
+
rexml (>= 3.2.5, < 4.0)
|
|
61
|
+
rubocop-ast (>= 1.28.0, < 2.0)
|
|
54
62
|
ruby-progressbar (~> 1.7)
|
|
55
|
-
unicode-display_width (>=
|
|
56
|
-
rubocop-ast (1.
|
|
57
|
-
parser (>= 3.
|
|
58
|
-
rubocop-
|
|
59
|
-
rubocop (
|
|
60
|
-
rubocop-
|
|
63
|
+
unicode-display_width (>= 2.4.0, < 3.0)
|
|
64
|
+
rubocop-ast (1.28.1)
|
|
65
|
+
parser (>= 3.2.1.0)
|
|
66
|
+
rubocop-capybara (2.18.0)
|
|
67
|
+
rubocop (~> 1.41)
|
|
68
|
+
rubocop-factory_bot (2.22.0)
|
|
69
|
+
rubocop (~> 1.33)
|
|
70
|
+
rubocop-packaging (0.5.2)
|
|
71
|
+
rubocop (>= 1.33, < 2.0)
|
|
72
|
+
rubocop-performance (1.17.1)
|
|
61
73
|
rubocop (>= 1.7.0, < 2.0)
|
|
62
74
|
rubocop-ast (>= 0.4.0)
|
|
63
75
|
rubocop-rake (0.6.0)
|
|
64
76
|
rubocop (~> 1.0)
|
|
65
|
-
rubocop-rspec (2.
|
|
66
|
-
rubocop (~> 1.
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
77
|
+
rubocop-rspec (2.22.0)
|
|
78
|
+
rubocop (~> 1.33)
|
|
79
|
+
rubocop-capybara (~> 2.17)
|
|
80
|
+
rubocop-factory_bot (~> 2.22)
|
|
81
|
+
ruby-progressbar (1.13.0)
|
|
82
|
+
syntax_tree (6.1.1)
|
|
83
|
+
prettier_print (>= 1.2.0)
|
|
84
|
+
syntax_tree-haml (4.0.3)
|
|
85
|
+
haml (>= 5.2)
|
|
86
|
+
prettier_print (>= 1.2.1)
|
|
87
|
+
syntax_tree (>= 6.0.0)
|
|
88
|
+
syntax_tree-rbs (1.0.0)
|
|
89
|
+
prettier_print
|
|
90
|
+
rbs
|
|
91
|
+
syntax_tree (>= 2.0.1)
|
|
92
|
+
temple (0.10.0)
|
|
93
|
+
thor (1.2.2)
|
|
94
|
+
tilt (2.1.0)
|
|
95
|
+
unicode-display_width (2.4.2)
|
|
70
96
|
|
|
71
97
|
PLATFORMS
|
|
72
98
|
arm64-darwin-20
|
|
73
99
|
arm64-darwin-21
|
|
100
|
+
arm64-darwin-22
|
|
74
101
|
x86_64-linux
|
|
75
102
|
|
|
76
103
|
DEPENDENCIES
|
|
77
104
|
pg_online_schema_change!
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
rubocop
|
|
83
|
-
rubocop-
|
|
84
|
-
rubocop-
|
|
85
|
-
rubocop-
|
|
105
|
+
prettier_print
|
|
106
|
+
pry
|
|
107
|
+
rake
|
|
108
|
+
rspec
|
|
109
|
+
rubocop
|
|
110
|
+
rubocop-packaging
|
|
111
|
+
rubocop-performance
|
|
112
|
+
rubocop-rake
|
|
113
|
+
rubocop-rspec
|
|
114
|
+
syntax_tree
|
|
115
|
+
syntax_tree-haml
|
|
116
|
+
syntax_tree-rbs
|
|
86
117
|
|
|
87
118
|
BUNDLED WITH
|
|
88
119
|
2.3.24
|
data/README.md
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
[](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml)
|
|
6
6
|
[](https://badge.fury.io/rb/pg_online_schema_change)
|
|
7
7
|
|
|
8
|
-
pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads.
|
|
8
|
+
pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads.
|
|
9
9
|
|
|
10
10
|
`pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it creates a shadow table that looks structurally the same as the primary table, performs the schema change on the shadow table, copies contents from the primary table to the shadow table and swaps the table names in the end while preserving all changes to the primary table using triggers (via audit table).
|
|
11
11
|
|
|
@@ -21,11 +21,11 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
|
|
|
21
21
|
- [Prominent features](#prominent-features)
|
|
22
22
|
- [Load test](#load-test)
|
|
23
23
|
- [Examples](#examples)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
24
|
+
- [Renaming a column](#renaming-a-column)
|
|
25
|
+
- [Multiple ALTER statements](#multiple-alter-statements)
|
|
26
|
+
- [Kill other backends after 5s](#kill-other-backends-after-5s)
|
|
27
|
+
- [Backfill data](#backfill-data)
|
|
28
|
+
- [Running using Docker](#running-using-docker)
|
|
29
29
|
- [Caveats](#caveats)
|
|
30
30
|
- [How does it work](#how-does-it-work)
|
|
31
31
|
- [Development](#development)
|
|
@@ -33,12 +33,13 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
|
|
|
33
33
|
- [Contributing](#contributing)
|
|
34
34
|
- [License](#license)
|
|
35
35
|
- [Code of Conduct](#code-of-conduct)
|
|
36
|
+
|
|
36
37
|
## Installation
|
|
37
38
|
|
|
38
39
|
Add this line to your application's Gemfile:
|
|
39
40
|
|
|
40
41
|
```ruby
|
|
41
|
-
gem
|
|
42
|
+
gem "pg_online_schema_change"
|
|
42
43
|
```
|
|
43
44
|
|
|
44
45
|
And then execute:
|
|
@@ -56,7 +57,9 @@ Or via Docker:
|
|
|
56
57
|
docker pull shayonj/pg-osc:latest
|
|
57
58
|
|
|
58
59
|
https://hub.docker.com/r/shayonj/pg-osc
|
|
60
|
+
|
|
59
61
|
## Requirements
|
|
62
|
+
|
|
60
63
|
- PostgreSQL 9.6 and later
|
|
61
64
|
- Ruby 2.6 and later
|
|
62
65
|
- Database user should have permissions for `TRIGGER` and/or a `SUPERUSER`
|
|
@@ -97,8 +100,10 @@ Usage:
|
|
|
97
100
|
|
|
98
101
|
print the version
|
|
99
102
|
```
|
|
103
|
+
|
|
100
104
|
## Prominent features
|
|
101
|
-
|
|
105
|
+
|
|
106
|
+
- `pg-osc` supports when a column is being added, dropped or renamed with no data loss.
|
|
102
107
|
- `pg-osc` acquires minimal locks throughout the process (read more below on the caveats).
|
|
103
108
|
- Copies over indexes and Foreign keys.
|
|
104
109
|
- Optionally drop or retain old tables in the end.
|
|
@@ -113,6 +118,7 @@ print the version
|
|
|
113
118
|
## Examples
|
|
114
119
|
|
|
115
120
|
### Renaming a column
|
|
121
|
+
|
|
116
122
|
```
|
|
117
123
|
export PGPASSWORD=""
|
|
118
124
|
pg-online-schema-change perform \
|
|
@@ -123,6 +129,7 @@ pg-online-schema-change perform \
|
|
|
123
129
|
```
|
|
124
130
|
|
|
125
131
|
### Multiple ALTER statements
|
|
132
|
+
|
|
126
133
|
```
|
|
127
134
|
export PGPASSWORD=""
|
|
128
135
|
pg-online-schema-change perform \
|
|
@@ -134,6 +141,7 @@ pg-online-schema-change perform \
|
|
|
134
141
|
```
|
|
135
142
|
|
|
136
143
|
### Kill other backends after 5s
|
|
144
|
+
|
|
137
145
|
If the operation is being performed on a busy table, you can use `pg-osc`'s `kill-backend` functionality to kill other backends that may be competing with the `pg-osc` operation to acquire a lock for a brief while. The `ACCESS EXCLUSIVE` lock acquired by `pg-osc` is only held for a brief while and released after. You can tune how long `pg-osc` should wait before killing other backends (or if at all `pg-osc` should kill backends in the first place).
|
|
138
146
|
|
|
139
147
|
```
|
|
@@ -149,6 +157,7 @@ pg-online-schema-change perform \
|
|
|
149
157
|
```
|
|
150
158
|
|
|
151
159
|
### Replaying larger workloads
|
|
160
|
+
|
|
152
161
|
If you have a table with high write volume, the default replay iteration may not suffice. That is - you may see that `pg-osc` is replaying 1000 rows (`pull-batch-count`) in one go from the audit table. `pg-osc` also waits until the remaining row count (`delta-count`) in audit table is 20 before making the swap. You can tune these values to be higher for faster catch up on these kind of workloads.
|
|
153
162
|
|
|
154
163
|
```
|
|
@@ -164,10 +173,13 @@ pg-online-schema-change perform \
|
|
|
164
173
|
--kill-backends \
|
|
165
174
|
--drop
|
|
166
175
|
```
|
|
176
|
+
|
|
167
177
|
### Backfill data
|
|
178
|
+
|
|
168
179
|
When inserting data into the shadow table, instead of just copying all columns and rows from the primary table, you can pass in a custom sql file to perform the copy and do any additional work. For instance - backfilling certain columns. By providing the `copy-statement`, `pg-osc` will instead play the query to perform the copy operation.
|
|
169
180
|
|
|
170
181
|
**IMPORTANT NOTES:**
|
|
182
|
+
|
|
171
183
|
- It is possible to violate a constraint accidentally or not copy data, **so proceed with caution**.
|
|
172
184
|
- You must use OUTER JOINs when joining in the custom SQL, or you will **lose rows** which do not match the joined table.
|
|
173
185
|
- The `ALTER` statement can change the table's structure, **so proceed with caution**.
|
|
@@ -204,22 +216,25 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
|
|
|
204
216
|
--username "jamesbond" \
|
|
205
217
|
--drop
|
|
206
218
|
```
|
|
219
|
+
|
|
207
220
|
## Caveats
|
|
221
|
+
|
|
208
222
|
- Partitioned tables are not supported as of yet. Pull requests and ideas welcome.
|
|
209
223
|
- A primary key should exist on the table; without it, `pg-osc` will raise an exception
|
|
210
|
-
|
|
224
|
+
- This is because - currently there is no other way to uniquely identify rows during replay.
|
|
211
225
|
- `pg-osc` will acquire `ACCESS EXCLUSIVE` lock on the parent table twice during the operation.
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
226
|
+
- First, when setting up the triggers and the shadow table.
|
|
227
|
+
- Next, when performing the swap and updating FK references.
|
|
228
|
+
- Note: If `kill-backends` is passed, it will attempt to terminate any competing operations during both times.
|
|
215
229
|
- By design, `pg-osc` doesn't kill any other DDLs being performed. It's best to not run any DDLs against the parent table during the operation.
|
|
216
230
|
- Due to the nature of duplicating a table, there needs to be enough space on the disk to support the operation.
|
|
217
231
|
- Index, constraints and sequence names will be altered and lose their original naming.
|
|
218
|
-
|
|
219
|
-
- Triggers are not carried over.
|
|
232
|
+
- Can be fixed in future releases. Feel free to open a feature req.
|
|
233
|
+
- Triggers are not carried over.
|
|
220
234
|
- Can be fixed in future releases. Feel free to open a feature req.
|
|
221
235
|
- Foreign keys are dropped & re-added to referencing tables with a `NOT VALID`. A follow on `VALIDATE CONSTRAINT` is run.
|
|
222
|
-
|
|
236
|
+
- Ensures that integrity is maintained and re-introducing FKs doesn't acquire additional locks, hence the `NOT VALID`.
|
|
237
|
+
|
|
223
238
|
## How does it work
|
|
224
239
|
|
|
225
240
|
- **Primary table**: A table against which a potential schema change is to be run
|
|
@@ -228,10 +243,9 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
|
|
|
228
243
|
|
|
229
244
|
|
|
230
245
|
|
|
231
|
-
|
|
232
246
|
1. Create an audit table to record changes made to the parent table.
|
|
233
247
|
2. Acquire a brief `ACCESS EXCLUSIVE` lock to add a trigger on the parent table (for inserts, updates, deletes) to the audit table.
|
|
234
|
-
3. Create a new shadow table and run ALTER/migration on the shadow table.
|
|
248
|
+
3. Create a new shadow table and run ALTER/migration on the shadow table.
|
|
235
249
|
4. Copy all rows from the old table.
|
|
236
250
|
5. Build indexes on the new table.
|
|
237
251
|
6. Replay all changes accumulated in the audit table against the shadow table.
|
|
@@ -245,22 +259,24 @@ docker run --network host -it --rm shayonj/pg-osc:latest \
|
|
|
245
259
|
|
|
246
260
|
## Development
|
|
247
261
|
|
|
248
|
-
- Install ruby 3.
|
|
262
|
+
- Install ruby 3.1.3
|
|
263
|
+
|
|
249
264
|
```
|
|
250
265
|
\curl -sSL https://get.rvm.io | bash
|
|
251
266
|
|
|
252
|
-
rvm install 3.
|
|
267
|
+
rvm install 3.1.3
|
|
253
268
|
|
|
254
|
-
rvm use 3.
|
|
269
|
+
rvm use 3.1.3
|
|
255
270
|
```
|
|
256
271
|
|
|
257
272
|
- Spin up postgres via Docker Compose - `docker compose up`
|
|
258
|
-
- `bundle exec rspec` to run the tests.
|
|
273
|
+
- `bundle exec rspec` to run the tests.
|
|
259
274
|
- You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
260
275
|
|
|
261
|
-
To install this gem onto your local machine, run `bundle exec rake install`.
|
|
276
|
+
To install this gem onto your local machine, run `bundle exec rake install`.
|
|
277
|
+
|
|
278
|
+
### Local testing
|
|
262
279
|
|
|
263
|
-
### Local testing
|
|
264
280
|
```
|
|
265
281
|
docker compose up
|
|
266
282
|
|
|
@@ -280,7 +296,7 @@ bundle exec bin/pg-online-schema-change perform -a 'ALTER TABLE pgbench_accounts
|
|
|
280
296
|
|
|
281
297
|
## Contributing
|
|
282
298
|
|
|
283
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-osc.
|
|
299
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-osc.
|
|
284
300
|
|
|
285
301
|
## License
|
|
286
302
|
|
data/Rakefile
CHANGED
data/docker-compose.yml
CHANGED
data/docs/load-test.md
CHANGED
|
@@ -14,15 +14,17 @@ Total time taken to run schema change: **<3mins**
|
|
|
14
14
|
## Simulating load with pgbench
|
|
15
15
|
|
|
16
16
|
**Initialize**
|
|
17
|
+
|
|
17
18
|
```
|
|
18
|
-
pgbench -p $PORT --initialize -s 20 -F 20 --foreign-keys --host $HOST -U $USERNAME -d $DB
|
|
19
|
+
pgbench -p $PORT --initialize -s 20 -F 20 --foreign-keys --host $HOST -U $USERNAME -d $DB
|
|
19
20
|
```
|
|
20
21
|
|
|
21
22
|
This creates bunch of pgbench tables. The table being used with `pg-osc` is `pgbench_accounts` which has FKs and also references by other tables with FKS, containing 2M rows.
|
|
22
23
|
|
|
23
24
|
**Begin**
|
|
25
|
+
|
|
24
26
|
```
|
|
25
|
-
pgbench -p $PORT -j 72 -c 288 -T 500 -r --host $DB_HOST -U $USERNAME -d $DB
|
|
27
|
+
pgbench -p $PORT -j 72 -c 288 -T 500 -r --host $DB_HOST -U $USERNAME -d $DB
|
|
26
28
|
```
|
|
27
29
|
|
|
28
30
|
## Running pg-osc
|
|
@@ -36,7 +38,7 @@ ALTER TABLE pgbench_accounts ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;
|
|
|
36
38
|
**Execution**
|
|
37
39
|
|
|
38
40
|
```bash
|
|
39
|
-
bundle exec bin/pg-online-schema-change perform \
|
|
41
|
+
bundle exec bin/pg-online-schema-change perform \
|
|
40
42
|
-a 'ALTER TABLE pgbench_accounts ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
|
|
41
43
|
-d "pool" \
|
|
42
44
|
-p 25061
|
|
@@ -79,13 +81,13 @@ Added `purchased` column.
|
|
|
79
81
|
```
|
|
80
82
|
defaultdb=> \d+ pgbench_accounts;
|
|
81
83
|
Table "public.pgbench_accounts"
|
|
82
|
-
Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
|
|
84
|
+
Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
|
|
83
85
|
-----------+---------------+-----------+----------+---------+----------+--------------+-------------
|
|
84
|
-
aid | integer | | not null | | plain | |
|
|
85
|
-
bid | integer | | | | plain | |
|
|
86
|
-
abalance | integer | | | | plain | |
|
|
87
|
-
filler | character(84) | | | | extended | |
|
|
88
|
-
purchased | boolean | | | false | plain | |
|
|
86
|
+
aid | integer | | not null | | plain | |
|
|
87
|
+
bid | integer | | | | plain | |
|
|
88
|
+
abalance | integer | | | | plain | |
|
|
89
|
+
filler | character(84) | | | | extended | |
|
|
90
|
+
purchased | boolean | | | false | plain | |
|
|
89
91
|
Indexes:
|
|
90
92
|
"pgosc_st_pgbench_accounts_815029_pkey" PRIMARY KEY, btree (aid)
|
|
91
93
|
Foreign-key constraints:
|
|
@@ -132,7 +134,6 @@ NOTICE: table "pgosc_st_pgbench_accounts_714a8b" does not exist, skipping
|
|
|
132
134
|
|
|
133
135
|
</details>
|
|
134
136
|
|
|
135
|
-
|
|
136
137
|
## Conclusion
|
|
137
138
|
|
|
138
139
|
By tweaking `--pull-batch-count` to `2000` (replay 2k rows at once) and `--delta-count` to `200` (time to swap when remaining rows is <200), `pg-osc` was able to perform the schema change with no impact within very quick time. Depending on the database size and load on the table, you can further tune them to achieve desired impact. At some point this is going to plateau - I can imagine the replay factor not working quite well for say 100k commits/s workloads. So, YMMV.
|
|
@@ -7,34 +7,98 @@ module PgOnlineSchemaChange
|
|
|
7
7
|
DELTA_COUNT = 20
|
|
8
8
|
class CLI < Thor
|
|
9
9
|
desc "perform", "Safely apply schema changes with minimal locks"
|
|
10
|
-
method_option :alter_statement,
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
method_option :
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
method_option :
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
10
|
+
method_option :alter_statement,
|
|
11
|
+
aliases: "-a",
|
|
12
|
+
type: :string,
|
|
13
|
+
required: true,
|
|
14
|
+
desc: "The ALTER statement to perform the schema change"
|
|
15
|
+
method_option :schema,
|
|
16
|
+
aliases: "-s",
|
|
17
|
+
type: :string,
|
|
18
|
+
required: true,
|
|
19
|
+
default: "public",
|
|
20
|
+
desc: "The schema in which the table is"
|
|
21
|
+
method_option :dbname,
|
|
22
|
+
aliases: "-d",
|
|
23
|
+
type: :string,
|
|
24
|
+
required: true,
|
|
25
|
+
desc: "Name of the database"
|
|
26
|
+
method_option :host,
|
|
27
|
+
aliases: "-h",
|
|
28
|
+
type: :string,
|
|
29
|
+
required: true,
|
|
30
|
+
desc: "Server host where the Database is located"
|
|
31
|
+
method_option :username,
|
|
32
|
+
aliases: "-u",
|
|
33
|
+
type: :string,
|
|
34
|
+
required: true,
|
|
35
|
+
desc: "Username for the Database"
|
|
36
|
+
method_option :port,
|
|
37
|
+
aliases: "-p",
|
|
38
|
+
type: :numeric,
|
|
39
|
+
required: true,
|
|
40
|
+
default: 5432,
|
|
41
|
+
desc: "Port for the Database"
|
|
42
|
+
method_option :password,
|
|
43
|
+
aliases: "-w",
|
|
44
|
+
type: :string,
|
|
45
|
+
required: false,
|
|
46
|
+
default: "",
|
|
47
|
+
desc:
|
|
48
|
+
"DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
|
|
49
|
+
method_option :verbose,
|
|
50
|
+
aliases: "-v",
|
|
51
|
+
type: :boolean,
|
|
52
|
+
default: false,
|
|
53
|
+
desc: "Emit logs in debug mode"
|
|
54
|
+
method_option :drop,
|
|
55
|
+
aliases: "-f",
|
|
56
|
+
type: :boolean,
|
|
57
|
+
default: false,
|
|
58
|
+
desc: "Drop the original table in the end after the swap"
|
|
59
|
+
method_option :kill_backends,
|
|
60
|
+
aliases: "-k",
|
|
61
|
+
type: :boolean,
|
|
62
|
+
default: false,
|
|
63
|
+
desc:
|
|
64
|
+
"Kill other competing queries/backends when trying to acquire lock for the shadow table creation and swap. It will wait for --wait-time-for-lock duration before killing backends and try upto 3 times."
|
|
65
|
+
method_option :wait_time_for_lock,
|
|
66
|
+
aliases: "-w",
|
|
67
|
+
type: :numeric,
|
|
68
|
+
default: 10,
|
|
69
|
+
desc:
|
|
70
|
+
"Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock."
|
|
71
|
+
method_option :copy_statement,
|
|
72
|
+
aliases: "-c",
|
|
73
|
+
type: :string,
|
|
74
|
+
required: false,
|
|
75
|
+
default: "",
|
|
76
|
+
desc:
|
|
77
|
+
"Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README."
|
|
78
|
+
method_option :pull_batch_count,
|
|
79
|
+
aliases: "-b",
|
|
80
|
+
type: :numeric,
|
|
81
|
+
required: false,
|
|
82
|
+
default: PULL_BATCH_COUNT,
|
|
83
|
+
desc:
|
|
84
|
+
"Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count."
|
|
85
|
+
method_option :delta_count,
|
|
86
|
+
aliases: "-e",
|
|
87
|
+
type: :numeric,
|
|
88
|
+
required: false,
|
|
89
|
+
default: DELTA_COUNT,
|
|
90
|
+
desc:
|
|
91
|
+
"Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count."
|
|
32
92
|
|
|
33
93
|
def perform
|
|
34
94
|
client_options = Struct.new(*options.keys.map(&:to_sym)).new(*options.values)
|
|
35
95
|
PgOnlineSchemaChange.logger(verbose: client_options.verbose)
|
|
36
96
|
|
|
37
|
-
|
|
97
|
+
if client_options.password
|
|
98
|
+
PgOnlineSchemaChange.logger.warn(
|
|
99
|
+
"DEPRECATED: -w is deprecated. Please pass PGPASSWORD environment variable instead.",
|
|
100
|
+
)
|
|
101
|
+
end
|
|
38
102
|
|
|
39
103
|
client_options.password = ENV["PGPASSWORD"] || client_options.password
|
|
40
104
|
|
|
@@ -4,8 +4,22 @@ require "pg"
|
|
|
4
4
|
|
|
5
5
|
module PgOnlineSchemaChange
|
|
6
6
|
class Client
|
|
7
|
-
attr_accessor :alter_statement,
|
|
8
|
-
:
|
|
7
|
+
attr_accessor :alter_statement,
|
|
8
|
+
:schema,
|
|
9
|
+
:dbname,
|
|
10
|
+
:host,
|
|
11
|
+
:username,
|
|
12
|
+
:port,
|
|
13
|
+
:password,
|
|
14
|
+
:connection,
|
|
15
|
+
:table,
|
|
16
|
+
:table_name,
|
|
17
|
+
:drop,
|
|
18
|
+
:kill_backends,
|
|
19
|
+
:wait_time_for_lock,
|
|
20
|
+
:copy_statement,
|
|
21
|
+
:pull_batch_count,
|
|
22
|
+
:delta_count
|
|
9
23
|
|
|
10
24
|
def initialize(options)
|
|
11
25
|
@alter_statement = options.alter_statement
|
|
@@ -24,13 +38,8 @@ module PgOnlineSchemaChange
|
|
|
24
38
|
handle_copy_statement(options.copy_statement)
|
|
25
39
|
handle_validations
|
|
26
40
|
|
|
27
|
-
@connection =
|
|
28
|
-
dbname: @dbname,
|
|
29
|
-
host: @host,
|
|
30
|
-
user: @username,
|
|
31
|
-
password: @password,
|
|
32
|
-
port: @port,
|
|
33
|
-
)
|
|
41
|
+
@connection =
|
|
42
|
+
PG.connect(dbname: @dbname, host: @host, user: @username, password: @password, port: @port)
|
|
34
43
|
|
|
35
44
|
@table = Query.table(@alter_statement)
|
|
36
45
|
@table_name = Query.table_name(@alter_statement, @table)
|
|
@@ -39,11 +48,13 @@ module PgOnlineSchemaChange
|
|
|
39
48
|
end
|
|
40
49
|
|
|
41
50
|
def handle_validations
|
|
42
|
-
|
|
51
|
+
unless Query.alter_statement?(@alter_statement)
|
|
52
|
+
raise Error, "Not a valid ALTER statement: #{@alter_statement}"
|
|
53
|
+
end
|
|
43
54
|
|
|
44
55
|
return if Query.same_table?(@alter_statement)
|
|
45
56
|
|
|
46
|
-
raise Error
|
|
57
|
+
raise Error("All statements should belong to the same table: #{@alter_statement}")
|
|
47
58
|
end
|
|
48
59
|
|
|
49
60
|
def handle_copy_statement(statement)
|
|
@@ -52,7 +63,7 @@ module PgOnlineSchemaChange
|
|
|
52
63
|
file_path = File.expand_path(statement)
|
|
53
64
|
raise Error, "File not found: #{file_path}" unless File.file?(file_path)
|
|
54
65
|
|
|
55
|
-
@copy_statement = File.
|
|
66
|
+
@copy_statement = File.binread(file_path)
|
|
56
67
|
end
|
|
57
68
|
end
|
|
58
69
|
end
|