pg_online_schema_change 0.2.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +57 -24
- data/.rubocop_todo.yml +44 -0
- data/CHANGELOG.md +15 -2
- data/Dockerfile +5 -0
- data/Gemfile +0 -13
- data/Gemfile.lock +12 -13
- data/README.md +103 -41
- data/{diagrams → docs}/how-it-works.excalidraw +500 -508
- data/docs/how-it-works.png +0 -0
- data/docs/load-test-1.png +0 -0
- data/docs/load-test.md +138 -0
- data/lib/pg_online_schema_change/cli.rb +15 -3
- data/lib/pg_online_schema_change/client.rb +15 -7
- data/lib/pg_online_schema_change/functions.rb +4 -2
- data/lib/pg_online_schema_change/helper.rb +10 -1
- data/lib/pg_online_schema_change/orchestrate.rb +25 -14
- data/lib/pg_online_schema_change/query.rb +16 -14
- data/lib/pg_online_schema_change/replay.rb +20 -14
- data/lib/pg_online_schema_change/store.rb +7 -3
- data/lib/pg_online_schema_change/version.rb +1 -1
- data/lib/pg_online_schema_change.rb +8 -12
- data/scripts/release.sh +28 -0
- metadata +182 -11
- data/diagrams/how-it-works.png +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b14af6aa2b98ab8f1b2aab5ae0a8555e017f92531ce1214e50f2ac62ff354224
|
|
4
|
+
data.tar.gz: 66ba81f4e90f4dc612d863d043838fec8676e468924eeb5c39d21094d002e48a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a272a3749f8f053a528d859cc35b97b5fde7d6353a3c0c710500e35d33df6b4bc8526b18e855fd154b742eef8189b661e8e710a6b25221df6e47d55a47381a2e
|
|
7
|
+
data.tar.gz: c0dd7f41e204840b54b768df1f1e8486f79038bede199021ef13684c4b1369e3de43be6fbfd8f03b7ca652251c59a8a580ecefb235e00e54858d77f88a7334de
|
data/.rubocop.yml
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
|
2
|
+
|
|
1
3
|
require:
|
|
2
4
|
- rubocop-rspec
|
|
3
5
|
- rubocop-packaging
|
|
@@ -14,71 +16,70 @@ AllCops:
|
|
|
14
16
|
- "vendor/**/*"
|
|
15
17
|
|
|
16
18
|
Layout/HashAlignment:
|
|
17
|
-
EnforcedColonStyle:
|
|
18
|
-
|
|
19
|
-
- key
|
|
20
|
-
EnforcedHashRocketStyle:
|
|
21
|
-
- table
|
|
22
|
-
- key
|
|
19
|
+
EnforcedColonStyle: key
|
|
20
|
+
EnforcedHashRocketStyle: key
|
|
23
21
|
|
|
24
22
|
Layout/SpaceAroundEqualsInParameterDefault:
|
|
25
|
-
EnforcedStyle:
|
|
23
|
+
EnforcedStyle: space
|
|
26
24
|
|
|
27
25
|
Metrics/AbcSize:
|
|
28
|
-
|
|
26
|
+
Enabled: true
|
|
27
|
+
Max: 40
|
|
29
28
|
Exclude:
|
|
30
|
-
- "
|
|
29
|
+
- "spec/**/*"
|
|
31
30
|
|
|
32
31
|
Metrics/BlockLength:
|
|
32
|
+
Max: 100
|
|
33
33
|
Exclude:
|
|
34
34
|
- "*.gemspec"
|
|
35
35
|
- "Rakefile"
|
|
36
|
+
- "spec/**/*"
|
|
36
37
|
|
|
37
38
|
Metrics/ClassLength:
|
|
38
39
|
Exclude:
|
|
39
40
|
- "test/**/*"
|
|
40
41
|
|
|
41
42
|
Metrics/MethodLength:
|
|
42
|
-
Max:
|
|
43
|
+
Max: 30
|
|
43
44
|
Exclude:
|
|
44
45
|
- "test/**/*"
|
|
45
46
|
|
|
46
47
|
Metrics/ParameterLists:
|
|
47
|
-
Max:
|
|
48
|
+
Max: 5
|
|
48
49
|
|
|
49
50
|
Naming/MemoizedInstanceVariableName:
|
|
50
|
-
Enabled:
|
|
51
|
+
Enabled: true
|
|
51
52
|
|
|
52
53
|
Naming/VariableNumber:
|
|
53
|
-
Enabled:
|
|
54
|
-
|
|
55
|
-
Rake/Desc:
|
|
56
|
-
Enabled: false
|
|
54
|
+
Enabled: true
|
|
57
55
|
|
|
58
56
|
Style/BarePercentLiterals:
|
|
59
57
|
EnforcedStyle: percent_q
|
|
60
58
|
|
|
61
59
|
Style/ClassAndModuleChildren:
|
|
62
|
-
Enabled:
|
|
60
|
+
Enabled: true
|
|
63
61
|
|
|
64
62
|
Style/Documentation:
|
|
65
63
|
Enabled: false
|
|
66
64
|
|
|
67
65
|
Style/DoubleNegation:
|
|
68
|
-
Enabled:
|
|
66
|
+
Enabled: true
|
|
69
67
|
|
|
70
68
|
Style/EmptyMethod:
|
|
71
|
-
Enabled:
|
|
69
|
+
Enabled: true
|
|
72
70
|
|
|
73
71
|
Style/FrozenStringLiteralComment:
|
|
74
|
-
Enabled:
|
|
72
|
+
Enabled: true
|
|
75
73
|
|
|
76
74
|
Style/NumericPredicate:
|
|
77
|
-
Enabled:
|
|
75
|
+
Enabled: true
|
|
78
76
|
|
|
79
77
|
Style/StringLiterals:
|
|
80
78
|
EnforcedStyle: double_quotes
|
|
81
79
|
|
|
80
|
+
Style/StringLiteralsInInterpolation:
|
|
81
|
+
EnforcedStyle: double_quotes
|
|
82
|
+
|
|
82
83
|
Style/TrivialAccessors:
|
|
83
84
|
AllowPredicates: true
|
|
84
85
|
|
|
@@ -91,9 +92,41 @@ Style/TrailingCommaInArrayLiteral:
|
|
|
91
92
|
Style/TrailingCommaInHashLiteral:
|
|
92
93
|
EnforcedStyleForMultiline: comma
|
|
93
94
|
|
|
94
|
-
|
|
95
|
-
|
|
95
|
+
Layout/MultilineArrayBraceLayout:
|
|
96
|
+
Enabled: true
|
|
97
|
+
EnforcedStyle: symmetrical
|
|
96
98
|
|
|
97
|
-
|
|
99
|
+
Layout/MultilineHashBraceLayout:
|
|
98
100
|
Enabled: true
|
|
99
101
|
EnforcedStyle: symmetrical
|
|
102
|
+
|
|
103
|
+
Layout/MultilineAssignmentLayout:
|
|
104
|
+
Enabled: true
|
|
105
|
+
EnforcedStyle: same_line
|
|
106
|
+
|
|
107
|
+
Layout/FirstArrayElementIndentation:
|
|
108
|
+
Enabled: true
|
|
109
|
+
EnforcedStyle: consistent
|
|
110
|
+
|
|
111
|
+
Layout/FirstHashElementIndentation:
|
|
112
|
+
Enabled: true
|
|
113
|
+
EnforcedStyle: consistent
|
|
114
|
+
|
|
115
|
+
Layout/MultilineHashKeyLineBreaks:
|
|
116
|
+
Enabled: true
|
|
117
|
+
|
|
118
|
+
Layout/LineLength:
|
|
119
|
+
Enabled: true
|
|
120
|
+
Max: 250
|
|
121
|
+
|
|
122
|
+
Style/FormatStringToken:
|
|
123
|
+
Enabled: true
|
|
124
|
+
EnforcedStyle: template
|
|
125
|
+
|
|
126
|
+
RSpec/MessageSpies:
|
|
127
|
+
Enabled: true
|
|
128
|
+
EnforcedStyle: receive
|
|
129
|
+
|
|
130
|
+
RSpec/FilePath:
|
|
131
|
+
Enabled: true
|
|
132
|
+
SpecSuffixOnly: true
|
data/.rubocop_todo.yml
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# This configuration was generated by
|
|
2
|
+
# `rubocop --auto-gen-config`
|
|
3
|
+
# on 2022-02-21 22:46:44 UTC using RuboCop version 1.23.0.
|
|
4
|
+
# The point is for the user to remove these configuration records
|
|
5
|
+
# one by one as the offenses are removed from the code base.
|
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
|
8
|
+
|
|
9
|
+
# Offense count: 2
|
|
10
|
+
# Configuration parameters: CountComments, CountAsOne.
|
|
11
|
+
Metrics/ClassLength:
|
|
12
|
+
Max: 233
|
|
13
|
+
|
|
14
|
+
# Offense count: 2
|
|
15
|
+
# Configuration parameters: IgnoredMethods.
|
|
16
|
+
Metrics/CyclomaticComplexity:
|
|
17
|
+
Max: 15
|
|
18
|
+
|
|
19
|
+
# Offense count: 2
|
|
20
|
+
# Configuration parameters: IgnoredMethods.
|
|
21
|
+
Metrics/PerceivedComplexity:
|
|
22
|
+
Max: 13
|
|
23
|
+
|
|
24
|
+
# Offense count: 1
|
|
25
|
+
Packaging/GemspecGit:
|
|
26
|
+
Exclude:
|
|
27
|
+
- 'pg_online_schema_change.gemspec'
|
|
28
|
+
|
|
29
|
+
# Offense count: 62
|
|
30
|
+
# Configuration parameters: CountAsOne.
|
|
31
|
+
RSpec/ExampleLength:
|
|
32
|
+
Max: 55
|
|
33
|
+
|
|
34
|
+
# Offense count: 38
|
|
35
|
+
RSpec/MultipleExpectations:
|
|
36
|
+
Max: 14
|
|
37
|
+
|
|
38
|
+
# Offense count: 6
|
|
39
|
+
# Configuration parameters: AllowedMethods.
|
|
40
|
+
# AllowedMethods: respond_to_missing?
|
|
41
|
+
Style/OptionalBooleanParameter:
|
|
42
|
+
Exclude:
|
|
43
|
+
- 'lib/pg_online_schema_change/query.rb'
|
|
44
|
+
- 'lib/pg_online_schema_change/replay.rb'
|
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,19 @@
|
|
|
1
|
+
## [0.4.0] - 2022-02-22
|
|
2
|
+
* Lint sourcecode, setup Rubocop proper and Lint in CI by @shayonj in https://github.com/shayonj/pg-osc/pull/46
|
|
3
|
+
* Uniquely identify operation_type column by @shayonj in https://github.com/shayonj/pg-osc/pull/50
|
|
4
|
+
* Introduce primary key on audit table for ordered reads by @shayonj in https://github.com/shayonj/pg-osc/pull/49
|
|
5
|
+
- This addresses an edge case with replay.
|
|
6
|
+
* Uniquely identify trigger_time column by @shayonj in https://github.com/shayonj/pg-osc/pull/51
|
|
7
|
+
* Abstract assertions into a helper function by @shayonj in https://github.com/shayonj/pg-osc/pull/52
|
|
8
|
+
|
|
9
|
+
## [0.3.0] - 2022-02-21
|
|
10
|
+
|
|
11
|
+
- Explicitly call dependencies and bump dependencies by @shayonj https://github.com/shayonj/pg-osc/pull/44
|
|
12
|
+
- Introduce Dockerfile and release process https://github.com/shayonj/pg-osc/pull/45
|
|
13
|
+
|
|
1
14
|
## [0.2.0] - 2022-02-17
|
|
2
15
|
|
|
3
|
-
- Use ISOLATION LEVEL SERIALIZABLE ([#42](https://github.com/shayonj/pg-
|
|
16
|
+
- Use ISOLATION LEVEL SERIALIZABLE ([#42](https://github.com/shayonj/pg-osc/pull/42)) (props to @jfrost)
|
|
4
17
|
|
|
5
18
|
## [0.1.0] - 2022-02-16
|
|
6
19
|
|
|
@@ -10,4 +23,4 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
|
|
|
10
23
|
|
|
11
24
|
`pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it copies the contents from a primary table to a shadow table, performs the schema change on the shadow table and swaps the table names in the end while preserving all changes to the primary table using triggers (via audit table).
|
|
12
25
|
|
|
13
|
-
Checkout [Readme](https://github.com/shayonj/pg-
|
|
26
|
+
Checkout [Readme](https://github.com/shayonj/pg-osc#readme) for more details.
|
data/Dockerfile
ADDED
data/Gemfile
CHANGED
|
@@ -3,16 +3,3 @@
|
|
|
3
3
|
source "https://rubygems.org"
|
|
4
4
|
|
|
5
5
|
gemspec
|
|
6
|
-
|
|
7
|
-
gem "ougai", "~> 2.0.0"
|
|
8
|
-
gem "pg", "~> 1.0"
|
|
9
|
-
gem "pg_query", "~> 2.1.2"
|
|
10
|
-
gem "pry"
|
|
11
|
-
gem "rake", "~> 13.0"
|
|
12
|
-
gem "rspec", "~> 3.0"
|
|
13
|
-
gem "rubocop", "~> 1.23.0"
|
|
14
|
-
gem "rubocop-packaging", "~> 0.5.1"
|
|
15
|
-
gem "rubocop-performance", "~> 1.12.0"
|
|
16
|
-
gem "rubocop-rake", "~> 0.6.0"
|
|
17
|
-
gem "rubocop-rspec", "~> 2.7.0"
|
|
18
|
-
gem "thor", "~> 1.1.0"
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
pg_online_schema_change (0.
|
|
4
|
+
pg_online_schema_change (0.4.0)
|
|
5
|
+
ougai (~> 2.0.0)
|
|
6
|
+
pg (~> 1.3.2)
|
|
7
|
+
pg_query (~> 2.1.3)
|
|
8
|
+
thor (~> 1.2.1)
|
|
5
9
|
|
|
6
10
|
GEM
|
|
7
11
|
remote: https://rubygems.org/
|
|
@@ -9,18 +13,17 @@ GEM
|
|
|
9
13
|
ast (2.4.2)
|
|
10
14
|
coderay (1.1.3)
|
|
11
15
|
diff-lcs (1.5.0)
|
|
12
|
-
google-protobuf (3.19.
|
|
13
|
-
google-protobuf (3.19.2-x86_64-linux)
|
|
16
|
+
google-protobuf (3.19.4)
|
|
14
17
|
method_source (1.0.0)
|
|
15
|
-
oj (3.13.
|
|
18
|
+
oj (3.13.11)
|
|
16
19
|
ougai (2.0.0)
|
|
17
20
|
oj (~> 3.10)
|
|
18
21
|
parallel (1.21.0)
|
|
19
22
|
parser (3.0.3.2)
|
|
20
23
|
ast (~> 2.4.1)
|
|
21
|
-
pg (1.2
|
|
22
|
-
pg_query (2.1.
|
|
23
|
-
google-protobuf (>= 3.
|
|
24
|
+
pg (1.3.2)
|
|
25
|
+
pg_query (2.1.3)
|
|
26
|
+
google-protobuf (>= 3.19.2)
|
|
24
27
|
pry (0.14.1)
|
|
25
28
|
coderay (~> 1.1)
|
|
26
29
|
method_source (~> 1.0)
|
|
@@ -62,7 +65,7 @@ GEM
|
|
|
62
65
|
rubocop-rspec (2.7.0)
|
|
63
66
|
rubocop (~> 1.19)
|
|
64
67
|
ruby-progressbar (1.11.0)
|
|
65
|
-
thor (1.1
|
|
68
|
+
thor (1.2.1)
|
|
66
69
|
unicode-display_width (2.1.0)
|
|
67
70
|
|
|
68
71
|
PLATFORMS
|
|
@@ -70,11 +73,8 @@ PLATFORMS
|
|
|
70
73
|
x86_64-linux
|
|
71
74
|
|
|
72
75
|
DEPENDENCIES
|
|
73
|
-
ougai (~> 2.0.0)
|
|
74
|
-
pg (~> 1.0)
|
|
75
76
|
pg_online_schema_change!
|
|
76
|
-
|
|
77
|
-
pry
|
|
77
|
+
pry (~> 0.14.1)
|
|
78
78
|
rake (~> 13.0)
|
|
79
79
|
rspec (~> 3.0)
|
|
80
80
|
rubocop (~> 1.23.0)
|
|
@@ -82,7 +82,6 @@ DEPENDENCIES
|
|
|
82
82
|
rubocop-performance (~> 1.12.0)
|
|
83
83
|
rubocop-rake (~> 0.6.0)
|
|
84
84
|
rubocop-rspec (~> 2.7.0)
|
|
85
|
-
thor (~> 1.1.0)
|
|
86
85
|
|
|
87
86
|
BUNDLED WITH
|
|
88
87
|
2.3.3
|
data/README.md
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
# pg-
|
|
2
|
-
|
|
1
|
+
# pg-osc
|
|
2
|
+
|
|
3
|
+
[](https://circleci.com/gh/shayonj/pg-osc/tree/main)
|
|
3
4
|
[](https://badge.fury.io/rb/pg_online_schema_change)
|
|
4
5
|
|
|
5
6
|
pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads.
|
|
@@ -10,6 +11,26 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
|
|
|
10
11
|
|
|
11
12
|
⚠️ Proceed with caution when using this on production like workloads. Best to try on similar setup or staging like environment first. Read on below for some examples and caveats.
|
|
12
13
|
|
|
14
|
+
## Table of Contents
|
|
15
|
+
|
|
16
|
+
- [Installation](#installation)
|
|
17
|
+
- [Requirements](#requirements)
|
|
18
|
+
- [Usage](#usage)
|
|
19
|
+
- [Prominent features](#prominent-features)
|
|
20
|
+
- [Load test](#load-test)
|
|
21
|
+
- [Examples](#examples)
|
|
22
|
+
* [Renaming a column](#renaming-a-column)
|
|
23
|
+
* [Multiple ALTER statements](#multiple-alter-statements)
|
|
24
|
+
* [Kill other backends after 5s](#kill-other-backends-after-5s)
|
|
25
|
+
* [Backfill data](#backfill-data)
|
|
26
|
+
* [Running using Docker](#running-using-docker)
|
|
27
|
+
- [Caveats](#caveats)
|
|
28
|
+
- [How does it work](#how-does-it-work)
|
|
29
|
+
- [Development](#development)
|
|
30
|
+
- [Releasing](#releasing)
|
|
31
|
+
- [Contributing](#contributing)
|
|
32
|
+
- [License](#license)
|
|
33
|
+
- [Code of Conduct](#code-of-conduct)
|
|
13
34
|
## Installation
|
|
14
35
|
|
|
15
36
|
Add this line to your application's Gemfile:
|
|
@@ -26,6 +47,13 @@ Or install it yourself as:
|
|
|
26
47
|
|
|
27
48
|
$ gem install pg_online_schema_change
|
|
28
49
|
|
|
50
|
+
This will include all dependencies accordingly as well. Make sure the following requirements are satisfied.
|
|
51
|
+
|
|
52
|
+
Or via Docker:
|
|
53
|
+
|
|
54
|
+
docker pull shayonj/pg-osc:latest
|
|
55
|
+
|
|
56
|
+
https://hub.docker.com/r/shayonj/pg-osc
|
|
29
57
|
## Requirements
|
|
30
58
|
- PostgreSQL 9.6 and later
|
|
31
59
|
- Ruby 2.6 and later
|
|
@@ -48,13 +76,17 @@ Options:
|
|
|
48
76
|
-u, --username=USERNAME # Username for the Database
|
|
49
77
|
-p, --port=N # Port for the Database
|
|
50
78
|
# Default: 5432
|
|
51
|
-
-w, --password=PASSWORD # Password for the Database
|
|
79
|
+
-w, --password=PASSWORD # DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead.
|
|
52
80
|
-v, [--verbose], [--no-verbose] # Emit logs in debug mode
|
|
53
81
|
-f, [--drop], [--no-drop] # Drop the original table in the end after the swap
|
|
54
82
|
-k, [--kill-backends], [--no-kill-backends] # Kill other competing queries/backends when trying to acquire lock for the shadow table creation and swap. It will wait for --wait-time-for-lock duration before killing backends and try upto 3 times.
|
|
55
83
|
-w, [--wait-time-for-lock=N] # Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock.
|
|
56
84
|
# Default: 10
|
|
57
|
-
-c, [--copy-statement=COPY_STATEMENT] # Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when
|
|
85
|
+
-c, [--copy-statement=COPY_STATEMENT] # Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README.
|
|
86
|
+
-b, [--pull-batch-count=N] # Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count.
|
|
87
|
+
# Default: 1000
|
|
88
|
+
-e, [--delta-count=N] # Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count.
|
|
89
|
+
# Default: 20
|
|
58
90
|
```
|
|
59
91
|
|
|
60
92
|
```
|
|
@@ -63,57 +95,38 @@ Usage:
|
|
|
63
95
|
|
|
64
96
|
print the version
|
|
65
97
|
```
|
|
66
|
-
## How does it work
|
|
67
|
-
|
|
68
|
-
- **Primary table**: A table against which a potential schema change is to be run
|
|
69
|
-
- **Shadow table**: A copy of an existing primary table
|
|
70
|
-
- **Audit table**: A table to store any updates/inserts/delete on a primary table
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
1. Create an audit table to record changes made to the parent table.
|
|
76
|
-
2. Acquire a brief `ACCESS EXCLUSIVE` lock to add a trigger on the parent table (for inserts, updates, deletes) to the audit table.
|
|
77
|
-
3. Create a new shadow table and run ALTER/migration on the shadow table.
|
|
78
|
-
4. Copy all rows from the old table.
|
|
79
|
-
5. Build indexes on the new table.
|
|
80
|
-
6. Replay all changes accumulated in the audit table against the shadow table.
|
|
81
|
-
- Delete rows in the audit table as they are replayed.
|
|
82
|
-
7. Once the delta (remaining rows) is ~20 rows, acquire an `ACCESS EXCLUSIVE` lock against the parent table within a transaction and:
|
|
83
|
-
- swap table names (shadow table <> parent table).
|
|
84
|
-
- update references in other tables (FKs) by dropping and re-creating the FKs with a `NOT VALID`.
|
|
85
|
-
8. Runs `ANALYZE` on the new table.
|
|
86
|
-
9. Validates all FKs that were added with `NOT VALID`.
|
|
87
|
-
10. Drop parent (now old) table (OPTIONAL).
|
|
88
|
-
|
|
89
98
|
## Prominent features
|
|
90
99
|
- `pg-osc` supports when a column is being added, dropped or renamed with no data loss.
|
|
91
100
|
- `pg-osc` acquires minimal locks throughout the process (read more below on the caveats).
|
|
92
101
|
- Copies over indexes and Foreign keys.
|
|
93
102
|
- Optionally drop or retain old tables in the end.
|
|
94
103
|
- Backfill old/new columns as data is copied from primary table to shadow table, and then perform the swap. [Example](#backfill-data)
|
|
95
|
-
- **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-
|
|
104
|
+
- **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-osc/issues/14)
|
|
105
|
+
|
|
106
|
+
## Load test
|
|
107
|
+
|
|
108
|
+
[More about the preliminary load test figures here](docs/load-test.md)
|
|
96
109
|
|
|
97
110
|
## Examples
|
|
98
111
|
|
|
99
112
|
### Renaming a column
|
|
100
113
|
```
|
|
114
|
+
export PGPASSWORD=""
|
|
101
115
|
pg-online-schema-change perform \
|
|
102
116
|
--alter-statement 'ALTER TABLE books RENAME COLUMN email TO new_email' \
|
|
103
117
|
--dbname "postgres" \
|
|
104
118
|
--host "localhost" \
|
|
105
119
|
--username "jamesbond" \
|
|
106
|
-
--password "" \
|
|
107
120
|
```
|
|
108
121
|
|
|
109
122
|
### Multiple ALTER statements
|
|
110
123
|
```
|
|
124
|
+
export PGPASSWORD=""
|
|
111
125
|
pg-online-schema-change perform \
|
|
112
126
|
--alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE; ALTER TABLE books RENAME COLUMN email TO new_email;' \
|
|
113
127
|
--dbname "postgres" \
|
|
114
128
|
--host "localhost" \
|
|
115
129
|
--username "jamesbond" \
|
|
116
|
-
--password "" \
|
|
117
130
|
--drop
|
|
118
131
|
```
|
|
119
132
|
|
|
@@ -121,13 +134,30 @@ pg-online-schema-change perform \
|
|
|
121
134
|
If the operation is being performed on a busy table, you can use `pg-osc`'s `kill-backend` functionality to kill other backends that may be competing with the `pg-osc` operation to acquire a lock for a brief while. The `ACCESS EXCLUSIVE` lock acquired by `pg-osc` is only held for a brief while and released after. You can tune how long `pg-osc` should wait before killing other backends (or if at all `pg-osc` should kill backends in the first place).
|
|
122
135
|
|
|
123
136
|
```
|
|
137
|
+
export PGPASSWORD=""
|
|
124
138
|
pg-online-schema-change perform \
|
|
125
139
|
--alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
|
|
126
140
|
--dbname "postgres" \
|
|
127
141
|
--host "localhost" \
|
|
128
142
|
--username "jamesbond" \
|
|
129
|
-
--
|
|
130
|
-
--
|
|
143
|
+
--wait-time-for-lock 5 \
|
|
144
|
+
--kill-backends \
|
|
145
|
+
--drop
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
### Replaying larger workloads
|
|
149
|
+
If you have a table with high write volume, the default replay iteration may not suffice. That is - you may see that `pg-osc` is replaying 1000 rows (`pull-batch-count`) in one go from the audit table. `pg-osc` also waits until the remaining row count (`delta-count`) in audit table is 20 before making the swap. You can tune these values to be higher for faster catch up on these kind of workloads.
|
|
150
|
+
|
|
151
|
+
```
|
|
152
|
+
export PGPASSWORD=""
|
|
153
|
+
pg-online-schema-change perform \
|
|
154
|
+
--alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE;' \
|
|
155
|
+
--dbname "postgres" \
|
|
156
|
+
--host "localhost" \
|
|
157
|
+
--username "jamesbond" \
|
|
158
|
+
--pull-batch-count 2000
|
|
159
|
+
--delta-count 500
|
|
160
|
+
--wait-time-for-lock 5 \
|
|
131
161
|
--kill-backends \
|
|
132
162
|
--drop
|
|
133
163
|
```
|
|
@@ -156,11 +186,22 @@ pg-online-schema-change perform \
|
|
|
156
186
|
--dbname "postgres" \
|
|
157
187
|
--host "localhost" \
|
|
158
188
|
--username "jamesbond" \
|
|
159
|
-
--password "" \
|
|
160
189
|
--copy-statement "/src/query.sql" \
|
|
161
190
|
--drop
|
|
162
191
|
```
|
|
163
|
-
|
|
192
|
+
|
|
193
|
+
### Running using Docker
|
|
194
|
+
|
|
195
|
+
```
|
|
196
|
+
docker run --network host -it --rm shayonj/pg-osc:latest \
|
|
197
|
+
pg-online-schema-change perform \
|
|
198
|
+
--alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE; ALTER TABLE books RENAME COLUMN email TO new_email;' \
|
|
199
|
+
--dbname "postgres" \
|
|
200
|
+
--host "localhost" \
|
|
201
|
+
--username "jamesbond" \
|
|
202
|
+
--drop
|
|
203
|
+
```
|
|
204
|
+
## Caveats
|
|
164
205
|
- A primary key should exist on the table; without it, `pg-osc` will raise an exception
|
|
165
206
|
- This is because - currently there is no other way to uniquely identify rows during replay.
|
|
166
207
|
- `pg-osc` will acquire `ACCESS EXCLUSIVE` lock on the parent table twice during the operation.
|
|
@@ -175,6 +216,29 @@ pg-online-schema-change perform \
|
|
|
175
216
|
- Can be fixed in future releases. Feel free to open a feature req.
|
|
176
217
|
- Foreign keys are dropped & re-added to referencing tables with a `NOT VALID`. A follow on `VALIDATE CONSTRAINT` is run.
|
|
177
218
|
- Ensures that integrity is maintained and re-introducing FKs doesn't acquire additional locks, hence the `NOT VALID`.
|
|
219
|
+
## How does it work
|
|
220
|
+
|
|
221
|
+
- **Primary table**: A table against which a potential schema change is to be run
|
|
222
|
+
- **Shadow table**: A copy of an existing primary table
|
|
223
|
+
- **Audit table**: A table to store any updates/inserts/delete on a primary table
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
1. Create an audit table to record changes made to the parent table.
|
|
229
|
+
2. Acquire a brief `ACCESS EXCLUSIVE` lock to add a trigger on the parent table (for inserts, updates, deletes) to the audit table.
|
|
230
|
+
3. Create a new shadow table and run ALTER/migration on the shadow table.
|
|
231
|
+
4. Copy all rows from the old table.
|
|
232
|
+
5. Build indexes on the new table.
|
|
233
|
+
6. Replay all changes accumulated in the audit table against the shadow table.
|
|
234
|
+
- Delete rows in the audit table as they are replayed.
|
|
235
|
+
7. Once the delta (remaining rows) is ~20 rows, acquire an `ACCESS EXCLUSIVE` lock against the parent table within a transaction and:
|
|
236
|
+
- swap table names (shadow table <> parent table).
|
|
237
|
+
- update references in other tables (FKs) by dropping and re-creating the FKs with a `NOT VALID`.
|
|
238
|
+
8. Runs `ANALYZE` on the new table.
|
|
239
|
+
9. Validates all FKs that were added with `NOT VALID`.
|
|
240
|
+
10. Drop parent (now old) table (OPTIONAL).
|
|
241
|
+
|
|
178
242
|
## Development
|
|
179
243
|
|
|
180
244
|
- Install ruby 3.0
|
|
@@ -195,16 +259,14 @@ To install this gem onto your local machine, run `bundle exec rake install`.
|
|
|
195
259
|
## Releasing
|
|
196
260
|
|
|
197
261
|
- Bump version in `version.rb`
|
|
198
|
-
-
|
|
199
|
-
-
|
|
200
|
-
- `gem build pg_online_schema_change.gemspec`
|
|
201
|
-
- `gem push pg_online_schema_change-0.1.0.gem`
|
|
262
|
+
- Commit
|
|
263
|
+
- `./scripts/release.sh 0.2.0`
|
|
202
264
|
- Update `CHANGELOG.md`
|
|
203
|
-
- Create a new release - https://github.com/shayonj/pg-
|
|
265
|
+
- Create a new release - https://github.com/shayonj/pg-osc/releases/new
|
|
204
266
|
|
|
205
267
|
## Contributing
|
|
206
268
|
|
|
207
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-
|
|
269
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-osc.
|
|
208
270
|
|
|
209
271
|
## License
|
|
210
272
|
|
|
@@ -212,4 +274,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
|
|
|
212
274
|
|
|
213
275
|
## Code of Conduct
|
|
214
276
|
|
|
215
|
-
Everyone interacting in the PgOnlineSchemaChange project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/shayonj/pg-
|
|
277
|
+
Everyone interacting in the PgOnlineSchemaChange project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/shayonj/pg-osc/blob/main/CODE_OF_CONDUCT.md).
|