pg_online_schema_change 0.1.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +57 -24
- data/.rubocop_todo.yml +44 -0
- data/CHANGELOG.md +10 -1
- data/Dockerfile +5 -0
- data/Gemfile +0 -13
- data/Gemfile.lock +13 -13
- data/README.md +53 -12
- data/diagrams/how-it-works.excalidraw +500 -508
- data/diagrams/how-it-works.png +0 -0
- data/lib/pg_online_schema_change/cli.rb +3 -1
- data/lib/pg_online_schema_change/client.rb +12 -6
- data/lib/pg_online_schema_change/functions.rb +4 -2
- data/lib/pg_online_schema_change/helper.rb +10 -1
- data/lib/pg_online_schema_change/orchestrate.rb +55 -27
- data/lib/pg_online_schema_change/query.rb +22 -20
- data/lib/pg_online_schema_change/replay.rb +19 -10
- data/lib/pg_online_schema_change/store.rb +7 -3
- data/lib/pg_online_schema_change/version.rb +1 -1
- data/lib/pg_online_schema_change.rb +7 -11
- data/scripts/release.sh +28 -0
- metadata +178 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8ef1069f7d159544838ec27cf518f98eb31609a0a2e1878ebd40246b10a6b534
|
4
|
+
data.tar.gz: 8e83cbac78164fb10870537bf4867020eae7cf06f4cd91930a2c8846fbc76c7c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2657c94b3b07730aa3bb282f34cd1ee95f0549d3649b39a61be3ce1b6d2ba5832a1914b6a9788778c912464cbd9773b5e967ed4e6c896127d60e62a0daa76c99
|
7
|
+
data.tar.gz: 4788523d691c25045b7f3c539f9743d880fabcbb01887ae34ec55a6c8a829804a9f4d43f619e79e0c44b1c9fff6ea88361fb5ac1111bc1f491aa17e6e43336d0
|
data/.rubocop.yml
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
2
|
+
|
1
3
|
require:
|
2
4
|
- rubocop-rspec
|
3
5
|
- rubocop-packaging
|
@@ -14,71 +16,70 @@ AllCops:
|
|
14
16
|
- "vendor/**/*"
|
15
17
|
|
16
18
|
Layout/HashAlignment:
|
17
|
-
EnforcedColonStyle:
|
18
|
-
|
19
|
-
- key
|
20
|
-
EnforcedHashRocketStyle:
|
21
|
-
- table
|
22
|
-
- key
|
19
|
+
EnforcedColonStyle: key
|
20
|
+
EnforcedHashRocketStyle: key
|
23
21
|
|
24
22
|
Layout/SpaceAroundEqualsInParameterDefault:
|
25
|
-
EnforcedStyle:
|
23
|
+
EnforcedStyle: space
|
26
24
|
|
27
25
|
Metrics/AbcSize:
|
28
|
-
|
26
|
+
Enabled: true
|
27
|
+
Max: 40
|
29
28
|
Exclude:
|
30
|
-
- "
|
29
|
+
- "spec/**/*"
|
31
30
|
|
32
31
|
Metrics/BlockLength:
|
32
|
+
Max: 100
|
33
33
|
Exclude:
|
34
34
|
- "*.gemspec"
|
35
35
|
- "Rakefile"
|
36
|
+
- "spec/**/*"
|
36
37
|
|
37
38
|
Metrics/ClassLength:
|
38
39
|
Exclude:
|
39
40
|
- "test/**/*"
|
40
41
|
|
41
42
|
Metrics/MethodLength:
|
42
|
-
Max:
|
43
|
+
Max: 30
|
43
44
|
Exclude:
|
44
45
|
- "test/**/*"
|
45
46
|
|
46
47
|
Metrics/ParameterLists:
|
47
|
-
Max:
|
48
|
+
Max: 5
|
48
49
|
|
49
50
|
Naming/MemoizedInstanceVariableName:
|
50
|
-
Enabled:
|
51
|
+
Enabled: true
|
51
52
|
|
52
53
|
Naming/VariableNumber:
|
53
|
-
Enabled:
|
54
|
-
|
55
|
-
Rake/Desc:
|
56
|
-
Enabled: false
|
54
|
+
Enabled: true
|
57
55
|
|
58
56
|
Style/BarePercentLiterals:
|
59
57
|
EnforcedStyle: percent_q
|
60
58
|
|
61
59
|
Style/ClassAndModuleChildren:
|
62
|
-
Enabled:
|
60
|
+
Enabled: true
|
63
61
|
|
64
62
|
Style/Documentation:
|
65
63
|
Enabled: false
|
66
64
|
|
67
65
|
Style/DoubleNegation:
|
68
|
-
Enabled:
|
66
|
+
Enabled: true
|
69
67
|
|
70
68
|
Style/EmptyMethod:
|
71
|
-
Enabled:
|
69
|
+
Enabled: true
|
72
70
|
|
73
71
|
Style/FrozenStringLiteralComment:
|
74
|
-
Enabled:
|
72
|
+
Enabled: true
|
75
73
|
|
76
74
|
Style/NumericPredicate:
|
77
|
-
Enabled:
|
75
|
+
Enabled: true
|
78
76
|
|
79
77
|
Style/StringLiterals:
|
80
78
|
EnforcedStyle: double_quotes
|
81
79
|
|
80
|
+
Style/StringLiteralsInInterpolation:
|
81
|
+
EnforcedStyle: double_quotes
|
82
|
+
|
82
83
|
Style/TrivialAccessors:
|
83
84
|
AllowPredicates: true
|
84
85
|
|
@@ -91,9 +92,41 @@ Style/TrailingCommaInArrayLiteral:
|
|
91
92
|
Style/TrailingCommaInHashLiteral:
|
92
93
|
EnforcedStyleForMultiline: comma
|
93
94
|
|
94
|
-
|
95
|
-
|
95
|
+
Layout/MultilineArrayBraceLayout:
|
96
|
+
Enabled: true
|
97
|
+
EnforcedStyle: symmetrical
|
96
98
|
|
97
|
-
|
99
|
+
Layout/MultilineHashBraceLayout:
|
98
100
|
Enabled: true
|
99
101
|
EnforcedStyle: symmetrical
|
102
|
+
|
103
|
+
Layout/MultilineAssignmentLayout:
|
104
|
+
Enabled: true
|
105
|
+
EnforcedStyle: same_line
|
106
|
+
|
107
|
+
Layout/FirstArrayElementIndentation:
|
108
|
+
Enabled: true
|
109
|
+
EnforcedStyle: consistent
|
110
|
+
|
111
|
+
Layout/FirstHashElementIndentation:
|
112
|
+
Enabled: true
|
113
|
+
EnforcedStyle: consistent
|
114
|
+
|
115
|
+
Layout/MultilineHashKeyLineBreaks:
|
116
|
+
Enabled: true
|
117
|
+
|
118
|
+
Layout/LineLength:
|
119
|
+
Enabled: true
|
120
|
+
Max: 250
|
121
|
+
|
122
|
+
Style/FormatStringToken:
|
123
|
+
Enabled: true
|
124
|
+
EnforcedStyle: template
|
125
|
+
|
126
|
+
RSpec/MessageSpies:
|
127
|
+
Enabled: true
|
128
|
+
EnforcedStyle: receive
|
129
|
+
|
130
|
+
RSpec/FilePath:
|
131
|
+
Enabled: true
|
132
|
+
SpecSuffixOnly: true
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config`
|
3
|
+
# on 2022-02-21 22:46:44 UTC using RuboCop version 1.23.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 2
|
10
|
+
# Configuration parameters: CountComments, CountAsOne.
|
11
|
+
Metrics/ClassLength:
|
12
|
+
Max: 233
|
13
|
+
|
14
|
+
# Offense count: 2
|
15
|
+
# Configuration parameters: IgnoredMethods.
|
16
|
+
Metrics/CyclomaticComplexity:
|
17
|
+
Max: 15
|
18
|
+
|
19
|
+
# Offense count: 2
|
20
|
+
# Configuration parameters: IgnoredMethods.
|
21
|
+
Metrics/PerceivedComplexity:
|
22
|
+
Max: 13
|
23
|
+
|
24
|
+
# Offense count: 1
|
25
|
+
Packaging/GemspecGit:
|
26
|
+
Exclude:
|
27
|
+
- 'pg_online_schema_change.gemspec'
|
28
|
+
|
29
|
+
# Offense count: 62
|
30
|
+
# Configuration parameters: CountAsOne.
|
31
|
+
RSpec/ExampleLength:
|
32
|
+
Max: 55
|
33
|
+
|
34
|
+
# Offense count: 38
|
35
|
+
RSpec/MultipleExpectations:
|
36
|
+
Max: 14
|
37
|
+
|
38
|
+
# Offense count: 6
|
39
|
+
# Configuration parameters: AllowedMethods.
|
40
|
+
# AllowedMethods: respond_to_missing?
|
41
|
+
Style/OptionalBooleanParameter:
|
42
|
+
Exclude:
|
43
|
+
- 'lib/pg_online_schema_change/query.rb'
|
44
|
+
- 'lib/pg_online_schema_change/replay.rb'
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
## [0.3.0] - 2022-02-21
|
2
|
+
|
3
|
+
- Explicitly call dependencies and bump dependencies by @shayonj https://github.com/shayonj/pg-osc/pull/44
|
4
|
+
- Introduce Dockerfile and release process https://github.com/shayonj/pg-osc/pull/45
|
5
|
+
|
6
|
+
## [0.2.0] - 2022-02-17
|
7
|
+
|
8
|
+
- Use ISOLATION LEVEL SERIALIZABLE ([#42](https://github.com/shayonj/pg-osc/pull/42)) (props to @jfrost)
|
9
|
+
|
1
10
|
## [0.1.0] - 2022-02-16
|
2
11
|
|
3
12
|
Initial release
|
@@ -6,4 +15,4 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
|
|
6
15
|
|
7
16
|
`pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it copies the contents from a primary table to a shadow table, performs the schema change on the shadow table and swaps the table names in the end while preserving all changes to the primary table using triggers (via audit table).
|
8
17
|
|
9
|
-
Checkout [Readme](https://github.com/shayonj/pg-
|
18
|
+
Checkout [Readme](https://github.com/shayonj/pg-osc#readme) for more details.
|
data/Dockerfile
ADDED
data/Gemfile
CHANGED
@@ -3,16 +3,3 @@
|
|
3
3
|
source "https://rubygems.org"
|
4
4
|
|
5
5
|
gemspec
|
6
|
-
|
7
|
-
gem "ougai", "~> 2.0.0"
|
8
|
-
gem "pg", "~> 1.0"
|
9
|
-
gem "pg_query", "~> 2.1.2"
|
10
|
-
gem "pry"
|
11
|
-
gem "rake", "~> 13.0"
|
12
|
-
gem "rspec", "~> 3.0"
|
13
|
-
gem "rubocop", "~> 1.23.0"
|
14
|
-
gem "rubocop-packaging", "~> 0.5.1"
|
15
|
-
gem "rubocop-performance", "~> 1.12.0"
|
16
|
-
gem "rubocop-rake", "~> 0.6.0"
|
17
|
-
gem "rubocop-rspec", "~> 2.7.0"
|
18
|
-
gem "thor", "~> 1.1.0"
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pg_online_schema_change (0.
|
4
|
+
pg_online_schema_change (0.3.0)
|
5
|
+
ougai (~> 2.0.0)
|
6
|
+
pg (~> 1.3.2)
|
7
|
+
pg_query (~> 2.1.3)
|
8
|
+
thor (~> 1.2.1)
|
5
9
|
|
6
10
|
GEM
|
7
11
|
remote: https://rubygems.org/
|
@@ -9,18 +13,18 @@ GEM
|
|
9
13
|
ast (2.4.2)
|
10
14
|
coderay (1.1.3)
|
11
15
|
diff-lcs (1.5.0)
|
12
|
-
google-protobuf (3.19.
|
13
|
-
google-protobuf (3.19.
|
16
|
+
google-protobuf (3.19.4)
|
17
|
+
google-protobuf (3.19.4-x86_64-linux)
|
14
18
|
method_source (1.0.0)
|
15
|
-
oj (3.13.
|
19
|
+
oj (3.13.11)
|
16
20
|
ougai (2.0.0)
|
17
21
|
oj (~> 3.10)
|
18
22
|
parallel (1.21.0)
|
19
23
|
parser (3.0.3.2)
|
20
24
|
ast (~> 2.4.1)
|
21
|
-
pg (1.2
|
22
|
-
pg_query (2.1.
|
23
|
-
google-protobuf (>= 3.
|
25
|
+
pg (1.3.2)
|
26
|
+
pg_query (2.1.3)
|
27
|
+
google-protobuf (>= 3.19.2)
|
24
28
|
pry (0.14.1)
|
25
29
|
coderay (~> 1.1)
|
26
30
|
method_source (~> 1.0)
|
@@ -62,7 +66,7 @@ GEM
|
|
62
66
|
rubocop-rspec (2.7.0)
|
63
67
|
rubocop (~> 1.19)
|
64
68
|
ruby-progressbar (1.11.0)
|
65
|
-
thor (1.1
|
69
|
+
thor (1.2.1)
|
66
70
|
unicode-display_width (2.1.0)
|
67
71
|
|
68
72
|
PLATFORMS
|
@@ -70,11 +74,8 @@ PLATFORMS
|
|
70
74
|
x86_64-linux
|
71
75
|
|
72
76
|
DEPENDENCIES
|
73
|
-
ougai (~> 2.0.0)
|
74
|
-
pg (~> 1.0)
|
75
77
|
pg_online_schema_change!
|
76
|
-
|
77
|
-
pry
|
78
|
+
pry (~> 0.14.1)
|
78
79
|
rake (~> 13.0)
|
79
80
|
rspec (~> 3.0)
|
80
81
|
rubocop (~> 1.23.0)
|
@@ -82,7 +83,6 @@ DEPENDENCIES
|
|
82
83
|
rubocop-performance (~> 1.12.0)
|
83
84
|
rubocop-rake (~> 0.6.0)
|
84
85
|
rubocop-rspec (~> 2.7.0)
|
85
|
-
thor (~> 1.1.0)
|
86
86
|
|
87
87
|
BUNDLED WITH
|
88
88
|
2.3.3
|
data/README.md
CHANGED
@@ -1,14 +1,35 @@
|
|
1
|
-
# pg-
|
2
|
-
|
1
|
+
# pg-osc
|
2
|
+
|
3
|
+
[![CircleCI](https://circleci.com/gh/shayonj/pg-osc/tree/main.svg?style=shield)](https://circleci.com/gh/shayonj/pg-osc/tree/main)
|
4
|
+
[![Gem Version](https://badge.fury.io/rb/pg_online_schema_change.svg)](https://badge.fury.io/rb/pg_online_schema_change)
|
3
5
|
|
4
6
|
pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads.
|
5
7
|
|
6
|
-
`pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it
|
8
|
+
`pg-osc` uses the concept of shadow table to perform schema changes. At a high level, it creates a shadow table that looks structurally the same as the primary table, performs the schema change on the shadow table, copies contents from the primary table to the shadow table and swaps the table names in the end while preserving all changes to the primary table using triggers (via audit table).
|
7
9
|
|
8
10
|
`pg-osc` is inspired by the design and workings of tools like `pg_repack` and `pt-online-schema-change` (MySQL). Read more below on [how does it work](#how-does-it-work), [prominent features](#prominent-features), the [caveats](#caveats) and [examples](#examples)
|
9
11
|
|
10
12
|
⚠️ Proceed with caution when using this on production like workloads. Best to try on similar setup or staging like environment first. Read on below for some examples and caveats.
|
11
13
|
|
14
|
+
## Table of Contents
|
15
|
+
|
16
|
+
- [Installation](#installation)
|
17
|
+
- [Requirements](#requirements)
|
18
|
+
- [Usage](#usage)
|
19
|
+
- [How does it work](#how-does-it-work)
|
20
|
+
- [Prominent features](#prominent-features)
|
21
|
+
- [Examples](#examples)
|
22
|
+
* [Renaming a column](#renaming-a-column)
|
23
|
+
* [Multiple ALTER statements](#multiple-alter-statements)
|
24
|
+
* [Kill other backends after 5s](#kill-other-backends-after-5s)
|
25
|
+
* [Backfill data](#backfill-data)
|
26
|
+
* [Running using Docker](#running-using-docker)
|
27
|
+
- [Caveats](#caveats)
|
28
|
+
- [Development](#development)
|
29
|
+
- [Releasing](#releasing)
|
30
|
+
- [Contributing](#contributing)
|
31
|
+
- [License](#license)
|
32
|
+
- [Code of Conduct](#code-of-conduct)
|
12
33
|
## Installation
|
13
34
|
|
14
35
|
Add this line to your application's Gemfile:
|
@@ -25,6 +46,13 @@ Or install it yourself as:
|
|
25
46
|
|
26
47
|
$ gem install pg_online_schema_change
|
27
48
|
|
49
|
+
This will include all dependencies accordingly as well. Make sure the following requirements are satisfied.
|
50
|
+
|
51
|
+
Or via Docker:
|
52
|
+
|
53
|
+
docker pull shayonj/pg-osc:latest
|
54
|
+
|
55
|
+
https://hub.docker.com/r/shayonj/pg-osc
|
28
56
|
## Requirements
|
29
57
|
- PostgreSQL 9.6 and later
|
30
58
|
- Ruby 2.6 and later
|
@@ -33,6 +61,8 @@ Or install it yourself as:
|
|
33
61
|
## Usage
|
34
62
|
|
35
63
|
```
|
64
|
+
pg-online-schema-change help perform
|
65
|
+
|
36
66
|
Usage:
|
37
67
|
pg-online-schema-change perform -a, --alter-statement=ALTER_STATEMENT -d, --dbname=DBNAME -h, --host=HOST -p, --port=N -s, --schema=SCHEMA -u, --username=USERNAME -w, --password=PASSWORD
|
38
68
|
|
@@ -89,7 +119,7 @@ print the version
|
|
89
119
|
- Copies over indexes and Foreign keys.
|
90
120
|
- Optionally drop or retain old tables in the end.
|
91
121
|
- Backfill old/new columns as data is copied from primary table to shadow table, and then perform the swap. [Example](#backfill-data)
|
92
|
-
- **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-
|
122
|
+
- **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-osc/issues/14)
|
93
123
|
|
94
124
|
## Examples
|
95
125
|
|
@@ -157,7 +187,20 @@ pg-online-schema-change perform \
|
|
157
187
|
--copy-statement "/src/query.sql" \
|
158
188
|
--drop
|
159
189
|
```
|
160
|
-
|
190
|
+
|
191
|
+
### Running using Docker
|
192
|
+
|
193
|
+
```
|
194
|
+
docker run --network host -it --rm shayonj/pg-osc:latest \
|
195
|
+
pg-online-schema-change perform \
|
196
|
+
--alter-statement 'ALTER TABLE books ADD COLUMN "purchased" BOOLEAN DEFAULT FALSE; ALTER TABLE books RENAME COLUMN email TO new_email;' \
|
197
|
+
--dbname "postgres" \
|
198
|
+
--host "localhost" \
|
199
|
+
--username "jamesbond" \
|
200
|
+
--password "" \
|
201
|
+
--drop
|
202
|
+
```
|
203
|
+
## Caveats
|
161
204
|
- A primary key should exist on the table; without it, `pg-osc` will raise an exception
|
162
205
|
- This is because - currently there is no other way to uniquely identify rows during replay.
|
163
206
|
- `pg-osc` will acquire `ACCESS EXCLUSIVE` lock on the parent table twice during the operation.
|
@@ -192,16 +235,14 @@ To install this gem onto your local machine, run `bundle exec rake install`.
|
|
192
235
|
## Releasing
|
193
236
|
|
194
237
|
- Bump version in `version.rb`
|
195
|
-
-
|
196
|
-
-
|
197
|
-
- `gem build pg_online_schema_change.gemspec`
|
198
|
-
- `gem push pg_online_schema_change-0.1.0.gem`
|
238
|
+
- Commit
|
239
|
+
- `./scripts/release.sh 0.2.0`
|
199
240
|
- Update `CHANGELOG.md`
|
200
|
-
- Create a new release - https://github.com/shayonj/pg-
|
241
|
+
- Create a new release - https://github.com/shayonj/pg-osc/releases/new
|
201
242
|
|
202
243
|
## Contributing
|
203
244
|
|
204
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-
|
245
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/shayonj/pg-osc.
|
205
246
|
|
206
247
|
## License
|
207
248
|
|
@@ -209,4 +250,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
|
|
209
250
|
|
210
251
|
## Code of Conduct
|
211
252
|
|
212
|
-
Everyone interacting in the PgOnlineSchemaChange project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/shayonj/pg-
|
253
|
+
Everyone interacting in the PgOnlineSchemaChange project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/shayonj/pg-osc/blob/main/CODE_OF_CONDUCT.md).
|