pg_online_schema_change 0.9.3 → 0.9.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -0
- data/Gemfile.lock +8 -8
- data/README.md +22 -22
- data/lib/pg_online_schema_change/cli.rb +7 -0
- data/lib/pg_online_schema_change/client.rb +4 -2
- data/lib/pg_online_schema_change/orchestrate.rb +15 -6
- data/lib/pg_online_schema_change/query.rb +7 -10
- data/lib/pg_online_schema_change/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b4b544005274743fc18f3147204997861e3ac55e7c619a6859849049eff42324
|
4
|
+
data.tar.gz: a4123844149ed79c7e8781588ca055f49954afa245a7a63c50886e076be9562e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d5b08b1f26c9d3f3c19d819cf65d0787bbbd6221845ce44bcf2aaf13a0389f8824139d3034507cb8118751b2e09713ec4116322e8b86b8988276d6ab8f4a2390
|
7
|
+
data.tar.gz: e5e589f58797e28fbe15fd355367a61c2673f6d90ccadb67c9b43427ea0a5b86afba9da4ec5199dd4656397f4c7a5e8b27a3805876915d7eddb8366a19892a4b
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
## [0.9.4] - 2023-09-17
|
2
|
+
|
3
|
+
- Resolving gem push and sync glitch in 0.9.3
|
4
|
+
|
5
|
+
## [0.9.3] - 2023-09-17
|
6
|
+
|
7
|
+
- Dependency updates
|
8
|
+
- Adding support for showing the gem version with -v or --version by @brycethornton #101
|
9
|
+
- Fix for INSERT's failing for long table names by @ahilmer #116
|
10
|
+
- Get view definition of a view from dedicated schema by @shayonj #117
|
11
|
+
|
1
12
|
## [0.9.2] - 2023-07-03
|
2
13
|
|
3
14
|
- Dependency updates
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pg_online_schema_change (0.9.
|
4
|
+
pg_online_schema_change (0.9.5)
|
5
5
|
ougai (~> 2.0.0)
|
6
6
|
pg (>= 1.3.2, < 1.6.0)
|
7
7
|
pg_query (>= 2.1.3, < 4.3.0)
|
@@ -14,8 +14,8 @@ GEM
|
|
14
14
|
base64 (0.1.1)
|
15
15
|
coderay (1.1.3)
|
16
16
|
diff-lcs (1.5.0)
|
17
|
-
google-protobuf (3.
|
18
|
-
google-protobuf (3.
|
17
|
+
google-protobuf (3.24.4-arm64-darwin)
|
18
|
+
google-protobuf (3.24.4-x86_64-linux)
|
19
19
|
haml (6.1.1)
|
20
20
|
temple (>= 0.8.2)
|
21
21
|
thor
|
@@ -23,7 +23,7 @@ GEM
|
|
23
23
|
json (2.6.3)
|
24
24
|
language_server-protocol (3.17.0.3)
|
25
25
|
method_source (1.0.0)
|
26
|
-
oj (3.
|
26
|
+
oj (3.16.1)
|
27
27
|
ougai (2.0.0)
|
28
28
|
oj (~> 3.10)
|
29
29
|
parallel (1.23.0)
|
@@ -56,7 +56,7 @@ GEM
|
|
56
56
|
diff-lcs (>= 1.2.0, < 2.0)
|
57
57
|
rspec-support (~> 3.12.0)
|
58
58
|
rspec-support (3.12.0)
|
59
|
-
rubocop (1.56.
|
59
|
+
rubocop (1.56.4)
|
60
60
|
base64 (~> 0.1.1)
|
61
61
|
json (~> 2.3)
|
62
62
|
language_server-protocol (>= 3.17.0)
|
@@ -76,7 +76,7 @@ GEM
|
|
76
76
|
rubocop (~> 1.33)
|
77
77
|
rubocop-packaging (0.5.2)
|
78
78
|
rubocop (>= 1.33, < 2.0)
|
79
|
-
rubocop-performance (1.19.
|
79
|
+
rubocop-performance (1.19.1)
|
80
80
|
rubocop (>= 1.7.0, < 2.0)
|
81
81
|
rubocop-ast (>= 0.4.0)
|
82
82
|
rubocop-rake (0.6.0)
|
@@ -86,7 +86,7 @@ GEM
|
|
86
86
|
rubocop-capybara (~> 2.17)
|
87
87
|
rubocop-factory_bot (~> 2.22)
|
88
88
|
ruby-progressbar (1.13.0)
|
89
|
-
syntax_tree (6.
|
89
|
+
syntax_tree (6.2.0)
|
90
90
|
prettier_print (>= 1.2.0)
|
91
91
|
syntax_tree-haml (4.0.3)
|
92
92
|
haml (>= 5.2)
|
@@ -99,7 +99,7 @@ GEM
|
|
99
99
|
temple (0.10.0)
|
100
100
|
thor (1.2.2)
|
101
101
|
tilt (2.1.0)
|
102
|
-
unicode-display_width (2.
|
102
|
+
unicode-display_width (2.5.0)
|
103
103
|
|
104
104
|
PLATFORMS
|
105
105
|
arm64-darwin-20
|
data/README.md
CHANGED
@@ -11,8 +11,6 @@ pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALT
|
|
11
11
|
|
12
12
|
`pg-osc` is inspired by the design and workings of tools like `pg_repack` and `pt-online-schema-change` (MySQL). Read more below on [how does it work](#how-does-it-work), [prominent features](#prominent-features), the [caveats](#caveats) and [examples](#examples)
|
13
13
|
|
14
|
-
⚠️ Proceed with caution when using this on production like workloads. Best to try on similar setup or staging like environment first. Read on below for some examples and caveats.
|
15
|
-
|
16
14
|
## Table of Contents
|
17
15
|
|
18
16
|
- [Installation](#installation)
|
@@ -70,28 +68,29 @@ https://hub.docker.com/r/shayonj/pg-osc
|
|
70
68
|
pg-online-schema-change help perform
|
71
69
|
|
72
70
|
Usage:
|
73
|
-
pg-online-schema-change perform -a, --alter-statement=ALTER_STATEMENT -d, --dbname=DBNAME -h, --host=HOST -p, --port=N -s, --schema=SCHEMA -u, --username=USERNAME
|
71
|
+
pg-online-schema-change perform -a, --alter-statement=ALTER_STATEMENT -d, --dbname=DBNAME -h, --host=HOST -p, --port=N -s, --schema=SCHEMA -u, --username=USERNAME
|
74
72
|
|
75
73
|
Options:
|
76
|
-
-a, --alter-statement=ALTER_STATEMENT
|
77
|
-
-s, --schema=SCHEMA
|
78
|
-
|
79
|
-
-d, --dbname=DBNAME
|
80
|
-
-h, --host=HOST
|
81
|
-
-u, --username=USERNAME
|
82
|
-
-p, --port=N
|
83
|
-
|
84
|
-
-w, --password=PASSWORD
|
85
|
-
-v, [--verbose], [--no-verbose]
|
86
|
-
-f, [--drop], [--no-drop]
|
87
|
-
-k, [--kill-backends], [--no-kill-backends]
|
88
|
-
-w, [--wait-time-for-lock=N]
|
89
|
-
|
90
|
-
-c, [--copy-statement=COPY_STATEMENT]
|
91
|
-
-b, [--pull-batch-count=N]
|
92
|
-
|
93
|
-
-e, [--delta-count=N]
|
94
|
-
|
74
|
+
-a, --alter-statement=ALTER_STATEMENT # The ALTER statement to perform the schema change
|
75
|
+
-s, --schema=SCHEMA # The schema in which the table is
|
76
|
+
# Default: public
|
77
|
+
-d, --dbname=DBNAME # Name of the database
|
78
|
+
-h, --host=HOST # Server host where the Database is located
|
79
|
+
-u, --username=USERNAME # Username for the Database
|
80
|
+
-p, --port=N # Port for the Database
|
81
|
+
# Default: 5432
|
82
|
+
-w, [--password=PASSWORD] # DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead.
|
83
|
+
-v, [--verbose], [--no-verbose] # Emit logs in debug mode
|
84
|
+
-f, [--drop], [--no-drop] # Drop the original table in the end after the swap
|
85
|
+
-k, [--kill-backends], [--no-kill-backends] # Kill other competing queries/backends when trying to acquire lock for the shadow table creation and swap. It will wait for --wait-time-for-lock duration before killing backends and try upto 3 times.
|
86
|
+
-w, [--wait-time-for-lock=N] # Time to wait before killing backends to acquire lock and/or retrying upto 3 times. It will kill backends if --kill-backends is true, otherwise try upto 3 times and exit if it cannot acquire a lock.
|
87
|
+
# Default: 10
|
88
|
+
-c, [--copy-statement=COPY_STATEMENT] # Takes a .sql file location where you can provide a custom query to be played (ex: backfills) when pgosc copies data from the primary to the shadow table. More examples in README.
|
89
|
+
-b, [--pull-batch-count=N] # Number of rows to be replayed on each iteration after copy. This can be tuned for faster catch up and swap. Best used with delta-count.
|
90
|
+
# Default: 1000
|
91
|
+
-e, [--delta-count=N] # Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count.
|
92
|
+
# Default: 20
|
93
|
+
-o, [--skip-foreign-key-validation], [--no-skip-foreign-key-validation] # Skip foreign key validation after swap. You shouldn't need this unless you have a very specific use case, like manually validating foreign key constraints after swap.
|
95
94
|
```
|
96
95
|
|
97
96
|
```
|
@@ -289,6 +288,7 @@ bundle exec bin/pg-online-schema-change perform -a 'ALTER TABLE pgbench_accounts
|
|
289
288
|
## Releasing
|
290
289
|
|
291
290
|
- Bump version in `version.rb`
|
291
|
+
- bundle install
|
292
292
|
- Commit
|
293
293
|
- `./scripts/release.sh 0.2.0`
|
294
294
|
- Update `CHANGELOG.md`
|
@@ -89,6 +89,13 @@ module PgOnlineSchemaChange
|
|
89
89
|
default: DELTA_COUNT,
|
90
90
|
desc:
|
91
91
|
"Indicates how many rows should be remaining before a swap should be performed. This can be tuned for faster catch up and swap, especially on highly volume tables. Best used with pull-batch-count."
|
92
|
+
method_option :skip_foreign_key_validation,
|
93
|
+
aliases: "-o",
|
94
|
+
type: :boolean,
|
95
|
+
required: false,
|
96
|
+
default: false,
|
97
|
+
desc:
|
98
|
+
"Skip foreign key validation after swap. You shouldn't need this unless you have a very specific use case, like manually validating foreign key constraints after swap."
|
92
99
|
|
93
100
|
def perform
|
94
101
|
client_options = Struct.new(*options.keys.map(&:to_sym)).new(*options.values)
|
@@ -19,7 +19,8 @@ module PgOnlineSchemaChange
|
|
19
19
|
:wait_time_for_lock,
|
20
20
|
:copy_statement,
|
21
21
|
:pull_batch_count,
|
22
|
-
:delta_count
|
22
|
+
:delta_count,
|
23
|
+
:skip_foreign_key_validation
|
23
24
|
|
24
25
|
def initialize(options)
|
25
26
|
@alter_statement = options.alter_statement
|
@@ -34,6 +35,7 @@ module PgOnlineSchemaChange
|
|
34
35
|
@wait_time_for_lock = options.wait_time_for_lock
|
35
36
|
@pull_batch_count = options.pull_batch_count
|
36
37
|
@delta_count = options.delta_count
|
38
|
+
@skip_foreign_key_validation = options.skip_foreign_key_validation
|
37
39
|
|
38
40
|
handle_copy_statement(options.copy_statement)
|
39
41
|
handle_validations
|
@@ -54,7 +56,7 @@ module PgOnlineSchemaChange
|
|
54
56
|
|
55
57
|
return if Query.same_table?(@alter_statement)
|
56
58
|
|
57
|
-
raise Error
|
59
|
+
raise Error, "All statements should belong to the same table: #{@alter_statement}"
|
58
60
|
end
|
59
61
|
|
60
62
|
def handle_copy_statement(statement)
|
@@ -36,7 +36,10 @@ module PgOnlineSchemaChange
|
|
36
36
|
Store.set(:trigger_time_column, "trigger_time_#{pgosc_identifier}")
|
37
37
|
Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
|
38
38
|
Store.set(:shadow_table, "pgosc_st_#{client.table.downcase}_#{pgosc_identifier}")
|
39
|
-
Store.set(
|
39
|
+
Store.set(
|
40
|
+
:primary_table_storage_parameters,
|
41
|
+
Query.storage_parameters_for(client, client.table_name, true) || "",
|
42
|
+
)
|
40
43
|
|
41
44
|
Store.set(
|
42
45
|
:referential_foreign_key_statements,
|
@@ -109,7 +112,10 @@ module PgOnlineSchemaChange
|
|
109
112
|
|
110
113
|
Query.run(client.connection, sql)
|
111
114
|
|
112
|
-
Store.set(
|
115
|
+
Store.set(
|
116
|
+
:audit_table_pk_sequence,
|
117
|
+
Query.get_sequence_name(client, audit_table, audit_table_pk),
|
118
|
+
)
|
113
119
|
end
|
114
120
|
|
115
121
|
def setup_trigger!
|
@@ -270,11 +276,14 @@ module PgOnlineSchemaChange
|
|
270
276
|
end
|
271
277
|
|
272
278
|
def validate_constraints!
|
273
|
-
|
274
|
-
|
275
|
-
validate_statements = Query.get_foreign_keys_to_validate(client, client.table_name)
|
279
|
+
return if client.skip_foreign_key_validation
|
276
280
|
|
277
|
-
Query
|
281
|
+
Query
|
282
|
+
.get_foreign_keys_to_validate(client, client.table_name)
|
283
|
+
.each do |statement|
|
284
|
+
logger.info("Validating constraints!", statement: statement)
|
285
|
+
Query.run(client.connection, statement)
|
286
|
+
end
|
278
287
|
end
|
279
288
|
|
280
289
|
def replace_views!
|
@@ -140,10 +140,9 @@ module PgOnlineSchemaChange
|
|
140
140
|
SELECT pg_get_serial_sequence('#{table}', '#{column}');
|
141
141
|
SQL
|
142
142
|
|
143
|
-
run(client.connection, query)
|
144
|
-
row["pg_get_serial_sequence"]
|
145
|
-
|
146
|
-
}.first
|
143
|
+
run(client.connection, query) do |result|
|
144
|
+
result.map { |row| row["pg_get_serial_sequence"] }
|
145
|
+
end.first
|
147
146
|
end
|
148
147
|
|
149
148
|
def get_triggers_for(client, table)
|
@@ -238,11 +237,9 @@ module PgOnlineSchemaChange
|
|
238
237
|
self_foreign_keys =
|
239
238
|
constraints.select { |row| row["table_on"] == table && row["constraint_type"] == "f" }
|
240
239
|
|
241
|
-
[referential_foreign_keys, self_foreign_keys].flatten
|
242
|
-
|
243
|
-
|
244
|
-
end
|
245
|
-
.join
|
240
|
+
[referential_foreign_keys, self_foreign_keys].flatten.map do |row|
|
241
|
+
"ALTER TABLE #{row["table_on"]} VALIDATE CONSTRAINT #{row["constraint_name"]};"
|
242
|
+
end
|
246
243
|
end
|
247
244
|
|
248
245
|
def dropped_columns(client)
|
@@ -327,7 +324,7 @@ module PgOnlineSchemaChange
|
|
327
324
|
|
328
325
|
definitions = []
|
329
326
|
run(client.connection, query) do |result|
|
330
|
-
definitions = result.map { |row| {row["view_name"] => row["view_definition"].strip} }
|
327
|
+
definitions = result.map { |row| { row["view_name"] => row["view_definition"].strip } }
|
331
328
|
end
|
332
329
|
|
333
330
|
definitions
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_online_schema_change
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shayon Mukherjee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-10-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ougai
|