pg_online_schema_change 0.5.0 → 0.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +2 -1
- data/.rubocop_todo.yml +5 -5
- data/CHANGELOG.md +17 -0
- data/CODE_OF_CONDUCT.md +1 -1
- data/Gemfile.lock +3 -2
- data/README.md +4 -1
- data/lib/pg_online_schema_change/cli.rb +1 -1
- data/lib/pg_online_schema_change/functions.rb +0 -12
- data/lib/pg_online_schema_change/orchestrate.rb +7 -3
- data/lib/pg_online_schema_change/query.rb +23 -3
- data/lib/pg_online_schema_change/replay.rb +4 -4
- data/lib/pg_online_schema_change/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c49f895f91c72ef491002660c994088b9d527db88d3b5d77cbb851771a2f2ac8
|
4
|
+
data.tar.gz: d57329d32657cd64edb60b8c174c6fc32f4279a202548b5132407e98c59f2f9e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 963c4c9751301d029857f021b40d033d50f9b8865548a5d8bd752ed4c03a4cf1e5b016c5129e771db45df1e2677a8e29b5c2b488f152932238e7158e4c3a2a8b
|
7
|
+
data.tar.gz: bc8c559ade623ea76794cf1756642474cf32eee08406f884503b9450e1d5b39233778dd77b7fcd56327c140ba046279b21a1a1a64b5ac606bf5c2119cea6b350
|
data/.rspec
CHANGED
data/.rubocop_todo.yml
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# This configuration was generated by
|
2
2
|
# `rubocop --auto-gen-config`
|
3
|
-
# on 2022-
|
3
|
+
# on 2022-03-13 19:35:49 UTC using RuboCop version 1.23.0.
|
4
4
|
# The point is for the user to remove these configuration records
|
5
5
|
# one by one as the offenses are removed from the code base.
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
@@ -9,7 +9,7 @@
|
|
9
9
|
# Offense count: 2
|
10
10
|
# Configuration parameters: CountComments, CountAsOne.
|
11
11
|
Metrics/ClassLength:
|
12
|
-
Max:
|
12
|
+
Max: 250
|
13
13
|
|
14
14
|
# Offense count: 2
|
15
15
|
# Configuration parameters: IgnoredMethods.
|
@@ -26,14 +26,14 @@ Packaging/GemspecGit:
|
|
26
26
|
Exclude:
|
27
27
|
- 'pg_online_schema_change.gemspec'
|
28
28
|
|
29
|
-
# Offense count:
|
29
|
+
# Offense count: 67
|
30
30
|
# Configuration parameters: CountAsOne.
|
31
31
|
RSpec/ExampleLength:
|
32
32
|
Max: 55
|
33
33
|
|
34
|
-
# Offense count:
|
34
|
+
# Offense count: 24
|
35
35
|
RSpec/MultipleExpectations:
|
36
|
-
Max:
|
36
|
+
Max: 13
|
37
37
|
|
38
38
|
# Offense count: 6
|
39
39
|
# Configuration parameters: AllowedMethods.
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,20 @@
|
|
1
|
+
|
2
|
+
## [0.6.0] - 2022-03-13
|
3
|
+
* Move CI to Github Actions in https://github.com/shayonj/pg-osc/pull/64
|
4
|
+
* Set --password as optional since it is deprecated in https://github.com/shayonj/pg-osc/pull/66
|
5
|
+
* Smoke tests in https://github.com/shayonj/pg-osc/pull/65
|
6
|
+
* Add foreign keys to parent during swap in https://github.com/shayonj/pg-osc/pull/67
|
7
|
+
|
8
|
+
## [0.6.0] - 2022-02-26
|
9
|
+
* Delete items by audit table PK when replaying by @shayonj @jfrost in https://github.com/shayonj/pg-osc/pull/60
|
10
|
+
- Fixes a race condition issue: https://github.com/shayonj/pg-osc/issues/58
|
11
|
+
|
12
|
+
## [0.5.0] - 2022-02-26
|
13
|
+
* Share some preliminary load test figures in https://github.com/shayonj/pg-osc/pull/54
|
14
|
+
* Reuse existing transaction open for reading table columns in https://github.com/shayonj/pg-osc/pull/53
|
15
|
+
* Start to deprecate --password with PGPASSWORD in https://github.com/shayonj/pg-osc/pull/56
|
16
|
+
* Introduce configurable PULL_BATCH_COUNT and DELTA_COUNT in https://github.com/shayonj/pg-osc/pull/57
|
17
|
+
|
1
18
|
## [0.4.0] - 2022-02-22
|
2
19
|
* Lint sourcecode, setup Rubocop proper and Lint in CI by @shayonj in https://github.com/shayonj/pg-osc/pull/46
|
3
20
|
* Uniquely identify operation_type column by @shayonj in https://github.com/shayonj/pg-osc/pull/50
|
data/CODE_OF_CONDUCT.md
CHANGED
@@ -39,7 +39,7 @@ This Code of Conduct applies within all community spaces, and also applies when
|
|
39
39
|
|
40
40
|
## Enforcement
|
41
41
|
|
42
|
-
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at
|
42
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at shayonj@gmail.com. All complaints will be reviewed and investigated promptly and fairly.
|
43
43
|
|
44
44
|
All community leaders are obligated to respect the privacy and security of the reporter of any incident.
|
45
45
|
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pg_online_schema_change (0.
|
4
|
+
pg_online_schema_change (0.7.0)
|
5
5
|
ougai (~> 2.0.0)
|
6
6
|
pg (~> 1.3.2)
|
7
7
|
pg_query (~> 2.1.3)
|
@@ -14,6 +14,7 @@ GEM
|
|
14
14
|
coderay (1.1.3)
|
15
15
|
diff-lcs (1.5.0)
|
16
16
|
google-protobuf (3.19.4)
|
17
|
+
google-protobuf (3.19.4-x86_64-linux)
|
17
18
|
method_source (1.0.0)
|
18
19
|
oj (3.13.11)
|
19
20
|
ougai (2.0.0)
|
@@ -21,7 +22,7 @@ GEM
|
|
21
22
|
parallel (1.21.0)
|
22
23
|
parser (3.0.3.2)
|
23
24
|
ast (~> 2.4.1)
|
24
|
-
pg (1.3.
|
25
|
+
pg (1.3.4)
|
25
26
|
pg_query (2.1.3)
|
26
27
|
google-protobuf (>= 3.19.2)
|
27
28
|
pry (0.14.1)
|
data/README.md
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
# pg-osc
|
2
2
|
|
3
|
-
[![
|
3
|
+
[![CI](https://github.com/shayonj/pg-osc/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/ci.yaml)
|
4
|
+
[![Smoke Test PG 9.6](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-9-6.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-9-6.yaml)
|
5
|
+
[![Smoke Test PG 13.6](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg-osc/actions/workflows/smoke-tests-13-6.yaml)
|
4
6
|
[![Gem Version](https://badge.fury.io/rb/pg_online_schema_change.svg)](https://badge.fury.io/rb/pg_online_schema_change)
|
5
7
|
|
6
8
|
pg-online-schema-change (`pg-osc`) is a tool for making schema changes (any `ALTER` statements) in Postgres tables with minimal locks, thus helping achieve zero downtime schema changes against production workloads.
|
@@ -100,6 +102,7 @@ print the version
|
|
100
102
|
- `pg-osc` acquires minimal locks throughout the process (read more below on the caveats).
|
101
103
|
- Copies over indexes and Foreign keys.
|
102
104
|
- Optionally drop or retain old tables in the end.
|
105
|
+
- Tune how slow or fast should replays be from the audit/log table ([Replaying larger workloads](#replaying-larger-workloads)).
|
103
106
|
- Backfill old/new columns as data is copied from primary table to shadow table, and then perform the swap. [Example](#backfill-data)
|
104
107
|
- **TBD**: Ability to reverse the change with no data loss. [tracking issue](https://github.com/shayonj/pg-osc/issues/14)
|
105
108
|
|
@@ -15,7 +15,7 @@ module PgOnlineSchemaChange
|
|
15
15
|
method_option :host, aliases: "-h", type: :string, required: true, desc: "Server host where the Database is located"
|
16
16
|
method_option :username, aliases: "-u", type: :string, required: true, desc: "Username for the Database"
|
17
17
|
method_option :port, aliases: "-p", type: :numeric, required: true, default: 5432, desc: "Port for the Database"
|
18
|
-
method_option :password, aliases: "-w", type: :string, required:
|
18
|
+
method_option :password, aliases: "-w", type: :string, required: false, default: "", desc: "DEPRECATED: Password for the Database. Please pass PGPASSWORD environment variable instead."
|
19
19
|
method_option :verbose, aliases: "-v", type: :boolean, default: false, desc: "Emit logs in debug mode"
|
20
20
|
method_option :drop, aliases: "-f", type: :boolean, default: false,
|
21
21
|
desc: "Drop the original table in the end after the swap"
|
@@ -47,18 +47,6 @@ FUNC_CREATE_TABLE_ALL = <<~SQL
|
|
47
47
|
EXECUTE format(
|
48
48
|
'CREATE TABLE %s (LIKE %s including all)',
|
49
49
|
newsource_table, source_table);
|
50
|
-
for rec in
|
51
|
-
SELECT oid, conname
|
52
|
-
FROM pg_constraint
|
53
|
-
WHERE contype = 'f'
|
54
|
-
AND conrelid = source_table::regclass
|
55
|
-
LOOP
|
56
|
-
EXECUTE format(
|
57
|
-
'ALTER TABLE %s add constraint %s %s',
|
58
|
-
newsource_table,
|
59
|
-
rec.conname,
|
60
|
-
pg_get_constraintdef(rec.oid));
|
61
|
-
END LOOP;
|
62
50
|
END
|
63
51
|
$$;
|
64
52
|
SQL
|
@@ -37,6 +37,9 @@ module PgOnlineSchemaChange
|
|
37
37
|
Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
|
38
38
|
Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
|
39
39
|
Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
|
40
|
+
|
41
|
+
Store.set(:referential_foreign_key_statements, Query.referential_foreign_keys_to_refresh(client, client.table))
|
42
|
+
Store.set(:self_foreign_key_statements, Query.self_foreign_keys_to_refresh(client, client.table))
|
40
43
|
end
|
41
44
|
|
42
45
|
def run!(options)
|
@@ -221,7 +224,6 @@ module PgOnlineSchemaChange
|
|
221
224
|
def swap!
|
222
225
|
logger.info("Performing swap!")
|
223
226
|
|
224
|
-
foreign_key_statements = Query.get_foreign_keys_to_refresh(client, client.table)
|
225
227
|
storage_params_reset = primary_table_storage_parameters.empty? ? "" : "ALTER TABLE #{client.table} SET (#{primary_table_storage_parameters});"
|
226
228
|
|
227
229
|
# From here on, all statements are carried out in a single
|
@@ -239,12 +241,13 @@ module PgOnlineSchemaChange
|
|
239
241
|
sql = <<~SQL
|
240
242
|
ALTER TABLE #{client.table} RENAME to #{old_primary_table};
|
241
243
|
ALTER TABLE #{shadow_table} RENAME to #{client.table};
|
242
|
-
#{
|
244
|
+
#{referential_foreign_key_statements}
|
245
|
+
#{self_foreign_key_statements}
|
243
246
|
#{storage_params_reset}
|
244
247
|
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
|
245
248
|
SQL
|
246
249
|
|
247
|
-
Query.run(client.connection, sql)
|
250
|
+
Query.run(client.connection, sql, opened)
|
248
251
|
ensure
|
249
252
|
Query.run(client.connection, "COMMIT;")
|
250
253
|
Query.run(client.connection, "SET statement_timeout = 0;")
|
@@ -270,6 +273,7 @@ module PgOnlineSchemaChange
|
|
270
273
|
shadow_table_drop = shadow_table ? "DROP TABLE IF EXISTS #{shadow_table}" : ""
|
271
274
|
|
272
275
|
sql = <<~SQL
|
276
|
+
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table};
|
273
277
|
#{audit_table_drop};
|
274
278
|
#{shadow_table_drop};
|
275
279
|
#{primary_drop}
|
@@ -140,7 +140,7 @@ module PgOnlineSchemaChange
|
|
140
140
|
end
|
141
141
|
end
|
142
142
|
|
143
|
-
def
|
143
|
+
def referential_foreign_keys_to_refresh(client, table)
|
144
144
|
references = get_all_constraints_for(client).select do |row|
|
145
145
|
row["table_from"] == table && row["constraint_type"] == "f"
|
146
146
|
end
|
@@ -158,12 +158,32 @@ module PgOnlineSchemaChange
|
|
158
158
|
end.join
|
159
159
|
end
|
160
160
|
|
161
|
-
def
|
161
|
+
def self_foreign_keys_to_refresh(client, table)
|
162
162
|
references = get_all_constraints_for(client).select do |row|
|
163
|
-
row["
|
163
|
+
row["table_on"] == table && row["constraint_type"] == "f"
|
164
164
|
end
|
165
165
|
|
166
166
|
references.map do |row|
|
167
|
+
add_statement = if row["definition"].end_with?("NOT VALID")
|
168
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
|
169
|
+
else
|
170
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
|
171
|
+
end
|
172
|
+
add_statement
|
173
|
+
end.join
|
174
|
+
end
|
175
|
+
|
176
|
+
def get_foreign_keys_to_validate(client, table)
|
177
|
+
constraints = get_all_constraints_for(client)
|
178
|
+
referential_foreign_keys = constraints.select do |row|
|
179
|
+
row["table_from"] == table && row["constraint_type"] == "f"
|
180
|
+
end
|
181
|
+
|
182
|
+
self_foreign_keys = constraints.select do |row|
|
183
|
+
row["table_on"] == table && row["constraint_type"] == "f"
|
184
|
+
end
|
185
|
+
|
186
|
+
[referential_foreign_keys, self_foreign_keys].flatten.map do |row|
|
167
187
|
"ALTER TABLE #{row["table_on"]} VALIDATE CONSTRAINT #{row["constraint_name"]};"
|
168
188
|
end.join
|
169
189
|
end
|
@@ -87,7 +87,7 @@ module PgOnlineSchemaChange
|
|
87
87
|
SQL
|
88
88
|
to_be_replayed << sql
|
89
89
|
|
90
|
-
to_be_deleted_rows << "'#{row[
|
90
|
+
to_be_deleted_rows << "'#{row[audit_table_pk]}'"
|
91
91
|
when "UPDATE"
|
92
92
|
set_values = new_row.map do |column, value|
|
93
93
|
"#{column} = '#{value}'"
|
@@ -100,14 +100,14 @@ module PgOnlineSchemaChange
|
|
100
100
|
SQL
|
101
101
|
to_be_replayed << sql
|
102
102
|
|
103
|
-
to_be_deleted_rows << "'#{row[
|
103
|
+
to_be_deleted_rows << "'#{row[audit_table_pk]}'"
|
104
104
|
when "DELETE"
|
105
105
|
sql = <<~SQL
|
106
106
|
DELETE FROM #{shadow_table} WHERE #{primary_key}=\'#{row[primary_key]}\';
|
107
107
|
SQL
|
108
108
|
to_be_replayed << sql
|
109
109
|
|
110
|
-
to_be_deleted_rows << "'#{row[
|
110
|
+
to_be_deleted_rows << "'#{row[audit_table_pk]}'"
|
111
111
|
end
|
112
112
|
end
|
113
113
|
|
@@ -117,7 +117,7 @@ module PgOnlineSchemaChange
|
|
117
117
|
return unless to_be_deleted_rows.count >= 1
|
118
118
|
|
119
119
|
delete_query = <<~SQL
|
120
|
-
DELETE FROM #{audit_table} WHERE #{
|
120
|
+
DELETE FROM #{audit_table} WHERE #{audit_table_pk} IN (#{to_be_deleted_rows.join(",")})
|
121
121
|
SQL
|
122
122
|
Query.run(client.connection, delete_query, reuse_trasaction)
|
123
123
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_online_schema_change
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shayon Mukherjee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-03-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ougai
|