pg_online_schema_change 0.7.4 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.prettierignore +4 -0
- data/.prettierrc +13 -0
- data/.rubocop.yml +204 -75
- data/.rubocop_todo.yml +11 -25
- data/.ruby-version +1 -1
- data/CHANGELOG.md +38 -20
- data/CODE_OF_CONDUCT.md +11 -11
- data/Gemfile.lock +73 -42
- data/README.md +40 -24
- data/Rakefile +1 -1
- data/docker-compose.yml +2 -2
- data/docs/load-test.md +11 -10
- data/lib/pg_online_schema_change/cli.rb +87 -23
- data/lib/pg_online_schema_change/client.rb +24 -12
- data/lib/pg_online_schema_change/helper.rb +1 -1
- data/lib/pg_online_schema_change/orchestrate.rb +78 -40
- data/lib/pg_online_schema_change/query.rb +148 -95
- data/lib/pg_online_schema_change/replay.rb +10 -20
- data/lib/pg_online_schema_change/version.rb +1 -1
- data/lib/pg_online_schema_change.rb +15 -9
- data/package.json +13 -0
- data/yarn.lock +15 -0
- metadata +96 -36
@@ -30,16 +30,23 @@ module PgOnlineSchemaChange
|
|
30
30
|
# Set this early on to ensure their creation and cleanup (unexpected)
|
31
31
|
# happens at all times. IOW, the calls from Store.get always return
|
32
32
|
# the same value.
|
33
|
-
Store.set(:old_primary_table, "pgosc_op_table_#{client.table}")
|
34
|
-
Store.set(:audit_table, "pgosc_at_#{client.table}_#{pgosc_identifier}")
|
33
|
+
Store.set(:old_primary_table, "pgosc_op_table_#{client.table.downcase}")
|
34
|
+
Store.set(:audit_table, "pgosc_at_#{client.table.downcase}_#{pgosc_identifier}")
|
35
35
|
Store.set(:operation_type_column, "operation_type_#{pgosc_identifier}")
|
36
36
|
Store.set(:trigger_time_column, "trigger_time_#{pgosc_identifier}")
|
37
37
|
Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
|
38
38
|
Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
|
39
|
-
Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
|
40
|
-
|
41
|
-
Store.set(
|
42
|
-
|
39
|
+
Store.set(:shadow_table, "pgosc_st_#{client.table.downcase}_#{pgosc_identifier}")
|
40
|
+
|
41
|
+
Store.set(
|
42
|
+
:referential_foreign_key_statements,
|
43
|
+
Query.referential_foreign_keys_to_refresh(client, client.table_name),
|
44
|
+
)
|
45
|
+
Store.set(
|
46
|
+
:self_foreign_key_statements,
|
47
|
+
Query.self_foreign_keys_to_refresh(client, client.table_name),
|
48
|
+
)
|
49
|
+
Store.set(:trigger_statements, Query.get_triggers_for(client, client.table_name))
|
43
50
|
end
|
44
51
|
|
45
52
|
def run!(options)
|
@@ -49,6 +56,7 @@ module PgOnlineSchemaChange
|
|
49
56
|
raise Error, "Parent table has no primary key, exiting..." if primary_key.nil?
|
50
57
|
|
51
58
|
setup_audit_table!
|
59
|
+
|
52
60
|
setup_trigger!
|
53
61
|
setup_shadow_table! # re-uses transaction with serializable
|
54
62
|
disable_vacuum! # re-uses transaction with serializable
|
@@ -72,9 +80,7 @@ module PgOnlineSchemaChange
|
|
72
80
|
def setup_signals!
|
73
81
|
reader, writer = IO.pipe
|
74
82
|
|
75
|
-
|
76
|
-
trap(sig) { writer.puts sig }
|
77
|
-
end
|
83
|
+
['TERM', 'QUIT', 'INT'].each { |sig| trap(sig) { writer.puts sig } }
|
78
84
|
|
79
85
|
reader
|
80
86
|
end
|
@@ -83,14 +89,14 @@ module PgOnlineSchemaChange
|
|
83
89
|
reader = setup_signals!
|
84
90
|
signal = reader.gets.chomp
|
85
91
|
|
86
|
-
while !reader.closed? &&
|
87
|
-
logger.info
|
92
|
+
while !reader.closed? && reader.wait_readable # rubocop:disable Lint/UnreachableLoop
|
93
|
+
logger.info("Signal #{signal} received, cleaning up")
|
88
94
|
|
89
95
|
client.connection.cancel
|
90
96
|
drop_and_cleanup!
|
91
97
|
reader.close
|
92
98
|
|
93
|
-
exit
|
99
|
+
exit(Signal.list[signal])
|
94
100
|
end
|
95
101
|
end
|
96
102
|
|
@@ -98,7 +104,7 @@ module PgOnlineSchemaChange
|
|
98
104
|
logger.info("Setting up audit table", { audit_table: audit_table })
|
99
105
|
|
100
106
|
sql = <<~SQL
|
101
|
-
CREATE TABLE #{audit_table} (#{audit_table_pk} SERIAL PRIMARY KEY, #{operation_type_column} text, #{trigger_time_column} timestamp, LIKE #{client.
|
107
|
+
CREATE TABLE #{audit_table} (#{audit_table_pk} SERIAL PRIMARY KEY, #{operation_type_column} text, #{trigger_time_column} timestamp, LIKE #{client.table_name});
|
102
108
|
SQL
|
103
109
|
|
104
110
|
Query.run(client.connection, sql)
|
@@ -108,34 +114,34 @@ module PgOnlineSchemaChange
|
|
108
114
|
# acquire access exclusive lock to ensure audit triggers
|
109
115
|
# are setup fine. This also calls kill_backends (if opted in via flag)
|
110
116
|
# so any competing backends will be killed to setup the trigger
|
111
|
-
opened = Query.open_lock_exclusive(client, client.
|
117
|
+
opened = Query.open_lock_exclusive(client, client.table_name)
|
112
118
|
|
113
119
|
raise AccessExclusiveLockNotAcquired unless opened
|
114
120
|
|
115
121
|
logger.info("Setting up triggers")
|
116
122
|
|
117
123
|
sql = <<~SQL
|
118
|
-
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.
|
124
|
+
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table_name};
|
119
125
|
|
120
126
|
CREATE OR REPLACE FUNCTION primary_to_audit_table_trigger()
|
121
127
|
RETURNS TRIGGER AS
|
122
128
|
$$
|
123
129
|
BEGIN
|
124
130
|
IF ( TG_OP = 'INSERT') THEN
|
125
|
-
INSERT INTO
|
131
|
+
INSERT INTO "#{audit_table}" select nextval('#{audit_table_pk_sequence}'), 'INSERT', clock_timestamp(), NEW.* ;
|
126
132
|
RETURN NEW;
|
127
133
|
ELSIF ( TG_OP = 'UPDATE') THEN
|
128
|
-
INSERT INTO
|
134
|
+
INSERT INTO "#{audit_table}" select nextval('#{audit_table_pk_sequence}'), 'UPDATE', clock_timestamp(), NEW.* ;
|
129
135
|
RETURN NEW;
|
130
136
|
ELSIF ( TG_OP = 'DELETE') THEN
|
131
|
-
INSERT INTO
|
137
|
+
INSERT INTO "#{audit_table}" select nextval('#{audit_table_pk_sequence}'), 'DELETE', clock_timestamp(), OLD.* ;
|
132
138
|
RETURN NEW;
|
133
139
|
END IF;
|
134
140
|
END;
|
135
141
|
$$ LANGUAGE PLPGSQL SECURITY DEFINER;
|
136
142
|
|
137
143
|
CREATE TRIGGER primary_to_audit_table_trigger
|
138
|
-
AFTER INSERT OR UPDATE OR DELETE ON #{client.
|
144
|
+
AFTER INSERT OR UPDATE OR DELETE ON #{client.table_name}
|
139
145
|
FOR EACH ROW EXECUTE PROCEDURE primary_to_audit_table_trigger();
|
140
146
|
SQL
|
141
147
|
|
@@ -156,20 +162,30 @@ module PgOnlineSchemaChange
|
|
156
162
|
Query.run(client.connection, "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", true)
|
157
163
|
logger.info("Setting up shadow table", { shadow_table: shadow_table })
|
158
164
|
|
159
|
-
Query.run(
|
165
|
+
Query.run(
|
166
|
+
client.connection,
|
167
|
+
"SELECT create_table_all('#{client.table_name}', '#{shadow_table}');",
|
168
|
+
true,
|
169
|
+
)
|
160
170
|
|
161
171
|
# update serials
|
162
|
-
Query.run(
|
172
|
+
Query.run(
|
173
|
+
client.connection,
|
174
|
+
"SELECT fix_serial_sequence('#{client.table_name}', '#{shadow_table}');",
|
175
|
+
true,
|
176
|
+
)
|
163
177
|
end
|
164
178
|
|
165
179
|
def disable_vacuum!
|
166
180
|
# re-uses transaction with serializable
|
167
181
|
# Disabling vacuum to avoid any issues during the process
|
168
|
-
result = Query.storage_parameters_for(client, client.
|
182
|
+
result = Query.storage_parameters_for(client, client.table_name, true) || ""
|
169
183
|
Store.set(:primary_table_storage_parameters, result)
|
170
184
|
|
171
|
-
logger.debug(
|
172
|
-
|
185
|
+
logger.debug(
|
186
|
+
"Disabling vacuum on shadow and audit table",
|
187
|
+
{ shadow_table: shadow_table, audit_table: audit_table },
|
188
|
+
)
|
173
189
|
sql = <<~SQL
|
174
190
|
ALTER TABLE #{shadow_table} SET (
|
175
191
|
autovacuum_enabled = false, toast.autovacuum_enabled = false
|
@@ -185,8 +201,10 @@ module PgOnlineSchemaChange
|
|
185
201
|
def run_alter_statement!
|
186
202
|
# re-uses transaction with serializable
|
187
203
|
statement = Query.alter_statement_for(client, shadow_table)
|
188
|
-
logger.info(
|
189
|
-
|
204
|
+
logger.info(
|
205
|
+
"Running alter statement on shadow table",
|
206
|
+
{ shadow_table: shadow_table, parent_table: client.table_name },
|
207
|
+
)
|
190
208
|
Query.run(client.connection, statement, true)
|
191
209
|
|
192
210
|
Store.set(:dropped_columns_list, Query.dropped_columns(client))
|
@@ -198,10 +216,16 @@ module PgOnlineSchemaChange
|
|
198
216
|
# Begin the process to copy data into copy table
|
199
217
|
# depending on the size of the table, this can be a time
|
200
218
|
# taking operation.
|
201
|
-
logger.info(
|
219
|
+
logger.info(
|
220
|
+
"Clearing contents of audit table before copy..",
|
221
|
+
{ shadow_table: shadow_table, parent_table: client.table_name },
|
222
|
+
)
|
202
223
|
Query.run(client.connection, "DELETE FROM #{audit_table}", true)
|
203
224
|
|
204
|
-
logger.info(
|
225
|
+
logger.info(
|
226
|
+
"Copying contents..",
|
227
|
+
{ shadow_table: shadow_table, parent_table: client.table_name },
|
228
|
+
)
|
205
229
|
if client.copy_statement
|
206
230
|
query = format(client.copy_statement, shadow_table: shadow_table)
|
207
231
|
return Query.run(client.connection, query, true)
|
@@ -223,31 +247,34 @@ module PgOnlineSchemaChange
|
|
223
247
|
|
224
248
|
def swap!
|
225
249
|
logger.info("Performing swap!")
|
226
|
-
|
227
|
-
storage_params_reset = primary_table_storage_parameters.empty? ? "" : "ALTER TABLE #{client.table} SET (#{primary_table_storage_parameters});"
|
228
|
-
|
229
250
|
# From here on, all statements are carried out in a single
|
230
251
|
# transaction with access exclusive lock
|
231
252
|
|
232
|
-
opened = Query.open_lock_exclusive(client, client.
|
253
|
+
opened = Query.open_lock_exclusive(client, client.table_name)
|
233
254
|
|
234
255
|
raise AccessExclusiveLockNotAcquired unless opened
|
235
256
|
|
236
|
-
Query.run(
|
257
|
+
Query.run(
|
258
|
+
client.connection,
|
259
|
+
"SET statement_timeout to '#{SWAP_STATEMENT_TIMEOUT}';",
|
260
|
+
opened,
|
261
|
+
)
|
237
262
|
|
238
263
|
rows = Replay.rows_to_play(opened)
|
239
264
|
Replay.play!(rows, opened)
|
240
265
|
|
241
|
-
query_for_primary_key_refresh =
|
266
|
+
query_for_primary_key_refresh =
|
267
|
+
Query.query_for_primary_key_refresh(shadow_table, primary_key, client.table_name, opened)
|
242
268
|
|
243
269
|
sql = <<~SQL
|
244
270
|
#{query_for_primary_key_refresh};
|
245
|
-
ALTER TABLE #{client.
|
246
|
-
ALTER TABLE #{shadow_table} RENAME to #{client.
|
271
|
+
ALTER TABLE #{client.table_name} RENAME to #{old_primary_table};
|
272
|
+
ALTER TABLE #{shadow_table} RENAME to #{client.table_name};
|
247
273
|
#{referential_foreign_key_statements}
|
248
274
|
#{self_foreign_key_statements}
|
275
|
+
#{trigger_statements}
|
249
276
|
#{storage_params_reset}
|
250
|
-
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.
|
277
|
+
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table_name};
|
251
278
|
SQL
|
252
279
|
|
253
280
|
Query.run(client.connection, sql, opened)
|
@@ -259,13 +286,13 @@ module PgOnlineSchemaChange
|
|
259
286
|
def run_analyze!
|
260
287
|
logger.info("Performing ANALYZE!")
|
261
288
|
|
262
|
-
Query.run(client.connection, "ANALYZE VERBOSE #{client.
|
289
|
+
Query.run(client.connection, "ANALYZE VERBOSE #{client.table_name};")
|
263
290
|
end
|
264
291
|
|
265
292
|
def validate_constraints!
|
266
293
|
logger.info("Validating constraints!")
|
267
294
|
|
268
|
-
validate_statements = Query.get_foreign_keys_to_validate(client, client.
|
295
|
+
validate_statements = Query.get_foreign_keys_to_validate(client, client.table_name)
|
269
296
|
|
270
297
|
Query.run(client.connection, validate_statements)
|
271
298
|
end
|
@@ -276,7 +303,7 @@ module PgOnlineSchemaChange
|
|
276
303
|
shadow_table_drop = shadow_table ? "DROP TABLE IF EXISTS #{shadow_table}" : ""
|
277
304
|
|
278
305
|
sql = <<~SQL
|
279
|
-
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.
|
306
|
+
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table_name};
|
280
307
|
#{audit_table_drop};
|
281
308
|
#{shadow_table_drop};
|
282
309
|
#{primary_drop}
|
@@ -293,6 +320,17 @@ module PgOnlineSchemaChange
|
|
293
320
|
def pgosc_identifier
|
294
321
|
@pgosc_identifier ||= SecureRandom.hex(3)
|
295
322
|
end
|
323
|
+
|
324
|
+
def storage_params_reset
|
325
|
+
"ALTER TABLE #{client.table_name} RESET (autovacuum_enabled, toast.autovacuum_enabled);" +
|
326
|
+
(
|
327
|
+
if primary_table_storage_parameters.empty?
|
328
|
+
""
|
329
|
+
else
|
330
|
+
"ALTER TABLE #{client.table_name} SET (#{primary_table_storage_parameters});"
|
331
|
+
end
|
332
|
+
)
|
333
|
+
end
|
296
334
|
end
|
297
335
|
end
|
298
336
|
end
|
@@ -14,21 +14,31 @@ module PgOnlineSchemaChange
|
|
14
14
|
|
15
15
|
class << self
|
16
16
|
def alter_statement?(query)
|
17
|
-
PgQuery
|
18
|
-
|
19
|
-
|
17
|
+
PgQuery
|
18
|
+
.parse(query)
|
19
|
+
.tree
|
20
|
+
.stmts
|
21
|
+
.all? do |statement|
|
22
|
+
statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt) ||
|
23
|
+
statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
|
24
|
+
end
|
20
25
|
rescue PgQuery::ParseError
|
21
26
|
false
|
22
27
|
end
|
23
28
|
|
24
29
|
def same_table?(query)
|
25
|
-
tables =
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
30
|
+
tables =
|
31
|
+
PgQuery
|
32
|
+
.parse(query)
|
33
|
+
.tree
|
34
|
+
.stmts
|
35
|
+
.filter_map do |statement|
|
36
|
+
if statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt)
|
37
|
+
statement.stmt.alter_table_stmt.relation.relname
|
38
|
+
elsif statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
|
39
|
+
statement.stmt.rename_stmt.relation.relname
|
40
|
+
end
|
41
|
+
end
|
32
42
|
|
33
43
|
tables.uniq.count == 1
|
34
44
|
rescue PgQuery::ParseError
|
@@ -36,14 +46,30 @@ module PgOnlineSchemaChange
|
|
36
46
|
end
|
37
47
|
|
38
48
|
def table(query)
|
39
|
-
from_rename_statement =
|
40
|
-
|
41
|
-
|
49
|
+
from_rename_statement =
|
50
|
+
PgQuery
|
51
|
+
.parse(query)
|
52
|
+
.tree
|
53
|
+
.stmts
|
54
|
+
.filter_map { |statement| statement.stmt.rename_stmt&.relation&.relname }[
|
55
|
+
0
|
56
|
+
]
|
42
57
|
PgQuery.parse(query).tables[0] || from_rename_statement
|
43
58
|
end
|
44
59
|
|
60
|
+
def table_name(query, table)
|
61
|
+
table_name = "\"#{table}\""
|
62
|
+
if table =~ /[A-Z]/ && query.include?(table_name) && table[0] != '"'
|
63
|
+
table_name
|
64
|
+
else
|
65
|
+
table
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
45
69
|
def run(connection, query, reuse_trasaction = false, &block)
|
46
|
-
|
70
|
+
if [PG::PQTRANS_INERROR, PG::PQTRANS_UNKNOWN].include?(connection.transaction_status)
|
71
|
+
connection.cancel
|
72
|
+
end
|
47
73
|
|
48
74
|
logger.debug("Running query", { query: query })
|
49
75
|
|
@@ -65,18 +91,19 @@ module PgOnlineSchemaChange
|
|
65
91
|
def table_columns(client, table = nil, reuse_trasaction = false)
|
66
92
|
sql = <<~SQL
|
67
93
|
SELECT attname as column_name, format_type(atttypid, atttypmod) as type, attnum as column_position FROM pg_attribute
|
68
|
-
WHERE attrelid =
|
94
|
+
WHERE attrelid = '#{table || client.table_name}'::regclass AND attnum > 0 AND NOT attisdropped
|
69
95
|
ORDER BY attnum;
|
70
96
|
SQL
|
71
97
|
mapped_columns = []
|
72
98
|
|
73
99
|
run(client.connection, sql, reuse_trasaction) do |result|
|
74
|
-
mapped_columns =
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
100
|
+
mapped_columns =
|
101
|
+
result.map do |row|
|
102
|
+
row["column_name_regular"] = row["column_name"]
|
103
|
+
row["column_name"] = client.connection.quote_ident(row["column_name"])
|
104
|
+
row["column_position"] = row["column_position"].to_i
|
105
|
+
row
|
106
|
+
end
|
80
107
|
end
|
81
108
|
|
82
109
|
mapped_columns
|
@@ -86,7 +113,9 @@ module PgOnlineSchemaChange
|
|
86
113
|
parsed_query = PgQuery.parse(client.alter_statement)
|
87
114
|
|
88
115
|
parsed_query.tree.stmts.each do |statement|
|
89
|
-
|
116
|
+
if statement.stmt.alter_table_stmt
|
117
|
+
statement.stmt.alter_table_stmt.relation.relname = shadow_table
|
118
|
+
end
|
90
119
|
|
91
120
|
statement.stmt.rename_stmt.relation.relname = shadow_table if statement.stmt.rename_stmt
|
92
121
|
end
|
@@ -97,17 +126,27 @@ module PgOnlineSchemaChange
|
|
97
126
|
query = <<~SQL
|
98
127
|
SELECT indexdef, schemaname
|
99
128
|
FROM pg_indexes
|
100
|
-
WHERE schemaname =
|
129
|
+
WHERE schemaname = '#{client.schema}' AND tablename = '#{table}'
|
101
130
|
SQL
|
102
131
|
|
103
132
|
indexes = []
|
104
|
-
run(client.connection, query)
|
105
|
-
indexes = result.map { |row| row["indexdef"] }
|
106
|
-
end
|
133
|
+
run(client.connection, query) { |result| indexes = result.map { |row| row["indexdef"] } }
|
107
134
|
|
108
135
|
indexes
|
109
136
|
end
|
110
137
|
|
138
|
+
def get_triggers_for(client, table)
|
139
|
+
query = <<~SQL
|
140
|
+
SELECT pg_get_triggerdef(oid) as tdef FROM pg_trigger
|
141
|
+
WHERE tgrelid = '#{client.schema}.#{table}'::regclass AND tgisinternal = FALSE;
|
142
|
+
SQL
|
143
|
+
|
144
|
+
triggers = []
|
145
|
+
run(client.connection, query) { |result| triggers = result.map { |row| "#{row["tdef"]};" } }
|
146
|
+
|
147
|
+
triggers.join(";")
|
148
|
+
end
|
149
|
+
|
111
150
|
def get_all_constraints_for(client)
|
112
151
|
query = <<~SQL
|
113
152
|
SELECT conrelid::regclass AS table_on,
|
@@ -121,9 +160,7 @@ module PgOnlineSchemaChange
|
|
121
160
|
SQL
|
122
161
|
|
123
162
|
constraints = []
|
124
|
-
run(client.connection, query)
|
125
|
-
constraints = result.map { |row| row }
|
126
|
-
end
|
163
|
+
run(client.connection, query) { |result| constraints = result.map { |row| row } }
|
127
164
|
|
128
165
|
constraints
|
129
166
|
end
|
@@ -141,72 +178,93 @@ module PgOnlineSchemaChange
|
|
141
178
|
end
|
142
179
|
|
143
180
|
def referential_foreign_keys_to_refresh(client, table)
|
144
|
-
references =
|
145
|
-
|
146
|
-
|
181
|
+
references =
|
182
|
+
get_all_constraints_for(client).select do |row|
|
183
|
+
row["table_from"] == table && row["constraint_type"] == "f"
|
184
|
+
end
|
147
185
|
|
148
|
-
references
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
186
|
+
references
|
187
|
+
.map do |row|
|
188
|
+
add_statement =
|
189
|
+
if row["definition"].end_with?("NOT VALID")
|
190
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
|
191
|
+
else
|
192
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
|
193
|
+
end
|
154
194
|
|
155
|
-
|
195
|
+
drop_statement =
|
196
|
+
"ALTER TABLE #{row["table_on"]} DROP CONSTRAINT #{row["constraint_name"]};"
|
156
197
|
|
157
|
-
|
158
|
-
|
198
|
+
"#{drop_statement} #{add_statement}"
|
199
|
+
end
|
200
|
+
.join
|
159
201
|
end
|
160
202
|
|
161
203
|
def self_foreign_keys_to_refresh(client, table)
|
162
|
-
references =
|
163
|
-
|
164
|
-
|
204
|
+
references =
|
205
|
+
get_all_constraints_for(client).select do |row|
|
206
|
+
row["table_on"] == table && row["constraint_type"] == "f"
|
207
|
+
end
|
165
208
|
|
166
|
-
references
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
209
|
+
references
|
210
|
+
.map do |row|
|
211
|
+
add_statement =
|
212
|
+
if row["definition"].end_with?("NOT VALID")
|
213
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
|
214
|
+
else
|
215
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
|
216
|
+
end
|
217
|
+
add_statement
|
218
|
+
end
|
219
|
+
.join
|
174
220
|
end
|
175
221
|
|
176
222
|
def get_foreign_keys_to_validate(client, table)
|
177
223
|
constraints = get_all_constraints_for(client)
|
178
|
-
referential_foreign_keys =
|
179
|
-
row["table_from"] == table && row["constraint_type"] == "f"
|
180
|
-
end
|
224
|
+
referential_foreign_keys =
|
225
|
+
constraints.select { |row| row["table_from"] == table && row["constraint_type"] == "f" }
|
181
226
|
|
182
|
-
self_foreign_keys =
|
183
|
-
row["table_on"] == table && row["constraint_type"] == "f"
|
184
|
-
end
|
227
|
+
self_foreign_keys =
|
228
|
+
constraints.select { |row| row["table_on"] == table && row["constraint_type"] == "f" }
|
185
229
|
|
186
|
-
[referential_foreign_keys, self_foreign_keys].flatten
|
187
|
-
|
188
|
-
|
230
|
+
[referential_foreign_keys, self_foreign_keys].flatten
|
231
|
+
.map do |row|
|
232
|
+
"ALTER TABLE #{row["table_on"]} VALIDATE CONSTRAINT #{row["constraint_name"]};"
|
233
|
+
end
|
234
|
+
.join
|
189
235
|
end
|
190
236
|
|
191
237
|
def dropped_columns(client)
|
192
|
-
PgQuery
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
238
|
+
PgQuery
|
239
|
+
.parse(client.alter_statement)
|
240
|
+
.tree
|
241
|
+
.stmts
|
242
|
+
.map do |statement|
|
243
|
+
next if statement.stmt.alter_table_stmt.nil?
|
244
|
+
|
245
|
+
statement.stmt.alter_table_stmt.cmds.map do |cmd|
|
246
|
+
cmd.alter_table_cmd.name if cmd.alter_table_cmd.subtype == DROPPED_COLUMN_TYPE
|
247
|
+
end
|
197
248
|
end
|
198
|
-
|
249
|
+
.flatten
|
250
|
+
.compact
|
199
251
|
end
|
200
252
|
|
201
253
|
def renamed_columns(client)
|
202
|
-
PgQuery
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
254
|
+
PgQuery
|
255
|
+
.parse(client.alter_statement)
|
256
|
+
.tree
|
257
|
+
.stmts
|
258
|
+
.map do |statement|
|
259
|
+
next if statement.stmt.rename_stmt.nil?
|
260
|
+
|
261
|
+
{
|
262
|
+
old_name: statement.stmt.rename_stmt.subname,
|
263
|
+
new_name: statement.stmt.rename_stmt.newname,
|
264
|
+
}
|
265
|
+
end
|
266
|
+
.flatten
|
267
|
+
.compact
|
210
268
|
end
|
211
269
|
|
212
270
|
def primary_key_for(client, table)
|
@@ -215,9 +273,9 @@ module PgOnlineSchemaChange
|
|
215
273
|
pg_attribute.attname as column_name
|
216
274
|
FROM pg_index, pg_class, pg_attribute, pg_namespace
|
217
275
|
WHERE
|
218
|
-
pg_class.oid =
|
276
|
+
pg_class.oid = '#{table}'::regclass AND
|
219
277
|
indrelid = pg_class.oid AND
|
220
|
-
nspname =
|
278
|
+
nspname = '#{client.schema}' AND
|
221
279
|
pg_class.relnamespace = pg_namespace.oid AND
|
222
280
|
pg_attribute.attrelid = pg_class.oid AND
|
223
281
|
pg_attribute.attnum = any(pg_index.indkey)
|
@@ -225,16 +283,14 @@ module PgOnlineSchemaChange
|
|
225
283
|
SQL
|
226
284
|
|
227
285
|
columns = []
|
228
|
-
run(client.connection, query)
|
229
|
-
columns = result.map { |row| row["column_name"] }
|
230
|
-
end
|
286
|
+
run(client.connection, query) { |result| columns = result.map { |row| row["column_name"] } }
|
231
287
|
|
232
288
|
columns.first
|
233
289
|
end
|
234
290
|
|
235
291
|
def storage_parameters_for(client, table, reuse_trasaction = false)
|
236
292
|
query = <<~SQL
|
237
|
-
SELECT array_to_string(reloptions, ',') as params FROM pg_class WHERE relname
|
293
|
+
SELECT array_to_string(reloptions, ',') as params FROM pg_class WHERE relname='#{table}';
|
238
294
|
SQL
|
239
295
|
|
240
296
|
columns = []
|
@@ -255,7 +311,7 @@ module PgOnlineSchemaChange
|
|
255
311
|
|
256
312
|
query = <<~SQL
|
257
313
|
SET lock_timeout = '#{client.wait_time_for_lock}s';
|
258
|
-
LOCK TABLE #{client.
|
314
|
+
LOCK TABLE #{client.table_name} IN ACCESS EXCLUSIVE MODE;
|
259
315
|
SQL
|
260
316
|
run(client.connection, query, true)
|
261
317
|
|
@@ -282,16 +338,17 @@ module PgOnlineSchemaChange
|
|
282
338
|
logger.info("Terminating other backends")
|
283
339
|
|
284
340
|
query = <<~SQL
|
285
|
-
SELECT pg_terminate_backend(pid) FROM pg_locks WHERE locktype = 'relation' AND relation =
|
341
|
+
SELECT pg_terminate_backend(pid) FROM pg_locks WHERE locktype = 'relation' AND relation = '#{table}'::regclass::oid AND pid <> pg_backend_pid()
|
286
342
|
SQL
|
287
343
|
|
288
344
|
run(client.connection, query, true)
|
289
345
|
end
|
290
346
|
|
291
347
|
def copy_data_statement(client, shadow_table, reuse_trasaction = false)
|
292
|
-
select_columns =
|
293
|
-
entry
|
294
|
-
|
348
|
+
select_columns =
|
349
|
+
table_columns(client, client.table_name, reuse_trasaction).map do |entry|
|
350
|
+
entry["column_name_regular"]
|
351
|
+
end
|
295
352
|
|
296
353
|
select_columns -= dropped_columns_list if dropped_columns_list.any?
|
297
354
|
|
@@ -309,27 +366,23 @@ module PgOnlineSchemaChange
|
|
309
366
|
client.connection.quote_ident(insert_into_column)
|
310
367
|
end
|
311
368
|
|
312
|
-
select_columns.map!
|
313
|
-
client.connection.quote_ident(select_column)
|
314
|
-
end
|
369
|
+
select_columns.map! { |select_column| client.connection.quote_ident(select_column) }
|
315
370
|
|
316
371
|
<<~SQL
|
317
372
|
INSERT INTO #{shadow_table}(#{insert_into_columns.join(", ")})
|
318
373
|
SELECT #{select_columns.join(", ")}
|
319
|
-
FROM ONLY #{client.
|
374
|
+
FROM ONLY #{client.table_name}
|
320
375
|
SQL
|
321
376
|
end
|
322
377
|
|
323
378
|
def primary_key_sequence(shadow_table, primary_key, opened)
|
324
379
|
query = <<~SQL
|
325
|
-
SELECT pg_get_serial_sequence(
|
380
|
+
SELECT pg_get_serial_sequence('#{shadow_table}', '#{primary_key}') as sequence_name
|
326
381
|
SQL
|
327
382
|
|
328
383
|
result = run(client.connection, query, opened)
|
329
384
|
|
330
|
-
result.map
|
331
|
-
row["sequence_name"]
|
332
|
-
end&.first
|
385
|
+
result.map { |row| row["sequence_name"] }&.first
|
333
386
|
end
|
334
387
|
|
335
388
|
def query_for_primary_key_refresh(shadow_table, primary_key, table, opened)
|
@@ -338,7 +391,7 @@ module PgOnlineSchemaChange
|
|
338
391
|
return "" if sequence_name.nil?
|
339
392
|
|
340
393
|
<<~SQL
|
341
|
-
SELECT setval((select pg_get_serial_sequence(
|
394
|
+
SELECT setval((select pg_get_serial_sequence('#{shadow_table}', '#{primary_key}')), (SELECT max(#{primary_key}) FROM #{table}));
|
342
395
|
SQL
|
343
396
|
end
|
344
397
|
end
|