pg_online_schema_change 0.7.4 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.prettierignore +4 -0
- data/.prettierrc +13 -0
- data/.rubocop.yml +204 -75
- data/.rubocop_todo.yml +11 -25
- data/.ruby-version +1 -1
- data/CHANGELOG.md +38 -20
- data/CODE_OF_CONDUCT.md +11 -11
- data/Gemfile.lock +73 -42
- data/README.md +40 -24
- data/Rakefile +1 -1
- data/docker-compose.yml +2 -2
- data/docs/load-test.md +11 -10
- data/lib/pg_online_schema_change/cli.rb +87 -23
- data/lib/pg_online_schema_change/client.rb +24 -12
- data/lib/pg_online_schema_change/helper.rb +1 -1
- data/lib/pg_online_schema_change/orchestrate.rb +78 -40
- data/lib/pg_online_schema_change/query.rb +148 -95
- data/lib/pg_online_schema_change/replay.rb +10 -20
- data/lib/pg_online_schema_change/version.rb +1 -1
- data/lib/pg_online_schema_change.rb +15 -9
- data/package.json +13 -0
- data/yarn.lock +15 -0
- metadata +96 -36
@@ -30,16 +30,23 @@ module PgOnlineSchemaChange
|
|
30
30
|
# Set this early on to ensure their creation and cleanup (unexpected)
|
31
31
|
# happens at all times. IOW, the calls from Store.get always return
|
32
32
|
# the same value.
|
33
|
-
Store.set(:old_primary_table, "pgosc_op_table_#{client.table}")
|
34
|
-
Store.set(:audit_table, "pgosc_at_#{client.table}_#{pgosc_identifier}")
|
33
|
+
Store.set(:old_primary_table, "pgosc_op_table_#{client.table.downcase}")
|
34
|
+
Store.set(:audit_table, "pgosc_at_#{client.table.downcase}_#{pgosc_identifier}")
|
35
35
|
Store.set(:operation_type_column, "operation_type_#{pgosc_identifier}")
|
36
36
|
Store.set(:trigger_time_column, "trigger_time_#{pgosc_identifier}")
|
37
37
|
Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
|
38
38
|
Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
|
39
|
-
Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
|
40
|
-
|
41
|
-
Store.set(
|
42
|
-
|
39
|
+
Store.set(:shadow_table, "pgosc_st_#{client.table.downcase}_#{pgosc_identifier}")
|
40
|
+
|
41
|
+
Store.set(
|
42
|
+
:referential_foreign_key_statements,
|
43
|
+
Query.referential_foreign_keys_to_refresh(client, client.table_name),
|
44
|
+
)
|
45
|
+
Store.set(
|
46
|
+
:self_foreign_key_statements,
|
47
|
+
Query.self_foreign_keys_to_refresh(client, client.table_name),
|
48
|
+
)
|
49
|
+
Store.set(:trigger_statements, Query.get_triggers_for(client, client.table_name))
|
43
50
|
end
|
44
51
|
|
45
52
|
def run!(options)
|
@@ -49,6 +56,7 @@ module PgOnlineSchemaChange
|
|
49
56
|
raise Error, "Parent table has no primary key, exiting..." if primary_key.nil?
|
50
57
|
|
51
58
|
setup_audit_table!
|
59
|
+
|
52
60
|
setup_trigger!
|
53
61
|
setup_shadow_table! # re-uses transaction with serializable
|
54
62
|
disable_vacuum! # re-uses transaction with serializable
|
@@ -72,9 +80,7 @@ module PgOnlineSchemaChange
|
|
72
80
|
def setup_signals!
|
73
81
|
reader, writer = IO.pipe
|
74
82
|
|
75
|
-
|
76
|
-
trap(sig) { writer.puts sig }
|
77
|
-
end
|
83
|
+
['TERM', 'QUIT', 'INT'].each { |sig| trap(sig) { writer.puts sig } }
|
78
84
|
|
79
85
|
reader
|
80
86
|
end
|
@@ -83,14 +89,14 @@ module PgOnlineSchemaChange
|
|
83
89
|
reader = setup_signals!
|
84
90
|
signal = reader.gets.chomp
|
85
91
|
|
86
|
-
while !reader.closed? &&
|
87
|
-
logger.info
|
92
|
+
while !reader.closed? && reader.wait_readable # rubocop:disable Lint/UnreachableLoop
|
93
|
+
logger.info("Signal #{signal} received, cleaning up")
|
88
94
|
|
89
95
|
client.connection.cancel
|
90
96
|
drop_and_cleanup!
|
91
97
|
reader.close
|
92
98
|
|
93
|
-
exit
|
99
|
+
exit(Signal.list[signal])
|
94
100
|
end
|
95
101
|
end
|
96
102
|
|
@@ -98,7 +104,7 @@ module PgOnlineSchemaChange
|
|
98
104
|
logger.info("Setting up audit table", { audit_table: audit_table })
|
99
105
|
|
100
106
|
sql = <<~SQL
|
101
|
-
CREATE TABLE #{audit_table} (#{audit_table_pk} SERIAL PRIMARY KEY, #{operation_type_column} text, #{trigger_time_column} timestamp, LIKE #{client.
|
107
|
+
CREATE TABLE #{audit_table} (#{audit_table_pk} SERIAL PRIMARY KEY, #{operation_type_column} text, #{trigger_time_column} timestamp, LIKE #{client.table_name});
|
102
108
|
SQL
|
103
109
|
|
104
110
|
Query.run(client.connection, sql)
|
@@ -108,34 +114,34 @@ module PgOnlineSchemaChange
|
|
108
114
|
# acquire access exclusive lock to ensure audit triggers
|
109
115
|
# are setup fine. This also calls kill_backends (if opted in via flag)
|
110
116
|
# so any competing backends will be killed to setup the trigger
|
111
|
-
opened = Query.open_lock_exclusive(client, client.
|
117
|
+
opened = Query.open_lock_exclusive(client, client.table_name)
|
112
118
|
|
113
119
|
raise AccessExclusiveLockNotAcquired unless opened
|
114
120
|
|
115
121
|
logger.info("Setting up triggers")
|
116
122
|
|
117
123
|
sql = <<~SQL
|
118
|
-
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.
|
124
|
+
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table_name};
|
119
125
|
|
120
126
|
CREATE OR REPLACE FUNCTION primary_to_audit_table_trigger()
|
121
127
|
RETURNS TRIGGER AS
|
122
128
|
$$
|
123
129
|
BEGIN
|
124
130
|
IF ( TG_OP = 'INSERT') THEN
|
125
|
-
INSERT INTO
|
131
|
+
INSERT INTO "#{audit_table}" select nextval('#{audit_table_pk_sequence}'), 'INSERT', clock_timestamp(), NEW.* ;
|
126
132
|
RETURN NEW;
|
127
133
|
ELSIF ( TG_OP = 'UPDATE') THEN
|
128
|
-
INSERT INTO
|
134
|
+
INSERT INTO "#{audit_table}" select nextval('#{audit_table_pk_sequence}'), 'UPDATE', clock_timestamp(), NEW.* ;
|
129
135
|
RETURN NEW;
|
130
136
|
ELSIF ( TG_OP = 'DELETE') THEN
|
131
|
-
INSERT INTO
|
137
|
+
INSERT INTO "#{audit_table}" select nextval('#{audit_table_pk_sequence}'), 'DELETE', clock_timestamp(), OLD.* ;
|
132
138
|
RETURN NEW;
|
133
139
|
END IF;
|
134
140
|
END;
|
135
141
|
$$ LANGUAGE PLPGSQL SECURITY DEFINER;
|
136
142
|
|
137
143
|
CREATE TRIGGER primary_to_audit_table_trigger
|
138
|
-
AFTER INSERT OR UPDATE OR DELETE ON #{client.
|
144
|
+
AFTER INSERT OR UPDATE OR DELETE ON #{client.table_name}
|
139
145
|
FOR EACH ROW EXECUTE PROCEDURE primary_to_audit_table_trigger();
|
140
146
|
SQL
|
141
147
|
|
@@ -156,20 +162,30 @@ module PgOnlineSchemaChange
|
|
156
162
|
Query.run(client.connection, "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", true)
|
157
163
|
logger.info("Setting up shadow table", { shadow_table: shadow_table })
|
158
164
|
|
159
|
-
Query.run(
|
165
|
+
Query.run(
|
166
|
+
client.connection,
|
167
|
+
"SELECT create_table_all('#{client.table_name}', '#{shadow_table}');",
|
168
|
+
true,
|
169
|
+
)
|
160
170
|
|
161
171
|
# update serials
|
162
|
-
Query.run(
|
172
|
+
Query.run(
|
173
|
+
client.connection,
|
174
|
+
"SELECT fix_serial_sequence('#{client.table_name}', '#{shadow_table}');",
|
175
|
+
true,
|
176
|
+
)
|
163
177
|
end
|
164
178
|
|
165
179
|
def disable_vacuum!
|
166
180
|
# re-uses transaction with serializable
|
167
181
|
# Disabling vacuum to avoid any issues during the process
|
168
|
-
result = Query.storage_parameters_for(client, client.
|
182
|
+
result = Query.storage_parameters_for(client, client.table_name, true) || ""
|
169
183
|
Store.set(:primary_table_storage_parameters, result)
|
170
184
|
|
171
|
-
logger.debug(
|
172
|
-
|
185
|
+
logger.debug(
|
186
|
+
"Disabling vacuum on shadow and audit table",
|
187
|
+
{ shadow_table: shadow_table, audit_table: audit_table },
|
188
|
+
)
|
173
189
|
sql = <<~SQL
|
174
190
|
ALTER TABLE #{shadow_table} SET (
|
175
191
|
autovacuum_enabled = false, toast.autovacuum_enabled = false
|
@@ -185,8 +201,10 @@ module PgOnlineSchemaChange
|
|
185
201
|
def run_alter_statement!
|
186
202
|
# re-uses transaction with serializable
|
187
203
|
statement = Query.alter_statement_for(client, shadow_table)
|
188
|
-
logger.info(
|
189
|
-
|
204
|
+
logger.info(
|
205
|
+
"Running alter statement on shadow table",
|
206
|
+
{ shadow_table: shadow_table, parent_table: client.table_name },
|
207
|
+
)
|
190
208
|
Query.run(client.connection, statement, true)
|
191
209
|
|
192
210
|
Store.set(:dropped_columns_list, Query.dropped_columns(client))
|
@@ -198,10 +216,16 @@ module PgOnlineSchemaChange
|
|
198
216
|
# Begin the process to copy data into copy table
|
199
217
|
# depending on the size of the table, this can be a time
|
200
218
|
# taking operation.
|
201
|
-
logger.info(
|
219
|
+
logger.info(
|
220
|
+
"Clearing contents of audit table before copy..",
|
221
|
+
{ shadow_table: shadow_table, parent_table: client.table_name },
|
222
|
+
)
|
202
223
|
Query.run(client.connection, "DELETE FROM #{audit_table}", true)
|
203
224
|
|
204
|
-
logger.info(
|
225
|
+
logger.info(
|
226
|
+
"Copying contents..",
|
227
|
+
{ shadow_table: shadow_table, parent_table: client.table_name },
|
228
|
+
)
|
205
229
|
if client.copy_statement
|
206
230
|
query = format(client.copy_statement, shadow_table: shadow_table)
|
207
231
|
return Query.run(client.connection, query, true)
|
@@ -223,31 +247,34 @@ module PgOnlineSchemaChange
|
|
223
247
|
|
224
248
|
def swap!
|
225
249
|
logger.info("Performing swap!")
|
226
|
-
|
227
|
-
storage_params_reset = primary_table_storage_parameters.empty? ? "" : "ALTER TABLE #{client.table} SET (#{primary_table_storage_parameters});"
|
228
|
-
|
229
250
|
# From here on, all statements are carried out in a single
|
230
251
|
# transaction with access exclusive lock
|
231
252
|
|
232
|
-
opened = Query.open_lock_exclusive(client, client.
|
253
|
+
opened = Query.open_lock_exclusive(client, client.table_name)
|
233
254
|
|
234
255
|
raise AccessExclusiveLockNotAcquired unless opened
|
235
256
|
|
236
|
-
Query.run(
|
257
|
+
Query.run(
|
258
|
+
client.connection,
|
259
|
+
"SET statement_timeout to '#{SWAP_STATEMENT_TIMEOUT}';",
|
260
|
+
opened,
|
261
|
+
)
|
237
262
|
|
238
263
|
rows = Replay.rows_to_play(opened)
|
239
264
|
Replay.play!(rows, opened)
|
240
265
|
|
241
|
-
query_for_primary_key_refresh =
|
266
|
+
query_for_primary_key_refresh =
|
267
|
+
Query.query_for_primary_key_refresh(shadow_table, primary_key, client.table_name, opened)
|
242
268
|
|
243
269
|
sql = <<~SQL
|
244
270
|
#{query_for_primary_key_refresh};
|
245
|
-
ALTER TABLE #{client.
|
246
|
-
ALTER TABLE #{shadow_table} RENAME to #{client.
|
271
|
+
ALTER TABLE #{client.table_name} RENAME to #{old_primary_table};
|
272
|
+
ALTER TABLE #{shadow_table} RENAME to #{client.table_name};
|
247
273
|
#{referential_foreign_key_statements}
|
248
274
|
#{self_foreign_key_statements}
|
275
|
+
#{trigger_statements}
|
249
276
|
#{storage_params_reset}
|
250
|
-
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.
|
277
|
+
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table_name};
|
251
278
|
SQL
|
252
279
|
|
253
280
|
Query.run(client.connection, sql, opened)
|
@@ -259,13 +286,13 @@ module PgOnlineSchemaChange
|
|
259
286
|
def run_analyze!
|
260
287
|
logger.info("Performing ANALYZE!")
|
261
288
|
|
262
|
-
Query.run(client.connection, "ANALYZE VERBOSE #{client.
|
289
|
+
Query.run(client.connection, "ANALYZE VERBOSE #{client.table_name};")
|
263
290
|
end
|
264
291
|
|
265
292
|
def validate_constraints!
|
266
293
|
logger.info("Validating constraints!")
|
267
294
|
|
268
|
-
validate_statements = Query.get_foreign_keys_to_validate(client, client.
|
295
|
+
validate_statements = Query.get_foreign_keys_to_validate(client, client.table_name)
|
269
296
|
|
270
297
|
Query.run(client.connection, validate_statements)
|
271
298
|
end
|
@@ -276,7 +303,7 @@ module PgOnlineSchemaChange
|
|
276
303
|
shadow_table_drop = shadow_table ? "DROP TABLE IF EXISTS #{shadow_table}" : ""
|
277
304
|
|
278
305
|
sql = <<~SQL
|
279
|
-
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.
|
306
|
+
DROP TRIGGER IF EXISTS primary_to_audit_table_trigger ON #{client.table_name};
|
280
307
|
#{audit_table_drop};
|
281
308
|
#{shadow_table_drop};
|
282
309
|
#{primary_drop}
|
@@ -293,6 +320,17 @@ module PgOnlineSchemaChange
|
|
293
320
|
def pgosc_identifier
|
294
321
|
@pgosc_identifier ||= SecureRandom.hex(3)
|
295
322
|
end
|
323
|
+
|
324
|
+
def storage_params_reset
|
325
|
+
"ALTER TABLE #{client.table_name} RESET (autovacuum_enabled, toast.autovacuum_enabled);" +
|
326
|
+
(
|
327
|
+
if primary_table_storage_parameters.empty?
|
328
|
+
""
|
329
|
+
else
|
330
|
+
"ALTER TABLE #{client.table_name} SET (#{primary_table_storage_parameters});"
|
331
|
+
end
|
332
|
+
)
|
333
|
+
end
|
296
334
|
end
|
297
335
|
end
|
298
336
|
end
|
@@ -14,21 +14,31 @@ module PgOnlineSchemaChange
|
|
14
14
|
|
15
15
|
class << self
|
16
16
|
def alter_statement?(query)
|
17
|
-
PgQuery
|
18
|
-
|
19
|
-
|
17
|
+
PgQuery
|
18
|
+
.parse(query)
|
19
|
+
.tree
|
20
|
+
.stmts
|
21
|
+
.all? do |statement|
|
22
|
+
statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt) ||
|
23
|
+
statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
|
24
|
+
end
|
20
25
|
rescue PgQuery::ParseError
|
21
26
|
false
|
22
27
|
end
|
23
28
|
|
24
29
|
def same_table?(query)
|
25
|
-
tables =
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
30
|
+
tables =
|
31
|
+
PgQuery
|
32
|
+
.parse(query)
|
33
|
+
.tree
|
34
|
+
.stmts
|
35
|
+
.filter_map do |statement|
|
36
|
+
if statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt)
|
37
|
+
statement.stmt.alter_table_stmt.relation.relname
|
38
|
+
elsif statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
|
39
|
+
statement.stmt.rename_stmt.relation.relname
|
40
|
+
end
|
41
|
+
end
|
32
42
|
|
33
43
|
tables.uniq.count == 1
|
34
44
|
rescue PgQuery::ParseError
|
@@ -36,14 +46,30 @@ module PgOnlineSchemaChange
|
|
36
46
|
end
|
37
47
|
|
38
48
|
def table(query)
|
39
|
-
from_rename_statement =
|
40
|
-
|
41
|
-
|
49
|
+
from_rename_statement =
|
50
|
+
PgQuery
|
51
|
+
.parse(query)
|
52
|
+
.tree
|
53
|
+
.stmts
|
54
|
+
.filter_map { |statement| statement.stmt.rename_stmt&.relation&.relname }[
|
55
|
+
0
|
56
|
+
]
|
42
57
|
PgQuery.parse(query).tables[0] || from_rename_statement
|
43
58
|
end
|
44
59
|
|
60
|
+
def table_name(query, table)
|
61
|
+
table_name = "\"#{table}\""
|
62
|
+
if table =~ /[A-Z]/ && query.include?(table_name) && table[0] != '"'
|
63
|
+
table_name
|
64
|
+
else
|
65
|
+
table
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
45
69
|
def run(connection, query, reuse_trasaction = false, &block)
|
46
|
-
|
70
|
+
if [PG::PQTRANS_INERROR, PG::PQTRANS_UNKNOWN].include?(connection.transaction_status)
|
71
|
+
connection.cancel
|
72
|
+
end
|
47
73
|
|
48
74
|
logger.debug("Running query", { query: query })
|
49
75
|
|
@@ -65,18 +91,19 @@ module PgOnlineSchemaChange
|
|
65
91
|
def table_columns(client, table = nil, reuse_trasaction = false)
|
66
92
|
sql = <<~SQL
|
67
93
|
SELECT attname as column_name, format_type(atttypid, atttypmod) as type, attnum as column_position FROM pg_attribute
|
68
|
-
WHERE attrelid =
|
94
|
+
WHERE attrelid = '#{table || client.table_name}'::regclass AND attnum > 0 AND NOT attisdropped
|
69
95
|
ORDER BY attnum;
|
70
96
|
SQL
|
71
97
|
mapped_columns = []
|
72
98
|
|
73
99
|
run(client.connection, sql, reuse_trasaction) do |result|
|
74
|
-
mapped_columns =
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
100
|
+
mapped_columns =
|
101
|
+
result.map do |row|
|
102
|
+
row["column_name_regular"] = row["column_name"]
|
103
|
+
row["column_name"] = client.connection.quote_ident(row["column_name"])
|
104
|
+
row["column_position"] = row["column_position"].to_i
|
105
|
+
row
|
106
|
+
end
|
80
107
|
end
|
81
108
|
|
82
109
|
mapped_columns
|
@@ -86,7 +113,9 @@ module PgOnlineSchemaChange
|
|
86
113
|
parsed_query = PgQuery.parse(client.alter_statement)
|
87
114
|
|
88
115
|
parsed_query.tree.stmts.each do |statement|
|
89
|
-
|
116
|
+
if statement.stmt.alter_table_stmt
|
117
|
+
statement.stmt.alter_table_stmt.relation.relname = shadow_table
|
118
|
+
end
|
90
119
|
|
91
120
|
statement.stmt.rename_stmt.relation.relname = shadow_table if statement.stmt.rename_stmt
|
92
121
|
end
|
@@ -97,17 +126,27 @@ module PgOnlineSchemaChange
|
|
97
126
|
query = <<~SQL
|
98
127
|
SELECT indexdef, schemaname
|
99
128
|
FROM pg_indexes
|
100
|
-
WHERE schemaname =
|
129
|
+
WHERE schemaname = '#{client.schema}' AND tablename = '#{table}'
|
101
130
|
SQL
|
102
131
|
|
103
132
|
indexes = []
|
104
|
-
run(client.connection, query)
|
105
|
-
indexes = result.map { |row| row["indexdef"] }
|
106
|
-
end
|
133
|
+
run(client.connection, query) { |result| indexes = result.map { |row| row["indexdef"] } }
|
107
134
|
|
108
135
|
indexes
|
109
136
|
end
|
110
137
|
|
138
|
+
def get_triggers_for(client, table)
|
139
|
+
query = <<~SQL
|
140
|
+
SELECT pg_get_triggerdef(oid) as tdef FROM pg_trigger
|
141
|
+
WHERE tgrelid = '#{client.schema}.#{table}'::regclass AND tgisinternal = FALSE;
|
142
|
+
SQL
|
143
|
+
|
144
|
+
triggers = []
|
145
|
+
run(client.connection, query) { |result| triggers = result.map { |row| "#{row["tdef"]};" } }
|
146
|
+
|
147
|
+
triggers.join(";")
|
148
|
+
end
|
149
|
+
|
111
150
|
def get_all_constraints_for(client)
|
112
151
|
query = <<~SQL
|
113
152
|
SELECT conrelid::regclass AS table_on,
|
@@ -121,9 +160,7 @@ module PgOnlineSchemaChange
|
|
121
160
|
SQL
|
122
161
|
|
123
162
|
constraints = []
|
124
|
-
run(client.connection, query)
|
125
|
-
constraints = result.map { |row| row }
|
126
|
-
end
|
163
|
+
run(client.connection, query) { |result| constraints = result.map { |row| row } }
|
127
164
|
|
128
165
|
constraints
|
129
166
|
end
|
@@ -141,72 +178,93 @@ module PgOnlineSchemaChange
|
|
141
178
|
end
|
142
179
|
|
143
180
|
def referential_foreign_keys_to_refresh(client, table)
|
144
|
-
references =
|
145
|
-
|
146
|
-
|
181
|
+
references =
|
182
|
+
get_all_constraints_for(client).select do |row|
|
183
|
+
row["table_from"] == table && row["constraint_type"] == "f"
|
184
|
+
end
|
147
185
|
|
148
|
-
references
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
186
|
+
references
|
187
|
+
.map do |row|
|
188
|
+
add_statement =
|
189
|
+
if row["definition"].end_with?("NOT VALID")
|
190
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
|
191
|
+
else
|
192
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
|
193
|
+
end
|
154
194
|
|
155
|
-
|
195
|
+
drop_statement =
|
196
|
+
"ALTER TABLE #{row["table_on"]} DROP CONSTRAINT #{row["constraint_name"]};"
|
156
197
|
|
157
|
-
|
158
|
-
|
198
|
+
"#{drop_statement} #{add_statement}"
|
199
|
+
end
|
200
|
+
.join
|
159
201
|
end
|
160
202
|
|
161
203
|
def self_foreign_keys_to_refresh(client, table)
|
162
|
-
references =
|
163
|
-
|
164
|
-
|
204
|
+
references =
|
205
|
+
get_all_constraints_for(client).select do |row|
|
206
|
+
row["table_on"] == table && row["constraint_type"] == "f"
|
207
|
+
end
|
165
208
|
|
166
|
-
references
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
209
|
+
references
|
210
|
+
.map do |row|
|
211
|
+
add_statement =
|
212
|
+
if row["definition"].end_with?("NOT VALID")
|
213
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
|
214
|
+
else
|
215
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
|
216
|
+
end
|
217
|
+
add_statement
|
218
|
+
end
|
219
|
+
.join
|
174
220
|
end
|
175
221
|
|
176
222
|
def get_foreign_keys_to_validate(client, table)
|
177
223
|
constraints = get_all_constraints_for(client)
|
178
|
-
referential_foreign_keys =
|
179
|
-
row["table_from"] == table && row["constraint_type"] == "f"
|
180
|
-
end
|
224
|
+
referential_foreign_keys =
|
225
|
+
constraints.select { |row| row["table_from"] == table && row["constraint_type"] == "f" }
|
181
226
|
|
182
|
-
self_foreign_keys =
|
183
|
-
row["table_on"] == table && row["constraint_type"] == "f"
|
184
|
-
end
|
227
|
+
self_foreign_keys =
|
228
|
+
constraints.select { |row| row["table_on"] == table && row["constraint_type"] == "f" }
|
185
229
|
|
186
|
-
[referential_foreign_keys, self_foreign_keys].flatten
|
187
|
-
|
188
|
-
|
230
|
+
[referential_foreign_keys, self_foreign_keys].flatten
|
231
|
+
.map do |row|
|
232
|
+
"ALTER TABLE #{row["table_on"]} VALIDATE CONSTRAINT #{row["constraint_name"]};"
|
233
|
+
end
|
234
|
+
.join
|
189
235
|
end
|
190
236
|
|
191
237
|
def dropped_columns(client)
|
192
|
-
PgQuery
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
238
|
+
PgQuery
|
239
|
+
.parse(client.alter_statement)
|
240
|
+
.tree
|
241
|
+
.stmts
|
242
|
+
.map do |statement|
|
243
|
+
next if statement.stmt.alter_table_stmt.nil?
|
244
|
+
|
245
|
+
statement.stmt.alter_table_stmt.cmds.map do |cmd|
|
246
|
+
cmd.alter_table_cmd.name if cmd.alter_table_cmd.subtype == DROPPED_COLUMN_TYPE
|
247
|
+
end
|
197
248
|
end
|
198
|
-
|
249
|
+
.flatten
|
250
|
+
.compact
|
199
251
|
end
|
200
252
|
|
201
253
|
def renamed_columns(client)
|
202
|
-
PgQuery
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
254
|
+
PgQuery
|
255
|
+
.parse(client.alter_statement)
|
256
|
+
.tree
|
257
|
+
.stmts
|
258
|
+
.map do |statement|
|
259
|
+
next if statement.stmt.rename_stmt.nil?
|
260
|
+
|
261
|
+
{
|
262
|
+
old_name: statement.stmt.rename_stmt.subname,
|
263
|
+
new_name: statement.stmt.rename_stmt.newname,
|
264
|
+
}
|
265
|
+
end
|
266
|
+
.flatten
|
267
|
+
.compact
|
210
268
|
end
|
211
269
|
|
212
270
|
def primary_key_for(client, table)
|
@@ -215,9 +273,9 @@ module PgOnlineSchemaChange
|
|
215
273
|
pg_attribute.attname as column_name
|
216
274
|
FROM pg_index, pg_class, pg_attribute, pg_namespace
|
217
275
|
WHERE
|
218
|
-
pg_class.oid =
|
276
|
+
pg_class.oid = '#{table}'::regclass AND
|
219
277
|
indrelid = pg_class.oid AND
|
220
|
-
nspname =
|
278
|
+
nspname = '#{client.schema}' AND
|
221
279
|
pg_class.relnamespace = pg_namespace.oid AND
|
222
280
|
pg_attribute.attrelid = pg_class.oid AND
|
223
281
|
pg_attribute.attnum = any(pg_index.indkey)
|
@@ -225,16 +283,14 @@ module PgOnlineSchemaChange
|
|
225
283
|
SQL
|
226
284
|
|
227
285
|
columns = []
|
228
|
-
run(client.connection, query)
|
229
|
-
columns = result.map { |row| row["column_name"] }
|
230
|
-
end
|
286
|
+
run(client.connection, query) { |result| columns = result.map { |row| row["column_name"] } }
|
231
287
|
|
232
288
|
columns.first
|
233
289
|
end
|
234
290
|
|
235
291
|
def storage_parameters_for(client, table, reuse_trasaction = false)
|
236
292
|
query = <<~SQL
|
237
|
-
SELECT array_to_string(reloptions, ',') as params FROM pg_class WHERE relname
|
293
|
+
SELECT array_to_string(reloptions, ',') as params FROM pg_class WHERE relname='#{table}';
|
238
294
|
SQL
|
239
295
|
|
240
296
|
columns = []
|
@@ -255,7 +311,7 @@ module PgOnlineSchemaChange
|
|
255
311
|
|
256
312
|
query = <<~SQL
|
257
313
|
SET lock_timeout = '#{client.wait_time_for_lock}s';
|
258
|
-
LOCK TABLE #{client.
|
314
|
+
LOCK TABLE #{client.table_name} IN ACCESS EXCLUSIVE MODE;
|
259
315
|
SQL
|
260
316
|
run(client.connection, query, true)
|
261
317
|
|
@@ -282,16 +338,17 @@ module PgOnlineSchemaChange
|
|
282
338
|
logger.info("Terminating other backends")
|
283
339
|
|
284
340
|
query = <<~SQL
|
285
|
-
SELECT pg_terminate_backend(pid) FROM pg_locks WHERE locktype = 'relation' AND relation =
|
341
|
+
SELECT pg_terminate_backend(pid) FROM pg_locks WHERE locktype = 'relation' AND relation = '#{table}'::regclass::oid AND pid <> pg_backend_pid()
|
286
342
|
SQL
|
287
343
|
|
288
344
|
run(client.connection, query, true)
|
289
345
|
end
|
290
346
|
|
291
347
|
def copy_data_statement(client, shadow_table, reuse_trasaction = false)
|
292
|
-
select_columns =
|
293
|
-
entry
|
294
|
-
|
348
|
+
select_columns =
|
349
|
+
table_columns(client, client.table_name, reuse_trasaction).map do |entry|
|
350
|
+
entry["column_name_regular"]
|
351
|
+
end
|
295
352
|
|
296
353
|
select_columns -= dropped_columns_list if dropped_columns_list.any?
|
297
354
|
|
@@ -309,27 +366,23 @@ module PgOnlineSchemaChange
|
|
309
366
|
client.connection.quote_ident(insert_into_column)
|
310
367
|
end
|
311
368
|
|
312
|
-
select_columns.map!
|
313
|
-
client.connection.quote_ident(select_column)
|
314
|
-
end
|
369
|
+
select_columns.map! { |select_column| client.connection.quote_ident(select_column) }
|
315
370
|
|
316
371
|
<<~SQL
|
317
372
|
INSERT INTO #{shadow_table}(#{insert_into_columns.join(", ")})
|
318
373
|
SELECT #{select_columns.join(", ")}
|
319
|
-
FROM ONLY #{client.
|
374
|
+
FROM ONLY #{client.table_name}
|
320
375
|
SQL
|
321
376
|
end
|
322
377
|
|
323
378
|
def primary_key_sequence(shadow_table, primary_key, opened)
|
324
379
|
query = <<~SQL
|
325
|
-
SELECT pg_get_serial_sequence(
|
380
|
+
SELECT pg_get_serial_sequence('#{shadow_table}', '#{primary_key}') as sequence_name
|
326
381
|
SQL
|
327
382
|
|
328
383
|
result = run(client.connection, query, opened)
|
329
384
|
|
330
|
-
result.map
|
331
|
-
row["sequence_name"]
|
332
|
-
end&.first
|
385
|
+
result.map { |row| row["sequence_name"] }&.first
|
333
386
|
end
|
334
387
|
|
335
388
|
def query_for_primary_key_refresh(shadow_table, primary_key, table, opened)
|
@@ -338,7 +391,7 @@ module PgOnlineSchemaChange
|
|
338
391
|
return "" if sequence_name.nil?
|
339
392
|
|
340
393
|
<<~SQL
|
341
|
-
SELECT setval((select pg_get_serial_sequence(
|
394
|
+
SELECT setval((select pg_get_serial_sequence('#{shadow_table}', '#{primary_key}')), (SELECT max(#{primary_key}) FROM #{table}));
|
342
395
|
SQL
|
343
396
|
end
|
344
397
|
end
|