pg_online_schema_change 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +57 -24
- data/.rubocop_todo.yml +44 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile.lock +1 -1
- data/lib/pg_online_schema_change/cli.rb +3 -1
- data/lib/pg_online_schema_change/client.rb +12 -6
- data/lib/pg_online_schema_change/functions.rb +4 -2
- data/lib/pg_online_schema_change/helper.rb +10 -1
- data/lib/pg_online_schema_change/orchestrate.rb +24 -13
- data/lib/pg_online_schema_change/query.rb +16 -14
- data/lib/pg_online_schema_change/replay.rb +19 -10
- data/lib/pg_online_schema_change/store.rb +7 -3
- data/lib/pg_online_schema_change/version.rb +1 -1
- data/lib/pg_online_schema_change.rb +7 -11
- data/scripts/release.sh +7 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8ef1069f7d159544838ec27cf518f98eb31609a0a2e1878ebd40246b10a6b534
|
4
|
+
data.tar.gz: 8e83cbac78164fb10870537bf4867020eae7cf06f4cd91930a2c8846fbc76c7c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2657c94b3b07730aa3bb282f34cd1ee95f0549d3649b39a61be3ce1b6d2ba5832a1914b6a9788778c912464cbd9773b5e967ed4e6c896127d60e62a0daa76c99
|
7
|
+
data.tar.gz: 4788523d691c25045b7f3c539f9743d880fabcbb01887ae34ec55a6c8a829804a9f4d43f619e79e0c44b1c9fff6ea88361fb5ac1111bc1f491aa17e6e43336d0
|
data/.rubocop.yml
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
2
|
+
|
1
3
|
require:
|
2
4
|
- rubocop-rspec
|
3
5
|
- rubocop-packaging
|
@@ -14,71 +16,70 @@ AllCops:
|
|
14
16
|
- "vendor/**/*"
|
15
17
|
|
16
18
|
Layout/HashAlignment:
|
17
|
-
EnforcedColonStyle:
|
18
|
-
|
19
|
-
- key
|
20
|
-
EnforcedHashRocketStyle:
|
21
|
-
- table
|
22
|
-
- key
|
19
|
+
EnforcedColonStyle: key
|
20
|
+
EnforcedHashRocketStyle: key
|
23
21
|
|
24
22
|
Layout/SpaceAroundEqualsInParameterDefault:
|
25
|
-
EnforcedStyle:
|
23
|
+
EnforcedStyle: space
|
26
24
|
|
27
25
|
Metrics/AbcSize:
|
28
|
-
|
26
|
+
Enabled: true
|
27
|
+
Max: 40
|
29
28
|
Exclude:
|
30
|
-
- "
|
29
|
+
- "spec/**/*"
|
31
30
|
|
32
31
|
Metrics/BlockLength:
|
32
|
+
Max: 100
|
33
33
|
Exclude:
|
34
34
|
- "*.gemspec"
|
35
35
|
- "Rakefile"
|
36
|
+
- "spec/**/*"
|
36
37
|
|
37
38
|
Metrics/ClassLength:
|
38
39
|
Exclude:
|
39
40
|
- "test/**/*"
|
40
41
|
|
41
42
|
Metrics/MethodLength:
|
42
|
-
Max:
|
43
|
+
Max: 30
|
43
44
|
Exclude:
|
44
45
|
- "test/**/*"
|
45
46
|
|
46
47
|
Metrics/ParameterLists:
|
47
|
-
Max:
|
48
|
+
Max: 5
|
48
49
|
|
49
50
|
Naming/MemoizedInstanceVariableName:
|
50
|
-
Enabled:
|
51
|
+
Enabled: true
|
51
52
|
|
52
53
|
Naming/VariableNumber:
|
53
|
-
Enabled:
|
54
|
-
|
55
|
-
Rake/Desc:
|
56
|
-
Enabled: false
|
54
|
+
Enabled: true
|
57
55
|
|
58
56
|
Style/BarePercentLiterals:
|
59
57
|
EnforcedStyle: percent_q
|
60
58
|
|
61
59
|
Style/ClassAndModuleChildren:
|
62
|
-
Enabled:
|
60
|
+
Enabled: true
|
63
61
|
|
64
62
|
Style/Documentation:
|
65
63
|
Enabled: false
|
66
64
|
|
67
65
|
Style/DoubleNegation:
|
68
|
-
Enabled:
|
66
|
+
Enabled: true
|
69
67
|
|
70
68
|
Style/EmptyMethod:
|
71
|
-
Enabled:
|
69
|
+
Enabled: true
|
72
70
|
|
73
71
|
Style/FrozenStringLiteralComment:
|
74
|
-
Enabled:
|
72
|
+
Enabled: true
|
75
73
|
|
76
74
|
Style/NumericPredicate:
|
77
|
-
Enabled:
|
75
|
+
Enabled: true
|
78
76
|
|
79
77
|
Style/StringLiterals:
|
80
78
|
EnforcedStyle: double_quotes
|
81
79
|
|
80
|
+
Style/StringLiteralsInInterpolation:
|
81
|
+
EnforcedStyle: double_quotes
|
82
|
+
|
82
83
|
Style/TrivialAccessors:
|
83
84
|
AllowPredicates: true
|
84
85
|
|
@@ -91,9 +92,41 @@ Style/TrailingCommaInArrayLiteral:
|
|
91
92
|
Style/TrailingCommaInHashLiteral:
|
92
93
|
EnforcedStyleForMultiline: comma
|
93
94
|
|
94
|
-
|
95
|
-
|
95
|
+
Layout/MultilineArrayBraceLayout:
|
96
|
+
Enabled: true
|
97
|
+
EnforcedStyle: symmetrical
|
96
98
|
|
97
|
-
|
99
|
+
Layout/MultilineHashBraceLayout:
|
98
100
|
Enabled: true
|
99
101
|
EnforcedStyle: symmetrical
|
102
|
+
|
103
|
+
Layout/MultilineAssignmentLayout:
|
104
|
+
Enabled: true
|
105
|
+
EnforcedStyle: same_line
|
106
|
+
|
107
|
+
Layout/FirstArrayElementIndentation:
|
108
|
+
Enabled: true
|
109
|
+
EnforcedStyle: consistent
|
110
|
+
|
111
|
+
Layout/FirstHashElementIndentation:
|
112
|
+
Enabled: true
|
113
|
+
EnforcedStyle: consistent
|
114
|
+
|
115
|
+
Layout/MultilineHashKeyLineBreaks:
|
116
|
+
Enabled: true
|
117
|
+
|
118
|
+
Layout/LineLength:
|
119
|
+
Enabled: true
|
120
|
+
Max: 250
|
121
|
+
|
122
|
+
Style/FormatStringToken:
|
123
|
+
Enabled: true
|
124
|
+
EnforcedStyle: template
|
125
|
+
|
126
|
+
RSpec/MessageSpies:
|
127
|
+
Enabled: true
|
128
|
+
EnforcedStyle: receive
|
129
|
+
|
130
|
+
RSpec/FilePath:
|
131
|
+
Enabled: true
|
132
|
+
SpecSuffixOnly: true
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config`
|
3
|
+
# on 2022-02-21 22:46:44 UTC using RuboCop version 1.23.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 2
|
10
|
+
# Configuration parameters: CountComments, CountAsOne.
|
11
|
+
Metrics/ClassLength:
|
12
|
+
Max: 233
|
13
|
+
|
14
|
+
# Offense count: 2
|
15
|
+
# Configuration parameters: IgnoredMethods.
|
16
|
+
Metrics/CyclomaticComplexity:
|
17
|
+
Max: 15
|
18
|
+
|
19
|
+
# Offense count: 2
|
20
|
+
# Configuration parameters: IgnoredMethods.
|
21
|
+
Metrics/PerceivedComplexity:
|
22
|
+
Max: 13
|
23
|
+
|
24
|
+
# Offense count: 1
|
25
|
+
Packaging/GemspecGit:
|
26
|
+
Exclude:
|
27
|
+
- 'pg_online_schema_change.gemspec'
|
28
|
+
|
29
|
+
# Offense count: 62
|
30
|
+
# Configuration parameters: CountAsOne.
|
31
|
+
RSpec/ExampleLength:
|
32
|
+
Max: 55
|
33
|
+
|
34
|
+
# Offense count: 38
|
35
|
+
RSpec/MultipleExpectations:
|
36
|
+
Max: 14
|
37
|
+
|
38
|
+
# Offense count: 6
|
39
|
+
# Configuration parameters: AllowedMethods.
|
40
|
+
# AllowedMethods: respond_to_missing?
|
41
|
+
Style/OptionalBooleanParameter:
|
42
|
+
Exclude:
|
43
|
+
- 'lib/pg_online_schema_change/query.rb'
|
44
|
+
- 'lib/pg_online_schema_change/replay.rb'
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## [0.3.0] - 2022-02-21
|
2
|
+
|
3
|
+
- Explicitly call dependencies and bump dependencies by @shayonj https://github.com/shayonj/pg-osc/pull/44
|
4
|
+
- Introduce Dockerfile and release process https://github.com/shayonj/pg-osc/pull/45
|
5
|
+
|
1
6
|
## [0.2.0] - 2022-02-17
|
2
7
|
|
3
8
|
- Use ISOLATION LEVEL SERIALIZABLE ([#42](https://github.com/shayonj/pg-osc/pull/42)) (props to @jfrost)
|
data/Gemfile.lock
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "thor"
|
2
4
|
|
3
5
|
module PgOnlineSchemaChange
|
@@ -25,7 +27,7 @@ module PgOnlineSchemaChange
|
|
25
27
|
def perform
|
26
28
|
client_options = Struct.new(*options.keys.map(&:to_sym)).new(*options.values)
|
27
29
|
|
28
|
-
PgOnlineSchemaChange.logger
|
30
|
+
PgOnlineSchemaChange.logger(verbose: client_options.verbose)
|
29
31
|
PgOnlineSchemaChange::Orchestrate.run!(client_options)
|
30
32
|
end
|
31
33
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "pg"
|
2
4
|
|
3
5
|
module PgOnlineSchemaChange
|
@@ -16,7 +18,9 @@ module PgOnlineSchemaChange
|
|
16
18
|
@drop = options.drop
|
17
19
|
@kill_backends = options.kill_backends
|
18
20
|
@wait_time_for_lock = options.wait_time_for_lock
|
21
|
+
|
19
22
|
handle_copy_statement(options.copy_statement)
|
23
|
+
handle_validations
|
20
24
|
|
21
25
|
@connection = PG.connect(
|
22
26
|
dbname: @dbname,
|
@@ -26,17 +30,19 @@ module PgOnlineSchemaChange
|
|
26
30
|
port: @port,
|
27
31
|
)
|
28
32
|
|
29
|
-
raise Error, "Not a valid ALTER statement: #{@alter_statement}" unless Query.alter_statement?(@alter_statement)
|
30
|
-
|
31
|
-
unless Query.same_table?(@alter_statement)
|
32
|
-
raise Error "All statements should belong to the same table: #{@alter_statement}"
|
33
|
-
end
|
34
|
-
|
35
33
|
@table = Query.table(@alter_statement)
|
36
34
|
|
37
35
|
PgOnlineSchemaChange.logger.debug("Connection established")
|
38
36
|
end
|
39
37
|
|
38
|
+
def handle_validations
|
39
|
+
raise Error, "Not a valid ALTER statement: #{@alter_statement}" unless Query.alter_statement?(@alter_statement)
|
40
|
+
|
41
|
+
return if Query.same_table?(@alter_statement)
|
42
|
+
|
43
|
+
raise Error "All statements should belong to the same table: #{@alter_statement}"
|
44
|
+
end
|
45
|
+
|
40
46
|
def handle_copy_statement(statement)
|
41
47
|
return if statement.nil? || statement == ""
|
42
48
|
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
FUNC_FIX_SERIAL_SEQUENCE = <<~SQL
|
2
4
|
CREATE OR REPLACE FUNCTION fix_serial_sequence(_table regclass, _newtable text)
|
3
5
|
RETURNS void AS
|
4
6
|
$func$
|
@@ -35,7 +37,7 @@ FUNC_FIX_SERIAL_SEQUENCE = <<~SQL.freeze
|
|
35
37
|
$func$ LANGUAGE plpgsql VOLATILE;
|
36
38
|
SQL
|
37
39
|
|
38
|
-
FUNC_CREATE_TABLE_ALL = <<~SQL
|
40
|
+
FUNC_CREATE_TABLE_ALL = <<~SQL
|
39
41
|
CREATE OR REPLACE FUNCTION create_table_all(source_table text, newsource_table text)
|
40
42
|
RETURNS void language plpgsql
|
41
43
|
as $$
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PgOnlineSchemaChange
|
2
4
|
module Helper
|
3
5
|
def primary_key
|
@@ -15,7 +17,14 @@ module PgOnlineSchemaChange
|
|
15
17
|
result = Store.send(:get, method)
|
16
18
|
return result if result
|
17
19
|
|
18
|
-
|
20
|
+
super
|
21
|
+
end
|
22
|
+
|
23
|
+
def respond_to_missing?(method_name, *args)
|
24
|
+
result = Store.send(:get, method)
|
25
|
+
return true if result
|
26
|
+
|
27
|
+
super
|
19
28
|
end
|
20
29
|
end
|
21
30
|
end
|
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "securerandom"
|
2
4
|
|
3
5
|
module PgOnlineSchemaChange
|
4
6
|
class Orchestrate
|
5
|
-
SWAP_STATEMENT_TIMEOUT = "5s"
|
7
|
+
SWAP_STATEMENT_TIMEOUT = "5s"
|
6
8
|
|
7
9
|
extend Helper
|
8
10
|
|
@@ -21,12 +23,20 @@ module PgOnlineSchemaChange
|
|
21
23
|
Query.run(client.connection, FUNC_FIX_SERIAL_SEQUENCE)
|
22
24
|
Query.run(client.connection, FUNC_CREATE_TABLE_ALL)
|
23
25
|
|
26
|
+
setup_store
|
27
|
+
end
|
28
|
+
|
29
|
+
def setup_store
|
24
30
|
# Set this early on to ensure their creation and cleanup (unexpected)
|
25
31
|
# happens at all times. IOW, the calls from Store.get always return
|
26
32
|
# the same value.
|
27
33
|
Store.set(:old_primary_table, "pgosc_op_table_#{client.table}")
|
28
|
-
Store.set(:audit_table, "pgosc_at_#{client.table}_#{
|
29
|
-
Store.set(:
|
34
|
+
Store.set(:audit_table, "pgosc_at_#{client.table}_#{pgosc_identifier}")
|
35
|
+
Store.set(:operation_type_column, "operation_type_#{pgosc_identifier}")
|
36
|
+
Store.set(:trigger_time_column, "trigger_time_#{pgosc_identifier}")
|
37
|
+
Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
|
38
|
+
Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
|
39
|
+
Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
|
30
40
|
end
|
31
41
|
|
32
42
|
def run!(options)
|
@@ -70,7 +80,7 @@ module PgOnlineSchemaChange
|
|
70
80
|
reader = setup_signals!
|
71
81
|
signal = reader.gets.chomp
|
72
82
|
|
73
|
-
while !reader.closed? && IO.select([reader])
|
83
|
+
while !reader.closed? && IO.select([reader]) # rubocop:disable Lint/UnreachableLoop
|
74
84
|
logger.info "Signal #{signal} received, cleaning up"
|
75
85
|
|
76
86
|
client.connection.cancel
|
@@ -85,7 +95,7 @@ module PgOnlineSchemaChange
|
|
85
95
|
logger.info("Setting up audit table", { audit_table: audit_table })
|
86
96
|
|
87
97
|
sql = <<~SQL
|
88
|
-
CREATE TABLE #{audit_table} (
|
98
|
+
CREATE TABLE #{audit_table} (#{audit_table_pk} SERIAL PRIMARY KEY, #{operation_type_column} text, #{trigger_time_column} timestamp, LIKE #{client.table});
|
89
99
|
SQL
|
90
100
|
|
91
101
|
Query.run(client.connection, sql)
|
@@ -109,13 +119,13 @@ module PgOnlineSchemaChange
|
|
109
119
|
$$
|
110
120
|
BEGIN
|
111
121
|
IF ( TG_OP = 'INSERT') THEN
|
112
|
-
INSERT INTO \"#{audit_table}\" select 'INSERT',
|
122
|
+
INSERT INTO \"#{audit_table}\" select nextval(\'#{audit_table_pk_sequence}\'), 'INSERT', clock_timestamp(), NEW.* ;
|
113
123
|
RETURN NEW;
|
114
124
|
ELSIF ( TG_OP = 'UPDATE') THEN
|
115
|
-
INSERT INTO \"#{audit_table}\" select 'UPDATE',
|
125
|
+
INSERT INTO \"#{audit_table}\" select nextval(\'#{audit_table_pk_sequence}\'), 'UPDATE', clock_timestamp(), NEW.* ;
|
116
126
|
RETURN NEW;
|
117
127
|
ELSIF ( TG_OP = 'DELETE') THEN
|
118
|
-
INSERT INTO \"#{audit_table}\" select 'DELETE',
|
128
|
+
INSERT INTO \"#{audit_table}\" select nextval(\'#{audit_table_pk_sequence}\'), 'DELETE', clock_timestamp(), OLD.* ;
|
119
129
|
RETURN NEW;
|
120
130
|
END IF;
|
121
131
|
END;
|
@@ -153,7 +163,7 @@ module PgOnlineSchemaChange
|
|
153
163
|
# re-uses transaction with serializable
|
154
164
|
# Disabling vacuum to avoid any issues during the process
|
155
165
|
result = Query.storage_parameters_for(client, client.table, true) || ""
|
156
|
-
|
166
|
+
Store.set(:primary_table_storage_parameters, result)
|
157
167
|
|
158
168
|
logger.debug("Disabling vacuum on shadow and audit table",
|
159
169
|
{ shadow_table: shadow_table, audit_table: audit_table })
|
@@ -185,8 +195,7 @@ module PgOnlineSchemaChange
|
|
185
195
|
# Begin the process to copy data into copy table
|
186
196
|
# depending on the size of the table, this can be a time
|
187
197
|
# taking operation.
|
188
|
-
logger.info("Clearing contents of audit table before copy..",
|
189
|
-
{ shadow_table: shadow_table, parent_table: client.table })
|
198
|
+
logger.info("Clearing contents of audit table before copy..", { shadow_table: shadow_table, parent_table: client.table })
|
190
199
|
Query.run(client.connection, "DELETE FROM #{audit_table}", true)
|
191
200
|
|
192
201
|
logger.info("Copying contents..", { shadow_table: shadow_table, parent_table: client.table })
|
@@ -272,8 +281,10 @@ module PgOnlineSchemaChange
|
|
272
281
|
Query.run(client.connection, sql)
|
273
282
|
end
|
274
283
|
|
275
|
-
private
|
276
|
-
|
284
|
+
private
|
285
|
+
|
286
|
+
def pgosc_identifier
|
287
|
+
@pgosc_identifier ||= SecureRandom.hex(3)
|
277
288
|
end
|
278
289
|
end
|
279
290
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "pg_query"
|
2
4
|
require "pg"
|
3
5
|
|
@@ -5,7 +7,7 @@ module PgOnlineSchemaChange
|
|
5
7
|
class Query
|
6
8
|
extend Helper
|
7
9
|
|
8
|
-
INDEX_SUFFIX = "_pgosc"
|
10
|
+
INDEX_SUFFIX = "_pgosc"
|
9
11
|
DROPPED_COLUMN_TYPE = :AT_DropColumn
|
10
12
|
RENAMED_COLUMN_TYPE = :AT_RenameColumn
|
11
13
|
LOCK_ATTEMPT = 4
|
@@ -15,28 +17,28 @@ module PgOnlineSchemaChange
|
|
15
17
|
PgQuery.parse(query).tree.stmts.all? do |statement|
|
16
18
|
statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt) || statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
|
17
19
|
end
|
18
|
-
rescue PgQuery::ParseError
|
20
|
+
rescue PgQuery::ParseError
|
19
21
|
false
|
20
22
|
end
|
21
23
|
|
22
24
|
def same_table?(query)
|
23
|
-
tables = PgQuery.parse(query).tree.stmts.
|
25
|
+
tables = PgQuery.parse(query).tree.stmts.filter_map do |statement|
|
24
26
|
if statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt)
|
25
27
|
statement.stmt.alter_table_stmt.relation.relname
|
26
28
|
elsif statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
|
27
29
|
statement.stmt.rename_stmt.relation.relname
|
28
30
|
end
|
29
|
-
end
|
31
|
+
end
|
30
32
|
|
31
33
|
tables.uniq.count == 1
|
32
|
-
rescue PgQuery::ParseError
|
34
|
+
rescue PgQuery::ParseError
|
33
35
|
false
|
34
36
|
end
|
35
37
|
|
36
38
|
def table(query)
|
37
|
-
from_rename_statement = PgQuery.parse(query).tree.stmts.
|
39
|
+
from_rename_statement = PgQuery.parse(query).tree.stmts.filter_map do |statement|
|
38
40
|
statement.stmt.rename_stmt&.relation&.relname
|
39
|
-
end
|
41
|
+
end[0]
|
40
42
|
PgQuery.parse(query).tables[0] || from_rename_statement
|
41
43
|
end
|
42
44
|
|
@@ -48,7 +50,7 @@ module PgOnlineSchemaChange
|
|
48
50
|
connection.async_exec("BEGIN;")
|
49
51
|
|
50
52
|
result = connection.async_exec(query, &block)
|
51
|
-
rescue Exception
|
53
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
52
54
|
connection.cancel if connection.transaction_status != PG::PQTRANS_IDLE
|
53
55
|
connection.block
|
54
56
|
logger.info("Exception raised, rolling back query", { rollback: true, query: query })
|
@@ -144,11 +146,11 @@ module PgOnlineSchemaChange
|
|
144
146
|
end
|
145
147
|
|
146
148
|
references.map do |row|
|
147
|
-
if row["definition"].end_with?("NOT VALID")
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
149
|
+
add_statement = if row["definition"].end_with?("NOT VALID")
|
150
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
|
151
|
+
else
|
152
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
|
153
|
+
end
|
152
154
|
|
153
155
|
drop_statement = "ALTER TABLE #{row["table_on"]} DROP CONSTRAINT #{row["constraint_name"]};"
|
154
156
|
|
@@ -291,7 +293,7 @@ module PgOnlineSchemaChange
|
|
291
293
|
client.connection.quote_ident(select_column)
|
292
294
|
end
|
293
295
|
|
294
|
-
|
296
|
+
<<~SQL
|
295
297
|
INSERT INTO #{shadow_table}(#{insert_into_columns.join(", ")})
|
296
298
|
SELECT #{select_columns.join(", ")}
|
297
299
|
FROM ONLY #{client.table}
|
@@ -1,3 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
4
|
+
|
1
5
|
module PgOnlineSchemaChange
|
2
6
|
class Replay
|
3
7
|
extend Helper
|
@@ -5,7 +9,6 @@ module PgOnlineSchemaChange
|
|
5
9
|
class << self
|
6
10
|
PULL_BATCH_COUNT = 1000
|
7
11
|
DELTA_COUNT = 20
|
8
|
-
RESERVED_COLUMNS = %w[operation_type trigger_time].freeze
|
9
12
|
|
10
13
|
# This, picks PULL_BATCH_COUNT rows by primary key from audit_table,
|
11
14
|
# replays it on the shadow_table. Once the batch is done,
|
@@ -25,7 +28,7 @@ module PgOnlineSchemaChange
|
|
25
28
|
|
26
29
|
def rows_to_play(reuse_trasaction = false)
|
27
30
|
select_query = <<~SQL
|
28
|
-
SELECT * FROM #{audit_table} ORDER BY #{
|
31
|
+
SELECT * FROM #{audit_table} ORDER BY #{audit_table_pk} LIMIT #{PULL_BATCH_COUNT};
|
29
32
|
SQL
|
30
33
|
|
31
34
|
rows = []
|
@@ -34,6 +37,10 @@ module PgOnlineSchemaChange
|
|
34
37
|
rows
|
35
38
|
end
|
36
39
|
|
40
|
+
def reserved_columns
|
41
|
+
@reserved_columns ||= [trigger_time_column, operation_type_column, audit_table_pk]
|
42
|
+
end
|
43
|
+
|
37
44
|
def play!(rows, reuse_trasaction = false)
|
38
45
|
logger.info("Replaying rows, count: #{rows.size}")
|
39
46
|
|
@@ -44,7 +51,7 @@ module PgOnlineSchemaChange
|
|
44
51
|
|
45
52
|
# Remove audit table cols, since we will be
|
46
53
|
# re-mapping them for inserts and updates
|
47
|
-
|
54
|
+
reserved_columns.each do |col|
|
48
55
|
new_row.delete(col)
|
49
56
|
end
|
50
57
|
|
@@ -73,7 +80,7 @@ module PgOnlineSchemaChange
|
|
73
80
|
client.connection.escape_string(value)
|
74
81
|
end
|
75
82
|
|
76
|
-
case row[
|
83
|
+
case row[operation_type_column]
|
77
84
|
when "INSERT"
|
78
85
|
values = new_row.map { |_, val| "'#{val}'" }.join(",")
|
79
86
|
|
@@ -110,13 +117,15 @@ module PgOnlineSchemaChange
|
|
110
117
|
Query.run(client.connection, to_be_replayed.join, reuse_trasaction)
|
111
118
|
|
112
119
|
# Delete items from the audit now that are replayed
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
120
|
+
return unless to_be_deleted_rows.count >= 1
|
121
|
+
|
122
|
+
delete_query = <<~SQL
|
123
|
+
DELETE FROM #{audit_table} WHERE #{primary_key} IN (#{to_be_deleted_rows.join(",")})
|
124
|
+
SQL
|
125
|
+
Query.run(client.connection, delete_query, reuse_trasaction)
|
119
126
|
end
|
120
127
|
end
|
121
128
|
end
|
122
129
|
end
|
130
|
+
|
131
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
@@ -1,17 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "pg_query"
|
2
4
|
require "pg"
|
3
5
|
|
4
6
|
module PgOnlineSchemaChange
|
5
7
|
class Store
|
6
8
|
class << self
|
7
|
-
|
9
|
+
@object = {}
|
8
10
|
|
9
11
|
def get(key)
|
10
|
-
|
12
|
+
@object ||= {}
|
13
|
+
@object[key.to_s] || @object[key.to_sym]
|
11
14
|
end
|
12
15
|
|
13
16
|
def set(key, value)
|
14
|
-
|
17
|
+
@object ||= {}
|
18
|
+
@object[key.to_sym] = value
|
15
19
|
end
|
16
20
|
end
|
17
21
|
end
|
@@ -18,16 +18,12 @@ module PgOnlineSchemaChange
|
|
18
18
|
class CountBelowDelta < StandardError; end
|
19
19
|
class AccessExclusiveLockNotAcquired < StandardError; end
|
20
20
|
|
21
|
-
def self.logger
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
end
|
29
|
-
|
30
|
-
def self.logger
|
31
|
-
@@logger
|
21
|
+
def self.logger(verbose: false)
|
22
|
+
@logger ||= begin
|
23
|
+
logger = Ougai::Logger.new($stdout)
|
24
|
+
logger.level = verbose ? Ougai::Logger::TRACE : Ougai::Logger::INFO
|
25
|
+
logger.with_fields = { version: PgOnlineSchemaChange::VERSION }
|
26
|
+
logger
|
27
|
+
end
|
32
28
|
end
|
33
29
|
end
|
data/scripts/release.sh
CHANGED
@@ -11,8 +11,11 @@ gem build pg_online_schema_change.gemspec
|
|
11
11
|
echo "=== Pushing gem ===="
|
12
12
|
gem push pg_online_schema_change-$VERSION.gem
|
13
13
|
|
14
|
+
echo "=== Sleeping for 5s ===="
|
15
|
+
sleep 5
|
16
|
+
|
14
17
|
echo "=== Building Image ===="
|
15
|
-
docker build . --build-arg VERSION=$VERSION -t pg-osc
|
18
|
+
docker build . --build-arg VERSION=$VERSION -t shayonj/pg-osc:$VERSION
|
16
19
|
|
17
20
|
echo "=== Tagging Image ===="
|
18
21
|
docker image tag shayonj/pg-osc:$VERSION shayonj/pg-osc:latest
|
@@ -20,3 +23,6 @@ docker image tag shayonj/pg-osc:$VERSION shayonj/pg-osc:latest
|
|
20
23
|
echo "=== Pushing Image ===="
|
21
24
|
docker push shayonj/pg-osc:$VERSION
|
22
25
|
docker push shayonj/pg-osc:latest
|
26
|
+
|
27
|
+
echo "=== Cleaning up ===="
|
28
|
+
rm pg_online_schema_change-$VERSION.gem
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_online_schema_change
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shayon Mukherjee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-02-
|
11
|
+
date: 2022-02-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ougai
|
@@ -191,6 +191,7 @@ extra_rdoc_files: []
|
|
191
191
|
files:
|
192
192
|
- ".rspec"
|
193
193
|
- ".rubocop.yml"
|
194
|
+
- ".rubocop_todo.yml"
|
194
195
|
- ".ruby-version"
|
195
196
|
- CHANGELOG.md
|
196
197
|
- CODE_OF_CONDUCT.md
|