pg_online_schema_change 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +57 -24
- data/.rubocop_todo.yml +44 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile.lock +1 -1
- data/lib/pg_online_schema_change/cli.rb +3 -1
- data/lib/pg_online_schema_change/client.rb +12 -6
- data/lib/pg_online_schema_change/functions.rb +4 -2
- data/lib/pg_online_schema_change/helper.rb +10 -1
- data/lib/pg_online_schema_change/orchestrate.rb +24 -13
- data/lib/pg_online_schema_change/query.rb +16 -14
- data/lib/pg_online_schema_change/replay.rb +19 -10
- data/lib/pg_online_schema_change/store.rb +7 -3
- data/lib/pg_online_schema_change/version.rb +1 -1
- data/lib/pg_online_schema_change.rb +7 -11
- data/scripts/release.sh +7 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8ef1069f7d159544838ec27cf518f98eb31609a0a2e1878ebd40246b10a6b534
|
4
|
+
data.tar.gz: 8e83cbac78164fb10870537bf4867020eae7cf06f4cd91930a2c8846fbc76c7c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2657c94b3b07730aa3bb282f34cd1ee95f0549d3649b39a61be3ce1b6d2ba5832a1914b6a9788778c912464cbd9773b5e967ed4e6c896127d60e62a0daa76c99
|
7
|
+
data.tar.gz: 4788523d691c25045b7f3c539f9743d880fabcbb01887ae34ec55a6c8a829804a9f4d43f619e79e0c44b1c9fff6ea88361fb5ac1111bc1f491aa17e6e43336d0
|
data/.rubocop.yml
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
2
|
+
|
1
3
|
require:
|
2
4
|
- rubocop-rspec
|
3
5
|
- rubocop-packaging
|
@@ -14,71 +16,70 @@ AllCops:
|
|
14
16
|
- "vendor/**/*"
|
15
17
|
|
16
18
|
Layout/HashAlignment:
|
17
|
-
EnforcedColonStyle:
|
18
|
-
|
19
|
-
- key
|
20
|
-
EnforcedHashRocketStyle:
|
21
|
-
- table
|
22
|
-
- key
|
19
|
+
EnforcedColonStyle: key
|
20
|
+
EnforcedHashRocketStyle: key
|
23
21
|
|
24
22
|
Layout/SpaceAroundEqualsInParameterDefault:
|
25
|
-
EnforcedStyle:
|
23
|
+
EnforcedStyle: space
|
26
24
|
|
27
25
|
Metrics/AbcSize:
|
28
|
-
|
26
|
+
Enabled: true
|
27
|
+
Max: 40
|
29
28
|
Exclude:
|
30
|
-
- "
|
29
|
+
- "spec/**/*"
|
31
30
|
|
32
31
|
Metrics/BlockLength:
|
32
|
+
Max: 100
|
33
33
|
Exclude:
|
34
34
|
- "*.gemspec"
|
35
35
|
- "Rakefile"
|
36
|
+
- "spec/**/*"
|
36
37
|
|
37
38
|
Metrics/ClassLength:
|
38
39
|
Exclude:
|
39
40
|
- "test/**/*"
|
40
41
|
|
41
42
|
Metrics/MethodLength:
|
42
|
-
Max:
|
43
|
+
Max: 30
|
43
44
|
Exclude:
|
44
45
|
- "test/**/*"
|
45
46
|
|
46
47
|
Metrics/ParameterLists:
|
47
|
-
Max:
|
48
|
+
Max: 5
|
48
49
|
|
49
50
|
Naming/MemoizedInstanceVariableName:
|
50
|
-
Enabled:
|
51
|
+
Enabled: true
|
51
52
|
|
52
53
|
Naming/VariableNumber:
|
53
|
-
Enabled:
|
54
|
-
|
55
|
-
Rake/Desc:
|
56
|
-
Enabled: false
|
54
|
+
Enabled: true
|
57
55
|
|
58
56
|
Style/BarePercentLiterals:
|
59
57
|
EnforcedStyle: percent_q
|
60
58
|
|
61
59
|
Style/ClassAndModuleChildren:
|
62
|
-
Enabled:
|
60
|
+
Enabled: true
|
63
61
|
|
64
62
|
Style/Documentation:
|
65
63
|
Enabled: false
|
66
64
|
|
67
65
|
Style/DoubleNegation:
|
68
|
-
Enabled:
|
66
|
+
Enabled: true
|
69
67
|
|
70
68
|
Style/EmptyMethod:
|
71
|
-
Enabled:
|
69
|
+
Enabled: true
|
72
70
|
|
73
71
|
Style/FrozenStringLiteralComment:
|
74
|
-
Enabled:
|
72
|
+
Enabled: true
|
75
73
|
|
76
74
|
Style/NumericPredicate:
|
77
|
-
Enabled:
|
75
|
+
Enabled: true
|
78
76
|
|
79
77
|
Style/StringLiterals:
|
80
78
|
EnforcedStyle: double_quotes
|
81
79
|
|
80
|
+
Style/StringLiteralsInInterpolation:
|
81
|
+
EnforcedStyle: double_quotes
|
82
|
+
|
82
83
|
Style/TrivialAccessors:
|
83
84
|
AllowPredicates: true
|
84
85
|
|
@@ -91,9 +92,41 @@ Style/TrailingCommaInArrayLiteral:
|
|
91
92
|
Style/TrailingCommaInHashLiteral:
|
92
93
|
EnforcedStyleForMultiline: comma
|
93
94
|
|
94
|
-
|
95
|
-
|
95
|
+
Layout/MultilineArrayBraceLayout:
|
96
|
+
Enabled: true
|
97
|
+
EnforcedStyle: symmetrical
|
96
98
|
|
97
|
-
|
99
|
+
Layout/MultilineHashBraceLayout:
|
98
100
|
Enabled: true
|
99
101
|
EnforcedStyle: symmetrical
|
102
|
+
|
103
|
+
Layout/MultilineAssignmentLayout:
|
104
|
+
Enabled: true
|
105
|
+
EnforcedStyle: same_line
|
106
|
+
|
107
|
+
Layout/FirstArrayElementIndentation:
|
108
|
+
Enabled: true
|
109
|
+
EnforcedStyle: consistent
|
110
|
+
|
111
|
+
Layout/FirstHashElementIndentation:
|
112
|
+
Enabled: true
|
113
|
+
EnforcedStyle: consistent
|
114
|
+
|
115
|
+
Layout/MultilineHashKeyLineBreaks:
|
116
|
+
Enabled: true
|
117
|
+
|
118
|
+
Layout/LineLength:
|
119
|
+
Enabled: true
|
120
|
+
Max: 250
|
121
|
+
|
122
|
+
Style/FormatStringToken:
|
123
|
+
Enabled: true
|
124
|
+
EnforcedStyle: template
|
125
|
+
|
126
|
+
RSpec/MessageSpies:
|
127
|
+
Enabled: true
|
128
|
+
EnforcedStyle: receive
|
129
|
+
|
130
|
+
RSpec/FilePath:
|
131
|
+
Enabled: true
|
132
|
+
SpecSuffixOnly: true
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
# This configuration was generated by
|
2
|
+
# `rubocop --auto-gen-config`
|
3
|
+
# on 2022-02-21 22:46:44 UTC using RuboCop version 1.23.0.
|
4
|
+
# The point is for the user to remove these configuration records
|
5
|
+
# one by one as the offenses are removed from the code base.
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
8
|
+
|
9
|
+
# Offense count: 2
|
10
|
+
# Configuration parameters: CountComments, CountAsOne.
|
11
|
+
Metrics/ClassLength:
|
12
|
+
Max: 233
|
13
|
+
|
14
|
+
# Offense count: 2
|
15
|
+
# Configuration parameters: IgnoredMethods.
|
16
|
+
Metrics/CyclomaticComplexity:
|
17
|
+
Max: 15
|
18
|
+
|
19
|
+
# Offense count: 2
|
20
|
+
# Configuration parameters: IgnoredMethods.
|
21
|
+
Metrics/PerceivedComplexity:
|
22
|
+
Max: 13
|
23
|
+
|
24
|
+
# Offense count: 1
|
25
|
+
Packaging/GemspecGit:
|
26
|
+
Exclude:
|
27
|
+
- 'pg_online_schema_change.gemspec'
|
28
|
+
|
29
|
+
# Offense count: 62
|
30
|
+
# Configuration parameters: CountAsOne.
|
31
|
+
RSpec/ExampleLength:
|
32
|
+
Max: 55
|
33
|
+
|
34
|
+
# Offense count: 38
|
35
|
+
RSpec/MultipleExpectations:
|
36
|
+
Max: 14
|
37
|
+
|
38
|
+
# Offense count: 6
|
39
|
+
# Configuration parameters: AllowedMethods.
|
40
|
+
# AllowedMethods: respond_to_missing?
|
41
|
+
Style/OptionalBooleanParameter:
|
42
|
+
Exclude:
|
43
|
+
- 'lib/pg_online_schema_change/query.rb'
|
44
|
+
- 'lib/pg_online_schema_change/replay.rb'
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## [0.3.0] - 2022-02-21
|
2
|
+
|
3
|
+
- Explicitly call dependencies and bump dependencies by @shayonj https://github.com/shayonj/pg-osc/pull/44
|
4
|
+
- Introduce Dockerfile and release process https://github.com/shayonj/pg-osc/pull/45
|
5
|
+
|
1
6
|
## [0.2.0] - 2022-02-17
|
2
7
|
|
3
8
|
- Use ISOLATION LEVEL SERIALIZABLE ([#42](https://github.com/shayonj/pg-osc/pull/42)) (props to @jfrost)
|
data/Gemfile.lock
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "thor"
|
2
4
|
|
3
5
|
module PgOnlineSchemaChange
|
@@ -25,7 +27,7 @@ module PgOnlineSchemaChange
|
|
25
27
|
def perform
|
26
28
|
client_options = Struct.new(*options.keys.map(&:to_sym)).new(*options.values)
|
27
29
|
|
28
|
-
PgOnlineSchemaChange.logger
|
30
|
+
PgOnlineSchemaChange.logger(verbose: client_options.verbose)
|
29
31
|
PgOnlineSchemaChange::Orchestrate.run!(client_options)
|
30
32
|
end
|
31
33
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "pg"
|
2
4
|
|
3
5
|
module PgOnlineSchemaChange
|
@@ -16,7 +18,9 @@ module PgOnlineSchemaChange
|
|
16
18
|
@drop = options.drop
|
17
19
|
@kill_backends = options.kill_backends
|
18
20
|
@wait_time_for_lock = options.wait_time_for_lock
|
21
|
+
|
19
22
|
handle_copy_statement(options.copy_statement)
|
23
|
+
handle_validations
|
20
24
|
|
21
25
|
@connection = PG.connect(
|
22
26
|
dbname: @dbname,
|
@@ -26,17 +30,19 @@ module PgOnlineSchemaChange
|
|
26
30
|
port: @port,
|
27
31
|
)
|
28
32
|
|
29
|
-
raise Error, "Not a valid ALTER statement: #{@alter_statement}" unless Query.alter_statement?(@alter_statement)
|
30
|
-
|
31
|
-
unless Query.same_table?(@alter_statement)
|
32
|
-
raise Error "All statements should belong to the same table: #{@alter_statement}"
|
33
|
-
end
|
34
|
-
|
35
33
|
@table = Query.table(@alter_statement)
|
36
34
|
|
37
35
|
PgOnlineSchemaChange.logger.debug("Connection established")
|
38
36
|
end
|
39
37
|
|
38
|
+
def handle_validations
|
39
|
+
raise Error, "Not a valid ALTER statement: #{@alter_statement}" unless Query.alter_statement?(@alter_statement)
|
40
|
+
|
41
|
+
return if Query.same_table?(@alter_statement)
|
42
|
+
|
43
|
+
raise Error "All statements should belong to the same table: #{@alter_statement}"
|
44
|
+
end
|
45
|
+
|
40
46
|
def handle_copy_statement(statement)
|
41
47
|
return if statement.nil? || statement == ""
|
42
48
|
|
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
FUNC_FIX_SERIAL_SEQUENCE = <<~SQL
|
2
4
|
CREATE OR REPLACE FUNCTION fix_serial_sequence(_table regclass, _newtable text)
|
3
5
|
RETURNS void AS
|
4
6
|
$func$
|
@@ -35,7 +37,7 @@ FUNC_FIX_SERIAL_SEQUENCE = <<~SQL.freeze
|
|
35
37
|
$func$ LANGUAGE plpgsql VOLATILE;
|
36
38
|
SQL
|
37
39
|
|
38
|
-
FUNC_CREATE_TABLE_ALL = <<~SQL
|
40
|
+
FUNC_CREATE_TABLE_ALL = <<~SQL
|
39
41
|
CREATE OR REPLACE FUNCTION create_table_all(source_table text, newsource_table text)
|
40
42
|
RETURNS void language plpgsql
|
41
43
|
as $$
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module PgOnlineSchemaChange
|
2
4
|
module Helper
|
3
5
|
def primary_key
|
@@ -15,7 +17,14 @@ module PgOnlineSchemaChange
|
|
15
17
|
result = Store.send(:get, method)
|
16
18
|
return result if result
|
17
19
|
|
18
|
-
|
20
|
+
super
|
21
|
+
end
|
22
|
+
|
23
|
+
def respond_to_missing?(method_name, *args)
|
24
|
+
result = Store.send(:get, method)
|
25
|
+
return true if result
|
26
|
+
|
27
|
+
super
|
19
28
|
end
|
20
29
|
end
|
21
30
|
end
|
@@ -1,8 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "securerandom"
|
2
4
|
|
3
5
|
module PgOnlineSchemaChange
|
4
6
|
class Orchestrate
|
5
|
-
SWAP_STATEMENT_TIMEOUT = "5s"
|
7
|
+
SWAP_STATEMENT_TIMEOUT = "5s"
|
6
8
|
|
7
9
|
extend Helper
|
8
10
|
|
@@ -21,12 +23,20 @@ module PgOnlineSchemaChange
|
|
21
23
|
Query.run(client.connection, FUNC_FIX_SERIAL_SEQUENCE)
|
22
24
|
Query.run(client.connection, FUNC_CREATE_TABLE_ALL)
|
23
25
|
|
26
|
+
setup_store
|
27
|
+
end
|
28
|
+
|
29
|
+
def setup_store
|
24
30
|
# Set this early on to ensure their creation and cleanup (unexpected)
|
25
31
|
# happens at all times. IOW, the calls from Store.get always return
|
26
32
|
# the same value.
|
27
33
|
Store.set(:old_primary_table, "pgosc_op_table_#{client.table}")
|
28
|
-
Store.set(:audit_table, "pgosc_at_#{client.table}_#{
|
29
|
-
Store.set(:
|
34
|
+
Store.set(:audit_table, "pgosc_at_#{client.table}_#{pgosc_identifier}")
|
35
|
+
Store.set(:operation_type_column, "operation_type_#{pgosc_identifier}")
|
36
|
+
Store.set(:trigger_time_column, "trigger_time_#{pgosc_identifier}")
|
37
|
+
Store.set(:audit_table_pk, "at_#{pgosc_identifier}_id")
|
38
|
+
Store.set(:audit_table_pk_sequence, "#{audit_table}_#{audit_table_pk}_seq")
|
39
|
+
Store.set(:shadow_table, "pgosc_st_#{client.table}_#{pgosc_identifier}")
|
30
40
|
end
|
31
41
|
|
32
42
|
def run!(options)
|
@@ -70,7 +80,7 @@ module PgOnlineSchemaChange
|
|
70
80
|
reader = setup_signals!
|
71
81
|
signal = reader.gets.chomp
|
72
82
|
|
73
|
-
while !reader.closed? && IO.select([reader])
|
83
|
+
while !reader.closed? && IO.select([reader]) # rubocop:disable Lint/UnreachableLoop
|
74
84
|
logger.info "Signal #{signal} received, cleaning up"
|
75
85
|
|
76
86
|
client.connection.cancel
|
@@ -85,7 +95,7 @@ module PgOnlineSchemaChange
|
|
85
95
|
logger.info("Setting up audit table", { audit_table: audit_table })
|
86
96
|
|
87
97
|
sql = <<~SQL
|
88
|
-
CREATE TABLE #{audit_table} (
|
98
|
+
CREATE TABLE #{audit_table} (#{audit_table_pk} SERIAL PRIMARY KEY, #{operation_type_column} text, #{trigger_time_column} timestamp, LIKE #{client.table});
|
89
99
|
SQL
|
90
100
|
|
91
101
|
Query.run(client.connection, sql)
|
@@ -109,13 +119,13 @@ module PgOnlineSchemaChange
|
|
109
119
|
$$
|
110
120
|
BEGIN
|
111
121
|
IF ( TG_OP = 'INSERT') THEN
|
112
|
-
INSERT INTO \"#{audit_table}\" select 'INSERT',
|
122
|
+
INSERT INTO \"#{audit_table}\" select nextval(\'#{audit_table_pk_sequence}\'), 'INSERT', clock_timestamp(), NEW.* ;
|
113
123
|
RETURN NEW;
|
114
124
|
ELSIF ( TG_OP = 'UPDATE') THEN
|
115
|
-
INSERT INTO \"#{audit_table}\" select 'UPDATE',
|
125
|
+
INSERT INTO \"#{audit_table}\" select nextval(\'#{audit_table_pk_sequence}\'), 'UPDATE', clock_timestamp(), NEW.* ;
|
116
126
|
RETURN NEW;
|
117
127
|
ELSIF ( TG_OP = 'DELETE') THEN
|
118
|
-
INSERT INTO \"#{audit_table}\" select 'DELETE',
|
128
|
+
INSERT INTO \"#{audit_table}\" select nextval(\'#{audit_table_pk_sequence}\'), 'DELETE', clock_timestamp(), OLD.* ;
|
119
129
|
RETURN NEW;
|
120
130
|
END IF;
|
121
131
|
END;
|
@@ -153,7 +163,7 @@ module PgOnlineSchemaChange
|
|
153
163
|
# re-uses transaction with serializable
|
154
164
|
# Disabling vacuum to avoid any issues during the process
|
155
165
|
result = Query.storage_parameters_for(client, client.table, true) || ""
|
156
|
-
|
166
|
+
Store.set(:primary_table_storage_parameters, result)
|
157
167
|
|
158
168
|
logger.debug("Disabling vacuum on shadow and audit table",
|
159
169
|
{ shadow_table: shadow_table, audit_table: audit_table })
|
@@ -185,8 +195,7 @@ module PgOnlineSchemaChange
|
|
185
195
|
# Begin the process to copy data into copy table
|
186
196
|
# depending on the size of the table, this can be a time
|
187
197
|
# taking operation.
|
188
|
-
logger.info("Clearing contents of audit table before copy..",
|
189
|
-
{ shadow_table: shadow_table, parent_table: client.table })
|
198
|
+
logger.info("Clearing contents of audit table before copy..", { shadow_table: shadow_table, parent_table: client.table })
|
190
199
|
Query.run(client.connection, "DELETE FROM #{audit_table}", true)
|
191
200
|
|
192
201
|
logger.info("Copying contents..", { shadow_table: shadow_table, parent_table: client.table })
|
@@ -272,8 +281,10 @@ module PgOnlineSchemaChange
|
|
272
281
|
Query.run(client.connection, sql)
|
273
282
|
end
|
274
283
|
|
275
|
-
private
|
276
|
-
|
284
|
+
private
|
285
|
+
|
286
|
+
def pgosc_identifier
|
287
|
+
@pgosc_identifier ||= SecureRandom.hex(3)
|
277
288
|
end
|
278
289
|
end
|
279
290
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "pg_query"
|
2
4
|
require "pg"
|
3
5
|
|
@@ -5,7 +7,7 @@ module PgOnlineSchemaChange
|
|
5
7
|
class Query
|
6
8
|
extend Helper
|
7
9
|
|
8
|
-
INDEX_SUFFIX = "_pgosc"
|
10
|
+
INDEX_SUFFIX = "_pgosc"
|
9
11
|
DROPPED_COLUMN_TYPE = :AT_DropColumn
|
10
12
|
RENAMED_COLUMN_TYPE = :AT_RenameColumn
|
11
13
|
LOCK_ATTEMPT = 4
|
@@ -15,28 +17,28 @@ module PgOnlineSchemaChange
|
|
15
17
|
PgQuery.parse(query).tree.stmts.all? do |statement|
|
16
18
|
statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt) || statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
|
17
19
|
end
|
18
|
-
rescue PgQuery::ParseError
|
20
|
+
rescue PgQuery::ParseError
|
19
21
|
false
|
20
22
|
end
|
21
23
|
|
22
24
|
def same_table?(query)
|
23
|
-
tables = PgQuery.parse(query).tree.stmts.
|
25
|
+
tables = PgQuery.parse(query).tree.stmts.filter_map do |statement|
|
24
26
|
if statement.stmt.alter_table_stmt.instance_of?(PgQuery::AlterTableStmt)
|
25
27
|
statement.stmt.alter_table_stmt.relation.relname
|
26
28
|
elsif statement.stmt.rename_stmt.instance_of?(PgQuery::RenameStmt)
|
27
29
|
statement.stmt.rename_stmt.relation.relname
|
28
30
|
end
|
29
|
-
end
|
31
|
+
end
|
30
32
|
|
31
33
|
tables.uniq.count == 1
|
32
|
-
rescue PgQuery::ParseError
|
34
|
+
rescue PgQuery::ParseError
|
33
35
|
false
|
34
36
|
end
|
35
37
|
|
36
38
|
def table(query)
|
37
|
-
from_rename_statement = PgQuery.parse(query).tree.stmts.
|
39
|
+
from_rename_statement = PgQuery.parse(query).tree.stmts.filter_map do |statement|
|
38
40
|
statement.stmt.rename_stmt&.relation&.relname
|
39
|
-
end
|
41
|
+
end[0]
|
40
42
|
PgQuery.parse(query).tables[0] || from_rename_statement
|
41
43
|
end
|
42
44
|
|
@@ -48,7 +50,7 @@ module PgOnlineSchemaChange
|
|
48
50
|
connection.async_exec("BEGIN;")
|
49
51
|
|
50
52
|
result = connection.async_exec(query, &block)
|
51
|
-
rescue Exception
|
53
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
52
54
|
connection.cancel if connection.transaction_status != PG::PQTRANS_IDLE
|
53
55
|
connection.block
|
54
56
|
logger.info("Exception raised, rolling back query", { rollback: true, query: query })
|
@@ -144,11 +146,11 @@ module PgOnlineSchemaChange
|
|
144
146
|
end
|
145
147
|
|
146
148
|
references.map do |row|
|
147
|
-
if row["definition"].end_with?("NOT VALID")
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
149
|
+
add_statement = if row["definition"].end_with?("NOT VALID")
|
150
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]};"
|
151
|
+
else
|
152
|
+
"ALTER TABLE #{row["table_on"]} ADD CONSTRAINT #{row["constraint_name"]} #{row["definition"]} NOT VALID;"
|
153
|
+
end
|
152
154
|
|
153
155
|
drop_statement = "ALTER TABLE #{row["table_on"]} DROP CONSTRAINT #{row["constraint_name"]};"
|
154
156
|
|
@@ -291,7 +293,7 @@ module PgOnlineSchemaChange
|
|
291
293
|
client.connection.quote_ident(select_column)
|
292
294
|
end
|
293
295
|
|
294
|
-
|
296
|
+
<<~SQL
|
295
297
|
INSERT INTO #{shadow_table}(#{insert_into_columns.join(", ")})
|
296
298
|
SELECT #{select_columns.join(", ")}
|
297
299
|
FROM ONLY #{client.table}
|
@@ -1,3 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
4
|
+
|
1
5
|
module PgOnlineSchemaChange
|
2
6
|
class Replay
|
3
7
|
extend Helper
|
@@ -5,7 +9,6 @@ module PgOnlineSchemaChange
|
|
5
9
|
class << self
|
6
10
|
PULL_BATCH_COUNT = 1000
|
7
11
|
DELTA_COUNT = 20
|
8
|
-
RESERVED_COLUMNS = %w[operation_type trigger_time].freeze
|
9
12
|
|
10
13
|
# This, picks PULL_BATCH_COUNT rows by primary key from audit_table,
|
11
14
|
# replays it on the shadow_table. Once the batch is done,
|
@@ -25,7 +28,7 @@ module PgOnlineSchemaChange
|
|
25
28
|
|
26
29
|
def rows_to_play(reuse_trasaction = false)
|
27
30
|
select_query = <<~SQL
|
28
|
-
SELECT * FROM #{audit_table} ORDER BY #{
|
31
|
+
SELECT * FROM #{audit_table} ORDER BY #{audit_table_pk} LIMIT #{PULL_BATCH_COUNT};
|
29
32
|
SQL
|
30
33
|
|
31
34
|
rows = []
|
@@ -34,6 +37,10 @@ module PgOnlineSchemaChange
|
|
34
37
|
rows
|
35
38
|
end
|
36
39
|
|
40
|
+
def reserved_columns
|
41
|
+
@reserved_columns ||= [trigger_time_column, operation_type_column, audit_table_pk]
|
42
|
+
end
|
43
|
+
|
37
44
|
def play!(rows, reuse_trasaction = false)
|
38
45
|
logger.info("Replaying rows, count: #{rows.size}")
|
39
46
|
|
@@ -44,7 +51,7 @@ module PgOnlineSchemaChange
|
|
44
51
|
|
45
52
|
# Remove audit table cols, since we will be
|
46
53
|
# re-mapping them for inserts and updates
|
47
|
-
|
54
|
+
reserved_columns.each do |col|
|
48
55
|
new_row.delete(col)
|
49
56
|
end
|
50
57
|
|
@@ -73,7 +80,7 @@ module PgOnlineSchemaChange
|
|
73
80
|
client.connection.escape_string(value)
|
74
81
|
end
|
75
82
|
|
76
|
-
case row[
|
83
|
+
case row[operation_type_column]
|
77
84
|
when "INSERT"
|
78
85
|
values = new_row.map { |_, val| "'#{val}'" }.join(",")
|
79
86
|
|
@@ -110,13 +117,15 @@ module PgOnlineSchemaChange
|
|
110
117
|
Query.run(client.connection, to_be_replayed.join, reuse_trasaction)
|
111
118
|
|
112
119
|
# Delete items from the audit now that are replayed
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
120
|
+
return unless to_be_deleted_rows.count >= 1
|
121
|
+
|
122
|
+
delete_query = <<~SQL
|
123
|
+
DELETE FROM #{audit_table} WHERE #{primary_key} IN (#{to_be_deleted_rows.join(",")})
|
124
|
+
SQL
|
125
|
+
Query.run(client.connection, delete_query, reuse_trasaction)
|
119
126
|
end
|
120
127
|
end
|
121
128
|
end
|
122
129
|
end
|
130
|
+
|
131
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
@@ -1,17 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "pg_query"
|
2
4
|
require "pg"
|
3
5
|
|
4
6
|
module PgOnlineSchemaChange
|
5
7
|
class Store
|
6
8
|
class << self
|
7
|
-
|
9
|
+
@object = {}
|
8
10
|
|
9
11
|
def get(key)
|
10
|
-
|
12
|
+
@object ||= {}
|
13
|
+
@object[key.to_s] || @object[key.to_sym]
|
11
14
|
end
|
12
15
|
|
13
16
|
def set(key, value)
|
14
|
-
|
17
|
+
@object ||= {}
|
18
|
+
@object[key.to_sym] = value
|
15
19
|
end
|
16
20
|
end
|
17
21
|
end
|
@@ -18,16 +18,12 @@ module PgOnlineSchemaChange
|
|
18
18
|
class CountBelowDelta < StandardError; end
|
19
19
|
class AccessExclusiveLockNotAcquired < StandardError; end
|
20
20
|
|
21
|
-
def self.logger
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
end
|
29
|
-
|
30
|
-
def self.logger
|
31
|
-
@@logger
|
21
|
+
def self.logger(verbose: false)
|
22
|
+
@logger ||= begin
|
23
|
+
logger = Ougai::Logger.new($stdout)
|
24
|
+
logger.level = verbose ? Ougai::Logger::TRACE : Ougai::Logger::INFO
|
25
|
+
logger.with_fields = { version: PgOnlineSchemaChange::VERSION }
|
26
|
+
logger
|
27
|
+
end
|
32
28
|
end
|
33
29
|
end
|
data/scripts/release.sh
CHANGED
@@ -11,8 +11,11 @@ gem build pg_online_schema_change.gemspec
|
|
11
11
|
echo "=== Pushing gem ===="
|
12
12
|
gem push pg_online_schema_change-$VERSION.gem
|
13
13
|
|
14
|
+
echo "=== Sleeping for 5s ===="
|
15
|
+
sleep 5
|
16
|
+
|
14
17
|
echo "=== Building Image ===="
|
15
|
-
docker build . --build-arg VERSION=$VERSION -t pg-osc
|
18
|
+
docker build . --build-arg VERSION=$VERSION -t shayonj/pg-osc:$VERSION
|
16
19
|
|
17
20
|
echo "=== Tagging Image ===="
|
18
21
|
docker image tag shayonj/pg-osc:$VERSION shayonj/pg-osc:latest
|
@@ -20,3 +23,6 @@ docker image tag shayonj/pg-osc:$VERSION shayonj/pg-osc:latest
|
|
20
23
|
echo "=== Pushing Image ===="
|
21
24
|
docker push shayonj/pg-osc:$VERSION
|
22
25
|
docker push shayonj/pg-osc:latest
|
26
|
+
|
27
|
+
echo "=== Cleaning up ===="
|
28
|
+
rm pg_online_schema_change-$VERSION.gem
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_online_schema_change
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shayon Mukherjee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-02-
|
11
|
+
date: 2022-02-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ougai
|
@@ -191,6 +191,7 @@ extra_rdoc_files: []
|
|
191
191
|
files:
|
192
192
|
- ".rspec"
|
193
193
|
- ".rubocop.yml"
|
194
|
+
- ".rubocop_todo.yml"
|
194
195
|
- ".ruby-version"
|
195
196
|
- CHANGELOG.md
|
196
197
|
- CODE_OF_CONDUCT.md
|