pg_easy_replicate 0.1.6 → 0.1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/Gemfile.lock +1 -1
- data/README.md +2 -1
- data/lib/pg_easy_replicate/orchestrate.rb +67 -20
- data/lib/pg_easy_replicate/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8b855bf14bd72bcf4ace60b2f746139d76408d6a00e0ae7ec3fd2d0c276cf22b
|
|
4
|
+
data.tar.gz: 1eff23811d7e9d8608c241bcc7a67367c8de8271ea53d6960f57859aa5c186fd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a7de21167ee2027115972442b9aa859fba7954bdc2e04b8e27e92d6fbadd2e7c9eb92c700d01f69ba9fecd5b40fe5593b5d0467ba4b2766106a7951311401842
|
|
7
|
+
data.tar.gz: c38c45df66c9a4433b558a41f120f8c3304b5cbd9ea1072348df61397b7e51b38d1709b83c160917d69fbcf460e18dc9f9355ea7c96ae3d109ba40b4430ff06c
|
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
# pg_easy_replicate
|
|
2
2
|
|
|
3
3
|
[](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml)
|
|
4
|
-
[](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml)
|
|
5
|
+
[](https://badge.fury.io/rb/pg_easy_replicate)
|
|
5
6
|
|
|
6
7
|
`pg_easy_replicate` is a CLI orchestrator tool that simplifies the process of setting up [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) between two PostgreSQL databases. `pg_easy_replicate` also supports switchover. After the source (primary database) is fully replicating, `pg_easy_replicate` puts it into read-only mode and via logical replication flushes all data to the new target database. This ensures zero data loss and minimal downtime for the application. This method can be useful for performing minimal downtime (up to <1min, depending) major version upgrades between two PostgreSQL databases, load testing with blue/green database setup and other similar use cases.
|
|
7
8
|
|
|
@@ -9,6 +9,14 @@ module PgEasyReplicate
|
|
|
9
9
|
DEFAULT_WAIT = 5 # seconds
|
|
10
10
|
|
|
11
11
|
def start_sync(options)
|
|
12
|
+
schema_name = options[:schema_name] || "public"
|
|
13
|
+
tables =
|
|
14
|
+
determine_tables(
|
|
15
|
+
schema: schema_name,
|
|
16
|
+
conn_string: source_db_url,
|
|
17
|
+
list: options[:tables],
|
|
18
|
+
)
|
|
19
|
+
|
|
12
20
|
create_publication(
|
|
13
21
|
group_name: options[:group_name],
|
|
14
22
|
conn_string: source_db_url,
|
|
@@ -16,9 +24,9 @@ module PgEasyReplicate
|
|
|
16
24
|
|
|
17
25
|
add_tables_to_publication(
|
|
18
26
|
group_name: options[:group_name],
|
|
19
|
-
tables:
|
|
27
|
+
tables: tables,
|
|
20
28
|
conn_string: source_db_url,
|
|
21
|
-
schema:
|
|
29
|
+
schema: schema_name,
|
|
22
30
|
)
|
|
23
31
|
|
|
24
32
|
create_subscription(
|
|
@@ -29,8 +37,8 @@ module PgEasyReplicate
|
|
|
29
37
|
|
|
30
38
|
Group.create(
|
|
31
39
|
name: options[:group_name],
|
|
32
|
-
table_names:
|
|
33
|
-
schema_name:
|
|
40
|
+
table_names: tables,
|
|
41
|
+
schema_name: schema_name,
|
|
34
42
|
started_at: Time.now.utc,
|
|
35
43
|
)
|
|
36
44
|
rescue => e
|
|
@@ -45,8 +53,8 @@ module PgEasyReplicate
|
|
|
45
53
|
else
|
|
46
54
|
Group.create(
|
|
47
55
|
name: options[:group_name],
|
|
48
|
-
table_names:
|
|
49
|
-
schema_name:
|
|
56
|
+
table_names: tables,
|
|
57
|
+
schema_name: schema_name,
|
|
50
58
|
started_at: Time.now.utc,
|
|
51
59
|
failed_at: Time.now.utc,
|
|
52
60
|
)
|
|
@@ -81,19 +89,16 @@ module PgEasyReplicate
|
|
|
81
89
|
{ publication_name: publication_name(group_name) },
|
|
82
90
|
)
|
|
83
91
|
|
|
84
|
-
tables
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
schema: schema,
|
|
95
|
-
)
|
|
96
|
-
end
|
|
92
|
+
tables
|
|
93
|
+
.split(",")
|
|
94
|
+
.map do |table_name|
|
|
95
|
+
Query.run(
|
|
96
|
+
query:
|
|
97
|
+
"ALTER PUBLICATION #{publication_name(group_name)} ADD TABLE \"#{table_name}\"",
|
|
98
|
+
connection_url: conn_string,
|
|
99
|
+
schema: schema,
|
|
100
|
+
)
|
|
101
|
+
end
|
|
97
102
|
rescue => e
|
|
98
103
|
raise "Unable to add tables to publication: #{e.message}"
|
|
99
104
|
end
|
|
@@ -102,11 +107,12 @@ module PgEasyReplicate
|
|
|
102
107
|
Query
|
|
103
108
|
.run(
|
|
104
109
|
query:
|
|
105
|
-
"SELECT table_name FROM information_schema.tables WHERE table_schema = '#{schema}'",
|
|
110
|
+
"SELECT table_name FROM information_schema.tables WHERE table_schema = '#{schema}' ORDER BY table_name",
|
|
106
111
|
connection_url: conn_string,
|
|
107
112
|
)
|
|
108
113
|
.map(&:values)
|
|
109
114
|
.flatten
|
|
115
|
+
.join(",")
|
|
110
116
|
end
|
|
111
117
|
|
|
112
118
|
def drop_publication(group_name:, conn_string:)
|
|
@@ -202,6 +208,11 @@ module PgEasyReplicate
|
|
|
202
208
|
)
|
|
203
209
|
group = Group.find(group_name)
|
|
204
210
|
|
|
211
|
+
run_vacuum_analyze(
|
|
212
|
+
conn_string: target_conn_string,
|
|
213
|
+
tables: group[:table_names],
|
|
214
|
+
schema: group[:schema_name],
|
|
215
|
+
)
|
|
205
216
|
watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
|
|
206
217
|
revoke_connections_on_source_db(group_name)
|
|
207
218
|
wait_for_remaining_catchup(group_name)
|
|
@@ -210,6 +221,12 @@ module PgEasyReplicate
|
|
|
210
221
|
schema: group[:schema_name],
|
|
211
222
|
)
|
|
212
223
|
mark_switchover_complete(group_name)
|
|
224
|
+
# Run vacuum analyze to refresh the planner post switchover
|
|
225
|
+
run_vacuum_analyze(
|
|
226
|
+
conn_string: target_conn_string,
|
|
227
|
+
tables: group[:table_names],
|
|
228
|
+
schema: group[:schema_name],
|
|
229
|
+
)
|
|
213
230
|
drop_subscription(
|
|
214
231
|
group_name: group_name,
|
|
215
232
|
target_conn_string: target_conn_string,
|
|
@@ -318,9 +335,39 @@ module PgEasyReplicate
|
|
|
318
335
|
raise "Unable to refresh sequences: #{e.message}"
|
|
319
336
|
end
|
|
320
337
|
|
|
338
|
+
def run_vacuum_analyze(conn_string:, tables:, schema:)
|
|
339
|
+
tables
|
|
340
|
+
.split(",")
|
|
341
|
+
.each do |t|
|
|
342
|
+
logger.info(
|
|
343
|
+
"Running vacuum analyze on #{t}",
|
|
344
|
+
schema: schema,
|
|
345
|
+
table: t,
|
|
346
|
+
)
|
|
347
|
+
Query.run(
|
|
348
|
+
query: "VACUUM VERBOSE ANALYZE #{t}",
|
|
349
|
+
connection_url: conn_string,
|
|
350
|
+
schema: schema,
|
|
351
|
+
transaction: false,
|
|
352
|
+
)
|
|
353
|
+
end
|
|
354
|
+
rescue => e
|
|
355
|
+
raise "Unable to run vacuum and analyze: #{e.message}"
|
|
356
|
+
end
|
|
357
|
+
|
|
321
358
|
def mark_switchover_complete(group_name)
|
|
322
359
|
Group.update(group_name: group_name, switchover_completed_at: Time.now)
|
|
323
360
|
end
|
|
361
|
+
|
|
362
|
+
private
|
|
363
|
+
|
|
364
|
+
def determine_tables(schema:, conn_string:, list: "")
|
|
365
|
+
tables = list&.split(",") || []
|
|
366
|
+
unless tables.size > 0
|
|
367
|
+
return list_all_tables(schema: schema, conn_string: conn_string)
|
|
368
|
+
end
|
|
369
|
+
""
|
|
370
|
+
end
|
|
324
371
|
end
|
|
325
372
|
end
|
|
326
373
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pg_easy_replicate
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Shayon Mukherjee
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2023-06-
|
|
11
|
+
date: 2023-06-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: ougai
|