pg_easy_replicate 0.2.2 → 0.2.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -1
- data/Gemfile.lock +16 -16
- data/README.md +14 -5
- data/lib/pg_easy_replicate/cli.rb +9 -2
- data/lib/pg_easy_replicate/orchestrate.rb +19 -12
- data/lib/pg_easy_replicate/query.rb +9 -3
- data/lib/pg_easy_replicate/version.rb +1 -1
- data/scripts/e2e-start.sh +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fa012c1a56df04d2aad4d61e901a7affb409c24480d8b93d1b2806a08963c93e
|
4
|
+
data.tar.gz: 04ac8a97564f1d179f46f83f16e1bac495b19c6c10fa3af6a8f94ec6252dec3b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f3accba4733b9eaeb5d1acde1f299a967f487f5b6dee20a9bab0ecc31f0a8f4622cbfef5d449fd05bcf1d0a4c9b396796ad3b2663bb078162bfd7f878fd1054a
|
7
|
+
data.tar.gz: 16961c9f095ca3491e1231fbd5b2b934362141bbddca12b13988b84656f5e89d5fc72a759085a9d5105722e413869eb12c0d12feaacd1c8938149012d2548363
|
data/CHANGELOG.md
CHANGED
@@ -1,4 +1,20 @@
|
|
1
|
-
## [0.2.
|
1
|
+
## [0.2.3] - 2024-01-21
|
2
|
+
|
3
|
+
- Fix tables check in config_check - #93
|
4
|
+
- add option to skip vacuum analyzing on switchover - #92
|
5
|
+
- Disable statement timeout and reset it before/after vacuum+analyze - #94
|
6
|
+
- Add spec for skip_vacuum_analyze - #95
|
7
|
+
|
8
|
+
Highlights
|
9
|
+
|
10
|
+
- You can now skip vacuum and analyze by passing `--skip-vacuum-analyze` to `switchover`. Thanks to @honzasterba
|
11
|
+
- Vacuum and Analyze won't run into timeouts. Thanks to the report from @TrueCarry
|
12
|
+
|
13
|
+
## [0.2.2] - 2024-01-21
|
14
|
+
|
15
|
+
- Extend config check to assert for REPLICA IDENTITY on tables and drop index bug - #88
|
16
|
+
|
17
|
+
## [0.2.1] - 2024-01-20
|
2
18
|
|
3
19
|
- Don't attempt to drop and recreate unique indices - #88
|
4
20
|
- Dependency updates
|
data/Gemfile.lock
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pg_easy_replicate (0.2.
|
4
|
+
pg_easy_replicate (0.2.4)
|
5
5
|
ougai (~> 2.0.0)
|
6
6
|
pg (~> 1.5.3)
|
7
|
-
sequel (>= 5.69, < 5.
|
7
|
+
sequel (>= 5.69, < 5.78)
|
8
8
|
thor (>= 1.2.2, < 1.4.0)
|
9
9
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
12
12
|
specs:
|
13
13
|
ast (2.4.2)
|
14
|
-
bigdecimal (3.1.
|
14
|
+
bigdecimal (3.1.6)
|
15
15
|
coderay (1.1.3)
|
16
|
-
diff-lcs (1.5.
|
16
|
+
diff-lcs (1.5.1)
|
17
17
|
haml (6.1.1)
|
18
18
|
temple (>= 0.8.2)
|
19
19
|
thor
|
@@ -39,19 +39,19 @@ GEM
|
|
39
39
|
rbs (3.1.0)
|
40
40
|
regexp_parser (2.9.0)
|
41
41
|
rexml (3.2.6)
|
42
|
-
rspec (3.
|
43
|
-
rspec-core (~> 3.
|
44
|
-
rspec-expectations (~> 3.
|
45
|
-
rspec-mocks (~> 3.
|
46
|
-
rspec-core (3.
|
47
|
-
rspec-support (~> 3.
|
48
|
-
rspec-expectations (3.
|
42
|
+
rspec (3.13.0)
|
43
|
+
rspec-core (~> 3.13.0)
|
44
|
+
rspec-expectations (~> 3.13.0)
|
45
|
+
rspec-mocks (~> 3.13.0)
|
46
|
+
rspec-core (3.13.0)
|
47
|
+
rspec-support (~> 3.13.0)
|
48
|
+
rspec-expectations (3.13.0)
|
49
49
|
diff-lcs (>= 1.2.0, < 2.0)
|
50
|
-
rspec-support (~> 3.
|
51
|
-
rspec-mocks (3.
|
50
|
+
rspec-support (~> 3.13.0)
|
51
|
+
rspec-mocks (3.13.0)
|
52
52
|
diff-lcs (>= 1.2.0, < 2.0)
|
53
|
-
rspec-support (~> 3.
|
54
|
-
rspec-support (3.
|
53
|
+
rspec-support (~> 3.13.0)
|
54
|
+
rspec-support (3.13.0)
|
55
55
|
rubocop (1.60.1)
|
56
56
|
json (~> 2.3)
|
57
57
|
language_server-protocol (>= 3.17.0)
|
@@ -81,7 +81,7 @@ GEM
|
|
81
81
|
rubocop-capybara (~> 2.17)
|
82
82
|
rubocop-factory_bot (~> 2.22)
|
83
83
|
ruby-progressbar (1.13.0)
|
84
|
-
sequel (5.
|
84
|
+
sequel (5.77.0)
|
85
85
|
bigdecimal
|
86
86
|
syntax_tree (6.2.0)
|
87
87
|
prettier_print (>= 1.2.0)
|
data/README.md
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
[![Smoke spec](https://github.com/shayonj/pg_easy_replicate/actions/workflows/smoke.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml)
|
5
5
|
[![Gem Version](https://badge.fury.io/rb/pg_easy_replicate.svg?2)](https://badge.fury.io/rb/pg_easy_replicate)
|
6
6
|
|
7
|
-
`pg_easy_replicate` is a CLI orchestrator tool that simplifies the process of setting up [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) between two PostgreSQL databases. `pg_easy_replicate` also supports switchover. After the source (primary database) is fully replicated, `pg_easy_replicate` puts it into read-only mode and via logical replication flushes all data to the new target database. This ensures zero data loss and minimal downtime for the application. This method can be useful for performing minimal downtime (up to <1min, depending) major version upgrades between
|
7
|
+
`pg_easy_replicate` is a CLI orchestrator tool that simplifies the process of setting up [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) between two PostgreSQL databases. `pg_easy_replicate` also supports switchover. After the source (primary database) is fully replicated, `pg_easy_replicate` puts it into read-only mode and via logical replication flushes all data to the new target database. This ensures zero data loss and minimal downtime for the application. This method can be useful for performing minimal downtime (up to <1min, depending) major version upgrades between a Blue/Green PostgreSQL database setup, load testing and other similar use cases.
|
8
8
|
|
9
9
|
Battle tested in production at [Tines](https://www.tines.com/) 🚀
|
10
10
|
|
@@ -19,7 +19,7 @@ Battle tested in production at [Tines](https://www.tines.com/) 🚀
|
|
19
19
|
- [Config check](#config-check)
|
20
20
|
- [Bootstrap](#bootstrap)
|
21
21
|
- [Bootstrap and Config Check with special user role in AWS or GCP](#bootstrap-and-config-check-with-special-user-role-in-aws-or-gcp)
|
22
|
-
- [Config Check](#config-check)
|
22
|
+
- [Config Check](#config-check-1)
|
23
23
|
- [Bootstrap](#bootstrap-1)
|
24
24
|
- [Start sync](#start-sync)
|
25
25
|
- [Stats](#stats)
|
@@ -29,7 +29,7 @@ Battle tested in production at [Tines](https://www.tines.com/) 🚀
|
|
29
29
|
- [Rolling restart strategy](#rolling-restart-strategy)
|
30
30
|
- [DNS Failover strategy](#dns-failover-strategy)
|
31
31
|
- [FAQ](#faq)
|
32
|
-
- [Adding internal user to pgBouncer `userlist`](#adding-internal-user-to-pgbouncer-userlist)
|
32
|
+
- [Adding internal user to `pg_hba` or pgBouncer `userlist`](#adding-internal-user-to-pg_hba-or-pgbouncer-userlist)
|
33
33
|
- [Contributing](#contributing)
|
34
34
|
|
35
35
|
## Installation
|
@@ -61,6 +61,7 @@ https://hub.docker.com/r/shayonj/pg_easy_replicate
|
|
61
61
|
- PostgreSQL 10 and later
|
62
62
|
- Ruby 3.0 and later
|
63
63
|
- Database users should have `SUPERUSER` permissions, or pass in a special user with privileges to create the needed role, schema, publication and subscription on both databases. More on `--special-user-role` section below.
|
64
|
+
- See more on [FAQ](#faq) below
|
64
65
|
|
65
66
|
## Limits
|
66
67
|
|
@@ -75,6 +76,14 @@ $ export SOURCE_DB_URL="postgres://USERNAME:PASSWORD@localhost:5432/DATABASE_NAM
|
|
75
76
|
$ export TARGET_DB_URL="postgres://USERNAME:PASSWORD@localhost:5433/DATABASE_NAME"
|
76
77
|
```
|
77
78
|
|
79
|
+
**Optional**
|
80
|
+
|
81
|
+
You can extend the default timeout by setting the following environment variable
|
82
|
+
|
83
|
+
```bash
|
84
|
+
$ export PG_EASY_REPLICATE_STATEMENT_TIMEOUT="10s" # default 5s
|
85
|
+
```
|
86
|
+
|
78
87
|
Any `pg_easy_replicate` command can be run the same way with the docker image as well. As long the container is running in an environment where it has access to both the databases. Example
|
79
88
|
|
80
89
|
```bash
|
@@ -256,9 +265,9 @@ Next, you can set up a program that watches the `stats` and waits until `switcho
|
|
256
265
|
|
257
266
|
## FAQ
|
258
267
|
|
259
|
-
### Adding internal user to pgBouncer `userlist`
|
268
|
+
### Adding internal user to `pg_hba` or pgBouncer `userlist`
|
260
269
|
|
261
|
-
`pg_easy_replicate`
|
270
|
+
`pg_easy_replicate` sets up a designated user for managing the replication process. In case you handle user permissions through `pg_hba`, it's necessary to modify this list to permit sessions from `pger_su_h1a4fb`. Similarly, with pgBouncer, you'll need to authorize `pger_su_h1a4fb` for login access by including it in the `userlist`.
|
262
271
|
|
263
272
|
## Contributing
|
264
273
|
|
@@ -18,6 +18,7 @@ module PgEasyReplicate
|
|
18
18
|
desc: "Copy schema to the new database"
|
19
19
|
method_option :tables,
|
20
20
|
aliases: "-t",
|
21
|
+
default: "",
|
21
22
|
desc:
|
22
23
|
"Comma separated list of table names. Default: All tables"
|
23
24
|
method_option :schema_name,
|
@@ -87,6 +88,7 @@ module PgEasyReplicate
|
|
87
88
|
"Name of the schema tables are in, only required if passing list of tables"
|
88
89
|
method_option :tables,
|
89
90
|
aliases: "-t",
|
91
|
+
default: "",
|
90
92
|
desc:
|
91
93
|
"Comma separated list of table names. Default: All tables"
|
92
94
|
method_option :recreate_indices_post_copy,
|
@@ -117,8 +119,12 @@ module PgEasyReplicate
|
|
117
119
|
desc: "Name of the group previously provisioned"
|
118
120
|
method_option :lag_delta_size,
|
119
121
|
aliases: "-l",
|
120
|
-
desc:
|
121
|
-
|
122
|
+
desc: "The size of the lag to watch for before switchover. Default 200KB."
|
123
|
+
method_option :skip_vacuum_analyze,
|
124
|
+
type: :boolean,
|
125
|
+
default: false,
|
126
|
+
aliases: "-s",
|
127
|
+
desc: "Skip vacuum analyzing tables before switchover."
|
122
128
|
# method_option :bi_directional,
|
123
129
|
# aliases: "-b",
|
124
130
|
# desc:
|
@@ -127,6 +133,7 @@ module PgEasyReplicate
|
|
127
133
|
PgEasyReplicate::Orchestrate.switchover(
|
128
134
|
group_name: options[:group_name],
|
129
135
|
lag_delta_size: options[:lag_delta_size],
|
136
|
+
skip_vacuum_analyze: options[:skip_vacuum_analyze]
|
130
137
|
)
|
131
138
|
end
|
132
139
|
|
@@ -204,16 +204,19 @@ module PgEasyReplicate
|
|
204
204
|
group_name:,
|
205
205
|
source_conn_string: source_db_url,
|
206
206
|
target_conn_string: target_db_url,
|
207
|
-
lag_delta_size: nil
|
207
|
+
lag_delta_size: nil,
|
208
|
+
skip_vacuum_analyze: false
|
208
209
|
)
|
209
210
|
group = Group.find(group_name)
|
210
211
|
tables_list = group[:table_names].split(",")
|
211
212
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
213
|
+
unless skip_vacuum_analyze
|
214
|
+
run_vacuum_analyze(
|
215
|
+
conn_string: target_conn_string,
|
216
|
+
tables: tables_list,
|
217
|
+
schema: group[:schema_name],
|
218
|
+
)
|
219
|
+
end
|
217
220
|
|
218
221
|
watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
|
219
222
|
|
@@ -238,11 +241,13 @@ module PgEasyReplicate
|
|
238
241
|
)
|
239
242
|
mark_switchover_complete(group_name)
|
240
243
|
# Run vacuum analyze to refresh the planner post switchover
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
244
|
+
unless skip_vacuum_analyze
|
245
|
+
run_vacuum_analyze(
|
246
|
+
conn_string: target_conn_string,
|
247
|
+
tables: tables_list,
|
248
|
+
schema: group[:schema_name],
|
249
|
+
)
|
250
|
+
end
|
246
251
|
drop_subscription(
|
247
252
|
group_name: group_name,
|
248
253
|
target_conn_string: target_conn_string,
|
@@ -358,11 +363,13 @@ module PgEasyReplicate
|
|
358
363
|
schema: schema,
|
359
364
|
table: t,
|
360
365
|
)
|
366
|
+
|
361
367
|
Query.run(
|
362
|
-
query: "VACUUM VERBOSE ANALYZE #{t}",
|
368
|
+
query: "VACUUM VERBOSE ANALYZE #{t};",
|
363
369
|
connection_url: conn_string,
|
364
370
|
schema: schema,
|
365
371
|
transaction: false,
|
372
|
+
using_vacuum_analyze: true,
|
366
373
|
)
|
367
374
|
end
|
368
375
|
rescue => e
|
@@ -10,20 +10,26 @@ module PgEasyReplicate
|
|
10
10
|
connection_url:,
|
11
11
|
user: internal_user_name,
|
12
12
|
schema: nil,
|
13
|
-
transaction: true
|
13
|
+
transaction: true,
|
14
|
+
using_vacuum_analyze: false
|
14
15
|
)
|
15
16
|
conn =
|
16
17
|
connect(connection_url: connection_url, schema: schema, user: user)
|
18
|
+
timeout ||= ENV["PG_EASY_REPLICATE_STATEMENT_TIMEOUT"] || "5s"
|
17
19
|
if transaction
|
18
20
|
r =
|
19
21
|
conn.transaction do
|
20
22
|
conn.run("SET search_path to #{quote_ident(schema)}") if schema
|
21
|
-
conn.run("SET statement_timeout to '
|
23
|
+
conn.run("SET statement_timeout to '#{timeout}'")
|
22
24
|
conn.fetch(query).to_a
|
23
25
|
end
|
24
26
|
else
|
25
27
|
conn.run("SET search_path to #{quote_ident(schema)}") if schema
|
26
|
-
|
28
|
+
if using_vacuum_analyze
|
29
|
+
conn.run("SET statement_timeout=0")
|
30
|
+
else
|
31
|
+
conn.run("SET statement_timeout to '5s'")
|
32
|
+
end
|
27
33
|
r = conn.fetch(query).to_a
|
28
34
|
end
|
29
35
|
conn.disconnect
|
data/scripts/e2e-start.sh
CHANGED
@@ -10,7 +10,7 @@ export SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhos
|
|
10
10
|
export TARGET_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5433/postgres-db"
|
11
11
|
export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR'
|
12
12
|
|
13
|
-
# Bootstrap and cleanup
|
13
|
+
# Config check, Bootstrap and cleanup
|
14
14
|
echo "===== Performing Bootstrap and cleanup"
|
15
15
|
bundle exec bin/pg_easy_replicate bootstrap -g cluster-1 --copy-schema
|
16
16
|
bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public --recreate-indices-post-copy
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_easy_replicate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shayon Mukherjee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-02-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ougai
|
@@ -47,7 +47,7 @@ dependencies:
|
|
47
47
|
version: '5.69'
|
48
48
|
- - "<"
|
49
49
|
- !ruby/object:Gem::Version
|
50
|
-
version: '5.
|
50
|
+
version: '5.78'
|
51
51
|
type: :runtime
|
52
52
|
prerelease: false
|
53
53
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -57,7 +57,7 @@ dependencies:
|
|
57
57
|
version: '5.69'
|
58
58
|
- - "<"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '5.
|
60
|
+
version: '5.78'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: thor
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|