pg_easy_replicate 0.1.12 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/CHANGELOG.md +9 -0
- data/Gemfile.lock +4 -4
- data/README.md +8 -2
- data/assets/mascot.png +0 -0
- data/lib/pg_easy_replicate/cli.rb +6 -0
- data/lib/pg_easy_replicate/group.rb +1 -0
- data/lib/pg_easy_replicate/index_manager.rb +94 -0
- data/lib/pg_easy_replicate/orchestrate.rb +25 -0
- data/lib/pg_easy_replicate/version.rb +1 -1
- data/lib/pg_easy_replicate.rb +2 -1
- data/scripts/e2e-start.sh +1 -1
- metadata +13 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da825eb796e5b827c9197440301d841f725a5bbaeac74b066fea67b5bbf039ac
|
4
|
+
data.tar.gz: 68da98e57eab118b7aea8f4e83171eaa8b18bd66a86bd5f39fb901f5aef9bfda
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e04ca73e6f6b63f68a37389cb788bc7a6fddd80aa75fa433d297d4a600b1929ee06341e456b8b2d12a0cb03952824c02dca63033f40675948c250bcba2849b2a
|
7
|
+
data.tar.gz: b0617c3d5d2defeab8f576eb6cceb1845c8c7fe7478a064f2109fbd4e63ebcf58867bd2557221810c53778e62806bacd65e92470fd368f96c5d38405bee4efec
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
## [0.1.12] - 2023-12-13
|
2
|
+
|
3
|
+
- Bump rubocop-rspec from 2.24.1 to 2.25.0 - #65
|
4
|
+
- Quote indent DB name - #76
|
5
|
+
|
6
|
+
## [0.1.12] - 2023-12-13
|
7
|
+
|
8
|
+
- Drop existing user with privileges when bootstrapping - #75
|
9
|
+
|
1
10
|
## [0.1.10] - 2023-12-12
|
2
11
|
|
3
12
|
- Reference the passed in URL and use source db url - #74
|
data/Gemfile.lock
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pg_easy_replicate (0.
|
4
|
+
pg_easy_replicate (0.2.0)
|
5
5
|
ougai (~> 2.0.0)
|
6
6
|
pg (~> 1.5.3)
|
7
7
|
sequel (>= 5.69, < 5.76)
|
8
|
-
thor (
|
8
|
+
thor (>= 1.2.2, < 1.4.0)
|
9
9
|
|
10
10
|
GEM
|
11
11
|
remote: https://rubygems.org/
|
@@ -35,7 +35,7 @@ GEM
|
|
35
35
|
method_source (~> 1.0)
|
36
36
|
racc (1.7.3)
|
37
37
|
rainbow (3.1.1)
|
38
|
-
rake (13.0
|
38
|
+
rake (13.1.0)
|
39
39
|
rbs (3.1.0)
|
40
40
|
regexp_parser (2.8.2)
|
41
41
|
rexml (3.2.6)
|
@@ -94,7 +94,7 @@ GEM
|
|
94
94
|
rbs
|
95
95
|
syntax_tree (>= 2.0.1)
|
96
96
|
temple (0.10.1)
|
97
|
-
thor (1.
|
97
|
+
thor (1.3.0)
|
98
98
|
tilt (2.1.0)
|
99
99
|
unicode-display_width (2.5.0)
|
100
100
|
|
data/README.md
CHANGED
@@ -8,6 +8,8 @@
|
|
8
8
|
|
9
9
|
Battle tested in production at [Tines](https://www.tines.com/) 🚀
|
10
10
|
|
11
|
+

|
12
|
+
|
11
13
|
- [Installation](#installation)
|
12
14
|
- [Requirements](#requirements)
|
13
15
|
- [Limits](#limits)
|
@@ -57,7 +59,7 @@ https://hub.docker.com/r/shayonj/pg_easy_replicate
|
|
57
59
|
## Requirements
|
58
60
|
|
59
61
|
- PostgreSQL 10 and later
|
60
|
-
- Ruby
|
62
|
+
- Ruby 3.0 and later
|
61
63
|
- Database users should have `SUPERUSER` permissions, or pass in a special user with privileges to create the needed role, schema, publication and subscription on both databases. More on `--special-user-role` section below.
|
62
64
|
|
63
65
|
## Limits
|
@@ -151,6 +153,8 @@ $ pg_easy_replicate bootstrap --group-name database-cluster-1 --special-user-rol
|
|
151
153
|
|
152
154
|
Once the bootstrap is complete, you can start the sync. Starting the sync sets up the publication, subscription and performs other minor housekeeping things.
|
153
155
|
|
156
|
+
**NOTE**: Start sync by default will drop all indices in the target database for performance reasons. And will automatically re-add the indices during `switchover`. It is turned on by default and you can opt out of this with `--no-recreate-indices-post-copy`
|
157
|
+
|
154
158
|
```bash
|
155
159
|
$ pg_easy_replicate start_sync --group-name database-cluster-1
|
156
160
|
|
@@ -203,7 +207,9 @@ $ pg_easy_replicate stats --group-name database-cluster-1
|
|
203
207
|
|
204
208
|
`switchover` will wait until all tables in the group are replicating and the delta for lag is <200kb (by calculating the `pg_wal_lsn_diff` between `sent_lsn` and `write_lsn`) and then perform the switch.
|
205
209
|
|
206
|
-
|
210
|
+
Additionally, `switchover` will take care of re-adding the indices (it had removed in `start_sync`) in the target database before hand. Depending on the size of the tables, the recreation of indexes (which happens `CONCURRENTLY`) may take a while. See `start_sync` for more details.
|
211
|
+
|
212
|
+
The switch is made by putting the user on the source database in `READ ONLY` mode, so that it is not accepting any more writes and waits for the flush lag to be `0`. It’s up to the user to kick off a rolling restart of their application containers or failover DNS (more on these below in strategies) after the switchover is complete, so that your application isn't sending any read + write requests to the old/source database.
|
207
213
|
|
208
214
|
```bash
|
209
215
|
$ pg_easy_replicate switchover --group-name database-cluster-1
|
data/assets/mascot.png
ADDED
Binary file
|
@@ -79,6 +79,12 @@ module PgEasyReplicate
|
|
79
79
|
aliases: "-t",
|
80
80
|
desc:
|
81
81
|
"Comma separated list of table names. Default: All tables"
|
82
|
+
method_option :recreate_indices_post_copy,
|
83
|
+
type: :boolean,
|
84
|
+
default: true,
|
85
|
+
aliases: "-r",
|
86
|
+
desc:
|
87
|
+
"Drop all non-primary indices before copy and recreate them post-copy"
|
82
88
|
def start_sync
|
83
89
|
PgEasyReplicate::Orchestrate.start_sync(options)
|
84
90
|
end
|
@@ -20,6 +20,7 @@ module PgEasyReplicate
|
|
20
20
|
column(:updated_at, Time, default: Sequel::CURRENT_TIMESTAMP)
|
21
21
|
column(:started_at, Time)
|
22
22
|
column(:failed_at, Time)
|
23
|
+
column(:recreate_indices_post_copy, TrueClass, default: true)
|
23
24
|
column(:switchover_completed_at, Time)
|
24
25
|
end
|
25
26
|
ensure
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PgEasyReplicate
|
4
|
+
module IndexManager
|
5
|
+
extend Helper
|
6
|
+
|
7
|
+
def self.drop_indices(
|
8
|
+
source_conn_string:,
|
9
|
+
target_conn_string:,
|
10
|
+
tables:,
|
11
|
+
schema:
|
12
|
+
)
|
13
|
+
logger.info("Dropping indices from target database")
|
14
|
+
|
15
|
+
fetch_indices(
|
16
|
+
conn_string: source_conn_string,
|
17
|
+
tables: tables,
|
18
|
+
schema: schema,
|
19
|
+
).each do |index|
|
20
|
+
drop_sql = "DROP INDEX CONCURRENTLY #{index[:index_name]};"
|
21
|
+
|
22
|
+
Query.run(
|
23
|
+
query: drop_sql,
|
24
|
+
connection_url: target_conn_string,
|
25
|
+
schema: schema,
|
26
|
+
transaction: false,
|
27
|
+
)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.recreate_indices(
|
32
|
+
source_conn_string:,
|
33
|
+
target_conn_string:,
|
34
|
+
tables:,
|
35
|
+
schema:
|
36
|
+
)
|
37
|
+
logger.info("Recreating indices on target database")
|
38
|
+
|
39
|
+
indices =
|
40
|
+
fetch_indices(
|
41
|
+
conn_string: source_conn_string,
|
42
|
+
tables: tables,
|
43
|
+
schema: schema,
|
44
|
+
)
|
45
|
+
indices.each do |index|
|
46
|
+
create_sql =
|
47
|
+
"#{index[:index_definition].gsub("CREATE INDEX", "CREATE INDEX CONCURRENTLY IF NOT EXISTS")};"
|
48
|
+
|
49
|
+
Query.run(
|
50
|
+
query: create_sql,
|
51
|
+
connection_url: target_conn_string,
|
52
|
+
schema: schema,
|
53
|
+
transaction: false,
|
54
|
+
)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.fetch_indices(conn_string:, tables:, schema:)
|
59
|
+
return [] if tables.split(",").empty?
|
60
|
+
table_list = tables.split(",").map { |table| "'#{table}'" }.join(",")
|
61
|
+
|
62
|
+
sql = <<-SQL
|
63
|
+
SELECT
|
64
|
+
t.relname AS table_name,
|
65
|
+
i.relname AS index_name,
|
66
|
+
pg_get_indexdef(i.oid) AS index_definition
|
67
|
+
FROM
|
68
|
+
pg_class t,
|
69
|
+
pg_class i,
|
70
|
+
pg_index ix,
|
71
|
+
pg_namespace n
|
72
|
+
WHERE
|
73
|
+
t.oid = ix.indrelid
|
74
|
+
AND i.oid = ix.indexrelid
|
75
|
+
AND n.oid = t.relnamespace
|
76
|
+
AND t.relkind = 'r' -- only find indexes of tables
|
77
|
+
AND ix.indisprimary = FALSE -- exclude primary keys
|
78
|
+
AND n.nspname = '#{schema}'
|
79
|
+
AND t.relname IN (#{table_list})
|
80
|
+
ORDER BY
|
81
|
+
t.relname,
|
82
|
+
i.relname;
|
83
|
+
SQL
|
84
|
+
Query.run(query: sql, connection_url: conn_string, schema: schema)
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.wait_for_replication_completion(group_name:)
|
88
|
+
loop do
|
89
|
+
break if Stats.all_tables_replicating?(group_name)
|
90
|
+
sleep(5)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -17,6 +17,15 @@ module PgEasyReplicate
|
|
17
17
|
list: options[:tables],
|
18
18
|
)
|
19
19
|
|
20
|
+
if options[:recreate_indices_post_copy]
|
21
|
+
IndexManager.drop_indices(
|
22
|
+
source_conn_string: source_db_url,
|
23
|
+
target_conn_string: target_db_url,
|
24
|
+
tables: tables,
|
25
|
+
schema: schema_name,
|
26
|
+
)
|
27
|
+
end
|
28
|
+
|
20
29
|
create_publication(
|
21
30
|
group_name: options[:group_name],
|
22
31
|
conn_string: source_db_url,
|
@@ -40,6 +49,7 @@ module PgEasyReplicate
|
|
40
49
|
table_names: tables,
|
41
50
|
schema_name: schema_name,
|
42
51
|
started_at: Time.now.utc,
|
52
|
+
recreate_indices_post_copy: options[:recreate_indices_post_copy],
|
43
53
|
)
|
44
54
|
rescue => e
|
45
55
|
stop_sync(
|
@@ -221,7 +231,22 @@ module PgEasyReplicate
|
|
221
231
|
tables: group[:table_names],
|
222
232
|
schema: group[:schema_name],
|
223
233
|
)
|
234
|
+
|
224
235
|
watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
|
236
|
+
|
237
|
+
if group[:recreate_indices_post_copy]
|
238
|
+
IndexManager.wait_for_replication_completion(group_name: group_name)
|
239
|
+
IndexManager.recreate_indices(
|
240
|
+
source_conn_string: source_db_url,
|
241
|
+
target_conn_string: target_db_url,
|
242
|
+
tables: group[:table_names],
|
243
|
+
schema: group[:schema_name],
|
244
|
+
)
|
245
|
+
end
|
246
|
+
|
247
|
+
# Watch for lag again, because it could've grown during index recreation
|
248
|
+
watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG)
|
249
|
+
|
225
250
|
revoke_connections_on_source_db(group_name)
|
226
251
|
wait_for_remaining_catchup(group_name)
|
227
252
|
refresh_sequences(
|
data/lib/pg_easy_replicate.rb
CHANGED
@@ -10,6 +10,7 @@ require "English"
|
|
10
10
|
require "pg_easy_replicate/helper"
|
11
11
|
require "pg_easy_replicate/version"
|
12
12
|
require "pg_easy_replicate/query"
|
13
|
+
require "pg_easy_replicate/index_manager"
|
13
14
|
require "pg_easy_replicate/orchestrate"
|
14
15
|
require "pg_easy_replicate/stats"
|
15
16
|
require "pg_easy_replicate/group"
|
@@ -90,7 +91,7 @@ module PgEasyReplicate
|
|
90
91
|
setup_internal_schema
|
91
92
|
|
92
93
|
if options[:copy_schema]
|
93
|
-
logger.info("Setting up schema on
|
94
|
+
logger.info("Setting up schema on target database")
|
94
95
|
copy_schema(
|
95
96
|
source_conn_string: source_db_url,
|
96
97
|
target_conn_string: target_db_url,
|
data/scripts/e2e-start.sh
CHANGED
@@ -13,6 +13,6 @@ export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR'
|
|
13
13
|
# Bootstrap and cleanup
|
14
14
|
echo "===== Performing Bootstrap and cleanup"
|
15
15
|
bundle exec bin/pg_easy_replicate bootstrap -g cluster-1 --copy-schema
|
16
|
-
bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public
|
16
|
+
bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public --recreate-indices-post-copy
|
17
17
|
bundle exec bin/pg_easy_replicate stats -g cluster-1
|
18
18
|
bundle exec bin/pg_easy_replicate switchover -g cluster-1
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_easy_replicate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shayon Mukherjee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-12-
|
11
|
+
date: 2023-12-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ougai
|
@@ -62,16 +62,22 @@ dependencies:
|
|
62
62
|
name: thor
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
64
64
|
requirements:
|
65
|
-
- - "
|
65
|
+
- - ">="
|
66
66
|
- !ruby/object:Gem::Version
|
67
67
|
version: 1.2.2
|
68
|
+
- - "<"
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: 1.4.0
|
68
71
|
type: :runtime
|
69
72
|
prerelease: false
|
70
73
|
version_requirements: !ruby/object:Gem::Requirement
|
71
74
|
requirements:
|
72
|
-
- - "
|
75
|
+
- - ">="
|
73
76
|
- !ruby/object:Gem::Version
|
74
77
|
version: 1.2.2
|
78
|
+
- - "<"
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: 1.4.0
|
75
81
|
- !ruby/object:Gem::Dependency
|
76
82
|
name: prettier_print
|
77
83
|
requirement: !ruby/object:Gem::Requirement
|
@@ -262,6 +268,7 @@ files:
|
|
262
268
|
- LICENSE.txt
|
263
269
|
- README.md
|
264
270
|
- Rakefile
|
271
|
+
- assets/mascot.png
|
265
272
|
- bin/pg_easy_replicate
|
266
273
|
- bin/pg_easy_replicate_console
|
267
274
|
- docker-compose.yml
|
@@ -269,6 +276,7 @@ files:
|
|
269
276
|
- lib/pg_easy_replicate/cli.rb
|
270
277
|
- lib/pg_easy_replicate/group.rb
|
271
278
|
- lib/pg_easy_replicate/helper.rb
|
279
|
+
- lib/pg_easy_replicate/index_manager.rb
|
272
280
|
- lib/pg_easy_replicate/orchestrate.rb
|
273
281
|
- lib/pg_easy_replicate/query.rb
|
274
282
|
- lib/pg_easy_replicate/stats.rb
|
@@ -291,7 +299,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
291
299
|
requirements:
|
292
300
|
- - ">="
|
293
301
|
- !ruby/object:Gem::Version
|
294
|
-
version:
|
302
|
+
version: 3.0.0
|
295
303
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
296
304
|
requirements:
|
297
305
|
- - ">="
|