pg_easy_replicate 0.2.6 → 0.2.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -1
- data/Gemfile.lock +9 -9
- data/README.md +34 -1
- data/lib/pg_easy_replicate/cli.rb +11 -0
- data/lib/pg_easy_replicate/helper.rb +29 -6
- data/lib/pg_easy_replicate/orchestrate.rb +1 -0
- data/lib/pg_easy_replicate/version.rb +1 -1
- data/lib/pg_easy_replicate.rb +64 -30
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 90a3489ff7f33639118b99b142c2dcdef491ad384df80de0cceee89b706a9073
|
4
|
+
data.tar.gz: 0ab288970a9eb6465878ef03aca0d67f7543426a4cc744a4e47592ac200f5b68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e487ac3b147855379c230d3b792d702d42776eb254f227cf31a8ebc5dee677cbf47b2efd7a9c0c218a87f064b1449e3c286d1dbc6c0f50f6b108cee3b1a5ccaa
|
7
|
+
data.tar.gz: 5ce88fff42f5ba68977b15261376d48e23173d210eddd1beccdad148cafab45f8e760a09d37991cef491313b4dee12dfa761b495fa1b90e8348c2b89ed2c3a6a
|
data/CHANGELOG.md
CHANGED
@@ -1,4 +1,10 @@
|
|
1
|
-
## [0.2.
|
1
|
+
## [0.2.6] - 2024-06-04
|
2
|
+
|
3
|
+
- Quote table name in the VACUUM SQL - #118
|
4
|
+
- Exclude tables created by extensions - #120
|
5
|
+
- Use db user when adding tables to replication - #130
|
6
|
+
|
7
|
+
## [0.2.5] - 2024-04-14
|
2
8
|
|
3
9
|
- List only permanent tables - #113
|
4
10
|
|
data/Gemfile.lock
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pg_easy_replicate (0.2.
|
4
|
+
pg_easy_replicate (0.2.7)
|
5
5
|
ougai (~> 2.0.0)
|
6
6
|
pg (~> 1.5.3)
|
7
|
-
sequel (>= 5.69, < 5.
|
7
|
+
sequel (>= 5.69, < 5.83)
|
8
8
|
thor (>= 1.2.2, < 1.4.0)
|
9
9
|
|
10
10
|
GEM
|
@@ -24,11 +24,11 @@ GEM
|
|
24
24
|
oj (3.14.3)
|
25
25
|
ougai (2.0.0)
|
26
26
|
oj (~> 3.10)
|
27
|
-
parallel (1.
|
28
|
-
parser (3.3.
|
27
|
+
parallel (1.25.1)
|
28
|
+
parser (3.3.3.0)
|
29
29
|
ast (~> 2.4.1)
|
30
30
|
racc
|
31
|
-
pg (1.5.
|
31
|
+
pg (1.5.7)
|
32
32
|
prettier_print (1.2.1)
|
33
33
|
pry (0.14.2)
|
34
34
|
coderay (~> 1.1)
|
@@ -38,8 +38,8 @@ GEM
|
|
38
38
|
rake (13.2.1)
|
39
39
|
rbs (3.1.0)
|
40
40
|
regexp_parser (2.9.2)
|
41
|
-
rexml (3.
|
42
|
-
strscan
|
41
|
+
rexml (3.3.3)
|
42
|
+
strscan
|
43
43
|
rspec (3.13.0)
|
44
44
|
rspec-core (~> 3.13.0)
|
45
45
|
rspec-expectations (~> 3.13.0)
|
@@ -72,7 +72,7 @@ GEM
|
|
72
72
|
rubocop (~> 1.41)
|
73
73
|
rubocop-packaging (0.5.2)
|
74
74
|
rubocop (>= 1.33, < 2.0)
|
75
|
-
rubocop-performance (1.21.
|
75
|
+
rubocop-performance (1.21.1)
|
76
76
|
rubocop (>= 1.48.1, < 2.0)
|
77
77
|
rubocop-ast (>= 1.31.1, < 2.0)
|
78
78
|
rubocop-rake (0.6.0)
|
@@ -85,7 +85,7 @@ GEM
|
|
85
85
|
rubocop-rspec_rails (2.28.2)
|
86
86
|
rubocop (~> 1.40)
|
87
87
|
ruby-progressbar (1.13.0)
|
88
|
-
sequel (5.
|
88
|
+
sequel (5.82.0)
|
89
89
|
bigdecimal
|
90
90
|
strscan (3.1.0)
|
91
91
|
syntax_tree (6.2.0)
|
data/README.md
CHANGED
@@ -69,8 +69,11 @@ All [Logical Replication Restrictions](https://www.postgresql.org/docs/current/l
|
|
69
69
|
|
70
70
|
## Usage
|
71
71
|
|
72
|
-
Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment.
|
72
|
+
Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment.
|
73
|
+
- `SOURCE_DB_URL` = The database that you want to replicate FROM.
|
74
|
+
- `TARGET_DB_URL` = The database that you want to replicate TO.
|
73
75
|
|
76
|
+
The URL should be in postgres connection string format. Example:
|
74
77
|
```bash
|
75
78
|
$ export SOURCE_DB_URL="postgres://USERNAME:PASSWORD@localhost:5432/DATABASE_NAME"
|
76
79
|
$ export TARGET_DB_URL="postgres://USERNAME:PASSWORD@localhost:5433/DATABASE_NAME"
|
@@ -84,6 +87,12 @@ You can extend the default timeout by setting the following environment variable
|
|
84
87
|
$ export PG_EASY_REPLICATE_STATEMENT_TIMEOUT="10s" # default 5s
|
85
88
|
```
|
86
89
|
|
90
|
+
You can get additional debug logging by adding the following environment variable
|
91
|
+
|
92
|
+
```bash
|
93
|
+
$ export DEBUG="true"
|
94
|
+
```
|
95
|
+
|
87
96
|
Any `pg_easy_replicate` command can be run the same way with the docker image as well. As long the container is running in an environment where it has access to both the databases. Example
|
88
97
|
|
89
98
|
```bash
|
@@ -247,6 +256,30 @@ $ pg_easy_replicate switchover --group-name database-cluster-2
|
|
247
256
|
...
|
248
257
|
```
|
249
258
|
|
259
|
+
## Exclude tables from replication
|
260
|
+
|
261
|
+
By default all tables are added for replication but you can exclude tables if necessary. Example
|
262
|
+
|
263
|
+
```bash
|
264
|
+
...
|
265
|
+
$ pg_easy_replicate bootstrap --group-name database-cluster-1 --copy-schema
|
266
|
+
$ pg_easy_replicate start_sync --group-name database-cluster-1 --schema-name public --exclude_tables "events"
|
267
|
+
...
|
268
|
+
```
|
269
|
+
|
270
|
+
### Cleanup
|
271
|
+
|
272
|
+
Use `cleanup` if you want to remove all bootstrapped data for the specified group. Additionally you can pass `-e` or `--everything` in order to clean up all schema changes for bootstrapped tables, users and any publication/subscription data.
|
273
|
+
|
274
|
+
```bash
|
275
|
+
$ pg_easy_replicate cleanup --group-name database-cluster-1 --everything
|
276
|
+
|
277
|
+
{"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping groups table","version":"0.1.0"}
|
278
|
+
|
279
|
+
{"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping schema","version":"0.1.0"}
|
280
|
+
...
|
281
|
+
```
|
282
|
+
|
250
283
|
## Switchover strategies with minimal downtime
|
251
284
|
|
252
285
|
For minimal downtime, it'd be best to watch/tail the stats and wait until `switchover_completed_at` is updated with a timestamp. Once that happens you can perform any of the following strategies. Note: These are just suggestions and `pg_easy_replicate` doesn't provide any functionalities for this.
|
@@ -21,6 +21,11 @@ module PgEasyReplicate
|
|
21
21
|
default: "",
|
22
22
|
desc:
|
23
23
|
"Comma separated list of table names. Default: All tables"
|
24
|
+
method_option :exclude_tables,
|
25
|
+
aliases: "-e",
|
26
|
+
default: "",
|
27
|
+
desc:
|
28
|
+
"Comma separated list of table names to exclude. Default: None"
|
24
29
|
method_option :schema_name,
|
25
30
|
aliases: "-s",
|
26
31
|
desc:
|
@@ -30,6 +35,7 @@ module PgEasyReplicate
|
|
30
35
|
special_user_role: options[:special_user_role],
|
31
36
|
copy_schema: options[:copy_schema],
|
32
37
|
tables: options[:tables],
|
38
|
+
exclude_tables: options[:exclude_tables],
|
33
39
|
schema_name: options[:schema_name],
|
34
40
|
)
|
35
41
|
|
@@ -91,6 +97,11 @@ module PgEasyReplicate
|
|
91
97
|
default: "",
|
92
98
|
desc:
|
93
99
|
"Comma separated list of table names. Default: All tables"
|
100
|
+
method_option :exclude_tables,
|
101
|
+
aliases: "-e",
|
102
|
+
default: "",
|
103
|
+
desc:
|
104
|
+
"Comma separated list of table names to exclude. Default: None"
|
94
105
|
method_option :recreate_indices_post_copy,
|
95
106
|
type: :boolean,
|
96
107
|
default: true,
|
@@ -73,14 +73,18 @@ module PgEasyReplicate
|
|
73
73
|
abort(msg)
|
74
74
|
end
|
75
75
|
|
76
|
-
def determine_tables(conn_string:, list: "", schema: nil)
|
76
|
+
def determine_tables(conn_string:, list: "", exclude_list: "", schema: nil)
|
77
77
|
schema ||= "public"
|
78
|
-
|
79
|
-
tables = list
|
80
|
-
|
81
|
-
|
78
|
+
|
79
|
+
tables = convert_to_array(list)
|
80
|
+
exclude_tables = convert_to_array(exclude_list)
|
81
|
+
validate_table_lists(tables, exclude_tables, schema)
|
82
|
+
|
83
|
+
if tables.empty?
|
84
|
+
all_tables = list_all_tables(schema: schema, conn_string: conn_string)
|
85
|
+
all_tables - (exclude_tables + %w[spatial_ref_sys])
|
82
86
|
else
|
83
|
-
|
87
|
+
tables
|
84
88
|
end
|
85
89
|
end
|
86
90
|
|
@@ -101,5 +105,24 @@ module PgEasyReplicate
|
|
101
105
|
.map(&:values)
|
102
106
|
.flatten
|
103
107
|
end
|
108
|
+
|
109
|
+
def convert_to_array(input)
|
110
|
+
input.is_a?(Array) ? input : input&.split(",") || []
|
111
|
+
end
|
112
|
+
|
113
|
+
def validate_table_lists(tables, exclude_tables, schema_name)
|
114
|
+
table_list = convert_to_array(tables)
|
115
|
+
exclude_table_list = convert_to_array(exclude_tables)
|
116
|
+
|
117
|
+
if !table_list.empty? && !exclude_table_list.empty?
|
118
|
+
abort_with("Options --tables(-t) and --exclude-tables(-e) cannot be used together.")
|
119
|
+
elsif !table_list.empty?
|
120
|
+
if table_list.size > 0 && (schema_name.nil? || schema_name == "")
|
121
|
+
abort_with("Schema name is required if tables are passed")
|
122
|
+
end
|
123
|
+
elsif exclude_table_list.size > 0 && (schema_name.nil? || schema_name == "")
|
124
|
+
abort_with("Schema name is required if exclude tables are passed")
|
125
|
+
end
|
126
|
+
end
|
104
127
|
end
|
105
128
|
end
|
data/lib/pg_easy_replicate.rb
CHANGED
@@ -30,11 +30,18 @@ module PgEasyReplicate
|
|
30
30
|
special_user_role: nil,
|
31
31
|
copy_schema: false,
|
32
32
|
tables: "",
|
33
|
+
exclude_tables: "",
|
33
34
|
schema_name: nil
|
34
35
|
)
|
35
36
|
abort_with("SOURCE_DB_URL is missing") if source_db_url.nil?
|
36
37
|
abort_with("TARGET_DB_URL is missing") if target_db_url.nil?
|
37
38
|
|
39
|
+
if !tables.empty? && !exclude_tables.empty?
|
40
|
+
abort_with(
|
41
|
+
"Options --tables(-t) and --exclude-tables(-e) cannot be used together.",
|
42
|
+
)
|
43
|
+
end
|
44
|
+
|
38
45
|
system("which pg_dump")
|
39
46
|
pg_dump_exists = $CHILD_STATUS.success?
|
40
47
|
|
@@ -65,6 +72,7 @@ module PgEasyReplicate
|
|
65
72
|
tables_have_replica_identity?(
|
66
73
|
conn_string: source_db_url,
|
67
74
|
tables: tables,
|
75
|
+
exclude_tables: exclude_tables,
|
68
76
|
schema_name: schema_name,
|
69
77
|
),
|
70
78
|
}
|
@@ -77,6 +85,7 @@ module PgEasyReplicate
|
|
77
85
|
special_user_role: nil,
|
78
86
|
copy_schema: false,
|
79
87
|
tables: "",
|
88
|
+
exclude_tables: "",
|
80
89
|
schema_name: nil
|
81
90
|
)
|
82
91
|
config_hash =
|
@@ -84,6 +93,7 @@ module PgEasyReplicate
|
|
84
93
|
special_user_role: special_user_role,
|
85
94
|
copy_schema: copy_schema,
|
86
95
|
tables: tables,
|
96
|
+
exclude_tables: exclude_tables,
|
87
97
|
schema_name: schema_name,
|
88
98
|
)
|
89
99
|
|
@@ -103,9 +113,7 @@ module PgEasyReplicate
|
|
103
113
|
abort_with("User on source database does not have super user privilege")
|
104
114
|
end
|
105
115
|
|
106
|
-
|
107
|
-
abort_with("Schema name is required if tables are passed")
|
108
|
-
end
|
116
|
+
validate_table_lists(tables, exclude_tables, schema_name)
|
109
117
|
|
110
118
|
unless config_hash.dig(:tables_have_replica_identity)
|
111
119
|
abort_with(
|
@@ -152,34 +160,58 @@ module PgEasyReplicate
|
|
152
160
|
end
|
153
161
|
|
154
162
|
def cleanup(options)
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
163
|
+
cleanup_steps = [
|
164
|
+
-> do
|
165
|
+
logger.info("Dropping groups table")
|
166
|
+
Group.drop
|
167
|
+
end,
|
168
|
+
-> do
|
169
|
+
if options[:everything]
|
170
|
+
logger.info("Dropping schema")
|
171
|
+
drop_internal_schema
|
172
|
+
end
|
173
|
+
end,
|
174
|
+
-> do
|
175
|
+
if options[:everything] || options[:sync]
|
176
|
+
logger.info("Dropping publication on source database")
|
177
|
+
Orchestrate.drop_publication(
|
178
|
+
group_name: options[:group_name],
|
179
|
+
conn_string: source_db_url,
|
180
|
+
)
|
181
|
+
end
|
182
|
+
end,
|
183
|
+
-> do
|
184
|
+
if options[:everything] || options[:sync]
|
185
|
+
logger.info("Dropping subscription on target database")
|
186
|
+
Orchestrate.drop_subscription(
|
187
|
+
group_name: options[:group_name],
|
188
|
+
target_conn_string: target_db_url,
|
189
|
+
)
|
190
|
+
end
|
191
|
+
end,
|
192
|
+
-> do
|
193
|
+
if options[:everything]
|
194
|
+
logger.info("Dropping replication user on source database")
|
195
|
+
drop_user(conn_string: source_db_url)
|
196
|
+
end
|
197
|
+
end,
|
198
|
+
-> do
|
199
|
+
if options[:everything]
|
200
|
+
logger.info("Dropping replication user on target database")
|
201
|
+
drop_user(conn_string: target_db_url)
|
202
|
+
end
|
203
|
+
end,
|
204
|
+
]
|
205
|
+
|
206
|
+
cleanup_steps.each do |step|
|
207
|
+
step.call
|
208
|
+
rescue => e
|
209
|
+
logger.warn(
|
210
|
+
"Part of the cleanup step failed with #{e.message}. Continuing...",
|
172
211
|
)
|
173
212
|
end
|
174
213
|
|
175
|
-
|
176
|
-
# Drop users at last
|
177
|
-
logger.info("Dropping replication user on source database")
|
178
|
-
drop_user(conn_string: source_db_url)
|
179
|
-
|
180
|
-
logger.info("Dropping replication user on target database")
|
181
|
-
drop_user(conn_string: target_db_url)
|
182
|
-
end
|
214
|
+
logger.info("Cleanup process completed.")
|
183
215
|
rescue => e
|
184
216
|
abort_with("Unable to cleanup: #{e.message}")
|
185
217
|
end
|
@@ -299,9 +331,9 @@ module PgEasyReplicate
|
|
299
331
|
special_user_role: nil,
|
300
332
|
grant_permissions_on_schema: false
|
301
333
|
)
|
302
|
-
|
334
|
+
return if user_exists?(conn_string: conn_string, user: internal_user_name)
|
303
335
|
|
304
|
-
|
336
|
+
password = connection_info(conn_string)[:password].gsub("'") { "''" }
|
305
337
|
|
306
338
|
sql = <<~SQL
|
307
339
|
create role #{quote_ident(internal_user_name)} with password '#{password}' login createdb createrole;
|
@@ -390,6 +422,7 @@ module PgEasyReplicate
|
|
390
422
|
def tables_have_replica_identity?(
|
391
423
|
conn_string:,
|
392
424
|
tables: "",
|
425
|
+
exclude_tables: "",
|
393
426
|
schema_name: nil
|
394
427
|
)
|
395
428
|
schema_name ||= "public"
|
@@ -399,6 +432,7 @@ module PgEasyReplicate
|
|
399
432
|
schema: schema_name,
|
400
433
|
conn_string: source_db_url,
|
401
434
|
list: tables,
|
435
|
+
exclude_list: exclude_tables,
|
402
436
|
)
|
403
437
|
return false if table_list.empty?
|
404
438
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_easy_replicate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shayon Mukherjee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-08-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ougai
|
@@ -47,7 +47,7 @@ dependencies:
|
|
47
47
|
version: '5.69'
|
48
48
|
- - "<"
|
49
49
|
- !ruby/object:Gem::Version
|
50
|
-
version: '5.
|
50
|
+
version: '5.83'
|
51
51
|
type: :runtime
|
52
52
|
prerelease: false
|
53
53
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -57,7 +57,7 @@ dependencies:
|
|
57
57
|
version: '5.69'
|
58
58
|
- - "<"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '5.
|
60
|
+
version: '5.83'
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: thor
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|