pg_easy_replicate 0.2.6 → 0.2.7

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 81be66e36a67bb505548d91dabf061ad83724c6d849bb611701769ffe3e7abc8
4
- data.tar.gz: 2223eb05e483c2481a68b157e75f9aca9fd62f5205de8fe5664ce91364bbaf73
3
+ metadata.gz: 90a3489ff7f33639118b99b142c2dcdef491ad384df80de0cceee89b706a9073
4
+ data.tar.gz: 0ab288970a9eb6465878ef03aca0d67f7543426a4cc744a4e47592ac200f5b68
5
5
  SHA512:
6
- metadata.gz: e3a93263182fc9c68c5b417d5512b021753ef356efb2e6c11ce2098e4c49b2ea41f21533f9aa590032a8eee0663a8ded1633f2d87399a11bd4c150da56c8f19e
7
- data.tar.gz: a24dc626daa748db907b07f8542008f2d5b6de558f6b80aa6f5879435a3eac92a6282e3a6bebc6d637502967adf3b77e9c57d8e67bd3a58136ca2c1f871b85be
6
+ metadata.gz: e487ac3b147855379c230d3b792d702d42776eb254f227cf31a8ebc5dee677cbf47b2efd7a9c0c218a87f064b1449e3c286d1dbc6c0f50f6b108cee3b1a5ccaa
7
+ data.tar.gz: 5ce88fff42f5ba68977b15261376d48e23173d210eddd1beccdad148cafab45f8e760a09d37991cef491313b4dee12dfa761b495fa1b90e8348c2b89ed2c3a6a
data/CHANGELOG.md CHANGED
@@ -1,4 +1,10 @@
1
- ## [0.2.4] - 2024-04-14
1
+ ## [0.2.6] - 2024-06-04
2
+
3
+ - Quote table name in the VACUUM SQL - #118
4
+ - Exclude tables created by extensions - #120
5
+ - Use db user when adding tables to replication - #130
6
+
7
+ ## [0.2.5] - 2024-04-14
2
8
 
3
9
  - List only permanent tables - #113
4
10
 
data/Gemfile.lock CHANGED
@@ -1,10 +1,10 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- pg_easy_replicate (0.2.6)
4
+ pg_easy_replicate (0.2.7)
5
5
  ougai (~> 2.0.0)
6
6
  pg (~> 1.5.3)
7
- sequel (>= 5.69, < 5.82)
7
+ sequel (>= 5.69, < 5.83)
8
8
  thor (>= 1.2.2, < 1.4.0)
9
9
 
10
10
  GEM
@@ -24,11 +24,11 @@ GEM
24
24
  oj (3.14.3)
25
25
  ougai (2.0.0)
26
26
  oj (~> 3.10)
27
- parallel (1.24.0)
28
- parser (3.3.2.0)
27
+ parallel (1.25.1)
28
+ parser (3.3.3.0)
29
29
  ast (~> 2.4.1)
30
30
  racc
31
- pg (1.5.6)
31
+ pg (1.5.7)
32
32
  prettier_print (1.2.1)
33
33
  pry (0.14.2)
34
34
  coderay (~> 1.1)
@@ -38,8 +38,8 @@ GEM
38
38
  rake (13.2.1)
39
39
  rbs (3.1.0)
40
40
  regexp_parser (2.9.2)
41
- rexml (3.2.8)
42
- strscan (>= 3.0.9)
41
+ rexml (3.3.3)
42
+ strscan
43
43
  rspec (3.13.0)
44
44
  rspec-core (~> 3.13.0)
45
45
  rspec-expectations (~> 3.13.0)
@@ -72,7 +72,7 @@ GEM
72
72
  rubocop (~> 1.41)
73
73
  rubocop-packaging (0.5.2)
74
74
  rubocop (>= 1.33, < 2.0)
75
- rubocop-performance (1.21.0)
75
+ rubocop-performance (1.21.1)
76
76
  rubocop (>= 1.48.1, < 2.0)
77
77
  rubocop-ast (>= 1.31.1, < 2.0)
78
78
  rubocop-rake (0.6.0)
@@ -85,7 +85,7 @@ GEM
85
85
  rubocop-rspec_rails (2.28.2)
86
86
  rubocop (~> 1.40)
87
87
  ruby-progressbar (1.13.0)
88
- sequel (5.81.0)
88
+ sequel (5.82.0)
89
89
  bigdecimal
90
90
  strscan (3.1.0)
91
91
  syntax_tree (6.2.0)
data/README.md CHANGED
@@ -69,8 +69,11 @@ All [Logical Replication Restrictions](https://www.postgresql.org/docs/current/l
69
69
 
70
70
  ## Usage
71
71
 
72
- Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment. The URL are of the postgres connection string format. Example:
72
+ Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment.
73
+ - `SOURCE_DB_URL` = The database that you want to replicate FROM.
74
+ - `TARGET_DB_URL` = The database that you want to replicate TO.
73
75
 
76
+ The URL should be in postgres connection string format. Example:
74
77
  ```bash
75
78
  $ export SOURCE_DB_URL="postgres://USERNAME:PASSWORD@localhost:5432/DATABASE_NAME"
76
79
  $ export TARGET_DB_URL="postgres://USERNAME:PASSWORD@localhost:5433/DATABASE_NAME"
@@ -84,6 +87,12 @@ You can extend the default timeout by setting the following environment variable
84
87
  $ export PG_EASY_REPLICATE_STATEMENT_TIMEOUT="10s" # default 5s
85
88
  ```
86
89
 
90
+ You can get additional debug logging by adding the following environment variable
91
+
92
+ ```bash
93
+ $ export DEBUG="true"
94
+ ```
95
+
87
96
  Any `pg_easy_replicate` command can be run the same way with the docker image as well. As long the container is running in an environment where it has access to both the databases. Example
88
97
 
89
98
  ```bash
@@ -247,6 +256,30 @@ $ pg_easy_replicate switchover --group-name database-cluster-2
247
256
  ...
248
257
  ```
249
258
 
259
+ ## Exclude tables from replication
260
+
261
+ By default all tables are added for replication but you can exclude tables if necessary. Example
262
+
263
+ ```bash
264
+ ...
265
+ $ pg_easy_replicate bootstrap --group-name database-cluster-1 --copy-schema
266
+ $ pg_easy_replicate start_sync --group-name database-cluster-1 --schema-name public --exclude_tables "events"
267
+ ...
268
+ ```
269
+
270
+ ### Cleanup
271
+
272
+ Use `cleanup` if you want to remove all bootstrapped data for the specified group. Additionally you can pass `-e` or `--everything` in order to clean up all schema changes for bootstrapped tables, users and any publication/subscription data.
273
+
274
+ ```bash
275
+ $ pg_easy_replicate cleanup --group-name database-cluster-1 --everything
276
+
277
+ {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping groups table","version":"0.1.0"}
278
+
279
+ {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping schema","version":"0.1.0"}
280
+ ...
281
+ ```
282
+
250
283
  ## Switchover strategies with minimal downtime
251
284
 
252
285
  For minimal downtime, it'd be best to watch/tail the stats and wait until `switchover_completed_at` is updated with a timestamp. Once that happens you can perform any of the following strategies. Note: These are just suggestions and `pg_easy_replicate` doesn't provide any functionalities for this.
@@ -21,6 +21,11 @@ module PgEasyReplicate
21
21
  default: "",
22
22
  desc:
23
23
  "Comma separated list of table names. Default: All tables"
24
+ method_option :exclude_tables,
25
+ aliases: "-e",
26
+ default: "",
27
+ desc:
28
+ "Comma separated list of table names to exclude. Default: None"
24
29
  method_option :schema_name,
25
30
  aliases: "-s",
26
31
  desc:
@@ -30,6 +35,7 @@ module PgEasyReplicate
30
35
  special_user_role: options[:special_user_role],
31
36
  copy_schema: options[:copy_schema],
32
37
  tables: options[:tables],
38
+ exclude_tables: options[:exclude_tables],
33
39
  schema_name: options[:schema_name],
34
40
  )
35
41
 
@@ -91,6 +97,11 @@ module PgEasyReplicate
91
97
  default: "",
92
98
  desc:
93
99
  "Comma separated list of table names. Default: All tables"
100
+ method_option :exclude_tables,
101
+ aliases: "-e",
102
+ default: "",
103
+ desc:
104
+ "Comma separated list of table names to exclude. Default: None"
94
105
  method_option :recreate_indices_post_copy,
95
106
  type: :boolean,
96
107
  default: true,
@@ -73,14 +73,18 @@ module PgEasyReplicate
73
73
  abort(msg)
74
74
  end
75
75
 
76
- def determine_tables(conn_string:, list: "", schema: nil)
76
+ def determine_tables(conn_string:, list: "", exclude_list: "", schema: nil)
77
77
  schema ||= "public"
78
-
79
- tables = list&.split(",") || []
80
- if tables.size > 0
81
- tables
78
+
79
+ tables = convert_to_array(list)
80
+ exclude_tables = convert_to_array(exclude_list)
81
+ validate_table_lists(tables, exclude_tables, schema)
82
+
83
+ if tables.empty?
84
+ all_tables = list_all_tables(schema: schema, conn_string: conn_string)
85
+ all_tables - (exclude_tables + %w[spatial_ref_sys])
82
86
  else
83
- list_all_tables(schema: schema, conn_string: conn_string) - %w[ spatial_ref_sys ]
87
+ tables
84
88
  end
85
89
  end
86
90
 
@@ -101,5 +105,24 @@ module PgEasyReplicate
101
105
  .map(&:values)
102
106
  .flatten
103
107
  end
108
+
109
+ def convert_to_array(input)
110
+ input.is_a?(Array) ? input : input&.split(",") || []
111
+ end
112
+
113
+ def validate_table_lists(tables, exclude_tables, schema_name)
114
+ table_list = convert_to_array(tables)
115
+ exclude_table_list = convert_to_array(exclude_tables)
116
+
117
+ if !table_list.empty? && !exclude_table_list.empty?
118
+ abort_with("Options --tables(-t) and --exclude-tables(-e) cannot be used together.")
119
+ elsif !table_list.empty?
120
+ if table_list.size > 0 && (schema_name.nil? || schema_name == "")
121
+ abort_with("Schema name is required if tables are passed")
122
+ end
123
+ elsif exclude_table_list.size > 0 && (schema_name.nil? || schema_name == "")
124
+ abort_with("Schema name is required if exclude tables are passed")
125
+ end
126
+ end
104
127
  end
105
128
  end
@@ -15,6 +15,7 @@ module PgEasyReplicate
15
15
  schema: schema_name,
16
16
  conn_string: source_db_url,
17
17
  list: options[:tables],
18
+ exclude_list: options[:exclude_tables]
18
19
  )
19
20
 
20
21
  if options[:recreate_indices_post_copy]
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PgEasyReplicate
4
- VERSION = "0.2.6"
4
+ VERSION = "0.2.7"
5
5
  end
@@ -30,11 +30,18 @@ module PgEasyReplicate
30
30
  special_user_role: nil,
31
31
  copy_schema: false,
32
32
  tables: "",
33
+ exclude_tables: "",
33
34
  schema_name: nil
34
35
  )
35
36
  abort_with("SOURCE_DB_URL is missing") if source_db_url.nil?
36
37
  abort_with("TARGET_DB_URL is missing") if target_db_url.nil?
37
38
 
39
+ if !tables.empty? && !exclude_tables.empty?
40
+ abort_with(
41
+ "Options --tables(-t) and --exclude-tables(-e) cannot be used together.",
42
+ )
43
+ end
44
+
38
45
  system("which pg_dump")
39
46
  pg_dump_exists = $CHILD_STATUS.success?
40
47
 
@@ -65,6 +72,7 @@ module PgEasyReplicate
65
72
  tables_have_replica_identity?(
66
73
  conn_string: source_db_url,
67
74
  tables: tables,
75
+ exclude_tables: exclude_tables,
68
76
  schema_name: schema_name,
69
77
  ),
70
78
  }
@@ -77,6 +85,7 @@ module PgEasyReplicate
77
85
  special_user_role: nil,
78
86
  copy_schema: false,
79
87
  tables: "",
88
+ exclude_tables: "",
80
89
  schema_name: nil
81
90
  )
82
91
  config_hash =
@@ -84,6 +93,7 @@ module PgEasyReplicate
84
93
  special_user_role: special_user_role,
85
94
  copy_schema: copy_schema,
86
95
  tables: tables,
96
+ exclude_tables: exclude_tables,
87
97
  schema_name: schema_name,
88
98
  )
89
99
 
@@ -103,9 +113,7 @@ module PgEasyReplicate
103
113
  abort_with("User on source database does not have super user privilege")
104
114
  end
105
115
 
106
- if tables.split(",").size > 0 && (schema_name.nil? || schema_name == "")
107
- abort_with("Schema name is required if tables are passed")
108
- end
116
+ validate_table_lists(tables, exclude_tables, schema_name)
109
117
 
110
118
  unless config_hash.dig(:tables_have_replica_identity)
111
119
  abort_with(
@@ -152,34 +160,58 @@ module PgEasyReplicate
152
160
  end
153
161
 
154
162
  def cleanup(options)
155
- logger.info("Dropping groups table")
156
- Group.drop
157
-
158
- if options[:everything]
159
- logger.info("Dropping schema")
160
- drop_internal_schema
161
- end
162
-
163
- if options[:everything] || options[:sync]
164
- Orchestrate.drop_publication(
165
- group_name: options[:group_name],
166
- conn_string: source_db_url,
167
- )
168
-
169
- Orchestrate.drop_subscription(
170
- group_name: options[:group_name],
171
- target_conn_string: target_db_url,
163
+ cleanup_steps = [
164
+ -> do
165
+ logger.info("Dropping groups table")
166
+ Group.drop
167
+ end,
168
+ -> do
169
+ if options[:everything]
170
+ logger.info("Dropping schema")
171
+ drop_internal_schema
172
+ end
173
+ end,
174
+ -> do
175
+ if options[:everything] || options[:sync]
176
+ logger.info("Dropping publication on source database")
177
+ Orchestrate.drop_publication(
178
+ group_name: options[:group_name],
179
+ conn_string: source_db_url,
180
+ )
181
+ end
182
+ end,
183
+ -> do
184
+ if options[:everything] || options[:sync]
185
+ logger.info("Dropping subscription on target database")
186
+ Orchestrate.drop_subscription(
187
+ group_name: options[:group_name],
188
+ target_conn_string: target_db_url,
189
+ )
190
+ end
191
+ end,
192
+ -> do
193
+ if options[:everything]
194
+ logger.info("Dropping replication user on source database")
195
+ drop_user(conn_string: source_db_url)
196
+ end
197
+ end,
198
+ -> do
199
+ if options[:everything]
200
+ logger.info("Dropping replication user on target database")
201
+ drop_user(conn_string: target_db_url)
202
+ end
203
+ end,
204
+ ]
205
+
206
+ cleanup_steps.each do |step|
207
+ step.call
208
+ rescue => e
209
+ logger.warn(
210
+ "Part of the cleanup step failed with #{e.message}. Continuing...",
172
211
  )
173
212
  end
174
213
 
175
- if options[:everything]
176
- # Drop users at last
177
- logger.info("Dropping replication user on source database")
178
- drop_user(conn_string: source_db_url)
179
-
180
- logger.info("Dropping replication user on target database")
181
- drop_user(conn_string: target_db_url)
182
- end
214
+ logger.info("Cleanup process completed.")
183
215
  rescue => e
184
216
  abort_with("Unable to cleanup: #{e.message}")
185
217
  end
@@ -299,9 +331,9 @@ module PgEasyReplicate
299
331
  special_user_role: nil,
300
332
  grant_permissions_on_schema: false
301
333
  )
302
- password = connection_info(conn_string)[:password].gsub("'") { "''" }
334
+ return if user_exists?(conn_string: conn_string, user: internal_user_name)
303
335
 
304
- drop_user(conn_string: conn_string)
336
+ password = connection_info(conn_string)[:password].gsub("'") { "''" }
305
337
 
306
338
  sql = <<~SQL
307
339
  create role #{quote_ident(internal_user_name)} with password '#{password}' login createdb createrole;
@@ -390,6 +422,7 @@ module PgEasyReplicate
390
422
  def tables_have_replica_identity?(
391
423
  conn_string:,
392
424
  tables: "",
425
+ exclude_tables: "",
393
426
  schema_name: nil
394
427
  )
395
428
  schema_name ||= "public"
@@ -399,6 +432,7 @@ module PgEasyReplicate
399
432
  schema: schema_name,
400
433
  conn_string: source_db_url,
401
434
  list: tables,
435
+ exclude_list: exclude_tables,
402
436
  )
403
437
  return false if table_list.empty?
404
438
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pg_easy_replicate
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.6
4
+ version: 0.2.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shayon Mukherjee
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-06-04 00:00:00.000000000 Z
11
+ date: 2024-08-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ougai
@@ -47,7 +47,7 @@ dependencies:
47
47
  version: '5.69'
48
48
  - - "<"
49
49
  - !ruby/object:Gem::Version
50
- version: '5.82'
50
+ version: '5.83'
51
51
  type: :runtime
52
52
  prerelease: false
53
53
  version_requirements: !ruby/object:Gem::Requirement
@@ -57,7 +57,7 @@ dependencies:
57
57
  version: '5.69'
58
58
  - - "<"
59
59
  - !ruby/object:Gem::Version
60
- version: '5.82'
60
+ version: '5.83'
61
61
  - !ruby/object:Gem::Dependency
62
62
  name: thor
63
63
  requirement: !ruby/object:Gem::Requirement