pg_logical_replicator 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +159 -0
- data/bin/pg_logical_replicator +6 -0
- data/lib/pg_logical_replicator/cli.rb +144 -0
- data/lib/pg_logical_replicator/logical_replication_initializer.rb +145 -0
- data/lib/pg_logical_replicator/replication_stopper.rb +48 -0
- data/lib/pg_logical_replicator/schema_transfer.rb +57 -0
- data/lib/pg_logical_replicator/version.rb +3 -0
- data/lib/pg_logical_replicator.rb +25 -0
- metadata +125 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 33cf7006d00c233de6e00b5f667e37d48ad68d776cd3405537bbf8ade48426a6
|
4
|
+
data.tar.gz: b876513ec2a3998f486a6228f876b2c9ebb644880f2ebaabe8572128b9dab226
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8c7849de36e5824832a9eb778ebd33bae52455f43a05de3f8e2cf1f832ea98f8f1cfc45aee2e9e6dfd335de8df3f0cfde8d8b4ed6680e9c95198dd911352259b
|
7
|
+
data.tar.gz: e66b10fcf714cfa22d8cd64e096d5dfe3b67d44923eeb77109ea0a311341cd9ae9be6c2e3dc9f5cededb869644c764a7233b5d487b3a48f8a35e33b72eacdf2b
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) [year] [fullname]
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,159 @@
|
|
1
|
+
# PgLogicalReplicator
|
2
|
+
|
3
|
+
PgLogicalReplicator is a Ruby gem that helps set up and manage logical replication for PostgreSQL databases. This gem was created so you can migrate databases with near 0 downtime. It uses the PG snapshot ability to create a backup of the current db and restores that backup to the target db. After the restore is done replication begins where the backup left off. This allows you to have a near 0 downtime migration.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'pg_logical_replicator'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle install
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install pg_logical_replicator
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
PgLogicalReplicator provides a command-line interface (CLI) to set up and stop logical replication between PostgreSQL databases.
|
24
|
+
|
25
|
+
### Creating a Replication User on the Target Database
|
26
|
+
|
27
|
+
Before setting up replication, you need to create a replication user on the target database. This user will have specific privileges to ensure that triggers and foreign keys are not enforced during replication. Use the following SQL commands to create and configure this user:
|
28
|
+
|
29
|
+
```sql
|
30
|
+
CREATE USER temp_replica_user WITH PASSWORD 'your_password';
|
31
|
+
|
32
|
+
GRANT ALL PRIVILEGES ON DATABASE your_target_database TO temp_replica_user;
|
33
|
+
GRANT ALL ON SCHEMA public TO temp_replica_user;
|
34
|
+
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO temp_replica_user;
|
35
|
+
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO temp_replica_user;
|
36
|
+
|
37
|
+
ALTER ROLE temp_replica_user SET session_replication_role = 'replica';
|
38
|
+
```
|
39
|
+
|
40
|
+
### Transferring Schema
|
41
|
+
|
42
|
+
To transfer the schema from the source to the target database, use the following command:
|
43
|
+
|
44
|
+
```
|
45
|
+
pg_logical_replicator transfer_schema [OPTIONS]
|
46
|
+
```
|
47
|
+
|
48
|
+
- `--source_host SOURCE_HOST` : Source database host (required)
|
49
|
+
- `--source_port SOURCE_PORT` : Source database port (default: 5432)
|
50
|
+
- `--source_database SOURCE_DATABASE` : Source database name (required)
|
51
|
+
- `--target_host TARGET_HOST` : Target database host (required)
|
52
|
+
- `--target_port TARGET_PORT` : Target database port (default: 5432)
|
53
|
+
- `--target_database TARGET_DATABASE` : Target database name (required)
|
54
|
+
- `--source_username SOURCE_USERNAME` : Source database username (required)
|
55
|
+
- `--source_password SOURCE_PASSWORD` : Source database password (required)
|
56
|
+
- `--target_username TARGET_USERNAME` : Target database username (required)
|
57
|
+
- `--target_password TARGET_PASSWORD` : Target database password (required)
|
58
|
+
|
59
|
+
```
|
60
|
+
pg_logical_replicator transfer_schema \
|
61
|
+
--source_host localhost \
|
62
|
+
--source_database mydb \
|
63
|
+
--source_username myuser \
|
64
|
+
--source_password mypass \
|
65
|
+
--target_host remotehost \
|
66
|
+
--target_database targetdb \
|
67
|
+
--target_username targetuser \
|
68
|
+
--target_password targetpass
|
69
|
+
```
|
70
|
+
|
71
|
+
This command will transfer the schema from the source database to the target database.
|
72
|
+
|
73
|
+
### Setting Up Replication
|
74
|
+
|
75
|
+
To set up logical replication, use the following command:
|
76
|
+
|
77
|
+
```
|
78
|
+
pg_logical_replicator setup [OPTIONS]
|
79
|
+
```
|
80
|
+
|
81
|
+
Options:
|
82
|
+
|
83
|
+
- `--source_host SOURCE_HOST` : Source database host (required)
|
84
|
+
- `--source_port SOURCE_PORT` : Source database port (default: 5432)
|
85
|
+
- `--source_database SOURCE_DATABASE` : Source database name (required)
|
86
|
+
- `--target_host TARGET_HOST` : Target database host (required)
|
87
|
+
- `--target_port TARGET_PORT` : Target database port (default: 5432)
|
88
|
+
- `--target_database TARGET_DATABASE` : Target database name (defaults to source database name)
|
89
|
+
- `--source_username SOURCE_USERNAME` : Source database username (required)
|
90
|
+
- `--source_password SOURCE_PASSWORD` : Source database password (required)
|
91
|
+
- `--target_username TARGET_USERNAME` : Target database username (defaults to source username)
|
92
|
+
- `--target_password TARGET_PASSWORD` : Target database password (defaults to source password)
|
93
|
+
- `--target_rep_username REP_USERNAME` : Target replication username (defaults to target username)
|
94
|
+
- `--target_rep_password REP_PASSWORD` : Target replication password (defaults to target password)
|
95
|
+
- `--num_slots NUM_SLOTS` : Number of replication slots (default: 10)
|
96
|
+
- `--groups GROUP1,GROUP2,...` : Groups to run (comma-separated list of numbers)
|
97
|
+
|
98
|
+
Example:
|
99
|
+
|
100
|
+
```
|
101
|
+
pg_logical_replicator setup \
|
102
|
+
--source_host localhost \
|
103
|
+
--source_database mydb \
|
104
|
+
--source_username myuser \
|
105
|
+
--source_password mypass \
|
106
|
+
--target_host remotehost \
|
107
|
+
--num_slots 5 \
|
108
|
+
--groups 1,3,5
|
109
|
+
```
|
110
|
+
|
111
|
+
This command will set up logical replication for the specified groups of tables between the source and target databases.
|
112
|
+
|
113
|
+
### Stopping Replication
|
114
|
+
|
115
|
+
To stop all replication, use the following command:
|
116
|
+
|
117
|
+
```
|
118
|
+
pg_logical_replicator stop_replication [OPTIONS]
|
119
|
+
```
|
120
|
+
|
121
|
+
Options:
|
122
|
+
|
123
|
+
- `--source_host SOURCE_HOST` : Source database host (required)
|
124
|
+
- `--source_port SOURCE_PORT` : Source database port (default: 5432)
|
125
|
+
- `--source_database SOURCE_DATABASE` : Source database name (required)
|
126
|
+
- `--target_host TARGET_HOST` : Target database host (required)
|
127
|
+
- `--target_port TARGET_PORT` : Target database port (default: 5432)
|
128
|
+
- `--target_database TARGET_DATABASE` : Target database name (defaults to source database name)
|
129
|
+
- `--source_username SOURCE_USERNAME` : Source database username (required)
|
130
|
+
- `--source_password SOURCE_PASSWORD` : Source database password (required)
|
131
|
+
- `--target_username TARGET_USERNAME` : Target database username (defaults to source username)
|
132
|
+
- `--target_password TARGET_PASSWORD` : Target database password (defaults to source password)
|
133
|
+
|
134
|
+
Example:
|
135
|
+
|
136
|
+
```
|
137
|
+
pg_logical_replicator stop_replication \
|
138
|
+
--source_host sourcehost \
|
139
|
+
--source_database mydb \
|
140
|
+
--source_username sourceuser \
|
141
|
+
--source_password sourcepass \
|
142
|
+
--target_host targethost
|
143
|
+
```
|
144
|
+
|
145
|
+
This command will drop all publications on the source database and all subscriptions on the target database, effectively stopping all logical replication.
|
146
|
+
|
147
|
+
## Development
|
148
|
+
|
149
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
150
|
+
|
151
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
152
|
+
|
153
|
+
## Contributing
|
154
|
+
|
155
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/yourusername/pg_logical_replicator.
|
156
|
+
|
157
|
+
## License
|
158
|
+
|
159
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'pg'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module PgLogicalReplicator
|
6
|
+
class CLI < Thor
|
7
|
+
desc 'setup', 'Set up logical replication'
|
8
|
+
method_option :source_host, type: :string, required: true
|
9
|
+
method_option :source_port, type: :numeric, default: 5432
|
10
|
+
method_option :source_database, type: :string, required: true
|
11
|
+
method_option :target_host, type: :string, required: true
|
12
|
+
method_option :target_port, type: :numeric, default: 5432
|
13
|
+
method_option :target_database, type: :string
|
14
|
+
method_option :source_username, type: :string, required: true
|
15
|
+
method_option :source_password, type: :string, required: true
|
16
|
+
method_option :target_username, type: :string
|
17
|
+
method_option :target_password, type: :string
|
18
|
+
method_option :target_rep_username, type: :string
|
19
|
+
method_option :target_rep_password, type: :string
|
20
|
+
method_option :num_slots, type: :numeric, default: 10
|
21
|
+
method_option :groups, type: :array
|
22
|
+
|
23
|
+
def setup
|
24
|
+
source_host = options[:source_host]
|
25
|
+
source_port = options[:source_port]
|
26
|
+
source_database = options[:source_database]
|
27
|
+
target_host = options[:target_host]
|
28
|
+
target_port = options[:target_port]
|
29
|
+
target_database = options[:target_database] || source_database
|
30
|
+
source_username = options[:source_username]
|
31
|
+
source_password = options[:source_password]
|
32
|
+
target_username = options[:target_username] || source_username
|
33
|
+
target_password = options[:target_password] || source_password
|
34
|
+
target_rep_username = options[:target_rep_username] || target_username
|
35
|
+
target_rep_password = options[:target_rep_password] || target_password
|
36
|
+
num_slots = options[:num_slots]
|
37
|
+
target_groups = options[:groups]
|
38
|
+
|
39
|
+
conn = PG.connect(dbname: source_database, user: source_username, password: source_password, host: source_host, port: source_port)
|
40
|
+
|
41
|
+
query = "SELECT tablename FROM pg_tables WHERE schemaname = 'public' order by tablename ASC;"
|
42
|
+
|
43
|
+
tables = begin
|
44
|
+
result = conn.exec(query)
|
45
|
+
puts "List of tables in the 'public' schema:"
|
46
|
+
result.map { |row| row['tablename'] }
|
47
|
+
ensure
|
48
|
+
conn.close if conn
|
49
|
+
end
|
50
|
+
puts "Total Tables: #{tables.size}"
|
51
|
+
|
52
|
+
tables_per_group = tables.size / num_slots
|
53
|
+
puts "Tables per group: #{tables_per_group}"
|
54
|
+
|
55
|
+
dump_dir_root = "~/db-dumps-#{Time.now.strftime('%Y%m%d%H%M%S')}"
|
56
|
+
system("mkdir -p #{dump_dir_root}")
|
57
|
+
|
58
|
+
tables.each_slice(tables_per_group).with_index do |table_group, group_idx|
|
59
|
+
puts "Processing group #{group_idx + 1} size: #{table_group.size}"
|
60
|
+
|
61
|
+
group_number = group_idx + 1
|
62
|
+
|
63
|
+
next unless target_groups.nil? || target_groups.include?(group_number)
|
64
|
+
|
65
|
+
table_names = table_group.join(',')
|
66
|
+
|
67
|
+
dump_dir = "#{dump_dir_root}/#{group_number}"
|
68
|
+
|
69
|
+
puts "Removing directory: #{dump_dir}"
|
70
|
+
|
71
|
+
FileUtils.rm_rf(dump_dir)
|
72
|
+
|
73
|
+
LogicalReplicationInitializer.new({
|
74
|
+
slot_name: "logical_sub_grp_#{group_number}",
|
75
|
+
publication_name: "logical_pub_grp_#{group_number}",
|
76
|
+
primary_conn_str: "host=#{source_host} port=#{source_port} dbname=#{source_database} user=#{source_username} password=#{source_password}",
|
77
|
+
target_conn_str: "host=#{target_host} port=#{target_port} dbname=#{target_database} user=#{target_username} password=#{target_password}",
|
78
|
+
target_rep_conn_str: "host=#{target_host} port=#{target_port} dbname=#{target_database} user=#{target_rep_username} password=#{target_rep_password}",
|
79
|
+
table_names: table_names,
|
80
|
+
dump_dir: dump_dir
|
81
|
+
}).start
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
desc 'stop_replication', 'Stop all replication'
|
86
|
+
method_option :source_host, type: :string, required: true
|
87
|
+
method_option :source_port, type: :numeric, default: 5432
|
88
|
+
method_option :source_database, type: :string, required: true
|
89
|
+
method_option :target_host, type: :string, required: true
|
90
|
+
method_option :target_port, type: :numeric, default: 5432
|
91
|
+
method_option :target_database, type: :string
|
92
|
+
method_option :source_username, type: :string, required: true
|
93
|
+
method_option :source_password, type: :string, required: true
|
94
|
+
method_option :target_username, type: :string
|
95
|
+
method_option :target_password, type: :string
|
96
|
+
|
97
|
+
def stop_replication
|
98
|
+
source_config = {
|
99
|
+
host: options[:source_host],
|
100
|
+
port: options[:source_port],
|
101
|
+
dbname: options[:source_database],
|
102
|
+
user: options[:source_username],
|
103
|
+
password: options[:source_password]
|
104
|
+
}
|
105
|
+
|
106
|
+
target_config = {
|
107
|
+
host: options[:target_host],
|
108
|
+
port: options[:target_port],
|
109
|
+
dbname: options[:target_database] || options[:source_database],
|
110
|
+
user: options[:target_username] || options[:source_username],
|
111
|
+
password: options[:target_password] || options[:source_password]
|
112
|
+
}
|
113
|
+
|
114
|
+
ReplicationStopper.new(source_config, target_config).stop_replication
|
115
|
+
end
|
116
|
+
|
117
|
+
desc 'transfer_schema', 'Transfer schema from source to target database'
|
118
|
+
method_option :source_host, type: :string, required: true
|
119
|
+
method_option :source_port, type: :numeric, default: 5432
|
120
|
+
method_option :source_database, type: :string, required: true
|
121
|
+
method_option :target_host, type: :string, required: true
|
122
|
+
method_option :target_port, type: :numeric, default: 5432
|
123
|
+
method_option :target_database, type: :string, required: true
|
124
|
+
method_option :source_username, type: :string, required: true
|
125
|
+
method_option :source_password, type: :string, required: true
|
126
|
+
method_option :target_username, type: :string, required: true
|
127
|
+
method_option :target_password, type: :string, required: true
|
128
|
+
|
129
|
+
def transfer_schema
|
130
|
+
SchemaTransfer.new(
|
131
|
+
source_host: options[:source_host],
|
132
|
+
source_port: options[:source_port],
|
133
|
+
source_database: options[:source_database],
|
134
|
+
source_username: options[:source_username],
|
135
|
+
source_password: options[:source_password],
|
136
|
+
target_host: options[:target_host],
|
137
|
+
target_port: options[:target_port],
|
138
|
+
target_database: options[:target_database],
|
139
|
+
target_username: options[:target_username],
|
140
|
+
target_password: options[:target_password]
|
141
|
+
).transfer
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,145 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'pg'
|
3
|
+
require 'logger'
|
4
|
+
require 'fileutils'
|
5
|
+
require 'open3'
|
6
|
+
|
7
|
+
module PgLogicalReplicator
|
8
|
+
class LogicalReplicationInitializer
|
9
|
+
def initialize(args)
|
10
|
+
@options = args
|
11
|
+
setup_logger
|
12
|
+
validate_options
|
13
|
+
connect_to_databases
|
14
|
+
end
|
15
|
+
|
16
|
+
def setup_logger
|
17
|
+
@log = Logger.new(STDOUT)
|
18
|
+
@log.level = Logger::INFO
|
19
|
+
end
|
20
|
+
|
21
|
+
def validate_options
|
22
|
+
[:primary_conn_str, :target_conn_str, :target_rep_conn_str, :table_names, :dump_dir].each do |opt|
|
23
|
+
raise OptionParser::MissingArgument, opt.to_s unless @options[opt]
|
24
|
+
end
|
25
|
+
|
26
|
+
@options[:slot_name] ||= 'migration_sub'
|
27
|
+
@options[:publication_name] ||= 'migration_pub'
|
28
|
+
@options[:secondary_conn_str] ||= @options[:primary_conn_str]
|
29
|
+
@options[:subscription_conn_str] ||= @options[:primary_conn_str]
|
30
|
+
end
|
31
|
+
|
32
|
+
def connect_to_databases
|
33
|
+
@primary_conn = PG.connect(@options[:primary_conn_str])
|
34
|
+
@secondary_conn = PG.connect(@options[:secondary_conn_str])
|
35
|
+
@target_conn = PG.connect(@options[:target_conn_str])
|
36
|
+
@target_rep_conn = PG.connect(@options[:target_rep_conn_str])
|
37
|
+
end
|
38
|
+
|
39
|
+
def start
|
40
|
+
begin
|
41
|
+
@log.info("Dumping to #{@options[:dump_dir]}")
|
42
|
+
|
43
|
+
drop_subscription
|
44
|
+
drop_publication
|
45
|
+
create_publication
|
46
|
+
create_logical_replication_slot
|
47
|
+
|
48
|
+
snapshot = create_snapshot
|
49
|
+
dump_database(snapshot)
|
50
|
+
restore_dump
|
51
|
+
create_subscription
|
52
|
+
advance_subscription_origin(snapshot[:lsn])
|
53
|
+
enable_subscription
|
54
|
+
rescue => e
|
55
|
+
@log.error(e.message)
|
56
|
+
exit 1
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def drop_subscription
|
61
|
+
@log.info("Dropping subscription #{@options[:slot_name]}")
|
62
|
+
@target_conn.exec("DROP SUBSCRIPTION IF EXISTS #{@options[:slot_name]}")
|
63
|
+
end
|
64
|
+
|
65
|
+
def drop_publication
|
66
|
+
@log.info("Dropping publication #{@options[:publication_name]}")
|
67
|
+
@primary_conn.exec("DROP PUBLICATION IF EXISTS #{@options[:publication_name]}")
|
68
|
+
end
|
69
|
+
|
70
|
+
def create_publication
|
71
|
+
@log.info("Creating publication #{@options[:publication_name]}")
|
72
|
+
table_names = @options[:table_names].split(',')
|
73
|
+
@primary_conn.exec("CREATE PUBLICATION #{@options[:publication_name]} FOR TABLE #{table_names.join(', ')}")
|
74
|
+
end
|
75
|
+
|
76
|
+
def create_logical_replication_slot
|
77
|
+
@log.info("Creating logical replication slot #{@options[:slot_name]}")
|
78
|
+
@primary_conn.exec("SELECT pg_create_logical_replication_slot('#{@options[:slot_name]}', 'pgoutput')")
|
79
|
+
end
|
80
|
+
|
81
|
+
def create_snapshot
|
82
|
+
@log.info("Creating snapshot")
|
83
|
+
@secondary_conn.transaction do |conn|
|
84
|
+
res = conn.exec("SELECT pg_current_wal_lsn(), pg_export_snapshot()")
|
85
|
+
{ lsn: res.getvalue(0, 0), snapshot: res.getvalue(0, 1) }
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def dump_database(snapshot)
|
90
|
+
@log.info("Dumping source DB")
|
91
|
+
table_names = @options[:table_names].split(',')
|
92
|
+
dump_command = [
|
93
|
+
'pg_dump',
|
94
|
+
'--no-publication',
|
95
|
+
'--no-subscription',
|
96
|
+
"--snapshot=#{snapshot[:snapshot]}",
|
97
|
+
'--format=d',
|
98
|
+
'--data-only',
|
99
|
+
'--jobs=8',
|
100
|
+
'-f', @options[:dump_dir],
|
101
|
+
@options[:secondary_conn_str]
|
102
|
+
] + table_names.flat_map { |table| ['-t', table] }
|
103
|
+
|
104
|
+
@log.info("Executing DB Dump Command: #{dump_command.join(' ')}")
|
105
|
+
system(*dump_command)
|
106
|
+
end
|
107
|
+
|
108
|
+
def restore_dump
|
109
|
+
@log.info("Restoring dump to target DB")
|
110
|
+
table_names = @options[:table_names].split(',')
|
111
|
+
table_names.each { |table| truncate_table(table) }
|
112
|
+
|
113
|
+
restore_command = [
|
114
|
+
'pg_restore',
|
115
|
+
'--format=d',
|
116
|
+
'--jobs=8',
|
117
|
+
'--data-only',
|
118
|
+
'-d', @options[:target_rep_conn_str],
|
119
|
+
@options[:dump_dir]
|
120
|
+
]
|
121
|
+
|
122
|
+
system(*restore_command)
|
123
|
+
end
|
124
|
+
|
125
|
+
def truncate_table(table)
|
126
|
+
@log.info("Truncating table #{table}")
|
127
|
+
@target_rep_conn.exec("TRUNCATE TABLE #{table}")
|
128
|
+
end
|
129
|
+
|
130
|
+
def create_subscription
|
131
|
+
@log.info("Creating subscription #{@options[:slot_name]}")
|
132
|
+
@target_conn.exec("CREATE SUBSCRIPTION #{@options[:slot_name]} CONNECTION '#{@options[:subscription_conn_str]}' PUBLICATION #{@options[:publication_name]} WITH (create_slot=false, slot_name='#{@options[:slot_name]}', enabled=false, copy_data=false)")
|
133
|
+
end
|
134
|
+
|
135
|
+
def advance_subscription_origin(lsn)
|
136
|
+
@log.info("Setting replication origin position to #{lsn}")
|
137
|
+
@target_conn.exec("SELECT pg_replication_origin_advance('pg_' || subid::text, '#{lsn}') FROM pg_stat_subscription WHERE subname = '#{@options[:slot_name]}'")
|
138
|
+
end
|
139
|
+
|
140
|
+
def enable_subscription
|
141
|
+
@log.info("Enabling subscription #{@options[:slot_name]}")
|
142
|
+
@target_conn.exec("ALTER SUBSCRIPTION #{@options[:slot_name]} ENABLE")
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'pg'
|
2
|
+
|
3
|
+
module PgLogicalReplicator
|
4
|
+
class ReplicationStopper
|
5
|
+
def initialize(source_config, target_config)
|
6
|
+
@source_config = source_config
|
7
|
+
@target_config = target_config
|
8
|
+
end
|
9
|
+
|
10
|
+
def stop_replication
|
11
|
+
source_conn = PG.connect(@source_config)
|
12
|
+
target_conn = PG.connect(@target_config)
|
13
|
+
|
14
|
+
begin
|
15
|
+
drop_all_publications(source_conn)
|
16
|
+
drop_all_subscriptions(target_conn)
|
17
|
+
puts "All replication stopped successfully."
|
18
|
+
ensure
|
19
|
+
source_conn.close if source_conn
|
20
|
+
target_conn.close if target_conn
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def drop_all_subscriptions(conn)
|
27
|
+
query = "SELECT subname FROM pg_subscription"
|
28
|
+
result = conn.exec(query)
|
29
|
+
result.each do |row|
|
30
|
+
subname = row['subname']
|
31
|
+
conn.exec("ALTER SUBSCRIPTION #{subname} DISABLE")
|
32
|
+
conn.exec("ALTER SUBSCRIPTION #{subname} SET (slot_name = NONE)")
|
33
|
+
conn.exec("DROP SUBSCRIPTION IF EXISTS #{subname}")
|
34
|
+
puts "Dropped subscription: #{subname}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def drop_all_publications(conn)
|
39
|
+
query = "SELECT pubname FROM pg_publication"
|
40
|
+
result = conn.exec(query)
|
41
|
+
result.each do |row|
|
42
|
+
pubname = row['pubname']
|
43
|
+
conn.exec("DROP PUBLICATION IF EXISTS #{pubname}")
|
44
|
+
puts "Dropped publication: #{pubname}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'tempfile'
|
5
|
+
|
6
|
+
module PgLogicalReplicator
|
7
|
+
class SchemaTransfer
|
8
|
+
def initialize(source_host:, source_port:, source_database:, target_host:, target_port:, target_database:, source_username:, source_password:, target_username:, target_password:)
|
9
|
+
@source_host = source_host
|
10
|
+
@source_port = source_port
|
11
|
+
@source_database = source_database
|
12
|
+
@source_username = source_username
|
13
|
+
@source_password = source_password
|
14
|
+
@target_host = target_host
|
15
|
+
@target_port = target_port
|
16
|
+
@target_database = target_database
|
17
|
+
@target_username = target_username
|
18
|
+
@target_password = target_password
|
19
|
+
end
|
20
|
+
|
21
|
+
def transfer
|
22
|
+
puts 'Starting schema transfer...'
|
23
|
+
|
24
|
+
ENV['PGPASSWORD_SOURCE'] = @source_password
|
25
|
+
ENV['PGPASSWORD_TARGET'] = @target_password
|
26
|
+
|
27
|
+
Tempfile.create('db_dump') do |tempfile|
|
28
|
+
pg_dump_command = [
|
29
|
+
"pg_dump",
|
30
|
+
"-s", "--no-owner", "--no-acl", "--no-privileges",
|
31
|
+
"-h", @source_host,
|
32
|
+
"-p", @source_port,
|
33
|
+
"-d", @source_database,
|
34
|
+
"-U", @source_username,
|
35
|
+
"> #{tempfile.path}"
|
36
|
+
].join(' ')
|
37
|
+
|
38
|
+
psql_command = [
|
39
|
+
"psql",
|
40
|
+
"-h", @target_host,
|
41
|
+
"-p", @target_port,
|
42
|
+
"-d", @target_database,
|
43
|
+
"-U", @target_username,
|
44
|
+
"< #{tempfile.path}"
|
45
|
+
].join(' ')
|
46
|
+
|
47
|
+
system({ "PGPASSWORD" => ENV['PGPASSWORD_SOURCE'] }, pg_dump_command)
|
48
|
+
system({ "PGPASSWORD" => ENV['PGPASSWORD_TARGET'] }, psql_command)
|
49
|
+
end
|
50
|
+
|
51
|
+
ENV.delete('PGPASSWORD_SOURCE')
|
52
|
+
ENV.delete('PGPASSWORD_TARGET')
|
53
|
+
|
54
|
+
puts 'Schema transfer completed successfully.'
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require_relative "pg_logical_replicator/version"
|
2
|
+
require_relative "pg_logical_replicator/logical_replication_initializer"
|
3
|
+
require_relative "pg_logical_replicator/replication_stopper"
|
4
|
+
require_relative "pg_logical_replicator/cli"
|
5
|
+
require_relative 'pg_logical_replicator/schema_transfer'
|
6
|
+
|
7
|
+
module PgLogicalReplicator
|
8
|
+
class Error < StandardError; end
|
9
|
+
|
10
|
+
# You can add any module-level methods or constants here if needed
|
11
|
+
|
12
|
+
# For example, you might want to add a method to get the gem's root directory:
|
13
|
+
def self.root
|
14
|
+
File.expand_path('..', __dir__)
|
15
|
+
end
|
16
|
+
|
17
|
+
# Or a method to get the gem's configuration if you decide to add config options in the future:
|
18
|
+
# def self.configuration
|
19
|
+
# @configuration ||= Configuration.new
|
20
|
+
# end
|
21
|
+
|
22
|
+
# def self.configure
|
23
|
+
# yield(configuration) if block_given?
|
24
|
+
# end
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,125 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pg_logical_replicator
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- eni9889
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-07-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: pg
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.2'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: thor
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.2'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.2'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '2.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '2.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '13.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '13.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rspec
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '3.0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '3.0'
|
83
|
+
description: A tool to set up and manage logical replication for PostgreSQL databases
|
84
|
+
email:
|
85
|
+
- enea09@gmail.com
|
86
|
+
executables:
|
87
|
+
- pg_logical_replicator
|
88
|
+
extensions: []
|
89
|
+
extra_rdoc_files: []
|
90
|
+
files:
|
91
|
+
- LICENSE.txt
|
92
|
+
- README.md
|
93
|
+
- bin/pg_logical_replicator
|
94
|
+
- lib/pg_logical_replicator.rb
|
95
|
+
- lib/pg_logical_replicator/cli.rb
|
96
|
+
- lib/pg_logical_replicator/logical_replication_initializer.rb
|
97
|
+
- lib/pg_logical_replicator/replication_stopper.rb
|
98
|
+
- lib/pg_logical_replicator/schema_transfer.rb
|
99
|
+
- lib/pg_logical_replicator/version.rb
|
100
|
+
homepage: https://github.com/eni9889/pg_logical_replicator
|
101
|
+
licenses:
|
102
|
+
- MIT
|
103
|
+
metadata:
|
104
|
+
homepage_uri: https://github.com/eni9889/pg_logical_replicator
|
105
|
+
source_code_uri: https://github.com/eni9889/pg_logical_replicator
|
106
|
+
post_install_message:
|
107
|
+
rdoc_options: []
|
108
|
+
require_paths:
|
109
|
+
- lib
|
110
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: 2.6.0
|
115
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
116
|
+
requirements:
|
117
|
+
- - ">="
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
version: '0'
|
120
|
+
requirements: []
|
121
|
+
rubygems_version: 3.1.6
|
122
|
+
signing_key:
|
123
|
+
specification_version: 4
|
124
|
+
summary: PostgreSQL logical replication setup tool
|
125
|
+
test_files: []
|