active_postgres 0.9.3 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/active_postgres/components/base.rb +8 -0
- data/lib/active_postgres/components/extensions.rb +2 -3
- data/lib/active_postgres/components/monitoring.rb +8 -10
- data/lib/active_postgres/components/pgbackrest.rb +87 -4
- data/lib/active_postgres/components/pgbouncer.rb +174 -21
- data/lib/active_postgres/components/repmgr.rb +21 -17
- data/lib/active_postgres/configuration.rb +36 -0
- data/lib/active_postgres/connection_pooler.rb +2 -2
- data/lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb +2 -0
- data/lib/active_postgres/secrets.rb +2 -0
- data/lib/active_postgres/ssh_executor.rb +11 -8
- data/lib/active_postgres/version.rb +1 -1
- data/lib/tasks/postgres.rake +3 -2
- data/lib/tasks/rolling_update.rake +17 -10
- data/templates/pgbackrest.conf.erb +2 -1
- data/templates/pgbouncer.ini.erb +2 -1
- data/templates/pgbouncer_follow_dns_primary.sh.erb +39 -0
- data/templates/pgbouncer_follow_primary.sh.erb +6 -4
- data/templates/postgres_log_archive.sh.erb +88 -0
- data/templates/postgresql.conf.erb +5 -3
- data/templates/repmgr.conf.erb +1 -1
- data/templates/repmgr_dns_failover.sh.erb +10 -3
- metadata +3 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8f31b928d75edd1c1c63a6534023a3c7255f4bce000487da5fb39ddd0bd357fb
|
|
4
|
+
data.tar.gz: ee835da34ead4b417ac2c5087e9fae0db12641c7dddb44e91fe795d083f4e61a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 81cc77508c361ab128d840dbaac52ae9f3fc50b431fa971aaede91fc573ed9186521a53f8098e038515b6c96c82141100fd7ce797b0eae5c4b2840bd01330212
|
|
7
|
+
data.tar.gz: bcdeb64f655cbc0e702f5b70fc3c749daa103ac74945a9432627aeff6926de9b1974fa52547b32e1d99acec687d30b252473a15a75ffadd299e6bc951e9e2bf2
|
|
@@ -43,6 +43,14 @@ module ActivePostgres
|
|
|
43
43
|
content = render_template(template_name, binding_context)
|
|
44
44
|
ssh_executor.upload_file(host, content, remote_path, mode: mode, owner: owner)
|
|
45
45
|
end
|
|
46
|
+
|
|
47
|
+
def install_apt_packages(host, *packages)
|
|
48
|
+
ssh_executor.execute_on_host(host) do
|
|
49
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
50
|
+
'-o', 'DPkg::Lock::Timeout=300',
|
|
51
|
+
'install', '-y', '-qq', *packages
|
|
52
|
+
end
|
|
53
|
+
end
|
|
46
54
|
end
|
|
47
55
|
end
|
|
48
56
|
end
|
|
@@ -65,10 +65,9 @@ module ActivePostgres
|
|
|
65
65
|
return if packages_to_install.empty?
|
|
66
66
|
|
|
67
67
|
ssh_executor.execute_on_host(host) do
|
|
68
|
-
execute :sudo, 'apt-get', 'update', '-qq'
|
|
69
|
-
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
|
|
70
|
-
*packages_to_install
|
|
68
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update', '-qq'
|
|
71
69
|
end
|
|
70
|
+
install_apt_packages(host, *packages_to_install)
|
|
72
71
|
end
|
|
73
72
|
|
|
74
73
|
def install_on_primary(extensions)
|
|
@@ -60,10 +60,7 @@ module ActivePostgres
|
|
|
60
60
|
exporter_port = monitoring_config[:exporter_port] || 9187
|
|
61
61
|
|
|
62
62
|
# Download and install postgres_exporter
|
|
63
|
-
|
|
64
|
-
# Install via package manager or download binary
|
|
65
|
-
execute :sudo, 'apt-get', 'install', '-y', '-qq', 'prometheus-postgres-exporter'
|
|
66
|
-
end
|
|
63
|
+
install_apt_packages(host, 'prometheus-postgres-exporter')
|
|
67
64
|
|
|
68
65
|
configure_exporter_service(host, monitoring_config, exporter_port)
|
|
69
66
|
|
|
@@ -157,9 +154,7 @@ module ActivePostgres
|
|
|
157
154
|
listen_address = monitoring_config[:node_exporter_listen_address].to_s.strip
|
|
158
155
|
puts " Installing node_exporter on #{host}..."
|
|
159
156
|
|
|
160
|
-
|
|
161
|
-
execute :sudo, 'apt-get', 'install', '-y', '-qq', 'prometheus-node-exporter'
|
|
162
|
-
end
|
|
157
|
+
install_apt_packages(host, 'prometheus-node-exporter')
|
|
163
158
|
|
|
164
159
|
configure_node_exporter_service(host, port, listen_address)
|
|
165
160
|
|
|
@@ -213,12 +208,15 @@ module ActivePostgres
|
|
|
213
208
|
puts "Installing Grafana on #{host}..."
|
|
214
209
|
|
|
215
210
|
ssh_executor.execute_on_host(host) do
|
|
216
|
-
execute :sudo, '
|
|
211
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
212
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq',
|
|
213
|
+
'apt-transport-https', 'software-properties-common', 'wget'
|
|
217
214
|
execute :sudo, 'wget', '-q', '-O', '/usr/share/keyrings/grafana.gpg', 'https://apt.grafana.com/gpg.key'
|
|
218
215
|
execute :sudo, 'sh', '-c',
|
|
219
216
|
'echo "deb [signed-by=/usr/share/keyrings/grafana.gpg] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list'
|
|
220
|
-
execute :sudo, 'apt-get', 'update', '-qq'
|
|
221
|
-
execute :sudo, '
|
|
217
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update', '-qq'
|
|
218
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
219
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq', 'grafana'
|
|
222
220
|
execute :sudo, 'systemctl', 'enable', '--now', 'grafana-server'
|
|
223
221
|
execute :sudo, 'grafana-cli', 'admin', 'reset-admin-password', admin_password
|
|
224
222
|
end
|
|
@@ -1,6 +1,12 @@
|
|
|
1
|
+
require 'shellwords'
|
|
2
|
+
|
|
1
3
|
module ActivePostgres
|
|
2
4
|
module Components
|
|
3
5
|
class PgBackRest < Base
|
|
6
|
+
LOG_ARCHIVE_CRON_FILE = '/etc/cron.d/postgres-log-archive'
|
|
7
|
+
LOG_ARCHIVE_ENV_FILE = '/etc/active-postgres-log-archive.env'
|
|
8
|
+
LOG_ARCHIVE_SCRIPT = '/usr/local/bin/active-postgres-archive-logs'
|
|
9
|
+
|
|
4
10
|
def install
|
|
5
11
|
puts 'Installing pgBackRest for backups...'
|
|
6
12
|
|
|
@@ -21,6 +27,9 @@ module ActivePostgres
|
|
|
21
27
|
execute :sudo, 'apt-get', 'remove', '-y', 'pgbackrest'
|
|
22
28
|
execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup'
|
|
23
29
|
execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental'
|
|
30
|
+
execute :sudo, 'rm', '-f', LOG_ARCHIVE_CRON_FILE
|
|
31
|
+
execute :sudo, 'rm', '-f', LOG_ARCHIVE_ENV_FILE
|
|
32
|
+
execute :sudo, 'rm', '-f', LOG_ARCHIVE_SCRIPT
|
|
24
33
|
end
|
|
25
34
|
end
|
|
26
35
|
end
|
|
@@ -105,10 +114,7 @@ module ActivePostgres
|
|
|
105
114
|
_ = pgbackrest_config # Used in ERB template
|
|
106
115
|
_ = secrets_obj # Used in ERB template
|
|
107
116
|
|
|
108
|
-
|
|
109
|
-
ssh_executor.execute_on_host(host) do
|
|
110
|
-
execute :sudo, 'apt-get', 'install', '-y', '-qq', 'pgbackrest'
|
|
111
|
-
end
|
|
117
|
+
install_apt_packages(host, 'pgbackrest')
|
|
112
118
|
|
|
113
119
|
# Upload configuration
|
|
114
120
|
upload_template(host, 'pgbackrest.conf.erb', '/etc/pgbackrest.conf', binding,
|
|
@@ -134,9 +140,17 @@ module ActivePostgres
|
|
|
134
140
|
end
|
|
135
141
|
end
|
|
136
142
|
|
|
143
|
+
if log_archive_enabled?(pgbackrest_config)
|
|
144
|
+
setup_log_archive(host, pgbackrest_config)
|
|
145
|
+
else
|
|
146
|
+
remove_log_archive(host)
|
|
147
|
+
end
|
|
148
|
+
|
|
137
149
|
# Set up scheduled backups on primary only
|
|
138
150
|
if create_stanza
|
|
139
151
|
setup_backup_schedules(host, pgbackrest_config)
|
|
152
|
+
else
|
|
153
|
+
remove_backup_schedule(host)
|
|
140
154
|
end
|
|
141
155
|
end
|
|
142
156
|
|
|
@@ -187,6 +201,75 @@ module ActivePostgres
|
|
|
187
201
|
execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental'
|
|
188
202
|
end
|
|
189
203
|
end
|
|
204
|
+
|
|
205
|
+
def setup_log_archive(host, pgbackrest_config)
|
|
206
|
+
log_archive_config = pgbackrest_config[:log_archive] || {}
|
|
207
|
+
unless pgbackrest_config[:repo_type] == 's3'
|
|
208
|
+
raise Error, 'pgbackrest.log_archive requires pgbackrest.repo_type: s3'
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
postgres_user = config.postgres_user
|
|
212
|
+
env_content = log_archive_env(pgbackrest_config, log_archive_config, host)
|
|
213
|
+
cron_content = log_archive_cron(log_archive_config)
|
|
214
|
+
|
|
215
|
+
install_apt_packages(host, 's3cmd')
|
|
216
|
+
|
|
217
|
+
ssh_executor.upload_file(host, env_content, LOG_ARCHIVE_ENV_FILE, mode: '640', owner: "root:#{postgres_user}")
|
|
218
|
+
|
|
219
|
+
upload_template(host, 'postgres_log_archive.sh.erb', LOG_ARCHIVE_SCRIPT, binding,
|
|
220
|
+
mode: '750', owner: "root:#{postgres_user}")
|
|
221
|
+
ssh_executor.upload_file(host, cron_content, LOG_ARCHIVE_CRON_FILE, mode: '644', owner: 'root:root')
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def remove_log_archive(host)
|
|
225
|
+
ssh_executor.execute_on_host(host) do
|
|
226
|
+
execute :sudo, 'rm', '-f', LOG_ARCHIVE_CRON_FILE
|
|
227
|
+
execute :sudo, 'rm', '-f', LOG_ARCHIVE_ENV_FILE
|
|
228
|
+
execute :sudo, 'rm', '-f', LOG_ARCHIVE_SCRIPT
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def log_archive_enabled?(pgbackrest_config)
|
|
233
|
+
(pgbackrest_config[:log_archive] || {})[:enabled] == true
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def log_archive_cron(log_archive_config)
|
|
237
|
+
schedule = log_archive_config[:schedule] || '17 3 * * *'
|
|
238
|
+
<<~CRON
|
|
239
|
+
# PostgreSQL text log archive (managed by active_postgres)
|
|
240
|
+
SHELL=/bin/bash
|
|
241
|
+
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
|
|
242
|
+
#{schedule} root #{LOG_ARCHIVE_SCRIPT}
|
|
243
|
+
CRON
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def log_archive_env(pgbackrest_config, log_archive_config, host)
|
|
247
|
+
access_key = secrets.resolve('s3_access_key')
|
|
248
|
+
secret_key = secrets.resolve('s3_secret_key')
|
|
249
|
+
raise Error, 's3_access_key secret is required for pgbackrest.log_archive' if access_key.to_s.empty?
|
|
250
|
+
raise Error, 's3_secret_key secret is required for pgbackrest.log_archive' if secret_key.to_s.empty?
|
|
251
|
+
|
|
252
|
+
env = {
|
|
253
|
+
'AWS_ACCESS_KEY_ID' => access_key,
|
|
254
|
+
'AWS_SECRET_ACCESS_KEY' => secret_key,
|
|
255
|
+
'AWS_DEFAULT_REGION' => pgbackrest_config[:s3_region] || 'us-east-1',
|
|
256
|
+
'AWS_BUCKET' => pgbackrest_config[:s3_bucket],
|
|
257
|
+
'AWS_ENDPOINT_URL' => s3_endpoint_url(pgbackrest_config),
|
|
258
|
+
'POSTGRES_LOG_ARCHIVE_LOG_DIR' => log_archive_config[:log_directory] || '/var/log/postgresql',
|
|
259
|
+
'POSTGRES_LOG_ARCHIVE_PREFIX' => log_archive_config[:prefix] || 'postgres-logs',
|
|
260
|
+
'POSTGRES_LOG_ARCHIVE_NODE' => config.node_label_for(host) || host
|
|
261
|
+
}.compact
|
|
262
|
+
|
|
263
|
+
env.map { |key, value| "#{key}=#{Shellwords.escape(value.to_s)}" }.join("\n") + "\n"
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def s3_endpoint_url(pgbackrest_config)
|
|
267
|
+
endpoint = pgbackrest_config[:s3_endpoint]
|
|
268
|
+
return nil if endpoint.to_s.empty?
|
|
269
|
+
return endpoint if endpoint.match?(/\Ahttps?:\/\//)
|
|
270
|
+
|
|
271
|
+
"https://#{endpoint}"
|
|
272
|
+
end
|
|
190
273
|
end
|
|
191
274
|
end
|
|
192
275
|
end
|
|
@@ -8,12 +8,16 @@ module ActivePostgres
|
|
|
8
8
|
is_standby = config.standby_hosts.include?(host)
|
|
9
9
|
install_on_host(host, is_standby: is_standby)
|
|
10
10
|
end
|
|
11
|
+
|
|
12
|
+
pgbouncer_app_hosts.each do |app_host|
|
|
13
|
+
install_on_app_host(app_host)
|
|
14
|
+
end
|
|
11
15
|
end
|
|
12
16
|
|
|
13
17
|
def uninstall
|
|
14
18
|
puts 'Uninstalling PgBouncer...'
|
|
15
19
|
|
|
16
|
-
config.all_hosts.each do |host|
|
|
20
|
+
(config.all_hosts + pgbouncer_app_hosts.map { |app_host| app_host.fetch('host') }).each do |host|
|
|
17
21
|
ssh_executor.execute_on_host(host) do
|
|
18
22
|
execute :sudo, 'systemctl', 'disable', '--now', 'pgbouncer-follow-primary.timer', '||', 'true'
|
|
19
23
|
execute :sudo, 'rm', '-f', '/usr/local/bin/pgbouncer-follow-primary',
|
|
@@ -29,7 +33,7 @@ module ActivePostgres
|
|
|
29
33
|
def restart
|
|
30
34
|
puts 'Restarting PgBouncer...'
|
|
31
35
|
|
|
32
|
-
config.all_hosts.each do |host|
|
|
36
|
+
(config.all_hosts + pgbouncer_app_hosts.map { |app_host| app_host.fetch('host') }).each do |host|
|
|
33
37
|
ssh_executor.execute_on_host(host) do
|
|
34
38
|
execute :sudo, 'systemctl', 'restart', 'pgbouncer'
|
|
35
39
|
end
|
|
@@ -46,6 +50,15 @@ module ActivePostgres
|
|
|
46
50
|
execute :sudo, 'systemctl', 'reload', 'pgbouncer'
|
|
47
51
|
end
|
|
48
52
|
end
|
|
53
|
+
|
|
54
|
+
pgbouncer_app_hosts.each do |app_host|
|
|
55
|
+
host = app_host.fetch('host')
|
|
56
|
+
create_userlist_from_primary(host)
|
|
57
|
+
|
|
58
|
+
ssh_executor.execute_on_host(host) do
|
|
59
|
+
execute :sudo, 'systemctl', 'reload', 'pgbouncer'
|
|
60
|
+
end
|
|
61
|
+
end
|
|
49
62
|
end
|
|
50
63
|
|
|
51
64
|
def install_on_standby(standby_host)
|
|
@@ -55,20 +68,18 @@ module ActivePostgres
|
|
|
55
68
|
|
|
56
69
|
private
|
|
57
70
|
|
|
71
|
+
def pgbouncer_app_hosts
|
|
72
|
+
return config.pgbouncer_app_hosts if config.respond_to?(:pgbouncer_app_hosts)
|
|
73
|
+
|
|
74
|
+
[]
|
|
75
|
+
end
|
|
76
|
+
|
|
58
77
|
def install_on_host(host, is_standby: false)
|
|
59
78
|
puts " Installing PgBouncer on #{host}..."
|
|
60
79
|
|
|
61
80
|
user_config = config.component_config(:pgbouncer)
|
|
62
81
|
|
|
63
|
-
|
|
64
|
-
# - Primary: use global follow_primary setting
|
|
65
|
-
# - Standby: check per-standby pgbouncer_follow_primary, default false (use localhost for read replicas)
|
|
66
|
-
if is_standby
|
|
67
|
-
standby_config = config.standby_config_for(host) || {}
|
|
68
|
-
follow_primary = standby_config['pgbouncer_follow_primary'] == true
|
|
69
|
-
else
|
|
70
|
-
follow_primary = user_config[:follow_primary] == true
|
|
71
|
-
end
|
|
82
|
+
follow_primary = follow_primary_for?(host, is_standby: is_standby, user_config: user_config)
|
|
72
83
|
|
|
73
84
|
if follow_primary && !config.component_enabled?(:repmgr)
|
|
74
85
|
raise Error, 'PgBouncer follow_primary requires repmgr to be enabled'
|
|
@@ -89,9 +100,7 @@ module ActivePostgres
|
|
|
89
100
|
|
|
90
101
|
puts " Calculated pool settings for max_connections=#{max_connections}"
|
|
91
102
|
|
|
92
|
-
|
|
93
|
-
execute :sudo, 'apt-get', 'install', '-y', '-qq', 'pgbouncer'
|
|
94
|
-
end
|
|
103
|
+
install_apt_packages(host, 'pgbouncer')
|
|
95
104
|
|
|
96
105
|
upload_template(host, 'pgbouncer.ini.erb', '/etc/pgbouncer/pgbouncer.ini', binding, mode: '644')
|
|
97
106
|
|
|
@@ -104,11 +113,68 @@ module ActivePostgres
|
|
|
104
113
|
ssh_executor.execute_on_host(host) do
|
|
105
114
|
execute :sudo, 'systemctl', 'enable', 'pgbouncer'
|
|
106
115
|
execute :sudo, 'systemctl', 'restart', 'pgbouncer'
|
|
116
|
+
unless follow_primary
|
|
117
|
+
execute :sudo, 'systemctl', 'disable', '--now', 'pgbouncer-follow-primary.timer', '||', 'true'
|
|
118
|
+
execute :sudo, 'rm', '-f', '/usr/local/bin/pgbouncer-follow-primary',
|
|
119
|
+
'/etc/systemd/system/pgbouncer-follow-primary.service',
|
|
120
|
+
'/etc/systemd/system/pgbouncer-follow-primary.timer'
|
|
121
|
+
execute :sudo, 'systemctl', 'daemon-reload'
|
|
122
|
+
end
|
|
107
123
|
end
|
|
108
124
|
|
|
109
125
|
install_follow_primary(host, pgbouncer_config) if follow_primary
|
|
110
126
|
end
|
|
111
127
|
|
|
128
|
+
def install_on_app_host(app_host)
|
|
129
|
+
host = app_host.fetch('host')
|
|
130
|
+
puts " Installing PgBouncer on app host #{host}..."
|
|
131
|
+
|
|
132
|
+
user_config = config.component_config(:pgbouncer)
|
|
133
|
+
max_connections = get_postgres_max_connections(config.primary_host)
|
|
134
|
+
optimal_pool = ConnectionPooler.calculate_optimal_pool_sizes(max_connections)
|
|
135
|
+
|
|
136
|
+
pgbouncer_config = optimal_pool.merge(user_config)
|
|
137
|
+
pgbouncer_config[:database_host] = app_host_value(app_host, :database_host) || pgbouncer_primary_record
|
|
138
|
+
pgbouncer_config[:database_port] = app_host_value(app_host, :database_port) || pgbouncer_config[:database_port] || 5432
|
|
139
|
+
pgbouncer_config[:listen_addr] = app_host_value(app_host, :listen_addr) || pgbouncer_config[:app_listen_addr] || '127.0.0.1'
|
|
140
|
+
pgbouncer_config[:listen_port] = app_host_value(app_host, :listen_port) || pgbouncer_config[:app_listen_port] ||
|
|
141
|
+
pgbouncer_config[:listen_port] || 6432
|
|
142
|
+
ssl_enabled = config.component_enabled?(:ssl)
|
|
143
|
+
has_ca_cert = ssl_enabled && secrets.resolve('ssl_chain')
|
|
144
|
+
secrets_obj = secrets
|
|
145
|
+
_ = pgbouncer_config
|
|
146
|
+
_ = has_ca_cert
|
|
147
|
+
_ = secrets_obj
|
|
148
|
+
|
|
149
|
+
puts " Calculated app pool settings for max_connections=#{max_connections}"
|
|
150
|
+
|
|
151
|
+
install_apt_packages(host, 'pgbouncer', 'postgresql-client')
|
|
152
|
+
|
|
153
|
+
upload_template(host, 'pgbouncer.ini.erb', '/etc/pgbouncer/pgbouncer.ini', binding, mode: '644')
|
|
154
|
+
|
|
155
|
+
setup_app_ssl_certs(host) if ssl_enabled
|
|
156
|
+
|
|
157
|
+
create_userlist_from_primary(host)
|
|
158
|
+
|
|
159
|
+
ssh_executor.execute_on_host(host) do
|
|
160
|
+
execute :sudo, 'systemctl', 'enable', 'pgbouncer'
|
|
161
|
+
execute :sudo, 'systemctl', 'restart', 'pgbouncer'
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
install_follow_dns_primary(host, pgbouncer_config)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def follow_primary_for?(host, is_standby:, user_config:)
|
|
168
|
+
if is_standby
|
|
169
|
+
standby_config = config.standby_config_for(host) || {}
|
|
170
|
+
standby_override = standby_config['pgbouncer_follow_primary']
|
|
171
|
+
standby_override = standby_config[:pgbouncer_follow_primary] if standby_override.nil?
|
|
172
|
+
return standby_override == true unless standby_override.nil?
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
user_config[:follow_primary] == true || user_config['follow_primary'] == true
|
|
176
|
+
end
|
|
177
|
+
|
|
112
178
|
def setup_ssl_certs(host)
|
|
113
179
|
puts ' Setting up SSL certificates for PgBouncer...'
|
|
114
180
|
version = config.version
|
|
@@ -127,6 +193,32 @@ module ActivePostgres
|
|
|
127
193
|
end
|
|
128
194
|
end
|
|
129
195
|
|
|
196
|
+
def setup_app_ssl_certs(host)
|
|
197
|
+
puts ' Setting up SSL certificates for app PgBouncer...'
|
|
198
|
+
|
|
199
|
+
ssl_cert = secrets.resolve('ssl_cert')
|
|
200
|
+
ssl_key = secrets.resolve('ssl_key')
|
|
201
|
+
ssl_chain = secrets.resolve('ssl_chain')
|
|
202
|
+
|
|
203
|
+
unless ssl_cert && ssl_key
|
|
204
|
+
raise Error, 'PgBouncer app_hosts require ssl_cert and ssl_key secrets when ssl is enabled'
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
full_cert = if ssl_chain
|
|
208
|
+
"#{ssl_cert.strip}\n#{ssl_chain.strip}\n"
|
|
209
|
+
else
|
|
210
|
+
ssl_cert
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
ssh_executor.execute_on_host(host) do
|
|
214
|
+
execute :sudo, 'mkdir', '-p', '/etc/pgbouncer'
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
ssh_executor.upload_file(host, full_cert, '/etc/pgbouncer/server.crt', mode: '644', owner: 'postgres:postgres')
|
|
218
|
+
ssh_executor.upload_file(host, ssl_key, '/etc/pgbouncer/server.key', mode: '600', owner: 'postgres:postgres')
|
|
219
|
+
ssh_executor.upload_file(host, ssl_chain, '/etc/pgbouncer/ca.crt', mode: '644', owner: 'postgres:postgres') if ssl_chain
|
|
220
|
+
end
|
|
221
|
+
|
|
130
222
|
def get_postgres_max_connections(host)
|
|
131
223
|
# Try to get max_connections from running PostgreSQL
|
|
132
224
|
postgres_user = config.postgres_user
|
|
@@ -146,18 +238,43 @@ module ActivePostgres
|
|
|
146
238
|
def create_userlist(host)
|
|
147
239
|
puts ' Creating PgBouncer userlist with database users...'
|
|
148
240
|
|
|
149
|
-
|
|
150
|
-
app_user = config.app_user
|
|
151
|
-
users_to_add = [postgres_user, (app_user if app_user != postgres_user)].compact
|
|
152
|
-
pgbouncer = self
|
|
241
|
+
component = self
|
|
153
242
|
|
|
154
243
|
ssh_executor.execute_on_host(host) do
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
pgbouncer.send(:write_userlist_file, backend, userlist_entries)
|
|
244
|
+
userlist_entries = component.send(:userlist_entries_from_backend, self)
|
|
245
|
+
component.send(:write_userlist_file, self, userlist_entries)
|
|
158
246
|
end
|
|
159
247
|
end
|
|
160
248
|
|
|
249
|
+
def create_userlist_from_primary(host)
|
|
250
|
+
puts ' Creating app PgBouncer userlist from primary database users...'
|
|
251
|
+
|
|
252
|
+
userlist_entries = nil
|
|
253
|
+
component = self
|
|
254
|
+
|
|
255
|
+
ssh_executor.execute_on_host(config.primary_host) do
|
|
256
|
+
userlist_entries = component.send(:userlist_entries_from_backend, self)
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
ssh_executor.execute_on_host(host) do
|
|
260
|
+
component.send(:write_userlist_file, self, userlist_entries || [])
|
|
261
|
+
end
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def userlist_entries_from_backend(backend)
|
|
265
|
+
postgres_user = config.postgres_user
|
|
266
|
+
users_to_add = pgbouncer_database_users
|
|
267
|
+
|
|
268
|
+
users_to_add.filter_map { |user| fetch_user_hash(backend, user, postgres_user) }
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def pgbouncer_database_users
|
|
272
|
+
postgres_user = config.postgres_user
|
|
273
|
+
app_user = config.app_user
|
|
274
|
+
|
|
275
|
+
[postgres_user, (app_user if app_user != postgres_user)].compact
|
|
276
|
+
end
|
|
277
|
+
|
|
161
278
|
def fetch_user_hash(backend, user, postgres_user)
|
|
162
279
|
sql = build_user_hash_sql(user)
|
|
163
280
|
|
|
@@ -213,9 +330,11 @@ module ActivePostgres
|
|
|
213
330
|
interval = 5 if interval <= 0
|
|
214
331
|
|
|
215
332
|
repmgr_conf = '/etc/repmgr.conf'
|
|
333
|
+
repmgr_database = config.repmgr_database
|
|
216
334
|
postgres_user = config.postgres_user
|
|
217
335
|
_ = interval
|
|
218
336
|
_ = repmgr_conf
|
|
337
|
+
_ = repmgr_database
|
|
219
338
|
_ = postgres_user
|
|
220
339
|
|
|
221
340
|
upload_template(host, 'pgbouncer_follow_primary.sh.erb', '/usr/local/bin/pgbouncer-follow-primary', binding,
|
|
@@ -231,6 +350,40 @@ module ActivePostgres
|
|
|
231
350
|
execute :sudo, 'systemctl', 'start', 'pgbouncer-follow-primary.service'
|
|
232
351
|
end
|
|
233
352
|
end
|
|
353
|
+
|
|
354
|
+
def install_follow_dns_primary(host, pgbouncer_config)
|
|
355
|
+
interval = pgbouncer_config[:follow_primary_interval] || 5
|
|
356
|
+
interval = interval.to_i
|
|
357
|
+
interval = 5 if interval <= 0
|
|
358
|
+
primary_record = pgbouncer_config[:database_host]
|
|
359
|
+
_ = interval
|
|
360
|
+
_ = primary_record
|
|
361
|
+
_ = pgbouncer_config
|
|
362
|
+
|
|
363
|
+
upload_template(host, 'pgbouncer_follow_dns_primary.sh.erb', '/usr/local/bin/pgbouncer-follow-primary',
|
|
364
|
+
binding, mode: '755', owner: 'root:root')
|
|
365
|
+
upload_template(host, 'pgbouncer-follow-primary.service.erb',
|
|
366
|
+
'/etc/systemd/system/pgbouncer-follow-primary.service', binding, mode: '644', owner: 'root:root')
|
|
367
|
+
upload_template(host, 'pgbouncer-follow-primary.timer.erb',
|
|
368
|
+
'/etc/systemd/system/pgbouncer-follow-primary.timer', binding, mode: '644', owner: 'root:root')
|
|
369
|
+
|
|
370
|
+
ssh_executor.execute_on_host(host) do
|
|
371
|
+
execute :sudo, 'systemctl', 'daemon-reload'
|
|
372
|
+
execute :sudo, 'systemctl', 'enable', '--now', 'pgbouncer-follow-primary.timer'
|
|
373
|
+
execute :sudo, 'systemctl', 'start', 'pgbouncer-follow-primary.service'
|
|
374
|
+
end
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
def pgbouncer_primary_record
|
|
378
|
+
return config.pgbouncer_primary_record if config.respond_to?(:pgbouncer_primary_record)
|
|
379
|
+
|
|
380
|
+
dns_failover = config.component_config(:repmgr)[:dns_failover] || {}
|
|
381
|
+
dns_failover[:primary_record] || config.primary_replication_host
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
def app_host_value(app_host, key)
|
|
385
|
+
app_host[key.to_s] || app_host[key]
|
|
386
|
+
end
|
|
234
387
|
end
|
|
235
388
|
end
|
|
236
389
|
end
|
|
@@ -11,6 +11,7 @@ module ActivePostgres
|
|
|
11
11
|
setup_standby(host)
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
+
setup_inter_node_ssh if config.standby_hosts.any?
|
|
14
15
|
setup_dns_failover if dns_failover_enabled?
|
|
15
16
|
|
|
16
17
|
# Verify the entire cluster is healthy after setup
|
|
@@ -64,10 +65,7 @@ module ActivePostgres
|
|
|
64
65
|
puts "Setting up standby #{standby_host} (primary will not be touched)..."
|
|
65
66
|
|
|
66
67
|
version = config.version
|
|
67
|
-
|
|
68
|
-
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
|
|
69
|
-
"postgresql-#{version}-repmgr"
|
|
70
|
-
end
|
|
68
|
+
install_apt_packages(standby_host, "postgresql-#{version}-repmgr")
|
|
71
69
|
|
|
72
70
|
setup_standby(standby_host)
|
|
73
71
|
|
|
@@ -81,10 +79,7 @@ module ActivePostgres
|
|
|
81
79
|
|
|
82
80
|
version = config.version
|
|
83
81
|
config.all_hosts.each do |host|
|
|
84
|
-
|
|
85
|
-
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
|
|
86
|
-
"postgresql-#{version}-repmgr"
|
|
87
|
-
end
|
|
82
|
+
install_apt_packages(host, "postgresql-#{version}-repmgr")
|
|
88
83
|
install_postgres_sudoers(host)
|
|
89
84
|
end
|
|
90
85
|
end
|
|
@@ -275,6 +270,7 @@ module ActivePostgres
|
|
|
275
270
|
update_postgres_configs_on_standby(standby_host, version)
|
|
276
271
|
ensure_ssl_certs(standby_host, version) if config.component_enabled?(:ssl)
|
|
277
272
|
register_standby_with_primary(standby_host)
|
|
273
|
+
setup_inter_node_ssh
|
|
278
274
|
enable_repmgrd_if_configured(standby_host, repmgr_config)
|
|
279
275
|
return
|
|
280
276
|
end
|
|
@@ -428,9 +424,7 @@ module ActivePostgres
|
|
|
428
424
|
pub_keys = {}
|
|
429
425
|
|
|
430
426
|
all_hosts.each do |host|
|
|
431
|
-
|
|
432
|
-
pub_keys[host] = capture(:sudo, '-u', postgres_user, 'cat', "#{key_path}.pub").strip
|
|
433
|
-
end
|
|
427
|
+
pub_keys[host] = ensure_dns_ssh_key(host, key_path, [])
|
|
434
428
|
end
|
|
435
429
|
|
|
436
430
|
all_hosts.each do |host|
|
|
@@ -1073,13 +1067,23 @@ module ActivePostgres
|
|
|
1073
1067
|
def enable_repmgrd_if_configured(host, repmgr_config)
|
|
1074
1068
|
return if repmgr_config[:auto_failover] == false
|
|
1075
1069
|
|
|
1070
|
+
default_config = <<~CONF
|
|
1071
|
+
REPMGRD_ENABLED=yes
|
|
1072
|
+
REPMGRD_CONF="/etc/repmgr.conf"
|
|
1073
|
+
REPMGRD_OPTS=""
|
|
1074
|
+
REPMGRD_USER=postgres
|
|
1075
|
+
REPMGRD_BIN=/usr/bin/repmgrd
|
|
1076
|
+
REPMGRD_PIDFILE=/var/run/repmgrd.pid
|
|
1077
|
+
CONF
|
|
1078
|
+
|
|
1076
1079
|
ssh_executor.execute_on_host(host) do
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1080
|
+
upload! StringIO.new(default_config), '/tmp/repmgrd-default'
|
|
1081
|
+
execute :sudo, 'mv', '/tmp/repmgrd-default', '/etc/default/repmgrd'
|
|
1082
|
+
execute :sudo, 'chown', 'root:root', '/etc/default/repmgrd'
|
|
1083
|
+
execute :sudo, 'chmod', '644', '/etc/default/repmgrd'
|
|
1084
|
+
execute :sudo, 'systemctl', 'enable', 'repmgrd'
|
|
1085
|
+
execute :sudo, 'systemctl', 'restart', 'repmgrd'
|
|
1086
|
+
execute :pgrep, '-x', 'repmgrd'
|
|
1083
1087
|
end
|
|
1084
1088
|
end
|
|
1085
1089
|
|
|
@@ -39,6 +39,35 @@ module ActivePostgres
|
|
|
39
39
|
[primary_host] + standby_hosts
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
+
def pgbouncer_app_hosts
|
|
43
|
+
hosts = component_config(:pgbouncer)[:app_hosts] || component_config(:pgbouncer)['app_hosts'] || []
|
|
44
|
+
hosts = [hosts] unless hosts.is_a?(Array)
|
|
45
|
+
|
|
46
|
+
hosts.filter_map do |entry|
|
|
47
|
+
case entry
|
|
48
|
+
when String
|
|
49
|
+
entry.strip.empty? ? nil : { 'host' => entry.strip }
|
|
50
|
+
when Hash
|
|
51
|
+
host = entry['host'] || entry[:host] || entry['ip'] || entry[:ip]
|
|
52
|
+
next if host.to_s.strip.empty?
|
|
53
|
+
|
|
54
|
+
entry.merge('host' => host.to_s.strip)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def pgbouncer_primary_record
|
|
60
|
+
pgbouncer = component_config(:pgbouncer)
|
|
61
|
+
explicit = pgbouncer[:primary_record] || pgbouncer['primary_record']
|
|
62
|
+
return explicit.to_s.strip unless explicit.to_s.strip.empty?
|
|
63
|
+
|
|
64
|
+
dns_failover = component_config(:repmgr)[:dns_failover] || component_config(:repmgr)['dns_failover'] || {}
|
|
65
|
+
record = dns_failover[:primary_record] || dns_failover['primary_record']
|
|
66
|
+
return record.to_s.strip unless record.to_s.strip.empty?
|
|
67
|
+
|
|
68
|
+
primary_replication_host
|
|
69
|
+
end
|
|
70
|
+
|
|
42
71
|
def primary_host
|
|
43
72
|
@primary['host']
|
|
44
73
|
end
|
|
@@ -114,6 +143,13 @@ module ActivePostgres
|
|
|
114
143
|
end
|
|
115
144
|
end
|
|
116
145
|
|
|
146
|
+
if component_enabled?(:pgbouncer)
|
|
147
|
+
pgbouncer_app_hosts.each do |app_host|
|
|
148
|
+
host = app_host['host'] || app_host[:host]
|
|
149
|
+
raise Error, 'pgbouncer.app_hosts entries must include host' if host.to_s.strip.empty?
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
117
153
|
if component_enabled?(:repmgr)
|
|
118
154
|
dns_failover = component_config(:repmgr)[:dns_failover]
|
|
119
155
|
if dns_failover && dns_failover[:enabled]
|
|
@@ -240,8 +240,8 @@ module ActivePostgres
|
|
|
240
240
|
|
|
241
241
|
@ssh_executor.execute_on_host(host) do
|
|
242
242
|
# Install PgBouncer package
|
|
243
|
-
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
244
|
-
'-y', '-qq', 'pgbouncer'
|
|
243
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
244
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq', 'pgbouncer'
|
|
245
245
|
|
|
246
246
|
# Create required directories
|
|
247
247
|
execute :sudo, 'mkdir', '-p', '/var/log/pgbouncer'
|
|
@@ -50,6 +50,8 @@ shared: &shared
|
|
|
50
50
|
enabled: false
|
|
51
51
|
# Optional: Override default pgbouncer user
|
|
52
52
|
# user: pgbouncer
|
|
53
|
+
# Keep PgBouncer on every node pointed at the current primary.
|
|
54
|
+
# Set standby.pgbouncer_follow_primary: false on a specific standby to keep it local.
|
|
53
55
|
# follow_primary: true
|
|
54
56
|
# follow_primary_interval: 5
|
|
55
57
|
|
|
@@ -96,6 +96,8 @@ module ActivePostgres
|
|
|
96
96
|
return if defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
|
|
97
97
|
|
|
98
98
|
@rails_boot_attempted = true
|
|
99
|
+
ENV['RAILS_ENV'] ||= config.environment
|
|
100
|
+
ENV['BORING_ENVIRONMENT'] ||= config.environment
|
|
99
101
|
|
|
100
102
|
# Try loading just the Rails application (without full initialization)
|
|
101
103
|
# This makes Rails.application available for credential access
|
|
@@ -50,9 +50,10 @@ module ActivePostgres
|
|
|
50
50
|
'/etc/apt/sources.list.d/pgdg.list'
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
-
execute :sudo, 'apt-get', 'update', '-qq'
|
|
54
|
-
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
55
|
-
'
|
|
53
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update', '-qq'
|
|
54
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
55
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq',
|
|
56
|
+
'gnupg', 'wget', 'lsb-release', 'locales'
|
|
56
57
|
|
|
57
58
|
info 'Generating locales...'
|
|
58
59
|
execute :sudo, 'locale-gen', 'en_US.UTF-8'
|
|
@@ -73,8 +74,9 @@ module ActivePostgres
|
|
|
73
74
|
execute :sudo, 'systemctl', 'stop', 'postgresql' if test('systemctl is-active postgresql')
|
|
74
75
|
|
|
75
76
|
# Remove all PostgreSQL packages
|
|
76
|
-
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
77
|
-
|
|
77
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
78
|
+
'-o', 'DPkg::Lock::Timeout=300', 'remove', '--purge', '-y', '-qq', 'postgresql*'
|
|
79
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'autoremove', '-y', '-qq'
|
|
78
80
|
|
|
79
81
|
# Clean up OLD version directories only (preserve target version SSL certs, etc.)
|
|
80
82
|
other_versions.each do |old_version|
|
|
@@ -103,9 +105,10 @@ module ActivePostgres
|
|
|
103
105
|
"/etc/apt/sources.list.d/pgdg.list'"
|
|
104
106
|
execute :sudo, 'sh', '-c', pgdg_repo
|
|
105
107
|
|
|
106
|
-
execute :sudo, 'apt-get', 'update', '-qq'
|
|
107
|
-
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
108
|
-
|
|
108
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update', '-qq'
|
|
109
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
110
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq',
|
|
111
|
+
"postgresql-#{version}", "postgresql-client-#{version}"
|
|
109
112
|
|
|
110
113
|
execute :sudo, 'systemctl', 'enable', 'postgresql'
|
|
111
114
|
execute :sudo, 'systemctl', 'start', 'postgresql' unless test('systemctl is-active postgresql')
|
data/lib/tasks/postgres.rake
CHANGED
|
@@ -97,13 +97,14 @@ namespace :postgres do
|
|
|
97
97
|
|
|
98
98
|
# Remove packages
|
|
99
99
|
begin
|
|
100
|
-
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
100
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
101
|
+
'-o', 'DPkg::Lock::Timeout=300', 'remove', '--purge', '-y',
|
|
101
102
|
'postgresql*', 'pgbouncer', 'repmgr', 'prometheus-postgres-exporter'
|
|
102
103
|
rescue StandardError
|
|
103
104
|
nil
|
|
104
105
|
end
|
|
105
106
|
begin
|
|
106
|
-
execute :sudo, 'apt-get', 'autoremove', '-y'
|
|
107
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'autoremove', '-y'
|
|
107
108
|
rescue StandardError
|
|
108
109
|
nil
|
|
109
110
|
end
|
|
@@ -56,8 +56,10 @@ namespace :postgres do
|
|
|
56
56
|
puts "\nUpdating #{standby}..."
|
|
57
57
|
ssh_executor.execute_on_host(standby) do
|
|
58
58
|
execute :sudo, 'systemctl', 'stop', "postgresql@#{current_version}-main"
|
|
59
|
-
execute :sudo, 'apt-get', 'update'
|
|
60
|
-
execute :sudo, '
|
|
59
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
|
|
60
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
61
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '-y',
|
|
62
|
+
"postgresql-#{new_version}", "postgresql-contrib-#{new_version}"
|
|
61
63
|
|
|
62
64
|
puts "Upgrading cluster from #{current_version} to #{new_version}..."
|
|
63
65
|
execute :sudo, 'pg_upgradecluster', current_version.to_s, 'main'
|
|
@@ -99,8 +101,10 @@ namespace :postgres do
|
|
|
99
101
|
puts "\nUpdating #{primary}..."
|
|
100
102
|
ssh_executor.execute_on_host(primary) do
|
|
101
103
|
execute :sudo, 'systemctl', 'stop', "postgresql@#{current_version}-main"
|
|
102
|
-
execute :sudo, 'apt-get', 'update'
|
|
103
|
-
execute :sudo, '
|
|
104
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
|
|
105
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
106
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '-y',
|
|
107
|
+
"postgresql-#{new_version}", "postgresql-contrib-#{new_version}"
|
|
104
108
|
|
|
105
109
|
puts "Upgrading cluster from #{current_version} to #{new_version}..."
|
|
106
110
|
execute :sudo, 'pg_upgradecluster', current_version.to_s, 'main'
|
|
@@ -156,8 +160,9 @@ namespace :postgres do
|
|
|
156
160
|
|
|
157
161
|
puts "\n📋 Step 1: Patch standby #{standby}"
|
|
158
162
|
ssh_executor.execute_on_host(standby) do
|
|
159
|
-
execute :sudo, 'apt-get', 'update'
|
|
160
|
-
execute :sudo, '
|
|
163
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
|
|
164
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
165
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '--only-upgrade', '-y', "postgresql-#{version}"
|
|
161
166
|
execute :sudo, 'systemctl', 'restart', "postgresql@#{version}-main"
|
|
162
167
|
end
|
|
163
168
|
puts '✓ Standby patched and restarted'
|
|
@@ -169,8 +174,9 @@ namespace :postgres do
|
|
|
169
174
|
|
|
170
175
|
puts "\n📋 Step 3: Patch old primary #{primary}"
|
|
171
176
|
ssh_executor.execute_on_host(primary) do
|
|
172
|
-
execute :sudo, 'apt-get', 'update'
|
|
173
|
-
execute :sudo, '
|
|
177
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
|
|
178
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
179
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '--only-upgrade', '-y', "postgresql-#{version}"
|
|
174
180
|
execute :sudo, 'systemctl', 'restart', "postgresql@#{version}-main"
|
|
175
181
|
end
|
|
176
182
|
puts '✓ Old primary patched'
|
|
@@ -211,8 +217,9 @@ namespace :postgres do
|
|
|
211
217
|
puts "\n🔄 Starting in-place upgrade..."
|
|
212
218
|
ssh_executor.execute_on_host(host) do
|
|
213
219
|
execute :sudo, 'systemctl', 'stop', "postgresql@#{current_version}-main"
|
|
214
|
-
execute :sudo, 'apt-get', 'update'
|
|
215
|
-
execute :sudo, '
|
|
220
|
+
execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
|
|
221
|
+
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
|
|
222
|
+
'-o', 'DPkg::Lock::Timeout=300', 'install', '-y', "postgresql-#{new_version}"
|
|
216
223
|
|
|
217
224
|
execute :sudo, 'pg_upgradecluster', current_version.to_s, 'main'
|
|
218
225
|
execute :sudo, 'pg_dropcluster', '--stop', current_version.to_s, 'main'
|
|
@@ -57,6 +57,8 @@ log-path=/var/log/pgbackrest
|
|
|
57
57
|
|
|
58
58
|
# Process settings
|
|
59
59
|
process-max=<%= pgbackrest_config[:process_max] || 2 %>
|
|
60
|
+
archive-async=y
|
|
61
|
+
spool-path=/var/spool/pgbackrest
|
|
60
62
|
|
|
61
63
|
# Start/Stop (for consistent backups)
|
|
62
64
|
start-fast=y
|
|
@@ -68,4 +70,3 @@ pg1-port=5432
|
|
|
68
70
|
pg1-socket-path=/var/run/postgresql
|
|
69
71
|
pg1-user=<%= config.postgres_user %>
|
|
70
72
|
|
|
71
|
-
|
data/templates/pgbouncer.ini.erb
CHANGED
|
@@ -36,6 +36,8 @@ client_idle_timeout = <%= pgbouncer_config[:client_idle_timeout] || 0 %>
|
|
|
36
36
|
client_login_timeout = <%= pgbouncer_config[:client_login_timeout] || 60 %>
|
|
37
37
|
query_timeout = <%= pgbouncer_config[:query_timeout] || 0 %>
|
|
38
38
|
query_wait_timeout = <%= pgbouncer_config[:query_wait_timeout] || 120 %>
|
|
39
|
+
dns_max_ttl = <%= pgbouncer_config[:dns_max_ttl] || 5 %>
|
|
40
|
+
dns_nxdomain_ttl = <%= pgbouncer_config[:dns_nxdomain_ttl] || 15 %>
|
|
39
41
|
|
|
40
42
|
# Logging and administration
|
|
41
43
|
admin_users = <%= pgbouncer_config[:admin_users] || config.postgres_user %>
|
|
@@ -62,4 +64,3 @@ pidfile = <%= pgbouncer_config[:pidfile] %>
|
|
|
62
64
|
logfile = <%= pgbouncer_config[:logfile] %>
|
|
63
65
|
<% end %>
|
|
64
66
|
|
|
65
|
-
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
PRIMARY_RECORD="<%= primary_record %>"
|
|
5
|
+
PGBOUNCER_INI="/etc/pgbouncer/pgbouncer.ini"
|
|
6
|
+
PGBOUNCER_SERVICE="pgbouncer"
|
|
7
|
+
STATE_DIR="/var/lib/active-postgres"
|
|
8
|
+
STATE_FILE="${STATE_DIR}/pgbouncer-primary-ip"
|
|
9
|
+
LOG_TAG="pgbouncer-follow-primary"
|
|
10
|
+
|
|
11
|
+
resolved_ip=$(getent ahostsv4 "$PRIMARY_RECORD" | awk '{print $1; exit}' || true)
|
|
12
|
+
if [[ -z "$resolved_ip" ]]; then
|
|
13
|
+
/usr/bin/logger -t "$LOG_TAG" "Could not resolve ${PRIMARY_RECORD}"
|
|
14
|
+
exit 1
|
|
15
|
+
fi
|
|
16
|
+
|
|
17
|
+
current_host=$(awk -F'host=' '/^\\* = host=/{print $2}' "$PGBOUNCER_INI" | awk '{print $1}')
|
|
18
|
+
previous_ip=""
|
|
19
|
+
if [[ -f "$STATE_FILE" ]]; then
|
|
20
|
+
previous_ip=$(cat "$STATE_FILE")
|
|
21
|
+
fi
|
|
22
|
+
|
|
23
|
+
changed=0
|
|
24
|
+
if [[ "$current_host" != "$PRIMARY_RECORD" ]]; then
|
|
25
|
+
/usr/bin/sed -i -E "s/^(\\* = host=)[^ ]+/\\1${PRIMARY_RECORD}/" "$PGBOUNCER_INI"
|
|
26
|
+
changed=1
|
|
27
|
+
fi
|
|
28
|
+
|
|
29
|
+
if [[ "$previous_ip" != "$resolved_ip" ]]; then
|
|
30
|
+
changed=1
|
|
31
|
+
fi
|
|
32
|
+
|
|
33
|
+
/usr/bin/install -d -m 0755 "$STATE_DIR"
|
|
34
|
+
printf "%s\n" "$resolved_ip" > "$STATE_FILE"
|
|
35
|
+
|
|
36
|
+
if [[ "$changed" == "1" ]]; then
|
|
37
|
+
/usr/bin/systemctl restart "$PGBOUNCER_SERVICE"
|
|
38
|
+
/usr/bin/logger -t "$LOG_TAG" "Refreshed PgBouncer primary target ${PRIMARY_RECORD} (${resolved_ip})"
|
|
39
|
+
fi
|
|
@@ -2,17 +2,19 @@
|
|
|
2
2
|
set -euo pipefail
|
|
3
3
|
|
|
4
4
|
REPMGR_CONF="<%= repmgr_conf %>"
|
|
5
|
+
REPMGR_DB="<%= repmgr_database %>"
|
|
5
6
|
PGBOUNCER_INI="/etc/pgbouncer/pgbouncer.ini"
|
|
6
7
|
PGBOUNCER_SERVICE="pgbouncer"
|
|
7
8
|
LOG_TAG="pgbouncer-follow-primary"
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
cluster_nodes=$(/usr/bin/sudo -u <%= postgres_user %> env HOME=/var/lib/postgresql \
|
|
11
|
+
psql -h /var/run/postgresql -d "$REPMGR_DB" -tAF',' \
|
|
12
|
+
-c "SELECT type, conninfo FROM repmgr.nodes WHERE active = true" 2>/dev/null || true)
|
|
13
|
+
if [[ -z "$cluster_nodes" ]]; then
|
|
12
14
|
exit 0
|
|
13
15
|
fi
|
|
14
16
|
|
|
15
|
-
primary_conninfo=$(printf "%s\n" "$
|
|
17
|
+
primary_conninfo=$(printf "%s\n" "$cluster_nodes" | awk -F',' '$1 == "primary" {print $2; exit}')
|
|
16
18
|
if [[ -z "$primary_conninfo" ]]; then
|
|
17
19
|
exit 0
|
|
18
20
|
fi
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
umask 077
|
|
5
|
+
|
|
6
|
+
ENV_FILE="/etc/active-postgres-log-archive.env"
|
|
7
|
+
if [[ ! -r "$ENV_FILE" ]]; then
|
|
8
|
+
echo "missing $ENV_FILE" >&2
|
|
9
|
+
exit 1
|
|
10
|
+
fi
|
|
11
|
+
|
|
12
|
+
set -a
|
|
13
|
+
source "$ENV_FILE"
|
|
14
|
+
set +a
|
|
15
|
+
|
|
16
|
+
LOG_DIR="${POSTGRES_LOG_ARCHIVE_LOG_DIR:-/var/log/postgresql}"
|
|
17
|
+
WORK_DIR="${POSTGRES_LOG_ARCHIVE_WORK_DIR:-/var/tmp/active-postgres-log-archive}"
|
|
18
|
+
PREFIX="${POSTGRES_LOG_ARCHIVE_PREFIX:-postgres-logs}"
|
|
19
|
+
NODE="${POSTGRES_LOG_ARCHIVE_NODE:-$(hostname -f 2>/dev/null || hostname)}"
|
|
20
|
+
DATE_PATH="$(date -u +%Y/%m/%d)"
|
|
21
|
+
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
22
|
+
LOCK_FILE="/var/run/active-postgres-log-archive.lock"
|
|
23
|
+
|
|
24
|
+
mkdir -p "$WORK_DIR"
|
|
25
|
+
exec 9>"$LOCK_FILE"
|
|
26
|
+
flock -n 9 || exit 0
|
|
27
|
+
|
|
28
|
+
shopt -s nullglob
|
|
29
|
+
log_files=("$LOG_DIR"/postgresql-*.log "$LOG_DIR"/repmgr.log)
|
|
30
|
+
if [[ ${#log_files[@]} -eq 0 ]]; then
|
|
31
|
+
exit 0
|
|
32
|
+
fi
|
|
33
|
+
|
|
34
|
+
archive="$WORK_DIR/postgresql-logs-${NODE}-${STAMP}.tar.gz"
|
|
35
|
+
cleanup() {
|
|
36
|
+
rm -f "$archive"
|
|
37
|
+
}
|
|
38
|
+
trap cleanup EXIT
|
|
39
|
+
|
|
40
|
+
tar_paths=()
|
|
41
|
+
for file in "${log_files[@]}"; do
|
|
42
|
+
[[ -f "$file" ]] && tar_paths+=("${file#/}")
|
|
43
|
+
done
|
|
44
|
+
|
|
45
|
+
if [[ ${#tar_paths[@]} -eq 0 ]]; then
|
|
46
|
+
exit 0
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
set +e
|
|
50
|
+
tar --warning=no-file-changed --ignore-failed-read -czf "$archive" -C / "${tar_paths[@]}"
|
|
51
|
+
tar_status=$?
|
|
52
|
+
set -e
|
|
53
|
+
|
|
54
|
+
if [[ $tar_status -gt 1 ]]; then
|
|
55
|
+
exit "$tar_status"
|
|
56
|
+
fi
|
|
57
|
+
|
|
58
|
+
if [[ ! -s "$archive" ]]; then
|
|
59
|
+
exit 0
|
|
60
|
+
fi
|
|
61
|
+
|
|
62
|
+
destination="s3://${AWS_BUCKET}/${PREFIX%/}/${NODE}/${DATE_PATH}/$(basename "$archive")"
|
|
63
|
+
if command -v aws >/dev/null 2>&1; then
|
|
64
|
+
aws_args=(s3 cp "$archive" "$destination" --only-show-errors)
|
|
65
|
+
if [[ -n "${AWS_ENDPOINT_URL:-}" ]]; then
|
|
66
|
+
aws_args+=(--endpoint-url "$AWS_ENDPOINT_URL")
|
|
67
|
+
fi
|
|
68
|
+
|
|
69
|
+
aws "${aws_args[@]}"
|
|
70
|
+
elif command -v s3cmd >/dev/null 2>&1; then
|
|
71
|
+
s3cmd_args=(
|
|
72
|
+
--access_key="$AWS_ACCESS_KEY_ID"
|
|
73
|
+
--secret_key="$AWS_SECRET_ACCESS_KEY"
|
|
74
|
+
--region="${AWS_DEFAULT_REGION:-us-east-1}"
|
|
75
|
+
--no-progress
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
if [[ -n "${AWS_ENDPOINT_URL:-}" ]]; then
|
|
79
|
+
endpoint="${AWS_ENDPOINT_URL#https://}"
|
|
80
|
+
endpoint="${endpoint#http://}"
|
|
81
|
+
s3cmd_args+=(--host="$endpoint" --host-bucket="${endpoint}/%(bucket)")
|
|
82
|
+
fi
|
|
83
|
+
|
|
84
|
+
s3cmd "${s3cmd_args[@]}" put "$archive" "$destination"
|
|
85
|
+
else
|
|
86
|
+
echo "missing aws or s3cmd command for PostgreSQL log archive" >&2
|
|
87
|
+
exit 1
|
|
88
|
+
fi
|
|
@@ -95,8 +95,12 @@ max_parallel_maintenance_workers = <%= pg_config[:max_parallel_maintenance_worke
|
|
|
95
95
|
logging_collector = <%= pg_config[:logging_collector] || 'on' %>
|
|
96
96
|
log_directory = '<%= pg_config[:log_directory] || 'log' %>'
|
|
97
97
|
log_filename = '<%= pg_config[:log_filename] || 'postgresql-%Y-%m-%d_%H%M%S.log' %>'
|
|
98
|
+
log_file_mode = <%= pg_config[:log_file_mode] || '0600' %>
|
|
98
99
|
log_rotation_age = <%= pg_config[:log_rotation_age] || '1d' %>
|
|
99
100
|
log_rotation_size = <%= pg_config[:log_rotation_size] || '100MB' %>
|
|
101
|
+
<% if pg_config[:log_truncate_on_rotation] %>
|
|
102
|
+
log_truncate_on_rotation = <%= pg_config[:log_truncate_on_rotation] %>
|
|
103
|
+
<% end %>
|
|
100
104
|
log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '
|
|
101
105
|
log_timezone = 'UTC'
|
|
102
106
|
<% if pg_config[:log_checkpoints] %>
|
|
@@ -143,7 +147,7 @@ jit = <%= pg_config[:jit] %>
|
|
|
143
147
|
<% end %>
|
|
144
148
|
|
|
145
149
|
# Additional settings
|
|
146
|
-
shared_preload_libraries = '<%= pg_config[:shared_preload_libraries] || 'pg_stat_statements' %>'
|
|
150
|
+
shared_preload_libraries = '<%= pg_config[:shared_preload_libraries] || (config.component_enabled?(:repmgr) ? 'pg_stat_statements, repmgr' : 'pg_stat_statements') %>'
|
|
147
151
|
<% if pg_config[:'pg_stat_statements.max'] %>
|
|
148
152
|
pg_stat_statements.max = <%= pg_config[:'pg_stat_statements.max'] %>
|
|
149
153
|
<% end %>
|
|
@@ -157,5 +161,3 @@ pg_stat_statements.track = <%= pg_config[:'pg_stat_statements.track'] %>
|
|
|
157
161
|
<%= key %> = <%= value %>
|
|
158
162
|
<% end %>
|
|
159
163
|
<% end %>
|
|
160
|
-
|
|
161
|
-
|
data/templates/repmgr.conf.erb
CHANGED
|
@@ -36,7 +36,7 @@ service_stop_command='sudo systemctl stop postgresql@<%= config.version %>-main'
|
|
|
36
36
|
service_restart_command='sudo systemctl restart postgresql@<%= config.version %>-main'
|
|
37
37
|
service_reload_command='sudo systemctl reload postgresql@<%= config.version %>-main'
|
|
38
38
|
|
|
39
|
-
ssh_options='-i /var/lib/postgresql/.ssh/active_postgres_dns -o StrictHostKeyChecking=no'
|
|
39
|
+
ssh_options='-l <%= repmgr_config[:ssh_user] || config.postgres_user %> -i /var/lib/postgresql/.ssh/active_postgres_dns -o IdentitiesOnly=yes -o StrictHostKeyChecking=no'
|
|
40
40
|
|
|
41
41
|
log_level=INFO
|
|
42
42
|
log_facility=STDERR
|
|
@@ -6,6 +6,7 @@ DNS_USER="<%= dns_user %>"
|
|
|
6
6
|
DNS_SERVERS=(<%= dns_servers.map(&:to_s).join(' ') %>)
|
|
7
7
|
DNS_SSH_KEY="<%= dns_ssh_key_path %>"
|
|
8
8
|
DNSMASQ_FILE="/etc/dnsmasq.d/active_postgres.conf"
|
|
9
|
+
DNSMASQ_STATIC_FILE="/etc/dnsmasq.d/messhy.conf"
|
|
9
10
|
PRIMARY_RECORDS=(<%= primary_records.map(&:to_s).join(' ') %>)
|
|
10
11
|
REPLICA_RECORDS=(<%= replica_records.map(&:to_s).join(' ') %>)
|
|
11
12
|
SSH_STRICT_HOST_KEY="<%= ssh_strict_host_key %>"
|
|
@@ -16,7 +17,7 @@ REPMGR_DB=$(grep -oP "dbname=\K[^ ']+" "$REPMGR_CONF" | head -1)
|
|
|
16
17
|
REPMGR_USER=$(grep -oP "user=\K[^ ']+" "$REPMGR_CONF" 2>/dev/null | head -1)
|
|
17
18
|
REPMGR_USER="${REPMGR_USER:-repmgr}"
|
|
18
19
|
|
|
19
|
-
cluster_data=$(
|
|
20
|
+
cluster_data=$(psql -h /var/run/postgresql -d "$REPMGR_DB" -tAF',' -c "SELECT type, conninfo FROM repmgr.nodes WHERE active = true" 2>/dev/null || true)
|
|
20
21
|
if [[ -z "$cluster_data" ]]; then
|
|
21
22
|
exit 0
|
|
22
23
|
fi
|
|
@@ -48,14 +49,20 @@ if [[ -n "$standby_hosts" ]]; then
|
|
|
48
49
|
fi
|
|
49
50
|
|
|
50
51
|
ssh_opts=(-i "$DNS_SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY" -o UserKnownHostsFile="$SSH_KNOWN_HOSTS")
|
|
52
|
+
cleanup_cmd=""
|
|
53
|
+
for record in "${PRIMARY_RECORDS[@]}" "${REPLICA_RECORDS[@]}"; do
|
|
54
|
+
if [[ -n "$record" ]]; then
|
|
55
|
+
cleanup_cmd="${cleanup_cmd}sudo sed -i '\\#^address=/${record}/#d' ${DNSMASQ_STATIC_FILE} 2>/dev/null || true; "
|
|
56
|
+
fi
|
|
57
|
+
done
|
|
51
58
|
|
|
52
59
|
for server in "${DNS_SERVERS[@]}"; do
|
|
53
60
|
if [[ -z "$server" ]]; then
|
|
54
61
|
continue
|
|
55
62
|
fi
|
|
56
63
|
|
|
57
|
-
if ! /usr/bin/ssh "${ssh_opts[@]}" "${DNS_USER}@${server}" \
|
|
58
|
-
"sudo
|
|
64
|
+
if ! printf "%s" "$content" | /usr/bin/ssh "${ssh_opts[@]}" "${DNS_USER}@${server}" \
|
|
65
|
+
"${cleanup_cmd}sudo tee ${DNSMASQ_FILE} >/dev/null && sudo systemctl restart dnsmasq"; then
|
|
59
66
|
/usr/bin/logger -t "$LOG_TAG" "Failed updating dnsmasq on ${server}"
|
|
60
67
|
fi
|
|
61
68
|
done
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: active_postgres
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.9.
|
|
4
|
+
version: 0.9.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- BoringCache
|
|
@@ -199,7 +199,9 @@ files:
|
|
|
199
199
|
- templates/pgbouncer-follow-primary.service.erb
|
|
200
200
|
- templates/pgbouncer-follow-primary.timer.erb
|
|
201
201
|
- templates/pgbouncer.ini.erb
|
|
202
|
+
- templates/pgbouncer_follow_dns_primary.sh.erb
|
|
202
203
|
- templates/pgbouncer_follow_primary.sh.erb
|
|
204
|
+
- templates/postgres_log_archive.sh.erb
|
|
203
205
|
- templates/postgresql.conf.erb
|
|
204
206
|
- templates/repmgr.conf.erb
|
|
205
207
|
- templates/repmgr_dns_failover.sh.erb
|