active_postgres 0.8.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +142 -5
- data/lib/active_postgres/cli.rb +7 -0
- data/lib/active_postgres/components/core.rb +2 -7
- data/lib/active_postgres/components/extensions.rb +40 -35
- data/lib/active_postgres/components/monitoring.rb +256 -4
- data/lib/active_postgres/components/pgbackrest.rb +91 -5
- data/lib/active_postgres/components/pgbouncer.rb +58 -7
- data/lib/active_postgres/components/repmgr.rb +448 -62
- data/lib/active_postgres/configuration.rb +66 -1
- data/lib/active_postgres/connection_pooler.rb +3 -12
- data/lib/active_postgres/credentials.rb +3 -3
- data/lib/active_postgres/direct_executor.rb +1 -2
- data/lib/active_postgres/generators/active_postgres/install_generator.rb +2 -0
- data/lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb +28 -1
- data/lib/active_postgres/health_checker.rb +29 -7
- data/lib/active_postgres/installer.rb +9 -0
- data/lib/active_postgres/overview.rb +351 -0
- data/lib/active_postgres/rollback_manager.rb +4 -8
- data/lib/active_postgres/secrets.rb +23 -5
- data/lib/active_postgres/ssh_executor.rb +44 -19
- data/lib/active_postgres/version.rb +1 -1
- data/lib/active_postgres.rb +1 -0
- data/lib/tasks/postgres.rake +77 -4
- data/lib/tasks/rotate_credentials.rake +4 -16
- data/templates/pg_hba.conf.erb +4 -1
- data/templates/pgbackrest.conf.erb +28 -0
- data/templates/pgbouncer-follow-primary.service.erb +8 -0
- data/templates/pgbouncer-follow-primary.timer.erb +11 -0
- data/templates/pgbouncer.ini.erb +2 -0
- data/templates/pgbouncer_follow_primary.sh.erb +34 -0
- data/templates/postgresql.conf.erb +4 -0
- data/templates/repmgr.conf.erb +10 -3
- data/templates/repmgr_dns_failover.sh.erb +59 -0
- metadata +6 -1
|
@@ -2,7 +2,8 @@ require 'yaml'
|
|
|
2
2
|
|
|
3
3
|
module ActivePostgres
|
|
4
4
|
class Configuration
|
|
5
|
-
attr_reader :environment, :version, :user, :ssh_key, :primary, :standbys, :components, :secrets_config,
|
|
5
|
+
attr_reader :environment, :version, :user, :ssh_key, :ssh_host_key_verification, :primary, :standbys, :components, :secrets_config,
|
|
6
|
+
:database_config
|
|
6
7
|
|
|
7
8
|
def initialize(config_hash, environment = 'development')
|
|
8
9
|
@environment = environment
|
|
@@ -13,6 +14,9 @@ module ActivePostgres
|
|
|
13
14
|
@version = env_config['version'] || 18
|
|
14
15
|
@user = env_config['user'] || 'ubuntu'
|
|
15
16
|
@ssh_key = File.expand_path(env_config['ssh_key'] || '~/.ssh/id_rsa')
|
|
17
|
+
@ssh_host_key_verification = normalize_ssh_host_key_verification(
|
|
18
|
+
env_config['ssh_host_key_verification'] || env_config['ssh_verify_host_key']
|
|
19
|
+
)
|
|
16
20
|
|
|
17
21
|
@primary = env_config['primary'] || {}
|
|
18
22
|
@standbys = env_config['standby'] || []
|
|
@@ -91,6 +95,46 @@ module ActivePostgres
|
|
|
91
95
|
|
|
92
96
|
# Validate required secrets if components are enabled
|
|
93
97
|
raise Error, 'Missing replication_password secret' if component_enabled?(:repmgr) && !secrets_config['replication_password']
|
|
98
|
+
raise Error, 'Missing monitoring_password secret' if component_enabled?(:monitoring) && !secrets_config['monitoring_password']
|
|
99
|
+
if component_enabled?(:monitoring)
|
|
100
|
+
grafana_config = component_config(:monitoring)[:grafana] || {}
|
|
101
|
+
if grafana_config[:enabled] && !secrets_config['grafana_admin_password']
|
|
102
|
+
raise Error, 'Missing grafana_admin_password secret'
|
|
103
|
+
end
|
|
104
|
+
if grafana_config[:enabled] && grafana_config[:host].to_s.strip.empty?
|
|
105
|
+
raise Error, 'monitoring.grafana.host is required when grafana is enabled'
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
if component_enabled?(:pgbackrest)
|
|
109
|
+
pg_config = component_config(:pgbackrest)
|
|
110
|
+
retention_full = pg_config[:retention_full]
|
|
111
|
+
retention_archive = pg_config[:retention_archive]
|
|
112
|
+
if retention_full && retention_archive && retention_archive.to_i < retention_full.to_i
|
|
113
|
+
raise Error, 'pgbackrest.retention_archive must be >= retention_full for PITR safety'
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
if component_enabled?(:repmgr)
|
|
118
|
+
dns_failover = component_config(:repmgr)[:dns_failover]
|
|
119
|
+
if dns_failover && dns_failover[:enabled]
|
|
120
|
+
domains = Array(dns_failover[:domains] || dns_failover[:domain]).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
121
|
+
servers = Array(dns_failover[:dns_servers])
|
|
122
|
+
provider = (dns_failover[:provider] || 'dnsmasq').to_s.strip
|
|
123
|
+
|
|
124
|
+
raise Error, 'dns_failover.domain or dns_failover.domains is required when enabled' if domains.empty?
|
|
125
|
+
raise Error, 'dns_failover.dns_servers is required when enabled' if servers.empty?
|
|
126
|
+
raise Error, "Unsupported dns_failover provider '#{provider}'" unless provider == 'dnsmasq'
|
|
127
|
+
|
|
128
|
+
servers.each do |server|
|
|
129
|
+
next unless server.is_a?(Hash)
|
|
130
|
+
|
|
131
|
+
ssh_host = server['ssh_host'] || server[:ssh_host] || server['host'] || server[:host]
|
|
132
|
+
private_ip = server['private_ip'] || server[:private_ip] || server['ip'] || server[:ip]
|
|
133
|
+
raise Error, 'dns_failover.dns_servers entries must include host/ssh_host or private_ip' if
|
|
134
|
+
(ssh_host.nil? || ssh_host.to_s.strip.empty?) && (private_ip.nil? || private_ip.to_s.strip.empty?)
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
94
138
|
|
|
95
139
|
true
|
|
96
140
|
end
|
|
@@ -108,6 +152,10 @@ module ActivePostgres
|
|
|
108
152
|
component_config(:repmgr)[:database] || 'repmgr'
|
|
109
153
|
end
|
|
110
154
|
|
|
155
|
+
def replication_user
|
|
156
|
+
component_config(:repmgr)[:replication_user] || 'replication'
|
|
157
|
+
end
|
|
158
|
+
|
|
111
159
|
def pgbouncer_user
|
|
112
160
|
component_config(:pgbouncer)[:user] || 'pgbouncer'
|
|
113
161
|
end
|
|
@@ -195,5 +243,22 @@ module ActivePostgres
|
|
|
195
243
|
value
|
|
196
244
|
end
|
|
197
245
|
end
|
|
246
|
+
|
|
247
|
+
def normalize_ssh_host_key_verification(value)
|
|
248
|
+
return :always if value.nil?
|
|
249
|
+
|
|
250
|
+
normalized = case value
|
|
251
|
+
when Symbol
|
|
252
|
+
value
|
|
253
|
+
else
|
|
254
|
+
value.to_s.strip.downcase.tr('-', '_').to_sym
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
return :always if normalized == :always
|
|
258
|
+
return :accept_new if %i[accept_new acceptnew new].include?(normalized)
|
|
259
|
+
|
|
260
|
+
raise Error,
|
|
261
|
+
"Invalid ssh_host_key_verification '#{value}'. Use 'always' or 'accept_new'."
|
|
262
|
+
end
|
|
198
263
|
end
|
|
199
264
|
end
|
|
@@ -321,10 +321,7 @@ module ActivePostgres
|
|
|
321
321
|
WHERE passwd IS NOT NULL
|
|
322
322
|
SQL
|
|
323
323
|
|
|
324
|
-
|
|
325
|
-
execute :chmod, '644', '/tmp/get_all_users.sql'
|
|
326
|
-
userlist = capture(:sudo, '-u', postgres_user, 'psql', '-t', '-f', '/tmp/get_all_users.sql').strip
|
|
327
|
-
execute :rm, '-f', '/tmp/get_all_users.sql'
|
|
324
|
+
userlist = @ssh_executor.run_sql_on_backend(self, sql, postgres_user: postgres_user).to_s.strip
|
|
328
325
|
|
|
329
326
|
unless userlist.include?(pgbouncer_user)
|
|
330
327
|
pgbouncer_pass = SecureRandom.hex(16)
|
|
@@ -335,10 +332,7 @@ module ActivePostgres
|
|
|
335
332
|
"GRANT CONNECT ON DATABASE postgres TO #{pgbouncer_user};"
|
|
336
333
|
].join("\n")
|
|
337
334
|
|
|
338
|
-
|
|
339
|
-
execute :chmod, '644', '/tmp/create_pgbouncer_user.sql'
|
|
340
|
-
execute :sudo, '-u', postgres_user, 'psql', '-f', '/tmp/create_pgbouncer_user.sql'
|
|
341
|
-
execute :rm, '-f', '/tmp/create_pgbouncer_user.sql'
|
|
335
|
+
@ssh_executor.run_sql_on_backend(self, sql, postgres_user: postgres_user, tuples_only: false, capture: false)
|
|
342
336
|
|
|
343
337
|
sql = <<~SQL.strip
|
|
344
338
|
SELECT passwd
|
|
@@ -346,10 +340,7 @@ module ActivePostgres
|
|
|
346
340
|
WHERE usename = '#{pgbouncer_user}'
|
|
347
341
|
SQL
|
|
348
342
|
|
|
349
|
-
|
|
350
|
-
execute :chmod, '644', '/tmp/get_pgbouncer_pass.sql'
|
|
351
|
-
encrypted = capture(:sudo, '-u', postgres_user, 'psql', '-t', '-f', '/tmp/get_pgbouncer_pass.sql').strip
|
|
352
|
-
execute :rm, '-f', '/tmp/get_pgbouncer_pass.sql'
|
|
343
|
+
encrypted = @ssh_executor.run_sql_on_backend(self, sql, postgres_user: postgres_user).to_s.strip
|
|
353
344
|
|
|
354
345
|
userlist += "\n\"#{pgbouncer_user}\" \"#{encrypted}\""
|
|
355
346
|
end
|
|
@@ -2,8 +2,8 @@ module ActivePostgres
|
|
|
2
2
|
class Credentials
|
|
3
3
|
def self.get(key_path)
|
|
4
4
|
# Try to get from Rails credentials if Rails is available
|
|
5
|
-
if defined?(Rails) && Rails.respond_to?(:application) && Rails.application
|
|
6
|
-
value = Rails.application.credentials.dig(*key_path.split('.').map(&:to_sym))
|
|
5
|
+
if defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
|
|
6
|
+
value = ::Rails.application.credentials.dig(*key_path.split('.').map(&:to_sym))
|
|
7
7
|
return value if value
|
|
8
8
|
end
|
|
9
9
|
|
|
@@ -11,7 +11,7 @@ module ActivePostgres
|
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def self.available?
|
|
14
|
-
defined?(Rails) && Rails.respond_to?(:application) && Rails.application&.credentials
|
|
14
|
+
defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application&.credentials
|
|
15
15
|
end
|
|
16
16
|
end
|
|
17
17
|
end
|
|
@@ -83,8 +83,7 @@ module ActivePostgres
|
|
|
83
83
|
path = value.sub('rails_credentials:', '')
|
|
84
84
|
keys = path.split('.').map(&:to_sym)
|
|
85
85
|
|
|
86
|
-
|
|
87
|
-
keys.reduce(credentials) { |obj, key| obj&.dig(key) }
|
|
86
|
+
Rails.application.credentials.dig(*keys)
|
|
88
87
|
rescue NameError
|
|
89
88
|
nil
|
|
90
89
|
end
|
|
@@ -112,6 +112,8 @@ module ActivePostgres
|
|
|
112
112
|
puts " superuser_password: \"#{generate_secure_password}\""
|
|
113
113
|
puts " replication_password: \"#{generate_secure_password}\""
|
|
114
114
|
puts " repmgr_password: \"#{generate_secure_password}\""
|
|
115
|
+
puts " monitoring_password: \"#{generate_secure_password}\""
|
|
116
|
+
puts " grafana_admin_password: \"#{generate_secure_password}\""
|
|
115
117
|
puts
|
|
116
118
|
puts '🔐 Secure passwords have been auto-generated above.'
|
|
117
119
|
puts '📌 Update primary_host and replica_host with your actual IPs after provisioning.'
|
|
@@ -11,6 +11,8 @@ shared: &shared
|
|
|
11
11
|
version: 18
|
|
12
12
|
user: ubuntu
|
|
13
13
|
ssh_key: ~/.ssh/id_rsa
|
|
14
|
+
# SSH host key verification: always (strict) or accept_new (first connection)
|
|
15
|
+
ssh_host_key_verification: always
|
|
14
16
|
|
|
15
17
|
components:
|
|
16
18
|
core:
|
|
@@ -31,17 +33,41 @@ shared: &shared
|
|
|
31
33
|
# Optional: Override default repmgr user/database
|
|
32
34
|
# user: repmgr
|
|
33
35
|
# database: repmgr
|
|
36
|
+
# replication_user: replication
|
|
37
|
+
# dns_failover:
|
|
38
|
+
# enabled: true
|
|
39
|
+
# provider: dnsmasq
|
|
40
|
+
# domain: mesh.internal
|
|
41
|
+
# dns_servers:
|
|
42
|
+
# - host: 18.170.173.14
|
|
43
|
+
# private_ip: 10.8.0.10
|
|
44
|
+
# - host: 98.85.183.175
|
|
45
|
+
# private_ip: 10.8.0.110
|
|
46
|
+
# primary_record: db-primary.mesh.internal
|
|
47
|
+
# replica_record: db-replica.mesh.internal
|
|
34
48
|
|
|
35
49
|
pgbouncer:
|
|
36
50
|
enabled: false
|
|
37
51
|
# Optional: Override default pgbouncer user
|
|
38
52
|
# user: pgbouncer
|
|
53
|
+
# follow_primary: true
|
|
54
|
+
# follow_primary_interval: 5
|
|
39
55
|
|
|
40
56
|
pgbackrest:
|
|
41
57
|
enabled: false
|
|
42
58
|
|
|
43
59
|
monitoring:
|
|
44
60
|
enabled: false
|
|
61
|
+
# user: postgres_exporter
|
|
62
|
+
# node_exporter: true
|
|
63
|
+
# node_exporter_port: 9100
|
|
64
|
+
# node_exporter_listen_address: 10.8.0.10
|
|
65
|
+
# grafana:
|
|
66
|
+
# enabled: true
|
|
67
|
+
# host: grafana.example.com
|
|
68
|
+
# listen_address: 10.8.0.10
|
|
69
|
+
# port: 3000
|
|
70
|
+
# prometheus_url: http://prometheus.example.com:9090
|
|
45
71
|
|
|
46
72
|
ssl:
|
|
47
73
|
enabled: false
|
|
@@ -80,6 +106,7 @@ production:
|
|
|
80
106
|
superuser_password: rails_credentials:postgres.superuser_password
|
|
81
107
|
replication_password: rails_credentials:postgres.replication_password
|
|
82
108
|
repmgr_password: rails_credentials:postgres.repmgr_password
|
|
109
|
+
monitoring_password: rails_credentials:postgres.monitoring_password
|
|
110
|
+
# grafana_admin_password: rails_credentials:postgres.grafana_admin_password
|
|
83
111
|
pgbouncer_password: rails_credentials:postgres.password
|
|
84
112
|
app_password: rails_credentials:postgres.password
|
|
85
|
-
|
|
@@ -13,7 +13,21 @@ module ActivePostgres
|
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
private def create_executor
|
|
16
|
-
|
|
16
|
+
if use_ssh_executor?
|
|
17
|
+
SSHExecutor.new(config, quiet: true)
|
|
18
|
+
else
|
|
19
|
+
DirectExecutor.new(config, quiet: true)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def use_ssh_executor?
|
|
24
|
+
mode = ENV['ACTIVE_POSTGRES_STATUS_MODE'].to_s.strip.downcase
|
|
25
|
+
return true if mode == 'ssh'
|
|
26
|
+
return false if mode == 'direct'
|
|
27
|
+
|
|
28
|
+
return false if %w[localhost 127.0.0.1 ::1].include?(config.primary_host.to_s)
|
|
29
|
+
|
|
30
|
+
true
|
|
17
31
|
end
|
|
18
32
|
|
|
19
33
|
def show_status
|
|
@@ -312,13 +326,21 @@ module ActivePostgres
|
|
|
312
326
|
end
|
|
313
327
|
|
|
314
328
|
def check_pgbouncer_direct(host)
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
329
|
+
if ssh_executor.respond_to?(:execute_on_host)
|
|
330
|
+
status = nil
|
|
331
|
+
ssh_executor.execute_on_host(host) do
|
|
332
|
+
status = capture(:systemctl, 'is-active', 'pgbouncer', raise_on_non_zero_exit: false).to_s.strip
|
|
333
|
+
end
|
|
334
|
+
status == 'active'
|
|
335
|
+
else
|
|
336
|
+
require 'socket'
|
|
337
|
+
connection_host = config.connection_host_for(host)
|
|
338
|
+
pgbouncer_port = config.component_config(:pgbouncer)[:listen_port] || 6432
|
|
318
339
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
340
|
+
socket = TCPSocket.new(connection_host, pgbouncer_port)
|
|
341
|
+
socket.close
|
|
342
|
+
true
|
|
343
|
+
end
|
|
322
344
|
rescue StandardError
|
|
323
345
|
false
|
|
324
346
|
end
|
|
@@ -91,6 +91,15 @@ module ActivePostgres
|
|
|
91
91
|
puts '✓ Restore complete'
|
|
92
92
|
end
|
|
93
93
|
|
|
94
|
+
def run_restore_at(target_time, target_action: 'promote')
|
|
95
|
+
puts "==> Restoring to #{target_time} (PITR)..."
|
|
96
|
+
|
|
97
|
+
component = Components::PgBackRest.new(config, ssh_executor, secrets)
|
|
98
|
+
component.run_restore_at(target_time, target_action: target_action)
|
|
99
|
+
|
|
100
|
+
puts '✓ Restore complete'
|
|
101
|
+
end
|
|
102
|
+
|
|
94
103
|
def list_backups
|
|
95
104
|
component = Components::PgBackRest.new(config, ssh_executor, secrets)
|
|
96
105
|
component.list_backups
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
require 'shellwords'
|
|
2
|
+
require 'timeout'
|
|
3
|
+
|
|
4
|
+
module ActivePostgres
|
|
5
|
+
class Overview
|
|
6
|
+
DEFAULT_TIMEOUT = 10
|
|
7
|
+
|
|
8
|
+
def initialize(config)
|
|
9
|
+
@config = config
|
|
10
|
+
@executor = SSHExecutor.new(config, quiet: true)
|
|
11
|
+
@health_checker = HealthChecker.new(config)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def show
|
|
15
|
+
puts
|
|
16
|
+
puts "ActivePostgres Control Tower (#{config.environment})"
|
|
17
|
+
puts '=' * 70
|
|
18
|
+
puts
|
|
19
|
+
|
|
20
|
+
health_checker.show_status
|
|
21
|
+
|
|
22
|
+
show_system_stats
|
|
23
|
+
show_repmgr_cluster if config.component_enabled?(:repmgr)
|
|
24
|
+
show_pgbouncer_targets if config.component_enabled?(:pgbouncer)
|
|
25
|
+
show_dns_status if dns_failover_enabled?
|
|
26
|
+
show_backups if config.component_enabled?(:pgbackrest)
|
|
27
|
+
|
|
28
|
+
puts
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
attr_reader :config, :executor, :health_checker
|
|
34
|
+
|
|
35
|
+
def show_repmgr_cluster
|
|
36
|
+
output = nil
|
|
37
|
+
postgres_user = config.postgres_user
|
|
38
|
+
with_timeout('repmgr cluster') do
|
|
39
|
+
executor.execute_on_host(config.primary_host) do
|
|
40
|
+
output = capture(:sudo, '-u', postgres_user, 'repmgr', '-f', '/etc/repmgr.conf',
|
|
41
|
+
'cluster', 'show', raise_on_non_zero_exit: false).to_s
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
puts '==> repmgr cluster'
|
|
46
|
+
puts LogSanitizer.sanitize(output) if output && !output.strip.empty?
|
|
47
|
+
puts
|
|
48
|
+
rescue StandardError => e
|
|
49
|
+
puts "==> repmgr cluster (error: #{e.message})"
|
|
50
|
+
puts
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def show_system_stats
|
|
54
|
+
hosts = config.all_hosts
|
|
55
|
+
return if hosts.empty?
|
|
56
|
+
|
|
57
|
+
paths = system_stat_paths
|
|
58
|
+
|
|
59
|
+
puts '==> System stats (DB nodes)'
|
|
60
|
+
hosts.each do |host|
|
|
61
|
+
label = config.node_label_for(host)
|
|
62
|
+
stats = fetch_system_stats(host, paths)
|
|
63
|
+
if stats.nil?
|
|
64
|
+
puts " #{host}#{label ? " (#{label})" : ''}: unavailable"
|
|
65
|
+
next
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
mem_used_kb = stats[:mem_total_kb] - stats[:mem_avail_kb]
|
|
69
|
+
mem_pct = stats[:mem_total_kb].positive? ? (mem_used_kb.to_f / stats[:mem_total_kb] * 100).round : 0
|
|
70
|
+
|
|
71
|
+
puts " #{host}#{label ? " (#{label})" : ''}: load #{stats[:loadavg]} | cpu #{stats[:cpu]}% | " \
|
|
72
|
+
"mem #{format_kb(mem_used_kb)}/#{format_kb(stats[:mem_total_kb])} (#{mem_pct}%)"
|
|
73
|
+
|
|
74
|
+
stats[:disks].each do |disk|
|
|
75
|
+
puts " disk #{disk[:mount]}: #{format_kb(disk[:used_kb])}/#{format_kb(disk[:total_kb])} (#{disk[:use_pct]})"
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
puts
|
|
79
|
+
rescue StandardError => e
|
|
80
|
+
puts "==> System stats (error: #{e.message})"
|
|
81
|
+
puts
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def show_pgbouncer_targets
|
|
85
|
+
puts '==> PgBouncer targets'
|
|
86
|
+
config.all_hosts.each do |host|
|
|
87
|
+
status = pgbouncer_status(host)
|
|
88
|
+
target = pgbouncer_target(host)
|
|
89
|
+
puts " #{host}: #{status} -> #{target || 'unknown'}"
|
|
90
|
+
end
|
|
91
|
+
puts
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def pgbouncer_status(host)
|
|
95
|
+
status = nil
|
|
96
|
+
executor.execute_on_host(host) do
|
|
97
|
+
status = capture(:systemctl, 'is-active', 'pgbouncer', raise_on_non_zero_exit: false).to_s.strip
|
|
98
|
+
end
|
|
99
|
+
status == 'active' ? '✓ running' : '✗ down'
|
|
100
|
+
rescue StandardError
|
|
101
|
+
'✗ down'
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def pgbouncer_target(host)
|
|
105
|
+
ini = nil
|
|
106
|
+
executor.execute_on_host(host) do
|
|
107
|
+
ini = capture(:sudo, 'cat', '/etc/pgbouncer/pgbouncer.ini', raise_on_non_zero_exit: false).to_s
|
|
108
|
+
end
|
|
109
|
+
ini[/^\* = host=([^\s]+)/, 1]
|
|
110
|
+
rescue StandardError
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def show_dns_status
|
|
115
|
+
dns_config = dns_failover_config
|
|
116
|
+
dns_servers = normalize_dns_servers(dns_config[:dns_servers])
|
|
117
|
+
dns_user = (dns_config[:dns_user] || config.user).to_s
|
|
118
|
+
|
|
119
|
+
primary_records = normalize_dns_records(dns_config[:primary_records] || dns_config[:primary_record],
|
|
120
|
+
default_prefix: 'db-primary',
|
|
121
|
+
domains: normalize_dns_domains(dns_config))
|
|
122
|
+
replica_records = normalize_dns_records(dns_config[:replica_records] || dns_config[:replica_record],
|
|
123
|
+
default_prefix: 'db-replica',
|
|
124
|
+
domains: normalize_dns_domains(dns_config))
|
|
125
|
+
records = primary_records + replica_records
|
|
126
|
+
|
|
127
|
+
puts '==> DNS (dnsmasq)'
|
|
128
|
+
puts " Servers: #{dns_servers.map { |s| s[:ssh_host] }.join(', ')}"
|
|
129
|
+
|
|
130
|
+
record_map = Hash.new { |h, k| h[k] = [] }
|
|
131
|
+
dns_servers.each do |server|
|
|
132
|
+
content = dnsmasq_config(server[:ssh_host], dns_user)
|
|
133
|
+
next if content.to_s.strip.empty?
|
|
134
|
+
|
|
135
|
+
content.each_line do |line|
|
|
136
|
+
next unless line.start_with?('address=/')
|
|
137
|
+
|
|
138
|
+
match = line.strip.match(%r{\Aaddress=/([^/]+)/(.+)\z})
|
|
139
|
+
next unless match
|
|
140
|
+
|
|
141
|
+
name = match[1]
|
|
142
|
+
ip = match[2]
|
|
143
|
+
next unless records.include?(name)
|
|
144
|
+
|
|
145
|
+
record_map[name] << ip
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
records.uniq.each do |record|
|
|
150
|
+
ips = record_map[record].uniq
|
|
151
|
+
display = ips.empty? ? 'missing' : ips.join(', ')
|
|
152
|
+
warn_suffix = if primary_records.include?(record) && ips.size > 1
|
|
153
|
+
' ⚠️'
|
|
154
|
+
else
|
|
155
|
+
''
|
|
156
|
+
end
|
|
157
|
+
puts " #{record}: #{display}#{warn_suffix}"
|
|
158
|
+
end
|
|
159
|
+
puts
|
|
160
|
+
rescue StandardError => e
|
|
161
|
+
puts "==> DNS (error: #{e.message})"
|
|
162
|
+
puts
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def dnsmasq_config(host, dns_user)
|
|
166
|
+
content = nil
|
|
167
|
+
with_timeout("dnsmasq #{host}") do
|
|
168
|
+
executor.execute_on_host_as(host, dns_user) do
|
|
169
|
+
content = capture(:sudo, 'sh', '-c',
|
|
170
|
+
'cat /etc/dnsmasq.d/active_postgres.conf 2>/dev/null || ' \
|
|
171
|
+
'cat /etc/dnsmasq.d/messhy.conf 2>/dev/null || true',
|
|
172
|
+
raise_on_non_zero_exit: false).to_s
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
content
|
|
176
|
+
rescue StandardError
|
|
177
|
+
nil
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def normalize_dns_servers(raw_servers)
|
|
181
|
+
Array(raw_servers).map do |server|
|
|
182
|
+
if server.is_a?(Hash)
|
|
183
|
+
ssh_host = server[:ssh_host] || server['ssh_host'] || server[:host] || server['host']
|
|
184
|
+
private_ip = server[:private_ip] || server['private_ip'] || server[:ip] || server['ip']
|
|
185
|
+
private_ip ||= ssh_host
|
|
186
|
+
ssh_host ||= private_ip
|
|
187
|
+
{ ssh_host: ssh_host.to_s, private_ip: private_ip.to_s }
|
|
188
|
+
else
|
|
189
|
+
value = server.to_s
|
|
190
|
+
{ ssh_host: value, private_ip: value }
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def normalize_dns_domains(dns_config)
|
|
196
|
+
Array(dns_config[:domains] || dns_config[:domain]).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def normalize_dns_records(value, default_prefix:, domains:)
|
|
200
|
+
records = Array(value).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
201
|
+
return records unless records.empty?
|
|
202
|
+
|
|
203
|
+
domains = ['mesh'] if domains.empty?
|
|
204
|
+
domains.map { |domain| "#{default_prefix}.#{domain}" }
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def show_backups
|
|
208
|
+
output = nil
|
|
209
|
+
postgres_user = config.postgres_user
|
|
210
|
+
with_timeout('pgbackrest info') do
|
|
211
|
+
executor.execute_on_host(config.primary_host) do
|
|
212
|
+
output = capture(:sudo, '-u', postgres_user, 'pgbackrest', 'info',
|
|
213
|
+
raise_on_non_zero_exit: false).to_s
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
puts '==> Backups (pgBackRest)'
|
|
218
|
+
puts output if output && !output.strip.empty?
|
|
219
|
+
puts
|
|
220
|
+
rescue StandardError => e
|
|
221
|
+
puts "==> Backups (error: #{e.message})"
|
|
222
|
+
puts
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def fetch_system_stats(host, paths)
|
|
226
|
+
output = nil
|
|
227
|
+
with_timeout("system stats #{host}") do
|
|
228
|
+
ssh_executor = executor
|
|
229
|
+
safe_paths = paths.map { |p| Shellwords.escape(p) }.join(' ')
|
|
230
|
+
script = <<~'BASH'
|
|
231
|
+
set -e
|
|
232
|
+
loadavg=$(cut -d' ' -f1-3 /proc/loadavg)
|
|
233
|
+
|
|
234
|
+
read _ user nice system idle iowait irq softirq steal _ _ < /proc/stat
|
|
235
|
+
sleep 0.2
|
|
236
|
+
read _ user2 nice2 system2 idle2 iowait2 irq2 softirq2 steal2 _ _ < /proc/stat
|
|
237
|
+
|
|
238
|
+
total1=$((user + nice + system + idle + iowait + irq + softirq + steal))
|
|
239
|
+
total2=$((user2 + nice2 + system2 + idle2 + iowait2 + irq2 + softirq2 + steal2))
|
|
240
|
+
total=$((total2 - total1))
|
|
241
|
+
idle_delta=$((idle2 + iowait2 - idle - iowait))
|
|
242
|
+
|
|
243
|
+
cpu=0
|
|
244
|
+
if [ "$total" -gt 0 ]; then
|
|
245
|
+
cpu=$(( (100 * (total - idle_delta)) / total ))
|
|
246
|
+
fi
|
|
247
|
+
|
|
248
|
+
mem_total=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
|
|
249
|
+
mem_avail=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
|
250
|
+
|
|
251
|
+
echo "loadavg=${loadavg}"
|
|
252
|
+
echo "cpu=${cpu}"
|
|
253
|
+
echo "mem_total_kb=${mem_total}"
|
|
254
|
+
echo "mem_avail_kb=${mem_avail}"
|
|
255
|
+
BASH
|
|
256
|
+
|
|
257
|
+
ssh_executor.execute_on_host(host) do
|
|
258
|
+
output = capture(:bash, '-lc', "#{script}\n df -kP #{safe_paths} 2>/dev/null | tail -n +2 | " \
|
|
259
|
+
"awk '{print \"disk=\" $6 \"|\" $2 \"|\" $3 \"|\" $4 \"|\" $5}'",
|
|
260
|
+
raise_on_non_zero_exit: false).to_s
|
|
261
|
+
end
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
parse_system_stats(output)
|
|
265
|
+
rescue StandardError
|
|
266
|
+
nil
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def parse_system_stats(output)
|
|
270
|
+
stats = { disks: [] }
|
|
271
|
+
output.to_s.each_line do |line|
|
|
272
|
+
line = line.strip
|
|
273
|
+
next if line.empty?
|
|
274
|
+
|
|
275
|
+
if line.start_with?('loadavg=')
|
|
276
|
+
stats[:loadavg] = line.split('=', 2)[1]
|
|
277
|
+
elsif line.start_with?('cpu=')
|
|
278
|
+
stats[:cpu] = line.split('=', 2)[1].to_i
|
|
279
|
+
elsif line.start_with?('mem_total_kb=')
|
|
280
|
+
stats[:mem_total_kb] = line.split('=', 2)[1].to_i
|
|
281
|
+
elsif line.start_with?('mem_avail_kb=')
|
|
282
|
+
stats[:mem_avail_kb] = line.split('=', 2)[1].to_i
|
|
283
|
+
elsif line.start_with?('disk=')
|
|
284
|
+
_, payload = line.split('=', 2)
|
|
285
|
+
mount, total, used, _avail, pct = payload.split('|')
|
|
286
|
+
stats[:disks] << {
|
|
287
|
+
mount: mount,
|
|
288
|
+
total_kb: total.to_i,
|
|
289
|
+
used_kb: used.to_i,
|
|
290
|
+
use_pct: pct
|
|
291
|
+
}
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
stats[:loadavg] ||= 'n/a'
|
|
296
|
+
stats[:cpu] ||= 0
|
|
297
|
+
stats[:mem_total_kb] ||= 0
|
|
298
|
+
stats[:mem_avail_kb] ||= 0
|
|
299
|
+
stats
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
def format_kb(kb)
|
|
303
|
+
kb = kb.to_f
|
|
304
|
+
gb = kb / 1024 / 1024
|
|
305
|
+
return format('%.1fG', gb) if gb >= 1
|
|
306
|
+
|
|
307
|
+
mb = kb / 1024
|
|
308
|
+
format('%.0fM', mb)
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
def system_stat_paths
|
|
312
|
+
paths = ['/']
|
|
313
|
+
paths << '/var/lib/postgresql'
|
|
314
|
+
repo_path = pgbackrest_repo_path
|
|
315
|
+
paths << repo_path if repo_path
|
|
316
|
+
paths.compact.uniq
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
def pgbackrest_repo_path
|
|
320
|
+
return nil unless config.component_enabled?(:pgbackrest)
|
|
321
|
+
|
|
322
|
+
pg_config = config.component_config(:pgbackrest)
|
|
323
|
+
return pg_config[:repo_path] if pg_config[:repo_path]
|
|
324
|
+
|
|
325
|
+
pg_config[:repo_type].to_s == 'local' ? '/var/lib/pgbackrest' : '/backups'
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
def dns_failover_config
|
|
329
|
+
repmgr_config = config.component_config(:repmgr)
|
|
330
|
+
dns_config = repmgr_config[:dns_failover]
|
|
331
|
+
return nil unless dns_config && dns_config[:enabled]
|
|
332
|
+
|
|
333
|
+
dns_config
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
def dns_failover_enabled?
|
|
337
|
+
dns_failover_config != nil
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def overview_timeout
|
|
341
|
+
value = ENV.fetch('ACTIVE_POSTGRES_OVERVIEW_TIMEOUT', DEFAULT_TIMEOUT.to_s).to_i
|
|
342
|
+
value.positive? ? value : DEFAULT_TIMEOUT
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
def with_timeout(label)
|
|
346
|
+
Timeout.timeout(overview_timeout) { yield }
|
|
347
|
+
rescue Timeout::Error
|
|
348
|
+
raise StandardError, "#{label} timed out after #{overview_timeout}s"
|
|
349
|
+
end
|
|
350
|
+
end
|
|
351
|
+
end
|
|
@@ -113,12 +113,10 @@ module ActivePostgres
|
|
|
113
113
|
|
|
114
114
|
def register_database_removal(host, database_name)
|
|
115
115
|
postgres_user = config.postgres_user
|
|
116
|
+
executor = ssh_executor
|
|
116
117
|
register("Drop database #{database_name} on #{host}", host: host) do
|
|
117
118
|
sql = "DROP DATABASE IF EXISTS #{database_name};"
|
|
118
|
-
|
|
119
|
-
execute :chmod, '644', '/tmp/drop_database.sql'
|
|
120
|
-
execute :sudo, '-u', postgres_user, 'psql', '-f', '/tmp/drop_database.sql'
|
|
121
|
-
execute :rm, '-f', '/tmp/drop_database.sql'
|
|
119
|
+
executor.run_sql_on_backend(self, sql, postgres_user: postgres_user, tuples_only: false, capture: false)
|
|
122
120
|
rescue StandardError
|
|
123
121
|
nil
|
|
124
122
|
end
|
|
@@ -126,12 +124,10 @@ module ActivePostgres
|
|
|
126
124
|
|
|
127
125
|
def register_postgres_user_removal(host, username)
|
|
128
126
|
postgres_user = config.postgres_user
|
|
127
|
+
executor = ssh_executor
|
|
129
128
|
register("Drop PostgreSQL user #{username} on #{host}", host: host) do
|
|
130
129
|
sql = "DROP USER IF EXISTS #{username};"
|
|
131
|
-
|
|
132
|
-
execute :chmod, '644', '/tmp/drop_user.sql'
|
|
133
|
-
execute :sudo, '-u', postgres_user, 'psql', '-f', '/tmp/drop_user.sql'
|
|
134
|
-
execute :rm, '-f', '/tmp/drop_user.sql'
|
|
130
|
+
executor.run_sql_on_backend(self, sql, postgres_user: postgres_user, tuples_only: false, capture: false)
|
|
135
131
|
rescue StandardError
|
|
136
132
|
nil
|
|
137
133
|
end
|