active_postgres 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +58 -3
- data/lib/active_postgres/cli.rb +7 -0
- data/lib/active_postgres/components/core.rb +5 -5
- data/lib/active_postgres/components/monitoring.rb +165 -0
- data/lib/active_postgres/components/pgbackrest.rb +59 -9
- data/lib/active_postgres/components/pgbouncer.rb +17 -5
- data/lib/active_postgres/components/repmgr.rb +103 -7
- data/lib/active_postgres/configuration.rb +19 -2
- data/lib/active_postgres/generators/active_postgres/install_generator.rb +1 -0
- data/lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb +10 -0
- data/lib/active_postgres/health_checker.rb +29 -7
- data/lib/active_postgres/installer.rb +9 -0
- data/lib/active_postgres/overview.rb +351 -0
- data/lib/active_postgres/secrets.rb +18 -0
- data/lib/active_postgres/ssh_executor.rb +9 -5
- data/lib/active_postgres/version.rb +1 -1
- data/lib/active_postgres.rb +1 -0
- data/lib/tasks/postgres.rake +67 -0
- data/templates/repmgr.conf.erb +10 -0
- data/templates/repmgr_dns_failover.sh.erb +25 -11
- metadata +2 -1
|
@@ -85,6 +85,22 @@ module ActivePostgres
|
|
|
85
85
|
execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
|
|
86
86
|
"postgresql-#{version}-repmgr"
|
|
87
87
|
end
|
|
88
|
+
install_postgres_sudoers(host)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def install_postgres_sudoers(host)
|
|
93
|
+
version = config.version
|
|
94
|
+
sudoers_line = "postgres ALL=(ALL) NOPASSWD: /usr/bin/systemctl start postgresql@#{version}-main, " \
|
|
95
|
+
"/usr/bin/systemctl stop postgresql@#{version}-main, " \
|
|
96
|
+
"/usr/bin/systemctl restart postgresql@#{version}-main, " \
|
|
97
|
+
"/usr/bin/systemctl reload postgresql@#{version}-main, " \
|
|
98
|
+
"/usr/bin/systemctl status postgresql@#{version}-main"
|
|
99
|
+
ssh_executor.execute_on_host(host) do
|
|
100
|
+
upload! StringIO.new("#{sudoers_line}\n"), '/tmp/postgres-repmgr-sudoers'
|
|
101
|
+
execute :sudo, 'cp', '/tmp/postgres-repmgr-sudoers', '/etc/sudoers.d/postgres-repmgr'
|
|
102
|
+
execute :sudo, 'chmod', '440', '/etc/sudoers.d/postgres-repmgr'
|
|
103
|
+
execute :rm, '-f', '/tmp/postgres-repmgr-sudoers'
|
|
88
104
|
end
|
|
89
105
|
end
|
|
90
106
|
|
|
@@ -247,6 +263,7 @@ module ActivePostgres
|
|
|
247
263
|
effective_replication_password = replication_user == repmgr_user ? repmgr_password : replication_password
|
|
248
264
|
|
|
249
265
|
ensure_primary_registered
|
|
266
|
+
ensure_primary_replication_ready(repmgr_password, effective_replication_password)
|
|
250
267
|
|
|
251
268
|
setup_pgpass_file(standby_host, repmgr_password, replication_password: effective_replication_password,
|
|
252
269
|
primary_ip: primary_replication_host)
|
|
@@ -399,9 +416,42 @@ module ActivePostgres
|
|
|
399
416
|
end
|
|
400
417
|
|
|
401
418
|
register_standby_with_primary(standby_host)
|
|
419
|
+
setup_inter_node_ssh
|
|
402
420
|
enable_repmgrd_if_configured(standby_host, repmgr_config)
|
|
403
421
|
end
|
|
404
422
|
|
|
423
|
+
def setup_inter_node_ssh
|
|
424
|
+
dns_config = dns_failover_config
|
|
425
|
+
key_path = dns_config && dns_config[:ssh_key_path] || '/var/lib/postgresql/.ssh/active_postgres_dns'
|
|
426
|
+
postgres_user = config.postgres_user
|
|
427
|
+
all_hosts = config.all_hosts
|
|
428
|
+
pub_keys = {}
|
|
429
|
+
|
|
430
|
+
all_hosts.each do |host|
|
|
431
|
+
ssh_executor.execute_on_host(host) do
|
|
432
|
+
pub_keys[host] = capture(:sudo, '-u', postgres_user, 'cat', "#{key_path}.pub").strip
|
|
433
|
+
end
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
all_hosts.each do |host|
|
|
437
|
+
ssh_executor.execute_on_host(host) do
|
|
438
|
+
other_keys = pub_keys.reject { |h, _| h == host }.values
|
|
439
|
+
other_keys.each do |key|
|
|
440
|
+
next if key.to_s.empty?
|
|
441
|
+
|
|
442
|
+
upload! StringIO.new("#{key}\n"), '/tmp/pg_peer_key.pub'
|
|
443
|
+
execute :sudo, '-u', postgres_user, 'bash', '-c',
|
|
444
|
+
"grep -qxF -f /tmp/pg_peer_key.pub /var/lib/postgresql/.ssh/authorized_keys 2>/dev/null || cat /tmp/pg_peer_key.pub >> /var/lib/postgresql/.ssh/authorized_keys"
|
|
445
|
+
execute :rm, '-f', '/tmp/pg_peer_key.pub'
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
peer_ips = all_hosts.reject { |h| h == host }.map { |h| config.replication_host_for(h) }
|
|
449
|
+
scan_cmd = "ssh-keyscan #{peer_ips.join(' ')} >> /var/lib/postgresql/.ssh/known_hosts 2>/dev/null || true"
|
|
450
|
+
execute :sudo, '-u', postgres_user, 'bash', '-c', scan_cmd
|
|
451
|
+
end
|
|
452
|
+
end
|
|
453
|
+
end
|
|
454
|
+
|
|
405
455
|
def setup_dns_failover
|
|
406
456
|
dns_config = dns_failover_config
|
|
407
457
|
return unless dns_config
|
|
@@ -424,6 +474,8 @@ module ActivePostgres
|
|
|
424
474
|
authorize_dns_keys(dns_server, dns_user, pub_keys.values.compact)
|
|
425
475
|
end
|
|
426
476
|
|
|
477
|
+
setup_inter_node_ssh
|
|
478
|
+
|
|
427
479
|
config.all_hosts.each do |host|
|
|
428
480
|
install_dns_failover_script(host, dns_config, dns_private_ips, dns_user, dns_ssh_key_path, ssh_strict_host_key)
|
|
429
481
|
end
|
|
@@ -512,12 +564,16 @@ module ActivePostgres
|
|
|
512
564
|
def install_dns_failover_script(host, dns_config, dns_servers, dns_user, dns_ssh_key_path, ssh_strict_host_key)
|
|
513
565
|
return if dns_servers.empty?
|
|
514
566
|
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
567
|
+
domains = normalize_dns_domains(dns_config)
|
|
568
|
+
primary_records = normalize_dns_records(dns_config[:primary_records] || dns_config[:primary_record],
|
|
569
|
+
default_prefix: 'db-primary',
|
|
570
|
+
domains: domains)
|
|
571
|
+
replica_records = normalize_dns_records(dns_config[:replica_records] || dns_config[:replica_record],
|
|
572
|
+
default_prefix: 'db-replica',
|
|
573
|
+
domains: domains)
|
|
518
574
|
|
|
519
|
-
_ =
|
|
520
|
-
_ =
|
|
575
|
+
_ = primary_records
|
|
576
|
+
_ = replica_records
|
|
521
577
|
_ = dns_servers
|
|
522
578
|
_ = dns_user
|
|
523
579
|
_ = dns_ssh_key_path
|
|
@@ -527,6 +583,19 @@ module ActivePostgres
|
|
|
527
583
|
mode: '755', owner: 'root:root')
|
|
528
584
|
end
|
|
529
585
|
|
|
586
|
+
def normalize_dns_domains(dns_config)
|
|
587
|
+
domains = Array(dns_config[:domains] || dns_config[:domain]).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
588
|
+
domains = ['mesh'] if domains.empty?
|
|
589
|
+
domains
|
|
590
|
+
end
|
|
591
|
+
|
|
592
|
+
def normalize_dns_records(value, default_prefix:, domains:)
|
|
593
|
+
records = Array(value).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
594
|
+
return records unless records.empty?
|
|
595
|
+
|
|
596
|
+
domains.map { |domain| "#{default_prefix}.#{domain}" }
|
|
597
|
+
end
|
|
598
|
+
|
|
530
599
|
def normalize_dns_host_key_verification(value)
|
|
531
600
|
normalized = case value
|
|
532
601
|
when Symbol
|
|
@@ -859,6 +928,33 @@ module ActivePostgres
|
|
|
859
928
|
is_registered
|
|
860
929
|
end
|
|
861
930
|
|
|
931
|
+
def ensure_primary_replication_ready(repmgr_password, effective_replication_password)
|
|
932
|
+
host = config.primary_host
|
|
933
|
+
repmgr_user = config.repmgr_user
|
|
934
|
+
repmgr_db = config.repmgr_database
|
|
935
|
+
replication_user = config.replication_user
|
|
936
|
+
repmgr_component = self
|
|
937
|
+
executor = ssh_executor
|
|
938
|
+
|
|
939
|
+
puts ' Ensuring repmgr user has correct password and privileges on primary...'
|
|
940
|
+
|
|
941
|
+
ssh_executor.execute_on_host(host) do
|
|
942
|
+
repmgr_sql = repmgr_component.send(:build_repmgr_setup_sql, repmgr_user, repmgr_db, repmgr_password)
|
|
943
|
+
executor.run_sql_on_backend(self, repmgr_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
|
|
944
|
+
capture: false)
|
|
945
|
+
|
|
946
|
+
if replication_user != repmgr_user
|
|
947
|
+
repl_sql = repmgr_component.send(:build_replication_user_sql, replication_user, effective_replication_password)
|
|
948
|
+
executor.run_sql_on_backend(self, repl_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
|
|
949
|
+
capture: false)
|
|
950
|
+
end
|
|
951
|
+
|
|
952
|
+
info '✓ Primary replication user is ready'
|
|
953
|
+
end
|
|
954
|
+
|
|
955
|
+
setup_pgpass_file(host, repmgr_password, replication_password: effective_replication_password)
|
|
956
|
+
end
|
|
957
|
+
|
|
862
958
|
def regenerate_ssl_certs(host, version)
|
|
863
959
|
ssl_config = config.component_config(:ssl)
|
|
864
960
|
ssl_cert = secrets.resolve('ssl_cert')
|
|
@@ -994,9 +1090,9 @@ module ActivePostgres
|
|
|
994
1090
|
'DO $$',
|
|
995
1091
|
'BEGIN',
|
|
996
1092
|
" IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '#{repmgr_user}') THEN",
|
|
997
|
-
" CREATE USER #{repmgr_user} WITH SUPERUSER PASSWORD '#{escaped_password}';",
|
|
1093
|
+
" CREATE USER #{repmgr_user} WITH SUPERUSER REPLICATION PASSWORD '#{escaped_password}';",
|
|
998
1094
|
' ELSE',
|
|
999
|
-
" ALTER USER #{repmgr_user} WITH SUPERUSER PASSWORD '#{escaped_password}';",
|
|
1095
|
+
" ALTER USER #{repmgr_user} WITH SUPERUSER REPLICATION PASSWORD '#{escaped_password}';",
|
|
1000
1096
|
' END IF;',
|
|
1001
1097
|
'END $$;',
|
|
1002
1098
|
'',
|
|
@@ -96,15 +96,32 @@ module ActivePostgres
|
|
|
96
96
|
# Validate required secrets if components are enabled
|
|
97
97
|
raise Error, 'Missing replication_password secret' if component_enabled?(:repmgr) && !secrets_config['replication_password']
|
|
98
98
|
raise Error, 'Missing monitoring_password secret' if component_enabled?(:monitoring) && !secrets_config['monitoring_password']
|
|
99
|
+
if component_enabled?(:monitoring)
|
|
100
|
+
grafana_config = component_config(:monitoring)[:grafana] || {}
|
|
101
|
+
if grafana_config[:enabled] && !secrets_config['grafana_admin_password']
|
|
102
|
+
raise Error, 'Missing grafana_admin_password secret'
|
|
103
|
+
end
|
|
104
|
+
if grafana_config[:enabled] && grafana_config[:host].to_s.strip.empty?
|
|
105
|
+
raise Error, 'monitoring.grafana.host is required when grafana is enabled'
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
if component_enabled?(:pgbackrest)
|
|
109
|
+
pg_config = component_config(:pgbackrest)
|
|
110
|
+
retention_full = pg_config[:retention_full]
|
|
111
|
+
retention_archive = pg_config[:retention_archive]
|
|
112
|
+
if retention_full && retention_archive && retention_archive.to_i < retention_full.to_i
|
|
113
|
+
raise Error, 'pgbackrest.retention_archive must be >= retention_full for PITR safety'
|
|
114
|
+
end
|
|
115
|
+
end
|
|
99
116
|
|
|
100
117
|
if component_enabled?(:repmgr)
|
|
101
118
|
dns_failover = component_config(:repmgr)[:dns_failover]
|
|
102
119
|
if dns_failover && dns_failover[:enabled]
|
|
103
|
-
|
|
120
|
+
domains = Array(dns_failover[:domains] || dns_failover[:domain]).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
104
121
|
servers = Array(dns_failover[:dns_servers])
|
|
105
122
|
provider = (dns_failover[:provider] || 'dnsmasq').to_s.strip
|
|
106
123
|
|
|
107
|
-
raise Error, 'dns_failover.domain is required when enabled' if
|
|
124
|
+
raise Error, 'dns_failover.domain or dns_failover.domains is required when enabled' if domains.empty?
|
|
108
125
|
raise Error, 'dns_failover.dns_servers is required when enabled' if servers.empty?
|
|
109
126
|
raise Error, "Unsupported dns_failover provider '#{provider}'" unless provider == 'dnsmasq'
|
|
110
127
|
|
|
@@ -113,6 +113,7 @@ module ActivePostgres
|
|
|
113
113
|
puts " replication_password: \"#{generate_secure_password}\""
|
|
114
114
|
puts " repmgr_password: \"#{generate_secure_password}\""
|
|
115
115
|
puts " monitoring_password: \"#{generate_secure_password}\""
|
|
116
|
+
puts " grafana_admin_password: \"#{generate_secure_password}\""
|
|
116
117
|
puts
|
|
117
118
|
puts '🔐 Secure passwords have been auto-generated above.'
|
|
118
119
|
puts '📌 Update primary_host and replica_host with your actual IPs after provisioning.'
|
|
@@ -59,6 +59,15 @@ shared: &shared
|
|
|
59
59
|
monitoring:
|
|
60
60
|
enabled: false
|
|
61
61
|
# user: postgres_exporter
|
|
62
|
+
# node_exporter: true
|
|
63
|
+
# node_exporter_port: 9100
|
|
64
|
+
# node_exporter_listen_address: 10.8.0.10
|
|
65
|
+
# grafana:
|
|
66
|
+
# enabled: true
|
|
67
|
+
# host: grafana.example.com
|
|
68
|
+
# listen_address: 10.8.0.10
|
|
69
|
+
# port: 3000
|
|
70
|
+
# prometheus_url: http://prometheus.example.com:9090
|
|
62
71
|
|
|
63
72
|
ssl:
|
|
64
73
|
enabled: false
|
|
@@ -98,5 +107,6 @@ production:
|
|
|
98
107
|
replication_password: rails_credentials:postgres.replication_password
|
|
99
108
|
repmgr_password: rails_credentials:postgres.repmgr_password
|
|
100
109
|
monitoring_password: rails_credentials:postgres.monitoring_password
|
|
110
|
+
# grafana_admin_password: rails_credentials:postgres.grafana_admin_password
|
|
101
111
|
pgbouncer_password: rails_credentials:postgres.password
|
|
102
112
|
app_password: rails_credentials:postgres.password
|
|
@@ -13,7 +13,21 @@ module ActivePostgres
|
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
private def create_executor
|
|
16
|
-
|
|
16
|
+
if use_ssh_executor?
|
|
17
|
+
SSHExecutor.new(config, quiet: true)
|
|
18
|
+
else
|
|
19
|
+
DirectExecutor.new(config, quiet: true)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def use_ssh_executor?
|
|
24
|
+
mode = ENV['ACTIVE_POSTGRES_STATUS_MODE'].to_s.strip.downcase
|
|
25
|
+
return true if mode == 'ssh'
|
|
26
|
+
return false if mode == 'direct'
|
|
27
|
+
|
|
28
|
+
return false if %w[localhost 127.0.0.1 ::1].include?(config.primary_host.to_s)
|
|
29
|
+
|
|
30
|
+
true
|
|
17
31
|
end
|
|
18
32
|
|
|
19
33
|
def show_status
|
|
@@ -312,13 +326,21 @@ module ActivePostgres
|
|
|
312
326
|
end
|
|
313
327
|
|
|
314
328
|
def check_pgbouncer_direct(host)
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
329
|
+
if ssh_executor.respond_to?(:execute_on_host)
|
|
330
|
+
status = nil
|
|
331
|
+
ssh_executor.execute_on_host(host) do
|
|
332
|
+
status = capture(:systemctl, 'is-active', 'pgbouncer', raise_on_non_zero_exit: false).to_s.strip
|
|
333
|
+
end
|
|
334
|
+
status == 'active'
|
|
335
|
+
else
|
|
336
|
+
require 'socket'
|
|
337
|
+
connection_host = config.connection_host_for(host)
|
|
338
|
+
pgbouncer_port = config.component_config(:pgbouncer)[:listen_port] || 6432
|
|
318
339
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
340
|
+
socket = TCPSocket.new(connection_host, pgbouncer_port)
|
|
341
|
+
socket.close
|
|
342
|
+
true
|
|
343
|
+
end
|
|
322
344
|
rescue StandardError
|
|
323
345
|
false
|
|
324
346
|
end
|
|
@@ -91,6 +91,15 @@ module ActivePostgres
|
|
|
91
91
|
puts '✓ Restore complete'
|
|
92
92
|
end
|
|
93
93
|
|
|
94
|
+
def run_restore_at(target_time, target_action: 'promote')
|
|
95
|
+
puts "==> Restoring to #{target_time} (PITR)..."
|
|
96
|
+
|
|
97
|
+
component = Components::PgBackRest.new(config, ssh_executor, secrets)
|
|
98
|
+
component.run_restore_at(target_time, target_action: target_action)
|
|
99
|
+
|
|
100
|
+
puts '✓ Restore complete'
|
|
101
|
+
end
|
|
102
|
+
|
|
94
103
|
def list_backups
|
|
95
104
|
component = Components::PgBackRest.new(config, ssh_executor, secrets)
|
|
96
105
|
component.list_backups
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
require 'shellwords'
|
|
2
|
+
require 'timeout'
|
|
3
|
+
|
|
4
|
+
module ActivePostgres
|
|
5
|
+
class Overview
|
|
6
|
+
DEFAULT_TIMEOUT = 10
|
|
7
|
+
|
|
8
|
+
def initialize(config)
|
|
9
|
+
@config = config
|
|
10
|
+
@executor = SSHExecutor.new(config, quiet: true)
|
|
11
|
+
@health_checker = HealthChecker.new(config)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def show
|
|
15
|
+
puts
|
|
16
|
+
puts "ActivePostgres Control Tower (#{config.environment})"
|
|
17
|
+
puts '=' * 70
|
|
18
|
+
puts
|
|
19
|
+
|
|
20
|
+
health_checker.show_status
|
|
21
|
+
|
|
22
|
+
show_system_stats
|
|
23
|
+
show_repmgr_cluster if config.component_enabled?(:repmgr)
|
|
24
|
+
show_pgbouncer_targets if config.component_enabled?(:pgbouncer)
|
|
25
|
+
show_dns_status if dns_failover_enabled?
|
|
26
|
+
show_backups if config.component_enabled?(:pgbackrest)
|
|
27
|
+
|
|
28
|
+
puts
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
attr_reader :config, :executor, :health_checker
|
|
34
|
+
|
|
35
|
+
def show_repmgr_cluster
|
|
36
|
+
output = nil
|
|
37
|
+
postgres_user = config.postgres_user
|
|
38
|
+
with_timeout('repmgr cluster') do
|
|
39
|
+
executor.execute_on_host(config.primary_host) do
|
|
40
|
+
output = capture(:sudo, '-u', postgres_user, 'repmgr', '-f', '/etc/repmgr.conf',
|
|
41
|
+
'cluster', 'show', raise_on_non_zero_exit: false).to_s
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
puts '==> repmgr cluster'
|
|
46
|
+
puts LogSanitizer.sanitize(output) if output && !output.strip.empty?
|
|
47
|
+
puts
|
|
48
|
+
rescue StandardError => e
|
|
49
|
+
puts "==> repmgr cluster (error: #{e.message})"
|
|
50
|
+
puts
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def show_system_stats
|
|
54
|
+
hosts = config.all_hosts
|
|
55
|
+
return if hosts.empty?
|
|
56
|
+
|
|
57
|
+
paths = system_stat_paths
|
|
58
|
+
|
|
59
|
+
puts '==> System stats (DB nodes)'
|
|
60
|
+
hosts.each do |host|
|
|
61
|
+
label = config.node_label_for(host)
|
|
62
|
+
stats = fetch_system_stats(host, paths)
|
|
63
|
+
if stats.nil?
|
|
64
|
+
puts " #{host}#{label ? " (#{label})" : ''}: unavailable"
|
|
65
|
+
next
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
mem_used_kb = stats[:mem_total_kb] - stats[:mem_avail_kb]
|
|
69
|
+
mem_pct = stats[:mem_total_kb].positive? ? (mem_used_kb.to_f / stats[:mem_total_kb] * 100).round : 0
|
|
70
|
+
|
|
71
|
+
puts " #{host}#{label ? " (#{label})" : ''}: load #{stats[:loadavg]} | cpu #{stats[:cpu]}% | " \
|
|
72
|
+
"mem #{format_kb(mem_used_kb)}/#{format_kb(stats[:mem_total_kb])} (#{mem_pct}%)"
|
|
73
|
+
|
|
74
|
+
stats[:disks].each do |disk|
|
|
75
|
+
puts " disk #{disk[:mount]}: #{format_kb(disk[:used_kb])}/#{format_kb(disk[:total_kb])} (#{disk[:use_pct]})"
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
puts
|
|
79
|
+
rescue StandardError => e
|
|
80
|
+
puts "==> System stats (error: #{e.message})"
|
|
81
|
+
puts
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def show_pgbouncer_targets
|
|
85
|
+
puts '==> PgBouncer targets'
|
|
86
|
+
config.all_hosts.each do |host|
|
|
87
|
+
status = pgbouncer_status(host)
|
|
88
|
+
target = pgbouncer_target(host)
|
|
89
|
+
puts " #{host}: #{status} -> #{target || 'unknown'}"
|
|
90
|
+
end
|
|
91
|
+
puts
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def pgbouncer_status(host)
|
|
95
|
+
status = nil
|
|
96
|
+
executor.execute_on_host(host) do
|
|
97
|
+
status = capture(:systemctl, 'is-active', 'pgbouncer', raise_on_non_zero_exit: false).to_s.strip
|
|
98
|
+
end
|
|
99
|
+
status == 'active' ? '✓ running' : '✗ down'
|
|
100
|
+
rescue StandardError
|
|
101
|
+
'✗ down'
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def pgbouncer_target(host)
|
|
105
|
+
ini = nil
|
|
106
|
+
executor.execute_on_host(host) do
|
|
107
|
+
ini = capture(:sudo, 'cat', '/etc/pgbouncer/pgbouncer.ini', raise_on_non_zero_exit: false).to_s
|
|
108
|
+
end
|
|
109
|
+
ini[/^\* = host=([^\s]+)/, 1]
|
|
110
|
+
rescue StandardError
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def show_dns_status
|
|
115
|
+
dns_config = dns_failover_config
|
|
116
|
+
dns_servers = normalize_dns_servers(dns_config[:dns_servers])
|
|
117
|
+
dns_user = (dns_config[:dns_user] || config.user).to_s
|
|
118
|
+
|
|
119
|
+
primary_records = normalize_dns_records(dns_config[:primary_records] || dns_config[:primary_record],
|
|
120
|
+
default_prefix: 'db-primary',
|
|
121
|
+
domains: normalize_dns_domains(dns_config))
|
|
122
|
+
replica_records = normalize_dns_records(dns_config[:replica_records] || dns_config[:replica_record],
|
|
123
|
+
default_prefix: 'db-replica',
|
|
124
|
+
domains: normalize_dns_domains(dns_config))
|
|
125
|
+
records = primary_records + replica_records
|
|
126
|
+
|
|
127
|
+
puts '==> DNS (dnsmasq)'
|
|
128
|
+
puts " Servers: #{dns_servers.map { |s| s[:ssh_host] }.join(', ')}"
|
|
129
|
+
|
|
130
|
+
record_map = Hash.new { |h, k| h[k] = [] }
|
|
131
|
+
dns_servers.each do |server|
|
|
132
|
+
content = dnsmasq_config(server[:ssh_host], dns_user)
|
|
133
|
+
next if content.to_s.strip.empty?
|
|
134
|
+
|
|
135
|
+
content.each_line do |line|
|
|
136
|
+
next unless line.start_with?('address=/')
|
|
137
|
+
|
|
138
|
+
match = line.strip.match(%r{\Aaddress=/([^/]+)/(.+)\z})
|
|
139
|
+
next unless match
|
|
140
|
+
|
|
141
|
+
name = match[1]
|
|
142
|
+
ip = match[2]
|
|
143
|
+
next unless records.include?(name)
|
|
144
|
+
|
|
145
|
+
record_map[name] << ip
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
records.uniq.each do |record|
|
|
150
|
+
ips = record_map[record].uniq
|
|
151
|
+
display = ips.empty? ? 'missing' : ips.join(', ')
|
|
152
|
+
warn_suffix = if primary_records.include?(record) && ips.size > 1
|
|
153
|
+
' ⚠️'
|
|
154
|
+
else
|
|
155
|
+
''
|
|
156
|
+
end
|
|
157
|
+
puts " #{record}: #{display}#{warn_suffix}"
|
|
158
|
+
end
|
|
159
|
+
puts
|
|
160
|
+
rescue StandardError => e
|
|
161
|
+
puts "==> DNS (error: #{e.message})"
|
|
162
|
+
puts
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def dnsmasq_config(host, dns_user)
|
|
166
|
+
content = nil
|
|
167
|
+
with_timeout("dnsmasq #{host}") do
|
|
168
|
+
executor.execute_on_host_as(host, dns_user) do
|
|
169
|
+
content = capture(:sudo, 'sh', '-c',
|
|
170
|
+
'cat /etc/dnsmasq.d/active_postgres.conf 2>/dev/null || ' \
|
|
171
|
+
'cat /etc/dnsmasq.d/messhy.conf 2>/dev/null || true',
|
|
172
|
+
raise_on_non_zero_exit: false).to_s
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
content
|
|
176
|
+
rescue StandardError
|
|
177
|
+
nil
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def normalize_dns_servers(raw_servers)
|
|
181
|
+
Array(raw_servers).map do |server|
|
|
182
|
+
if server.is_a?(Hash)
|
|
183
|
+
ssh_host = server[:ssh_host] || server['ssh_host'] || server[:host] || server['host']
|
|
184
|
+
private_ip = server[:private_ip] || server['private_ip'] || server[:ip] || server['ip']
|
|
185
|
+
private_ip ||= ssh_host
|
|
186
|
+
ssh_host ||= private_ip
|
|
187
|
+
{ ssh_host: ssh_host.to_s, private_ip: private_ip.to_s }
|
|
188
|
+
else
|
|
189
|
+
value = server.to_s
|
|
190
|
+
{ ssh_host: value, private_ip: value }
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def normalize_dns_domains(dns_config)
|
|
196
|
+
Array(dns_config[:domains] || dns_config[:domain]).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def normalize_dns_records(value, default_prefix:, domains:)
|
|
200
|
+
records = Array(value).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
201
|
+
return records unless records.empty?
|
|
202
|
+
|
|
203
|
+
domains = ['mesh'] if domains.empty?
|
|
204
|
+
domains.map { |domain| "#{default_prefix}.#{domain}" }
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def show_backups
|
|
208
|
+
output = nil
|
|
209
|
+
postgres_user = config.postgres_user
|
|
210
|
+
with_timeout('pgbackrest info') do
|
|
211
|
+
executor.execute_on_host(config.primary_host) do
|
|
212
|
+
output = capture(:sudo, '-u', postgres_user, 'pgbackrest', 'info',
|
|
213
|
+
raise_on_non_zero_exit: false).to_s
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
puts '==> Backups (pgBackRest)'
|
|
218
|
+
puts output if output && !output.strip.empty?
|
|
219
|
+
puts
|
|
220
|
+
rescue StandardError => e
|
|
221
|
+
puts "==> Backups (error: #{e.message})"
|
|
222
|
+
puts
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def fetch_system_stats(host, paths)
|
|
226
|
+
output = nil
|
|
227
|
+
with_timeout("system stats #{host}") do
|
|
228
|
+
ssh_executor = executor
|
|
229
|
+
safe_paths = paths.map { |p| Shellwords.escape(p) }.join(' ')
|
|
230
|
+
script = <<~'BASH'
|
|
231
|
+
set -e
|
|
232
|
+
loadavg=$(cut -d' ' -f1-3 /proc/loadavg)
|
|
233
|
+
|
|
234
|
+
read _ user nice system idle iowait irq softirq steal _ _ < /proc/stat
|
|
235
|
+
sleep 0.2
|
|
236
|
+
read _ user2 nice2 system2 idle2 iowait2 irq2 softirq2 steal2 _ _ < /proc/stat
|
|
237
|
+
|
|
238
|
+
total1=$((user + nice + system + idle + iowait + irq + softirq + steal))
|
|
239
|
+
total2=$((user2 + nice2 + system2 + idle2 + iowait2 + irq2 + softirq2 + steal2))
|
|
240
|
+
total=$((total2 - total1))
|
|
241
|
+
idle_delta=$((idle2 + iowait2 - idle - iowait))
|
|
242
|
+
|
|
243
|
+
cpu=0
|
|
244
|
+
if [ "$total" -gt 0 ]; then
|
|
245
|
+
cpu=$(( (100 * (total - idle_delta)) / total ))
|
|
246
|
+
fi
|
|
247
|
+
|
|
248
|
+
mem_total=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
|
|
249
|
+
mem_avail=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
|
250
|
+
|
|
251
|
+
echo "loadavg=${loadavg}"
|
|
252
|
+
echo "cpu=${cpu}"
|
|
253
|
+
echo "mem_total_kb=${mem_total}"
|
|
254
|
+
echo "mem_avail_kb=${mem_avail}"
|
|
255
|
+
BASH
|
|
256
|
+
|
|
257
|
+
ssh_executor.execute_on_host(host) do
|
|
258
|
+
output = capture(:bash, '-lc', "#{script}\n df -kP #{safe_paths} 2>/dev/null | tail -n +2 | " \
|
|
259
|
+
"awk '{print \"disk=\" $6 \"|\" $2 \"|\" $3 \"|\" $4 \"|\" $5}'",
|
|
260
|
+
raise_on_non_zero_exit: false).to_s
|
|
261
|
+
end
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
parse_system_stats(output)
|
|
265
|
+
rescue StandardError
|
|
266
|
+
nil
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def parse_system_stats(output)
|
|
270
|
+
stats = { disks: [] }
|
|
271
|
+
output.to_s.each_line do |line|
|
|
272
|
+
line = line.strip
|
|
273
|
+
next if line.empty?
|
|
274
|
+
|
|
275
|
+
if line.start_with?('loadavg=')
|
|
276
|
+
stats[:loadavg] = line.split('=', 2)[1]
|
|
277
|
+
elsif line.start_with?('cpu=')
|
|
278
|
+
stats[:cpu] = line.split('=', 2)[1].to_i
|
|
279
|
+
elsif line.start_with?('mem_total_kb=')
|
|
280
|
+
stats[:mem_total_kb] = line.split('=', 2)[1].to_i
|
|
281
|
+
elsif line.start_with?('mem_avail_kb=')
|
|
282
|
+
stats[:mem_avail_kb] = line.split('=', 2)[1].to_i
|
|
283
|
+
elsif line.start_with?('disk=')
|
|
284
|
+
_, payload = line.split('=', 2)
|
|
285
|
+
mount, total, used, _avail, pct = payload.split('|')
|
|
286
|
+
stats[:disks] << {
|
|
287
|
+
mount: mount,
|
|
288
|
+
total_kb: total.to_i,
|
|
289
|
+
used_kb: used.to_i,
|
|
290
|
+
use_pct: pct
|
|
291
|
+
}
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
stats[:loadavg] ||= 'n/a'
|
|
296
|
+
stats[:cpu] ||= 0
|
|
297
|
+
stats[:mem_total_kb] ||= 0
|
|
298
|
+
stats[:mem_avail_kb] ||= 0
|
|
299
|
+
stats
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
def format_kb(kb)
|
|
303
|
+
kb = kb.to_f
|
|
304
|
+
gb = kb / 1024 / 1024
|
|
305
|
+
return format('%.1fG', gb) if gb >= 1
|
|
306
|
+
|
|
307
|
+
mb = kb / 1024
|
|
308
|
+
format('%.0fM', mb)
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
def system_stat_paths
|
|
312
|
+
paths = ['/']
|
|
313
|
+
paths << '/var/lib/postgresql'
|
|
314
|
+
repo_path = pgbackrest_repo_path
|
|
315
|
+
paths << repo_path if repo_path
|
|
316
|
+
paths.compact.uniq
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
def pgbackrest_repo_path
|
|
320
|
+
return nil unless config.component_enabled?(:pgbackrest)
|
|
321
|
+
|
|
322
|
+
pg_config = config.component_config(:pgbackrest)
|
|
323
|
+
return pg_config[:repo_path] if pg_config[:repo_path]
|
|
324
|
+
|
|
325
|
+
pg_config[:repo_type].to_s == 'local' ? '/var/lib/pgbackrest' : '/backups'
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
def dns_failover_config
|
|
329
|
+
repmgr_config = config.component_config(:repmgr)
|
|
330
|
+
dns_config = repmgr_config[:dns_failover]
|
|
331
|
+
return nil unless dns_config && dns_config[:enabled]
|
|
332
|
+
|
|
333
|
+
dns_config
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
def dns_failover_enabled?
|
|
337
|
+
dns_failover_config != nil
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def overview_timeout
|
|
341
|
+
value = ENV.fetch('ACTIVE_POSTGRES_OVERVIEW_TIMEOUT', DEFAULT_TIMEOUT.to_s).to_i
|
|
342
|
+
value.positive? ? value : DEFAULT_TIMEOUT
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
def with_timeout(label)
|
|
346
|
+
Timeout.timeout(overview_timeout) { yield }
|
|
347
|
+
rescue Timeout::Error
|
|
348
|
+
raise StandardError, "#{label} timed out after #{overview_timeout}s"
|
|
349
|
+
end
|
|
350
|
+
end
|
|
351
|
+
end
|
|
@@ -81,6 +81,8 @@ module ActivePostgres
|
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
def fetch_from_rails_credentials(key_path)
|
|
84
|
+
ensure_rails_credentials_available
|
|
85
|
+
|
|
84
86
|
return nil unless defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
|
|
85
87
|
|
|
86
88
|
keys = key_path.split('.').map(&:to_sym)
|
|
@@ -89,6 +91,22 @@ module ActivePostgres
|
|
|
89
91
|
nil
|
|
90
92
|
end
|
|
91
93
|
|
|
94
|
+
def ensure_rails_credentials_available
|
|
95
|
+
return if @rails_boot_attempted
|
|
96
|
+
return if defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
|
|
97
|
+
|
|
98
|
+
@rails_boot_attempted = true
|
|
99
|
+
|
|
100
|
+
# Try loading just the Rails application (without full initialization)
|
|
101
|
+
# This makes Rails.application available for credential access
|
|
102
|
+
if File.exist?('./config/application.rb')
|
|
103
|
+
require './config/application'
|
|
104
|
+
end
|
|
105
|
+
rescue StandardError
|
|
106
|
+
# Rails boot failed — CLI running outside a Rails project
|
|
107
|
+
nil
|
|
108
|
+
end
|
|
109
|
+
|
|
92
110
|
def execute_command(command)
|
|
93
111
|
# Preserve RAILS_ENV if set
|
|
94
112
|
env_prefix = ENV['RAILS_ENV'] ? "RAILS_ENV=#{ENV['RAILS_ENV']} " : ''
|