active_postgres 0.9.4 → 0.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +23 -0
- data/README.md +20 -4
- data/SECURITY.md +21 -0
- data/lib/active_postgres/components/base.rb +2 -0
- data/lib/active_postgres/components/core.rb +11 -4
- data/lib/active_postgres/components/monitoring.rb +2 -2
- data/lib/active_postgres/components/pgbackrest.rb +10 -18
- data/lib/active_postgres/components/pgbouncer.rb +153 -14
- data/lib/active_postgres/components/repmgr.rb +75 -39
- data/lib/active_postgres/configuration.rb +57 -22
- data/lib/active_postgres/direct_executor.rb +3 -9
- data/lib/active_postgres/error_handler.rb +18 -18
- data/lib/active_postgres/health_checker.rb +10 -12
- data/lib/active_postgres/overview.rb +10 -10
- data/lib/active_postgres/secrets.rb +8 -7
- data/lib/active_postgres/ssh_executor.rb +5 -5
- data/lib/active_postgres/version.rb +1 -1
- data/lib/tasks/postgres.rake +26 -26
- data/templates/pgbackrest.conf.erb +2 -1
- data/templates/pgbouncer.ini.erb +2 -1
- data/templates/pgbouncer_follow_dns_primary.sh.erb +39 -0
- data/templates/postgres_log_archive.sh.erb +37 -6
- data/templates/postgresql.conf.erb +5 -0
- data/templates/repmgr_dns_failover.sh.erb +2 -2
- metadata +17 -56
|
@@ -114,6 +114,7 @@ module ActivePostgres
|
|
|
114
114
|
if replication_user == repmgr_user && replication_password != repmgr_password
|
|
115
115
|
raise Error, 'replication_user matches repmgr user but passwords differ. Use a distinct replication_user or the same password.'
|
|
116
116
|
end
|
|
117
|
+
|
|
117
118
|
effective_replication_password = replication_user == repmgr_user ? repmgr_password : replication_password
|
|
118
119
|
|
|
119
120
|
# Variables used in ERB templates via binding
|
|
@@ -174,13 +175,13 @@ module ActivePostgres
|
|
|
174
175
|
info 'Creating repmgr database and user...'
|
|
175
176
|
repmgr_sql = repmgr_component.send(:build_repmgr_setup_sql, repmgr_user, repmgr_db, repmgr_password)
|
|
176
177
|
executor.run_sql_on_backend(self, repmgr_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
|
|
177
|
-
|
|
178
|
+
capture: false)
|
|
178
179
|
|
|
179
180
|
if replication_user != repmgr_user
|
|
180
181
|
info 'Ensuring replication user exists...'
|
|
181
182
|
repl_sql = repmgr_component.send(:build_replication_user_sql, replication_user, effective_replication_password)
|
|
182
183
|
executor.run_sql_on_backend(self, repl_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
|
|
183
|
-
|
|
184
|
+
capture: false)
|
|
184
185
|
end
|
|
185
186
|
|
|
186
187
|
info 'Reloading PostgreSQL configuration to apply pg_hba.conf changes...'
|
|
@@ -202,14 +203,14 @@ module ActivePostgres
|
|
|
202
203
|
cluster_show = nil
|
|
203
204
|
5.times do |attempt|
|
|
204
205
|
cluster_show = capture(:sudo, '-u', 'postgres', 'bash', '-lc',
|
|
205
|
-
|
|
206
|
+
'repmgr cluster show -f /etc/repmgr.conf 2>&1', raise_on_non_zero_exit: false).to_s
|
|
206
207
|
|
|
207
208
|
break if cluster_show.match?(/primary/i)
|
|
208
209
|
|
|
209
210
|
sleep 2 if attempt < 4
|
|
210
211
|
end
|
|
211
212
|
|
|
212
|
-
unless cluster_show
|
|
213
|
+
unless cluster_show&.match?(/primary/i)
|
|
213
214
|
# Fallback: verify via repmgr metadata in the repmgr database
|
|
214
215
|
db_check = executor.run_sql_on_backend(self,
|
|
215
216
|
'SELECT type FROM repmgr.nodes WHERE node_id = 1;',
|
|
@@ -255,13 +256,14 @@ module ActivePostgres
|
|
|
255
256
|
if replication_user == repmgr_user && replication_password != repmgr_password
|
|
256
257
|
raise Error, 'replication_user matches repmgr user but passwords differ. Use a distinct replication_user or the same password.'
|
|
257
258
|
end
|
|
259
|
+
|
|
258
260
|
effective_replication_password = replication_user == repmgr_user ? repmgr_password : replication_password
|
|
259
261
|
|
|
260
262
|
ensure_primary_registered
|
|
261
263
|
ensure_primary_replication_ready(repmgr_password, effective_replication_password)
|
|
262
264
|
|
|
263
265
|
setup_pgpass_file(standby_host, repmgr_password, replication_password: effective_replication_password,
|
|
264
|
-
|
|
266
|
+
primary_ip: primary_replication_host)
|
|
265
267
|
|
|
266
268
|
if standby_already_configured?(standby_host)
|
|
267
269
|
puts ' Standby already configured, updating configs...'
|
|
@@ -418,9 +420,10 @@ module ActivePostgres
|
|
|
418
420
|
|
|
419
421
|
def setup_inter_node_ssh
|
|
420
422
|
dns_config = dns_failover_config
|
|
421
|
-
key_path = dns_config && dns_config[:ssh_key_path] || '/var/lib/postgresql/.ssh/active_postgres_dns'
|
|
423
|
+
key_path = (dns_config && dns_config[:ssh_key_path]) || '/var/lib/postgresql/.ssh/active_postgres_dns'
|
|
422
424
|
postgres_user = config.postgres_user
|
|
423
425
|
all_hosts = config.all_hosts
|
|
426
|
+
replication_hosts = all_hosts.to_h { |host| [host, config.replication_host_for(host)] }
|
|
424
427
|
pub_keys = {}
|
|
425
428
|
|
|
426
429
|
all_hosts.each do |host|
|
|
@@ -430,18 +433,35 @@ module ActivePostgres
|
|
|
430
433
|
all_hosts.each do |host|
|
|
431
434
|
ssh_executor.execute_on_host(host) do
|
|
432
435
|
other_keys = pub_keys.reject { |h, _| h == host }.values
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
436
|
+
authorized_keys_path = '/var/lib/postgresql/.ssh/authorized_keys'
|
|
437
|
+
existing_keys = if test(:sudo, 'test', '-f', authorized_keys_path)
|
|
438
|
+
capture(:sudo, 'cat', authorized_keys_path).lines.map(&:strip)
|
|
439
|
+
else
|
|
440
|
+
[]
|
|
441
|
+
end
|
|
442
|
+
authorized_keys = (existing_keys + other_keys.map(&:to_s).map(&:strip)).reject(&:empty?).uniq.join("\n")
|
|
443
|
+
unless authorized_keys.empty?
|
|
444
|
+
upload! StringIO.new("#{authorized_keys}\n"), '/tmp/pg_authorized_keys'
|
|
445
|
+
execute :sudo, 'mv', '/tmp/pg_authorized_keys', authorized_keys_path
|
|
446
|
+
execute :sudo, 'chown', "#{postgres_user}:#{postgres_user}", authorized_keys_path
|
|
447
|
+
execute :sudo, 'chmod', '600', authorized_keys_path
|
|
440
448
|
end
|
|
441
449
|
|
|
442
|
-
peer_ips = all_hosts.reject { |h| h == host }.map { |h|
|
|
443
|
-
|
|
444
|
-
|
|
450
|
+
peer_ips = all_hosts.reject { |h| h == host }.map { |h| replication_hosts[h] }
|
|
451
|
+
known_hosts_path = '/var/lib/postgresql/.ssh/known_hosts'
|
|
452
|
+
existing_hosts = if test(:sudo, 'test', '-f', known_hosts_path)
|
|
453
|
+
capture(:sudo, 'cat', known_hosts_path).lines.map(&:strip)
|
|
454
|
+
else
|
|
455
|
+
[]
|
|
456
|
+
end
|
|
457
|
+
scanned_hosts = capture(:sudo, '-u', postgres_user, 'ssh-keyscan', *peer_ips, raise_on_non_zero_exit: false)
|
|
458
|
+
known_hosts = (existing_hosts + scanned_hosts.lines.map(&:strip)).reject(&:empty?).uniq.join("\n")
|
|
459
|
+
unless known_hosts.empty?
|
|
460
|
+
upload! StringIO.new("#{known_hosts}\n"), '/tmp/pg_known_hosts'
|
|
461
|
+
execute :sudo, 'mv', '/tmp/pg_known_hosts', known_hosts_path
|
|
462
|
+
execute :sudo, 'chown', "#{postgres_user}:#{postgres_user}", known_hosts_path
|
|
463
|
+
execute :sudo, 'chmod', '600', known_hosts_path
|
|
464
|
+
end
|
|
445
465
|
end
|
|
446
466
|
end
|
|
447
467
|
end
|
|
@@ -518,10 +538,21 @@ module ActivePostgres
|
|
|
518
538
|
public_key = capture(:sudo, '-u', postgres_user, 'cat', "#{key_path}.pub").strip
|
|
519
539
|
|
|
520
540
|
unless dns_servers.empty?
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
541
|
+
known_hosts_path = '/var/lib/postgresql/.ssh/known_hosts'
|
|
542
|
+
existing_hosts = if test(:sudo, 'test', '-f', known_hosts_path)
|
|
543
|
+
capture(:sudo, 'cat', known_hosts_path).lines.map(&:strip)
|
|
544
|
+
else
|
|
545
|
+
[]
|
|
546
|
+
end
|
|
547
|
+
scanned_hosts = capture(:sudo, '-u', postgres_user, 'ssh-keyscan', '-H', *dns_servers,
|
|
548
|
+
raise_on_non_zero_exit: false)
|
|
549
|
+
known_hosts = (existing_hosts + scanned_hosts.lines.map(&:strip)).reject(&:empty?).uniq.join("\n")
|
|
550
|
+
unless known_hosts.empty?
|
|
551
|
+
upload! StringIO.new("#{known_hosts}\n"), '/tmp/pg_known_hosts'
|
|
552
|
+
execute :sudo, 'mv', '/tmp/pg_known_hosts', known_hosts_path
|
|
553
|
+
execute :sudo, 'chown', "#{postgres_user}:#{postgres_user}", known_hosts_path
|
|
554
|
+
execute :sudo, 'chmod', '600', known_hosts_path
|
|
555
|
+
end
|
|
525
556
|
end
|
|
526
557
|
end
|
|
527
558
|
|
|
@@ -542,8 +573,8 @@ module ActivePostgres
|
|
|
542
573
|
|
|
543
574
|
upload! StringIO.new("#{key}\n"), '/tmp/active_postgres_dns_key.pub'
|
|
544
575
|
execute :bash, '-c',
|
|
545
|
-
|
|
546
|
-
|
|
576
|
+
'grep -qxF -f /tmp/active_postgres_dns_key.pub ~/.ssh/authorized_keys || ' \
|
|
577
|
+
'cat /tmp/active_postgres_dns_key.pub >> ~/.ssh/authorized_keys'
|
|
547
578
|
execute :rm, '-f', '/tmp/active_postgres_dns_key.pub'
|
|
548
579
|
end
|
|
549
580
|
end
|
|
@@ -560,11 +591,11 @@ module ActivePostgres
|
|
|
560
591
|
|
|
561
592
|
domains = normalize_dns_domains(dns_config)
|
|
562
593
|
primary_records = normalize_dns_records(dns_config[:primary_records] || dns_config[:primary_record],
|
|
563
|
-
|
|
564
|
-
|
|
594
|
+
default_prefix: 'db-primary',
|
|
595
|
+
domains: domains)
|
|
565
596
|
replica_records = normalize_dns_records(dns_config[:replica_records] || dns_config[:replica_record],
|
|
566
|
-
|
|
567
|
-
|
|
597
|
+
default_prefix: 'db-replica',
|
|
598
|
+
domains: domains)
|
|
568
599
|
|
|
569
600
|
_ = primary_records
|
|
570
601
|
_ = replica_records
|
|
@@ -732,10 +763,10 @@ module ActivePostgres
|
|
|
732
763
|
# Check replication status
|
|
733
764
|
begin
|
|
734
765
|
rep_status = executor.run_sql_on_backend(self,
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
766
|
+
'SELECT pg_is_in_recovery();',
|
|
767
|
+
postgres_user: postgres_user,
|
|
768
|
+
tuples_only: true,
|
|
769
|
+
capture: true).to_s
|
|
739
770
|
if rep_status.include?('t')
|
|
740
771
|
info '✓ Standby is in recovery mode (receiving replication)'
|
|
741
772
|
else
|
|
@@ -750,10 +781,10 @@ module ActivePostgres
|
|
|
750
781
|
# Check lag
|
|
751
782
|
begin
|
|
752
783
|
lag_result = executor.run_sql_on_backend(self,
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
784
|
+
'SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::int AS lag;',
|
|
785
|
+
postgres_user: postgres_user,
|
|
786
|
+
tuples_only: true,
|
|
787
|
+
capture: true).to_s.strip
|
|
757
788
|
info "Replication lag: #{lag_result}"
|
|
758
789
|
rescue StandardError => e
|
|
759
790
|
error "Failed to get replication lag: #{e.message}"
|
|
@@ -785,7 +816,7 @@ module ActivePostgres
|
|
|
785
816
|
# Create .pgpass file for postgres user to avoid password exposure in logs
|
|
786
817
|
# Format: hostname:port:database:username:password
|
|
787
818
|
pgpass_content = build_pgpass_content(host, repmgr_password, replication_password: replication_password,
|
|
788
|
-
|
|
819
|
+
primary_ip: primary_ip)
|
|
789
820
|
|
|
790
821
|
ssh_executor.execute_on_host(host) do
|
|
791
822
|
# Create .pgpass in postgres user's home directory
|
|
@@ -854,8 +885,13 @@ module ActivePostgres
|
|
|
854
885
|
def escape_pgpass_value(value)
|
|
855
886
|
return '' if value.nil?
|
|
856
887
|
|
|
857
|
-
|
|
858
|
-
|
|
888
|
+
value.to_s.each_char.map do |char|
|
|
889
|
+
case char
|
|
890
|
+
when '\\' then '\\\\'
|
|
891
|
+
when ':' then '\\:'
|
|
892
|
+
else char
|
|
893
|
+
end
|
|
894
|
+
end.join
|
|
859
895
|
end
|
|
860
896
|
|
|
861
897
|
def build_primary_conninfo(standby_label)
|
|
@@ -935,12 +971,12 @@ module ActivePostgres
|
|
|
935
971
|
ssh_executor.execute_on_host(host) do
|
|
936
972
|
repmgr_sql = repmgr_component.send(:build_repmgr_setup_sql, repmgr_user, repmgr_db, repmgr_password)
|
|
937
973
|
executor.run_sql_on_backend(self, repmgr_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
|
|
938
|
-
|
|
974
|
+
capture: false)
|
|
939
975
|
|
|
940
976
|
if replication_user != repmgr_user
|
|
941
977
|
repl_sql = repmgr_component.send(:build_replication_user_sql, replication_user, effective_replication_password)
|
|
942
978
|
executor.run_sql_on_backend(self, repl_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
|
|
943
|
-
|
|
979
|
+
capture: false)
|
|
944
980
|
end
|
|
945
981
|
|
|
946
982
|
info '✓ Primary replication user is ready'
|
|
@@ -39,6 +39,35 @@ module ActivePostgres
|
|
|
39
39
|
[primary_host] + standby_hosts
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
+
def pgbouncer_app_hosts
|
|
43
|
+
hosts = component_config(:pgbouncer)[:app_hosts] || component_config(:pgbouncer)['app_hosts'] || []
|
|
44
|
+
hosts = [hosts] unless hosts.is_a?(Array)
|
|
45
|
+
|
|
46
|
+
hosts.filter_map do |entry|
|
|
47
|
+
case entry
|
|
48
|
+
when String
|
|
49
|
+
entry.strip.empty? ? nil : { 'host' => entry.strip }
|
|
50
|
+
when Hash
|
|
51
|
+
host = entry['host'] || entry[:host] || entry['ip'] || entry[:ip]
|
|
52
|
+
next if host.to_s.strip.empty?
|
|
53
|
+
|
|
54
|
+
entry.merge('host' => host.to_s.strip)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def pgbouncer_primary_record
|
|
60
|
+
pgbouncer = component_config(:pgbouncer)
|
|
61
|
+
explicit = pgbouncer[:primary_record] || pgbouncer['primary_record']
|
|
62
|
+
return explicit.to_s.strip unless explicit.to_s.strip.empty?
|
|
63
|
+
|
|
64
|
+
dns_failover = component_config(:repmgr)[:dns_failover] || component_config(:repmgr)['dns_failover'] || {}
|
|
65
|
+
record = dns_failover[:primary_record] || dns_failover['primary_record']
|
|
66
|
+
return record.to_s.strip unless record.to_s.strip.empty?
|
|
67
|
+
|
|
68
|
+
primary_replication_host
|
|
69
|
+
end
|
|
70
|
+
|
|
42
71
|
def primary_host
|
|
43
72
|
@primary['host']
|
|
44
73
|
end
|
|
@@ -96,11 +125,10 @@ module ActivePostgres
|
|
|
96
125
|
# Validate required secrets if components are enabled
|
|
97
126
|
raise Error, 'Missing replication_password secret' if component_enabled?(:repmgr) && !secrets_config['replication_password']
|
|
98
127
|
raise Error, 'Missing monitoring_password secret' if component_enabled?(:monitoring) && !secrets_config['monitoring_password']
|
|
128
|
+
|
|
99
129
|
if component_enabled?(:monitoring)
|
|
100
130
|
grafana_config = component_config(:monitoring)[:grafana] || {}
|
|
101
|
-
if grafana_config[:enabled] && !secrets_config['grafana_admin_password']
|
|
102
|
-
raise Error, 'Missing grafana_admin_password secret'
|
|
103
|
-
end
|
|
131
|
+
raise Error, 'Missing grafana_admin_password secret' if grafana_config[:enabled] && !secrets_config['grafana_admin_password']
|
|
104
132
|
if grafana_config[:enabled] && grafana_config[:host].to_s.strip.empty?
|
|
105
133
|
raise Error, 'monitoring.grafana.host is required when grafana is enabled'
|
|
106
134
|
end
|
|
@@ -114,27 +142,34 @@ module ActivePostgres
|
|
|
114
142
|
end
|
|
115
143
|
end
|
|
116
144
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
145
|
+
if component_enabled?(:pgbouncer)
|
|
146
|
+
pgbouncer_app_hosts.each do |app_host|
|
|
147
|
+
host = app_host['host'] || app_host[:host]
|
|
148
|
+
raise Error, 'pgbouncer.app_hosts entries must include host' if host.to_s.strip.empty?
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
if component_enabled?(:repmgr)
|
|
153
|
+
dns_failover = component_config(:repmgr)[:dns_failover]
|
|
154
|
+
if dns_failover && dns_failover[:enabled]
|
|
155
|
+
domains = Array(dns_failover[:domains] || dns_failover[:domain]).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
156
|
+
servers = Array(dns_failover[:dns_servers])
|
|
157
|
+
provider = (dns_failover[:provider] || 'dnsmasq').to_s.strip
|
|
158
|
+
|
|
159
|
+
raise Error, 'dns_failover.domain or dns_failover.domains is required when enabled' if domains.empty?
|
|
160
|
+
raise Error, 'dns_failover.dns_servers is required when enabled' if servers.empty?
|
|
161
|
+
raise Error, "Unsupported dns_failover provider '#{provider}'" unless provider == 'dnsmasq'
|
|
162
|
+
|
|
163
|
+
servers.each do |server|
|
|
164
|
+
next unless server.is_a?(Hash)
|
|
165
|
+
|
|
166
|
+
ssh_host = server['ssh_host'] || server[:ssh_host] || server['host'] || server[:host]
|
|
167
|
+
private_ip = server['private_ip'] || server[:private_ip] || server['ip'] || server[:ip]
|
|
168
|
+
raise Error, 'dns_failover.dns_servers entries must include host/ssh_host or private_ip' if
|
|
169
|
+
(ssh_host.nil? || ssh_host.to_s.strip.empty?) && (private_ip.nil? || private_ip.to_s.strip.empty?)
|
|
136
170
|
end
|
|
137
171
|
end
|
|
172
|
+
end
|
|
138
173
|
|
|
139
174
|
true
|
|
140
175
|
end
|
|
@@ -62,19 +62,13 @@ module ActivePostgres
|
|
|
62
62
|
return nil if value.nil?
|
|
63
63
|
|
|
64
64
|
# Handle Rails credentials
|
|
65
|
-
if value.start_with?('rails_credentials:')
|
|
66
|
-
return resolve_rails_credentials(value)
|
|
67
|
-
end
|
|
65
|
+
return resolve_rails_credentials(value) if value.start_with?('rails_credentials:')
|
|
68
66
|
|
|
69
67
|
# Handle environment variables
|
|
70
|
-
if value.start_with?('$')
|
|
71
|
-
return ENV[value[1..]]
|
|
72
|
-
end
|
|
68
|
+
return ENV.fetch(value[1..], nil) if value.start_with?('$')
|
|
73
69
|
|
|
74
70
|
# Handle shell commands
|
|
75
|
-
if value.start_with?('$(') && value.end_with?(')')
|
|
76
|
-
return `#{value[2..-2]}`.strip
|
|
77
|
-
end
|
|
71
|
+
return `#{value[2..-2]}`.strip if value.start_with?('$(') && value.end_with?(')')
|
|
78
72
|
|
|
79
73
|
value
|
|
80
74
|
end
|
|
@@ -127,24 +127,24 @@ module ActivePostgres
|
|
|
127
127
|
|
|
128
128
|
# Identify error type from error message
|
|
129
129
|
def identify_error_type(message)
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
130
|
+
normalized = message.to_s.downcase
|
|
131
|
+
|
|
132
|
+
return :ssh_connection if contains_any?(normalized, 'ssh', 'connection refused', 'network unreachable')
|
|
133
|
+
return :private_network_connectivity if contains_any?(normalized, 'private network', 'vpn', 'wireguard network')
|
|
134
|
+
return :postgresql_not_starting if normalized.include?('postgresql') && normalized.include?('not') && normalized.include?('start')
|
|
135
|
+
return :postgresql_not_starting if normalized.include?('cluster') && normalized.include?('not') && normalized.include?('running')
|
|
136
|
+
return :repmgr_clone_failed if normalized.include?('repmgr') && normalized.include?('clone')
|
|
137
|
+
return :repmgr_clone_failed if normalized.include?('data directory')
|
|
138
|
+
return :repmgr_register_failed if normalized.include?('repmgr') && normalized.include?('register')
|
|
139
|
+
return :repmgr_register_failed if normalized.include?('unable to connect to') && normalized.include?('primary')
|
|
140
|
+
return :ssl_certificate_error if contains_any?(normalized, 'ssl', 'certificate', 'tls')
|
|
141
|
+
return :disk_space_error if contains_any?(normalized, 'no space', 'disk full')
|
|
142
|
+
|
|
143
|
+
:authentication_failed if contains_any?(normalized, 'authentication', 'password', 'pg_hba')
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def contains_any?(value, *needles)
|
|
147
|
+
needles.any? { |needle| value.include?(needle) }
|
|
148
148
|
end
|
|
149
149
|
|
|
150
150
|
# Show troubleshooting guide for a specific error type
|
|
@@ -12,14 +12,6 @@ module ActivePostgres
|
|
|
12
12
|
@executor
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
-
private def create_executor
|
|
16
|
-
if use_ssh_executor?
|
|
17
|
-
SSHExecutor.new(config, quiet: true)
|
|
18
|
-
else
|
|
19
|
-
DirectExecutor.new(config, quiet: true)
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
|
|
23
15
|
def use_ssh_executor?
|
|
24
16
|
mode = ENV['ACTIVE_POSTGRES_STATUS_MODE'].to_s.strip.downcase
|
|
25
17
|
return true if mode == 'ssh'
|
|
@@ -131,6 +123,14 @@ module ActivePostgres
|
|
|
131
123
|
|
|
132
124
|
private
|
|
133
125
|
|
|
126
|
+
def create_executor
|
|
127
|
+
if use_ssh_executor?
|
|
128
|
+
SSHExecutor.new(config, quiet: true)
|
|
129
|
+
else
|
|
130
|
+
DirectExecutor.new(config, quiet: true)
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
134
|
def collect_node_status
|
|
135
135
|
nodes = []
|
|
136
136
|
|
|
@@ -184,9 +184,7 @@ module ActivePostgres
|
|
|
184
184
|
lag: [3, nodes.map { |n| n[:lag].to_s.length }.max].max
|
|
185
185
|
}
|
|
186
186
|
|
|
187
|
-
if config.component_enabled?(:pgbouncer)
|
|
188
|
-
cols[:pgbouncer] = [9, nodes.map { |n| n[:pgbouncer].to_s.length }.max].max
|
|
189
|
-
end
|
|
187
|
+
cols[:pgbouncer] = [9, nodes.map { |n| n[:pgbouncer].to_s.length }.max].max if config.component_enabled?(:pgbouncer)
|
|
190
188
|
|
|
191
189
|
cols
|
|
192
190
|
end
|
|
@@ -194,7 +192,7 @@ module ActivePostgres
|
|
|
194
192
|
def print_table_header(cols)
|
|
195
193
|
fmt = "%-#{cols[:role]}s %-#{cols[:host]}s %-#{cols[:private_ip]}s " \
|
|
196
194
|
"%-#{cols[:label]}s %-#{cols[:status]}s %#{cols[:conn]}s %#{cols[:lag]}s"
|
|
197
|
-
headers =
|
|
195
|
+
headers = ['Role', 'Host', 'Private IP', 'Label', 'Status', 'Conn', 'Lag']
|
|
198
196
|
|
|
199
197
|
if cols[:pgbouncer]
|
|
200
198
|
fmt += " %-#{cols[:pgbouncer]}s"
|
|
@@ -61,14 +61,14 @@ module ActivePostgres
|
|
|
61
61
|
label = config.node_label_for(host)
|
|
62
62
|
stats = fetch_system_stats(host, paths)
|
|
63
63
|
if stats.nil?
|
|
64
|
-
puts " #{host}#{
|
|
64
|
+
puts " #{host}#{" (#{label})" if label}: unavailable"
|
|
65
65
|
next
|
|
66
66
|
end
|
|
67
67
|
|
|
68
68
|
mem_used_kb = stats[:mem_total_kb] - stats[:mem_avail_kb]
|
|
69
69
|
mem_pct = stats[:mem_total_kb].positive? ? (mem_used_kb.to_f / stats[:mem_total_kb] * 100).round : 0
|
|
70
70
|
|
|
71
|
-
puts " #{host}#{
|
|
71
|
+
puts " #{host}#{" (#{label})" if label}: load #{stats[:loadavg]} | cpu #{stats[:cpu]}% | " \
|
|
72
72
|
"mem #{format_kb(mem_used_kb)}/#{format_kb(stats[:mem_total_kb])} (#{mem_pct}%)"
|
|
73
73
|
|
|
74
74
|
stats[:disks].each do |disk|
|
|
@@ -227,7 +227,7 @@ module ActivePostgres
|
|
|
227
227
|
with_timeout("system stats #{host}") do
|
|
228
228
|
ssh_executor = executor
|
|
229
229
|
safe_paths = paths.map { |p| Shellwords.escape(p) }.join(' ')
|
|
230
|
-
script = <<~
|
|
230
|
+
script = <<~BASH
|
|
231
231
|
set -e
|
|
232
232
|
loadavg=$(cut -d' ' -f1-3 /proc/loadavg)
|
|
233
233
|
|
|
@@ -256,7 +256,7 @@ module ActivePostgres
|
|
|
256
256
|
|
|
257
257
|
ssh_executor.execute_on_host(host) do
|
|
258
258
|
output = capture(:bash, '-lc', "#{script}\n df -kP #{safe_paths} 2>/dev/null | tail -n +2 | " \
|
|
259
|
-
|
|
259
|
+
"awk '{print \"disk=\" $6 \"|\" $2 \"|\" $3 \"|\" $4 \"|\" $5}'",
|
|
260
260
|
raise_on_non_zero_exit: false).to_s
|
|
261
261
|
end
|
|
262
262
|
end
|
|
@@ -299,12 +299,12 @@ module ActivePostgres
|
|
|
299
299
|
stats
|
|
300
300
|
end
|
|
301
301
|
|
|
302
|
-
def format_kb(
|
|
303
|
-
|
|
304
|
-
gb =
|
|
302
|
+
def format_kb(kilobytes)
|
|
303
|
+
kilobytes = kilobytes.to_f
|
|
304
|
+
gb = kilobytes / 1024 / 1024
|
|
305
305
|
return format('%.1fG', gb) if gb >= 1
|
|
306
306
|
|
|
307
|
-
mb =
|
|
307
|
+
mb = kilobytes / 1024
|
|
308
308
|
format('%.0fM', mb)
|
|
309
309
|
end
|
|
310
310
|
|
|
@@ -342,8 +342,8 @@ module ActivePostgres
|
|
|
342
342
|
value.positive? ? value : DEFAULT_TIMEOUT
|
|
343
343
|
end
|
|
344
344
|
|
|
345
|
-
def with_timeout(label)
|
|
346
|
-
Timeout.timeout(overview_timeout)
|
|
345
|
+
def with_timeout(label, &)
|
|
346
|
+
Timeout.timeout(overview_timeout, &)
|
|
347
347
|
rescue Timeout::Error
|
|
348
348
|
raise StandardError, "#{label} timed out after #{overview_timeout}s"
|
|
349
349
|
end
|
|
@@ -20,8 +20,8 @@ module ActivePostgres
|
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
def resolve_all
|
|
23
|
-
config.secrets_config.keys.
|
|
24
|
-
|
|
23
|
+
config.secrets_config.keys.to_h do |key|
|
|
24
|
+
[key, resolve(key)]
|
|
25
25
|
end
|
|
26
26
|
end
|
|
27
27
|
|
|
@@ -47,8 +47,9 @@ module ActivePostgres
|
|
|
47
47
|
|
|
48
48
|
resolve_all.each do |key, value|
|
|
49
49
|
file_path = File.join(directory, key)
|
|
50
|
-
File.
|
|
51
|
-
|
|
50
|
+
File.open(file_path, File::WRONLY | File::CREAT | File::TRUNC, 0o600) do |file|
|
|
51
|
+
file.write(value)
|
|
52
|
+
end
|
|
52
53
|
puts "Cached #{key} to #{file_path}"
|
|
53
54
|
end
|
|
54
55
|
|
|
@@ -96,12 +97,12 @@ module ActivePostgres
|
|
|
96
97
|
return if defined?(::Rails) && ::Rails.respond_to?(:application) && ::Rails.application
|
|
97
98
|
|
|
98
99
|
@rails_boot_attempted = true
|
|
100
|
+
ENV['RAILS_ENV'] ||= config.environment
|
|
101
|
+
ENV['BORING_ENVIRONMENT'] ||= config.environment
|
|
99
102
|
|
|
100
103
|
# Try loading just the Rails application (without full initialization)
|
|
101
104
|
# This makes Rails.application available for credential access
|
|
102
|
-
if File.exist?('./config/application.rb')
|
|
103
|
-
require './config/application'
|
|
104
|
-
end
|
|
105
|
+
require './config/application' if File.exist?('./config/application.rb')
|
|
105
106
|
rescue StandardError
|
|
106
107
|
# Rails boot failed — CLI running outside a Rails project
|
|
107
108
|
nil
|
|
@@ -215,11 +215,11 @@ module ActivePostgres
|
|
|
215
215
|
execute_on_host(host) do
|
|
216
216
|
backend = self
|
|
217
217
|
result = executor.run_sql_on_backend(backend, sql,
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
218
|
+
postgres_user: postgres_user,
|
|
219
|
+
port: port,
|
|
220
|
+
database: database,
|
|
221
|
+
tuples_only: tuples_only,
|
|
222
|
+
capture: capture)
|
|
223
223
|
end
|
|
224
224
|
result
|
|
225
225
|
end
|