active_postgres 0.9.5 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +40 -0
- data/README.md +47 -6
- data/SECURITY.md +21 -0
- data/lib/active_postgres/components/base.rb +2 -0
- data/lib/active_postgres/components/core.rb +11 -4
- data/lib/active_postgres/components/monitoring.rb +2 -2
- data/lib/active_postgres/components/pgbackrest.rb +13 -17
- data/lib/active_postgres/components/pgbouncer.rb +5 -9
- data/lib/active_postgres/components/repmgr.rb +165 -105
- data/lib/active_postgres/configuration/pgbackrest_policy.rb +19 -0
- data/lib/active_postgres/configuration/standby_policy.rb +24 -0
- data/lib/active_postgres/configuration.rb +64 -38
- data/lib/active_postgres/deployment_flow.rb +5 -1
- data/lib/active_postgres/direct_executor.rb +3 -9
- data/lib/active_postgres/error_handler.rb +18 -18
- data/lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb +7 -0
- data/lib/active_postgres/health_checker.rb +10 -12
- data/lib/active_postgres/installer.rb +2 -2
- data/lib/active_postgres/overview.rb +10 -10
- data/lib/active_postgres/secrets.rb +6 -7
- data/lib/active_postgres/ssh_executor.rb +66 -9
- data/lib/active_postgres/standby_deployment_flow.rb +46 -8
- data/lib/active_postgres/version.rb +1 -1
- data/lib/tasks/postgres.rake +26 -26
- data/templates/pgbackrest.conf.erb +9 -1
- data/templates/postgresql.conf.erb +5 -0
- data/templates/repmgr_dns_failover.sh.erb +2 -2
- metadata +18 -56
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
require 'yaml'
|
|
2
|
+
require_relative 'configuration/pgbackrest_policy'
|
|
3
|
+
require_relative 'configuration/standby_policy'
|
|
2
4
|
|
|
3
5
|
module ActivePostgres
|
|
4
6
|
class Configuration
|
|
5
|
-
|
|
6
|
-
|
|
7
|
+
include PgBackRestPolicy
|
|
8
|
+
include StandbyPolicy
|
|
9
|
+
|
|
10
|
+
attr_reader :environment, :version, :user, :ssh_key, :ssh_known_hosts_file, :ssh_host_key_verification,
|
|
11
|
+
:primary, :standbys, :jump_hosts, :components, :secrets_config, :database_config
|
|
7
12
|
|
|
8
13
|
def initialize(config_hash, environment = 'development')
|
|
9
14
|
@environment = environment
|
|
@@ -14,6 +19,7 @@ module ActivePostgres
|
|
|
14
19
|
@version = env_config['version'] || 18
|
|
15
20
|
@user = env_config['user'] || 'ubuntu'
|
|
16
21
|
@ssh_key = File.expand_path(env_config['ssh_key'] || '~/.ssh/id_rsa')
|
|
22
|
+
@ssh_known_hosts_file = optional_path(env_config['ssh_known_hosts_file'])
|
|
17
23
|
@ssh_host_key_verification = normalize_ssh_host_key_verification(
|
|
18
24
|
env_config['ssh_host_key_verification'] || env_config['ssh_verify_host_key']
|
|
19
25
|
)
|
|
@@ -21,6 +27,7 @@ module ActivePostgres
|
|
|
21
27
|
@primary = env_config['primary'] || {}
|
|
22
28
|
@standbys = env_config['standby'] || []
|
|
23
29
|
@standbys = [@standbys] unless @standbys.is_a?(Array)
|
|
30
|
+
@jump_hosts = env_config['jump_hosts'] || {}
|
|
24
31
|
|
|
25
32
|
@components = parse_components(env_config['components'] || {})
|
|
26
33
|
@secrets_config = env_config['secrets'] || {}
|
|
@@ -111,6 +118,24 @@ module ActivePostgres
|
|
|
111
118
|
@standbys.find { |s| s['host'] == host }
|
|
112
119
|
end
|
|
113
120
|
|
|
121
|
+
def node_config_for(host)
|
|
122
|
+
return @primary if host == primary_host
|
|
123
|
+
|
|
124
|
+
standby_config_for(host)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def jump_host_config_for(host)
|
|
128
|
+
jump_host_name = node_config_for(host)&.dig('jump_host')
|
|
129
|
+
return unless jump_host_name
|
|
130
|
+
|
|
131
|
+
@jump_hosts[jump_host_name] || raise(Error, "Host #{host} jump_host not found: #{jump_host_name}")
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def repmgr_config_for(host)
|
|
135
|
+
overrides = node_config_for(host)&.dig('repmgr') || {}
|
|
136
|
+
component_config(:repmgr).merge(symbolize_keys(overrides))
|
|
137
|
+
end
|
|
138
|
+
|
|
114
139
|
def node_label_for(host)
|
|
115
140
|
if host == primary_host
|
|
116
141
|
@primary['label']
|
|
@@ -125,23 +150,15 @@ module ActivePostgres
|
|
|
125
150
|
# Validate required secrets if components are enabled
|
|
126
151
|
raise Error, 'Missing replication_password secret' if component_enabled?(:repmgr) && !secrets_config['replication_password']
|
|
127
152
|
raise Error, 'Missing monitoring_password secret' if component_enabled?(:monitoring) && !secrets_config['monitoring_password']
|
|
153
|
+
|
|
128
154
|
if component_enabled?(:monitoring)
|
|
129
155
|
grafana_config = component_config(:monitoring)[:grafana] || {}
|
|
130
|
-
if grafana_config[:enabled] && !secrets_config['grafana_admin_password']
|
|
131
|
-
raise Error, 'Missing grafana_admin_password secret'
|
|
132
|
-
end
|
|
156
|
+
raise Error, 'Missing grafana_admin_password secret' if grafana_config[:enabled] && !secrets_config['grafana_admin_password']
|
|
133
157
|
if grafana_config[:enabled] && grafana_config[:host].to_s.strip.empty?
|
|
134
158
|
raise Error, 'monitoring.grafana.host is required when grafana is enabled'
|
|
135
159
|
end
|
|
136
160
|
end
|
|
137
|
-
|
|
138
|
-
pg_config = component_config(:pgbackrest)
|
|
139
|
-
retention_full = pg_config[:retention_full]
|
|
140
|
-
retention_archive = pg_config[:retention_archive]
|
|
141
|
-
if retention_full && retention_archive && retention_archive.to_i < retention_full.to_i
|
|
142
|
-
raise Error, 'pgbackrest.retention_archive must be >= retention_full for PITR safety'
|
|
143
|
-
end
|
|
144
|
-
end
|
|
161
|
+
validate_pgbackrest!
|
|
145
162
|
|
|
146
163
|
if component_enabled?(:pgbouncer)
|
|
147
164
|
pgbouncer_app_hosts.each do |app_host|
|
|
@@ -150,27 +167,38 @@ module ActivePostgres
|
|
|
150
167
|
end
|
|
151
168
|
end
|
|
152
169
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
170
|
+
all_hosts.each do |host|
|
|
171
|
+
jump_host_name = node_config_for(host)&.dig('jump_host')
|
|
172
|
+
next unless jump_host_name
|
|
173
|
+
|
|
174
|
+
jump_host = @jump_hosts[jump_host_name]
|
|
175
|
+
raise Error, "Host #{host} jump_host not found: #{jump_host_name}" unless jump_host
|
|
176
|
+
raise Error, "Jump host #{jump_host_name} must include host" if jump_host['host'].to_s.strip.empty?
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
validate_standby_seed_methods!
|
|
180
|
+
|
|
181
|
+
if component_enabled?(:repmgr)
|
|
182
|
+
dns_failover = component_config(:repmgr)[:dns_failover]
|
|
183
|
+
if dns_failover && dns_failover[:enabled]
|
|
184
|
+
domains = Array(dns_failover[:domains] || dns_failover[:domain]).map(&:to_s).map(&:strip).reject(&:empty?)
|
|
185
|
+
servers = Array(dns_failover[:dns_servers])
|
|
186
|
+
provider = (dns_failover[:provider] || 'dnsmasq').to_s.strip
|
|
187
|
+
|
|
188
|
+
raise Error, 'dns_failover.domain or dns_failover.domains is required when enabled' if domains.empty?
|
|
189
|
+
raise Error, 'dns_failover.dns_servers is required when enabled' if servers.empty?
|
|
190
|
+
raise Error, "Unsupported dns_failover provider '#{provider}'" unless provider == 'dnsmasq'
|
|
191
|
+
|
|
192
|
+
servers.each do |server|
|
|
193
|
+
next unless server.is_a?(Hash)
|
|
194
|
+
|
|
195
|
+
ssh_host = server['ssh_host'] || server[:ssh_host] || server['host'] || server[:host]
|
|
196
|
+
private_ip = server['private_ip'] || server[:private_ip] || server['ip'] || server[:ip]
|
|
197
|
+
raise Error, 'dns_failover.dns_servers entries must include host/ssh_host or private_ip' if
|
|
198
|
+
(ssh_host.nil? || ssh_host.to_s.strip.empty?) && (private_ip.nil? || private_ip.to_s.strip.empty?)
|
|
172
199
|
end
|
|
173
200
|
end
|
|
201
|
+
end
|
|
174
202
|
|
|
175
203
|
true
|
|
176
204
|
end
|
|
@@ -247,12 +275,6 @@ module ActivePostgres
|
|
|
247
275
|
result
|
|
248
276
|
end
|
|
249
277
|
|
|
250
|
-
def node_config_for(host)
|
|
251
|
-
return @primary if host == primary_host
|
|
252
|
-
|
|
253
|
-
standby_config_for(host)
|
|
254
|
-
end
|
|
255
|
-
|
|
256
278
|
def private_ip_for(node_config)
|
|
257
279
|
return unless node_config
|
|
258
280
|
|
|
@@ -296,5 +318,9 @@ module ActivePostgres
|
|
|
296
318
|
raise Error,
|
|
297
319
|
"Invalid ssh_host_key_verification '#{value}'. Use 'always' or 'accept_new'."
|
|
298
320
|
end
|
|
321
|
+
|
|
322
|
+
def optional_path(path)
|
|
323
|
+
File.expand_path(path) if path
|
|
324
|
+
end
|
|
299
325
|
end
|
|
300
326
|
end
|
|
@@ -13,12 +13,16 @@ module ActivePostgres
|
|
|
13
13
|
@skip_validation = skip_validation
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
-
def execute
|
|
16
|
+
def execute(dry_run: false)
|
|
17
17
|
ErrorHandler.with_handling(context: { operation: operation_name }) do
|
|
18
18
|
print_header
|
|
19
19
|
validate_prerequisites
|
|
20
20
|
run_preflight_checks unless skip_validation
|
|
21
21
|
print_deployment_plan
|
|
22
|
+
if dry_run
|
|
23
|
+
logger.info "\n[DRY RUN] No changes made."
|
|
24
|
+
return
|
|
25
|
+
end
|
|
22
26
|
return unless confirm_deployment
|
|
23
27
|
|
|
24
28
|
rollback_manager.with_rollback(description: operation_name) do
|
|
@@ -62,19 +62,13 @@ module ActivePostgres
|
|
|
62
62
|
return nil if value.nil?
|
|
63
63
|
|
|
64
64
|
# Handle Rails credentials
|
|
65
|
-
if value.start_with?('rails_credentials:')
|
|
66
|
-
return resolve_rails_credentials(value)
|
|
67
|
-
end
|
|
65
|
+
return resolve_rails_credentials(value) if value.start_with?('rails_credentials:')
|
|
68
66
|
|
|
69
67
|
# Handle environment variables
|
|
70
|
-
if value.start_with?('$')
|
|
71
|
-
return ENV[value[1..]]
|
|
72
|
-
end
|
|
68
|
+
return ENV.fetch(value[1..], nil) if value.start_with?('$')
|
|
73
69
|
|
|
74
70
|
# Handle shell commands
|
|
75
|
-
if value.start_with?('$(') && value.end_with?(')')
|
|
76
|
-
return `#{value[2..-2]}`.strip
|
|
77
|
-
end
|
|
71
|
+
return `#{value[2..-2]}`.strip if value.start_with?('$(') && value.end_with?(')')
|
|
78
72
|
|
|
79
73
|
value
|
|
80
74
|
end
|
|
@@ -127,24 +127,24 @@ module ActivePostgres
|
|
|
127
127
|
|
|
128
128
|
# Identify error type from error message
|
|
129
129
|
def identify_error_type(message)
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
130
|
+
normalized = message.to_s.downcase
|
|
131
|
+
|
|
132
|
+
return :ssh_connection if contains_any?(normalized, 'ssh', 'connection refused', 'network unreachable')
|
|
133
|
+
return :private_network_connectivity if contains_any?(normalized, 'private network', 'vpn', 'wireguard network')
|
|
134
|
+
return :postgresql_not_starting if normalized.include?('postgresql') && normalized.include?('not') && normalized.include?('start')
|
|
135
|
+
return :postgresql_not_starting if normalized.include?('cluster') && normalized.include?('not') && normalized.include?('running')
|
|
136
|
+
return :repmgr_clone_failed if normalized.include?('repmgr') && normalized.include?('clone')
|
|
137
|
+
return :repmgr_clone_failed if normalized.include?('data directory')
|
|
138
|
+
return :repmgr_register_failed if normalized.include?('repmgr') && normalized.include?('register')
|
|
139
|
+
return :repmgr_register_failed if normalized.include?('unable to connect to') && normalized.include?('primary')
|
|
140
|
+
return :ssl_certificate_error if contains_any?(normalized, 'ssl', 'certificate', 'tls')
|
|
141
|
+
return :disk_space_error if contains_any?(normalized, 'no space', 'disk full')
|
|
142
|
+
|
|
143
|
+
:authentication_failed if contains_any?(normalized, 'authentication', 'password', 'pg_hba')
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def contains_any?(value, *needles)
|
|
147
|
+
needles.any? { |needle| value.include?(needle) }
|
|
148
148
|
end
|
|
149
149
|
|
|
150
150
|
# Show troubleshooting guide for a specific error type
|
|
@@ -57,6 +57,13 @@ shared: &shared
|
|
|
57
57
|
|
|
58
58
|
pgbackrest:
|
|
59
59
|
enabled: false
|
|
60
|
+
# For large, frequently changed relation files, bundle repository objects
|
|
61
|
+
# and back up changed blocks instead of copying each whole file.
|
|
62
|
+
# repo_bundle: true
|
|
63
|
+
# repo_block: true
|
|
64
|
+
# schedule_full: "0 2 * * 0"
|
|
65
|
+
# schedule_differential: "35 2 * * 1-6"
|
|
66
|
+
# schedule_incremental: "5 */4 * * *"
|
|
60
67
|
|
|
61
68
|
monitoring:
|
|
62
69
|
enabled: false
|
|
@@ -12,14 +12,6 @@ module ActivePostgres
|
|
|
12
12
|
@executor
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
-
private def create_executor
|
|
16
|
-
if use_ssh_executor?
|
|
17
|
-
SSHExecutor.new(config, quiet: true)
|
|
18
|
-
else
|
|
19
|
-
DirectExecutor.new(config, quiet: true)
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
|
|
23
15
|
def use_ssh_executor?
|
|
24
16
|
mode = ENV['ACTIVE_POSTGRES_STATUS_MODE'].to_s.strip.downcase
|
|
25
17
|
return true if mode == 'ssh'
|
|
@@ -131,6 +123,14 @@ module ActivePostgres
|
|
|
131
123
|
|
|
132
124
|
private
|
|
133
125
|
|
|
126
|
+
def create_executor
|
|
127
|
+
if use_ssh_executor?
|
|
128
|
+
SSHExecutor.new(config, quiet: true)
|
|
129
|
+
else
|
|
130
|
+
DirectExecutor.new(config, quiet: true)
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
134
|
def collect_node_status
|
|
135
135
|
nodes = []
|
|
136
136
|
|
|
@@ -184,9 +184,7 @@ module ActivePostgres
|
|
|
184
184
|
lag: [3, nodes.map { |n| n[:lag].to_s.length }.max].max
|
|
185
185
|
}
|
|
186
186
|
|
|
187
|
-
if config.component_enabled?(:pgbouncer)
|
|
188
|
-
cols[:pgbouncer] = [9, nodes.map { |n| n[:pgbouncer].to_s.length }.max].max
|
|
189
|
-
end
|
|
187
|
+
cols[:pgbouncer] = [9, nodes.map { |n| n[:pgbouncer].to_s.length }.max].max if config.component_enabled?(:pgbouncer)
|
|
190
188
|
|
|
191
189
|
cols
|
|
192
190
|
end
|
|
@@ -194,7 +192,7 @@ module ActivePostgres
|
|
|
194
192
|
def print_table_header(cols)
|
|
195
193
|
fmt = "%-#{cols[:role]}s %-#{cols[:host]}s %-#{cols[:private_ip]}s " \
|
|
196
194
|
"%-#{cols[:label]}s %-#{cols[:status]}s %#{cols[:conn]}s %#{cols[:lag]}s"
|
|
197
|
-
headers =
|
|
195
|
+
headers = ['Role', 'Host', 'Private IP', 'Label', 'Status', 'Conn', 'Lag']
|
|
198
196
|
|
|
199
197
|
if cols[:pgbouncer]
|
|
200
198
|
fmt += " %-#{cols[:pgbouncer]}s"
|
|
@@ -28,7 +28,7 @@ module ActivePostgres
|
|
|
28
28
|
rollback_manager: rollback_manager,
|
|
29
29
|
skip_validation: skip_validation
|
|
30
30
|
)
|
|
31
|
-
flow.execute
|
|
31
|
+
flow.execute(dry_run: dry_run)
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
def setup_component(component_name)
|
|
@@ -117,7 +117,7 @@ module ActivePostgres
|
|
|
117
117
|
rollback_manager: rollback_manager,
|
|
118
118
|
skip_validation: skip_validation
|
|
119
119
|
)
|
|
120
|
-
flow.execute
|
|
120
|
+
flow.execute(dry_run: dry_run)
|
|
121
121
|
end
|
|
122
122
|
end
|
|
123
123
|
end
|
|
@@ -61,14 +61,14 @@ module ActivePostgres
|
|
|
61
61
|
label = config.node_label_for(host)
|
|
62
62
|
stats = fetch_system_stats(host, paths)
|
|
63
63
|
if stats.nil?
|
|
64
|
-
puts " #{host}#{
|
|
64
|
+
puts " #{host}#{" (#{label})" if label}: unavailable"
|
|
65
65
|
next
|
|
66
66
|
end
|
|
67
67
|
|
|
68
68
|
mem_used_kb = stats[:mem_total_kb] - stats[:mem_avail_kb]
|
|
69
69
|
mem_pct = stats[:mem_total_kb].positive? ? (mem_used_kb.to_f / stats[:mem_total_kb] * 100).round : 0
|
|
70
70
|
|
|
71
|
-
puts " #{host}#{
|
|
71
|
+
puts " #{host}#{" (#{label})" if label}: load #{stats[:loadavg]} | cpu #{stats[:cpu]}% | " \
|
|
72
72
|
"mem #{format_kb(mem_used_kb)}/#{format_kb(stats[:mem_total_kb])} (#{mem_pct}%)"
|
|
73
73
|
|
|
74
74
|
stats[:disks].each do |disk|
|
|
@@ -227,7 +227,7 @@ module ActivePostgres
|
|
|
227
227
|
with_timeout("system stats #{host}") do
|
|
228
228
|
ssh_executor = executor
|
|
229
229
|
safe_paths = paths.map { |p| Shellwords.escape(p) }.join(' ')
|
|
230
|
-
script = <<~
|
|
230
|
+
script = <<~BASH
|
|
231
231
|
set -e
|
|
232
232
|
loadavg=$(cut -d' ' -f1-3 /proc/loadavg)
|
|
233
233
|
|
|
@@ -256,7 +256,7 @@ module ActivePostgres
|
|
|
256
256
|
|
|
257
257
|
ssh_executor.execute_on_host(host) do
|
|
258
258
|
output = capture(:bash, '-lc', "#{script}\n df -kP #{safe_paths} 2>/dev/null | tail -n +2 | " \
|
|
259
|
-
|
|
259
|
+
"awk '{print \"disk=\" $6 \"|\" $2 \"|\" $3 \"|\" $4 \"|\" $5}'",
|
|
260
260
|
raise_on_non_zero_exit: false).to_s
|
|
261
261
|
end
|
|
262
262
|
end
|
|
@@ -299,12 +299,12 @@ module ActivePostgres
|
|
|
299
299
|
stats
|
|
300
300
|
end
|
|
301
301
|
|
|
302
|
-
def format_kb(
|
|
303
|
-
|
|
304
|
-
gb =
|
|
302
|
+
def format_kb(kilobytes)
|
|
303
|
+
kilobytes = kilobytes.to_f
|
|
304
|
+
gb = kilobytes / 1024 / 1024
|
|
305
305
|
return format('%.1fG', gb) if gb >= 1
|
|
306
306
|
|
|
307
|
-
mb =
|
|
307
|
+
mb = kilobytes / 1024
|
|
308
308
|
format('%.0fM', mb)
|
|
309
309
|
end
|
|
310
310
|
|
|
@@ -342,8 +342,8 @@ module ActivePostgres
|
|
|
342
342
|
value.positive? ? value : DEFAULT_TIMEOUT
|
|
343
343
|
end
|
|
344
344
|
|
|
345
|
-
def with_timeout(label)
|
|
346
|
-
Timeout.timeout(overview_timeout)
|
|
345
|
+
def with_timeout(label, &)
|
|
346
|
+
Timeout.timeout(overview_timeout, &)
|
|
347
347
|
rescue Timeout::Error
|
|
348
348
|
raise StandardError, "#{label} timed out after #{overview_timeout}s"
|
|
349
349
|
end
|
|
@@ -20,8 +20,8 @@ module ActivePostgres
|
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
def resolve_all
|
|
23
|
-
config.secrets_config.keys.
|
|
24
|
-
|
|
23
|
+
config.secrets_config.keys.to_h do |key|
|
|
24
|
+
[key, resolve(key)]
|
|
25
25
|
end
|
|
26
26
|
end
|
|
27
27
|
|
|
@@ -47,8 +47,9 @@ module ActivePostgres
|
|
|
47
47
|
|
|
48
48
|
resolve_all.each do |key, value|
|
|
49
49
|
file_path = File.join(directory, key)
|
|
50
|
-
File.
|
|
51
|
-
|
|
50
|
+
File.open(file_path, File::WRONLY | File::CREAT | File::TRUNC, 0o600) do |file|
|
|
51
|
+
file.write(value)
|
|
52
|
+
end
|
|
52
53
|
puts "Cached #{key} to #{file_path}"
|
|
53
54
|
end
|
|
54
55
|
|
|
@@ -101,9 +102,7 @@ module ActivePostgres
|
|
|
101
102
|
|
|
102
103
|
# Try loading just the Rails application (without full initialization)
|
|
103
104
|
# This makes Rails.application available for credential access
|
|
104
|
-
if File.exist?('./config/application.rb')
|
|
105
|
-
require './config/application'
|
|
106
|
-
end
|
|
105
|
+
require './config/application' if File.exist?('./config/application.rb')
|
|
107
106
|
rescue StandardError
|
|
108
107
|
# Rails boot failed — CLI running outside a Rails project
|
|
109
108
|
nil
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
require 'sshkit'
|
|
2
2
|
require 'sshkit/dsl'
|
|
3
|
+
require 'net/ssh/proxy/command'
|
|
3
4
|
require 'securerandom'
|
|
5
|
+
require 'shellwords'
|
|
4
6
|
require 'stringio'
|
|
5
7
|
|
|
6
8
|
module ActivePostgres
|
|
@@ -20,11 +22,11 @@ module ActivePostgres
|
|
|
20
22
|
end
|
|
21
23
|
|
|
22
24
|
def execute_on_host(host, &)
|
|
23
|
-
on(
|
|
25
|
+
on(ssh_host_for(host), &)
|
|
24
26
|
end
|
|
25
27
|
|
|
26
28
|
def execute_on_host_as(host, user, &)
|
|
27
|
-
on(
|
|
29
|
+
on(ssh_host_for(host, user:), &)
|
|
28
30
|
end
|
|
29
31
|
|
|
30
32
|
def execute_on_primary(&)
|
|
@@ -32,12 +34,12 @@ module ActivePostgres
|
|
|
32
34
|
end
|
|
33
35
|
|
|
34
36
|
def execute_on_standbys(&)
|
|
35
|
-
hosts = config.standby_hosts.map { |
|
|
37
|
+
hosts = config.standby_hosts.map { |host| ssh_host_for(host) }
|
|
36
38
|
on(hosts, in: :parallel, &)
|
|
37
39
|
end
|
|
38
40
|
|
|
39
41
|
def execute_on_all_hosts(&)
|
|
40
|
-
hosts = config.all_hosts.map { |
|
|
42
|
+
hosts = config.all_hosts.map { |host| ssh_host_for(host) }
|
|
41
43
|
on(hosts, in: :parallel, &)
|
|
42
44
|
end
|
|
43
45
|
|
|
@@ -215,11 +217,11 @@ module ActivePostgres
|
|
|
215
217
|
execute_on_host(host) do
|
|
216
218
|
backend = self
|
|
217
219
|
result = executor.run_sql_on_backend(backend, sql,
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
220
|
+
postgres_user: postgres_user,
|
|
221
|
+
port: port,
|
|
222
|
+
database: database,
|
|
223
|
+
tuples_only: tuples_only,
|
|
224
|
+
capture: capture)
|
|
223
225
|
end
|
|
224
226
|
result
|
|
225
227
|
end
|
|
@@ -307,6 +309,8 @@ module ActivePostgres
|
|
|
307
309
|
number_of_password_prompts: 0
|
|
308
310
|
}
|
|
309
311
|
|
|
312
|
+
options[:user_known_hosts_file] = [config.ssh_known_hosts_file] if config.ssh_known_hosts_file
|
|
313
|
+
|
|
310
314
|
if config.ssh_key && File.exist?(config.ssh_key)
|
|
311
315
|
options[:keys] = [config.ssh_key]
|
|
312
316
|
options[:keys_only] = true
|
|
@@ -315,5 +319,58 @@ module ActivePostgres
|
|
|
315
319
|
ssh.ssh_options = options
|
|
316
320
|
end
|
|
317
321
|
end
|
|
322
|
+
|
|
323
|
+
def ssh_host_for(host, user: config.user)
|
|
324
|
+
ssh_host = SSHKit::Host.new(host)
|
|
325
|
+
ssh_host.user = user
|
|
326
|
+
|
|
327
|
+
node_config = config.node_config_for(host) || {}
|
|
328
|
+
ssh_host.port = node_config['ssh_port'] || node_config['port'] if node_config['ssh_port'] || node_config['port']
|
|
329
|
+
|
|
330
|
+
options = {
|
|
331
|
+
verify_host_key: config.ssh_host_key_verification || :always
|
|
332
|
+
}
|
|
333
|
+
options[:user_known_hosts_file] = [config.ssh_known_hosts_file] if config.ssh_known_hosts_file
|
|
334
|
+
|
|
335
|
+
if node_config['ssh_key']
|
|
336
|
+
options[:keys] = [File.expand_path(node_config['ssh_key'])]
|
|
337
|
+
options[:keys_only] = true
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
if (jump_host_config = config.jump_host_config_for(host))
|
|
341
|
+
options[:proxy] = jump_proxy(jump_host_config)
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
ssh_host.ssh_options = options
|
|
345
|
+
ssh_host
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
def jump_proxy(jump_host_config)
|
|
349
|
+
jump_user = jump_host_config['ssh_user'] || jump_host_config['user'] || config.user
|
|
350
|
+
jump_port = jump_host_config['ssh_port'] || jump_host_config['port']
|
|
351
|
+
jump_key = jump_host_config['ssh_key'] || config.ssh_key
|
|
352
|
+
command = [
|
|
353
|
+
'ssh',
|
|
354
|
+
'-F', '/dev/null',
|
|
355
|
+
'-o', 'BatchMode=yes',
|
|
356
|
+
'-o', 'ForwardAgent=no',
|
|
357
|
+
'-o', "StrictHostKeyChecking=#{open_ssh_host_key_mode}"
|
|
358
|
+
]
|
|
359
|
+
|
|
360
|
+
command.push('-o', "UserKnownHostsFile=#{config.ssh_known_hosts_file}") if config.ssh_known_hosts_file
|
|
361
|
+
|
|
362
|
+
command.push('-i', File.expand_path(jump_key), '-o', 'IdentitiesOnly=yes') if
|
|
363
|
+
jump_key && File.exist?(File.expand_path(jump_key))
|
|
364
|
+
|
|
365
|
+
command.push('-p', jump_port.to_s) if jump_port
|
|
366
|
+
command.push('-W', '%h:%p', "#{jump_user}@#{jump_host_config.fetch('host')}")
|
|
367
|
+
|
|
368
|
+
command_line = Shellwords.join(command).gsub('\\%h', '%h').gsub('\\%p', '%p')
|
|
369
|
+
Net::SSH::Proxy::Command.new(command_line)
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
def open_ssh_host_key_mode
|
|
373
|
+
config.ssh_host_key_verification == :accept_new ? 'accept-new' : 'yes'
|
|
374
|
+
end
|
|
318
375
|
end
|
|
319
376
|
end
|
|
@@ -21,9 +21,9 @@ module ActivePostgres
|
|
|
21
21
|
def validate_specific_requirements
|
|
22
22
|
abort "❌ #{standby_host} is not configured as a standby in config/postgres.yml" unless config.standby_hosts.include?(standby_host)
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
abort '❌ repmgr component must be enabled to setup standbys' unless config.component_enabled?(:repmgr)
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
validate_required_secrets!
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
def run_preflight_checks
|
|
@@ -49,17 +49,22 @@ module ActivePostgres
|
|
|
49
49
|
|
|
50
50
|
def list_deployment_steps
|
|
51
51
|
logger.info " • Install PostgreSQL #{config.version} packages on #{standby_host}"
|
|
52
|
-
|
|
52
|
+
if config.standby_seed_method_for(standby_host) == :pgbackrest
|
|
53
|
+
logger.info ' • Restore the latest valid pgBackRest backup as a standby'
|
|
54
|
+
else
|
|
55
|
+
logger.info " • Clone data from primary: #{config.primary_host}"
|
|
56
|
+
end
|
|
53
57
|
logger.info ' • Register standby with repmgr cluster'
|
|
54
58
|
end
|
|
55
59
|
|
|
56
60
|
def list_warnings
|
|
57
|
-
logger.info "\n⚠️ Primary
|
|
61
|
+
logger.info "\n⚠️ Primary stays online; setup performs idempotent role and repmgr checks"
|
|
58
62
|
end
|
|
59
63
|
|
|
60
64
|
def deploy_components
|
|
61
65
|
deploy_ssl if config.component_enabled?(:ssl)
|
|
62
66
|
deploy_core
|
|
67
|
+
deploy_pgbackrest if config.component_enabled?(:pgbackrest)
|
|
63
68
|
deploy_repmgr
|
|
64
69
|
deploy_optional_components
|
|
65
70
|
end
|
|
@@ -79,8 +84,15 @@ module ActivePostgres
|
|
|
79
84
|
end
|
|
80
85
|
end
|
|
81
86
|
|
|
87
|
+
def deploy_pgbackrest
|
|
88
|
+
logger.task('Installing pgBackRest on standby') do
|
|
89
|
+
component = Components::PgBackRest.new(config, ssh_executor, secrets)
|
|
90
|
+
component.install_on_standby(standby_host)
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
82
94
|
def deploy_repmgr
|
|
83
|
-
logger.task('Setting up repmgr and
|
|
95
|
+
logger.task('Setting up repmgr and seeding standby') do
|
|
84
96
|
component = Components::Repmgr.new(config, ssh_executor, secrets)
|
|
85
97
|
register_rollback('repmgr', component)
|
|
86
98
|
component.setup_standby_only(standby_host)
|
|
@@ -107,13 +119,39 @@ module ActivePostgres
|
|
|
107
119
|
end
|
|
108
120
|
|
|
109
121
|
def register_rollback(component_name, component)
|
|
110
|
-
|
|
111
|
-
|
|
122
|
+
target_host = standby_host
|
|
123
|
+
captured_logger = logger
|
|
124
|
+
|
|
125
|
+
rollback_manager.register("Uninstall #{component_name} on #{target_host}") do
|
|
126
|
+
if component.respond_to?(:uninstall_from)
|
|
127
|
+
component.uninstall_from(target_host)
|
|
128
|
+
else
|
|
129
|
+
component.uninstall
|
|
130
|
+
end
|
|
112
131
|
rescue StandardError => e
|
|
113
|
-
|
|
132
|
+
captured_logger.warn "Failed to uninstall #{component_name} on #{target_host}: #{e.message}"
|
|
114
133
|
end
|
|
115
134
|
end
|
|
116
135
|
|
|
136
|
+
def validate_required_secrets!
|
|
137
|
+
required = %w[repmgr_password replication_password]
|
|
138
|
+
required.push('ssl_cert', 'ssl_key') if config.component_enabled?(:ssl)
|
|
139
|
+
required << 'monitoring_password' if config.component_enabled?(:monitoring)
|
|
140
|
+
|
|
141
|
+
if config.component_enabled?(:pgbackrest)
|
|
142
|
+
pgbackrest = secrets.resolve_value(config.component_config(:pgbackrest))
|
|
143
|
+
if pgbackrest[:repo_type] == 's3'
|
|
144
|
+
required.push('s3_access_key', 's3_secret_key')
|
|
145
|
+
raise Error, 'pgbackrest.s3_bucket did not resolve' if pgbackrest[:s3_bucket].to_s.empty?
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
missing = required.uniq.select { |key| secrets.resolve(key).to_s.empty? }
|
|
150
|
+
return if missing.empty?
|
|
151
|
+
|
|
152
|
+
raise Error, "Required standby secrets did not resolve: #{missing.join(', ')}"
|
|
153
|
+
end
|
|
154
|
+
|
|
117
155
|
def list_next_steps
|
|
118
156
|
logger.info ' 1. Check cluster status: active_postgres status'
|
|
119
157
|
logger.info ' 2. Verify replication: Check repmgr cluster show'
|