manageiq-appliance_console 6.0.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +24 -25
- data/.rspec_ci +2 -0
- data/.travis.yml +3 -2
- data/Gemfile +0 -2
- data/Rakefile +20 -1
- data/bin/appliance_console +30 -6
- data/lib/manageiq-appliance_console.rb +2 -6
- data/lib/manageiq/appliance_console/certificate_authority.rb +1 -1
- data/lib/manageiq/appliance_console/cli.rb +64 -20
- data/lib/manageiq/appliance_console/database_configuration.rb +2 -1
- data/lib/manageiq/appliance_console/database_replication.rb +1 -1
- data/lib/manageiq/appliance_console/database_replication_standby.rb +1 -1
- data/lib/manageiq/appliance_console/internal_database_configuration.rb +1 -1
- data/lib/manageiq/appliance_console/logfile_configuration.rb +2 -2
- data/lib/manageiq/appliance_console/message_configuration.rb +199 -0
- data/lib/manageiq/appliance_console/message_configuration_client.rb +96 -0
- data/lib/manageiq/appliance_console/message_configuration_server.rb +319 -0
- data/lib/manageiq/appliance_console/postgres_admin.rb +325 -0
- data/lib/manageiq/appliance_console/utilities.rb +45 -1
- data/lib/manageiq/appliance_console/version.rb +1 -1
- data/manageiq-appliance_console.gemspec +2 -2
- metadata +19 -16
- data/lib/manageiq/appliance_console/messaging_configuration.rb +0 -92
@@ -0,0 +1,96 @@
|
|
1
|
+
require "awesome_spawn"
|
2
|
+
require "fileutils"
|
3
|
+
require "linux_admin"
|
4
|
+
require 'net/scp'
|
5
|
+
require "manageiq/appliance_console/message_configuration"
|
6
|
+
|
7
|
+
module ManageIQ
|
8
|
+
module ApplianceConsole
|
9
|
+
class MessageClientConfiguration < MessageConfiguration
|
10
|
+
attr_reader :message_server_password, :message_server_username, :installed_files,
|
11
|
+
:message_truststore_path_src, :message_ca_cert_path_src
|
12
|
+
|
13
|
+
def initialize(options = {})
|
14
|
+
super(options)
|
15
|
+
|
16
|
+
@message_server_host = options[:message_server_host]
|
17
|
+
@message_server_username = options[:message_server_usernamed] || "root"
|
18
|
+
@message_server_password = options[:message_server_password]
|
19
|
+
|
20
|
+
@message_truststore_path_src = options[:message_truststore_path_src] || truststore_path
|
21
|
+
@message_ca_cert_path_src = options[:message_ca_cert_path_src] || ca_cert_path
|
22
|
+
|
23
|
+
@installed_files = [client_properties_path, messaging_yaml_path, truststore_path]
|
24
|
+
end
|
25
|
+
|
26
|
+
def configure
|
27
|
+
begin
|
28
|
+
MessageServerConfiguration.new.unconfigure if MessageServerConfiguration.configured?
|
29
|
+
configure_messaging_yaml # Set up the local message client in case EVM is actually running on this, Message Server
|
30
|
+
create_client_properties # Create the client.properties configuration fle
|
31
|
+
fetch_truststore_from_server # Fetch the Java Keystore from the Kafka Server
|
32
|
+
configure_messaging_type("kafka") # Settings.prototype.messaging_type = 'kafka'
|
33
|
+
restart_evmserverd
|
34
|
+
rescue AwesomeSpawn::CommandResultError => e
|
35
|
+
say(e.result.output)
|
36
|
+
say(e.result.error)
|
37
|
+
say("")
|
38
|
+
say("Failed to Configure the Message Client- #{e}")
|
39
|
+
return false
|
40
|
+
rescue => e
|
41
|
+
say("Failed to Configure the Message Client- #{e}")
|
42
|
+
return false
|
43
|
+
end
|
44
|
+
true
|
45
|
+
end
|
46
|
+
|
47
|
+
def ask_for_parameters
|
48
|
+
say("\nMessage Client Parameters:\n\n")
|
49
|
+
|
50
|
+
@message_server_host = ask_for_string("Message Server Hostname or IP address")
|
51
|
+
@message_server_port = ask_for_integer("Message Server Port number", (1..65_535), 9_093).to_i
|
52
|
+
@message_server_username = ask_for_string("Message Server Username", message_server_username)
|
53
|
+
@message_server_password = ask_for_password("Message Server Password")
|
54
|
+
@message_truststore_path_src = ask_for_string("Message Server Truststore Path", truststore_path)
|
55
|
+
@message_ca_cert_path_src = ask_for_string("Message Server CA Cert Path", ca_cert_path)
|
56
|
+
@message_keystore_username = ask_for_string("Message Keystore Username", message_keystore_username) if secure?
|
57
|
+
@message_keystore_password = ask_for_password("Message Keystore Password") if secure?
|
58
|
+
end
|
59
|
+
|
60
|
+
def show_parameters
|
61
|
+
say("\nMessage Client Configuration:\n")
|
62
|
+
say("Message Client Details:\n")
|
63
|
+
say(" Message Server Hostname: #{message_server_host}\n")
|
64
|
+
say(" Message Server Username: #{message_server_username}\n")
|
65
|
+
say(" Message Keystore Username: #{message_keystore_username}\n")
|
66
|
+
end
|
67
|
+
|
68
|
+
def fetch_truststore_from_server
|
69
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
70
|
+
|
71
|
+
fetch_from_server(message_truststore_path_src, truststore_path)
|
72
|
+
end
|
73
|
+
|
74
|
+
def fetch_ca_cert_from_server
|
75
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
76
|
+
|
77
|
+
fetch_from_server(message_ca_cert_path_src, ca_cert_path)
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
def fetch_from_server(src_file, dst_file)
|
83
|
+
return if file_found?(dst_file)
|
84
|
+
|
85
|
+
Net::SCP.start(message_server_host, message_server_username, :password => message_server_password) do |scp|
|
86
|
+
scp.download!(src_file, dst_file)
|
87
|
+
end
|
88
|
+
|
89
|
+
File.exist?(dst_file)
|
90
|
+
rescue => e
|
91
|
+
say("Failed to fetch #{src_file} from server: #{e.message}")
|
92
|
+
false
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,319 @@
|
|
1
|
+
require "awesome_spawn"
|
2
|
+
require "fileutils"
|
3
|
+
require "linux_admin"
|
4
|
+
require "manageiq/appliance_console/message_configuration"
|
5
|
+
|
6
|
+
module ManageIQ
|
7
|
+
module ApplianceConsole
|
8
|
+
class MessageServerConfiguration < MessageConfiguration
|
9
|
+
attr_reader :jaas_config_path,
|
10
|
+
:server_properties_path, :server_properties_sample_path,
|
11
|
+
:ca_cert_srl_path, :ca_key_path, :cert_file_path, :cert_signed_path,
|
12
|
+
:keystore_files, :installed_files, :message_persistent_disk
|
13
|
+
|
14
|
+
PERSISTENT_DIRECTORY = Pathname.new("/var/lib/kafka/persistent_data").freeze
|
15
|
+
PERSISTENT_NAME = "kafka_messages".freeze
|
16
|
+
|
17
|
+
def initialize(options = {})
|
18
|
+
super(options)
|
19
|
+
|
20
|
+
@message_server_host = options[:message_server_use_ipaddr] == true ? my_ipaddr : options[:message_server_host] || my_hostname
|
21
|
+
@message_persistent_disk = LinuxAdmin::Disk.new(:path => options[:message_persistent_disk]) unless options[:message_persistent_disk].nil?
|
22
|
+
|
23
|
+
@jaas_config_path = config_dir_path.join("kafka_server_jaas.conf")
|
24
|
+
@server_properties_path = config_dir_path.join("server.properties")
|
25
|
+
@server_properties_sample_path = sample_config_dir_path.join("server.properties")
|
26
|
+
|
27
|
+
@ca_cert_srl_path = keystore_dir_path.join("ca-cert.srl")
|
28
|
+
@ca_key_path = keystore_dir_path.join("ca-key")
|
29
|
+
@cert_file_path = keystore_dir_path.join("cert-file")
|
30
|
+
@cert_signed_path = keystore_dir_path.join("cert-signed")
|
31
|
+
|
32
|
+
@keystore_files = [ca_cert_path, ca_cert_srl_path, ca_key_path, cert_file_path, cert_signed_path, truststore_path, keystore_path]
|
33
|
+
@installed_files = [jaas_config_path, client_properties_path, server_properties_path, messaging_yaml_path, LOGS_DIR] + keystore_files
|
34
|
+
end
|
35
|
+
|
36
|
+
def configure
|
37
|
+
begin
|
38
|
+
configure_persistent_disk # Configure the persistent message store on a different disk
|
39
|
+
create_jaas_config # Create the message server jaas config file
|
40
|
+
create_client_properties # Create the client.properties config
|
41
|
+
create_logs_directory # Create the logs directory:
|
42
|
+
configure_firewall # Open the firewall for message port 9093
|
43
|
+
configure_keystore # Populate the Java Keystore
|
44
|
+
create_server_properties # Update the /opt/message/config/server.properties
|
45
|
+
configure_messaging_yaml # Set up the local message client in case EVM is actually running on this, Message Server
|
46
|
+
configure_messaging_type("kafka") # Settings.prototype.messaging_type = 'kafka'
|
47
|
+
restart_services
|
48
|
+
rescue AwesomeSpawn::CommandResultError => e
|
49
|
+
say(e.result.output)
|
50
|
+
say(e.result.error)
|
51
|
+
say("")
|
52
|
+
say("Failed to Configure the Message Server- #{e}")
|
53
|
+
return false
|
54
|
+
rescue => e
|
55
|
+
say("Failed to Configure the Message Server- #{e}")
|
56
|
+
return false
|
57
|
+
end
|
58
|
+
true
|
59
|
+
end
|
60
|
+
|
61
|
+
def restart_services
|
62
|
+
say("Starting zookeeper and configure it to start on reboots ...")
|
63
|
+
LinuxAdmin::Service.new("zookeeper").start.enable
|
64
|
+
|
65
|
+
say("Starting kafka and configure it to start on reboots ...")
|
66
|
+
LinuxAdmin::Service.new("kafka").start.enable
|
67
|
+
|
68
|
+
restart_evmserverd
|
69
|
+
end
|
70
|
+
|
71
|
+
def ask_for_parameters
|
72
|
+
say("\nMessage Server Parameters:\n\n")
|
73
|
+
|
74
|
+
@message_server_host = ask_for_string("Message Server Hostname or IP address", message_server_host)
|
75
|
+
@message_keystore_username = ask_for_string("Message Keystore Username", message_keystore_username)
|
76
|
+
@message_keystore_password = ask_for_password("Message Keystore Password")
|
77
|
+
@message_persistent_disk = ask_for_persistent_disk
|
78
|
+
end
|
79
|
+
|
80
|
+
def ask_for_persistent_disk
|
81
|
+
choose_disk if use_new_disk
|
82
|
+
end
|
83
|
+
|
84
|
+
def use_new_disk
|
85
|
+
agree("Configure a new persistent disk volume? (Y/N): ")
|
86
|
+
end
|
87
|
+
|
88
|
+
def choose_disk
|
89
|
+
ask_for_disk("Persistent disk")
|
90
|
+
end
|
91
|
+
|
92
|
+
def show_parameters
|
93
|
+
say("\nMessage Server Configuration:\n")
|
94
|
+
say("Message Server Details:\n")
|
95
|
+
say(" Message Server Hostname: #{message_server_host}\n")
|
96
|
+
say(" Message Keystore Username: #{message_keystore_username}\n")
|
97
|
+
say(" Persistent message disk: #{message_persistent_disk.path}\n") if message_persistent_disk
|
98
|
+
end
|
99
|
+
|
100
|
+
def unconfigure
|
101
|
+
super
|
102
|
+
|
103
|
+
unconfigure_firewall
|
104
|
+
deactivate_services
|
105
|
+
end
|
106
|
+
|
107
|
+
def self.configured?
|
108
|
+
LinuxAdmin::Service.new("kafka").running? ||
|
109
|
+
LinuxAdmin::Service.new("zookeeper").running?
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
|
114
|
+
def my_ipaddr
|
115
|
+
LinuxAdmin::IpAddress.new.address
|
116
|
+
end
|
117
|
+
|
118
|
+
def my_hostname
|
119
|
+
LinuxAdmin::Hosts.new.hostname
|
120
|
+
end
|
121
|
+
|
122
|
+
def configure_persistent_disk
|
123
|
+
return true unless message_persistent_disk
|
124
|
+
|
125
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
126
|
+
|
127
|
+
deactivate_services # Just in case they are running.
|
128
|
+
|
129
|
+
FileUtils.mkdir_p(PERSISTENT_DIRECTORY)
|
130
|
+
LogicalVolumeManagement.new(:disk => message_persistent_disk, :mount_point => PERSISTENT_DIRECTORY, :name => PERSISTENT_NAME).setup
|
131
|
+
FileUtils.chmod(0o755, PERSISTENT_DIRECTORY)
|
132
|
+
FileUtils.chown("kafka", "kafka", PERSISTENT_DIRECTORY)
|
133
|
+
|
134
|
+
true
|
135
|
+
end
|
136
|
+
|
137
|
+
def activate_new_persistent_disk
|
138
|
+
return true unless message_persistent_disk
|
139
|
+
|
140
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
141
|
+
|
142
|
+
data = File.read(server_properties_path)
|
143
|
+
data.gsub!(/^log.dirs=.*$/, "log.dirs=#{PERSISTENT_DIRECTORY}")
|
144
|
+
File.write(server_properties_path, data)
|
145
|
+
|
146
|
+
true
|
147
|
+
end
|
148
|
+
|
149
|
+
def create_jaas_config
|
150
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
151
|
+
|
152
|
+
content = <<~JAAS
|
153
|
+
KafkaServer {
|
154
|
+
org.apache.kafka.common.security.plain.PlainLoginModule required
|
155
|
+
username=#{message_keystore_username}
|
156
|
+
password=#{message_keystore_password}
|
157
|
+
user_admin=#{message_keystore_password} ;
|
158
|
+
};
|
159
|
+
JAAS
|
160
|
+
|
161
|
+
File.write(jaas_config_path, content) unless file_found?(jaas_config_path)
|
162
|
+
end
|
163
|
+
|
164
|
+
def create_logs_directory
|
165
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
166
|
+
|
167
|
+
return if file_found?(LOGS_DIR)
|
168
|
+
|
169
|
+
FileUtils.mkdir_p(LOGS_DIR)
|
170
|
+
FileUtils.chmod(0o755, LOGS_DIR)
|
171
|
+
FileUtils.chown("kafka", "kafka", LOGS_DIR)
|
172
|
+
end
|
173
|
+
|
174
|
+
def configure_firewall
|
175
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
176
|
+
|
177
|
+
modify_firewall(:add_port)
|
178
|
+
end
|
179
|
+
|
180
|
+
def configure_keystore
|
181
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
182
|
+
|
183
|
+
return if files_found?(keystore_files)
|
184
|
+
|
185
|
+
keystore_params = assemble_keystore_params
|
186
|
+
|
187
|
+
# Generte a Java keystore and key pair, creating keystore.jks
|
188
|
+
# :stdin_data provides the -storepass twice to confirm and an extra CR to accept the same password for -keypass
|
189
|
+
AwesomeSpawn.run!("keytool", :params => keystore_params, :stdin_data => "#{message_keystore_password}\n#{message_keystore_password}\n\n")
|
190
|
+
|
191
|
+
# Use openssl to create a new CA cert, creating ca-cert and ca-key
|
192
|
+
AwesomeSpawn.run!("openssl", :env => {"PASSWORD" => message_keystore_password},
|
193
|
+
:params => ["req", "-new", "-x509", {"-keyout" => ca_key_path,
|
194
|
+
"-out" => ca_cert_path,
|
195
|
+
"-days" => 10_000,
|
196
|
+
"-passout" => "env:PASSWORD",
|
197
|
+
"-subj" => '/CN=something'}])
|
198
|
+
|
199
|
+
# Import the CA cert into the trust store, creating truststore.jks
|
200
|
+
# :stdin_data provides the -storepass argument and yes to confirm
|
201
|
+
AwesomeSpawn.run!("keytool", :params => {"-keystore" => truststore_path,
|
202
|
+
"-alias" => "CARoot",
|
203
|
+
"-import" => nil,
|
204
|
+
"-file" => ca_cert_path},
|
205
|
+
:stdin_data => "#{message_keystore_password}\n#{message_keystore_password}\nyes\n")
|
206
|
+
|
207
|
+
# Generate a certificate signing request (CSR) for an existing Java keystore, creating cert-file
|
208
|
+
# :stdin_data provides the -storepass argument
|
209
|
+
AwesomeSpawn.run!("keytool", :params => {"-keystore" => keystore_path,
|
210
|
+
"-alias" => keystore_params["-alias"],
|
211
|
+
"-certreq" => nil,
|
212
|
+
"-file" => cert_file_path},
|
213
|
+
:stdin_data => "#{message_keystore_password}\n")
|
214
|
+
|
215
|
+
# Use openssl to sign the certificate with the "CA" certificate, creating ca-cert.srl and cert-signed
|
216
|
+
AwesomeSpawn.run!("openssl", :env => {"PASSWORD" => message_keystore_password},
|
217
|
+
:params => ["x509", "-req", {"-CA" => ca_cert_path,
|
218
|
+
"-CAkey" => ca_key_path,
|
219
|
+
"-in" => cert_file_path,
|
220
|
+
"-out" => cert_signed_path,
|
221
|
+
"-days" => 10_000,
|
222
|
+
"-CAcreateserial" => nil,
|
223
|
+
"-passin" => "env:PASSWORD"}])
|
224
|
+
|
225
|
+
# Import a root or intermediate CA certificate to an existing Java keystore, updating keystore.jks
|
226
|
+
# :stdin_data provides the -storepass argument and yes to confirm
|
227
|
+
AwesomeSpawn.run!("keytool", :params => {"-keystore" => keystore_path,
|
228
|
+
"-alias" => "CARoot",
|
229
|
+
"-import" => nil,
|
230
|
+
"-file" => ca_cert_path},
|
231
|
+
:stdin_data => "#{message_keystore_password}\nyes\n")
|
232
|
+
|
233
|
+
# Import a signed primary certificate to an existing Java keystore, updating keystore.jks
|
234
|
+
# :stdin_data provides the -storepass argument
|
235
|
+
AwesomeSpawn.run!("keytool", :params => {"-keystore" => keystore_path,
|
236
|
+
"-alias" => keystore_params["-alias"],
|
237
|
+
"-import" => nil,
|
238
|
+
"-file" => cert_signed_path},
|
239
|
+
:stdin_data => "#{message_keystore_password}\n")
|
240
|
+
end
|
241
|
+
|
242
|
+
def create_server_properties
|
243
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
244
|
+
|
245
|
+
if message_server_host.ipaddress?
|
246
|
+
ident_algorithm = ""
|
247
|
+
client_auth = "none"
|
248
|
+
else
|
249
|
+
ident_algorithm = "HTTPS"
|
250
|
+
client_auth = "required"
|
251
|
+
end
|
252
|
+
|
253
|
+
content = <<~SERVER_PROPERTIES
|
254
|
+
|
255
|
+
listeners=SASL_SSL://:#{message_server_port}
|
256
|
+
|
257
|
+
ssl.endpoint.identification.algorithm=#{ident_algorithm}
|
258
|
+
ssl.keystore.location=#{keystore_path}
|
259
|
+
ssl.keystore.password=#{message_keystore_password}
|
260
|
+
ssl.key.password=#{message_keystore_password}
|
261
|
+
|
262
|
+
ssl.truststore.location=#{truststore_path}
|
263
|
+
ssl.truststore.password=#{message_keystore_password}
|
264
|
+
|
265
|
+
ssl.client.auth=#{client_auth}
|
266
|
+
|
267
|
+
sasl.enabled.mechanisms=PLAIN
|
268
|
+
sasl.mechanism.inter.broker.protocol=PLAIN
|
269
|
+
|
270
|
+
security.inter.broker.protocol=SASL_SSL
|
271
|
+
SERVER_PROPERTIES
|
272
|
+
|
273
|
+
return if file_contains?(server_properties_path, content)
|
274
|
+
|
275
|
+
FileUtils.cp(server_properties_sample_path, server_properties_path)
|
276
|
+
File.write(server_properties_path, content, :mode => "a")
|
277
|
+
|
278
|
+
activate_new_persistent_disk
|
279
|
+
end
|
280
|
+
|
281
|
+
def unconfigure_firewall
|
282
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
283
|
+
|
284
|
+
modify_firewall(:remove_port)
|
285
|
+
end
|
286
|
+
|
287
|
+
def deactivate_services
|
288
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
289
|
+
|
290
|
+
LinuxAdmin::Service.new("zookeeper").stop
|
291
|
+
LinuxAdmin::Service.new("kafka").stop
|
292
|
+
end
|
293
|
+
|
294
|
+
def assemble_keystore_params
|
295
|
+
keystore_params = {"-keystore" => keystore_path,
|
296
|
+
"-validity" => 10_000,
|
297
|
+
"-genkey" => nil,
|
298
|
+
"-keyalg" => "RSA"}
|
299
|
+
|
300
|
+
if message_server_host.ipaddress?
|
301
|
+
keystore_params["-alias"] = "localhost"
|
302
|
+
keystore_params["-ext"] = "san=ip:#{message_server_host}"
|
303
|
+
else
|
304
|
+
keystore_params["-alias"] = message_server_host
|
305
|
+
keystore_params["-ext"] = "san=dns:#{message_server_host}"
|
306
|
+
end
|
307
|
+
|
308
|
+
keystore_params["-dname"] = "cn=#{keystore_params["-alias"]}"
|
309
|
+
|
310
|
+
keystore_params
|
311
|
+
end
|
312
|
+
|
313
|
+
def modify_firewall(action)
|
314
|
+
AwesomeSpawn.run!("firewall-cmd", :params => {action => "#{message_server_port}/tcp", :permanent => nil})
|
315
|
+
AwesomeSpawn.run!("firewall-cmd --reload")
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
end
|
@@ -0,0 +1,325 @@
|
|
1
|
+
require 'awesome_spawn'
|
2
|
+
require 'pathname'
|
3
|
+
require 'linux_admin'
|
4
|
+
|
5
|
+
module ManageIQ
|
6
|
+
module ApplianceConsole
|
7
|
+
class PostgresAdmin
|
8
|
+
def self.data_directory
|
9
|
+
Pathname.new(ENV.fetch("APPLIANCE_PG_DATA"))
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.mount_point
|
13
|
+
Pathname.new(ENV.fetch("APPLIANCE_PG_MOUNT_POINT"))
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.template_directory
|
17
|
+
Pathname.new(ENV.fetch("APPLIANCE_TEMPLATE_DIRECTORY"))
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.service_name
|
21
|
+
ENV.fetch("APPLIANCE_PG_SERVICE")
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.package_name
|
25
|
+
ENV.fetch('APPLIANCE_PG_PACKAGE_NAME')
|
26
|
+
end
|
27
|
+
|
28
|
+
# Unprivileged user to run postgresql
|
29
|
+
def self.user
|
30
|
+
"postgres".freeze
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.group
|
34
|
+
user
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.logical_volume_name
|
38
|
+
"lv_pg".freeze
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.volume_group_name
|
42
|
+
"vg_data".freeze
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.database_disk_filesystem
|
46
|
+
"xfs".freeze
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.initialized?
|
50
|
+
!Dir[data_directory.join("*")].empty?
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.service_running?
|
54
|
+
LinuxAdmin::Service.new(service_name).running?
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.local_server_in_recovery?
|
58
|
+
data_directory.join("recovery.conf").exist?
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.local_server_status
|
62
|
+
if service_running?
|
63
|
+
"running (#{local_server_in_recovery? ? "standby" : "primary"})"
|
64
|
+
elsif initialized?
|
65
|
+
"initialized and stopped"
|
66
|
+
else
|
67
|
+
"not initialized"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.logical_volume_path
|
72
|
+
Pathname.new("/dev").join(volume_group_name, logical_volume_name)
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.database_size(opts)
|
76
|
+
result = run_command("psql", opts, :command => "SELECT pg_database_size('#{opts[:dbname]}');")
|
77
|
+
result.match(/^\s+([0-9]+)\n/)[1].to_i
|
78
|
+
end
|
79
|
+
|
80
|
+
def self.prep_data_directory
|
81
|
+
# initdb will fail if the database directory is not empty or not owned by the PostgresAdmin.user
|
82
|
+
FileUtils.mkdir(PostgresAdmin.data_directory) unless Dir.exist?(PostgresAdmin.data_directory)
|
83
|
+
FileUtils.chown_R(PostgresAdmin.user, PostgresAdmin.group, PostgresAdmin.data_directory)
|
84
|
+
FileUtils.rm_rf(PostgresAdmin.data_directory.children.map(&:to_s))
|
85
|
+
end
|
86
|
+
|
87
|
+
PG_DUMP_MAGIC = "PGDMP".force_encoding(Encoding::BINARY).freeze
|
88
|
+
def self.pg_dump_file?(file)
|
89
|
+
File.open(file, "rb") { |f| f.readpartial(5) } == PG_DUMP_MAGIC
|
90
|
+
end
|
91
|
+
|
92
|
+
BASE_BACKUP_MAGIC = "\037\213".force_encoding(Encoding::BINARY).freeze # just the first 2 bits of gzip magic
|
93
|
+
def self.base_backup_file?(file)
|
94
|
+
File.open(file, "rb") { |f| f.readpartial(2) } == BASE_BACKUP_MAGIC
|
95
|
+
end
|
96
|
+
|
97
|
+
def self.backup(opts)
|
98
|
+
backup_pg_compress(opts)
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.restore(opts)
|
102
|
+
file = opts[:local_file]
|
103
|
+
backup_type = opts.delete(:backup_type)
|
104
|
+
|
105
|
+
case
|
106
|
+
when backup_type == :pgdump then restore_pg_dump(opts)
|
107
|
+
when backup_type == :basebackup then restore_pg_basebackup(file)
|
108
|
+
when pg_dump_file?(file) then restore_pg_dump(opts)
|
109
|
+
when base_backup_file?(file) then restore_pg_basebackup(file)
|
110
|
+
else
|
111
|
+
raise "#{file} is not a database backup"
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def self.restore_pg_basebackup(file)
|
116
|
+
pg_service = LinuxAdmin::Service.new(service_name)
|
117
|
+
|
118
|
+
pg_service.stop
|
119
|
+
prep_data_directory
|
120
|
+
|
121
|
+
require 'rubygems/package'
|
122
|
+
|
123
|
+
# Using a Gem::Package instance for the #extract_tar_gz method, so we don't
|
124
|
+
# have to re-write all of that logic. Mostly making use of
|
125
|
+
# `Gem::Package::TarReader` + `Zlib::GzipReader` that is already part of
|
126
|
+
# rubygems/stdlib and integrated there.
|
127
|
+
unpacker = Gem::Package.new("obviously_not_a_gem")
|
128
|
+
File.open(file, IO::RDONLY | IO::NONBLOCK) do |backup_file|
|
129
|
+
unpacker.extract_tar_gz(backup_file, data_directory.to_s)
|
130
|
+
end
|
131
|
+
|
132
|
+
FileUtils.chown_R(PostgresAdmin.user, PostgresAdmin.group, PostgresAdmin.data_directory)
|
133
|
+
|
134
|
+
pg_service.start
|
135
|
+
file
|
136
|
+
end
|
137
|
+
|
138
|
+
def self.backup_pg_dump(opts)
|
139
|
+
opts = opts.dup
|
140
|
+
dbname = opts.delete(:dbname)
|
141
|
+
|
142
|
+
args = combine_command_args(opts, :format => "c", :file => opts[:local_file], nil => dbname)
|
143
|
+
args = handle_multi_value_pg_dump_args!(opts, args)
|
144
|
+
|
145
|
+
run_command_with_logging("pg_dump", opts, args)
|
146
|
+
opts[:local_file]
|
147
|
+
end
|
148
|
+
|
149
|
+
def self.backup_pg_compress(opts)
|
150
|
+
opts = opts.dup
|
151
|
+
|
152
|
+
# discard dbname as pg_basebackup does not connect to a specific database
|
153
|
+
opts.delete(:dbname)
|
154
|
+
|
155
|
+
path = Pathname.new(opts.delete(:local_file))
|
156
|
+
FileUtils.mkdir_p(path.dirname)
|
157
|
+
|
158
|
+
# Build commandline from AwesomeSpawn
|
159
|
+
args = {:z => nil, :format => "t", :wal_method => "fetch", :pgdata => "-"}
|
160
|
+
cmd = AwesomeSpawn.build_command_line("pg_basebackup", combine_command_args(opts, args))
|
161
|
+
logger.info("MIQ(#{name}.#{__method__}) Running command... #{cmd}")
|
162
|
+
|
163
|
+
# Run command in a separate thread
|
164
|
+
read, write = IO.pipe
|
165
|
+
error_path = Dir::Tmpname.create("") { |tmpname| tmpname }
|
166
|
+
process_thread = Process.detach(Kernel.spawn(pg_env(opts), cmd, :out => write, :err => error_path))
|
167
|
+
stream_reader = Thread.new { IO.copy_stream(read, path) } # Copy output to path
|
168
|
+
write.close
|
169
|
+
|
170
|
+
# Wait for them to finish
|
171
|
+
process_status = process_thread.value
|
172
|
+
stream_reader.join
|
173
|
+
read.close
|
174
|
+
|
175
|
+
handle_error(cmd, process_status.exitstatus, error_path)
|
176
|
+
path.to_s
|
177
|
+
end
|
178
|
+
|
179
|
+
def self.recreate_db(opts)
|
180
|
+
dbname = opts[:dbname]
|
181
|
+
opts = opts.merge(:dbname => 'postgres')
|
182
|
+
run_command("psql", opts, :command => "DROP DATABASE IF EXISTS #{dbname}")
|
183
|
+
run_command("psql", opts, :command => "CREATE DATABASE #{dbname} WITH OWNER = #{opts[:username] || 'root'} ENCODING = 'UTF8'")
|
184
|
+
end
|
185
|
+
|
186
|
+
def self.restore_pg_dump(opts)
|
187
|
+
recreate_db(opts)
|
188
|
+
args = { :verbose => nil, :exit_on_error => nil }
|
189
|
+
|
190
|
+
if File.pipe?(opts[:local_file])
|
191
|
+
cmd_args = combine_command_args(opts, args)
|
192
|
+
cmd = AwesomeSpawn.build_command_line("pg_restore", cmd_args)
|
193
|
+
error_path = Dir::Tmpname.create("") { |tmpname| tmpname }
|
194
|
+
spawn_args = { :err => error_path, :in => [opts[:local_file].to_s, "rb"] }
|
195
|
+
|
196
|
+
logger.info("MIQ(#{name}.#{__method__}) Running command... #{cmd}")
|
197
|
+
process_thread = Process.detach(Kernel.spawn(pg_env(opts), cmd, spawn_args))
|
198
|
+
process_status = process_thread.value
|
199
|
+
|
200
|
+
handle_error(cmd, process_status.exitstatus, error_path)
|
201
|
+
else
|
202
|
+
args[nil] = opts[:local_file]
|
203
|
+
run_command("pg_restore", opts, args)
|
204
|
+
end
|
205
|
+
opts[:local_file]
|
206
|
+
end
|
207
|
+
|
208
|
+
GC_DEFAULTS = {
|
209
|
+
:analyze => false,
|
210
|
+
:full => false,
|
211
|
+
:verbose => false,
|
212
|
+
:table => nil,
|
213
|
+
:dbname => nil,
|
214
|
+
:username => nil,
|
215
|
+
:reindex => false
|
216
|
+
}
|
217
|
+
|
218
|
+
GC_AGGRESSIVE_DEFAULTS = {
|
219
|
+
:analyze => true,
|
220
|
+
:full => true,
|
221
|
+
:verbose => false,
|
222
|
+
:table => nil,
|
223
|
+
:dbname => nil,
|
224
|
+
:username => nil,
|
225
|
+
:reindex => true
|
226
|
+
}
|
227
|
+
|
228
|
+
def self.gc(options = {})
|
229
|
+
options = (options[:aggressive] ? GC_AGGRESSIVE_DEFAULTS : GC_DEFAULTS).merge(options)
|
230
|
+
|
231
|
+
result = vacuum(options)
|
232
|
+
logger.info("MIQ(#{name}.#{__method__}) Output... #{result}") if result.to_s.length > 0
|
233
|
+
|
234
|
+
if options[:reindex]
|
235
|
+
result = reindex(options)
|
236
|
+
logger.info("MIQ(#{name}.#{__method__}) Output... #{result}") if result.to_s.length > 0
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def self.vacuum(opts)
|
241
|
+
# TODO: Add a real exception here
|
242
|
+
raise "Vacuum requires database" unless opts[:dbname]
|
243
|
+
|
244
|
+
args = {}
|
245
|
+
args[:analyze] = nil if opts[:analyze]
|
246
|
+
args[:full] = nil if opts[:full]
|
247
|
+
args[:verbose] = nil if opts[:verbose]
|
248
|
+
args[:table] = opts[:table] if opts[:table]
|
249
|
+
run_command("vacuumdb", opts, args)
|
250
|
+
end
|
251
|
+
|
252
|
+
def self.reindex(opts)
|
253
|
+
args = {}
|
254
|
+
args[:table] = opts[:table] if opts[:table]
|
255
|
+
run_command("reindexdb", opts, args)
|
256
|
+
end
|
257
|
+
|
258
|
+
def self.run_command(cmd_str, opts, args)
|
259
|
+
run_command_with_logging(cmd_str, opts, combine_command_args(opts, args))
|
260
|
+
end
|
261
|
+
|
262
|
+
def self.run_command_with_logging(cmd_str, opts, params = {})
|
263
|
+
logger.info("MIQ(#{name}.#{__method__}) Running command... #{AwesomeSpawn.build_command_line(cmd_str, params)}")
|
264
|
+
AwesomeSpawn.run!(cmd_str, :params => params, :env => pg_env(opts)).output
|
265
|
+
end
|
266
|
+
|
267
|
+
class << self
|
268
|
+
# Temporary alias due to manageiq core stubbing this method
|
269
|
+
alias runcmd_with_logging run_command_with_logging
|
270
|
+
end
|
271
|
+
|
272
|
+
private_class_method def self.combine_command_args(opts, args)
|
273
|
+
default_args = {:no_password => nil}
|
274
|
+
default_args[:dbname] = opts[:dbname] if opts[:dbname]
|
275
|
+
default_args[:username] = opts[:username] if opts[:username]
|
276
|
+
default_args[:host] = opts[:hostname] if opts[:hostname]
|
277
|
+
default_args[:port] = opts[:port] if opts[:port]
|
278
|
+
default_args.merge(args)
|
279
|
+
end
|
280
|
+
|
281
|
+
private_class_method def self.logger
|
282
|
+
ManageIQ::ApplianceConsole.logger
|
283
|
+
end
|
284
|
+
|
285
|
+
private_class_method def self.pg_env(opts)
|
286
|
+
{
|
287
|
+
"PGUSER" => opts[:username],
|
288
|
+
"PGPASSWORD" => opts[:password]
|
289
|
+
}.delete_blanks
|
290
|
+
end
|
291
|
+
# rubocop:disable Style/SymbolArray
|
292
|
+
PG_DUMP_MULTI_VALUE_ARGS = [
|
293
|
+
:t, :table, :T, :exclude_table, :"exclude-table", :exclude_table_data, :"exclude-table-data",
|
294
|
+
:n, :schema, :N, :exclude_schema, :"exclude-schema"
|
295
|
+
].freeze
|
296
|
+
# rubocop:enable Style/SymbolArray
|
297
|
+
#
|
298
|
+
# NOTE: Potentially mutates opts hash (args becomes new array and not
|
299
|
+
# mutated by this method)
|
300
|
+
private_class_method def self.handle_multi_value_pg_dump_args!(opts, args)
|
301
|
+
if opts.keys.any? { |key| PG_DUMP_MULTI_VALUE_ARGS.include?(key) }
|
302
|
+
args = args.to_a
|
303
|
+
PG_DUMP_MULTI_VALUE_ARGS.each do |table_key|
|
304
|
+
next unless opts.key?(table_key)
|
305
|
+
table_val = opts.delete(table_key)
|
306
|
+
args += Array.wrap(table_val).map! { |v| [table_key, v] }
|
307
|
+
end
|
308
|
+
end
|
309
|
+
args
|
310
|
+
end
|
311
|
+
|
312
|
+
private_class_method def self.handle_error(cmd, exit_status, error_path)
|
313
|
+
if exit_status != 0
|
314
|
+
result = AwesomeSpawn::CommandResult.new(cmd, "", File.read(error_path), exit_status)
|
315
|
+
message = AwesomeSpawn::CommandResultError.default_message(cmd, exit_status)
|
316
|
+
logger.error("AwesomeSpawn: #{message}")
|
317
|
+
logger.error("AwesomeSpawn: #{result.error}")
|
318
|
+
raise AwesomeSpawn::CommandResultError.new(message, result)
|
319
|
+
end
|
320
|
+
ensure
|
321
|
+
File.delete(error_path) if File.exist?(error_path)
|
322
|
+
end
|
323
|
+
end
|
324
|
+
end
|
325
|
+
end
|