manageiq-appliance_console 6.0.0 → 6.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.codeclimate.yml +24 -25
- data/.rspec_ci +2 -0
- data/.travis.yml +3 -2
- data/Gemfile +0 -2
- data/Rakefile +20 -1
- data/bin/appliance_console +30 -6
- data/lib/manageiq-appliance_console.rb +2 -6
- data/lib/manageiq/appliance_console/certificate_authority.rb +1 -1
- data/lib/manageiq/appliance_console/cli.rb +64 -20
- data/lib/manageiq/appliance_console/database_configuration.rb +2 -1
- data/lib/manageiq/appliance_console/database_replication.rb +1 -1
- data/lib/manageiq/appliance_console/database_replication_standby.rb +1 -1
- data/lib/manageiq/appliance_console/internal_database_configuration.rb +1 -1
- data/lib/manageiq/appliance_console/logfile_configuration.rb +2 -2
- data/lib/manageiq/appliance_console/message_configuration.rb +199 -0
- data/lib/manageiq/appliance_console/message_configuration_client.rb +96 -0
- data/lib/manageiq/appliance_console/message_configuration_server.rb +319 -0
- data/lib/manageiq/appliance_console/postgres_admin.rb +325 -0
- data/lib/manageiq/appliance_console/utilities.rb +45 -1
- data/lib/manageiq/appliance_console/version.rb +1 -1
- data/manageiq-appliance_console.gemspec +2 -2
- metadata +19 -16
- data/lib/manageiq/appliance_console/messaging_configuration.rb +0 -92
@@ -0,0 +1,96 @@
|
|
1
|
+
require "awesome_spawn"
|
2
|
+
require "fileutils"
|
3
|
+
require "linux_admin"
|
4
|
+
require 'net/scp'
|
5
|
+
require "manageiq/appliance_console/message_configuration"
|
6
|
+
|
7
|
+
module ManageIQ
|
8
|
+
module ApplianceConsole
|
9
|
+
class MessageClientConfiguration < MessageConfiguration
|
10
|
+
attr_reader :message_server_password, :message_server_username, :installed_files,
|
11
|
+
:message_truststore_path_src, :message_ca_cert_path_src
|
12
|
+
|
13
|
+
def initialize(options = {})
|
14
|
+
super(options)
|
15
|
+
|
16
|
+
@message_server_host = options[:message_server_host]
|
17
|
+
@message_server_username = options[:message_server_usernamed] || "root"
|
18
|
+
@message_server_password = options[:message_server_password]
|
19
|
+
|
20
|
+
@message_truststore_path_src = options[:message_truststore_path_src] || truststore_path
|
21
|
+
@message_ca_cert_path_src = options[:message_ca_cert_path_src] || ca_cert_path
|
22
|
+
|
23
|
+
@installed_files = [client_properties_path, messaging_yaml_path, truststore_path]
|
24
|
+
end
|
25
|
+
|
26
|
+
def configure
|
27
|
+
begin
|
28
|
+
MessageServerConfiguration.new.unconfigure if MessageServerConfiguration.configured?
|
29
|
+
configure_messaging_yaml # Set up the local message client in case EVM is actually running on this, Message Server
|
30
|
+
create_client_properties # Create the client.properties configuration fle
|
31
|
+
fetch_truststore_from_server # Fetch the Java Keystore from the Kafka Server
|
32
|
+
configure_messaging_type("kafka") # Settings.prototype.messaging_type = 'kafka'
|
33
|
+
restart_evmserverd
|
34
|
+
rescue AwesomeSpawn::CommandResultError => e
|
35
|
+
say(e.result.output)
|
36
|
+
say(e.result.error)
|
37
|
+
say("")
|
38
|
+
say("Failed to Configure the Message Client- #{e}")
|
39
|
+
return false
|
40
|
+
rescue => e
|
41
|
+
say("Failed to Configure the Message Client- #{e}")
|
42
|
+
return false
|
43
|
+
end
|
44
|
+
true
|
45
|
+
end
|
46
|
+
|
47
|
+
def ask_for_parameters
|
48
|
+
say("\nMessage Client Parameters:\n\n")
|
49
|
+
|
50
|
+
@message_server_host = ask_for_string("Message Server Hostname or IP address")
|
51
|
+
@message_server_port = ask_for_integer("Message Server Port number", (1..65_535), 9_093).to_i
|
52
|
+
@message_server_username = ask_for_string("Message Server Username", message_server_username)
|
53
|
+
@message_server_password = ask_for_password("Message Server Password")
|
54
|
+
@message_truststore_path_src = ask_for_string("Message Server Truststore Path", truststore_path)
|
55
|
+
@message_ca_cert_path_src = ask_for_string("Message Server CA Cert Path", ca_cert_path)
|
56
|
+
@message_keystore_username = ask_for_string("Message Keystore Username", message_keystore_username) if secure?
|
57
|
+
@message_keystore_password = ask_for_password("Message Keystore Password") if secure?
|
58
|
+
end
|
59
|
+
|
60
|
+
def show_parameters
|
61
|
+
say("\nMessage Client Configuration:\n")
|
62
|
+
say("Message Client Details:\n")
|
63
|
+
say(" Message Server Hostname: #{message_server_host}\n")
|
64
|
+
say(" Message Server Username: #{message_server_username}\n")
|
65
|
+
say(" Message Keystore Username: #{message_keystore_username}\n")
|
66
|
+
end
|
67
|
+
|
68
|
+
def fetch_truststore_from_server
|
69
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
70
|
+
|
71
|
+
fetch_from_server(message_truststore_path_src, truststore_path)
|
72
|
+
end
|
73
|
+
|
74
|
+
def fetch_ca_cert_from_server
|
75
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
76
|
+
|
77
|
+
fetch_from_server(message_ca_cert_path_src, ca_cert_path)
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
|
82
|
+
def fetch_from_server(src_file, dst_file)
|
83
|
+
return if file_found?(dst_file)
|
84
|
+
|
85
|
+
Net::SCP.start(message_server_host, message_server_username, :password => message_server_password) do |scp|
|
86
|
+
scp.download!(src_file, dst_file)
|
87
|
+
end
|
88
|
+
|
89
|
+
File.exist?(dst_file)
|
90
|
+
rescue => e
|
91
|
+
say("Failed to fetch #{src_file} from server: #{e.message}")
|
92
|
+
false
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,319 @@
|
|
1
|
+
require "awesome_spawn"
|
2
|
+
require "fileutils"
|
3
|
+
require "linux_admin"
|
4
|
+
require "manageiq/appliance_console/message_configuration"
|
5
|
+
|
6
|
+
module ManageIQ
|
7
|
+
module ApplianceConsole
|
8
|
+
class MessageServerConfiguration < MessageConfiguration
|
9
|
+
attr_reader :jaas_config_path,
|
10
|
+
:server_properties_path, :server_properties_sample_path,
|
11
|
+
:ca_cert_srl_path, :ca_key_path, :cert_file_path, :cert_signed_path,
|
12
|
+
:keystore_files, :installed_files, :message_persistent_disk
|
13
|
+
|
14
|
+
PERSISTENT_DIRECTORY = Pathname.new("/var/lib/kafka/persistent_data").freeze
|
15
|
+
PERSISTENT_NAME = "kafka_messages".freeze
|
16
|
+
|
17
|
+
def initialize(options = {})
|
18
|
+
super(options)
|
19
|
+
|
20
|
+
@message_server_host = options[:message_server_use_ipaddr] == true ? my_ipaddr : options[:message_server_host] || my_hostname
|
21
|
+
@message_persistent_disk = LinuxAdmin::Disk.new(:path => options[:message_persistent_disk]) unless options[:message_persistent_disk].nil?
|
22
|
+
|
23
|
+
@jaas_config_path = config_dir_path.join("kafka_server_jaas.conf")
|
24
|
+
@server_properties_path = config_dir_path.join("server.properties")
|
25
|
+
@server_properties_sample_path = sample_config_dir_path.join("server.properties")
|
26
|
+
|
27
|
+
@ca_cert_srl_path = keystore_dir_path.join("ca-cert.srl")
|
28
|
+
@ca_key_path = keystore_dir_path.join("ca-key")
|
29
|
+
@cert_file_path = keystore_dir_path.join("cert-file")
|
30
|
+
@cert_signed_path = keystore_dir_path.join("cert-signed")
|
31
|
+
|
32
|
+
@keystore_files = [ca_cert_path, ca_cert_srl_path, ca_key_path, cert_file_path, cert_signed_path, truststore_path, keystore_path]
|
33
|
+
@installed_files = [jaas_config_path, client_properties_path, server_properties_path, messaging_yaml_path, LOGS_DIR] + keystore_files
|
34
|
+
end
|
35
|
+
|
36
|
+
def configure
|
37
|
+
begin
|
38
|
+
configure_persistent_disk # Configure the persistent message store on a different disk
|
39
|
+
create_jaas_config # Create the message server jaas config file
|
40
|
+
create_client_properties # Create the client.properties config
|
41
|
+
create_logs_directory # Create the logs directory:
|
42
|
+
configure_firewall # Open the firewall for message port 9093
|
43
|
+
configure_keystore # Populate the Java Keystore
|
44
|
+
create_server_properties # Update the /opt/message/config/server.properties
|
45
|
+
configure_messaging_yaml # Set up the local message client in case EVM is actually running on this, Message Server
|
46
|
+
configure_messaging_type("kafka") # Settings.prototype.messaging_type = 'kafka'
|
47
|
+
restart_services
|
48
|
+
rescue AwesomeSpawn::CommandResultError => e
|
49
|
+
say(e.result.output)
|
50
|
+
say(e.result.error)
|
51
|
+
say("")
|
52
|
+
say("Failed to Configure the Message Server- #{e}")
|
53
|
+
return false
|
54
|
+
rescue => e
|
55
|
+
say("Failed to Configure the Message Server- #{e}")
|
56
|
+
return false
|
57
|
+
end
|
58
|
+
true
|
59
|
+
end
|
60
|
+
|
61
|
+
def restart_services
|
62
|
+
say("Starting zookeeper and configure it to start on reboots ...")
|
63
|
+
LinuxAdmin::Service.new("zookeeper").start.enable
|
64
|
+
|
65
|
+
say("Starting kafka and configure it to start on reboots ...")
|
66
|
+
LinuxAdmin::Service.new("kafka").start.enable
|
67
|
+
|
68
|
+
restart_evmserverd
|
69
|
+
end
|
70
|
+
|
71
|
+
def ask_for_parameters
|
72
|
+
say("\nMessage Server Parameters:\n\n")
|
73
|
+
|
74
|
+
@message_server_host = ask_for_string("Message Server Hostname or IP address", message_server_host)
|
75
|
+
@message_keystore_username = ask_for_string("Message Keystore Username", message_keystore_username)
|
76
|
+
@message_keystore_password = ask_for_password("Message Keystore Password")
|
77
|
+
@message_persistent_disk = ask_for_persistent_disk
|
78
|
+
end
|
79
|
+
|
80
|
+
def ask_for_persistent_disk
|
81
|
+
choose_disk if use_new_disk
|
82
|
+
end
|
83
|
+
|
84
|
+
def use_new_disk
|
85
|
+
agree("Configure a new persistent disk volume? (Y/N): ")
|
86
|
+
end
|
87
|
+
|
88
|
+
def choose_disk
|
89
|
+
ask_for_disk("Persistent disk")
|
90
|
+
end
|
91
|
+
|
92
|
+
def show_parameters
|
93
|
+
say("\nMessage Server Configuration:\n")
|
94
|
+
say("Message Server Details:\n")
|
95
|
+
say(" Message Server Hostname: #{message_server_host}\n")
|
96
|
+
say(" Message Keystore Username: #{message_keystore_username}\n")
|
97
|
+
say(" Persistent message disk: #{message_persistent_disk.path}\n") if message_persistent_disk
|
98
|
+
end
|
99
|
+
|
100
|
+
def unconfigure
|
101
|
+
super
|
102
|
+
|
103
|
+
unconfigure_firewall
|
104
|
+
deactivate_services
|
105
|
+
end
|
106
|
+
|
107
|
+
def self.configured?
|
108
|
+
LinuxAdmin::Service.new("kafka").running? ||
|
109
|
+
LinuxAdmin::Service.new("zookeeper").running?
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
|
114
|
+
def my_ipaddr
|
115
|
+
LinuxAdmin::IpAddress.new.address
|
116
|
+
end
|
117
|
+
|
118
|
+
def my_hostname
|
119
|
+
LinuxAdmin::Hosts.new.hostname
|
120
|
+
end
|
121
|
+
|
122
|
+
def configure_persistent_disk
|
123
|
+
return true unless message_persistent_disk
|
124
|
+
|
125
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
126
|
+
|
127
|
+
deactivate_services # Just in case they are running.
|
128
|
+
|
129
|
+
FileUtils.mkdir_p(PERSISTENT_DIRECTORY)
|
130
|
+
LogicalVolumeManagement.new(:disk => message_persistent_disk, :mount_point => PERSISTENT_DIRECTORY, :name => PERSISTENT_NAME).setup
|
131
|
+
FileUtils.chmod(0o755, PERSISTENT_DIRECTORY)
|
132
|
+
FileUtils.chown("kafka", "kafka", PERSISTENT_DIRECTORY)
|
133
|
+
|
134
|
+
true
|
135
|
+
end
|
136
|
+
|
137
|
+
def activate_new_persistent_disk
|
138
|
+
return true unless message_persistent_disk
|
139
|
+
|
140
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
141
|
+
|
142
|
+
data = File.read(server_properties_path)
|
143
|
+
data.gsub!(/^log.dirs=.*$/, "log.dirs=#{PERSISTENT_DIRECTORY}")
|
144
|
+
File.write(server_properties_path, data)
|
145
|
+
|
146
|
+
true
|
147
|
+
end
|
148
|
+
|
149
|
+
def create_jaas_config
|
150
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
151
|
+
|
152
|
+
content = <<~JAAS
|
153
|
+
KafkaServer {
|
154
|
+
org.apache.kafka.common.security.plain.PlainLoginModule required
|
155
|
+
username=#{message_keystore_username}
|
156
|
+
password=#{message_keystore_password}
|
157
|
+
user_admin=#{message_keystore_password} ;
|
158
|
+
};
|
159
|
+
JAAS
|
160
|
+
|
161
|
+
File.write(jaas_config_path, content) unless file_found?(jaas_config_path)
|
162
|
+
end
|
163
|
+
|
164
|
+
def create_logs_directory
|
165
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
166
|
+
|
167
|
+
return if file_found?(LOGS_DIR)
|
168
|
+
|
169
|
+
FileUtils.mkdir_p(LOGS_DIR)
|
170
|
+
FileUtils.chmod(0o755, LOGS_DIR)
|
171
|
+
FileUtils.chown("kafka", "kafka", LOGS_DIR)
|
172
|
+
end
|
173
|
+
|
174
|
+
def configure_firewall
|
175
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
176
|
+
|
177
|
+
modify_firewall(:add_port)
|
178
|
+
end
|
179
|
+
|
180
|
+
def configure_keystore
|
181
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
182
|
+
|
183
|
+
return if files_found?(keystore_files)
|
184
|
+
|
185
|
+
keystore_params = assemble_keystore_params
|
186
|
+
|
187
|
+
# Generte a Java keystore and key pair, creating keystore.jks
|
188
|
+
# :stdin_data provides the -storepass twice to confirm and an extra CR to accept the same password for -keypass
|
189
|
+
AwesomeSpawn.run!("keytool", :params => keystore_params, :stdin_data => "#{message_keystore_password}\n#{message_keystore_password}\n\n")
|
190
|
+
|
191
|
+
# Use openssl to create a new CA cert, creating ca-cert and ca-key
|
192
|
+
AwesomeSpawn.run!("openssl", :env => {"PASSWORD" => message_keystore_password},
|
193
|
+
:params => ["req", "-new", "-x509", {"-keyout" => ca_key_path,
|
194
|
+
"-out" => ca_cert_path,
|
195
|
+
"-days" => 10_000,
|
196
|
+
"-passout" => "env:PASSWORD",
|
197
|
+
"-subj" => '/CN=something'}])
|
198
|
+
|
199
|
+
# Import the CA cert into the trust store, creating truststore.jks
|
200
|
+
# :stdin_data provides the -storepass argument and yes to confirm
|
201
|
+
AwesomeSpawn.run!("keytool", :params => {"-keystore" => truststore_path,
|
202
|
+
"-alias" => "CARoot",
|
203
|
+
"-import" => nil,
|
204
|
+
"-file" => ca_cert_path},
|
205
|
+
:stdin_data => "#{message_keystore_password}\n#{message_keystore_password}\nyes\n")
|
206
|
+
|
207
|
+
# Generate a certificate signing request (CSR) for an existing Java keystore, creating cert-file
|
208
|
+
# :stdin_data provides the -storepass argument
|
209
|
+
AwesomeSpawn.run!("keytool", :params => {"-keystore" => keystore_path,
|
210
|
+
"-alias" => keystore_params["-alias"],
|
211
|
+
"-certreq" => nil,
|
212
|
+
"-file" => cert_file_path},
|
213
|
+
:stdin_data => "#{message_keystore_password}\n")
|
214
|
+
|
215
|
+
# Use openssl to sign the certificate with the "CA" certificate, creating ca-cert.srl and cert-signed
|
216
|
+
AwesomeSpawn.run!("openssl", :env => {"PASSWORD" => message_keystore_password},
|
217
|
+
:params => ["x509", "-req", {"-CA" => ca_cert_path,
|
218
|
+
"-CAkey" => ca_key_path,
|
219
|
+
"-in" => cert_file_path,
|
220
|
+
"-out" => cert_signed_path,
|
221
|
+
"-days" => 10_000,
|
222
|
+
"-CAcreateserial" => nil,
|
223
|
+
"-passin" => "env:PASSWORD"}])
|
224
|
+
|
225
|
+
# Import a root or intermediate CA certificate to an existing Java keystore, updating keystore.jks
|
226
|
+
# :stdin_data provides the -storepass argument and yes to confirm
|
227
|
+
AwesomeSpawn.run!("keytool", :params => {"-keystore" => keystore_path,
|
228
|
+
"-alias" => "CARoot",
|
229
|
+
"-import" => nil,
|
230
|
+
"-file" => ca_cert_path},
|
231
|
+
:stdin_data => "#{message_keystore_password}\nyes\n")
|
232
|
+
|
233
|
+
# Import a signed primary certificate to an existing Java keystore, updating keystore.jks
|
234
|
+
# :stdin_data provides the -storepass argument
|
235
|
+
AwesomeSpawn.run!("keytool", :params => {"-keystore" => keystore_path,
|
236
|
+
"-alias" => keystore_params["-alias"],
|
237
|
+
"-import" => nil,
|
238
|
+
"-file" => cert_signed_path},
|
239
|
+
:stdin_data => "#{message_keystore_password}\n")
|
240
|
+
end
|
241
|
+
|
242
|
+
def create_server_properties
|
243
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
244
|
+
|
245
|
+
if message_server_host.ipaddress?
|
246
|
+
ident_algorithm = ""
|
247
|
+
client_auth = "none"
|
248
|
+
else
|
249
|
+
ident_algorithm = "HTTPS"
|
250
|
+
client_auth = "required"
|
251
|
+
end
|
252
|
+
|
253
|
+
content = <<~SERVER_PROPERTIES
|
254
|
+
|
255
|
+
listeners=SASL_SSL://:#{message_server_port}
|
256
|
+
|
257
|
+
ssl.endpoint.identification.algorithm=#{ident_algorithm}
|
258
|
+
ssl.keystore.location=#{keystore_path}
|
259
|
+
ssl.keystore.password=#{message_keystore_password}
|
260
|
+
ssl.key.password=#{message_keystore_password}
|
261
|
+
|
262
|
+
ssl.truststore.location=#{truststore_path}
|
263
|
+
ssl.truststore.password=#{message_keystore_password}
|
264
|
+
|
265
|
+
ssl.client.auth=#{client_auth}
|
266
|
+
|
267
|
+
sasl.enabled.mechanisms=PLAIN
|
268
|
+
sasl.mechanism.inter.broker.protocol=PLAIN
|
269
|
+
|
270
|
+
security.inter.broker.protocol=SASL_SSL
|
271
|
+
SERVER_PROPERTIES
|
272
|
+
|
273
|
+
return if file_contains?(server_properties_path, content)
|
274
|
+
|
275
|
+
FileUtils.cp(server_properties_sample_path, server_properties_path)
|
276
|
+
File.write(server_properties_path, content, :mode => "a")
|
277
|
+
|
278
|
+
activate_new_persistent_disk
|
279
|
+
end
|
280
|
+
|
281
|
+
def unconfigure_firewall
|
282
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
283
|
+
|
284
|
+
modify_firewall(:remove_port)
|
285
|
+
end
|
286
|
+
|
287
|
+
def deactivate_services
|
288
|
+
say(__method__.to_s.tr("_", " ").titleize)
|
289
|
+
|
290
|
+
LinuxAdmin::Service.new("zookeeper").stop
|
291
|
+
LinuxAdmin::Service.new("kafka").stop
|
292
|
+
end
|
293
|
+
|
294
|
+
def assemble_keystore_params
|
295
|
+
keystore_params = {"-keystore" => keystore_path,
|
296
|
+
"-validity" => 10_000,
|
297
|
+
"-genkey" => nil,
|
298
|
+
"-keyalg" => "RSA"}
|
299
|
+
|
300
|
+
if message_server_host.ipaddress?
|
301
|
+
keystore_params["-alias"] = "localhost"
|
302
|
+
keystore_params["-ext"] = "san=ip:#{message_server_host}"
|
303
|
+
else
|
304
|
+
keystore_params["-alias"] = message_server_host
|
305
|
+
keystore_params["-ext"] = "san=dns:#{message_server_host}"
|
306
|
+
end
|
307
|
+
|
308
|
+
keystore_params["-dname"] = "cn=#{keystore_params["-alias"]}"
|
309
|
+
|
310
|
+
keystore_params
|
311
|
+
end
|
312
|
+
|
313
|
+
def modify_firewall(action)
|
314
|
+
AwesomeSpawn.run!("firewall-cmd", :params => {action => "#{message_server_port}/tcp", :permanent => nil})
|
315
|
+
AwesomeSpawn.run!("firewall-cmd --reload")
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
end
|
@@ -0,0 +1,325 @@
|
|
1
|
+
require 'awesome_spawn'
|
2
|
+
require 'pathname'
|
3
|
+
require 'linux_admin'
|
4
|
+
|
5
|
+
module ManageIQ
|
6
|
+
module ApplianceConsole
|
7
|
+
class PostgresAdmin
|
8
|
+
def self.data_directory
|
9
|
+
Pathname.new(ENV.fetch("APPLIANCE_PG_DATA"))
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.mount_point
|
13
|
+
Pathname.new(ENV.fetch("APPLIANCE_PG_MOUNT_POINT"))
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.template_directory
|
17
|
+
Pathname.new(ENV.fetch("APPLIANCE_TEMPLATE_DIRECTORY"))
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.service_name
|
21
|
+
ENV.fetch("APPLIANCE_PG_SERVICE")
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.package_name
|
25
|
+
ENV.fetch('APPLIANCE_PG_PACKAGE_NAME')
|
26
|
+
end
|
27
|
+
|
28
|
+
# Unprivileged user to run postgresql
|
29
|
+
def self.user
|
30
|
+
"postgres".freeze
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.group
|
34
|
+
user
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.logical_volume_name
|
38
|
+
"lv_pg".freeze
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.volume_group_name
|
42
|
+
"vg_data".freeze
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.database_disk_filesystem
|
46
|
+
"xfs".freeze
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.initialized?
|
50
|
+
!Dir[data_directory.join("*")].empty?
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.service_running?
|
54
|
+
LinuxAdmin::Service.new(service_name).running?
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.local_server_in_recovery?
|
58
|
+
data_directory.join("recovery.conf").exist?
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.local_server_status
|
62
|
+
if service_running?
|
63
|
+
"running (#{local_server_in_recovery? ? "standby" : "primary"})"
|
64
|
+
elsif initialized?
|
65
|
+
"initialized and stopped"
|
66
|
+
else
|
67
|
+
"not initialized"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.logical_volume_path
|
72
|
+
Pathname.new("/dev").join(volume_group_name, logical_volume_name)
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.database_size(opts)
|
76
|
+
result = run_command("psql", opts, :command => "SELECT pg_database_size('#{opts[:dbname]}');")
|
77
|
+
result.match(/^\s+([0-9]+)\n/)[1].to_i
|
78
|
+
end
|
79
|
+
|
80
|
+
def self.prep_data_directory
|
81
|
+
# initdb will fail if the database directory is not empty or not owned by the PostgresAdmin.user
|
82
|
+
FileUtils.mkdir(PostgresAdmin.data_directory) unless Dir.exist?(PostgresAdmin.data_directory)
|
83
|
+
FileUtils.chown_R(PostgresAdmin.user, PostgresAdmin.group, PostgresAdmin.data_directory)
|
84
|
+
FileUtils.rm_rf(PostgresAdmin.data_directory.children.map(&:to_s))
|
85
|
+
end
|
86
|
+
|
87
|
+
PG_DUMP_MAGIC = "PGDMP".force_encoding(Encoding::BINARY).freeze
|
88
|
+
def self.pg_dump_file?(file)
|
89
|
+
File.open(file, "rb") { |f| f.readpartial(5) } == PG_DUMP_MAGIC
|
90
|
+
end
|
91
|
+
|
92
|
+
BASE_BACKUP_MAGIC = "\037\213".force_encoding(Encoding::BINARY).freeze # just the first 2 bits of gzip magic
|
93
|
+
def self.base_backup_file?(file)
|
94
|
+
File.open(file, "rb") { |f| f.readpartial(2) } == BASE_BACKUP_MAGIC
|
95
|
+
end
|
96
|
+
|
97
|
+
def self.backup(opts)
|
98
|
+
backup_pg_compress(opts)
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.restore(opts)
|
102
|
+
file = opts[:local_file]
|
103
|
+
backup_type = opts.delete(:backup_type)
|
104
|
+
|
105
|
+
case
|
106
|
+
when backup_type == :pgdump then restore_pg_dump(opts)
|
107
|
+
when backup_type == :basebackup then restore_pg_basebackup(file)
|
108
|
+
when pg_dump_file?(file) then restore_pg_dump(opts)
|
109
|
+
when base_backup_file?(file) then restore_pg_basebackup(file)
|
110
|
+
else
|
111
|
+
raise "#{file} is not a database backup"
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def self.restore_pg_basebackup(file)
|
116
|
+
pg_service = LinuxAdmin::Service.new(service_name)
|
117
|
+
|
118
|
+
pg_service.stop
|
119
|
+
prep_data_directory
|
120
|
+
|
121
|
+
require 'rubygems/package'
|
122
|
+
|
123
|
+
# Using a Gem::Package instance for the #extract_tar_gz method, so we don't
|
124
|
+
# have to re-write all of that logic. Mostly making use of
|
125
|
+
# `Gem::Package::TarReader` + `Zlib::GzipReader` that is already part of
|
126
|
+
# rubygems/stdlib and integrated there.
|
127
|
+
unpacker = Gem::Package.new("obviously_not_a_gem")
|
128
|
+
File.open(file, IO::RDONLY | IO::NONBLOCK) do |backup_file|
|
129
|
+
unpacker.extract_tar_gz(backup_file, data_directory.to_s)
|
130
|
+
end
|
131
|
+
|
132
|
+
FileUtils.chown_R(PostgresAdmin.user, PostgresAdmin.group, PostgresAdmin.data_directory)
|
133
|
+
|
134
|
+
pg_service.start
|
135
|
+
file
|
136
|
+
end
|
137
|
+
|
138
|
+
def self.backup_pg_dump(opts)
|
139
|
+
opts = opts.dup
|
140
|
+
dbname = opts.delete(:dbname)
|
141
|
+
|
142
|
+
args = combine_command_args(opts, :format => "c", :file => opts[:local_file], nil => dbname)
|
143
|
+
args = handle_multi_value_pg_dump_args!(opts, args)
|
144
|
+
|
145
|
+
run_command_with_logging("pg_dump", opts, args)
|
146
|
+
opts[:local_file]
|
147
|
+
end
|
148
|
+
|
149
|
+
def self.backup_pg_compress(opts)
|
150
|
+
opts = opts.dup
|
151
|
+
|
152
|
+
# discard dbname as pg_basebackup does not connect to a specific database
|
153
|
+
opts.delete(:dbname)
|
154
|
+
|
155
|
+
path = Pathname.new(opts.delete(:local_file))
|
156
|
+
FileUtils.mkdir_p(path.dirname)
|
157
|
+
|
158
|
+
# Build commandline from AwesomeSpawn
|
159
|
+
args = {:z => nil, :format => "t", :wal_method => "fetch", :pgdata => "-"}
|
160
|
+
cmd = AwesomeSpawn.build_command_line("pg_basebackup", combine_command_args(opts, args))
|
161
|
+
logger.info("MIQ(#{name}.#{__method__}) Running command... #{cmd}")
|
162
|
+
|
163
|
+
# Run command in a separate thread
|
164
|
+
read, write = IO.pipe
|
165
|
+
error_path = Dir::Tmpname.create("") { |tmpname| tmpname }
|
166
|
+
process_thread = Process.detach(Kernel.spawn(pg_env(opts), cmd, :out => write, :err => error_path))
|
167
|
+
stream_reader = Thread.new { IO.copy_stream(read, path) } # Copy output to path
|
168
|
+
write.close
|
169
|
+
|
170
|
+
# Wait for them to finish
|
171
|
+
process_status = process_thread.value
|
172
|
+
stream_reader.join
|
173
|
+
read.close
|
174
|
+
|
175
|
+
handle_error(cmd, process_status.exitstatus, error_path)
|
176
|
+
path.to_s
|
177
|
+
end
|
178
|
+
|
179
|
+
def self.recreate_db(opts)
|
180
|
+
dbname = opts[:dbname]
|
181
|
+
opts = opts.merge(:dbname => 'postgres')
|
182
|
+
run_command("psql", opts, :command => "DROP DATABASE IF EXISTS #{dbname}")
|
183
|
+
run_command("psql", opts, :command => "CREATE DATABASE #{dbname} WITH OWNER = #{opts[:username] || 'root'} ENCODING = 'UTF8'")
|
184
|
+
end
|
185
|
+
|
186
|
+
def self.restore_pg_dump(opts)
|
187
|
+
recreate_db(opts)
|
188
|
+
args = { :verbose => nil, :exit_on_error => nil }
|
189
|
+
|
190
|
+
if File.pipe?(opts[:local_file])
|
191
|
+
cmd_args = combine_command_args(opts, args)
|
192
|
+
cmd = AwesomeSpawn.build_command_line("pg_restore", cmd_args)
|
193
|
+
error_path = Dir::Tmpname.create("") { |tmpname| tmpname }
|
194
|
+
spawn_args = { :err => error_path, :in => [opts[:local_file].to_s, "rb"] }
|
195
|
+
|
196
|
+
logger.info("MIQ(#{name}.#{__method__}) Running command... #{cmd}")
|
197
|
+
process_thread = Process.detach(Kernel.spawn(pg_env(opts), cmd, spawn_args))
|
198
|
+
process_status = process_thread.value
|
199
|
+
|
200
|
+
handle_error(cmd, process_status.exitstatus, error_path)
|
201
|
+
else
|
202
|
+
args[nil] = opts[:local_file]
|
203
|
+
run_command("pg_restore", opts, args)
|
204
|
+
end
|
205
|
+
opts[:local_file]
|
206
|
+
end
|
207
|
+
|
208
|
+
GC_DEFAULTS = {
|
209
|
+
:analyze => false,
|
210
|
+
:full => false,
|
211
|
+
:verbose => false,
|
212
|
+
:table => nil,
|
213
|
+
:dbname => nil,
|
214
|
+
:username => nil,
|
215
|
+
:reindex => false
|
216
|
+
}
|
217
|
+
|
218
|
+
GC_AGGRESSIVE_DEFAULTS = {
|
219
|
+
:analyze => true,
|
220
|
+
:full => true,
|
221
|
+
:verbose => false,
|
222
|
+
:table => nil,
|
223
|
+
:dbname => nil,
|
224
|
+
:username => nil,
|
225
|
+
:reindex => true
|
226
|
+
}
|
227
|
+
|
228
|
+
def self.gc(options = {})
|
229
|
+
options = (options[:aggressive] ? GC_AGGRESSIVE_DEFAULTS : GC_DEFAULTS).merge(options)
|
230
|
+
|
231
|
+
result = vacuum(options)
|
232
|
+
logger.info("MIQ(#{name}.#{__method__}) Output... #{result}") if result.to_s.length > 0
|
233
|
+
|
234
|
+
if options[:reindex]
|
235
|
+
result = reindex(options)
|
236
|
+
logger.info("MIQ(#{name}.#{__method__}) Output... #{result}") if result.to_s.length > 0
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def self.vacuum(opts)
|
241
|
+
# TODO: Add a real exception here
|
242
|
+
raise "Vacuum requires database" unless opts[:dbname]
|
243
|
+
|
244
|
+
args = {}
|
245
|
+
args[:analyze] = nil if opts[:analyze]
|
246
|
+
args[:full] = nil if opts[:full]
|
247
|
+
args[:verbose] = nil if opts[:verbose]
|
248
|
+
args[:table] = opts[:table] if opts[:table]
|
249
|
+
run_command("vacuumdb", opts, args)
|
250
|
+
end
|
251
|
+
|
252
|
+
def self.reindex(opts)
|
253
|
+
args = {}
|
254
|
+
args[:table] = opts[:table] if opts[:table]
|
255
|
+
run_command("reindexdb", opts, args)
|
256
|
+
end
|
257
|
+
|
258
|
+
def self.run_command(cmd_str, opts, args)
|
259
|
+
run_command_with_logging(cmd_str, opts, combine_command_args(opts, args))
|
260
|
+
end
|
261
|
+
|
262
|
+
def self.run_command_with_logging(cmd_str, opts, params = {})
|
263
|
+
logger.info("MIQ(#{name}.#{__method__}) Running command... #{AwesomeSpawn.build_command_line(cmd_str, params)}")
|
264
|
+
AwesomeSpawn.run!(cmd_str, :params => params, :env => pg_env(opts)).output
|
265
|
+
end
|
266
|
+
|
267
|
+
class << self
|
268
|
+
# Temporary alias due to manageiq core stubbing this method
|
269
|
+
alias runcmd_with_logging run_command_with_logging
|
270
|
+
end
|
271
|
+
|
272
|
+
private_class_method def self.combine_command_args(opts, args)
|
273
|
+
default_args = {:no_password => nil}
|
274
|
+
default_args[:dbname] = opts[:dbname] if opts[:dbname]
|
275
|
+
default_args[:username] = opts[:username] if opts[:username]
|
276
|
+
default_args[:host] = opts[:hostname] if opts[:hostname]
|
277
|
+
default_args[:port] = opts[:port] if opts[:port]
|
278
|
+
default_args.merge(args)
|
279
|
+
end
|
280
|
+
|
281
|
+
private_class_method def self.logger
|
282
|
+
ManageIQ::ApplianceConsole.logger
|
283
|
+
end
|
284
|
+
|
285
|
+
private_class_method def self.pg_env(opts)
|
286
|
+
{
|
287
|
+
"PGUSER" => opts[:username],
|
288
|
+
"PGPASSWORD" => opts[:password]
|
289
|
+
}.delete_blanks
|
290
|
+
end
|
291
|
+
# rubocop:disable Style/SymbolArray
|
292
|
+
PG_DUMP_MULTI_VALUE_ARGS = [
|
293
|
+
:t, :table, :T, :exclude_table, :"exclude-table", :exclude_table_data, :"exclude-table-data",
|
294
|
+
:n, :schema, :N, :exclude_schema, :"exclude-schema"
|
295
|
+
].freeze
|
296
|
+
# rubocop:enable Style/SymbolArray
|
297
|
+
#
|
298
|
+
# NOTE: Potentially mutates opts hash (args becomes new array and not
|
299
|
+
# mutated by this method)
|
300
|
+
private_class_method def self.handle_multi_value_pg_dump_args!(opts, args)
|
301
|
+
if opts.keys.any? { |key| PG_DUMP_MULTI_VALUE_ARGS.include?(key) }
|
302
|
+
args = args.to_a
|
303
|
+
PG_DUMP_MULTI_VALUE_ARGS.each do |table_key|
|
304
|
+
next unless opts.key?(table_key)
|
305
|
+
table_val = opts.delete(table_key)
|
306
|
+
args += Array.wrap(table_val).map! { |v| [table_key, v] }
|
307
|
+
end
|
308
|
+
end
|
309
|
+
args
|
310
|
+
end
|
311
|
+
|
312
|
+
private_class_method def self.handle_error(cmd, exit_status, error_path)
|
313
|
+
if exit_status != 0
|
314
|
+
result = AwesomeSpawn::CommandResult.new(cmd, "", File.read(error_path), exit_status)
|
315
|
+
message = AwesomeSpawn::CommandResultError.default_message(cmd, exit_status)
|
316
|
+
logger.error("AwesomeSpawn: #{message}")
|
317
|
+
logger.error("AwesomeSpawn: #{result.error}")
|
318
|
+
raise AwesomeSpawn::CommandResultError.new(message, result)
|
319
|
+
end
|
320
|
+
ensure
|
321
|
+
File.delete(error_path) if File.exist?(error_path)
|
322
|
+
end
|
323
|
+
end
|
324
|
+
end
|
325
|
+
end
|