hybrid_platforms_conductor 32.4.2 → 32.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/hybrid_platforms_conductor/deployer.rb +9 -8
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +157 -81
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +6 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +7 -6
- data/lib/hybrid_platforms_conductor/nodes_handler.rb +37 -0
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +35 -0
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +65 -5
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +50 -6
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +47 -9
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +28 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +71 -0
- data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb +2 -1
- data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +29 -11
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +50 -11
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +14 -14
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +70 -11
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +2 -2
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3abcfb2c500d444ed5ea2486bda7150aa451986c502cafdf68afdd802e0af866
|
|
4
|
+
data.tar.gz: 06cf63f0c1e5a187ca53e259133f7e14e4e73d6b4e2d7a3b79180b4797ffaf82
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f3c4891e7add29c7b424256888fb959967f5b7ab428bb4564aecab19a2de0c34f7dc4d05cf232cea944a4aa4008bd5a541a2a90786e2d4cf2fdf7750eb439b12
|
|
7
|
+
data.tar.gz: 254f4f7c9c3e9638be1632cd27d7eb0aa296a6be5b13167e1926ab0de52bfb9f8490c6b6251bef7cc20876f686fe161d67fb48bace985e743ee80f52359e3093
|
|
@@ -480,6 +480,7 @@ module HybridPlatformsConductor
|
|
|
480
480
|
outputs = @actions_executor.execute_actions(
|
|
481
481
|
Hash[services.map do |node, node_services|
|
|
482
482
|
image_id = @nodes_handler.get_image_of(node)
|
|
483
|
+
sudo = (ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} ")
|
|
483
484
|
# Install My_company corporate certificates if present
|
|
484
485
|
certificate_actions =
|
|
485
486
|
if @local_environment && ENV['hpc_certificates']
|
|
@@ -489,20 +490,20 @@ module HybridPlatformsConductor
|
|
|
489
490
|
when 'debian_9', 'debian_10'
|
|
490
491
|
[
|
|
491
492
|
{
|
|
492
|
-
remote_bash: "#{
|
|
493
|
+
remote_bash: "#{sudo}apt update && #{sudo}apt install -y ca-certificates"
|
|
493
494
|
},
|
|
494
495
|
{
|
|
495
496
|
scp: {
|
|
496
497
|
ENV['hpc_certificates'] => '/usr/local/share/ca-certificates',
|
|
497
498
|
:sudo => ssh_user != 'root'
|
|
498
499
|
},
|
|
499
|
-
remote_bash: "#{
|
|
500
|
+
remote_bash: "#{sudo}update-ca-certificates"
|
|
500
501
|
}
|
|
501
502
|
]
|
|
502
503
|
when 'centos_7'
|
|
503
504
|
[
|
|
504
505
|
{
|
|
505
|
-
remote_bash: "#{
|
|
506
|
+
remote_bash: "#{sudo}yum install -y ca-certificates"
|
|
506
507
|
},
|
|
507
508
|
{
|
|
508
509
|
scp: Hash[Dir.glob("#{ENV['hpc_certificates']}/*.crt").map do |cert_file|
|
|
@@ -512,8 +513,8 @@ module HybridPlatformsConductor
|
|
|
512
513
|
]
|
|
513
514
|
end].merge(sudo: ssh_user != 'root'),
|
|
514
515
|
remote_bash: [
|
|
515
|
-
"#{
|
|
516
|
-
"#{
|
|
516
|
+
"#{sudo}update-ca-trust enable",
|
|
517
|
+
"#{sudo}update-ca-trust extract"
|
|
517
518
|
]
|
|
518
519
|
}
|
|
519
520
|
]
|
|
@@ -532,7 +533,7 @@ module HybridPlatformsConductor
|
|
|
532
533
|
# Install the mutex lock and acquire it
|
|
533
534
|
{
|
|
534
535
|
scp: { "#{__dir__}/mutex_dir" => '.' },
|
|
535
|
-
remote_bash: "while ! #{
|
|
536
|
+
remote_bash: "while ! #{sudo}./mutex_dir lock /tmp/hybrid_platforms_conductor_deploy_lock \"$(ps -o ppid= -p $$)\"; do echo -e 'Another deployment is running on #{node}. Waiting for it to finish to continue...' ; sleep 5 ; done"
|
|
536
537
|
}
|
|
537
538
|
] +
|
|
538
539
|
certificate_actions +
|
|
@@ -548,7 +549,7 @@ module HybridPlatformsConductor
|
|
|
548
549
|
Hash[services.keys.map do |node|
|
|
549
550
|
[
|
|
550
551
|
node,
|
|
551
|
-
{ remote_bash: "#{ssh_user == 'root' ? '' :
|
|
552
|
+
{ remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
|
|
552
553
|
]
|
|
553
554
|
end],
|
|
554
555
|
timeout: 10,
|
|
@@ -595,7 +596,7 @@ module HybridPlatformsConductor
|
|
|
595
596
|
[
|
|
596
597
|
node,
|
|
597
598
|
{
|
|
598
|
-
remote_bash: "#{ssh_user == 'root' ? '' :
|
|
599
|
+
remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}mkdir -p /var/log/deployments",
|
|
599
600
|
scp: {
|
|
600
601
|
log_file => '/var/log/deployments',
|
|
601
602
|
:sudo => ssh_user != 'root',
|
|
@@ -16,11 +16,40 @@ module HybridPlatformsConductor
|
|
|
16
16
|
|
|
17
17
|
module PlatformsDslSsh
|
|
18
18
|
|
|
19
|
+
# List of SSH connection transformations:
|
|
20
|
+
# * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
|
|
21
|
+
# * *transform* (Proc): Code called to transform SSH connection info:
|
|
22
|
+
# Parameters::
|
|
23
|
+
# * *node* (String): Node for which we transform the SSH connection
|
|
24
|
+
# * *connection* (String or nil): The connection host or IP, or nil if none
|
|
25
|
+
# * *connection_user* (String): The connection user
|
|
26
|
+
# * *gateway* (String or nil): The gateway name, or nil if none
|
|
27
|
+
# * *gateway_user* (String or nil): The gateway user, or nil if none
|
|
28
|
+
# Result::
|
|
29
|
+
# * String: The transformed connection host or IP, or nil if none
|
|
30
|
+
# * String: The transformed connection user
|
|
31
|
+
# * String or nil: The transformed gateway name, or nil if none
|
|
32
|
+
# * String or nil: The transformed gateway user, or nil if none
|
|
33
|
+
# Array< Hash<Symbol, Object> >
|
|
34
|
+
attr_reader :ssh_connection_transforms
|
|
35
|
+
|
|
19
36
|
# Initialize the DSL
|
|
20
37
|
def init_ssh
|
|
21
38
|
# List of gateway configurations, per gateway config name
|
|
22
39
|
# Hash<Symbol, String>
|
|
23
40
|
@gateways = {}
|
|
41
|
+
@ssh_connection_transforms = []
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Define a transformation of SSH connection.
|
|
45
|
+
#
|
|
46
|
+
# Parameters::
|
|
47
|
+
# * *transform* (Proc): Code to be called to transform an SSH connection (see ssh_connection_transforms signature for details)
|
|
48
|
+
def transform_ssh_connection(&transform)
|
|
49
|
+
@ssh_connection_transforms << {
|
|
50
|
+
nodes_selectors_stack: current_nodes_selectors_stack,
|
|
51
|
+
transform: transform
|
|
52
|
+
}
|
|
24
53
|
end
|
|
25
54
|
|
|
26
55
|
# Register a new gateway configuration
|
|
@@ -204,7 +233,13 @@ module HybridPlatformsConductor
|
|
|
204
233
|
# Parameters::
|
|
205
234
|
# * *bash_cmds* (String): Bash commands to execute
|
|
206
235
|
def remote_bash(bash_cmds)
|
|
207
|
-
ssh_cmd =
|
|
236
|
+
ssh_cmd =
|
|
237
|
+
if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
|
|
238
|
+
# When ExecSession is disabled we need to use stdin directly
|
|
239
|
+
"{ cat | #{ssh_exec} #{ssh_url} -T; } <<'EOF'\n#{bash_cmds}\nEOF"
|
|
240
|
+
else
|
|
241
|
+
"#{ssh_exec} #{ssh_url} /bin/bash <<'EOF'\n#{bash_cmds}\nEOF"
|
|
242
|
+
end
|
|
208
243
|
# Due to a limitation of Process.spawn, each individual argument is limited to 128KB of size.
|
|
209
244
|
# Therefore we need to make sure that if bash_cmds exceeds MAX_CMD_ARG_LENGTH bytes (considering EOF chars) then we use an intermediary shell script to store the commands.
|
|
210
245
|
if bash_cmds.size > MAX_CMD_ARG_LENGTH
|
|
@@ -261,25 +296,30 @@ module HybridPlatformsConductor
|
|
|
261
296
|
# * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
|
|
262
297
|
# * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
|
|
263
298
|
def remote_copy(from, to, sudo: false, owner: nil, group: nil)
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
#{ssh_exec} \
|
|
274
|
-
#{ssh_url} \
|
|
275
|
-
\"#{sudo ? 'sudo ' : ''}tar \
|
|
276
|
-
--extract \
|
|
277
|
-
--gunzip \
|
|
299
|
+
if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
|
|
300
|
+
# We don't have ExecSession, so don't use ssh, but scp instead.
|
|
301
|
+
run_cmd "scp -S #{ssh_exec} #{from} #{ssh_url}:#{to}"
|
|
302
|
+
else
|
|
303
|
+
run_cmd <<~EOS
|
|
304
|
+
cd #{File.dirname(from)} && \
|
|
305
|
+
tar \
|
|
306
|
+
--create \
|
|
307
|
+
--gzip \
|
|
278
308
|
--file - \
|
|
279
|
-
--
|
|
280
|
-
--
|
|
281
|
-
|
|
282
|
-
|
|
309
|
+
#{owner.nil? ? '' : "--owner #{owner}"} \
|
|
310
|
+
#{group.nil? ? '' : "--group #{group}"} \
|
|
311
|
+
#{File.basename(from)} | \
|
|
312
|
+
#{ssh_exec} \
|
|
313
|
+
#{ssh_url} \
|
|
314
|
+
\"#{sudo ? "#{@nodes_handler.sudo_on(@node)} " : ''}tar \
|
|
315
|
+
--extract \
|
|
316
|
+
--gunzip \
|
|
317
|
+
--file - \
|
|
318
|
+
--directory #{to} \
|
|
319
|
+
--owner root \
|
|
320
|
+
\"
|
|
321
|
+
EOS
|
|
322
|
+
end
|
|
283
323
|
end
|
|
284
324
|
|
|
285
325
|
# Get the ssh executable to be used when connecting to the current node
|
|
@@ -295,7 +335,7 @@ module HybridPlatformsConductor
|
|
|
295
335
|
# Result::
|
|
296
336
|
# * String: The ssh URL connecting to the current node
|
|
297
337
|
def ssh_url
|
|
298
|
-
"
|
|
338
|
+
"hpc.#{@node}"
|
|
299
339
|
end
|
|
300
340
|
|
|
301
341
|
# Get an SSH configuration content giving access to nodes of the platforms with the current configuration
|
|
@@ -318,14 +358,6 @@ module HybridPlatformsConductor
|
|
|
318
358
|
# ENDPOINTS #
|
|
319
359
|
#############
|
|
320
360
|
|
|
321
|
-
Host *
|
|
322
|
-
User #{@ssh_user}
|
|
323
|
-
# Default control socket path to be used when multiplexing SSH connections
|
|
324
|
-
ControlPath #{control_master_file('%h', '%p', '%r')}
|
|
325
|
-
#{open_ssh_major_version >= 7 ? 'PubkeyAcceptedKeyTypes +ssh-dss' : ''}
|
|
326
|
-
#{known_hosts_file.nil? ? '' : "UserKnownHostsFile #{known_hosts_file}"}
|
|
327
|
-
#{@ssh_strict_host_key_checking ? '' : 'StrictHostKeyChecking no'}
|
|
328
|
-
|
|
329
361
|
EOS
|
|
330
362
|
|
|
331
363
|
# Add each node
|
|
@@ -333,21 +365,37 @@ module HybridPlatformsConductor
|
|
|
333
365
|
@nodes_handler.prefetch_metadata_of nodes, %i[private_ips hostname host_ip description]
|
|
334
366
|
nodes.sort.each do |node|
|
|
335
367
|
# Generate the conf for the node
|
|
336
|
-
|
|
337
|
-
|
|
368
|
+
connection, connection_user, gateway, gateway_user = connection_info_for(node, no_exception: true)
|
|
369
|
+
if connection.nil?
|
|
370
|
+
config_content << "# #{node} - Not connectable using SSH - #{@nodes_handler.get_description_of(node) || ''}\n"
|
|
371
|
+
else
|
|
338
372
|
config_content << "# #{node} - #{connection} - #{@nodes_handler.get_description_of(node) || ''}\n"
|
|
339
373
|
config_content << "Host #{ssh_aliases_for(node).join(' ')}\n"
|
|
340
374
|
config_content << " Hostname #{connection}\n"
|
|
375
|
+
config_content << " User \"#{connection_user}\"\n" if connection_user != @ssh_user
|
|
341
376
|
config_content << " ProxyCommand #{ssh_exec} -q -W %h:%p #{gateway_user}@#{gateway}\n" unless gateway.nil?
|
|
342
377
|
if @passwords.key?(node)
|
|
343
378
|
config_content << " PreferredAuthentications password\n"
|
|
344
379
|
config_content << " PubkeyAuthentication no\n"
|
|
345
380
|
end
|
|
346
|
-
rescue NotConnectableError
|
|
347
|
-
config_content << "# #{node} - Not connectable using SSH - #{@nodes_handler.get_description_of(node) || ''}\n"
|
|
348
381
|
end
|
|
349
382
|
config_content << "\n"
|
|
350
383
|
end
|
|
384
|
+
# Add global definitions at the end of the SSH config, as they might be overriden by previous ones, and first match wins.
|
|
385
|
+
config_content << <<~EOS
|
|
386
|
+
###########
|
|
387
|
+
# GLOBALS #
|
|
388
|
+
###########
|
|
389
|
+
|
|
390
|
+
Host *
|
|
391
|
+
User #{@ssh_user}
|
|
392
|
+
# Default control socket path to be used when multiplexing SSH connections
|
|
393
|
+
ControlPath #{control_master_file('%h', '%p', '%r')}
|
|
394
|
+
#{open_ssh_major_version >= 7 ? 'PubkeyAcceptedKeyTypes +ssh-dss' : ''}
|
|
395
|
+
#{known_hosts_file.nil? ? '' : "UserKnownHostsFile #{known_hosts_file}"}
|
|
396
|
+
#{@ssh_strict_host_key_checking ? '' : 'StrictHostKeyChecking no'}
|
|
397
|
+
|
|
398
|
+
EOS
|
|
351
399
|
config_content
|
|
352
400
|
end
|
|
353
401
|
|
|
@@ -450,34 +498,48 @@ module HybridPlatformsConductor
|
|
|
450
498
|
with_lock_on_control_master_for(node) do |current_users, user_id|
|
|
451
499
|
working_master = false
|
|
452
500
|
ssh_exec = ssh_exec_for(node)
|
|
453
|
-
ssh_url = "
|
|
501
|
+
ssh_url = "hpc.#{node}"
|
|
454
502
|
if current_users.empty?
|
|
455
503
|
log_debug "[ ControlMaster - #{ssh_url} ] - Creating SSH ControlMaster..."
|
|
456
|
-
# Create the control master
|
|
457
|
-
ssh_control_master_start_cmd = "#{ssh_exec}#{@passwords.key?(node) || @auth_password ? '' : ' -o BatchMode=yes'} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url} true"
|
|
458
504
|
exit_status = nil
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
505
|
+
if @nodes_handler.get_ssh_session_exec_of(node) == 'false'
|
|
506
|
+
# Here we have to create a ControlMaster using an interactive session, as the SSH server prohibits ExecSession, and so command executions.
|
|
507
|
+
# We'll do that using another terminal spawned in the background.
|
|
508
|
+
Thread.new do
|
|
509
|
+
log_debug "[ ControlMaster - #{ssh_url} ] - Spawn interactive ControlMaster in separate terminal"
|
|
510
|
+
@cmd_runner.run_cmd "xterm -e '#{ssh_exec} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url}'", log_to_stdout: log_debug?
|
|
511
|
+
log_debug "[ ControlMaster - #{ssh_url} ] - Separate interactive ControlMaster closed"
|
|
512
|
+
end
|
|
513
|
+
out 'External ControlMaster has been spawned.'
|
|
514
|
+
out 'Please login into it, keep its session opened and press enter here when done...'
|
|
515
|
+
$stdin.gets
|
|
516
|
+
exit_status = 0
|
|
517
|
+
else
|
|
518
|
+
# Create the control master
|
|
519
|
+
ssh_control_master_start_cmd = "#{ssh_exec}#{@passwords.key?(node) || @auth_password ? '' : ' -o BatchMode=yes'} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url} true"
|
|
520
|
+
idx_try = 0
|
|
521
|
+
loop do
|
|
522
|
+
stderr = nil
|
|
523
|
+
exit_status, _stdout, stderr = @cmd_runner.run_cmd ssh_control_master_start_cmd, log_to_stdout: log_debug?, no_exception: true, timeout: timeout
|
|
524
|
+
if exit_status == 0
|
|
525
|
+
break
|
|
526
|
+
elsif stderr =~ /System is booting up/
|
|
527
|
+
if idx_try == MAX_RETRIES_FOR_BOOT
|
|
528
|
+
if no_exception
|
|
529
|
+
break
|
|
530
|
+
else
|
|
531
|
+
raise ActionsExecutor::ConnectionError, "Tried #{idx_try} times to create SSH Control Master with #{ssh_control_master_start_cmd} but system says it's booting up."
|
|
532
|
+
end
|
|
471
533
|
end
|
|
534
|
+
# Wait a bit and try again
|
|
535
|
+
idx_try += 1
|
|
536
|
+
log_debug "[ ControlMaster - #{ssh_url} ] - System is booting up (try ##{idx_try}). Wait #{WAIT_TIME_FOR_BOOT} seconds before trying ControlMaster's creation again."
|
|
537
|
+
sleep WAIT_TIME_FOR_BOOT
|
|
538
|
+
elsif no_exception
|
|
539
|
+
break
|
|
540
|
+
else
|
|
541
|
+
raise ActionsExecutor::ConnectionError, "Error while starting SSH Control Master with #{ssh_control_master_start_cmd}: #{stderr.strip}"
|
|
472
542
|
end
|
|
473
|
-
# Wait a bit and try again
|
|
474
|
-
idx_try += 1
|
|
475
|
-
log_debug "[ ControlMaster - #{ssh_url} ] - System is booting up (try ##{idx_try}). Wait #{WAIT_TIME_FOR_BOOT} seconds before trying ControlMaster's creation again."
|
|
476
|
-
sleep WAIT_TIME_FOR_BOOT
|
|
477
|
-
elsif no_exception
|
|
478
|
-
break
|
|
479
|
-
else
|
|
480
|
-
raise ActionsExecutor::ConnectionError, "Error while starting SSH Control Master with #{ssh_control_master_start_cmd}: #{stderr.strip}"
|
|
481
543
|
end
|
|
482
544
|
end
|
|
483
545
|
if exit_status == 0
|
|
@@ -512,28 +574,33 @@ module HybridPlatformsConductor
|
|
|
512
574
|
end
|
|
513
575
|
end
|
|
514
576
|
end
|
|
577
|
+
else
|
|
578
|
+
# We have not created any ControlMaster, but still consider the nodes to be ready to connect
|
|
579
|
+
user_locks = Hash[nodes.map { |node| [node, nil]} ]
|
|
515
580
|
end
|
|
516
581
|
yield user_locks.keys
|
|
517
582
|
ensure
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
583
|
+
if @ssh_use_control_master
|
|
584
|
+
user_locks_mutex.synchronize do
|
|
585
|
+
user_locks.each do |node, user_id|
|
|
586
|
+
with_lock_on_control_master_for(node, user_id: user_id) do |current_users, user_id|
|
|
587
|
+
ssh_url = "hpc.#{node}"
|
|
588
|
+
log_warn "[ ControlMaster - #{ssh_url} ] - Current process/thread was not part of the ControlMaster users anymore whereas it should have been" unless current_users.include?(user_id)
|
|
589
|
+
remaining_users = current_users - [user_id]
|
|
590
|
+
if remaining_users.empty?
|
|
591
|
+
# Stop the ControlMaster
|
|
592
|
+
log_debug "[ ControlMaster - #{ssh_url} ] - Stopping ControlMaster..."
|
|
593
|
+
# Dumb verbose ssh! Tricky trick to just silence what is useless.
|
|
594
|
+
# Don't fail if the connection close fails (but still log the error), as it can be seen as only a warning: it means the connection was closed anyway.
|
|
595
|
+
@cmd_runner.run_cmd "#{ssh_exec_for(node)} -O exit #{ssh_url} 2>&1 | grep -v 'Exit request sent.'", log_to_stdout: log_debug?, expected_code: 1, timeout: timeout, no_exception: true
|
|
596
|
+
log_debug "[ ControlMaster - #{ssh_url} ] - ControlMaster stopped"
|
|
597
|
+
# Uncomment if you want to test that the connection has been closed
|
|
598
|
+
# @cmd_runner.run_cmd "#{ssh_exec_for(node)} -O check #{ssh_url}", log_to_stdout: log_debug?, expected_code: 255, timeout: timeout
|
|
599
|
+
else
|
|
600
|
+
log_debug "[ ControlMaster - #{ssh_url} ] - Leaving ControlMaster started as #{remaining_users.size} processes/threads are still using it."
|
|
601
|
+
end
|
|
602
|
+
false
|
|
535
603
|
end
|
|
536
|
-
false
|
|
537
604
|
end
|
|
538
605
|
end
|
|
539
606
|
end
|
|
@@ -562,8 +629,9 @@ module HybridPlatformsConductor
|
|
|
562
629
|
# TODO: Add test case when control file is missing ad when it is stale
|
|
563
630
|
# Get the list of existing process/thread ids using this control master
|
|
564
631
|
existing_users = File.exist?(control_master_users_file) ? File.read(control_master_users_file).split("\n") : []
|
|
565
|
-
ssh_url = "
|
|
566
|
-
|
|
632
|
+
ssh_url = "hpc.#{node}"
|
|
633
|
+
connection, connection_user, _gateway, _gateway_user = connection_info_for(node)
|
|
634
|
+
control_path_file = control_master_file(connection, '22', connection_user)
|
|
567
635
|
if existing_users.empty?
|
|
568
636
|
# Make sure there is no stale one.
|
|
569
637
|
if File.exist?(control_path_file)
|
|
@@ -595,7 +663,7 @@ module HybridPlatformsConductor
|
|
|
595
663
|
# * *port* (String): The port. Can be a string as ssh config uses wildchars.
|
|
596
664
|
# * *user* (String): The user
|
|
597
665
|
def control_master_file(host, port, user)
|
|
598
|
-
"#{@tmp_dir}/
|
|
666
|
+
"#{@tmp_dir}/hpc_ssh_mux_#{host}_#{port}_#{user}"
|
|
599
667
|
end
|
|
600
668
|
|
|
601
669
|
# Provide a bootstrapped ssh executable that includes an SSH config allowing access to nodes.
|
|
@@ -627,7 +695,7 @@ module HybridPlatformsConductor
|
|
|
627
695
|
nodes.sort.each do |node|
|
|
628
696
|
host_keys = @nodes_handler.get_host_keys_of(node)
|
|
629
697
|
if host_keys && !host_keys.empty?
|
|
630
|
-
connection, _gateway, _gateway_user = connection_info_for(node)
|
|
698
|
+
connection, _connection_user, _gateway, _gateway_user = connection_info_for(node)
|
|
631
699
|
host_keys.each do |host_key|
|
|
632
700
|
file.puts "#{connection} #{host_key}"
|
|
633
701
|
end
|
|
@@ -660,11 +728,13 @@ module HybridPlatformsConductor
|
|
|
660
728
|
#
|
|
661
729
|
# Parameters::
|
|
662
730
|
# * *node* (String): The node to access
|
|
731
|
+
# * *no_exception* (Boolean): Should we skip exceptions in case of no connection possible? [default: false]
|
|
663
732
|
# Result::
|
|
664
|
-
# * String: The real hostname or IP to be used to connect
|
|
733
|
+
# * String: The real hostname or IP to be used to connect, or nil if none and no_exception is true
|
|
734
|
+
# * String: The real user to be used to connect, or nil if none and no_exception is true
|
|
665
735
|
# * String or nil: The gateway name to be used (should be defined by the gateways configurations), or nil if no gateway to be used.
|
|
666
736
|
# * String or nil: The gateway user to be used, or nil if none.
|
|
667
|
-
def connection_info_for(node)
|
|
737
|
+
def connection_info_for(node, no_exception: false)
|
|
668
738
|
connection =
|
|
669
739
|
if @nodes_handler.get_host_ip_of(node)
|
|
670
740
|
@nodes_handler.get_host_ip_of(node)
|
|
@@ -673,12 +743,18 @@ module HybridPlatformsConductor
|
|
|
673
743
|
elsif @nodes_handler.get_hostname_of(node)
|
|
674
744
|
@nodes_handler.get_hostname_of(node)
|
|
675
745
|
else
|
|
676
|
-
|
|
746
|
+
nil
|
|
677
747
|
end
|
|
748
|
+
connection_user = @ssh_user
|
|
678
749
|
gateway = @nodes_handler.get_gateway_of node
|
|
679
750
|
gateway_user = @nodes_handler.get_gateway_user_of node
|
|
680
751
|
gateway_user = @ssh_gateway_user if !gateway.nil? && gateway_user.nil?
|
|
681
|
-
|
|
752
|
+
# In case we want to transform the connection info, do it here.
|
|
753
|
+
@nodes_handler.select_confs_for_node(node, @config.ssh_connection_transforms).each do |transform_info|
|
|
754
|
+
connection, connection_user, gateway, gateway_user = transform_info[:transform].call(node, connection, connection_user, gateway, gateway_user)
|
|
755
|
+
end
|
|
756
|
+
raise NotConnectableError, "No connection possible to #{node}" if connection.nil? && !no_exception
|
|
757
|
+
[connection, connection_user, gateway, gateway_user]
|
|
682
758
|
end
|
|
683
759
|
|
|
684
760
|
# Get the possible SSH aliases for a given node.
|
|
@@ -404,7 +404,7 @@ module HybridPlatformsConductor
|
|
|
404
404
|
{
|
|
405
405
|
proxmox_test_info[:sync_node] => {
|
|
406
406
|
remote_bash: {
|
|
407
|
-
commands: "#{@actions_executor.connector(:ssh).ssh_user == 'root' ? '' :
|
|
407
|
+
commands: "#{@actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(proxmox_test_info[:sync_node])} -E "}./proxmox/#{cmd}",
|
|
408
408
|
env: {
|
|
409
409
|
'hpc_user_for_proxmox' => user,
|
|
410
410
|
'hpc_password_for_proxmox' => password,
|
|
@@ -15,7 +15,7 @@ module HybridPlatformsConductor
|
|
|
15
15
|
def test_on_node
|
|
16
16
|
now = Time.now
|
|
17
17
|
{
|
|
18
|
-
|
|
18
|
+
"#{@nodes_handler.sudo_on(@node)} ls -t /var/log/deployments" => proc do |stdout|
|
|
19
19
|
if stdout.empty?
|
|
20
20
|
error 'Node has never been deployed using deploy (/var/log/deployments is empty)'
|
|
21
21
|
elsif stdout.first =~ /No such file or directory/
|