hybrid_platforms_conductor 32.4.0 → 32.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/nodes_to_deploy +11 -5
- data/lib/hybrid_platforms_conductor/deployer.rb +9 -8
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +147 -68
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/platform_handler_plugin.rb.sample +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +6 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +7 -6
- data/lib/hybrid_platforms_conductor/nodes_handler.rb +45 -1
- data/lib/hybrid_platforms_conductor/services_handler.rb +9 -13
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +35 -0
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +57 -2
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +68 -12
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +47 -9
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +28 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +71 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/git_diff_impacts_spec.rb +10 -0
- data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +25 -0
- data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +1 -5
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +37 -9
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +14 -14
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +70 -11
- data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +2 -2
- metadata +12 -11
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 34ae09b6107332c4f51418677736cabed358f01c3e5c47ff2f5752f6c597a809
|
|
4
|
+
data.tar.gz: 99358ffeb3d2f70819c5408c18b5b8d8e5531331da0c96073e5577f02cebcb72
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 873851a3e54538b2915df041e9dca3decde7cbfe4f7e8b82ea7102bbe5e75668ef5d7adb828979f4eab8493245f982b3e5da0bb492c116688b9151d10bc979a2
|
|
7
|
+
data.tar.gz: 72418b66806746d05589f08f773eb065b0811d760e35908410405f1cbd9a6fadc94aa2ec85a83ac7f314f53d6a7ebc6396460894c4f595aee9c9b0ffd65625c0
|
data/bin/nodes_to_deploy
CHANGED
|
@@ -72,11 +72,17 @@ unless ignore_deploy_info
|
|
|
72
72
|
commit_id = node_deploy_info["commit_id_#{repo_idx}".to_sym]
|
|
73
73
|
impacted_nodes = cache_impacted_nodes.dig(repo_name, commit_id)
|
|
74
74
|
if impacted_nodes.nil?
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
75
|
+
begin
|
|
76
|
+
impacted_nodes, _single_impacted_nodes, _impacted_services, _impact_global = nodes_handler.impacted_nodes_from_git_diff(
|
|
77
|
+
repo_name,
|
|
78
|
+
from_commit: commit_id,
|
|
79
|
+
to_commit: 'master'
|
|
80
|
+
)
|
|
81
|
+
rescue HybridPlatformsConductor::NodesHandler::GitError
|
|
82
|
+
# Consider the node was deployed with a non-release branch commit (as it is missing)
|
|
83
|
+
# So we have to make sure we deploy it again
|
|
84
|
+
impacted_nodes = [node]
|
|
85
|
+
end
|
|
80
86
|
cache_impacted_nodes[repo_name] = {} unless cache_impacted_nodes.key?(repo_name)
|
|
81
87
|
cache_impacted_nodes[repo_name][commit_id] = impacted_nodes
|
|
82
88
|
end
|
|
@@ -480,6 +480,7 @@ module HybridPlatformsConductor
|
|
|
480
480
|
outputs = @actions_executor.execute_actions(
|
|
481
481
|
Hash[services.map do |node, node_services|
|
|
482
482
|
image_id = @nodes_handler.get_image_of(node)
|
|
483
|
+
sudo = (ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} ")
|
|
483
484
|
# Install My_company corporate certificates if present
|
|
484
485
|
certificate_actions =
|
|
485
486
|
if @local_environment && ENV['hpc_certificates']
|
|
@@ -489,20 +490,20 @@ module HybridPlatformsConductor
|
|
|
489
490
|
when 'debian_9', 'debian_10'
|
|
490
491
|
[
|
|
491
492
|
{
|
|
492
|
-
remote_bash: "#{
|
|
493
|
+
remote_bash: "#{sudo}apt update && #{sudo}apt install -y ca-certificates"
|
|
493
494
|
},
|
|
494
495
|
{
|
|
495
496
|
scp: {
|
|
496
497
|
ENV['hpc_certificates'] => '/usr/local/share/ca-certificates',
|
|
497
498
|
:sudo => ssh_user != 'root'
|
|
498
499
|
},
|
|
499
|
-
remote_bash: "#{
|
|
500
|
+
remote_bash: "#{sudo}update-ca-certificates"
|
|
500
501
|
}
|
|
501
502
|
]
|
|
502
503
|
when 'centos_7'
|
|
503
504
|
[
|
|
504
505
|
{
|
|
505
|
-
remote_bash: "#{
|
|
506
|
+
remote_bash: "#{sudo}yum install -y ca-certificates"
|
|
506
507
|
},
|
|
507
508
|
{
|
|
508
509
|
scp: Hash[Dir.glob("#{ENV['hpc_certificates']}/*.crt").map do |cert_file|
|
|
@@ -512,8 +513,8 @@ module HybridPlatformsConductor
|
|
|
512
513
|
]
|
|
513
514
|
end].merge(sudo: ssh_user != 'root'),
|
|
514
515
|
remote_bash: [
|
|
515
|
-
"#{
|
|
516
|
-
"#{
|
|
516
|
+
"#{sudo}update-ca-trust enable",
|
|
517
|
+
"#{sudo}update-ca-trust extract"
|
|
517
518
|
]
|
|
518
519
|
}
|
|
519
520
|
]
|
|
@@ -532,7 +533,7 @@ module HybridPlatformsConductor
|
|
|
532
533
|
# Install the mutex lock and acquire it
|
|
533
534
|
{
|
|
534
535
|
scp: { "#{__dir__}/mutex_dir" => '.' },
|
|
535
|
-
remote_bash: "while ! #{
|
|
536
|
+
remote_bash: "while ! #{sudo}./mutex_dir lock /tmp/hybrid_platforms_conductor_deploy_lock \"$(ps -o ppid= -p $$)\"; do echo -e 'Another deployment is running on #{node}. Waiting for it to finish to continue...' ; sleep 5 ; done"
|
|
536
537
|
}
|
|
537
538
|
] +
|
|
538
539
|
certificate_actions +
|
|
@@ -548,7 +549,7 @@ module HybridPlatformsConductor
|
|
|
548
549
|
Hash[services.keys.map do |node|
|
|
549
550
|
[
|
|
550
551
|
node,
|
|
551
|
-
{ remote_bash: "#{ssh_user == 'root' ? '' :
|
|
552
|
+
{ remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
|
|
552
553
|
]
|
|
553
554
|
end],
|
|
554
555
|
timeout: 10,
|
|
@@ -595,7 +596,7 @@ module HybridPlatformsConductor
|
|
|
595
596
|
[
|
|
596
597
|
node,
|
|
597
598
|
{
|
|
598
|
-
remote_bash: "#{ssh_user == 'root' ? '' :
|
|
599
|
+
remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}mkdir -p /var/log/deployments",
|
|
599
600
|
scp: {
|
|
600
601
|
log_file => '/var/log/deployments',
|
|
601
602
|
:sudo => ssh_user != 'root',
|
|
@@ -10,13 +10,46 @@ module HybridPlatformsConductor
|
|
|
10
10
|
# Connect to node using SSH
|
|
11
11
|
class Ssh < HybridPlatformsConductor::Connector
|
|
12
12
|
|
|
13
|
+
# Exception raise when a node is not connectable using SSH
|
|
14
|
+
class NotConnectableError < RuntimeError
|
|
15
|
+
end
|
|
16
|
+
|
|
13
17
|
module PlatformsDslSsh
|
|
14
18
|
|
|
19
|
+
# List of SSH connection transformations:
|
|
20
|
+
# * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
|
|
21
|
+
# * *transform* (Proc): Code called to transform SSH connection info:
|
|
22
|
+
# Parameters::
|
|
23
|
+
# * *node* (String): Node for which we transform the SSH connection
|
|
24
|
+
# * *connection* (String or nil): The connection host or IP, or nil if none
|
|
25
|
+
# * *connection_user* (String): The connection user
|
|
26
|
+
# * *gateway* (String or nil): The gateway name, or nil if none
|
|
27
|
+
# * *gateway_user* (String or nil): The gateway user, or nil if none
|
|
28
|
+
# Result::
|
|
29
|
+
# * String: The transformed connection host or IP, or nil if none
|
|
30
|
+
# * String: The transformed connection user
|
|
31
|
+
# * String or nil: The transformed gateway name, or nil if none
|
|
32
|
+
# * String or nil: The transformed gateway user, or nil if none
|
|
33
|
+
# Array< Hash<Symbol, Object> >
|
|
34
|
+
attr_reader :ssh_connection_transforms
|
|
35
|
+
|
|
15
36
|
# Initialize the DSL
|
|
16
37
|
def init_ssh
|
|
17
38
|
# List of gateway configurations, per gateway config name
|
|
18
39
|
# Hash<Symbol, String>
|
|
19
40
|
@gateways = {}
|
|
41
|
+
@ssh_connection_transforms = []
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Define a transformation of SSH connection.
|
|
45
|
+
#
|
|
46
|
+
# Parameters::
|
|
47
|
+
# * *transform* (Proc): Code to be called to transform an SSH connection (see ssh_connection_transforms signature for details)
|
|
48
|
+
def transform_ssh_connection(&transform)
|
|
49
|
+
@ssh_connection_transforms << {
|
|
50
|
+
nodes_selectors_stack: current_nodes_selectors_stack,
|
|
51
|
+
transform: transform
|
|
52
|
+
}
|
|
20
53
|
end
|
|
21
54
|
|
|
22
55
|
# Register a new gateway configuration
|
|
@@ -200,7 +233,13 @@ module HybridPlatformsConductor
|
|
|
200
233
|
# Parameters::
|
|
201
234
|
# * *bash_cmds* (String): Bash commands to execute
|
|
202
235
|
def remote_bash(bash_cmds)
|
|
203
|
-
ssh_cmd =
|
|
236
|
+
ssh_cmd =
|
|
237
|
+
if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
|
|
238
|
+
# When ExecSession is disabled we need to use stdin directly
|
|
239
|
+
"{ cat | #{ssh_exec} #{ssh_url} -T; } <<'EOF'\n#{bash_cmds}\nEOF"
|
|
240
|
+
else
|
|
241
|
+
"#{ssh_exec} #{ssh_url} /bin/bash <<'EOF'\n#{bash_cmds}\nEOF"
|
|
242
|
+
end
|
|
204
243
|
# Due to a limitation of Process.spawn, each individual argument is limited to 128KB of size.
|
|
205
244
|
# Therefore we need to make sure that if bash_cmds exceeds MAX_CMD_ARG_LENGTH bytes (considering EOF chars) then we use an intermediary shell script to store the commands.
|
|
206
245
|
if bash_cmds.size > MAX_CMD_ARG_LENGTH
|
|
@@ -257,25 +296,30 @@ module HybridPlatformsConductor
|
|
|
257
296
|
# * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
|
|
258
297
|
# * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
|
|
259
298
|
def remote_copy(from, to, sudo: false, owner: nil, group: nil)
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
#{ssh_exec} \
|
|
270
|
-
#{ssh_url} \
|
|
271
|
-
\"#{sudo ? 'sudo ' : ''}tar \
|
|
272
|
-
--extract \
|
|
273
|
-
--gunzip \
|
|
299
|
+
if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
|
|
300
|
+
# We don't have ExecSession, so don't use ssh, but scp instead.
|
|
301
|
+
run_cmd "scp -S #{ssh_exec} #{from} #{ssh_url}:#{to}"
|
|
302
|
+
else
|
|
303
|
+
run_cmd <<~EOS
|
|
304
|
+
cd #{File.dirname(from)} && \
|
|
305
|
+
tar \
|
|
306
|
+
--create \
|
|
307
|
+
--gzip \
|
|
274
308
|
--file - \
|
|
275
|
-
--
|
|
276
|
-
--
|
|
277
|
-
|
|
278
|
-
|
|
309
|
+
#{owner.nil? ? '' : "--owner #{owner}"} \
|
|
310
|
+
#{group.nil? ? '' : "--group #{group}"} \
|
|
311
|
+
#{File.basename(from)} | \
|
|
312
|
+
#{ssh_exec} \
|
|
313
|
+
#{ssh_url} \
|
|
314
|
+
\"#{sudo ? "#{@nodes_handler.sudo_on(@node)} " : ''}tar \
|
|
315
|
+
--extract \
|
|
316
|
+
--gunzip \
|
|
317
|
+
--file - \
|
|
318
|
+
--directory #{to} \
|
|
319
|
+
--owner root \
|
|
320
|
+
\"
|
|
321
|
+
EOS
|
|
322
|
+
end
|
|
279
323
|
end
|
|
280
324
|
|
|
281
325
|
# Get the ssh executable to be used when connecting to the current node
|
|
@@ -291,7 +335,7 @@ module HybridPlatformsConductor
|
|
|
291
335
|
# Result::
|
|
292
336
|
# * String: The ssh URL connecting to the current node
|
|
293
337
|
def ssh_url
|
|
294
|
-
"
|
|
338
|
+
"hpc.#{@node}"
|
|
295
339
|
end
|
|
296
340
|
|
|
297
341
|
# Get an SSH configuration content giving access to nodes of the platforms with the current configuration
|
|
@@ -314,14 +358,6 @@ module HybridPlatformsConductor
|
|
|
314
358
|
# ENDPOINTS #
|
|
315
359
|
#############
|
|
316
360
|
|
|
317
|
-
Host *
|
|
318
|
-
User #{@ssh_user}
|
|
319
|
-
# Default control socket path to be used when multiplexing SSH connections
|
|
320
|
-
ControlPath #{control_master_file('%h', '%p', '%r')}
|
|
321
|
-
#{open_ssh_major_version >= 7 ? 'PubkeyAcceptedKeyTypes +ssh-dss' : ''}
|
|
322
|
-
#{known_hosts_file.nil? ? '' : "UserKnownHostsFile #{known_hosts_file}"}
|
|
323
|
-
#{@ssh_strict_host_key_checking ? '' : 'StrictHostKeyChecking no'}
|
|
324
|
-
|
|
325
361
|
EOS
|
|
326
362
|
|
|
327
363
|
# Add each node
|
|
@@ -329,17 +365,37 @@ module HybridPlatformsConductor
|
|
|
329
365
|
@nodes_handler.prefetch_metadata_of nodes, %i[private_ips hostname host_ip description]
|
|
330
366
|
nodes.sort.each do |node|
|
|
331
367
|
# Generate the conf for the node
|
|
332
|
-
connection, gateway, gateway_user = connection_info_for(node)
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
config_content << "
|
|
339
|
-
config_content << "
|
|
368
|
+
connection, connection_user, gateway, gateway_user = connection_info_for(node, no_exception: true)
|
|
369
|
+
if connection.nil?
|
|
370
|
+
config_content << "# #{node} - Not connectable using SSH - #{@nodes_handler.get_description_of(node) || ''}\n"
|
|
371
|
+
else
|
|
372
|
+
config_content << "# #{node} - #{connection} - #{@nodes_handler.get_description_of(node) || ''}\n"
|
|
373
|
+
config_content << "Host #{ssh_aliases_for(node).join(' ')}\n"
|
|
374
|
+
config_content << " Hostname #{connection}\n"
|
|
375
|
+
config_content << " User \"#{connection_user}\"\n" if connection_user != @ssh_user
|
|
376
|
+
config_content << " ProxyCommand #{ssh_exec} -q -W %h:%p #{gateway_user}@#{gateway}\n" unless gateway.nil?
|
|
377
|
+
if @passwords.key?(node)
|
|
378
|
+
config_content << " PreferredAuthentications password\n"
|
|
379
|
+
config_content << " PubkeyAuthentication no\n"
|
|
380
|
+
end
|
|
340
381
|
end
|
|
341
382
|
config_content << "\n"
|
|
342
383
|
end
|
|
384
|
+
# Add global definitions at the end of the SSH config, as they might be overriden by previous ones, and first match wins.
|
|
385
|
+
config_content << <<~EOS
|
|
386
|
+
###########
|
|
387
|
+
# GLOBALS #
|
|
388
|
+
###########
|
|
389
|
+
|
|
390
|
+
Host *
|
|
391
|
+
User #{@ssh_user}
|
|
392
|
+
# Default control socket path to be used when multiplexing SSH connections
|
|
393
|
+
ControlPath #{control_master_file('%h', '%p', '%r')}
|
|
394
|
+
#{open_ssh_major_version >= 7 ? 'PubkeyAcceptedKeyTypes +ssh-dss' : ''}
|
|
395
|
+
#{known_hosts_file.nil? ? '' : "UserKnownHostsFile #{known_hosts_file}"}
|
|
396
|
+
#{@ssh_strict_host_key_checking ? '' : 'StrictHostKeyChecking no'}
|
|
397
|
+
|
|
398
|
+
EOS
|
|
343
399
|
config_content
|
|
344
400
|
end
|
|
345
401
|
|
|
@@ -442,34 +498,48 @@ module HybridPlatformsConductor
|
|
|
442
498
|
with_lock_on_control_master_for(node) do |current_users, user_id|
|
|
443
499
|
working_master = false
|
|
444
500
|
ssh_exec = ssh_exec_for(node)
|
|
445
|
-
ssh_url = "
|
|
501
|
+
ssh_url = "hpc.#{node}"
|
|
446
502
|
if current_users.empty?
|
|
447
503
|
log_debug "[ ControlMaster - #{ssh_url} ] - Creating SSH ControlMaster..."
|
|
448
|
-
# Create the control master
|
|
449
|
-
ssh_control_master_start_cmd = "#{ssh_exec}#{@passwords.key?(node) || @auth_password ? '' : ' -o BatchMode=yes'} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url} true"
|
|
450
504
|
exit_status = nil
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
505
|
+
if @nodes_handler.get_ssh_session_exec_of(node) == 'false'
|
|
506
|
+
# Here we have to create a ControlMaster using an interactive session, as the SSH server prohibits ExecSession, and so command executions.
|
|
507
|
+
# We'll do that using another terminal spawned in the background.
|
|
508
|
+
Thread.new do
|
|
509
|
+
log_debug "[ ControlMaster - #{ssh_url} ] - Spawn interactive ControlMaster in separate terminal"
|
|
510
|
+
@cmd_runner.run_cmd "xterm -e '#{ssh_exec} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url}'", log_to_stdout: log_debug?
|
|
511
|
+
log_debug "[ ControlMaster - #{ssh_url} ] - Separate interactive ControlMaster closed"
|
|
512
|
+
end
|
|
513
|
+
out 'External ControlMaster has been spawned.'
|
|
514
|
+
out 'Please login into it, keep its session opened and press enter here when done...'
|
|
515
|
+
$stdin.gets
|
|
516
|
+
exit_status = 0
|
|
517
|
+
else
|
|
518
|
+
# Create the control master
|
|
519
|
+
ssh_control_master_start_cmd = "#{ssh_exec}#{@passwords.key?(node) || @auth_password ? '' : ' -o BatchMode=yes'} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url} true"
|
|
520
|
+
idx_try = 0
|
|
521
|
+
loop do
|
|
522
|
+
stderr = nil
|
|
523
|
+
exit_status, _stdout, stderr = @cmd_runner.run_cmd ssh_control_master_start_cmd, log_to_stdout: log_debug?, no_exception: true, timeout: timeout
|
|
524
|
+
if exit_status == 0
|
|
525
|
+
break
|
|
526
|
+
elsif stderr =~ /System is booting up/
|
|
527
|
+
if idx_try == MAX_RETRIES_FOR_BOOT
|
|
528
|
+
if no_exception
|
|
529
|
+
break
|
|
530
|
+
else
|
|
531
|
+
raise ActionsExecutor::ConnectionError, "Tried #{idx_try} times to create SSH Control Master with #{ssh_control_master_start_cmd} but system says it's booting up."
|
|
532
|
+
end
|
|
463
533
|
end
|
|
534
|
+
# Wait a bit and try again
|
|
535
|
+
idx_try += 1
|
|
536
|
+
log_debug "[ ControlMaster - #{ssh_url} ] - System is booting up (try ##{idx_try}). Wait #{WAIT_TIME_FOR_BOOT} seconds before trying ControlMaster's creation again."
|
|
537
|
+
sleep WAIT_TIME_FOR_BOOT
|
|
538
|
+
elsif no_exception
|
|
539
|
+
break
|
|
540
|
+
else
|
|
541
|
+
raise ActionsExecutor::ConnectionError, "Error while starting SSH Control Master with #{ssh_control_master_start_cmd}: #{stderr.strip}"
|
|
464
542
|
end
|
|
465
|
-
# Wait a bit and try again
|
|
466
|
-
idx_try += 1
|
|
467
|
-
log_debug "[ ControlMaster - #{ssh_url} ] - System is booting up (try ##{idx_try}). Wait #{WAIT_TIME_FOR_BOOT} seconds before trying ControlMaster's creation again."
|
|
468
|
-
sleep WAIT_TIME_FOR_BOOT
|
|
469
|
-
elsif no_exception
|
|
470
|
-
break
|
|
471
|
-
else
|
|
472
|
-
raise ActionsExecutor::ConnectionError, "Error while starting SSH Control Master with #{ssh_control_master_start_cmd}: #{stderr.strip}"
|
|
473
543
|
end
|
|
474
544
|
end
|
|
475
545
|
if exit_status == 0
|
|
@@ -510,7 +580,7 @@ module HybridPlatformsConductor
|
|
|
510
580
|
user_locks_mutex.synchronize do
|
|
511
581
|
user_locks.each do |node, user_id|
|
|
512
582
|
with_lock_on_control_master_for(node, user_id: user_id) do |current_users, user_id|
|
|
513
|
-
ssh_url = "
|
|
583
|
+
ssh_url = "hpc.#{node}"
|
|
514
584
|
log_warn "[ ControlMaster - #{ssh_url} ] - Current process/thread was not part of the ControlMaster users anymore whereas it should have been" unless current_users.include?(user_id)
|
|
515
585
|
remaining_users = current_users - [user_id]
|
|
516
586
|
if remaining_users.empty?
|
|
@@ -554,8 +624,9 @@ module HybridPlatformsConductor
|
|
|
554
624
|
# TODO: Add test case when control file is missing ad when it is stale
|
|
555
625
|
# Get the list of existing process/thread ids using this control master
|
|
556
626
|
existing_users = File.exist?(control_master_users_file) ? File.read(control_master_users_file).split("\n") : []
|
|
557
|
-
ssh_url = "
|
|
558
|
-
|
|
627
|
+
ssh_url = "hpc.#{node}"
|
|
628
|
+
connection, connection_user, _gateway, _gateway_user = connection_info_for(node)
|
|
629
|
+
control_path_file = control_master_file(connection, '22', connection_user)
|
|
559
630
|
if existing_users.empty?
|
|
560
631
|
# Make sure there is no stale one.
|
|
561
632
|
if File.exist?(control_path_file)
|
|
@@ -587,7 +658,7 @@ module HybridPlatformsConductor
|
|
|
587
658
|
# * *port* (String): The port. Can be a string as ssh config uses wildchars.
|
|
588
659
|
# * *user* (String): The user
|
|
589
660
|
def control_master_file(host, port, user)
|
|
590
|
-
"#{@tmp_dir}/
|
|
661
|
+
"#{@tmp_dir}/hpc_ssh_mux_#{host}_#{port}_#{user}"
|
|
591
662
|
end
|
|
592
663
|
|
|
593
664
|
# Provide a bootstrapped ssh executable that includes an SSH config allowing access to nodes.
|
|
@@ -619,7 +690,7 @@ module HybridPlatformsConductor
|
|
|
619
690
|
nodes.sort.each do |node|
|
|
620
691
|
host_keys = @nodes_handler.get_host_keys_of(node)
|
|
621
692
|
if host_keys && !host_keys.empty?
|
|
622
|
-
connection, _gateway, _gateway_user = connection_info_for(node)
|
|
693
|
+
connection, _connection_user, _gateway, _gateway_user = connection_info_for(node)
|
|
623
694
|
host_keys.each do |host_key|
|
|
624
695
|
file.puts "#{connection} #{host_key}"
|
|
625
696
|
end
|
|
@@ -652,11 +723,13 @@ module HybridPlatformsConductor
|
|
|
652
723
|
#
|
|
653
724
|
# Parameters::
|
|
654
725
|
# * *node* (String): The node to access
|
|
726
|
+
# * *no_exception* (Boolean): Should we skip exceptions in case of no connection possible? [default: false]
|
|
655
727
|
# Result::
|
|
656
|
-
# * String: The real hostname or IP to be used to connect
|
|
728
|
+
# * String: The real hostname or IP to be used to connect, or nil if none and no_exception is true
|
|
729
|
+
# * String: The real user to be used to connect, or nil if none and no_exception is true
|
|
657
730
|
# * String or nil: The gateway name to be used (should be defined by the gateways configurations), or nil if no gateway to be used.
|
|
658
731
|
# * String or nil: The gateway user to be used, or nil if none.
|
|
659
|
-
def connection_info_for(node)
|
|
732
|
+
def connection_info_for(node, no_exception: false)
|
|
660
733
|
connection =
|
|
661
734
|
if @nodes_handler.get_host_ip_of(node)
|
|
662
735
|
@nodes_handler.get_host_ip_of(node)
|
|
@@ -665,12 +738,18 @@ module HybridPlatformsConductor
|
|
|
665
738
|
elsif @nodes_handler.get_hostname_of(node)
|
|
666
739
|
@nodes_handler.get_hostname_of(node)
|
|
667
740
|
else
|
|
668
|
-
|
|
741
|
+
nil
|
|
669
742
|
end
|
|
743
|
+
connection_user = @ssh_user
|
|
670
744
|
gateway = @nodes_handler.get_gateway_of node
|
|
671
745
|
gateway_user = @nodes_handler.get_gateway_user_of node
|
|
672
746
|
gateway_user = @ssh_gateway_user if !gateway.nil? && gateway_user.nil?
|
|
673
|
-
|
|
747
|
+
# In case we want to transform the connection info, do it here.
|
|
748
|
+
@nodes_handler.select_confs_for_node(node, @config.ssh_connection_transforms).each do |transform_info|
|
|
749
|
+
connection, connection_user, gateway, gateway_user = transform_info[:transform].call(node, connection, connection_user, gateway, gateway_user)
|
|
750
|
+
end
|
|
751
|
+
raise NotConnectableError, "No connection possible to #{node}" if connection.nil? && !no_exception
|
|
752
|
+
[connection, connection_user, gateway, gateway_user]
|
|
674
753
|
end
|
|
675
754
|
|
|
676
755
|
# Get the possible SSH aliases for a given node.
|
data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/platform_handler_plugin.rb.sample
CHANGED
|
@@ -173,7 +173,7 @@ module HybridPlatformsConductor
|
|
|
173
173
|
# * *local_environment* (Boolean): Are we deploying to a local environment?
|
|
174
174
|
def package(services:, secrets:, local_environment:)
|
|
175
175
|
# This method should take all actions to prepare the repository to be deployed on nodes later.
|
|
176
|
-
File.write(
|
|
176
|
+
File.write("#{@repository_path}/temporary_secrets_to_be_deployed.json", secrets.to_json)
|
|
177
177
|
# Usually it is meant to package the deployment scripts.
|
|
178
178
|
@cmd_runner.run_cmd "cd #{@repository_path} && ./scripts/package_in_debian_format.sh"
|
|
179
179
|
end
|
|
@@ -404,7 +404,7 @@ module HybridPlatformsConductor
|
|
|
404
404
|
{
|
|
405
405
|
proxmox_test_info[:sync_node] => {
|
|
406
406
|
remote_bash: {
|
|
407
|
-
commands: "#{@actions_executor.connector(:ssh).ssh_user == 'root' ? '' :
|
|
407
|
+
commands: "#{@actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(proxmox_test_info[:sync_node])} -E "}./proxmox/#{cmd}",
|
|
408
408
|
env: {
|
|
409
409
|
'hpc_user_for_proxmox' => user,
|
|
410
410
|
'hpc_password_for_proxmox' => password,
|
|
@@ -15,7 +15,7 @@ module HybridPlatformsConductor
|
|
|
15
15
|
def test_on_node
|
|
16
16
|
now = Time.now
|
|
17
17
|
{
|
|
18
|
-
|
|
18
|
+
"#{@nodes_handler.sudo_on(@node)} ls -t /var/log/deployments" => proc do |stdout|
|
|
19
19
|
if stdout.empty?
|
|
20
20
|
error 'Node has never been deployed using deploy (/var/log/deployments is empty)'
|
|
21
21
|
elsif stdout.first =~ /No such file or directory/
|