hybrid_platforms_conductor 33.9.0 → 33.9.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +39 -0
- data/bin/last_deploys +3 -4
- data/bin/nodes_to_deploy +19 -9
- data/docs/gen/mermaid/README.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_veids.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/setup.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
- data/lib/hybrid_platforms_conductor/actions_executor.rb +2 -2
- data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +2 -2
- data/lib/hybrid_platforms_conductor/config.rb +1 -0
- data/lib/hybrid_platforms_conductor/deployer.rb +19 -13
- data/lib/hybrid_platforms_conductor/executable.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +21 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +6 -6
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/keepass.rb +6 -6
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +2 -2
- data/lib/hybrid_platforms_conductor/plugins.rb +4 -3
- data/lib/hybrid_platforms_conductor/tests_runner.rb +2 -2
- data/lib/hybrid_platforms_conductor/topographer.rb +5 -6
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/helpers_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/deployer/deploy_spec.rb +44 -0
- data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +12 -12
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/docker_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +27 -27
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +25 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +9 -9
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +30 -2
- data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +7 -7
- data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +18 -18
- data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +30 -30
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_2.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others_2.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/check_node_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +75 -0
- data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +7 -7
- data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +11 -12
- data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem_with_lib/sub1/lib/sub2/hpc_plugins/test_plugin_type_3/test_plugin_id_5.rb +34 -0
- data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/test_connector.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/test_log_plugin.rb +2 -2
- data/spec/hybrid_platforms_conductor_test.rb +2 -0
- metadata +152 -136
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 433e01481e26d03667cde79b93697f0f14e949a6201931c6e570ebb84670e79a
|
4
|
+
data.tar.gz: 61c1eb5c9b9ec3df221528e8ae1c08a7a6bdf3628c2a6a83b1756e11a84a2e88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 31ff562ab7108537b8ba57b8c657ef516ef5665fb40fc81d4afe4f14a54400f62324597136845af8c39bd8788318d3d12c54362562cb73dff9d769c60cf8bce1
|
7
|
+
data.tar.gz: cb7dfea944204a89f9522703fb588ec8085dcd9a92816abe0cc012f0e7440432ee4a3831825517ae9b74f59e1ac1ff7d4995b6fa75220c28096de1a8e95f31b6
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,42 @@
|
|
1
|
+
# [v33.9.4](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.3...v33.9.4) (2022-03-20 16:20:04)
|
2
|
+
|
3
|
+
### Patches
|
4
|
+
|
5
|
+
* [Added missing tests and corrections from PR #119](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/e86be2f99e273e65bf9073c1b389a10ad53d7353)
|
6
|
+
* [Merge branch 'rebase_pull_116'](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/4030ba58e0e9cf4ef96328a32cf4daa6226270c0)
|
7
|
+
|
8
|
+
# [v33.9.3](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.2...v33.9.3) (2022-03-20 00:06:14)
|
9
|
+
|
10
|
+
### Patches
|
11
|
+
|
12
|
+
* [Fix semantic-release dependencies](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/68ec49fbec772c1de6971173830a85b76398de42)
|
13
|
+
* [Work-around IceCube dependency issue and correct rubocop errors](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/68858e4d5fbb9d91fcee0525f56dfc40850af7df)
|
14
|
+
* [fixing issue 118](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/bb917101c336119104889e5d6cedd34aef538f54)
|
15
|
+
|
16
|
+
# [v33.9.2](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.1...v33.9.2) (2021-09-01 17:46:34)
|
17
|
+
|
18
|
+
## Global changes
|
19
|
+
### Patches
|
20
|
+
|
21
|
+
* [[Fix(nodes_to_deploy)] [#112] Support UTF-8 encoding from log files](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/26a7bd2fd53d387bd972b5e1b60cac9be6541ce5)
|
22
|
+
|
23
|
+
## Changes for nodes_to_deploy
|
24
|
+
### Patches
|
25
|
+
|
26
|
+
* [[Fix(nodes_to_deploy)] [#112] Support UTF-8 encoding from log files](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/26a7bd2fd53d387bd972b5e1b60cac9be6541ce5)
|
27
|
+
|
28
|
+
# [v33.9.1](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.0...v33.9.1) (2021-09-01 16:59:59)
|
29
|
+
|
30
|
+
## Global changes
|
31
|
+
### Patches
|
32
|
+
|
33
|
+
* [[Fix(nodes_to_deploy)] [#110] Make sure unknown platforms are taken into account when detecting nodes to be deployed](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/c1b8312c5d473a1c5d044f6cda3ad9bed2b3ca61)
|
34
|
+
|
35
|
+
## Changes for nodes_to_deploy
|
36
|
+
### Patches
|
37
|
+
|
38
|
+
* [[Fix(nodes_to_deploy)] [#110] Make sure unknown platforms are taken into account when detecting nodes to be deployed](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/c1b8312c5d473a1c5d044f6cda3ad9bed2b3ca61)
|
39
|
+
|
1
40
|
# [v33.9.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.8.4...v33.9.0) (2021-08-24 13:15:44)
|
2
41
|
|
3
42
|
## Global changes
|
data/bin/last_deploys
CHANGED
@@ -8,12 +8,12 @@ possible_string_sorts = %i[
|
|
8
8
|
node
|
9
9
|
user
|
10
10
|
]
|
11
|
-
possible_sorts = possible_string_sorts.
|
11
|
+
possible_sorts = possible_string_sorts.to_h do |property_name|
|
12
12
|
[
|
13
13
|
property_name,
|
14
14
|
proc { |node, deploy_info| [deploy_info.key?(:error) || !deploy_info.key?(property_name) ? '' : deploy_info[property_name], node] }
|
15
15
|
]
|
16
|
-
end
|
16
|
+
end
|
17
17
|
|
18
18
|
sort_by = :node
|
19
19
|
sort_desc = false
|
@@ -34,7 +34,7 @@ raise "Unknown sort name: #{sort_by}. Should be one of #{possible_sorts.keys.joi
|
|
34
34
|
|
35
35
|
sorted_deploy_info = deployer.
|
36
36
|
deployment_info_from(nodes_handler.select_nodes(executable.selected_nodes.empty? ? [{ all: true }] : executable.selected_nodes)).
|
37
|
-
|
37
|
+
to_h do |node, deploy_info|
|
38
38
|
decorated_deploy_info = deploy_info.merge(node: node)
|
39
39
|
if deploy_info.key?(:deployment_info)
|
40
40
|
decorated_deploy_info.merge!(deploy_info[:deployment_info])
|
@@ -43,7 +43,6 @@ sorted_deploy_info = deployer.
|
|
43
43
|
decorated_deploy_info[:services] = deploy_info[:services].join(', ') if deploy_info.key?(:services)
|
44
44
|
[node, decorated_deploy_info]
|
45
45
|
end.
|
46
|
-
to_h.
|
47
46
|
sort_by(&possible_sorts[sort_by])
|
48
47
|
sorted_deploy_info.reverse! if sort_desc
|
49
48
|
info_displayed = {
|
data/bin/nodes_to_deploy
CHANGED
@@ -17,6 +17,7 @@ executable = HybridPlatformsConductor::Executable.new(deploy_options: false) do
|
|
17
17
|
end
|
18
18
|
end
|
19
19
|
nodes_handler = executable.nodes_handler
|
20
|
+
platforms_handler = executable.platforms_handler
|
20
21
|
deployer = executable.deployer
|
21
22
|
|
22
23
|
executable.parse_options!
|
@@ -72,20 +73,29 @@ unless ignore_deploy_info
|
|
72
73
|
commit_id = node_deploy_info[:deployment_info]["commit_id_#{repo_idx}".to_sym]
|
73
74
|
impacted_nodes = cache_impacted_nodes.dig(repo_name, commit_id)
|
74
75
|
if impacted_nodes.nil?
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
76
|
+
if platforms_handler.platform(repo_name)
|
77
|
+
begin
|
78
|
+
impacted_nodes, _single_impacted_nodes, _impacted_services, _impact_global = nodes_handler.impacted_nodes_from_git_diff(
|
79
|
+
repo_name,
|
80
|
+
from_commit: commit_id,
|
81
|
+
to_commit: 'master'
|
82
|
+
)
|
83
|
+
rescue HybridPlatformsConductor::NodesHandler::GitError
|
84
|
+
# Consider the node was deployed with a non-release branch commit (as it is missing)
|
85
|
+
# So we have to make sure we deploy it again
|
86
|
+
executable.log_warn "[ #{node} ] - Unknown commit ID from deployment logs: #{repo_name} / #{commit_id}."
|
87
|
+
impacted_nodes = :unknown
|
88
|
+
end
|
89
|
+
else
|
90
|
+
# Consider the node was deployed with an obdsolete platform (as it is unknown)
|
83
91
|
# So we have to make sure we deploy it again
|
84
|
-
|
92
|
+
executable.log_warn "[ #{node} ] - Unknown platform from deployment logs: #{repo_name}."
|
93
|
+
impacted_nodes = :unknown
|
85
94
|
end
|
86
95
|
cache_impacted_nodes[repo_name] = {} unless cache_impacted_nodes.key?(repo_name)
|
87
96
|
cache_impacted_nodes[repo_name][commit_id] = impacted_nodes
|
88
97
|
end
|
98
|
+
impacted_nodes = [node] if impacted_nodes == :unknown
|
89
99
|
if impacted_nodes.include?(node)
|
90
100
|
executable.log_debug "[ #{node} ] - Diffs on #{repo_name} between #{commit_id} and master are impacting this node."
|
91
101
|
node_impacted = true
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
@@ -146,7 +146,7 @@ module HybridPlatformsConductor
|
|
146
146
|
actions_per_node[node].concat(resolved_nodes_actions)
|
147
147
|
end
|
148
148
|
end
|
149
|
-
result = actions_per_node.keys.
|
149
|
+
result = actions_per_node.keys.to_h { |node| [node, nil] }
|
150
150
|
with_connections_prepared_to(nodes_needing_connectors, no_exception: true) do |connected_nodes|
|
151
151
|
missing_nodes = []
|
152
152
|
connected_nodes.each do |node, connector|
|
@@ -194,7 +194,7 @@ module HybridPlatformsConductor
|
|
194
194
|
# * *connected_nodes* (Hash<String, Connector or Symbol>): Prepared connectors (or Symbol in case of failure with no_exception), per node name
|
195
195
|
def with_connections_prepared_to(nodes, no_exception: false)
|
196
196
|
# Make sure every node needing connectors finds a connector
|
197
|
-
nodes_needing_connectors = nodes.
|
197
|
+
nodes_needing_connectors = nodes.to_h { |node| [node, nil] }
|
198
198
|
@connector_plugins.each_value do |connector|
|
199
199
|
nodes_without_connectors = nodes_needing_connectors.select { |_node, selected_connector| selected_connector.nil? }.keys
|
200
200
|
break if nodes_without_connectors.empty?
|
@@ -84,7 +84,7 @@ module HybridPlatformsConductor
|
|
84
84
|
select_confs_for_node(node, fs_paths_rules).
|
85
85
|
inject({}) do |merged_paths, paths_info|
|
86
86
|
if paths_info[:context][:file_system_type] == file_system_type
|
87
|
-
merged_paths.merge(paths_info[:paths].
|
87
|
+
merged_paths.merge(paths_info[:paths].to_h do |path|
|
88
88
|
[
|
89
89
|
path,
|
90
90
|
{
|
@@ -92,7 +92,7 @@ module HybridPlatformsConductor
|
|
92
92
|
context: paths_info[:context]
|
93
93
|
}
|
94
94
|
]
|
95
|
-
end
|
95
|
+
end) do |path, rule_info_1, rule_info_2|
|
96
96
|
# Just check that configuration is not inconsistent
|
97
97
|
raise "Inconsistent rule for #{file_system_type} file system checks in configuration for #{node}: #{path} is marked as being both #{rule_info_1[:state]} and #{rule_info_2[:state]}" if rule_info_1[:state] != rule_info_2[:state]
|
98
98
|
|
@@ -263,9 +263,9 @@ module HybridPlatformsConductor
|
|
263
263
|
def deploy_on(*nodes_selectors)
|
264
264
|
# Get the sorted list of services to be deployed, per node
|
265
265
|
# Hash<String, Array<String> >
|
266
|
-
services_to_deploy = @nodes_handler.select_nodes(nodes_selectors.flatten).
|
266
|
+
services_to_deploy = @nodes_handler.select_nodes(nodes_selectors.flatten).to_h do |node|
|
267
267
|
[node, @nodes_handler.get_services_of(node)]
|
268
|
-
end
|
268
|
+
end
|
269
269
|
|
270
270
|
# Get the secrets to be deployed
|
271
271
|
secrets = {}
|
@@ -474,12 +474,18 @@ module HybridPlatformsConductor
|
|
474
474
|
timeout: 10,
|
475
475
|
progress_name: 'Read deployment logs'
|
476
476
|
)
|
477
|
-
nodes.
|
477
|
+
nodes.to_h do |node|
|
478
|
+
exit_code, stdout, stderr = read_actions_results[node] || [nil, nil, nil]
|
478
479
|
[
|
479
480
|
node,
|
480
|
-
@log_plugins[log_plugins_for(node).first].logs_for(
|
481
|
+
@log_plugins[log_plugins_for(node).first].logs_for(
|
482
|
+
node,
|
483
|
+
exit_code,
|
484
|
+
stdout&.force_encoding(Encoding::UTF_8),
|
485
|
+
stderr&.force_encoding(Encoding::UTF_8)
|
486
|
+
)
|
481
487
|
]
|
482
|
-
end
|
488
|
+
end
|
483
489
|
end
|
484
490
|
|
485
491
|
# Parse stdout and stderr of a given deploy run and get the list of tasks with their status
|
@@ -547,7 +553,7 @@ module HybridPlatformsConductor
|
|
547
553
|
# Deploy for real
|
548
554
|
@nodes_handler.prefetch_metadata_of services.keys, :image
|
549
555
|
outputs = @actions_executor.execute_actions(
|
550
|
-
services.
|
556
|
+
services.to_h do |node, node_services|
|
551
557
|
image_id = @nodes_handler.get_image_of(node)
|
552
558
|
need_sudo = !@actions_executor.privileged_access?(node)
|
553
559
|
sudo = @actions_executor.sudo_prefix(node)
|
@@ -577,12 +583,12 @@ module HybridPlatformsConductor
|
|
577
583
|
remote_bash: "#{sudo}yum install -y ca-certificates"
|
578
584
|
},
|
579
585
|
{
|
580
|
-
scp: Dir.glob("#{ENV['hpc_certificates']}/*.crt").
|
586
|
+
scp: Dir.glob("#{ENV['hpc_certificates']}/*.crt").to_h do |cert_file|
|
581
587
|
[
|
582
588
|
cert_file,
|
583
589
|
'/etc/pki/ca-trust/source/anchors'
|
584
590
|
]
|
585
|
-
end.
|
591
|
+
end.merge(sudo: need_sudo),
|
586
592
|
remote_bash: [
|
587
593
|
"#{sudo}update-ca-trust enable",
|
588
594
|
"#{sudo}update-ca-trust extract"
|
@@ -607,19 +613,19 @@ module HybridPlatformsConductor
|
|
607
613
|
certificate_actions +
|
608
614
|
@services_handler.actions_to_deploy_on(node, node_services, @use_why_run)
|
609
615
|
]
|
610
|
-
end
|
616
|
+
end,
|
611
617
|
timeout: @timeout,
|
612
618
|
concurrent: @concurrent_execution,
|
613
619
|
log_to_stdout: !@concurrent_execution
|
614
620
|
)
|
615
621
|
# Free eventual locks
|
616
622
|
@actions_executor.execute_actions(
|
617
|
-
services.keys.
|
623
|
+
services.keys.to_h do |node|
|
618
624
|
[
|
619
625
|
node,
|
620
626
|
{ remote_bash: "#{@actions_executor.sudo_prefix(node)}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
|
621
627
|
]
|
622
|
-
end
|
628
|
+
end,
|
623
629
|
timeout: 10,
|
624
630
|
concurrent: true,
|
625
631
|
log_to_dir: nil
|
@@ -641,7 +647,7 @@ module HybridPlatformsConductor
|
|
641
647
|
section "Saving deployment logs for #{logs.size} nodes" do
|
642
648
|
ssh_user = @actions_executor.connector(:ssh).ssh_user
|
643
649
|
@actions_executor.execute_actions(
|
644
|
-
logs.
|
650
|
+
logs.to_h do |node, (exit_status, stdout, stderr)|
|
645
651
|
[
|
646
652
|
node,
|
647
653
|
log_plugins_for(node).
|
@@ -660,7 +666,7 @@ module HybridPlatformsConductor
|
|
660
666
|
end.
|
661
667
|
flatten(1)
|
662
668
|
]
|
663
|
-
end
|
669
|
+
end,
|
664
670
|
timeout: 10,
|
665
671
|
concurrent: true,
|
666
672
|
log_to_dir: nil,
|
@@ -99,7 +99,7 @@ module HybridPlatformsConductor
|
|
99
99
|
@instantiated_components[component] = HybridPlatformsConductor.const_get(component.to_s.split('_').collect(&:capitalize).join.to_sym).new(
|
100
100
|
logger: @logger,
|
101
101
|
logger_stderr: @logger_stderr,
|
102
|
-
**dependencies.
|
102
|
+
**dependencies.to_h { |dependency| [dependency, send(dependency)] }
|
103
103
|
)
|
104
104
|
end
|
105
105
|
@instantiated_components[component]
|
@@ -73,19 +73,33 @@ module HybridPlatformsConductor
|
|
73
73
|
def ip_for(*hosts)
|
74
74
|
results = {}
|
75
75
|
log_debug "Get IPs of #{hosts.size} hosts..."
|
76
|
+
exit_status, _stdout, _stderr = @cmd_runner.run_cmd('command -v getent', no_exception: true)
|
77
|
+
getent_present = exit_status.zero?
|
76
78
|
for_each_element_in(
|
77
79
|
hosts,
|
78
80
|
parallel: true,
|
79
81
|
nbr_threads_max: MAX_THREADS_GETENT,
|
80
82
|
progress: log_debug? ? 'Gather IPs' : nil
|
81
83
|
) do |host|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
84
|
+
ip =
|
85
|
+
if getent_present
|
86
|
+
_exit_status, stdout, _stderr = @cmd_runner.run_cmd(
|
87
|
+
"getent hosts #{host}",
|
88
|
+
timeout: TIMEOUT_GETENT,
|
89
|
+
log_to_stdout: log_debug?,
|
90
|
+
no_exception: true
|
91
|
+
)
|
92
|
+
stdout.strip.split(/\s+/).first
|
93
|
+
else
|
94
|
+
_exit_status, stdout, _stderr = @cmd_runner.run_cmd(
|
95
|
+
"host #{host} | grep 'has address'",
|
96
|
+
timeout: TIMEOUT_GETENT,
|
97
|
+
log_to_stdout: log_debug?,
|
98
|
+
no_exception: true
|
99
|
+
)
|
100
|
+
stdout.strip.split(/\s+/).last
|
101
|
+
end
|
102
|
+
|
89
103
|
if ip.nil?
|
90
104
|
log_warn "Host #{host} has no IP."
|
91
105
|
else
|
@@ -23,7 +23,7 @@ module HybridPlatformsConductor
|
|
23
23
|
# * Hash<String, Object>: The corresponding property, per required node.
|
24
24
|
# Nodes for which the property can't be fetched can be ommitted.
|
25
25
|
def get_services(nodes, _metadata)
|
26
|
-
nodes.
|
26
|
+
nodes.to_h { |node| [node, platform_for(node).services_for(node)] }
|
27
27
|
end
|
28
28
|
|
29
29
|
# Get other properties for a given set of nodes.
|
@@ -42,7 +42,7 @@ module HybridPlatformsConductor
|
|
42
42
|
# * Hash<String, Hash<Symbol,Object> >: The corresponding properties, per required node.
|
43
43
|
# Nodes for which the property can't be fetched can be ommitted.
|
44
44
|
def get_others(nodes, _metadata)
|
45
|
-
nodes.
|
45
|
+
nodes.to_h { |node| [node, platform_for(node).metadata_for(node)] }
|
46
46
|
end
|
47
47
|
|
48
48
|
private
|
@@ -605,7 +605,7 @@ module HybridPlatformsConductor
|
|
605
605
|
end
|
606
606
|
else
|
607
607
|
# We have not created any ControlMaster, but still consider the nodes to be ready to connect
|
608
|
-
user_locks = nodes.
|
608
|
+
user_locks = nodes.to_h { |node| [node, nil] }
|
609
609
|
end
|
610
610
|
yield user_locks.keys
|
611
611
|
ensure
|
@@ -132,8 +132,7 @@ module HybridPlatformsConductor
|
|
132
132
|
else
|
133
133
|
(info[:status][:added_files] + info[:status][:changed_files] + info[:status][:untracked_files]).
|
134
134
|
sort.
|
135
|
-
|
136
|
-
to_h
|
135
|
+
to_h { |f| [f, File.mtime("#{@repository_path}/#{f}").strftime('%F %T')] }
|
137
136
|
end,
|
138
137
|
deleted_files: info[:status].nil? ? [] : info[:status][:deleted_files].sort
|
139
138
|
}
|
@@ -371,7 +371,7 @@ class ProxmoxWaiter
|
|
371
371
|
# Result::
|
372
372
|
# * Hash<String, [Float or nil, Float or nil]>: The set of 2 scores, per PVE node name
|
373
373
|
def pve_scores_for(_nbr_cpus, ram_mb, disk_gb)
|
374
|
-
@config['pve_nodes'].
|
374
|
+
@config['pve_nodes'].to_h do |pve_node|
|
375
375
|
# Get some resource usages stats from the node directly
|
376
376
|
status_info = api_get("nodes/#{pve_node}/status")
|
377
377
|
load_average = status_info['loadavg'].map { |load_str| Float(load_str) }
|
@@ -418,11 +418,11 @@ class ProxmoxWaiter
|
|
418
418
|
[
|
419
419
|
if expected_ram_percent_used <= @config['limits']['ram_percent_used_max'] &&
|
420
420
|
expected_disk_percent_used <= @config['limits']['disk_percent_used_max']
|
421
|
-
expected_ram_percent_used * @config['coeff_ram_consumption'] + expected_disk_percent_used * @config['coeff_disk_consumption']
|
421
|
+
(expected_ram_percent_used * @config['coeff_ram_consumption']) + (expected_disk_percent_used * @config['coeff_disk_consumption'])
|
422
422
|
end,
|
423
423
|
if expected_ram_percent_used_without_expired <= @config['limits']['ram_percent_used_max'] &&
|
424
424
|
expected_disk_percent_used_without_expired <= @config['limits']['disk_percent_used_max']
|
425
|
-
expected_ram_percent_used_without_expired * @config['coeff_ram_consumption'] + expected_disk_percent_used_without_expired * @config['coeff_disk_consumption']
|
425
|
+
(expected_ram_percent_used_without_expired * @config['coeff_ram_consumption']) + (expected_disk_percent_used_without_expired * @config['coeff_disk_consumption'])
|
426
426
|
end
|
427
427
|
]
|
428
428
|
else
|
@@ -431,7 +431,7 @@ class ProxmoxWaiter
|
|
431
431
|
[nil, nil]
|
432
432
|
end
|
433
433
|
]
|
434
|
-
end
|
434
|
+
end
|
435
435
|
end
|
436
436
|
|
437
437
|
# Is a given VM expired?
|
@@ -499,10 +499,10 @@ class ProxmoxWaiter
|
|
499
499
|
if hpc_marker_idx.nil?
|
500
500
|
{}
|
501
501
|
else
|
502
|
-
vm_description_lines[hpc_marker_idx + 1..].
|
502
|
+
vm_description_lines[hpc_marker_idx + 1..].to_h do |line|
|
503
503
|
property, value = line.split(': ')
|
504
504
|
[property.to_sym, value]
|
505
|
-
end
|
505
|
+
end
|
506
506
|
end
|
507
507
|
end
|
508
508
|
|
@@ -101,10 +101,10 @@ module HybridPlatformsConductor
|
|
101
101
|
|
102
102
|
# Get the HPC info associated to this VM
|
103
103
|
# Hash<Symbol,String>
|
104
|
-
vm_hpc_info = vm_description_lines[hpc_marker_idx + 1..].
|
104
|
+
vm_hpc_info = vm_description_lines[hpc_marker_idx + 1..].to_h do |line|
|
105
105
|
property, value = line.split(': ')
|
106
106
|
[property.to_sym, value]
|
107
|
-
end
|
107
|
+
end
|
108
108
|
next unless vm_hpc_info[:node] == @node && vm_hpc_info[:environment] == @environment
|
109
109
|
|
110
110
|
# Found it
|
@@ -87,7 +87,7 @@ module HybridPlatformsConductor
|
|
87
87
|
@nodes_handler.prefetch_metadata_of nodes, locale.keys
|
88
88
|
nodes.
|
89
89
|
map do |node|
|
90
|
-
{ node: node }.merge(all_properties.
|
90
|
+
{ node: node }.merge(all_properties.to_h { |property| [property, @nodes_handler.metadata_of(node, property)] })
|
91
91
|
end.
|
92
92
|
# Group them by physical / VMs
|
93
93
|
group_by do |node_info|
|
@@ -125,7 +125,7 @@ module HybridPlatformsConductor
|
|
125
125
|
# * Hash: The JSON secrets parsed from this group
|
126
126
|
def parse_xml_secrets(group)
|
127
127
|
# Parse all entries
|
128
|
-
group.xpath('Entry').
|
128
|
+
group.xpath('Entry').to_h do |entry|
|
129
129
|
[
|
130
130
|
entry.at_xpath('String/Key[contains(.,"Title")]/../Value').text,
|
131
131
|
FIELDS.map do |property, field|
|
@@ -139,7 +139,7 @@ module HybridPlatformsConductor
|
|
139
139
|
]
|
140
140
|
end
|
141
141
|
end.compact.to_h.merge(
|
142
|
-
entry.xpath('Binary').
|
142
|
+
entry.xpath('Binary').to_h do |binary|
|
143
143
|
binary_meta = group.document.at_xpath("KeePassFile/Meta/Binaries/Binary[@ID=#{Integer(binary.xpath('Value').attr('Ref').value)}]")
|
144
144
|
binary_content = Base64.decode64(binary_meta.text)
|
145
145
|
if binary_meta.attr('Compressed') == 'True'
|
@@ -151,17 +151,17 @@ module HybridPlatformsConductor
|
|
151
151
|
binary.xpath('Key').text,
|
152
152
|
binary_content
|
153
153
|
]
|
154
|
-
end
|
154
|
+
end
|
155
155
|
)
|
156
156
|
]
|
157
|
-
end.
|
157
|
+
end.merge(
|
158
158
|
# Add children groups
|
159
|
-
group.xpath('Group').
|
159
|
+
group.xpath('Group').to_h do |sub_group|
|
160
160
|
[
|
161
161
|
sub_group.at_xpath('Name').text,
|
162
162
|
parse_xml_secrets(sub_group)
|
163
163
|
]
|
164
|
-
end
|
164
|
+
end
|
165
165
|
)
|
166
166
|
end
|
167
167
|
|
@@ -15,7 +15,7 @@ module HybridPlatformsConductor
|
|
15
15
|
# Check my_test_plugin.rb.sample documentation for signature details.
|
16
16
|
def test_on_node
|
17
17
|
# Flatten the paths rules so that we can spot inconsistencies in configuration
|
18
|
-
@config.aggregate_files_rules(@nodes_handler, @node).
|
18
|
+
@config.aggregate_files_rules(@nodes_handler, @node).to_h do |path, rule_info|
|
19
19
|
[
|
20
20
|
"if #{@actions_executor.sudo_prefix(@node)}/bin/bash -c '[[ -d \"#{path}\" ]]' ; then echo 1 ; else echo 0 ; fi",
|
21
21
|
{
|
@@ -32,7 +32,7 @@ module HybridPlatformsConductor
|
|
32
32
|
timeout: 2
|
33
33
|
}
|
34
34
|
]
|
35
|
-
end
|
35
|
+
end
|
36
36
|
end
|
37
37
|
|
38
38
|
end
|
@@ -14,7 +14,7 @@ module HybridPlatformsConductor
|
|
14
14
|
# Check my_test_plugin.rb.sample documentation for signature details.
|
15
15
|
def test_on_node
|
16
16
|
# Flatten the paths rules so that we can spot inconsistencies in configuration
|
17
|
-
@config.aggregate_files_rules(@nodes_handler, @node, file_system_type: :hdfs).
|
17
|
+
@config.aggregate_files_rules(@nodes_handler, @node, file_system_type: :hdfs).to_h do |path, rule_info|
|
18
18
|
[
|
19
19
|
"if sudo#{rule_info[:context][:sudo_user] ? " -u #{rule_info[:context][:sudo_user]}" : ''} hdfs dfs -ls \"#{path}\" ; then echo 1 ; else echo 0 ; fi",
|
20
20
|
{
|
@@ -31,7 +31,7 @@ module HybridPlatformsConductor
|
|
31
31
|
timeout: 5
|
32
32
|
}
|
33
33
|
]
|
34
|
-
end
|
34
|
+
end
|
35
35
|
end
|
36
36
|
|
37
37
|
end
|
@@ -18,7 +18,7 @@ module HybridPlatformsConductor
|
|
18
18
|
else
|
19
19
|
private_ips = @nodes_handler.get_private_ips_of @node
|
20
20
|
if private_ips
|
21
|
-
host_ips = stdout.first.split.
|
21
|
+
host_ips = stdout.first.split.grep(/^172\.16\.\d+\.\d+$/).sort
|
22
22
|
ref_ips = private_ips.sort
|
23
23
|
assert_equal(
|
24
24
|
host_ips,
|
@@ -31,7 +31,7 @@ module HybridPlatformsConductor
|
|
31
31
|
next if stdout.empty?
|
32
32
|
|
33
33
|
_exit_status, stdout, _stderr = @cmd_runner.run_cmd(
|
34
|
-
"cd #{@platform.repository_path} && git --no-pager log #{merge_commit_id} --pretty=format:%aI",
|
34
|
+
"cd #{@platform.repository_path} && git --no-pager log #{merge_commit_id} --max-count 1 --pretty=format:%aI",
|
35
35
|
log_to_stdout: log_debug?
|
36
36
|
)
|
37
37
|
error "Git history is not linear because of Merge commit #{merge_commit_id}" if Time.now - Time.parse(stdout.strip) < LOOKING_PERIOD
|
@@ -13,8 +13,7 @@ module HybridPlatformsConductor
|
|
13
13
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, :private_ips
|
14
14
|
private_ips = @nodes_handler.
|
15
15
|
known_nodes.
|
16
|
-
|
17
|
-
to_h
|
16
|
+
to_h { |node| [node, @nodes_handler.get_private_ips_of(node) || []] }
|
18
17
|
|
19
18
|
# Check there are no duplicates
|
20
19
|
nodes_per_private_ip = {}
|
@@ -13,8 +13,7 @@ module HybridPlatformsConductor
|
|
13
13
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, :public_ips
|
14
14
|
public_ips = @nodes_handler.
|
15
15
|
known_nodes.
|
16
|
-
|
17
|
-
to_h
|
16
|
+
to_h { |node| [node, @nodes_handler.get_public_ips_of(node) || []] }
|
18
17
|
|
19
18
|
# Check there are no duplicates
|
20
19
|
nodes_per_public_ip = {}
|
@@ -13,8 +13,7 @@ module HybridPlatformsConductor
|
|
13
13
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, :veid
|
14
14
|
veids = @nodes_handler.
|
15
15
|
known_nodes.
|
16
|
-
|
17
|
-
to_h
|
16
|
+
to_h { |node| [node, @nodes_handler.get_veid_of(node) ? @nodes_handler.get_veid_of(node).to_i : nil] }
|
18
17
|
|
19
18
|
# Check there are no duplicates
|
20
19
|
veids.group_by { |_node, veid| veid }.each do |veid, nodes|
|
@@ -57,7 +57,7 @@ module HybridPlatformsConductor
|
|
57
57
|
end
|
58
58
|
)
|
59
59
|
sudo = @actions_executor.sudo_prefix(@node)
|
60
|
-
urls.
|
60
|
+
urls.to_h do |url|
|
61
61
|
# 1. Get the OVAL file on the node to be tested (uncompress it if needed)
|
62
62
|
# 2. Make sure oscap is installed
|
63
63
|
# 3. Generate the report for this OVAL file using oscap
|
@@ -143,7 +143,7 @@ module HybridPlatformsConductor
|
|
143
143
|
timeout: 240
|
144
144
|
}
|
145
145
|
]
|
146
|
-
end
|
146
|
+
end
|
147
147
|
else
|
148
148
|
error "No OVAL file defined for image #{image} at #{oval_file}"
|
149
149
|
{}
|