hybrid_platforms_conductor 33.9.2 → 33.9.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/bin/last_deploys +3 -4
- data/docs/gen/mermaid/README.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_veids.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/setup.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
- data/lib/hybrid_platforms_conductor/actions_executor.rb +2 -2
- data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +2 -2
- data/lib/hybrid_platforms_conductor/config.rb +1 -0
- data/lib/hybrid_platforms_conductor/deployer.rb +12 -12
- data/lib/hybrid_platforms_conductor/executable.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +21 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +6 -6
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/keepass.rb +6 -6
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +2 -2
- data/lib/hybrid_platforms_conductor/plugins.rb +4 -3
- data/lib/hybrid_platforms_conductor/tests_runner.rb +2 -2
- data/lib/hybrid_platforms_conductor/topographer.rb +5 -6
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/helpers_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +12 -12
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/docker_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +27 -27
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +25 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +9 -9
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +30 -2
- data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +7 -7
- data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +18 -18
- data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +30 -30
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_2.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others_2.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/check_node_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +7 -7
- data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +11 -12
- data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem_with_lib/sub1/lib/sub2/hpc_plugins/test_plugin_type_3/test_plugin_id_5.rb +34 -0
- data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/test_connector.rb +1 -1
- data/spec/hybrid_platforms_conductor_test.rb +1 -0
- metadata +152 -136
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 433e01481e26d03667cde79b93697f0f14e949a6201931c6e570ebb84670e79a
|
4
|
+
data.tar.gz: 61c1eb5c9b9ec3df221528e8ae1c08a7a6bdf3628c2a6a83b1756e11a84a2e88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 31ff562ab7108537b8ba57b8c657ef516ef5665fb40fc81d4afe4f14a54400f62324597136845af8c39bd8788318d3d12c54362562cb73dff9d769c60cf8bce1
|
7
|
+
data.tar.gz: cb7dfea944204a89f9522703fb588ec8085dcd9a92816abe0cc012f0e7440432ee4a3831825517ae9b74f59e1ac1ff7d4995b6fa75220c28096de1a8e95f31b6
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,18 @@
|
|
1
|
+
# [v33.9.4](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.3...v33.9.4) (2022-03-20 16:20:04)
|
2
|
+
|
3
|
+
### Patches
|
4
|
+
|
5
|
+
* [Added missing tests and corrections from PR #119](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/e86be2f99e273e65bf9073c1b389a10ad53d7353)
|
6
|
+
* [Merge branch 'rebase_pull_116'](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/4030ba58e0e9cf4ef96328a32cf4daa6226270c0)
|
7
|
+
|
8
|
+
# [v33.9.3](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.2...v33.9.3) (2022-03-20 00:06:14)
|
9
|
+
|
10
|
+
### Patches
|
11
|
+
|
12
|
+
* [Fix semantic-release dependencies](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/68ec49fbec772c1de6971173830a85b76398de42)
|
13
|
+
* [Work-around IceCube dependency issue and correct rubocop errors](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/68858e4d5fbb9d91fcee0525f56dfc40850af7df)
|
14
|
+
* [fixing issue 118](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/bb917101c336119104889e5d6cedd34aef538f54)
|
15
|
+
|
1
16
|
# [v33.9.2](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.1...v33.9.2) (2021-09-01 17:46:34)
|
2
17
|
|
3
18
|
## Global changes
|
data/bin/last_deploys
CHANGED
@@ -8,12 +8,12 @@ possible_string_sorts = %i[
|
|
8
8
|
node
|
9
9
|
user
|
10
10
|
]
|
11
|
-
possible_sorts = possible_string_sorts.
|
11
|
+
possible_sorts = possible_string_sorts.to_h do |property_name|
|
12
12
|
[
|
13
13
|
property_name,
|
14
14
|
proc { |node, deploy_info| [deploy_info.key?(:error) || !deploy_info.key?(property_name) ? '' : deploy_info[property_name], node] }
|
15
15
|
]
|
16
|
-
end
|
16
|
+
end
|
17
17
|
|
18
18
|
sort_by = :node
|
19
19
|
sort_desc = false
|
@@ -34,7 +34,7 @@ raise "Unknown sort name: #{sort_by}. Should be one of #{possible_sorts.keys.joi
|
|
34
34
|
|
35
35
|
sorted_deploy_info = deployer.
|
36
36
|
deployment_info_from(nodes_handler.select_nodes(executable.selected_nodes.empty? ? [{ all: true }] : executable.selected_nodes)).
|
37
|
-
|
37
|
+
to_h do |node, deploy_info|
|
38
38
|
decorated_deploy_info = deploy_info.merge(node: node)
|
39
39
|
if deploy_info.key?(:deployment_info)
|
40
40
|
decorated_deploy_info.merge!(deploy_info[:deployment_info])
|
@@ -43,7 +43,6 @@ sorted_deploy_info = deployer.
|
|
43
43
|
decorated_deploy_info[:services] = deploy_info[:services].join(', ') if deploy_info.key?(:services)
|
44
44
|
[node, decorated_deploy_info]
|
45
45
|
end.
|
46
|
-
to_h.
|
47
46
|
sort_by(&possible_sorts[sort_by])
|
48
47
|
sorted_deploy_info.reverse! if sort_desc
|
49
48
|
info_displayed = {
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
@@ -146,7 +146,7 @@ module HybridPlatformsConductor
|
|
146
146
|
actions_per_node[node].concat(resolved_nodes_actions)
|
147
147
|
end
|
148
148
|
end
|
149
|
-
result = actions_per_node.keys.
|
149
|
+
result = actions_per_node.keys.to_h { |node| [node, nil] }
|
150
150
|
with_connections_prepared_to(nodes_needing_connectors, no_exception: true) do |connected_nodes|
|
151
151
|
missing_nodes = []
|
152
152
|
connected_nodes.each do |node, connector|
|
@@ -194,7 +194,7 @@ module HybridPlatformsConductor
|
|
194
194
|
# * *connected_nodes* (Hash<String, Connector or Symbol>): Prepared connectors (or Symbol in case of failure with no_exception), per node name
|
195
195
|
def with_connections_prepared_to(nodes, no_exception: false)
|
196
196
|
# Make sure every node needing connectors finds a connector
|
197
|
-
nodes_needing_connectors = nodes.
|
197
|
+
nodes_needing_connectors = nodes.to_h { |node| [node, nil] }
|
198
198
|
@connector_plugins.each_value do |connector|
|
199
199
|
nodes_without_connectors = nodes_needing_connectors.select { |_node, selected_connector| selected_connector.nil? }.keys
|
200
200
|
break if nodes_without_connectors.empty?
|
@@ -84,7 +84,7 @@ module HybridPlatformsConductor
|
|
84
84
|
select_confs_for_node(node, fs_paths_rules).
|
85
85
|
inject({}) do |merged_paths, paths_info|
|
86
86
|
if paths_info[:context][:file_system_type] == file_system_type
|
87
|
-
merged_paths.merge(paths_info[:paths].
|
87
|
+
merged_paths.merge(paths_info[:paths].to_h do |path|
|
88
88
|
[
|
89
89
|
path,
|
90
90
|
{
|
@@ -92,7 +92,7 @@ module HybridPlatformsConductor
|
|
92
92
|
context: paths_info[:context]
|
93
93
|
}
|
94
94
|
]
|
95
|
-
end
|
95
|
+
end) do |path, rule_info_1, rule_info_2|
|
96
96
|
# Just check that configuration is not inconsistent
|
97
97
|
raise "Inconsistent rule for #{file_system_type} file system checks in configuration for #{node}: #{path} is marked as being both #{rule_info_1[:state]} and #{rule_info_2[:state]}" if rule_info_1[:state] != rule_info_2[:state]
|
98
98
|
|
@@ -263,9 +263,9 @@ module HybridPlatformsConductor
|
|
263
263
|
def deploy_on(*nodes_selectors)
|
264
264
|
# Get the sorted list of services to be deployed, per node
|
265
265
|
# Hash<String, Array<String> >
|
266
|
-
services_to_deploy = @nodes_handler.select_nodes(nodes_selectors.flatten).
|
266
|
+
services_to_deploy = @nodes_handler.select_nodes(nodes_selectors.flatten).to_h do |node|
|
267
267
|
[node, @nodes_handler.get_services_of(node)]
|
268
|
-
end
|
268
|
+
end
|
269
269
|
|
270
270
|
# Get the secrets to be deployed
|
271
271
|
secrets = {}
|
@@ -474,7 +474,7 @@ module HybridPlatformsConductor
|
|
474
474
|
timeout: 10,
|
475
475
|
progress_name: 'Read deployment logs'
|
476
476
|
)
|
477
|
-
nodes.
|
477
|
+
nodes.to_h do |node|
|
478
478
|
exit_code, stdout, stderr = read_actions_results[node] || [nil, nil, nil]
|
479
479
|
[
|
480
480
|
node,
|
@@ -485,7 +485,7 @@ module HybridPlatformsConductor
|
|
485
485
|
stderr&.force_encoding(Encoding::UTF_8)
|
486
486
|
)
|
487
487
|
]
|
488
|
-
end
|
488
|
+
end
|
489
489
|
end
|
490
490
|
|
491
491
|
# Parse stdout and stderr of a given deploy run and get the list of tasks with their status
|
@@ -553,7 +553,7 @@ module HybridPlatformsConductor
|
|
553
553
|
# Deploy for real
|
554
554
|
@nodes_handler.prefetch_metadata_of services.keys, :image
|
555
555
|
outputs = @actions_executor.execute_actions(
|
556
|
-
services.
|
556
|
+
services.to_h do |node, node_services|
|
557
557
|
image_id = @nodes_handler.get_image_of(node)
|
558
558
|
need_sudo = !@actions_executor.privileged_access?(node)
|
559
559
|
sudo = @actions_executor.sudo_prefix(node)
|
@@ -583,12 +583,12 @@ module HybridPlatformsConductor
|
|
583
583
|
remote_bash: "#{sudo}yum install -y ca-certificates"
|
584
584
|
},
|
585
585
|
{
|
586
|
-
scp: Dir.glob("#{ENV['hpc_certificates']}/*.crt").
|
586
|
+
scp: Dir.glob("#{ENV['hpc_certificates']}/*.crt").to_h do |cert_file|
|
587
587
|
[
|
588
588
|
cert_file,
|
589
589
|
'/etc/pki/ca-trust/source/anchors'
|
590
590
|
]
|
591
|
-
end.
|
591
|
+
end.merge(sudo: need_sudo),
|
592
592
|
remote_bash: [
|
593
593
|
"#{sudo}update-ca-trust enable",
|
594
594
|
"#{sudo}update-ca-trust extract"
|
@@ -613,19 +613,19 @@ module HybridPlatformsConductor
|
|
613
613
|
certificate_actions +
|
614
614
|
@services_handler.actions_to_deploy_on(node, node_services, @use_why_run)
|
615
615
|
]
|
616
|
-
end
|
616
|
+
end,
|
617
617
|
timeout: @timeout,
|
618
618
|
concurrent: @concurrent_execution,
|
619
619
|
log_to_stdout: !@concurrent_execution
|
620
620
|
)
|
621
621
|
# Free eventual locks
|
622
622
|
@actions_executor.execute_actions(
|
623
|
-
services.keys.
|
623
|
+
services.keys.to_h do |node|
|
624
624
|
[
|
625
625
|
node,
|
626
626
|
{ remote_bash: "#{@actions_executor.sudo_prefix(node)}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
|
627
627
|
]
|
628
|
-
end
|
628
|
+
end,
|
629
629
|
timeout: 10,
|
630
630
|
concurrent: true,
|
631
631
|
log_to_dir: nil
|
@@ -647,7 +647,7 @@ module HybridPlatformsConductor
|
|
647
647
|
section "Saving deployment logs for #{logs.size} nodes" do
|
648
648
|
ssh_user = @actions_executor.connector(:ssh).ssh_user
|
649
649
|
@actions_executor.execute_actions(
|
650
|
-
logs.
|
650
|
+
logs.to_h do |node, (exit_status, stdout, stderr)|
|
651
651
|
[
|
652
652
|
node,
|
653
653
|
log_plugins_for(node).
|
@@ -666,7 +666,7 @@ module HybridPlatformsConductor
|
|
666
666
|
end.
|
667
667
|
flatten(1)
|
668
668
|
]
|
669
|
-
end
|
669
|
+
end,
|
670
670
|
timeout: 10,
|
671
671
|
concurrent: true,
|
672
672
|
log_to_dir: nil,
|
@@ -99,7 +99,7 @@ module HybridPlatformsConductor
|
|
99
99
|
@instantiated_components[component] = HybridPlatformsConductor.const_get(component.to_s.split('_').collect(&:capitalize).join.to_sym).new(
|
100
100
|
logger: @logger,
|
101
101
|
logger_stderr: @logger_stderr,
|
102
|
-
**dependencies.
|
102
|
+
**dependencies.to_h { |dependency| [dependency, send(dependency)] }
|
103
103
|
)
|
104
104
|
end
|
105
105
|
@instantiated_components[component]
|
@@ -73,19 +73,33 @@ module HybridPlatformsConductor
|
|
73
73
|
def ip_for(*hosts)
|
74
74
|
results = {}
|
75
75
|
log_debug "Get IPs of #{hosts.size} hosts..."
|
76
|
+
exit_status, _stdout, _stderr = @cmd_runner.run_cmd('command -v getent', no_exception: true)
|
77
|
+
getent_present = exit_status.zero?
|
76
78
|
for_each_element_in(
|
77
79
|
hosts,
|
78
80
|
parallel: true,
|
79
81
|
nbr_threads_max: MAX_THREADS_GETENT,
|
80
82
|
progress: log_debug? ? 'Gather IPs' : nil
|
81
83
|
) do |host|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
84
|
+
ip =
|
85
|
+
if getent_present
|
86
|
+
_exit_status, stdout, _stderr = @cmd_runner.run_cmd(
|
87
|
+
"getent hosts #{host}",
|
88
|
+
timeout: TIMEOUT_GETENT,
|
89
|
+
log_to_stdout: log_debug?,
|
90
|
+
no_exception: true
|
91
|
+
)
|
92
|
+
stdout.strip.split(/\s+/).first
|
93
|
+
else
|
94
|
+
_exit_status, stdout, _stderr = @cmd_runner.run_cmd(
|
95
|
+
"host #{host} | grep 'has address'",
|
96
|
+
timeout: TIMEOUT_GETENT,
|
97
|
+
log_to_stdout: log_debug?,
|
98
|
+
no_exception: true
|
99
|
+
)
|
100
|
+
stdout.strip.split(/\s+/).last
|
101
|
+
end
|
102
|
+
|
89
103
|
if ip.nil?
|
90
104
|
log_warn "Host #{host} has no IP."
|
91
105
|
else
|
@@ -23,7 +23,7 @@ module HybridPlatformsConductor
|
|
23
23
|
# * Hash<String, Object>: The corresponding property, per required node.
|
24
24
|
# Nodes for which the property can't be fetched can be ommitted.
|
25
25
|
def get_services(nodes, _metadata)
|
26
|
-
nodes.
|
26
|
+
nodes.to_h { |node| [node, platform_for(node).services_for(node)] }
|
27
27
|
end
|
28
28
|
|
29
29
|
# Get other properties for a given set of nodes.
|
@@ -42,7 +42,7 @@ module HybridPlatformsConductor
|
|
42
42
|
# * Hash<String, Hash<Symbol,Object> >: The corresponding properties, per required node.
|
43
43
|
# Nodes for which the property can't be fetched can be ommitted.
|
44
44
|
def get_others(nodes, _metadata)
|
45
|
-
nodes.
|
45
|
+
nodes.to_h { |node| [node, platform_for(node).metadata_for(node)] }
|
46
46
|
end
|
47
47
|
|
48
48
|
private
|
@@ -605,7 +605,7 @@ module HybridPlatformsConductor
|
|
605
605
|
end
|
606
606
|
else
|
607
607
|
# We have not created any ControlMaster, but still consider the nodes to be ready to connect
|
608
|
-
user_locks = nodes.
|
608
|
+
user_locks = nodes.to_h { |node| [node, nil] }
|
609
609
|
end
|
610
610
|
yield user_locks.keys
|
611
611
|
ensure
|
@@ -132,8 +132,7 @@ module HybridPlatformsConductor
|
|
132
132
|
else
|
133
133
|
(info[:status][:added_files] + info[:status][:changed_files] + info[:status][:untracked_files]).
|
134
134
|
sort.
|
135
|
-
|
136
|
-
to_h
|
135
|
+
to_h { |f| [f, File.mtime("#{@repository_path}/#{f}").strftime('%F %T')] }
|
137
136
|
end,
|
138
137
|
deleted_files: info[:status].nil? ? [] : info[:status][:deleted_files].sort
|
139
138
|
}
|
@@ -371,7 +371,7 @@ class ProxmoxWaiter
|
|
371
371
|
# Result::
|
372
372
|
# * Hash<String, [Float or nil, Float or nil]>: The set of 2 scores, per PVE node name
|
373
373
|
def pve_scores_for(_nbr_cpus, ram_mb, disk_gb)
|
374
|
-
@config['pve_nodes'].
|
374
|
+
@config['pve_nodes'].to_h do |pve_node|
|
375
375
|
# Get some resource usages stats from the node directly
|
376
376
|
status_info = api_get("nodes/#{pve_node}/status")
|
377
377
|
load_average = status_info['loadavg'].map { |load_str| Float(load_str) }
|
@@ -418,11 +418,11 @@ class ProxmoxWaiter
|
|
418
418
|
[
|
419
419
|
if expected_ram_percent_used <= @config['limits']['ram_percent_used_max'] &&
|
420
420
|
expected_disk_percent_used <= @config['limits']['disk_percent_used_max']
|
421
|
-
expected_ram_percent_used * @config['coeff_ram_consumption'] + expected_disk_percent_used * @config['coeff_disk_consumption']
|
421
|
+
(expected_ram_percent_used * @config['coeff_ram_consumption']) + (expected_disk_percent_used * @config['coeff_disk_consumption'])
|
422
422
|
end,
|
423
423
|
if expected_ram_percent_used_without_expired <= @config['limits']['ram_percent_used_max'] &&
|
424
424
|
expected_disk_percent_used_without_expired <= @config['limits']['disk_percent_used_max']
|
425
|
-
expected_ram_percent_used_without_expired * @config['coeff_ram_consumption'] + expected_disk_percent_used_without_expired * @config['coeff_disk_consumption']
|
425
|
+
(expected_ram_percent_used_without_expired * @config['coeff_ram_consumption']) + (expected_disk_percent_used_without_expired * @config['coeff_disk_consumption'])
|
426
426
|
end
|
427
427
|
]
|
428
428
|
else
|
@@ -431,7 +431,7 @@ class ProxmoxWaiter
|
|
431
431
|
[nil, nil]
|
432
432
|
end
|
433
433
|
]
|
434
|
-
end
|
434
|
+
end
|
435
435
|
end
|
436
436
|
|
437
437
|
# Is a given VM expired?
|
@@ -499,10 +499,10 @@ class ProxmoxWaiter
|
|
499
499
|
if hpc_marker_idx.nil?
|
500
500
|
{}
|
501
501
|
else
|
502
|
-
vm_description_lines[hpc_marker_idx + 1..].
|
502
|
+
vm_description_lines[hpc_marker_idx + 1..].to_h do |line|
|
503
503
|
property, value = line.split(': ')
|
504
504
|
[property.to_sym, value]
|
505
|
-
end
|
505
|
+
end
|
506
506
|
end
|
507
507
|
end
|
508
508
|
|
@@ -101,10 +101,10 @@ module HybridPlatformsConductor
|
|
101
101
|
|
102
102
|
# Get the HPC info associated to this VM
|
103
103
|
# Hash<Symbol,String>
|
104
|
-
vm_hpc_info = vm_description_lines[hpc_marker_idx + 1..].
|
104
|
+
vm_hpc_info = vm_description_lines[hpc_marker_idx + 1..].to_h do |line|
|
105
105
|
property, value = line.split(': ')
|
106
106
|
[property.to_sym, value]
|
107
|
-
end
|
107
|
+
end
|
108
108
|
next unless vm_hpc_info[:node] == @node && vm_hpc_info[:environment] == @environment
|
109
109
|
|
110
110
|
# Found it
|
@@ -87,7 +87,7 @@ module HybridPlatformsConductor
|
|
87
87
|
@nodes_handler.prefetch_metadata_of nodes, locale.keys
|
88
88
|
nodes.
|
89
89
|
map do |node|
|
90
|
-
{ node: node }.merge(all_properties.
|
90
|
+
{ node: node }.merge(all_properties.to_h { |property| [property, @nodes_handler.metadata_of(node, property)] })
|
91
91
|
end.
|
92
92
|
# Group them by physical / VMs
|
93
93
|
group_by do |node_info|
|
@@ -125,7 +125,7 @@ module HybridPlatformsConductor
|
|
125
125
|
# * Hash: The JSON secrets parsed from this group
|
126
126
|
def parse_xml_secrets(group)
|
127
127
|
# Parse all entries
|
128
|
-
group.xpath('Entry').
|
128
|
+
group.xpath('Entry').to_h do |entry|
|
129
129
|
[
|
130
130
|
entry.at_xpath('String/Key[contains(.,"Title")]/../Value').text,
|
131
131
|
FIELDS.map do |property, field|
|
@@ -139,7 +139,7 @@ module HybridPlatformsConductor
|
|
139
139
|
]
|
140
140
|
end
|
141
141
|
end.compact.to_h.merge(
|
142
|
-
entry.xpath('Binary').
|
142
|
+
entry.xpath('Binary').to_h do |binary|
|
143
143
|
binary_meta = group.document.at_xpath("KeePassFile/Meta/Binaries/Binary[@ID=#{Integer(binary.xpath('Value').attr('Ref').value)}]")
|
144
144
|
binary_content = Base64.decode64(binary_meta.text)
|
145
145
|
if binary_meta.attr('Compressed') == 'True'
|
@@ -151,17 +151,17 @@ module HybridPlatformsConductor
|
|
151
151
|
binary.xpath('Key').text,
|
152
152
|
binary_content
|
153
153
|
]
|
154
|
-
end
|
154
|
+
end
|
155
155
|
)
|
156
156
|
]
|
157
|
-
end.
|
157
|
+
end.merge(
|
158
158
|
# Add children groups
|
159
|
-
group.xpath('Group').
|
159
|
+
group.xpath('Group').to_h do |sub_group|
|
160
160
|
[
|
161
161
|
sub_group.at_xpath('Name').text,
|
162
162
|
parse_xml_secrets(sub_group)
|
163
163
|
]
|
164
|
-
end
|
164
|
+
end
|
165
165
|
)
|
166
166
|
end
|
167
167
|
|
@@ -15,7 +15,7 @@ module HybridPlatformsConductor
|
|
15
15
|
# Check my_test_plugin.rb.sample documentation for signature details.
|
16
16
|
def test_on_node
|
17
17
|
# Flatten the paths rules so that we can spot inconsistencies in configuration
|
18
|
-
@config.aggregate_files_rules(@nodes_handler, @node).
|
18
|
+
@config.aggregate_files_rules(@nodes_handler, @node).to_h do |path, rule_info|
|
19
19
|
[
|
20
20
|
"if #{@actions_executor.sudo_prefix(@node)}/bin/bash -c '[[ -d \"#{path}\" ]]' ; then echo 1 ; else echo 0 ; fi",
|
21
21
|
{
|
@@ -32,7 +32,7 @@ module HybridPlatformsConductor
|
|
32
32
|
timeout: 2
|
33
33
|
}
|
34
34
|
]
|
35
|
-
end
|
35
|
+
end
|
36
36
|
end
|
37
37
|
|
38
38
|
end
|
@@ -14,7 +14,7 @@ module HybridPlatformsConductor
|
|
14
14
|
# Check my_test_plugin.rb.sample documentation for signature details.
|
15
15
|
def test_on_node
|
16
16
|
# Flatten the paths rules so that we can spot inconsistencies in configuration
|
17
|
-
@config.aggregate_files_rules(@nodes_handler, @node, file_system_type: :hdfs).
|
17
|
+
@config.aggregate_files_rules(@nodes_handler, @node, file_system_type: :hdfs).to_h do |path, rule_info|
|
18
18
|
[
|
19
19
|
"if sudo#{rule_info[:context][:sudo_user] ? " -u #{rule_info[:context][:sudo_user]}" : ''} hdfs dfs -ls \"#{path}\" ; then echo 1 ; else echo 0 ; fi",
|
20
20
|
{
|
@@ -31,7 +31,7 @@ module HybridPlatformsConductor
|
|
31
31
|
timeout: 5
|
32
32
|
}
|
33
33
|
]
|
34
|
-
end
|
34
|
+
end
|
35
35
|
end
|
36
36
|
|
37
37
|
end
|
@@ -18,7 +18,7 @@ module HybridPlatformsConductor
|
|
18
18
|
else
|
19
19
|
private_ips = @nodes_handler.get_private_ips_of @node
|
20
20
|
if private_ips
|
21
|
-
host_ips = stdout.first.split.
|
21
|
+
host_ips = stdout.first.split.grep(/^172\.16\.\d+\.\d+$/).sort
|
22
22
|
ref_ips = private_ips.sort
|
23
23
|
assert_equal(
|
24
24
|
host_ips,
|
@@ -31,7 +31,7 @@ module HybridPlatformsConductor
|
|
31
31
|
next if stdout.empty?
|
32
32
|
|
33
33
|
_exit_status, stdout, _stderr = @cmd_runner.run_cmd(
|
34
|
-
"cd #{@platform.repository_path} && git --no-pager log #{merge_commit_id} --pretty=format:%aI",
|
34
|
+
"cd #{@platform.repository_path} && git --no-pager log #{merge_commit_id} --max-count 1 --pretty=format:%aI",
|
35
35
|
log_to_stdout: log_debug?
|
36
36
|
)
|
37
37
|
error "Git history is not linear because of Merge commit #{merge_commit_id}" if Time.now - Time.parse(stdout.strip) < LOOKING_PERIOD
|
@@ -13,8 +13,7 @@ module HybridPlatformsConductor
|
|
13
13
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, :private_ips
|
14
14
|
private_ips = @nodes_handler.
|
15
15
|
known_nodes.
|
16
|
-
|
17
|
-
to_h
|
16
|
+
to_h { |node| [node, @nodes_handler.get_private_ips_of(node) || []] }
|
18
17
|
|
19
18
|
# Check there are no duplicates
|
20
19
|
nodes_per_private_ip = {}
|
@@ -13,8 +13,7 @@ module HybridPlatformsConductor
|
|
13
13
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, :public_ips
|
14
14
|
public_ips = @nodes_handler.
|
15
15
|
known_nodes.
|
16
|
-
|
17
|
-
to_h
|
16
|
+
to_h { |node| [node, @nodes_handler.get_public_ips_of(node) || []] }
|
18
17
|
|
19
18
|
# Check there are no duplicates
|
20
19
|
nodes_per_public_ip = {}
|
@@ -13,8 +13,7 @@ module HybridPlatformsConductor
|
|
13
13
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, :veid
|
14
14
|
veids = @nodes_handler.
|
15
15
|
known_nodes.
|
16
|
-
|
17
|
-
to_h
|
16
|
+
to_h { |node| [node, @nodes_handler.get_veid_of(node) ? @nodes_handler.get_veid_of(node).to_i : nil] }
|
18
17
|
|
19
18
|
# Check there are no duplicates
|
20
19
|
veids.group_by { |_node, veid| veid }.each do |veid, nodes|
|
@@ -57,7 +57,7 @@ module HybridPlatformsConductor
|
|
57
57
|
end
|
58
58
|
)
|
59
59
|
sudo = @actions_executor.sudo_prefix(@node)
|
60
|
-
urls.
|
60
|
+
urls.to_h do |url|
|
61
61
|
# 1. Get the OVAL file on the node to be tested (uncompress it if needed)
|
62
62
|
# 2. Make sure oscap is installed
|
63
63
|
# 3. Generate the report for this OVAL file using oscap
|
@@ -143,7 +143,7 @@ module HybridPlatformsConductor
|
|
143
143
|
timeout: 240
|
144
144
|
}
|
145
145
|
]
|
146
|
-
end
|
146
|
+
end
|
147
147
|
else
|
148
148
|
error "No OVAL file defined for image #{image} at #{oval_file}"
|
149
149
|
{}
|
@@ -75,10 +75,11 @@ module HybridPlatformsConductor
|
|
75
75
|
# Register plugins by parsing gems
|
76
76
|
def register_plugins_from_gems
|
77
77
|
# Require all possible files that could define such a plugin, from all gems
|
78
|
-
|
79
|
-
|
78
|
+
Gem.loaded_specs.clone.each do |gem_name, gem_specs|
|
79
|
+
gem_path = gem_specs.full_gem_path
|
80
|
+
files_regexp = %r{#{Regexp.escape(gem_path)}/lib/(.*hpc_plugins/#{Regexp.escape(@plugins_type.to_s)}/[^/]+)\.rb$}
|
80
81
|
# Careful to not use gem_specs.files here as if your gem name contains "-" or other weird characters, files won't appear in the gemspec list.
|
81
|
-
Dir.glob("#{
|
82
|
+
Dir.glob("#{gem_path}/lib/**/*.rb").each do |file|
|
82
83
|
next unless file =~ files_regexp
|
83
84
|
|
84
85
|
require_name = Regexp.last_match(1)
|
@@ -524,14 +524,14 @@ module HybridPlatformsConductor
|
|
524
524
|
nodes_to_test = selected_tests.map(&:node).uniq.sort
|
525
525
|
@outputs =
|
526
526
|
if @skip_run
|
527
|
-
nodes_to_test.
|
527
|
+
nodes_to_test.to_h do |node|
|
528
528
|
run_log_file_name = "#{@config.hybrid_platforms_dir}/run_logs/#{node}.stdout"
|
529
529
|
[
|
530
530
|
node,
|
531
531
|
# TODO: Find a way to also save stderr and the status code
|
532
532
|
[0, File.exist?(run_log_file_name) ? File.read(run_log_file_name) : nil, '']
|
533
533
|
]
|
534
|
-
end
|
534
|
+
end
|
535
535
|
else
|
536
536
|
# Why-run deploy on all nodes
|
537
537
|
@deployer.concurrent_execution = !log_debug?
|
@@ -112,15 +112,14 @@ module HybridPlatformsConductor
|
|
112
112
|
# Parse plugins
|
113
113
|
@plugins = Dir.
|
114
114
|
glob("#{__dir__}/topographer/plugins/*.rb").
|
115
|
-
|
115
|
+
to_h do |file_name|
|
116
116
|
plugin_name = File.basename(file_name)[0..-4].to_sym
|
117
117
|
require file_name
|
118
118
|
[
|
119
119
|
plugin_name,
|
120
120
|
Topographer::Plugins.const_get(plugin_name.to_s.split('_').collect(&:capitalize).join.to_sym)
|
121
121
|
]
|
122
|
-
end
|
123
|
-
to_h
|
122
|
+
end
|
124
123
|
|
125
124
|
@ips_to_host = known_ips.clone
|
126
125
|
|
@@ -132,11 +131,11 @@ module HybridPlatformsConductor
|
|
132
131
|
]
|
133
132
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, metadata_properties
|
134
133
|
@nodes_handler.known_nodes.each do |hostname|
|
135
|
-
@node_metadata[hostname] = metadata_properties.
|
134
|
+
@node_metadata[hostname] = metadata_properties.to_h { |property| [property, @nodes_handler.metadata_of(hostname, property)] }
|
136
135
|
end
|
137
136
|
|
138
137
|
# Small cache of hostnames used a lot to parse JSON
|
139
|
-
@known_nodes = @nodes_handler.known_nodes.
|
138
|
+
@known_nodes = @nodes_handler.known_nodes.to_h { |hostname| [hostname, nil] }
|
140
139
|
# Cache of objects being used a lot in parsing for performance
|
141
140
|
@non_word_regexp = /\W+/
|
142
141
|
@ip_regexp = %r{(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(/(\d{1,2})|[^\d/]|$)}
|
@@ -600,7 +599,7 @@ module HybridPlatformsConductor
|
|
600
599
|
@ips_mask[ip_def] = {} unless @ips_mask.key?(ip_def)
|
601
600
|
unless @ips_mask[ip_def].key?(ip_mask)
|
602
601
|
# For performance, keep a cache of all the IPAddress::IPv4 objects
|
603
|
-
@ip_v4_cache = known_ips.keys.
|
602
|
+
@ip_v4_cache = known_ips.keys.to_h { |ip, _node| [ip, IPAddress::IPv4.new(ip)] } unless defined?(@ip_v4_cache)
|
604
603
|
ip_range = IPAddress::IPv4.new("#{ip_def}/#{ip_mask}")
|
605
604
|
@ips_mask[ip_def][ip_mask] = @ip_v4_cache.select { |_ip, ipv4| ip_range.include?(ipv4) }.keys
|
606
605
|
end
|