hybrid_platforms_conductor 33.9.2 → 33.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/bin/last_deploys +3 -4
- data/docs/gen/mermaid/README.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_veids.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/setup.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
- data/lib/hybrid_platforms_conductor/actions_executor.rb +2 -2
- data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +2 -2
- data/lib/hybrid_platforms_conductor/config.rb +1 -0
- data/lib/hybrid_platforms_conductor/deployer.rb +12 -12
- data/lib/hybrid_platforms_conductor/executable.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +21 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +6 -6
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/keepass.rb +6 -6
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +1 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +2 -2
- data/lib/hybrid_platforms_conductor/plugins.rb +4 -3
- data/lib/hybrid_platforms_conductor/tests_runner.rb +2 -2
- data/lib/hybrid_platforms_conductor/topographer.rb +5 -6
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/helpers_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +12 -12
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/docker_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +27 -27
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +25 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +9 -9
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +30 -2
- data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +7 -7
- data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +18 -18
- data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +30 -30
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_2.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others_2.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/check_node_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +7 -7
- data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +11 -12
- data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem_with_lib/sub1/lib/sub2/hpc_plugins/test_plugin_type_3/test_plugin_id_5.rb +34 -0
- data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/test_connector.rb +1 -1
- data/spec/hybrid_platforms_conductor_test.rb +1 -0
- metadata +152 -136
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 433e01481e26d03667cde79b93697f0f14e949a6201931c6e570ebb84670e79a
|
|
4
|
+
data.tar.gz: 61c1eb5c9b9ec3df221528e8ae1c08a7a6bdf3628c2a6a83b1756e11a84a2e88
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 31ff562ab7108537b8ba57b8c657ef516ef5665fb40fc81d4afe4f14a54400f62324597136845af8c39bd8788318d3d12c54362562cb73dff9d769c60cf8bce1
|
|
7
|
+
data.tar.gz: cb7dfea944204a89f9522703fb588ec8085dcd9a92816abe0cc012f0e7440432ee4a3831825517ae9b74f59e1ac1ff7d4995b6fa75220c28096de1a8e95f31b6
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,18 @@
|
|
|
1
|
+
# [v33.9.4](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.3...v33.9.4) (2022-03-20 16:20:04)
|
|
2
|
+
|
|
3
|
+
### Patches
|
|
4
|
+
|
|
5
|
+
* [Added missing tests and corrections from PR #119](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/e86be2f99e273e65bf9073c1b389a10ad53d7353)
|
|
6
|
+
* [Merge branch 'rebase_pull_116'](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/4030ba58e0e9cf4ef96328a32cf4daa6226270c0)
|
|
7
|
+
|
|
8
|
+
# [v33.9.3](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.2...v33.9.3) (2022-03-20 00:06:14)
|
|
9
|
+
|
|
10
|
+
### Patches
|
|
11
|
+
|
|
12
|
+
* [Fix semantic-release dependencies](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/68ec49fbec772c1de6971173830a85b76398de42)
|
|
13
|
+
* [Work-around IceCube dependency issue and correct rubocop errors](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/68858e4d5fbb9d91fcee0525f56dfc40850af7df)
|
|
14
|
+
* [fixing issue 118](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/bb917101c336119104889e5d6cedd34aef538f54)
|
|
15
|
+
|
|
1
16
|
# [v33.9.2](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.9.1...v33.9.2) (2021-09-01 17:46:34)
|
|
2
17
|
|
|
3
18
|
## Global changes
|
data/bin/last_deploys
CHANGED
|
@@ -8,12 +8,12 @@ possible_string_sorts = %i[
|
|
|
8
8
|
node
|
|
9
9
|
user
|
|
10
10
|
]
|
|
11
|
-
possible_sorts = possible_string_sorts.
|
|
11
|
+
possible_sorts = possible_string_sorts.to_h do |property_name|
|
|
12
12
|
[
|
|
13
13
|
property_name,
|
|
14
14
|
proc { |node, deploy_info| [deploy_info.key?(:error) || !deploy_info.key?(property_name) ? '' : deploy_info[property_name], node] }
|
|
15
15
|
]
|
|
16
|
-
end
|
|
16
|
+
end
|
|
17
17
|
|
|
18
18
|
sort_by = :node
|
|
19
19
|
sort_desc = false
|
|
@@ -34,7 +34,7 @@ raise "Unknown sort name: #{sort_by}. Should be one of #{possible_sorts.keys.joi
|
|
|
34
34
|
|
|
35
35
|
sorted_deploy_info = deployer.
|
|
36
36
|
deployment_info_from(nodes_handler.select_nodes(executable.selected_nodes.empty? ? [{ all: true }] : executable.selected_nodes)).
|
|
37
|
-
|
|
37
|
+
to_h do |node, deploy_info|
|
|
38
38
|
decorated_deploy_info = deploy_info.merge(node: node)
|
|
39
39
|
if deploy_info.key?(:deployment_info)
|
|
40
40
|
decorated_deploy_info.merge!(deploy_info[:deployment_info])
|
|
@@ -43,7 +43,6 @@ sorted_deploy_info = deployer.
|
|
|
43
43
|
decorated_deploy_info[:services] = deploy_info[:services].join(', ') if deploy_info.key?(:services)
|
|
44
44
|
[node, decorated_deploy_info]
|
|
45
45
|
end.
|
|
46
|
-
to_h.
|
|
47
46
|
sort_by(&possible_sorts[sort_by])
|
|
48
47
|
sorted_deploy_info.reverse! if sort_desc
|
|
49
48
|
info_displayed = {
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -146,7 +146,7 @@ module HybridPlatformsConductor
|
|
|
146
146
|
actions_per_node[node].concat(resolved_nodes_actions)
|
|
147
147
|
end
|
|
148
148
|
end
|
|
149
|
-
result = actions_per_node.keys.
|
|
149
|
+
result = actions_per_node.keys.to_h { |node| [node, nil] }
|
|
150
150
|
with_connections_prepared_to(nodes_needing_connectors, no_exception: true) do |connected_nodes|
|
|
151
151
|
missing_nodes = []
|
|
152
152
|
connected_nodes.each do |node, connector|
|
|
@@ -194,7 +194,7 @@ module HybridPlatformsConductor
|
|
|
194
194
|
# * *connected_nodes* (Hash<String, Connector or Symbol>): Prepared connectors (or Symbol in case of failure with no_exception), per node name
|
|
195
195
|
def with_connections_prepared_to(nodes, no_exception: false)
|
|
196
196
|
# Make sure every node needing connectors finds a connector
|
|
197
|
-
nodes_needing_connectors = nodes.
|
|
197
|
+
nodes_needing_connectors = nodes.to_h { |node| [node, nil] }
|
|
198
198
|
@connector_plugins.each_value do |connector|
|
|
199
199
|
nodes_without_connectors = nodes_needing_connectors.select { |_node, selected_connector| selected_connector.nil? }.keys
|
|
200
200
|
break if nodes_without_connectors.empty?
|
|
@@ -84,7 +84,7 @@ module HybridPlatformsConductor
|
|
|
84
84
|
select_confs_for_node(node, fs_paths_rules).
|
|
85
85
|
inject({}) do |merged_paths, paths_info|
|
|
86
86
|
if paths_info[:context][:file_system_type] == file_system_type
|
|
87
|
-
merged_paths.merge(paths_info[:paths].
|
|
87
|
+
merged_paths.merge(paths_info[:paths].to_h do |path|
|
|
88
88
|
[
|
|
89
89
|
path,
|
|
90
90
|
{
|
|
@@ -92,7 +92,7 @@ module HybridPlatformsConductor
|
|
|
92
92
|
context: paths_info[:context]
|
|
93
93
|
}
|
|
94
94
|
]
|
|
95
|
-
end
|
|
95
|
+
end) do |path, rule_info_1, rule_info_2|
|
|
96
96
|
# Just check that configuration is not inconsistent
|
|
97
97
|
raise "Inconsistent rule for #{file_system_type} file system checks in configuration for #{node}: #{path} is marked as being both #{rule_info_1[:state]} and #{rule_info_2[:state]}" if rule_info_1[:state] != rule_info_2[:state]
|
|
98
98
|
|
|
@@ -263,9 +263,9 @@ module HybridPlatformsConductor
|
|
|
263
263
|
def deploy_on(*nodes_selectors)
|
|
264
264
|
# Get the sorted list of services to be deployed, per node
|
|
265
265
|
# Hash<String, Array<String> >
|
|
266
|
-
services_to_deploy = @nodes_handler.select_nodes(nodes_selectors.flatten).
|
|
266
|
+
services_to_deploy = @nodes_handler.select_nodes(nodes_selectors.flatten).to_h do |node|
|
|
267
267
|
[node, @nodes_handler.get_services_of(node)]
|
|
268
|
-
end
|
|
268
|
+
end
|
|
269
269
|
|
|
270
270
|
# Get the secrets to be deployed
|
|
271
271
|
secrets = {}
|
|
@@ -474,7 +474,7 @@ module HybridPlatformsConductor
|
|
|
474
474
|
timeout: 10,
|
|
475
475
|
progress_name: 'Read deployment logs'
|
|
476
476
|
)
|
|
477
|
-
nodes.
|
|
477
|
+
nodes.to_h do |node|
|
|
478
478
|
exit_code, stdout, stderr = read_actions_results[node] || [nil, nil, nil]
|
|
479
479
|
[
|
|
480
480
|
node,
|
|
@@ -485,7 +485,7 @@ module HybridPlatformsConductor
|
|
|
485
485
|
stderr&.force_encoding(Encoding::UTF_8)
|
|
486
486
|
)
|
|
487
487
|
]
|
|
488
|
-
end
|
|
488
|
+
end
|
|
489
489
|
end
|
|
490
490
|
|
|
491
491
|
# Parse stdout and stderr of a given deploy run and get the list of tasks with their status
|
|
@@ -553,7 +553,7 @@ module HybridPlatformsConductor
|
|
|
553
553
|
# Deploy for real
|
|
554
554
|
@nodes_handler.prefetch_metadata_of services.keys, :image
|
|
555
555
|
outputs = @actions_executor.execute_actions(
|
|
556
|
-
services.
|
|
556
|
+
services.to_h do |node, node_services|
|
|
557
557
|
image_id = @nodes_handler.get_image_of(node)
|
|
558
558
|
need_sudo = !@actions_executor.privileged_access?(node)
|
|
559
559
|
sudo = @actions_executor.sudo_prefix(node)
|
|
@@ -583,12 +583,12 @@ module HybridPlatformsConductor
|
|
|
583
583
|
remote_bash: "#{sudo}yum install -y ca-certificates"
|
|
584
584
|
},
|
|
585
585
|
{
|
|
586
|
-
scp: Dir.glob("#{ENV['hpc_certificates']}/*.crt").
|
|
586
|
+
scp: Dir.glob("#{ENV['hpc_certificates']}/*.crt").to_h do |cert_file|
|
|
587
587
|
[
|
|
588
588
|
cert_file,
|
|
589
589
|
'/etc/pki/ca-trust/source/anchors'
|
|
590
590
|
]
|
|
591
|
-
end.
|
|
591
|
+
end.merge(sudo: need_sudo),
|
|
592
592
|
remote_bash: [
|
|
593
593
|
"#{sudo}update-ca-trust enable",
|
|
594
594
|
"#{sudo}update-ca-trust extract"
|
|
@@ -613,19 +613,19 @@ module HybridPlatformsConductor
|
|
|
613
613
|
certificate_actions +
|
|
614
614
|
@services_handler.actions_to_deploy_on(node, node_services, @use_why_run)
|
|
615
615
|
]
|
|
616
|
-
end
|
|
616
|
+
end,
|
|
617
617
|
timeout: @timeout,
|
|
618
618
|
concurrent: @concurrent_execution,
|
|
619
619
|
log_to_stdout: !@concurrent_execution
|
|
620
620
|
)
|
|
621
621
|
# Free eventual locks
|
|
622
622
|
@actions_executor.execute_actions(
|
|
623
|
-
services.keys.
|
|
623
|
+
services.keys.to_h do |node|
|
|
624
624
|
[
|
|
625
625
|
node,
|
|
626
626
|
{ remote_bash: "#{@actions_executor.sudo_prefix(node)}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
|
|
627
627
|
]
|
|
628
|
-
end
|
|
628
|
+
end,
|
|
629
629
|
timeout: 10,
|
|
630
630
|
concurrent: true,
|
|
631
631
|
log_to_dir: nil
|
|
@@ -647,7 +647,7 @@ module HybridPlatformsConductor
|
|
|
647
647
|
section "Saving deployment logs for #{logs.size} nodes" do
|
|
648
648
|
ssh_user = @actions_executor.connector(:ssh).ssh_user
|
|
649
649
|
@actions_executor.execute_actions(
|
|
650
|
-
logs.
|
|
650
|
+
logs.to_h do |node, (exit_status, stdout, stderr)|
|
|
651
651
|
[
|
|
652
652
|
node,
|
|
653
653
|
log_plugins_for(node).
|
|
@@ -666,7 +666,7 @@ module HybridPlatformsConductor
|
|
|
666
666
|
end.
|
|
667
667
|
flatten(1)
|
|
668
668
|
]
|
|
669
|
-
end
|
|
669
|
+
end,
|
|
670
670
|
timeout: 10,
|
|
671
671
|
concurrent: true,
|
|
672
672
|
log_to_dir: nil,
|
|
@@ -99,7 +99,7 @@ module HybridPlatformsConductor
|
|
|
99
99
|
@instantiated_components[component] = HybridPlatformsConductor.const_get(component.to_s.split('_').collect(&:capitalize).join.to_sym).new(
|
|
100
100
|
logger: @logger,
|
|
101
101
|
logger_stderr: @logger_stderr,
|
|
102
|
-
**dependencies.
|
|
102
|
+
**dependencies.to_h { |dependency| [dependency, send(dependency)] }
|
|
103
103
|
)
|
|
104
104
|
end
|
|
105
105
|
@instantiated_components[component]
|
|
@@ -73,19 +73,33 @@ module HybridPlatformsConductor
|
|
|
73
73
|
def ip_for(*hosts)
|
|
74
74
|
results = {}
|
|
75
75
|
log_debug "Get IPs of #{hosts.size} hosts..."
|
|
76
|
+
exit_status, _stdout, _stderr = @cmd_runner.run_cmd('command -v getent', no_exception: true)
|
|
77
|
+
getent_present = exit_status.zero?
|
|
76
78
|
for_each_element_in(
|
|
77
79
|
hosts,
|
|
78
80
|
parallel: true,
|
|
79
81
|
nbr_threads_max: MAX_THREADS_GETENT,
|
|
80
82
|
progress: log_debug? ? 'Gather IPs' : nil
|
|
81
83
|
) do |host|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
84
|
+
ip =
|
|
85
|
+
if getent_present
|
|
86
|
+
_exit_status, stdout, _stderr = @cmd_runner.run_cmd(
|
|
87
|
+
"getent hosts #{host}",
|
|
88
|
+
timeout: TIMEOUT_GETENT,
|
|
89
|
+
log_to_stdout: log_debug?,
|
|
90
|
+
no_exception: true
|
|
91
|
+
)
|
|
92
|
+
stdout.strip.split(/\s+/).first
|
|
93
|
+
else
|
|
94
|
+
_exit_status, stdout, _stderr = @cmd_runner.run_cmd(
|
|
95
|
+
"host #{host} | grep 'has address'",
|
|
96
|
+
timeout: TIMEOUT_GETENT,
|
|
97
|
+
log_to_stdout: log_debug?,
|
|
98
|
+
no_exception: true
|
|
99
|
+
)
|
|
100
|
+
stdout.strip.split(/\s+/).last
|
|
101
|
+
end
|
|
102
|
+
|
|
89
103
|
if ip.nil?
|
|
90
104
|
log_warn "Host #{host} has no IP."
|
|
91
105
|
else
|
|
@@ -23,7 +23,7 @@ module HybridPlatformsConductor
|
|
|
23
23
|
# * Hash<String, Object>: The corresponding property, per required node.
|
|
24
24
|
# Nodes for which the property can't be fetched can be ommitted.
|
|
25
25
|
def get_services(nodes, _metadata)
|
|
26
|
-
nodes.
|
|
26
|
+
nodes.to_h { |node| [node, platform_for(node).services_for(node)] }
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
# Get other properties for a given set of nodes.
|
|
@@ -42,7 +42,7 @@ module HybridPlatformsConductor
|
|
|
42
42
|
# * Hash<String, Hash<Symbol,Object> >: The corresponding properties, per required node.
|
|
43
43
|
# Nodes for which the property can't be fetched can be ommitted.
|
|
44
44
|
def get_others(nodes, _metadata)
|
|
45
|
-
nodes.
|
|
45
|
+
nodes.to_h { |node| [node, platform_for(node).metadata_for(node)] }
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
private
|
|
@@ -605,7 +605,7 @@ module HybridPlatformsConductor
|
|
|
605
605
|
end
|
|
606
606
|
else
|
|
607
607
|
# We have not created any ControlMaster, but still consider the nodes to be ready to connect
|
|
608
|
-
user_locks = nodes.
|
|
608
|
+
user_locks = nodes.to_h { |node| [node, nil] }
|
|
609
609
|
end
|
|
610
610
|
yield user_locks.keys
|
|
611
611
|
ensure
|
|
@@ -132,8 +132,7 @@ module HybridPlatformsConductor
|
|
|
132
132
|
else
|
|
133
133
|
(info[:status][:added_files] + info[:status][:changed_files] + info[:status][:untracked_files]).
|
|
134
134
|
sort.
|
|
135
|
-
|
|
136
|
-
to_h
|
|
135
|
+
to_h { |f| [f, File.mtime("#{@repository_path}/#{f}").strftime('%F %T')] }
|
|
137
136
|
end,
|
|
138
137
|
deleted_files: info[:status].nil? ? [] : info[:status][:deleted_files].sort
|
|
139
138
|
}
|
|
@@ -371,7 +371,7 @@ class ProxmoxWaiter
|
|
|
371
371
|
# Result::
|
|
372
372
|
# * Hash<String, [Float or nil, Float or nil]>: The set of 2 scores, per PVE node name
|
|
373
373
|
def pve_scores_for(_nbr_cpus, ram_mb, disk_gb)
|
|
374
|
-
@config['pve_nodes'].
|
|
374
|
+
@config['pve_nodes'].to_h do |pve_node|
|
|
375
375
|
# Get some resource usages stats from the node directly
|
|
376
376
|
status_info = api_get("nodes/#{pve_node}/status")
|
|
377
377
|
load_average = status_info['loadavg'].map { |load_str| Float(load_str) }
|
|
@@ -418,11 +418,11 @@ class ProxmoxWaiter
|
|
|
418
418
|
[
|
|
419
419
|
if expected_ram_percent_used <= @config['limits']['ram_percent_used_max'] &&
|
|
420
420
|
expected_disk_percent_used <= @config['limits']['disk_percent_used_max']
|
|
421
|
-
expected_ram_percent_used * @config['coeff_ram_consumption'] + expected_disk_percent_used * @config['coeff_disk_consumption']
|
|
421
|
+
(expected_ram_percent_used * @config['coeff_ram_consumption']) + (expected_disk_percent_used * @config['coeff_disk_consumption'])
|
|
422
422
|
end,
|
|
423
423
|
if expected_ram_percent_used_without_expired <= @config['limits']['ram_percent_used_max'] &&
|
|
424
424
|
expected_disk_percent_used_without_expired <= @config['limits']['disk_percent_used_max']
|
|
425
|
-
expected_ram_percent_used_without_expired * @config['coeff_ram_consumption'] + expected_disk_percent_used_without_expired * @config['coeff_disk_consumption']
|
|
425
|
+
(expected_ram_percent_used_without_expired * @config['coeff_ram_consumption']) + (expected_disk_percent_used_without_expired * @config['coeff_disk_consumption'])
|
|
426
426
|
end
|
|
427
427
|
]
|
|
428
428
|
else
|
|
@@ -431,7 +431,7 @@ class ProxmoxWaiter
|
|
|
431
431
|
[nil, nil]
|
|
432
432
|
end
|
|
433
433
|
]
|
|
434
|
-
end
|
|
434
|
+
end
|
|
435
435
|
end
|
|
436
436
|
|
|
437
437
|
# Is a given VM expired?
|
|
@@ -499,10 +499,10 @@ class ProxmoxWaiter
|
|
|
499
499
|
if hpc_marker_idx.nil?
|
|
500
500
|
{}
|
|
501
501
|
else
|
|
502
|
-
vm_description_lines[hpc_marker_idx + 1..].
|
|
502
|
+
vm_description_lines[hpc_marker_idx + 1..].to_h do |line|
|
|
503
503
|
property, value = line.split(': ')
|
|
504
504
|
[property.to_sym, value]
|
|
505
|
-
end
|
|
505
|
+
end
|
|
506
506
|
end
|
|
507
507
|
end
|
|
508
508
|
|
|
@@ -101,10 +101,10 @@ module HybridPlatformsConductor
|
|
|
101
101
|
|
|
102
102
|
# Get the HPC info associated to this VM
|
|
103
103
|
# Hash<Symbol,String>
|
|
104
|
-
vm_hpc_info = vm_description_lines[hpc_marker_idx + 1..].
|
|
104
|
+
vm_hpc_info = vm_description_lines[hpc_marker_idx + 1..].to_h do |line|
|
|
105
105
|
property, value = line.split(': ')
|
|
106
106
|
[property.to_sym, value]
|
|
107
|
-
end
|
|
107
|
+
end
|
|
108
108
|
next unless vm_hpc_info[:node] == @node && vm_hpc_info[:environment] == @environment
|
|
109
109
|
|
|
110
110
|
# Found it
|
|
@@ -87,7 +87,7 @@ module HybridPlatformsConductor
|
|
|
87
87
|
@nodes_handler.prefetch_metadata_of nodes, locale.keys
|
|
88
88
|
nodes.
|
|
89
89
|
map do |node|
|
|
90
|
-
{ node: node }.merge(all_properties.
|
|
90
|
+
{ node: node }.merge(all_properties.to_h { |property| [property, @nodes_handler.metadata_of(node, property)] })
|
|
91
91
|
end.
|
|
92
92
|
# Group them by physical / VMs
|
|
93
93
|
group_by do |node_info|
|
|
@@ -125,7 +125,7 @@ module HybridPlatformsConductor
|
|
|
125
125
|
# * Hash: The JSON secrets parsed from this group
|
|
126
126
|
def parse_xml_secrets(group)
|
|
127
127
|
# Parse all entries
|
|
128
|
-
group.xpath('Entry').
|
|
128
|
+
group.xpath('Entry').to_h do |entry|
|
|
129
129
|
[
|
|
130
130
|
entry.at_xpath('String/Key[contains(.,"Title")]/../Value').text,
|
|
131
131
|
FIELDS.map do |property, field|
|
|
@@ -139,7 +139,7 @@ module HybridPlatformsConductor
|
|
|
139
139
|
]
|
|
140
140
|
end
|
|
141
141
|
end.compact.to_h.merge(
|
|
142
|
-
entry.xpath('Binary').
|
|
142
|
+
entry.xpath('Binary').to_h do |binary|
|
|
143
143
|
binary_meta = group.document.at_xpath("KeePassFile/Meta/Binaries/Binary[@ID=#{Integer(binary.xpath('Value').attr('Ref').value)}]")
|
|
144
144
|
binary_content = Base64.decode64(binary_meta.text)
|
|
145
145
|
if binary_meta.attr('Compressed') == 'True'
|
|
@@ -151,17 +151,17 @@ module HybridPlatformsConductor
|
|
|
151
151
|
binary.xpath('Key').text,
|
|
152
152
|
binary_content
|
|
153
153
|
]
|
|
154
|
-
end
|
|
154
|
+
end
|
|
155
155
|
)
|
|
156
156
|
]
|
|
157
|
-
end.
|
|
157
|
+
end.merge(
|
|
158
158
|
# Add children groups
|
|
159
|
-
group.xpath('Group').
|
|
159
|
+
group.xpath('Group').to_h do |sub_group|
|
|
160
160
|
[
|
|
161
161
|
sub_group.at_xpath('Name').text,
|
|
162
162
|
parse_xml_secrets(sub_group)
|
|
163
163
|
]
|
|
164
|
-
end
|
|
164
|
+
end
|
|
165
165
|
)
|
|
166
166
|
end
|
|
167
167
|
|
|
@@ -15,7 +15,7 @@ module HybridPlatformsConductor
|
|
|
15
15
|
# Check my_test_plugin.rb.sample documentation for signature details.
|
|
16
16
|
def test_on_node
|
|
17
17
|
# Flatten the paths rules so that we can spot inconsistencies in configuration
|
|
18
|
-
@config.aggregate_files_rules(@nodes_handler, @node).
|
|
18
|
+
@config.aggregate_files_rules(@nodes_handler, @node).to_h do |path, rule_info|
|
|
19
19
|
[
|
|
20
20
|
"if #{@actions_executor.sudo_prefix(@node)}/bin/bash -c '[[ -d \"#{path}\" ]]' ; then echo 1 ; else echo 0 ; fi",
|
|
21
21
|
{
|
|
@@ -32,7 +32,7 @@ module HybridPlatformsConductor
|
|
|
32
32
|
timeout: 2
|
|
33
33
|
}
|
|
34
34
|
]
|
|
35
|
-
end
|
|
35
|
+
end
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
end
|
|
@@ -14,7 +14,7 @@ module HybridPlatformsConductor
|
|
|
14
14
|
# Check my_test_plugin.rb.sample documentation for signature details.
|
|
15
15
|
def test_on_node
|
|
16
16
|
# Flatten the paths rules so that we can spot inconsistencies in configuration
|
|
17
|
-
@config.aggregate_files_rules(@nodes_handler, @node, file_system_type: :hdfs).
|
|
17
|
+
@config.aggregate_files_rules(@nodes_handler, @node, file_system_type: :hdfs).to_h do |path, rule_info|
|
|
18
18
|
[
|
|
19
19
|
"if sudo#{rule_info[:context][:sudo_user] ? " -u #{rule_info[:context][:sudo_user]}" : ''} hdfs dfs -ls \"#{path}\" ; then echo 1 ; else echo 0 ; fi",
|
|
20
20
|
{
|
|
@@ -31,7 +31,7 @@ module HybridPlatformsConductor
|
|
|
31
31
|
timeout: 5
|
|
32
32
|
}
|
|
33
33
|
]
|
|
34
|
-
end
|
|
34
|
+
end
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
end
|
|
@@ -18,7 +18,7 @@ module HybridPlatformsConductor
|
|
|
18
18
|
else
|
|
19
19
|
private_ips = @nodes_handler.get_private_ips_of @node
|
|
20
20
|
if private_ips
|
|
21
|
-
host_ips = stdout.first.split.
|
|
21
|
+
host_ips = stdout.first.split.grep(/^172\.16\.\d+\.\d+$/).sort
|
|
22
22
|
ref_ips = private_ips.sort
|
|
23
23
|
assert_equal(
|
|
24
24
|
host_ips,
|
|
@@ -31,7 +31,7 @@ module HybridPlatformsConductor
|
|
|
31
31
|
next if stdout.empty?
|
|
32
32
|
|
|
33
33
|
_exit_status, stdout, _stderr = @cmd_runner.run_cmd(
|
|
34
|
-
"cd #{@platform.repository_path} && git --no-pager log #{merge_commit_id} --pretty=format:%aI",
|
|
34
|
+
"cd #{@platform.repository_path} && git --no-pager log #{merge_commit_id} --max-count 1 --pretty=format:%aI",
|
|
35
35
|
log_to_stdout: log_debug?
|
|
36
36
|
)
|
|
37
37
|
error "Git history is not linear because of Merge commit #{merge_commit_id}" if Time.now - Time.parse(stdout.strip) < LOOKING_PERIOD
|
|
@@ -13,8 +13,7 @@ module HybridPlatformsConductor
|
|
|
13
13
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, :private_ips
|
|
14
14
|
private_ips = @nodes_handler.
|
|
15
15
|
known_nodes.
|
|
16
|
-
|
|
17
|
-
to_h
|
|
16
|
+
to_h { |node| [node, @nodes_handler.get_private_ips_of(node) || []] }
|
|
18
17
|
|
|
19
18
|
# Check there are no duplicates
|
|
20
19
|
nodes_per_private_ip = {}
|
|
@@ -13,8 +13,7 @@ module HybridPlatformsConductor
|
|
|
13
13
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, :public_ips
|
|
14
14
|
public_ips = @nodes_handler.
|
|
15
15
|
known_nodes.
|
|
16
|
-
|
|
17
|
-
to_h
|
|
16
|
+
to_h { |node| [node, @nodes_handler.get_public_ips_of(node) || []] }
|
|
18
17
|
|
|
19
18
|
# Check there are no duplicates
|
|
20
19
|
nodes_per_public_ip = {}
|
|
@@ -13,8 +13,7 @@ module HybridPlatformsConductor
|
|
|
13
13
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, :veid
|
|
14
14
|
veids = @nodes_handler.
|
|
15
15
|
known_nodes.
|
|
16
|
-
|
|
17
|
-
to_h
|
|
16
|
+
to_h { |node| [node, @nodes_handler.get_veid_of(node) ? @nodes_handler.get_veid_of(node).to_i : nil] }
|
|
18
17
|
|
|
19
18
|
# Check there are no duplicates
|
|
20
19
|
veids.group_by { |_node, veid| veid }.each do |veid, nodes|
|
|
@@ -57,7 +57,7 @@ module HybridPlatformsConductor
|
|
|
57
57
|
end
|
|
58
58
|
)
|
|
59
59
|
sudo = @actions_executor.sudo_prefix(@node)
|
|
60
|
-
urls.
|
|
60
|
+
urls.to_h do |url|
|
|
61
61
|
# 1. Get the OVAL file on the node to be tested (uncompress it if needed)
|
|
62
62
|
# 2. Make sure oscap is installed
|
|
63
63
|
# 3. Generate the report for this OVAL file using oscap
|
|
@@ -143,7 +143,7 @@ module HybridPlatformsConductor
|
|
|
143
143
|
timeout: 240
|
|
144
144
|
}
|
|
145
145
|
]
|
|
146
|
-
end
|
|
146
|
+
end
|
|
147
147
|
else
|
|
148
148
|
error "No OVAL file defined for image #{image} at #{oval_file}"
|
|
149
149
|
{}
|
|
@@ -75,10 +75,11 @@ module HybridPlatformsConductor
|
|
|
75
75
|
# Register plugins by parsing gems
|
|
76
76
|
def register_plugins_from_gems
|
|
77
77
|
# Require all possible files that could define such a plugin, from all gems
|
|
78
|
-
|
|
79
|
-
|
|
78
|
+
Gem.loaded_specs.clone.each do |gem_name, gem_specs|
|
|
79
|
+
gem_path = gem_specs.full_gem_path
|
|
80
|
+
files_regexp = %r{#{Regexp.escape(gem_path)}/lib/(.*hpc_plugins/#{Regexp.escape(@plugins_type.to_s)}/[^/]+)\.rb$}
|
|
80
81
|
# Careful to not use gem_specs.files here as if your gem name contains "-" or other weird characters, files won't appear in the gemspec list.
|
|
81
|
-
Dir.glob("#{
|
|
82
|
+
Dir.glob("#{gem_path}/lib/**/*.rb").each do |file|
|
|
82
83
|
next unless file =~ files_regexp
|
|
83
84
|
|
|
84
85
|
require_name = Regexp.last_match(1)
|
|
@@ -524,14 +524,14 @@ module HybridPlatformsConductor
|
|
|
524
524
|
nodes_to_test = selected_tests.map(&:node).uniq.sort
|
|
525
525
|
@outputs =
|
|
526
526
|
if @skip_run
|
|
527
|
-
nodes_to_test.
|
|
527
|
+
nodes_to_test.to_h do |node|
|
|
528
528
|
run_log_file_name = "#{@config.hybrid_platforms_dir}/run_logs/#{node}.stdout"
|
|
529
529
|
[
|
|
530
530
|
node,
|
|
531
531
|
# TODO: Find a way to also save stderr and the status code
|
|
532
532
|
[0, File.exist?(run_log_file_name) ? File.read(run_log_file_name) : nil, '']
|
|
533
533
|
]
|
|
534
|
-
end
|
|
534
|
+
end
|
|
535
535
|
else
|
|
536
536
|
# Why-run deploy on all nodes
|
|
537
537
|
@deployer.concurrent_execution = !log_debug?
|
|
@@ -112,15 +112,14 @@ module HybridPlatformsConductor
|
|
|
112
112
|
# Parse plugins
|
|
113
113
|
@plugins = Dir.
|
|
114
114
|
glob("#{__dir__}/topographer/plugins/*.rb").
|
|
115
|
-
|
|
115
|
+
to_h do |file_name|
|
|
116
116
|
plugin_name = File.basename(file_name)[0..-4].to_sym
|
|
117
117
|
require file_name
|
|
118
118
|
[
|
|
119
119
|
plugin_name,
|
|
120
120
|
Topographer::Plugins.const_get(plugin_name.to_s.split('_').collect(&:capitalize).join.to_sym)
|
|
121
121
|
]
|
|
122
|
-
end
|
|
123
|
-
to_h
|
|
122
|
+
end
|
|
124
123
|
|
|
125
124
|
@ips_to_host = known_ips.clone
|
|
126
125
|
|
|
@@ -132,11 +131,11 @@ module HybridPlatformsConductor
|
|
|
132
131
|
]
|
|
133
132
|
@nodes_handler.prefetch_metadata_of @nodes_handler.known_nodes, metadata_properties
|
|
134
133
|
@nodes_handler.known_nodes.each do |hostname|
|
|
135
|
-
@node_metadata[hostname] = metadata_properties.
|
|
134
|
+
@node_metadata[hostname] = metadata_properties.to_h { |property| [property, @nodes_handler.metadata_of(hostname, property)] }
|
|
136
135
|
end
|
|
137
136
|
|
|
138
137
|
# Small cache of hostnames used a lot to parse JSON
|
|
139
|
-
@known_nodes = @nodes_handler.known_nodes.
|
|
138
|
+
@known_nodes = @nodes_handler.known_nodes.to_h { |hostname| [hostname, nil] }
|
|
140
139
|
# Cache of objects being used a lot in parsing for performance
|
|
141
140
|
@non_word_regexp = /\W+/
|
|
142
141
|
@ip_regexp = %r{(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(/(\d{1,2})|[^\d/]|$)}
|
|
@@ -600,7 +599,7 @@ module HybridPlatformsConductor
|
|
|
600
599
|
@ips_mask[ip_def] = {} unless @ips_mask.key?(ip_def)
|
|
601
600
|
unless @ips_mask[ip_def].key?(ip_mask)
|
|
602
601
|
# For performance, keep a cache of all the IPAddress::IPv4 objects
|
|
603
|
-
@ip_v4_cache = known_ips.keys.
|
|
602
|
+
@ip_v4_cache = known_ips.keys.to_h { |ip, _node| [ip, IPAddress::IPv4.new(ip)] } unless defined?(@ip_v4_cache)
|
|
604
603
|
ip_range = IPAddress::IPv4.new("#{ip_def}/#{ip_mask}")
|
|
605
604
|
@ips_mask[ip_def][ip_mask] = @ip_v4_cache.select { |_ip, ipv4| ip_range.include?(ipv4) }.keys
|
|
606
605
|
end
|