hybrid_platforms_conductor 33.0.0 → 33.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/bin/check-node +0 -1
- data/bin/deploy +0 -1
- data/bin/get_impacted_nodes +0 -1
- data/bin/last_deploys +8 -7
- data/bin/nodes_to_deploy +2 -2
- data/bin/setup +6 -6
- data/bin/topograph +1 -1
- data/lib/hybrid_platforms_conductor/action.rb +4 -4
- data/lib/hybrid_platforms_conductor/actions_executor.rb +37 -42
- data/lib/hybrid_platforms_conductor/bitbucket.rb +5 -4
- data/lib/hybrid_platforms_conductor/cmd_runner.rb +13 -12
- data/lib/hybrid_platforms_conductor/cmdb.rb +2 -2
- data/lib/hybrid_platforms_conductor/common_config_dsl/bitbucket.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/confluence.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +5 -4
- data/lib/hybrid_platforms_conductor/common_config_dsl/github.rb +4 -3
- data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb +2 -2
- data/lib/hybrid_platforms_conductor/config.rb +7 -4
- data/lib/hybrid_platforms_conductor/confluence.rb +1 -1
- data/lib/hybrid_platforms_conductor/connector.rb +5 -2
- data/lib/hybrid_platforms_conductor/core_extensions/symbol/zero.rb +24 -0
- data/lib/hybrid_platforms_conductor/credentials.rb +39 -36
- data/lib/hybrid_platforms_conductor/current_dir_monitor.rb +4 -1
- data/lib/hybrid_platforms_conductor/deployer.rb +95 -96
- data/lib/hybrid_platforms_conductor/executable.rb +20 -15
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/config.rb +10 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +4 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb +2 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +45 -49
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb +6 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +68 -66
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb +13 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb +39 -38
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb +5 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb +43 -45
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/podman.rb +18 -20
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +118 -117
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +39 -43
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container +9 -13
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +28 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/stdout.rb +26 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/cli.rb +8 -6
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb +7 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb +4 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +0 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb +21 -19
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +19 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +19 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb +2 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb +21 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb +10 -12
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +9 -9
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +10 -9
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +4 -6
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +25 -24
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/stdout.rb +8 -6
- data/lib/hybrid_platforms_conductor/io_router.rb +14 -13
- data/lib/hybrid_platforms_conductor/json_dumper.rb +2 -2
- data/lib/hybrid_platforms_conductor/log.rb +2 -2
- data/lib/hybrid_platforms_conductor/logger_helpers.rb +19 -16
- data/lib/hybrid_platforms_conductor/nodes_handler.rb +89 -71
- data/lib/hybrid_platforms_conductor/parallel_threads.rb +7 -11
- data/lib/hybrid_platforms_conductor/platform_handler.rb +7 -7
- data/lib/hybrid_platforms_conductor/platforms_handler.rb +5 -3
- data/lib/hybrid_platforms_conductor/plugin.rb +2 -2
- data/lib/hybrid_platforms_conductor/plugins.rb +13 -8
- data/lib/hybrid_platforms_conductor/provisioner.rb +4 -4
- data/lib/hybrid_platforms_conductor/report.rb +2 -2
- data/lib/hybrid_platforms_conductor/reports_handler.rb +3 -2
- data/lib/hybrid_platforms_conductor/secrets_reader.rb +2 -2
- data/lib/hybrid_platforms_conductor/services_handler.rb +32 -29
- data/lib/hybrid_platforms_conductor/test_only_remote_node.rb +1 -1
- data/lib/hybrid_platforms_conductor/test_report.rb +15 -18
- data/lib/hybrid_platforms_conductor/tests_runner.rb +116 -118
- data/lib/hybrid_platforms_conductor/thycotic.rb +28 -19
- data/lib/hybrid_platforms_conductor/topographer.rb +200 -190
- data/lib/hybrid_platforms_conductor/topographer/plugins/graphviz.rb +8 -8
- data/lib/hybrid_platforms_conductor/topographer/plugins/json.rb +4 -4
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +16 -11
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/interactive_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb +23 -18
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +67 -49
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/scp_spec.rb +20 -14
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions_spec.rb +63 -50
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connection_spec.rb +35 -35
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/connectable_nodes_spec.rb +12 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb +4 -7
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +21 -22
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +23 -24
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connectable_nodes_spec.rb +10 -6
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +73 -54
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +145 -126
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +29 -25
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/logging_spec.rb +106 -91
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/parallel_spec.rb +272 -244
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +10 -12
- data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +36 -36
- data/spec/hybrid_platforms_conductor_test/api/config_spec.rb +24 -22
- data/spec/hybrid_platforms_conductor_test/api/deployer/check_spec.rb +4 -2
- data/spec/hybrid_platforms_conductor_test/api/deployer/config_dsl_spec.rb +7 -11
- data/spec/hybrid_platforms_conductor_test/api/deployer/deploy_spec.rb +34 -26
- data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +21 -21
- data/spec/hybrid_platforms_conductor_test/api/deployer/parse_deploy_output_spec.rb +55 -59
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +36 -62
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/podman_spec.rb +17 -17
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/config_dsl_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/create_spec.rb +44 -51
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/destroy_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/ip_spec.rb +12 -16
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/destroy_vm_spec.rb +31 -19
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +324 -266
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/ips_assignment_spec.rb +89 -61
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/other_lxc_containers_resources_spec.rb +117 -93
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/pve_node_resources_spec.rb +71 -54
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +10 -8
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb +80 -60
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/start_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/stop_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/cli_spec.rb +21 -20
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/thycotic_spec.rb +70 -55
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/config_spec.rb +8 -10
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +33 -24
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_keys_spec.rb +64 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/platform_handlers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +50 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +91 -81
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +14 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/git_diff_impacts_spec.rb +51 -75
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/nodes_selectors_spec.rb +35 -26
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/platform_handlers_plugins_api_spec.rb +24 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/several_platforms_spec.rb +29 -19
- data/spec/hybrid_platforms_conductor_test/api/platform_handler_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb +57 -99
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +32 -35
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +10 -10
- data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +38 -27
- data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +46 -52
- data/spec/hybrid_platforms_conductor_test/api/reports_handler_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/services_handler/actions_to_deploy_spec.rb +90 -58
- data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +38 -34
- data/spec/hybrid_platforms_conductor_test/api/services_handler/log_info_spec.rb +11 -9
- data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +193 -171
- data/spec/hybrid_platforms_conductor_test/api/services_handler/parse_deploy_output_spec.rb +66 -54
- data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +147 -133
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +69 -49
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/global_spec.rb +4 -3
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +30 -27
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/platform_spec.rb +12 -9
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +48 -47
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/bitbucket_conf_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/github_ci_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_reports_plugins/confluence_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +9 -9
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb2.rb → test_cmdb_2.rb} +6 -6
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb_others2.rb → test_cmdb_others_2.rb} +2 -2
- data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/{check-node_spec.rb → check_node_spec.rb} +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/get_impacted_nodes_spec.rb +76 -77
- data/spec/hybrid_platforms_conductor_test/executables/last_deploys_spec.rb +13 -15
- data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +289 -307
- data/spec/hybrid_platforms_conductor_test/executables/options/actions_executor_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb +8 -9
- data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +12 -14
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_selectors_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/reports_handler_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +22 -22
- data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +22 -16
- data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +32 -32
- data/spec/hybrid_platforms_conductor_test/executables/ssh_config_spec.rb +7 -9
- data/spec/hybrid_platforms_conductor_test/executables/test_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +4 -3
- data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/config_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +12 -13
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +216 -14
- data/spec/hybrid_platforms_conductor_test/helpers/executables_helpers.rb +11 -11
- data/spec/hybrid_platforms_conductor_test/helpers/nodes_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +39 -28
- data/spec/hybrid_platforms_conductor_test/helpers/plugins_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +86 -111
- data/spec/hybrid_platforms_conductor_test/helpers/reports_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/helpers/services_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/tests_runner_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id1.rb → test_plugin_id_1.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id2.rb → test_plugin_id_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/test_plugin_type/{test_plugin_id3.rb → test_plugin_id_3.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/{test_plugin_type2/test_plugin_id4.rb → test_plugin_type_2/test_plugin_id_4.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/test.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/{test2.rb → test_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/rubocop_spec.rb +31 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +952 -0
- data/spec/hybrid_platforms_conductor_test/test_connector.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/test_log_no_read_plugin.rb +3 -1
- data/spec/hybrid_platforms_conductor_test/test_log_plugin.rb +3 -1
- data/spec/hybrid_platforms_conductor_test/test_plugins/global.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node_check.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/platform.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/several_checks.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/test_secrets_reader_plugin.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/tests_report_plugin.rb +5 -6
- data/spec/spec_helper.rb +17 -18
- data/tools/check_md +16 -20
- data/tools/generate_mermaid +1 -1
- metadata +43 -13
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +0 -1152
@@ -4,9 +4,10 @@ module HybridPlatformsConductor
|
|
4
4
|
|
5
5
|
module CommonConfigDsl
|
6
6
|
|
7
|
+
# Add common Bitbucket config DSL to declare known Bitbucket repositories
|
7
8
|
module Bitbucket
|
8
9
|
|
9
|
-
# Initialize the DSL
|
10
|
+
# Initialize the DSL
|
10
11
|
def init_bitbucket
|
11
12
|
# List of Bitbucket repositories definitions
|
12
13
|
# Array< Hash<Symbol, Object> >
|
@@ -2,9 +2,10 @@ module HybridPlatformsConductor
|
|
2
2
|
|
3
3
|
module CommonConfigDsl
|
4
4
|
|
5
|
+
# Add common Confluence config DSL to declare known Confluence installations
|
5
6
|
module Confluence
|
6
7
|
|
7
|
-
# Initialize the DSL
|
8
|
+
# Initialize the DSL
|
8
9
|
def init_confluence
|
9
10
|
# Confluence configuration (can be nil if none)
|
10
11
|
# Hash<Symbol, Object> or nil. See #confluence_info to know details.
|
@@ -19,7 +19,7 @@ module HybridPlatformsConductor
|
|
19
19
|
# Array< Hash<Symbol, Object> >
|
20
20
|
attr_reader :fs_paths_rules
|
21
21
|
|
22
|
-
# Initialize the DSL
|
22
|
+
# Initialize the DSL
|
23
23
|
def init_file_system_tests
|
24
24
|
@fs_paths_rules = []
|
25
25
|
@context = {
|
@@ -49,7 +49,7 @@ module HybridPlatformsConductor
|
|
49
49
|
paths: paths.flatten,
|
50
50
|
nodes_selectors_stack: current_nodes_selectors_stack,
|
51
51
|
context: @context.clone,
|
52
|
-
state: :present
|
52
|
+
state: :present
|
53
53
|
}
|
54
54
|
end
|
55
55
|
|
@@ -84,7 +84,7 @@ module HybridPlatformsConductor
|
|
84
84
|
select_confs_for_node(node, fs_paths_rules).
|
85
85
|
inject({}) do |merged_paths, paths_info|
|
86
86
|
if paths_info[:context][:file_system_type] == file_system_type
|
87
|
-
merged_paths.merge(
|
87
|
+
merged_paths.merge(paths_info[:paths].map do |path|
|
88
88
|
[
|
89
89
|
path,
|
90
90
|
{
|
@@ -92,9 +92,10 @@ module HybridPlatformsConductor
|
|
92
92
|
context: paths_info[:context]
|
93
93
|
}
|
94
94
|
]
|
95
|
-
end
|
95
|
+
end.to_h) do |path, rule_info_1, rule_info_2|
|
96
96
|
# Just check that configuration is not inconsistent
|
97
97
|
raise "Inconsistent rule for #{file_system_type} file system checks in configuration for #{node}: #{path} is marked as being both #{rule_info_1[:state]} and #{rule_info_2[:state]}" if rule_info_1[:state] != rule_info_2[:state]
|
98
|
+
|
98
99
|
rule_info_2
|
99
100
|
end
|
100
101
|
else
|
@@ -5,9 +5,10 @@ module HybridPlatformsConductor
|
|
5
5
|
|
6
6
|
module CommonConfigDsl
|
7
7
|
|
8
|
+
# Add common Github config DSL to declare known Github repositories
|
8
9
|
module Github
|
9
10
|
|
10
|
-
# Initialize the DSL
|
11
|
+
# Initialize the DSL
|
11
12
|
def init_github
|
12
13
|
# List of Github repositories definitions
|
13
14
|
# Array< Hash<Symbol, Object> >
|
@@ -18,10 +19,10 @@ module HybridPlatformsConductor
|
|
18
19
|
# Register new Github repositories
|
19
20
|
#
|
20
21
|
# Parameters::
|
21
|
-
# * *url* (String): URL to the Github API [default: 'https://api.github.com']
|
22
22
|
# * *user* (String): User or organization name, storing repositories
|
23
|
+
# * *url* (String): URL to the Github API [default: 'https://api.github.com']
|
23
24
|
# * *repos* (Array<String> or Symbol): List of repository names from this project, or :all for all [default: :all]
|
24
|
-
def github_repos(url: 'https://api.github.com',
|
25
|
+
def github_repos(user:, url: 'https://api.github.com', repos: :all)
|
25
26
|
@github_repos << {
|
26
27
|
url: url,
|
27
28
|
user: user,
|
@@ -38,7 +38,7 @@ module HybridPlatformsConductor
|
|
38
38
|
def ignore_idempotence_tasks(tasks_to_ignore)
|
39
39
|
@ignored_idempotence_tasks << {
|
40
40
|
ignored_tasks: tasks_to_ignore,
|
41
|
-
nodes_selectors_stack: current_nodes_selectors_stack
|
41
|
+
nodes_selectors_stack: current_nodes_selectors_stack
|
42
42
|
}
|
43
43
|
end
|
44
44
|
|
@@ -49,7 +49,7 @@ module HybridPlatformsConductor
|
|
49
49
|
def ignore_divergent_tasks(tasks_to_ignore)
|
50
50
|
@ignored_divergent_tasks << {
|
51
51
|
ignored_tasks: tasks_to_ignore,
|
52
|
-
nodes_selectors_stack: current_nodes_selectors_stack
|
52
|
+
nodes_selectors_stack: current_nodes_selectors_stack
|
53
53
|
}
|
54
54
|
end
|
55
55
|
|
@@ -8,7 +8,8 @@ module HybridPlatformsConductor
|
|
8
8
|
# Object used to access the whole configuration
|
9
9
|
class Config
|
10
10
|
|
11
|
-
include
|
11
|
+
include Cleanroom
|
12
|
+
include LoggerHelpers
|
12
13
|
|
13
14
|
class << self
|
14
15
|
|
@@ -35,6 +36,7 @@ module HybridPlatformsConductor
|
|
35
36
|
# Directory of the definition of the platforms
|
36
37
|
# String
|
37
38
|
attr_reader :hybrid_platforms_dir
|
39
|
+
|
38
40
|
expose :hybrid_platforms_dir
|
39
41
|
|
40
42
|
# List of expected failures info. Each info has the following properties:
|
@@ -60,7 +62,7 @@ module HybridPlatformsConductor
|
|
60
62
|
# Parameters::
|
61
63
|
# * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
|
62
64
|
# * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
|
63
|
-
def initialize(logger: Logger.new(
|
65
|
+
def initialize(logger: Logger.new($stdout), logger_stderr: Logger.new($stderr))
|
64
66
|
init_loggers(logger, logger_stderr)
|
65
67
|
@hybrid_platforms_dir = File.expand_path(ENV['hpc_platforms'].nil? ? '.' : ENV['hpc_platforms'])
|
66
68
|
# Stack of the nodes selectors scopes
|
@@ -94,7 +96,7 @@ module HybridPlatformsConductor
|
|
94
96
|
end
|
95
97
|
# Call initializers if needed
|
96
98
|
Config.mixin_initializers.each do |mixin_init_method|
|
97
|
-
|
99
|
+
send(mixin_init_method)
|
98
100
|
end
|
99
101
|
include_config_from "#{@hybrid_platforms_dir}/hpc_config.rb"
|
100
102
|
end
|
@@ -105,7 +107,7 @@ module HybridPlatformsConductor
|
|
105
107
|
# * *dsl_file* (String): Path to the DSL file
|
106
108
|
def include_config_from(dsl_file)
|
107
109
|
log_debug "Include config from #{dsl_file}"
|
108
|
-
|
110
|
+
evaluate_file(dsl_file)
|
109
111
|
end
|
110
112
|
expose :include_config_from
|
111
113
|
|
@@ -116,6 +118,7 @@ module HybridPlatformsConductor
|
|
116
118
|
# * *dir* (String): Directory containing the Dockerfile defining the image
|
117
119
|
def os_image(image, dir)
|
118
120
|
raise "OS image #{image} already defined to #{@os_images[image]}" if @os_images.key?(image)
|
121
|
+
|
119
122
|
@os_images[image] = dir
|
120
123
|
end
|
121
124
|
expose :os_image
|
@@ -34,7 +34,7 @@ module HybridPlatformsConductor
|
|
34
34
|
# * *confluence_password* (String): Confluence password to be used when querying the API
|
35
35
|
# * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
|
36
36
|
# * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
|
37
|
-
def initialize(confluence_url, confluence_user_name, confluence_password, logger: Logger.new(
|
37
|
+
def initialize(confluence_url, confluence_user_name, confluence_password, logger: Logger.new($stdout), logger_stderr: Logger.new($stderr))
|
38
38
|
init_loggers(logger, logger_stderr)
|
39
39
|
@confluence_url = confluence_url
|
40
40
|
@confluence_user_name = confluence_user_name
|
@@ -15,8 +15,8 @@ module HybridPlatformsConductor
|
|
15
15
|
# * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
|
16
16
|
# * *nodes_handler* (NodesHandler): NodesHandler to be used. [default: NodesHandler.new]
|
17
17
|
def initialize(
|
18
|
-
logger: Logger.new(
|
19
|
-
logger_stderr: Logger.new(
|
18
|
+
logger: Logger.new($stdout),
|
19
|
+
logger_stderr: Logger.new($stderr),
|
20
20
|
config: Config.new,
|
21
21
|
cmd_runner: CmdRunner.new,
|
22
22
|
nodes_handler: NodesHandler.new
|
@@ -45,8 +45,10 @@ module HybridPlatformsConductor
|
|
45
45
|
@stderr_io = stderr_io
|
46
46
|
end
|
47
47
|
|
48
|
+
# rubocop:disable Lint/UnusedMethodArgument
|
48
49
|
# Prepare connections to a given set of nodes.
|
49
50
|
# Useful to prefetch metadata or open bulk connections.
|
51
|
+
# This method is supposed to be overridden by sub-classes (hence the rubocop exception).
|
50
52
|
#
|
51
53
|
# Parameters::
|
52
54
|
# * *nodes* (Array<String>): Nodes to prepare the connection to
|
@@ -57,6 +59,7 @@ module HybridPlatformsConductor
|
|
57
59
|
def with_connection_to(nodes, no_exception: false)
|
58
60
|
yield nodes
|
59
61
|
end
|
62
|
+
# rubocop:enable Lint/UnusedMethodArgument
|
60
63
|
|
61
64
|
private
|
62
65
|
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module HybridPlatformsConductor
|
2
|
+
|
3
|
+
module CoreExtensions
|
4
|
+
|
5
|
+
module Symbol
|
6
|
+
|
7
|
+
# As it is better to test status code 0 with zero? and as we use status codes as symbols in case of errors, make the zero? call return appropriately.
|
8
|
+
module Zero
|
9
|
+
|
10
|
+
# Does the symbol equal zero?
|
11
|
+
#
|
12
|
+
# Result::
|
13
|
+
# * false: It does not.
|
14
|
+
def zero?
|
15
|
+
false
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -42,7 +42,7 @@ module HybridPlatformsConductor
|
|
42
42
|
# * *url* (String or nil): The URL for which we want the credentials, or nil if not associated to a URL [default: nil]
|
43
43
|
# * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
|
44
44
|
# * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
|
45
|
-
def initialize(id, url: nil, logger: Logger.new(
|
45
|
+
def initialize(id, url: nil, logger: Logger.new($stdout), logger_stderr: Logger.new($stderr))
|
46
46
|
init_loggers(logger, logger_stderr)
|
47
47
|
@id = id
|
48
48
|
@url = url
|
@@ -54,7 +54,7 @@ module HybridPlatformsConductor
|
|
54
54
|
# Provide a helper to clear password from memory for security.
|
55
55
|
# To be used when the client knows it won't use the password anymore.
|
56
56
|
def clear_password
|
57
|
-
@password
|
57
|
+
@password&.replace('gotyou!' * 100)
|
58
58
|
GC.start
|
59
59
|
end
|
60
60
|
|
@@ -82,46 +82,49 @@ module HybridPlatformsConductor
|
|
82
82
|
# Do it only once.
|
83
83
|
# Make sure the retrieved credentials are not linked to other objects in memory, so that we can remove any other trace of secrets.
|
84
84
|
def retrieve_credentials
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
85
|
+
return if @retrieved
|
86
|
+
|
87
|
+
# Check environment variables
|
88
|
+
@user = ENV["hpc_user_for_#{@id}"].dup
|
89
|
+
@password = ENV["hpc_password_for_#{@id}"].dup
|
90
|
+
if @user.nil? || @user.empty? || @password.nil? || @password.empty?
|
91
|
+
log_debug "[ Credentials for #{@id} ] - Credentials not found from environment variables."
|
92
|
+
if @url.nil?
|
93
|
+
log_debug "[ Credentials for #{@id} ] - No URL associated to this credentials, so .netrc can't be used."
|
94
|
+
else
|
95
|
+
# Check Netrc
|
96
|
+
netrc = ::Netrc.read
|
97
|
+
begin
|
98
|
+
netrc_user, netrc_password = netrc[URI.parse(@url).host.downcase]
|
99
|
+
if netrc_user.nil?
|
100
|
+
log_debug "[ Credentials for #{@id} ] - No credentials retrieved from .netrc."
|
101
|
+
# TODO: Add more credentials source if needed here
|
102
|
+
log_warn "[ Credentials for #{@id} ] - Unable to get credentials for #{@id} (URL: #{@url})."
|
103
|
+
else
|
104
|
+
@user = netrc_user.dup
|
105
|
+
@password = netrc_password.dup
|
106
|
+
log_debug "[ Credentials for #{@id} ] - Credentials retrieved from .netrc using #{@url}."
|
107
|
+
end
|
108
|
+
ensure
|
109
|
+
# Make sure the password does not stay in Netrc memory
|
110
|
+
# Wipe out any memory trace that might contain passwords in clear
|
111
|
+
netrc.instance_variable_get(:@data).each do |data_line|
|
112
|
+
data_line.each do |data_string|
|
113
|
+
data_string.replace('GotYou!!!' * 100)
|
114
114
|
end
|
115
|
-
netrc = nil
|
116
115
|
end
|
116
|
+
# We don this assignment on purpose so that GC can remove sensitive data later
|
117
|
+
# rubocop:disable Lint/UselessAssignment
|
118
|
+
netrc = nil
|
119
|
+
# rubocop:enable Lint/UselessAssignment
|
117
120
|
end
|
118
|
-
else
|
119
|
-
log_debug "[ Credentials for #{@id} ] - Credentials retrieved from environment variables."
|
120
121
|
end
|
121
|
-
|
122
|
+
else
|
123
|
+
log_debug "[ Credentials for #{@id} ] - Credentials retrieved from environment variables."
|
122
124
|
end
|
125
|
+
GC.start
|
123
126
|
end
|
124
127
|
|
125
128
|
end
|
126
129
|
|
127
|
-
end
|
130
|
+
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'monitor'
|
2
2
|
|
3
|
+
# Decorate methods changing the process' current directory with a mutex to ensure they have an exclusive access
|
3
4
|
module HybridPlatformsConductor
|
4
5
|
|
5
6
|
# Implement a global monitor to protect accesses to the current directory.
|
@@ -7,7 +8,9 @@ module HybridPlatformsConductor
|
|
7
8
|
module CurrentDirMonitor
|
8
9
|
|
9
10
|
class << self
|
11
|
+
|
10
12
|
attr_reader :monitor
|
13
|
+
|
11
14
|
end
|
12
15
|
|
13
16
|
@monitor = Monitor.new
|
@@ -24,7 +27,7 @@ module HybridPlatformsConductor
|
|
24
27
|
result = nil
|
25
28
|
CurrentDirMonitor.monitor.synchronize do
|
26
29
|
# puts "TID #{Thread.current.object_id} from #{caller[2]} - Current dir monitor taken from #{Dir.pwd}"
|
27
|
-
result =
|
30
|
+
result = send(original_method_name, *args, &block)
|
28
31
|
# puts "TID #{Thread.current.object_id} from #{caller[2]} - Current dir monitor released back to #{Dir.pwd}"
|
29
32
|
end
|
30
33
|
result
|
@@ -3,7 +3,6 @@ require 'futex'
|
|
3
3
|
require 'json'
|
4
4
|
require 'securerandom'
|
5
5
|
require 'time'
|
6
|
-
require 'thread'
|
7
6
|
require 'hybrid_platforms_conductor/actions_executor'
|
8
7
|
require 'hybrid_platforms_conductor/cmd_runner'
|
9
8
|
require 'hybrid_platforms_conductor/executable'
|
@@ -109,8 +108,8 @@ module HybridPlatformsConductor
|
|
109
108
|
# * *actions_executor* (ActionsExecutor): Actions Executor to be used. [default: ActionsExecutor.new]
|
110
109
|
# * *services_handler* (ServicesHandler): Services Handler to be used. [default: ServicesHandler.new]
|
111
110
|
def initialize(
|
112
|
-
logger: Logger.new(
|
113
|
-
logger_stderr: Logger.new(
|
111
|
+
logger: Logger.new($stdout),
|
112
|
+
logger_stderr: Logger.new($stderr),
|
114
113
|
config: Config.new,
|
115
114
|
cmd_runner: CmdRunner.new,
|
116
115
|
nodes_handler: NodesHandler.new,
|
@@ -171,25 +170,31 @@ module HybridPlatformsConductor
|
|
171
170
|
def options_parse(options_parser, parallel_switch: true, why_run_switch: false, timeout_options: true)
|
172
171
|
options_parser.separator ''
|
173
172
|
options_parser.separator 'Deployer options:'
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
173
|
+
if parallel_switch
|
174
|
+
options_parser.on('-p', '--parallel', 'Execute the commands in parallel (put the standard output in files <hybrid-platforms-dir>/run_logs/*.stdout)') do
|
175
|
+
@concurrent_execution = true
|
176
|
+
end
|
177
|
+
end
|
178
|
+
if timeout_options
|
179
|
+
options_parser.on('-t', '--timeout SECS', "Timeout in seconds to wait for each chef run. Only used in why-run mode. (defaults to #{@timeout.nil? ? 'no timeout' : @timeout})") do |nbr_secs|
|
180
|
+
@timeout = nbr_secs.to_i
|
181
|
+
end
|
182
|
+
end
|
183
|
+
if why_run_switch
|
184
|
+
options_parser.on('-W', '--why-run', 'Use the why-run mode to see what would be the result of the deploy instead of deploying it for real.') do
|
185
|
+
@use_why_run = true
|
186
|
+
end
|
187
|
+
end
|
183
188
|
options_parser.on('--retries-on-error NBR', "Number of retries in case of non-deterministic errors (defaults to #{@nbr_retries_on_error})") do |nbr_retries|
|
184
189
|
@nbr_retries_on_error = nbr_retries.to_i
|
185
190
|
end
|
186
191
|
# Display options secrets readers might have
|
187
192
|
@secrets_readers.each do |secret_reader_name, secret_reader|
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
+
next unless secret_reader.respond_to?(:options_parse)
|
194
|
+
|
195
|
+
options_parser.separator ''
|
196
|
+
options_parser.separator "Secrets reader #{secret_reader_name} options:"
|
197
|
+
secret_reader.options_parse(options_parser)
|
193
198
|
end
|
194
199
|
end
|
195
200
|
|
@@ -224,9 +229,9 @@ module HybridPlatformsConductor
|
|
224
229
|
def deploy_on(*nodes_selectors)
|
225
230
|
# Get the sorted list of services to be deployed, per node
|
226
231
|
# Hash<String, Array<String> >
|
227
|
-
services_to_deploy =
|
232
|
+
services_to_deploy = @nodes_handler.select_nodes(nodes_selectors.flatten).map do |node|
|
228
233
|
[node, @nodes_handler.get_services_of(node)]
|
229
|
-
end
|
234
|
+
end.to_h
|
230
235
|
|
231
236
|
# Get the secrets to be deployed
|
232
237
|
secrets = {}
|
@@ -251,7 +256,6 @@ module HybridPlatformsConductor
|
|
251
256
|
unless @use_why_run
|
252
257
|
reason_for_interdiction = @services_handler.deploy_allowed?(
|
253
258
|
services: services_to_deploy,
|
254
|
-
secrets: secrets,
|
255
259
|
local_environment: @local_environment
|
256
260
|
)
|
257
261
|
raise "Deployment not allowed: #{reason_for_interdiction}" unless reason_for_interdiction.nil?
|
@@ -289,51 +293,50 @@ module HybridPlatformsConductor
|
|
289
293
|
remaining_nodes_to_deploy = services_to_deploy.keys
|
290
294
|
while nbr_retries >= 0 && !remaining_nodes_to_deploy.empty?
|
291
295
|
last_deploy_results = deploy(services_to_deploy.slice(*remaining_nodes_to_deploy))
|
292
|
-
if nbr_retries
|
296
|
+
if nbr_retries.positive?
|
293
297
|
# Check if we need to retry deployment on some nodes
|
294
298
|
# Only parse the last deployment attempt logs
|
295
|
-
retriable_nodes =
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
299
|
+
retriable_nodes = remaining_nodes_to_deploy.
|
300
|
+
map do |node|
|
301
|
+
exit_status, stdout, stderr = last_deploy_results[node]
|
302
|
+
if exit_status.zero?
|
303
|
+
nil
|
304
|
+
else
|
305
|
+
retriable_errors = retriable_errors_from(node, exit_status, stdout, stderr)
|
306
|
+
if retriable_errors.empty?
|
300
307
|
nil
|
301
308
|
else
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
else
|
306
|
-
# Log the issue in the stderr of the deployment
|
307
|
-
stderr << "!!! #{retriable_errors.size} retriable errors detected in this deployment:\n#{retriable_errors.map { |error| "* #{error}" }.join("\n")}\n"
|
308
|
-
[node, retriable_errors]
|
309
|
-
end
|
309
|
+
# Log the issue in the stderr of the deployment
|
310
|
+
stderr << "!!! #{retriable_errors.size} retriable errors detected in this deployment:\n#{retriable_errors.map { |error| "* #{error}" }.join("\n")}\n"
|
311
|
+
[node, retriable_errors]
|
310
312
|
end
|
311
|
-
end
|
312
|
-
|
313
|
-
|
313
|
+
end
|
314
|
+
end.
|
315
|
+
compact.
|
316
|
+
to_h
|
314
317
|
unless retriable_nodes.empty?
|
315
|
-
log_warn <<~
|
318
|
+
log_warn <<~EO_LOG.strip
|
316
319
|
Retry deployment for #{retriable_nodes.size} nodes as they got non-deterministic errors (#{nbr_retries} retries remaining):
|
317
320
|
#{retriable_nodes.map { |node, retriable_errors| " * #{node}:\n#{retriable_errors.map { |error| " - #{error}" }.join("\n")}" }.join("\n")}
|
318
|
-
|
321
|
+
EO_LOG
|
319
322
|
end
|
320
323
|
remaining_nodes_to_deploy = retriable_nodes.keys
|
321
324
|
end
|
322
325
|
# Merge deployment results
|
323
|
-
results.merge!(last_deploy_results) do |
|
326
|
+
results.merge!(last_deploy_results) do |_node, (exit_status_1, stdout_1, stderr_1), (exit_status_2, stdout_2, stderr_2)|
|
324
327
|
[
|
325
328
|
exit_status_2,
|
326
|
-
<<~
|
329
|
+
<<~EO_STDOUT,
|
327
330
|
#{stdout_1}
|
328
331
|
Deployment exit status code: #{exit_status_1}
|
329
332
|
!!! Retry deployment due to non-deterministic error (#{nbr_retries} remaining attempts)...
|
330
333
|
#{stdout_2}
|
331
|
-
|
332
|
-
<<~
|
334
|
+
EO_STDOUT
|
335
|
+
<<~EO_STDERR
|
333
336
|
#{stderr_1}
|
334
337
|
!!! Retry deployment due to non-deterministic error (#{nbr_retries} remaining attempts)...
|
335
338
|
#{stderr_2}
|
336
|
-
|
339
|
+
EO_STDERR
|
337
340
|
]
|
338
341
|
end
|
339
342
|
nbr_retries -= 1
|
@@ -388,7 +391,7 @@ module HybridPlatformsConductor
|
|
388
391
|
sub_executable.config.sudo_procs.replace(sub_executable.config.sudo_procs.map do |sudo_proc_info|
|
389
392
|
{
|
390
393
|
nodes_selectors_stack: sudo_proc_info[:nodes_selectors_stack].map do |nodes_selector|
|
391
|
-
@nodes_handler.select_nodes(nodes_selector).
|
394
|
+
@nodes_handler.select_nodes(nodes_selector).reject { |selected_node| selected_node == node }
|
392
395
|
end,
|
393
396
|
sudo_proc: sudo_proc_info[:sudo_proc]
|
394
397
|
}
|
@@ -407,7 +410,7 @@ module HybridPlatformsConductor
|
|
407
410
|
rescue
|
408
411
|
# Make sure Docker logs are being output to better investigate errors if we were not already outputing them in debug mode
|
409
412
|
stdouts = sub_executable.stdouts_to_s
|
410
|
-
log_error "[ #{node}/#{environment} ] - Encountered unhandled exception #{
|
413
|
+
log_error "[ #{node}/#{environment} ] - Encountered unhandled exception #{$ERROR_INFO}\n#{$ERROR_INFO.backtrace.join("\n")}\n-----\n#{stdouts}" unless stdouts.nil?
|
411
414
|
raise
|
412
415
|
end
|
413
416
|
end
|
@@ -428,21 +431,21 @@ module HybridPlatformsConductor
|
|
428
431
|
nodes = nodes.flatten
|
429
432
|
@actions_executor.max_threads = 64
|
430
433
|
read_actions_results = @actions_executor.execute_actions(
|
431
|
-
|
434
|
+
nodes.map do |node|
|
432
435
|
master_log_plugin = @log_plugins[log_plugins_for(node).first]
|
433
436
|
master_log_plugin.respond_to?(:actions_to_read_logs) ? [node, master_log_plugin.actions_to_read_logs(node)] : nil
|
434
|
-
end.compact
|
437
|
+
end.compact.to_h,
|
435
438
|
log_to_stdout: false,
|
436
439
|
concurrent: true,
|
437
440
|
timeout: 10,
|
438
441
|
progress_name: 'Read deployment logs'
|
439
442
|
)
|
440
|
-
|
443
|
+
nodes.map do |node|
|
441
444
|
[
|
442
445
|
node,
|
443
446
|
@log_plugins[log_plugins_for(node).first].logs_for(node, *(read_actions_results[node] || [nil, nil, nil]))
|
444
447
|
]
|
445
|
-
end
|
448
|
+
end.to_h
|
446
449
|
end
|
447
450
|
|
448
451
|
# Parse stdout and stderr of a given deploy run and get the list of tasks with their status
|
@@ -458,7 +461,7 @@ module HybridPlatformsConductor
|
|
458
461
|
# * *:changed*: The task has been changed
|
459
462
|
# * *:identical*: The task has not been changed
|
460
463
|
# * *diffs* (String): Differences, if any
|
461
|
-
def parse_deploy_output(
|
464
|
+
def parse_deploy_output(_node, stdout, stderr)
|
462
465
|
@services_handler.parse_deploy_output(stdout, stderr).map { |deploy_info| deploy_info[:tasks] }.flatten
|
463
466
|
end
|
464
467
|
|
@@ -503,7 +506,7 @@ module HybridPlatformsConductor
|
|
503
506
|
# * *stderr* (String): Deployment stderr
|
504
507
|
# Result::
|
505
508
|
# * Array<String>: List of retriable errors that have been matched
|
506
|
-
def retriable_errors_from(node,
|
509
|
+
def retriable_errors_from(node, _exit_status, stdout, stderr)
|
507
510
|
# List of retriable errors for this node, as exact string match or regexps.
|
508
511
|
# Array<String or Regexp>
|
509
512
|
retriable_errors_on_stdout = []
|
@@ -534,59 +537,55 @@ module HybridPlatformsConductor
|
|
534
537
|
# Result::
|
535
538
|
# * Hash<String, [Integer or Symbol, String, String]>: Exit status code (or Symbol in case of error or dry run), standard output and error for each node.
|
536
539
|
def deploy(services)
|
537
|
-
outputs = {}
|
538
|
-
|
539
540
|
# Get the ssh user directly from the connector
|
540
541
|
ssh_user = @actions_executor.connector(:ssh).ssh_user
|
541
542
|
|
542
543
|
# Deploy for real
|
543
544
|
@nodes_handler.prefetch_metadata_of services.keys, :image
|
544
545
|
outputs = @actions_executor.execute_actions(
|
545
|
-
|
546
|
+
services.map do |node, node_services|
|
546
547
|
image_id = @nodes_handler.get_image_of(node)
|
547
548
|
sudo = (ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} ")
|
548
549
|
# Install corporate certificates if present
|
549
550
|
certificate_actions =
|
550
551
|
if @local_environment && ENV['hpc_certificates']
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
}
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
remote_bash: "#{sudo}update-ca-certificates"
|
565
|
-
}
|
566
|
-
]
|
567
|
-
when 'centos_7'
|
568
|
-
[
|
569
|
-
{
|
570
|
-
remote_bash: "#{sudo}yum install -y ca-certificates"
|
552
|
+
raise "Missing path referenced by the hpc_certificates environment variable: #{ENV['hpc_certificates']}" unless File.exist?(ENV['hpc_certificates'])
|
553
|
+
|
554
|
+
log_debug "Deploy certificates from #{ENV['hpc_certificates']}"
|
555
|
+
case image_id
|
556
|
+
when 'debian_9', 'debian_10'
|
557
|
+
[
|
558
|
+
{
|
559
|
+
remote_bash: "#{sudo}apt update && #{sudo}apt install -y ca-certificates"
|
560
|
+
},
|
561
|
+
{
|
562
|
+
scp: {
|
563
|
+
ENV['hpc_certificates'] => '/usr/local/share/ca-certificates',
|
564
|
+
:sudo => ssh_user != 'root'
|
571
565
|
},
|
572
|
-
{
|
573
|
-
|
574
|
-
|
575
|
-
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
566
|
+
remote_bash: "#{sudo}update-ca-certificates"
|
567
|
+
}
|
568
|
+
]
|
569
|
+
when 'centos_7'
|
570
|
+
[
|
571
|
+
{
|
572
|
+
remote_bash: "#{sudo}yum install -y ca-certificates"
|
573
|
+
},
|
574
|
+
{
|
575
|
+
scp: Dir.glob("#{ENV['hpc_certificates']}/*.crt").map do |cert_file|
|
576
|
+
[
|
577
|
+
cert_file,
|
578
|
+
'/etc/pki/ca-trust/source/anchors'
|
582
579
|
]
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
|
580
|
+
end.to_h.merge(sudo: ssh_user != 'root'),
|
581
|
+
remote_bash: [
|
582
|
+
"#{sudo}update-ca-trust enable",
|
583
|
+
"#{sudo}update-ca-trust extract"
|
584
|
+
]
|
585
|
+
}
|
586
|
+
]
|
588
587
|
else
|
589
|
-
raise "
|
588
|
+
raise "Unknown image ID for node #{node}: #{image_id}. Check metadata for this node."
|
590
589
|
end
|
591
590
|
else
|
592
591
|
[]
|
@@ -603,19 +602,19 @@ module HybridPlatformsConductor
|
|
603
602
|
certificate_actions +
|
604
603
|
@services_handler.actions_to_deploy_on(node, node_services, @use_why_run)
|
605
604
|
]
|
606
|
-
end
|
605
|
+
end.to_h,
|
607
606
|
timeout: @timeout,
|
608
607
|
concurrent: @concurrent_execution,
|
609
608
|
log_to_stdout: !@concurrent_execution
|
610
609
|
)
|
611
610
|
# Free eventual locks
|
612
611
|
@actions_executor.execute_actions(
|
613
|
-
|
612
|
+
services.keys.map do |node|
|
614
613
|
[
|
615
614
|
node,
|
616
615
|
{ remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
|
617
616
|
]
|
618
|
-
end
|
617
|
+
end.to_h,
|
619
618
|
timeout: 10,
|
620
619
|
concurrent: true,
|
621
620
|
log_to_dir: nil
|
@@ -637,7 +636,7 @@ module HybridPlatformsConductor
|
|
637
636
|
section "Saving deployment logs for #{logs.size} nodes" do
|
638
637
|
ssh_user = @actions_executor.connector(:ssh).ssh_user
|
639
638
|
@actions_executor.execute_actions(
|
640
|
-
|
639
|
+
logs.map do |node, (exit_status, stdout, stderr)|
|
641
640
|
[
|
642
641
|
node,
|
643
642
|
log_plugins_for(node).
|
@@ -656,7 +655,7 @@ module HybridPlatformsConductor
|
|
656
655
|
end.
|
657
656
|
flatten(1)
|
658
657
|
]
|
659
|
-
end
|
658
|
+
end.to_h,
|
660
659
|
timeout: 10,
|
661
660
|
concurrent: true,
|
662
661
|
log_to_dir: nil,
|