hybrid_platforms_conductor 32.17.0 → 33.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -0
- data/README.md +6 -3
- data/bin/check-node +0 -1
- data/bin/deploy +0 -1
- data/bin/get_impacted_nodes +0 -1
- data/bin/last_deploys +12 -8
- data/bin/nodes_to_deploy +6 -6
- data/bin/setup +6 -6
- data/bin/topograph +1 -1
- data/docs/config_dsl.md +45 -1
- data/docs/executables.md +6 -7
- data/docs/executables/check-node.md +3 -3
- data/docs/executables/deploy.md +3 -3
- data/docs/executables/dump_nodes_json.md +3 -3
- data/docs/executables/test.md +3 -3
- data/docs/executables/topograph.md +3 -3
- data/docs/gen/mermaid/README.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
- data/docs/plugins.md +46 -0
- data/docs/plugins/connector/ssh.md +1 -1
- data/docs/plugins/log/remote_fs.md +26 -0
- data/docs/plugins/secrets_reader/cli.md +31 -0
- data/docs/plugins/secrets_reader/thycotic.md +46 -0
- data/docs/plugins/test/check_deploy_and_idempotence.md +1 -1
- data/docs/plugins/test/connection.md +1 -0
- data/docs/plugins/test/deploy_removes_root_access.md +1 -1
- data/docs/plugins/test/file_system.md +1 -0
- data/docs/plugins/test/hostname.md +1 -0
- data/docs/plugins/test/ip.md +1 -0
- data/docs/plugins/test/local_users.md +1 -0
- data/docs/plugins/test/mounts.md +1 -0
- data/docs/plugins/test/orphan_files.md +1 -0
- data/docs/plugins/test/ports.md +1 -0
- data/docs/plugins/test/spectre.md +1 -0
- data/docs/plugins/test/vulnerabilities.md +1 -0
- data/lib/hybrid_platforms_conductor/action.rb +4 -4
- data/lib/hybrid_platforms_conductor/actions_executor.rb +45 -43
- data/lib/hybrid_platforms_conductor/bitbucket.rb +5 -4
- data/lib/hybrid_platforms_conductor/cmd_runner.rb +13 -12
- data/lib/hybrid_platforms_conductor/cmdb.rb +2 -2
- data/lib/hybrid_platforms_conductor/common_config_dsl/bitbucket.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/confluence.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +5 -4
- data/lib/hybrid_platforms_conductor/common_config_dsl/github.rb +4 -3
- data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb +2 -2
- data/lib/hybrid_platforms_conductor/config.rb +8 -4
- data/lib/hybrid_platforms_conductor/confluence.rb +1 -1
- data/lib/hybrid_platforms_conductor/connector.rb +5 -2
- data/lib/hybrid_platforms_conductor/core_extensions/cleanroom/fix_kwargs.rb +116 -0
- data/lib/hybrid_platforms_conductor/core_extensions/symbol/zero.rb +24 -0
- data/lib/hybrid_platforms_conductor/credentials.rb +39 -36
- data/lib/hybrid_platforms_conductor/current_dir_monitor.rb +4 -1
- data/lib/hybrid_platforms_conductor/deployer.rb +275 -224
- data/lib/hybrid_platforms_conductor/executable.rb +20 -15
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/config.rb +10 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +4 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb +2 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +45 -49
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/my_log_plugin.rb.sample +100 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb +180 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +68 -66
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb +13 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb +39 -38
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb +5 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb +43 -45
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/podman.rb +18 -20
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +118 -117
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +39 -43
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container +9 -13
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +28 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/stdout.rb +26 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/cli.rb +77 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/my_secrets_reader_plugin.rb.sample +46 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb +90 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb +4 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb +3 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +7 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb +21 -19
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +21 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +19 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb +2 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +3 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +4 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb +21 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb +10 -12
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +9 -9
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +13 -10
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +7 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +27 -25
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/stdout.rb +8 -6
- data/lib/hybrid_platforms_conductor/io_router.rb +14 -13
- data/lib/hybrid_platforms_conductor/json_dumper.rb +2 -2
- data/lib/hybrid_platforms_conductor/log.rb +31 -0
- data/lib/hybrid_platforms_conductor/logger_helpers.rb +19 -16
- data/lib/hybrid_platforms_conductor/nodes_handler.rb +89 -71
- data/lib/hybrid_platforms_conductor/parallel_threads.rb +7 -11
- data/lib/hybrid_platforms_conductor/platform_handler.rb +7 -7
- data/lib/hybrid_platforms_conductor/platforms_handler.rb +5 -3
- data/lib/hybrid_platforms_conductor/plugin.rb +2 -2
- data/lib/hybrid_platforms_conductor/plugins.rb +14 -8
- data/lib/hybrid_platforms_conductor/provisioner.rb +4 -4
- data/lib/hybrid_platforms_conductor/report.rb +2 -2
- data/lib/hybrid_platforms_conductor/reports_handler.rb +3 -2
- data/lib/hybrid_platforms_conductor/secrets_reader.rb +31 -0
- data/lib/hybrid_platforms_conductor/services_handler.rb +32 -29
- data/lib/hybrid_platforms_conductor/test_only_remote_node.rb +18 -0
- data/lib/hybrid_platforms_conductor/test_report.rb +15 -18
- data/lib/hybrid_platforms_conductor/tests_runner.rb +116 -118
- data/lib/hybrid_platforms_conductor/thycotic.rb +28 -19
- data/lib/hybrid_platforms_conductor/topographer.rb +200 -190
- data/lib/hybrid_platforms_conductor/topographer/plugins/graphviz.rb +8 -8
- data/lib/hybrid_platforms_conductor/topographer/plugins/json.rb +4 -4
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test.rb +33 -12
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +18 -11
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/interactive_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb +32 -21
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +75 -49
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/scp_spec.rb +27 -15
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions_spec.rb +90 -59
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connection_spec.rb +46 -44
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/connectable_nodes_spec.rb +12 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb +4 -7
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +21 -22
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +23 -24
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connectable_nodes_spec.rb +10 -6
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +106 -75
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +145 -126
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +29 -25
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/logging_spec.rb +167 -142
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/parallel_spec.rb +272 -244
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +16 -16
- data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +36 -36
- data/spec/hybrid_platforms_conductor_test/api/config_spec.rb +24 -22
- data/spec/hybrid_platforms_conductor_test/api/deployer/check_spec.rb +4 -2
- data/spec/hybrid_platforms_conductor_test/api/deployer/config_dsl_spec.rb +43 -5
- data/spec/hybrid_platforms_conductor_test/api/deployer/deploy_spec.rb +199 -216
- data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +223 -0
- data/spec/hybrid_platforms_conductor_test/api/deployer/parse_deploy_output_spec.rb +55 -59
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +36 -62
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/podman_spec.rb +17 -17
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/config_dsl_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/create_spec.rb +44 -51
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/destroy_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/ip_spec.rb +12 -16
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/destroy_vm_spec.rb +31 -19
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +324 -266
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/ips_assignment_spec.rb +89 -61
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/other_lxc_containers_resources_spec.rb +117 -93
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/pve_node_resources_spec.rb +71 -54
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +10 -8
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb +80 -60
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/start_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/stop_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/cli_spec.rb +64 -0
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/thycotic_spec.rb +268 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/config_spec.rb +8 -10
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +33 -24
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_keys_spec.rb +64 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/platform_handlers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +50 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +91 -81
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +14 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/git_diff_impacts_spec.rb +51 -75
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/nodes_selectors_spec.rb +35 -26
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/platform_handlers_plugins_api_spec.rb +24 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/several_platforms_spec.rb +29 -19
- data/spec/hybrid_platforms_conductor_test/api/platform_handler_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb +57 -99
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +32 -35
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +10 -10
- data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +38 -27
- data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +46 -52
- data/spec/hybrid_platforms_conductor_test/api/reports_handler_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/services_handler/actions_to_deploy_spec.rb +90 -58
- data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +38 -34
- data/spec/hybrid_platforms_conductor_test/api/services_handler/log_info_spec.rb +11 -9
- data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +193 -171
- data/spec/hybrid_platforms_conductor_test/api/services_handler/parse_deploy_output_spec.rb +66 -54
- data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +147 -133
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +69 -49
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/global_spec.rb +5 -4
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +30 -27
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/platform_spec.rb +12 -9
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +48 -47
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/bitbucket_conf_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/github_ci_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_reports_plugins/confluence_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +9 -9
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb2.rb → test_cmdb_2.rb} +6 -6
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb_others2.rb → test_cmdb_others_2.rb} +2 -2
- data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/{check-node_spec.rb → check_node_spec.rb} +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/get_impacted_nodes_spec.rb +76 -77
- data/spec/hybrid_platforms_conductor_test/executables/last_deploys_spec.rb +159 -113
- data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +299 -160
- data/spec/hybrid_platforms_conductor_test/executables/options/actions_executor_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +12 -196
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_selectors_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/reports_handler_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +22 -22
- data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +22 -16
- data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +32 -32
- data/spec/hybrid_platforms_conductor_test/executables/ssh_config_spec.rb +7 -9
- data/spec/hybrid_platforms_conductor_test/executables/test_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +4 -3
- data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/config_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +12 -13
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +245 -56
- data/spec/hybrid_platforms_conductor_test/helpers/executables_helpers.rb +11 -11
- data/spec/hybrid_platforms_conductor_test/helpers/nodes_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +39 -28
- data/spec/hybrid_platforms_conductor_test/helpers/plugins_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +86 -111
- data/spec/hybrid_platforms_conductor_test/helpers/reports_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/helpers/services_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/tests_runner_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id1.rb → test_plugin_id_1.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id2.rb → test_plugin_id_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/test_plugin_type/{test_plugin_id3.rb → test_plugin_id_3.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/{test_plugin_type2/test_plugin_id4.rb → test_plugin_type_2/test_plugin_id_4.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/test.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/{test2.rb → test_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/rubocop_spec.rb +31 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +952 -0
- data/spec/hybrid_platforms_conductor_test/test_connector.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/test_log_no_read_plugin.rb +84 -0
- data/spec/hybrid_platforms_conductor_test/test_log_plugin.rb +105 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/global.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node_check.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/platform.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/several_checks.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/test_secrets_reader_plugin.rb +45 -0
- data/spec/hybrid_platforms_conductor_test/tests_report_plugin.rb +5 -6
- data/spec/spec_helper.rb +17 -18
- data/tools/check_md +16 -20
- data/tools/generate_mermaid +1 -1
- metadata +195 -144
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +0 -916
data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb
CHANGED
|
@@ -29,6 +29,10 @@ module HybridPlatformsConductor
|
|
|
29
29
|
instance_eval(File.read(source))
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
+
# Intercept all missing methods
|
|
33
|
+
#
|
|
34
|
+
# Parameters::
|
|
35
|
+
# * *method_name* (Symbol): The missing method being called
|
|
32
36
|
def method_missing(method_name, *args, &block)
|
|
33
37
|
sub_calls = []
|
|
34
38
|
@calls << {
|
|
@@ -40,6 +44,15 @@ module HybridPlatformsConductor
|
|
|
40
44
|
DslParser.new(sub_calls)
|
|
41
45
|
end
|
|
42
46
|
|
|
47
|
+
# Make sure we register the methods we handle in method_missing
|
|
48
|
+
#
|
|
49
|
+
# Parameters::
|
|
50
|
+
# * *name* (Symbol): The missing method name
|
|
51
|
+
# * *include_private* (Boolean): Should we include private methods in the search?
|
|
52
|
+
def respond_to_missing?(_name, _include_private)
|
|
53
|
+
true
|
|
54
|
+
end
|
|
55
|
+
|
|
43
56
|
end
|
|
44
57
|
|
|
45
58
|
end
|
|
@@ -40,11 +40,14 @@ module HybridPlatformsConductor
|
|
|
40
40
|
add_recipe_in_tree(cookbook_dir, cookbook, recipe)
|
|
41
41
|
end
|
|
42
42
|
end
|
|
43
|
+
# Make sure we don't combine those 2 loops
|
|
44
|
+
# rubocop:disable Style/CombinableLoops
|
|
43
45
|
@platform.deployable_services.each do |service|
|
|
44
|
-
@platform.policy_run_list(service).each do |(
|
|
46
|
+
@platform.policy_run_list(service).each do |(_cookbook_dir, cookbook, recipe)|
|
|
45
47
|
mark_recipe_used_by_policy(cookbook, recipe, service)
|
|
46
48
|
end
|
|
47
49
|
end
|
|
50
|
+
# rubocop:enable Style/CombinableLoops
|
|
48
51
|
@recipes_tree
|
|
49
52
|
end
|
|
50
53
|
|
|
@@ -58,25 +61,25 @@ module HybridPlatformsConductor
|
|
|
58
61
|
# * *recipe* (Symbol): The recipe name
|
|
59
62
|
def add_recipe_in_tree(cookbook_dir, cookbook, recipe)
|
|
60
63
|
@recipes_tree[cookbook] = {} unless @recipes_tree.key?(cookbook)
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
@recipes_tree[cookbook][recipe] = recipe_info.merge(
|
|
75
|
-
used_by_policies: []
|
|
76
|
-
)
|
|
77
|
-
recipe_info[:included_recipes].each do |(sub_cookbook_dir, sub_cookbook, sub_recipe)|
|
|
78
|
-
add_recipe_in_tree(sub_cookbook_dir, sub_cookbook, sub_recipe)
|
|
64
|
+
return if @recipes_tree[cookbook].key?(recipe)
|
|
65
|
+
|
|
66
|
+
recipe_info =
|
|
67
|
+
if cookbook_dir.nil?
|
|
68
|
+
# This recipe comes from an external cookbook, we won't get into it.
|
|
69
|
+
{
|
|
70
|
+
included_recipes: [],
|
|
71
|
+
used_templates: [],
|
|
72
|
+
used_files: [],
|
|
73
|
+
used_cookbooks: []
|
|
74
|
+
}
|
|
75
|
+
else
|
|
76
|
+
recipe_usage(cookbook_dir, cookbook, recipe)
|
|
79
77
|
end
|
|
78
|
+
@recipes_tree[cookbook][recipe] = recipe_info.merge(
|
|
79
|
+
used_by_policies: []
|
|
80
|
+
)
|
|
81
|
+
recipe_info[:included_recipes].each do |(sub_cookbook_dir, sub_cookbook, sub_recipe)|
|
|
82
|
+
add_recipe_in_tree(sub_cookbook_dir, sub_cookbook, sub_recipe)
|
|
80
83
|
end
|
|
81
84
|
end
|
|
82
85
|
|
|
@@ -115,18 +118,18 @@ module HybridPlatformsConductor
|
|
|
115
118
|
recipe_content.
|
|
116
119
|
scan(/source\s+(["'])(.+?)\1/).
|
|
117
120
|
each do |(_sub_grp, source)|
|
|
118
|
-
sources << source unless source =~
|
|
121
|
+
sources << source unless source =~ %r{^https?://}
|
|
119
122
|
end
|
|
120
123
|
erb_sources = sources.select { |source| File.extname(source).downcase == '.erb' }
|
|
121
124
|
non_erb_sources = sources - erb_sources
|
|
122
125
|
erb_sources.concat(recipe_content.scan(/template:?\s+(["'])(.+?)\1/).map { |(_sub_grp, source)| source })
|
|
123
126
|
# Check for known resources and library methods
|
|
124
127
|
used_cookbooks = []
|
|
125
|
-
known_resources.each do |
|
|
126
|
-
used_cookbooks <<
|
|
128
|
+
known_resources.each do |itr_cookbook, methods|
|
|
129
|
+
used_cookbooks << itr_cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
|
|
127
130
|
end
|
|
128
|
-
known_library_methods.each do |
|
|
129
|
-
used_cookbooks <<
|
|
131
|
+
known_library_methods.each do |itr_cookbook, methods|
|
|
132
|
+
used_cookbooks << itr_cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
|
|
130
133
|
end
|
|
131
134
|
{
|
|
132
135
|
included_recipes: used_recipes,
|
|
@@ -171,9 +174,9 @@ module HybridPlatformsConductor
|
|
|
171
174
|
@known_library_methods = {}
|
|
172
175
|
for_each_cookbook do |cookbook, cookbook_dir|
|
|
173
176
|
if File.exist?("#{cookbook_dir}/libraries")
|
|
174
|
-
found_methods =
|
|
175
|
-
|
|
176
|
-
|
|
177
|
+
found_methods = Dir.glob("#{cookbook_dir}/libraries/*.rb").
|
|
178
|
+
map { |lib_file| File.read(lib_file).scan(/\bdef\s+(\w+)\b/).map { |(method_name)| method_name } }.
|
|
179
|
+
flatten - INVALID_LIBRARY_METHODS
|
|
177
180
|
@known_library_methods[cookbook] = found_methods unless found_methods.empty?
|
|
178
181
|
end
|
|
179
182
|
end
|
|
@@ -184,15 +187,13 @@ module HybridPlatformsConductor
|
|
|
184
187
|
# Iterate over all cookbooks
|
|
185
188
|
#
|
|
186
189
|
# Parameters::
|
|
187
|
-
# * Proc: Code called for each cookbook:
|
|
190
|
+
# * *block* (Proc): Code called for each cookbook:
|
|
188
191
|
# * Parameters::
|
|
189
192
|
# * *cookbook* (Symbol): Cookbook name
|
|
190
193
|
# * *cookbook_dir* (String): Cookbook directory
|
|
191
|
-
def for_each_cookbook
|
|
194
|
+
def for_each_cookbook(&block)
|
|
192
195
|
@platform.known_cookbook_paths.each do |cookbook_path|
|
|
193
|
-
cookbooks_in(cookbook_path).each
|
|
194
|
-
yield cookbook, cookbook_dir
|
|
195
|
-
end
|
|
196
|
+
cookbooks_in(cookbook_path).each(&block)
|
|
196
197
|
end
|
|
197
198
|
end
|
|
198
199
|
|
|
@@ -203,7 +204,7 @@ module HybridPlatformsConductor
|
|
|
203
204
|
# Result::
|
|
204
205
|
# * Hash<Symbol, String>: List of cookbook directories, per cookbook name
|
|
205
206
|
def cookbooks_in(cookbook_type)
|
|
206
|
-
|
|
207
|
+
Dir.glob("#{@platform.repository_path}/#{cookbook_type}/*").map { |dir| [File.basename(dir).to_sym, dir] }.sort.to_h
|
|
207
208
|
end
|
|
208
209
|
|
|
209
210
|
# Mark a recipe (and its included recipes) as used by a policy
|
|
@@ -213,11 +214,11 @@ module HybridPlatformsConductor
|
|
|
213
214
|
# * *recipe* (Symbol): The recipe
|
|
214
215
|
# * *used_by_policy* (String): The policy using this recipe
|
|
215
216
|
def mark_recipe_used_by_policy(cookbook, recipe, used_by_policy)
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
217
|
+
return if @recipes_tree[cookbook][recipe][:used_by_policies].include?(used_by_policy)
|
|
218
|
+
|
|
219
|
+
@recipes_tree[cookbook][recipe][:used_by_policies] << used_by_policy
|
|
220
|
+
@recipes_tree[cookbook][recipe][:included_recipes].each do |(_sub_cookbook_dir, sub_cookbook, sub_recipe)|
|
|
221
|
+
mark_recipe_used_by_policy(sub_cookbook, sub_recipe, used_by_policy)
|
|
221
222
|
end
|
|
222
223
|
end
|
|
223
224
|
|
|
@@ -16,7 +16,7 @@ module HybridPlatformsConductor
|
|
|
16
16
|
def init
|
|
17
17
|
# This method is called when initializing a new instance of this platform handler, for a given repository.
|
|
18
18
|
inv_file = "#{@repository_path}/inventory.yaml"
|
|
19
|
-
@inventory = File.exist?(inv_file) ? YAML.
|
|
19
|
+
@inventory = File.exist?(inv_file) ? YAML.safe_load(File.read(inv_file)) : {}
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
# Get the list of known nodes.
|
|
@@ -89,12 +89,13 @@ module HybridPlatformsConductor
|
|
|
89
89
|
# * *cmd_runner* (CmdRunner): CmdRunner to be used [default: CmdRunner.new]
|
|
90
90
|
def initialize(
|
|
91
91
|
platform_handler,
|
|
92
|
-
logger: Logger.new(
|
|
93
|
-
logger_stderr: Logger.new(
|
|
92
|
+
logger: Logger.new($stdout),
|
|
93
|
+
logger_stderr: Logger.new($stderr),
|
|
94
94
|
config: Config.new,
|
|
95
95
|
nodes_handler: NodesHandler.new,
|
|
96
96
|
cmd_runner: CmdRunner.new
|
|
97
97
|
)
|
|
98
|
+
super
|
|
98
99
|
init_loggers(logger, logger_stderr)
|
|
99
100
|
@platform_handler = platform_handler
|
|
100
101
|
@config = config
|
|
@@ -127,7 +128,7 @@ module HybridPlatformsConductor
|
|
|
127
128
|
# * *:changed*: The task has been changed
|
|
128
129
|
# * *:identical*: The task has not been changed
|
|
129
130
|
# * *diffs* (String): Differences, if any
|
|
130
|
-
def parse_deploy_output(
|
|
131
|
+
def parse_deploy_output(_stdout, _stderr)
|
|
131
132
|
[]
|
|
132
133
|
end
|
|
133
134
|
|
|
@@ -21,7 +21,7 @@ module HybridPlatformsConductor
|
|
|
21
21
|
::Docker.validate_version!
|
|
22
22
|
docker_ok = true
|
|
23
23
|
rescue
|
|
24
|
-
log_error "[ #{@node}/#{@environment} ] - Docker is not installed correctly. Please install it. Error: #{
|
|
24
|
+
log_error "[ #{@node}/#{@environment} ] - Docker is not installed correctly. Please install it. Error: #{$ERROR_INFO}"
|
|
25
25
|
end
|
|
26
26
|
docker_ok
|
|
27
27
|
end
|
|
@@ -33,47 +33,45 @@ module HybridPlatformsConductor
|
|
|
33
33
|
# Get the image name for this node
|
|
34
34
|
image = @nodes_handler.get_image_of(@node).to_sym
|
|
35
35
|
# Find if we have such an image registered
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
end
|
|
51
|
-
container_name = "hpc_docker_container_#{@node}_#{@environment}"
|
|
52
|
-
container_futex_file = "#{Dir.tmpdir}/hpc_docker_container_futexes/#{image_tag}"
|
|
53
|
-
FileUtils.mkdir_p File.dirname(container_futex_file)
|
|
54
|
-
Futex.new(container_futex_file).open do
|
|
55
|
-
old_docker_container = ::Docker::Container.all(all: true).find { |container| container.info['Names'].include? "/#{container_name}" }
|
|
56
|
-
@container =
|
|
57
|
-
if old_docker_container
|
|
58
|
-
old_docker_container
|
|
59
|
-
else
|
|
60
|
-
log_debug "[ #{@node}/#{@environment} ] - Creating Docker container #{container_name}..."
|
|
61
|
-
# We add the SYS_PTRACE capability as some images need to restart services (for example postfix) and those services need the rights to ls in /proc/{PID}/exe to check if a status is running. Without SYS_PTRACE such ls returns permission denied and the service can't be stopped (as init.d always returns it as stopped even when running).
|
|
62
|
-
# We add the privileges as some containers need to install and configure the udev package, which needs RW access to /sys.
|
|
63
|
-
# We add the bind to cgroup volume to be able to test systemd specifics (enabling/disabling services for example).
|
|
64
|
-
::Docker::Container.create(
|
|
65
|
-
name: container_name,
|
|
66
|
-
image: image_tag,
|
|
67
|
-
CapAdd: 'SYS_PTRACE',
|
|
68
|
-
Privileged: true,
|
|
69
|
-
Binds: ['/sys/fs/cgroup:/sys/fs/cgroup:ro'],
|
|
70
|
-
# Some playbooks need the hostname to be set to a correct FQDN
|
|
71
|
-
Hostname: "#{@node}.testdomain"
|
|
72
|
-
)
|
|
73
|
-
end
|
|
36
|
+
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}" unless @config.known_os_images.include?(image)
|
|
37
|
+
|
|
38
|
+
# Build the image if it does not exist
|
|
39
|
+
image_tag = "hpc_image_#{image}"
|
|
40
|
+
docker_image = nil
|
|
41
|
+
image_futex_file = "#{Dir.tmpdir}/hpc_docker_image_futexes/#{image_tag}"
|
|
42
|
+
FileUtils.mkdir_p File.dirname(image_futex_file)
|
|
43
|
+
Futex.new(image_futex_file).open do
|
|
44
|
+
docker_image = ::Docker::Image.all.find { |search_image| !search_image.info['RepoTags'].nil? && search_image.info['RepoTags'].include?("#{image_tag}:latest") }
|
|
45
|
+
unless docker_image
|
|
46
|
+
log_debug "[ #{@node}/#{@environment} ] - Creating Docker image #{image_tag}..."
|
|
47
|
+
Excon.defaults[:read_timeout] = 600
|
|
48
|
+
docker_image = ::Docker::Image.build_from_dir(@config.os_image_dir(image))
|
|
49
|
+
docker_image.tag repo: image_tag
|
|
74
50
|
end
|
|
75
|
-
|
|
76
|
-
|
|
51
|
+
end
|
|
52
|
+
container_name = "hpc_docker_container_#{@node}_#{@environment}"
|
|
53
|
+
container_futex_file = "#{Dir.tmpdir}/hpc_docker_container_futexes/#{image_tag}"
|
|
54
|
+
FileUtils.mkdir_p File.dirname(container_futex_file)
|
|
55
|
+
Futex.new(container_futex_file).open do
|
|
56
|
+
old_docker_container = ::Docker::Container.all(all: true).find { |container| container.info['Names'].include? "/#{container_name}" }
|
|
57
|
+
@container =
|
|
58
|
+
if old_docker_container
|
|
59
|
+
old_docker_container
|
|
60
|
+
else
|
|
61
|
+
log_debug "[ #{@node}/#{@environment} ] - Creating Docker container #{container_name}..."
|
|
62
|
+
# We add the SYS_PTRACE capability as some images need to restart services (for example postfix) and those services need the rights to ls in /proc/{PID}/exe to check if a status is running. Without SYS_PTRACE such ls returns permission denied and the service can't be stopped (as init.d always returns it as stopped even when running).
|
|
63
|
+
# We add the privileges as some containers need to install and configure the udev package, which needs RW access to /sys.
|
|
64
|
+
# We add the bind to cgroup volume to be able to test systemd specifics (enabling/disabling services for example).
|
|
65
|
+
::Docker::Container.create(
|
|
66
|
+
name: container_name,
|
|
67
|
+
image: image_tag,
|
|
68
|
+
CapAdd: 'SYS_PTRACE',
|
|
69
|
+
Privileged: true,
|
|
70
|
+
Binds: ['/sys/fs/cgroup:/sys/fs/cgroup:ro'],
|
|
71
|
+
# Some playbooks need the hostname to be set to a correct FQDN
|
|
72
|
+
Hostname: "#{@node}.testdomain"
|
|
73
|
+
)
|
|
74
|
+
end
|
|
77
75
|
end
|
|
78
76
|
end
|
|
79
77
|
|
|
@@ -81,7 +79,7 @@ module HybridPlatformsConductor
|
|
|
81
79
|
# Prerequisite: create has been called before
|
|
82
80
|
# [API] - This method is mandatory
|
|
83
81
|
def start
|
|
84
|
-
log_debug "[ #{@node}/#{@environment} ] - Start Docker Container #{@container.refresh!.info['Name'][1
|
|
82
|
+
log_debug "[ #{@node}/#{@environment} ] - Start Docker Container #{@container.refresh!.info['Name'][1..]} ..."
|
|
85
83
|
@container.start
|
|
86
84
|
end
|
|
87
85
|
|
|
@@ -89,7 +87,7 @@ module HybridPlatformsConductor
|
|
|
89
87
|
# Prerequisite: create has been called before
|
|
90
88
|
# [API] - This method is mandatory
|
|
91
89
|
def stop
|
|
92
|
-
log_debug "[ #{@node}/#{@environment} ] - Stop Docker Container #{@container.refresh!.info['Name'][1
|
|
90
|
+
log_debug "[ #{@node}/#{@environment} ] - Stop Docker Container #{@container.refresh!.info['Name'][1..]} ..."
|
|
93
91
|
@container.stop
|
|
94
92
|
end
|
|
95
93
|
|
|
@@ -97,7 +95,7 @@ module HybridPlatformsConductor
|
|
|
97
95
|
# Prerequisite: create has been called before
|
|
98
96
|
# [API] - This method is mandatory
|
|
99
97
|
def destroy
|
|
100
|
-
log_debug "[ #{@node}/#{@environment} ] - Destroy Docker Container #{@container.refresh!.info['Name'][1
|
|
98
|
+
log_debug "[ #{@node}/#{@environment} ] - Destroy Docker Container #{@container.refresh!.info['Name'][1..]} ..."
|
|
101
99
|
@container.remove
|
|
102
100
|
@container = nil
|
|
103
101
|
end
|
|
@@ -119,7 +117,7 @@ module HybridPlatformsConductor
|
|
|
119
117
|
begin
|
|
120
118
|
@container.refresh!.info['State']['Status'].to_sym
|
|
121
119
|
rescue
|
|
122
|
-
log_error "[ #{@node}/#{@environment} ] - Error while reading state of Docker container: #{
|
|
120
|
+
log_error "[ #{@node}/#{@environment} ] - Error while reading state of Docker container: #{$ERROR_INFO}"
|
|
123
121
|
:error
|
|
124
122
|
end
|
|
125
123
|
end
|
|
@@ -17,25 +17,23 @@ module HybridPlatformsConductor
|
|
|
17
17
|
# Get the image name for this node
|
|
18
18
|
image = @nodes_handler.get_image_of(@node).to_sym
|
|
19
19
|
# Find if we have such an image registered
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
else
|
|
38
|
-
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}"
|
|
20
|
+
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}" unless @config.known_os_images.include?(image)
|
|
21
|
+
|
|
22
|
+
# Build the image if it does not exist
|
|
23
|
+
image_tag = "hpc_image_#{image}"
|
|
24
|
+
image_futex_file = "#{Dir.tmpdir}/hpc_podman_image_futexes/#{image_tag}"
|
|
25
|
+
FileUtils.mkdir_p File.dirname(image_futex_file)
|
|
26
|
+
Futex.new(image_futex_file).open do
|
|
27
|
+
@cmd_runner.run_cmd "cd #{@config.os_image_dir(image)} && #{podman_cmd} build --tag #{image_tag} --security-opt seccomp=/usr/share/containers/seccomp.json --cgroup-manager=cgroupfs ."
|
|
28
|
+
end
|
|
29
|
+
container_name = "hpc_container_#{@node}_#{@environment}"
|
|
30
|
+
container_futex_file = "#{Dir.tmpdir}/hpc_podman_container_futexes/#{image_tag}"
|
|
31
|
+
FileUtils.mkdir_p File.dirname(container_futex_file)
|
|
32
|
+
Futex.new(container_futex_file).open do
|
|
33
|
+
_exit_status, stdout, _stderr = @cmd_runner.run_cmd "#{podman_cmd} container list --all | grep #{container_name}", expected_code: [0, 1]
|
|
34
|
+
existing_container = !stdout.strip.empty?
|
|
35
|
+
@cmd_runner.run_cmd "#{podman_cmd} container create --name #{container_name} #{image_tag}" unless existing_container
|
|
36
|
+
@container = container_name
|
|
39
37
|
end
|
|
40
38
|
end
|
|
41
39
|
|
|
@@ -83,7 +81,7 @@ module HybridPlatformsConductor
|
|
|
83
81
|
status = :created if status == :configured
|
|
84
82
|
status
|
|
85
83
|
rescue
|
|
86
|
-
log_warn "Error while reading state of Podman container #{@container}: #{
|
|
84
|
+
log_warn "Error while reading state of Podman container #{@container}: #{$ERROR_INFO}"
|
|
87
85
|
:error
|
|
88
86
|
end
|
|
89
87
|
end
|
|
@@ -8,6 +8,7 @@ module HybridPlatformsConductor
|
|
|
8
8
|
|
|
9
9
|
module HpcPlugins
|
|
10
10
|
|
|
11
|
+
# Patch proxmox lib
|
|
11
12
|
module Provisioner
|
|
12
13
|
|
|
13
14
|
# Monkey patch some Proxmox methods
|
|
@@ -68,7 +69,9 @@ module HybridPlatformsConductor
|
|
|
68
69
|
extend_config_dsl_with PlatformsDSLProxmox, :init_proxmox
|
|
69
70
|
|
|
70
71
|
class << self
|
|
72
|
+
|
|
71
73
|
attr_accessor :proxmox_waiter_files_mutex
|
|
74
|
+
|
|
72
75
|
end
|
|
73
76
|
@proxmox_waiter_files_mutex = Mutex.new
|
|
74
77
|
|
|
@@ -83,105 +86,100 @@ module HybridPlatformsConductor
|
|
|
83
86
|
@lxc_details = nil
|
|
84
87
|
with_proxmox do |proxmox|
|
|
85
88
|
proxmox_get(proxmox, 'nodes').each do |node_info|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
pve_node: node_info['node'],
|
|
115
|
-
vm_id: vm_id,
|
|
116
|
-
vm_ip: ip_found
|
|
117
|
-
}
|
|
118
|
-
break
|
|
119
|
-
end
|
|
120
|
-
end
|
|
89
|
+
next unless proxmox_test_info[:test_config][:pve_nodes].include?(node_info['node']) && node_info['status'] == 'online'
|
|
90
|
+
|
|
91
|
+
proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc").each do |lxc_info|
|
|
92
|
+
vm_id = Integer(lxc_info['vmid'])
|
|
93
|
+
next unless vm_id.between?(*proxmox_test_info[:test_config][:vm_ids_range])
|
|
94
|
+
|
|
95
|
+
# Check if the description contains our ID
|
|
96
|
+
lxc_config = proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc/#{vm_id}/config")
|
|
97
|
+
vm_description_lines = (lxc_config['description'] || '').split("\n")
|
|
98
|
+
hpc_marker_idx = vm_description_lines.index('===== HPC info =====')
|
|
99
|
+
next if hpc_marker_idx.nil?
|
|
100
|
+
|
|
101
|
+
# Get the HPC info associated to this VM
|
|
102
|
+
# Hash<Symbol,String>
|
|
103
|
+
vm_hpc_info = vm_description_lines[hpc_marker_idx + 1..].map do |line|
|
|
104
|
+
property, value = line.split(': ')
|
|
105
|
+
[property.to_sym, value]
|
|
106
|
+
end.to_h
|
|
107
|
+
next unless vm_hpc_info[:node] == @node && vm_hpc_info[:environment] == @environment
|
|
108
|
+
|
|
109
|
+
# Found it
|
|
110
|
+
# Get back the IP
|
|
111
|
+
ip_found = nil
|
|
112
|
+
lxc_config['net0'].split(',').each do |net_info|
|
|
113
|
+
property, value = net_info.split('=')
|
|
114
|
+
if property == 'ip'
|
|
115
|
+
ip_found = value.split('/').first
|
|
116
|
+
break
|
|
121
117
|
end
|
|
122
118
|
end
|
|
123
|
-
|
|
119
|
+
raise "[ #{@node}/#{@environment} ] - Unable to get IP back from LXC container nodes/#{node_info['node']}/lxc/#{vm_id}/config" if ip_found.nil?
|
|
120
|
+
|
|
121
|
+
@lxc_details = {
|
|
122
|
+
pve_node: node_info['node'],
|
|
123
|
+
vm_id: vm_id,
|
|
124
|
+
vm_ip: ip_found
|
|
125
|
+
}
|
|
126
|
+
break
|
|
124
127
|
end
|
|
128
|
+
break if @lxc_details
|
|
125
129
|
end
|
|
126
130
|
end
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
cores: min_resources_to_deploy[:cpus],
|
|
161
|
-
cpulimit: min_resources_to_deploy[:cpus],
|
|
162
|
-
memory: min_resources_to_deploy[:ram_mb],
|
|
163
|
-
rootfs: "local-lvm:#{min_resources_to_deploy[:disk_gb]}",
|
|
164
|
-
nameserver: vm_config[:vm_dns_servers].join(' '),
|
|
165
|
-
searchdomain: vm_config[:vm_search_domain],
|
|
166
|
-
net0: "name=eth0,bridge=vmbr0,gw=#{vm_config[:vm_gateway]}",
|
|
167
|
-
password: 'root_pwd',
|
|
168
|
-
description: <<~EOS
|
|
169
|
-
===== HPC info =====
|
|
170
|
-
node: #{@node}
|
|
171
|
-
environment: #{@environment}
|
|
172
|
-
debug: #{log_debug? ? 'true' : 'false'}
|
|
173
|
-
EOS
|
|
174
|
-
})
|
|
175
|
-
else
|
|
176
|
-
raise "[ #{@node}/#{@environment} ] - No template found in #{proxmox_conf}"
|
|
177
|
-
end
|
|
178
|
-
else
|
|
179
|
-
raise "[ #{@node}/#{@environment} ] - No Proxmox configuration found at #{proxmox_conf}"
|
|
180
|
-
end
|
|
181
|
-
else
|
|
182
|
-
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}"
|
|
183
|
-
end
|
|
131
|
+
return if @lxc_details
|
|
132
|
+
|
|
133
|
+
# We couldn't find an existing LXC container for this node/environment.
|
|
134
|
+
# We have to create one.
|
|
135
|
+
# Get the image name for this node
|
|
136
|
+
image = @nodes_handler.get_image_of(@node).to_sym
|
|
137
|
+
# Find if we have such an image registered
|
|
138
|
+
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}" unless @config.known_os_images.include?(image)
|
|
139
|
+
|
|
140
|
+
proxmox_conf = "#{@config.os_image_dir(image)}/proxmox.json"
|
|
141
|
+
raise "[ #{@node}/#{@environment} ] - No Proxmox configuration found at #{proxmox_conf}" unless File.exist?(proxmox_conf)
|
|
142
|
+
|
|
143
|
+
pve_template = JSON.parse(File.read(proxmox_conf))['template']
|
|
144
|
+
raise "[ #{@node}/#{@environment} ] - No template found in #{proxmox_conf}" unless pve_template
|
|
145
|
+
|
|
146
|
+
# Query the inventory to know about minimum resources needed to deploy the node.
|
|
147
|
+
# Provide default values if they are not part of the metadata.
|
|
148
|
+
min_resources_to_deploy = {
|
|
149
|
+
cpus: 2,
|
|
150
|
+
ram_mb: 1024,
|
|
151
|
+
disk_gb: 10
|
|
152
|
+
}.merge(@nodes_handler.get_deploy_resources_min_of(@node) || {})
|
|
153
|
+
# Create the Proxmox container from the sync node.
|
|
154
|
+
vm_config = proxmox_test_info[:vm_config]
|
|
155
|
+
# Hostname in Proxmox is capped at 65 chars.
|
|
156
|
+
# Make sure we don't get over it, but still use a unique one.
|
|
157
|
+
hostname = "#{@node}.#{@environment}.hpc-test.com"
|
|
158
|
+
if hostname.size > MAX_PROXMOX_HOSTNAME_SIZE
|
|
159
|
+
# Truncate it, but add a unique ID in it.
|
|
160
|
+
# In the end the hostname looks like:
|
|
161
|
+
# <truncated_node_environment>.<unique_id>.hpc-test.com
|
|
162
|
+
hostname = "-#{Digest::MD5.hexdigest(hostname)[0..7]}.hpc-test.com"
|
|
163
|
+
hostname = "#{@node}.#{@environment}"[0..MAX_PROXMOX_HOSTNAME_SIZE - hostname.size - 1] + hostname
|
|
184
164
|
end
|
|
165
|
+
@lxc_details = request_lxc_creation_for(
|
|
166
|
+
ostemplate: pve_template,
|
|
167
|
+
hostname: hostname.gsub('_', '-'),
|
|
168
|
+
cores: min_resources_to_deploy[:cpus],
|
|
169
|
+
cpulimit: min_resources_to_deploy[:cpus],
|
|
170
|
+
memory: min_resources_to_deploy[:ram_mb],
|
|
171
|
+
rootfs: "local-lvm:#{min_resources_to_deploy[:disk_gb]}",
|
|
172
|
+
nameserver: vm_config[:vm_dns_servers].join(' '),
|
|
173
|
+
searchdomain: vm_config[:vm_search_domain],
|
|
174
|
+
net0: "name=eth0,bridge=vmbr0,gw=#{vm_config[:vm_gateway]}",
|
|
175
|
+
password: 'root_pwd',
|
|
176
|
+
description: <<~EO_DESCRIPTION
|
|
177
|
+
===== HPC info =====
|
|
178
|
+
node: #{@node}
|
|
179
|
+
environment: #{@environment}
|
|
180
|
+
debug: #{log_debug? ? 'true' : 'false'}
|
|
181
|
+
EO_DESCRIPTION
|
|
182
|
+
)
|
|
185
183
|
end
|
|
186
184
|
|
|
187
185
|
# Start an instance
|
|
@@ -274,6 +272,7 @@ module HybridPlatformsConductor
|
|
|
274
272
|
def with_proxmox
|
|
275
273
|
url = proxmox_test_info[:api_url]
|
|
276
274
|
raise 'No Proxmox server defined' if url.nil?
|
|
275
|
+
|
|
277
276
|
Credentials.with_credentials_for(:proxmox, @logger, @logger_stderr, url: url) do |user, password|
|
|
278
277
|
log_debug "[ #{@node}/#{@environment} ] - Connect to Proxmox #{url}"
|
|
279
278
|
proxmox_logs = StringIO.new
|
|
@@ -313,9 +312,11 @@ module HybridPlatformsConductor
|
|
|
313
312
|
idx_try = 0
|
|
314
313
|
loop do
|
|
315
314
|
response = proxmox.get(path)
|
|
316
|
-
break if !
|
|
315
|
+
break if !response.is_a?(String) || response !~ /^NOK: error code = 5\d\d$/
|
|
316
|
+
|
|
317
317
|
log_warn "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returned error #{response} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
|
|
318
318
|
raise "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returns #{response} continuously (tried #{idx_try + 1} times)" if idx_try >= proxmox_test_info[:api_max_retries]
|
|
319
|
+
|
|
319
320
|
idx_try += 1
|
|
320
321
|
# We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
|
|
321
322
|
proxmox.reauthenticate
|
|
@@ -336,23 +337,22 @@ module HybridPlatformsConductor
|
|
|
336
337
|
def run_proxmox_task(proxmox, http_method, pve_node, sub_path, *args)
|
|
337
338
|
task = nil
|
|
338
339
|
idx_try = 0
|
|
339
|
-
while task.nil?
|
|
340
|
+
while task.nil?
|
|
340
341
|
task = proxmox.send(http_method, "nodes/#{pve_node}/#{sub_path}", *args)
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
if task.nil?
|
|
352
|
-
raise "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} is constantly failing. Giving up."
|
|
353
|
-
else
|
|
354
|
-
wait_for_proxmox_task(proxmox, pve_node, task)
|
|
342
|
+
next unless task =~ /^NOK: error code = 5\d\d$/
|
|
343
|
+
|
|
344
|
+
log_warn "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} returned error #{task} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
|
|
345
|
+
task = nil
|
|
346
|
+
break if idx_try >= proxmox_test_info[:api_max_retries]
|
|
347
|
+
|
|
348
|
+
idx_try += 1
|
|
349
|
+
# We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
|
|
350
|
+
proxmox.reauthenticate
|
|
351
|
+
sleep proxmox_test_info[:api_wait_between_retries_secs] + rand(5)
|
|
355
352
|
end
|
|
353
|
+
raise "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} is constantly failing. Giving up." if task.nil?
|
|
354
|
+
|
|
355
|
+
wait_for_proxmox_task(proxmox, pve_node, task)
|
|
356
356
|
end
|
|
357
357
|
|
|
358
358
|
# Wait for a given Proxmox task completion
|
|
@@ -363,18 +363,18 @@ module HybridPlatformsConductor
|
|
|
363
363
|
# * *task* (String): The task ID
|
|
364
364
|
def wait_for_proxmox_task(proxmox, pve_node, task)
|
|
365
365
|
raise "Invalid task: #{task}" if task[0..3] == 'NOK:'
|
|
366
|
+
|
|
366
367
|
status = nil
|
|
367
368
|
loop do
|
|
368
369
|
status = task_status(proxmox, pve_node, task)
|
|
369
370
|
break unless status == 'running'
|
|
371
|
+
|
|
370
372
|
log_debug "[ #{@node}/#{@environment} ] - Wait for Proxmox task #{task} to complete..."
|
|
371
373
|
sleep 1
|
|
372
374
|
end
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
raise "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed with status #{status}"
|
|
377
|
-
end
|
|
375
|
+
raise "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed with status #{status}" unless status.split(':').last == 'OK'
|
|
376
|
+
|
|
377
|
+
log_debug "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed."
|
|
378
378
|
end
|
|
379
379
|
|
|
380
380
|
# Get task status
|
|
@@ -402,13 +402,13 @@ module HybridPlatformsConductor
|
|
|
402
402
|
config_file = "#{Dir.tmpdir}/config_#{file_id}.json"
|
|
403
403
|
File.write(
|
|
404
404
|
config_file,
|
|
405
|
-
|
|
405
|
+
proxmox_test_info[:test_config].merge(
|
|
406
406
|
proxmox_api_url: proxmox_test_info[:api_url],
|
|
407
407
|
futex_file: '/tmp/hpc_proxmox_allocations.futex',
|
|
408
408
|
logs_dir: '/tmp/hpc_proxmox_waiter_logs',
|
|
409
409
|
api_max_retries: proxmox_test_info[:api_max_retries],
|
|
410
410
|
api_wait_between_retries_secs: proxmox_test_info[:api_wait_between_retries_secs]
|
|
411
|
-
)
|
|
411
|
+
).to_json
|
|
412
412
|
)
|
|
413
413
|
result = nil
|
|
414
414
|
begin
|
|
@@ -447,7 +447,7 @@ module HybridPlatformsConductor
|
|
|
447
447
|
)[proxmox_test_info[:sync_node]]
|
|
448
448
|
end
|
|
449
449
|
stdout_lines = stdout.split("\n")
|
|
450
|
-
result = JSON.parse(stdout_lines[stdout_lines.index('===== JSON =====') + 1
|
|
450
|
+
result = JSON.parse(stdout_lines[stdout_lines.index('===== JSON =====') + 1..].join("\n")).transform_keys(&:to_sym)
|
|
451
451
|
raise "[ #{@node}/#{@environment} ] - Error returned by #{cmd}: #{result[:error]}" if result.key?(:error)
|
|
452
452
|
ensure
|
|
453
453
|
File.unlink(config_file)
|
|
@@ -548,6 +548,7 @@ module HybridPlatformsConductor
|
|
|
548
548
|
def proxmox_test_info
|
|
549
549
|
@config.proxmox_servers.first
|
|
550
550
|
end
|
|
551
|
+
|
|
551
552
|
end
|
|
552
553
|
|
|
553
554
|
end
|