hybrid_platforms_conductor 32.16.4 → 33.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +42 -0
- data/README.md +6 -3
- data/bin/check-node +0 -1
- data/bin/deploy +0 -1
- data/bin/get_impacted_nodes +0 -1
- data/bin/last_deploys +12 -8
- data/bin/nodes_to_deploy +6 -6
- data/bin/setup +6 -6
- data/bin/topograph +1 -1
- data/docs/config_dsl.md +45 -1
- data/docs/executables.md +6 -7
- data/docs/executables/check-node.md +3 -3
- data/docs/executables/deploy.md +3 -3
- data/docs/executables/dump_nodes_json.md +3 -3
- data/docs/executables/test.md +3 -3
- data/docs/executables/topograph.md +3 -3
- data/docs/gen/mermaid/README.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
- data/docs/plugins.md +47 -0
- data/docs/plugins/connector/ssh.md +1 -1
- data/docs/plugins/log/remote_fs.md +26 -0
- data/docs/plugins/secrets_reader/cli.md +31 -0
- data/docs/plugins/secrets_reader/thycotic.md +46 -0
- data/docs/plugins/test/bitbucket_conf.md +1 -1
- data/docs/plugins/test/check_deploy_and_idempotence.md +1 -1
- data/docs/plugins/test/connection.md +1 -0
- data/docs/plugins/test/deploy_removes_root_access.md +1 -1
- data/docs/plugins/test/file_system.md +1 -0
- data/docs/plugins/test/github_ci.md +48 -0
- data/docs/plugins/test/hostname.md +1 -0
- data/docs/plugins/test/ip.md +1 -0
- data/docs/plugins/test/jenkins_ci_conf.md +1 -1
- data/docs/plugins/test/jenkins_ci_masters_ok.md +1 -1
- data/docs/plugins/test/local_users.md +1 -0
- data/docs/plugins/test/mounts.md +1 -0
- data/docs/plugins/test/orphan_files.md +1 -0
- data/docs/plugins/test/ports.md +1 -0
- data/docs/plugins/test/spectre.md +1 -0
- data/docs/plugins/test/vulnerabilities.md +1 -0
- data/lib/hybrid_platforms_conductor/action.rb +4 -4
- data/lib/hybrid_platforms_conductor/actions_executor.rb +45 -43
- data/lib/hybrid_platforms_conductor/bitbucket.rb +5 -4
- data/lib/hybrid_platforms_conductor/cmd_runner.rb +13 -12
- data/lib/hybrid_platforms_conductor/cmdb.rb +2 -2
- data/lib/hybrid_platforms_conductor/common_config_dsl/bitbucket.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/confluence.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +5 -4
- data/lib/hybrid_platforms_conductor/common_config_dsl/github.rb +63 -0
- data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb +2 -2
- data/lib/hybrid_platforms_conductor/config.rb +7 -4
- data/lib/hybrid_platforms_conductor/confluence.rb +1 -1
- data/lib/hybrid_platforms_conductor/connector.rb +5 -2
- data/lib/hybrid_platforms_conductor/core_extensions/symbol/zero.rb +24 -0
- data/lib/hybrid_platforms_conductor/credentials.rb +39 -36
- data/lib/hybrid_platforms_conductor/current_dir_monitor.rb +4 -1
- data/lib/hybrid_platforms_conductor/deployer.rb +275 -224
- data/lib/hybrid_platforms_conductor/executable.rb +20 -15
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/config.rb +10 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +4 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb +2 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +45 -49
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/my_log_plugin.rb.sample +100 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb +180 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +68 -66
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb +13 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb +39 -38
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb +5 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb +43 -45
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/podman.rb +18 -20
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +118 -117
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +39 -43
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container +9 -13
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +28 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/stdout.rb +26 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/cli.rb +77 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/my_secrets_reader_plugin.rb.sample +46 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb +90 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb +4 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb +3 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +7 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb +21 -19
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +21 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +19 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb +31 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +3 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +4 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb +21 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb +10 -12
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +9 -9
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +13 -10
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +7 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +27 -25
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/stdout.rb +8 -6
- data/lib/hybrid_platforms_conductor/io_router.rb +14 -13
- data/lib/hybrid_platforms_conductor/json_dumper.rb +2 -2
- data/lib/hybrid_platforms_conductor/log.rb +31 -0
- data/lib/hybrid_platforms_conductor/logger_helpers.rb +19 -16
- data/lib/hybrid_platforms_conductor/nodes_handler.rb +89 -71
- data/lib/hybrid_platforms_conductor/parallel_threads.rb +7 -11
- data/lib/hybrid_platforms_conductor/platform_handler.rb +7 -7
- data/lib/hybrid_platforms_conductor/platforms_handler.rb +5 -3
- data/lib/hybrid_platforms_conductor/plugin.rb +2 -2
- data/lib/hybrid_platforms_conductor/plugins.rb +14 -8
- data/lib/hybrid_platforms_conductor/provisioner.rb +4 -4
- data/lib/hybrid_platforms_conductor/report.rb +2 -2
- data/lib/hybrid_platforms_conductor/reports_handler.rb +3 -2
- data/lib/hybrid_platforms_conductor/secrets_reader.rb +31 -0
- data/lib/hybrid_platforms_conductor/services_handler.rb +32 -29
- data/lib/hybrid_platforms_conductor/test_only_remote_node.rb +18 -0
- data/lib/hybrid_platforms_conductor/test_report.rb +15 -18
- data/lib/hybrid_platforms_conductor/tests_runner.rb +116 -118
- data/lib/hybrid_platforms_conductor/thycotic.rb +28 -19
- data/lib/hybrid_platforms_conductor/topographer.rb +200 -190
- data/lib/hybrid_platforms_conductor/topographer/plugins/graphviz.rb +8 -8
- data/lib/hybrid_platforms_conductor/topographer/plugins/json.rb +4 -4
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test.rb +33 -12
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +16 -11
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/interactive_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb +23 -18
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +67 -49
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/scp_spec.rb +20 -14
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions_spec.rb +63 -50
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connection_spec.rb +35 -35
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/connectable_nodes_spec.rb +12 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb +4 -7
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +21 -22
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +23 -24
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connectable_nodes_spec.rb +10 -6
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +73 -54
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +145 -126
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +29 -25
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/logging_spec.rb +106 -91
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/parallel_spec.rb +272 -244
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +10 -12
- data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +36 -36
- data/spec/hybrid_platforms_conductor_test/api/config_spec.rb +24 -22
- data/spec/hybrid_platforms_conductor_test/api/deployer/check_spec.rb +4 -2
- data/spec/hybrid_platforms_conductor_test/api/deployer/config_dsl_spec.rb +43 -5
- data/spec/hybrid_platforms_conductor_test/api/deployer/deploy_spec.rb +199 -216
- data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +223 -0
- data/spec/hybrid_platforms_conductor_test/api/deployer/parse_deploy_output_spec.rb +55 -59
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +36 -62
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/podman_spec.rb +17 -17
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/config_dsl_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/create_spec.rb +44 -51
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/destroy_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/ip_spec.rb +12 -16
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/destroy_vm_spec.rb +31 -19
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +324 -266
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/ips_assignment_spec.rb +89 -61
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/other_lxc_containers_resources_spec.rb +117 -93
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/pve_node_resources_spec.rb +71 -54
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +10 -8
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb +80 -60
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/start_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/stop_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/cli_spec.rb +64 -0
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/thycotic_spec.rb +268 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/config_spec.rb +8 -10
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +33 -24
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_keys_spec.rb +64 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/platform_handlers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +50 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +91 -81
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +14 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/git_diff_impacts_spec.rb +51 -75
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/nodes_selectors_spec.rb +35 -26
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/platform_handlers_plugins_api_spec.rb +24 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/several_platforms_spec.rb +29 -19
- data/spec/hybrid_platforms_conductor_test/api/platform_handler_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb +57 -99
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +32 -35
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +10 -10
- data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +38 -27
- data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +46 -52
- data/spec/hybrid_platforms_conductor_test/api/reports_handler_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/services_handler/actions_to_deploy_spec.rb +90 -58
- data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +38 -34
- data/spec/hybrid_platforms_conductor_test/api/services_handler/log_info_spec.rb +11 -9
- data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +193 -171
- data/spec/hybrid_platforms_conductor_test/api/services_handler/parse_deploy_output_spec.rb +66 -54
- data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +147 -133
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +69 -49
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/global_spec.rb +5 -4
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +30 -27
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/platform_spec.rb +12 -9
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +48 -47
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/bitbucket_conf_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/github_ci_spec.rb +72 -0
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_reports_plugins/confluence_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +9 -9
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb2.rb → test_cmdb_2.rb} +6 -6
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb_others2.rb → test_cmdb_others_2.rb} +2 -2
- data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/{check-node_spec.rb → check_node_spec.rb} +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/get_impacted_nodes_spec.rb +76 -77
- data/spec/hybrid_platforms_conductor_test/executables/last_deploys_spec.rb +159 -113
- data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +299 -160
- data/spec/hybrid_platforms_conductor_test/executables/options/actions_executor_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +12 -196
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_selectors_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/reports_handler_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +22 -22
- data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +22 -16
- data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +32 -32
- data/spec/hybrid_platforms_conductor_test/executables/ssh_config_spec.rb +7 -9
- data/spec/hybrid_platforms_conductor_test/executables/test_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +4 -3
- data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/config_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +12 -13
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +245 -56
- data/spec/hybrid_platforms_conductor_test/helpers/executables_helpers.rb +11 -11
- data/spec/hybrid_platforms_conductor_test/helpers/nodes_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +39 -28
- data/spec/hybrid_platforms_conductor_test/helpers/plugins_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +86 -111
- data/spec/hybrid_platforms_conductor_test/helpers/reports_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/helpers/services_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/tests_runner_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id1.rb → test_plugin_id_1.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id2.rb → test_plugin_id_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/test_plugin_type/{test_plugin_id3.rb → test_plugin_id_3.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/{test_plugin_type2/test_plugin_id4.rb → test_plugin_type_2/test_plugin_id_4.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/test.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/{test2.rb → test_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/rubocop_spec.rb +31 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +952 -0
- data/spec/hybrid_platforms_conductor_test/test_connector.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/test_log_no_read_plugin.rb +84 -0
- data/spec/hybrid_platforms_conductor_test/test_log_plugin.rb +105 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/global.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node_check.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/platform.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/several_checks.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/test_secrets_reader_plugin.rb +45 -0
- data/spec/hybrid_platforms_conductor_test/tests_report_plugin.rb +5 -6
- data/spec/spec_helper.rb +17 -18
- data/tools/check_md +16 -20
- data/tools/generate_mermaid +1 -1
- metadata +82 -13
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +0 -916
data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb
CHANGED
@@ -29,6 +29,10 @@ module HybridPlatformsConductor
|
|
29
29
|
instance_eval(File.read(source))
|
30
30
|
end
|
31
31
|
|
32
|
+
# Intercept all missing methods
|
33
|
+
#
|
34
|
+
# Parameters::
|
35
|
+
# * *method_name* (Symbol): The missing method being called
|
32
36
|
def method_missing(method_name, *args, &block)
|
33
37
|
sub_calls = []
|
34
38
|
@calls << {
|
@@ -40,6 +44,15 @@ module HybridPlatformsConductor
|
|
40
44
|
DslParser.new(sub_calls)
|
41
45
|
end
|
42
46
|
|
47
|
+
# Make sure we register the methods we handle in method_missing
|
48
|
+
#
|
49
|
+
# Parameters::
|
50
|
+
# * *name* (Symbol): The missing method name
|
51
|
+
# * *include_private* (Boolean): Should we include private methods in the search?
|
52
|
+
def respond_to_missing?(_name, _include_private)
|
53
|
+
true
|
54
|
+
end
|
55
|
+
|
43
56
|
end
|
44
57
|
|
45
58
|
end
|
@@ -40,11 +40,14 @@ module HybridPlatformsConductor
|
|
40
40
|
add_recipe_in_tree(cookbook_dir, cookbook, recipe)
|
41
41
|
end
|
42
42
|
end
|
43
|
+
# Make sure we don't combine those 2 loops
|
44
|
+
# rubocop:disable Style/CombinableLoops
|
43
45
|
@platform.deployable_services.each do |service|
|
44
|
-
@platform.policy_run_list(service).each do |(
|
46
|
+
@platform.policy_run_list(service).each do |(_cookbook_dir, cookbook, recipe)|
|
45
47
|
mark_recipe_used_by_policy(cookbook, recipe, service)
|
46
48
|
end
|
47
49
|
end
|
50
|
+
# rubocop:enable Style/CombinableLoops
|
48
51
|
@recipes_tree
|
49
52
|
end
|
50
53
|
|
@@ -58,25 +61,25 @@ module HybridPlatformsConductor
|
|
58
61
|
# * *recipe* (Symbol): The recipe name
|
59
62
|
def add_recipe_in_tree(cookbook_dir, cookbook, recipe)
|
60
63
|
@recipes_tree[cookbook] = {} unless @recipes_tree.key?(cookbook)
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
@recipes_tree[cookbook][recipe] = recipe_info.merge(
|
75
|
-
used_by_policies: []
|
76
|
-
)
|
77
|
-
recipe_info[:included_recipes].each do |(sub_cookbook_dir, sub_cookbook, sub_recipe)|
|
78
|
-
add_recipe_in_tree(sub_cookbook_dir, sub_cookbook, sub_recipe)
|
64
|
+
return if @recipes_tree[cookbook].key?(recipe)
|
65
|
+
|
66
|
+
recipe_info =
|
67
|
+
if cookbook_dir.nil?
|
68
|
+
# This recipe comes from an external cookbook, we won't get into it.
|
69
|
+
{
|
70
|
+
included_recipes: [],
|
71
|
+
used_templates: [],
|
72
|
+
used_files: [],
|
73
|
+
used_cookbooks: []
|
74
|
+
}
|
75
|
+
else
|
76
|
+
recipe_usage(cookbook_dir, cookbook, recipe)
|
79
77
|
end
|
78
|
+
@recipes_tree[cookbook][recipe] = recipe_info.merge(
|
79
|
+
used_by_policies: []
|
80
|
+
)
|
81
|
+
recipe_info[:included_recipes].each do |(sub_cookbook_dir, sub_cookbook, sub_recipe)|
|
82
|
+
add_recipe_in_tree(sub_cookbook_dir, sub_cookbook, sub_recipe)
|
80
83
|
end
|
81
84
|
end
|
82
85
|
|
@@ -115,18 +118,18 @@ module HybridPlatformsConductor
|
|
115
118
|
recipe_content.
|
116
119
|
scan(/source\s+(["'])(.+?)\1/).
|
117
120
|
each do |(_sub_grp, source)|
|
118
|
-
sources << source unless source =~
|
121
|
+
sources << source unless source =~ %r{^https?://}
|
119
122
|
end
|
120
123
|
erb_sources = sources.select { |source| File.extname(source).downcase == '.erb' }
|
121
124
|
non_erb_sources = sources - erb_sources
|
122
125
|
erb_sources.concat(recipe_content.scan(/template:?\s+(["'])(.+?)\1/).map { |(_sub_grp, source)| source })
|
123
126
|
# Check for known resources and library methods
|
124
127
|
used_cookbooks = []
|
125
|
-
known_resources.each do |
|
126
|
-
used_cookbooks <<
|
128
|
+
known_resources.each do |itr_cookbook, methods|
|
129
|
+
used_cookbooks << itr_cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
|
127
130
|
end
|
128
|
-
known_library_methods.each do |
|
129
|
-
used_cookbooks <<
|
131
|
+
known_library_methods.each do |itr_cookbook, methods|
|
132
|
+
used_cookbooks << itr_cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
|
130
133
|
end
|
131
134
|
{
|
132
135
|
included_recipes: used_recipes,
|
@@ -171,9 +174,9 @@ module HybridPlatformsConductor
|
|
171
174
|
@known_library_methods = {}
|
172
175
|
for_each_cookbook do |cookbook, cookbook_dir|
|
173
176
|
if File.exist?("#{cookbook_dir}/libraries")
|
174
|
-
found_methods =
|
175
|
-
|
176
|
-
|
177
|
+
found_methods = Dir.glob("#{cookbook_dir}/libraries/*.rb").
|
178
|
+
map { |lib_file| File.read(lib_file).scan(/\bdef\s+(\w+)\b/).map { |(method_name)| method_name } }.
|
179
|
+
flatten - INVALID_LIBRARY_METHODS
|
177
180
|
@known_library_methods[cookbook] = found_methods unless found_methods.empty?
|
178
181
|
end
|
179
182
|
end
|
@@ -184,15 +187,13 @@ module HybridPlatformsConductor
|
|
184
187
|
# Iterate over all cookbooks
|
185
188
|
#
|
186
189
|
# Parameters::
|
187
|
-
# * Proc: Code called for each cookbook:
|
190
|
+
# * *block* (Proc): Code called for each cookbook:
|
188
191
|
# * Parameters::
|
189
192
|
# * *cookbook* (Symbol): Cookbook name
|
190
193
|
# * *cookbook_dir* (String): Cookbook directory
|
191
|
-
def for_each_cookbook
|
194
|
+
def for_each_cookbook(&block)
|
192
195
|
@platform.known_cookbook_paths.each do |cookbook_path|
|
193
|
-
cookbooks_in(cookbook_path).each
|
194
|
-
yield cookbook, cookbook_dir
|
195
|
-
end
|
196
|
+
cookbooks_in(cookbook_path).each(&block)
|
196
197
|
end
|
197
198
|
end
|
198
199
|
|
@@ -203,7 +204,7 @@ module HybridPlatformsConductor
|
|
203
204
|
# Result::
|
204
205
|
# * Hash<Symbol, String>: List of cookbook directories, per cookbook name
|
205
206
|
def cookbooks_in(cookbook_type)
|
206
|
-
|
207
|
+
Dir.glob("#{@platform.repository_path}/#{cookbook_type}/*").map { |dir| [File.basename(dir).to_sym, dir] }.sort.to_h
|
207
208
|
end
|
208
209
|
|
209
210
|
# Mark a recipe (and its included recipes) as used by a policy
|
@@ -213,11 +214,11 @@ module HybridPlatformsConductor
|
|
213
214
|
# * *recipe* (Symbol): The recipe
|
214
215
|
# * *used_by_policy* (String): The policy using this recipe
|
215
216
|
def mark_recipe_used_by_policy(cookbook, recipe, used_by_policy)
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
217
|
+
return if @recipes_tree[cookbook][recipe][:used_by_policies].include?(used_by_policy)
|
218
|
+
|
219
|
+
@recipes_tree[cookbook][recipe][:used_by_policies] << used_by_policy
|
220
|
+
@recipes_tree[cookbook][recipe][:included_recipes].each do |(_sub_cookbook_dir, sub_cookbook, sub_recipe)|
|
221
|
+
mark_recipe_used_by_policy(sub_cookbook, sub_recipe, used_by_policy)
|
221
222
|
end
|
222
223
|
end
|
223
224
|
|
@@ -16,7 +16,7 @@ module HybridPlatformsConductor
|
|
16
16
|
def init
|
17
17
|
# This method is called when initializing a new instance of this platform handler, for a given repository.
|
18
18
|
inv_file = "#{@repository_path}/inventory.yaml"
|
19
|
-
@inventory = File.exist?(inv_file) ? YAML.
|
19
|
+
@inventory = File.exist?(inv_file) ? YAML.safe_load(File.read(inv_file)) : {}
|
20
20
|
end
|
21
21
|
|
22
22
|
# Get the list of known nodes.
|
@@ -89,12 +89,13 @@ module HybridPlatformsConductor
|
|
89
89
|
# * *cmd_runner* (CmdRunner): CmdRunner to be used [default: CmdRunner.new]
|
90
90
|
def initialize(
|
91
91
|
platform_handler,
|
92
|
-
logger: Logger.new(
|
93
|
-
logger_stderr: Logger.new(
|
92
|
+
logger: Logger.new($stdout),
|
93
|
+
logger_stderr: Logger.new($stderr),
|
94
94
|
config: Config.new,
|
95
95
|
nodes_handler: NodesHandler.new,
|
96
96
|
cmd_runner: CmdRunner.new
|
97
97
|
)
|
98
|
+
super
|
98
99
|
init_loggers(logger, logger_stderr)
|
99
100
|
@platform_handler = platform_handler
|
100
101
|
@config = config
|
@@ -127,7 +128,7 @@ module HybridPlatformsConductor
|
|
127
128
|
# * *:changed*: The task has been changed
|
128
129
|
# * *:identical*: The task has not been changed
|
129
130
|
# * *diffs* (String): Differences, if any
|
130
|
-
def parse_deploy_output(
|
131
|
+
def parse_deploy_output(_stdout, _stderr)
|
131
132
|
[]
|
132
133
|
end
|
133
134
|
|
@@ -21,7 +21,7 @@ module HybridPlatformsConductor
|
|
21
21
|
::Docker.validate_version!
|
22
22
|
docker_ok = true
|
23
23
|
rescue
|
24
|
-
log_error "[ #{@node}/#{@environment} ] - Docker is not installed correctly. Please install it. Error: #{
|
24
|
+
log_error "[ #{@node}/#{@environment} ] - Docker is not installed correctly. Please install it. Error: #{$ERROR_INFO}"
|
25
25
|
end
|
26
26
|
docker_ok
|
27
27
|
end
|
@@ -33,47 +33,45 @@ module HybridPlatformsConductor
|
|
33
33
|
# Get the image name for this node
|
34
34
|
image = @nodes_handler.get_image_of(@node).to_sym
|
35
35
|
# Find if we have such an image registered
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
end
|
51
|
-
container_name = "hpc_docker_container_#{@node}_#{@environment}"
|
52
|
-
container_futex_file = "#{Dir.tmpdir}/hpc_docker_container_futexes/#{image_tag}"
|
53
|
-
FileUtils.mkdir_p File.dirname(container_futex_file)
|
54
|
-
Futex.new(container_futex_file).open do
|
55
|
-
old_docker_container = ::Docker::Container.all(all: true).find { |container| container.info['Names'].include? "/#{container_name}" }
|
56
|
-
@container =
|
57
|
-
if old_docker_container
|
58
|
-
old_docker_container
|
59
|
-
else
|
60
|
-
log_debug "[ #{@node}/#{@environment} ] - Creating Docker container #{container_name}..."
|
61
|
-
# We add the SYS_PTRACE capability as some images need to restart services (for example postfix) and those services need the rights to ls in /proc/{PID}/exe to check if a status is running. Without SYS_PTRACE such ls returns permission denied and the service can't be stopped (as init.d always returns it as stopped even when running).
|
62
|
-
# We add the privileges as some containers need to install and configure the udev package, which needs RW access to /sys.
|
63
|
-
# We add the bind to cgroup volume to be able to test systemd specifics (enabling/disabling services for example).
|
64
|
-
::Docker::Container.create(
|
65
|
-
name: container_name,
|
66
|
-
image: image_tag,
|
67
|
-
CapAdd: 'SYS_PTRACE',
|
68
|
-
Privileged: true,
|
69
|
-
Binds: ['/sys/fs/cgroup:/sys/fs/cgroup:ro'],
|
70
|
-
# Some playbooks need the hostname to be set to a correct FQDN
|
71
|
-
Hostname: "#{@node}.testdomain"
|
72
|
-
)
|
73
|
-
end
|
36
|
+
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}" unless @config.known_os_images.include?(image)
|
37
|
+
|
38
|
+
# Build the image if it does not exist
|
39
|
+
image_tag = "hpc_image_#{image}"
|
40
|
+
docker_image = nil
|
41
|
+
image_futex_file = "#{Dir.tmpdir}/hpc_docker_image_futexes/#{image_tag}"
|
42
|
+
FileUtils.mkdir_p File.dirname(image_futex_file)
|
43
|
+
Futex.new(image_futex_file).open do
|
44
|
+
docker_image = ::Docker::Image.all.find { |search_image| !search_image.info['RepoTags'].nil? && search_image.info['RepoTags'].include?("#{image_tag}:latest") }
|
45
|
+
unless docker_image
|
46
|
+
log_debug "[ #{@node}/#{@environment} ] - Creating Docker image #{image_tag}..."
|
47
|
+
Excon.defaults[:read_timeout] = 600
|
48
|
+
docker_image = ::Docker::Image.build_from_dir(@config.os_image_dir(image))
|
49
|
+
docker_image.tag repo: image_tag
|
74
50
|
end
|
75
|
-
|
76
|
-
|
51
|
+
end
|
52
|
+
container_name = "hpc_docker_container_#{@node}_#{@environment}"
|
53
|
+
container_futex_file = "#{Dir.tmpdir}/hpc_docker_container_futexes/#{image_tag}"
|
54
|
+
FileUtils.mkdir_p File.dirname(container_futex_file)
|
55
|
+
Futex.new(container_futex_file).open do
|
56
|
+
old_docker_container = ::Docker::Container.all(all: true).find { |container| container.info['Names'].include? "/#{container_name}" }
|
57
|
+
@container =
|
58
|
+
if old_docker_container
|
59
|
+
old_docker_container
|
60
|
+
else
|
61
|
+
log_debug "[ #{@node}/#{@environment} ] - Creating Docker container #{container_name}..."
|
62
|
+
# We add the SYS_PTRACE capability as some images need to restart services (for example postfix) and those services need the rights to ls in /proc/{PID}/exe to check if a status is running. Without SYS_PTRACE such ls returns permission denied and the service can't be stopped (as init.d always returns it as stopped even when running).
|
63
|
+
# We add the privileges as some containers need to install and configure the udev package, which needs RW access to /sys.
|
64
|
+
# We add the bind to cgroup volume to be able to test systemd specifics (enabling/disabling services for example).
|
65
|
+
::Docker::Container.create(
|
66
|
+
name: container_name,
|
67
|
+
image: image_tag,
|
68
|
+
CapAdd: 'SYS_PTRACE',
|
69
|
+
Privileged: true,
|
70
|
+
Binds: ['/sys/fs/cgroup:/sys/fs/cgroup:ro'],
|
71
|
+
# Some playbooks need the hostname to be set to a correct FQDN
|
72
|
+
Hostname: "#{@node}.testdomain"
|
73
|
+
)
|
74
|
+
end
|
77
75
|
end
|
78
76
|
end
|
79
77
|
|
@@ -81,7 +79,7 @@ module HybridPlatformsConductor
|
|
81
79
|
# Prerequisite: create has been called before
|
82
80
|
# [API] - This method is mandatory
|
83
81
|
def start
|
84
|
-
log_debug "[ #{@node}/#{@environment} ] - Start Docker Container #{@container.refresh!.info['Name'][1
|
82
|
+
log_debug "[ #{@node}/#{@environment} ] - Start Docker Container #{@container.refresh!.info['Name'][1..]} ..."
|
85
83
|
@container.start
|
86
84
|
end
|
87
85
|
|
@@ -89,7 +87,7 @@ module HybridPlatformsConductor
|
|
89
87
|
# Prerequisite: create has been called before
|
90
88
|
# [API] - This method is mandatory
|
91
89
|
def stop
|
92
|
-
log_debug "[ #{@node}/#{@environment} ] - Stop Docker Container #{@container.refresh!.info['Name'][1
|
90
|
+
log_debug "[ #{@node}/#{@environment} ] - Stop Docker Container #{@container.refresh!.info['Name'][1..]} ..."
|
93
91
|
@container.stop
|
94
92
|
end
|
95
93
|
|
@@ -97,7 +95,7 @@ module HybridPlatformsConductor
|
|
97
95
|
# Prerequisite: create has been called before
|
98
96
|
# [API] - This method is mandatory
|
99
97
|
def destroy
|
100
|
-
log_debug "[ #{@node}/#{@environment} ] - Destroy Docker Container #{@container.refresh!.info['Name'][1
|
98
|
+
log_debug "[ #{@node}/#{@environment} ] - Destroy Docker Container #{@container.refresh!.info['Name'][1..]} ..."
|
101
99
|
@container.remove
|
102
100
|
@container = nil
|
103
101
|
end
|
@@ -119,7 +117,7 @@ module HybridPlatformsConductor
|
|
119
117
|
begin
|
120
118
|
@container.refresh!.info['State']['Status'].to_sym
|
121
119
|
rescue
|
122
|
-
log_error "[ #{@node}/#{@environment} ] - Error while reading state of Docker container: #{
|
120
|
+
log_error "[ #{@node}/#{@environment} ] - Error while reading state of Docker container: #{$ERROR_INFO}"
|
123
121
|
:error
|
124
122
|
end
|
125
123
|
end
|
@@ -17,25 +17,23 @@ module HybridPlatformsConductor
|
|
17
17
|
# Get the image name for this node
|
18
18
|
image = @nodes_handler.get_image_of(@node).to_sym
|
19
19
|
# Find if we have such an image registered
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
else
|
38
|
-
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}"
|
20
|
+
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}" unless @config.known_os_images.include?(image)
|
21
|
+
|
22
|
+
# Build the image if it does not exist
|
23
|
+
image_tag = "hpc_image_#{image}"
|
24
|
+
image_futex_file = "#{Dir.tmpdir}/hpc_podman_image_futexes/#{image_tag}"
|
25
|
+
FileUtils.mkdir_p File.dirname(image_futex_file)
|
26
|
+
Futex.new(image_futex_file).open do
|
27
|
+
@cmd_runner.run_cmd "cd #{@config.os_image_dir(image)} && #{podman_cmd} build --tag #{image_tag} --security-opt seccomp=/usr/share/containers/seccomp.json --cgroup-manager=cgroupfs ."
|
28
|
+
end
|
29
|
+
container_name = "hpc_container_#{@node}_#{@environment}"
|
30
|
+
container_futex_file = "#{Dir.tmpdir}/hpc_podman_container_futexes/#{image_tag}"
|
31
|
+
FileUtils.mkdir_p File.dirname(container_futex_file)
|
32
|
+
Futex.new(container_futex_file).open do
|
33
|
+
_exit_status, stdout, _stderr = @cmd_runner.run_cmd "#{podman_cmd} container list --all | grep #{container_name}", expected_code: [0, 1]
|
34
|
+
existing_container = !stdout.strip.empty?
|
35
|
+
@cmd_runner.run_cmd "#{podman_cmd} container create --name #{container_name} #{image_tag}" unless existing_container
|
36
|
+
@container = container_name
|
39
37
|
end
|
40
38
|
end
|
41
39
|
|
@@ -83,7 +81,7 @@ module HybridPlatformsConductor
|
|
83
81
|
status = :created if status == :configured
|
84
82
|
status
|
85
83
|
rescue
|
86
|
-
log_warn "Error while reading state of Podman container #{@container}: #{
|
84
|
+
log_warn "Error while reading state of Podman container #{@container}: #{$ERROR_INFO}"
|
87
85
|
:error
|
88
86
|
end
|
89
87
|
end
|
@@ -8,6 +8,7 @@ module HybridPlatformsConductor
|
|
8
8
|
|
9
9
|
module HpcPlugins
|
10
10
|
|
11
|
+
# Patch proxmox lib
|
11
12
|
module Provisioner
|
12
13
|
|
13
14
|
# Monkey patch some Proxmox methods
|
@@ -68,7 +69,9 @@ module HybridPlatformsConductor
|
|
68
69
|
extend_config_dsl_with PlatformsDSLProxmox, :init_proxmox
|
69
70
|
|
70
71
|
class << self
|
72
|
+
|
71
73
|
attr_accessor :proxmox_waiter_files_mutex
|
74
|
+
|
72
75
|
end
|
73
76
|
@proxmox_waiter_files_mutex = Mutex.new
|
74
77
|
|
@@ -83,105 +86,100 @@ module HybridPlatformsConductor
|
|
83
86
|
@lxc_details = nil
|
84
87
|
with_proxmox do |proxmox|
|
85
88
|
proxmox_get(proxmox, 'nodes').each do |node_info|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
pve_node: node_info['node'],
|
115
|
-
vm_id: vm_id,
|
116
|
-
vm_ip: ip_found
|
117
|
-
}
|
118
|
-
break
|
119
|
-
end
|
120
|
-
end
|
89
|
+
next unless proxmox_test_info[:test_config][:pve_nodes].include?(node_info['node']) && node_info['status'] == 'online'
|
90
|
+
|
91
|
+
proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc").each do |lxc_info|
|
92
|
+
vm_id = Integer(lxc_info['vmid'])
|
93
|
+
next unless vm_id.between?(*proxmox_test_info[:test_config][:vm_ids_range])
|
94
|
+
|
95
|
+
# Check if the description contains our ID
|
96
|
+
lxc_config = proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc/#{vm_id}/config")
|
97
|
+
vm_description_lines = (lxc_config['description'] || '').split("\n")
|
98
|
+
hpc_marker_idx = vm_description_lines.index('===== HPC info =====')
|
99
|
+
next if hpc_marker_idx.nil?
|
100
|
+
|
101
|
+
# Get the HPC info associated to this VM
|
102
|
+
# Hash<Symbol,String>
|
103
|
+
vm_hpc_info = vm_description_lines[hpc_marker_idx + 1..].map do |line|
|
104
|
+
property, value = line.split(': ')
|
105
|
+
[property.to_sym, value]
|
106
|
+
end.to_h
|
107
|
+
next unless vm_hpc_info[:node] == @node && vm_hpc_info[:environment] == @environment
|
108
|
+
|
109
|
+
# Found it
|
110
|
+
# Get back the IP
|
111
|
+
ip_found = nil
|
112
|
+
lxc_config['net0'].split(',').each do |net_info|
|
113
|
+
property, value = net_info.split('=')
|
114
|
+
if property == 'ip'
|
115
|
+
ip_found = value.split('/').first
|
116
|
+
break
|
121
117
|
end
|
122
118
|
end
|
123
|
-
|
119
|
+
raise "[ #{@node}/#{@environment} ] - Unable to get IP back from LXC container nodes/#{node_info['node']}/lxc/#{vm_id}/config" if ip_found.nil?
|
120
|
+
|
121
|
+
@lxc_details = {
|
122
|
+
pve_node: node_info['node'],
|
123
|
+
vm_id: vm_id,
|
124
|
+
vm_ip: ip_found
|
125
|
+
}
|
126
|
+
break
|
124
127
|
end
|
128
|
+
break if @lxc_details
|
125
129
|
end
|
126
130
|
end
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
cores: min_resources_to_deploy[:cpus],
|
161
|
-
cpulimit: min_resources_to_deploy[:cpus],
|
162
|
-
memory: min_resources_to_deploy[:ram_mb],
|
163
|
-
rootfs: "local-lvm:#{min_resources_to_deploy[:disk_gb]}",
|
164
|
-
nameserver: vm_config[:vm_dns_servers].join(' '),
|
165
|
-
searchdomain: vm_config[:vm_search_domain],
|
166
|
-
net0: "name=eth0,bridge=vmbr0,gw=#{vm_config[:vm_gateway]}",
|
167
|
-
password: 'root_pwd',
|
168
|
-
description: <<~EOS
|
169
|
-
===== HPC info =====
|
170
|
-
node: #{@node}
|
171
|
-
environment: #{@environment}
|
172
|
-
debug: #{log_debug? ? 'true' : 'false'}
|
173
|
-
EOS
|
174
|
-
})
|
175
|
-
else
|
176
|
-
raise "[ #{@node}/#{@environment} ] - No template found in #{proxmox_conf}"
|
177
|
-
end
|
178
|
-
else
|
179
|
-
raise "[ #{@node}/#{@environment} ] - No Proxmox configuration found at #{proxmox_conf}"
|
180
|
-
end
|
181
|
-
else
|
182
|
-
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}"
|
183
|
-
end
|
131
|
+
return if @lxc_details
|
132
|
+
|
133
|
+
# We couldn't find an existing LXC container for this node/environment.
|
134
|
+
# We have to create one.
|
135
|
+
# Get the image name for this node
|
136
|
+
image = @nodes_handler.get_image_of(@node).to_sym
|
137
|
+
# Find if we have such an image registered
|
138
|
+
raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}" unless @config.known_os_images.include?(image)
|
139
|
+
|
140
|
+
proxmox_conf = "#{@config.os_image_dir(image)}/proxmox.json"
|
141
|
+
raise "[ #{@node}/#{@environment} ] - No Proxmox configuration found at #{proxmox_conf}" unless File.exist?(proxmox_conf)
|
142
|
+
|
143
|
+
pve_template = JSON.parse(File.read(proxmox_conf))['template']
|
144
|
+
raise "[ #{@node}/#{@environment} ] - No template found in #{proxmox_conf}" unless pve_template
|
145
|
+
|
146
|
+
# Query the inventory to know about minimum resources needed to deploy the node.
|
147
|
+
# Provide default values if they are not part of the metadata.
|
148
|
+
min_resources_to_deploy = {
|
149
|
+
cpus: 2,
|
150
|
+
ram_mb: 1024,
|
151
|
+
disk_gb: 10
|
152
|
+
}.merge(@nodes_handler.get_deploy_resources_min_of(@node) || {})
|
153
|
+
# Create the Proxmox container from the sync node.
|
154
|
+
vm_config = proxmox_test_info[:vm_config]
|
155
|
+
# Hostname in Proxmox is capped at 65 chars.
|
156
|
+
# Make sure we don't get over it, but still use a unique one.
|
157
|
+
hostname = "#{@node}.#{@environment}.hpc-test.com"
|
158
|
+
if hostname.size > MAX_PROXMOX_HOSTNAME_SIZE
|
159
|
+
# Truncate it, but add a unique ID in it.
|
160
|
+
# In the end the hostname looks like:
|
161
|
+
# <truncated_node_environment>.<unique_id>.hpc-test.com
|
162
|
+
hostname = "-#{Digest::MD5.hexdigest(hostname)[0..7]}.hpc-test.com"
|
163
|
+
hostname = "#{@node}.#{@environment}"[0..MAX_PROXMOX_HOSTNAME_SIZE - hostname.size - 1] + hostname
|
184
164
|
end
|
165
|
+
@lxc_details = request_lxc_creation_for(
|
166
|
+
ostemplate: pve_template,
|
167
|
+
hostname: hostname.gsub('_', '-'),
|
168
|
+
cores: min_resources_to_deploy[:cpus],
|
169
|
+
cpulimit: min_resources_to_deploy[:cpus],
|
170
|
+
memory: min_resources_to_deploy[:ram_mb],
|
171
|
+
rootfs: "local-lvm:#{min_resources_to_deploy[:disk_gb]}",
|
172
|
+
nameserver: vm_config[:vm_dns_servers].join(' '),
|
173
|
+
searchdomain: vm_config[:vm_search_domain],
|
174
|
+
net0: "name=eth0,bridge=vmbr0,gw=#{vm_config[:vm_gateway]}",
|
175
|
+
password: 'root_pwd',
|
176
|
+
description: <<~EO_DESCRIPTION
|
177
|
+
===== HPC info =====
|
178
|
+
node: #{@node}
|
179
|
+
environment: #{@environment}
|
180
|
+
debug: #{log_debug? ? 'true' : 'false'}
|
181
|
+
EO_DESCRIPTION
|
182
|
+
)
|
185
183
|
end
|
186
184
|
|
187
185
|
# Start an instance
|
@@ -274,6 +272,7 @@ module HybridPlatformsConductor
|
|
274
272
|
def with_proxmox
|
275
273
|
url = proxmox_test_info[:api_url]
|
276
274
|
raise 'No Proxmox server defined' if url.nil?
|
275
|
+
|
277
276
|
Credentials.with_credentials_for(:proxmox, @logger, @logger_stderr, url: url) do |user, password|
|
278
277
|
log_debug "[ #{@node}/#{@environment} ] - Connect to Proxmox #{url}"
|
279
278
|
proxmox_logs = StringIO.new
|
@@ -313,9 +312,11 @@ module HybridPlatformsConductor
|
|
313
312
|
idx_try = 0
|
314
313
|
loop do
|
315
314
|
response = proxmox.get(path)
|
316
|
-
break if !
|
315
|
+
break if !response.is_a?(String) || response !~ /^NOK: error code = 5\d\d$/
|
316
|
+
|
317
317
|
log_warn "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returned error #{response} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
|
318
318
|
raise "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returns #{response} continuously (tried #{idx_try + 1} times)" if idx_try >= proxmox_test_info[:api_max_retries]
|
319
|
+
|
319
320
|
idx_try += 1
|
320
321
|
# We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
|
321
322
|
proxmox.reauthenticate
|
@@ -336,23 +337,22 @@ module HybridPlatformsConductor
|
|
336
337
|
def run_proxmox_task(proxmox, http_method, pve_node, sub_path, *args)
|
337
338
|
task = nil
|
338
339
|
idx_try = 0
|
339
|
-
while task.nil?
|
340
|
+
while task.nil?
|
340
341
|
task = proxmox.send(http_method, "nodes/#{pve_node}/#{sub_path}", *args)
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
if task.nil?
|
352
|
-
raise "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} is constantly failing. Giving up."
|
353
|
-
else
|
354
|
-
wait_for_proxmox_task(proxmox, pve_node, task)
|
342
|
+
next unless task =~ /^NOK: error code = 5\d\d$/
|
343
|
+
|
344
|
+
log_warn "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} returned error #{task} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
|
345
|
+
task = nil
|
346
|
+
break if idx_try >= proxmox_test_info[:api_max_retries]
|
347
|
+
|
348
|
+
idx_try += 1
|
349
|
+
# We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
|
350
|
+
proxmox.reauthenticate
|
351
|
+
sleep proxmox_test_info[:api_wait_between_retries_secs] + rand(5)
|
355
352
|
end
|
353
|
+
raise "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} is constantly failing. Giving up." if task.nil?
|
354
|
+
|
355
|
+
wait_for_proxmox_task(proxmox, pve_node, task)
|
356
356
|
end
|
357
357
|
|
358
358
|
# Wait for a given Proxmox task completion
|
@@ -363,18 +363,18 @@ module HybridPlatformsConductor
|
|
363
363
|
# * *task* (String): The task ID
|
364
364
|
def wait_for_proxmox_task(proxmox, pve_node, task)
|
365
365
|
raise "Invalid task: #{task}" if task[0..3] == 'NOK:'
|
366
|
+
|
366
367
|
status = nil
|
367
368
|
loop do
|
368
369
|
status = task_status(proxmox, pve_node, task)
|
369
370
|
break unless status == 'running'
|
371
|
+
|
370
372
|
log_debug "[ #{@node}/#{@environment} ] - Wait for Proxmox task #{task} to complete..."
|
371
373
|
sleep 1
|
372
374
|
end
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
raise "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed with status #{status}"
|
377
|
-
end
|
375
|
+
raise "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed with status #{status}" unless status.split(':').last == 'OK'
|
376
|
+
|
377
|
+
log_debug "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed."
|
378
378
|
end
|
379
379
|
|
380
380
|
# Get task status
|
@@ -402,13 +402,13 @@ module HybridPlatformsConductor
|
|
402
402
|
config_file = "#{Dir.tmpdir}/config_#{file_id}.json"
|
403
403
|
File.write(
|
404
404
|
config_file,
|
405
|
-
|
405
|
+
proxmox_test_info[:test_config].merge(
|
406
406
|
proxmox_api_url: proxmox_test_info[:api_url],
|
407
407
|
futex_file: '/tmp/hpc_proxmox_allocations.futex',
|
408
408
|
logs_dir: '/tmp/hpc_proxmox_waiter_logs',
|
409
409
|
api_max_retries: proxmox_test_info[:api_max_retries],
|
410
410
|
api_wait_between_retries_secs: proxmox_test_info[:api_wait_between_retries_secs]
|
411
|
-
)
|
411
|
+
).to_json
|
412
412
|
)
|
413
413
|
result = nil
|
414
414
|
begin
|
@@ -447,7 +447,7 @@ module HybridPlatformsConductor
|
|
447
447
|
)[proxmox_test_info[:sync_node]]
|
448
448
|
end
|
449
449
|
stdout_lines = stdout.split("\n")
|
450
|
-
result = JSON.parse(stdout_lines[stdout_lines.index('===== JSON =====') + 1
|
450
|
+
result = JSON.parse(stdout_lines[stdout_lines.index('===== JSON =====') + 1..].join("\n")).transform_keys(&:to_sym)
|
451
451
|
raise "[ #{@node}/#{@environment} ] - Error returned by #{cmd}: #{result[:error]}" if result.key?(:error)
|
452
452
|
ensure
|
453
453
|
File.unlink(config_file)
|
@@ -548,6 +548,7 @@ module HybridPlatformsConductor
|
|
548
548
|
def proxmox_test_info
|
549
549
|
@config.proxmox_servers.first
|
550
550
|
end
|
551
|
+
|
551
552
|
end
|
552
553
|
|
553
554
|
end
|