hybrid_platforms_conductor 32.17.0 → 33.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -0
- data/README.md +6 -3
- data/bin/check-node +0 -1
- data/bin/deploy +0 -1
- data/bin/get_impacted_nodes +0 -1
- data/bin/last_deploys +12 -8
- data/bin/nodes_to_deploy +6 -6
- data/bin/setup +6 -6
- data/bin/topograph +1 -1
- data/docs/config_dsl.md +45 -1
- data/docs/executables.md +6 -7
- data/docs/executables/check-node.md +3 -3
- data/docs/executables/deploy.md +3 -3
- data/docs/executables/dump_nodes_json.md +3 -3
- data/docs/executables/test.md +3 -3
- data/docs/executables/topograph.md +3 -3
- data/docs/gen/mermaid/README.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
- data/docs/plugins.md +46 -0
- data/docs/plugins/connector/ssh.md +1 -1
- data/docs/plugins/log/remote_fs.md +26 -0
- data/docs/plugins/secrets_reader/cli.md +31 -0
- data/docs/plugins/secrets_reader/thycotic.md +46 -0
- data/docs/plugins/test/check_deploy_and_idempotence.md +1 -1
- data/docs/plugins/test/connection.md +1 -0
- data/docs/plugins/test/deploy_removes_root_access.md +1 -1
- data/docs/plugins/test/file_system.md +1 -0
- data/docs/plugins/test/hostname.md +1 -0
- data/docs/plugins/test/ip.md +1 -0
- data/docs/plugins/test/local_users.md +1 -0
- data/docs/plugins/test/mounts.md +1 -0
- data/docs/plugins/test/orphan_files.md +1 -0
- data/docs/plugins/test/ports.md +1 -0
- data/docs/plugins/test/spectre.md +1 -0
- data/docs/plugins/test/vulnerabilities.md +1 -0
- data/lib/hybrid_platforms_conductor/action.rb +4 -4
- data/lib/hybrid_platforms_conductor/actions_executor.rb +45 -43
- data/lib/hybrid_platforms_conductor/bitbucket.rb +5 -4
- data/lib/hybrid_platforms_conductor/cmd_runner.rb +13 -12
- data/lib/hybrid_platforms_conductor/cmdb.rb +2 -2
- data/lib/hybrid_platforms_conductor/common_config_dsl/bitbucket.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/confluence.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +5 -4
- data/lib/hybrid_platforms_conductor/common_config_dsl/github.rb +4 -3
- data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb +2 -2
- data/lib/hybrid_platforms_conductor/config.rb +8 -4
- data/lib/hybrid_platforms_conductor/confluence.rb +1 -1
- data/lib/hybrid_platforms_conductor/connector.rb +5 -2
- data/lib/hybrid_platforms_conductor/core_extensions/cleanroom/fix_kwargs.rb +116 -0
- data/lib/hybrid_platforms_conductor/core_extensions/symbol/zero.rb +24 -0
- data/lib/hybrid_platforms_conductor/credentials.rb +39 -36
- data/lib/hybrid_platforms_conductor/current_dir_monitor.rb +4 -1
- data/lib/hybrid_platforms_conductor/deployer.rb +275 -224
- data/lib/hybrid_platforms_conductor/executable.rb +20 -15
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/config.rb +10 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +4 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb +2 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +45 -49
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/my_log_plugin.rb.sample +100 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb +180 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +68 -66
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb +13 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb +39 -38
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb +5 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb +43 -45
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/podman.rb +18 -20
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +118 -117
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +39 -43
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container +9 -13
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +28 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/stdout.rb +26 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/cli.rb +77 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/my_secrets_reader_plugin.rb.sample +46 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb +90 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb +4 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb +3 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +7 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb +21 -19
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +21 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +19 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb +2 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +3 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +4 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb +21 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb +10 -12
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +9 -9
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +13 -10
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +7 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +27 -25
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/stdout.rb +8 -6
- data/lib/hybrid_platforms_conductor/io_router.rb +14 -13
- data/lib/hybrid_platforms_conductor/json_dumper.rb +2 -2
- data/lib/hybrid_platforms_conductor/log.rb +31 -0
- data/lib/hybrid_platforms_conductor/logger_helpers.rb +19 -16
- data/lib/hybrid_platforms_conductor/nodes_handler.rb +89 -71
- data/lib/hybrid_platforms_conductor/parallel_threads.rb +7 -11
- data/lib/hybrid_platforms_conductor/platform_handler.rb +7 -7
- data/lib/hybrid_platforms_conductor/platforms_handler.rb +5 -3
- data/lib/hybrid_platforms_conductor/plugin.rb +2 -2
- data/lib/hybrid_platforms_conductor/plugins.rb +14 -8
- data/lib/hybrid_platforms_conductor/provisioner.rb +4 -4
- data/lib/hybrid_platforms_conductor/report.rb +2 -2
- data/lib/hybrid_platforms_conductor/reports_handler.rb +3 -2
- data/lib/hybrid_platforms_conductor/secrets_reader.rb +31 -0
- data/lib/hybrid_platforms_conductor/services_handler.rb +32 -29
- data/lib/hybrid_platforms_conductor/test_only_remote_node.rb +18 -0
- data/lib/hybrid_platforms_conductor/test_report.rb +15 -18
- data/lib/hybrid_platforms_conductor/tests_runner.rb +116 -118
- data/lib/hybrid_platforms_conductor/thycotic.rb +28 -19
- data/lib/hybrid_platforms_conductor/topographer.rb +200 -190
- data/lib/hybrid_platforms_conductor/topographer/plugins/graphviz.rb +8 -8
- data/lib/hybrid_platforms_conductor/topographer/plugins/json.rb +4 -4
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test.rb +33 -12
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +18 -11
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/interactive_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb +32 -21
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +75 -49
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/scp_spec.rb +27 -15
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions_spec.rb +90 -59
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connection_spec.rb +46 -44
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/connectable_nodes_spec.rb +12 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb +4 -7
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +21 -22
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +23 -24
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connectable_nodes_spec.rb +10 -6
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +106 -75
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +145 -126
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +29 -25
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/logging_spec.rb +167 -142
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/parallel_spec.rb +272 -244
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +16 -16
- data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +36 -36
- data/spec/hybrid_platforms_conductor_test/api/config_spec.rb +24 -22
- data/spec/hybrid_platforms_conductor_test/api/deployer/check_spec.rb +4 -2
- data/spec/hybrid_platforms_conductor_test/api/deployer/config_dsl_spec.rb +43 -5
- data/spec/hybrid_platforms_conductor_test/api/deployer/deploy_spec.rb +199 -216
- data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +223 -0
- data/spec/hybrid_platforms_conductor_test/api/deployer/parse_deploy_output_spec.rb +55 -59
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +36 -62
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/podman_spec.rb +17 -17
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/config_dsl_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/create_spec.rb +44 -51
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/destroy_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/ip_spec.rb +12 -16
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/destroy_vm_spec.rb +31 -19
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +324 -266
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/ips_assignment_spec.rb +89 -61
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/other_lxc_containers_resources_spec.rb +117 -93
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/pve_node_resources_spec.rb +71 -54
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +10 -8
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb +80 -60
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/start_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/stop_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/cli_spec.rb +64 -0
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/thycotic_spec.rb +268 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/config_spec.rb +8 -10
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +33 -24
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_keys_spec.rb +64 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/platform_handlers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +50 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +91 -81
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +14 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/git_diff_impacts_spec.rb +51 -75
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/nodes_selectors_spec.rb +35 -26
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/platform_handlers_plugins_api_spec.rb +24 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/several_platforms_spec.rb +29 -19
- data/spec/hybrid_platforms_conductor_test/api/platform_handler_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb +57 -99
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +32 -35
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +10 -10
- data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +38 -27
- data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +46 -52
- data/spec/hybrid_platforms_conductor_test/api/reports_handler_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/services_handler/actions_to_deploy_spec.rb +90 -58
- data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +38 -34
- data/spec/hybrid_platforms_conductor_test/api/services_handler/log_info_spec.rb +11 -9
- data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +193 -171
- data/spec/hybrid_platforms_conductor_test/api/services_handler/parse_deploy_output_spec.rb +66 -54
- data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +147 -133
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +69 -49
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/global_spec.rb +5 -4
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +30 -27
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/platform_spec.rb +12 -9
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +48 -47
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/bitbucket_conf_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/github_ci_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_reports_plugins/confluence_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +9 -9
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb2.rb → test_cmdb_2.rb} +6 -6
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb_others2.rb → test_cmdb_others_2.rb} +2 -2
- data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/{check-node_spec.rb → check_node_spec.rb} +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/get_impacted_nodes_spec.rb +76 -77
- data/spec/hybrid_platforms_conductor_test/executables/last_deploys_spec.rb +159 -113
- data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +299 -160
- data/spec/hybrid_platforms_conductor_test/executables/options/actions_executor_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +12 -196
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_selectors_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/reports_handler_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +22 -22
- data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +22 -16
- data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +32 -32
- data/spec/hybrid_platforms_conductor_test/executables/ssh_config_spec.rb +7 -9
- data/spec/hybrid_platforms_conductor_test/executables/test_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +4 -3
- data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/config_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +12 -13
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +245 -56
- data/spec/hybrid_platforms_conductor_test/helpers/executables_helpers.rb +11 -11
- data/spec/hybrid_platforms_conductor_test/helpers/nodes_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +39 -28
- data/spec/hybrid_platforms_conductor_test/helpers/plugins_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +86 -111
- data/spec/hybrid_platforms_conductor_test/helpers/reports_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/helpers/services_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/tests_runner_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id1.rb → test_plugin_id_1.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id2.rb → test_plugin_id_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/test_plugin_type/{test_plugin_id3.rb → test_plugin_id_3.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/{test_plugin_type2/test_plugin_id4.rb → test_plugin_type_2/test_plugin_id_4.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/test.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/{test2.rb → test_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/rubocop_spec.rb +31 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +952 -0
- data/spec/hybrid_platforms_conductor_test/test_connector.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/test_log_no_read_plugin.rb +84 -0
- data/spec/hybrid_platforms_conductor_test/test_log_plugin.rb +105 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/global.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node_check.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/platform.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/several_checks.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/test_secrets_reader_plugin.rb +45 -0
- data/spec/hybrid_platforms_conductor_test/tests_report_plugin.rb +5 -6
- data/spec/spec_helper.rb +17 -18
- data/tools/check_md +16 -20
- data/tools/generate_mermaid +1 -1
- metadata +195 -144
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +0 -916
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require 'nokogiri'
|
|
2
|
+
require 'hybrid_platforms_conductor/test_only_remote_node'
|
|
2
3
|
|
|
3
4
|
module HybridPlatformsConductor
|
|
4
5
|
|
|
@@ -17,7 +18,7 @@ module HybridPlatformsConductor
|
|
|
17
18
|
# Each final OVAL URL can be directly an XML file, either raw or compressed with .gz or .bz2.
|
|
18
19
|
# This is useful to follow repository links, such as jFrog or web servers serving common file systems structure storing several versions of the OVAL file.
|
|
19
20
|
# * *reported_severities* (Array<String> or nil): List of severities to report, if any (use Unknown when the severity is not known), or nil for all [default: nil]
|
|
20
|
-
class Vulnerabilities <
|
|
21
|
+
class Vulnerabilities < TestOnlyRemoteNode
|
|
21
22
|
|
|
22
23
|
# Known compression methods, per file extension, and their corresponding uncompress bash script
|
|
23
24
|
KNOWN_COMPRESSIONS = {
|
|
@@ -47,8 +48,9 @@ module HybridPlatformsConductor
|
|
|
47
48
|
# Follow the last link recursively until we find a .xml or compressed file
|
|
48
49
|
current_url = artifactory_url
|
|
49
50
|
loop do
|
|
50
|
-
current_url = "#{current_url}#{current_url.end_with?('/') ? '' : '/'}#{Nokogiri::HTML.parse(URI.
|
|
51
|
+
current_url = "#{current_url}#{current_url.end_with?('/') ? '' : '/'}#{Nokogiri::HTML.parse(URI.parse(current_url).open).css('a').last['href']}"
|
|
51
52
|
break if current_url.end_with?('.xml') || KNOWN_COMPRESSIONS.keys.any? { |file_ext| current_url.end_with?(".#{file_ext}") }
|
|
53
|
+
|
|
52
54
|
log_debug "Follow last link to #{current_url}"
|
|
53
55
|
end
|
|
54
56
|
current_url
|
|
@@ -56,7 +58,7 @@ module HybridPlatformsConductor
|
|
|
56
58
|
)
|
|
57
59
|
# TODO: Access the user correctly when the user notion will be moved out of the ssh connector
|
|
58
60
|
sudo = @deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "
|
|
59
|
-
|
|
61
|
+
urls.map do |url|
|
|
60
62
|
# 1. Get the OVAL file on the node to be tested (uncompress it if needed)
|
|
61
63
|
# 2. Make sure oscap is installed
|
|
62
64
|
# 3. Generate the report for this OVAL file using oscap
|
|
@@ -66,13 +68,13 @@ module HybridPlatformsConductor
|
|
|
66
68
|
packages_to_install = []
|
|
67
69
|
KNOWN_COMPRESSIONS.each do |file_ext, compress_info|
|
|
68
70
|
file_ending = ".#{file_ext}"
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
71
|
+
next unless local_oval_file.end_with?(file_ending)
|
|
72
|
+
|
|
73
|
+
uncompress_cmds << compress_info[:cmd].call(local_oval_file)
|
|
74
|
+
packages_to_install.concat(compress_info[:packages])
|
|
75
|
+
local_oval_file = File.basename(local_oval_file, file_ending)
|
|
74
76
|
end
|
|
75
|
-
cmds = <<~
|
|
77
|
+
cmds = <<~EO_BASH
|
|
76
78
|
set -e
|
|
77
79
|
#{
|
|
78
80
|
case image
|
|
@@ -84,7 +86,7 @@ module HybridPlatformsConductor
|
|
|
84
86
|
# On Debian 10 we have to compile it from sources, as the packaged official version has core dumps.
|
|
85
87
|
# cf https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1688223.html
|
|
86
88
|
# TODO: Remove this Debian 10 specificity when the official libopenscap8 will be corrected
|
|
87
|
-
<<~
|
|
89
|
+
<<~EO_BASH2
|
|
88
90
|
if [ ! -x "$(command -v oscap)" ] || [ "$(oscap --version | head -n 1 | awk '{print $6}')" != "1.3.4" ]; then
|
|
89
91
|
rm -rf openscap
|
|
90
92
|
git clone --recurse-submodules https://github.com/OpenSCAP/openscap.git
|
|
@@ -96,7 +98,7 @@ module HybridPlatformsConductor
|
|
|
96
98
|
#{sudo}make install
|
|
97
99
|
fi
|
|
98
100
|
#{sudo}apt install -y wget #{packages_to_install.join(' ')}
|
|
99
|
-
|
|
101
|
+
EO_BASH2
|
|
100
102
|
else
|
|
101
103
|
raise "Non supported image: #{image}. Please adapt this test's code."
|
|
102
104
|
end
|
|
@@ -110,7 +112,7 @@ module HybridPlatformsConductor
|
|
|
110
112
|
echo "===== RESULTS ====="
|
|
111
113
|
cat "#{local_oval_file}.results.xml"
|
|
112
114
|
cd ..
|
|
113
|
-
|
|
115
|
+
EO_BASH
|
|
114
116
|
[
|
|
115
117
|
cmds,
|
|
116
118
|
{
|
|
@@ -119,21 +121,21 @@ module HybridPlatformsConductor
|
|
|
119
121
|
if idx_results.nil?
|
|
120
122
|
error 'No results given by the oscap run', stdout.join("\n")
|
|
121
123
|
else
|
|
122
|
-
results = Nokogiri::XML(stdout[idx_results + 1
|
|
124
|
+
results = Nokogiri::XML(stdout[idx_results + 1..].join("\n"))
|
|
123
125
|
results.remove_namespaces!
|
|
124
126
|
oval_definitions = results.css('oval_results oval_definitions definitions definition')
|
|
125
127
|
results.css('results system definitions definition').each do |definition_xml|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
128
|
+
next unless definition_xml['result'] == 'true'
|
|
129
|
+
|
|
130
|
+
# Just found an OVAL item to be patched.
|
|
131
|
+
definition_id = definition_xml['definition_id']
|
|
132
|
+
oval_definition = oval_definitions.find { |el| el['id'] == definition_id }
|
|
133
|
+
# We don't forcefully want to report all missing patches. Only the most important ones.
|
|
134
|
+
severity = oval_definition.css('metadata advisory severity').text
|
|
135
|
+
severity = 'Unknown' if severity.empty?
|
|
136
|
+
if !oval_info.key?('reported_severities') || oval_info['reported_severities'].include?(severity)
|
|
137
|
+
# Only consider the first line of the description, as sometimes it's very long
|
|
138
|
+
error "Non-patched #{severity} vulnerability found: #{oval_definition.css('metadata title').text} - #{oval_definition.css('metadata description').text.split("\n").first}"
|
|
137
139
|
end
|
|
138
140
|
end
|
|
139
141
|
end
|
|
@@ -142,7 +144,7 @@ module HybridPlatformsConductor
|
|
|
142
144
|
timeout: 240
|
|
143
145
|
}
|
|
144
146
|
]
|
|
145
|
-
end
|
|
147
|
+
end.to_h
|
|
146
148
|
else
|
|
147
149
|
error "No OVAL file defined for image #{image} at #{oval_file}"
|
|
148
150
|
{}
|
|
@@ -12,7 +12,7 @@ module HybridPlatformsConductor
|
|
|
12
12
|
# Report tests results on a generated Confluence page
|
|
13
13
|
class Confluence < HybridPlatformsConductor::TestReport
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
extend_config_dsl_with CommonConfigDsl::Confluence, :init_confluence
|
|
16
16
|
|
|
17
17
|
# Maximum errors to be reported by item
|
|
18
18
|
MAX_ERROR_ITEMS_DISPLAYED = 10
|
|
@@ -34,7 +34,7 @@ module HybridPlatformsConductor
|
|
|
34
34
|
at('h1:contains("Evolution")').
|
|
35
35
|
search('~ structured-macro:first-of-type').
|
|
36
36
|
css('table td').
|
|
37
|
-
map
|
|
37
|
+
map(&:text).
|
|
38
38
|
each_slice(2).
|
|
39
39
|
to_a.
|
|
40
40
|
map { |(time_str, value_str)| [Time.parse("#{time_str} UTC"), value_str.to_f] }
|
|
@@ -97,18 +97,20 @@ module HybridPlatformsConductor
|
|
|
97
97
|
table << [
|
|
98
98
|
nodes_list,
|
|
99
99
|
nodes_info[:nodes].size,
|
|
100
|
-
nodes_info[:nodes].empty? ? '' : "#{(nodes_info[:tested_nodes].size*100.0/nodes_info[:nodes].size).to_i} %",
|
|
100
|
+
nodes_info[:nodes].empty? ? '' : "#{(nodes_info[:tested_nodes].size * 100.0 / nodes_info[:nodes].size).to_i} %",
|
|
101
101
|
nodes_info[:tested_nodes].empty? ? '' : "#{((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error_as_expected].size) * 100.0 / nodes_info[:tested_nodes].size).to_i} %",
|
|
102
102
|
nodes_info[:tested_nodes].empty? ? '' : "#{((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * 100.0 / nodes_info[:tested_nodes].size).to_i} %",
|
|
103
|
-
nodes_info[:nodes].empty?
|
|
103
|
+
if nodes_info[:nodes].empty?
|
|
104
|
+
''
|
|
105
|
+
else
|
|
104
106
|
('=' * ((nodes_info[:tested_nodes_in_error_as_expected].size * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).yellow.bold +
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
107
|
+
('=' * (((nodes_info[:tested_nodes_in_error].size - nodes_info[:tested_nodes_in_error_as_expected].size).abs * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).red.bold +
|
|
108
|
+
('=' * (((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).green.bold +
|
|
109
|
+
('=' * (((nodes_info[:nodes].size - nodes_info[:tested_nodes].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).white.bold
|
|
110
|
+
end
|
|
108
111
|
]
|
|
109
112
|
end
|
|
110
113
|
end)
|
|
111
|
-
|
|
112
114
|
end
|
|
113
115
|
|
|
114
116
|
end
|
|
@@ -30,6 +30,7 @@ module HybridPlatformsConductor
|
|
|
30
30
|
# Start routing messages asynchronously
|
|
31
31
|
def start
|
|
32
32
|
raise 'IO router is already started. Can\'t start it again.' unless @reading_thread.nil?
|
|
33
|
+
|
|
33
34
|
@end_read = false
|
|
34
35
|
# Create a thread to handle routes asynchronously
|
|
35
36
|
@reading_thread = Thread.new do
|
|
@@ -37,22 +38,21 @@ module HybridPlatformsConductor
|
|
|
37
38
|
need_to_stop = @end_read.clone
|
|
38
39
|
data_found = false
|
|
39
40
|
@routes.each do |src_io, dst_ios|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
else
|
|
52
|
-
raise "Unknown type of source IO: #{src_io}"
|
|
41
|
+
raise "Unknown type of source IO: #{src_io}" unless src_io.is_a?(Queue)
|
|
42
|
+
|
|
43
|
+
queue_size = src_io.size
|
|
44
|
+
next unless queue_size.positive?
|
|
45
|
+
|
|
46
|
+
# There is data to be read from src_io
|
|
47
|
+
data_found = true
|
|
48
|
+
data_chunk_str = queue_size.times.map { src_io.pop }.join
|
|
49
|
+
dst_ios.each do |dst_io|
|
|
50
|
+
dst_io << data_chunk_str
|
|
51
|
+
dst_io.flush if dst_io.respond_to?(:flush)
|
|
53
52
|
end
|
|
54
53
|
end
|
|
55
54
|
break if need_to_stop && !data_found
|
|
55
|
+
|
|
56
56
|
sleep 0.1
|
|
57
57
|
end
|
|
58
58
|
end
|
|
@@ -61,6 +61,7 @@ module HybridPlatformsConductor
|
|
|
61
61
|
# Stop routing messages asynchronously
|
|
62
62
|
def stop
|
|
63
63
|
raise 'IO router is not started. Can\'t stop it.' if @reading_thread.nil?
|
|
64
|
+
|
|
64
65
|
@end_read = true
|
|
65
66
|
@reading_thread.join
|
|
66
67
|
end
|
|
@@ -23,7 +23,7 @@ module HybridPlatformsConductor
|
|
|
23
23
|
# * *config* (Config): Config to be used. [default = Config.new]
|
|
24
24
|
# * *nodes_handler* (NodesHandler): Nodes handler to be used. [default = NodesHandler.new]
|
|
25
25
|
# * *deployer* (Deployer): Deployer to be used. [default = Deployer.new]
|
|
26
|
-
def initialize(logger: Logger.new(
|
|
26
|
+
def initialize(logger: Logger.new($stdout), logger_stderr: Logger.new($stderr), config: Config.new, nodes_handler: NodesHandler.new, deployer: Deployer.new)
|
|
27
27
|
init_loggers(logger, logger_stderr)
|
|
28
28
|
@config = config
|
|
29
29
|
@nodes_handler = nodes_handler
|
|
@@ -74,7 +74,7 @@ module HybridPlatformsConductor
|
|
|
74
74
|
out "[ #{node} ] - Error while dumping JSON. Check #{stdout_file_name}"
|
|
75
75
|
else
|
|
76
76
|
json_file_name = "#{@dump_dir}/#{node}.json"
|
|
77
|
-
File.write(json_file_name, stdout[dump_begin_idx+1..dump_end_idx-1].join("\n"))
|
|
77
|
+
File.write(json_file_name, stdout[dump_begin_idx + 1..dump_end_idx - 1].join("\n"))
|
|
78
78
|
out "[ #{node} ] - OK. Check #{json_file_name}"
|
|
79
79
|
end
|
|
80
80
|
else
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
require 'hybrid_platforms_conductor/logger_helpers'
|
|
2
|
+
require 'hybrid_platforms_conductor/plugin'
|
|
3
|
+
|
|
4
|
+
module HybridPlatformsConductor
|
|
5
|
+
|
|
6
|
+
# Ancestor of all log plugins
|
|
7
|
+
class Log < Plugin
|
|
8
|
+
|
|
9
|
+
# Constructor
|
|
10
|
+
#
|
|
11
|
+
# Parameters::
|
|
12
|
+
# * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
|
|
13
|
+
# * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
|
|
14
|
+
# * *config* (Config): Config to be used. [default: Config.new]
|
|
15
|
+
# * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
|
|
16
|
+
# * *actions_executor* (ActionsExecutor): Actions executor to be used. [default: ActionsExecutor.new]
|
|
17
|
+
def initialize(
|
|
18
|
+
logger: Logger.new($stdout),
|
|
19
|
+
logger_stderr: Logger.new($stderr),
|
|
20
|
+
config: Config.new,
|
|
21
|
+
nodes_handler: NodesHandler.new,
|
|
22
|
+
actions_executor: ActionsExecutor.new
|
|
23
|
+
)
|
|
24
|
+
super(logger: logger, logger_stderr: logger_stderr, config: config)
|
|
25
|
+
@nodes_handler = nodes_handler
|
|
26
|
+
@actions_executor = actions_executor
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
end
|
|
@@ -55,25 +55,27 @@ module HybridPlatformsConductor
|
|
|
55
55
|
|
|
56
56
|
# Make sure if the current line is not flushed we still do it
|
|
57
57
|
def flush
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
return if @current_line.nil?
|
|
59
|
+
|
|
60
|
+
@stream << @current_line
|
|
61
|
+
@current_line = nil
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
end
|
|
65
65
|
|
|
66
66
|
class << self
|
|
67
|
+
|
|
67
68
|
attr_reader :progress_bar_semaphore
|
|
69
|
+
|
|
68
70
|
end
|
|
69
71
|
# Make sure the progress bar setting is protected by a Mutex
|
|
70
72
|
@progress_bar_semaphore = Mutex.new
|
|
71
73
|
|
|
72
74
|
# Sorted list of levels and their corresponding modifiers.
|
|
73
75
|
LEVELS_MODIFIERS = {
|
|
74
|
-
fatal: [
|
|
75
|
-
error: [
|
|
76
|
-
warn: [
|
|
76
|
+
fatal: %i[red bold],
|
|
77
|
+
error: %i[red bold],
|
|
78
|
+
warn: %i[yellow bold],
|
|
77
79
|
info: [:white],
|
|
78
80
|
debug: [:white],
|
|
79
81
|
unknown: [:white]
|
|
@@ -82,7 +84,7 @@ module HybridPlatformsConductor
|
|
|
82
84
|
# List of levels that will output on stderr
|
|
83
85
|
LEVELS_TO_STDERR = %i[warn error fatal]
|
|
84
86
|
|
|
85
|
-
LEVELS_MODIFIERS.
|
|
87
|
+
LEVELS_MODIFIERS.each_key do |level|
|
|
86
88
|
define_method("log_#{level}") do |message|
|
|
87
89
|
(LEVELS_TO_STDERR.include?(level) ? @logger_stderr : @logger).send(
|
|
88
90
|
level,
|
|
@@ -105,10 +107,10 @@ module HybridPlatformsConductor
|
|
|
105
107
|
# Set loggers to the desired format
|
|
106
108
|
def set_loggers_format
|
|
107
109
|
[@logger, @logger_stderr].each do |logger|
|
|
108
|
-
logger.formatter = proc do |severity,
|
|
110
|
+
logger.formatter = proc do |severity, _datetime, progname, msg|
|
|
109
111
|
# If the message already has control characters, don't colorize it
|
|
110
112
|
keep_original_color = msg.include? "\u001b"
|
|
111
|
-
message = "[#{Time.now.utc.strftime('%F %T')} (PID #{
|
|
113
|
+
message = "[#{Time.now.utc.strftime('%F %T')} (PID #{$PROCESS_ID} / TID #{Thread.current.object_id})] #{severity.rjust(5)} - [ #{progname} ] - "
|
|
112
114
|
message << "#{msg}\n" unless keep_original_color
|
|
113
115
|
LEVELS_MODIFIERS[severity.downcase.to_sym].each do |modifier|
|
|
114
116
|
message = message.send(modifier)
|
|
@@ -282,8 +284,8 @@ module HybridPlatformsConductor
|
|
|
282
284
|
yield progress_bar
|
|
283
285
|
ensure
|
|
284
286
|
LoggerHelpers.progress_bar_semaphore.synchronize do
|
|
285
|
-
|
|
286
|
-
|
|
287
|
+
stdout_device.flush
|
|
288
|
+
stderr_device.flush
|
|
287
289
|
self.stdout_device = previous_stdout_device unless previous_stdout_device.nil?
|
|
288
290
|
self.stderr_device = previous_stderr_device unless previous_stderr_device.nil?
|
|
289
291
|
end
|
|
@@ -298,15 +300,16 @@ module HybridPlatformsConductor
|
|
|
298
300
|
def stdouts_to_s
|
|
299
301
|
messages = []
|
|
300
302
|
{
|
|
301
|
-
'STDOUT' =>
|
|
302
|
-
'STDERR' =>
|
|
303
|
+
'STDOUT' => stdout_device,
|
|
304
|
+
'STDERR' => stderr_device
|
|
303
305
|
}.each do |name, device|
|
|
304
|
-
|
|
306
|
+
case device
|
|
307
|
+
when File
|
|
305
308
|
if File.exist?(device.path)
|
|
306
309
|
content = File.read(device.path).strip
|
|
307
310
|
messages << "----- #{name} BEGIN - #{device.path} -----\n#{content}\n----- #{name} END - #{device.path} -----" unless content.empty?
|
|
308
311
|
end
|
|
309
|
-
|
|
312
|
+
when StringIO
|
|
310
313
|
content = device.string
|
|
311
314
|
messages << "----- #{name} BEGIN -----\n#{content}\n----- #{name} END -----" unless content.empty?
|
|
312
315
|
end
|
|
@@ -40,7 +40,7 @@ module HybridPlatformsConductor
|
|
|
40
40
|
# * *master_cmdbs_info* (Hash< Symbol, Symbol or Array<Symbol> >): List of metadata properties (or single one) per CMDB name considered as master for those properties.
|
|
41
41
|
def master_cmdbs(master_cmdbs_info)
|
|
42
42
|
@cmdb_masters << {
|
|
43
|
-
cmdb_masters:
|
|
43
|
+
cmdb_masters: master_cmdbs_info.transform_values { |properties| properties.is_a?(Array) ? properties : [properties] },
|
|
44
44
|
nodes_selectors_stack: current_nodes_selectors_stack
|
|
45
45
|
}
|
|
46
46
|
end
|
|
@@ -60,7 +60,8 @@ module HybridPlatformsConductor
|
|
|
60
60
|
|
|
61
61
|
Config.extend_config_dsl_with ConfigDSLExtension, :init_nodes_handler_config
|
|
62
62
|
|
|
63
|
-
include
|
|
63
|
+
include ParallelThreads
|
|
64
|
+
include LoggerHelpers
|
|
64
65
|
|
|
65
66
|
class GitError < RuntimeError
|
|
66
67
|
end
|
|
@@ -74,8 +75,8 @@ module HybridPlatformsConductor
|
|
|
74
75
|
# * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
|
|
75
76
|
# * *platforms_handler* (PlatformsHandler): Platforms Handler to be used. [default: PlatformsHandler.new]
|
|
76
77
|
def initialize(
|
|
77
|
-
logger: Logger.new(
|
|
78
|
-
logger_stderr: Logger.new(
|
|
78
|
+
logger: Logger.new($stdout),
|
|
79
|
+
logger_stderr: Logger.new($stderr),
|
|
79
80
|
config: Config.new,
|
|
80
81
|
cmd_runner: CmdRunner.new,
|
|
81
82
|
platforms_handler: PlatformsHandler.new
|
|
@@ -111,11 +112,11 @@ module HybridPlatformsConductor
|
|
|
111
112
|
)
|
|
112
113
|
@cmdbs_others << cmdb if cmdb.respond_to?(:get_others)
|
|
113
114
|
cmdb.methods.each do |method|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
115
|
+
next unless method.to_s =~ /^get_(.*)$/
|
|
116
|
+
|
|
117
|
+
property = Regexp.last_match(1).to_sym
|
|
118
|
+
@cmdbs_per_property[property] = [] unless @cmdbs_per_property.key?(property)
|
|
119
|
+
@cmdbs_per_property[property] << cmdb
|
|
119
120
|
end
|
|
120
121
|
cmdb
|
|
121
122
|
end
|
|
@@ -133,13 +134,17 @@ module HybridPlatformsConductor
|
|
|
133
134
|
# Register all known nodes for this platform
|
|
134
135
|
platform.known_nodes.each do |node|
|
|
135
136
|
raise "Can't register #{node} to platform #{platform.repository_path}, as it is already defined in platform #{@nodes_platform[node].repository_path}." if @nodes_platform.key?(node)
|
|
137
|
+
|
|
136
138
|
@nodes_platform[node] = platform
|
|
137
139
|
end
|
|
138
140
|
# Register all known nodes lists
|
|
141
|
+
next unless platform.respond_to?(:known_nodes_lists)
|
|
142
|
+
|
|
139
143
|
platform.known_nodes_lists.each do |nodes_list|
|
|
140
144
|
raise "Can't register nodes list #{nodes_list} to platform #{platform.repository_path}, as it is already defined in platform #{@nodes_list_platform[nodes_list].repository_path}." if @nodes_list_platform.key?(nodes_list)
|
|
145
|
+
|
|
141
146
|
@nodes_list_platform[nodes_list] = platform
|
|
142
|
-
end
|
|
147
|
+
end
|
|
143
148
|
end
|
|
144
149
|
end
|
|
145
150
|
|
|
@@ -147,7 +152,7 @@ module HybridPlatformsConductor
|
|
|
147
152
|
#
|
|
148
153
|
# Parameters::
|
|
149
154
|
# * *options_parser* (OptionParser): The option parser to complete
|
|
150
|
-
def options_parse(options_parser
|
|
155
|
+
def options_parse(options_parser)
|
|
151
156
|
options_parser.separator ''
|
|
152
157
|
options_parser.separator 'Nodes handler options:'
|
|
153
158
|
options_parser.on('-o', '--show-nodes', 'Display the list of possible nodes and exit') do
|
|
@@ -221,6 +226,7 @@ module HybridPlatformsConductor
|
|
|
221
226
|
platform_name, from_commit, to_commit, flags = nodes_git_impact.split(':')
|
|
222
227
|
flags = (flags || '').split(',')
|
|
223
228
|
raise "Invalid platform in --nodes-git-impact: #{platform_name}. Possible values are: #{platform_names.join(', ')}." unless platform_names.include?(platform_name)
|
|
229
|
+
|
|
224
230
|
nodes_selector = { platform: platform_name }
|
|
225
231
|
nodes_selector[:from_commit] = from_commit if from_commit && !from_commit.empty?
|
|
226
232
|
nodes_selector[:to_commit] = to_commit if to_commit && !to_commit.empty?
|
|
@@ -329,7 +335,7 @@ module HybridPlatformsConductor
|
|
|
329
335
|
# * *block* (Proc): Code block given to the call
|
|
330
336
|
def method_missing(method, *args, &block)
|
|
331
337
|
if method.to_s =~ /^get_(.*)_of$/
|
|
332
|
-
property =
|
|
338
|
+
property = Regexp.last_match(1).to_sym
|
|
333
339
|
# Define the method so that we don't go trough method_missing next time (more efficient).
|
|
334
340
|
define_property_method_for(property)
|
|
335
341
|
# Then call it
|
|
@@ -341,6 +347,15 @@ module HybridPlatformsConductor
|
|
|
341
347
|
end
|
|
342
348
|
end
|
|
343
349
|
|
|
350
|
+
# Make sure we register the methods we handle in method_missing
|
|
351
|
+
#
|
|
352
|
+
# Parameters::
|
|
353
|
+
# * *name* (Symbol): The missing method name
|
|
354
|
+
# * *include_private* (Boolean): Should we include private methods in the search?
|
|
355
|
+
def respond_to_missing?(name, include_private)
|
|
356
|
+
name.to_s =~ /^get_(.*)_of$/ || super
|
|
357
|
+
end
|
|
358
|
+
|
|
344
359
|
# Prefetch some metadata properties for a given list of nodes.
|
|
345
360
|
# Useful for performance reasons when clients know they will need to use a lot of properties on nodes.
|
|
346
361
|
# Keep a thread-safe memory cache of it.
|
|
@@ -352,52 +367,51 @@ module HybridPlatformsConductor
|
|
|
352
367
|
(properties.is_a?(Symbol) ? [properties] : properties).each do |property|
|
|
353
368
|
# Gather the list of nodes missing this property
|
|
354
369
|
missing_nodes = nodes.select { |node| !@metadata.key?(node) || !@metadata[node].key?(property) }
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
370
|
+
next if missing_nodes.empty?
|
|
371
|
+
|
|
372
|
+
# Query the CMDBs having first the get_<property> method, then the ones having the get_others method till we have our property set for all missing nodes
|
|
373
|
+
# Metadata being retrieved by the different CMDBs, per node
|
|
374
|
+
# Hash< String, Object >
|
|
375
|
+
updated_metadata = {}
|
|
376
|
+
(
|
|
377
|
+
(@cmdbs_per_property.key?(property) ? @cmdbs_per_property[property] : []).map { |cmdb| [cmdb, property] } +
|
|
378
|
+
@cmdbs_others.map { |cmdb| [cmdb, :others] }
|
|
379
|
+
).each do |(cmdb, cmdb_property)|
|
|
380
|
+
# If among the missing nodes some of them have some master CMDB declared for this property, filter them out unless we are dealing with their master CMDB.
|
|
381
|
+
nodes_to_query = missing_nodes.select do |node|
|
|
382
|
+
master_cmdb = cmdb_master_for(node, property)
|
|
383
|
+
master_cmdb.nil? || master_cmdb == cmdb
|
|
384
|
+
end
|
|
385
|
+
next if nodes_to_query.empty?
|
|
386
|
+
|
|
387
|
+
# Check first if this property depends on other ones for this cmdb
|
|
388
|
+
if cmdb.respond_to?(:property_dependencies)
|
|
389
|
+
property_deps = cmdb.property_dependencies
|
|
390
|
+
prefetch_metadata_of nodes_to_query, property_deps[property] if property_deps.key?(property)
|
|
391
|
+
end
|
|
392
|
+
# Property values, per node name
|
|
358
393
|
# Hash< String, Object >
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
end
|
|
369
|
-
unless nodes_to_query.empty?
|
|
370
|
-
# Check first if this property depends on other ones for this cmdb
|
|
371
|
-
if cmdb.respond_to?(:property_dependencies)
|
|
372
|
-
property_deps = cmdb.property_dependencies
|
|
373
|
-
prefetch_metadata_of nodes_to_query, property_deps[property] if property_deps.key?(property)
|
|
374
|
-
end
|
|
375
|
-
# Property values, per node name
|
|
376
|
-
# Hash< String, Object >
|
|
377
|
-
metadata_from_cmdb = Hash[
|
|
378
|
-
cmdb.send("get_#{cmdb_property}".to_sym, nodes_to_query, @metadata.slice(*nodes_to_query)).map do |node, cmdb_result|
|
|
379
|
-
[node, cmdb_property == :others ? cmdb_result[property] : cmdb_result]
|
|
380
|
-
end
|
|
381
|
-
].compact
|
|
382
|
-
cmdb_log_header = "[CMDB #{cmdb.class.name.split('::').last}.#{cmdb_property}] -"
|
|
383
|
-
log_debug "#{cmdb_log_header} Query property #{property} for #{nodes_to_query.size} nodes (#{nodes_to_query[0..7].join(', ')}...) => Found metadata for #{metadata_from_cmdb.size} nodes."
|
|
384
|
-
updated_metadata.merge!(metadata_from_cmdb) do |node, existing_value, new_value|
|
|
385
|
-
raise "#{cmdb_log_header} Returned a conflicting value for metadata #{property} of node #{node}: #{new_value} whereas the value was already set to #{existing_value}" if !existing_value.nil? && new_value != existing_value
|
|
386
|
-
new_value
|
|
387
|
-
end
|
|
388
|
-
end
|
|
394
|
+
metadata_from_cmdb = cmdb.send("get_#{cmdb_property}".to_sym, nodes_to_query, @metadata.slice(*nodes_to_query)).transform_values do |cmdb_result|
|
|
395
|
+
cmdb_property == :others ? cmdb_result[property] : cmdb_result
|
|
396
|
+
end.compact
|
|
397
|
+
cmdb_log_header = "[CMDB #{cmdb.class.name.split('::').last}.#{cmdb_property}] -"
|
|
398
|
+
log_debug "#{cmdb_log_header} Query property #{property} for #{nodes_to_query.size} nodes (#{nodes_to_query[0..7].join(', ')}...) => Found metadata for #{metadata_from_cmdb.size} nodes."
|
|
399
|
+
updated_metadata.merge!(metadata_from_cmdb) do |node, existing_value, new_value|
|
|
400
|
+
raise "#{cmdb_log_header} Returned a conflicting value for metadata #{property} of node #{node}: #{new_value} whereas the value was already set to #{existing_value}" if !existing_value.nil? && new_value != existing_value
|
|
401
|
+
|
|
402
|
+
new_value
|
|
389
403
|
end
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
404
|
+
end
|
|
405
|
+
# Avoid conflicts in metadata while merging and make sure this update is thread-safe
|
|
406
|
+
# As @metadata is only appending data and never deleting it, protecting the update only is enough.
|
|
407
|
+
# At worst several threads will query several times the same CMDBs to update the same data several times.
|
|
408
|
+
# If we also want to be thread-safe in this regard, we should protect the whole CMDB call with mutexes, at the granularity of the node + property bein read.
|
|
409
|
+
@metadata_mutex.synchronize do
|
|
410
|
+
missing_nodes.each do |node|
|
|
411
|
+
@metadata[node] = {} unless @metadata.key?(node)
|
|
412
|
+
# Here, explicitely store nil if nothing has been found for a node because we know there is no value to be fetched.
|
|
413
|
+
# This way we won't query again all CMDBs thanks to the cache.
|
|
414
|
+
@metadata[node][property] = updated_metadata[node]
|
|
401
415
|
end
|
|
402
416
|
end
|
|
403
417
|
end
|
|
@@ -426,6 +440,7 @@ module HybridPlatformsConductor
|
|
|
426
440
|
nodes_selectors = nodes_selectors.flatten
|
|
427
441
|
# 1. Check for the presence of all
|
|
428
442
|
return known_nodes if nodes_selectors.any? { |nodes_selector| nodes_selector.is_a?(Hash) && nodes_selector.key?(:all) && nodes_selector[:all] }
|
|
443
|
+
|
|
429
444
|
# 2. Expand the nodes lists, platforms and services contents
|
|
430
445
|
string_nodes = []
|
|
431
446
|
nodes_selectors.each do |nodes_selector|
|
|
@@ -435,6 +450,7 @@ module HybridPlatformsConductor
|
|
|
435
450
|
if nodes_selector.key?(:list)
|
|
436
451
|
platform = @nodes_list_platform[nodes_selector[:list]]
|
|
437
452
|
raise "Unknown nodes list: #{nodes_selector[:list]}" if platform.nil?
|
|
453
|
+
|
|
438
454
|
string_nodes.concat(platform.nodes_selectors_from_nodes_list(nodes_selector[:list]))
|
|
439
455
|
end
|
|
440
456
|
string_nodes.concat(@platforms_handler.platform(nodes_selector[:platform]).known_nodes) if nodes_selector.key?(:platform)
|
|
@@ -462,8 +478,8 @@ module HybridPlatformsConductor
|
|
|
462
478
|
# 3. Expand the Regexps
|
|
463
479
|
real_nodes = []
|
|
464
480
|
string_nodes.each do |node|
|
|
465
|
-
if node =~
|
|
466
|
-
node_regexp = Regexp.new(
|
|
481
|
+
if node =~ %r{^/(.+)/$}
|
|
482
|
+
node_regexp = Regexp.new(Regexp.last_match(1))
|
|
467
483
|
real_nodes.concat(known_nodes.select { |known_node| known_node[node_regexp] })
|
|
468
484
|
else
|
|
469
485
|
real_nodes << node
|
|
@@ -489,13 +505,11 @@ module HybridPlatformsConductor
|
|
|
489
505
|
# * *parallel* (Boolean): Iterate in a multithreaded way? [default: false]
|
|
490
506
|
# * *nbr_threads_max* (Integer or nil): Maximum number of threads to be used in case of parallel, or nil for no limit [default: nil]
|
|
491
507
|
# * *progress* (String or nil): Name of a progress bar to follow the progression, or nil for no progress bar [default: 'Processing nodes']
|
|
492
|
-
# * Proc: The code called for each node being iterated on.
|
|
508
|
+
# * *block* (Proc): The code called for each node being iterated on.
|
|
493
509
|
# * Parameters::
|
|
494
510
|
# * *node* (String): The node name
|
|
495
|
-
def for_each_node_in(nodes, parallel: false, nbr_threads_max: nil, progress: 'Processing nodes')
|
|
496
|
-
for_each_element_in(nodes.sort, parallel: parallel, nbr_threads_max: nbr_threads_max, progress: progress)
|
|
497
|
-
yield node
|
|
498
|
-
end
|
|
511
|
+
def for_each_node_in(nodes, parallel: false, nbr_threads_max: nil, progress: 'Processing nodes', &block)
|
|
512
|
+
for_each_element_in(nodes.sort, parallel: parallel, nbr_threads_max: nbr_threads_max, progress: progress, &block)
|
|
499
513
|
end
|
|
500
514
|
|
|
501
515
|
# Get the list of impacted nodes from a git diff on a platform
|
|
@@ -513,10 +527,11 @@ module HybridPlatformsConductor
|
|
|
513
527
|
def impacted_nodes_from_git_diff(platform_name, from_commit: 'master', to_commit: nil, smallest_set: false)
|
|
514
528
|
platform = @platforms_handler.platform(platform_name)
|
|
515
529
|
raise "Unkown platform #{platform_name}. Possible platforms are #{@platforms_handler.known_platforms.map(&:name).sort.join(', ')}" if platform.nil?
|
|
530
|
+
|
|
516
531
|
begin
|
|
517
532
|
_exit_status, stdout, _stderr = @cmd_runner.run_cmd "cd #{platform.repository_path} && git --no-pager diff --no-color #{from_commit} #{to_commit.nil? ? '' : to_commit}", log_to_stdout: log_debug?
|
|
518
533
|
rescue CmdRunner::UnexpectedExitCodeError
|
|
519
|
-
raise GitError,
|
|
534
|
+
raise GitError, $ERROR_INFO.to_s
|
|
520
535
|
end
|
|
521
536
|
# Parse the git diff output to create a structured diff
|
|
522
537
|
# Hash< String, Hash< Symbol, Object > >: List of diffs info, per file name having a diff. Diffs info have the following properties:
|
|
@@ -526,9 +541,10 @@ module HybridPlatformsConductor
|
|
|
526
541
|
current_file_diff = nil
|
|
527
542
|
stdout.split("\n").each do |line|
|
|
528
543
|
case line
|
|
529
|
-
when
|
|
544
|
+
when %r{^diff --git a/(.+) b/(.+)$}
|
|
530
545
|
# A new file diff
|
|
531
|
-
from
|
|
546
|
+
from = Regexp.last_match(1)
|
|
547
|
+
to = Regexp.last_match(2)
|
|
532
548
|
current_file_diff = {
|
|
533
549
|
diff: ''
|
|
534
550
|
}
|
|
@@ -626,11 +642,13 @@ module HybridPlatformsConductor
|
|
|
626
642
|
cmdb_masters_cache = {}
|
|
627
643
|
select_confs_for_node(node, @config.cmdb_masters).each do |cmdb_masters_info|
|
|
628
644
|
cmdb_masters_info[:cmdb_masters].each do |cmdb, properties|
|
|
629
|
-
properties.each do |
|
|
630
|
-
raise "Property #{
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
645
|
+
properties.each do |itr_property|
|
|
646
|
+
raise "Property #{itr_property} have conflicting CMDB masters for #{node} declared in the configuration: #{cmdb_masters_cache[itr_property].class.name} and #{@cmdbs[cmdb].class.name}" if cmdb_masters_cache.key?(itr_property) && cmdb_masters_cache[itr_property] != @cmdbs[cmdb]
|
|
647
|
+
|
|
648
|
+
log_debug "CMDB master for #{node} / #{itr_property}: #{cmdb}"
|
|
649
|
+
raise "CMDB #{cmdb} is configured as a master for property #{itr_property} on node #{node} but it does not implement the needed API to retrieve it" unless (@cmdbs_per_property[itr_property] || []).include?(@cmdbs[cmdb]) || @cmdbs_others.include?(@cmdbs[cmdb])
|
|
650
|
+
|
|
651
|
+
cmdb_masters_cache[itr_property] = @cmdbs[cmdb]
|
|
634
652
|
end
|
|
635
653
|
end
|
|
636
654
|
end
|