hybrid_platforms_conductor 32.16.4 → 33.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +42 -0
- data/README.md +6 -3
- data/bin/check-node +0 -1
- data/bin/deploy +0 -1
- data/bin/get_impacted_nodes +0 -1
- data/bin/last_deploys +12 -8
- data/bin/nodes_to_deploy +6 -6
- data/bin/setup +6 -6
- data/bin/topograph +1 -1
- data/docs/config_dsl.md +45 -1
- data/docs/executables.md +6 -7
- data/docs/executables/check-node.md +3 -3
- data/docs/executables/deploy.md +3 -3
- data/docs/executables/dump_nodes_json.md +3 -3
- data/docs/executables/test.md +3 -3
- data/docs/executables/topograph.md +3 -3
- data/docs/gen/mermaid/README.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
- data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
- data/docs/plugins.md +47 -0
- data/docs/plugins/connector/ssh.md +1 -1
- data/docs/plugins/log/remote_fs.md +26 -0
- data/docs/plugins/secrets_reader/cli.md +31 -0
- data/docs/plugins/secrets_reader/thycotic.md +46 -0
- data/docs/plugins/test/bitbucket_conf.md +1 -1
- data/docs/plugins/test/check_deploy_and_idempotence.md +1 -1
- data/docs/plugins/test/connection.md +1 -0
- data/docs/plugins/test/deploy_removes_root_access.md +1 -1
- data/docs/plugins/test/file_system.md +1 -0
- data/docs/plugins/test/github_ci.md +48 -0
- data/docs/plugins/test/hostname.md +1 -0
- data/docs/plugins/test/ip.md +1 -0
- data/docs/plugins/test/jenkins_ci_conf.md +1 -1
- data/docs/plugins/test/jenkins_ci_masters_ok.md +1 -1
- data/docs/plugins/test/local_users.md +1 -0
- data/docs/plugins/test/mounts.md +1 -0
- data/docs/plugins/test/orphan_files.md +1 -0
- data/docs/plugins/test/ports.md +1 -0
- data/docs/plugins/test/spectre.md +1 -0
- data/docs/plugins/test/vulnerabilities.md +1 -0
- data/lib/hybrid_platforms_conductor/action.rb +4 -4
- data/lib/hybrid_platforms_conductor/actions_executor.rb +45 -43
- data/lib/hybrid_platforms_conductor/bitbucket.rb +5 -4
- data/lib/hybrid_platforms_conductor/cmd_runner.rb +13 -12
- data/lib/hybrid_platforms_conductor/cmdb.rb +2 -2
- data/lib/hybrid_platforms_conductor/common_config_dsl/bitbucket.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/confluence.rb +2 -1
- data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +5 -4
- data/lib/hybrid_platforms_conductor/common_config_dsl/github.rb +63 -0
- data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb +2 -2
- data/lib/hybrid_platforms_conductor/config.rb +7 -4
- data/lib/hybrid_platforms_conductor/confluence.rb +1 -1
- data/lib/hybrid_platforms_conductor/connector.rb +5 -2
- data/lib/hybrid_platforms_conductor/core_extensions/symbol/zero.rb +24 -0
- data/lib/hybrid_platforms_conductor/credentials.rb +39 -36
- data/lib/hybrid_platforms_conductor/current_dir_monitor.rb +4 -1
- data/lib/hybrid_platforms_conductor/deployer.rb +275 -224
- data/lib/hybrid_platforms_conductor/executable.rb +20 -15
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/config.rb +10 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +1 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +4 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb +2 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +45 -49
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/my_log_plugin.rb.sample +100 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb +180 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +68 -66
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb +13 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb +39 -38
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb +5 -4
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb +43 -45
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/podman.rb +18 -20
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +118 -117
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +39 -43
- data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container +9 -13
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +28 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/report/stdout.rb +26 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/cli.rb +77 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/my_secrets_reader_plugin.rb.sample +46 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb +90 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb +4 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb +3 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +7 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb +21 -19
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +21 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +19 -21
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb +31 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +3 -1
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +4 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb +21 -22
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb +10 -12
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +9 -9
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +13 -10
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb +5 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +5 -5
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +7 -7
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +3 -3
- data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +27 -25
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/confluence.rb +2 -2
- data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/stdout.rb +8 -6
- data/lib/hybrid_platforms_conductor/io_router.rb +14 -13
- data/lib/hybrid_platforms_conductor/json_dumper.rb +2 -2
- data/lib/hybrid_platforms_conductor/log.rb +31 -0
- data/lib/hybrid_platforms_conductor/logger_helpers.rb +19 -16
- data/lib/hybrid_platforms_conductor/nodes_handler.rb +89 -71
- data/lib/hybrid_platforms_conductor/parallel_threads.rb +7 -11
- data/lib/hybrid_platforms_conductor/platform_handler.rb +7 -7
- data/lib/hybrid_platforms_conductor/platforms_handler.rb +5 -3
- data/lib/hybrid_platforms_conductor/plugin.rb +2 -2
- data/lib/hybrid_platforms_conductor/plugins.rb +14 -8
- data/lib/hybrid_platforms_conductor/provisioner.rb +4 -4
- data/lib/hybrid_platforms_conductor/report.rb +2 -2
- data/lib/hybrid_platforms_conductor/reports_handler.rb +3 -2
- data/lib/hybrid_platforms_conductor/secrets_reader.rb +31 -0
- data/lib/hybrid_platforms_conductor/services_handler.rb +32 -29
- data/lib/hybrid_platforms_conductor/test_only_remote_node.rb +18 -0
- data/lib/hybrid_platforms_conductor/test_report.rb +15 -18
- data/lib/hybrid_platforms_conductor/tests_runner.rb +116 -118
- data/lib/hybrid_platforms_conductor/thycotic.rb +28 -19
- data/lib/hybrid_platforms_conductor/topographer.rb +200 -190
- data/lib/hybrid_platforms_conductor/topographer/plugins/graphviz.rb +8 -8
- data/lib/hybrid_platforms_conductor/topographer/plugins/json.rb +4 -4
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test.rb +33 -12
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +16 -11
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/interactive_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb +23 -18
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +67 -49
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/scp_spec.rb +20 -14
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions_spec.rb +63 -50
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connection_spec.rb +35 -35
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/connectable_nodes_spec.rb +12 -8
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb +4 -7
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +21 -22
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +23 -24
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connectable_nodes_spec.rb +10 -6
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +73 -54
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +145 -126
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +29 -25
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/logging_spec.rb +106 -91
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/parallel_spec.rb +272 -244
- data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +10 -12
- data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +36 -36
- data/spec/hybrid_platforms_conductor_test/api/config_spec.rb +24 -22
- data/spec/hybrid_platforms_conductor_test/api/deployer/check_spec.rb +4 -2
- data/spec/hybrid_platforms_conductor_test/api/deployer/config_dsl_spec.rb +43 -5
- data/spec/hybrid_platforms_conductor_test/api/deployer/deploy_spec.rb +199 -216
- data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +223 -0
- data/spec/hybrid_platforms_conductor_test/api/deployer/parse_deploy_output_spec.rb +55 -59
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +36 -62
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/podman_spec.rb +17 -17
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/config_dsl_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/create_spec.rb +44 -51
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/destroy_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/ip_spec.rb +12 -16
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/destroy_vm_spec.rb +31 -19
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +324 -266
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/ips_assignment_spec.rb +89 -61
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/other_lxc_containers_resources_spec.rb +117 -93
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/pve_node_resources_spec.rb +71 -54
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +10 -8
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb +80 -60
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/start_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/stop_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/cli_spec.rb +64 -0
- data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/thycotic_spec.rb +268 -0
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/config_spec.rb +8 -10
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +33 -24
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_keys_spec.rb +64 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/platform_handlers_spec.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +50 -51
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +91 -81
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +14 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/git_diff_impacts_spec.rb +51 -75
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/nodes_selectors_spec.rb +35 -26
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/platform_handlers_plugins_api_spec.rb +24 -16
- data/spec/hybrid_platforms_conductor_test/api/nodes_handler/several_platforms_spec.rb +29 -19
- data/spec/hybrid_platforms_conductor_test/api/platform_handler_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb +6 -6
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb +57 -99
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +4 -4
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +32 -35
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +10 -10
- data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +38 -27
- data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +46 -52
- data/spec/hybrid_platforms_conductor_test/api/reports_handler_spec.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/api/services_handler/actions_to_deploy_spec.rb +90 -58
- data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +38 -34
- data/spec/hybrid_platforms_conductor_test/api/services_handler/log_info_spec.rb +11 -9
- data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +193 -171
- data/spec/hybrid_platforms_conductor_test/api/services_handler/parse_deploy_output_spec.rb +66 -54
- data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +147 -133
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +69 -49
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/global_spec.rb +5 -4
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_spec.rb +8 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +30 -27
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/platform_spec.rb +12 -9
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +48 -47
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/bitbucket_conf_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/github_ci_spec.rb +72 -0
- data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_reports_plugins/confluence_spec.rb +5 -5
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +9 -9
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb2.rb → test_cmdb_2.rb} +6 -6
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb_others2.rb → test_cmdb_others_2.rb} +2 -2
- data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/{check-node_spec.rb → check_node_spec.rb} +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/get_impacted_nodes_spec.rb +76 -77
- data/spec/hybrid_platforms_conductor_test/executables/last_deploys_spec.rb +159 -113
- data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +299 -160
- data/spec/hybrid_platforms_conductor_test/executables/options/actions_executor_spec.rb +4 -6
- data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb +8 -8
- data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +12 -196
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/nodes_selectors_spec.rb +9 -10
- data/spec/hybrid_platforms_conductor_test/executables/options/reports_handler_spec.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +22 -22
- data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +22 -16
- data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +32 -32
- data/spec/hybrid_platforms_conductor_test/executables/ssh_config_spec.rb +7 -9
- data/spec/hybrid_platforms_conductor_test/executables/test_spec.rb +3 -5
- data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +4 -3
- data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/helpers/config_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +12 -13
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +245 -56
- data/spec/hybrid_platforms_conductor_test/helpers/executables_helpers.rb +11 -11
- data/spec/hybrid_platforms_conductor_test/helpers/nodes_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +39 -28
- data/spec/hybrid_platforms_conductor_test/helpers/plugins_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +86 -111
- data/spec/hybrid_platforms_conductor_test/helpers/reports_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/helpers/services_handler_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/helpers/tests_runner_helpers.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id1.rb → test_plugin_id_1.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id2.rb → test_plugin_id_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/test_plugin_type/{test_plugin_id3.rb → test_plugin_id_3.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/{test_plugin_type2/test_plugin_id4.rb → test_plugin_type_2/test_plugin_id_4.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/test.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/{test2.rb → test_2.rb} +0 -0
- data/spec/hybrid_platforms_conductor_test/rubocop_spec.rb +31 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json +3 -3
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +952 -0
- data/spec/hybrid_platforms_conductor_test/test_connector.rb +3 -3
- data/spec/hybrid_platforms_conductor_test/test_log_no_read_plugin.rb +84 -0
- data/spec/hybrid_platforms_conductor_test/test_log_plugin.rb +105 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/global.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/node_check.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/platform.rb +1 -0
- data/spec/hybrid_platforms_conductor_test/test_plugins/several_checks.rb +2 -2
- data/spec/hybrid_platforms_conductor_test/test_secrets_reader_plugin.rb +45 -0
- data/spec/hybrid_platforms_conductor_test/tests_report_plugin.rb +5 -6
- data/spec/spec_helper.rb +17 -18
- data/tools/check_md +16 -20
- data/tools/generate_mermaid +1 -1
- metadata +82 -13
- data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +0 -916
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'nokogiri'
|
2
|
+
require 'hybrid_platforms_conductor/test_only_remote_node'
|
2
3
|
|
3
4
|
module HybridPlatformsConductor
|
4
5
|
|
@@ -17,7 +18,7 @@ module HybridPlatformsConductor
|
|
17
18
|
# Each final OVAL URL can be directly an XML file, either raw or compressed with .gz or .bz2.
|
18
19
|
# This is useful to follow repository links, such as jFrog or web servers serving common file systems structure storing several versions of the OVAL file.
|
19
20
|
# * *reported_severities* (Array<String> or nil): List of severities to report, if any (use Unknown when the severity is not known), or nil for all [default: nil]
|
20
|
-
class Vulnerabilities <
|
21
|
+
class Vulnerabilities < TestOnlyRemoteNode
|
21
22
|
|
22
23
|
# Known compression methods, per file extension, and their corresponding uncompress bash script
|
23
24
|
KNOWN_COMPRESSIONS = {
|
@@ -47,8 +48,9 @@ module HybridPlatformsConductor
|
|
47
48
|
# Follow the last link recursively until we find a .xml or compressed file
|
48
49
|
current_url = artifactory_url
|
49
50
|
loop do
|
50
|
-
current_url = "#{current_url}#{current_url.end_with?('/') ? '' : '/'}#{Nokogiri::HTML.parse(URI.
|
51
|
+
current_url = "#{current_url}#{current_url.end_with?('/') ? '' : '/'}#{Nokogiri::HTML.parse(URI.parse(current_url).open).css('a').last['href']}"
|
51
52
|
break if current_url.end_with?('.xml') || KNOWN_COMPRESSIONS.keys.any? { |file_ext| current_url.end_with?(".#{file_ext}") }
|
53
|
+
|
52
54
|
log_debug "Follow last link to #{current_url}"
|
53
55
|
end
|
54
56
|
current_url
|
@@ -56,7 +58,7 @@ module HybridPlatformsConductor
|
|
56
58
|
)
|
57
59
|
# TODO: Access the user correctly when the user notion will be moved out of the ssh connector
|
58
60
|
sudo = @deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "
|
59
|
-
|
61
|
+
urls.map do |url|
|
60
62
|
# 1. Get the OVAL file on the node to be tested (uncompress it if needed)
|
61
63
|
# 2. Make sure oscap is installed
|
62
64
|
# 3. Generate the report for this OVAL file using oscap
|
@@ -66,13 +68,13 @@ module HybridPlatformsConductor
|
|
66
68
|
packages_to_install = []
|
67
69
|
KNOWN_COMPRESSIONS.each do |file_ext, compress_info|
|
68
70
|
file_ending = ".#{file_ext}"
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
71
|
+
next unless local_oval_file.end_with?(file_ending)
|
72
|
+
|
73
|
+
uncompress_cmds << compress_info[:cmd].call(local_oval_file)
|
74
|
+
packages_to_install.concat(compress_info[:packages])
|
75
|
+
local_oval_file = File.basename(local_oval_file, file_ending)
|
74
76
|
end
|
75
|
-
cmds = <<~
|
77
|
+
cmds = <<~EO_BASH
|
76
78
|
set -e
|
77
79
|
#{
|
78
80
|
case image
|
@@ -84,7 +86,7 @@ module HybridPlatformsConductor
|
|
84
86
|
# On Debian 10 we have to compile it from sources, as the packaged official version has core dumps.
|
85
87
|
# cf https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1688223.html
|
86
88
|
# TODO: Remove this Debian 10 specificity when the official libopenscap8 will be corrected
|
87
|
-
<<~
|
89
|
+
<<~EO_BASH2
|
88
90
|
if [ ! -x "$(command -v oscap)" ] || [ "$(oscap --version | head -n 1 | awk '{print $6}')" != "1.3.4" ]; then
|
89
91
|
rm -rf openscap
|
90
92
|
git clone --recurse-submodules https://github.com/OpenSCAP/openscap.git
|
@@ -96,7 +98,7 @@ module HybridPlatformsConductor
|
|
96
98
|
#{sudo}make install
|
97
99
|
fi
|
98
100
|
#{sudo}apt install -y wget #{packages_to_install.join(' ')}
|
99
|
-
|
101
|
+
EO_BASH2
|
100
102
|
else
|
101
103
|
raise "Non supported image: #{image}. Please adapt this test's code."
|
102
104
|
end
|
@@ -110,7 +112,7 @@ module HybridPlatformsConductor
|
|
110
112
|
echo "===== RESULTS ====="
|
111
113
|
cat "#{local_oval_file}.results.xml"
|
112
114
|
cd ..
|
113
|
-
|
115
|
+
EO_BASH
|
114
116
|
[
|
115
117
|
cmds,
|
116
118
|
{
|
@@ -119,21 +121,21 @@ module HybridPlatformsConductor
|
|
119
121
|
if idx_results.nil?
|
120
122
|
error 'No results given by the oscap run', stdout.join("\n")
|
121
123
|
else
|
122
|
-
results = Nokogiri::XML(stdout[idx_results + 1
|
124
|
+
results = Nokogiri::XML(stdout[idx_results + 1..].join("\n"))
|
123
125
|
results.remove_namespaces!
|
124
126
|
oval_definitions = results.css('oval_results oval_definitions definitions definition')
|
125
127
|
results.css('results system definitions definition').each do |definition_xml|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
128
|
+
next unless definition_xml['result'] == 'true'
|
129
|
+
|
130
|
+
# Just found an OVAL item to be patched.
|
131
|
+
definition_id = definition_xml['definition_id']
|
132
|
+
oval_definition = oval_definitions.find { |el| el['id'] == definition_id }
|
133
|
+
# We don't forcefully want to report all missing patches. Only the most important ones.
|
134
|
+
severity = oval_definition.css('metadata advisory severity').text
|
135
|
+
severity = 'Unknown' if severity.empty?
|
136
|
+
if !oval_info.key?('reported_severities') || oval_info['reported_severities'].include?(severity)
|
137
|
+
# Only consider the first line of the description, as sometimes it's very long
|
138
|
+
error "Non-patched #{severity} vulnerability found: #{oval_definition.css('metadata title').text} - #{oval_definition.css('metadata description').text.split("\n").first}"
|
137
139
|
end
|
138
140
|
end
|
139
141
|
end
|
@@ -142,7 +144,7 @@ module HybridPlatformsConductor
|
|
142
144
|
timeout: 240
|
143
145
|
}
|
144
146
|
]
|
145
|
-
end
|
147
|
+
end.to_h
|
146
148
|
else
|
147
149
|
error "No OVAL file defined for image #{image} at #{oval_file}"
|
148
150
|
{}
|
@@ -12,7 +12,7 @@ module HybridPlatformsConductor
|
|
12
12
|
# Report tests results on a generated Confluence page
|
13
13
|
class Confluence < HybridPlatformsConductor::TestReport
|
14
14
|
|
15
|
-
|
15
|
+
extend_config_dsl_with CommonConfigDsl::Confluence, :init_confluence
|
16
16
|
|
17
17
|
# Maximum errors to be reported by item
|
18
18
|
MAX_ERROR_ITEMS_DISPLAYED = 10
|
@@ -34,7 +34,7 @@ module HybridPlatformsConductor
|
|
34
34
|
at('h1:contains("Evolution")').
|
35
35
|
search('~ structured-macro:first-of-type').
|
36
36
|
css('table td').
|
37
|
-
map
|
37
|
+
map(&:text).
|
38
38
|
each_slice(2).
|
39
39
|
to_a.
|
40
40
|
map { |(time_str, value_str)| [Time.parse("#{time_str} UTC"), value_str.to_f] }
|
@@ -97,18 +97,20 @@ module HybridPlatformsConductor
|
|
97
97
|
table << [
|
98
98
|
nodes_list,
|
99
99
|
nodes_info[:nodes].size,
|
100
|
-
nodes_info[:nodes].empty? ? '' : "#{(nodes_info[:tested_nodes].size*100.0/nodes_info[:nodes].size).to_i} %",
|
100
|
+
nodes_info[:nodes].empty? ? '' : "#{(nodes_info[:tested_nodes].size * 100.0 / nodes_info[:nodes].size).to_i} %",
|
101
101
|
nodes_info[:tested_nodes].empty? ? '' : "#{((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error_as_expected].size) * 100.0 / nodes_info[:tested_nodes].size).to_i} %",
|
102
102
|
nodes_info[:tested_nodes].empty? ? '' : "#{((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * 100.0 / nodes_info[:tested_nodes].size).to_i} %",
|
103
|
-
nodes_info[:nodes].empty?
|
103
|
+
if nodes_info[:nodes].empty?
|
104
|
+
''
|
105
|
+
else
|
104
106
|
('=' * ((nodes_info[:tested_nodes_in_error_as_expected].size * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).yellow.bold +
|
105
|
-
|
106
|
-
|
107
|
-
|
107
|
+
('=' * (((nodes_info[:tested_nodes_in_error].size - nodes_info[:tested_nodes_in_error_as_expected].size).abs * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).red.bold +
|
108
|
+
('=' * (((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).green.bold +
|
109
|
+
('=' * (((nodes_info[:nodes].size - nodes_info[:tested_nodes].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).white.bold
|
110
|
+
end
|
108
111
|
]
|
109
112
|
end
|
110
113
|
end)
|
111
|
-
|
112
114
|
end
|
113
115
|
|
114
116
|
end
|
@@ -30,6 +30,7 @@ module HybridPlatformsConductor
|
|
30
30
|
# Start routing messages asynchronously
|
31
31
|
def start
|
32
32
|
raise 'IO router is already started. Can\'t start it again.' unless @reading_thread.nil?
|
33
|
+
|
33
34
|
@end_read = false
|
34
35
|
# Create a thread to handle routes asynchronously
|
35
36
|
@reading_thread = Thread.new do
|
@@ -37,22 +38,21 @@ module HybridPlatformsConductor
|
|
37
38
|
need_to_stop = @end_read.clone
|
38
39
|
data_found = false
|
39
40
|
@routes.each do |src_io, dst_ios|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
else
|
52
|
-
raise "Unknown type of source IO: #{src_io}"
|
41
|
+
raise "Unknown type of source IO: #{src_io}" unless src_io.is_a?(Queue)
|
42
|
+
|
43
|
+
queue_size = src_io.size
|
44
|
+
next unless queue_size.positive?
|
45
|
+
|
46
|
+
# There is data to be read from src_io
|
47
|
+
data_found = true
|
48
|
+
data_chunk_str = queue_size.times.map { src_io.pop }.join
|
49
|
+
dst_ios.each do |dst_io|
|
50
|
+
dst_io << data_chunk_str
|
51
|
+
dst_io.flush if dst_io.respond_to?(:flush)
|
53
52
|
end
|
54
53
|
end
|
55
54
|
break if need_to_stop && !data_found
|
55
|
+
|
56
56
|
sleep 0.1
|
57
57
|
end
|
58
58
|
end
|
@@ -61,6 +61,7 @@ module HybridPlatformsConductor
|
|
61
61
|
# Stop routing messages asynchronously
|
62
62
|
def stop
|
63
63
|
raise 'IO router is not started. Can\'t stop it.' if @reading_thread.nil?
|
64
|
+
|
64
65
|
@end_read = true
|
65
66
|
@reading_thread.join
|
66
67
|
end
|
@@ -23,7 +23,7 @@ module HybridPlatformsConductor
|
|
23
23
|
# * *config* (Config): Config to be used. [default = Config.new]
|
24
24
|
# * *nodes_handler* (NodesHandler): Nodes handler to be used. [default = NodesHandler.new]
|
25
25
|
# * *deployer* (Deployer): Deployer to be used. [default = Deployer.new]
|
26
|
-
def initialize(logger: Logger.new(
|
26
|
+
def initialize(logger: Logger.new($stdout), logger_stderr: Logger.new($stderr), config: Config.new, nodes_handler: NodesHandler.new, deployer: Deployer.new)
|
27
27
|
init_loggers(logger, logger_stderr)
|
28
28
|
@config = config
|
29
29
|
@nodes_handler = nodes_handler
|
@@ -74,7 +74,7 @@ module HybridPlatformsConductor
|
|
74
74
|
out "[ #{node} ] - Error while dumping JSON. Check #{stdout_file_name}"
|
75
75
|
else
|
76
76
|
json_file_name = "#{@dump_dir}/#{node}.json"
|
77
|
-
File.write(json_file_name, stdout[dump_begin_idx+1..dump_end_idx-1].join("\n"))
|
77
|
+
File.write(json_file_name, stdout[dump_begin_idx + 1..dump_end_idx - 1].join("\n"))
|
78
78
|
out "[ #{node} ] - OK. Check #{json_file_name}"
|
79
79
|
end
|
80
80
|
else
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'hybrid_platforms_conductor/logger_helpers'
|
2
|
+
require 'hybrid_platforms_conductor/plugin'
|
3
|
+
|
4
|
+
module HybridPlatformsConductor
|
5
|
+
|
6
|
+
# Ancestor of all log plugins
|
7
|
+
class Log < Plugin
|
8
|
+
|
9
|
+
# Constructor
|
10
|
+
#
|
11
|
+
# Parameters::
|
12
|
+
# * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
|
13
|
+
# * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
|
14
|
+
# * *config* (Config): Config to be used. [default: Config.new]
|
15
|
+
# * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
|
16
|
+
# * *actions_executor* (ActionsExecutor): Actions executor to be used. [default: ActionsExecutor.new]
|
17
|
+
def initialize(
|
18
|
+
logger: Logger.new($stdout),
|
19
|
+
logger_stderr: Logger.new($stderr),
|
20
|
+
config: Config.new,
|
21
|
+
nodes_handler: NodesHandler.new,
|
22
|
+
actions_executor: ActionsExecutor.new
|
23
|
+
)
|
24
|
+
super(logger: logger, logger_stderr: logger_stderr, config: config)
|
25
|
+
@nodes_handler = nodes_handler
|
26
|
+
@actions_executor = actions_executor
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
@@ -55,25 +55,27 @@ module HybridPlatformsConductor
|
|
55
55
|
|
56
56
|
# Make sure if the current line is not flushed we still do it
|
57
57
|
def flush
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
58
|
+
return if @current_line.nil?
|
59
|
+
|
60
|
+
@stream << @current_line
|
61
|
+
@current_line = nil
|
62
62
|
end
|
63
63
|
|
64
64
|
end
|
65
65
|
|
66
66
|
class << self
|
67
|
+
|
67
68
|
attr_reader :progress_bar_semaphore
|
69
|
+
|
68
70
|
end
|
69
71
|
# Make sure the progress bar setting is protected by a Mutex
|
70
72
|
@progress_bar_semaphore = Mutex.new
|
71
73
|
|
72
74
|
# Sorted list of levels and their corresponding modifiers.
|
73
75
|
LEVELS_MODIFIERS = {
|
74
|
-
fatal: [
|
75
|
-
error: [
|
76
|
-
warn: [
|
76
|
+
fatal: %i[red bold],
|
77
|
+
error: %i[red bold],
|
78
|
+
warn: %i[yellow bold],
|
77
79
|
info: [:white],
|
78
80
|
debug: [:white],
|
79
81
|
unknown: [:white]
|
@@ -82,7 +84,7 @@ module HybridPlatformsConductor
|
|
82
84
|
# List of levels that will output on stderr
|
83
85
|
LEVELS_TO_STDERR = %i[warn error fatal]
|
84
86
|
|
85
|
-
LEVELS_MODIFIERS.
|
87
|
+
LEVELS_MODIFIERS.each_key do |level|
|
86
88
|
define_method("log_#{level}") do |message|
|
87
89
|
(LEVELS_TO_STDERR.include?(level) ? @logger_stderr : @logger).send(
|
88
90
|
level,
|
@@ -105,10 +107,10 @@ module HybridPlatformsConductor
|
|
105
107
|
# Set loggers to the desired format
|
106
108
|
def set_loggers_format
|
107
109
|
[@logger, @logger_stderr].each do |logger|
|
108
|
-
logger.formatter = proc do |severity,
|
110
|
+
logger.formatter = proc do |severity, _datetime, progname, msg|
|
109
111
|
# If the message already has control characters, don't colorize it
|
110
112
|
keep_original_color = msg.include? "\u001b"
|
111
|
-
message = "[#{Time.now.utc.strftime('%F %T')} (PID #{
|
113
|
+
message = "[#{Time.now.utc.strftime('%F %T')} (PID #{$PROCESS_ID} / TID #{Thread.current.object_id})] #{severity.rjust(5)} - [ #{progname} ] - "
|
112
114
|
message << "#{msg}\n" unless keep_original_color
|
113
115
|
LEVELS_MODIFIERS[severity.downcase.to_sym].each do |modifier|
|
114
116
|
message = message.send(modifier)
|
@@ -282,8 +284,8 @@ module HybridPlatformsConductor
|
|
282
284
|
yield progress_bar
|
283
285
|
ensure
|
284
286
|
LoggerHelpers.progress_bar_semaphore.synchronize do
|
285
|
-
|
286
|
-
|
287
|
+
stdout_device.flush
|
288
|
+
stderr_device.flush
|
287
289
|
self.stdout_device = previous_stdout_device unless previous_stdout_device.nil?
|
288
290
|
self.stderr_device = previous_stderr_device unless previous_stderr_device.nil?
|
289
291
|
end
|
@@ -298,15 +300,16 @@ module HybridPlatformsConductor
|
|
298
300
|
def stdouts_to_s
|
299
301
|
messages = []
|
300
302
|
{
|
301
|
-
'STDOUT' =>
|
302
|
-
'STDERR' =>
|
303
|
+
'STDOUT' => stdout_device,
|
304
|
+
'STDERR' => stderr_device
|
303
305
|
}.each do |name, device|
|
304
|
-
|
306
|
+
case device
|
307
|
+
when File
|
305
308
|
if File.exist?(device.path)
|
306
309
|
content = File.read(device.path).strip
|
307
310
|
messages << "----- #{name} BEGIN - #{device.path} -----\n#{content}\n----- #{name} END - #{device.path} -----" unless content.empty?
|
308
311
|
end
|
309
|
-
|
312
|
+
when StringIO
|
310
313
|
content = device.string
|
311
314
|
messages << "----- #{name} BEGIN -----\n#{content}\n----- #{name} END -----" unless content.empty?
|
312
315
|
end
|
@@ -40,7 +40,7 @@ module HybridPlatformsConductor
|
|
40
40
|
# * *master_cmdbs_info* (Hash< Symbol, Symbol or Array<Symbol> >): List of metadata properties (or single one) per CMDB name considered as master for those properties.
|
41
41
|
def master_cmdbs(master_cmdbs_info)
|
42
42
|
@cmdb_masters << {
|
43
|
-
cmdb_masters:
|
43
|
+
cmdb_masters: master_cmdbs_info.transform_values { |properties| properties.is_a?(Array) ? properties : [properties] },
|
44
44
|
nodes_selectors_stack: current_nodes_selectors_stack
|
45
45
|
}
|
46
46
|
end
|
@@ -60,7 +60,8 @@ module HybridPlatformsConductor
|
|
60
60
|
|
61
61
|
Config.extend_config_dsl_with ConfigDSLExtension, :init_nodes_handler_config
|
62
62
|
|
63
|
-
include
|
63
|
+
include ParallelThreads
|
64
|
+
include LoggerHelpers
|
64
65
|
|
65
66
|
class GitError < RuntimeError
|
66
67
|
end
|
@@ -74,8 +75,8 @@ module HybridPlatformsConductor
|
|
74
75
|
# * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
|
75
76
|
# * *platforms_handler* (PlatformsHandler): Platforms Handler to be used. [default: PlatformsHandler.new]
|
76
77
|
def initialize(
|
77
|
-
logger: Logger.new(
|
78
|
-
logger_stderr: Logger.new(
|
78
|
+
logger: Logger.new($stdout),
|
79
|
+
logger_stderr: Logger.new($stderr),
|
79
80
|
config: Config.new,
|
80
81
|
cmd_runner: CmdRunner.new,
|
81
82
|
platforms_handler: PlatformsHandler.new
|
@@ -111,11 +112,11 @@ module HybridPlatformsConductor
|
|
111
112
|
)
|
112
113
|
@cmdbs_others << cmdb if cmdb.respond_to?(:get_others)
|
113
114
|
cmdb.methods.each do |method|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
115
|
+
next unless method.to_s =~ /^get_(.*)$/
|
116
|
+
|
117
|
+
property = Regexp.last_match(1).to_sym
|
118
|
+
@cmdbs_per_property[property] = [] unless @cmdbs_per_property.key?(property)
|
119
|
+
@cmdbs_per_property[property] << cmdb
|
119
120
|
end
|
120
121
|
cmdb
|
121
122
|
end
|
@@ -133,13 +134,17 @@ module HybridPlatformsConductor
|
|
133
134
|
# Register all known nodes for this platform
|
134
135
|
platform.known_nodes.each do |node|
|
135
136
|
raise "Can't register #{node} to platform #{platform.repository_path}, as it is already defined in platform #{@nodes_platform[node].repository_path}." if @nodes_platform.key?(node)
|
137
|
+
|
136
138
|
@nodes_platform[node] = platform
|
137
139
|
end
|
138
140
|
# Register all known nodes lists
|
141
|
+
next unless platform.respond_to?(:known_nodes_lists)
|
142
|
+
|
139
143
|
platform.known_nodes_lists.each do |nodes_list|
|
140
144
|
raise "Can't register nodes list #{nodes_list} to platform #{platform.repository_path}, as it is already defined in platform #{@nodes_list_platform[nodes_list].repository_path}." if @nodes_list_platform.key?(nodes_list)
|
145
|
+
|
141
146
|
@nodes_list_platform[nodes_list] = platform
|
142
|
-
end
|
147
|
+
end
|
143
148
|
end
|
144
149
|
end
|
145
150
|
|
@@ -147,7 +152,7 @@ module HybridPlatformsConductor
|
|
147
152
|
#
|
148
153
|
# Parameters::
|
149
154
|
# * *options_parser* (OptionParser): The option parser to complete
|
150
|
-
def options_parse(options_parser
|
155
|
+
def options_parse(options_parser)
|
151
156
|
options_parser.separator ''
|
152
157
|
options_parser.separator 'Nodes handler options:'
|
153
158
|
options_parser.on('-o', '--show-nodes', 'Display the list of possible nodes and exit') do
|
@@ -221,6 +226,7 @@ module HybridPlatformsConductor
|
|
221
226
|
platform_name, from_commit, to_commit, flags = nodes_git_impact.split(':')
|
222
227
|
flags = (flags || '').split(',')
|
223
228
|
raise "Invalid platform in --nodes-git-impact: #{platform_name}. Possible values are: #{platform_names.join(', ')}." unless platform_names.include?(platform_name)
|
229
|
+
|
224
230
|
nodes_selector = { platform: platform_name }
|
225
231
|
nodes_selector[:from_commit] = from_commit if from_commit && !from_commit.empty?
|
226
232
|
nodes_selector[:to_commit] = to_commit if to_commit && !to_commit.empty?
|
@@ -329,7 +335,7 @@ module HybridPlatformsConductor
|
|
329
335
|
# * *block* (Proc): Code block given to the call
|
330
336
|
def method_missing(method, *args, &block)
|
331
337
|
if method.to_s =~ /^get_(.*)_of$/
|
332
|
-
property =
|
338
|
+
property = Regexp.last_match(1).to_sym
|
333
339
|
# Define the method so that we don't go trough method_missing next time (more efficient).
|
334
340
|
define_property_method_for(property)
|
335
341
|
# Then call it
|
@@ -341,6 +347,15 @@ module HybridPlatformsConductor
|
|
341
347
|
end
|
342
348
|
end
|
343
349
|
|
350
|
+
# Make sure we register the methods we handle in method_missing
|
351
|
+
#
|
352
|
+
# Parameters::
|
353
|
+
# * *name* (Symbol): The missing method name
|
354
|
+
# * *include_private* (Boolean): Should we include private methods in the search?
|
355
|
+
def respond_to_missing?(name, include_private)
|
356
|
+
name.to_s =~ /^get_(.*)_of$/ || super
|
357
|
+
end
|
358
|
+
|
344
359
|
# Prefetch some metadata properties for a given list of nodes.
|
345
360
|
# Useful for performance reasons when clients know they will need to use a lot of properties on nodes.
|
346
361
|
# Keep a thread-safe memory cache of it.
|
@@ -352,52 +367,51 @@ module HybridPlatformsConductor
|
|
352
367
|
(properties.is_a?(Symbol) ? [properties] : properties).each do |property|
|
353
368
|
# Gather the list of nodes missing this property
|
354
369
|
missing_nodes = nodes.select { |node| !@metadata.key?(node) || !@metadata[node].key?(property) }
|
355
|
-
|
356
|
-
|
357
|
-
|
370
|
+
next if missing_nodes.empty?
|
371
|
+
|
372
|
+
# Query the CMDBs having first the get_<property> method, then the ones having the get_others method till we have our property set for all missing nodes
|
373
|
+
# Metadata being retrieved by the different CMDBs, per node
|
374
|
+
# Hash< String, Object >
|
375
|
+
updated_metadata = {}
|
376
|
+
(
|
377
|
+
(@cmdbs_per_property.key?(property) ? @cmdbs_per_property[property] : []).map { |cmdb| [cmdb, property] } +
|
378
|
+
@cmdbs_others.map { |cmdb| [cmdb, :others] }
|
379
|
+
).each do |(cmdb, cmdb_property)|
|
380
|
+
# If among the missing nodes some of them have some master CMDB declared for this property, filter them out unless we are dealing with their master CMDB.
|
381
|
+
nodes_to_query = missing_nodes.select do |node|
|
382
|
+
master_cmdb = cmdb_master_for(node, property)
|
383
|
+
master_cmdb.nil? || master_cmdb == cmdb
|
384
|
+
end
|
385
|
+
next if nodes_to_query.empty?
|
386
|
+
|
387
|
+
# Check first if this property depends on other ones for this cmdb
|
388
|
+
if cmdb.respond_to?(:property_dependencies)
|
389
|
+
property_deps = cmdb.property_dependencies
|
390
|
+
prefetch_metadata_of nodes_to_query, property_deps[property] if property_deps.key?(property)
|
391
|
+
end
|
392
|
+
# Property values, per node name
|
358
393
|
# Hash< String, Object >
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
end
|
369
|
-
unless nodes_to_query.empty?
|
370
|
-
# Check first if this property depends on other ones for this cmdb
|
371
|
-
if cmdb.respond_to?(:property_dependencies)
|
372
|
-
property_deps = cmdb.property_dependencies
|
373
|
-
prefetch_metadata_of nodes_to_query, property_deps[property] if property_deps.key?(property)
|
374
|
-
end
|
375
|
-
# Property values, per node name
|
376
|
-
# Hash< String, Object >
|
377
|
-
metadata_from_cmdb = Hash[
|
378
|
-
cmdb.send("get_#{cmdb_property}".to_sym, nodes_to_query, @metadata.slice(*nodes_to_query)).map do |node, cmdb_result|
|
379
|
-
[node, cmdb_property == :others ? cmdb_result[property] : cmdb_result]
|
380
|
-
end
|
381
|
-
].compact
|
382
|
-
cmdb_log_header = "[CMDB #{cmdb.class.name.split('::').last}.#{cmdb_property}] -"
|
383
|
-
log_debug "#{cmdb_log_header} Query property #{property} for #{nodes_to_query.size} nodes (#{nodes_to_query[0..7].join(', ')}...) => Found metadata for #{metadata_from_cmdb.size} nodes."
|
384
|
-
updated_metadata.merge!(metadata_from_cmdb) do |node, existing_value, new_value|
|
385
|
-
raise "#{cmdb_log_header} Returned a conflicting value for metadata #{property} of node #{node}: #{new_value} whereas the value was already set to #{existing_value}" if !existing_value.nil? && new_value != existing_value
|
386
|
-
new_value
|
387
|
-
end
|
388
|
-
end
|
394
|
+
metadata_from_cmdb = cmdb.send("get_#{cmdb_property}".to_sym, nodes_to_query, @metadata.slice(*nodes_to_query)).transform_values do |cmdb_result|
|
395
|
+
cmdb_property == :others ? cmdb_result[property] : cmdb_result
|
396
|
+
end.compact
|
397
|
+
cmdb_log_header = "[CMDB #{cmdb.class.name.split('::').last}.#{cmdb_property}] -"
|
398
|
+
log_debug "#{cmdb_log_header} Query property #{property} for #{nodes_to_query.size} nodes (#{nodes_to_query[0..7].join(', ')}...) => Found metadata for #{metadata_from_cmdb.size} nodes."
|
399
|
+
updated_metadata.merge!(metadata_from_cmdb) do |node, existing_value, new_value|
|
400
|
+
raise "#{cmdb_log_header} Returned a conflicting value for metadata #{property} of node #{node}: #{new_value} whereas the value was already set to #{existing_value}" if !existing_value.nil? && new_value != existing_value
|
401
|
+
|
402
|
+
new_value
|
389
403
|
end
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
404
|
+
end
|
405
|
+
# Avoid conflicts in metadata while merging and make sure this update is thread-safe
|
406
|
+
# As @metadata is only appending data and never deleting it, protecting the update only is enough.
|
407
|
+
# At worst several threads will query several times the same CMDBs to update the same data several times.
|
408
|
+
# If we also want to be thread-safe in this regard, we should protect the whole CMDB call with mutexes, at the granularity of the node + property bein read.
|
409
|
+
@metadata_mutex.synchronize do
|
410
|
+
missing_nodes.each do |node|
|
411
|
+
@metadata[node] = {} unless @metadata.key?(node)
|
412
|
+
# Here, explicitely store nil if nothing has been found for a node because we know there is no value to be fetched.
|
413
|
+
# This way we won't query again all CMDBs thanks to the cache.
|
414
|
+
@metadata[node][property] = updated_metadata[node]
|
401
415
|
end
|
402
416
|
end
|
403
417
|
end
|
@@ -426,6 +440,7 @@ module HybridPlatformsConductor
|
|
426
440
|
nodes_selectors = nodes_selectors.flatten
|
427
441
|
# 1. Check for the presence of all
|
428
442
|
return known_nodes if nodes_selectors.any? { |nodes_selector| nodes_selector.is_a?(Hash) && nodes_selector.key?(:all) && nodes_selector[:all] }
|
443
|
+
|
429
444
|
# 2. Expand the nodes lists, platforms and services contents
|
430
445
|
string_nodes = []
|
431
446
|
nodes_selectors.each do |nodes_selector|
|
@@ -435,6 +450,7 @@ module HybridPlatformsConductor
|
|
435
450
|
if nodes_selector.key?(:list)
|
436
451
|
platform = @nodes_list_platform[nodes_selector[:list]]
|
437
452
|
raise "Unknown nodes list: #{nodes_selector[:list]}" if platform.nil?
|
453
|
+
|
438
454
|
string_nodes.concat(platform.nodes_selectors_from_nodes_list(nodes_selector[:list]))
|
439
455
|
end
|
440
456
|
string_nodes.concat(@platforms_handler.platform(nodes_selector[:platform]).known_nodes) if nodes_selector.key?(:platform)
|
@@ -462,8 +478,8 @@ module HybridPlatformsConductor
|
|
462
478
|
# 3. Expand the Regexps
|
463
479
|
real_nodes = []
|
464
480
|
string_nodes.each do |node|
|
465
|
-
if node =~
|
466
|
-
node_regexp = Regexp.new(
|
481
|
+
if node =~ %r{^/(.+)/$}
|
482
|
+
node_regexp = Regexp.new(Regexp.last_match(1))
|
467
483
|
real_nodes.concat(known_nodes.select { |known_node| known_node[node_regexp] })
|
468
484
|
else
|
469
485
|
real_nodes << node
|
@@ -489,13 +505,11 @@ module HybridPlatformsConductor
|
|
489
505
|
# * *parallel* (Boolean): Iterate in a multithreaded way? [default: false]
|
490
506
|
# * *nbr_threads_max* (Integer or nil): Maximum number of threads to be used in case of parallel, or nil for no limit [default: nil]
|
491
507
|
# * *progress* (String or nil): Name of a progress bar to follow the progression, or nil for no progress bar [default: 'Processing nodes']
|
492
|
-
# * Proc: The code called for each node being iterated on.
|
508
|
+
# * *block* (Proc): The code called for each node being iterated on.
|
493
509
|
# * Parameters::
|
494
510
|
# * *node* (String): The node name
|
495
|
-
def for_each_node_in(nodes, parallel: false, nbr_threads_max: nil, progress: 'Processing nodes')
|
496
|
-
for_each_element_in(nodes.sort, parallel: parallel, nbr_threads_max: nbr_threads_max, progress: progress)
|
497
|
-
yield node
|
498
|
-
end
|
511
|
+
def for_each_node_in(nodes, parallel: false, nbr_threads_max: nil, progress: 'Processing nodes', &block)
|
512
|
+
for_each_element_in(nodes.sort, parallel: parallel, nbr_threads_max: nbr_threads_max, progress: progress, &block)
|
499
513
|
end
|
500
514
|
|
501
515
|
# Get the list of impacted nodes from a git diff on a platform
|
@@ -513,10 +527,11 @@ module HybridPlatformsConductor
|
|
513
527
|
def impacted_nodes_from_git_diff(platform_name, from_commit: 'master', to_commit: nil, smallest_set: false)
|
514
528
|
platform = @platforms_handler.platform(platform_name)
|
515
529
|
raise "Unkown platform #{platform_name}. Possible platforms are #{@platforms_handler.known_platforms.map(&:name).sort.join(', ')}" if platform.nil?
|
530
|
+
|
516
531
|
begin
|
517
532
|
_exit_status, stdout, _stderr = @cmd_runner.run_cmd "cd #{platform.repository_path} && git --no-pager diff --no-color #{from_commit} #{to_commit.nil? ? '' : to_commit}", log_to_stdout: log_debug?
|
518
533
|
rescue CmdRunner::UnexpectedExitCodeError
|
519
|
-
raise GitError,
|
534
|
+
raise GitError, $ERROR_INFO.to_s
|
520
535
|
end
|
521
536
|
# Parse the git diff output to create a structured diff
|
522
537
|
# Hash< String, Hash< Symbol, Object > >: List of diffs info, per file name having a diff. Diffs info have the following properties:
|
@@ -526,9 +541,10 @@ module HybridPlatformsConductor
|
|
526
541
|
current_file_diff = nil
|
527
542
|
stdout.split("\n").each do |line|
|
528
543
|
case line
|
529
|
-
when
|
544
|
+
when %r{^diff --git a/(.+) b/(.+)$}
|
530
545
|
# A new file diff
|
531
|
-
from
|
546
|
+
from = Regexp.last_match(1)
|
547
|
+
to = Regexp.last_match(2)
|
532
548
|
current_file_diff = {
|
533
549
|
diff: ''
|
534
550
|
}
|
@@ -626,11 +642,13 @@ module HybridPlatformsConductor
|
|
626
642
|
cmdb_masters_cache = {}
|
627
643
|
select_confs_for_node(node, @config.cmdb_masters).each do |cmdb_masters_info|
|
628
644
|
cmdb_masters_info[:cmdb_masters].each do |cmdb, properties|
|
629
|
-
properties.each do |
|
630
|
-
raise "Property #{
|
631
|
-
|
632
|
-
|
633
|
-
|
645
|
+
properties.each do |itr_property|
|
646
|
+
raise "Property #{itr_property} have conflicting CMDB masters for #{node} declared in the configuration: #{cmdb_masters_cache[itr_property].class.name} and #{@cmdbs[cmdb].class.name}" if cmdb_masters_cache.key?(itr_property) && cmdb_masters_cache[itr_property] != @cmdbs[cmdb]
|
647
|
+
|
648
|
+
log_debug "CMDB master for #{node} / #{itr_property}: #{cmdb}"
|
649
|
+
raise "CMDB #{cmdb} is configured as a master for property #{itr_property} on node #{node} but it does not implement the needed API to retrieve it" unless (@cmdbs_per_property[itr_property] || []).include?(@cmdbs[cmdb]) || @cmdbs_others.include?(@cmdbs[cmdb])
|
650
|
+
|
651
|
+
cmdb_masters_cache[itr_property] = @cmdbs[cmdb]
|
634
652
|
end
|
635
653
|
end
|
636
654
|
end
|