hybrid_platforms_conductor 32.17.0 → 33.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +36 -0
  3. data/README.md +6 -3
  4. data/bin/check-node +0 -1
  5. data/bin/deploy +0 -1
  6. data/bin/get_impacted_nodes +0 -1
  7. data/bin/last_deploys +12 -8
  8. data/bin/nodes_to_deploy +6 -6
  9. data/bin/setup +6 -6
  10. data/bin/topograph +1 -1
  11. data/docs/config_dsl.md +45 -1
  12. data/docs/executables.md +6 -7
  13. data/docs/executables/check-node.md +3 -3
  14. data/docs/executables/deploy.md +3 -3
  15. data/docs/executables/dump_nodes_json.md +3 -3
  16. data/docs/executables/test.md +3 -3
  17. data/docs/executables/topograph.md +3 -3
  18. data/docs/gen/mermaid/README.md-0.png +0 -0
  19. data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
  20. data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
  21. data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
  22. data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
  23. data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
  24. data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
  25. data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
  26. data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
  27. data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
  28. data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
  29. data/docs/plugins.md +46 -0
  30. data/docs/plugins/connector/ssh.md +1 -1
  31. data/docs/plugins/log/remote_fs.md +26 -0
  32. data/docs/plugins/secrets_reader/cli.md +31 -0
  33. data/docs/plugins/secrets_reader/thycotic.md +46 -0
  34. data/docs/plugins/test/check_deploy_and_idempotence.md +1 -1
  35. data/docs/plugins/test/connection.md +1 -0
  36. data/docs/plugins/test/deploy_removes_root_access.md +1 -1
  37. data/docs/plugins/test/file_system.md +1 -0
  38. data/docs/plugins/test/hostname.md +1 -0
  39. data/docs/plugins/test/ip.md +1 -0
  40. data/docs/plugins/test/local_users.md +1 -0
  41. data/docs/plugins/test/mounts.md +1 -0
  42. data/docs/plugins/test/orphan_files.md +1 -0
  43. data/docs/plugins/test/ports.md +1 -0
  44. data/docs/plugins/test/spectre.md +1 -0
  45. data/docs/plugins/test/vulnerabilities.md +1 -0
  46. data/lib/hybrid_platforms_conductor/action.rb +4 -4
  47. data/lib/hybrid_platforms_conductor/actions_executor.rb +45 -43
  48. data/lib/hybrid_platforms_conductor/bitbucket.rb +5 -4
  49. data/lib/hybrid_platforms_conductor/cmd_runner.rb +13 -12
  50. data/lib/hybrid_platforms_conductor/cmdb.rb +2 -2
  51. data/lib/hybrid_platforms_conductor/common_config_dsl/bitbucket.rb +2 -1
  52. data/lib/hybrid_platforms_conductor/common_config_dsl/confluence.rb +2 -1
  53. data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +5 -4
  54. data/lib/hybrid_platforms_conductor/common_config_dsl/github.rb +4 -3
  55. data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb +2 -2
  56. data/lib/hybrid_platforms_conductor/config.rb +8 -4
  57. data/lib/hybrid_platforms_conductor/confluence.rb +1 -1
  58. data/lib/hybrid_platforms_conductor/connector.rb +5 -2
  59. data/lib/hybrid_platforms_conductor/core_extensions/cleanroom/fix_kwargs.rb +116 -0
  60. data/lib/hybrid_platforms_conductor/core_extensions/symbol/zero.rb +24 -0
  61. data/lib/hybrid_platforms_conductor/credentials.rb +39 -36
  62. data/lib/hybrid_platforms_conductor/current_dir_monitor.rb +4 -1
  63. data/lib/hybrid_platforms_conductor/deployer.rb +275 -224
  64. data/lib/hybrid_platforms_conductor/executable.rb +20 -15
  65. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/config.rb +10 -7
  66. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +1 -1
  67. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb +2 -2
  68. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +4 -4
  69. data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb +2 -0
  70. data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +45 -49
  71. data/lib/hybrid_platforms_conductor/hpc_plugins/log/my_log_plugin.rb.sample +100 -0
  72. data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb +180 -0
  73. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +68 -66
  74. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb +13 -0
  75. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb +39 -38
  76. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb +5 -4
  77. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb +43 -45
  78. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/podman.rb +18 -20
  79. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +118 -117
  80. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +39 -43
  81. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container +9 -13
  82. data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb +2 -2
  83. data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +28 -21
  84. data/lib/hybrid_platforms_conductor/hpc_plugins/report/stdout.rb +26 -22
  85. data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/cli.rb +77 -0
  86. data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/my_secrets_reader_plugin.rb.sample +46 -0
  87. data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb +90 -0
  88. data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb +3 -3
  89. data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb +4 -2
  90. data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb +3 -1
  91. data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +7 -21
  92. data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb +21 -19
  93. data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb +2 -2
  94. data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb +2 -2
  95. data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +21 -22
  96. data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +19 -21
  97. data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb +2 -3
  98. data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +3 -1
  99. data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb +2 -2
  100. data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +4 -2
  101. data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb +21 -22
  102. data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb +10 -12
  103. data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +9 -9
  104. data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +5 -3
  105. data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +5 -3
  106. data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +13 -10
  107. data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb +5 -3
  108. data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +5 -5
  109. data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +5 -5
  110. data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +7 -7
  111. data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +3 -3
  112. data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +27 -25
  113. data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/confluence.rb +2 -2
  114. data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/stdout.rb +8 -6
  115. data/lib/hybrid_platforms_conductor/io_router.rb +14 -13
  116. data/lib/hybrid_platforms_conductor/json_dumper.rb +2 -2
  117. data/lib/hybrid_platforms_conductor/log.rb +31 -0
  118. data/lib/hybrid_platforms_conductor/logger_helpers.rb +19 -16
  119. data/lib/hybrid_platforms_conductor/nodes_handler.rb +89 -71
  120. data/lib/hybrid_platforms_conductor/parallel_threads.rb +7 -11
  121. data/lib/hybrid_platforms_conductor/platform_handler.rb +7 -7
  122. data/lib/hybrid_platforms_conductor/platforms_handler.rb +5 -3
  123. data/lib/hybrid_platforms_conductor/plugin.rb +2 -2
  124. data/lib/hybrid_platforms_conductor/plugins.rb +14 -8
  125. data/lib/hybrid_platforms_conductor/provisioner.rb +4 -4
  126. data/lib/hybrid_platforms_conductor/report.rb +2 -2
  127. data/lib/hybrid_platforms_conductor/reports_handler.rb +3 -2
  128. data/lib/hybrid_platforms_conductor/secrets_reader.rb +31 -0
  129. data/lib/hybrid_platforms_conductor/services_handler.rb +32 -29
  130. data/lib/hybrid_platforms_conductor/test_only_remote_node.rb +18 -0
  131. data/lib/hybrid_platforms_conductor/test_report.rb +15 -18
  132. data/lib/hybrid_platforms_conductor/tests_runner.rb +116 -118
  133. data/lib/hybrid_platforms_conductor/thycotic.rb +28 -19
  134. data/lib/hybrid_platforms_conductor/topographer.rb +200 -190
  135. data/lib/hybrid_platforms_conductor/topographer/plugins/graphviz.rb +8 -8
  136. data/lib/hybrid_platforms_conductor/topographer/plugins/json.rb +4 -4
  137. data/lib/hybrid_platforms_conductor/version.rb +1 -1
  138. data/spec/hybrid_platforms_conductor_test.rb +33 -12
  139. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +18 -11
  140. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/interactive_spec.rb +2 -2
  141. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb +32 -21
  142. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +75 -49
  143. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/scp_spec.rb +27 -15
  144. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions_spec.rb +90 -59
  145. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connection_spec.rb +46 -44
  146. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/connectable_nodes_spec.rb +12 -8
  147. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb +4 -7
  148. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +21 -22
  149. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +23 -24
  150. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connectable_nodes_spec.rb +10 -6
  151. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +106 -75
  152. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +145 -126
  153. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +3 -3
  154. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +29 -25
  155. data/spec/hybrid_platforms_conductor_test/api/actions_executor/logging_spec.rb +167 -142
  156. data/spec/hybrid_platforms_conductor_test/api/actions_executor/parallel_spec.rb +272 -244
  157. data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +16 -16
  158. data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +36 -36
  159. data/spec/hybrid_platforms_conductor_test/api/config_spec.rb +24 -22
  160. data/spec/hybrid_platforms_conductor_test/api/deployer/check_spec.rb +4 -2
  161. data/spec/hybrid_platforms_conductor_test/api/deployer/config_dsl_spec.rb +43 -5
  162. data/spec/hybrid_platforms_conductor_test/api/deployer/deploy_spec.rb +199 -216
  163. data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +223 -0
  164. data/spec/hybrid_platforms_conductor_test/api/deployer/parse_deploy_output_spec.rb +55 -59
  165. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +36 -62
  166. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/podman_spec.rb +17 -17
  167. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/config_dsl_spec.rb +4 -4
  168. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/create_spec.rb +44 -51
  169. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/destroy_spec.rb +3 -3
  170. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/ip_spec.rb +12 -16
  171. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/destroy_vm_spec.rb +31 -19
  172. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +324 -266
  173. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/ips_assignment_spec.rb +89 -61
  174. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/other_lxc_containers_resources_spec.rb +117 -93
  175. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/pve_node_resources_spec.rb +71 -54
  176. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +10 -8
  177. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb +80 -60
  178. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/start_spec.rb +1 -1
  179. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb +1 -1
  180. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/stop_spec.rb +1 -1
  181. data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/cli_spec.rb +64 -0
  182. data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/thycotic_spec.rb +268 -0
  183. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/config_spec.rb +8 -10
  184. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +33 -24
  185. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_keys_spec.rb +64 -51
  186. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/platform_handlers_spec.rb +3 -3
  187. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +50 -51
  188. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +91 -81
  189. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +14 -16
  190. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/git_diff_impacts_spec.rb +51 -75
  191. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/nodes_selectors_spec.rb +35 -26
  192. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/platform_handlers_plugins_api_spec.rb +24 -16
  193. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/several_platforms_spec.rb +29 -19
  194. data/spec/hybrid_platforms_conductor_test/api/platform_handler_spec.rb +4 -4
  195. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb +2 -2
  196. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb +6 -6
  197. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb +57 -99
  198. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +4 -4
  199. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +32 -35
  200. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +10 -10
  201. data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +38 -27
  202. data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +46 -52
  203. data/spec/hybrid_platforms_conductor_test/api/reports_handler_spec.rb +2 -2
  204. data/spec/hybrid_platforms_conductor_test/api/services_handler/actions_to_deploy_spec.rb +90 -58
  205. data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +38 -34
  206. data/spec/hybrid_platforms_conductor_test/api/services_handler/log_info_spec.rb +11 -9
  207. data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +193 -171
  208. data/spec/hybrid_platforms_conductor_test/api/services_handler/parse_deploy_output_spec.rb +66 -54
  209. data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +147 -133
  210. data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +69 -49
  211. data/spec/hybrid_platforms_conductor_test/api/tests_runner/global_spec.rb +5 -4
  212. data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +8 -5
  213. data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_spec.rb +8 -5
  214. data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +30 -27
  215. data/spec/hybrid_platforms_conductor_test/api/tests_runner/platform_spec.rb +12 -9
  216. data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +48 -47
  217. data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/bitbucket_conf_spec.rb +5 -5
  218. data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/github_ci_spec.rb +5 -5
  219. data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_reports_plugins/confluence_spec.rb +5 -5
  220. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +9 -9
  221. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb2.rb → test_cmdb_2.rb} +6 -6
  222. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
  223. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb_others2.rb → test_cmdb_others_2.rb} +2 -2
  224. data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
  225. data/spec/hybrid_platforms_conductor_test/executables/{check-node_spec.rb → check_node_spec.rb} +4 -6
  226. data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +4 -6
  227. data/spec/hybrid_platforms_conductor_test/executables/get_impacted_nodes_spec.rb +76 -77
  228. data/spec/hybrid_platforms_conductor_test/executables/last_deploys_spec.rb +159 -113
  229. data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +299 -160
  230. data/spec/hybrid_platforms_conductor_test/executables/options/actions_executor_spec.rb +4 -6
  231. data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +3 -5
  232. data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb +8 -8
  233. data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +12 -196
  234. data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +9 -10
  235. data/spec/hybrid_platforms_conductor_test/executables/options/nodes_selectors_spec.rb +9 -10
  236. data/spec/hybrid_platforms_conductor_test/executables/options/reports_handler_spec.rb +1 -1
  237. data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +22 -22
  238. data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +22 -16
  239. data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +32 -32
  240. data/spec/hybrid_platforms_conductor_test/executables/ssh_config_spec.rb +7 -9
  241. data/spec/hybrid_platforms_conductor_test/executables/test_spec.rb +3 -5
  242. data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +2 -2
  243. data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +4 -3
  244. data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
  245. data/spec/hybrid_platforms_conductor_test/helpers/config_helpers.rb +1 -1
  246. data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +12 -13
  247. data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +245 -56
  248. data/spec/hybrid_platforms_conductor_test/helpers/executables_helpers.rb +11 -11
  249. data/spec/hybrid_platforms_conductor_test/helpers/nodes_handler_helpers.rb +1 -1
  250. data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +39 -28
  251. data/spec/hybrid_platforms_conductor_test/helpers/plugins_helpers.rb +1 -1
  252. data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +86 -111
  253. data/spec/hybrid_platforms_conductor_test/helpers/reports_handler_helpers.rb +1 -1
  254. data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +3 -3
  255. data/spec/hybrid_platforms_conductor_test/helpers/services_handler_helpers.rb +1 -1
  256. data/spec/hybrid_platforms_conductor_test/helpers/tests_runner_helpers.rb +1 -1
  257. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id1.rb → test_plugin_id_1.rb} +0 -0
  258. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id2.rb → test_plugin_id_2.rb} +0 -0
  259. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/test_plugin_type/{test_plugin_id3.rb → test_plugin_id_3.rb} +0 -0
  260. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/{test_plugin_type2/test_plugin_id4.rb → test_plugin_type_2/test_plugin_id_4.rb} +0 -0
  261. data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/test.rb +1 -1
  262. data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/{test2.rb → test_2.rb} +0 -0
  263. data/spec/hybrid_platforms_conductor_test/rubocop_spec.rb +31 -0
  264. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json +3 -3
  265. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json +3 -3
  266. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb +1 -0
  267. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb +1 -0
  268. data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +952 -0
  269. data/spec/hybrid_platforms_conductor_test/test_connector.rb +3 -3
  270. data/spec/hybrid_platforms_conductor_test/test_log_no_read_plugin.rb +84 -0
  271. data/spec/hybrid_platforms_conductor_test/test_log_plugin.rb +105 -0
  272. data/spec/hybrid_platforms_conductor_test/test_plugins/global.rb +1 -0
  273. data/spec/hybrid_platforms_conductor_test/test_plugins/node.rb +1 -0
  274. data/spec/hybrid_platforms_conductor_test/test_plugins/node_check.rb +1 -0
  275. data/spec/hybrid_platforms_conductor_test/test_plugins/platform.rb +1 -0
  276. data/spec/hybrid_platforms_conductor_test/test_plugins/several_checks.rb +2 -2
  277. data/spec/hybrid_platforms_conductor_test/test_secrets_reader_plugin.rb +45 -0
  278. data/spec/hybrid_platforms_conductor_test/tests_report_plugin.rb +5 -6
  279. data/spec/spec_helper.rb +17 -18
  280. data/tools/check_md +16 -20
  281. data/tools/generate_mermaid +1 -1
  282. metadata +195 -144
  283. data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +0 -916
@@ -1,4 +1,5 @@
1
1
  require 'nokogiri'
2
+ require 'hybrid_platforms_conductor/test_only_remote_node'
2
3
 
3
4
  module HybridPlatformsConductor
4
5
 
@@ -17,7 +18,7 @@ module HybridPlatformsConductor
17
18
  # Each final OVAL URL can be directly an XML file, either raw or compressed with .gz or .bz2.
18
19
  # This is useful to follow repository links, such as jFrog or web servers serving common file systems structure storing several versions of the OVAL file.
19
20
  # * *reported_severities* (Array<String> or nil): List of severities to report, if any (use Unknown when the severity is not known), or nil for all [default: nil]
20
- class Vulnerabilities < HybridPlatformsConductor::Test
21
+ class Vulnerabilities < TestOnlyRemoteNode
21
22
 
22
23
  # Known compression methods, per file extension, and their corresponding uncompress bash script
23
24
  KNOWN_COMPRESSIONS = {
@@ -47,8 +48,9 @@ module HybridPlatformsConductor
47
48
  # Follow the last link recursively until we find a .xml or compressed file
48
49
  current_url = artifactory_url
49
50
  loop do
50
- current_url = "#{current_url}#{current_url.end_with?('/') ? '' : '/'}#{Nokogiri::HTML.parse(URI.open(current_url)).css('a').last['href']}"
51
+ current_url = "#{current_url}#{current_url.end_with?('/') ? '' : '/'}#{Nokogiri::HTML.parse(URI.parse(current_url).open).css('a').last['href']}"
51
52
  break if current_url.end_with?('.xml') || KNOWN_COMPRESSIONS.keys.any? { |file_ext| current_url.end_with?(".#{file_ext}") }
53
+
52
54
  log_debug "Follow last link to #{current_url}"
53
55
  end
54
56
  current_url
@@ -56,7 +58,7 @@ module HybridPlatformsConductor
56
58
  )
57
59
  # TODO: Access the user correctly when the user notion will be moved out of the ssh connector
58
60
  sudo = @deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "
59
- Hash[urls.map do |url|
61
+ urls.map do |url|
60
62
  # 1. Get the OVAL file on the node to be tested (uncompress it if needed)
61
63
  # 2. Make sure oscap is installed
62
64
  # 3. Generate the report for this OVAL file using oscap
@@ -66,13 +68,13 @@ module HybridPlatformsConductor
66
68
  packages_to_install = []
67
69
  KNOWN_COMPRESSIONS.each do |file_ext, compress_info|
68
70
  file_ending = ".#{file_ext}"
69
- if local_oval_file.end_with?(file_ending)
70
- uncompress_cmds << compress_info[:cmd].call(local_oval_file)
71
- packages_to_install.concat(compress_info[:packages])
72
- local_oval_file = File.basename(local_oval_file, file_ending)
73
- end
71
+ next unless local_oval_file.end_with?(file_ending)
72
+
73
+ uncompress_cmds << compress_info[:cmd].call(local_oval_file)
74
+ packages_to_install.concat(compress_info[:packages])
75
+ local_oval_file = File.basename(local_oval_file, file_ending)
74
76
  end
75
- cmds = <<~EOS
77
+ cmds = <<~EO_BASH
76
78
  set -e
77
79
  #{
78
80
  case image
@@ -84,7 +86,7 @@ module HybridPlatformsConductor
84
86
  # On Debian 10 we have to compile it from sources, as the packaged official version has core dumps.
85
87
  # cf https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1688223.html
86
88
  # TODO: Remove this Debian 10 specificity when the official libopenscap8 will be corrected
87
- <<~EOS2
89
+ <<~EO_BASH2
88
90
  if [ ! -x "$(command -v oscap)" ] || [ "$(oscap --version | head -n 1 | awk '{print $6}')" != "1.3.4" ]; then
89
91
  rm -rf openscap
90
92
  git clone --recurse-submodules https://github.com/OpenSCAP/openscap.git
@@ -96,7 +98,7 @@ module HybridPlatformsConductor
96
98
  #{sudo}make install
97
99
  fi
98
100
  #{sudo}apt install -y wget #{packages_to_install.join(' ')}
99
- EOS2
101
+ EO_BASH2
100
102
  else
101
103
  raise "Non supported image: #{image}. Please adapt this test's code."
102
104
  end
@@ -110,7 +112,7 @@ module HybridPlatformsConductor
110
112
  echo "===== RESULTS ====="
111
113
  cat "#{local_oval_file}.results.xml"
112
114
  cd ..
113
- EOS
115
+ EO_BASH
114
116
  [
115
117
  cmds,
116
118
  {
@@ -119,21 +121,21 @@ module HybridPlatformsConductor
119
121
  if idx_results.nil?
120
122
  error 'No results given by the oscap run', stdout.join("\n")
121
123
  else
122
- results = Nokogiri::XML(stdout[idx_results + 1..-1].join("\n"))
124
+ results = Nokogiri::XML(stdout[idx_results + 1..].join("\n"))
123
125
  results.remove_namespaces!
124
126
  oval_definitions = results.css('oval_results oval_definitions definitions definition')
125
127
  results.css('results system definitions definition').each do |definition_xml|
126
- if definition_xml['result'] == 'true'
127
- # Just found an OVAL item to be patched.
128
- definition_id = definition_xml['definition_id']
129
- oval_definition = oval_definitions.find { |el| el['id'] == definition_id }
130
- # We don't forcefully want to report all missing patches. Only the most important ones.
131
- severity = oval_definition.css('metadata advisory severity').text
132
- severity = 'Unknown' if severity.empty?
133
- if !oval_info.key?('reported_severities') || oval_info['reported_severities'].include?(severity)
134
- # Only consider the first line of the description, as sometimes it's very long
135
- error "Non-patched #{severity} vulnerability found: #{oval_definition.css('metadata title').text} - #{oval_definition.css('metadata description').text.split("\n").first}"
136
- end
128
+ next unless definition_xml['result'] == 'true'
129
+
130
+ # Just found an OVAL item to be patched.
131
+ definition_id = definition_xml['definition_id']
132
+ oval_definition = oval_definitions.find { |el| el['id'] == definition_id }
133
+ # We don't forcefully want to report all missing patches. Only the most important ones.
134
+ severity = oval_definition.css('metadata advisory severity').text
135
+ severity = 'Unknown' if severity.empty?
136
+ if !oval_info.key?('reported_severities') || oval_info['reported_severities'].include?(severity)
137
+ # Only consider the first line of the description, as sometimes it's very long
138
+ error "Non-patched #{severity} vulnerability found: #{oval_definition.css('metadata title').text} - #{oval_definition.css('metadata description').text.split("\n").first}"
137
139
  end
138
140
  end
139
141
  end
@@ -142,7 +144,7 @@ module HybridPlatformsConductor
142
144
  timeout: 240
143
145
  }
144
146
  ]
145
- end]
147
+ end.to_h
146
148
  else
147
149
  error "No OVAL file defined for image #{image} at #{oval_file}"
148
150
  {}
@@ -12,7 +12,7 @@ module HybridPlatformsConductor
12
12
  # Report tests results on a generated Confluence page
13
13
  class Confluence < HybridPlatformsConductor::TestReport
14
14
 
15
- self.extend_config_dsl_with CommonConfigDsl::Confluence, :init_confluence
15
+ extend_config_dsl_with CommonConfigDsl::Confluence, :init_confluence
16
16
 
17
17
  # Maximum errors to be reported by item
18
18
  MAX_ERROR_ITEMS_DISPLAYED = 10
@@ -34,7 +34,7 @@ module HybridPlatformsConductor
34
34
  at('h1:contains("Evolution")').
35
35
  search('~ structured-macro:first-of-type').
36
36
  css('table td').
37
- map { |td_element| td_element.text }.
37
+ map(&:text).
38
38
  each_slice(2).
39
39
  to_a.
40
40
  map { |(time_str, value_str)| [Time.parse("#{time_str} UTC"), value_str.to_f] }
@@ -97,18 +97,20 @@ module HybridPlatformsConductor
97
97
  table << [
98
98
  nodes_list,
99
99
  nodes_info[:nodes].size,
100
- nodes_info[:nodes].empty? ? '' : "#{(nodes_info[:tested_nodes].size*100.0/nodes_info[:nodes].size).to_i} %",
100
+ nodes_info[:nodes].empty? ? '' : "#{(nodes_info[:tested_nodes].size * 100.0 / nodes_info[:nodes].size).to_i} %",
101
101
  nodes_info[:tested_nodes].empty? ? '' : "#{((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error_as_expected].size) * 100.0 / nodes_info[:tested_nodes].size).to_i} %",
102
102
  nodes_info[:tested_nodes].empty? ? '' : "#{((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * 100.0 / nodes_info[:tested_nodes].size).to_i} %",
103
- nodes_info[:nodes].empty? ? '' :
103
+ if nodes_info[:nodes].empty?
104
+ ''
105
+ else
104
106
  ('=' * ((nodes_info[:tested_nodes_in_error_as_expected].size * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).yellow.bold +
105
- ('=' * (((nodes_info[:tested_nodes_in_error].size - nodes_info[:tested_nodes_in_error_as_expected].size).abs * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).red.bold +
106
- ('=' * (((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).green.bold +
107
- ('=' * (((nodes_info[:nodes].size - nodes_info[:tested_nodes].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).white.bold
107
+ ('=' * (((nodes_info[:tested_nodes_in_error].size - nodes_info[:tested_nodes_in_error_as_expected].size).abs * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).red.bold +
108
+ ('=' * (((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).green.bold +
109
+ ('=' * (((nodes_info[:nodes].size - nodes_info[:tested_nodes].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).white.bold
110
+ end
108
111
  ]
109
112
  end
110
113
  end)
111
-
112
114
  end
113
115
 
114
116
  end
@@ -30,6 +30,7 @@ module HybridPlatformsConductor
30
30
  # Start routing messages asynchronously
31
31
  def start
32
32
  raise 'IO router is already started. Can\'t start it again.' unless @reading_thread.nil?
33
+
33
34
  @end_read = false
34
35
  # Create a thread to handle routes asynchronously
35
36
  @reading_thread = Thread.new do
@@ -37,22 +38,21 @@ module HybridPlatformsConductor
37
38
  need_to_stop = @end_read.clone
38
39
  data_found = false
39
40
  @routes.each do |src_io, dst_ios|
40
- if src_io.is_a?(Queue)
41
- queue_size = src_io.size
42
- if queue_size > 0
43
- # There is data to be read from src_io
44
- data_found = true
45
- data_chunk_str = queue_size.times.map { src_io.pop }.join
46
- dst_ios.each do |dst_io|
47
- dst_io << data_chunk_str
48
- dst_io.flush if dst_io.respond_to?(:flush)
49
- end
50
- end
51
- else
52
- raise "Unknown type of source IO: #{src_io}"
41
+ raise "Unknown type of source IO: #{src_io}" unless src_io.is_a?(Queue)
42
+
43
+ queue_size = src_io.size
44
+ next unless queue_size.positive?
45
+
46
+ # There is data to be read from src_io
47
+ data_found = true
48
+ data_chunk_str = queue_size.times.map { src_io.pop }.join
49
+ dst_ios.each do |dst_io|
50
+ dst_io << data_chunk_str
51
+ dst_io.flush if dst_io.respond_to?(:flush)
53
52
  end
54
53
  end
55
54
  break if need_to_stop && !data_found
55
+
56
56
  sleep 0.1
57
57
  end
58
58
  end
@@ -61,6 +61,7 @@ module HybridPlatformsConductor
61
61
  # Stop routing messages asynchronously
62
62
  def stop
63
63
  raise 'IO router is not started. Can\'t stop it.' if @reading_thread.nil?
64
+
64
65
  @end_read = true
65
66
  @reading_thread.join
66
67
  end
@@ -23,7 +23,7 @@ module HybridPlatformsConductor
23
23
  # * *config* (Config): Config to be used. [default = Config.new]
24
24
  # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default = NodesHandler.new]
25
25
  # * *deployer* (Deployer): Deployer to be used. [default = Deployer.new]
26
- def initialize(logger: Logger.new(STDOUT), logger_stderr: Logger.new(STDERR), config: Config.new, nodes_handler: NodesHandler.new, deployer: Deployer.new)
26
+ def initialize(logger: Logger.new($stdout), logger_stderr: Logger.new($stderr), config: Config.new, nodes_handler: NodesHandler.new, deployer: Deployer.new)
27
27
  init_loggers(logger, logger_stderr)
28
28
  @config = config
29
29
  @nodes_handler = nodes_handler
@@ -74,7 +74,7 @@ module HybridPlatformsConductor
74
74
  out "[ #{node} ] - Error while dumping JSON. Check #{stdout_file_name}"
75
75
  else
76
76
  json_file_name = "#{@dump_dir}/#{node}.json"
77
- File.write(json_file_name, stdout[dump_begin_idx+1..dump_end_idx-1].join("\n"))
77
+ File.write(json_file_name, stdout[dump_begin_idx + 1..dump_end_idx - 1].join("\n"))
78
78
  out "[ #{node} ] - OK. Check #{json_file_name}"
79
79
  end
80
80
  else
@@ -0,0 +1,31 @@
1
+ require 'hybrid_platforms_conductor/logger_helpers'
2
+ require 'hybrid_platforms_conductor/plugin'
3
+
4
+ module HybridPlatformsConductor
5
+
6
+ # Ancestor of all log plugins
7
+ class Log < Plugin
8
+
9
+ # Constructor
10
+ #
11
+ # Parameters::
12
+ # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
13
+ # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
14
+ # * *config* (Config): Config to be used. [default: Config.new]
15
+ # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
16
+ # * *actions_executor* (ActionsExecutor): Actions executor to be used. [default: ActionsExecutor.new]
17
+ def initialize(
18
+ logger: Logger.new($stdout),
19
+ logger_stderr: Logger.new($stderr),
20
+ config: Config.new,
21
+ nodes_handler: NodesHandler.new,
22
+ actions_executor: ActionsExecutor.new
23
+ )
24
+ super(logger: logger, logger_stderr: logger_stderr, config: config)
25
+ @nodes_handler = nodes_handler
26
+ @actions_executor = actions_executor
27
+ end
28
+
29
+ end
30
+
31
+ end
@@ -55,25 +55,27 @@ module HybridPlatformsConductor
55
55
 
56
56
  # Make sure if the current line is not flushed we still do it
57
57
  def flush
58
- unless @current_line.nil?
59
- @stream << @current_line
60
- @current_line = nil
61
- end
58
+ return if @current_line.nil?
59
+
60
+ @stream << @current_line
61
+ @current_line = nil
62
62
  end
63
63
 
64
64
  end
65
65
 
66
66
  class << self
67
+
67
68
  attr_reader :progress_bar_semaphore
69
+
68
70
  end
69
71
  # Make sure the progress bar setting is protected by a Mutex
70
72
  @progress_bar_semaphore = Mutex.new
71
73
 
72
74
  # Sorted list of levels and their corresponding modifiers.
73
75
  LEVELS_MODIFIERS = {
74
- fatal: [:red, :bold],
75
- error: [:red, :bold],
76
- warn: [:yellow, :bold],
76
+ fatal: %i[red bold],
77
+ error: %i[red bold],
78
+ warn: %i[yellow bold],
77
79
  info: [:white],
78
80
  debug: [:white],
79
81
  unknown: [:white]
@@ -82,7 +84,7 @@ module HybridPlatformsConductor
82
84
  # List of levels that will output on stderr
83
85
  LEVELS_TO_STDERR = %i[warn error fatal]
84
86
 
85
- LEVELS_MODIFIERS.keys.each do |level|
87
+ LEVELS_MODIFIERS.each_key do |level|
86
88
  define_method("log_#{level}") do |message|
87
89
  (LEVELS_TO_STDERR.include?(level) ? @logger_stderr : @logger).send(
88
90
  level,
@@ -105,10 +107,10 @@ module HybridPlatformsConductor
105
107
  # Set loggers to the desired format
106
108
  def set_loggers_format
107
109
  [@logger, @logger_stderr].each do |logger|
108
- logger.formatter = proc do |severity, datetime, progname, msg|
110
+ logger.formatter = proc do |severity, _datetime, progname, msg|
109
111
  # If the message already has control characters, don't colorize it
110
112
  keep_original_color = msg.include? "\u001b"
111
- message = "[#{Time.now.utc.strftime('%F %T')} (PID #{$$} / TID #{Thread.current.object_id})] #{severity.rjust(5)} - [ #{progname} ] - "
113
+ message = "[#{Time.now.utc.strftime('%F %T')} (PID #{$PROCESS_ID} / TID #{Thread.current.object_id})] #{severity.rjust(5)} - [ #{progname} ] - "
112
114
  message << "#{msg}\n" unless keep_original_color
113
115
  LEVELS_MODIFIERS[severity.downcase.to_sym].each do |modifier|
114
116
  message = message.send(modifier)
@@ -282,8 +284,8 @@ module HybridPlatformsConductor
282
284
  yield progress_bar
283
285
  ensure
284
286
  LoggerHelpers.progress_bar_semaphore.synchronize do
285
- self.stdout_device.flush
286
- self.stderr_device.flush
287
+ stdout_device.flush
288
+ stderr_device.flush
287
289
  self.stdout_device = previous_stdout_device unless previous_stdout_device.nil?
288
290
  self.stderr_device = previous_stderr_device unless previous_stderr_device.nil?
289
291
  end
@@ -298,15 +300,16 @@ module HybridPlatformsConductor
298
300
  def stdouts_to_s
299
301
  messages = []
300
302
  {
301
- 'STDOUT' => self.stdout_device,
302
- 'STDERR' => self.stderr_device
303
+ 'STDOUT' => stdout_device,
304
+ 'STDERR' => stderr_device
303
305
  }.each do |name, device|
304
- if device.is_a?(File)
306
+ case device
307
+ when File
305
308
  if File.exist?(device.path)
306
309
  content = File.read(device.path).strip
307
310
  messages << "----- #{name} BEGIN - #{device.path} -----\n#{content}\n----- #{name} END - #{device.path} -----" unless content.empty?
308
311
  end
309
- elsif device.is_a?(StringIO)
312
+ when StringIO
310
313
  content = device.string
311
314
  messages << "----- #{name} BEGIN -----\n#{content}\n----- #{name} END -----" unless content.empty?
312
315
  end
@@ -40,7 +40,7 @@ module HybridPlatformsConductor
40
40
  # * *master_cmdbs_info* (Hash< Symbol, Symbol or Array<Symbol> >): List of metadata properties (or single one) per CMDB name considered as master for those properties.
41
41
  def master_cmdbs(master_cmdbs_info)
42
42
  @cmdb_masters << {
43
- cmdb_masters: Hash[master_cmdbs_info.map { |cmdb, properties| [cmdb, properties.is_a?(Array) ? properties : [properties]] }],
43
+ cmdb_masters: master_cmdbs_info.transform_values { |properties| properties.is_a?(Array) ? properties : [properties] },
44
44
  nodes_selectors_stack: current_nodes_selectors_stack
45
45
  }
46
46
  end
@@ -60,7 +60,8 @@ module HybridPlatformsConductor
60
60
 
61
61
  Config.extend_config_dsl_with ConfigDSLExtension, :init_nodes_handler_config
62
62
 
63
- include LoggerHelpers, ParallelThreads
63
+ include ParallelThreads
64
+ include LoggerHelpers
64
65
 
65
66
  class GitError < RuntimeError
66
67
  end
@@ -74,8 +75,8 @@ module HybridPlatformsConductor
74
75
  # * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
75
76
  # * *platforms_handler* (PlatformsHandler): Platforms Handler to be used. [default: PlatformsHandler.new]
76
77
  def initialize(
77
- logger: Logger.new(STDOUT),
78
- logger_stderr: Logger.new(STDERR),
78
+ logger: Logger.new($stdout),
79
+ logger_stderr: Logger.new($stderr),
79
80
  config: Config.new,
80
81
  cmd_runner: CmdRunner.new,
81
82
  platforms_handler: PlatformsHandler.new
@@ -111,11 +112,11 @@ module HybridPlatformsConductor
111
112
  )
112
113
  @cmdbs_others << cmdb if cmdb.respond_to?(:get_others)
113
114
  cmdb.methods.each do |method|
114
- if method.to_s =~ /^get_(.*)$/
115
- property = $1.to_sym
116
- @cmdbs_per_property[property] = [] unless @cmdbs_per_property.key?(property)
117
- @cmdbs_per_property[property] << cmdb
118
- end
115
+ next unless method.to_s =~ /^get_(.*)$/
116
+
117
+ property = Regexp.last_match(1).to_sym
118
+ @cmdbs_per_property[property] = [] unless @cmdbs_per_property.key?(property)
119
+ @cmdbs_per_property[property] << cmdb
119
120
  end
120
121
  cmdb
121
122
  end
@@ -133,13 +134,17 @@ module HybridPlatformsConductor
133
134
  # Register all known nodes for this platform
134
135
  platform.known_nodes.each do |node|
135
136
  raise "Can't register #{node} to platform #{platform.repository_path}, as it is already defined in platform #{@nodes_platform[node].repository_path}." if @nodes_platform.key?(node)
137
+
136
138
  @nodes_platform[node] = platform
137
139
  end
138
140
  # Register all known nodes lists
141
+ next unless platform.respond_to?(:known_nodes_lists)
142
+
139
143
  platform.known_nodes_lists.each do |nodes_list|
140
144
  raise "Can't register nodes list #{nodes_list} to platform #{platform.repository_path}, as it is already defined in platform #{@nodes_list_platform[nodes_list].repository_path}." if @nodes_list_platform.key?(nodes_list)
145
+
141
146
  @nodes_list_platform[nodes_list] = platform
142
- end if platform.respond_to?(:known_nodes_lists)
147
+ end
143
148
  end
144
149
  end
145
150
 
@@ -147,7 +152,7 @@ module HybridPlatformsConductor
147
152
  #
148
153
  # Parameters::
149
154
  # * *options_parser* (OptionParser): The option parser to complete
150
- def options_parse(options_parser, parallel: true)
155
+ def options_parse(options_parser)
151
156
  options_parser.separator ''
152
157
  options_parser.separator 'Nodes handler options:'
153
158
  options_parser.on('-o', '--show-nodes', 'Display the list of possible nodes and exit') do
@@ -221,6 +226,7 @@ module HybridPlatformsConductor
221
226
  platform_name, from_commit, to_commit, flags = nodes_git_impact.split(':')
222
227
  flags = (flags || '').split(',')
223
228
  raise "Invalid platform in --nodes-git-impact: #{platform_name}. Possible values are: #{platform_names.join(', ')}." unless platform_names.include?(platform_name)
229
+
224
230
  nodes_selector = { platform: platform_name }
225
231
  nodes_selector[:from_commit] = from_commit if from_commit && !from_commit.empty?
226
232
  nodes_selector[:to_commit] = to_commit if to_commit && !to_commit.empty?
@@ -329,7 +335,7 @@ module HybridPlatformsConductor
329
335
  # * *block* (Proc): Code block given to the call
330
336
  def method_missing(method, *args, &block)
331
337
  if method.to_s =~ /^get_(.*)_of$/
332
- property = $1.to_sym
338
+ property = Regexp.last_match(1).to_sym
333
339
  # Define the method so that we don't go trough method_missing next time (more efficient).
334
340
  define_property_method_for(property)
335
341
  # Then call it
@@ -341,6 +347,15 @@ module HybridPlatformsConductor
341
347
  end
342
348
  end
343
349
 
350
+ # Make sure we register the methods we handle in method_missing
351
+ #
352
+ # Parameters::
353
+ # * *name* (Symbol): The missing method name
354
+ # * *include_private* (Boolean): Should we include private methods in the search?
355
+ def respond_to_missing?(name, include_private)
356
+ name.to_s =~ /^get_(.*)_of$/ || super
357
+ end
358
+
344
359
  # Prefetch some metadata properties for a given list of nodes.
345
360
  # Useful for performance reasons when clients know they will need to use a lot of properties on nodes.
346
361
  # Keep a thread-safe memory cache of it.
@@ -352,52 +367,51 @@ module HybridPlatformsConductor
352
367
  (properties.is_a?(Symbol) ? [properties] : properties).each do |property|
353
368
  # Gather the list of nodes missing this property
354
369
  missing_nodes = nodes.select { |node| !@metadata.key?(node) || !@metadata[node].key?(property) }
355
- unless missing_nodes.empty?
356
- # Query the CMDBs having first the get_<property> method, then the ones having the get_others method till we have our property set for all missing nodes
357
- # Metadata being retrieved by the different CMDBs, per node
370
+ next if missing_nodes.empty?
371
+
372
+ # Query the CMDBs having first the get_<property> method, then the ones having the get_others method till we have our property set for all missing nodes
373
+ # Metadata being retrieved by the different CMDBs, per node
374
+ # Hash< String, Object >
375
+ updated_metadata = {}
376
+ (
377
+ (@cmdbs_per_property.key?(property) ? @cmdbs_per_property[property] : []).map { |cmdb| [cmdb, property] } +
378
+ @cmdbs_others.map { |cmdb| [cmdb, :others] }
379
+ ).each do |(cmdb, cmdb_property)|
380
+ # If among the missing nodes some of them have some master CMDB declared for this property, filter them out unless we are dealing with their master CMDB.
381
+ nodes_to_query = missing_nodes.select do |node|
382
+ master_cmdb = cmdb_master_for(node, property)
383
+ master_cmdb.nil? || master_cmdb == cmdb
384
+ end
385
+ next if nodes_to_query.empty?
386
+
387
+ # Check first if this property depends on other ones for this cmdb
388
+ if cmdb.respond_to?(:property_dependencies)
389
+ property_deps = cmdb.property_dependencies
390
+ prefetch_metadata_of nodes_to_query, property_deps[property] if property_deps.key?(property)
391
+ end
392
+ # Property values, per node name
358
393
  # Hash< String, Object >
359
- updated_metadata = {}
360
- (
361
- (@cmdbs_per_property.key?(property) ? @cmdbs_per_property[property] : []).map { |cmdb| [cmdb, property] } +
362
- @cmdbs_others.map { |cmdb| [cmdb, :others] }
363
- ).each do |(cmdb, cmdb_property)|
364
- # If among the missing nodes some of them have some master CMDB declared for this property, filter them out unless we are dealing with their master CMDB.
365
- nodes_to_query = missing_nodes.select do |node|
366
- master_cmdb = cmdb_master_for(node, property)
367
- master_cmdb.nil? || master_cmdb == cmdb
368
- end
369
- unless nodes_to_query.empty?
370
- # Check first if this property depends on other ones for this cmdb
371
- if cmdb.respond_to?(:property_dependencies)
372
- property_deps = cmdb.property_dependencies
373
- prefetch_metadata_of nodes_to_query, property_deps[property] if property_deps.key?(property)
374
- end
375
- # Property values, per node name
376
- # Hash< String, Object >
377
- metadata_from_cmdb = Hash[
378
- cmdb.send("get_#{cmdb_property}".to_sym, nodes_to_query, @metadata.slice(*nodes_to_query)).map do |node, cmdb_result|
379
- [node, cmdb_property == :others ? cmdb_result[property] : cmdb_result]
380
- end
381
- ].compact
382
- cmdb_log_header = "[CMDB #{cmdb.class.name.split('::').last}.#{cmdb_property}] -"
383
- log_debug "#{cmdb_log_header} Query property #{property} for #{nodes_to_query.size} nodes (#{nodes_to_query[0..7].join(', ')}...) => Found metadata for #{metadata_from_cmdb.size} nodes."
384
- updated_metadata.merge!(metadata_from_cmdb) do |node, existing_value, new_value|
385
- raise "#{cmdb_log_header} Returned a conflicting value for metadata #{property} of node #{node}: #{new_value} whereas the value was already set to #{existing_value}" if !existing_value.nil? && new_value != existing_value
386
- new_value
387
- end
388
- end
394
+ metadata_from_cmdb = cmdb.send("get_#{cmdb_property}".to_sym, nodes_to_query, @metadata.slice(*nodes_to_query)).transform_values do |cmdb_result|
395
+ cmdb_property == :others ? cmdb_result[property] : cmdb_result
396
+ end.compact
397
+ cmdb_log_header = "[CMDB #{cmdb.class.name.split('::').last}.#{cmdb_property}] -"
398
+ log_debug "#{cmdb_log_header} Query property #{property} for #{nodes_to_query.size} nodes (#{nodes_to_query[0..7].join(', ')}...) => Found metadata for #{metadata_from_cmdb.size} nodes."
399
+ updated_metadata.merge!(metadata_from_cmdb) do |node, existing_value, new_value|
400
+ raise "#{cmdb_log_header} Returned a conflicting value for metadata #{property} of node #{node}: #{new_value} whereas the value was already set to #{existing_value}" if !existing_value.nil? && new_value != existing_value
401
+
402
+ new_value
389
403
  end
390
- # Avoid conflicts in metadata while merging and make sure this update is thread-safe
391
- # As @metadata is only appending data and never deleting it, protecting the update only is enough.
392
- # At worst several threads will query several times the same CMDBs to update the same data several times.
393
- # If we also want to be thread-safe in this regard, we should protect the whole CMDB call with mutexes, at the granularity of the node + property bein read.
394
- @metadata_mutex.synchronize do
395
- missing_nodes.each do |node|
396
- @metadata[node] = {} unless @metadata.key?(node)
397
- # Here, explicitely store nil if nothing has been found for a node because we know there is no value to be fetched.
398
- # This way we won't query again all CMDBs thanks to the cache.
399
- @metadata[node][property] = updated_metadata[node]
400
- end
404
+ end
405
+ # Avoid conflicts in metadata while merging and make sure this update is thread-safe
406
+ # As @metadata is only appending data and never deleting it, protecting the update only is enough.
407
+ # At worst several threads will query several times the same CMDBs to update the same data several times.
408
+ # If we also want to be thread-safe in this regard, we should protect the whole CMDB call with mutexes, at the granularity of the node + property bein read.
409
+ @metadata_mutex.synchronize do
410
+ missing_nodes.each do |node|
411
+ @metadata[node] = {} unless @metadata.key?(node)
412
+ # Here, explicitely store nil if nothing has been found for a node because we know there is no value to be fetched.
413
+ # This way we won't query again all CMDBs thanks to the cache.
414
+ @metadata[node][property] = updated_metadata[node]
401
415
  end
402
416
  end
403
417
  end
@@ -426,6 +440,7 @@ module HybridPlatformsConductor
426
440
  nodes_selectors = nodes_selectors.flatten
427
441
  # 1. Check for the presence of all
428
442
  return known_nodes if nodes_selectors.any? { |nodes_selector| nodes_selector.is_a?(Hash) && nodes_selector.key?(:all) && nodes_selector[:all] }
443
+
429
444
  # 2. Expand the nodes lists, platforms and services contents
430
445
  string_nodes = []
431
446
  nodes_selectors.each do |nodes_selector|
@@ -435,6 +450,7 @@ module HybridPlatformsConductor
435
450
  if nodes_selector.key?(:list)
436
451
  platform = @nodes_list_platform[nodes_selector[:list]]
437
452
  raise "Unknown nodes list: #{nodes_selector[:list]}" if platform.nil?
453
+
438
454
  string_nodes.concat(platform.nodes_selectors_from_nodes_list(nodes_selector[:list]))
439
455
  end
440
456
  string_nodes.concat(@platforms_handler.platform(nodes_selector[:platform]).known_nodes) if nodes_selector.key?(:platform)
@@ -462,8 +478,8 @@ module HybridPlatformsConductor
462
478
  # 3. Expand the Regexps
463
479
  real_nodes = []
464
480
  string_nodes.each do |node|
465
- if node =~ /^\/(.+)\/$/
466
- node_regexp = Regexp.new($1)
481
+ if node =~ %r{^/(.+)/$}
482
+ node_regexp = Regexp.new(Regexp.last_match(1))
467
483
  real_nodes.concat(known_nodes.select { |known_node| known_node[node_regexp] })
468
484
  else
469
485
  real_nodes << node
@@ -489,13 +505,11 @@ module HybridPlatformsConductor
489
505
  # * *parallel* (Boolean): Iterate in a multithreaded way? [default: false]
490
506
  # * *nbr_threads_max* (Integer or nil): Maximum number of threads to be used in case of parallel, or nil for no limit [default: nil]
491
507
  # * *progress* (String or nil): Name of a progress bar to follow the progression, or nil for no progress bar [default: 'Processing nodes']
492
- # * Proc: The code called for each node being iterated on.
508
+ # * *block* (Proc): The code called for each node being iterated on.
493
509
  # * Parameters::
494
510
  # * *node* (String): The node name
495
- def for_each_node_in(nodes, parallel: false, nbr_threads_max: nil, progress: 'Processing nodes')
496
- for_each_element_in(nodes.sort, parallel: parallel, nbr_threads_max: nbr_threads_max, progress: progress) do |node|
497
- yield node
498
- end
511
+ def for_each_node_in(nodes, parallel: false, nbr_threads_max: nil, progress: 'Processing nodes', &block)
512
+ for_each_element_in(nodes.sort, parallel: parallel, nbr_threads_max: nbr_threads_max, progress: progress, &block)
499
513
  end
500
514
 
501
515
  # Get the list of impacted nodes from a git diff on a platform
@@ -513,10 +527,11 @@ module HybridPlatformsConductor
513
527
  def impacted_nodes_from_git_diff(platform_name, from_commit: 'master', to_commit: nil, smallest_set: false)
514
528
  platform = @platforms_handler.platform(platform_name)
515
529
  raise "Unkown platform #{platform_name}. Possible platforms are #{@platforms_handler.known_platforms.map(&:name).sort.join(', ')}" if platform.nil?
530
+
516
531
  begin
517
532
  _exit_status, stdout, _stderr = @cmd_runner.run_cmd "cd #{platform.repository_path} && git --no-pager diff --no-color #{from_commit} #{to_commit.nil? ? '' : to_commit}", log_to_stdout: log_debug?
518
533
  rescue CmdRunner::UnexpectedExitCodeError
519
- raise GitError, $!.to_s
534
+ raise GitError, $ERROR_INFO.to_s
520
535
  end
521
536
  # Parse the git diff output to create a structured diff
522
537
  # Hash< String, Hash< Symbol, Object > >: List of diffs info, per file name having a diff. Diffs info have the following properties:
@@ -526,9 +541,10 @@ module HybridPlatformsConductor
526
541
  current_file_diff = nil
527
542
  stdout.split("\n").each do |line|
528
543
  case line
529
- when /^diff --git a\/(.+) b\/(.+)$/
544
+ when %r{^diff --git a/(.+) b/(.+)$}
530
545
  # A new file diff
531
- from, to = $1, $2
546
+ from = Regexp.last_match(1)
547
+ to = Regexp.last_match(2)
532
548
  current_file_diff = {
533
549
  diff: ''
534
550
  }
@@ -626,11 +642,13 @@ module HybridPlatformsConductor
626
642
  cmdb_masters_cache = {}
627
643
  select_confs_for_node(node, @config.cmdb_masters).each do |cmdb_masters_info|
628
644
  cmdb_masters_info[:cmdb_masters].each do |cmdb, properties|
629
- properties.each do |property|
630
- raise "Property #{property} have conflicting CMDB masters for #{node} declared in the configuration: #{cmdb_masters_cache[property].class.name} and #{@cmdbs[cmdb].class.name}" if cmdb_masters_cache.key?(property) && cmdb_masters_cache[property] != @cmdbs[cmdb]
631
- log_debug "CMDB master for #{node} / #{property}: #{cmdb}"
632
- raise "CMDB #{cmdb} is configured as a master for property #{property} on node #{node} but it does not implement the needed API to retrieve it" unless (@cmdbs_per_property[property] || []).include?(@cmdbs[cmdb]) || @cmdbs_others.include?(@cmdbs[cmdb])
633
- cmdb_masters_cache[property] = @cmdbs[cmdb]
645
+ properties.each do |itr_property|
646
+ raise "Property #{itr_property} have conflicting CMDB masters for #{node} declared in the configuration: #{cmdb_masters_cache[itr_property].class.name} and #{@cmdbs[cmdb].class.name}" if cmdb_masters_cache.key?(itr_property) && cmdb_masters_cache[itr_property] != @cmdbs[cmdb]
647
+
648
+ log_debug "CMDB master for #{node} / #{itr_property}: #{cmdb}"
649
+ raise "CMDB #{cmdb} is configured as a master for property #{itr_property} on node #{node} but it does not implement the needed API to retrieve it" unless (@cmdbs_per_property[itr_property] || []).include?(@cmdbs[cmdb]) || @cmdbs_others.include?(@cmdbs[cmdb])
650
+
651
+ cmdb_masters_cache[itr_property] = @cmdbs[cmdb]
634
652
  end
635
653
  end
636
654
  end