hybrid_platforms_conductor 32.17.0 → 33.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (283) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +36 -0
  3. data/README.md +6 -3
  4. data/bin/check-node +0 -1
  5. data/bin/deploy +0 -1
  6. data/bin/get_impacted_nodes +0 -1
  7. data/bin/last_deploys +12 -8
  8. data/bin/nodes_to_deploy +6 -6
  9. data/bin/setup +6 -6
  10. data/bin/topograph +1 -1
  11. data/docs/config_dsl.md +45 -1
  12. data/docs/executables.md +6 -7
  13. data/docs/executables/check-node.md +3 -3
  14. data/docs/executables/deploy.md +3 -3
  15. data/docs/executables/dump_nodes_json.md +3 -3
  16. data/docs/executables/test.md +3 -3
  17. data/docs/executables/topograph.md +3 -3
  18. data/docs/gen/mermaid/README.md-0.png +0 -0
  19. data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
  20. data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
  21. data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
  22. data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
  23. data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
  24. data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
  25. data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
  26. data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
  27. data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
  28. data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
  29. data/docs/plugins.md +46 -0
  30. data/docs/plugins/connector/ssh.md +1 -1
  31. data/docs/plugins/log/remote_fs.md +26 -0
  32. data/docs/plugins/secrets_reader/cli.md +31 -0
  33. data/docs/plugins/secrets_reader/thycotic.md +46 -0
  34. data/docs/plugins/test/check_deploy_and_idempotence.md +1 -1
  35. data/docs/plugins/test/connection.md +1 -0
  36. data/docs/plugins/test/deploy_removes_root_access.md +1 -1
  37. data/docs/plugins/test/file_system.md +1 -0
  38. data/docs/plugins/test/hostname.md +1 -0
  39. data/docs/plugins/test/ip.md +1 -0
  40. data/docs/plugins/test/local_users.md +1 -0
  41. data/docs/plugins/test/mounts.md +1 -0
  42. data/docs/plugins/test/orphan_files.md +1 -0
  43. data/docs/plugins/test/ports.md +1 -0
  44. data/docs/plugins/test/spectre.md +1 -0
  45. data/docs/plugins/test/vulnerabilities.md +1 -0
  46. data/lib/hybrid_platforms_conductor/action.rb +4 -4
  47. data/lib/hybrid_platforms_conductor/actions_executor.rb +45 -43
  48. data/lib/hybrid_platforms_conductor/bitbucket.rb +5 -4
  49. data/lib/hybrid_platforms_conductor/cmd_runner.rb +13 -12
  50. data/lib/hybrid_platforms_conductor/cmdb.rb +2 -2
  51. data/lib/hybrid_platforms_conductor/common_config_dsl/bitbucket.rb +2 -1
  52. data/lib/hybrid_platforms_conductor/common_config_dsl/confluence.rb +2 -1
  53. data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +5 -4
  54. data/lib/hybrid_platforms_conductor/common_config_dsl/github.rb +4 -3
  55. data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb +2 -2
  56. data/lib/hybrid_platforms_conductor/config.rb +8 -4
  57. data/lib/hybrid_platforms_conductor/confluence.rb +1 -1
  58. data/lib/hybrid_platforms_conductor/connector.rb +5 -2
  59. data/lib/hybrid_platforms_conductor/core_extensions/cleanroom/fix_kwargs.rb +116 -0
  60. data/lib/hybrid_platforms_conductor/core_extensions/symbol/zero.rb +24 -0
  61. data/lib/hybrid_platforms_conductor/credentials.rb +39 -36
  62. data/lib/hybrid_platforms_conductor/current_dir_monitor.rb +4 -1
  63. data/lib/hybrid_platforms_conductor/deployer.rb +275 -224
  64. data/lib/hybrid_platforms_conductor/executable.rb +20 -15
  65. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/config.rb +10 -7
  66. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +1 -1
  67. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb +2 -2
  68. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +4 -4
  69. data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb +2 -0
  70. data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +45 -49
  71. data/lib/hybrid_platforms_conductor/hpc_plugins/log/my_log_plugin.rb.sample +100 -0
  72. data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb +180 -0
  73. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +68 -66
  74. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb +13 -0
  75. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb +39 -38
  76. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb +5 -4
  77. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb +43 -45
  78. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/podman.rb +18 -20
  79. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +118 -117
  80. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +39 -43
  81. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container +9 -13
  82. data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb +2 -2
  83. data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +28 -21
  84. data/lib/hybrid_platforms_conductor/hpc_plugins/report/stdout.rb +26 -22
  85. data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/cli.rb +77 -0
  86. data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/my_secrets_reader_plugin.rb.sample +46 -0
  87. data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb +90 -0
  88. data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb +3 -3
  89. data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb +4 -2
  90. data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb +3 -1
  91. data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +7 -21
  92. data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb +21 -19
  93. data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb +2 -2
  94. data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb +2 -2
  95. data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +21 -22
  96. data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +19 -21
  97. data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb +2 -3
  98. data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +3 -1
  99. data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb +2 -2
  100. data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +4 -2
  101. data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb +21 -22
  102. data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb +10 -12
  103. data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +9 -9
  104. data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +5 -3
  105. data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +5 -3
  106. data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +13 -10
  107. data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb +5 -3
  108. data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +5 -5
  109. data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +5 -5
  110. data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +7 -7
  111. data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +3 -3
  112. data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +27 -25
  113. data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/confluence.rb +2 -2
  114. data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/stdout.rb +8 -6
  115. data/lib/hybrid_platforms_conductor/io_router.rb +14 -13
  116. data/lib/hybrid_platforms_conductor/json_dumper.rb +2 -2
  117. data/lib/hybrid_platforms_conductor/log.rb +31 -0
  118. data/lib/hybrid_platforms_conductor/logger_helpers.rb +19 -16
  119. data/lib/hybrid_platforms_conductor/nodes_handler.rb +89 -71
  120. data/lib/hybrid_platforms_conductor/parallel_threads.rb +7 -11
  121. data/lib/hybrid_platforms_conductor/platform_handler.rb +7 -7
  122. data/lib/hybrid_platforms_conductor/platforms_handler.rb +5 -3
  123. data/lib/hybrid_platforms_conductor/plugin.rb +2 -2
  124. data/lib/hybrid_platforms_conductor/plugins.rb +14 -8
  125. data/lib/hybrid_platforms_conductor/provisioner.rb +4 -4
  126. data/lib/hybrid_platforms_conductor/report.rb +2 -2
  127. data/lib/hybrid_platforms_conductor/reports_handler.rb +3 -2
  128. data/lib/hybrid_platforms_conductor/secrets_reader.rb +31 -0
  129. data/lib/hybrid_platforms_conductor/services_handler.rb +32 -29
  130. data/lib/hybrid_platforms_conductor/test_only_remote_node.rb +18 -0
  131. data/lib/hybrid_platforms_conductor/test_report.rb +15 -18
  132. data/lib/hybrid_platforms_conductor/tests_runner.rb +116 -118
  133. data/lib/hybrid_platforms_conductor/thycotic.rb +28 -19
  134. data/lib/hybrid_platforms_conductor/topographer.rb +200 -190
  135. data/lib/hybrid_platforms_conductor/topographer/plugins/graphviz.rb +8 -8
  136. data/lib/hybrid_platforms_conductor/topographer/plugins/json.rb +4 -4
  137. data/lib/hybrid_platforms_conductor/version.rb +1 -1
  138. data/spec/hybrid_platforms_conductor_test.rb +33 -12
  139. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +18 -11
  140. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/interactive_spec.rb +2 -2
  141. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb +32 -21
  142. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +75 -49
  143. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/scp_spec.rb +27 -15
  144. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions_spec.rb +90 -59
  145. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connection_spec.rb +46 -44
  146. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/connectable_nodes_spec.rb +12 -8
  147. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb +4 -7
  148. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +21 -22
  149. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +23 -24
  150. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connectable_nodes_spec.rb +10 -6
  151. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +106 -75
  152. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +145 -126
  153. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +3 -3
  154. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +29 -25
  155. data/spec/hybrid_platforms_conductor_test/api/actions_executor/logging_spec.rb +167 -142
  156. data/spec/hybrid_platforms_conductor_test/api/actions_executor/parallel_spec.rb +272 -244
  157. data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +16 -16
  158. data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +36 -36
  159. data/spec/hybrid_platforms_conductor_test/api/config_spec.rb +24 -22
  160. data/spec/hybrid_platforms_conductor_test/api/deployer/check_spec.rb +4 -2
  161. data/spec/hybrid_platforms_conductor_test/api/deployer/config_dsl_spec.rb +43 -5
  162. data/spec/hybrid_platforms_conductor_test/api/deployer/deploy_spec.rb +199 -216
  163. data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +223 -0
  164. data/spec/hybrid_platforms_conductor_test/api/deployer/parse_deploy_output_spec.rb +55 -59
  165. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +36 -62
  166. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/podman_spec.rb +17 -17
  167. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/config_dsl_spec.rb +4 -4
  168. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/create_spec.rb +44 -51
  169. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/destroy_spec.rb +3 -3
  170. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/ip_spec.rb +12 -16
  171. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/destroy_vm_spec.rb +31 -19
  172. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +324 -266
  173. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/ips_assignment_spec.rb +89 -61
  174. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/other_lxc_containers_resources_spec.rb +117 -93
  175. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/pve_node_resources_spec.rb +71 -54
  176. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +10 -8
  177. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb +80 -60
  178. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/start_spec.rb +1 -1
  179. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb +1 -1
  180. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/stop_spec.rb +1 -1
  181. data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/cli_spec.rb +64 -0
  182. data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/thycotic_spec.rb +268 -0
  183. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/config_spec.rb +8 -10
  184. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +33 -24
  185. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_keys_spec.rb +64 -51
  186. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/platform_handlers_spec.rb +3 -3
  187. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +50 -51
  188. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +91 -81
  189. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +14 -16
  190. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/git_diff_impacts_spec.rb +51 -75
  191. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/nodes_selectors_spec.rb +35 -26
  192. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/platform_handlers_plugins_api_spec.rb +24 -16
  193. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/several_platforms_spec.rb +29 -19
  194. data/spec/hybrid_platforms_conductor_test/api/platform_handler_spec.rb +4 -4
  195. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb +2 -2
  196. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb +6 -6
  197. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb +57 -99
  198. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +4 -4
  199. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +32 -35
  200. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +10 -10
  201. data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +38 -27
  202. data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +46 -52
  203. data/spec/hybrid_platforms_conductor_test/api/reports_handler_spec.rb +2 -2
  204. data/spec/hybrid_platforms_conductor_test/api/services_handler/actions_to_deploy_spec.rb +90 -58
  205. data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +38 -34
  206. data/spec/hybrid_platforms_conductor_test/api/services_handler/log_info_spec.rb +11 -9
  207. data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +193 -171
  208. data/spec/hybrid_platforms_conductor_test/api/services_handler/parse_deploy_output_spec.rb +66 -54
  209. data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +147 -133
  210. data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +69 -49
  211. data/spec/hybrid_platforms_conductor_test/api/tests_runner/global_spec.rb +5 -4
  212. data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +8 -5
  213. data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_spec.rb +8 -5
  214. data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +30 -27
  215. data/spec/hybrid_platforms_conductor_test/api/tests_runner/platform_spec.rb +12 -9
  216. data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +48 -47
  217. data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/bitbucket_conf_spec.rb +5 -5
  218. data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/github_ci_spec.rb +5 -5
  219. data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_reports_plugins/confluence_spec.rb +5 -5
  220. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +9 -9
  221. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb2.rb → test_cmdb_2.rb} +6 -6
  222. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
  223. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb_others2.rb → test_cmdb_others_2.rb} +2 -2
  224. data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
  225. data/spec/hybrid_platforms_conductor_test/executables/{check-node_spec.rb → check_node_spec.rb} +4 -6
  226. data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +4 -6
  227. data/spec/hybrid_platforms_conductor_test/executables/get_impacted_nodes_spec.rb +76 -77
  228. data/spec/hybrid_platforms_conductor_test/executables/last_deploys_spec.rb +159 -113
  229. data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +299 -160
  230. data/spec/hybrid_platforms_conductor_test/executables/options/actions_executor_spec.rb +4 -6
  231. data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +3 -5
  232. data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb +8 -8
  233. data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +12 -196
  234. data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +9 -10
  235. data/spec/hybrid_platforms_conductor_test/executables/options/nodes_selectors_spec.rb +9 -10
  236. data/spec/hybrid_platforms_conductor_test/executables/options/reports_handler_spec.rb +1 -1
  237. data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +22 -22
  238. data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +22 -16
  239. data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +32 -32
  240. data/spec/hybrid_platforms_conductor_test/executables/ssh_config_spec.rb +7 -9
  241. data/spec/hybrid_platforms_conductor_test/executables/test_spec.rb +3 -5
  242. data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +2 -2
  243. data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +4 -3
  244. data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
  245. data/spec/hybrid_platforms_conductor_test/helpers/config_helpers.rb +1 -1
  246. data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +12 -13
  247. data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +245 -56
  248. data/spec/hybrid_platforms_conductor_test/helpers/executables_helpers.rb +11 -11
  249. data/spec/hybrid_platforms_conductor_test/helpers/nodes_handler_helpers.rb +1 -1
  250. data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +39 -28
  251. data/spec/hybrid_platforms_conductor_test/helpers/plugins_helpers.rb +1 -1
  252. data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +86 -111
  253. data/spec/hybrid_platforms_conductor_test/helpers/reports_handler_helpers.rb +1 -1
  254. data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +3 -3
  255. data/spec/hybrid_platforms_conductor_test/helpers/services_handler_helpers.rb +1 -1
  256. data/spec/hybrid_platforms_conductor_test/helpers/tests_runner_helpers.rb +1 -1
  257. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id1.rb → test_plugin_id_1.rb} +0 -0
  258. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id2.rb → test_plugin_id_2.rb} +0 -0
  259. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/test_plugin_type/{test_plugin_id3.rb → test_plugin_id_3.rb} +0 -0
  260. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/{test_plugin_type2/test_plugin_id4.rb → test_plugin_type_2/test_plugin_id_4.rb} +0 -0
  261. data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/test.rb +1 -1
  262. data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/{test2.rb → test_2.rb} +0 -0
  263. data/spec/hybrid_platforms_conductor_test/rubocop_spec.rb +31 -0
  264. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json +3 -3
  265. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json +3 -3
  266. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb +1 -0
  267. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb +1 -0
  268. data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +952 -0
  269. data/spec/hybrid_platforms_conductor_test/test_connector.rb +3 -3
  270. data/spec/hybrid_platforms_conductor_test/test_log_no_read_plugin.rb +84 -0
  271. data/spec/hybrid_platforms_conductor_test/test_log_plugin.rb +105 -0
  272. data/spec/hybrid_platforms_conductor_test/test_plugins/global.rb +1 -0
  273. data/spec/hybrid_platforms_conductor_test/test_plugins/node.rb +1 -0
  274. data/spec/hybrid_platforms_conductor_test/test_plugins/node_check.rb +1 -0
  275. data/spec/hybrid_platforms_conductor_test/test_plugins/platform.rb +1 -0
  276. data/spec/hybrid_platforms_conductor_test/test_plugins/several_checks.rb +2 -2
  277. data/spec/hybrid_platforms_conductor_test/test_secrets_reader_plugin.rb +45 -0
  278. data/spec/hybrid_platforms_conductor_test/tests_report_plugin.rb +5 -6
  279. data/spec/spec_helper.rb +17 -18
  280. data/tools/check_md +16 -20
  281. data/tools/generate_mermaid +1 -1
  282. metadata +195 -144
  283. data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +0 -916
@@ -1,4 +1,5 @@
1
1
  require 'nokogiri'
2
+ require 'hybrid_platforms_conductor/test_only_remote_node'
2
3
 
3
4
  module HybridPlatformsConductor
4
5
 
@@ -17,7 +18,7 @@ module HybridPlatformsConductor
17
18
  # Each final OVAL URL can be directly an XML file, either raw or compressed with .gz or .bz2.
18
19
  # This is useful to follow repository links, such as jFrog or web servers serving common file systems structure storing several versions of the OVAL file.
19
20
  # * *reported_severities* (Array<String> or nil): List of severities to report, if any (use Unknown when the severity is not known), or nil for all [default: nil]
20
- class Vulnerabilities < HybridPlatformsConductor::Test
21
+ class Vulnerabilities < TestOnlyRemoteNode
21
22
 
22
23
  # Known compression methods, per file extension, and their corresponding uncompress bash script
23
24
  KNOWN_COMPRESSIONS = {
@@ -47,8 +48,9 @@ module HybridPlatformsConductor
47
48
  # Follow the last link recursively until we find a .xml or compressed file
48
49
  current_url = artifactory_url
49
50
  loop do
50
- current_url = "#{current_url}#{current_url.end_with?('/') ? '' : '/'}#{Nokogiri::HTML.parse(URI.open(current_url)).css('a').last['href']}"
51
+ current_url = "#{current_url}#{current_url.end_with?('/') ? '' : '/'}#{Nokogiri::HTML.parse(URI.parse(current_url).open).css('a').last['href']}"
51
52
  break if current_url.end_with?('.xml') || KNOWN_COMPRESSIONS.keys.any? { |file_ext| current_url.end_with?(".#{file_ext}") }
53
+
52
54
  log_debug "Follow last link to #{current_url}"
53
55
  end
54
56
  current_url
@@ -56,7 +58,7 @@ module HybridPlatformsConductor
56
58
  )
57
59
  # TODO: Access the user correctly when the user notion will be moved out of the ssh connector
58
60
  sudo = @deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "
59
- Hash[urls.map do |url|
61
+ urls.map do |url|
60
62
  # 1. Get the OVAL file on the node to be tested (uncompress it if needed)
61
63
  # 2. Make sure oscap is installed
62
64
  # 3. Generate the report for this OVAL file using oscap
@@ -66,13 +68,13 @@ module HybridPlatformsConductor
66
68
  packages_to_install = []
67
69
  KNOWN_COMPRESSIONS.each do |file_ext, compress_info|
68
70
  file_ending = ".#{file_ext}"
69
- if local_oval_file.end_with?(file_ending)
70
- uncompress_cmds << compress_info[:cmd].call(local_oval_file)
71
- packages_to_install.concat(compress_info[:packages])
72
- local_oval_file = File.basename(local_oval_file, file_ending)
73
- end
71
+ next unless local_oval_file.end_with?(file_ending)
72
+
73
+ uncompress_cmds << compress_info[:cmd].call(local_oval_file)
74
+ packages_to_install.concat(compress_info[:packages])
75
+ local_oval_file = File.basename(local_oval_file, file_ending)
74
76
  end
75
- cmds = <<~EOS
77
+ cmds = <<~EO_BASH
76
78
  set -e
77
79
  #{
78
80
  case image
@@ -84,7 +86,7 @@ module HybridPlatformsConductor
84
86
  # On Debian 10 we have to compile it from sources, as the packaged official version has core dumps.
85
87
  # cf https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1688223.html
86
88
  # TODO: Remove this Debian 10 specificity when the official libopenscap8 will be corrected
87
- <<~EOS2
89
+ <<~EO_BASH2
88
90
  if [ ! -x "$(command -v oscap)" ] || [ "$(oscap --version | head -n 1 | awk '{print $6}')" != "1.3.4" ]; then
89
91
  rm -rf openscap
90
92
  git clone --recurse-submodules https://github.com/OpenSCAP/openscap.git
@@ -96,7 +98,7 @@ module HybridPlatformsConductor
96
98
  #{sudo}make install
97
99
  fi
98
100
  #{sudo}apt install -y wget #{packages_to_install.join(' ')}
99
- EOS2
101
+ EO_BASH2
100
102
  else
101
103
  raise "Non supported image: #{image}. Please adapt this test's code."
102
104
  end
@@ -110,7 +112,7 @@ module HybridPlatformsConductor
110
112
  echo "===== RESULTS ====="
111
113
  cat "#{local_oval_file}.results.xml"
112
114
  cd ..
113
- EOS
115
+ EO_BASH
114
116
  [
115
117
  cmds,
116
118
  {
@@ -119,21 +121,21 @@ module HybridPlatformsConductor
119
121
  if idx_results.nil?
120
122
  error 'No results given by the oscap run', stdout.join("\n")
121
123
  else
122
- results = Nokogiri::XML(stdout[idx_results + 1..-1].join("\n"))
124
+ results = Nokogiri::XML(stdout[idx_results + 1..].join("\n"))
123
125
  results.remove_namespaces!
124
126
  oval_definitions = results.css('oval_results oval_definitions definitions definition')
125
127
  results.css('results system definitions definition').each do |definition_xml|
126
- if definition_xml['result'] == 'true'
127
- # Just found an OVAL item to be patched.
128
- definition_id = definition_xml['definition_id']
129
- oval_definition = oval_definitions.find { |el| el['id'] == definition_id }
130
- # We don't forcefully want to report all missing patches. Only the most important ones.
131
- severity = oval_definition.css('metadata advisory severity').text
132
- severity = 'Unknown' if severity.empty?
133
- if !oval_info.key?('reported_severities') || oval_info['reported_severities'].include?(severity)
134
- # Only consider the first line of the description, as sometimes it's very long
135
- error "Non-patched #{severity} vulnerability found: #{oval_definition.css('metadata title').text} - #{oval_definition.css('metadata description').text.split("\n").first}"
136
- end
128
+ next unless definition_xml['result'] == 'true'
129
+
130
+ # Just found an OVAL item to be patched.
131
+ definition_id = definition_xml['definition_id']
132
+ oval_definition = oval_definitions.find { |el| el['id'] == definition_id }
133
+ # We don't forcefully want to report all missing patches. Only the most important ones.
134
+ severity = oval_definition.css('metadata advisory severity').text
135
+ severity = 'Unknown' if severity.empty?
136
+ if !oval_info.key?('reported_severities') || oval_info['reported_severities'].include?(severity)
137
+ # Only consider the first line of the description, as sometimes it's very long
138
+ error "Non-patched #{severity} vulnerability found: #{oval_definition.css('metadata title').text} - #{oval_definition.css('metadata description').text.split("\n").first}"
137
139
  end
138
140
  end
139
141
  end
@@ -142,7 +144,7 @@ module HybridPlatformsConductor
142
144
  timeout: 240
143
145
  }
144
146
  ]
145
- end]
147
+ end.to_h
146
148
  else
147
149
  error "No OVAL file defined for image #{image} at #{oval_file}"
148
150
  {}
@@ -12,7 +12,7 @@ module HybridPlatformsConductor
12
12
  # Report tests results on a generated Confluence page
13
13
  class Confluence < HybridPlatformsConductor::TestReport
14
14
 
15
- self.extend_config_dsl_with CommonConfigDsl::Confluence, :init_confluence
15
+ extend_config_dsl_with CommonConfigDsl::Confluence, :init_confluence
16
16
 
17
17
  # Maximum errors to be reported by item
18
18
  MAX_ERROR_ITEMS_DISPLAYED = 10
@@ -34,7 +34,7 @@ module HybridPlatformsConductor
34
34
  at('h1:contains("Evolution")').
35
35
  search('~ structured-macro:first-of-type').
36
36
  css('table td').
37
- map { |td_element| td_element.text }.
37
+ map(&:text).
38
38
  each_slice(2).
39
39
  to_a.
40
40
  map { |(time_str, value_str)| [Time.parse("#{time_str} UTC"), value_str.to_f] }
@@ -97,18 +97,20 @@ module HybridPlatformsConductor
97
97
  table << [
98
98
  nodes_list,
99
99
  nodes_info[:nodes].size,
100
- nodes_info[:nodes].empty? ? '' : "#{(nodes_info[:tested_nodes].size*100.0/nodes_info[:nodes].size).to_i} %",
100
+ nodes_info[:nodes].empty? ? '' : "#{(nodes_info[:tested_nodes].size * 100.0 / nodes_info[:nodes].size).to_i} %",
101
101
  nodes_info[:tested_nodes].empty? ? '' : "#{((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error_as_expected].size) * 100.0 / nodes_info[:tested_nodes].size).to_i} %",
102
102
  nodes_info[:tested_nodes].empty? ? '' : "#{((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * 100.0 / nodes_info[:tested_nodes].size).to_i} %",
103
- nodes_info[:nodes].empty? ? '' :
103
+ if nodes_info[:nodes].empty?
104
+ ''
105
+ else
104
106
  ('=' * ((nodes_info[:tested_nodes_in_error_as_expected].size * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).yellow.bold +
105
- ('=' * (((nodes_info[:tested_nodes_in_error].size - nodes_info[:tested_nodes_in_error_as_expected].size).abs * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).red.bold +
106
- ('=' * (((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).green.bold +
107
- ('=' * (((nodes_info[:nodes].size - nodes_info[:tested_nodes].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).white.bold
107
+ ('=' * (((nodes_info[:tested_nodes_in_error].size - nodes_info[:tested_nodes_in_error_as_expected].size).abs * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).red.bold +
108
+ ('=' * (((nodes_info[:tested_nodes].size - nodes_info[:tested_nodes_in_error].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).green.bold +
109
+ ('=' * (((nodes_info[:nodes].size - nodes_info[:tested_nodes].size) * PROGRESS_BAR_SIZE.to_f) / nodes_info[:nodes].size).round).white.bold
110
+ end
108
111
  ]
109
112
  end
110
113
  end)
111
-
112
114
  end
113
115
 
114
116
  end
@@ -30,6 +30,7 @@ module HybridPlatformsConductor
30
30
  # Start routing messages asynchronously
31
31
  def start
32
32
  raise 'IO router is already started. Can\'t start it again.' unless @reading_thread.nil?
33
+
33
34
  @end_read = false
34
35
  # Create a thread to handle routes asynchronously
35
36
  @reading_thread = Thread.new do
@@ -37,22 +38,21 @@ module HybridPlatformsConductor
37
38
  need_to_stop = @end_read.clone
38
39
  data_found = false
39
40
  @routes.each do |src_io, dst_ios|
40
- if src_io.is_a?(Queue)
41
- queue_size = src_io.size
42
- if queue_size > 0
43
- # There is data to be read from src_io
44
- data_found = true
45
- data_chunk_str = queue_size.times.map { src_io.pop }.join
46
- dst_ios.each do |dst_io|
47
- dst_io << data_chunk_str
48
- dst_io.flush if dst_io.respond_to?(:flush)
49
- end
50
- end
51
- else
52
- raise "Unknown type of source IO: #{src_io}"
41
+ raise "Unknown type of source IO: #{src_io}" unless src_io.is_a?(Queue)
42
+
43
+ queue_size = src_io.size
44
+ next unless queue_size.positive?
45
+
46
+ # There is data to be read from src_io
47
+ data_found = true
48
+ data_chunk_str = queue_size.times.map { src_io.pop }.join
49
+ dst_ios.each do |dst_io|
50
+ dst_io << data_chunk_str
51
+ dst_io.flush if dst_io.respond_to?(:flush)
53
52
  end
54
53
  end
55
54
  break if need_to_stop && !data_found
55
+
56
56
  sleep 0.1
57
57
  end
58
58
  end
@@ -61,6 +61,7 @@ module HybridPlatformsConductor
61
61
  # Stop routing messages asynchronously
62
62
  def stop
63
63
  raise 'IO router is not started. Can\'t stop it.' if @reading_thread.nil?
64
+
64
65
  @end_read = true
65
66
  @reading_thread.join
66
67
  end
@@ -23,7 +23,7 @@ module HybridPlatformsConductor
23
23
  # * *config* (Config): Config to be used. [default = Config.new]
24
24
  # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default = NodesHandler.new]
25
25
  # * *deployer* (Deployer): Deployer to be used. [default = Deployer.new]
26
- def initialize(logger: Logger.new(STDOUT), logger_stderr: Logger.new(STDERR), config: Config.new, nodes_handler: NodesHandler.new, deployer: Deployer.new)
26
+ def initialize(logger: Logger.new($stdout), logger_stderr: Logger.new($stderr), config: Config.new, nodes_handler: NodesHandler.new, deployer: Deployer.new)
27
27
  init_loggers(logger, logger_stderr)
28
28
  @config = config
29
29
  @nodes_handler = nodes_handler
@@ -74,7 +74,7 @@ module HybridPlatformsConductor
74
74
  out "[ #{node} ] - Error while dumping JSON. Check #{stdout_file_name}"
75
75
  else
76
76
  json_file_name = "#{@dump_dir}/#{node}.json"
77
- File.write(json_file_name, stdout[dump_begin_idx+1..dump_end_idx-1].join("\n"))
77
+ File.write(json_file_name, stdout[dump_begin_idx + 1..dump_end_idx - 1].join("\n"))
78
78
  out "[ #{node} ] - OK. Check #{json_file_name}"
79
79
  end
80
80
  else
@@ -0,0 +1,31 @@
1
+ require 'hybrid_platforms_conductor/logger_helpers'
2
+ require 'hybrid_platforms_conductor/plugin'
3
+
4
+ module HybridPlatformsConductor
5
+
6
+ # Ancestor of all log plugins
7
+ class Log < Plugin
8
+
9
+ # Constructor
10
+ #
11
+ # Parameters::
12
+ # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
13
+ # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
14
+ # * *config* (Config): Config to be used. [default: Config.new]
15
+ # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
16
+ # * *actions_executor* (ActionsExecutor): Actions executor to be used. [default: ActionsExecutor.new]
17
+ def initialize(
18
+ logger: Logger.new($stdout),
19
+ logger_stderr: Logger.new($stderr),
20
+ config: Config.new,
21
+ nodes_handler: NodesHandler.new,
22
+ actions_executor: ActionsExecutor.new
23
+ )
24
+ super(logger: logger, logger_stderr: logger_stderr, config: config)
25
+ @nodes_handler = nodes_handler
26
+ @actions_executor = actions_executor
27
+ end
28
+
29
+ end
30
+
31
+ end
@@ -55,25 +55,27 @@ module HybridPlatformsConductor
55
55
 
56
56
  # Make sure if the current line is not flushed we still do it
57
57
  def flush
58
- unless @current_line.nil?
59
- @stream << @current_line
60
- @current_line = nil
61
- end
58
+ return if @current_line.nil?
59
+
60
+ @stream << @current_line
61
+ @current_line = nil
62
62
  end
63
63
 
64
64
  end
65
65
 
66
66
  class << self
67
+
67
68
  attr_reader :progress_bar_semaphore
69
+
68
70
  end
69
71
  # Make sure the progress bar setting is protected by a Mutex
70
72
  @progress_bar_semaphore = Mutex.new
71
73
 
72
74
  # Sorted list of levels and their corresponding modifiers.
73
75
  LEVELS_MODIFIERS = {
74
- fatal: [:red, :bold],
75
- error: [:red, :bold],
76
- warn: [:yellow, :bold],
76
+ fatal: %i[red bold],
77
+ error: %i[red bold],
78
+ warn: %i[yellow bold],
77
79
  info: [:white],
78
80
  debug: [:white],
79
81
  unknown: [:white]
@@ -82,7 +84,7 @@ module HybridPlatformsConductor
82
84
  # List of levels that will output on stderr
83
85
  LEVELS_TO_STDERR = %i[warn error fatal]
84
86
 
85
- LEVELS_MODIFIERS.keys.each do |level|
87
+ LEVELS_MODIFIERS.each_key do |level|
86
88
  define_method("log_#{level}") do |message|
87
89
  (LEVELS_TO_STDERR.include?(level) ? @logger_stderr : @logger).send(
88
90
  level,
@@ -105,10 +107,10 @@ module HybridPlatformsConductor
105
107
  # Set loggers to the desired format
106
108
  def set_loggers_format
107
109
  [@logger, @logger_stderr].each do |logger|
108
- logger.formatter = proc do |severity, datetime, progname, msg|
110
+ logger.formatter = proc do |severity, _datetime, progname, msg|
109
111
  # If the message already has control characters, don't colorize it
110
112
  keep_original_color = msg.include? "\u001b"
111
- message = "[#{Time.now.utc.strftime('%F %T')} (PID #{$$} / TID #{Thread.current.object_id})] #{severity.rjust(5)} - [ #{progname} ] - "
113
+ message = "[#{Time.now.utc.strftime('%F %T')} (PID #{$PROCESS_ID} / TID #{Thread.current.object_id})] #{severity.rjust(5)} - [ #{progname} ] - "
112
114
  message << "#{msg}\n" unless keep_original_color
113
115
  LEVELS_MODIFIERS[severity.downcase.to_sym].each do |modifier|
114
116
  message = message.send(modifier)
@@ -282,8 +284,8 @@ module HybridPlatformsConductor
282
284
  yield progress_bar
283
285
  ensure
284
286
  LoggerHelpers.progress_bar_semaphore.synchronize do
285
- self.stdout_device.flush
286
- self.stderr_device.flush
287
+ stdout_device.flush
288
+ stderr_device.flush
287
289
  self.stdout_device = previous_stdout_device unless previous_stdout_device.nil?
288
290
  self.stderr_device = previous_stderr_device unless previous_stderr_device.nil?
289
291
  end
@@ -298,15 +300,16 @@ module HybridPlatformsConductor
298
300
  def stdouts_to_s
299
301
  messages = []
300
302
  {
301
- 'STDOUT' => self.stdout_device,
302
- 'STDERR' => self.stderr_device
303
+ 'STDOUT' => stdout_device,
304
+ 'STDERR' => stderr_device
303
305
  }.each do |name, device|
304
- if device.is_a?(File)
306
+ case device
307
+ when File
305
308
  if File.exist?(device.path)
306
309
  content = File.read(device.path).strip
307
310
  messages << "----- #{name} BEGIN - #{device.path} -----\n#{content}\n----- #{name} END - #{device.path} -----" unless content.empty?
308
311
  end
309
- elsif device.is_a?(StringIO)
312
+ when StringIO
310
313
  content = device.string
311
314
  messages << "----- #{name} BEGIN -----\n#{content}\n----- #{name} END -----" unless content.empty?
312
315
  end
@@ -40,7 +40,7 @@ module HybridPlatformsConductor
40
40
  # * *master_cmdbs_info* (Hash< Symbol, Symbol or Array<Symbol> >): List of metadata properties (or single one) per CMDB name considered as master for those properties.
41
41
  def master_cmdbs(master_cmdbs_info)
42
42
  @cmdb_masters << {
43
- cmdb_masters: Hash[master_cmdbs_info.map { |cmdb, properties| [cmdb, properties.is_a?(Array) ? properties : [properties]] }],
43
+ cmdb_masters: master_cmdbs_info.transform_values { |properties| properties.is_a?(Array) ? properties : [properties] },
44
44
  nodes_selectors_stack: current_nodes_selectors_stack
45
45
  }
46
46
  end
@@ -60,7 +60,8 @@ module HybridPlatformsConductor
60
60
 
61
61
  Config.extend_config_dsl_with ConfigDSLExtension, :init_nodes_handler_config
62
62
 
63
- include LoggerHelpers, ParallelThreads
63
+ include ParallelThreads
64
+ include LoggerHelpers
64
65
 
65
66
  class GitError < RuntimeError
66
67
  end
@@ -74,8 +75,8 @@ module HybridPlatformsConductor
74
75
  # * *cmd_runner* (CmdRunner): Command executor to be used. [default: CmdRunner.new]
75
76
  # * *platforms_handler* (PlatformsHandler): Platforms Handler to be used. [default: PlatformsHandler.new]
76
77
  def initialize(
77
- logger: Logger.new(STDOUT),
78
- logger_stderr: Logger.new(STDERR),
78
+ logger: Logger.new($stdout),
79
+ logger_stderr: Logger.new($stderr),
79
80
  config: Config.new,
80
81
  cmd_runner: CmdRunner.new,
81
82
  platforms_handler: PlatformsHandler.new
@@ -111,11 +112,11 @@ module HybridPlatformsConductor
111
112
  )
112
113
  @cmdbs_others << cmdb if cmdb.respond_to?(:get_others)
113
114
  cmdb.methods.each do |method|
114
- if method.to_s =~ /^get_(.*)$/
115
- property = $1.to_sym
116
- @cmdbs_per_property[property] = [] unless @cmdbs_per_property.key?(property)
117
- @cmdbs_per_property[property] << cmdb
118
- end
115
+ next unless method.to_s =~ /^get_(.*)$/
116
+
117
+ property = Regexp.last_match(1).to_sym
118
+ @cmdbs_per_property[property] = [] unless @cmdbs_per_property.key?(property)
119
+ @cmdbs_per_property[property] << cmdb
119
120
  end
120
121
  cmdb
121
122
  end
@@ -133,13 +134,17 @@ module HybridPlatformsConductor
133
134
  # Register all known nodes for this platform
134
135
  platform.known_nodes.each do |node|
135
136
  raise "Can't register #{node} to platform #{platform.repository_path}, as it is already defined in platform #{@nodes_platform[node].repository_path}." if @nodes_platform.key?(node)
137
+
136
138
  @nodes_platform[node] = platform
137
139
  end
138
140
  # Register all known nodes lists
141
+ next unless platform.respond_to?(:known_nodes_lists)
142
+
139
143
  platform.known_nodes_lists.each do |nodes_list|
140
144
  raise "Can't register nodes list #{nodes_list} to platform #{platform.repository_path}, as it is already defined in platform #{@nodes_list_platform[nodes_list].repository_path}." if @nodes_list_platform.key?(nodes_list)
145
+
141
146
  @nodes_list_platform[nodes_list] = platform
142
- end if platform.respond_to?(:known_nodes_lists)
147
+ end
143
148
  end
144
149
  end
145
150
 
@@ -147,7 +152,7 @@ module HybridPlatformsConductor
147
152
  #
148
153
  # Parameters::
149
154
  # * *options_parser* (OptionParser): The option parser to complete
150
- def options_parse(options_parser, parallel: true)
155
+ def options_parse(options_parser)
151
156
  options_parser.separator ''
152
157
  options_parser.separator 'Nodes handler options:'
153
158
  options_parser.on('-o', '--show-nodes', 'Display the list of possible nodes and exit') do
@@ -221,6 +226,7 @@ module HybridPlatformsConductor
221
226
  platform_name, from_commit, to_commit, flags = nodes_git_impact.split(':')
222
227
  flags = (flags || '').split(',')
223
228
  raise "Invalid platform in --nodes-git-impact: #{platform_name}. Possible values are: #{platform_names.join(', ')}." unless platform_names.include?(platform_name)
229
+
224
230
  nodes_selector = { platform: platform_name }
225
231
  nodes_selector[:from_commit] = from_commit if from_commit && !from_commit.empty?
226
232
  nodes_selector[:to_commit] = to_commit if to_commit && !to_commit.empty?
@@ -329,7 +335,7 @@ module HybridPlatformsConductor
329
335
  # * *block* (Proc): Code block given to the call
330
336
  def method_missing(method, *args, &block)
331
337
  if method.to_s =~ /^get_(.*)_of$/
332
- property = $1.to_sym
338
+ property = Regexp.last_match(1).to_sym
333
339
  # Define the method so that we don't go trough method_missing next time (more efficient).
334
340
  define_property_method_for(property)
335
341
  # Then call it
@@ -341,6 +347,15 @@ module HybridPlatformsConductor
341
347
  end
342
348
  end
343
349
 
350
+ # Make sure we register the methods we handle in method_missing
351
+ #
352
+ # Parameters::
353
+ # * *name* (Symbol): The missing method name
354
+ # * *include_private* (Boolean): Should we include private methods in the search?
355
+ def respond_to_missing?(name, include_private)
356
+ name.to_s =~ /^get_(.*)_of$/ || super
357
+ end
358
+
344
359
  # Prefetch some metadata properties for a given list of nodes.
345
360
  # Useful for performance reasons when clients know they will need to use a lot of properties on nodes.
346
361
  # Keep a thread-safe memory cache of it.
@@ -352,52 +367,51 @@ module HybridPlatformsConductor
352
367
  (properties.is_a?(Symbol) ? [properties] : properties).each do |property|
353
368
  # Gather the list of nodes missing this property
354
369
  missing_nodes = nodes.select { |node| !@metadata.key?(node) || !@metadata[node].key?(property) }
355
- unless missing_nodes.empty?
356
- # Query the CMDBs having first the get_<property> method, then the ones having the get_others method till we have our property set for all missing nodes
357
- # Metadata being retrieved by the different CMDBs, per node
370
+ next if missing_nodes.empty?
371
+
372
+ # Query the CMDBs having first the get_<property> method, then the ones having the get_others method till we have our property set for all missing nodes
373
+ # Metadata being retrieved by the different CMDBs, per node
374
+ # Hash< String, Object >
375
+ updated_metadata = {}
376
+ (
377
+ (@cmdbs_per_property.key?(property) ? @cmdbs_per_property[property] : []).map { |cmdb| [cmdb, property] } +
378
+ @cmdbs_others.map { |cmdb| [cmdb, :others] }
379
+ ).each do |(cmdb, cmdb_property)|
380
+ # If among the missing nodes some of them have some master CMDB declared for this property, filter them out unless we are dealing with their master CMDB.
381
+ nodes_to_query = missing_nodes.select do |node|
382
+ master_cmdb = cmdb_master_for(node, property)
383
+ master_cmdb.nil? || master_cmdb == cmdb
384
+ end
385
+ next if nodes_to_query.empty?
386
+
387
+ # Check first if this property depends on other ones for this cmdb
388
+ if cmdb.respond_to?(:property_dependencies)
389
+ property_deps = cmdb.property_dependencies
390
+ prefetch_metadata_of nodes_to_query, property_deps[property] if property_deps.key?(property)
391
+ end
392
+ # Property values, per node name
358
393
  # Hash< String, Object >
359
- updated_metadata = {}
360
- (
361
- (@cmdbs_per_property.key?(property) ? @cmdbs_per_property[property] : []).map { |cmdb| [cmdb, property] } +
362
- @cmdbs_others.map { |cmdb| [cmdb, :others] }
363
- ).each do |(cmdb, cmdb_property)|
364
- # If among the missing nodes some of them have some master CMDB declared for this property, filter them out unless we are dealing with their master CMDB.
365
- nodes_to_query = missing_nodes.select do |node|
366
- master_cmdb = cmdb_master_for(node, property)
367
- master_cmdb.nil? || master_cmdb == cmdb
368
- end
369
- unless nodes_to_query.empty?
370
- # Check first if this property depends on other ones for this cmdb
371
- if cmdb.respond_to?(:property_dependencies)
372
- property_deps = cmdb.property_dependencies
373
- prefetch_metadata_of nodes_to_query, property_deps[property] if property_deps.key?(property)
374
- end
375
- # Property values, per node name
376
- # Hash< String, Object >
377
- metadata_from_cmdb = Hash[
378
- cmdb.send("get_#{cmdb_property}".to_sym, nodes_to_query, @metadata.slice(*nodes_to_query)).map do |node, cmdb_result|
379
- [node, cmdb_property == :others ? cmdb_result[property] : cmdb_result]
380
- end
381
- ].compact
382
- cmdb_log_header = "[CMDB #{cmdb.class.name.split('::').last}.#{cmdb_property}] -"
383
- log_debug "#{cmdb_log_header} Query property #{property} for #{nodes_to_query.size} nodes (#{nodes_to_query[0..7].join(', ')}...) => Found metadata for #{metadata_from_cmdb.size} nodes."
384
- updated_metadata.merge!(metadata_from_cmdb) do |node, existing_value, new_value|
385
- raise "#{cmdb_log_header} Returned a conflicting value for metadata #{property} of node #{node}: #{new_value} whereas the value was already set to #{existing_value}" if !existing_value.nil? && new_value != existing_value
386
- new_value
387
- end
388
- end
394
+ metadata_from_cmdb = cmdb.send("get_#{cmdb_property}".to_sym, nodes_to_query, @metadata.slice(*nodes_to_query)).transform_values do |cmdb_result|
395
+ cmdb_property == :others ? cmdb_result[property] : cmdb_result
396
+ end.compact
397
+ cmdb_log_header = "[CMDB #{cmdb.class.name.split('::').last}.#{cmdb_property}] -"
398
+ log_debug "#{cmdb_log_header} Query property #{property} for #{nodes_to_query.size} nodes (#{nodes_to_query[0..7].join(', ')}...) => Found metadata for #{metadata_from_cmdb.size} nodes."
399
+ updated_metadata.merge!(metadata_from_cmdb) do |node, existing_value, new_value|
400
+ raise "#{cmdb_log_header} Returned a conflicting value for metadata #{property} of node #{node}: #{new_value} whereas the value was already set to #{existing_value}" if !existing_value.nil? && new_value != existing_value
401
+
402
+ new_value
389
403
  end
390
- # Avoid conflicts in metadata while merging and make sure this update is thread-safe
391
- # As @metadata is only appending data and never deleting it, protecting the update only is enough.
392
- # At worst several threads will query several times the same CMDBs to update the same data several times.
393
- # If we also want to be thread-safe in this regard, we should protect the whole CMDB call with mutexes, at the granularity of the node + property bein read.
394
- @metadata_mutex.synchronize do
395
- missing_nodes.each do |node|
396
- @metadata[node] = {} unless @metadata.key?(node)
397
- # Here, explicitely store nil if nothing has been found for a node because we know there is no value to be fetched.
398
- # This way we won't query again all CMDBs thanks to the cache.
399
- @metadata[node][property] = updated_metadata[node]
400
- end
404
+ end
405
+ # Avoid conflicts in metadata while merging and make sure this update is thread-safe
406
+ # As @metadata is only appending data and never deleting it, protecting the update only is enough.
407
+ # At worst several threads will query several times the same CMDBs to update the same data several times.
408
+ # If we also want to be thread-safe in this regard, we should protect the whole CMDB call with mutexes, at the granularity of the node + property bein read.
409
+ @metadata_mutex.synchronize do
410
+ missing_nodes.each do |node|
411
+ @metadata[node] = {} unless @metadata.key?(node)
412
+ # Here, explicitely store nil if nothing has been found for a node because we know there is no value to be fetched.
413
+ # This way we won't query again all CMDBs thanks to the cache.
414
+ @metadata[node][property] = updated_metadata[node]
401
415
  end
402
416
  end
403
417
  end
@@ -426,6 +440,7 @@ module HybridPlatformsConductor
426
440
  nodes_selectors = nodes_selectors.flatten
427
441
  # 1. Check for the presence of all
428
442
  return known_nodes if nodes_selectors.any? { |nodes_selector| nodes_selector.is_a?(Hash) && nodes_selector.key?(:all) && nodes_selector[:all] }
443
+
429
444
  # 2. Expand the nodes lists, platforms and services contents
430
445
  string_nodes = []
431
446
  nodes_selectors.each do |nodes_selector|
@@ -435,6 +450,7 @@ module HybridPlatformsConductor
435
450
  if nodes_selector.key?(:list)
436
451
  platform = @nodes_list_platform[nodes_selector[:list]]
437
452
  raise "Unknown nodes list: #{nodes_selector[:list]}" if platform.nil?
453
+
438
454
  string_nodes.concat(platform.nodes_selectors_from_nodes_list(nodes_selector[:list]))
439
455
  end
440
456
  string_nodes.concat(@platforms_handler.platform(nodes_selector[:platform]).known_nodes) if nodes_selector.key?(:platform)
@@ -462,8 +478,8 @@ module HybridPlatformsConductor
462
478
  # 3. Expand the Regexps
463
479
  real_nodes = []
464
480
  string_nodes.each do |node|
465
- if node =~ /^\/(.+)\/$/
466
- node_regexp = Regexp.new($1)
481
+ if node =~ %r{^/(.+)/$}
482
+ node_regexp = Regexp.new(Regexp.last_match(1))
467
483
  real_nodes.concat(known_nodes.select { |known_node| known_node[node_regexp] })
468
484
  else
469
485
  real_nodes << node
@@ -489,13 +505,11 @@ module HybridPlatformsConductor
489
505
  # * *parallel* (Boolean): Iterate in a multithreaded way? [default: false]
490
506
  # * *nbr_threads_max* (Integer or nil): Maximum number of threads to be used in case of parallel, or nil for no limit [default: nil]
491
507
  # * *progress* (String or nil): Name of a progress bar to follow the progression, or nil for no progress bar [default: 'Processing nodes']
492
- # * Proc: The code called for each node being iterated on.
508
+ # * *block* (Proc): The code called for each node being iterated on.
493
509
  # * Parameters::
494
510
  # * *node* (String): The node name
495
- def for_each_node_in(nodes, parallel: false, nbr_threads_max: nil, progress: 'Processing nodes')
496
- for_each_element_in(nodes.sort, parallel: parallel, nbr_threads_max: nbr_threads_max, progress: progress) do |node|
497
- yield node
498
- end
511
+ def for_each_node_in(nodes, parallel: false, nbr_threads_max: nil, progress: 'Processing nodes', &block)
512
+ for_each_element_in(nodes.sort, parallel: parallel, nbr_threads_max: nbr_threads_max, progress: progress, &block)
499
513
  end
500
514
 
501
515
  # Get the list of impacted nodes from a git diff on a platform
@@ -513,10 +527,11 @@ module HybridPlatformsConductor
513
527
  def impacted_nodes_from_git_diff(platform_name, from_commit: 'master', to_commit: nil, smallest_set: false)
514
528
  platform = @platforms_handler.platform(platform_name)
515
529
  raise "Unkown platform #{platform_name}. Possible platforms are #{@platforms_handler.known_platforms.map(&:name).sort.join(', ')}" if platform.nil?
530
+
516
531
  begin
517
532
  _exit_status, stdout, _stderr = @cmd_runner.run_cmd "cd #{platform.repository_path} && git --no-pager diff --no-color #{from_commit} #{to_commit.nil? ? '' : to_commit}", log_to_stdout: log_debug?
518
533
  rescue CmdRunner::UnexpectedExitCodeError
519
- raise GitError, $!.to_s
534
+ raise GitError, $ERROR_INFO.to_s
520
535
  end
521
536
  # Parse the git diff output to create a structured diff
522
537
  # Hash< String, Hash< Symbol, Object > >: List of diffs info, per file name having a diff. Diffs info have the following properties:
@@ -526,9 +541,10 @@ module HybridPlatformsConductor
526
541
  current_file_diff = nil
527
542
  stdout.split("\n").each do |line|
528
543
  case line
529
- when /^diff --git a\/(.+) b\/(.+)$/
544
+ when %r{^diff --git a/(.+) b/(.+)$}
530
545
  # A new file diff
531
- from, to = $1, $2
546
+ from = Regexp.last_match(1)
547
+ to = Regexp.last_match(2)
532
548
  current_file_diff = {
533
549
  diff: ''
534
550
  }
@@ -626,11 +642,13 @@ module HybridPlatformsConductor
626
642
  cmdb_masters_cache = {}
627
643
  select_confs_for_node(node, @config.cmdb_masters).each do |cmdb_masters_info|
628
644
  cmdb_masters_info[:cmdb_masters].each do |cmdb, properties|
629
- properties.each do |property|
630
- raise "Property #{property} have conflicting CMDB masters for #{node} declared in the configuration: #{cmdb_masters_cache[property].class.name} and #{@cmdbs[cmdb].class.name}" if cmdb_masters_cache.key?(property) && cmdb_masters_cache[property] != @cmdbs[cmdb]
631
- log_debug "CMDB master for #{node} / #{property}: #{cmdb}"
632
- raise "CMDB #{cmdb} is configured as a master for property #{property} on node #{node} but it does not implement the needed API to retrieve it" unless (@cmdbs_per_property[property] || []).include?(@cmdbs[cmdb]) || @cmdbs_others.include?(@cmdbs[cmdb])
633
- cmdb_masters_cache[property] = @cmdbs[cmdb]
645
+ properties.each do |itr_property|
646
+ raise "Property #{itr_property} have conflicting CMDB masters for #{node} declared in the configuration: #{cmdb_masters_cache[itr_property].class.name} and #{@cmdbs[cmdb].class.name}" if cmdb_masters_cache.key?(itr_property) && cmdb_masters_cache[itr_property] != @cmdbs[cmdb]
647
+
648
+ log_debug "CMDB master for #{node} / #{itr_property}: #{cmdb}"
649
+ raise "CMDB #{cmdb} is configured as a master for property #{itr_property} on node #{node} but it does not implement the needed API to retrieve it" unless (@cmdbs_per_property[itr_property] || []).include?(@cmdbs[cmdb]) || @cmdbs_others.include?(@cmdbs[cmdb])
650
+
651
+ cmdb_masters_cache[itr_property] = @cmdbs[cmdb]
634
652
  end
635
653
  end
636
654
  end