hybrid_platforms_conductor 32.17.0 → 33.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +36 -0
  3. data/README.md +6 -3
  4. data/bin/check-node +0 -1
  5. data/bin/deploy +0 -1
  6. data/bin/get_impacted_nodes +0 -1
  7. data/bin/last_deploys +12 -8
  8. data/bin/nodes_to_deploy +6 -6
  9. data/bin/setup +6 -6
  10. data/bin/topograph +1 -1
  11. data/docs/config_dsl.md +45 -1
  12. data/docs/executables.md +6 -7
  13. data/docs/executables/check-node.md +3 -3
  14. data/docs/executables/deploy.md +3 -3
  15. data/docs/executables/dump_nodes_json.md +3 -3
  16. data/docs/executables/test.md +3 -3
  17. data/docs/executables/topograph.md +3 -3
  18. data/docs/gen/mermaid/README.md-0.png +0 -0
  19. data/docs/gen/mermaid/docs/executables/check-node.md-0.png +0 -0
  20. data/docs/gen/mermaid/docs/executables/deploy.md-0.png +0 -0
  21. data/docs/gen/mermaid/docs/executables/free_ips.md-0.png +0 -0
  22. data/docs/gen/mermaid/docs/executables/get_impacted_nodes.md-0.png +0 -0
  23. data/docs/gen/mermaid/docs/executables/last_deploys.md-0.png +0 -0
  24. data/docs/gen/mermaid/docs/executables/nodes_to_deploy.md-0.png +0 -0
  25. data/docs/gen/mermaid/docs/executables/report.md-0.png +0 -0
  26. data/docs/gen/mermaid/docs/executables/run.md-0.png +0 -0
  27. data/docs/gen/mermaid/docs/executables/ssh_config.md-0.png +0 -0
  28. data/docs/gen/mermaid/docs/executables/test.md-0.png +0 -0
  29. data/docs/plugins.md +46 -0
  30. data/docs/plugins/connector/ssh.md +1 -1
  31. data/docs/plugins/log/remote_fs.md +26 -0
  32. data/docs/plugins/secrets_reader/cli.md +31 -0
  33. data/docs/plugins/secrets_reader/thycotic.md +46 -0
  34. data/docs/plugins/test/check_deploy_and_idempotence.md +1 -1
  35. data/docs/plugins/test/connection.md +1 -0
  36. data/docs/plugins/test/deploy_removes_root_access.md +1 -1
  37. data/docs/plugins/test/file_system.md +1 -0
  38. data/docs/plugins/test/hostname.md +1 -0
  39. data/docs/plugins/test/ip.md +1 -0
  40. data/docs/plugins/test/local_users.md +1 -0
  41. data/docs/plugins/test/mounts.md +1 -0
  42. data/docs/plugins/test/orphan_files.md +1 -0
  43. data/docs/plugins/test/ports.md +1 -0
  44. data/docs/plugins/test/spectre.md +1 -0
  45. data/docs/plugins/test/vulnerabilities.md +1 -0
  46. data/lib/hybrid_platforms_conductor/action.rb +4 -4
  47. data/lib/hybrid_platforms_conductor/actions_executor.rb +45 -43
  48. data/lib/hybrid_platforms_conductor/bitbucket.rb +5 -4
  49. data/lib/hybrid_platforms_conductor/cmd_runner.rb +13 -12
  50. data/lib/hybrid_platforms_conductor/cmdb.rb +2 -2
  51. data/lib/hybrid_platforms_conductor/common_config_dsl/bitbucket.rb +2 -1
  52. data/lib/hybrid_platforms_conductor/common_config_dsl/confluence.rb +2 -1
  53. data/lib/hybrid_platforms_conductor/common_config_dsl/file_system_tests.rb +5 -4
  54. data/lib/hybrid_platforms_conductor/common_config_dsl/github.rb +4 -3
  55. data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb +2 -2
  56. data/lib/hybrid_platforms_conductor/config.rb +8 -4
  57. data/lib/hybrid_platforms_conductor/confluence.rb +1 -1
  58. data/lib/hybrid_platforms_conductor/connector.rb +5 -2
  59. data/lib/hybrid_platforms_conductor/core_extensions/cleanroom/fix_kwargs.rb +116 -0
  60. data/lib/hybrid_platforms_conductor/core_extensions/symbol/zero.rb +24 -0
  61. data/lib/hybrid_platforms_conductor/credentials.rb +39 -36
  62. data/lib/hybrid_platforms_conductor/current_dir_monitor.rb +4 -1
  63. data/lib/hybrid_platforms_conductor/deployer.rb +275 -224
  64. data/lib/hybrid_platforms_conductor/executable.rb +20 -15
  65. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/config.rb +10 -7
  66. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb +1 -1
  67. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb +2 -2
  68. data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/platform_handlers.rb +4 -4
  69. data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb +2 -0
  70. data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb +45 -49
  71. data/lib/hybrid_platforms_conductor/hpc_plugins/log/my_log_plugin.rb.sample +100 -0
  72. data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb +180 -0
  73. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +68 -66
  74. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb +13 -0
  75. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb +39 -38
  76. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb +5 -4
  77. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb +43 -45
  78. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/podman.rb +18 -20
  79. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb +118 -117
  80. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/proxmox_waiter.rb +39 -43
  81. data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox/reserve_proxmox_container +9 -13
  82. data/lib/hybrid_platforms_conductor/hpc_plugins/report/confluence.rb +2 -2
  83. data/lib/hybrid_platforms_conductor/hpc_plugins/report/mediawiki.rb +28 -21
  84. data/lib/hybrid_platforms_conductor/hpc_plugins/report/stdout.rb +26 -22
  85. data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/cli.rb +77 -0
  86. data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/my_secrets_reader_plugin.rb.sample +46 -0
  87. data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb +90 -0
  88. data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb +3 -3
  89. data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb +4 -2
  90. data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb +3 -1
  91. data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb +7 -21
  92. data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb +21 -19
  93. data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb +2 -2
  94. data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb +2 -2
  95. data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb +21 -22
  96. data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb +19 -21
  97. data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb +2 -3
  98. data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb +3 -1
  99. data/lib/hybrid_platforms_conductor/hpc_plugins/test/idempotence.rb +2 -2
  100. data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb +4 -2
  101. data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_conf.rb +21 -22
  102. data/lib/hybrid_platforms_conductor/hpc_plugins/test/jenkins_ci_masters_ok.rb +10 -12
  103. data/lib/hybrid_platforms_conductor/hpc_plugins/test/linear_strategy.rb +9 -9
  104. data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb +5 -3
  105. data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb +5 -3
  106. data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb +13 -10
  107. data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb +5 -3
  108. data/lib/hybrid_platforms_conductor/hpc_plugins/test/private_ips.rb +5 -5
  109. data/lib/hybrid_platforms_conductor/hpc_plugins/test/public_ips.rb +5 -5
  110. data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb +7 -7
  111. data/lib/hybrid_platforms_conductor/hpc_plugins/test/veids.rb +3 -3
  112. data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb +27 -25
  113. data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/confluence.rb +2 -2
  114. data/lib/hybrid_platforms_conductor/hpc_plugins/test_report/stdout.rb +8 -6
  115. data/lib/hybrid_platforms_conductor/io_router.rb +14 -13
  116. data/lib/hybrid_platforms_conductor/json_dumper.rb +2 -2
  117. data/lib/hybrid_platforms_conductor/log.rb +31 -0
  118. data/lib/hybrid_platforms_conductor/logger_helpers.rb +19 -16
  119. data/lib/hybrid_platforms_conductor/nodes_handler.rb +89 -71
  120. data/lib/hybrid_platforms_conductor/parallel_threads.rb +7 -11
  121. data/lib/hybrid_platforms_conductor/platform_handler.rb +7 -7
  122. data/lib/hybrid_platforms_conductor/platforms_handler.rb +5 -3
  123. data/lib/hybrid_platforms_conductor/plugin.rb +2 -2
  124. data/lib/hybrid_platforms_conductor/plugins.rb +14 -8
  125. data/lib/hybrid_platforms_conductor/provisioner.rb +4 -4
  126. data/lib/hybrid_platforms_conductor/report.rb +2 -2
  127. data/lib/hybrid_platforms_conductor/reports_handler.rb +3 -2
  128. data/lib/hybrid_platforms_conductor/secrets_reader.rb +31 -0
  129. data/lib/hybrid_platforms_conductor/services_handler.rb +32 -29
  130. data/lib/hybrid_platforms_conductor/test_only_remote_node.rb +18 -0
  131. data/lib/hybrid_platforms_conductor/test_report.rb +15 -18
  132. data/lib/hybrid_platforms_conductor/tests_runner.rb +116 -118
  133. data/lib/hybrid_platforms_conductor/thycotic.rb +28 -19
  134. data/lib/hybrid_platforms_conductor/topographer.rb +200 -190
  135. data/lib/hybrid_platforms_conductor/topographer/plugins/graphviz.rb +8 -8
  136. data/lib/hybrid_platforms_conductor/topographer/plugins/json.rb +4 -4
  137. data/lib/hybrid_platforms_conductor/version.rb +1 -1
  138. data/spec/hybrid_platforms_conductor_test.rb +33 -12
  139. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/bash_spec.rb +18 -11
  140. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/interactive_spec.rb +2 -2
  141. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/remote_bash_spec.rb +32 -21
  142. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/ruby_spec.rb +75 -49
  143. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions/scp_spec.rb +27 -15
  144. data/spec/hybrid_platforms_conductor_test/api/actions_executor/actions_spec.rb +90 -59
  145. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connection_spec.rb +46 -44
  146. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/connectable_nodes_spec.rb +12 -8
  147. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/local/remote_actions_spec.rb +4 -7
  148. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb +21 -22
  149. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb +23 -24
  150. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connectable_nodes_spec.rb +10 -6
  151. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb +106 -75
  152. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb +145 -126
  153. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb +3 -3
  154. data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb +29 -25
  155. data/spec/hybrid_platforms_conductor_test/api/actions_executor/logging_spec.rb +167 -142
  156. data/spec/hybrid_platforms_conductor_test/api/actions_executor/parallel_spec.rb +272 -244
  157. data/spec/hybrid_platforms_conductor_test/api/actions_executor/timeout_spec.rb +16 -16
  158. data/spec/hybrid_platforms_conductor_test/api/cmd_runner_spec.rb +36 -36
  159. data/spec/hybrid_platforms_conductor_test/api/config_spec.rb +24 -22
  160. data/spec/hybrid_platforms_conductor_test/api/deployer/check_spec.rb +4 -2
  161. data/spec/hybrid_platforms_conductor_test/api/deployer/config_dsl_spec.rb +43 -5
  162. data/spec/hybrid_platforms_conductor_test/api/deployer/deploy_spec.rb +199 -216
  163. data/spec/hybrid_platforms_conductor_test/api/deployer/log_plugins/remote_fs_spec.rb +223 -0
  164. data/spec/hybrid_platforms_conductor_test/api/deployer/parse_deploy_output_spec.rb +55 -59
  165. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +36 -62
  166. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/podman_spec.rb +17 -17
  167. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/config_dsl_spec.rb +4 -4
  168. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/create_spec.rb +44 -51
  169. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/destroy_spec.rb +3 -3
  170. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/ip_spec.rb +12 -16
  171. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/destroy_vm_spec.rb +31 -19
  172. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/expired_containers_spec.rb +324 -266
  173. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/ips_assignment_spec.rb +89 -61
  174. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/other_lxc_containers_resources_spec.rb +117 -93
  175. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/pve_node_resources_spec.rb +71 -54
  176. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/retries_spec.rb +10 -8
  177. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/reserve_proxmox_container/vm_ids_assignment_spec.rb +80 -60
  178. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/start_spec.rb +1 -1
  179. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/state_spec.rb +1 -1
  180. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioners/proxmox/stop_spec.rb +1 -1
  181. data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/cli_spec.rb +64 -0
  182. data/spec/hybrid_platforms_conductor_test/api/deployer/secrets_reader_plugins/thycotic_spec.rb +268 -0
  183. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/config_spec.rb +8 -10
  184. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_ip_spec.rb +33 -24
  185. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/host_keys_spec.rb +64 -51
  186. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs/platform_handlers_spec.rb +3 -3
  187. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +50 -51
  188. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/common_spec.rb +91 -81
  189. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/config_dsl_spec.rb +14 -16
  190. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/git_diff_impacts_spec.rb +51 -75
  191. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/nodes_selectors_spec.rb +35 -26
  192. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/platform_handlers_plugins_api_spec.rb +24 -16
  193. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/several_platforms_spec.rb +29 -19
  194. data/spec/hybrid_platforms_conductor_test/api/platform_handler_spec.rb +4 -4
  195. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb +2 -2
  196. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb +6 -6
  197. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb +57 -99
  198. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +4 -4
  199. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +32 -35
  200. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +10 -10
  201. data/spec/hybrid_platforms_conductor_test/api/platforms_handler_spec.rb +38 -27
  202. data/spec/hybrid_platforms_conductor_test/api/plugins_spec.rb +46 -52
  203. data/spec/hybrid_platforms_conductor_test/api/reports_handler_spec.rb +2 -2
  204. data/spec/hybrid_platforms_conductor_test/api/services_handler/actions_to_deploy_spec.rb +90 -58
  205. data/spec/hybrid_platforms_conductor_test/api/services_handler/deploy_allowed_spec.rb +38 -34
  206. data/spec/hybrid_platforms_conductor_test/api/services_handler/log_info_spec.rb +11 -9
  207. data/spec/hybrid_platforms_conductor_test/api/services_handler/package_spec.rb +193 -171
  208. data/spec/hybrid_platforms_conductor_test/api/services_handler/parse_deploy_output_spec.rb +66 -54
  209. data/spec/hybrid_platforms_conductor_test/api/services_handler/prepare_for_deploy_spec.rb +147 -133
  210. data/spec/hybrid_platforms_conductor_test/api/tests_runner/common_spec.rb +69 -49
  211. data/spec/hybrid_platforms_conductor_test/api/tests_runner/global_spec.rb +5 -4
  212. data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_check_spec.rb +8 -5
  213. data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_spec.rb +8 -5
  214. data/spec/hybrid_platforms_conductor_test/api/tests_runner/node_ssh_spec.rb +30 -27
  215. data/spec/hybrid_platforms_conductor_test/api/tests_runner/platform_spec.rb +12 -9
  216. data/spec/hybrid_platforms_conductor_test/api/tests_runner/reports_spec.rb +48 -47
  217. data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/bitbucket_conf_spec.rb +5 -5
  218. data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_plugins/github_ci_spec.rb +5 -5
  219. data/spec/hybrid_platforms_conductor_test/api/tests_runner/test_reports_plugins/confluence_spec.rb +5 -5
  220. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb.rb +9 -9
  221. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb2.rb → test_cmdb_2.rb} +6 -6
  222. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/test_cmdb_others.rb +3 -3
  223. data/spec/hybrid_platforms_conductor_test/cmdb_plugins/{test_cmdb_others2.rb → test_cmdb_others_2.rb} +2 -2
  224. data/spec/hybrid_platforms_conductor_test/docs_spec.rb +1 -1
  225. data/spec/hybrid_platforms_conductor_test/executables/{check-node_spec.rb → check_node_spec.rb} +4 -6
  226. data/spec/hybrid_platforms_conductor_test/executables/deploy_spec.rb +4 -6
  227. data/spec/hybrid_platforms_conductor_test/executables/get_impacted_nodes_spec.rb +76 -77
  228. data/spec/hybrid_platforms_conductor_test/executables/last_deploys_spec.rb +159 -113
  229. data/spec/hybrid_platforms_conductor_test/executables/nodes_to_deploy_spec.rb +299 -160
  230. data/spec/hybrid_platforms_conductor_test/executables/options/actions_executor_spec.rb +4 -6
  231. data/spec/hybrid_platforms_conductor_test/executables/options/cmd_runner_spec.rb +3 -5
  232. data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb +8 -8
  233. data/spec/hybrid_platforms_conductor_test/executables/options/deployer_spec.rb +12 -196
  234. data/spec/hybrid_platforms_conductor_test/executables/options/nodes_handler_spec.rb +9 -10
  235. data/spec/hybrid_platforms_conductor_test/executables/options/nodes_selectors_spec.rb +9 -10
  236. data/spec/hybrid_platforms_conductor_test/executables/options/reports_handler_spec.rb +1 -1
  237. data/spec/hybrid_platforms_conductor_test/executables/options/tests_runner_spec.rb +22 -22
  238. data/spec/hybrid_platforms_conductor_test/executables/report_spec.rb +22 -16
  239. data/spec/hybrid_platforms_conductor_test/executables/run_spec.rb +32 -32
  240. data/spec/hybrid_platforms_conductor_test/executables/ssh_config_spec.rb +7 -9
  241. data/spec/hybrid_platforms_conductor_test/executables/test_spec.rb +3 -5
  242. data/spec/hybrid_platforms_conductor_test/helpers/actions_executor_helpers.rb +2 -2
  243. data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb +4 -3
  244. data/spec/hybrid_platforms_conductor_test/helpers/cmdb_helpers.rb +2 -2
  245. data/spec/hybrid_platforms_conductor_test/helpers/config_helpers.rb +1 -1
  246. data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb +12 -13
  247. data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb +245 -56
  248. data/spec/hybrid_platforms_conductor_test/helpers/executables_helpers.rb +11 -11
  249. data/spec/hybrid_platforms_conductor_test/helpers/nodes_handler_helpers.rb +1 -1
  250. data/spec/hybrid_platforms_conductor_test/helpers/platforms_handler_helpers.rb +39 -28
  251. data/spec/hybrid_platforms_conductor_test/helpers/plugins_helpers.rb +1 -1
  252. data/spec/hybrid_platforms_conductor_test/helpers/provisioner_proxmox_helpers.rb +86 -111
  253. data/spec/hybrid_platforms_conductor_test/helpers/reports_handler_helpers.rb +1 -1
  254. data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +3 -3
  255. data/spec/hybrid_platforms_conductor_test/helpers/services_handler_helpers.rb +1 -1
  256. data/spec/hybrid_platforms_conductor_test/helpers/tests_runner_helpers.rb +1 -1
  257. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id1.rb → test_plugin_id_1.rb} +0 -0
  258. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem/hpc_plugins/test_plugin_type/{test_plugin_id2.rb → test_plugin_id_2.rb} +0 -0
  259. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/test_plugin_type/{test_plugin_id3.rb → test_plugin_id_3.rb} +0 -0
  260. data/spec/hybrid_platforms_conductor_test/mocked_lib/my_test_gem2/sub_dir/hpc_plugins/{test_plugin_type2/test_plugin_id4.rb → test_plugin_type_2/test_plugin_id_4.rb} +0 -0
  261. data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/test.rb +1 -1
  262. data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/{test2.rb → test_2.rb} +0 -0
  263. data/spec/hybrid_platforms_conductor_test/rubocop_spec.rb +31 -0
  264. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json +3 -3
  265. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json +3 -3
  266. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb +1 -0
  267. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb +1 -0
  268. data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb +952 -0
  269. data/spec/hybrid_platforms_conductor_test/test_connector.rb +3 -3
  270. data/spec/hybrid_platforms_conductor_test/test_log_no_read_plugin.rb +84 -0
  271. data/spec/hybrid_platforms_conductor_test/test_log_plugin.rb +105 -0
  272. data/spec/hybrid_platforms_conductor_test/test_plugins/global.rb +1 -0
  273. data/spec/hybrid_platforms_conductor_test/test_plugins/node.rb +1 -0
  274. data/spec/hybrid_platforms_conductor_test/test_plugins/node_check.rb +1 -0
  275. data/spec/hybrid_platforms_conductor_test/test_plugins/platform.rb +1 -0
  276. data/spec/hybrid_platforms_conductor_test/test_plugins/several_checks.rb +2 -2
  277. data/spec/hybrid_platforms_conductor_test/test_secrets_reader_plugin.rb +45 -0
  278. data/spec/hybrid_platforms_conductor_test/tests_report_plugin.rb +5 -6
  279. data/spec/spec_helper.rb +17 -18
  280. data/tools/check_md +16 -20
  281. data/tools/generate_mermaid +1 -1
  282. metadata +195 -144
  283. data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb +0 -916
@@ -29,6 +29,10 @@ module HybridPlatformsConductor
29
29
  instance_eval(File.read(source))
30
30
  end
31
31
 
32
+ # Intercept all missing methods
33
+ #
34
+ # Parameters::
35
+ # * *method_name* (Symbol): The missing method being called
32
36
  def method_missing(method_name, *args, &block)
33
37
  sub_calls = []
34
38
  @calls << {
@@ -40,6 +44,15 @@ module HybridPlatformsConductor
40
44
  DslParser.new(sub_calls)
41
45
  end
42
46
 
47
+ # Make sure we register the methods we handle in method_missing
48
+ #
49
+ # Parameters::
50
+ # * *name* (Symbol): The missing method name
51
+ # * *include_private* (Boolean): Should we include private methods in the search?
52
+ def respond_to_missing?(_name, _include_private)
53
+ true
54
+ end
55
+
43
56
  end
44
57
 
45
58
  end
@@ -40,11 +40,14 @@ module HybridPlatformsConductor
40
40
  add_recipe_in_tree(cookbook_dir, cookbook, recipe)
41
41
  end
42
42
  end
43
+ # Make sure we don't combine those 2 loops
44
+ # rubocop:disable Style/CombinableLoops
43
45
  @platform.deployable_services.each do |service|
44
- @platform.policy_run_list(service).each do |(cookbook_dir, cookbook, recipe)|
46
+ @platform.policy_run_list(service).each do |(_cookbook_dir, cookbook, recipe)|
45
47
  mark_recipe_used_by_policy(cookbook, recipe, service)
46
48
  end
47
49
  end
50
+ # rubocop:enable Style/CombinableLoops
48
51
  @recipes_tree
49
52
  end
50
53
 
@@ -58,25 +61,25 @@ module HybridPlatformsConductor
58
61
  # * *recipe* (Symbol): The recipe name
59
62
  def add_recipe_in_tree(cookbook_dir, cookbook, recipe)
60
63
  @recipes_tree[cookbook] = {} unless @recipes_tree.key?(cookbook)
61
- unless @recipes_tree[cookbook].key?(recipe)
62
- recipe_info =
63
- if cookbook_dir.nil?
64
- # This recipe comes from an external cookbook, we won't get into it.
65
- {
66
- included_recipes: [],
67
- used_templates: [],
68
- used_files: [],
69
- used_cookbooks: []
70
- }
71
- else
72
- recipe_usage(cookbook_dir, cookbook, recipe)
73
- end
74
- @recipes_tree[cookbook][recipe] = recipe_info.merge(
75
- used_by_policies: []
76
- )
77
- recipe_info[:included_recipes].each do |(sub_cookbook_dir, sub_cookbook, sub_recipe)|
78
- add_recipe_in_tree(sub_cookbook_dir, sub_cookbook, sub_recipe)
64
+ return if @recipes_tree[cookbook].key?(recipe)
65
+
66
+ recipe_info =
67
+ if cookbook_dir.nil?
68
+ # This recipe comes from an external cookbook, we won't get into it.
69
+ {
70
+ included_recipes: [],
71
+ used_templates: [],
72
+ used_files: [],
73
+ used_cookbooks: []
74
+ }
75
+ else
76
+ recipe_usage(cookbook_dir, cookbook, recipe)
79
77
  end
78
+ @recipes_tree[cookbook][recipe] = recipe_info.merge(
79
+ used_by_policies: []
80
+ )
81
+ recipe_info[:included_recipes].each do |(sub_cookbook_dir, sub_cookbook, sub_recipe)|
82
+ add_recipe_in_tree(sub_cookbook_dir, sub_cookbook, sub_recipe)
80
83
  end
81
84
  end
82
85
 
@@ -115,18 +118,18 @@ module HybridPlatformsConductor
115
118
  recipe_content.
116
119
  scan(/source\s+(["'])(.+?)\1/).
117
120
  each do |(_sub_grp, source)|
118
- sources << source unless source =~ /^https?:\/\//
121
+ sources << source unless source =~ %r{^https?://}
119
122
  end
120
123
  erb_sources = sources.select { |source| File.extname(source).downcase == '.erb' }
121
124
  non_erb_sources = sources - erb_sources
122
125
  erb_sources.concat(recipe_content.scan(/template:?\s+(["'])(.+?)\1/).map { |(_sub_grp, source)| source })
123
126
  # Check for known resources and library methods
124
127
  used_cookbooks = []
125
- known_resources.each do |cookbook, methods|
126
- used_cookbooks << cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
128
+ known_resources.each do |itr_cookbook, methods|
129
+ used_cookbooks << itr_cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
127
130
  end
128
- known_library_methods.each do |cookbook, methods|
129
- used_cookbooks << cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
131
+ known_library_methods.each do |itr_cookbook, methods|
132
+ used_cookbooks << itr_cookbook if methods.any? { |method_name| recipe_content.include?(method_name) }
130
133
  end
131
134
  {
132
135
  included_recipes: used_recipes,
@@ -171,9 +174,9 @@ module HybridPlatformsConductor
171
174
  @known_library_methods = {}
172
175
  for_each_cookbook do |cookbook, cookbook_dir|
173
176
  if File.exist?("#{cookbook_dir}/libraries")
174
- found_methods = Dir.glob("#{cookbook_dir}/libraries/*.rb").
175
- map { |lib_file| File.read(lib_file).scan(/\bdef\s+(\w+)\b/).map { |(method_name)| method_name } }.
176
- flatten - INVALID_LIBRARY_METHODS
177
+ found_methods = Dir.glob("#{cookbook_dir}/libraries/*.rb").
178
+ map { |lib_file| File.read(lib_file).scan(/\bdef\s+(\w+)\b/).map { |(method_name)| method_name } }.
179
+ flatten - INVALID_LIBRARY_METHODS
177
180
  @known_library_methods[cookbook] = found_methods unless found_methods.empty?
178
181
  end
179
182
  end
@@ -184,15 +187,13 @@ module HybridPlatformsConductor
184
187
  # Iterate over all cookbooks
185
188
  #
186
189
  # Parameters::
187
- # * Proc: Code called for each cookbook:
190
+ # * *block* (Proc): Code called for each cookbook:
188
191
  # * Parameters::
189
192
  # * *cookbook* (Symbol): Cookbook name
190
193
  # * *cookbook_dir* (String): Cookbook directory
191
- def for_each_cookbook
194
+ def for_each_cookbook(&block)
192
195
  @platform.known_cookbook_paths.each do |cookbook_path|
193
- cookbooks_in(cookbook_path).each do |cookbook, cookbook_dir|
194
- yield cookbook, cookbook_dir
195
- end
196
+ cookbooks_in(cookbook_path).each(&block)
196
197
  end
197
198
  end
198
199
 
@@ -203,7 +204,7 @@ module HybridPlatformsConductor
203
204
  # Result::
204
205
  # * Hash<Symbol, String>: List of cookbook directories, per cookbook name
205
206
  def cookbooks_in(cookbook_type)
206
- Hash[Dir.glob("#{@platform.repository_path}/#{cookbook_type}/*").map { |dir| [File.basename(dir).to_sym, dir] }.sort]
207
+ Dir.glob("#{@platform.repository_path}/#{cookbook_type}/*").map { |dir| [File.basename(dir).to_sym, dir] }.sort.to_h
207
208
  end
208
209
 
209
210
  # Mark a recipe (and its included recipes) as used by a policy
@@ -213,11 +214,11 @@ module HybridPlatformsConductor
213
214
  # * *recipe* (Symbol): The recipe
214
215
  # * *used_by_policy* (String): The policy using this recipe
215
216
  def mark_recipe_used_by_policy(cookbook, recipe, used_by_policy)
216
- unless @recipes_tree[cookbook][recipe][:used_by_policies].include?(used_by_policy)
217
- @recipes_tree[cookbook][recipe][:used_by_policies] << used_by_policy
218
- @recipes_tree[cookbook][recipe][:included_recipes].each do |(_sub_cookbook_dir, sub_cookbook, sub_recipe)|
219
- mark_recipe_used_by_policy(sub_cookbook, sub_recipe, used_by_policy)
220
- end
217
+ return if @recipes_tree[cookbook][recipe][:used_by_policies].include?(used_by_policy)
218
+
219
+ @recipes_tree[cookbook][recipe][:used_by_policies] << used_by_policy
220
+ @recipes_tree[cookbook][recipe][:included_recipes].each do |(_sub_cookbook_dir, sub_cookbook, sub_recipe)|
221
+ mark_recipe_used_by_policy(sub_cookbook, sub_recipe, used_by_policy)
221
222
  end
222
223
  end
223
224
 
@@ -16,7 +16,7 @@ module HybridPlatformsConductor
16
16
  def init
17
17
  # This method is called when initializing a new instance of this platform handler, for a given repository.
18
18
  inv_file = "#{@repository_path}/inventory.yaml"
19
- @inventory = File.exist?(inv_file) ? YAML.load(File.read(inv_file)) : {}
19
+ @inventory = File.exist?(inv_file) ? YAML.safe_load(File.read(inv_file)) : {}
20
20
  end
21
21
 
22
22
  # Get the list of known nodes.
@@ -89,12 +89,13 @@ module HybridPlatformsConductor
89
89
  # * *cmd_runner* (CmdRunner): CmdRunner to be used [default: CmdRunner.new]
90
90
  def initialize(
91
91
  platform_handler,
92
- logger: Logger.new(STDOUT),
93
- logger_stderr: Logger.new(STDERR),
92
+ logger: Logger.new($stdout),
93
+ logger_stderr: Logger.new($stderr),
94
94
  config: Config.new,
95
95
  nodes_handler: NodesHandler.new,
96
96
  cmd_runner: CmdRunner.new
97
97
  )
98
+ super
98
99
  init_loggers(logger, logger_stderr)
99
100
  @platform_handler = platform_handler
100
101
  @config = config
@@ -127,7 +128,7 @@ module HybridPlatformsConductor
127
128
  # * *:changed*: The task has been changed
128
129
  # * *:identical*: The task has not been changed
129
130
  # * *diffs* (String): Differences, if any
130
- def parse_deploy_output(stdout, stderr)
131
+ def parse_deploy_output(_stdout, _stderr)
131
132
  []
132
133
  end
133
134
 
@@ -21,7 +21,7 @@ module HybridPlatformsConductor
21
21
  ::Docker.validate_version!
22
22
  docker_ok = true
23
23
  rescue
24
- log_error "[ #{@node}/#{@environment} ] - Docker is not installed correctly. Please install it. Error: #{$!}"
24
+ log_error "[ #{@node}/#{@environment} ] - Docker is not installed correctly. Please install it. Error: #{$ERROR_INFO}"
25
25
  end
26
26
  docker_ok
27
27
  end
@@ -33,47 +33,45 @@ module HybridPlatformsConductor
33
33
  # Get the image name for this node
34
34
  image = @nodes_handler.get_image_of(@node).to_sym
35
35
  # Find if we have such an image registered
36
- if @config.known_os_images.include?(image)
37
- # Build the image if it does not exist
38
- image_tag = "hpc_image_#{image}"
39
- docker_image = nil
40
- image_futex_file = "#{Dir.tmpdir}/hpc_docker_image_futexes/#{image_tag}"
41
- FileUtils.mkdir_p File.dirname(image_futex_file)
42
- Futex.new(image_futex_file).open do
43
- docker_image = ::Docker::Image.all.find { |search_image| !search_image.info['RepoTags'].nil? && search_image.info['RepoTags'].include?("#{image_tag}:latest") }
44
- unless docker_image
45
- log_debug "[ #{@node}/#{@environment} ] - Creating Docker image #{image_tag}..."
46
- Excon.defaults[:read_timeout] = 600
47
- docker_image = ::Docker::Image.build_from_dir(@config.os_image_dir(image))
48
- docker_image.tag repo: image_tag
49
- end
50
- end
51
- container_name = "hpc_docker_container_#{@node}_#{@environment}"
52
- container_futex_file = "#{Dir.tmpdir}/hpc_docker_container_futexes/#{image_tag}"
53
- FileUtils.mkdir_p File.dirname(container_futex_file)
54
- Futex.new(container_futex_file).open do
55
- old_docker_container = ::Docker::Container.all(all: true).find { |container| container.info['Names'].include? "/#{container_name}" }
56
- @container =
57
- if old_docker_container
58
- old_docker_container
59
- else
60
- log_debug "[ #{@node}/#{@environment} ] - Creating Docker container #{container_name}..."
61
- # We add the SYS_PTRACE capability as some images need to restart services (for example postfix) and those services need the rights to ls in /proc/{PID}/exe to check if a status is running. Without SYS_PTRACE such ls returns permission denied and the service can't be stopped (as init.d always returns it as stopped even when running).
62
- # We add the privileges as some containers need to install and configure the udev package, which needs RW access to /sys.
63
- # We add the bind to cgroup volume to be able to test systemd specifics (enabling/disabling services for example).
64
- ::Docker::Container.create(
65
- name: container_name,
66
- image: image_tag,
67
- CapAdd: 'SYS_PTRACE',
68
- Privileged: true,
69
- Binds: ['/sys/fs/cgroup:/sys/fs/cgroup:ro'],
70
- # Some playbooks need the hostname to be set to a correct FQDN
71
- Hostname: "#{@node}.testdomain"
72
- )
73
- end
36
+ raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}" unless @config.known_os_images.include?(image)
37
+
38
+ # Build the image if it does not exist
39
+ image_tag = "hpc_image_#{image}"
40
+ docker_image = nil
41
+ image_futex_file = "#{Dir.tmpdir}/hpc_docker_image_futexes/#{image_tag}"
42
+ FileUtils.mkdir_p File.dirname(image_futex_file)
43
+ Futex.new(image_futex_file).open do
44
+ docker_image = ::Docker::Image.all.find { |search_image| !search_image.info['RepoTags'].nil? && search_image.info['RepoTags'].include?("#{image_tag}:latest") }
45
+ unless docker_image
46
+ log_debug "[ #{@node}/#{@environment} ] - Creating Docker image #{image_tag}..."
47
+ Excon.defaults[:read_timeout] = 600
48
+ docker_image = ::Docker::Image.build_from_dir(@config.os_image_dir(image))
49
+ docker_image.tag repo: image_tag
74
50
  end
75
- else
76
- raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}"
51
+ end
52
+ container_name = "hpc_docker_container_#{@node}_#{@environment}"
53
+ container_futex_file = "#{Dir.tmpdir}/hpc_docker_container_futexes/#{image_tag}"
54
+ FileUtils.mkdir_p File.dirname(container_futex_file)
55
+ Futex.new(container_futex_file).open do
56
+ old_docker_container = ::Docker::Container.all(all: true).find { |container| container.info['Names'].include? "/#{container_name}" }
57
+ @container =
58
+ if old_docker_container
59
+ old_docker_container
60
+ else
61
+ log_debug "[ #{@node}/#{@environment} ] - Creating Docker container #{container_name}..."
62
+ # We add the SYS_PTRACE capability as some images need to restart services (for example postfix) and those services need the rights to ls in /proc/{PID}/exe to check if a status is running. Without SYS_PTRACE such ls returns permission denied and the service can't be stopped (as init.d always returns it as stopped even when running).
63
+ # We add the privileges as some containers need to install and configure the udev package, which needs RW access to /sys.
64
+ # We add the bind to cgroup volume to be able to test systemd specifics (enabling/disabling services for example).
65
+ ::Docker::Container.create(
66
+ name: container_name,
67
+ image: image_tag,
68
+ CapAdd: 'SYS_PTRACE',
69
+ Privileged: true,
70
+ Binds: ['/sys/fs/cgroup:/sys/fs/cgroup:ro'],
71
+ # Some playbooks need the hostname to be set to a correct FQDN
72
+ Hostname: "#{@node}.testdomain"
73
+ )
74
+ end
77
75
  end
78
76
  end
79
77
 
@@ -81,7 +79,7 @@ module HybridPlatformsConductor
81
79
  # Prerequisite: create has been called before
82
80
  # [API] - This method is mandatory
83
81
  def start
84
- log_debug "[ #{@node}/#{@environment} ] - Start Docker Container #{@container.refresh!.info['Name'][1..-1]} ..."
82
+ log_debug "[ #{@node}/#{@environment} ] - Start Docker Container #{@container.refresh!.info['Name'][1..]} ..."
85
83
  @container.start
86
84
  end
87
85
 
@@ -89,7 +87,7 @@ module HybridPlatformsConductor
89
87
  # Prerequisite: create has been called before
90
88
  # [API] - This method is mandatory
91
89
  def stop
92
- log_debug "[ #{@node}/#{@environment} ] - Stop Docker Container #{@container.refresh!.info['Name'][1..-1]} ..."
90
+ log_debug "[ #{@node}/#{@environment} ] - Stop Docker Container #{@container.refresh!.info['Name'][1..]} ..."
93
91
  @container.stop
94
92
  end
95
93
 
@@ -97,7 +95,7 @@ module HybridPlatformsConductor
97
95
  # Prerequisite: create has been called before
98
96
  # [API] - This method is mandatory
99
97
  def destroy
100
- log_debug "[ #{@node}/#{@environment} ] - Destroy Docker Container #{@container.refresh!.info['Name'][1..-1]} ..."
98
+ log_debug "[ #{@node}/#{@environment} ] - Destroy Docker Container #{@container.refresh!.info['Name'][1..]} ..."
101
99
  @container.remove
102
100
  @container = nil
103
101
  end
@@ -119,7 +117,7 @@ module HybridPlatformsConductor
119
117
  begin
120
118
  @container.refresh!.info['State']['Status'].to_sym
121
119
  rescue
122
- log_error "[ #{@node}/#{@environment} ] - Error while reading state of Docker container: #{$!}"
120
+ log_error "[ #{@node}/#{@environment} ] - Error while reading state of Docker container: #{$ERROR_INFO}"
123
121
  :error
124
122
  end
125
123
  end
@@ -17,25 +17,23 @@ module HybridPlatformsConductor
17
17
  # Get the image name for this node
18
18
  image = @nodes_handler.get_image_of(@node).to_sym
19
19
  # Find if we have such an image registered
20
- if @config.known_os_images.include?(image)
21
- # Build the image if it does not exist
22
- image_tag = "hpc_image_#{image}"
23
- image_futex_file = "#{Dir.tmpdir}/hpc_podman_image_futexes/#{image_tag}"
24
- FileUtils.mkdir_p File.dirname(image_futex_file)
25
- Futex.new(image_futex_file).open do
26
- @cmd_runner.run_cmd "cd #{@config.os_image_dir(image)} && #{podman_cmd} build --tag #{image_tag} --security-opt seccomp=/usr/share/containers/seccomp.json --cgroup-manager=cgroupfs ."
27
- end
28
- container_name = "hpc_container_#{@node}_#{@environment}"
29
- container_futex_file = "#{Dir.tmpdir}/hpc_podman_container_futexes/#{image_tag}"
30
- FileUtils.mkdir_p File.dirname(container_futex_file)
31
- Futex.new(container_futex_file).open do
32
- _exit_status, stdout, _stderr = @cmd_runner.run_cmd "#{podman_cmd} container list --all | grep #{container_name}", expected_code: [0, 1]
33
- existing_container = !stdout.strip.empty?
34
- @cmd_runner.run_cmd "#{podman_cmd} container create --name #{container_name} #{image_tag}" unless existing_container
35
- @container = container_name
36
- end
37
- else
38
- raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}"
20
+ raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}" unless @config.known_os_images.include?(image)
21
+
22
+ # Build the image if it does not exist
23
+ image_tag = "hpc_image_#{image}"
24
+ image_futex_file = "#{Dir.tmpdir}/hpc_podman_image_futexes/#{image_tag}"
25
+ FileUtils.mkdir_p File.dirname(image_futex_file)
26
+ Futex.new(image_futex_file).open do
27
+ @cmd_runner.run_cmd "cd #{@config.os_image_dir(image)} && #{podman_cmd} build --tag #{image_tag} --security-opt seccomp=/usr/share/containers/seccomp.json --cgroup-manager=cgroupfs ."
28
+ end
29
+ container_name = "hpc_container_#{@node}_#{@environment}"
30
+ container_futex_file = "#{Dir.tmpdir}/hpc_podman_container_futexes/#{image_tag}"
31
+ FileUtils.mkdir_p File.dirname(container_futex_file)
32
+ Futex.new(container_futex_file).open do
33
+ _exit_status, stdout, _stderr = @cmd_runner.run_cmd "#{podman_cmd} container list --all | grep #{container_name}", expected_code: [0, 1]
34
+ existing_container = !stdout.strip.empty?
35
+ @cmd_runner.run_cmd "#{podman_cmd} container create --name #{container_name} #{image_tag}" unless existing_container
36
+ @container = container_name
39
37
  end
40
38
  end
41
39
 
@@ -83,7 +81,7 @@ module HybridPlatformsConductor
83
81
  status = :created if status == :configured
84
82
  status
85
83
  rescue
86
- log_warn "Error while reading state of Podman container #{@container}: #{$!}"
84
+ log_warn "Error while reading state of Podman container #{@container}: #{$ERROR_INFO}"
87
85
  :error
88
86
  end
89
87
  end
@@ -8,6 +8,7 @@ module HybridPlatformsConductor
8
8
 
9
9
  module HpcPlugins
10
10
 
11
+ # Patch proxmox lib
11
12
  module Provisioner
12
13
 
13
14
  # Monkey patch some Proxmox methods
@@ -68,7 +69,9 @@ module HybridPlatformsConductor
68
69
  extend_config_dsl_with PlatformsDSLProxmox, :init_proxmox
69
70
 
70
71
  class << self
72
+
71
73
  attr_accessor :proxmox_waiter_files_mutex
74
+
72
75
  end
73
76
  @proxmox_waiter_files_mutex = Mutex.new
74
77
 
@@ -83,105 +86,100 @@ module HybridPlatformsConductor
83
86
  @lxc_details = nil
84
87
  with_proxmox do |proxmox|
85
88
  proxmox_get(proxmox, 'nodes').each do |node_info|
86
- if proxmox_test_info[:test_config][:pve_nodes].include?(node_info['node']) && node_info['status'] == 'online'
87
- proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc").each do |lxc_info|
88
- vm_id = Integer(lxc_info['vmid'])
89
- if vm_id.between?(*proxmox_test_info[:test_config][:vm_ids_range])
90
- # Check if the description contains our ID
91
- lxc_config = proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc/#{vm_id}/config")
92
- vm_description_lines = (lxc_config['description'] || '').split("\n")
93
- hpc_marker_idx = vm_description_lines.index('===== HPC info =====')
94
- unless hpc_marker_idx.nil?
95
- # Get the HPC info associated to this VM
96
- # Hash<Symbol,String>
97
- vm_hpc_info = Hash[vm_description_lines[hpc_marker_idx + 1..-1].map do |line|
98
- property, value = line.split(': ')
99
- [property.to_sym, value]
100
- end]
101
- if vm_hpc_info[:node] == @node && vm_hpc_info[:environment] == @environment
102
- # Found it
103
- # Get back the IP
104
- ip_found = nil
105
- lxc_config['net0'].split(',').each do |net_info|
106
- property, value = net_info.split('=')
107
- if property == 'ip'
108
- ip_found = value.split('/').first
109
- break
110
- end
111
- end
112
- raise "[ #{@node}/#{@environment} ] - Unable to get IP back from LXC container nodes/#{node_info['node']}/lxc/#{vm_id}/config" if ip_found.nil?
113
- @lxc_details = {
114
- pve_node: node_info['node'],
115
- vm_id: vm_id,
116
- vm_ip: ip_found
117
- }
118
- break
119
- end
120
- end
89
+ next unless proxmox_test_info[:test_config][:pve_nodes].include?(node_info['node']) && node_info['status'] == 'online'
90
+
91
+ proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc").each do |lxc_info|
92
+ vm_id = Integer(lxc_info['vmid'])
93
+ next unless vm_id.between?(*proxmox_test_info[:test_config][:vm_ids_range])
94
+
95
+ # Check if the description contains our ID
96
+ lxc_config = proxmox_get(proxmox, "nodes/#{node_info['node']}/lxc/#{vm_id}/config")
97
+ vm_description_lines = (lxc_config['description'] || '').split("\n")
98
+ hpc_marker_idx = vm_description_lines.index('===== HPC info =====')
99
+ next if hpc_marker_idx.nil?
100
+
101
+ # Get the HPC info associated to this VM
102
+ # Hash<Symbol,String>
103
+ vm_hpc_info = vm_description_lines[hpc_marker_idx + 1..].map do |line|
104
+ property, value = line.split(': ')
105
+ [property.to_sym, value]
106
+ end.to_h
107
+ next unless vm_hpc_info[:node] == @node && vm_hpc_info[:environment] == @environment
108
+
109
+ # Found it
110
+ # Get back the IP
111
+ ip_found = nil
112
+ lxc_config['net0'].split(',').each do |net_info|
113
+ property, value = net_info.split('=')
114
+ if property == 'ip'
115
+ ip_found = value.split('/').first
116
+ break
121
117
  end
122
118
  end
123
- break if @lxc_details
119
+ raise "[ #{@node}/#{@environment} ] - Unable to get IP back from LXC container nodes/#{node_info['node']}/lxc/#{vm_id}/config" if ip_found.nil?
120
+
121
+ @lxc_details = {
122
+ pve_node: node_info['node'],
123
+ vm_id: vm_id,
124
+ vm_ip: ip_found
125
+ }
126
+ break
124
127
  end
128
+ break if @lxc_details
125
129
  end
126
130
  end
127
- unless @lxc_details
128
- # We couldn't find an existing LXC container for this node/environment.
129
- # We have to create one.
130
- # Get the image name for this node
131
- image = @nodes_handler.get_image_of(@node).to_sym
132
- # Find if we have such an image registered
133
- if @config.known_os_images.include?(image)
134
- proxmox_conf = "#{@config.os_image_dir(image)}/proxmox.json"
135
- if File.exist?(proxmox_conf)
136
- pve_template = JSON.parse(File.read(proxmox_conf)).dig 'template'
137
- if pve_template
138
- # Query the inventory to know about minimum resources needed to deploy the node.
139
- # Provide default values if they are not part of the metadata.
140
- min_resources_to_deploy = {
141
- cpus: 2,
142
- ram_mb: 1024,
143
- disk_gb: 10
144
- }.merge(@nodes_handler.get_deploy_resources_min_of(@node) || {})
145
- # Create the Proxmox container from the sync node.
146
- vm_config = proxmox_test_info[:vm_config]
147
- # Hostname in Proxmox is capped at 65 chars.
148
- # Make sure we don't get over it, but still use a unique one.
149
- hostname = "#{@node}.#{@environment}.hpc-test.com"
150
- if hostname.size > MAX_PROXMOX_HOSTNAME_SIZE
151
- # Truncate it, but add a unique ID in it.
152
- # In the end the hostname looks like:
153
- # <truncated_node_environment>.<unique_id>.hpc-test.com
154
- hostname = "-#{Digest::MD5.hexdigest(hostname)[0..7]}.hpc-test.com"
155
- hostname = "#{@node}.#{@environment}"[0..MAX_PROXMOX_HOSTNAME_SIZE - hostname.size - 1] + hostname
156
- end
157
- @lxc_details = request_lxc_creation_for({
158
- ostemplate: pve_template,
159
- hostname: hostname.gsub('_', '-'),
160
- cores: min_resources_to_deploy[:cpus],
161
- cpulimit: min_resources_to_deploy[:cpus],
162
- memory: min_resources_to_deploy[:ram_mb],
163
- rootfs: "local-lvm:#{min_resources_to_deploy[:disk_gb]}",
164
- nameserver: vm_config[:vm_dns_servers].join(' '),
165
- searchdomain: vm_config[:vm_search_domain],
166
- net0: "name=eth0,bridge=vmbr0,gw=#{vm_config[:vm_gateway]}",
167
- password: 'root_pwd',
168
- description: <<~EOS
169
- ===== HPC info =====
170
- node: #{@node}
171
- environment: #{@environment}
172
- debug: #{log_debug? ? 'true' : 'false'}
173
- EOS
174
- })
175
- else
176
- raise "[ #{@node}/#{@environment} ] - No template found in #{proxmox_conf}"
177
- end
178
- else
179
- raise "[ #{@node}/#{@environment} ] - No Proxmox configuration found at #{proxmox_conf}"
180
- end
181
- else
182
- raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}"
183
- end
131
+ return if @lxc_details
132
+
133
+ # We couldn't find an existing LXC container for this node/environment.
134
+ # We have to create one.
135
+ # Get the image name for this node
136
+ image = @nodes_handler.get_image_of(@node).to_sym
137
+ # Find if we have such an image registered
138
+ raise "[ #{@node}/#{@environment} ] - Unknown OS image #{image} defined for node #{@node}" unless @config.known_os_images.include?(image)
139
+
140
+ proxmox_conf = "#{@config.os_image_dir(image)}/proxmox.json"
141
+ raise "[ #{@node}/#{@environment} ] - No Proxmox configuration found at #{proxmox_conf}" unless File.exist?(proxmox_conf)
142
+
143
+ pve_template = JSON.parse(File.read(proxmox_conf))['template']
144
+ raise "[ #{@node}/#{@environment} ] - No template found in #{proxmox_conf}" unless pve_template
145
+
146
+ # Query the inventory to know about minimum resources needed to deploy the node.
147
+ # Provide default values if they are not part of the metadata.
148
+ min_resources_to_deploy = {
149
+ cpus: 2,
150
+ ram_mb: 1024,
151
+ disk_gb: 10
152
+ }.merge(@nodes_handler.get_deploy_resources_min_of(@node) || {})
153
+ # Create the Proxmox container from the sync node.
154
+ vm_config = proxmox_test_info[:vm_config]
155
+ # Hostname in Proxmox is capped at 65 chars.
156
+ # Make sure we don't get over it, but still use a unique one.
157
+ hostname = "#{@node}.#{@environment}.hpc-test.com"
158
+ if hostname.size > MAX_PROXMOX_HOSTNAME_SIZE
159
+ # Truncate it, but add a unique ID in it.
160
+ # In the end the hostname looks like:
161
+ # <truncated_node_environment>.<unique_id>.hpc-test.com
162
+ hostname = "-#{Digest::MD5.hexdigest(hostname)[0..7]}.hpc-test.com"
163
+ hostname = "#{@node}.#{@environment}"[0..MAX_PROXMOX_HOSTNAME_SIZE - hostname.size - 1] + hostname
184
164
  end
165
+ @lxc_details = request_lxc_creation_for(
166
+ ostemplate: pve_template,
167
+ hostname: hostname.gsub('_', '-'),
168
+ cores: min_resources_to_deploy[:cpus],
169
+ cpulimit: min_resources_to_deploy[:cpus],
170
+ memory: min_resources_to_deploy[:ram_mb],
171
+ rootfs: "local-lvm:#{min_resources_to_deploy[:disk_gb]}",
172
+ nameserver: vm_config[:vm_dns_servers].join(' '),
173
+ searchdomain: vm_config[:vm_search_domain],
174
+ net0: "name=eth0,bridge=vmbr0,gw=#{vm_config[:vm_gateway]}",
175
+ password: 'root_pwd',
176
+ description: <<~EO_DESCRIPTION
177
+ ===== HPC info =====
178
+ node: #{@node}
179
+ environment: #{@environment}
180
+ debug: #{log_debug? ? 'true' : 'false'}
181
+ EO_DESCRIPTION
182
+ )
185
183
  end
186
184
 
187
185
  # Start an instance
@@ -274,6 +272,7 @@ module HybridPlatformsConductor
274
272
  def with_proxmox
275
273
  url = proxmox_test_info[:api_url]
276
274
  raise 'No Proxmox server defined' if url.nil?
275
+
277
276
  Credentials.with_credentials_for(:proxmox, @logger, @logger_stderr, url: url) do |user, password|
278
277
  log_debug "[ #{@node}/#{@environment} ] - Connect to Proxmox #{url}"
279
278
  proxmox_logs = StringIO.new
@@ -313,9 +312,11 @@ module HybridPlatformsConductor
313
312
  idx_try = 0
314
313
  loop do
315
314
  response = proxmox.get(path)
316
- break if !(response.is_a?(String)) || !(response =~ /^NOK: error code = 5\d\d$/)
315
+ break if !response.is_a?(String) || response !~ /^NOK: error code = 5\d\d$/
316
+
317
317
  log_warn "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returned error #{response} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
318
318
  raise "[ #{@node}/#{@environment} ] - Proxmox API call get #{path} returns #{response} continuously (tried #{idx_try + 1} times)" if idx_try >= proxmox_test_info[:api_max_retries]
319
+
319
320
  idx_try += 1
320
321
  # We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
321
322
  proxmox.reauthenticate
@@ -336,23 +337,22 @@ module HybridPlatformsConductor
336
337
  def run_proxmox_task(proxmox, http_method, pve_node, sub_path, *args)
337
338
  task = nil
338
339
  idx_try = 0
339
- while task.nil? do
340
+ while task.nil?
340
341
  task = proxmox.send(http_method, "nodes/#{pve_node}/#{sub_path}", *args)
341
- if task =~ /^NOK: error code = 5\d\d$/
342
- log_warn "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} returned error #{task} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
343
- task = nil
344
- break if idx_try >= proxmox_test_info[:api_max_retries]
345
- idx_try += 1
346
- # We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
347
- proxmox.reauthenticate
348
- sleep proxmox_test_info[:api_wait_between_retries_secs] + rand(5)
349
- end
350
- end
351
- if task.nil?
352
- raise "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} is constantly failing. Giving up."
353
- else
354
- wait_for_proxmox_task(proxmox, pve_node, task)
342
+ next unless task =~ /^NOK: error code = 5\d\d$/
343
+
344
+ log_warn "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} returned error #{task} (attempt ##{idx_try}/#{proxmox_test_info[:api_max_retries]})"
345
+ task = nil
346
+ break if idx_try >= proxmox_test_info[:api_max_retries]
347
+
348
+ idx_try += 1
349
+ # We have to reauthenticate: error 500 raised by Proxmox are often due to token being invalidated wrongly
350
+ proxmox.reauthenticate
351
+ sleep proxmox_test_info[:api_wait_between_retries_secs] + rand(5)
355
352
  end
353
+ raise "[ #{@node}/#{@environment} ] - Proxmox API call #{http_method} nodes/#{pve_node}/#{sub_path} #{args} is constantly failing. Giving up." if task.nil?
354
+
355
+ wait_for_proxmox_task(proxmox, pve_node, task)
356
356
  end
357
357
 
358
358
  # Wait for a given Proxmox task completion
@@ -363,18 +363,18 @@ module HybridPlatformsConductor
363
363
  # * *task* (String): The task ID
364
364
  def wait_for_proxmox_task(proxmox, pve_node, task)
365
365
  raise "Invalid task: #{task}" if task[0..3] == 'NOK:'
366
+
366
367
  status = nil
367
368
  loop do
368
369
  status = task_status(proxmox, pve_node, task)
369
370
  break unless status == 'running'
371
+
370
372
  log_debug "[ #{@node}/#{@environment} ] - Wait for Proxmox task #{task} to complete..."
371
373
  sleep 1
372
374
  end
373
- if status.split(':').last == 'OK'
374
- log_debug "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed."
375
- else
376
- raise "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed with status #{status}"
377
- end
375
+ raise "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed with status #{status}" unless status.split(':').last == 'OK'
376
+
377
+ log_debug "[ #{@node}/#{@environment} ] - Proxmox task #{task} completed."
378
378
  end
379
379
 
380
380
  # Get task status
@@ -402,13 +402,13 @@ module HybridPlatformsConductor
402
402
  config_file = "#{Dir.tmpdir}/config_#{file_id}.json"
403
403
  File.write(
404
404
  config_file,
405
- (proxmox_test_info[:test_config].merge(
405
+ proxmox_test_info[:test_config].merge(
406
406
  proxmox_api_url: proxmox_test_info[:api_url],
407
407
  futex_file: '/tmp/hpc_proxmox_allocations.futex',
408
408
  logs_dir: '/tmp/hpc_proxmox_waiter_logs',
409
409
  api_max_retries: proxmox_test_info[:api_max_retries],
410
410
  api_wait_between_retries_secs: proxmox_test_info[:api_wait_between_retries_secs]
411
- )).to_json
411
+ ).to_json
412
412
  )
413
413
  result = nil
414
414
  begin
@@ -447,7 +447,7 @@ module HybridPlatformsConductor
447
447
  )[proxmox_test_info[:sync_node]]
448
448
  end
449
449
  stdout_lines = stdout.split("\n")
450
- result = JSON.parse(stdout_lines[stdout_lines.index('===== JSON =====') + 1..-1].join("\n")).transform_keys(&:to_sym)
450
+ result = JSON.parse(stdout_lines[stdout_lines.index('===== JSON =====') + 1..].join("\n")).transform_keys(&:to_sym)
451
451
  raise "[ #{@node}/#{@environment} ] - Error returned by #{cmd}: #{result[:error]}" if result.key?(:error)
452
452
  ensure
453
453
  File.unlink(config_file)
@@ -548,6 +548,7 @@ module HybridPlatformsConductor
548
548
  def proxmox_test_info
549
549
  @config.proxmox_servers.first
550
550
  end
551
+
551
552
  end
552
553
 
553
554
  end