hybrid_platforms_conductor 32.13.4 → 32.14.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +12 -0
  3. data/bin/get_impacted_nodes +1 -1
  4. data/bin/setup +6 -1
  5. data/docs/plugins.md +1 -0
  6. data/docs/plugins/platform_handler/serverless_chef.md +105 -0
  7. data/lib/hybrid_platforms_conductor/deployer.rb +2 -1
  8. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +440 -0
  9. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/dsl_parser.rb +51 -0
  10. data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef/recipes_tree_builder.rb +271 -0
  11. data/lib/hybrid_platforms_conductor/nodes_handler.rb +9 -5
  12. data/lib/hybrid_platforms_conductor/version.rb +1 -1
  13. data/spec/hybrid_platforms_conductor_test.rb +3 -0
  14. data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb +23 -0
  15. data/spec/hybrid_platforms_conductor_test/api/nodes_handler/cmdbs_plugins_api_spec.rb +11 -0
  16. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/config_dsl_spec.rb +17 -0
  17. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/deploy_output_parsing_spec.rb +94 -0
  18. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/diff_impacts_spec.rb +317 -0
  19. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/inventory_spec.rb +65 -0
  20. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +213 -0
  21. data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +268 -0
  22. data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb +53 -0
  23. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/chef_versions.yml +3 -0
  24. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json +14 -0
  25. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/policyfiles/test_policy.rb +3 -0
  26. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/chef_versions.yml +3 -0
  27. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/data_bags/my_bag/my_item.json +4 -0
  28. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json +14 -0
  29. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/policyfiles/test_policy.rb +3 -0
  30. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_1/recipes/default.rb +1 -0
  31. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb +4 -0
  32. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/recipes/default.rb +1 -0
  33. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/recipes/other_recipe.rb +1 -0
  34. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/resources/my_resource.rb +1 -0
  35. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/nodes/node1.json +10 -0
  36. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/nodes/node2.json +10 -0
  37. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/policyfiles/test_policy_1.rb +4 -0
  38. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/policyfiles/test_policy_2.rb +4 -0
  39. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/config.rb +1 -0
  40. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/cookbooks/test_cookbook_1/recipes/default.rb +1 -0
  41. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/nodes/node1.json +10 -0
  42. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/nodes/node2.json +10 -0
  43. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb +4 -0
  44. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/recipes/default.rb +1 -0
  45. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/recipes/other_recipe.rb +1 -0
  46. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/resources/my_resource.rb +1 -0
  47. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/policyfiles/test_policy_1.rb +4 -0
  48. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/policyfiles/test_policy_2.rb +4 -0
  49. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_nodes/chef_versions.yml +3 -0
  50. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_nodes/nodes/local.json +10 -0
  51. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_nodes/nodes/node1.json +10 -0
  52. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_nodes/nodes/node2.json +10 -0
  53. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_nodes/policyfiles/test_policy_1.rb +3 -0
  54. data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_nodes/policyfiles/test_policy_2.rb +3 -0
  55. metadata +187 -143
@@ -0,0 +1,268 @@
1
+ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef do
2
+
3
+ context 'checking services deployment' do
4
+
5
+ # Simulate a packaging of a given repository
6
+ #
7
+ # Parameters::
8
+ # * *repository* (String): The repository we package
9
+ # * *service* (String): The service being packaged in this repository [default: 'test_policy']
10
+ def mock_package(repository, service: 'test_policy')
11
+ FileUtils.mkdir_p "#{repository}/dist/prod/#{service}"
12
+ end
13
+
14
+ # Get expected actions to deploy a service on a given node
15
+ #
16
+ # Parameters::
17
+ # * *repository* (String): Platform repository
18
+ # * *check_mode* (Boolean): Are we expected check-mode? [default: false]
19
+ # * *sudo* (String): sudo prefix command [default: 'sudo -u root ']
20
+ # * *env* (String): Environment expected to be packaged [default: 'prod']
21
+ # * *policy* (String): Expected policy to be packaged [default: 'test_policy']
22
+ # * *node* (String): Expected node to be deployed [default: 'node']
23
+ # Result::
24
+ # * Array: Expected actions
25
+ def expected_actions_to_deploy_chef(
26
+ repository,
27
+ check_mode: false,
28
+ sudo: 'sudo -u root ',
29
+ env: 'prod',
30
+ policy: 'test_policy',
31
+ node: 'node'
32
+ )
33
+ [
34
+ {
35
+ remote_bash: [
36
+ 'set -e',
37
+ 'set -o pipefail',
38
+ "if [ -n \"$(command -v apt)\" ]; then #{sudo}apt update && #{sudo}apt install -y curl build-essential ; else #{sudo}yum groupinstall 'Development Tools' && #{sudo}yum install -y curl ; fi",
39
+ 'mkdir -p ./hpc_deploy',
40
+ "curl --location https://omnitruck.chef.io/install.sh | tac | tac | #{sudo}bash -s -- -d /opt/artefacts -v 17.0 -s once"
41
+ ]
42
+ },
43
+ {
44
+ scp: { "#{repository}/dist/#{env}/#{policy}" => './hpc_deploy' },
45
+ remote_bash: [
46
+ 'set -e',
47
+ "cd ./hpc_deploy/#{policy}",
48
+ "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client --local-mode --chef-license=accept --json-attributes nodes/#{node}.json#{check_mode ? ' --why-run' : ''}",
49
+ 'cd ..',
50
+ "#{sudo}rm -rf #{policy}"
51
+ ]
52
+ }
53
+ ]
54
+ end
55
+
56
+ context 'with an empty platform' do
57
+
58
+ it 'prepares for deploy' do
59
+ with_serverless_chef_platforms('empty') do |platform, repository|
60
+ platform.prepare_for_deploy(
61
+ services: {},
62
+ secrets: {},
63
+ local_environment: false,
64
+ why_run: false
65
+ )
66
+ end
67
+ end
68
+
69
+ it 'prepares for deploy in why-run mode' do
70
+ with_serverless_chef_platforms('empty') do |platform, repository|
71
+ platform.prepare_for_deploy(
72
+ services: {},
73
+ secrets: {},
74
+ local_environment: false,
75
+ why_run: true
76
+ )
77
+ end
78
+ end
79
+
80
+ it 'prepares for deploy in local mode' do
81
+ with_serverless_chef_platforms('empty') do |platform, repository|
82
+ platform.prepare_for_deploy(
83
+ services: {},
84
+ secrets: {},
85
+ local_environment: true,
86
+ why_run: false
87
+ )
88
+ end
89
+ end
90
+
91
+ end
92
+
93
+ context 'with a platform having 1 node' do
94
+
95
+ it 'returns actions to deploy on this node' do
96
+ with_serverless_chef_platforms('1_node') do |platform, repository|
97
+ mock_package(repository)
98
+ platform.prepare_for_deploy(
99
+ services: { 'node' => %w[test_policy] },
100
+ secrets: {},
101
+ local_environment: false,
102
+ why_run: false
103
+ )
104
+ expect(platform.actions_to_deploy_on('node', 'test_policy', use_why_run: false)).to eq expected_actions_to_deploy_chef(repository)
105
+ end
106
+ end
107
+
108
+ it 'returns actions to deploy on this node with node attributes setup from metadata' do
109
+ with_serverless_chef_platforms('1_node') do |platform, repository|
110
+ test_nodes_handler.override_metadata_of 'node', :new_metadata, 'new_value'
111
+ mock_package(repository)
112
+ platform.prepare_for_deploy(
113
+ services: { 'node' => %w[test_policy] },
114
+ secrets: {},
115
+ local_environment: false,
116
+ why_run: false
117
+ )
118
+ expect(platform.actions_to_deploy_on('node', 'test_policy', use_why_run: false)).to eq expected_actions_to_deploy_chef(repository)
119
+ attributes_file = "#{repository}/dist/prod/test_policy/nodes/node.json"
120
+ expect(File.exist?(attributes_file)).to eq true
121
+ expect(JSON.parse(File.read(attributes_file))).to eq(
122
+ 'description' => 'Single test node',
123
+ 'image' => 'debian_9',
124
+ 'new_metadata' => 'new_value',
125
+ 'private_ips' => ['172.16.0.1'],
126
+ 'property1' => { 'property11' => 'value11' },
127
+ 'property2' => 'value2',
128
+ )
129
+ end
130
+ end
131
+
132
+ it 'returns actions to deploy on this node with secrets' do
133
+ with_serverless_chef_platforms('1_node') do |platform, repository|
134
+ mock_package(repository)
135
+ platform.prepare_for_deploy(
136
+ services: { 'node' => %w[test_policy] },
137
+ secrets: { 'my_secret' => 'secret_value' },
138
+ local_environment: false,
139
+ why_run: false
140
+ )
141
+ expect(platform.actions_to_deploy_on('node', 'test_policy', use_why_run: false)).to eq expected_actions_to_deploy_chef(repository)
142
+ end
143
+ end
144
+
145
+ it 'returns actions to deploy on this node in why-run mode' do
146
+ with_serverless_chef_platforms('1_node') do |platform, repository|
147
+ mock_package(repository)
148
+ platform.prepare_for_deploy(
149
+ services: { 'node' => %w[test_policy] },
150
+ secrets: {},
151
+ local_environment: false,
152
+ why_run: true
153
+ )
154
+ expect(platform.actions_to_deploy_on('node', 'test_policy', use_why_run: true)).to eq expected_actions_to_deploy_chef(repository, check_mode: true)
155
+ end
156
+ end
157
+
158
+ it 'returns actions to deploy on this node using local mode' do
159
+ with_serverless_chef_platforms('1_node') do |platform, repository|
160
+ mock_package(repository)
161
+ platform.prepare_for_deploy(
162
+ services: { 'node' => %w[test_policy] },
163
+ secrets: {},
164
+ local_environment: true,
165
+ why_run: false
166
+ )
167
+ expect(platform.actions_to_deploy_on('node', 'test_policy', use_why_run: false)).to eq expected_actions_to_deploy_chef(repository, env: 'local')
168
+ end
169
+ end
170
+
171
+ it 'returns actions to deploy on this node in why-run mode and local mode' do
172
+ with_serverless_chef_platforms('1_node') do |platform, repository|
173
+ mock_package(repository)
174
+ platform.prepare_for_deploy(
175
+ services: { 'node' => %w[test_policy] },
176
+ secrets: {},
177
+ local_environment: true,
178
+ why_run: true
179
+ )
180
+ expect(platform.actions_to_deploy_on('node', 'test_policy', use_why_run: true)).to eq expected_actions_to_deploy_chef(repository, env: 'local', check_mode: true)
181
+ end
182
+ end
183
+
184
+ it 'returns actions to deploy on this node using root user' do
185
+ with_serverless_chef_platforms('1_node') do |platform, repository|
186
+ test_actions_executor.connector(:ssh).ssh_user = 'root'
187
+ mock_package(repository)
188
+ platform.prepare_for_deploy(
189
+ services: { 'node' => %w[test_policy] },
190
+ secrets: {},
191
+ local_environment: false,
192
+ why_run: false
193
+ )
194
+ expect(platform.actions_to_deploy_on('node', 'test_policy', use_why_run: false)).to eq expected_actions_to_deploy_chef(repository, sudo: '')
195
+ end
196
+ end
197
+
198
+ it 'fails with a nice message when chef_versions.yml is missing' do
199
+ with_serverless_chef_platforms('1_node') do |platform, repository|
200
+ mock_package(repository)
201
+ platform.prepare_for_deploy(
202
+ services: { 'node' => %w[test_policy] },
203
+ secrets: {},
204
+ local_environment: false,
205
+ why_run: false
206
+ )
207
+ File.unlink("#{repository}/chef_versions.yml")
208
+ expect { platform.actions_to_deploy_on('node', 'test_policy', use_why_run: false) }.to raise_error "Missing file #{repository}/chef_versions.yml specifying the Chef Infra Client version to be deployed"
209
+ end
210
+ end
211
+
212
+ end
213
+
214
+ context 'with a platform having several nodes' do
215
+
216
+ it 'deploys services declared on 1 node on another node if asked' do
217
+ with_serverless_chef_platforms('several_nodes') do |platform, repository|
218
+ mock_package(repository)
219
+ platform.prepare_for_deploy(
220
+ services: { 'node2' => %w[test_policy_1] },
221
+ secrets: {},
222
+ local_environment: false,
223
+ why_run: false
224
+ )
225
+ expect(platform.actions_to_deploy_on('node2', 'test_policy_1', use_why_run: false)).to eq expected_actions_to_deploy_chef(repository, policy: 'test_policy_1', node: 'node2')
226
+ end
227
+ end
228
+
229
+ it 'deploys local nodes' do
230
+ with_serverless_chef_platforms('several_nodes') do |platform, repository|
231
+ mock_package(repository)
232
+ platform.prepare_for_deploy(
233
+ services: { 'local' => %w[test_policy_1] },
234
+ secrets: {},
235
+ local_environment: false,
236
+ why_run: false
237
+ )
238
+ expect(platform.actions_to_deploy_on('local', 'test_policy_1', use_why_run: false)).to eq [
239
+ {
240
+ bash: "cd #{repository}/dist/prod/test_policy_1 && sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client --local-mode --json-attributes nodes/local.json"
241
+ }
242
+ ]
243
+ end
244
+ end
245
+
246
+ end
247
+
248
+ context 'with 2 platforms' do
249
+
250
+ it 'deploys a service on a node belonging to another platform' do
251
+ with_serverless_chef_platforms({ 'p1' => '1_node', 'p2' => 'several_nodes' }) do |repositories|
252
+ platform_p1, repository_p1 = repositories.find { |platform, _repository| platform.name == 'p1' }
253
+ mock_package(repository_p1)
254
+ platform_p1.prepare_for_deploy(
255
+ services: { 'node2' => %w[test_policy_1] },
256
+ secrets: {},
257
+ local_environment: false,
258
+ why_run: false
259
+ )
260
+ expect(platform_p1.actions_to_deploy_on('node2', 'test_policy_1', use_why_run: false)).to eq expected_actions_to_deploy_chef(repository_p1, policy: 'test_policy_1', node: 'node2')
261
+ end
262
+ end
263
+
264
+ end
265
+
266
+ end
267
+
268
+ end
@@ -0,0 +1,53 @@
1
+ require 'hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef'
2
+
3
+ module HybridPlatformsConductorTest
4
+
5
+ module Helpers
6
+
7
+ module ServerlessChefHelpers
8
+
9
+ # Setup a platforms config using test repository names
10
+ #
11
+ # Parameters::
12
+ # * *names* (String or Hash<String, String>): The test repository name (taken from the repositories/ folder), or a Hash of names and their corresponding test repository source name
13
+ # * *additional_config* (String): Additional config to append after the platform declaration [default: '']
14
+ # * *as_git* (Boolean): Should we initialize the repository as a git repo? [default: false]
15
+ # * Proc: Code called when repository is setup
16
+ # * Parameters::
17
+ # If there was only 1 repository:
18
+ # * *platform* (PlatformHandler): The platform handler to be tested
19
+ # * *repository* (String): Repository path
20
+ # If there was multiple repositories:
21
+ # * *repositories* (Hash<PlatformHandler,String>): Set of repositories, per platform handler
22
+ def with_serverless_chef_platforms(names, additional_config: '', as_git: false)
23
+ names = { names => names } unless names.is_a?(Hash)
24
+ with_repositories(names.keys, as_git: as_git) do |repositories|
25
+ repositories.each do |name, repository|
26
+ # Copy the content of the test repository in the temporary one
27
+ FileUtils.cp_r "#{__dir__}/../serverless_chef_repositories/#{names[name]}/.", repository
28
+ end
29
+ with_platforms(repositories.values.map { |repository| "serverless_chef_platform path: '#{repository}'\n" }.join + additional_config) do
30
+ repositories = Hash[names.keys.map do |name|
31
+ [
32
+ test_platforms_handler.platform(name),
33
+ repositories[name]
34
+ ]
35
+ end]
36
+ test_platforms_handler.inject_dependencies(
37
+ nodes_handler: test_nodes_handler,
38
+ actions_executor: test_actions_executor
39
+ )
40
+ if repositories.size == 1
41
+ yield *repositories.first
42
+ else
43
+ yield repositories
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ end
50
+
51
+ end
52
+
53
+ end
@@ -0,0 +1,3 @@
1
+ ---
2
+ workstation: '21.5'
3
+ client: '17.0'
@@ -0,0 +1,14 @@
1
+ {
2
+ "name": "node",
3
+ "normal": {
4
+ "description": "Single test node",
5
+ "image": "debian_9",
6
+ "private_ips": ["172.16.0.1"],
7
+ "property1": {
8
+ "property11": "value11"
9
+ },
10
+ "property2": "value2"
11
+ },
12
+ "policy_name": "test_policy",
13
+ "policy_group": "test_group"
14
+ }
@@ -0,0 +1,3 @@
1
+ name File.basename(__FILE__, '.rb')
2
+ default_source :supermarket
3
+ run_list 'recipe[test_cookbook]'
@@ -0,0 +1,3 @@
1
+ ---
2
+ workstation: '21.5'
3
+ client: '17.0'
@@ -0,0 +1,14 @@
1
+ {
2
+ "name": "node",
3
+ "normal": {
4
+ "description": "Single test node",
5
+ "image": "debian_9",
6
+ "private_ips": ["172.16.0.1"],
7
+ "property1": {
8
+ "property11": "value11"
9
+ },
10
+ "property2": "value2"
11
+ },
12
+ "policy_name": "test_policy",
13
+ "policy_group": "test_group"
14
+ }
@@ -0,0 +1,3 @@
1
+ name File.basename(__FILE__, '.rb')
2
+ default_source :supermarket
3
+ run_list 'recipe[test_cookbook]'
@@ -0,0 +1,10 @@
1
+ {
2
+ "name": "node1",
3
+ "normal": {
4
+ "description": "Node 1",
5
+ "image": "debian_9",
6
+ "private_ips": ["172.16.0.1"]
7
+ },
8
+ "policy_name": "test_policy_1",
9
+ "policy_group": "test_group"
10
+ }
@@ -0,0 +1,10 @@
1
+ {
2
+ "name": "node2",
3
+ "normal": {
4
+ "description": "Node 2",
5
+ "image": "debian_9",
6
+ "private_ips": ["172.16.0.2"]
7
+ },
8
+ "policy_name": "test_policy_2",
9
+ "policy_group": "test_group"
10
+ }
@@ -0,0 +1,4 @@
1
+ name File.basename(__FILE__, '.rb')
2
+ default_source :supermarket
3
+ default_source :chef_repo, '..'
4
+ run_list 'recipe[test_cookbook_1]'
@@ -0,0 +1,4 @@
1
+ name File.basename(__FILE__, '.rb')
2
+ default_source :supermarket
3
+ default_source :chef_repo, '..'
4
+ run_list 'recipe[test_cookbook_2]'
@@ -0,0 +1 @@
1
+ cookbook_path %w[cookbooks] + (ENV['hpc_test_cookbooks_path'] ? ENV['hpc_test_cookbooks_path'].split(':') : [])
@@ -0,0 +1,10 @@
1
+ {
2
+ "name": "node1",
3
+ "normal": {
4
+ "description": "Node 1",
5
+ "image": "debian_9",
6
+ "private_ips": ["172.16.0.1"]
7
+ },
8
+ "policy_name": "test_policy_1",
9
+ "policy_group": "test_group"
10
+ }