hybrid_platforms_conductor 33.1.1 → 33.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb +30 -5
- data/lib/hybrid_platforms_conductor/version.rb +1 -1
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb +48 -2
- data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/services_deployment_spec.rb +92 -15
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9f284e0fbe4ad4a3f1731a4c3b1bc61aa0bbe315d37241845bebdfe384e5f366
|
4
|
+
data.tar.gz: 1f731fef465da4333a435ebfbbc1a4b4f592d85110e02e4ed4f38984275502b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d061127dcc4caeb2e5f5984c4e91b50e46fc9687f4ae4353fb37812232dd69d1289196303f4586c0ad5e6311559679c4937844951ca6daf7a3f902e9b9fe519
|
7
|
+
data.tar.gz: 5561ebc29c31d61fb0c009faf3b0dc661ce72c7078f55dad2e6e8a5630027137121ea69c536928646f3ac7271a1cf6a2b492d6a3ed0979ee1f1d35e85aed9ac3
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
# [v33.2.0](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.1.1...v33.2.0) (2021-06-18 23:22:21)
|
2
|
+
|
3
|
+
## Global changes
|
4
|
+
### Patches
|
5
|
+
|
6
|
+
* [[Feature(platform_handler_serverless_chef)] [#70] Install dependency gems from cookbook metadata before calling chef-client](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/6dfe7aa053db63489f3d0a236433304606051ecd)
|
7
|
+
|
8
|
+
## Changes for platform_handler_serverless_chef
|
9
|
+
### Features
|
10
|
+
|
11
|
+
* [[Feature(platform_handler_serverless_chef)] [#70] Install dependency gems from cookbook metadata before calling chef-client](https://github.com/sweet-delights/hybrid-platforms-conductor/commit/6dfe7aa053db63489f3d0a236433304606051ecd)
|
12
|
+
|
1
13
|
# [v33.1.1](https://github.com/sweet-delights/hybrid-platforms-conductor/compare/v33.1.0...v33.1.1) (2021-06-18 13:19:39)
|
2
14
|
|
3
15
|
### Patches
|
@@ -179,6 +179,13 @@ module HybridPlatformsConductor
|
|
179
179
|
/opt/chef-workstation/bin/chef export #{policy_file} #{package_dir} --chef-license accept#{extra_cp_data_bags}"
|
180
180
|
next if @cmd_runner.dry_run
|
181
181
|
|
182
|
+
# Write the list of gems to be installed for this package
|
183
|
+
File.write(
|
184
|
+
"#{@repository_path}/#{package_dir}/gems.json",
|
185
|
+
Dir.glob("#{@repository_path}/#{package_dir}/cookbook_artifacts/*/metadata.json").map do |metadata|
|
186
|
+
JSON.parse(File.read(metadata))['gems']
|
187
|
+
end.flatten(1).to_json
|
188
|
+
)
|
182
189
|
# Create secrets file
|
183
190
|
secrets_file = "#{@repository_path}/#{package_dir}/data_bags/hpc_secrets/hpc_secrets.json"
|
184
191
|
FileUtils.mkdir_p(File.dirname(secrets_file))
|
@@ -225,6 +232,8 @@ module HybridPlatformsConductor
|
|
225
232
|
FileUtils.mkdir_p "#{package_dir}/nodes"
|
226
233
|
File.write("#{package_dir}/nodes/#{node}.json", (known_nodes.include?(node) ? metadata_for(node) : {}).merge(@nodes_handler.metadata_of(node)).to_json)
|
227
234
|
end
|
235
|
+
# Get the gems to be installed
|
236
|
+
gems_to_install = JSON.parse(File.read("#{package_dir}/gems.json"))
|
228
237
|
client_options = [
|
229
238
|
'--local-mode',
|
230
239
|
'--chef-license', 'accept',
|
@@ -233,7 +242,19 @@ module HybridPlatformsConductor
|
|
233
242
|
client_options << '--why-run' if use_why_run
|
234
243
|
if @nodes_handler.get_use_local_chef_of(node)
|
235
244
|
# Just run the chef-client directly from the packaged repository
|
236
|
-
|
245
|
+
sudo_prefix = @cmd_runner.root? ? '' : 'sudo '
|
246
|
+
[
|
247
|
+
{
|
248
|
+
bash: [
|
249
|
+
'set -e',
|
250
|
+
"cd #{package_dir}"
|
251
|
+
] +
|
252
|
+
gems_to_install.map { |(gem_name, gem_version)| "#{sudo_prefix}SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef gem install #{gem_name} --version \"#{gem_version}\"" } +
|
253
|
+
[
|
254
|
+
"#{sudo_prefix}SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client #{client_options.join(' ')}"
|
255
|
+
]
|
256
|
+
}
|
257
|
+
]
|
237
258
|
else
|
238
259
|
# Upload the package and run it from the node
|
239
260
|
package_name = File.basename(package_dir)
|
@@ -261,10 +282,14 @@ module HybridPlatformsConductor
|
|
261
282
|
scp: { package_dir => './hpc_deploy' },
|
262
283
|
remote_bash: [
|
263
284
|
'set -e',
|
264
|
-
"cd ./hpc_deploy/#{package_name}"
|
265
|
-
|
266
|
-
|
267
|
-
|
285
|
+
"cd ./hpc_deploy/#{package_name}"
|
286
|
+
] +
|
287
|
+
gems_to_install.map { |(gem_name, gem_version)| "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/embedded/bin/gem install #{gem_name} --version \"#{gem_version}\"" } +
|
288
|
+
[
|
289
|
+
"#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client #{client_options.join(' ')}",
|
290
|
+
'cd ..'
|
291
|
+
] +
|
292
|
+
(log_debug? ? [] : ["#{sudo}rm -rf ./hpc_deploy/#{package_name}"])
|
268
293
|
}
|
269
294
|
]
|
270
295
|
end
|
data/spec/hybrid_platforms_conductor_test/api/platform_handlers/serverless_chef/packaging_spec.rb
CHANGED
@@ -12,6 +12,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
12
12
|
# * *export* (Boolean): Are we expecting the chef export stage? [default: true]
|
13
13
|
# * *data_bags* (Boolean): Do we expect data bags copy? [default: false]
|
14
14
|
# * *env* (String): Expected environment being packaged [default: 'prod']
|
15
|
+
# * *cookbook_metadata* (Hash<String, Hash>): JSON metadata to generate for packaged cookbooks [default: {}]
|
15
16
|
# * *block* (Proc): Code called with mock in place
|
16
17
|
def with_packaging_mocked(
|
17
18
|
repository,
|
@@ -21,6 +22,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
21
22
|
export: true,
|
22
23
|
data_bags: false,
|
23
24
|
env: 'prod',
|
25
|
+
cookbook_metadata: {},
|
24
26
|
&block
|
25
27
|
)
|
26
28
|
with_cmd_runner_mocked(
|
@@ -49,8 +51,14 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
49
51
|
[
|
50
52
|
%r{^cd #{Regexp.escape(repository)} &&\s+sudo rm -rf dist/#{Regexp.escape(env)}/#{Regexp.escape(policy)} &&\s+/opt/chef-workstation/bin/chef export #{Regexp.escape(policy_file)} dist/#{Regexp.escape(env)}/#{Regexp.escape(policy)} --chef-license accept#{data_bags ? " && cp -ar data_bags/ dist/#{Regexp.escape(env)}/#{Regexp.escape(policy)}/" : ''}$},
|
51
53
|
proc do
|
52
|
-
|
53
|
-
FileUtils.
|
54
|
+
package_dir = "#{repository}/dist/#{env}/#{policy}"
|
55
|
+
FileUtils.mkdir_p package_dir
|
56
|
+
FileUtils.cp_r("#{repository}/data_bags", "#{package_dir}/") if data_bags
|
57
|
+
cookbook_metadata.each do |cookbook, metadata|
|
58
|
+
metadata_file = "#{package_dir}/cookbook_artifacts/#{cookbook}/metadata.json"
|
59
|
+
FileUtils.mkdir_p File.dirname(metadata_file)
|
60
|
+
File.write(metadata_file, metadata.to_json)
|
61
|
+
end
|
54
62
|
[0, 'Chef export done', '']
|
55
63
|
end
|
56
64
|
]
|
@@ -80,6 +88,9 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
80
88
|
with_serverless_chef_platforms('1_node') do |platform, repository|
|
81
89
|
with_packaging_mocked(repository) do
|
82
90
|
platform.package(services: { 'node' => %w[test_policy] }, secrets: {}, local_environment: false)
|
91
|
+
gems_file = "#{repository}/dist/prod/test_policy/gems.json"
|
92
|
+
expect(File.exist?(gems_file)).to eq true
|
93
|
+
expect(JSON.parse(File.read(gems_file))).to eq []
|
83
94
|
end
|
84
95
|
end
|
85
96
|
end
|
@@ -212,6 +223,41 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
212
223
|
|
213
224
|
end
|
214
225
|
|
226
|
+
context 'with a platform having several cookbooks' do
|
227
|
+
|
228
|
+
it 'generates the gems info to be installed' do
|
229
|
+
with_serverless_chef_platforms('several_cookbooks') do |platform, repository|
|
230
|
+
with_packaging_mocked(
|
231
|
+
repository,
|
232
|
+
policy: 'test_policy_1',
|
233
|
+
cookbook_metadata: {
|
234
|
+
'test_cookbook_1' => {
|
235
|
+
gems: [
|
236
|
+
['my_gem_1', '0.0.1'],
|
237
|
+
['my_gem_2', '0.0.2']
|
238
|
+
]
|
239
|
+
},
|
240
|
+
'dependency_cookbook' => {
|
241
|
+
gems: [
|
242
|
+
['my_gem_3', '~> 1.3']
|
243
|
+
]
|
244
|
+
}
|
245
|
+
}
|
246
|
+
) do
|
247
|
+
platform.package(services: { 'node1' => %w[test_policy_1] }, secrets: {}, local_environment: false)
|
248
|
+
gems_file = "#{repository}/dist/prod/test_policy_1/gems.json"
|
249
|
+
expect(File.exist?(gems_file)).to eq true
|
250
|
+
expect(JSON.parse(File.read(gems_file)).sort).to eq [
|
251
|
+
['my_gem_1', '0.0.1'],
|
252
|
+
['my_gem_2', '0.0.2'],
|
253
|
+
['my_gem_3', '~> 1.3']
|
254
|
+
].sort
|
255
|
+
end
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
end
|
260
|
+
|
215
261
|
context 'with a platform having data bags' do
|
216
262
|
|
217
263
|
it 'packages data bags' do
|
@@ -6,9 +6,18 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
6
6
|
#
|
7
7
|
# Parameters::
|
8
8
|
# * *repository* (String): The repository we package
|
9
|
+
# * *env* (String): The environment for which this repository is packaged [default: 'prod']
|
9
10
|
# * *service* (String): The service being packaged in this repository [default: 'test_policy']
|
10
|
-
|
11
|
-
|
11
|
+
# * *gems* (Array<String, String>): The [<name>, <version>] gems info to be installed as generated by the packaging [default: []]
|
12
|
+
def mock_package(
|
13
|
+
repository,
|
14
|
+
env: 'prod',
|
15
|
+
service: 'test_policy',
|
16
|
+
gems: []
|
17
|
+
)
|
18
|
+
package_dir = "#{repository}/dist/#{env}/#{service}"
|
19
|
+
FileUtils.mkdir_p package_dir
|
20
|
+
File.write("#{package_dir}/gems.json", gems.to_json)
|
12
21
|
end
|
13
22
|
|
14
23
|
# Get expected actions to deploy a service on a given node
|
@@ -20,6 +29,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
20
29
|
# * *env* (String): Environment expected to be packaged [default: 'prod']
|
21
30
|
# * *policy* (String): Expected policy to be packaged [default: 'test_policy']
|
22
31
|
# * *node* (String): Expected node to be deployed [default: 'node']
|
32
|
+
# * *gems_install_cmds* (Array<String>): Expected gem install commands [default: []]
|
23
33
|
# Result::
|
24
34
|
# * Array: Expected actions
|
25
35
|
def expected_actions_to_deploy_chef(
|
@@ -28,7 +38,8 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
28
38
|
sudo: 'sudo -u root ',
|
29
39
|
env: 'prod',
|
30
40
|
policy: 'test_policy',
|
31
|
-
node: 'node'
|
41
|
+
node: 'node',
|
42
|
+
gems_install_cmds: []
|
32
43
|
)
|
33
44
|
[
|
34
45
|
{
|
@@ -48,11 +59,14 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
48
59
|
scp: { "#{repository}/dist/#{env}/#{policy}" => './hpc_deploy' },
|
49
60
|
remote_bash: [
|
50
61
|
'set -e',
|
51
|
-
"cd ./hpc_deploy/#{policy}"
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
62
|
+
"cd ./hpc_deploy/#{policy}"
|
63
|
+
] +
|
64
|
+
gems_install_cmds.map { |gem_install_cmd| "#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/embedded/bin/#{gem_install_cmd}" } +
|
65
|
+
[
|
66
|
+
"#{sudo}SSL_CERT_DIR=/etc/ssl/certs /opt/chef/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/#{node}.json#{check_mode ? ' --why-run' : ''}",
|
67
|
+
'cd ..',
|
68
|
+
"#{sudo}rm -rf ./hpc_deploy/#{policy}"
|
69
|
+
]
|
56
70
|
}
|
57
71
|
]
|
58
72
|
end
|
@@ -109,6 +123,33 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
109
123
|
end
|
110
124
|
end
|
111
125
|
|
126
|
+
it 'returns actions to deploy on this node with gems to be installed' do
|
127
|
+
with_serverless_chef_platforms('1_node') do |platform, repository|
|
128
|
+
mock_package(
|
129
|
+
repository,
|
130
|
+
gems: [
|
131
|
+
['my_gem_1', '0.0.1'],
|
132
|
+
['my_gem_2', '0.0.2'],
|
133
|
+
['my_gem_3', '~> 1.3']
|
134
|
+
]
|
135
|
+
)
|
136
|
+
platform.prepare_for_deploy(
|
137
|
+
services: { 'node' => %w[test_policy] },
|
138
|
+
secrets: {},
|
139
|
+
local_environment: false,
|
140
|
+
why_run: false
|
141
|
+
)
|
142
|
+
expect(platform.actions_to_deploy_on('node', 'test_policy', use_why_run: false)).to eq expected_actions_to_deploy_chef(
|
143
|
+
repository,
|
144
|
+
gems_install_cmds: [
|
145
|
+
'gem install my_gem_1 --version "0.0.1"',
|
146
|
+
'gem install my_gem_2 --version "0.0.2"',
|
147
|
+
'gem install my_gem_3 --version "~> 1.3"'
|
148
|
+
]
|
149
|
+
)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
112
153
|
it 'returns actions to deploy on this node with node attributes setup from metadata' do
|
113
154
|
with_serverless_chef_platforms('1_node') do |platform, repository|
|
114
155
|
test_nodes_handler.override_metadata_of 'node', :new_metadata, 'new_value'
|
@@ -161,7 +202,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
161
202
|
|
162
203
|
it 'returns actions to deploy on this node using local mode' do
|
163
204
|
with_serverless_chef_platforms('1_node') do |platform, repository|
|
164
|
-
mock_package(repository)
|
205
|
+
mock_package(repository, env: 'local')
|
165
206
|
platform.prepare_for_deploy(
|
166
207
|
services: { 'node' => %w[test_policy] },
|
167
208
|
secrets: {},
|
@@ -174,7 +215,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
174
215
|
|
175
216
|
it 'returns actions to deploy on this node in why-run mode and local mode' do
|
176
217
|
with_serverless_chef_platforms('1_node') do |platform, repository|
|
177
|
-
mock_package(repository)
|
218
|
+
mock_package(repository, env: 'local')
|
178
219
|
platform.prepare_for_deploy(
|
179
220
|
services: { 'node' => %w[test_policy] },
|
180
221
|
secrets: {},
|
@@ -219,7 +260,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
219
260
|
|
220
261
|
it 'deploys services declared on 1 node on another node if asked' do
|
221
262
|
with_serverless_chef_platforms('several_nodes') do |platform, repository|
|
222
|
-
mock_package(repository)
|
263
|
+
mock_package(repository, service: 'test_policy_1')
|
223
264
|
platform.prepare_for_deploy(
|
224
265
|
services: { 'node2' => %w[test_policy_1] },
|
225
266
|
secrets: {},
|
@@ -230,9 +271,38 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
230
271
|
end
|
231
272
|
end
|
232
273
|
|
233
|
-
it 'deploys local
|
274
|
+
it 'deploys using the local chef workstation when use_local_chef is set' do
|
234
275
|
with_serverless_chef_platforms('several_nodes') do |platform, repository|
|
235
|
-
mock_package(repository)
|
276
|
+
mock_package(repository, service: 'test_policy_1')
|
277
|
+
platform.prepare_for_deploy(
|
278
|
+
services: { 'local' => %w[test_policy_1] },
|
279
|
+
secrets: {},
|
280
|
+
local_environment: false,
|
281
|
+
why_run: false
|
282
|
+
)
|
283
|
+
expect(platform.actions_to_deploy_on('local', 'test_policy_1', use_why_run: false)).to eq [
|
284
|
+
{
|
285
|
+
bash: [
|
286
|
+
'set -e',
|
287
|
+
"cd #{repository}/dist/prod/test_policy_1",
|
288
|
+
'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/local.json'
|
289
|
+
]
|
290
|
+
}
|
291
|
+
]
|
292
|
+
end
|
293
|
+
end
|
294
|
+
|
295
|
+
it 'deploys using the local chef workstation with gems to be installed when use_local_chef is set' do
|
296
|
+
with_serverless_chef_platforms('several_nodes') do |platform, repository|
|
297
|
+
mock_package(
|
298
|
+
repository,
|
299
|
+
service: 'test_policy_1',
|
300
|
+
gems: [
|
301
|
+
['my_gem_1', '0.0.1'],
|
302
|
+
['my_gem_2', '0.0.2'],
|
303
|
+
['my_gem_3', '~> 1.3']
|
304
|
+
]
|
305
|
+
)
|
236
306
|
platform.prepare_for_deploy(
|
237
307
|
services: { 'local' => %w[test_policy_1] },
|
238
308
|
secrets: {},
|
@@ -241,7 +311,14 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
241
311
|
)
|
242
312
|
expect(platform.actions_to_deploy_on('local', 'test_policy_1', use_why_run: false)).to eq [
|
243
313
|
{
|
244
|
-
bash:
|
314
|
+
bash: [
|
315
|
+
'set -e',
|
316
|
+
"cd #{repository}/dist/prod/test_policy_1",
|
317
|
+
'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef gem install my_gem_1 --version "0.0.1"',
|
318
|
+
'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef gem install my_gem_2 --version "0.0.2"',
|
319
|
+
'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef gem install my_gem_3 --version "~> 1.3"',
|
320
|
+
'sudo SSL_CERT_DIR=/etc/ssl/certs /opt/chef-workstation/bin/chef-client --local-mode --chef-license accept --json-attributes nodes/local.json'
|
321
|
+
]
|
245
322
|
}
|
246
323
|
]
|
247
324
|
end
|
@@ -254,7 +331,7 @@ describe HybridPlatformsConductor::HpcPlugins::PlatformHandler::ServerlessChef d
|
|
254
331
|
it 'deploys a service on a node belonging to another platform' do
|
255
332
|
with_serverless_chef_platforms({ 'p1' => '1_node', 'p2' => 'several_nodes' }) do |repositories|
|
256
333
|
platform_1, repository_1 = repositories.find { |platform, _repository| platform.name == 'p1' }
|
257
|
-
mock_package(repository_1)
|
334
|
+
mock_package(repository_1, service: 'test_policy_1')
|
258
335
|
platform_1.prepare_for_deploy(
|
259
336
|
services: { 'node2' => %w[test_policy_1] },
|
260
337
|
secrets: {},
|