vmpooler 2.1.0 → 2.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/vmpooler/api/helpers.rb +365 -299
- data/lib/vmpooler/api/reroute.rb +16 -0
- data/lib/vmpooler/api/v1.rb +501 -378
- data/lib/vmpooler/api/v2.rb +505 -0
- data/lib/vmpooler/api.rb +2 -1
- data/lib/vmpooler/pool_manager.rb +84 -63
- data/lib/vmpooler/providers/base.rb +1 -1
- data/lib/vmpooler/util/parsing.rb +21 -1
- data/lib/vmpooler/version.rb +1 -1
- data/lib/vmpooler.rb +6 -1
- metadata +103 -83
data/lib/vmpooler/api/v1.rb
CHANGED
@@ -24,6 +24,10 @@ module Vmpooler
|
|
24
24
|
Vmpooler::API.settings.config[:config]
|
25
25
|
end
|
26
26
|
|
27
|
+
def full_config
|
28
|
+
Vmpooler::API.settings.config
|
29
|
+
end
|
30
|
+
|
27
31
|
def pools
|
28
32
|
Vmpooler::API.settings.config[:pools]
|
29
33
|
end
|
@@ -49,13 +53,15 @@ module Vmpooler
|
|
49
53
|
end
|
50
54
|
|
51
55
|
def get_template_aliases(template)
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
56
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
57
|
+
result = []
|
58
|
+
aliases = Vmpooler::API.settings.config[:alias]
|
59
|
+
if aliases
|
60
|
+
result += aliases[template] if aliases[template].is_a?(Array)
|
61
|
+
template_backends << aliases[template] if aliases[template].is_a?(String)
|
62
|
+
end
|
63
|
+
result
|
57
64
|
end
|
58
|
-
result
|
59
65
|
end
|
60
66
|
|
61
67
|
def get_pool_weights(template_backends)
|
@@ -109,398 +115,463 @@ module Vmpooler
|
|
109
115
|
end
|
110
116
|
|
111
117
|
def fetch_single_vm(template)
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
118
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
119
|
+
template_backends = [template]
|
120
|
+
aliases = Vmpooler::API.settings.config[:alias]
|
121
|
+
if aliases
|
122
|
+
template_backends += aliases[template] if aliases[template].is_a?(Array)
|
123
|
+
template_backends << aliases[template] if aliases[template].is_a?(String)
|
124
|
+
pool_index = pool_index(pools)
|
125
|
+
weighted_pools = {}
|
126
|
+
template_backends.each do |t|
|
127
|
+
next unless pool_index.key? t
|
128
|
+
|
129
|
+
index = pool_index[t]
|
130
|
+
clone_target = pools[index]['clone_target'] || config['clone_target']
|
131
|
+
next unless config.key?('backend_weight')
|
132
|
+
|
133
|
+
weight = config['backend_weight'][clone_target]
|
134
|
+
if weight
|
135
|
+
weighted_pools[t] = weight
|
136
|
+
end
|
129
137
|
end
|
130
|
-
end
|
131
138
|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
139
|
+
if weighted_pools.count == template_backends.count
|
140
|
+
pickup = Pickup.new(weighted_pools)
|
141
|
+
selection = pickup.pick
|
142
|
+
template_backends.delete(selection)
|
143
|
+
template_backends.unshift(selection)
|
144
|
+
else
|
145
|
+
first = template_backends.sample
|
146
|
+
template_backends.delete(first)
|
147
|
+
template_backends.unshift(first)
|
148
|
+
end
|
141
149
|
end
|
142
|
-
end
|
143
|
-
|
144
|
-
checkoutlock.synchronize do
|
145
|
-
template_backends.each do |template_backend|
|
146
|
-
vms = backend.smembers("vmpooler__ready__#{template_backend}")
|
147
|
-
next if vms.empty?
|
148
150
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
151
|
+
checkoutlock.synchronize do
|
152
|
+
template_backends.each do |template_backend|
|
153
|
+
vms = backend.smembers("vmpooler__ready__#{template_backend}")
|
154
|
+
next if vms.empty?
|
155
|
+
|
156
|
+
vms.reverse.each do |vm|
|
157
|
+
ready = vm_ready?(vm, config['domain'])
|
158
|
+
if ready
|
159
|
+
smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
|
160
|
+
if smoved
|
161
|
+
return [vm, template_backend, template]
|
162
|
+
else
|
163
|
+
metrics.increment("checkout.smove.failed.#{template_backend}")
|
164
|
+
return [nil, nil, nil]
|
165
|
+
end
|
155
166
|
else
|
156
|
-
|
157
|
-
|
167
|
+
backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
|
168
|
+
metrics.increment("checkout.nonresponsive.#{template_backend}")
|
158
169
|
end
|
159
|
-
else
|
160
|
-
backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
|
161
|
-
metrics.increment("checkout.nonresponsive.#{template_backend}")
|
162
170
|
end
|
163
171
|
end
|
172
|
+
[nil, nil, nil]
|
164
173
|
end
|
165
|
-
[nil, nil, nil]
|
166
174
|
end
|
167
175
|
end
|
168
176
|
|
169
177
|
def return_vm_to_ready_state(template, vm)
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
178
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
179
|
+
backend.srem("vmpooler__migrating__#{template}", vm)
|
180
|
+
backend.hdel("vmpooler__active__#{template}", vm)
|
181
|
+
backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user')
|
182
|
+
backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm)
|
183
|
+
end
|
174
184
|
end
|
175
185
|
|
176
186
|
def account_for_starting_vm(template, vm)
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
187
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
188
|
+
user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user')
|
189
|
+
span.set_attribute('enduser.id', user)
|
190
|
+
has_token_result = has_token?
|
191
|
+
backend.sadd("vmpooler__migrating__#{template}", vm)
|
192
|
+
backend.hset("vmpooler__active__#{template}", vm, Time.now)
|
193
|
+
backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
|
194
|
+
|
195
|
+
if Vmpooler::API.settings.config[:auth] and has_token_result
|
196
|
+
backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
|
197
|
+
backend.hset("vmpooler__vm__#{vm}", 'token:user', user)
|
198
|
+
|
199
|
+
if config['vm_lifetime_auth'].to_i > 0
|
200
|
+
backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i)
|
201
|
+
end
|
189
202
|
end
|
190
203
|
end
|
191
204
|
end
|
192
205
|
|
193
206
|
def update_result_hosts(result, template, vm)
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
207
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
208
|
+
result[template] ||= {}
|
209
|
+
if result[template]['hostname']
|
210
|
+
result[template]['hostname'] = Array(result[template]['hostname'])
|
211
|
+
result[template]['hostname'].push(vm)
|
212
|
+
else
|
213
|
+
result[template]['hostname'] = vm
|
214
|
+
end
|
200
215
|
end
|
201
216
|
end
|
202
217
|
|
203
218
|
def atomically_allocate_vms(payload)
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
219
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
220
|
+
result = { 'ok' => false }
|
221
|
+
failed = false
|
222
|
+
vms = []
|
223
|
+
|
224
|
+
validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token?
|
225
|
+
|
226
|
+
payload.each do |requested, count|
|
227
|
+
count.to_i.times do |_i|
|
228
|
+
vmname, vmpool, vmtemplate = fetch_single_vm(requested)
|
229
|
+
if vmname
|
230
|
+
account_for_starting_vm(vmpool, vmname)
|
231
|
+
vms << [vmpool, vmname, vmtemplate]
|
232
|
+
metrics.increment("checkout.success.#{vmpool}")
|
233
|
+
update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats']
|
234
|
+
else
|
235
|
+
failed = true
|
236
|
+
metrics.increment("checkout.empty.#{requested}")
|
237
|
+
break
|
238
|
+
end
|
222
239
|
end
|
223
240
|
end
|
224
|
-
end
|
225
241
|
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
242
|
+
if failed
|
243
|
+
vms.each do |(vmpool, vmname, _vmtemplate)|
|
244
|
+
return_vm_to_ready_state(vmpool, vmname)
|
245
|
+
end
|
246
|
+
span.add_event('error', attributes: {
|
247
|
+
'error.type' => 'Vmpooler::API::V1.atomically_allocate_vms',
|
248
|
+
'error.message' => '503 due to failing to allocate one or more vms'
|
249
|
+
})
|
250
|
+
status 503
|
251
|
+
else
|
252
|
+
vm_names = []
|
253
|
+
vms.each do |(_vmpool, vmname, vmtemplate)|
|
254
|
+
update_result_hosts(result, vmtemplate, vmname)
|
255
|
+
vm_names.append(vmname)
|
256
|
+
end
|
257
|
+
|
258
|
+
span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty?
|
259
|
+
|
260
|
+
result['ok'] = true
|
261
|
+
result['domain'] = config['domain'] if config['domain']
|
234
262
|
end
|
235
263
|
|
236
|
-
result
|
237
|
-
result['domain'] = config['domain'] if config['domain']
|
264
|
+
result
|
238
265
|
end
|
239
|
-
|
240
|
-
result
|
241
266
|
end
|
242
267
|
|
243
268
|
def component_to_test(match, labels_string)
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
269
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
270
|
+
return if labels_string.nil?
|
271
|
+
|
272
|
+
labels_string_parts = labels_string.split(',')
|
273
|
+
labels_string_parts.each do |part|
|
274
|
+
key, value = part.split('=')
|
275
|
+
next if value.nil?
|
276
|
+
return value if key == match
|
277
|
+
end
|
278
|
+
'none'
|
251
279
|
end
|
252
|
-
'none'
|
253
280
|
end
|
254
281
|
|
255
282
|
def update_user_metrics(operation, vmname)
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
283
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
284
|
+
begin
|
285
|
+
backend.multi
|
286
|
+
backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url')
|
287
|
+
backend.hget("vmpooler__vm__#{vmname}", 'token:user')
|
288
|
+
backend.hget("vmpooler__vm__#{vmname}", 'template')
|
289
|
+
jenkins_build_url, user, poolname = backend.exec
|
290
|
+
poolname = poolname.gsub('.', '_')
|
291
|
+
|
292
|
+
if user
|
293
|
+
user = user.gsub('.', '_')
|
294
|
+
else
|
295
|
+
user = 'unauthenticated'
|
296
|
+
end
|
297
|
+
metrics.increment("user.#{user}.#{operation}.#{poolname}")
|
269
298
|
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
299
|
+
if jenkins_build_url
|
300
|
+
if jenkins_build_url.include? 'litmus'
|
301
|
+
# Very simple filter for Litmus jobs - just count them coming through for the moment.
|
302
|
+
metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}")
|
303
|
+
else
|
304
|
+
url_parts = jenkins_build_url.split('/')[2..-1]
|
305
|
+
jenkins_instance = url_parts[0].gsub('.', '_')
|
306
|
+
value_stream_parts = url_parts[2].split('_')
|
307
|
+
value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
|
308
|
+
value_stream = value_stream_parts.shift
|
309
|
+
branch = value_stream_parts.pop
|
310
|
+
project = value_stream_parts.shift
|
311
|
+
job_name = value_stream_parts.join('_')
|
312
|
+
build_metadata_parts = url_parts[3]
|
313
|
+
component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
|
314
|
+
|
315
|
+
metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
|
316
|
+
metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
|
317
|
+
metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
|
318
|
+
end
|
319
|
+
end
|
320
|
+
rescue StandardError => e
|
321
|
+
puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
|
322
|
+
span.record_exception(e)
|
323
|
+
span.status = OpenTelemetry::Trace::Status.error(e.to_s)
|
324
|
+
span.add_event('log', attributes: {
|
325
|
+
'log.severity' => 'debug',
|
326
|
+
'log.message' => "[#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
|
327
|
+
})
|
275
328
|
end
|
276
|
-
|
277
|
-
url_parts = jenkins_build_url.split('/')[2..-1]
|
278
|
-
jenkins_instance = url_parts[0].gsub('.', '_')
|
279
|
-
value_stream_parts = url_parts[2].split('_')
|
280
|
-
value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
|
281
|
-
value_stream = value_stream_parts.shift
|
282
|
-
branch = value_stream_parts.pop
|
283
|
-
project = value_stream_parts.shift
|
284
|
-
job_name = value_stream_parts.join('_')
|
285
|
-
build_metadata_parts = url_parts[3]
|
286
|
-
component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
|
287
|
-
|
288
|
-
metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
|
289
|
-
metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
|
290
|
-
metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
|
291
329
|
end
|
292
|
-
rescue StandardError => e
|
293
|
-
puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
|
294
330
|
end
|
295
331
|
|
296
332
|
def reset_pool_size(poolname)
|
297
|
-
|
333
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
334
|
+
result = { 'ok' => false }
|
298
335
|
|
299
|
-
|
336
|
+
pool_index = pool_index(pools)
|
300
337
|
|
301
|
-
|
302
|
-
|
338
|
+
pools_updated = 0
|
339
|
+
sync_pool_sizes
|
303
340
|
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
341
|
+
pool_size_now = pools[pool_index[poolname]]['size'].to_i
|
342
|
+
pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i
|
343
|
+
result['pool_size_before_reset'] = pool_size_now
|
344
|
+
result['pool_size_before_overrides'] = pool_size_original
|
308
345
|
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
346
|
+
unless pool_size_now == pool_size_original
|
347
|
+
pools[pool_index[poolname]]['size'] = pool_size_original
|
348
|
+
backend.hdel('vmpooler__config__poolsize', poolname)
|
349
|
+
backend.sadd('vmpooler__pool__undo_size_override', poolname)
|
350
|
+
pools_updated += 1
|
351
|
+
status 201
|
352
|
+
end
|
316
353
|
|
317
|
-
|
318
|
-
|
319
|
-
|
354
|
+
status 200 unless pools_updated > 0
|
355
|
+
result['ok'] = true
|
356
|
+
result
|
357
|
+
end
|
320
358
|
end
|
321
359
|
|
322
360
|
def update_pool_size(payload)
|
323
|
-
|
361
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
362
|
+
result = { 'ok' => false }
|
324
363
|
|
325
|
-
|
326
|
-
|
327
|
-
|
364
|
+
pool_index = pool_index(pools)
|
365
|
+
pools_updated = 0
|
366
|
+
sync_pool_sizes
|
328
367
|
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
368
|
+
payload.each do |poolname, size|
|
369
|
+
unless pools[pool_index[poolname]]['size'] == size.to_i
|
370
|
+
pools[pool_index[poolname]]['size'] = size.to_i
|
371
|
+
backend.hset('vmpooler__config__poolsize', poolname, size)
|
372
|
+
pools_updated += 1
|
373
|
+
status 201
|
374
|
+
end
|
335
375
|
end
|
376
|
+
status 200 unless pools_updated > 0
|
377
|
+
result['ok'] = true
|
378
|
+
result
|
336
379
|
end
|
337
|
-
status 200 unless pools_updated > 0
|
338
|
-
result['ok'] = true
|
339
|
-
result
|
340
380
|
end
|
341
381
|
|
342
382
|
def reset_pool_template(poolname)
|
343
|
-
|
383
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
384
|
+
result = { 'ok' => false }
|
344
385
|
|
345
|
-
|
346
|
-
|
386
|
+
pool_index_live = pool_index(pools)
|
387
|
+
pool_index_original = pool_index(pools_at_startup)
|
347
388
|
|
348
|
-
|
349
|
-
|
389
|
+
pools_updated = 0
|
390
|
+
sync_pool_templates
|
350
391
|
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
392
|
+
template_now = pools[pool_index_live[poolname]]['template']
|
393
|
+
template_original = pools_at_startup[pool_index_original[poolname]]['template']
|
394
|
+
result['template_before_reset'] = template_now
|
395
|
+
result['template_before_overrides'] = template_original
|
355
396
|
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
397
|
+
unless template_now == template_original
|
398
|
+
pools[pool_index_live[poolname]]['template'] = template_original
|
399
|
+
backend.hdel('vmpooler__config__template', poolname)
|
400
|
+
backend.sadd('vmpooler__pool__undo_template_override', poolname)
|
401
|
+
pools_updated += 1
|
402
|
+
status 201
|
403
|
+
end
|
363
404
|
|
364
|
-
|
365
|
-
|
366
|
-
|
405
|
+
status 200 unless pools_updated > 0
|
406
|
+
result['ok'] = true
|
407
|
+
result
|
408
|
+
end
|
367
409
|
end
|
368
410
|
|
369
411
|
def update_pool_template(payload)
|
370
|
-
|
412
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
413
|
+
result = { 'ok' => false }
|
371
414
|
|
372
|
-
|
373
|
-
|
374
|
-
|
415
|
+
pool_index = pool_index(pools)
|
416
|
+
pools_updated = 0
|
417
|
+
sync_pool_templates
|
375
418
|
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
419
|
+
payload.each do |poolname, template|
|
420
|
+
unless pools[pool_index[poolname]]['template'] == template
|
421
|
+
pools[pool_index[poolname]]['template'] = template
|
422
|
+
backend.hset('vmpooler__config__template', poolname, template)
|
423
|
+
pools_updated += 1
|
424
|
+
status 201
|
425
|
+
end
|
382
426
|
end
|
427
|
+
status 200 unless pools_updated > 0
|
428
|
+
result['ok'] = true
|
429
|
+
result
|
383
430
|
end
|
384
|
-
status 200 unless pools_updated > 0
|
385
|
-
result['ok'] = true
|
386
|
-
result
|
387
431
|
end
|
388
432
|
|
389
433
|
def reset_pool(payload)
|
390
|
-
|
434
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
435
|
+
result = { 'ok' => false }
|
391
436
|
|
392
|
-
|
393
|
-
|
437
|
+
payload.each do |poolname, _count|
|
438
|
+
backend.sadd('vmpooler__poolreset', poolname)
|
439
|
+
end
|
440
|
+
status 201
|
441
|
+
result['ok'] = true
|
442
|
+
result
|
394
443
|
end
|
395
|
-
status 201
|
396
|
-
result['ok'] = true
|
397
|
-
result
|
398
444
|
end
|
399
445
|
|
400
446
|
def update_clone_target(payload)
|
401
|
-
|
447
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
448
|
+
result = { 'ok' => false }
|
402
449
|
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
450
|
+
pool_index = pool_index(pools)
|
451
|
+
pools_updated = 0
|
452
|
+
sync_clone_targets
|
453
|
+
|
454
|
+
payload.each do |poolname, clone_target|
|
455
|
+
unless pools[pool_index[poolname]]['clone_target'] == clone_target
|
456
|
+
pools[pool_index[poolname]]['clone_target'] = clone_target
|
457
|
+
backend.hset('vmpooler__config__clone_target', poolname, clone_target)
|
458
|
+
pools_updated += 1
|
459
|
+
status 201
|
460
|
+
end
|
413
461
|
end
|
462
|
+
status 200 unless pools_updated > 0
|
463
|
+
result['ok'] = true
|
464
|
+
result
|
414
465
|
end
|
415
|
-
status 200 unless pools_updated > 0
|
416
|
-
result['ok'] = true
|
417
|
-
result
|
418
466
|
end
|
419
467
|
|
420
468
|
def sync_pool_templates
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
469
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
470
|
+
pool_index = pool_index(pools)
|
471
|
+
template_configs = backend.hgetall('vmpooler__config__template')
|
472
|
+
template_configs&.each do |poolname, template|
|
473
|
+
next unless pool_index.include? poolname
|
425
474
|
|
426
|
-
|
475
|
+
pools[pool_index[poolname]]['template'] = template
|
476
|
+
end
|
427
477
|
end
|
428
478
|
end
|
429
479
|
|
430
480
|
def sync_pool_sizes
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
481
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
482
|
+
pool_index = pool_index(pools)
|
483
|
+
poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
|
484
|
+
poolsize_configs&.each do |poolname, size|
|
485
|
+
next unless pool_index.include? poolname
|
435
486
|
|
436
|
-
|
487
|
+
pools[pool_index[poolname]]['size'] = size.to_i
|
488
|
+
end
|
437
489
|
end
|
438
490
|
end
|
439
491
|
|
440
492
|
def sync_clone_targets
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
493
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
494
|
+
pool_index = pool_index(pools)
|
495
|
+
clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
|
496
|
+
clone_target_configs&.each do |poolname, clone_target|
|
497
|
+
next unless pool_index.include? poolname
|
445
498
|
|
446
|
-
|
499
|
+
pools[pool_index[poolname]]['clone_target'] = clone_target
|
500
|
+
end
|
447
501
|
end
|
448
502
|
end
|
449
503
|
|
450
504
|
def too_many_requested?(payload)
|
451
|
-
|
452
|
-
|
505
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
506
|
+
payload&.each do |poolname, count|
|
507
|
+
next unless count.to_i > config['max_ondemand_instances_per_request']
|
453
508
|
|
454
|
-
|
455
|
-
|
509
|
+
metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}")
|
510
|
+
return true
|
511
|
+
end
|
512
|
+
false
|
456
513
|
end
|
457
|
-
false
|
458
514
|
end
|
459
515
|
|
460
516
|
def generate_ondemand_request(payload)
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
517
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
518
|
+
result = { 'ok': false }
|
519
|
+
|
520
|
+
requested_instances = payload.reject { |k, _v| k == 'request_id' }
|
521
|
+
if too_many_requested?(requested_instances)
|
522
|
+
e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
|
523
|
+
result['message'] = e_message
|
524
|
+
status 403
|
525
|
+
span.add_event('error', attributes: {
|
526
|
+
'error.type' => 'Vmpooler::API::V1.generate_ondemand_request',
|
527
|
+
'error.message' => "403 due to #{e_message}"
|
528
|
+
})
|
529
|
+
return result
|
530
|
+
end
|
469
531
|
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
532
|
+
score = Time.now.to_i
|
533
|
+
request_id = payload['request_id']
|
534
|
+
request_id ||= generate_request_id
|
535
|
+
result['request_id'] = request_id
|
536
|
+
span.set_attribute('vmpooler.request_id', request_id)
|
537
|
+
|
538
|
+
if backend.exists?("vmpooler__odrequest__#{request_id}")
|
539
|
+
e_message = "request_id '#{request_id}' has already been created"
|
540
|
+
result['message'] = e_message
|
541
|
+
status 409
|
542
|
+
span.add_event('error', attributes: {
|
543
|
+
'error.type' => 'Vmpooler::API::V1.generate_ondemand_request',
|
544
|
+
'error.message' => "409 due to #{e_message}"
|
545
|
+
})
|
546
|
+
metrics.increment('ondemandrequest_generate.duplicaterequests')
|
547
|
+
return result
|
548
|
+
end
|
474
549
|
|
475
|
-
|
476
|
-
result['message'] = "request_id '#{request_id}' has already been created"
|
477
|
-
status 409
|
478
|
-
metrics.increment('ondemandrequest_generate.duplicaterequests')
|
479
|
-
return result
|
480
|
-
end
|
550
|
+
status 201
|
481
551
|
|
482
|
-
|
552
|
+
platforms_with_aliases = []
|
553
|
+
requested_instances.each do |poolname, count|
|
554
|
+
selection = evaluate_template_aliases(poolname, count)
|
555
|
+
selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
|
556
|
+
end
|
557
|
+
platforms_string = platforms_with_aliases.join(',')
|
483
558
|
|
484
|
-
|
485
|
-
requested_instances.each do |poolname, count|
|
486
|
-
selection = evaluate_template_aliases(poolname, count)
|
487
|
-
selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
|
488
|
-
end
|
489
|
-
platforms_string = platforms_with_aliases.join(',')
|
559
|
+
return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
|
490
560
|
|
491
|
-
|
561
|
+
backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
|
562
|
+
if Vmpooler::API.settings.config[:auth] and has_token?
|
563
|
+
token_token = request.env['HTTP_X_AUTH_TOKEN']
|
564
|
+
token_user = backend.hget("vmpooler__token__#{token_token}", 'user')
|
565
|
+
backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token)
|
566
|
+
backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user)
|
567
|
+
span.set_attribute('enduser.id', token_user)
|
568
|
+
end
|
492
569
|
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user'))
|
570
|
+
result['domain'] = config['domain'] if config['domain']
|
571
|
+
result[:ok] = true
|
572
|
+
metrics.increment('ondemandrequest_generate.success')
|
573
|
+
result
|
498
574
|
end
|
499
|
-
|
500
|
-
result['domain'] = config['domain'] if config['domain']
|
501
|
-
result[:ok] = true
|
502
|
-
metrics.increment('ondemandrequest_generate.success')
|
503
|
-
result
|
504
575
|
end
|
505
576
|
|
506
577
|
def generate_request_id
|
@@ -813,6 +884,8 @@ module Vmpooler
|
|
813
884
|
data = backend.hgetall(key)
|
814
885
|
|
815
886
|
if data['user'] == Rack::Auth::Basic::Request.new(request.env).username
|
887
|
+
span = OpenTelemetry::Trace.current_span
|
888
|
+
span.set_attribute('enduser.id', data['user'])
|
816
889
|
token = key.split('__').last
|
817
890
|
|
818
891
|
result[token] ||= {}
|
@@ -899,6 +972,8 @@ module Vmpooler
|
|
899
972
|
|
900
973
|
backend.hset("vmpooler__token__#{result['token']}", 'user', @auth.username)
|
901
974
|
backend.hset("vmpooler__token__#{result['token']}", 'created', Time.now)
|
975
|
+
span = OpenTelemetry::Trace.current_span
|
976
|
+
span.set_attribute('enduser.id', @auth.username)
|
902
977
|
|
903
978
|
status 200
|
904
979
|
result['ok'] = true
|
@@ -946,6 +1021,8 @@ module Vmpooler
|
|
946
1021
|
status 404
|
947
1022
|
end
|
948
1023
|
rescue JSON::ParserError
|
1024
|
+
span = OpenTelemetry::Trace.current_span
|
1025
|
+
span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
|
949
1026
|
status 400
|
950
1027
|
result = {
|
951
1028
|
'ok' => false,
|
@@ -1031,134 +1108,160 @@ module Vmpooler
|
|
1031
1108
|
end
|
1032
1109
|
|
1033
1110
|
def extract_templates_from_query_params(params)
|
1034
|
-
|
1111
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1112
|
+
payload = {}
|
1035
1113
|
|
1036
|
-
|
1037
|
-
|
1038
|
-
|
1039
|
-
|
1114
|
+
params.split('+').each do |template|
|
1115
|
+
payload[template] ||= 0
|
1116
|
+
payload[template] += 1
|
1117
|
+
end
|
1040
1118
|
|
1041
|
-
|
1119
|
+
payload
|
1120
|
+
end
|
1042
1121
|
end
|
1043
1122
|
|
1044
1123
|
def invalid_templates(payload)
|
1045
|
-
|
1046
|
-
|
1047
|
-
|
1124
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1125
|
+
invalid = []
|
1126
|
+
payload.keys.each do |template|
|
1127
|
+
invalid << template unless pool_exists?(template)
|
1128
|
+
end
|
1129
|
+
invalid
|
1048
1130
|
end
|
1049
|
-
invalid
|
1050
1131
|
end
|
1051
1132
|
|
1052
1133
|
def invalid_template_or_size(payload)
|
1053
|
-
|
1054
|
-
|
1055
|
-
|
1056
|
-
|
1057
|
-
|
1058
|
-
|
1134
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1135
|
+
invalid = []
|
1136
|
+
payload.each do |pool, size|
|
1137
|
+
invalid << pool unless pool_exists?(pool)
|
1138
|
+
unless is_integer?(size)
|
1139
|
+
invalid << pool
|
1140
|
+
next
|
1141
|
+
end
|
1142
|
+
invalid << pool unless Integer(size) >= 0
|
1059
1143
|
end
|
1060
|
-
invalid
|
1144
|
+
invalid
|
1061
1145
|
end
|
1062
|
-
invalid
|
1063
1146
|
end
|
1064
1147
|
|
1065
1148
|
def invalid_template_or_path(payload)
|
1066
|
-
|
1067
|
-
|
1068
|
-
|
1069
|
-
|
1070
|
-
|
1071
|
-
|
1149
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1150
|
+
invalid = []
|
1151
|
+
payload.each do |pool, template|
|
1152
|
+
invalid << pool unless pool_exists?(pool)
|
1153
|
+
invalid << pool unless template.include? '/'
|
1154
|
+
invalid << pool if template[0] == '/'
|
1155
|
+
invalid << pool if template[-1] == '/'
|
1156
|
+
end
|
1157
|
+
invalid
|
1072
1158
|
end
|
1073
|
-
invalid
|
1074
1159
|
end
|
1075
1160
|
|
1076
1161
|
def invalid_pool(payload)
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1162
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1163
|
+
invalid = []
|
1164
|
+
payload.each do |pool, _clone_target|
|
1165
|
+
invalid << pool unless pool_exists?(pool)
|
1166
|
+
end
|
1167
|
+
invalid
|
1080
1168
|
end
|
1081
|
-
invalid
|
1082
1169
|
end
|
1083
1170
|
|
1084
1171
|
def check_ondemand_request(request_id)
|
1085
|
-
|
1086
|
-
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
1090
|
-
|
1172
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
1173
|
+
span.set_attribute('vmpooler.request_id', request_id)
|
1174
|
+
result = { 'ok' => false }
|
1175
|
+
request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
|
1176
|
+
if request_hash.empty?
|
1177
|
+
e_message = "no request found for request_id '#{request_id}'"
|
1178
|
+
result['message'] = e_message
|
1179
|
+
span.add_event('error', attributes: {
|
1180
|
+
'error.type' => 'Vmpooler::API::V1.check_ondemand_request',
|
1181
|
+
'error.message' => e_message
|
1182
|
+
})
|
1183
|
+
return result
|
1184
|
+
end
|
1091
1185
|
|
1092
|
-
|
1093
|
-
|
1094
|
-
|
1095
|
-
|
1186
|
+
result['request_id'] = request_id
|
1187
|
+
result['ready'] = false
|
1188
|
+
result['ok'] = true
|
1189
|
+
status 202
|
1096
1190
|
|
1097
|
-
|
1098
|
-
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1191
|
+
case request_hash['status']
|
1192
|
+
when 'ready'
|
1193
|
+
result['ready'] = true
|
1194
|
+
Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
|
1195
|
+
instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
1102
1196
|
|
1103
|
-
|
1104
|
-
|
1105
|
-
|
1106
|
-
|
1197
|
+
if result.key?(platform_alias)
|
1198
|
+
result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
|
1199
|
+
else
|
1200
|
+
result[platform_alias] = { 'hostname': instances }
|
1201
|
+
end
|
1107
1202
|
end
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1111
|
-
|
1112
|
-
|
1113
|
-
|
1114
|
-
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
instances_pending = count.to_i - instance_count.to_i
|
1203
|
+
result['domain'] = config['domain'] if config['domain']
|
1204
|
+
status 200
|
1205
|
+
when 'failed'
|
1206
|
+
result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
|
1207
|
+
status 200
|
1208
|
+
when 'deleted'
|
1209
|
+
result['message'] = 'The request has been deleted'
|
1210
|
+
status 200
|
1211
|
+
else
|
1212
|
+
Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
|
1213
|
+
instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
1214
|
+
instances_pending = count.to_i - instance_count.to_i
|
1121
1215
|
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1126
|
-
|
1127
|
-
|
1128
|
-
|
1129
|
-
|
1216
|
+
if result.key?(platform_alias) && result[platform_alias].key?(:ready)
|
1217
|
+
result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
|
1218
|
+
result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
|
1219
|
+
else
|
1220
|
+
result[platform_alias] = {
|
1221
|
+
'ready': instance_count.to_s,
|
1222
|
+
'pending': instances_pending.to_s
|
1223
|
+
}
|
1224
|
+
end
|
1130
1225
|
end
|
1131
1226
|
end
|
1132
|
-
end
|
1133
1227
|
|
1134
|
-
|
1228
|
+
result
|
1229
|
+
end
|
1135
1230
|
end
|
1136
1231
|
|
1137
1232
|
def delete_ondemand_request(request_id)
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1233
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
1234
|
+
span.set_attribute('vmpooler.request_id', request_id)
|
1235
|
+
result = { 'ok' => false }
|
1236
|
+
|
1237
|
+
platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
|
1238
|
+
unless platforms
|
1239
|
+
e_message = "no request found for request_id '#{request_id}'"
|
1240
|
+
result['message'] = e_message
|
1241
|
+
span.add_event('error', attributes: {
|
1242
|
+
'error.type' => 'Vmpooler::API::V1.delete_ondemand_request',
|
1243
|
+
'error.message' => e_message
|
1244
|
+
})
|
1245
|
+
return result
|
1246
|
+
end
|
1145
1247
|
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1248
|
+
if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
|
1249
|
+
result['message'] = 'the request has already been deleted'
|
1250
|
+
else
|
1251
|
+
backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
|
1150
1252
|
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1253
|
+
Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count|
|
1254
|
+
backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm|
|
1255
|
+
backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
|
1256
|
+
end
|
1257
|
+
backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
1154
1258
|
end
|
1155
|
-
backend.
|
1259
|
+
backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
|
1156
1260
|
end
|
1157
|
-
|
1261
|
+
status 200
|
1262
|
+
result['ok'] = true
|
1263
|
+
result
|
1158
1264
|
end
|
1159
|
-
status 200
|
1160
|
-
result['ok'] = true
|
1161
|
-
result
|
1162
1265
|
end
|
1163
1266
|
|
1164
1267
|
post "#{api_prefix}/vm/:template/?" do
|
@@ -1303,7 +1406,10 @@ module Vmpooler
|
|
1303
1406
|
if backend.exists?("vmpooler__vm__#{params[:hostname]}")
|
1304
1407
|
begin
|
1305
1408
|
jdata = JSON.parse(request.body.read)
|
1306
|
-
rescue StandardError
|
1409
|
+
rescue StandardError => e
|
1410
|
+
span = OpenTelemetry::Trace.current_span
|
1411
|
+
span.record_exception(e)
|
1412
|
+
span.status = OpenTelemetry::Trace::Status.error(e.to_s)
|
1307
1413
|
halt 400, JSON.pretty_generate(result)
|
1308
1414
|
end
|
1309
1415
|
|
@@ -1413,7 +1519,7 @@ module Vmpooler
|
|
1413
1519
|
|
1414
1520
|
post "#{api_prefix}/vm/:hostname/snapshot/:snapshot/?" do
|
1415
1521
|
content_type :json
|
1416
|
-
metrics.increment('http_requests_vm_total.post.vm.
|
1522
|
+
metrics.increment('http_requests_vm_total.post.vm.snapshot')
|
1417
1523
|
|
1418
1524
|
need_token! if Vmpooler::API.settings.config[:auth]
|
1419
1525
|
|
@@ -1559,6 +1665,9 @@ module Vmpooler
|
|
1559
1665
|
status 404
|
1560
1666
|
end
|
1561
1667
|
rescue JSON::ParserError
|
1668
|
+
span = OpenTelemetry::Trace.current_span
|
1669
|
+
span.record_exception(e)
|
1670
|
+
span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
|
1562
1671
|
status 400
|
1563
1672
|
result = {
|
1564
1673
|
'ok' => false,
|
@@ -1629,6 +1738,20 @@ module Vmpooler
|
|
1629
1738
|
end
|
1630
1739
|
JSON.pretty_generate(result)
|
1631
1740
|
end
|
1741
|
+
|
1742
|
+
get "#{api_prefix}/full_config/?" do
|
1743
|
+
content_type :json
|
1744
|
+
|
1745
|
+
result = {
|
1746
|
+
full_config: full_config,
|
1747
|
+
status: {
|
1748
|
+
ok: true
|
1749
|
+
}
|
1750
|
+
}
|
1751
|
+
|
1752
|
+
status 200
|
1753
|
+
JSON.pretty_generate(result)
|
1754
|
+
end
|
1632
1755
|
end
|
1633
1756
|
end
|
1634
1757
|
end
|