vmpooler 2.0.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/vmpooler/api/helpers.rb +365 -294
- data/lib/vmpooler/api/reroute.rb +16 -0
- data/lib/vmpooler/api/v1.rb +500 -377
- data/lib/vmpooler/api/v2.rb +429 -0
- data/lib/vmpooler/api.rb +2 -1
- data/lib/vmpooler/pool_manager.rb +105 -88
- data/lib/vmpooler/providers/base.rb +26 -3
- data/lib/vmpooler/util/parsing.rb +21 -1
- data/lib/vmpooler/version.rb +1 -1
- data/lib/vmpooler.rb +11 -3
- metadata +82 -48
data/lib/vmpooler/api/v1.rb
CHANGED
@@ -24,6 +24,10 @@ module Vmpooler
|
|
24
24
|
Vmpooler::API.settings.config[:config]
|
25
25
|
end
|
26
26
|
|
27
|
+
def full_config
|
28
|
+
Vmpooler::API.settings.config
|
29
|
+
end
|
30
|
+
|
27
31
|
def pools
|
28
32
|
Vmpooler::API.settings.config[:pools]
|
29
33
|
end
|
@@ -49,13 +53,15 @@ module Vmpooler
|
|
49
53
|
end
|
50
54
|
|
51
55
|
def get_template_aliases(template)
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
56
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
57
|
+
result = []
|
58
|
+
aliases = Vmpooler::API.settings.config[:alias]
|
59
|
+
if aliases
|
60
|
+
result += aliases[template] if aliases[template].is_a?(Array)
|
61
|
+
template_backends << aliases[template] if aliases[template].is_a?(String)
|
62
|
+
end
|
63
|
+
result
|
57
64
|
end
|
58
|
-
result
|
59
65
|
end
|
60
66
|
|
61
67
|
def get_pool_weights(template_backends)
|
@@ -109,398 +115,463 @@ module Vmpooler
|
|
109
115
|
end
|
110
116
|
|
111
117
|
def fetch_single_vm(template)
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
118
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
119
|
+
template_backends = [template]
|
120
|
+
aliases = Vmpooler::API.settings.config[:alias]
|
121
|
+
if aliases
|
122
|
+
template_backends += aliases[template] if aliases[template].is_a?(Array)
|
123
|
+
template_backends << aliases[template] if aliases[template].is_a?(String)
|
124
|
+
pool_index = pool_index(pools)
|
125
|
+
weighted_pools = {}
|
126
|
+
template_backends.each do |t|
|
127
|
+
next unless pool_index.key? t
|
128
|
+
|
129
|
+
index = pool_index[t]
|
130
|
+
clone_target = pools[index]['clone_target'] || config['clone_target']
|
131
|
+
next unless config.key?('backend_weight')
|
132
|
+
|
133
|
+
weight = config['backend_weight'][clone_target]
|
134
|
+
if weight
|
135
|
+
weighted_pools[t] = weight
|
136
|
+
end
|
129
137
|
end
|
130
|
-
end
|
131
138
|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
139
|
+
if weighted_pools.count == template_backends.count
|
140
|
+
pickup = Pickup.new(weighted_pools)
|
141
|
+
selection = pickup.pick
|
142
|
+
template_backends.delete(selection)
|
143
|
+
template_backends.unshift(selection)
|
144
|
+
else
|
145
|
+
first = template_backends.sample
|
146
|
+
template_backends.delete(first)
|
147
|
+
template_backends.unshift(first)
|
148
|
+
end
|
141
149
|
end
|
142
|
-
end
|
143
|
-
|
144
|
-
checkoutlock.synchronize do
|
145
|
-
template_backends.each do |template_backend|
|
146
|
-
vms = backend.smembers("vmpooler__ready__#{template_backend}")
|
147
|
-
next if vms.empty?
|
148
150
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
151
|
+
checkoutlock.synchronize do
|
152
|
+
template_backends.each do |template_backend|
|
153
|
+
vms = backend.smembers("vmpooler__ready__#{template_backend}")
|
154
|
+
next if vms.empty?
|
155
|
+
|
156
|
+
vms.reverse.each do |vm|
|
157
|
+
ready = vm_ready?(vm, config['domain'])
|
158
|
+
if ready
|
159
|
+
smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
|
160
|
+
if smoved
|
161
|
+
return [vm, template_backend, template]
|
162
|
+
else
|
163
|
+
metrics.increment("checkout.smove.failed.#{template_backend}")
|
164
|
+
return [nil, nil, nil]
|
165
|
+
end
|
155
166
|
else
|
156
|
-
|
157
|
-
|
167
|
+
backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
|
168
|
+
metrics.increment("checkout.nonresponsive.#{template_backend}")
|
158
169
|
end
|
159
|
-
else
|
160
|
-
backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
|
161
|
-
metrics.increment("checkout.nonresponsive.#{template_backend}")
|
162
170
|
end
|
163
171
|
end
|
172
|
+
[nil, nil, nil]
|
164
173
|
end
|
165
|
-
[nil, nil, nil]
|
166
174
|
end
|
167
175
|
end
|
168
176
|
|
169
177
|
def return_vm_to_ready_state(template, vm)
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
178
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
179
|
+
backend.srem("vmpooler__migrating__#{template}", vm)
|
180
|
+
backend.hdel("vmpooler__active__#{template}", vm)
|
181
|
+
backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user')
|
182
|
+
backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm)
|
183
|
+
end
|
174
184
|
end
|
175
185
|
|
176
186
|
def account_for_starting_vm(template, vm)
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
187
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
188
|
+
user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user')
|
189
|
+
span.set_attribute('enduser.id', user)
|
190
|
+
has_token_result = has_token?
|
191
|
+
backend.sadd("vmpooler__migrating__#{template}", vm)
|
192
|
+
backend.hset("vmpooler__active__#{template}", vm, Time.now)
|
193
|
+
backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
|
194
|
+
|
195
|
+
if Vmpooler::API.settings.config[:auth] and has_token_result
|
196
|
+
backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
|
197
|
+
backend.hset("vmpooler__vm__#{vm}", 'token:user', user)
|
198
|
+
|
199
|
+
if config['vm_lifetime_auth'].to_i > 0
|
200
|
+
backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i)
|
201
|
+
end
|
189
202
|
end
|
190
203
|
end
|
191
204
|
end
|
192
205
|
|
193
206
|
def update_result_hosts(result, template, vm)
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
207
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
208
|
+
result[template] ||= {}
|
209
|
+
if result[template]['hostname']
|
210
|
+
result[template]['hostname'] = Array(result[template]['hostname'])
|
211
|
+
result[template]['hostname'].push(vm)
|
212
|
+
else
|
213
|
+
result[template]['hostname'] = vm
|
214
|
+
end
|
200
215
|
end
|
201
216
|
end
|
202
217
|
|
203
218
|
def atomically_allocate_vms(payload)
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
219
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
220
|
+
result = { 'ok' => false }
|
221
|
+
failed = false
|
222
|
+
vms = []
|
223
|
+
|
224
|
+
validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token?
|
225
|
+
|
226
|
+
payload.each do |requested, count|
|
227
|
+
count.to_i.times do |_i|
|
228
|
+
vmname, vmpool, vmtemplate = fetch_single_vm(requested)
|
229
|
+
if vmname
|
230
|
+
account_for_starting_vm(vmpool, vmname)
|
231
|
+
vms << [vmpool, vmname, vmtemplate]
|
232
|
+
metrics.increment("checkout.success.#{vmpool}")
|
233
|
+
update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats']
|
234
|
+
else
|
235
|
+
failed = true
|
236
|
+
metrics.increment("checkout.empty.#{requested}")
|
237
|
+
break
|
238
|
+
end
|
222
239
|
end
|
223
240
|
end
|
224
|
-
end
|
225
241
|
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
242
|
+
if failed
|
243
|
+
vms.each do |(vmpool, vmname, _vmtemplate)|
|
244
|
+
return_vm_to_ready_state(vmpool, vmname)
|
245
|
+
end
|
246
|
+
span.add_event('error', attributes: {
|
247
|
+
'error.type' => 'Vmpooler::API::V1.atomically_allocate_vms',
|
248
|
+
'error.message' => '503 due to failing to allocate one or more vms'
|
249
|
+
})
|
250
|
+
status 503
|
251
|
+
else
|
252
|
+
vm_names = []
|
253
|
+
vms.each do |(_vmpool, vmname, vmtemplate)|
|
254
|
+
update_result_hosts(result, vmtemplate, vmname)
|
255
|
+
vm_names.append(vmname)
|
256
|
+
end
|
257
|
+
|
258
|
+
span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty?
|
259
|
+
|
260
|
+
result['ok'] = true
|
261
|
+
result['domain'] = config['domain'] if config['domain']
|
234
262
|
end
|
235
263
|
|
236
|
-
result
|
237
|
-
result['domain'] = config['domain'] if config['domain']
|
264
|
+
result
|
238
265
|
end
|
239
|
-
|
240
|
-
result
|
241
266
|
end
|
242
267
|
|
243
268
|
def component_to_test(match, labels_string)
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
269
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
270
|
+
return if labels_string.nil?
|
271
|
+
|
272
|
+
labels_string_parts = labels_string.split(',')
|
273
|
+
labels_string_parts.each do |part|
|
274
|
+
key, value = part.split('=')
|
275
|
+
next if value.nil?
|
276
|
+
return value if key == match
|
277
|
+
end
|
278
|
+
'none'
|
251
279
|
end
|
252
|
-
'none'
|
253
280
|
end
|
254
281
|
|
255
282
|
def update_user_metrics(operation, vmname)
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
283
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
284
|
+
begin
|
285
|
+
backend.multi
|
286
|
+
backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url')
|
287
|
+
backend.hget("vmpooler__vm__#{vmname}", 'token:user')
|
288
|
+
backend.hget("vmpooler__vm__#{vmname}", 'template')
|
289
|
+
jenkins_build_url, user, poolname = backend.exec
|
290
|
+
poolname = poolname.gsub('.', '_')
|
291
|
+
|
292
|
+
if user
|
293
|
+
user = user.gsub('.', '_')
|
294
|
+
else
|
295
|
+
user = 'unauthenticated'
|
296
|
+
end
|
297
|
+
metrics.increment("user.#{user}.#{operation}.#{poolname}")
|
269
298
|
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
299
|
+
if jenkins_build_url
|
300
|
+
if jenkins_build_url.include? 'litmus'
|
301
|
+
# Very simple filter for Litmus jobs - just count them coming through for the moment.
|
302
|
+
metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}")
|
303
|
+
else
|
304
|
+
url_parts = jenkins_build_url.split('/')[2..-1]
|
305
|
+
jenkins_instance = url_parts[0].gsub('.', '_')
|
306
|
+
value_stream_parts = url_parts[2].split('_')
|
307
|
+
value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
|
308
|
+
value_stream = value_stream_parts.shift
|
309
|
+
branch = value_stream_parts.pop
|
310
|
+
project = value_stream_parts.shift
|
311
|
+
job_name = value_stream_parts.join('_')
|
312
|
+
build_metadata_parts = url_parts[3]
|
313
|
+
component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
|
314
|
+
|
315
|
+
metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
|
316
|
+
metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
|
317
|
+
metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
|
318
|
+
end
|
319
|
+
end
|
320
|
+
rescue StandardError => e
|
321
|
+
puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
|
322
|
+
span.record_exception(e)
|
323
|
+
span.status = OpenTelemetry::Trace::Status.error(e.to_s)
|
324
|
+
span.add_event('log', attributes: {
|
325
|
+
'log.severity' => 'debug',
|
326
|
+
'log.message' => "[#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
|
327
|
+
})
|
275
328
|
end
|
276
|
-
|
277
|
-
url_parts = jenkins_build_url.split('/')[2..-1]
|
278
|
-
jenkins_instance = url_parts[0].gsub('.', '_')
|
279
|
-
value_stream_parts = url_parts[2].split('_')
|
280
|
-
value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
|
281
|
-
value_stream = value_stream_parts.shift
|
282
|
-
branch = value_stream_parts.pop
|
283
|
-
project = value_stream_parts.shift
|
284
|
-
job_name = value_stream_parts.join('_')
|
285
|
-
build_metadata_parts = url_parts[3]
|
286
|
-
component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
|
287
|
-
|
288
|
-
metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
|
289
|
-
metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
|
290
|
-
metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
|
291
329
|
end
|
292
|
-
rescue StandardError => e
|
293
|
-
puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
|
294
330
|
end
|
295
331
|
|
296
332
|
def reset_pool_size(poolname)
|
297
|
-
|
333
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
334
|
+
result = { 'ok' => false }
|
298
335
|
|
299
|
-
|
336
|
+
pool_index = pool_index(pools)
|
300
337
|
|
301
|
-
|
302
|
-
|
338
|
+
pools_updated = 0
|
339
|
+
sync_pool_sizes
|
303
340
|
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
341
|
+
pool_size_now = pools[pool_index[poolname]]['size'].to_i
|
342
|
+
pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i
|
343
|
+
result['pool_size_before_reset'] = pool_size_now
|
344
|
+
result['pool_size_before_overrides'] = pool_size_original
|
308
345
|
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
346
|
+
unless pool_size_now == pool_size_original
|
347
|
+
pools[pool_index[poolname]]['size'] = pool_size_original
|
348
|
+
backend.hdel('vmpooler__config__poolsize', poolname)
|
349
|
+
backend.sadd('vmpooler__pool__undo_size_override', poolname)
|
350
|
+
pools_updated += 1
|
351
|
+
status 201
|
352
|
+
end
|
316
353
|
|
317
|
-
|
318
|
-
|
319
|
-
|
354
|
+
status 200 unless pools_updated > 0
|
355
|
+
result['ok'] = true
|
356
|
+
result
|
357
|
+
end
|
320
358
|
end
|
321
359
|
|
322
360
|
def update_pool_size(payload)
|
323
|
-
|
361
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
362
|
+
result = { 'ok' => false }
|
324
363
|
|
325
|
-
|
326
|
-
|
327
|
-
|
364
|
+
pool_index = pool_index(pools)
|
365
|
+
pools_updated = 0
|
366
|
+
sync_pool_sizes
|
328
367
|
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
368
|
+
payload.each do |poolname, size|
|
369
|
+
unless pools[pool_index[poolname]]['size'] == size.to_i
|
370
|
+
pools[pool_index[poolname]]['size'] = size.to_i
|
371
|
+
backend.hset('vmpooler__config__poolsize', poolname, size)
|
372
|
+
pools_updated += 1
|
373
|
+
status 201
|
374
|
+
end
|
335
375
|
end
|
376
|
+
status 200 unless pools_updated > 0
|
377
|
+
result['ok'] = true
|
378
|
+
result
|
336
379
|
end
|
337
|
-
status 200 unless pools_updated > 0
|
338
|
-
result['ok'] = true
|
339
|
-
result
|
340
380
|
end
|
341
381
|
|
342
382
|
def reset_pool_template(poolname)
|
343
|
-
|
383
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
384
|
+
result = { 'ok' => false }
|
344
385
|
|
345
|
-
|
346
|
-
|
386
|
+
pool_index_live = pool_index(pools)
|
387
|
+
pool_index_original = pool_index(pools_at_startup)
|
347
388
|
|
348
|
-
|
349
|
-
|
389
|
+
pools_updated = 0
|
390
|
+
sync_pool_templates
|
350
391
|
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
392
|
+
template_now = pools[pool_index_live[poolname]]['template']
|
393
|
+
template_original = pools_at_startup[pool_index_original[poolname]]['template']
|
394
|
+
result['template_before_reset'] = template_now
|
395
|
+
result['template_before_overrides'] = template_original
|
355
396
|
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
397
|
+
unless template_now == template_original
|
398
|
+
pools[pool_index_live[poolname]]['template'] = template_original
|
399
|
+
backend.hdel('vmpooler__config__template', poolname)
|
400
|
+
backend.sadd('vmpooler__pool__undo_template_override', poolname)
|
401
|
+
pools_updated += 1
|
402
|
+
status 201
|
403
|
+
end
|
363
404
|
|
364
|
-
|
365
|
-
|
366
|
-
|
405
|
+
status 200 unless pools_updated > 0
|
406
|
+
result['ok'] = true
|
407
|
+
result
|
408
|
+
end
|
367
409
|
end
|
368
410
|
|
369
411
|
def update_pool_template(payload)
|
370
|
-
|
412
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
413
|
+
result = { 'ok' => false }
|
371
414
|
|
372
|
-
|
373
|
-
|
374
|
-
|
415
|
+
pool_index = pool_index(pools)
|
416
|
+
pools_updated = 0
|
417
|
+
sync_pool_templates
|
375
418
|
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
419
|
+
payload.each do |poolname, template|
|
420
|
+
unless pools[pool_index[poolname]]['template'] == template
|
421
|
+
pools[pool_index[poolname]]['template'] = template
|
422
|
+
backend.hset('vmpooler__config__template', poolname, template)
|
423
|
+
pools_updated += 1
|
424
|
+
status 201
|
425
|
+
end
|
382
426
|
end
|
427
|
+
status 200 unless pools_updated > 0
|
428
|
+
result['ok'] = true
|
429
|
+
result
|
383
430
|
end
|
384
|
-
status 200 unless pools_updated > 0
|
385
|
-
result['ok'] = true
|
386
|
-
result
|
387
431
|
end
|
388
432
|
|
389
433
|
def reset_pool(payload)
|
390
|
-
|
434
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
435
|
+
result = { 'ok' => false }
|
391
436
|
|
392
|
-
|
393
|
-
|
437
|
+
payload.each do |poolname, _count|
|
438
|
+
backend.sadd('vmpooler__poolreset', poolname)
|
439
|
+
end
|
440
|
+
status 201
|
441
|
+
result['ok'] = true
|
442
|
+
result
|
394
443
|
end
|
395
|
-
status 201
|
396
|
-
result['ok'] = true
|
397
|
-
result
|
398
444
|
end
|
399
445
|
|
400
446
|
def update_clone_target(payload)
|
401
|
-
|
402
|
-
|
403
|
-
pool_index = pool_index(pools)
|
404
|
-
pools_updated = 0
|
405
|
-
sync_clone_targets
|
447
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
448
|
+
result = { 'ok' => false }
|
406
449
|
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
450
|
+
pool_index = pool_index(pools)
|
451
|
+
pools_updated = 0
|
452
|
+
sync_clone_targets
|
453
|
+
|
454
|
+
payload.each do |poolname, clone_target|
|
455
|
+
unless pools[pool_index[poolname]]['clone_target'] == clone_target
|
456
|
+
pools[pool_index[poolname]]['clone_target'] = clone_target
|
457
|
+
backend.hset('vmpooler__config__clone_target', poolname, clone_target)
|
458
|
+
pools_updated += 1
|
459
|
+
status 201
|
460
|
+
end
|
413
461
|
end
|
462
|
+
status 200 unless pools_updated > 0
|
463
|
+
result['ok'] = true
|
464
|
+
result
|
414
465
|
end
|
415
|
-
status 200 unless pools_updated > 0
|
416
|
-
result['ok'] = true
|
417
|
-
result
|
418
466
|
end
|
419
467
|
|
420
468
|
def sync_pool_templates
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
469
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
470
|
+
pool_index = pool_index(pools)
|
471
|
+
template_configs = backend.hgetall('vmpooler__config__template')
|
472
|
+
template_configs&.each do |poolname, template|
|
473
|
+
next unless pool_index.include? poolname
|
425
474
|
|
426
|
-
|
475
|
+
pools[pool_index[poolname]]['template'] = template
|
476
|
+
end
|
427
477
|
end
|
428
478
|
end
|
429
479
|
|
430
480
|
def sync_pool_sizes
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
481
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
482
|
+
pool_index = pool_index(pools)
|
483
|
+
poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
|
484
|
+
poolsize_configs&.each do |poolname, size|
|
485
|
+
next unless pool_index.include? poolname
|
435
486
|
|
436
|
-
|
487
|
+
pools[pool_index[poolname]]['size'] = size.to_i
|
488
|
+
end
|
437
489
|
end
|
438
490
|
end
|
439
491
|
|
440
492
|
def sync_clone_targets
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
493
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
494
|
+
pool_index = pool_index(pools)
|
495
|
+
clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
|
496
|
+
clone_target_configs&.each do |poolname, clone_target|
|
497
|
+
next unless pool_index.include? poolname
|
445
498
|
|
446
|
-
|
499
|
+
pools[pool_index[poolname]]['clone_target'] = clone_target
|
500
|
+
end
|
447
501
|
end
|
448
502
|
end
|
449
503
|
|
450
504
|
def too_many_requested?(payload)
|
451
|
-
|
452
|
-
|
505
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
506
|
+
payload&.each do |poolname, count|
|
507
|
+
next unless count.to_i > config['max_ondemand_instances_per_request']
|
453
508
|
|
454
|
-
|
455
|
-
|
509
|
+
metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}")
|
510
|
+
return true
|
511
|
+
end
|
512
|
+
false
|
456
513
|
end
|
457
|
-
false
|
458
514
|
end
|
459
515
|
|
460
516
|
def generate_ondemand_request(payload)
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
517
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
518
|
+
result = { 'ok': false }
|
519
|
+
|
520
|
+
requested_instances = payload.reject { |k, _v| k == 'request_id' }
|
521
|
+
if too_many_requested?(requested_instances)
|
522
|
+
e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
|
523
|
+
result['message'] = e_message
|
524
|
+
status 403
|
525
|
+
span.add_event('error', attributes: {
|
526
|
+
'error.type' => 'Vmpooler::API::V1.generate_ondemand_request',
|
527
|
+
'error.message' => "403 due to #{e_message}"
|
528
|
+
})
|
529
|
+
return result
|
530
|
+
end
|
469
531
|
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
532
|
+
score = Time.now.to_i
|
533
|
+
request_id = payload['request_id']
|
534
|
+
request_id ||= generate_request_id
|
535
|
+
result['request_id'] = request_id
|
536
|
+
span.set_attribute('vmpooler.request_id', request_id)
|
537
|
+
|
538
|
+
if backend.exists?("vmpooler__odrequest__#{request_id}")
|
539
|
+
e_message = "request_id '#{request_id}' has already been created"
|
540
|
+
result['message'] = e_message
|
541
|
+
status 409
|
542
|
+
span.add_event('error', attributes: {
|
543
|
+
'error.type' => 'Vmpooler::API::V1.generate_ondemand_request',
|
544
|
+
'error.message' => "409 due to #{e_message}"
|
545
|
+
})
|
546
|
+
metrics.increment('ondemandrequest_generate.duplicaterequests')
|
547
|
+
return result
|
548
|
+
end
|
474
549
|
|
475
|
-
|
476
|
-
result['message'] = "request_id '#{request_id}' has already been created"
|
477
|
-
status 409
|
478
|
-
metrics.increment('ondemandrequest_generate.duplicaterequests')
|
479
|
-
return result
|
480
|
-
end
|
550
|
+
status 201
|
481
551
|
|
482
|
-
|
552
|
+
platforms_with_aliases = []
|
553
|
+
requested_instances.each do |poolname, count|
|
554
|
+
selection = evaluate_template_aliases(poolname, count)
|
555
|
+
selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
|
556
|
+
end
|
557
|
+
platforms_string = platforms_with_aliases.join(',')
|
483
558
|
|
484
|
-
|
485
|
-
requested_instances.each do |poolname, count|
|
486
|
-
selection = evaluate_template_aliases(poolname, count)
|
487
|
-
selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
|
488
|
-
end
|
489
|
-
platforms_string = platforms_with_aliases.join(',')
|
559
|
+
return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
|
490
560
|
|
491
|
-
|
561
|
+
backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
|
562
|
+
if Vmpooler::API.settings.config[:auth] and has_token?
|
563
|
+
token_token = request.env['HTTP_X_AUTH_TOKEN']
|
564
|
+
token_user = backend.hget("vmpooler__token__#{token_token}", 'user')
|
565
|
+
backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token)
|
566
|
+
backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user)
|
567
|
+
span.set_attribute('enduser.id', token_user)
|
568
|
+
end
|
492
569
|
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user'))
|
570
|
+
result['domain'] = config['domain'] if config['domain']
|
571
|
+
result[:ok] = true
|
572
|
+
metrics.increment('ondemandrequest_generate.success')
|
573
|
+
result
|
498
574
|
end
|
499
|
-
|
500
|
-
result['domain'] = config['domain'] if config['domain']
|
501
|
-
result[:ok] = true
|
502
|
-
metrics.increment('ondemandrequest_generate.success')
|
503
|
-
result
|
504
575
|
end
|
505
576
|
|
506
577
|
def generate_request_id
|
@@ -813,6 +884,8 @@ module Vmpooler
|
|
813
884
|
data = backend.hgetall(key)
|
814
885
|
|
815
886
|
if data['user'] == Rack::Auth::Basic::Request.new(request.env).username
|
887
|
+
span = OpenTelemetry::Trace.current_span
|
888
|
+
span.set_attribute('enduser.id', data['user'])
|
816
889
|
token = key.split('__').last
|
817
890
|
|
818
891
|
result[token] ||= {}
|
@@ -899,6 +972,8 @@ module Vmpooler
|
|
899
972
|
|
900
973
|
backend.hset("vmpooler__token__#{result['token']}", 'user', @auth.username)
|
901
974
|
backend.hset("vmpooler__token__#{result['token']}", 'created', Time.now)
|
975
|
+
span = OpenTelemetry::Trace.current_span
|
976
|
+
span.set_attribute('enduser.id', @auth.username)
|
902
977
|
|
903
978
|
status 200
|
904
979
|
result['ok'] = true
|
@@ -946,6 +1021,8 @@ module Vmpooler
|
|
946
1021
|
status 404
|
947
1022
|
end
|
948
1023
|
rescue JSON::ParserError
|
1024
|
+
span = OpenTelemetry::Trace.current_span
|
1025
|
+
span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
|
949
1026
|
status 400
|
950
1027
|
result = {
|
951
1028
|
'ok' => false,
|
@@ -1031,134 +1108,160 @@ module Vmpooler
|
|
1031
1108
|
end
|
1032
1109
|
|
1033
1110
|
def extract_templates_from_query_params(params)
|
1034
|
-
|
1111
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1112
|
+
payload = {}
|
1035
1113
|
|
1036
|
-
|
1037
|
-
|
1038
|
-
|
1039
|
-
|
1114
|
+
params.split('+').each do |template|
|
1115
|
+
payload[template] ||= 0
|
1116
|
+
payload[template] += 1
|
1117
|
+
end
|
1040
1118
|
|
1041
|
-
|
1119
|
+
payload
|
1120
|
+
end
|
1042
1121
|
end
|
1043
1122
|
|
1044
1123
|
def invalid_templates(payload)
|
1045
|
-
|
1046
|
-
|
1047
|
-
|
1124
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1125
|
+
invalid = []
|
1126
|
+
payload.keys.each do |template|
|
1127
|
+
invalid << template unless pool_exists?(template)
|
1128
|
+
end
|
1129
|
+
invalid
|
1048
1130
|
end
|
1049
|
-
invalid
|
1050
1131
|
end
|
1051
1132
|
|
1052
1133
|
def invalid_template_or_size(payload)
|
1053
|
-
|
1054
|
-
|
1055
|
-
|
1056
|
-
|
1057
|
-
|
1058
|
-
|
1134
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1135
|
+
invalid = []
|
1136
|
+
payload.each do |pool, size|
|
1137
|
+
invalid << pool unless pool_exists?(pool)
|
1138
|
+
unless is_integer?(size)
|
1139
|
+
invalid << pool
|
1140
|
+
next
|
1141
|
+
end
|
1142
|
+
invalid << pool unless Integer(size) >= 0
|
1059
1143
|
end
|
1060
|
-
invalid
|
1144
|
+
invalid
|
1061
1145
|
end
|
1062
|
-
invalid
|
1063
1146
|
end
|
1064
1147
|
|
1065
1148
|
def invalid_template_or_path(payload)
|
1066
|
-
|
1067
|
-
|
1068
|
-
|
1069
|
-
|
1070
|
-
|
1071
|
-
|
1149
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1150
|
+
invalid = []
|
1151
|
+
payload.each do |pool, template|
|
1152
|
+
invalid << pool unless pool_exists?(pool)
|
1153
|
+
invalid << pool unless template.include? '/'
|
1154
|
+
invalid << pool if template[0] == '/'
|
1155
|
+
invalid << pool if template[-1] == '/'
|
1156
|
+
end
|
1157
|
+
invalid
|
1072
1158
|
end
|
1073
|
-
invalid
|
1074
1159
|
end
|
1075
1160
|
|
1076
1161
|
def invalid_pool(payload)
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1162
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do
|
1163
|
+
invalid = []
|
1164
|
+
payload.each do |pool, _clone_target|
|
1165
|
+
invalid << pool unless pool_exists?(pool)
|
1166
|
+
end
|
1167
|
+
invalid
|
1080
1168
|
end
|
1081
|
-
invalid
|
1082
1169
|
end
|
1083
1170
|
|
1084
1171
|
def check_ondemand_request(request_id)
|
1085
|
-
|
1086
|
-
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
1090
|
-
|
1172
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
1173
|
+
span.set_attribute('vmpooler.request_id', request_id)
|
1174
|
+
result = { 'ok' => false }
|
1175
|
+
request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
|
1176
|
+
if request_hash.empty?
|
1177
|
+
e_message = "no request found for request_id '#{request_id}'"
|
1178
|
+
result['message'] = e_message
|
1179
|
+
span.add_event('error', attributes: {
|
1180
|
+
'error.type' => 'Vmpooler::API::V1.check_ondemand_request',
|
1181
|
+
'error.message' => e_message
|
1182
|
+
})
|
1183
|
+
return result
|
1184
|
+
end
|
1091
1185
|
|
1092
|
-
|
1093
|
-
|
1094
|
-
|
1095
|
-
|
1186
|
+
result['request_id'] = request_id
|
1187
|
+
result['ready'] = false
|
1188
|
+
result['ok'] = true
|
1189
|
+
status 202
|
1096
1190
|
|
1097
|
-
|
1098
|
-
|
1099
|
-
|
1100
|
-
|
1101
|
-
|
1191
|
+
case request_hash['status']
|
1192
|
+
when 'ready'
|
1193
|
+
result['ready'] = true
|
1194
|
+
Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
|
1195
|
+
instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
1102
1196
|
|
1103
|
-
|
1104
|
-
|
1105
|
-
|
1106
|
-
|
1197
|
+
if result.key?(platform_alias)
|
1198
|
+
result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
|
1199
|
+
else
|
1200
|
+
result[platform_alias] = { 'hostname': instances }
|
1201
|
+
end
|
1107
1202
|
end
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1111
|
-
|
1112
|
-
|
1113
|
-
|
1114
|
-
|
1115
|
-
|
1116
|
-
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
instances_pending = count.to_i - instance_count.to_i
|
1203
|
+
result['domain'] = config['domain'] if config['domain']
|
1204
|
+
status 200
|
1205
|
+
when 'failed'
|
1206
|
+
result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
|
1207
|
+
status 200
|
1208
|
+
when 'deleted'
|
1209
|
+
result['message'] = 'The request has been deleted'
|
1210
|
+
status 200
|
1211
|
+
else
|
1212
|
+
Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
|
1213
|
+
instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
1214
|
+
instances_pending = count.to_i - instance_count.to_i
|
1121
1215
|
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1126
|
-
|
1127
|
-
|
1128
|
-
|
1129
|
-
|
1216
|
+
if result.key?(platform_alias) && result[platform_alias].key?(:ready)
|
1217
|
+
result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
|
1218
|
+
result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
|
1219
|
+
else
|
1220
|
+
result[platform_alias] = {
|
1221
|
+
'ready': instance_count.to_s,
|
1222
|
+
'pending': instances_pending.to_s
|
1223
|
+
}
|
1224
|
+
end
|
1130
1225
|
end
|
1131
1226
|
end
|
1132
|
-
end
|
1133
1227
|
|
1134
|
-
|
1228
|
+
result
|
1229
|
+
end
|
1135
1230
|
end
|
1136
1231
|
|
1137
1232
|
def delete_ondemand_request(request_id)
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1233
|
+
tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
|
1234
|
+
span.set_attribute('vmpooler.request_id', request_id)
|
1235
|
+
result = { 'ok' => false }
|
1236
|
+
|
1237
|
+
platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
|
1238
|
+
unless platforms
|
1239
|
+
e_message = "no request found for request_id '#{request_id}'"
|
1240
|
+
result['message'] = e_message
|
1241
|
+
span.add_event('error', attributes: {
|
1242
|
+
'error.type' => 'Vmpooler::API::V1.delete_ondemand_request',
|
1243
|
+
'error.message' => e_message
|
1244
|
+
})
|
1245
|
+
return result
|
1246
|
+
end
|
1145
1247
|
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1248
|
+
if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
|
1249
|
+
result['message'] = 'the request has already been deleted'
|
1250
|
+
else
|
1251
|
+
backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
|
1150
1252
|
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1253
|
+
Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count|
|
1254
|
+
backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm|
|
1255
|
+
backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
|
1256
|
+
end
|
1257
|
+
backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
1154
1258
|
end
|
1155
|
-
backend.
|
1259
|
+
backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
|
1156
1260
|
end
|
1157
|
-
|
1261
|
+
status 200
|
1262
|
+
result['ok'] = true
|
1263
|
+
result
|
1158
1264
|
end
|
1159
|
-
status 200
|
1160
|
-
result['ok'] = true
|
1161
|
-
result
|
1162
1265
|
end
|
1163
1266
|
|
1164
1267
|
post "#{api_prefix}/vm/:template/?" do
|
@@ -1303,7 +1406,10 @@ module Vmpooler
|
|
1303
1406
|
if backend.exists?("vmpooler__vm__#{params[:hostname]}")
|
1304
1407
|
begin
|
1305
1408
|
jdata = JSON.parse(request.body.read)
|
1306
|
-
rescue StandardError
|
1409
|
+
rescue StandardError => e
|
1410
|
+
span = OpenTelemetry::Trace.current_span
|
1411
|
+
span.record_exception(e)
|
1412
|
+
span.status = OpenTelemetry::Trace::Status.error(e.to_s)
|
1307
1413
|
halt 400, JSON.pretty_generate(result)
|
1308
1414
|
end
|
1309
1415
|
|
@@ -1559,6 +1665,9 @@ module Vmpooler
|
|
1559
1665
|
status 404
|
1560
1666
|
end
|
1561
1667
|
rescue JSON::ParserError
|
1668
|
+
span = OpenTelemetry::Trace.current_span
|
1669
|
+
span.record_exception(e)
|
1670
|
+
span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
|
1562
1671
|
status 400
|
1563
1672
|
result = {
|
1564
1673
|
'ok' => false,
|
@@ -1629,6 +1738,20 @@ module Vmpooler
|
|
1629
1738
|
end
|
1630
1739
|
JSON.pretty_generate(result)
|
1631
1740
|
end
|
1741
|
+
|
1742
|
+
get "#{api_prefix}/full_config/?" do
|
1743
|
+
content_type :json
|
1744
|
+
|
1745
|
+
result = {
|
1746
|
+
full_config: full_config,
|
1747
|
+
status: {
|
1748
|
+
ok: true
|
1749
|
+
}
|
1750
|
+
}
|
1751
|
+
|
1752
|
+
status 200
|
1753
|
+
JSON.pretty_generate(result)
|
1754
|
+
end
|
1632
1755
|
end
|
1633
1756
|
end
|
1634
1757
|
end
|