vmpooler 2.0.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,6 +24,10 @@ module Vmpooler
24
24
  Vmpooler::API.settings.config[:config]
25
25
  end
26
26
 
27
+ def full_config
28
+ Vmpooler::API.settings.config
29
+ end
30
+
27
31
  def pools
28
32
  Vmpooler::API.settings.config[:pools]
29
33
  end
@@ -49,13 +53,15 @@ module Vmpooler
49
53
  end
50
54
 
51
55
  def get_template_aliases(template)
52
- result = []
53
- aliases = Vmpooler::API.settings.config[:alias]
54
- if aliases
55
- result += aliases[template] if aliases[template].is_a?(Array)
56
- template_backends << aliases[template] if aliases[template].is_a?(String)
56
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
57
+ result = []
58
+ aliases = Vmpooler::API.settings.config[:alias]
59
+ if aliases
60
+ result += aliases[template] if aliases[template].is_a?(Array)
61
+ template_backends << aliases[template] if aliases[template].is_a?(String)
62
+ end
63
+ result
57
64
  end
58
- result
59
65
  end
60
66
 
61
67
  def get_pool_weights(template_backends)
@@ -109,398 +115,463 @@ module Vmpooler
109
115
  end
110
116
 
111
117
  def fetch_single_vm(template)
112
- template_backends = [template]
113
- aliases = Vmpooler::API.settings.config[:alias]
114
- if aliases
115
- template_backends += aliases[template] if aliases[template].is_a?(Array)
116
- template_backends << aliases[template] if aliases[template].is_a?(String)
117
- pool_index = pool_index(pools)
118
- weighted_pools = {}
119
- template_backends.each do |t|
120
- next unless pool_index.key? t
121
-
122
- index = pool_index[t]
123
- clone_target = pools[index]['clone_target'] || config['clone_target']
124
- next unless config.key?('backend_weight')
125
-
126
- weight = config['backend_weight'][clone_target]
127
- if weight
128
- weighted_pools[t] = weight
118
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
119
+ template_backends = [template]
120
+ aliases = Vmpooler::API.settings.config[:alias]
121
+ if aliases
122
+ template_backends += aliases[template] if aliases[template].is_a?(Array)
123
+ template_backends << aliases[template] if aliases[template].is_a?(String)
124
+ pool_index = pool_index(pools)
125
+ weighted_pools = {}
126
+ template_backends.each do |t|
127
+ next unless pool_index.key? t
128
+
129
+ index = pool_index[t]
130
+ clone_target = pools[index]['clone_target'] || config['clone_target']
131
+ next unless config.key?('backend_weight')
132
+
133
+ weight = config['backend_weight'][clone_target]
134
+ if weight
135
+ weighted_pools[t] = weight
136
+ end
129
137
  end
130
- end
131
138
 
132
- if weighted_pools.count == template_backends.count
133
- pickup = Pickup.new(weighted_pools)
134
- selection = pickup.pick
135
- template_backends.delete(selection)
136
- template_backends.unshift(selection)
137
- else
138
- first = template_backends.sample
139
- template_backends.delete(first)
140
- template_backends.unshift(first)
139
+ if weighted_pools.count == template_backends.count
140
+ pickup = Pickup.new(weighted_pools)
141
+ selection = pickup.pick
142
+ template_backends.delete(selection)
143
+ template_backends.unshift(selection)
144
+ else
145
+ first = template_backends.sample
146
+ template_backends.delete(first)
147
+ template_backends.unshift(first)
148
+ end
141
149
  end
142
- end
143
-
144
- checkoutlock.synchronize do
145
- template_backends.each do |template_backend|
146
- vms = backend.smembers("vmpooler__ready__#{template_backend}")
147
- next if vms.empty?
148
150
 
149
- vms.reverse.each do |vm|
150
- ready = vm_ready?(vm, config['domain'])
151
- if ready
152
- smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
153
- if smoved
154
- return [vm, template_backend, template]
151
+ checkoutlock.synchronize do
152
+ template_backends.each do |template_backend|
153
+ vms = backend.smembers("vmpooler__ready__#{template_backend}")
154
+ next if vms.empty?
155
+
156
+ vms.reverse.each do |vm|
157
+ ready = vm_ready?(vm, config['domain'])
158
+ if ready
159
+ smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
160
+ if smoved
161
+ return [vm, template_backend, template]
162
+ else
163
+ metrics.increment("checkout.smove.failed.#{template_backend}")
164
+ return [nil, nil, nil]
165
+ end
155
166
  else
156
- metrics.increment("checkout.smove.failed.#{template_backend}")
157
- return [nil, nil, nil]
167
+ backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
168
+ metrics.increment("checkout.nonresponsive.#{template_backend}")
158
169
  end
159
- else
160
- backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
161
- metrics.increment("checkout.nonresponsive.#{template_backend}")
162
170
  end
163
171
  end
172
+ [nil, nil, nil]
164
173
  end
165
- [nil, nil, nil]
166
174
  end
167
175
  end
168
176
 
169
177
  def return_vm_to_ready_state(template, vm)
170
- backend.srem("vmpooler__migrating__#{template}", vm)
171
- backend.hdel("vmpooler__active__#{template}", vm)
172
- backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user')
173
- backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm)
178
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
179
+ backend.srem("vmpooler__migrating__#{template}", vm)
180
+ backend.hdel("vmpooler__active__#{template}", vm)
181
+ backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user')
182
+ backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm)
183
+ end
174
184
  end
175
185
 
176
186
  def account_for_starting_vm(template, vm)
177
- user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user')
178
- has_token_result = has_token?
179
- backend.sadd("vmpooler__migrating__#{template}", vm)
180
- backend.hset("vmpooler__active__#{template}", vm, Time.now)
181
- backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
182
-
183
- if Vmpooler::API.settings.config[:auth] and has_token_result
184
- backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
185
- backend.hset("vmpooler__vm__#{vm}", 'token:user', user)
186
-
187
- if config['vm_lifetime_auth'].to_i > 0
188
- backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i)
187
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
188
+ user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user')
189
+ span.set_attribute('enduser.id', user)
190
+ has_token_result = has_token?
191
+ backend.sadd("vmpooler__migrating__#{template}", vm)
192
+ backend.hset("vmpooler__active__#{template}", vm, Time.now)
193
+ backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
194
+
195
+ if Vmpooler::API.settings.config[:auth] and has_token_result
196
+ backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
197
+ backend.hset("vmpooler__vm__#{vm}", 'token:user', user)
198
+
199
+ if config['vm_lifetime_auth'].to_i > 0
200
+ backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i)
201
+ end
189
202
  end
190
203
  end
191
204
  end
192
205
 
193
206
  def update_result_hosts(result, template, vm)
194
- result[template] ||= {}
195
- if result[template]['hostname']
196
- result[template]['hostname'] = Array(result[template]['hostname'])
197
- result[template]['hostname'].push(vm)
198
- else
199
- result[template]['hostname'] = vm
207
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
208
+ result[template] ||= {}
209
+ if result[template]['hostname']
210
+ result[template]['hostname'] = Array(result[template]['hostname'])
211
+ result[template]['hostname'].push(vm)
212
+ else
213
+ result[template]['hostname'] = vm
214
+ end
200
215
  end
201
216
  end
202
217
 
203
218
  def atomically_allocate_vms(payload)
204
- result = { 'ok' => false }
205
- failed = false
206
- vms = []
207
-
208
- validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token?
209
-
210
- payload.each do |requested, count|
211
- count.to_i.times do |_i|
212
- vmname, vmpool, vmtemplate = fetch_single_vm(requested)
213
- if vmname
214
- account_for_starting_vm(vmpool, vmname)
215
- vms << [vmpool, vmname, vmtemplate]
216
- metrics.increment("checkout.success.#{vmpool}")
217
- update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats']
218
- else
219
- failed = true
220
- metrics.increment("checkout.empty.#{requested}")
221
- break
219
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
220
+ result = { 'ok' => false }
221
+ failed = false
222
+ vms = []
223
+
224
+ validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token?
225
+
226
+ payload.each do |requested, count|
227
+ count.to_i.times do |_i|
228
+ vmname, vmpool, vmtemplate = fetch_single_vm(requested)
229
+ if vmname
230
+ account_for_starting_vm(vmpool, vmname)
231
+ vms << [vmpool, vmname, vmtemplate]
232
+ metrics.increment("checkout.success.#{vmpool}")
233
+ update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats']
234
+ else
235
+ failed = true
236
+ metrics.increment("checkout.empty.#{requested}")
237
+ break
238
+ end
222
239
  end
223
240
  end
224
- end
225
241
 
226
- if failed
227
- vms.each do |(vmpool, vmname, _vmtemplate)|
228
- return_vm_to_ready_state(vmpool, vmname)
229
- end
230
- status 503
231
- else
232
- vms.each do |(_vmpool, vmname, vmtemplate)|
233
- update_result_hosts(result, vmtemplate, vmname)
242
+ if failed
243
+ vms.each do |(vmpool, vmname, _vmtemplate)|
244
+ return_vm_to_ready_state(vmpool, vmname)
245
+ end
246
+ span.add_event('error', attributes: {
247
+ 'error.type' => 'Vmpooler::API::V1.atomically_allocate_vms',
248
+ 'error.message' => '503 due to failing to allocate one or more vms'
249
+ })
250
+ status 503
251
+ else
252
+ vm_names = []
253
+ vms.each do |(_vmpool, vmname, vmtemplate)|
254
+ update_result_hosts(result, vmtemplate, vmname)
255
+ vm_names.append(vmname)
256
+ end
257
+
258
+ span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty?
259
+
260
+ result['ok'] = true
261
+ result['domain'] = config['domain'] if config['domain']
234
262
  end
235
263
 
236
- result['ok'] = true
237
- result['domain'] = config['domain'] if config['domain']
264
+ result
238
265
  end
239
-
240
- result
241
266
  end
242
267
 
243
268
  def component_to_test(match, labels_string)
244
- return if labels_string.nil?
245
-
246
- labels_string_parts = labels_string.split(',')
247
- labels_string_parts.each do |part|
248
- key, value = part.split('=')
249
- next if value.nil?
250
- return value if key == match
269
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
270
+ return if labels_string.nil?
271
+
272
+ labels_string_parts = labels_string.split(',')
273
+ labels_string_parts.each do |part|
274
+ key, value = part.split('=')
275
+ next if value.nil?
276
+ return value if key == match
277
+ end
278
+ 'none'
251
279
  end
252
- 'none'
253
280
  end
254
281
 
255
282
  def update_user_metrics(operation, vmname)
256
- backend.multi
257
- backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url')
258
- backend.hget("vmpooler__vm__#{vmname}", 'token:user')
259
- backend.hget("vmpooler__vm__#{vmname}", 'template')
260
- jenkins_build_url, user, poolname = backend.exec
261
- poolname = poolname.gsub('.', '_')
262
-
263
- if user
264
- user = user.gsub('.', '_')
265
- else
266
- user = 'unauthenticated'
267
- end
268
- metrics.increment("user.#{user}.#{operation}.#{poolname}")
283
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
284
+ begin
285
+ backend.multi
286
+ backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url')
287
+ backend.hget("vmpooler__vm__#{vmname}", 'token:user')
288
+ backend.hget("vmpooler__vm__#{vmname}", 'template')
289
+ jenkins_build_url, user, poolname = backend.exec
290
+ poolname = poolname.gsub('.', '_')
291
+
292
+ if user
293
+ user = user.gsub('.', '_')
294
+ else
295
+ user = 'unauthenticated'
296
+ end
297
+ metrics.increment("user.#{user}.#{operation}.#{poolname}")
269
298
 
270
- if jenkins_build_url
271
- if jenkins_build_url.include? 'litmus'
272
- # Very simple filter for Litmus jobs - just count them coming through for the moment.
273
- metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}")
274
- return
299
+ if jenkins_build_url
300
+ if jenkins_build_url.include? 'litmus'
301
+ # Very simple filter for Litmus jobs - just count them coming through for the moment.
302
+ metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}")
303
+ else
304
+ url_parts = jenkins_build_url.split('/')[2..-1]
305
+ jenkins_instance = url_parts[0].gsub('.', '_')
306
+ value_stream_parts = url_parts[2].split('_')
307
+ value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
308
+ value_stream = value_stream_parts.shift
309
+ branch = value_stream_parts.pop
310
+ project = value_stream_parts.shift
311
+ job_name = value_stream_parts.join('_')
312
+ build_metadata_parts = url_parts[3]
313
+ component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
314
+
315
+ metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
316
+ metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
317
+ metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
318
+ end
319
+ end
320
+ rescue StandardError => e
321
+ puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
322
+ span.record_exception(e)
323
+ span.status = OpenTelemetry::Trace::Status.error(e.to_s)
324
+ span.add_event('log', attributes: {
325
+ 'log.severity' => 'debug',
326
+ 'log.message' => "[#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
327
+ })
275
328
  end
276
-
277
- url_parts = jenkins_build_url.split('/')[2..-1]
278
- jenkins_instance = url_parts[0].gsub('.', '_')
279
- value_stream_parts = url_parts[2].split('_')
280
- value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
281
- value_stream = value_stream_parts.shift
282
- branch = value_stream_parts.pop
283
- project = value_stream_parts.shift
284
- job_name = value_stream_parts.join('_')
285
- build_metadata_parts = url_parts[3]
286
- component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
287
-
288
- metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}")
289
- metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}")
290
- metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}")
291
329
  end
292
- rescue StandardError => e
293
- puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}"
294
330
  end
295
331
 
296
332
  def reset_pool_size(poolname)
297
- result = { 'ok' => false }
333
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
334
+ result = { 'ok' => false }
298
335
 
299
- pool_index = pool_index(pools)
336
+ pool_index = pool_index(pools)
300
337
 
301
- pools_updated = 0
302
- sync_pool_sizes
338
+ pools_updated = 0
339
+ sync_pool_sizes
303
340
 
304
- pool_size_now = pools[pool_index[poolname]]['size'].to_i
305
- pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i
306
- result['pool_size_before_reset'] = pool_size_now
307
- result['pool_size_before_overrides'] = pool_size_original
341
+ pool_size_now = pools[pool_index[poolname]]['size'].to_i
342
+ pool_size_original = pools_at_startup[pool_index[poolname]]['size'].to_i
343
+ result['pool_size_before_reset'] = pool_size_now
344
+ result['pool_size_before_overrides'] = pool_size_original
308
345
 
309
- unless pool_size_now == pool_size_original
310
- pools[pool_index[poolname]]['size'] = pool_size_original
311
- backend.hdel('vmpooler__config__poolsize', poolname)
312
- backend.sadd('vmpooler__pool__undo_size_override', poolname)
313
- pools_updated += 1
314
- status 201
315
- end
346
+ unless pool_size_now == pool_size_original
347
+ pools[pool_index[poolname]]['size'] = pool_size_original
348
+ backend.hdel('vmpooler__config__poolsize', poolname)
349
+ backend.sadd('vmpooler__pool__undo_size_override', poolname)
350
+ pools_updated += 1
351
+ status 201
352
+ end
316
353
 
317
- status 200 unless pools_updated > 0
318
- result['ok'] = true
319
- result
354
+ status 200 unless pools_updated > 0
355
+ result['ok'] = true
356
+ result
357
+ end
320
358
  end
321
359
 
322
360
  def update_pool_size(payload)
323
- result = { 'ok' => false }
361
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
362
+ result = { 'ok' => false }
324
363
 
325
- pool_index = pool_index(pools)
326
- pools_updated = 0
327
- sync_pool_sizes
364
+ pool_index = pool_index(pools)
365
+ pools_updated = 0
366
+ sync_pool_sizes
328
367
 
329
- payload.each do |poolname, size|
330
- unless pools[pool_index[poolname]]['size'] == size.to_i
331
- pools[pool_index[poolname]]['size'] = size.to_i
332
- backend.hset('vmpooler__config__poolsize', poolname, size)
333
- pools_updated += 1
334
- status 201
368
+ payload.each do |poolname, size|
369
+ unless pools[pool_index[poolname]]['size'] == size.to_i
370
+ pools[pool_index[poolname]]['size'] = size.to_i
371
+ backend.hset('vmpooler__config__poolsize', poolname, size)
372
+ pools_updated += 1
373
+ status 201
374
+ end
335
375
  end
376
+ status 200 unless pools_updated > 0
377
+ result['ok'] = true
378
+ result
336
379
  end
337
- status 200 unless pools_updated > 0
338
- result['ok'] = true
339
- result
340
380
  end
341
381
 
342
382
  def reset_pool_template(poolname)
343
- result = { 'ok' => false }
383
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
384
+ result = { 'ok' => false }
344
385
 
345
- pool_index_live = pool_index(pools)
346
- pool_index_original = pool_index(pools_at_startup)
386
+ pool_index_live = pool_index(pools)
387
+ pool_index_original = pool_index(pools_at_startup)
347
388
 
348
- pools_updated = 0
349
- sync_pool_templates
389
+ pools_updated = 0
390
+ sync_pool_templates
350
391
 
351
- template_now = pools[pool_index_live[poolname]]['template']
352
- template_original = pools_at_startup[pool_index_original[poolname]]['template']
353
- result['template_before_reset'] = template_now
354
- result['template_before_overrides'] = template_original
392
+ template_now = pools[pool_index_live[poolname]]['template']
393
+ template_original = pools_at_startup[pool_index_original[poolname]]['template']
394
+ result['template_before_reset'] = template_now
395
+ result['template_before_overrides'] = template_original
355
396
 
356
- unless template_now == template_original
357
- pools[pool_index_live[poolname]]['template'] = template_original
358
- backend.hdel('vmpooler__config__template', poolname)
359
- backend.sadd('vmpooler__pool__undo_template_override', poolname)
360
- pools_updated += 1
361
- status 201
362
- end
397
+ unless template_now == template_original
398
+ pools[pool_index_live[poolname]]['template'] = template_original
399
+ backend.hdel('vmpooler__config__template', poolname)
400
+ backend.sadd('vmpooler__pool__undo_template_override', poolname)
401
+ pools_updated += 1
402
+ status 201
403
+ end
363
404
 
364
- status 200 unless pools_updated > 0
365
- result['ok'] = true
366
- result
405
+ status 200 unless pools_updated > 0
406
+ result['ok'] = true
407
+ result
408
+ end
367
409
  end
368
410
 
369
411
  def update_pool_template(payload)
370
- result = { 'ok' => false }
412
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
413
+ result = { 'ok' => false }
371
414
 
372
- pool_index = pool_index(pools)
373
- pools_updated = 0
374
- sync_pool_templates
415
+ pool_index = pool_index(pools)
416
+ pools_updated = 0
417
+ sync_pool_templates
375
418
 
376
- payload.each do |poolname, template|
377
- unless pools[pool_index[poolname]]['template'] == template
378
- pools[pool_index[poolname]]['template'] = template
379
- backend.hset('vmpooler__config__template', poolname, template)
380
- pools_updated += 1
381
- status 201
419
+ payload.each do |poolname, template|
420
+ unless pools[pool_index[poolname]]['template'] == template
421
+ pools[pool_index[poolname]]['template'] = template
422
+ backend.hset('vmpooler__config__template', poolname, template)
423
+ pools_updated += 1
424
+ status 201
425
+ end
382
426
  end
427
+ status 200 unless pools_updated > 0
428
+ result['ok'] = true
429
+ result
383
430
  end
384
- status 200 unless pools_updated > 0
385
- result['ok'] = true
386
- result
387
431
  end
388
432
 
389
433
  def reset_pool(payload)
390
- result = { 'ok' => false }
434
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
435
+ result = { 'ok' => false }
391
436
 
392
- payload.each do |poolname, _count|
393
- backend.sadd('vmpooler__poolreset', poolname)
437
+ payload.each do |poolname, _count|
438
+ backend.sadd('vmpooler__poolreset', poolname)
439
+ end
440
+ status 201
441
+ result['ok'] = true
442
+ result
394
443
  end
395
- status 201
396
- result['ok'] = true
397
- result
398
444
  end
399
445
 
400
446
  def update_clone_target(payload)
401
- result = { 'ok' => false }
402
-
403
- pool_index = pool_index(pools)
404
- pools_updated = 0
405
- sync_clone_targets
447
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
448
+ result = { 'ok' => false }
406
449
 
407
- payload.each do |poolname, clone_target|
408
- unless pools[pool_index[poolname]]['clone_target'] == clone_target
409
- pools[pool_index[poolname]]['clone_target'] = clone_target
410
- backend.hset('vmpooler__config__clone_target', poolname, clone_target)
411
- pools_updated += 1
412
- status 201
450
+ pool_index = pool_index(pools)
451
+ pools_updated = 0
452
+ sync_clone_targets
453
+
454
+ payload.each do |poolname, clone_target|
455
+ unless pools[pool_index[poolname]]['clone_target'] == clone_target
456
+ pools[pool_index[poolname]]['clone_target'] = clone_target
457
+ backend.hset('vmpooler__config__clone_target', poolname, clone_target)
458
+ pools_updated += 1
459
+ status 201
460
+ end
413
461
  end
462
+ status 200 unless pools_updated > 0
463
+ result['ok'] = true
464
+ result
414
465
  end
415
- status 200 unless pools_updated > 0
416
- result['ok'] = true
417
- result
418
466
  end
419
467
 
420
468
  def sync_pool_templates
421
- pool_index = pool_index(pools)
422
- template_configs = backend.hgetall('vmpooler__config__template')
423
- template_configs&.each do |poolname, template|
424
- next unless pool_index.include? poolname
469
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
470
+ pool_index = pool_index(pools)
471
+ template_configs = backend.hgetall('vmpooler__config__template')
472
+ template_configs&.each do |poolname, template|
473
+ next unless pool_index.include? poolname
425
474
 
426
- pools[pool_index[poolname]]['template'] = template
475
+ pools[pool_index[poolname]]['template'] = template
476
+ end
427
477
  end
428
478
  end
429
479
 
430
480
  def sync_pool_sizes
431
- pool_index = pool_index(pools)
432
- poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
433
- poolsize_configs&.each do |poolname, size|
434
- next unless pool_index.include? poolname
481
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
482
+ pool_index = pool_index(pools)
483
+ poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
484
+ poolsize_configs&.each do |poolname, size|
485
+ next unless pool_index.include? poolname
435
486
 
436
- pools[pool_index[poolname]]['size'] = size.to_i
487
+ pools[pool_index[poolname]]['size'] = size.to_i
488
+ end
437
489
  end
438
490
  end
439
491
 
440
492
  def sync_clone_targets
441
- pool_index = pool_index(pools)
442
- clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
443
- clone_target_configs&.each do |poolname, clone_target|
444
- next unless pool_index.include? poolname
493
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
494
+ pool_index = pool_index(pools)
495
+ clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
496
+ clone_target_configs&.each do |poolname, clone_target|
497
+ next unless pool_index.include? poolname
445
498
 
446
- pools[pool_index[poolname]]['clone_target'] = clone_target
499
+ pools[pool_index[poolname]]['clone_target'] = clone_target
500
+ end
447
501
  end
448
502
  end
449
503
 
450
504
  def too_many_requested?(payload)
451
- payload&.each do |poolname, count|
452
- next unless count.to_i > config['max_ondemand_instances_per_request']
505
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
506
+ payload&.each do |poolname, count|
507
+ next unless count.to_i > config['max_ondemand_instances_per_request']
453
508
 
454
- metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}")
455
- return true
509
+ metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}")
510
+ return true
511
+ end
512
+ false
456
513
  end
457
- false
458
514
  end
459
515
 
460
516
  def generate_ondemand_request(payload)
461
- result = { 'ok': false }
462
-
463
- requested_instances = payload.reject { |k, _v| k == 'request_id' }
464
- if too_many_requested?(requested_instances)
465
- result['message'] = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
466
- status 403
467
- return result
468
- end
517
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
518
+ result = { 'ok': false }
519
+
520
+ requested_instances = payload.reject { |k, _v| k == 'request_id' }
521
+ if too_many_requested?(requested_instances)
522
+ e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
523
+ result['message'] = e_message
524
+ status 403
525
+ span.add_event('error', attributes: {
526
+ 'error.type' => 'Vmpooler::API::V1.generate_ondemand_request',
527
+ 'error.message' => "403 due to #{e_message}"
528
+ })
529
+ return result
530
+ end
469
531
 
470
- score = Time.now.to_i
471
- request_id = payload['request_id']
472
- request_id ||= generate_request_id
473
- result['request_id'] = request_id
532
+ score = Time.now.to_i
533
+ request_id = payload['request_id']
534
+ request_id ||= generate_request_id
535
+ result['request_id'] = request_id
536
+ span.set_attribute('vmpooler.request_id', request_id)
537
+
538
+ if backend.exists?("vmpooler__odrequest__#{request_id}")
539
+ e_message = "request_id '#{request_id}' has already been created"
540
+ result['message'] = e_message
541
+ status 409
542
+ span.add_event('error', attributes: {
543
+ 'error.type' => 'Vmpooler::API::V1.generate_ondemand_request',
544
+ 'error.message' => "409 due to #{e_message}"
545
+ })
546
+ metrics.increment('ondemandrequest_generate.duplicaterequests')
547
+ return result
548
+ end
474
549
 
475
- if backend.exists?("vmpooler__odrequest__#{request_id}")
476
- result['message'] = "request_id '#{request_id}' has already been created"
477
- status 409
478
- metrics.increment('ondemandrequest_generate.duplicaterequests')
479
- return result
480
- end
550
+ status 201
481
551
 
482
- status 201
552
+ platforms_with_aliases = []
553
+ requested_instances.each do |poolname, count|
554
+ selection = evaluate_template_aliases(poolname, count)
555
+ selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
556
+ end
557
+ platforms_string = platforms_with_aliases.join(',')
483
558
 
484
- platforms_with_aliases = []
485
- requested_instances.each do |poolname, count|
486
- selection = evaluate_template_aliases(poolname, count)
487
- selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
488
- end
489
- platforms_string = platforms_with_aliases.join(',')
559
+ return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
490
560
 
491
- return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
561
+ backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
562
+ if Vmpooler::API.settings.config[:auth] and has_token?
563
+ token_token = request.env['HTTP_X_AUTH_TOKEN']
564
+ token_user = backend.hget("vmpooler__token__#{token_token}", 'user')
565
+ backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token)
566
+ backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user)
567
+ span.set_attribute('enduser.id', token_user)
568
+ end
492
569
 
493
- backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
494
- if Vmpooler::API.settings.config[:auth] and has_token?
495
- backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
496
- backend.hset("vmpooler__odrequest__#{request_id}", 'token:user',
497
- backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user'))
570
+ result['domain'] = config['domain'] if config['domain']
571
+ result[:ok] = true
572
+ metrics.increment('ondemandrequest_generate.success')
573
+ result
498
574
  end
499
-
500
- result['domain'] = config['domain'] if config['domain']
501
- result[:ok] = true
502
- metrics.increment('ondemandrequest_generate.success')
503
- result
504
575
  end
505
576
 
506
577
  def generate_request_id
@@ -813,6 +884,8 @@ module Vmpooler
813
884
  data = backend.hgetall(key)
814
885
 
815
886
  if data['user'] == Rack::Auth::Basic::Request.new(request.env).username
887
+ span = OpenTelemetry::Trace.current_span
888
+ span.set_attribute('enduser.id', data['user'])
816
889
  token = key.split('__').last
817
890
 
818
891
  result[token] ||= {}
@@ -899,6 +972,8 @@ module Vmpooler
899
972
 
900
973
  backend.hset("vmpooler__token__#{result['token']}", 'user', @auth.username)
901
974
  backend.hset("vmpooler__token__#{result['token']}", 'created', Time.now)
975
+ span = OpenTelemetry::Trace.current_span
976
+ span.set_attribute('enduser.id', @auth.username)
902
977
 
903
978
  status 200
904
979
  result['ok'] = true
@@ -946,6 +1021,8 @@ module Vmpooler
946
1021
  status 404
947
1022
  end
948
1023
  rescue JSON::ParserError
1024
+ span = OpenTelemetry::Trace.current_span
1025
+ span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
949
1026
  status 400
950
1027
  result = {
951
1028
  'ok' => false,
@@ -1031,134 +1108,160 @@ module Vmpooler
1031
1108
  end
1032
1109
 
1033
1110
  def extract_templates_from_query_params(params)
1034
- payload = {}
1111
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
1112
+ payload = {}
1035
1113
 
1036
- params.split('+').each do |template|
1037
- payload[template] ||= 0
1038
- payload[template] += 1
1039
- end
1114
+ params.split('+').each do |template|
1115
+ payload[template] ||= 0
1116
+ payload[template] += 1
1117
+ end
1040
1118
 
1041
- payload
1119
+ payload
1120
+ end
1042
1121
  end
1043
1122
 
1044
1123
  def invalid_templates(payload)
1045
- invalid = []
1046
- payload.keys.each do |template|
1047
- invalid << template unless pool_exists?(template)
1124
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
1125
+ invalid = []
1126
+ payload.keys.each do |template|
1127
+ invalid << template unless pool_exists?(template)
1128
+ end
1129
+ invalid
1048
1130
  end
1049
- invalid
1050
1131
  end
1051
1132
 
1052
1133
  def invalid_template_or_size(payload)
1053
- invalid = []
1054
- payload.each do |pool, size|
1055
- invalid << pool unless pool_exists?(pool)
1056
- unless is_integer?(size)
1057
- invalid << pool
1058
- next
1134
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
1135
+ invalid = []
1136
+ payload.each do |pool, size|
1137
+ invalid << pool unless pool_exists?(pool)
1138
+ unless is_integer?(size)
1139
+ invalid << pool
1140
+ next
1141
+ end
1142
+ invalid << pool unless Integer(size) >= 0
1059
1143
  end
1060
- invalid << pool unless Integer(size) >= 0
1144
+ invalid
1061
1145
  end
1062
- invalid
1063
1146
  end
1064
1147
 
1065
1148
  def invalid_template_or_path(payload)
1066
- invalid = []
1067
- payload.each do |pool, template|
1068
- invalid << pool unless pool_exists?(pool)
1069
- invalid << pool unless template.include? '/'
1070
- invalid << pool if template[0] == '/'
1071
- invalid << pool if template[-1] == '/'
1149
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
1150
+ invalid = []
1151
+ payload.each do |pool, template|
1152
+ invalid << pool unless pool_exists?(pool)
1153
+ invalid << pool unless template.include? '/'
1154
+ invalid << pool if template[0] == '/'
1155
+ invalid << pool if template[-1] == '/'
1156
+ end
1157
+ invalid
1072
1158
  end
1073
- invalid
1074
1159
  end
1075
1160
 
1076
1161
  def invalid_pool(payload)
1077
- invalid = []
1078
- payload.each do |pool, _clone_target|
1079
- invalid << pool unless pool_exists?(pool)
1162
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do
1163
+ invalid = []
1164
+ payload.each do |pool, _clone_target|
1165
+ invalid << pool unless pool_exists?(pool)
1166
+ end
1167
+ invalid
1080
1168
  end
1081
- invalid
1082
1169
  end
1083
1170
 
1084
1171
  def check_ondemand_request(request_id)
1085
- result = { 'ok' => false }
1086
- request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
1087
- if request_hash.empty?
1088
- result['message'] = "no request found for request_id '#{request_id}'"
1089
- return result
1090
- end
1172
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
1173
+ span.set_attribute('vmpooler.request_id', request_id)
1174
+ result = { 'ok' => false }
1175
+ request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
1176
+ if request_hash.empty?
1177
+ e_message = "no request found for request_id '#{request_id}'"
1178
+ result['message'] = e_message
1179
+ span.add_event('error', attributes: {
1180
+ 'error.type' => 'Vmpooler::API::V1.check_ondemand_request',
1181
+ 'error.message' => e_message
1182
+ })
1183
+ return result
1184
+ end
1091
1185
 
1092
- result['request_id'] = request_id
1093
- result['ready'] = false
1094
- result['ok'] = true
1095
- status 202
1186
+ result['request_id'] = request_id
1187
+ result['ready'] = false
1188
+ result['ok'] = true
1189
+ status 202
1096
1190
 
1097
- case request_hash['status']
1098
- when 'ready'
1099
- result['ready'] = true
1100
- Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
1101
- instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1191
+ case request_hash['status']
1192
+ when 'ready'
1193
+ result['ready'] = true
1194
+ Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
1195
+ instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1102
1196
 
1103
- if result.key?(platform_alias)
1104
- result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
1105
- else
1106
- result[platform_alias] = { 'hostname': instances }
1197
+ if result.key?(platform_alias)
1198
+ result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
1199
+ else
1200
+ result[platform_alias] = { 'hostname': instances }
1201
+ end
1107
1202
  end
1108
- end
1109
- result['domain'] = config['domain'] if config['domain']
1110
- status 200
1111
- when 'failed'
1112
- result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
1113
- status 200
1114
- when 'deleted'
1115
- result['message'] = 'The request has been deleted'
1116
- status 200
1117
- else
1118
- Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
1119
- instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1120
- instances_pending = count.to_i - instance_count.to_i
1203
+ result['domain'] = config['domain'] if config['domain']
1204
+ status 200
1205
+ when 'failed'
1206
+ result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
1207
+ status 200
1208
+ when 'deleted'
1209
+ result['message'] = 'The request has been deleted'
1210
+ status 200
1211
+ else
1212
+ Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
1213
+ instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1214
+ instances_pending = count.to_i - instance_count.to_i
1121
1215
 
1122
- if result.key?(platform_alias) && result[platform_alias].key?(:ready)
1123
- result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
1124
- result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
1125
- else
1126
- result[platform_alias] = {
1127
- 'ready': instance_count.to_s,
1128
- 'pending': instances_pending.to_s
1129
- }
1216
+ if result.key?(platform_alias) && result[platform_alias].key?(:ready)
1217
+ result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
1218
+ result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
1219
+ else
1220
+ result[platform_alias] = {
1221
+ 'ready': instance_count.to_s,
1222
+ 'pending': instances_pending.to_s
1223
+ }
1224
+ end
1130
1225
  end
1131
1226
  end
1132
- end
1133
1227
 
1134
- result
1228
+ result
1229
+ end
1135
1230
  end
1136
1231
 
1137
1232
  def delete_ondemand_request(request_id)
1138
- result = { 'ok' => false }
1139
-
1140
- platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
1141
- unless platforms
1142
- result['message'] = "no request found for request_id '#{request_id}'"
1143
- return result
1144
- end
1233
+ tracer.in_span("Vmpooler::API::V1.#{__method__}") do |span|
1234
+ span.set_attribute('vmpooler.request_id', request_id)
1235
+ result = { 'ok' => false }
1236
+
1237
+ platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested')
1238
+ unless platforms
1239
+ e_message = "no request found for request_id '#{request_id}'"
1240
+ result['message'] = e_message
1241
+ span.add_event('error', attributes: {
1242
+ 'error.type' => 'Vmpooler::API::V1.delete_ondemand_request',
1243
+ 'error.message' => e_message
1244
+ })
1245
+ return result
1246
+ end
1145
1247
 
1146
- if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
1147
- result['message'] = 'the request has already been deleted'
1148
- else
1149
- backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
1248
+ if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted'
1249
+ result['message'] = 'the request has already been deleted'
1250
+ else
1251
+ backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted')
1150
1252
 
1151
- Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count|
1152
- backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm|
1153
- backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
1253
+ Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count|
1254
+ backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm|
1255
+ backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm)
1256
+ end
1257
+ backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1154
1258
  end
1155
- backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1259
+ backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
1156
1260
  end
1157
- backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0)
1261
+ status 200
1262
+ result['ok'] = true
1263
+ result
1158
1264
  end
1159
- status 200
1160
- result['ok'] = true
1161
- result
1162
1265
  end
1163
1266
 
1164
1267
  post "#{api_prefix}/vm/:template/?" do
@@ -1303,7 +1406,10 @@ module Vmpooler
1303
1406
  if backend.exists?("vmpooler__vm__#{params[:hostname]}")
1304
1407
  begin
1305
1408
  jdata = JSON.parse(request.body.read)
1306
- rescue StandardError
1409
+ rescue StandardError => e
1410
+ span = OpenTelemetry::Trace.current_span
1411
+ span.record_exception(e)
1412
+ span.status = OpenTelemetry::Trace::Status.error(e.to_s)
1307
1413
  halt 400, JSON.pretty_generate(result)
1308
1414
  end
1309
1415
 
@@ -1559,6 +1665,9 @@ module Vmpooler
1559
1665
  status 404
1560
1666
  end
1561
1667
  rescue JSON::ParserError
1668
+ span = OpenTelemetry::Trace.current_span
1669
+ span.record_exception(e)
1670
+ span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
1562
1671
  status 400
1563
1672
  result = {
1564
1673
  'ok' => false,
@@ -1629,6 +1738,20 @@ module Vmpooler
1629
1738
  end
1630
1739
  JSON.pretty_generate(result)
1631
1740
  end
1741
+
1742
+ get "#{api_prefix}/full_config/?" do
1743
+ content_type :json
1744
+
1745
+ result = {
1746
+ full_config: full_config,
1747
+ status: {
1748
+ ok: true
1749
+ }
1750
+ }
1751
+
1752
+ status 200
1753
+ JSON.pretty_generate(result)
1754
+ end
1632
1755
  end
1633
1756
  end
1634
1757
  end