vmpooler 2.1.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,505 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'vmpooler/api/v1'
4
+
5
+ module Vmpooler
6
+ class API
7
+ class V2 < Vmpooler::API::V1
8
+ api_version = '2'
9
+ api_prefix = "/api/v#{api_version}"
10
+
11
+ def full_config
12
+ Vmpooler::API.settings.config
13
+ end
14
+
15
+ def get_template_aliases(template)
16
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do
17
+ result = []
18
+ aliases = Vmpooler::API.settings.config[:alias]
19
+ if aliases
20
+ result += aliases[template] if aliases[template].is_a?(Array)
21
+ template_backends << aliases[template] if aliases[template].is_a?(String)
22
+ end
23
+ result
24
+ end
25
+ end
26
+
27
+ # Fetch a single vm from a pool
28
+ #
29
+ # @param [String] template
30
+ # The template that the vm should be created from
31
+ #
32
+ # @return [Tuple] vmname, vmpool, vmtemplate
33
+ # Returns a tuple containing the vm's name, the pool it came from, and
34
+ # what template was used, if successful. Otherwise the tuple contains.
35
+ # nil values.
36
+ def fetch_single_vm(template)
37
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do
38
+ template_backends = [template]
39
+ aliases = Vmpooler::API.settings.config[:alias]
40
+ if aliases
41
+ template_backends += aliases[template] if aliases[template].is_a?(Array)
42
+ template_backends << aliases[template] if aliases[template].is_a?(String)
43
+ pool_index = pool_index(pools)
44
+ weighted_pools = {}
45
+ template_backends.each do |t|
46
+ next unless pool_index.key? t
47
+
48
+ index = pool_index[t]
49
+ clone_target = pools[index]['clone_target'] || config['clone_target']
50
+ next unless config.key?('backend_weight')
51
+
52
+ weight = config['backend_weight'][clone_target]
53
+ if weight
54
+ weighted_pools[t] = weight
55
+ end
56
+ end
57
+
58
+ if weighted_pools.count == template_backends.count
59
+ pickup = Pickup.new(weighted_pools)
60
+ selection = pickup.pick
61
+ template_backends.delete(selection)
62
+ template_backends.unshift(selection)
63
+ else
64
+ first = template_backends.sample
65
+ template_backends.delete(first)
66
+ template_backends.unshift(first)
67
+ end
68
+ end
69
+
70
+ checkoutlock.synchronize do
71
+ template_backends.each do |template_backend|
72
+ vms = backend.smembers("vmpooler__ready__#{template_backend}")
73
+ next if vms.empty?
74
+
75
+ vm = vms.pop
76
+ smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
77
+ if smoved
78
+ return [vm, template_backend, template]
79
+ end
80
+ end
81
+ [nil, nil, nil]
82
+ end
83
+ end
84
+ end
85
+
86
+ # The domain in the result body will be set to the one associated with the
87
+ # last vm added. The part of the response is only being retained for
88
+ # backwards compatibility as the hostnames are now fqdn's instead of bare
89
+ # hostnames. This change is a result of now being able to specify a domain
90
+ # per pool. If no vm's in the result had a domain sepcified then the
91
+ # domain key will be omitted similar to how it was previously omitted if
92
+ # the global option domain wasn't specified.
93
+ def atomically_allocate_vms(payload)
94
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span|
95
+ result = { 'ok' => false }
96
+ failed = false
97
+ vms = [] # vmpool, vmname, vmtemplate
98
+
99
+ validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token?
100
+
101
+ payload.each do |requested, count|
102
+ count.to_i.times do |_i|
103
+ vmname, vmpool, vmtemplate = fetch_single_vm(requested)
104
+ if vmname
105
+ account_for_starting_vm(vmpool, vmname)
106
+ vms << [vmpool, vmname, vmtemplate]
107
+ metrics.increment("checkout.success.#{vmpool}")
108
+ update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats']
109
+ else
110
+ failed = true
111
+ metrics.increment("checkout.empty.#{requested}")
112
+ break
113
+ end
114
+ end
115
+ end
116
+
117
+ if failed
118
+ vms.each do |(vmpool, vmname, _vmtemplate)|
119
+ return_vm_to_ready_state(vmpool, vmname)
120
+ end
121
+ span.add_event('error', attributes: {
122
+ 'error.type' => 'Vmpooler::API::V2.atomically_allocate_vms',
123
+ 'error.message' => '503 due to failing to allocate one or more vms'
124
+ })
125
+ status 503
126
+ else
127
+ vm_names = []
128
+ vms.each do |(vmpool, vmname, vmtemplate)|
129
+ vmdomain = Parsing.get_domain_for_pool(full_config, vmpool)
130
+ if vmdomain
131
+ vmfqdn = "#{vmname}.#{vmdomain}"
132
+ update_result_hosts(result, vmtemplate, vmfqdn)
133
+ vm_names.append(vmfqdn)
134
+ else
135
+ update_result_hosts(result, vmtemplate, vmname)
136
+ vm_names.append(vmname)
137
+ end
138
+ end
139
+
140
+ span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty?
141
+
142
+ result['ok'] = true
143
+ end
144
+
145
+ result
146
+ end
147
+ end
148
+
149
+ def generate_ondemand_request(payload)
150
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span|
151
+ result = { 'ok': false }
152
+
153
+ requested_instances = payload.reject { |k, _v| k == 'request_id' }
154
+ if too_many_requested?(requested_instances)
155
+ e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
156
+ result['message'] = e_message
157
+ status 403
158
+ span.add_event('error', attributes: {
159
+ 'error.type' => 'Vmpooler::API::V2.generate_ondemand_request',
160
+ 'error.message' => "403 due to #{e_message}"
161
+ })
162
+ return result
163
+ end
164
+
165
+ score = Time.now.to_i
166
+ request_id = payload['request_id']
167
+ request_id ||= generate_request_id
168
+ result['request_id'] = request_id
169
+ span.set_attribute('vmpooler.request_id', request_id)
170
+
171
+ if backend.exists?("vmpooler__odrequest__#{request_id}")
172
+ e_message = "request_id '#{request_id}' has already been created"
173
+ result['message'] = e_message
174
+ status 409
175
+ span.add_event('error', attributes: {
176
+ 'error.type' => 'Vmpooler::API::V2.generate_ondemand_request',
177
+ 'error.message' => "409 due to #{e_message}"
178
+ })
179
+ metrics.increment('ondemandrequest_generate.duplicaterequests')
180
+ return result
181
+ end
182
+
183
+ status 201
184
+
185
+ platforms_with_aliases = []
186
+ requested_instances.each do |poolname, count|
187
+ selection = evaluate_template_aliases(poolname, count)
188
+ selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
189
+ end
190
+ platforms_string = platforms_with_aliases.join(',')
191
+
192
+ return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
193
+
194
+ backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
195
+ if Vmpooler::API.settings.config[:auth] and has_token?
196
+ token_token = request.env['HTTP_X_AUTH_TOKEN']
197
+ token_user = backend.hget("vmpooler__token__#{token_token}", 'user')
198
+ backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token)
199
+ backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user)
200
+ span.set_attribute('enduser.id', token_user)
201
+ end
202
+
203
+ result[:ok] = true
204
+ metrics.increment('ondemandrequest_generate.success')
205
+ result
206
+ end
207
+ end
208
+
209
+ # Endpoints that use overridden methods
210
+
211
+ post "#{api_prefix}/vm/?" do
212
+ content_type :json
213
+ result = { 'ok' => false }
214
+ metrics.increment('http_requests_vm_total.post.vm.checkout')
215
+
216
+ payload = JSON.parse(request.body.read)
217
+
218
+ if payload
219
+ invalid = invalid_templates(payload)
220
+ if invalid.empty?
221
+ result = atomically_allocate_vms(payload)
222
+ else
223
+ invalid.each do |bad_template|
224
+ metrics.increment("checkout.invalid.#{bad_template}")
225
+ end
226
+ status 404
227
+ end
228
+ else
229
+ metrics.increment('checkout.invalid.unknown')
230
+ status 404
231
+ end
232
+
233
+ JSON.pretty_generate(result)
234
+ end
235
+
236
+ post "#{api_prefix}/vm/:template/?" do
237
+ content_type :json
238
+ result = { 'ok' => false }
239
+ metrics.increment('http_requests_vm_total.get.vm.template')
240
+
241
+ payload = extract_templates_from_query_params(params[:template])
242
+
243
+ if payload
244
+ invalid = invalid_templates(payload)
245
+ if invalid.empty?
246
+ result = atomically_allocate_vms(payload)
247
+ else
248
+ invalid.each do |bad_template|
249
+ metrics.increment("checkout.invalid.#{bad_template}")
250
+ end
251
+ status 404
252
+ end
253
+ else
254
+ metrics.increment('checkout.invalid.unknown')
255
+ status 404
256
+ end
257
+
258
+ JSON.pretty_generate(result)
259
+ end
260
+
261
+ get "#{api_prefix}/vm/:hostname/?" do
262
+ content_type :json
263
+ metrics.increment('http_requests_vm_total.get.vm.hostname')
264
+
265
+ result = {}
266
+
267
+ status 404
268
+ result['ok'] = false
269
+
270
+ params[:hostname] = hostname_shorten(params[:hostname], nil)
271
+
272
+ rdata = backend.hgetall("vmpooler__vm__#{params[:hostname]}")
273
+ unless rdata.empty?
274
+ status 200
275
+ result['ok'] = true
276
+
277
+ result[params[:hostname]] = {}
278
+
279
+ result[params[:hostname]]['template'] = rdata['template']
280
+ result[params[:hostname]]['lifetime'] = (rdata['lifetime'] || config['vm_lifetime']).to_i
281
+
282
+ if rdata['destroy']
283
+ result[params[:hostname]]['running'] = ((Time.parse(rdata['destroy']) - Time.parse(rdata['checkout'])) / 60 / 60).round(2) if rdata['checkout']
284
+ result[params[:hostname]]['state'] = 'destroyed'
285
+ elsif rdata['checkout']
286
+ result[params[:hostname]]['running'] = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
287
+ result[params[:hostname]]['remaining'] = ((Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60 - Time.now) / 60 / 60).round(2)
288
+ result[params[:hostname]]['start_time'] = Time.parse(rdata['checkout']).to_datetime.rfc3339
289
+ result[params[:hostname]]['end_time'] = (Time.parse(rdata['checkout']) + rdata['lifetime'].to_i*60*60).to_datetime.rfc3339
290
+ result[params[:hostname]]['state'] = 'running'
291
+ elsif rdata['check']
292
+ result[params[:hostname]]['state'] = 'ready'
293
+ else
294
+ result[params[:hostname]]['state'] = 'pending'
295
+ end
296
+
297
+ rdata.keys.each do |key|
298
+ if key.match('^tag\:(.+?)$')
299
+ result[params[:hostname]]['tags'] ||= {}
300
+ result[params[:hostname]]['tags'][$1] = rdata[key]
301
+ end
302
+
303
+ if key.match('^snapshot\:(.+?)$')
304
+ result[params[:hostname]]['snapshots'] ||= []
305
+ result[params[:hostname]]['snapshots'].push($1)
306
+ end
307
+ end
308
+
309
+ if rdata['disk']
310
+ result[params[:hostname]]['disk'] = rdata['disk'].split(':')
311
+ end
312
+
313
+ # Look up IP address of the hostname
314
+ begin
315
+ ipAddress = TCPSocket.gethostbyname(params[:hostname])[3]
316
+ rescue StandardError
317
+ ipAddress = ""
318
+ end
319
+
320
+ result[params[:hostname]]['ip'] = ipAddress
321
+
322
+ if rdata['pool']
323
+ vmdomain = Parsing.get_domain_for_pool(full_config, rdata['pool'])
324
+ if vmdomain
325
+ result[params[:hostname]]['fqdn'] = "#{params[:hostname]}.#{vmdomain}"
326
+ end
327
+ end
328
+
329
+ result[params[:hostname]]['host'] = rdata['host'] if rdata['host']
330
+ result[params[:hostname]]['migrated'] = rdata['migrated'] if rdata['migrated']
331
+
332
+ end
333
+
334
+ JSON.pretty_generate(result)
335
+ end
336
+
337
+ post "#{api_prefix}/ondemandvm/?" do
338
+ content_type :json
339
+ metrics.increment('http_requests_vm_total.post.ondemand.requestid')
340
+
341
+ need_token! if Vmpooler::API.settings.config[:auth]
342
+
343
+ result = { 'ok' => false }
344
+
345
+ begin
346
+ payload = JSON.parse(request.body.read)
347
+
348
+ if payload
349
+ invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
350
+ if invalid.empty?
351
+ result = generate_ondemand_request(payload)
352
+ else
353
+ result[:bad_templates] = invalid
354
+ invalid.each do |bad_template|
355
+ metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
356
+ end
357
+ status 404
358
+ end
359
+ else
360
+ metrics.increment('ondemandrequest_fail.invalid.unknown')
361
+ status 404
362
+ end
363
+ rescue JSON::ParserError
364
+ span = OpenTelemetry::Trace.current_span
365
+ span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
366
+ status 400
367
+ result = {
368
+ 'ok' => false,
369
+ 'message' => 'JSON payload could not be parsed'
370
+ }
371
+ end
372
+
373
+ JSON.pretty_generate(result)
374
+ end
375
+
376
+ post "#{api_prefix}/ondemandvm/:template/?" do
377
+ content_type :json
378
+ result = { 'ok' => false }
379
+ metrics.increment('http_requests_vm_total.delete.ondemand.template')
380
+
381
+ need_token! if Vmpooler::API.settings.config[:auth]
382
+
383
+ payload = extract_templates_from_query_params(params[:template])
384
+
385
+ if payload
386
+ invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
387
+ if invalid.empty?
388
+ result = generate_ondemand_request(payload)
389
+ else
390
+ result[:bad_templates] = invalid
391
+ invalid.each do |bad_template|
392
+ metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
393
+ end
394
+ status 404
395
+ end
396
+ else
397
+ metrics.increment('ondemandrequest_fail.invalid.unknown')
398
+ status 404
399
+ end
400
+
401
+ JSON.pretty_generate(result)
402
+ end
403
+
404
+ get "#{api_prefix}/ondemandvm/:requestid/?" do
405
+ content_type :json
406
+ metrics.increment('http_requests_vm_total.get.ondemand.request')
407
+
408
+ status 404
409
+ result = check_ondemand_request(params[:requestid])
410
+
411
+ JSON.pretty_generate(result)
412
+ end
413
+
414
+ def check_ondemand_request(request_id)
415
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span|
416
+ span.set_attribute('vmpooler.request_id', request_id)
417
+ result = { 'ok' => false }
418
+ request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
419
+ if request_hash.empty?
420
+ e_message = "no request found for request_id '#{request_id}'"
421
+ result['message'] = e_message
422
+ span.add_event('error', attributes: {
423
+ 'error.type' => 'Vmpooler::API::V2.check_ondemand_request',
424
+ 'error.message' => e_message
425
+ })
426
+ return result
427
+ end
428
+
429
+ result['request_id'] = request_id
430
+ result['ready'] = false
431
+ result['ok'] = true
432
+ status 202
433
+
434
+ case request_hash['status']
435
+ when 'ready'
436
+ result['ready'] = true
437
+ Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
438
+ instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
439
+ domain = Parsing.get_domain_for_pool(full_config, pool)
440
+ instances.map! { |instance| instance.concat(".#{domain}") } if domain
441
+
442
+ if result.key?(platform_alias)
443
+ result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
444
+ else
445
+ result[platform_alias] = { 'hostname': instances }
446
+ end
447
+ end
448
+ status 200
449
+ when 'failed'
450
+ result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
451
+ status 200
452
+ when 'deleted'
453
+ result['message'] = 'The request has been deleted'
454
+ status 200
455
+ else
456
+ Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
457
+ instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
458
+ instances_pending = count.to_i - instance_count.to_i
459
+
460
+ if result.key?(platform_alias) && result[platform_alias].key?(:ready)
461
+ result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
462
+ result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
463
+ else
464
+ result[platform_alias] = {
465
+ 'ready': instance_count.to_s,
466
+ 'pending': instances_pending.to_s
467
+ }
468
+ end
469
+ end
470
+ end
471
+
472
+ result
473
+ end
474
+ end
475
+
476
+ # Endpoints that only use bits from the V1 api are called here
477
+ # Note that traces will be named based on the route used in the V1 api
478
+ # but the http.url trace attribute will still have the actual requested url in it
479
+
480
+ delete "#{api_prefix}/*" do
481
+ versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
482
+ request.path_info = "/api/v1/#{versionless_path_info}"
483
+ call env
484
+ end
485
+
486
+ get "#{api_prefix}/*" do
487
+ versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
488
+ request.path_info = "/api/v1/#{versionless_path_info}"
489
+ call env
490
+ end
491
+
492
+ post "#{api_prefix}/*" do
493
+ versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
494
+ request.path_info = "/api/v1/#{versionless_path_info}"
495
+ call env
496
+ end
497
+
498
+ put "#{api_prefix}/*" do
499
+ versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
500
+ request.path_info = "/api/v1/#{versionless_path_info}"
501
+ call env
502
+ end
503
+ end
504
+ end
505
+ end
data/lib/vmpooler/api.rb CHANGED
@@ -3,7 +3,7 @@
3
3
  module Vmpooler
4
4
  class API < Sinatra::Base
5
5
  # Load API components
6
- %w[helpers dashboard reroute v1 request_logger healthcheck].each do |lib|
6
+ %w[helpers dashboard reroute v1 v2 request_logger healthcheck].each do |lib|
7
7
  require "vmpooler/api/#{lib}"
8
8
  end
9
9
  # Load dashboard components
@@ -54,6 +54,7 @@ module Vmpooler
54
54
  use Vmpooler::API::Dashboard
55
55
  use Vmpooler::API::Reroute
56
56
  use Vmpooler::API::V1
57
+ use Vmpooler::API::V2
57
58
  end
58
59
 
59
60
  # Get thee started O WebServer