vmpooler 2.0.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,429 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'vmpooler/api/v1'
4
+
5
+ module Vmpooler
6
+ class API
7
+ class V2 < Vmpooler::API::V1
8
+ api_version = '2'
9
+ api_prefix = "/api/v#{api_version}"
10
+
11
+ def full_config
12
+ Vmpooler::API.settings.config
13
+ end
14
+
15
+ def get_template_aliases(template)
16
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do
17
+ result = []
18
+ aliases = Vmpooler::API.settings.config[:alias]
19
+ if aliases
20
+ result += aliases[template] if aliases[template].is_a?(Array)
21
+ template_backends << aliases[template] if aliases[template].is_a?(String)
22
+ end
23
+ result
24
+ end
25
+ end
26
+
27
+ # Fetch a single vm from a pool
28
+ #
29
+ # @param [String] template
30
+ # The template that the vm should be created from
31
+ #
32
+ # @return [Tuple] vmname, vmpool, vmtemplate
33
+ # Returns a tuple containing the vm's name, the pool it came from, and
34
+ # what template was used, if successful. Otherwise the tuple contains.
35
+ # nil values.
36
+ def fetch_single_vm(template)
37
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do
38
+ template_backends = [template]
39
+ aliases = Vmpooler::API.settings.config[:alias]
40
+ if aliases
41
+ template_backends += aliases[template] if aliases[template].is_a?(Array)
42
+ template_backends << aliases[template] if aliases[template].is_a?(String)
43
+ pool_index = pool_index(pools)
44
+ weighted_pools = {}
45
+ template_backends.each do |t|
46
+ next unless pool_index.key? t
47
+
48
+ index = pool_index[t]
49
+ clone_target = pools[index]['clone_target'] || config['clone_target']
50
+ next unless config.key?('backend_weight')
51
+
52
+ weight = config['backend_weight'][clone_target]
53
+ if weight
54
+ weighted_pools[t] = weight
55
+ end
56
+ end
57
+
58
+ if weighted_pools.count == template_backends.count
59
+ pickup = Pickup.new(weighted_pools)
60
+ selection = pickup.pick
61
+ template_backends.delete(selection)
62
+ template_backends.unshift(selection)
63
+ else
64
+ first = template_backends.sample
65
+ template_backends.delete(first)
66
+ template_backends.unshift(first)
67
+ end
68
+ end
69
+
70
+ checkoutlock.synchronize do
71
+ template_backends.each do |template_backend|
72
+ vms = backend.smembers("vmpooler__ready__#{template_backend}")
73
+ next if vms.empty?
74
+
75
+ vm = vms.pop
76
+ smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
77
+ if smoved
78
+ return [vm, template_backend, template]
79
+ end
80
+ end
81
+ [nil, nil, nil]
82
+ end
83
+ end
84
+ end
85
+
86
+ # The domain in the result body will be set to the one associated with the
87
+ # last vm added. The part of the response is only being retained for
88
+ # backwards compatibility as the hostnames are now fqdn's instead of bare
89
+ # hostnames. This change is a result of now being able to specify a domain
90
+ # per pool. If no vm's in the result had a domain sepcified then the
91
+ # domain key will be omitted similar to how it was previously omitted if
92
+ # the global option domain wasn't specified.
93
+ def atomically_allocate_vms(payload)
94
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span|
95
+ result = { 'ok' => false }
96
+ failed = false
97
+ vms = [] # vmpool, vmname, vmtemplate
98
+
99
+ validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token?
100
+
101
+ payload.each do |requested, count|
102
+ count.to_i.times do |_i|
103
+ vmname, vmpool, vmtemplate = fetch_single_vm(requested)
104
+ if vmname
105
+ account_for_starting_vm(vmpool, vmname)
106
+ vms << [vmpool, vmname, vmtemplate]
107
+ metrics.increment("checkout.success.#{vmpool}")
108
+ update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats']
109
+ else
110
+ failed = true
111
+ metrics.increment("checkout.empty.#{requested}")
112
+ break
113
+ end
114
+ end
115
+ end
116
+
117
+ if failed
118
+ vms.each do |(vmpool, vmname, _vmtemplate)|
119
+ return_vm_to_ready_state(vmpool, vmname)
120
+ end
121
+ span.add_event('error', attributes: {
122
+ 'error.type' => 'Vmpooler::API::V2.atomically_allocate_vms',
123
+ 'error.message' => '503 due to failing to allocate one or more vms'
124
+ })
125
+ status 503
126
+ else
127
+ vm_names = []
128
+ vms.each do |(vmpool, vmname, vmtemplate)|
129
+ vmdomain = Parsing.get_domain_for_pool(full_config, vmpool)
130
+ if vmdomain
131
+ vmfqdn = "#{vmname}.#{vmdomain}"
132
+ update_result_hosts(result, vmtemplate, vmfqdn)
133
+ vm_names.append(vmfqdn)
134
+ else
135
+ update_result_hosts(result, vmtemplate, vmname)
136
+ vm_names.append(vmname)
137
+ end
138
+ end
139
+
140
+ span.set_attribute('vmpooler.vm_names', vm_names.join(',')) unless vm_names.empty?
141
+
142
+ result['ok'] = true
143
+ end
144
+
145
+ result
146
+ end
147
+ end
148
+
149
+ def generate_ondemand_request(payload)
150
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span|
151
+ result = { 'ok': false }
152
+
153
+ requested_instances = payload.reject { |k, _v| k == 'request_id' }
154
+ if too_many_requested?(requested_instances)
155
+ e_message = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}"
156
+ result['message'] = e_message
157
+ status 403
158
+ span.add_event('error', attributes: {
159
+ 'error.type' => 'Vmpooler::API::V2.generate_ondemand_request',
160
+ 'error.message' => "403 due to #{e_message}"
161
+ })
162
+ return result
163
+ end
164
+
165
+ score = Time.now.to_i
166
+ request_id = payload['request_id']
167
+ request_id ||= generate_request_id
168
+ result['request_id'] = request_id
169
+ span.set_attribute('vmpooler.request_id', request_id)
170
+
171
+ if backend.exists?("vmpooler__odrequest__#{request_id}")
172
+ e_message = "request_id '#{request_id}' has already been created"
173
+ result['message'] = e_message
174
+ status 409
175
+ span.add_event('error', attributes: {
176
+ 'error.type' => 'Vmpooler::API::V2.generate_ondemand_request',
177
+ 'error.message' => "409 due to #{e_message}"
178
+ })
179
+ metrics.increment('ondemandrequest_generate.duplicaterequests')
180
+ return result
181
+ end
182
+
183
+ status 201
184
+
185
+ platforms_with_aliases = []
186
+ requested_instances.each do |poolname, count|
187
+ selection = evaluate_template_aliases(poolname, count)
188
+ selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" }
189
+ end
190
+ platforms_string = platforms_with_aliases.join(',')
191
+
192
+ return result unless backend.zadd('vmpooler__provisioning__request', score, request_id)
193
+
194
+ backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string)
195
+ if Vmpooler::API.settings.config[:auth] and has_token?
196
+ token_token = request.env['HTTP_X_AUTH_TOKEN']
197
+ token_user = backend.hget("vmpooler__token__#{token_token}", 'user')
198
+ backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', token_token)
199
+ backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', token_user)
200
+ span.set_attribute('enduser.id', token_user)
201
+ end
202
+
203
+ result[:ok] = true
204
+ metrics.increment('ondemandrequest_generate.success')
205
+ result
206
+ end
207
+ end
208
+
209
+ # Endpoints that use overridden methods
210
+
211
+ post "#{api_prefix}/vm/?" do
212
+ content_type :json
213
+ result = { 'ok' => false }
214
+ metrics.increment('http_requests_vm_total.post.vm.checkout')
215
+
216
+ payload = JSON.parse(request.body.read)
217
+
218
+ if payload
219
+ invalid = invalid_templates(payload)
220
+ if invalid.empty?
221
+ result = atomically_allocate_vms(payload)
222
+ else
223
+ invalid.each do |bad_template|
224
+ metrics.increment("checkout.invalid.#{bad_template}")
225
+ end
226
+ status 404
227
+ end
228
+ else
229
+ metrics.increment('checkout.invalid.unknown')
230
+ status 404
231
+ end
232
+
233
+ JSON.pretty_generate(result)
234
+ end
235
+
236
+ post "#{api_prefix}/vm/:template/?" do
237
+ content_type :json
238
+ result = { 'ok' => false }
239
+ metrics.increment('http_requests_vm_total.get.vm.template')
240
+
241
+ payload = extract_templates_from_query_params(params[:template])
242
+
243
+ if payload
244
+ invalid = invalid_templates(payload)
245
+ if invalid.empty?
246
+ result = atomically_allocate_vms(payload)
247
+ else
248
+ invalid.each do |bad_template|
249
+ metrics.increment("checkout.invalid.#{bad_template}")
250
+ end
251
+ status 404
252
+ end
253
+ else
254
+ metrics.increment('checkout.invalid.unknown')
255
+ status 404
256
+ end
257
+
258
+ JSON.pretty_generate(result)
259
+ end
260
+
261
+ post "#{api_prefix}/ondemandvm/?" do
262
+ content_type :json
263
+ metrics.increment('http_requests_vm_total.post.ondemand.requestid')
264
+
265
+ need_token! if Vmpooler::API.settings.config[:auth]
266
+
267
+ result = { 'ok' => false }
268
+
269
+ begin
270
+ payload = JSON.parse(request.body.read)
271
+
272
+ if payload
273
+ invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
274
+ if invalid.empty?
275
+ result = generate_ondemand_request(payload)
276
+ else
277
+ result[:bad_templates] = invalid
278
+ invalid.each do |bad_template|
279
+ metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
280
+ end
281
+ status 404
282
+ end
283
+ else
284
+ metrics.increment('ondemandrequest_fail.invalid.unknown')
285
+ status 404
286
+ end
287
+ rescue JSON::ParserError
288
+ span = OpenTelemetry::Trace.current_span
289
+ span.status = OpenTelemetry::Trace::Status.error('JSON payload could not be parsed')
290
+ status 400
291
+ result = {
292
+ 'ok' => false,
293
+ 'message' => 'JSON payload could not be parsed'
294
+ }
295
+ end
296
+
297
+ JSON.pretty_generate(result)
298
+ end
299
+
300
+ post "#{api_prefix}/ondemandvm/:template/?" do
301
+ content_type :json
302
+ result = { 'ok' => false }
303
+ metrics.increment('http_requests_vm_total.delete.ondemand.template')
304
+
305
+ need_token! if Vmpooler::API.settings.config[:auth]
306
+
307
+ payload = extract_templates_from_query_params(params[:template])
308
+
309
+ if payload
310
+ invalid = invalid_templates(payload.reject { |k, _v| k == 'request_id' })
311
+ if invalid.empty?
312
+ result = generate_ondemand_request(payload)
313
+ else
314
+ result[:bad_templates] = invalid
315
+ invalid.each do |bad_template|
316
+ metrics.increment("ondemandrequest_fail.invalid.#{bad_template}")
317
+ end
318
+ status 404
319
+ end
320
+ else
321
+ metrics.increment('ondemandrequest_fail.invalid.unknown')
322
+ status 404
323
+ end
324
+
325
+ JSON.pretty_generate(result)
326
+ end
327
+
328
+ get "#{api_prefix}/ondemandvm/:requestid/?" do
329
+ content_type :json
330
+ metrics.increment('http_requests_vm_total.get.ondemand.request')
331
+
332
+ status 404
333
+ result = check_ondemand_request(params[:requestid])
334
+
335
+ JSON.pretty_generate(result)
336
+ end
337
+
338
+ def check_ondemand_request(request_id)
339
+ tracer.in_span("Vmpooler::API::V2.#{__method__}") do |span|
340
+ span.set_attribute('vmpooler.request_id', request_id)
341
+ result = { 'ok' => false }
342
+ request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}")
343
+ if request_hash.empty?
344
+ e_message = "no request found for request_id '#{request_id}'"
345
+ result['message'] = e_message
346
+ span.add_event('error', attributes: {
347
+ 'error.type' => 'Vmpooler::API::V2.check_ondemand_request',
348
+ 'error.message' => e_message
349
+ })
350
+ return result
351
+ end
352
+
353
+ result['request_id'] = request_id
354
+ result['ready'] = false
355
+ result['ok'] = true
356
+ status 202
357
+
358
+ case request_hash['status']
359
+ when 'ready'
360
+ result['ready'] = true
361
+ Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
362
+ instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
363
+ domain = Parsing.get_domain_for_pool(full_config, pool)
364
+ instances.map! { |instance| instance.concat(".#{domain}") } if domain
365
+
366
+ if result.key?(platform_alias)
367
+ result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances
368
+ else
369
+ result[platform_alias] = { 'hostname': instances }
370
+ end
371
+ end
372
+ status 200
373
+ when 'failed'
374
+ result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'"
375
+ status 200
376
+ when 'deleted'
377
+ result['message'] = 'The request has been deleted'
378
+ status 200
379
+ else
380
+ Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
381
+ instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
382
+ instances_pending = count.to_i - instance_count.to_i
383
+
384
+ if result.key?(platform_alias) && result[platform_alias].key?(:ready)
385
+ result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s
386
+ result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s
387
+ else
388
+ result[platform_alias] = {
389
+ 'ready': instance_count.to_s,
390
+ 'pending': instances_pending.to_s
391
+ }
392
+ end
393
+ end
394
+ end
395
+
396
+ result
397
+ end
398
+ end
399
+
400
+ # Endpoints that only use bits from the V1 api are called here
401
+ # Note that traces will be named based on the route used in the V1 api
402
+ # but the http.url trace attribute will still have the actual requested url in it
403
+
404
+ delete "#{api_prefix}/*" do
405
+ versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
406
+ request.path_info = "/api/v1/#{versionless_path_info}"
407
+ call env
408
+ end
409
+
410
+ get "#{api_prefix}/*" do
411
+ versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
412
+ request.path_info = "/api/v1/#{versionless_path_info}"
413
+ call env
414
+ end
415
+
416
+ post "#{api_prefix}/*" do
417
+ versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
418
+ request.path_info = "/api/v1/#{versionless_path_info}"
419
+ call env
420
+ end
421
+
422
+ put "#{api_prefix}/*" do
423
+ versionless_path_info = request.path_info.delete_prefix("#{api_prefix}/")
424
+ request.path_info = "/api/v1/#{versionless_path_info}"
425
+ call env
426
+ end
427
+ end
428
+ end
429
+ end
data/lib/vmpooler/api.rb CHANGED
@@ -3,7 +3,7 @@
3
3
  module Vmpooler
4
4
  class API < Sinatra::Base
5
5
  # Load API components
6
- %w[helpers dashboard reroute v1 request_logger healthcheck].each do |lib|
6
+ %w[helpers dashboard reroute v1 v2 request_logger healthcheck].each do |lib|
7
7
  require "vmpooler/api/#{lib}"
8
8
  end
9
9
  # Load dashboard components
@@ -54,6 +54,7 @@ module Vmpooler
54
54
  use Vmpooler::API::Dashboard
55
55
  use Vmpooler::API::Reroute
56
56
  use Vmpooler::API::V1
57
+ use Vmpooler::API::V2
57
58
  end
58
59
 
59
60
  # Get thee started O WebServer