vmpooler 2.2.0 → 2.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/vmpooler/api/helpers.rb +11 -9
- data/lib/vmpooler/api/reroute.rb +16 -0
- data/lib/vmpooler/api/v1.rb +18 -0
- data/lib/vmpooler/api/v2.rb +429 -0
- data/lib/vmpooler/api.rb +2 -1
- data/lib/vmpooler/pool_manager.rb +70 -55
- data/lib/vmpooler/util/parsing.rb +21 -1
- data/lib/vmpooler/version.rb +1 -1
- data/lib/vmpooler.rb +4 -0
- metadata +51 -37
@@ -60,6 +60,7 @@ module Vmpooler
|
|
60
60
|
to_set[k] = pool[k]
|
61
61
|
end
|
62
62
|
to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
|
63
|
+
to_set['domain'] = Parsing.get_domain_for_pool(config, pool['name'])
|
63
64
|
redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
|
64
65
|
end
|
65
66
|
previously_configured_pools.each do |pool|
|
@@ -148,15 +149,15 @@ module Vmpooler
|
|
148
149
|
end
|
149
150
|
pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
|
150
151
|
|
151
|
-
redis.pipelined do
|
152
|
-
|
153
|
-
|
152
|
+
redis.pipelined do |pipeline|
|
153
|
+
pipeline.hset("vmpooler__active__#{pool}", vm, Time.now)
|
154
|
+
pipeline.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
|
154
155
|
if ondemandrequest_hash['token:token']
|
155
|
-
|
156
|
-
|
157
|
-
|
156
|
+
pipeline.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token'])
|
157
|
+
pipeline.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user'])
|
158
|
+
pipeline.hset("vmpooler__vm__#{vm}", 'lifetime', $config[:config]['vm_lifetime_auth'].to_i)
|
158
159
|
end
|
159
|
-
|
160
|
+
pipeline.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
|
160
161
|
end
|
161
162
|
move_vm_queue(pool, vm, 'pending', 'running', redis)
|
162
163
|
check_ondemand_request_ready(request_id, redis)
|
@@ -164,12 +165,12 @@ module Vmpooler
|
|
164
165
|
redis.smove("vmpooler__pending__#{pool}", "vmpooler__ready__#{pool}", vm)
|
165
166
|
end
|
166
167
|
|
167
|
-
redis.pipelined do
|
168
|
-
|
169
|
-
|
168
|
+
redis.pipelined do |pipeline|
|
169
|
+
pipeline.hset("vmpooler__boot__#{Date.today}", "#{pool}:#{vm}", finish) # maybe remove as this is never used by vmpooler itself?
|
170
|
+
pipeline.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
|
170
171
|
|
171
172
|
# last boot time is displayed in API, and used by alarming script
|
172
|
-
|
173
|
+
pipeline.hset('vmpooler__lastboot', pool, Time.now)
|
173
174
|
end
|
174
175
|
|
175
176
|
$metrics.timing("time_to_ready_state.#{pool}", finish)
|
@@ -361,35 +362,47 @@ module Vmpooler
|
|
361
362
|
max_hostname_retries = 3
|
362
363
|
while hostname_retries < max_hostname_retries
|
363
364
|
hostname, hostname_available = generate_and_check_hostname
|
364
|
-
domain =
|
365
|
-
|
365
|
+
domain = Parsing.get_domain_for_pool(config, pool_name)
|
366
|
+
if domain
|
367
|
+
fqdn = "#{hostname}.#{domain}"
|
368
|
+
else
|
369
|
+
fqdn = hostname
|
370
|
+
end
|
371
|
+
|
372
|
+
# skip dns check if the provider is set to skip_dns_check_before_creating_vm
|
373
|
+
provider = get_provider_for_pool(pool_name)
|
374
|
+
if provider && provider.provider_config['skip_dns_check_before_creating_vm']
|
375
|
+
dns_available = true
|
376
|
+
else
|
377
|
+
dns_ip, dns_available = check_dns_available(fqdn)
|
378
|
+
end
|
379
|
+
|
366
380
|
break if hostname_available && dns_available
|
367
381
|
|
368
382
|
hostname_retries += 1
|
369
383
|
|
370
384
|
if !hostname_available
|
371
385
|
$metrics.increment("errors.duplicatehostname.#{pool_name}")
|
372
|
-
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{
|
386
|
+
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
|
373
387
|
elsif !dns_available
|
374
388
|
$metrics.increment("errors.staledns.#{pool_name}")
|
375
|
-
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{
|
389
|
+
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} already exists in DNS records (#{dns_ip}), stale DNS")
|
376
390
|
end
|
377
391
|
end
|
378
392
|
|
379
|
-
raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{
|
393
|
+
raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{fqdn}" unless hostname_available && dns_available
|
380
394
|
|
381
395
|
hostname
|
382
396
|
end
|
383
397
|
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
vm_name = "#{vm_name}.#{domain}" if domain
|
398
|
+
# Query the DNS for the name we want to create and if it already exists, mark it unavailable
|
399
|
+
# This protects against stale DNS records
|
400
|
+
def check_dns_available(vm_name)
|
388
401
|
begin
|
389
402
|
dns_ip = Resolv.getaddress(vm_name)
|
390
403
|
rescue Resolv::ResolvError
|
391
404
|
# this is the expected case, swallow the error
|
392
|
-
# eg "no address for blah-daisy"
|
405
|
+
# eg "no address for blah-daisy.example.com"
|
393
406
|
return ['', true]
|
394
407
|
end
|
395
408
|
[dns_ip, false]
|
@@ -397,6 +410,7 @@ module Vmpooler
|
|
397
410
|
|
398
411
|
def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
|
399
412
|
new_vmname = find_unique_hostname(pool_name)
|
413
|
+
pool_domain = Parsing.get_domain_for_pool(config, pool_name)
|
400
414
|
mutex = vm_mutex(new_vmname)
|
401
415
|
mutex.synchronize do
|
402
416
|
@redis.with_metrics do |redis|
|
@@ -406,6 +420,7 @@ module Vmpooler
|
|
406
420
|
redis.hset("vmpooler__vm__#{new_vmname}", 'clone', Time.now)
|
407
421
|
redis.hset("vmpooler__vm__#{new_vmname}", 'template', pool_name) # This value is used to represent the pool.
|
408
422
|
redis.hset("vmpooler__vm__#{new_vmname}", 'pool', pool_name)
|
423
|
+
redis.hset("vmpooler__vm__#{new_vmname}", 'domain', pool_domain) if pool_domain
|
409
424
|
redis.hset("vmpooler__vm__#{new_vmname}", 'request_id', request_id) if request_id
|
410
425
|
redis.hset("vmpooler__vm__#{new_vmname}", 'pool_alias', pool_alias) if pool_alias
|
411
426
|
redis.exec
|
@@ -418,9 +433,9 @@ module Vmpooler
|
|
418
433
|
finish = format('%<time>.2f', time: Time.now - start)
|
419
434
|
|
420
435
|
@redis.with_metrics do |redis|
|
421
|
-
redis.pipelined do
|
422
|
-
|
423
|
-
|
436
|
+
redis.pipelined do |pipeline|
|
437
|
+
pipeline.hset("vmpooler__clone__#{Date.today}", "#{pool_name}:#{new_vmname}", finish)
|
438
|
+
pipeline.hset("vmpooler__vm__#{new_vmname}", 'clone_time', finish)
|
424
439
|
end
|
425
440
|
end
|
426
441
|
$logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
|
@@ -428,10 +443,10 @@ module Vmpooler
|
|
428
443
|
$metrics.timing("clone.#{pool_name}", finish)
|
429
444
|
rescue StandardError
|
430
445
|
@redis.with_metrics do |redis|
|
431
|
-
redis.pipelined do
|
432
|
-
|
446
|
+
redis.pipelined do |pipeline|
|
447
|
+
pipeline.srem("vmpooler__pending__#{pool_name}", new_vmname)
|
433
448
|
expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
|
434
|
-
|
449
|
+
pipeline.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
|
435
450
|
end
|
436
451
|
end
|
437
452
|
raise
|
@@ -462,12 +477,12 @@ module Vmpooler
|
|
462
477
|
|
463
478
|
mutex.synchronize do
|
464
479
|
@redis.with_metrics do |redis|
|
465
|
-
redis.pipelined do
|
466
|
-
|
467
|
-
|
480
|
+
redis.pipelined do |pipeline|
|
481
|
+
pipeline.hdel("vmpooler__active__#{pool}", vm)
|
482
|
+
pipeline.hset("vmpooler__vm__#{vm}", 'destroy', Time.now)
|
468
483
|
|
469
484
|
# Auto-expire metadata key
|
470
|
-
|
485
|
+
pipeline.expire("vmpooler__vm__#{vm}", ($config[:redis]['data_ttl'].to_i * 60 * 60))
|
471
486
|
end
|
472
487
|
|
473
488
|
start = Time.now
|
@@ -879,7 +894,7 @@ module Vmpooler
|
|
879
894
|
loop_count = 1
|
880
895
|
loop_delay = loop_delay_min
|
881
896
|
provider = get_provider_for_pool(pool['name'])
|
882
|
-
raise("Could not find provider '#{pool['provider']}") if provider.nil?
|
897
|
+
raise("Could not find provider '#{pool['provider']}'") if provider.nil?
|
883
898
|
|
884
899
|
sync_pool_template(pool)
|
885
900
|
loop do
|
@@ -1204,19 +1219,19 @@ module Vmpooler
|
|
1204
1219
|
pool_check_response[:destroyed_vms] += 1
|
1205
1220
|
destroy_vm(vm, pool_name, provider)
|
1206
1221
|
rescue StandardError => e
|
1207
|
-
redis.pipelined do
|
1208
|
-
|
1209
|
-
|
1210
|
-
|
1222
|
+
redis.pipelined do |pipeline|
|
1223
|
+
pipeline.srem("vmpooler__completed__#{pool_name}", vm)
|
1224
|
+
pipeline.hdel("vmpooler__active__#{pool_name}", vm)
|
1225
|
+
pipeline.del("vmpooler__vm__#{vm}")
|
1211
1226
|
end
|
1212
1227
|
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
|
1213
1228
|
end
|
1214
1229
|
else
|
1215
1230
|
$logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
1216
|
-
redis.pipelined do
|
1217
|
-
|
1218
|
-
|
1219
|
-
|
1231
|
+
redis.pipelined do |pipeline|
|
1232
|
+
pipeline.srem("vmpooler__completed__#{pool_name}", vm)
|
1233
|
+
pipeline.hdel("vmpooler__active__#{pool_name}", vm)
|
1234
|
+
pipeline.del("vmpooler__vm__#{vm}")
|
1220
1235
|
end
|
1221
1236
|
end
|
1222
1237
|
end
|
@@ -1366,7 +1381,7 @@ module Vmpooler
|
|
1366
1381
|
|
1367
1382
|
return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options)
|
1368
1383
|
end
|
1369
|
-
raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if
|
1384
|
+
raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider_klass.nil?
|
1370
1385
|
end
|
1371
1386
|
|
1372
1387
|
def check_ondemand_requests(maxloop = 0,
|
@@ -1432,12 +1447,12 @@ module Vmpooler
|
|
1432
1447
|
score = redis.zscore('vmpooler__provisioning__request', request_id)
|
1433
1448
|
requested = requested.split(',')
|
1434
1449
|
|
1435
|
-
redis.pipelined do
|
1450
|
+
redis.pipelined do |pipeline|
|
1436
1451
|
requested.each do |request|
|
1437
|
-
|
1452
|
+
pipeline.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
|
1438
1453
|
end
|
1439
|
-
|
1440
|
-
|
1454
|
+
pipeline.zrem('vmpooler__provisioning__request', request_id)
|
1455
|
+
pipeline.zadd('vmpooler__provisioning__processing', score, request_id)
|
1441
1456
|
end
|
1442
1457
|
end
|
1443
1458
|
|
@@ -1467,9 +1482,9 @@ module Vmpooler
|
|
1467
1482
|
redis.incr('vmpooler__tasks__ondemandclone')
|
1468
1483
|
clone_vm(pool, provider, request_id, pool_alias)
|
1469
1484
|
end
|
1470
|
-
redis.pipelined do
|
1471
|
-
|
1472
|
-
|
1485
|
+
redis.pipelined do |pipeline|
|
1486
|
+
pipeline.zrem(queue_key, request)
|
1487
|
+
pipeline.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
|
1473
1488
|
end
|
1474
1489
|
end
|
1475
1490
|
end
|
@@ -1520,10 +1535,10 @@ module Vmpooler
|
|
1520
1535
|
|
1521
1536
|
$logger.log('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
|
1522
1537
|
expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
|
1523
|
-
redis.pipelined do
|
1524
|
-
|
1525
|
-
|
1526
|
-
|
1538
|
+
redis.pipelined do |pipeline|
|
1539
|
+
pipeline.zrem('vmpooler__provisioning__processing', request_id)
|
1540
|
+
pipeline.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
|
1541
|
+
pipeline.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
|
1527
1542
|
end
|
1528
1543
|
remove_vms_for_failed_request(request_id, expiration_ttl, redis)
|
1529
1544
|
true
|
@@ -1533,11 +1548,11 @@ module Vmpooler
|
|
1533
1548
|
request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
1534
1549
|
Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
|
1535
1550
|
pools_filled = redis.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
1536
|
-
redis.pipelined do
|
1551
|
+
redis.pipelined do |pipeline|
|
1537
1552
|
pools_filled&.each do |vm|
|
1538
|
-
move_vm_queue(pool, vm, 'running', 'completed',
|
1553
|
+
move_vm_queue(pool, vm, 'running', 'completed', pipeline, "moved to completed queue. '#{request_id}' could not be filled in time")
|
1539
1554
|
end
|
1540
|
-
|
1555
|
+
pipeline.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
|
1541
1556
|
end
|
1542
1557
|
end
|
1543
1558
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
# utility class shared between apps
|
3
|
+
# utility class shared between apps api and pool_manager
|
4
4
|
module Vmpooler
|
5
5
|
class Parsing
|
6
6
|
def self.get_platform_pool_count(requested, &_block)
|
@@ -12,5 +12,25 @@ module Vmpooler
|
|
12
12
|
yield platform_alias, pool, count
|
13
13
|
end
|
14
14
|
end
|
15
|
+
|
16
|
+
# @param config [String] - the full config structure
|
17
|
+
# @param pool_name [String] - the name of the pool
|
18
|
+
# @return [String] - domain name for pool, if set in the provider for the pool or in the config block
|
19
|
+
def self.get_domain_for_pool(config, pool_name)
|
20
|
+
pool = config[:pools].find { |p| p['name'] == pool_name }
|
21
|
+
return nil unless pool
|
22
|
+
|
23
|
+
provider_name = pool.fetch('provider', 'vsphere') # see vmpooler.yaml.example where it states defaulting to vsphere
|
24
|
+
|
25
|
+
if config[:providers] && config[:providers][provider_name.to_sym] && config[:providers][provider_name.to_sym]['domain']
|
26
|
+
domain = config[:providers][provider_name.to_sym]['domain']
|
27
|
+
elsif config[:config] && config[:config]['domain']
|
28
|
+
domain = config[:config]['domain']
|
29
|
+
else
|
30
|
+
domain = nil
|
31
|
+
end
|
32
|
+
|
33
|
+
domain
|
34
|
+
end
|
15
35
|
end
|
16
36
|
end
|
data/lib/vmpooler/version.rb
CHANGED
data/lib/vmpooler.rb
CHANGED
@@ -17,6 +17,7 @@ module Vmpooler
|
|
17
17
|
|
18
18
|
# Dependencies for tracing
|
19
19
|
require 'opentelemetry-instrumentation-concurrent_ruby'
|
20
|
+
require 'opentelemetry-instrumentation-http_client'
|
20
21
|
require 'opentelemetry-instrumentation-redis'
|
21
22
|
require 'opentelemetry-instrumentation-sinatra'
|
22
23
|
require 'opentelemetry-sdk'
|
@@ -42,6 +43,7 @@ module Vmpooler
|
|
42
43
|
if parsed_config[:config]['extra_config']
|
43
44
|
extra_configs = parsed_config[:config]['extra_config'].split(',')
|
44
45
|
extra_configs.each do |config|
|
46
|
+
puts "loading extra_config file #{config}"
|
45
47
|
extra_config = YAML.load_file(config)
|
46
48
|
parsed_config.deep_merge(extra_config)
|
47
49
|
end
|
@@ -132,6 +134,7 @@ module Vmpooler
|
|
132
134
|
# Create an index of pool aliases
|
133
135
|
parsed_config[:pool_names] = Set.new
|
134
136
|
unless parsed_config[:pools]
|
137
|
+
puts 'loading pools configuration from redis, since the config[:pools] is empty'
|
135
138
|
redis = new_redis(parsed_config[:redis]['server'], parsed_config[:redis]['port'], parsed_config[:redis]['password'])
|
136
139
|
parsed_config[:pools] = load_pools_from_redis(redis)
|
137
140
|
end
|
@@ -265,6 +268,7 @@ module Vmpooler
|
|
265
268
|
OpenTelemetry::SDK.configure do |c|
|
266
269
|
c.use 'OpenTelemetry::Instrumentation::Sinatra'
|
267
270
|
c.use 'OpenTelemetry::Instrumentation::ConcurrentRuby'
|
271
|
+
c.use 'OpenTelemetry::Instrumentation::HttpClient'
|
268
272
|
c.use 'OpenTelemetry::Instrumentation::Redis'
|
269
273
|
|
270
274
|
c.add_span_processor(span_processor)
|