vmpooler 0.10.3 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/vmpooler +2 -3
- data/lib/vmpooler/api/dashboard.rb +12 -5
- data/lib/vmpooler/api/helpers.rb +47 -42
- data/lib/vmpooler/api/reroute.rb +2 -0
- data/lib/vmpooler/api/v1.rb +49 -49
- data/lib/vmpooler/api.rb +2 -0
- data/lib/vmpooler/dashboard.rb +2 -1
- data/lib/vmpooler/dummy_statsd.rb +2 -0
- data/lib/vmpooler/generic_connection_pool.rb +8 -6
- data/lib/vmpooler/graphite.rb +5 -5
- data/lib/vmpooler/logger.rb +3 -1
- data/lib/vmpooler/pool_manager.rb +133 -120
- data/lib/vmpooler/providers/base.rb +5 -3
- data/lib/vmpooler/providers/dummy.rb +16 -12
- data/lib/vmpooler/providers/vsphere.rb +114 -95
- data/lib/vmpooler/providers.rb +13 -15
- data/lib/vmpooler/statsd.rb +9 -9
- data/lib/vmpooler/version.rb +3 -1
- data/lib/vmpooler.rb +20 -11
- metadata +2 -2
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'vmpooler/providers'
|
2
4
|
require 'spicy-proton'
|
3
5
|
|
@@ -54,7 +56,7 @@ module Vmpooler
|
|
54
56
|
pool_keys.each do |k|
|
55
57
|
to_set[k] = pool[k]
|
56
58
|
end
|
57
|
-
to_set['alias'] = pool['alias'].join(',') if to_set.
|
59
|
+
to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
|
58
60
|
$redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
|
59
61
|
end
|
60
62
|
previously_configured_pools.each do |pool|
|
@@ -63,7 +65,7 @@ module Vmpooler
|
|
63
65
|
$redis.del("vmpooler__pool__#{pool}")
|
64
66
|
end
|
65
67
|
end
|
66
|
-
|
68
|
+
nil
|
67
69
|
end
|
68
70
|
|
69
71
|
# Check the state of a VM
|
@@ -71,8 +73,8 @@ module Vmpooler
|
|
71
73
|
Thread.new do
|
72
74
|
begin
|
73
75
|
_check_pending_vm(vm, pool, timeout, provider)
|
74
|
-
rescue =>
|
75
|
-
$logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{
|
76
|
+
rescue StandardError => e
|
77
|
+
$logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
|
76
78
|
fail_pending_vm(vm, pool, timeout)
|
77
79
|
raise
|
78
80
|
end
|
@@ -82,6 +84,7 @@ module Vmpooler
|
|
82
84
|
def _check_pending_vm(vm, pool, timeout, provider)
|
83
85
|
mutex = vm_mutex(vm)
|
84
86
|
return if mutex.locked?
|
87
|
+
|
85
88
|
mutex.synchronize do
|
86
89
|
if provider.vm_ready?(pool, vm)
|
87
90
|
move_pending_vm_to_ready(vm, pool)
|
@@ -111,14 +114,14 @@ module Vmpooler
|
|
111
114
|
end
|
112
115
|
end
|
113
116
|
true
|
114
|
-
rescue =>
|
115
|
-
$logger.log('d', "Fail pending VM failed with an error: #{
|
117
|
+
rescue StandardError => e
|
118
|
+
$logger.log('d', "Fail pending VM failed with an error: #{e}")
|
116
119
|
false
|
117
120
|
end
|
118
121
|
|
119
122
|
def move_pending_vm_to_ready(vm, pool)
|
120
123
|
clone_time = $redis.hget('vmpooler__vm__' + vm, 'clone')
|
121
|
-
finish = format('
|
124
|
+
finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time)) if clone_time
|
122
125
|
|
123
126
|
$redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
|
124
127
|
$redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
|
@@ -134,8 +137,9 @@ module Vmpooler
|
|
134
137
|
def vm_still_ready?(pool_name, vm_name, provider)
|
135
138
|
# Check if the VM is still ready/available
|
136
139
|
return true if provider.vm_ready?(pool_name, vm_name)
|
140
|
+
|
137
141
|
raise("VM #{vm_name} is not ready")
|
138
|
-
rescue
|
142
|
+
rescue StandardError
|
139
143
|
move_vm_queue(pool_name, vm_name, 'ready', 'completed', "is unreachable, removed from 'ready' queue")
|
140
144
|
end
|
141
145
|
|
@@ -143,8 +147,8 @@ module Vmpooler
|
|
143
147
|
Thread.new do
|
144
148
|
begin
|
145
149
|
_check_ready_vm(vm, pool_name, ttl, provider)
|
146
|
-
rescue =>
|
147
|
-
$logger.log('s', "[!] [#{pool_name}] '#{vm}' failed while checking a ready vm : #{
|
150
|
+
rescue StandardError => e
|
151
|
+
$logger.log('s', "[!] [#{pool_name}] '#{vm}' failed while checking a ready vm : #{e}")
|
148
152
|
raise
|
149
153
|
end
|
150
154
|
end
|
@@ -154,6 +158,7 @@ module Vmpooler
|
|
154
158
|
# Periodically check that the VM is available
|
155
159
|
mutex = vm_mutex(vm)
|
156
160
|
return if mutex.locked?
|
161
|
+
|
157
162
|
mutex.synchronize do
|
158
163
|
check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check')
|
159
164
|
return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime'])
|
@@ -176,13 +181,13 @@ module Vmpooler
|
|
176
181
|
end
|
177
182
|
end
|
178
183
|
|
179
|
-
return if
|
184
|
+
return if mismatched_hostname?(vm, pool_name, provider)
|
180
185
|
|
181
186
|
vm_still_ready?(pool_name, vm, provider)
|
182
187
|
end
|
183
188
|
end
|
184
189
|
|
185
|
-
def
|
190
|
+
def mismatched_hostname?(vm, pool_name, provider)
|
186
191
|
pool_config = $config[:pools][$config[:pool_index][pool_name]]
|
187
192
|
check_hostname = pool_config['check_hostname_for_mismatch']
|
188
193
|
check_hostname = $config[:config]['check_ready_vm_hostname_for_mismatch'] if check_hostname.nil?
|
@@ -201,22 +206,24 @@ module Vmpooler
|
|
201
206
|
# Check if the hostname has magically changed from underneath Pooler
|
202
207
|
vm_hash = provider.get_vm(pool_name, vm)
|
203
208
|
return unless vm_hash.is_a? Hash
|
209
|
+
|
204
210
|
hostname = vm_hash['hostname']
|
205
211
|
|
206
212
|
return if hostname.nil?
|
207
213
|
return if hostname.empty?
|
208
214
|
return if hostname == vm
|
215
|
+
|
209
216
|
$redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
|
210
217
|
$logger.log('d', "[!] [#{pool_name}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
|
211
|
-
|
218
|
+
true
|
212
219
|
end
|
213
220
|
|
214
221
|
def check_running_vm(vm, pool, ttl, provider)
|
215
222
|
Thread.new do
|
216
223
|
begin
|
217
224
|
_check_running_vm(vm, pool, ttl, provider)
|
218
|
-
rescue =>
|
219
|
-
$logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{
|
225
|
+
rescue StandardError => e
|
226
|
+
$logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{e}")
|
220
227
|
raise
|
221
228
|
end
|
222
229
|
end
|
@@ -225,6 +232,7 @@ module Vmpooler
|
|
225
232
|
def _check_running_vm(vm, pool, ttl, provider)
|
226
233
|
mutex = vm_mutex(vm)
|
227
234
|
return if mutex.locked?
|
235
|
+
|
228
236
|
mutex.synchronize do
|
229
237
|
# Check that VM is within defined lifetime
|
230
238
|
checkouttime = $redis.hget('vmpooler__active__' + pool, vm)
|
@@ -245,7 +253,7 @@ module Vmpooler
|
|
245
253
|
if host
|
246
254
|
return
|
247
255
|
else
|
248
|
-
move_vm_queue(pool, vm, 'running', 'completed',
|
256
|
+
move_vm_queue(pool, vm, 'running', 'completed', 'is no longer in inventory, removing from running')
|
249
257
|
end
|
250
258
|
end
|
251
259
|
end
|
@@ -261,14 +269,14 @@ module Vmpooler
|
|
261
269
|
Thread.new do
|
262
270
|
begin
|
263
271
|
_clone_vm(pool_name, provider)
|
264
|
-
rescue =>
|
265
|
-
$logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{
|
272
|
+
rescue StandardError => e
|
273
|
+
$logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
|
266
274
|
raise
|
267
275
|
end
|
268
276
|
end
|
269
277
|
end
|
270
278
|
|
271
|
-
def generate_and_check_hostname(
|
279
|
+
def generate_and_check_hostname(_pool_name)
|
272
280
|
# Generate a randomized hostname. The total name must no longer than 15
|
273
281
|
# character including the hyphen. The shortest adjective in the corpus is
|
274
282
|
# three characters long. Therefore, we can technically select a noun up to 11
|
@@ -278,12 +286,12 @@ module Vmpooler
|
|
278
286
|
# inviting more conflicts. We favor selecting a longer noun rather than a
|
279
287
|
# longer adjective because longer adjectives tend to be less fun.
|
280
288
|
noun = @name_generator.noun(max: 10)
|
281
|
-
adjective = @name_generator.adjective(max: 14-noun.length)
|
289
|
+
adjective = @name_generator.adjective(max: 14 - noun.length)
|
282
290
|
random_name = [adjective, noun].join('-')
|
283
291
|
hostname = $config[:config]['prefix'] + random_name
|
284
292
|
available = $redis.hlen('vmpooler__vm__' + hostname) == 0
|
285
293
|
|
286
|
-
|
294
|
+
[hostname, available]
|
287
295
|
end
|
288
296
|
|
289
297
|
def find_unique_hostname(pool_name)
|
@@ -297,9 +305,9 @@ module Vmpooler
|
|
297
305
|
$metrics.increment("errors.duplicatehostname.#{pool_name}")
|
298
306
|
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
|
299
307
|
end
|
300
|
-
|
308
|
+
|
301
309
|
raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless available
|
302
|
-
|
310
|
+
|
303
311
|
hostname
|
304
312
|
end
|
305
313
|
|
@@ -315,18 +323,18 @@ module Vmpooler
|
|
315
323
|
$logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
|
316
324
|
start = Time.now
|
317
325
|
provider.create_vm(pool_name, new_vmname)
|
318
|
-
finish = format('
|
326
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
319
327
|
|
320
328
|
$redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
|
321
329
|
$redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
|
322
330
|
$logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
|
323
331
|
|
324
332
|
$metrics.timing("clone.#{pool_name}", finish)
|
325
|
-
rescue
|
333
|
+
rescue StandardError
|
326
334
|
$redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
|
327
335
|
expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
|
328
336
|
$redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
|
329
|
-
raise
|
337
|
+
raise
|
330
338
|
ensure
|
331
339
|
$redis.decr('vmpooler__tasks__clone')
|
332
340
|
end
|
@@ -337,8 +345,8 @@ module Vmpooler
|
|
337
345
|
Thread.new do
|
338
346
|
begin
|
339
347
|
_destroy_vm(vm, pool, provider)
|
340
|
-
rescue =>
|
341
|
-
$logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{
|
348
|
+
rescue StandardError => e
|
349
|
+
$logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{e}")
|
342
350
|
raise
|
343
351
|
end
|
344
352
|
end
|
@@ -347,6 +355,7 @@ module Vmpooler
|
|
347
355
|
def _destroy_vm(vm, pool, provider)
|
348
356
|
mutex = vm_mutex(vm)
|
349
357
|
return if mutex.locked?
|
358
|
+
|
350
359
|
mutex.synchronize do
|
351
360
|
$redis.hdel('vmpooler__active__' + pool, vm)
|
352
361
|
$redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
|
@@ -360,7 +369,7 @@ module Vmpooler
|
|
360
369
|
|
361
370
|
$redis.srem('vmpooler__completed__' + pool, vm)
|
362
371
|
|
363
|
-
finish = format('
|
372
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
364
373
|
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
|
365
374
|
$metrics.timing("destroy.#{pool}", finish)
|
366
375
|
get_vm_usage_labels(vm)
|
@@ -370,11 +379,13 @@ module Vmpooler
|
|
370
379
|
|
371
380
|
def get_vm_usage_labels(vm)
|
372
381
|
return unless $config[:config]['usage_stats']
|
382
|
+
|
373
383
|
checkout = $redis.hget("vmpooler__vm__#{vm}", 'checkout')
|
374
384
|
return if checkout.nil?
|
385
|
+
|
375
386
|
jenkins_build_url = $redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
|
376
387
|
user = $redis.hget("vmpooler__vm__#{vm}", 'token:user') || 'unauthenticated'
|
377
|
-
poolname = $redis.hget("vmpooler__vm__#{vm}",
|
388
|
+
poolname = $redis.hget("vmpooler__vm__#{vm}", 'template')
|
378
389
|
|
379
390
|
unless jenkins_build_url
|
380
391
|
user = user.gsub('.', '_')
|
@@ -404,25 +415,24 @@ module Vmpooler
|
|
404
415
|
poolname
|
405
416
|
]
|
406
417
|
|
407
|
-
metric_parts = metric_parts.reject
|
418
|
+
metric_parts = metric_parts.reject(&:nil?)
|
408
419
|
metric_parts = metric_parts.map { |s| s.gsub('.', '_') }
|
409
420
|
|
410
421
|
$metrics.increment(metric_parts.join('.'))
|
411
|
-
rescue =>
|
412
|
-
logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{
|
422
|
+
rescue StandardError => e
|
423
|
+
logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
|
413
424
|
end
|
414
425
|
|
415
426
|
def component_to_test(match, labels_string)
|
416
427
|
return if labels_string.nil?
|
428
|
+
|
417
429
|
labels_string_parts = labels_string.split(',')
|
418
430
|
labels_string_parts.each do |part|
|
419
431
|
key, value = part.split('=')
|
420
432
|
next if value.nil?
|
421
|
-
if key == match
|
422
|
-
return value
|
423
|
-
end
|
433
|
+
return value if key == match
|
424
434
|
end
|
425
|
-
|
435
|
+
nil
|
426
436
|
end
|
427
437
|
|
428
438
|
def purge_unused_vms_and_folders
|
@@ -435,13 +445,13 @@ module Vmpooler
|
|
435
445
|
Thread.new do
|
436
446
|
begin
|
437
447
|
purge_vms_and_folders($providers[provider.to_s])
|
438
|
-
rescue =>
|
439
|
-
$logger.log('s', "[!] failed while purging provider #{provider
|
448
|
+
rescue StandardError => e
|
449
|
+
$logger.log('s', "[!] failed while purging provider #{provider} VMs and folders with an error: #{e}")
|
440
450
|
end
|
441
451
|
end
|
442
452
|
end
|
443
453
|
end
|
444
|
-
|
454
|
+
nil
|
445
455
|
end
|
446
456
|
|
447
457
|
# Return a list of pool folders
|
@@ -450,6 +460,7 @@ module Vmpooler
|
|
450
460
|
folders = {}
|
451
461
|
$config[:pools].each do |pool|
|
452
462
|
next unless pool['provider'] == provider_name
|
463
|
+
|
453
464
|
folder_parts = pool['folder'].split('/')
|
454
465
|
datacenter = provider.get_target_datacenter_from_config(pool['name'])
|
455
466
|
folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
|
@@ -459,7 +470,7 @@ module Vmpooler
|
|
459
470
|
|
460
471
|
def get_base_folders(folders)
|
461
472
|
base = []
|
462
|
-
folders.each do |
|
473
|
+
folders.each do |_key, value|
|
463
474
|
base << value
|
464
475
|
end
|
465
476
|
base.uniq
|
@@ -476,8 +487,8 @@ module Vmpooler
|
|
476
487
|
Thread.new do
|
477
488
|
begin
|
478
489
|
_create_vm_disk(pool_name, vm, disk_size, provider)
|
479
|
-
rescue =>
|
480
|
-
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating disk: #{
|
490
|
+
rescue StandardError => e
|
491
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating disk: #{e}")
|
481
492
|
raise
|
482
493
|
end
|
483
494
|
end
|
@@ -492,7 +503,7 @@ module Vmpooler
|
|
492
503
|
|
493
504
|
result = provider.create_disk(pool_name, vm_name, disk_size.to_i)
|
494
505
|
|
495
|
-
finish = format('
|
506
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
496
507
|
|
497
508
|
if result
|
498
509
|
rdisks = $redis.hget('vmpooler__vm__' + vm_name, 'disk')
|
@@ -512,8 +523,8 @@ module Vmpooler
|
|
512
523
|
Thread.new do
|
513
524
|
begin
|
514
525
|
_create_vm_snapshot(pool_name, vm, snapshot_name, provider)
|
515
|
-
rescue =>
|
516
|
-
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating snapshot: #{
|
526
|
+
rescue StandardError => e
|
527
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating snapshot: #{e}")
|
517
528
|
raise
|
518
529
|
end
|
519
530
|
end
|
@@ -525,7 +536,7 @@ module Vmpooler
|
|
525
536
|
|
526
537
|
result = provider.create_snapshot(pool_name, vm_name, snapshot_name)
|
527
538
|
|
528
|
-
finish = format('
|
539
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
529
540
|
|
530
541
|
if result
|
531
542
|
$redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
|
@@ -541,8 +552,8 @@ module Vmpooler
|
|
541
552
|
Thread.new do
|
542
553
|
begin
|
543
554
|
_revert_vm_snapshot(pool_name, vm, snapshot_name, provider)
|
544
|
-
rescue =>
|
545
|
-
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while reverting snapshot: #{
|
555
|
+
rescue StandardError => e
|
556
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while reverting snapshot: #{e}")
|
546
557
|
raise
|
547
558
|
end
|
548
559
|
end
|
@@ -554,7 +565,7 @@ module Vmpooler
|
|
554
565
|
|
555
566
|
result = provider.revert_snapshot(pool_name, vm_name, snapshot_name)
|
556
567
|
|
557
|
-
finish = format('
|
568
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
558
569
|
|
559
570
|
if result
|
560
571
|
$logger.log('s', "[+] [snapshot_manager] '#{vm_name}' reverted to snapshot '#{snapshot_name}' in #{finish} seconds")
|
@@ -574,14 +585,14 @@ module Vmpooler
|
|
574
585
|
# ie. ["vsphere", "dummy"]
|
575
586
|
def used_providers
|
576
587
|
pools = config[:pools] || []
|
577
|
-
@used_providers ||= (pools.map { |pool| pool[:provider] || pool['provider'] }.compact + default_providers
|
588
|
+
@used_providers ||= (pools.map { |pool| pool[:provider] || pool['provider'] }.compact + default_providers).uniq
|
578
589
|
end
|
579
590
|
|
580
591
|
# @return [Array] - returns a list of providers that should always be loaded
|
581
592
|
# note: vsphere is the default if user does not specify although this should not be
|
582
593
|
# if vsphere is to no longer be loaded by default please remove
|
583
594
|
def default_providers
|
584
|
-
@default_providers ||= %w
|
595
|
+
@default_providers ||= %w[vsphere dummy]
|
585
596
|
end
|
586
597
|
|
587
598
|
def get_pool_name_for_vm(vm_name)
|
@@ -592,8 +603,9 @@ module Vmpooler
|
|
592
603
|
# @param pool_name [String] - the name of the pool
|
593
604
|
# @return [Provider] - returns the provider class Object
|
594
605
|
def get_provider_for_pool(pool_name)
|
595
|
-
pool = $config[:pools].find { |
|
606
|
+
pool = $config[:pools].find { |p| p['name'] == pool_name }
|
596
607
|
return nil unless pool
|
608
|
+
|
597
609
|
provider_name = pool.fetch('provider', nil)
|
598
610
|
$providers[provider_name]
|
599
611
|
end
|
@@ -607,8 +619,9 @@ module Vmpooler
|
|
607
619
|
_check_disk_queue
|
608
620
|
sleep(loop_delay)
|
609
621
|
|
610
|
-
unless maxloop
|
622
|
+
unless maxloop == 0
|
611
623
|
break if loop_count >= maxloop
|
624
|
+
|
612
625
|
loop_count += 1
|
613
626
|
end
|
614
627
|
end
|
@@ -627,8 +640,8 @@ module Vmpooler
|
|
627
640
|
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
628
641
|
|
629
642
|
create_vm_disk(pool_name, vm_name, disk_size, provider)
|
630
|
-
rescue =>
|
631
|
-
$logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{
|
643
|
+
rescue StandardError => e
|
644
|
+
$logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{e}")
|
632
645
|
end
|
633
646
|
end
|
634
647
|
end
|
@@ -642,8 +655,9 @@ module Vmpooler
|
|
642
655
|
_check_snapshot_queue
|
643
656
|
sleep(loop_delay)
|
644
657
|
|
645
|
-
unless maxloop
|
658
|
+
unless maxloop == 0
|
646
659
|
break if loop_count >= maxloop
|
660
|
+
|
647
661
|
loop_count += 1
|
648
662
|
end
|
649
663
|
end
|
@@ -663,8 +677,8 @@ module Vmpooler
|
|
663
677
|
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
664
678
|
|
665
679
|
create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
|
666
|
-
rescue =>
|
667
|
-
$logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{
|
680
|
+
rescue StandardError => e
|
681
|
+
$logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{e}")
|
668
682
|
end
|
669
683
|
end
|
670
684
|
|
@@ -680,8 +694,8 @@ module Vmpooler
|
|
680
694
|
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
681
695
|
|
682
696
|
revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
|
683
|
-
rescue =>
|
684
|
-
$logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{
|
697
|
+
rescue StandardError => e
|
698
|
+
$logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{e}")
|
685
699
|
end
|
686
700
|
end
|
687
701
|
end
|
@@ -694,8 +708,8 @@ module Vmpooler
|
|
694
708
|
$redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
|
695
709
|
provider.migrate_vm(pool_name, vm_name)
|
696
710
|
end
|
697
|
-
rescue =>
|
698
|
-
$logger.log('s', "[x] [#{pool_name}] '#{vm_name}' migration failed with an error: #{
|
711
|
+
rescue StandardError => e
|
712
|
+
$logger.log('s', "[x] [#{pool_name}] '#{vm_name}' migration failed with an error: #{e}")
|
699
713
|
end
|
700
714
|
end
|
701
715
|
end
|
@@ -724,17 +738,11 @@ module Vmpooler
|
|
724
738
|
wakeup_by = Time.now + wakeup_period
|
725
739
|
return if time_passed?(:exit_by, exit_by)
|
726
740
|
|
727
|
-
if options[:pool_size_change]
|
728
|
-
initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
|
729
|
-
end
|
741
|
+
initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
|
730
742
|
|
731
|
-
if options[:clone_target_change]
|
732
|
-
initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target])
|
733
|
-
end
|
743
|
+
initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
|
734
744
|
|
735
|
-
if options[:pool_template_change]
|
736
|
-
initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname])
|
737
|
-
end
|
745
|
+
initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
|
738
746
|
|
739
747
|
loop do
|
740
748
|
sleep(1)
|
@@ -751,7 +759,7 @@ module Vmpooler
|
|
751
759
|
end
|
752
760
|
|
753
761
|
if options[:clone_target_change]
|
754
|
-
clone_target = $redis.hget(
|
762
|
+
clone_target = $redis.hget('vmpooler__config__clone_target}', options[:poolname])
|
755
763
|
if clone_target
|
756
764
|
break unless clone_target == initial_clone_target
|
757
765
|
end
|
@@ -795,6 +803,7 @@ module Vmpooler
|
|
795
803
|
loop_delay = loop_delay_min
|
796
804
|
provider = get_provider_for_pool(pool['name'])
|
797
805
|
raise("Could not find provider '#{pool['provider']}") if provider.nil?
|
806
|
+
|
798
807
|
sync_pool_template(pool)
|
799
808
|
loop do
|
800
809
|
result = _check_pool(pool, provider)
|
@@ -807,13 +816,14 @@ module Vmpooler
|
|
807
816
|
end
|
808
817
|
sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pool_reset: true)
|
809
818
|
|
810
|
-
unless maxloop
|
819
|
+
unless maxloop == 0
|
811
820
|
break if loop_count >= maxloop
|
821
|
+
|
812
822
|
loop_count += 1
|
813
823
|
end
|
814
824
|
end
|
815
|
-
rescue =>
|
816
|
-
$logger.log('s', "[!] [#{pool['name']}] Error while checking the pool: #{
|
825
|
+
rescue StandardError => e
|
826
|
+
$logger.log('s', "[!] [#{pool['name']}] Error while checking the pool: #{e}")
|
817
827
|
raise
|
818
828
|
end
|
819
829
|
end
|
@@ -828,19 +838,13 @@ module Vmpooler
|
|
828
838
|
end
|
829
839
|
|
830
840
|
def dereference_mutex(vmname)
|
831
|
-
if @vm_mutex.delete(vmname)
|
832
|
-
return true
|
833
|
-
else
|
834
|
-
return
|
835
|
-
end
|
841
|
+
true if @vm_mutex.delete(vmname)
|
836
842
|
end
|
837
843
|
|
838
844
|
def sync_pool_template(pool)
|
839
845
|
pool_template = $redis.hget('vmpooler__config__template', pool['name'])
|
840
846
|
if pool_template
|
841
|
-
unless pool['template'] == pool_template
|
842
|
-
pool['template'] = pool_template
|
843
|
-
end
|
847
|
+
pool['template'] = pool_template unless pool['template'] == pool_template
|
844
848
|
end
|
845
849
|
end
|
846
850
|
|
@@ -850,8 +854,8 @@ module Vmpooler
|
|
850
854
|
begin
|
851
855
|
provider.create_template_delta_disks(pool)
|
852
856
|
$redis.sadd('vmpooler__template__deltas', pool['template'])
|
853
|
-
rescue =>
|
854
|
-
$logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{
|
857
|
+
rescue StandardError => e
|
858
|
+
$logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{e}")
|
855
859
|
end
|
856
860
|
end
|
857
861
|
end
|
@@ -863,6 +867,7 @@ module Vmpooler
|
|
863
867
|
prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
|
864
868
|
configured_template = $redis.hget('vmpooler__config__template', pool['name'])
|
865
869
|
return if mutex.locked?
|
870
|
+
|
866
871
|
if prepared_template.nil?
|
867
872
|
mutex.synchronize do
|
868
873
|
prepare_template(pool, provider)
|
@@ -878,6 +883,7 @@ module Vmpooler
|
|
878
883
|
end
|
879
884
|
return if configured_template.nil?
|
880
885
|
return if configured_template == prepared_template
|
886
|
+
|
881
887
|
mutex.synchronize do
|
882
888
|
update_pool_template(pool, provider, configured_template, prepared_template)
|
883
889
|
end
|
@@ -913,9 +919,11 @@ module Vmpooler
|
|
913
919
|
def update_clone_target(pool)
|
914
920
|
mutex = pool_mutex(pool['name'])
|
915
921
|
return if mutex.locked?
|
922
|
+
|
916
923
|
clone_target = $redis.hget('vmpooler__config__clone_target', pool['name'])
|
917
924
|
return if clone_target.nil?
|
918
925
|
return if clone_target == pool['clone_target']
|
926
|
+
|
919
927
|
$logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
|
920
928
|
mutex.synchronize do
|
921
929
|
pool['clone_target'] = clone_target
|
@@ -930,9 +938,11 @@ module Vmpooler
|
|
930
938
|
total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
|
931
939
|
return if total.nil?
|
932
940
|
return if total == 0
|
941
|
+
|
933
942
|
mutex = pool_mutex(pool['name'])
|
934
943
|
return if mutex.locked?
|
935
944
|
return unless ready > pool['size']
|
945
|
+
|
936
946
|
mutex.synchronize do
|
937
947
|
difference = ready - pool['size']
|
938
948
|
difference.times do
|
@@ -950,10 +960,13 @@ module Vmpooler
|
|
950
960
|
def update_pool_size(pool)
|
951
961
|
mutex = pool_mutex(pool['name'])
|
952
962
|
return if mutex.locked?
|
963
|
+
|
953
964
|
poolsize = $redis.hget('vmpooler__config__poolsize', pool['name'])
|
954
965
|
return if poolsize.nil?
|
966
|
+
|
955
967
|
poolsize = Integer(poolsize)
|
956
968
|
return if poolsize == pool['size']
|
969
|
+
|
957
970
|
mutex.synchronize do
|
958
971
|
pool['size'] = poolsize
|
959
972
|
end
|
@@ -962,6 +975,7 @@ module Vmpooler
|
|
962
975
|
def reset_pool(pool)
|
963
976
|
poolname = pool['name']
|
964
977
|
return unless $redis.sismember('vmpooler__poolreset', poolname)
|
978
|
+
|
965
979
|
$redis.srem('vmpooler__poolreset', poolname)
|
966
980
|
mutex = pool_mutex(poolname)
|
967
981
|
mutex.synchronize do
|
@@ -992,9 +1006,9 @@ module Vmpooler
|
|
992
1006
|
inventory[vm['name']] = 1
|
993
1007
|
end
|
994
1008
|
end
|
995
|
-
rescue =>
|
996
|
-
$logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while running create_inventory: #{
|
997
|
-
raise(
|
1009
|
+
rescue StandardError => e
|
1010
|
+
$logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while running create_inventory: #{e}")
|
1011
|
+
raise(e)
|
998
1012
|
end
|
999
1013
|
inventory
|
1000
1014
|
end
|
@@ -1006,8 +1020,8 @@ module Vmpooler
|
|
1006
1020
|
vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
|
1007
1021
|
pool_check_response[:checked_running_vms] += 1
|
1008
1022
|
check_running_vm(vm, pool_name, vm_lifetime, provider)
|
1009
|
-
rescue =>
|
1010
|
-
$logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{
|
1023
|
+
rescue StandardError => e
|
1024
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{e}")
|
1011
1025
|
end
|
1012
1026
|
else
|
1013
1027
|
move_vm_queue(pool_name, vm, 'running', 'completed', 'is a running VM but is missing from inventory. Marking as completed.')
|
@@ -1021,8 +1035,8 @@ module Vmpooler
|
|
1021
1035
|
begin
|
1022
1036
|
pool_check_response[:checked_ready_vms] += 1
|
1023
1037
|
check_ready_vm(vm, pool_name, pool_ttl || 0, provider)
|
1024
|
-
rescue =>
|
1025
|
-
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{
|
1038
|
+
rescue StandardError => e
|
1039
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
|
1026
1040
|
end
|
1027
1041
|
else
|
1028
1042
|
move_vm_queue(pool_name, vm, 'ready', 'completed', 'is a ready VM but is missing from inventory. Marking as completed.')
|
@@ -1037,8 +1051,8 @@ module Vmpooler
|
|
1037
1051
|
begin
|
1038
1052
|
pool_check_response[:checked_pending_vms] += 1
|
1039
1053
|
check_pending_vm(vm, pool_name, pool_timeout, provider)
|
1040
|
-
rescue =>
|
1041
|
-
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{
|
1054
|
+
rescue StandardError => e
|
1055
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
|
1042
1056
|
end
|
1043
1057
|
else
|
1044
1058
|
fail_pending_vm(vm, pool_name, pool_timeout, false)
|
@@ -1052,11 +1066,11 @@ module Vmpooler
|
|
1052
1066
|
begin
|
1053
1067
|
pool_check_response[:destroyed_vms] += 1
|
1054
1068
|
destroy_vm(vm, pool_name, provider)
|
1055
|
-
rescue =>
|
1069
|
+
rescue StandardError => e
|
1056
1070
|
$redis.srem("vmpooler__completed__#{pool_name}", vm)
|
1057
1071
|
$redis.hdel("vmpooler__active__#{pool_name}", vm)
|
1058
1072
|
$redis.del("vmpooler__vm__#{vm}")
|
1059
|
-
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{
|
1073
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
|
1060
1074
|
end
|
1061
1075
|
else
|
1062
1076
|
$logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
@@ -1068,22 +1082,18 @@ module Vmpooler
|
|
1068
1082
|
end
|
1069
1083
|
|
1070
1084
|
def check_discovered_pool_vms(pool_name)
|
1071
|
-
|
1072
|
-
|
1073
|
-
|
1074
|
-
|
1075
|
-
|
1076
|
-
$redis.srem("vmpooler__discovered__#{pool_name}", vm)
|
1077
|
-
end
|
1078
|
-
end
|
1079
|
-
|
1080
|
-
if $redis.sismember("vmpooler__discovered__#{pool_name}", vm)
|
1081
|
-
$redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm)
|
1085
|
+
$redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
|
1086
|
+
%w[pending ready running completed].each do |queue|
|
1087
|
+
if $redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
|
1088
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
|
1089
|
+
$redis.srem("vmpooler__discovered__#{pool_name}", vm)
|
1082
1090
|
end
|
1083
1091
|
end
|
1084
|
-
|
1085
|
-
$
|
1092
|
+
|
1093
|
+
$redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm) if $redis.sismember("vmpooler__discovered__#{pool_name}", vm)
|
1086
1094
|
end
|
1095
|
+
rescue StandardError => e
|
1096
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating discovered VMs: #{e}")
|
1087
1097
|
end
|
1088
1098
|
|
1089
1099
|
def check_migrating_pool_vms(pool_name, provider, pool_check_response, inventory)
|
@@ -1092,8 +1102,8 @@ module Vmpooler
|
|
1092
1102
|
begin
|
1093
1103
|
pool_check_response[:migrated_vms] += 1
|
1094
1104
|
migrate_vm(vm, pool_name, provider)
|
1095
|
-
rescue =>
|
1096
|
-
$logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{
|
1105
|
+
rescue StandardError => e
|
1106
|
+
$logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{e}")
|
1097
1107
|
end
|
1098
1108
|
end
|
1099
1109
|
end
|
@@ -1101,6 +1111,7 @@ module Vmpooler
|
|
1101
1111
|
|
1102
1112
|
def repopulate_pool_vms(pool_name, provider, pool_check_response, pool_size)
|
1103
1113
|
return if pool_mutex(pool_name).locked?
|
1114
|
+
|
1104
1115
|
ready = $redis.scard("vmpooler__ready__#{pool_name}")
|
1105
1116
|
total = $redis.scard("vmpooler__pending__#{pool_name}") + ready
|
1106
1117
|
|
@@ -1108,8 +1119,8 @@ module Vmpooler
|
|
1108
1119
|
$metrics.gauge("running.#{pool_name}", $redis.scard("vmpooler__running__#{pool_name}"))
|
1109
1120
|
|
1110
1121
|
if $redis.get("vmpooler__empty__#{pool_name}")
|
1111
|
-
$redis.del("vmpooler__empty__#{pool_name}") unless ready
|
1112
|
-
elsif ready
|
1122
|
+
$redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
|
1123
|
+
elsif ready == 0
|
1113
1124
|
$redis.set("vmpooler__empty__#{pool_name}", 'true')
|
1114
1125
|
$logger.log('s', "[!] [#{pool_name}] is empty")
|
1115
1126
|
end
|
@@ -1120,8 +1131,8 @@ module Vmpooler
|
|
1120
1131
|
$redis.incr('vmpooler__tasks__clone')
|
1121
1132
|
pool_check_response[:cloned_vms] += 1
|
1122
1133
|
clone_vm(pool_name, provider)
|
1123
|
-
rescue =>
|
1124
|
-
$logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{
|
1134
|
+
rescue StandardError => e
|
1135
|
+
$logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
|
1125
1136
|
$redis.decr('vmpooler__tasks__clone')
|
1126
1137
|
raise
|
1127
1138
|
end
|
@@ -1142,7 +1153,7 @@ module Vmpooler
|
|
1142
1153
|
|
1143
1154
|
begin
|
1144
1155
|
inventory = create_inventory(pool, provider, pool_check_response)
|
1145
|
-
rescue
|
1156
|
+
rescue StandardError
|
1146
1157
|
return(pool_check_response)
|
1147
1158
|
end
|
1148
1159
|
|
@@ -1195,6 +1206,7 @@ module Vmpooler
|
|
1195
1206
|
provider_klass = Vmpooler::PoolManager::Provider
|
1196
1207
|
provider_klass.constants.each do |classname|
|
1197
1208
|
next unless classname.to_s.casecmp(provider_class) == 0
|
1209
|
+
|
1198
1210
|
return provider_klass.const_get(classname).new(config, logger, metrics, provider_name, options)
|
1199
1211
|
end
|
1200
1212
|
raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
|
@@ -1257,8 +1269,8 @@ module Vmpooler
|
|
1257
1269
|
end
|
1258
1270
|
begin
|
1259
1271
|
$providers[provider_name] = create_provider_object($config, $logger, $metrics, provider_class, provider_name, {}) if $providers[provider_name].nil?
|
1260
|
-
rescue =>
|
1261
|
-
$logger.log('s', "Error while creating provider for pool #{pool['name']}: #{
|
1272
|
+
rescue StandardError => e
|
1273
|
+
$logger.log('s', "Error while creating provider for pool #{pool['name']}: #{e}")
|
1262
1274
|
raise
|
1263
1275
|
end
|
1264
1276
|
end
|
@@ -1292,8 +1304,9 @@ module Vmpooler
|
|
1292
1304
|
|
1293
1305
|
sleep(loop_delay)
|
1294
1306
|
|
1295
|
-
unless maxloop
|
1307
|
+
unless maxloop == 0
|
1296
1308
|
break if loop_count >= maxloop
|
1309
|
+
|
1297
1310
|
loop_count += 1
|
1298
1311
|
end
|
1299
1312
|
end
|