vmpooler 0.10.3 → 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/vmpooler +2 -3
- data/lib/vmpooler/api/dashboard.rb +12 -5
- data/lib/vmpooler/api/helpers.rb +47 -42
- data/lib/vmpooler/api/reroute.rb +2 -0
- data/lib/vmpooler/api/v1.rb +49 -49
- data/lib/vmpooler/api.rb +2 -0
- data/lib/vmpooler/dashboard.rb +2 -1
- data/lib/vmpooler/dummy_statsd.rb +2 -0
- data/lib/vmpooler/generic_connection_pool.rb +8 -6
- data/lib/vmpooler/graphite.rb +5 -5
- data/lib/vmpooler/logger.rb +3 -1
- data/lib/vmpooler/pool_manager.rb +133 -120
- data/lib/vmpooler/providers/base.rb +5 -3
- data/lib/vmpooler/providers/dummy.rb +16 -12
- data/lib/vmpooler/providers/vsphere.rb +114 -95
- data/lib/vmpooler/providers.rb +13 -15
- data/lib/vmpooler/statsd.rb +9 -9
- data/lib/vmpooler/version.rb +3 -1
- data/lib/vmpooler.rb +20 -11
- metadata +2 -2
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'vmpooler/providers'
|
2
4
|
require 'spicy-proton'
|
3
5
|
|
@@ -54,7 +56,7 @@ module Vmpooler
|
|
54
56
|
pool_keys.each do |k|
|
55
57
|
to_set[k] = pool[k]
|
56
58
|
end
|
57
|
-
to_set['alias'] = pool['alias'].join(',') if to_set.
|
59
|
+
to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
|
58
60
|
$redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
|
59
61
|
end
|
60
62
|
previously_configured_pools.each do |pool|
|
@@ -63,7 +65,7 @@ module Vmpooler
|
|
63
65
|
$redis.del("vmpooler__pool__#{pool}")
|
64
66
|
end
|
65
67
|
end
|
66
|
-
|
68
|
+
nil
|
67
69
|
end
|
68
70
|
|
69
71
|
# Check the state of a VM
|
@@ -71,8 +73,8 @@ module Vmpooler
|
|
71
73
|
Thread.new do
|
72
74
|
begin
|
73
75
|
_check_pending_vm(vm, pool, timeout, provider)
|
74
|
-
rescue =>
|
75
|
-
$logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{
|
76
|
+
rescue StandardError => e
|
77
|
+
$logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
|
76
78
|
fail_pending_vm(vm, pool, timeout)
|
77
79
|
raise
|
78
80
|
end
|
@@ -82,6 +84,7 @@ module Vmpooler
|
|
82
84
|
def _check_pending_vm(vm, pool, timeout, provider)
|
83
85
|
mutex = vm_mutex(vm)
|
84
86
|
return if mutex.locked?
|
87
|
+
|
85
88
|
mutex.synchronize do
|
86
89
|
if provider.vm_ready?(pool, vm)
|
87
90
|
move_pending_vm_to_ready(vm, pool)
|
@@ -111,14 +114,14 @@ module Vmpooler
|
|
111
114
|
end
|
112
115
|
end
|
113
116
|
true
|
114
|
-
rescue =>
|
115
|
-
$logger.log('d', "Fail pending VM failed with an error: #{
|
117
|
+
rescue StandardError => e
|
118
|
+
$logger.log('d', "Fail pending VM failed with an error: #{e}")
|
116
119
|
false
|
117
120
|
end
|
118
121
|
|
119
122
|
def move_pending_vm_to_ready(vm, pool)
|
120
123
|
clone_time = $redis.hget('vmpooler__vm__' + vm, 'clone')
|
121
|
-
finish = format('
|
124
|
+
finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time)) if clone_time
|
122
125
|
|
123
126
|
$redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
|
124
127
|
$redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
|
@@ -134,8 +137,9 @@ module Vmpooler
|
|
134
137
|
def vm_still_ready?(pool_name, vm_name, provider)
|
135
138
|
# Check if the VM is still ready/available
|
136
139
|
return true if provider.vm_ready?(pool_name, vm_name)
|
140
|
+
|
137
141
|
raise("VM #{vm_name} is not ready")
|
138
|
-
rescue
|
142
|
+
rescue StandardError
|
139
143
|
move_vm_queue(pool_name, vm_name, 'ready', 'completed', "is unreachable, removed from 'ready' queue")
|
140
144
|
end
|
141
145
|
|
@@ -143,8 +147,8 @@ module Vmpooler
|
|
143
147
|
Thread.new do
|
144
148
|
begin
|
145
149
|
_check_ready_vm(vm, pool_name, ttl, provider)
|
146
|
-
rescue =>
|
147
|
-
$logger.log('s', "[!] [#{pool_name}] '#{vm}' failed while checking a ready vm : #{
|
150
|
+
rescue StandardError => e
|
151
|
+
$logger.log('s', "[!] [#{pool_name}] '#{vm}' failed while checking a ready vm : #{e}")
|
148
152
|
raise
|
149
153
|
end
|
150
154
|
end
|
@@ -154,6 +158,7 @@ module Vmpooler
|
|
154
158
|
# Periodically check that the VM is available
|
155
159
|
mutex = vm_mutex(vm)
|
156
160
|
return if mutex.locked?
|
161
|
+
|
157
162
|
mutex.synchronize do
|
158
163
|
check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check')
|
159
164
|
return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime'])
|
@@ -176,13 +181,13 @@ module Vmpooler
|
|
176
181
|
end
|
177
182
|
end
|
178
183
|
|
179
|
-
return if
|
184
|
+
return if mismatched_hostname?(vm, pool_name, provider)
|
180
185
|
|
181
186
|
vm_still_ready?(pool_name, vm, provider)
|
182
187
|
end
|
183
188
|
end
|
184
189
|
|
185
|
-
def
|
190
|
+
def mismatched_hostname?(vm, pool_name, provider)
|
186
191
|
pool_config = $config[:pools][$config[:pool_index][pool_name]]
|
187
192
|
check_hostname = pool_config['check_hostname_for_mismatch']
|
188
193
|
check_hostname = $config[:config]['check_ready_vm_hostname_for_mismatch'] if check_hostname.nil?
|
@@ -201,22 +206,24 @@ module Vmpooler
|
|
201
206
|
# Check if the hostname has magically changed from underneath Pooler
|
202
207
|
vm_hash = provider.get_vm(pool_name, vm)
|
203
208
|
return unless vm_hash.is_a? Hash
|
209
|
+
|
204
210
|
hostname = vm_hash['hostname']
|
205
211
|
|
206
212
|
return if hostname.nil?
|
207
213
|
return if hostname.empty?
|
208
214
|
return if hostname == vm
|
215
|
+
|
209
216
|
$redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
|
210
217
|
$logger.log('d', "[!] [#{pool_name}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
|
211
|
-
|
218
|
+
true
|
212
219
|
end
|
213
220
|
|
214
221
|
def check_running_vm(vm, pool, ttl, provider)
|
215
222
|
Thread.new do
|
216
223
|
begin
|
217
224
|
_check_running_vm(vm, pool, ttl, provider)
|
218
|
-
rescue =>
|
219
|
-
$logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{
|
225
|
+
rescue StandardError => e
|
226
|
+
$logger.log('s', "[!] [#{pool}] '#{vm}' failed while checking VM with an error: #{e}")
|
220
227
|
raise
|
221
228
|
end
|
222
229
|
end
|
@@ -225,6 +232,7 @@ module Vmpooler
|
|
225
232
|
def _check_running_vm(vm, pool, ttl, provider)
|
226
233
|
mutex = vm_mutex(vm)
|
227
234
|
return if mutex.locked?
|
235
|
+
|
228
236
|
mutex.synchronize do
|
229
237
|
# Check that VM is within defined lifetime
|
230
238
|
checkouttime = $redis.hget('vmpooler__active__' + pool, vm)
|
@@ -245,7 +253,7 @@ module Vmpooler
|
|
245
253
|
if host
|
246
254
|
return
|
247
255
|
else
|
248
|
-
move_vm_queue(pool, vm, 'running', 'completed',
|
256
|
+
move_vm_queue(pool, vm, 'running', 'completed', 'is no longer in inventory, removing from running')
|
249
257
|
end
|
250
258
|
end
|
251
259
|
end
|
@@ -261,14 +269,14 @@ module Vmpooler
|
|
261
269
|
Thread.new do
|
262
270
|
begin
|
263
271
|
_clone_vm(pool_name, provider)
|
264
|
-
rescue =>
|
265
|
-
$logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{
|
272
|
+
rescue StandardError => e
|
273
|
+
$logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
|
266
274
|
raise
|
267
275
|
end
|
268
276
|
end
|
269
277
|
end
|
270
278
|
|
271
|
-
def generate_and_check_hostname(
|
279
|
+
def generate_and_check_hostname(_pool_name)
|
272
280
|
# Generate a randomized hostname. The total name must no longer than 15
|
273
281
|
# character including the hyphen. The shortest adjective in the corpus is
|
274
282
|
# three characters long. Therefore, we can technically select a noun up to 11
|
@@ -278,12 +286,12 @@ module Vmpooler
|
|
278
286
|
# inviting more conflicts. We favor selecting a longer noun rather than a
|
279
287
|
# longer adjective because longer adjectives tend to be less fun.
|
280
288
|
noun = @name_generator.noun(max: 10)
|
281
|
-
adjective = @name_generator.adjective(max: 14-noun.length)
|
289
|
+
adjective = @name_generator.adjective(max: 14 - noun.length)
|
282
290
|
random_name = [adjective, noun].join('-')
|
283
291
|
hostname = $config[:config]['prefix'] + random_name
|
284
292
|
available = $redis.hlen('vmpooler__vm__' + hostname) == 0
|
285
293
|
|
286
|
-
|
294
|
+
[hostname, available]
|
287
295
|
end
|
288
296
|
|
289
297
|
def find_unique_hostname(pool_name)
|
@@ -297,9 +305,9 @@ module Vmpooler
|
|
297
305
|
$metrics.increment("errors.duplicatehostname.#{pool_name}")
|
298
306
|
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
|
299
307
|
end
|
300
|
-
|
308
|
+
|
301
309
|
raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless available
|
302
|
-
|
310
|
+
|
303
311
|
hostname
|
304
312
|
end
|
305
313
|
|
@@ -315,18 +323,18 @@ module Vmpooler
|
|
315
323
|
$logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
|
316
324
|
start = Time.now
|
317
325
|
provider.create_vm(pool_name, new_vmname)
|
318
|
-
finish = format('
|
326
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
319
327
|
|
320
328
|
$redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
|
321
329
|
$redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
|
322
330
|
$logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
|
323
331
|
|
324
332
|
$metrics.timing("clone.#{pool_name}", finish)
|
325
|
-
rescue
|
333
|
+
rescue StandardError
|
326
334
|
$redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
|
327
335
|
expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
|
328
336
|
$redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
|
329
|
-
raise
|
337
|
+
raise
|
330
338
|
ensure
|
331
339
|
$redis.decr('vmpooler__tasks__clone')
|
332
340
|
end
|
@@ -337,8 +345,8 @@ module Vmpooler
|
|
337
345
|
Thread.new do
|
338
346
|
begin
|
339
347
|
_destroy_vm(vm, pool, provider)
|
340
|
-
rescue =>
|
341
|
-
$logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{
|
348
|
+
rescue StandardError => e
|
349
|
+
$logger.log('d', "[!] [#{pool}] '#{vm}' failed while destroying the VM with an error: #{e}")
|
342
350
|
raise
|
343
351
|
end
|
344
352
|
end
|
@@ -347,6 +355,7 @@ module Vmpooler
|
|
347
355
|
def _destroy_vm(vm, pool, provider)
|
348
356
|
mutex = vm_mutex(vm)
|
349
357
|
return if mutex.locked?
|
358
|
+
|
350
359
|
mutex.synchronize do
|
351
360
|
$redis.hdel('vmpooler__active__' + pool, vm)
|
352
361
|
$redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
|
@@ -360,7 +369,7 @@ module Vmpooler
|
|
360
369
|
|
361
370
|
$redis.srem('vmpooler__completed__' + pool, vm)
|
362
371
|
|
363
|
-
finish = format('
|
372
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
364
373
|
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
|
365
374
|
$metrics.timing("destroy.#{pool}", finish)
|
366
375
|
get_vm_usage_labels(vm)
|
@@ -370,11 +379,13 @@ module Vmpooler
|
|
370
379
|
|
371
380
|
def get_vm_usage_labels(vm)
|
372
381
|
return unless $config[:config]['usage_stats']
|
382
|
+
|
373
383
|
checkout = $redis.hget("vmpooler__vm__#{vm}", 'checkout')
|
374
384
|
return if checkout.nil?
|
385
|
+
|
375
386
|
jenkins_build_url = $redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
|
376
387
|
user = $redis.hget("vmpooler__vm__#{vm}", 'token:user') || 'unauthenticated'
|
377
|
-
poolname = $redis.hget("vmpooler__vm__#{vm}",
|
388
|
+
poolname = $redis.hget("vmpooler__vm__#{vm}", 'template')
|
378
389
|
|
379
390
|
unless jenkins_build_url
|
380
391
|
user = user.gsub('.', '_')
|
@@ -404,25 +415,24 @@ module Vmpooler
|
|
404
415
|
poolname
|
405
416
|
]
|
406
417
|
|
407
|
-
metric_parts = metric_parts.reject
|
418
|
+
metric_parts = metric_parts.reject(&:nil?)
|
408
419
|
metric_parts = metric_parts.map { |s| s.gsub('.', '_') }
|
409
420
|
|
410
421
|
$metrics.increment(metric_parts.join('.'))
|
411
|
-
rescue =>
|
412
|
-
logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{
|
422
|
+
rescue StandardError => e
|
423
|
+
logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
|
413
424
|
end
|
414
425
|
|
415
426
|
def component_to_test(match, labels_string)
|
416
427
|
return if labels_string.nil?
|
428
|
+
|
417
429
|
labels_string_parts = labels_string.split(',')
|
418
430
|
labels_string_parts.each do |part|
|
419
431
|
key, value = part.split('=')
|
420
432
|
next if value.nil?
|
421
|
-
if key == match
|
422
|
-
return value
|
423
|
-
end
|
433
|
+
return value if key == match
|
424
434
|
end
|
425
|
-
|
435
|
+
nil
|
426
436
|
end
|
427
437
|
|
428
438
|
def purge_unused_vms_and_folders
|
@@ -435,13 +445,13 @@ module Vmpooler
|
|
435
445
|
Thread.new do
|
436
446
|
begin
|
437
447
|
purge_vms_and_folders($providers[provider.to_s])
|
438
|
-
rescue =>
|
439
|
-
$logger.log('s', "[!] failed while purging provider #{provider
|
448
|
+
rescue StandardError => e
|
449
|
+
$logger.log('s', "[!] failed while purging provider #{provider} VMs and folders with an error: #{e}")
|
440
450
|
end
|
441
451
|
end
|
442
452
|
end
|
443
453
|
end
|
444
|
-
|
454
|
+
nil
|
445
455
|
end
|
446
456
|
|
447
457
|
# Return a list of pool folders
|
@@ -450,6 +460,7 @@ module Vmpooler
|
|
450
460
|
folders = {}
|
451
461
|
$config[:pools].each do |pool|
|
452
462
|
next unless pool['provider'] == provider_name
|
463
|
+
|
453
464
|
folder_parts = pool['folder'].split('/')
|
454
465
|
datacenter = provider.get_target_datacenter_from_config(pool['name'])
|
455
466
|
folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
|
@@ -459,7 +470,7 @@ module Vmpooler
|
|
459
470
|
|
460
471
|
def get_base_folders(folders)
|
461
472
|
base = []
|
462
|
-
folders.each do |
|
473
|
+
folders.each do |_key, value|
|
463
474
|
base << value
|
464
475
|
end
|
465
476
|
base.uniq
|
@@ -476,8 +487,8 @@ module Vmpooler
|
|
476
487
|
Thread.new do
|
477
488
|
begin
|
478
489
|
_create_vm_disk(pool_name, vm, disk_size, provider)
|
479
|
-
rescue =>
|
480
|
-
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating disk: #{
|
490
|
+
rescue StandardError => e
|
491
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating disk: #{e}")
|
481
492
|
raise
|
482
493
|
end
|
483
494
|
end
|
@@ -492,7 +503,7 @@ module Vmpooler
|
|
492
503
|
|
493
504
|
result = provider.create_disk(pool_name, vm_name, disk_size.to_i)
|
494
505
|
|
495
|
-
finish = format('
|
506
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
496
507
|
|
497
508
|
if result
|
498
509
|
rdisks = $redis.hget('vmpooler__vm__' + vm_name, 'disk')
|
@@ -512,8 +523,8 @@ module Vmpooler
|
|
512
523
|
Thread.new do
|
513
524
|
begin
|
514
525
|
_create_vm_snapshot(pool_name, vm, snapshot_name, provider)
|
515
|
-
rescue =>
|
516
|
-
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating snapshot: #{
|
526
|
+
rescue StandardError => e
|
527
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while creating snapshot: #{e}")
|
517
528
|
raise
|
518
529
|
end
|
519
530
|
end
|
@@ -525,7 +536,7 @@ module Vmpooler
|
|
525
536
|
|
526
537
|
result = provider.create_snapshot(pool_name, vm_name, snapshot_name)
|
527
538
|
|
528
|
-
finish = format('
|
539
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
529
540
|
|
530
541
|
if result
|
531
542
|
$redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
|
@@ -541,8 +552,8 @@ module Vmpooler
|
|
541
552
|
Thread.new do
|
542
553
|
begin
|
543
554
|
_revert_vm_snapshot(pool_name, vm, snapshot_name, provider)
|
544
|
-
rescue =>
|
545
|
-
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while reverting snapshot: #{
|
555
|
+
rescue StandardError => e
|
556
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' failed while reverting snapshot: #{e}")
|
546
557
|
raise
|
547
558
|
end
|
548
559
|
end
|
@@ -554,7 +565,7 @@ module Vmpooler
|
|
554
565
|
|
555
566
|
result = provider.revert_snapshot(pool_name, vm_name, snapshot_name)
|
556
567
|
|
557
|
-
finish = format('
|
568
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
558
569
|
|
559
570
|
if result
|
560
571
|
$logger.log('s', "[+] [snapshot_manager] '#{vm_name}' reverted to snapshot '#{snapshot_name}' in #{finish} seconds")
|
@@ -574,14 +585,14 @@ module Vmpooler
|
|
574
585
|
# ie. ["vsphere", "dummy"]
|
575
586
|
def used_providers
|
576
587
|
pools = config[:pools] || []
|
577
|
-
@used_providers ||= (pools.map { |pool| pool[:provider] || pool['provider'] }.compact + default_providers
|
588
|
+
@used_providers ||= (pools.map { |pool| pool[:provider] || pool['provider'] }.compact + default_providers).uniq
|
578
589
|
end
|
579
590
|
|
580
591
|
# @return [Array] - returns a list of providers that should always be loaded
|
581
592
|
# note: vsphere is the default if user does not specify although this should not be
|
582
593
|
# if vsphere is to no longer be loaded by default please remove
|
583
594
|
def default_providers
|
584
|
-
@default_providers ||= %w
|
595
|
+
@default_providers ||= %w[vsphere dummy]
|
585
596
|
end
|
586
597
|
|
587
598
|
def get_pool_name_for_vm(vm_name)
|
@@ -592,8 +603,9 @@ module Vmpooler
|
|
592
603
|
# @param pool_name [String] - the name of the pool
|
593
604
|
# @return [Provider] - returns the provider class Object
|
594
605
|
def get_provider_for_pool(pool_name)
|
595
|
-
pool = $config[:pools].find { |
|
606
|
+
pool = $config[:pools].find { |p| p['name'] == pool_name }
|
596
607
|
return nil unless pool
|
608
|
+
|
597
609
|
provider_name = pool.fetch('provider', nil)
|
598
610
|
$providers[provider_name]
|
599
611
|
end
|
@@ -607,8 +619,9 @@ module Vmpooler
|
|
607
619
|
_check_disk_queue
|
608
620
|
sleep(loop_delay)
|
609
621
|
|
610
|
-
unless maxloop
|
622
|
+
unless maxloop == 0
|
611
623
|
break if loop_count >= maxloop
|
624
|
+
|
612
625
|
loop_count += 1
|
613
626
|
end
|
614
627
|
end
|
@@ -627,8 +640,8 @@ module Vmpooler
|
|
627
640
|
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
628
641
|
|
629
642
|
create_vm_disk(pool_name, vm_name, disk_size, provider)
|
630
|
-
rescue =>
|
631
|
-
$logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{
|
643
|
+
rescue StandardError => e
|
644
|
+
$logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{e}")
|
632
645
|
end
|
633
646
|
end
|
634
647
|
end
|
@@ -642,8 +655,9 @@ module Vmpooler
|
|
642
655
|
_check_snapshot_queue
|
643
656
|
sleep(loop_delay)
|
644
657
|
|
645
|
-
unless maxloop
|
658
|
+
unless maxloop == 0
|
646
659
|
break if loop_count >= maxloop
|
660
|
+
|
647
661
|
loop_count += 1
|
648
662
|
end
|
649
663
|
end
|
@@ -663,8 +677,8 @@ module Vmpooler
|
|
663
677
|
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
664
678
|
|
665
679
|
create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
|
666
|
-
rescue =>
|
667
|
-
$logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{
|
680
|
+
rescue StandardError => e
|
681
|
+
$logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{e}")
|
668
682
|
end
|
669
683
|
end
|
670
684
|
|
@@ -680,8 +694,8 @@ module Vmpooler
|
|
680
694
|
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
681
695
|
|
682
696
|
revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
|
683
|
-
rescue =>
|
684
|
-
$logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{
|
697
|
+
rescue StandardError => e
|
698
|
+
$logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{e}")
|
685
699
|
end
|
686
700
|
end
|
687
701
|
end
|
@@ -694,8 +708,8 @@ module Vmpooler
|
|
694
708
|
$redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
|
695
709
|
provider.migrate_vm(pool_name, vm_name)
|
696
710
|
end
|
697
|
-
rescue =>
|
698
|
-
$logger.log('s', "[x] [#{pool_name}] '#{vm_name}' migration failed with an error: #{
|
711
|
+
rescue StandardError => e
|
712
|
+
$logger.log('s', "[x] [#{pool_name}] '#{vm_name}' migration failed with an error: #{e}")
|
699
713
|
end
|
700
714
|
end
|
701
715
|
end
|
@@ -724,17 +738,11 @@ module Vmpooler
|
|
724
738
|
wakeup_by = Time.now + wakeup_period
|
725
739
|
return if time_passed?(:exit_by, exit_by)
|
726
740
|
|
727
|
-
if options[:pool_size_change]
|
728
|
-
initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
|
729
|
-
end
|
741
|
+
initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
|
730
742
|
|
731
|
-
if options[:clone_target_change]
|
732
|
-
initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target])
|
733
|
-
end
|
743
|
+
initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
|
734
744
|
|
735
|
-
if options[:pool_template_change]
|
736
|
-
initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname])
|
737
|
-
end
|
745
|
+
initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
|
738
746
|
|
739
747
|
loop do
|
740
748
|
sleep(1)
|
@@ -751,7 +759,7 @@ module Vmpooler
|
|
751
759
|
end
|
752
760
|
|
753
761
|
if options[:clone_target_change]
|
754
|
-
clone_target = $redis.hget(
|
762
|
+
clone_target = $redis.hget('vmpooler__config__clone_target}', options[:poolname])
|
755
763
|
if clone_target
|
756
764
|
break unless clone_target == initial_clone_target
|
757
765
|
end
|
@@ -795,6 +803,7 @@ module Vmpooler
|
|
795
803
|
loop_delay = loop_delay_min
|
796
804
|
provider = get_provider_for_pool(pool['name'])
|
797
805
|
raise("Could not find provider '#{pool['provider']}") if provider.nil?
|
806
|
+
|
798
807
|
sync_pool_template(pool)
|
799
808
|
loop do
|
800
809
|
result = _check_pool(pool, provider)
|
@@ -807,13 +816,14 @@ module Vmpooler
|
|
807
816
|
end
|
808
817
|
sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pool_reset: true)
|
809
818
|
|
810
|
-
unless maxloop
|
819
|
+
unless maxloop == 0
|
811
820
|
break if loop_count >= maxloop
|
821
|
+
|
812
822
|
loop_count += 1
|
813
823
|
end
|
814
824
|
end
|
815
|
-
rescue =>
|
816
|
-
$logger.log('s', "[!] [#{pool['name']}] Error while checking the pool: #{
|
825
|
+
rescue StandardError => e
|
826
|
+
$logger.log('s', "[!] [#{pool['name']}] Error while checking the pool: #{e}")
|
817
827
|
raise
|
818
828
|
end
|
819
829
|
end
|
@@ -828,19 +838,13 @@ module Vmpooler
|
|
828
838
|
end
|
829
839
|
|
830
840
|
def dereference_mutex(vmname)
|
831
|
-
if @vm_mutex.delete(vmname)
|
832
|
-
return true
|
833
|
-
else
|
834
|
-
return
|
835
|
-
end
|
841
|
+
true if @vm_mutex.delete(vmname)
|
836
842
|
end
|
837
843
|
|
838
844
|
def sync_pool_template(pool)
|
839
845
|
pool_template = $redis.hget('vmpooler__config__template', pool['name'])
|
840
846
|
if pool_template
|
841
|
-
unless pool['template'] == pool_template
|
842
|
-
pool['template'] = pool_template
|
843
|
-
end
|
847
|
+
pool['template'] = pool_template unless pool['template'] == pool_template
|
844
848
|
end
|
845
849
|
end
|
846
850
|
|
@@ -850,8 +854,8 @@ module Vmpooler
|
|
850
854
|
begin
|
851
855
|
provider.create_template_delta_disks(pool)
|
852
856
|
$redis.sadd('vmpooler__template__deltas', pool['template'])
|
853
|
-
rescue =>
|
854
|
-
$logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{
|
857
|
+
rescue StandardError => e
|
858
|
+
$logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{e}")
|
855
859
|
end
|
856
860
|
end
|
857
861
|
end
|
@@ -863,6 +867,7 @@ module Vmpooler
|
|
863
867
|
prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
|
864
868
|
configured_template = $redis.hget('vmpooler__config__template', pool['name'])
|
865
869
|
return if mutex.locked?
|
870
|
+
|
866
871
|
if prepared_template.nil?
|
867
872
|
mutex.synchronize do
|
868
873
|
prepare_template(pool, provider)
|
@@ -878,6 +883,7 @@ module Vmpooler
|
|
878
883
|
end
|
879
884
|
return if configured_template.nil?
|
880
885
|
return if configured_template == prepared_template
|
886
|
+
|
881
887
|
mutex.synchronize do
|
882
888
|
update_pool_template(pool, provider, configured_template, prepared_template)
|
883
889
|
end
|
@@ -913,9 +919,11 @@ module Vmpooler
|
|
913
919
|
def update_clone_target(pool)
|
914
920
|
mutex = pool_mutex(pool['name'])
|
915
921
|
return if mutex.locked?
|
922
|
+
|
916
923
|
clone_target = $redis.hget('vmpooler__config__clone_target', pool['name'])
|
917
924
|
return if clone_target.nil?
|
918
925
|
return if clone_target == pool['clone_target']
|
926
|
+
|
919
927
|
$logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
|
920
928
|
mutex.synchronize do
|
921
929
|
pool['clone_target'] = clone_target
|
@@ -930,9 +938,11 @@ module Vmpooler
|
|
930
938
|
total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
|
931
939
|
return if total.nil?
|
932
940
|
return if total == 0
|
941
|
+
|
933
942
|
mutex = pool_mutex(pool['name'])
|
934
943
|
return if mutex.locked?
|
935
944
|
return unless ready > pool['size']
|
945
|
+
|
936
946
|
mutex.synchronize do
|
937
947
|
difference = ready - pool['size']
|
938
948
|
difference.times do
|
@@ -950,10 +960,13 @@ module Vmpooler
|
|
950
960
|
def update_pool_size(pool)
|
951
961
|
mutex = pool_mutex(pool['name'])
|
952
962
|
return if mutex.locked?
|
963
|
+
|
953
964
|
poolsize = $redis.hget('vmpooler__config__poolsize', pool['name'])
|
954
965
|
return if poolsize.nil?
|
966
|
+
|
955
967
|
poolsize = Integer(poolsize)
|
956
968
|
return if poolsize == pool['size']
|
969
|
+
|
957
970
|
mutex.synchronize do
|
958
971
|
pool['size'] = poolsize
|
959
972
|
end
|
@@ -962,6 +975,7 @@ module Vmpooler
|
|
962
975
|
def reset_pool(pool)
|
963
976
|
poolname = pool['name']
|
964
977
|
return unless $redis.sismember('vmpooler__poolreset', poolname)
|
978
|
+
|
965
979
|
$redis.srem('vmpooler__poolreset', poolname)
|
966
980
|
mutex = pool_mutex(poolname)
|
967
981
|
mutex.synchronize do
|
@@ -992,9 +1006,9 @@ module Vmpooler
|
|
992
1006
|
inventory[vm['name']] = 1
|
993
1007
|
end
|
994
1008
|
end
|
995
|
-
rescue =>
|
996
|
-
$logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while running create_inventory: #{
|
997
|
-
raise(
|
1009
|
+
rescue StandardError => e
|
1010
|
+
$logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while running create_inventory: #{e}")
|
1011
|
+
raise(e)
|
998
1012
|
end
|
999
1013
|
inventory
|
1000
1014
|
end
|
@@ -1006,8 +1020,8 @@ module Vmpooler
|
|
1006
1020
|
vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
|
1007
1021
|
pool_check_response[:checked_running_vms] += 1
|
1008
1022
|
check_running_vm(vm, pool_name, vm_lifetime, provider)
|
1009
|
-
rescue =>
|
1010
|
-
$logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{
|
1023
|
+
rescue StandardError => e
|
1024
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{e}")
|
1011
1025
|
end
|
1012
1026
|
else
|
1013
1027
|
move_vm_queue(pool_name, vm, 'running', 'completed', 'is a running VM but is missing from inventory. Marking as completed.')
|
@@ -1021,8 +1035,8 @@ module Vmpooler
|
|
1021
1035
|
begin
|
1022
1036
|
pool_check_response[:checked_ready_vms] += 1
|
1023
1037
|
check_ready_vm(vm, pool_name, pool_ttl || 0, provider)
|
1024
|
-
rescue =>
|
1025
|
-
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{
|
1038
|
+
rescue StandardError => e
|
1039
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
|
1026
1040
|
end
|
1027
1041
|
else
|
1028
1042
|
move_vm_queue(pool_name, vm, 'ready', 'completed', 'is a ready VM but is missing from inventory. Marking as completed.')
|
@@ -1037,8 +1051,8 @@ module Vmpooler
|
|
1037
1051
|
begin
|
1038
1052
|
pool_check_response[:checked_pending_vms] += 1
|
1039
1053
|
check_pending_vm(vm, pool_name, pool_timeout, provider)
|
1040
|
-
rescue =>
|
1041
|
-
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{
|
1054
|
+
rescue StandardError => e
|
1055
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
|
1042
1056
|
end
|
1043
1057
|
else
|
1044
1058
|
fail_pending_vm(vm, pool_name, pool_timeout, false)
|
@@ -1052,11 +1066,11 @@ module Vmpooler
|
|
1052
1066
|
begin
|
1053
1067
|
pool_check_response[:destroyed_vms] += 1
|
1054
1068
|
destroy_vm(vm, pool_name, provider)
|
1055
|
-
rescue =>
|
1069
|
+
rescue StandardError => e
|
1056
1070
|
$redis.srem("vmpooler__completed__#{pool_name}", vm)
|
1057
1071
|
$redis.hdel("vmpooler__active__#{pool_name}", vm)
|
1058
1072
|
$redis.del("vmpooler__vm__#{vm}")
|
1059
|
-
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{
|
1073
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
|
1060
1074
|
end
|
1061
1075
|
else
|
1062
1076
|
$logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
@@ -1068,22 +1082,18 @@ module Vmpooler
|
|
1068
1082
|
end
|
1069
1083
|
|
1070
1084
|
def check_discovered_pool_vms(pool_name)
|
1071
|
-
|
1072
|
-
|
1073
|
-
|
1074
|
-
|
1075
|
-
|
1076
|
-
$redis.srem("vmpooler__discovered__#{pool_name}", vm)
|
1077
|
-
end
|
1078
|
-
end
|
1079
|
-
|
1080
|
-
if $redis.sismember("vmpooler__discovered__#{pool_name}", vm)
|
1081
|
-
$redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm)
|
1085
|
+
$redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
|
1086
|
+
%w[pending ready running completed].each do |queue|
|
1087
|
+
if $redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
|
1088
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
|
1089
|
+
$redis.srem("vmpooler__discovered__#{pool_name}", vm)
|
1082
1090
|
end
|
1083
1091
|
end
|
1084
|
-
|
1085
|
-
$
|
1092
|
+
|
1093
|
+
$redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm) if $redis.sismember("vmpooler__discovered__#{pool_name}", vm)
|
1086
1094
|
end
|
1095
|
+
rescue StandardError => e
|
1096
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating discovered VMs: #{e}")
|
1087
1097
|
end
|
1088
1098
|
|
1089
1099
|
def check_migrating_pool_vms(pool_name, provider, pool_check_response, inventory)
|
@@ -1092,8 +1102,8 @@ module Vmpooler
|
|
1092
1102
|
begin
|
1093
1103
|
pool_check_response[:migrated_vms] += 1
|
1094
1104
|
migrate_vm(vm, pool_name, provider)
|
1095
|
-
rescue =>
|
1096
|
-
$logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{
|
1105
|
+
rescue StandardError => e
|
1106
|
+
$logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{e}")
|
1097
1107
|
end
|
1098
1108
|
end
|
1099
1109
|
end
|
@@ -1101,6 +1111,7 @@ module Vmpooler
|
|
1101
1111
|
|
1102
1112
|
def repopulate_pool_vms(pool_name, provider, pool_check_response, pool_size)
|
1103
1113
|
return if pool_mutex(pool_name).locked?
|
1114
|
+
|
1104
1115
|
ready = $redis.scard("vmpooler__ready__#{pool_name}")
|
1105
1116
|
total = $redis.scard("vmpooler__pending__#{pool_name}") + ready
|
1106
1117
|
|
@@ -1108,8 +1119,8 @@ module Vmpooler
|
|
1108
1119
|
$metrics.gauge("running.#{pool_name}", $redis.scard("vmpooler__running__#{pool_name}"))
|
1109
1120
|
|
1110
1121
|
if $redis.get("vmpooler__empty__#{pool_name}")
|
1111
|
-
$redis.del("vmpooler__empty__#{pool_name}") unless ready
|
1112
|
-
elsif ready
|
1122
|
+
$redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
|
1123
|
+
elsif ready == 0
|
1113
1124
|
$redis.set("vmpooler__empty__#{pool_name}", 'true')
|
1114
1125
|
$logger.log('s', "[!] [#{pool_name}] is empty")
|
1115
1126
|
end
|
@@ -1120,8 +1131,8 @@ module Vmpooler
|
|
1120
1131
|
$redis.incr('vmpooler__tasks__clone')
|
1121
1132
|
pool_check_response[:cloned_vms] += 1
|
1122
1133
|
clone_vm(pool_name, provider)
|
1123
|
-
rescue =>
|
1124
|
-
$logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{
|
1134
|
+
rescue StandardError => e
|
1135
|
+
$logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
|
1125
1136
|
$redis.decr('vmpooler__tasks__clone')
|
1126
1137
|
raise
|
1127
1138
|
end
|
@@ -1142,7 +1153,7 @@ module Vmpooler
|
|
1142
1153
|
|
1143
1154
|
begin
|
1144
1155
|
inventory = create_inventory(pool, provider, pool_check_response)
|
1145
|
-
rescue
|
1156
|
+
rescue StandardError
|
1146
1157
|
return(pool_check_response)
|
1147
1158
|
end
|
1148
1159
|
|
@@ -1195,6 +1206,7 @@ module Vmpooler
|
|
1195
1206
|
provider_klass = Vmpooler::PoolManager::Provider
|
1196
1207
|
provider_klass.constants.each do |classname|
|
1197
1208
|
next unless classname.to_s.casecmp(provider_class) == 0
|
1209
|
+
|
1198
1210
|
return provider_klass.const_get(classname).new(config, logger, metrics, provider_name, options)
|
1199
1211
|
end
|
1200
1212
|
raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
|
@@ -1257,8 +1269,8 @@ module Vmpooler
|
|
1257
1269
|
end
|
1258
1270
|
begin
|
1259
1271
|
$providers[provider_name] = create_provider_object($config, $logger, $metrics, provider_class, provider_name, {}) if $providers[provider_name].nil?
|
1260
|
-
rescue =>
|
1261
|
-
$logger.log('s', "Error while creating provider for pool #{pool['name']}: #{
|
1272
|
+
rescue StandardError => e
|
1273
|
+
$logger.log('s', "Error while creating provider for pool #{pool['name']}: #{e}")
|
1262
1274
|
raise
|
1263
1275
|
end
|
1264
1276
|
end
|
@@ -1292,8 +1304,9 @@ module Vmpooler
|
|
1292
1304
|
|
1293
1305
|
sleep(loop_delay)
|
1294
1306
|
|
1295
|
-
unless maxloop
|
1307
|
+
unless maxloop == 0
|
1296
1308
|
break if loop_count >= maxloop
|
1309
|
+
|
1297
1310
|
loop_count += 1
|
1298
1311
|
end
|
1299
1312
|
end
|