vmpooler 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,35 +15,17 @@ module Vmpooler
15
15
  @metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == ''
16
16
  end
17
17
 
18
- if Thread.respond_to?(:handle_interrupt)
19
- # MRI
20
- def with_metrics(options = {})
21
- Thread.handle_interrupt(Exception => :never) do
22
- start = Time.now
23
- conn = checkout(options)
24
- timespan_ms = ((Time.now - start) * 1000).to_i
25
- @metrics&.gauge(@metric_prefix + '.available', @available.length)
26
- @metrics&.timing(@metric_prefix + '.waited', timespan_ms)
27
- begin
28
- Thread.handle_interrupt(Exception => :immediate) do
29
- yield conn
30
- end
31
- ensure
32
- checkin
33
- @metrics&.gauge(@metric_prefix + '.available', @available.length)
34
- end
35
- end
36
- end
37
- else
38
- # jruby 1.7.x
39
- def with_metrics(options = {})
18
+ def with_metrics(options = {})
19
+ Thread.handle_interrupt(Exception => :never) do
40
20
  start = Time.now
41
21
  conn = checkout(options)
42
22
  timespan_ms = ((Time.now - start) * 1000).to_i
43
23
  @metrics&.gauge(@metric_prefix + '.available', @available.length)
44
24
  @metrics&.timing(@metric_prefix + '.waited', timespan_ms)
45
25
  begin
46
- yield conn
26
+ Thread.handle_interrupt(Exception => :immediate) do
27
+ yield conn
28
+ end
47
29
  ensure
48
30
  checkin
49
31
  @metrics&.gauge(@metric_prefix + '.available', @available.length)
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'vmpooler/providers'
4
4
  require 'spicy-proton'
5
+ require 'resolv' # ruby standard lib
5
6
 
6
7
  module Vmpooler
7
8
  class PoolManager
@@ -9,7 +10,7 @@ module Vmpooler
9
10
  CHECK_LOOP_DELAY_MAX_DEFAULT = 60
10
11
  CHECK_LOOP_DELAY_DECAY_DEFAULT = 2.0
11
12
 
12
- def initialize(config, logger, redis, metrics)
13
+ def initialize(config, logger, redis_connection_pool, metrics)
13
14
  $config = config
14
15
 
15
16
  # Load logger library
@@ -18,19 +19,19 @@ module Vmpooler
18
19
  # metrics logging handle
19
20
  $metrics = metrics
20
21
 
21
- # Connect to Redis
22
- $redis = redis
22
+ # Redis connection pool
23
+ @redis = redis_connection_pool
23
24
 
24
25
  # VM Provider objects
25
- $providers = {}
26
+ $providers = Concurrent::Hash.new
26
27
 
27
28
  # Our thread-tracker object
28
- $threads = {}
29
+ $threads = Concurrent::Hash.new
29
30
 
30
31
  # Pool mutex
31
- @reconfigure_pool = {}
32
+ @reconfigure_pool = Concurrent::Hash.new
32
33
 
33
- @vm_mutex = {}
34
+ @vm_mutex = Concurrent::Hash.new
34
35
 
35
36
  # Name generator for generating host names
36
37
  @name_generator = Spicy::Proton.new
@@ -45,24 +46,26 @@ module Vmpooler
45
46
 
46
47
  # Place pool configuration in redis so an API instance can discover running pool configuration
47
48
  def load_pools_to_redis
48
- previously_configured_pools = $redis.smembers('vmpooler__pools')
49
- currently_configured_pools = []
50
- config[:pools].each do |pool|
51
- currently_configured_pools << pool['name']
52
- $redis.sadd('vmpooler__pools', pool['name'])
53
- pool_keys = pool.keys
54
- pool_keys.delete('alias')
55
- to_set = {}
56
- pool_keys.each do |k|
57
- to_set[k] = pool[k]
49
+ @redis.with_metrics do |redis|
50
+ previously_configured_pools = redis.smembers('vmpooler__pools')
51
+ currently_configured_pools = []
52
+ config[:pools].each do |pool|
53
+ currently_configured_pools << pool['name']
54
+ redis.sadd('vmpooler__pools', pool['name'])
55
+ pool_keys = pool.keys
56
+ pool_keys.delete('alias')
57
+ to_set = {}
58
+ pool_keys.each do |k|
59
+ to_set[k] = pool[k]
60
+ end
61
+ to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
62
+ redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
58
63
  end
59
- to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
60
- $redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
61
- end
62
- previously_configured_pools.each do |pool|
63
- unless currently_configured_pools.include? pool
64
- $redis.srem('vmpooler__pools', pool)
65
- $redis.del("vmpooler__pool__#{pool}")
64
+ previously_configured_pools.each do |pool|
65
+ unless currently_configured_pools.include? pool
66
+ redis.srem('vmpooler__pools', pool)
67
+ redis.del("vmpooler__pool__#{pool}")
68
+ end
66
69
  end
67
70
  end
68
71
  nil
@@ -75,7 +78,9 @@ module Vmpooler
75
78
  _check_pending_vm(vm, pool, timeout, provider)
76
79
  rescue StandardError => e
77
80
  $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
78
- fail_pending_vm(vm, pool, timeout)
81
+ @redis.with_metrics do |redis|
82
+ fail_pending_vm(vm, pool, timeout, redis)
83
+ end
79
84
  raise
80
85
  end
81
86
  end
@@ -86,31 +91,38 @@ module Vmpooler
86
91
  return if mutex.locked?
87
92
 
88
93
  mutex.synchronize do
89
- if provider.vm_ready?(pool, vm)
90
- move_pending_vm_to_ready(vm, pool)
91
- else
92
- fail_pending_vm(vm, pool, timeout)
94
+ @redis.with_metrics do |redis|
95
+ request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
96
+ if provider.vm_ready?(pool, vm)
97
+ move_pending_vm_to_ready(vm, pool, redis, request_id)
98
+ else
99
+ fail_pending_vm(vm, pool, timeout, redis)
100
+ end
93
101
  end
94
102
  end
95
103
  end
96
104
 
97
- def remove_nonexistent_vm(vm, pool)
98
- $redis.srem("vmpooler__pending__#{pool}", vm)
105
+ def remove_nonexistent_vm(vm, pool, redis)
106
+ redis.srem("vmpooler__pending__#{pool}", vm)
99
107
  $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
100
108
  end
101
109
 
102
- def fail_pending_vm(vm, pool, timeout, exists = true)
103
- clone_stamp = $redis.hget("vmpooler__vm__#{vm}", 'clone')
104
- return true unless clone_stamp
110
+ def fail_pending_vm(vm, pool, timeout, redis, exists = true)
111
+ clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
105
112
 
106
113
  time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
107
114
  if time_since_clone > timeout
108
115
  if exists
109
- $redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm)
116
+ request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
117
+ pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
118
+ redis.multi
119
+ redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm)
120
+ redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}") if request_id
121
+ redis.exec
110
122
  $metrics.increment("errors.markedasfailed.#{pool}")
111
123
  $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
112
124
  else
113
- remove_nonexistent_vm(vm, pool)
125
+ remove_nonexistent_vm(vm, pool, redis)
114
126
  end
115
127
  end
116
128
  true
@@ -119,28 +131,54 @@ module Vmpooler
119
131
  false
120
132
  end
121
133
 
122
- def move_pending_vm_to_ready(vm, pool)
123
- clone_time = $redis.hget('vmpooler__vm__' + vm, 'clone')
124
- finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time)) if clone_time
134
+ def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
135
+ clone_time = redis.hget('vmpooler__vm__' + vm, 'clone')
136
+ finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
125
137
 
126
- $redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
127
- $redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
128
- $redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
138
+ if request_id
139
+ ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
140
+ if ondemandrequest_hash['status'] == 'failed'
141
+ move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
142
+ return nil
143
+ elsif ondemandrequest_hash['status'] == 'deleted'
144
+ move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' has been deleted")
145
+ return nil
146
+ end
147
+ pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
148
+
149
+ redis.pipelined do
150
+ redis.hset("vmpooler__active__#{pool}", vm, Time.now)
151
+ redis.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
152
+ redis.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token']) if ondemandrequest_hash['token:token']
153
+ redis.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user']) if ondemandrequest_hash['token:user']
154
+ redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
155
+ end
156
+ move_vm_queue(pool, vm, 'pending', 'running', redis)
157
+ check_ondemand_request_ready(request_id, redis)
158
+ else
159
+ redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
160
+ end
129
161
 
130
- # last boot time is displayed in API, and used by alarming script
131
- $redis.hset('vmpooler__lastboot', pool, Time.now)
162
+ redis.pipelined do
163
+ redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
164
+ redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
165
+
166
+ # last boot time is displayed in API, and used by alarming script
167
+ redis.hset('vmpooler__lastboot', pool, Time.now)
168
+ end
132
169
 
133
170
  $metrics.timing("time_to_ready_state.#{pool}", finish)
134
- $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
171
+ $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") unless request_id
172
+ $logger.log('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'") if request_id
135
173
  end
136
174
 
137
- def vm_still_ready?(pool_name, vm_name, provider)
175
+ def vm_still_ready?(pool_name, vm_name, provider, redis)
138
176
  # Check if the VM is still ready/available
139
177
  return true if provider.vm_ready?(pool_name, vm_name)
140
178
 
141
179
  raise("VM #{vm_name} is not ready")
142
180
  rescue StandardError
143
- move_vm_queue(pool_name, vm_name, 'ready', 'completed', "is unreachable, removed from 'ready' queue")
181
+ move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "is unreachable, removed from 'ready' queue")
144
182
  end
145
183
 
146
184
  def check_ready_vm(vm, pool_name, ttl, provider)
@@ -160,34 +198,35 @@ module Vmpooler
160
198
  return if mutex.locked?
161
199
 
162
200
  mutex.synchronize do
163
- check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check')
164
- return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime'])
201
+ @redis.with_metrics do |redis|
202
+ check_stamp = redis.hget('vmpooler__vm__' + vm, 'check')
203
+ last_checked_too_soon = ((Time.now - Time.parse(check_stamp)).to_i < $config[:config]['vm_checktime'] * 60) if check_stamp
204
+ break if check_stamp && last_checked_too_soon
165
205
 
166
- $redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
167
- # Check if the hosts TTL has expired
168
- if ttl > 0
206
+ redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
207
+ # Check if the hosts TTL has expired
169
208
  # if 'boottime' is nil, set bootime to beginning of unix epoch, forces TTL to be assumed expired
170
- boottime = $redis.hget("vmpooler__vm__#{vm}", 'ready')
209
+ boottime = redis.hget("vmpooler__vm__#{vm}", 'ready')
171
210
  if boottime
172
211
  boottime = Time.parse(boottime)
173
212
  else
174
213
  boottime = Time.at(0)
175
214
  end
176
- if ((Time.now - boottime) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl
177
- $redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
215
+ if (Time.now - boottime).to_i > ttl * 60
216
+ redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
178
217
 
179
218
  $logger.log('d', "[!] [#{pool_name}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
180
- return
219
+ return nil
181
220
  end
182
- end
183
221
 
184
- return if mismatched_hostname?(vm, pool_name, provider)
222
+ break if mismatched_hostname?(vm, pool_name, provider, redis)
185
223
 
186
- vm_still_ready?(pool_name, vm, provider)
224
+ vm_still_ready?(pool_name, vm, provider, redis)
225
+ end
187
226
  end
188
227
  end
189
228
 
190
- def mismatched_hostname?(vm, pool_name, provider)
229
+ def mismatched_hostname?(vm, pool_name, provider, redis)
191
230
  pool_config = $config[:pools][$config[:pool_index][pool_name]]
192
231
  check_hostname = pool_config['check_hostname_for_mismatch']
193
232
  check_hostname = $config[:config]['check_ready_vm_hostname_for_mismatch'] if check_hostname.nil?
@@ -196,7 +235,7 @@ module Vmpooler
196
235
  # Wait one minute before checking a VM for hostname mismatch
197
236
  # When checking as soon as the VM passes the ready test the instance
198
237
  # often doesn't report its hostname yet causing the VM to be removed immediately
199
- vm_ready_time = $redis.hget("vmpooler__vm__#{vm}", 'ready')
238
+ vm_ready_time = redis.hget("vmpooler__vm__#{vm}", 'ready')
200
239
  if vm_ready_time
201
240
  wait_before_checking = 60
202
241
  time_since_ready = (Time.now - Time.parse(vm_ready_time)).to_i
@@ -213,7 +252,7 @@ module Vmpooler
213
252
  return if hostname.empty?
214
253
  return if hostname == vm
215
254
 
216
- $redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
255
+ redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
217
256
  $logger.log('d', "[!] [#{pool_name}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
218
257
  true
219
258
  end
@@ -234,49 +273,61 @@ module Vmpooler
234
273
  return if mutex.locked?
235
274
 
236
275
  mutex.synchronize do
237
- # Check that VM is within defined lifetime
238
- checkouttime = $redis.hget('vmpooler__active__' + pool, vm)
239
- if checkouttime
240
- running = (Time.now - Time.parse(checkouttime)) / 60 / 60
241
-
242
- if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
243
- move_vm_queue(pool, vm, 'running', 'completed', "reached end of TTL after #{ttl} hours")
244
- return
245
- end
246
- end
276
+ catch :stop_checking do
277
+ @redis.with_metrics do |redis|
278
+ # Check that VM is within defined lifetime
279
+ checkouttime = redis.hget('vmpooler__active__' + pool, vm)
280
+ if checkouttime
281
+ time_since_checkout = Time.now - Time.parse(checkouttime)
282
+ running = time_since_checkout / 60 / 60
283
+
284
+ if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
285
+ move_vm_queue(pool, vm, 'running', 'completed', redis, "reached end of TTL after #{ttl} hours")
286
+ throw :stop_checking
287
+ end
288
+ end
247
289
 
248
- if provider.vm_ready?(pool, vm)
249
- return
250
- else
251
- host = provider.get_vm(pool, vm)
290
+ if provider.vm_ready?(pool, vm)
291
+ throw :stop_checking
292
+ else
293
+ host = provider.get_vm(pool, vm)
252
294
 
253
- if host
254
- return
255
- else
256
- move_vm_queue(pool, vm, 'running', 'completed', 'is no longer in inventory, removing from running')
295
+ if host
296
+ throw :stop_checking
297
+ else
298
+ move_vm_queue(pool, vm, 'running', 'completed', redis, 'is no longer in inventory, removing from running')
299
+ end
300
+ end
257
301
  end
258
302
  end
259
303
  end
260
304
  end
261
305
 
262
- def move_vm_queue(pool, vm, queue_from, queue_to, msg = nil)
263
- $redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
306
+ def move_vm_queue(pool, vm, queue_from, queue_to, redis, msg = nil)
307
+ redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
264
308
  $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") if msg
265
309
  end
266
310
 
267
311
  # Clone a VM
268
- def clone_vm(pool_name, provider)
312
+ def clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
269
313
  Thread.new do
270
314
  begin
271
- _clone_vm(pool_name, provider)
315
+ _clone_vm(pool_name, provider, request_id, pool_alias)
272
316
  rescue StandardError => e
273
- $logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
317
+ if request_id
318
+ $logger.log('s', "[!] [#{pool_name}] failed while cloning VM for request #{request_id} with an error: #{e}")
319
+ @redis.with_metrics do |redis|
320
+ redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
321
+ end
322
+ else
323
+ $logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
324
+ end
274
325
  raise
275
326
  end
276
327
  end
277
328
  end
278
329
 
279
- def generate_and_check_hostname(_pool_name)
330
+ def generate_and_check_hostname
280
331
  # Generate a randomized hostname. The total name must no longer than 15
281
332
  # character including the hyphen. The shortest adjective in the corpus is
282
333
  # three characters long. Therefore, we can technically select a noun up to 11
@@ -285,58 +336,104 @@ module Vmpooler
285
336
  # letter adjectives, we actually limit the noun to 10 letters to avoid
286
337
  # inviting more conflicts. We favor selecting a longer noun rather than a
287
338
  # longer adjective because longer adjectives tend to be less fun.
288
- noun = @name_generator.noun(max: 10)
289
- adjective = @name_generator.adjective(max: 14 - noun.length)
290
- random_name = [adjective, noun].join('-')
291
- hostname = $config[:config]['prefix'] + random_name
292
- available = $redis.hlen('vmpooler__vm__' + hostname) == 0
293
-
294
- [hostname, available]
339
+ @redis.with do |redis|
340
+ noun = @name_generator.noun(max: 10)
341
+ adjective = @name_generator.adjective(max: 14 - noun.length)
342
+ random_name = [adjective, noun].join('-')
343
+ hostname = $config[:config]['prefix'] + random_name
344
+ available = redis.hlen('vmpooler__vm__' + hostname) == 0
345
+
346
+ [hostname, available]
347
+ end
295
348
  end
296
349
 
297
350
  def find_unique_hostname(pool_name)
351
+ # generate hostname that is not already in use in vmpooler
352
+ # also check that no dns record already exists
298
353
  hostname_retries = 0
299
354
  max_hostname_retries = 3
300
355
  while hostname_retries < max_hostname_retries
301
- hostname, available = generate_and_check_hostname(pool_name)
302
- break if available
356
+ hostname, hostname_available = generate_and_check_hostname(pool_name)
357
+ domain = $config[:config]['domain']
358
+ dns_ip, dns_available = check_dns_available(hostname, domain)
359
+ break if hostname_available && dns_available
303
360
 
304
361
  hostname_retries += 1
305
- $metrics.increment("errors.duplicatehostname.#{pool_name}")
306
- $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
362
+
363
+ if !hostname_available
364
+ $metrics.increment("errors.duplicatehostname.#{pool_name}")
365
+ $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
366
+ elsif !dns_available
367
+ $metrics.increment("errors.staledns.#{hostname}")
368
+ $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} already exists in DNS records (#{dns_ip}), stale DNS")
369
+ end
307
370
  end
308
371
 
309
- raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless available
372
+ raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless hostname_available && dns_available
310
373
 
311
374
  hostname
312
375
  end
313
376
 
314
- def _clone_vm(pool_name, provider)
315
- new_vmname = find_unique_hostname(pool_name)
316
-
317
- # Add VM to Redis inventory ('pending' pool)
318
- $redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
319
- $redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
320
- $redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name)
321
-
377
+ def check_dns_available(vm_name, domain = nil)
378
+ # Query the DNS for the name we want to create and if it already exists, mark it unavailable
379
+ # This protects against stale DNS records
380
+ vm_name = "#{vm_name}.#{domain}" if domain
322
381
  begin
323
- $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
324
- start = Time.now
325
- provider.create_vm(pool_name, new_vmname)
326
- finish = format('%<time>.2f', time: Time.now - start)
382
+ dns_ip = Resolv.getaddress(vm_name)
383
+ rescue Resolv::ResolvError
384
+ # this is the expected case, swallow the error
385
+ # eg "no address for blah-daisy"
386
+ return ['', true]
387
+ end
388
+ [dns_ip, false]
389
+ end
327
390
 
328
- $redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
329
- $redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
330
- $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
391
+ def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
392
+ new_vmname = find_unique_hostname(pool_name)
393
+ mutex = vm_mutex(new_vmname)
394
+ mutex.synchronize do
395
+ @redis.with_metrics do |redis|
396
+ # Add VM to Redis inventory ('pending' pool)
397
+ redis.multi
398
+ redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
399
+ redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
400
+ redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name) # This value is used to represent the pool.
401
+ redis.hset('vmpooler__vm__' + new_vmname, 'pool', pool_name)
402
+ redis.hset('vmpooler__vm__' + new_vmname, 'request_id', request_id) if request_id
403
+ redis.hset('vmpooler__vm__' + new_vmname, 'pool_alias', pool_alias) if pool_alias
404
+ redis.exec
405
+ end
331
406
 
332
- $metrics.timing("clone.#{pool_name}", finish)
333
- rescue StandardError
334
- $redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
335
- expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
336
- $redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
337
- raise
338
- ensure
339
- $redis.decr('vmpooler__tasks__clone')
407
+ begin
408
+ $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
409
+ start = Time.now
410
+ provider.create_vm(pool_name, new_vmname)
411
+ finish = format('%<time>.2f', time: Time.now - start)
412
+
413
+ @redis.with_metrics do |redis|
414
+ redis.pipelined do
415
+ redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
416
+ redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
417
+ end
418
+ end
419
+ $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
420
+
421
+ $metrics.timing("clone.#{pool_name}", finish)
422
+ rescue StandardError
423
+ @redis.with_metrics do |redis|
424
+ redis.pipelined do
425
+ redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
426
+ expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
427
+ redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
428
+ end
429
+ end
430
+ raise
431
+ ensure
432
+ @redis.with_metrics do |redis|
433
+ redis.decr('vmpooler__tasks__ondemandclone') if request_id
434
+ redis.decr('vmpooler__tasks__clone') unless request_id
435
+ end
436
+ end
340
437
  end
341
438
  end
342
439
 
@@ -357,36 +454,42 @@ module Vmpooler
357
454
  return if mutex.locked?
358
455
 
359
456
  mutex.synchronize do
360
- $redis.hdel('vmpooler__active__' + pool, vm)
361
- $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
457
+ @redis.with_metrics do |redis|
458
+ redis.pipelined do
459
+ redis.hdel('vmpooler__active__' + pool, vm)
460
+ redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
362
461
 
363
- # Auto-expire metadata key
364
- $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
462
+ # Auto-expire metadata key
463
+ redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
464
+ end
365
465
 
366
- start = Time.now
466
+ start = Time.now
367
467
 
368
- provider.destroy_vm(pool, vm)
468
+ provider.destroy_vm(pool, vm)
369
469
 
370
- $redis.srem('vmpooler__completed__' + pool, vm)
470
+ redis.srem('vmpooler__completed__' + pool, vm)
371
471
 
372
- finish = format('%<time>.2f', time: Time.now - start)
373
- $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
374
- $metrics.timing("destroy.#{pool}", finish)
375
- get_vm_usage_labels(vm)
472
+ finish = format('%<time>.2f', time: Time.now - start)
473
+ $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
474
+ $metrics.timing("destroy.#{pool}", finish)
475
+ get_vm_usage_labels(vm, redis)
476
+ end
376
477
  end
377
478
  dereference_mutex(vm)
378
479
  end
379
480
 
380
- def get_vm_usage_labels(vm)
481
+ def get_vm_usage_labels(vm, redis)
381
482
  return unless $config[:config]['usage_stats']
382
483
 
383
- checkout = $redis.hget("vmpooler__vm__#{vm}", 'checkout')
484
+ redis.multi
485
+ redis.hget("vmpooler__vm__#{vm}", 'checkout')
486
+ redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
487
+ redis.hget("vmpooler__vm__#{vm}", 'token:user')
488
+ redis.hget("vmpooler__vm__#{vm}", 'template')
489
+ checkout, jenkins_build_url, user, poolname = redis.exec
384
490
  return if checkout.nil?
385
491
 
386
- jenkins_build_url = $redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
387
- user = $redis.hget("vmpooler__vm__#{vm}", 'token:user') || 'unauthenticated'
388
- poolname = $redis.hget("vmpooler__vm__#{vm}", 'template')
389
-
492
+ user ||= 'unauthenticated'
390
493
  unless jenkins_build_url
391
494
  user = user.gsub('.', '_')
392
495
  $metrics.increment("usage.#{user}.#{poolname}")
@@ -420,7 +523,8 @@ module Vmpooler
420
523
 
421
524
  $metrics.increment(metric_parts.join('.'))
422
525
  rescue StandardError => e
423
- logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
526
+ $logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
527
+ raise
424
528
  end
425
529
 
426
530
  def component_to_test(match, labels_string)
@@ -444,7 +548,7 @@ module Vmpooler
444
548
  if provider_purge
445
549
  Thread.new do
446
550
  begin
447
- purge_vms_and_folders(provider.to_s)
551
+ purge_vms_and_folders($providers[provider.to_s])
448
552
  rescue StandardError => e
449
553
  $logger.log('s', "[!] failed while purging provider #{provider} VMs and folders with an error: #{e}")
450
554
  end
@@ -455,13 +559,14 @@ module Vmpooler
455
559
  end
456
560
 
457
561
  # Return a list of pool folders
458
- def pool_folders(provider_name)
562
+ def pool_folders(provider)
563
+ provider_name = provider.name
459
564
  folders = {}
460
565
  $config[:pools].each do |pool|
461
566
  next unless pool['provider'] == provider_name
462
567
 
463
568
  folder_parts = pool['folder'].split('/')
464
- datacenter = $providers[provider_name].get_target_datacenter_from_config(pool['name'])
569
+ datacenter = provider.get_target_datacenter_from_config(pool['name'])
465
570
  folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
466
571
  end
467
572
  folders
@@ -478,8 +583,8 @@ module Vmpooler
478
583
  def purge_vms_and_folders(provider)
479
584
  configured_folders = pool_folders(provider)
480
585
  base_folders = get_base_folders(configured_folders)
481
- whitelist = $providers[provider].provider_config['folder_whitelist']
482
- $providers[provider].purge_unconfigured_folders(base_folders, configured_folders, whitelist)
586
+ whitelist = provider.provider_config['folder_whitelist']
587
+ provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
483
588
  end
484
589
 
485
590
  def create_vm_disk(pool_name, vm, disk_size, provider)
@@ -505,10 +610,12 @@ module Vmpooler
505
610
  finish = format('%<time>.2f', time: Time.now - start)
506
611
 
507
612
  if result
508
- rdisks = $redis.hget('vmpooler__vm__' + vm_name, 'disk')
509
- disks = rdisks ? rdisks.split(':') : []
510
- disks.push("+#{disk_size}gb")
511
- $redis.hset('vmpooler__vm__' + vm_name, 'disk', disks.join(':'))
613
+ @redis.with_metrics do |redis|
614
+ rdisks = redis.hget('vmpooler__vm__' + vm_name, 'disk')
615
+ disks = rdisks ? rdisks.split(':') : []
616
+ disks.push("+#{disk_size}gb")
617
+ redis.hset('vmpooler__vm__' + vm_name, 'disk', disks.join(':'))
618
+ end
512
619
 
513
620
  $logger.log('s', "[+] [disk_manager] '#{vm_name}' attached #{disk_size}gb disk in #{finish} seconds")
514
621
  else
@@ -538,7 +645,9 @@ module Vmpooler
538
645
  finish = format('%<time>.2f', time: Time.now - start)
539
646
 
540
647
  if result
541
- $redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
648
+ @redis.with_metrics do |redis|
649
+ redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
650
+ end
542
651
  $logger.log('s', "[+] [snapshot_manager] '#{vm_name}' snapshot created in #{finish} seconds")
543
652
  else
544
653
  $logger.log('s', "[+] [snapshot_manager] Failed to snapshot '#{vm_name}'")
@@ -594,9 +703,9 @@ module Vmpooler
594
703
  @default_providers ||= %w[vsphere dummy]
595
704
  end
596
705
 
597
- def get_pool_name_for_vm(vm_name)
706
+ def get_pool_name_for_vm(vm_name, redis)
598
707
  # the 'template' is a bad name. Should really be 'poolname'
599
- $redis.hget('vmpooler__vm__' + vm_name, 'template')
708
+ redis.hget('vmpooler__vm__' + vm_name, 'template')
600
709
  end
601
710
 
602
711
  # @param pool_name [String] - the name of the pool
@@ -628,19 +737,21 @@ module Vmpooler
628
737
  end
629
738
 
630
739
  def _check_disk_queue
631
- task_detail = $redis.spop('vmpooler__tasks__disk')
632
- unless task_detail.nil?
633
- begin
634
- vm_name, disk_size = task_detail.split(':')
635
- pool_name = get_pool_name_for_vm(vm_name)
636
- raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
740
+ @redis.with_metrics do |redis|
741
+ task_detail = redis.spop('vmpooler__tasks__disk')
742
+ unless task_detail.nil?
743
+ begin
744
+ vm_name, disk_size = task_detail.split(':')
745
+ pool_name = get_pool_name_for_vm(vm_name, redis)
746
+ raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
637
747
 
638
- provider = get_provider_for_pool(pool_name)
639
- raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
748
+ provider = get_provider_for_pool(pool_name)
749
+ raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
640
750
 
641
- create_vm_disk(pool_name, vm_name, disk_size, provider)
642
- rescue StandardError => e
643
- $logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{e}")
751
+ create_vm_disk(pool_name, vm_name, disk_size, provider)
752
+ rescue StandardError => e
753
+ $logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{e}")
754
+ end
644
755
  end
645
756
  end
646
757
  end
@@ -664,37 +775,39 @@ module Vmpooler
664
775
  end
665
776
 
666
777
  def _check_snapshot_queue
667
- task_detail = $redis.spop('vmpooler__tasks__snapshot')
778
+ @redis.with_metrics do |redis|
779
+ task_detail = redis.spop('vmpooler__tasks__snapshot')
668
780
 
669
- unless task_detail.nil?
670
- begin
671
- vm_name, snapshot_name = task_detail.split(':')
672
- pool_name = get_pool_name_for_vm(vm_name)
673
- raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
781
+ unless task_detail.nil?
782
+ begin
783
+ vm_name, snapshot_name = task_detail.split(':')
784
+ pool_name = get_pool_name_for_vm(vm_name, redis)
785
+ raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
674
786
 
675
- provider = get_provider_for_pool(pool_name)
676
- raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
787
+ provider = get_provider_for_pool(pool_name)
788
+ raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
677
789
 
678
- create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
679
- rescue StandardError => e
680
- $logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{e}")
790
+ create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
791
+ rescue StandardError => e
792
+ $logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{e}")
793
+ end
681
794
  end
682
- end
683
795
 
684
- task_detail = $redis.spop('vmpooler__tasks__snapshot-revert')
796
+ task_detail = redis.spop('vmpooler__tasks__snapshot-revert')
685
797
 
686
- unless task_detail.nil?
687
- begin
688
- vm_name, snapshot_name = task_detail.split(':')
689
- pool_name = get_pool_name_for_vm(vm_name)
690
- raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
798
+ unless task_detail.nil?
799
+ begin
800
+ vm_name, snapshot_name = task_detail.split(':')
801
+ pool_name = get_pool_name_for_vm(vm_name, redis)
802
+ raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
691
803
 
692
- provider = get_provider_for_pool(pool_name)
693
- raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
804
+ provider = get_provider_for_pool(pool_name)
805
+ raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
694
806
 
695
- revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
696
- rescue StandardError => e
697
- $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{e}")
807
+ revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
808
+ rescue StandardError => e
809
+ $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{e}")
810
+ end
698
811
  end
699
812
  end
700
813
  end
@@ -704,7 +817,9 @@ module Vmpooler
704
817
  begin
705
818
  mutex = vm_mutex(vm_name)
706
819
  mutex.synchronize do
707
- $redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
820
+ @redis.with_metrics do |redis|
821
+ redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
822
+ end
708
823
  provider.migrate_vm(pool_name, vm_name)
709
824
  end
710
825
  rescue StandardError => e
@@ -737,47 +852,65 @@ module Vmpooler
737
852
  wakeup_by = Time.now + wakeup_period
738
853
  return if time_passed?(:exit_by, exit_by)
739
854
 
740
- initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
855
+ @redis.with_metrics do |redis|
856
+ initial_ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
741
857
 
742
- initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
858
+ initial_clone_target = redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
743
859
 
744
- initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
860
+ initial_template = redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
745
861
 
746
- loop do
747
- sleep(1)
748
- break if time_passed?(:exit_by, exit_by)
862
+ loop do
863
+ sleep(1)
864
+ break if time_passed?(:exit_by, exit_by)
749
865
 
750
- # Check for wakeup events
751
- if time_passed?(:wakeup_by, wakeup_by)
752
- wakeup_by = Time.now + wakeup_period
866
+ # Check for wakeup events
867
+ if time_passed?(:wakeup_by, wakeup_by)
868
+ wakeup_by = Time.now + wakeup_period
753
869
 
754
- # Wakeup if the number of ready VMs has changed
755
- if options[:pool_size_change]
756
- ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
757
- break unless ready_size == initial_ready_size
758
- end
870
+ # Wakeup if the number of ready VMs has changed
871
+ if options[:pool_size_change]
872
+ ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}")
873
+ break unless ready_size == initial_ready_size
874
+ end
759
875
 
760
- if options[:clone_target_change]
761
- clone_target = $redis.hget('vmpooler__config__clone_target}', options[:poolname])
762
- if clone_target
763
- break unless clone_target == initial_clone_target
876
+ if options[:clone_target_change]
877
+ clone_target = redis.hget('vmpooler__config__clone_target}', options[:poolname])
878
+ if clone_target
879
+ break unless clone_target == initial_clone_target
880
+ end
764
881
  end
765
- end
766
882
 
767
- if options[:pool_template_change]
768
- configured_template = $redis.hget('vmpooler__config__template', options[:poolname])
769
- if configured_template
770
- break unless initial_template == configured_template
883
+ if options[:pool_template_change]
884
+ configured_template = redis.hget('vmpooler__config__template', options[:poolname])
885
+ if configured_template
886
+ break unless initial_template == configured_template
887
+ end
888
+ end
889
+
890
+ if options[:pool_reset]
891
+ pending = redis.sismember('vmpooler__poolreset', options[:poolname])
892
+ break if pending
771
893
  end
772
- end
773
894
 
774
- if options[:pool_reset]
775
- break if $redis.sismember('vmpooler__poolreset', options[:poolname])
895
+ if options[:pending_vm]
896
+ pending_vm_count = redis.scard("vmpooler__pending__#{options[:poolname]}")
897
+ break unless pending_vm_count == 0
898
+ end
899
+
900
+ if options[:ondemand_request]
901
+ redis.multi
902
+ redis.zcard('vmpooler__provisioning__request')
903
+ redis.zcard('vmpooler__provisioning__processing')
904
+ redis.zcard('vmpooler__odcreate__task')
905
+ od_request, od_processing, od_createtask = redis.exec
906
+ break unless od_request == 0
907
+ break unless od_processing == 0
908
+ break unless od_createtask == 0
909
+ end
776
910
  end
777
911
 
912
+ break if time_passed?(:exit_by, exit_by)
778
913
  end
779
-
780
- break if time_passed?(:exit_by, exit_by)
781
914
  end
782
915
  end
783
916
 
@@ -813,7 +946,7 @@ module Vmpooler
813
946
  loop_delay = (loop_delay * loop_delay_decay).to_i
814
947
  loop_delay = loop_delay_max if loop_delay > loop_delay_max
815
948
  end
816
- sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pool_reset: true)
949
+ sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pending_vm: true, pool_reset: true)
817
950
 
818
951
  unless maxloop == 0
819
952
  break if loop_count >= maxloop
@@ -843,77 +976,84 @@ module Vmpooler
843
976
  end
844
977
 
845
978
  def sync_pool_template(pool)
846
- pool_template = $redis.hget('vmpooler__config__template', pool['name'])
847
- if pool_template
848
- pool['template'] = pool_template unless pool['template'] == pool_template
979
+ @redis.with_metrics do |redis|
980
+ pool_template = redis.hget('vmpooler__config__template', pool['name'])
981
+ if pool_template
982
+ pool['template'] = pool_template unless pool['template'] == pool_template
983
+ end
849
984
  end
850
985
  end
851
986
 
852
- def prepare_template(pool, provider)
987
+ def prepare_template(pool, provider, redis)
853
988
  if $config[:config]['create_template_delta_disks']
854
- unless $redis.sismember('vmpooler__template__deltas', pool['template'])
989
+ unless redis.sismember('vmpooler__template__deltas', pool['template'])
855
990
  begin
856
991
  provider.create_template_delta_disks(pool)
857
- $redis.sadd('vmpooler__template__deltas', pool['template'])
992
+ redis.sadd('vmpooler__template__deltas', pool['template'])
858
993
  rescue StandardError => e
859
994
  $logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{e}")
860
995
  end
861
996
  end
862
997
  end
863
- $redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
998
+ redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
864
999
  end
865
1000
 
866
1001
  def evaluate_template(pool, provider)
867
1002
  mutex = pool_mutex(pool['name'])
868
- prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
869
- configured_template = $redis.hget('vmpooler__config__template', pool['name'])
870
1003
  return if mutex.locked?
871
1004
 
872
- if prepared_template.nil?
873
- mutex.synchronize do
874
- prepare_template(pool, provider)
875
- prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
876
- end
877
- elsif prepared_template != pool['template']
878
- if configured_template.nil?
1005
+ catch :update_not_needed do
1006
+ @redis.with_metrics do |redis|
1007
+ prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
1008
+ configured_template = redis.hget('vmpooler__config__template', pool['name'])
1009
+
1010
+ if prepared_template.nil?
1011
+ mutex.synchronize do
1012
+ prepare_template(pool, provider, redis)
1013
+ prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
1014
+ end
1015
+ elsif prepared_template != pool['template']
1016
+ if configured_template.nil?
1017
+ mutex.synchronize do
1018
+ prepare_template(pool, provider, redis)
1019
+ prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
1020
+ end
1021
+ end
1022
+ end
1023
+ throw :update_not_needed if configured_template.nil?
1024
+ throw :update_not_needed if configured_template == prepared_template
1025
+
879
1026
  mutex.synchronize do
880
- prepare_template(pool, provider)
881
- prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
1027
+ update_pool_template(pool, provider, configured_template, prepared_template, redis)
882
1028
  end
883
1029
  end
884
1030
  end
885
- return if configured_template.nil?
886
- return if configured_template == prepared_template
887
-
888
- mutex.synchronize do
889
- update_pool_template(pool, provider, configured_template, prepared_template)
890
- end
891
1031
  end
892
1032
 
893
- def drain_pool(poolname)
1033
+ def drain_pool(poolname, redis)
894
1034
  # Clear a pool of ready and pending instances
895
- if $redis.scard("vmpooler__ready__#{poolname}") > 0
1035
+ if redis.scard("vmpooler__ready__#{poolname}") > 0
896
1036
  $logger.log('s', "[*] [#{poolname}] removing ready instances")
897
- $redis.smembers("vmpooler__ready__#{poolname}").each do |vm|
898
- move_vm_queue(poolname, vm, 'ready', 'completed')
1037
+ redis.smembers("vmpooler__ready__#{poolname}").each do |vm|
1038
+ move_vm_queue(poolname, vm, 'ready', 'completed', redis)
899
1039
  end
900
1040
  end
901
- if $redis.scard("vmpooler__pending__#{poolname}") > 0
1041
+ if redis.scard("vmpooler__pending__#{poolname}") > 0
902
1042
  $logger.log('s', "[*] [#{poolname}] removing pending instances")
903
- $redis.smembers("vmpooler__pending__#{poolname}").each do |vm|
904
- move_vm_queue(poolname, vm, 'pending', 'completed')
1043
+ redis.smembers("vmpooler__pending__#{poolname}").each do |vm|
1044
+ move_vm_queue(poolname, vm, 'pending', 'completed', redis)
905
1045
  end
906
1046
  end
907
1047
  end
908
1048
 
909
- def update_pool_template(pool, provider, configured_template, prepared_template)
1049
+ def update_pool_template(pool, provider, configured_template, prepared_template, redis)
910
1050
  pool['template'] = configured_template
911
1051
  $logger.log('s', "[*] [#{pool['name']}] template updated from #{prepared_template} to #{configured_template}")
912
1052
  # Remove all ready and pending VMs so new instances are created from the new template
913
- drain_pool(pool['name'])
1053
+ drain_pool(pool['name'], redis)
914
1054
  # Prepare template for deployment
915
1055
  $logger.log('s', "[*] [#{pool['name']}] preparing pool template for deployment")
916
- prepare_template(pool, provider)
1056
+ prepare_template(pool, provider, redis)
917
1057
  $logger.log('s', "[*] [#{pool['name']}] is ready for use")
918
1058
  end
919
1059
 
@@ -921,38 +1061,45 @@ module Vmpooler
921
1061
  mutex = pool_mutex(pool['name'])
922
1062
  return if mutex.locked?
923
1063
 
924
- clone_target = $redis.hget('vmpooler__config__clone_target', pool['name'])
925
- return if clone_target.nil?
926
- return if clone_target == pool['clone_target']
1064
+ @redis.with_metrics do |redis|
1065
+ clone_target = redis.hget('vmpooler__config__clone_target', pool['name'])
1066
+ break if clone_target.nil?
1067
+ break if clone_target == pool['clone_target']
927
1068
 
928
- $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
929
- mutex.synchronize do
930
- pool['clone_target'] = clone_target
931
- # Remove all ready and pending VMs so new instances are created for the new clone_target
932
- drain_pool(pool['name'])
1069
+ $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
1070
+ mutex.synchronize do
1071
+ pool['clone_target'] = clone_target
1072
+ # Remove all ready and pending VMs so new instances are created for the new clone_target
1073
+ drain_pool(pool['name'], redis)
1074
+ end
1075
+ $logger.log('s', "[*] [#{pool['name']}] is ready for use")
933
1076
  end
934
- $logger.log('s', "[*] [#{pool['name']}] is ready for use")
935
1077
  end
936
1078
 
937
1079
  def remove_excess_vms(pool)
938
- ready = $redis.scard("vmpooler__ready__#{pool['name']}")
939
- total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
940
- return if total.nil?
941
- return if total == 0
1080
+ @redis.with_metrics do |redis|
1081
+ redis.multi
1082
+ redis.scard("vmpooler__ready__#{pool['name']}")
1083
+ redis.scard("vmpooler__pending__#{pool['name']}")
1084
+ ready, pending = redis.exec
1085
+ total = pending.to_i + ready.to_i
1086
+ break if total.nil?
1087
+ break if total == 0
942
1088
 
943
- mutex = pool_mutex(pool['name'])
944
- return if mutex.locked?
945
- return unless ready > pool['size']
1089
+ mutex = pool_mutex(pool['name'])
1090
+ break if mutex.locked?
1091
+ break unless ready.to_i > pool['size']
946
1092
 
947
- mutex.synchronize do
948
- difference = ready - pool['size']
949
- difference.times do
950
- next_vm = $redis.spop("vmpooler__ready__#{pool['name']}")
951
- move_vm_queue(pool['name'], next_vm, 'ready', 'completed')
952
- end
953
- if total > ready
954
- $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
955
- move_vm_queue(pool['name'], vm, 'pending', 'completed')
1093
+ mutex.synchronize do
1094
+ difference = ready.to_i - pool['size']
1095
+ difference.times do
1096
+ next_vm = redis.spop("vmpooler__ready__#{pool['name']}")
1097
+ move_vm_queue(pool['name'], next_vm, 'ready', 'completed', redis)
1098
+ end
1099
+ if total > ready
1100
+ redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
1101
+ move_vm_queue(pool['name'], vm, 'pending', 'completed', redis)
1102
+ end
956
1103
  end
957
1104
  end
958
1105
  end
@@ -962,26 +1109,30 @@ module Vmpooler
962
1109
  mutex = pool_mutex(pool['name'])
963
1110
  return if mutex.locked?
964
1111
 
965
- poolsize = $redis.hget('vmpooler__config__poolsize', pool['name'])
966
- return if poolsize.nil?
1112
+ @redis.with_metrics do |redis|
1113
+ poolsize = redis.hget('vmpooler__config__poolsize', pool['name'])
1114
+ break if poolsize.nil?
967
1115
 
968
- poolsize = Integer(poolsize)
969
- return if poolsize == pool['size']
1116
+ poolsize = Integer(poolsize)
1117
+ break if poolsize == pool['size']
970
1118
 
971
- mutex.synchronize do
972
- pool['size'] = poolsize
1119
+ mutex.synchronize do
1120
+ pool['size'] = poolsize
1121
+ end
973
1122
  end
974
1123
  end
975
1124
 
976
1125
  def reset_pool(pool)
977
1126
  poolname = pool['name']
978
- return unless $redis.sismember('vmpooler__poolreset', poolname)
1127
+ @redis.with_metrics do |redis|
1128
+ break unless redis.sismember('vmpooler__poolreset', poolname)
979
1129
 
980
- $redis.srem('vmpooler__poolreset', poolname)
981
- mutex = pool_mutex(poolname)
982
- mutex.synchronize do
983
- drain_pool(poolname)
984
- $logger.log('s', "[*] [#{poolname}] reset has cleared ready and pending instances")
1130
+ redis.srem('vmpooler__poolreset', poolname)
1131
+ mutex = pool_mutex(poolname)
1132
+ mutex.synchronize do
1133
+ drain_pool(poolname, redis)
1134
+ $logger.log('s', "[*] [#{poolname}] reset has cleared ready and pending instances")
1135
+ end
985
1136
  end
986
1137
  end
987
1138
 
@@ -990,21 +1141,23 @@ module Vmpooler
990
1141
  begin
991
1142
  mutex = pool_mutex(pool['name'])
992
1143
  mutex.synchronize do
993
- provider.vms_in_pool(pool['name']).each do |vm|
994
- if !$redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
995
- !$redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
996
- !$redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) &&
997
- !$redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) &&
998
- !$redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) &&
999
- !$redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])
1000
-
1001
- pool_check_response[:discovered_vms] += 1
1002
- $redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
1003
-
1004
- $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
1144
+ @redis.with_metrics do |redis|
1145
+ provider.vms_in_pool(pool['name']).each do |vm|
1146
+ if !redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
1147
+ !redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
1148
+ !redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) &&
1149
+ !redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) &&
1150
+ !redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) &&
1151
+ !redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])
1152
+
1153
+ pool_check_response[:discovered_vms] += 1
1154
+ redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
1155
+
1156
+ $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
1157
+ end
1158
+
1159
+ inventory[vm['name']] = 1
1005
1160
  end
1006
-
1007
- inventory[vm['name']] = 1
1008
1161
  end
1009
1162
  end
1010
1163
  rescue StandardError => e
@@ -1015,96 +1168,112 @@ module Vmpooler
1015
1168
  end
1016
1169
 
1017
1170
  def check_running_pool_vms(pool_name, provider, pool_check_response, inventory)
1018
- $redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
1019
- if inventory[vm]
1020
- begin
1021
- vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
1022
- pool_check_response[:checked_running_vms] += 1
1023
- check_running_vm(vm, pool_name, vm_lifetime, provider)
1024
- rescue StandardError => e
1025
- $logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{e}")
1171
+ @redis.with_metrics do |redis|
1172
+ redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
1173
+ if inventory[vm]
1174
+ begin
1175
+ vm_lifetime = redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
1176
+ pool_check_response[:checked_running_vms] += 1
1177
+ check_running_vm(vm, pool_name, vm_lifetime, provider)
1178
+ rescue StandardError => e
1179
+ $logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{e}")
1180
+ end
1181
+ else
1182
+ move_vm_queue(pool_name, vm, 'running', 'completed', redis, 'is a running VM but is missing from inventory. Marking as completed.')
1026
1183
  end
1027
- else
1028
- move_vm_queue(pool_name, vm, 'running', 'completed', 'is a running VM but is missing from inventory. Marking as completed.')
1029
1184
  end
1030
1185
  end
1031
1186
  end
1032
1187
 
1033
- def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl = 0)
1034
- $redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
1035
- if inventory[vm]
1036
- begin
1037
- pool_check_response[:checked_ready_vms] += 1
1038
- check_ready_vm(vm, pool_name, pool_ttl || 0, provider)
1039
- rescue StandardError => e
1040
- $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
1188
+ def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl)
1189
+ @redis.with_metrics do |redis|
1190
+ redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
1191
+ if inventory[vm]
1192
+ begin
1193
+ pool_check_response[:checked_ready_vms] += 1
1194
+ check_ready_vm(vm, pool_name, pool_ttl, provider)
1195
+ rescue StandardError => e
1196
+ $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
1197
+ end
1198
+ else
1199
+ move_vm_queue(pool_name, vm, 'ready', 'completed', redis, 'is a ready VM but is missing from inventory. Marking as completed.')
1041
1200
  end
1042
- else
1043
- move_vm_queue(pool_name, vm, 'ready', 'completed', 'is a ready VM but is missing from inventory. Marking as completed.')
1044
1201
  end
1045
1202
  end
1046
1203
  end
1047
1204
 
1048
- def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout = nil)
1205
+ def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout)
1049
1206
  pool_timeout ||= $config[:config]['timeout'] || 15
1050
- $redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
1051
- if inventory[vm]
1052
- begin
1053
- pool_check_response[:checked_pending_vms] += 1
1054
- check_pending_vm(vm, pool_name, pool_timeout, provider)
1055
- rescue StandardError => e
1056
- $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
1207
+ @redis.with_metrics do |redis|
1208
+ redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
1209
+ if inventory[vm]
1210
+ begin
1211
+ pool_check_response[:checked_pending_vms] += 1
1212
+ check_pending_vm(vm, pool_name, pool_timeout, provider)
1213
+ rescue StandardError => e
1214
+ $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
1215
+ end
1216
+ else
1217
+ fail_pending_vm(vm, pool_name, pool_timeout, redis, false)
1057
1218
  end
1058
- else
1059
- fail_pending_vm(vm, pool_name, pool_timeout, false)
1060
1219
  end
1061
1220
  end
1062
1221
  end
1063
1222
 
1064
1223
  def check_completed_pool_vms(pool_name, provider, pool_check_response, inventory)
1065
- $redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
1066
- if inventory[vm]
1067
- begin
1068
- pool_check_response[:destroyed_vms] += 1
1069
- destroy_vm(vm, pool_name, provider)
1070
- rescue StandardError => e
1071
- $redis.srem("vmpooler__completed__#{pool_name}", vm)
1072
- $redis.hdel("vmpooler__active__#{pool_name}", vm)
1073
- $redis.del("vmpooler__vm__#{vm}")
1074
- $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
1224
+ @redis.with_metrics do |redis|
1225
+ redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
1226
+ if inventory[vm]
1227
+ begin
1228
+ pool_check_response[:destroyed_vms] += 1
1229
+ destroy_vm(vm, pool_name, provider)
1230
+ rescue StandardError => e
1231
+ redis.pipelined do
1232
+ redis.srem("vmpooler__completed__#{pool_name}", vm)
1233
+ redis.hdel("vmpooler__active__#{pool_name}", vm)
1234
+ redis.del("vmpooler__vm__#{vm}")
1235
+ end
1236
+ $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
1237
+ end
1238
+ else
1239
+ $logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
1240
+ redis.pipelined do
1241
+ redis.srem("vmpooler__completed__#{pool_name}", vm)
1242
+ redis.hdel("vmpooler__active__#{pool_name}", vm)
1243
+ redis.del("vmpooler__vm__#{vm}")
1244
+ end
1075
1245
  end
1076
- else
1077
- $logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
1078
- $redis.srem("vmpooler__completed__#{pool_name}", vm)
1079
- $redis.hdel("vmpooler__active__#{pool_name}", vm)
1080
- $redis.del("vmpooler__vm__#{vm}")
1081
1246
  end
1082
1247
  end
1083
1248
  end
1084
1249
 
1085
1250
  def check_discovered_pool_vms(pool_name)
1086
- $redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
1087
- %w[pending ready running completed].each do |queue|
1088
- if $redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
1089
- $logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
1090
- $redis.srem("vmpooler__discovered__#{pool_name}", vm)
1251
+ @redis.with_metrics do |redis|
1252
+ redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
1253
+ %w[pending ready running completed].each do |queue|
1254
+ if redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
1255
+ $logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
1256
+ redis.srem("vmpooler__discovered__#{pool_name}", vm)
1257
+ end
1091
1258
  end
1092
- end
1093
1259
 
1094
- $redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm) if $redis.sismember("vmpooler__discovered__#{pool_name}", vm)
1260
+ redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm) if redis.sismember("vmpooler__discovered__#{pool_name}", vm)
1261
+ end
1095
1262
  end
1096
1263
  rescue StandardError => e
1097
1264
  $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating discovered VMs: #{e}")
1098
1265
  end
1099
1266
 
1100
1267
  def check_migrating_pool_vms(pool_name, provider, pool_check_response, inventory)
1101
- $redis.smembers("vmpooler__migrating__#{pool_name}").reverse.each do |vm|
1102
- if inventory[vm]
1103
- begin
1104
- pool_check_response[:migrated_vms] += 1
1105
- migrate_vm(vm, pool_name, provider)
1106
- rescue StandardError => e
1107
- $logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{e}")
1268
+ @redis.with_metrics do |redis|
1269
+ redis.smembers("vmpooler__migrating__#{pool_name}").reverse.each do |vm|
1270
+ if inventory[vm]
1271
+ begin
1272
+ pool_check_response[:migrated_vms] += 1
1273
+ migrate_vm(vm, pool_name, provider)
1274
+ rescue StandardError => e
1275
+ $logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{e}")
1276
+ end
1108
1277
  end
1109
1278
  end
1110
1279
  end
@@ -1113,29 +1282,37 @@ module Vmpooler
1113
1282
  def repopulate_pool_vms(pool_name, provider, pool_check_response, pool_size)
1114
1283
  return if pool_mutex(pool_name).locked?
1115
1284
 
1116
- ready = $redis.scard("vmpooler__ready__#{pool_name}")
1117
- total = $redis.scard("vmpooler__pending__#{pool_name}") + ready
1118
-
1119
- $metrics.gauge("ready.#{pool_name}", $redis.scard("vmpooler__ready__#{pool_name}"))
1120
- $metrics.gauge("running.#{pool_name}", $redis.scard("vmpooler__running__#{pool_name}"))
1121
-
1122
- if $redis.get("vmpooler__empty__#{pool_name}")
1123
- $redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
1124
- elsif ready == 0
1125
- $redis.set("vmpooler__empty__#{pool_name}", 'true')
1126
- $logger.log('s', "[!] [#{pool_name}] is empty")
1127
- end
1285
+ @redis.with_metrics do |redis|
1286
+ redis.multi
1287
+ redis.scard("vmpooler__ready__#{pool_name}")
1288
+ redis.scard("vmpooler__pending__#{pool_name}")
1289
+ redis.scard("vmpooler__running__#{pool_name}")
1290
+ ready, pending, running = redis.exec
1291
+ total = pending.to_i + ready.to_i
1292
+
1293
+ $metrics.gauge("ready.#{pool_name}", ready)
1294
+ $metrics.gauge("running.#{pool_name}", running)
1295
+
1296
+ unless pool_size == 0
1297
+ if redis.get("vmpooler__empty__#{pool_name}")
1298
+ redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
1299
+ elsif ready == 0
1300
+ redis.set("vmpooler__empty__#{pool_name}", 'true')
1301
+ $logger.log('s', "[!] [#{pool_name}] is empty")
1302
+ end
1303
+ end
1128
1304
 
1129
- (pool_size - total).times do
1130
- if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
1131
- begin
1132
- $redis.incr('vmpooler__tasks__clone')
1133
- pool_check_response[:cloned_vms] += 1
1134
- clone_vm(pool_name, provider)
1135
- rescue StandardError => e
1136
- $logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
1137
- $redis.decr('vmpooler__tasks__clone')
1138
- raise
1305
+ (pool_size - total.to_i).times do
1306
+ if redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
1307
+ begin
1308
+ redis.incr('vmpooler__tasks__clone')
1309
+ pool_check_response[:cloned_vms] += 1
1310
+ clone_vm(pool_name, provider)
1311
+ rescue StandardError => e
1312
+ $logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
1313
+ redis.decr('vmpooler__tasks__clone')
1314
+ raise
1315
+ end
1139
1316
  end
1140
1317
  end
1141
1318
  end
@@ -1160,7 +1337,7 @@ module Vmpooler
1160
1337
 
1161
1338
  check_running_pool_vms(pool['name'], provider, pool_check_response, inventory)
1162
1339
 
1163
- check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'])
1340
+ check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
1164
1341
 
1165
1342
  check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
1166
1343
 
@@ -1203,23 +1380,203 @@ module Vmpooler
1203
1380
  #
1204
1381
  # returns an object Vmpooler::PoolManager::Provider::*
1205
1382
  # or raises an error if the class does not exist
1206
- def create_provider_object(config, logger, metrics, provider_class, provider_name, options)
1383
+ def create_provider_object(config, logger, metrics, redis_connection_pool, provider_class, provider_name, options)
1207
1384
  provider_klass = Vmpooler::PoolManager::Provider
1208
1385
  provider_klass.constants.each do |classname|
1209
1386
  next unless classname.to_s.casecmp(provider_class) == 0
1210
1387
 
1211
- return provider_klass.const_get(classname).new(config, logger, metrics, provider_name, options)
1388
+ return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options)
1212
1389
  end
1213
1390
  raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
1214
1391
  end
1215
1392
 
1393
+ def check_ondemand_requests(maxloop = 0,
1394
+ loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
1395
+ loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
1396
+ loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
1397
+
1398
+ $logger.log('d', '[*] [ondemand_provisioner] starting worker thread')
1399
+
1400
+ $threads['ondemand_provisioner'] = Thread.new do
1401
+ _check_ondemand_requests(maxloop, loop_delay_min, loop_delay_max, loop_delay_decay)
1402
+ end
1403
+ end
1404
+
1405
+ def _check_ondemand_requests(maxloop = 0,
1406
+ loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
1407
+ loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
1408
+ loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
1409
+
1410
+ loop_delay_min = $config[:config]['check_loop_delay_min'] unless $config[:config]['check_loop_delay_min'].nil?
1411
+ loop_delay_max = $config[:config]['check_loop_delay_max'] unless $config[:config]['check_loop_delay_max'].nil?
1412
+ loop_delay_decay = $config[:config]['check_loop_delay_decay'] unless $config[:config]['check_loop_delay_decay'].nil?
1413
+
1414
+ loop_delay_decay = 2.0 if loop_delay_decay <= 1.0
1415
+ loop_delay_max = loop_delay_min if loop_delay_max.nil? || loop_delay_max < loop_delay_min
1416
+
1417
+ loop_count = 1
1418
+ loop_delay = loop_delay_min
1419
+
1420
+ loop do
1421
+ result = process_ondemand_requests
1422
+
1423
+ loop_delay = (loop_delay * loop_delay_decay).to_i
1424
+ loop_delay = loop_delay_min if result > 0
1425
+ loop_delay = loop_delay_max if loop_delay > loop_delay_max
1426
+ sleep_with_wakeup_events(loop_delay, loop_delay_min, ondemand_request: true)
1427
+
1428
+ unless maxloop == 0
1429
+ break if loop_count >= maxloop
1430
+
1431
+ loop_count += 1
1432
+ end
1433
+ end
1434
+ end
1435
+
1436
+ def process_ondemand_requests
1437
+ @redis.with_metrics do |redis|
1438
+ requests = redis.zrange('vmpooler__provisioning__request', 0, -1)
1439
+ requests&.map { |request_id| create_ondemand_vms(request_id, redis) }
1440
+ provisioning_tasks = process_ondemand_vms(redis)
1441
+ requests_ready = check_ondemand_requests_ready(redis)
1442
+ requests.length + provisioning_tasks + requests_ready
1443
+ end
1444
+ end
1445
+
1446
+ def create_ondemand_vms(request_id, redis)
1447
+ requested = redis.hget("vmpooler__odrequest__#{request_id}", 'requested')
1448
+ unless requested
1449
+ $logger.log('s', "Failed to find odrequest for request_id '#{request_id}'")
1450
+ redis.zrem('vmpooler__provisioning__request', request_id)
1451
+ return
1452
+ end
1453
+ score = redis.zscore('vmpooler__provisioning__request', request_id)
1454
+ requested = requested.split(',')
1455
+
1456
+ redis.pipelined do
1457
+ requested.each do |request|
1458
+ redis.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
1459
+ end
1460
+ redis.zrem('vmpooler__provisioning__request', request_id)
1461
+ redis.zadd('vmpooler__provisioning__processing', score, request_id)
1462
+ end
1463
+ end
1464
+
1465
+ def process_ondemand_vms(redis)
1466
+ queue_key = 'vmpooler__odcreate__task'
1467
+ queue = redis.zrange(queue_key, 0, -1, with_scores: true)
1468
+ ondemand_clone_limit = $config[:config]['ondemand_clone_limit']
1469
+ queue.each do |request, score|
1470
+ clone_count = redis.get('vmpooler__tasks__ondemandclone').to_i
1471
+ break unless clone_count < ondemand_clone_limit
1472
+
1473
+ pool_alias, pool, count, request_id = request.split(':')
1474
+ count = count.to_i
1475
+ provider = get_provider_for_pool(pool)
1476
+ slots = ondemand_clone_limit - clone_count
1477
+ break if slots == 0
1478
+
1479
+ if slots >= count
1480
+ count.times do
1481
+ redis.incr('vmpooler__tasks__ondemandclone')
1482
+ clone_vm(pool, provider, request_id, pool_alias)
1483
+ end
1484
+ redis.zrem(queue_key, request)
1485
+ else
1486
+ remaining_count = count - slots
1487
+ slots.times do
1488
+ redis.incr('vmpooler__tasks__ondemandclone')
1489
+ clone_vm(pool, provider, request_id, pool_alias)
1490
+ end
1491
+ redis.pipelined do
1492
+ redis.zrem(queue_key, request)
1493
+ redis.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
1494
+ end
1495
+ end
1496
+ end
1497
+ queue.length
1498
+ end
1499
+
1500
+ def vms_ready?(request_id, redis)
1501
+ catch :request_not_ready do
1502
+ request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
1503
+ requested_platforms = request_hash['requested'].split(',')
1504
+ requested_platforms.each do |platform|
1505
+ platform_alias, pool, count = platform.split(':')
1506
+ pools_filled = redis.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1507
+ throw :request_not_ready unless pools_filled.to_i == count.to_i
1508
+ end
1509
+ return true
1510
+ end
1511
+ false
1512
+ end
1513
+
1514
+ def check_ondemand_requests_ready(redis)
1515
+ in_progress_requests = redis.zrange('vmpooler__provisioning__processing', 0, -1, with_scores: true)
1516
+ in_progress_requests&.each do |request_id, score|
1517
+ check_ondemand_request_ready(request_id, redis, score)
1518
+ end
1519
+ in_progress_requests.length
1520
+ end
1521
+
1522
+ def check_ondemand_request_ready(request_id, redis, score = nil)
1523
+ # default expiration is one month to ensure the data does not stay in redis forever
1524
+ default_expiration = 259_200_0
1525
+ processing_key = 'vmpooler__provisioning__processing'
1526
+ ondemand_hash_key = "vmpooler__odrequest__#{request_id}"
1527
+ score ||= redis.zscore(processing_key, request_id)
1528
+ return if request_expired?(request_id, score, redis)
1529
+
1530
+ return unless vms_ready?(request_id, redis)
1531
+
1532
+ redis.multi
1533
+ redis.hset(ondemand_hash_key, 'status', 'ready')
1534
+ redis.expire(ondemand_hash_key, default_expiration)
1535
+ redis.zrem(processing_key, request_id)
1536
+ redis.exec
1537
+ end
1538
+
1539
+ def request_expired?(request_id, score, redis)
1540
+ delta = Time.now.to_i - score.to_i
1541
+ ondemand_request_ttl = $config[:config]['ondemand_request_ttl']
1542
+ return false unless delta > ondemand_request_ttl * 60
1543
+
1544
+ $logger.log('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
1545
+ expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
1546
+ redis.pipelined do
1547
+ redis.zrem('vmpooler__provisioning__processing', request_id)
1548
+ redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
1549
+ redis.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
1550
+ end
1551
+ remove_vms_for_failed_request(request_id, expiration_ttl, redis)
1552
+ true
1553
+ end
1554
+
1555
+ def remove_vms_for_failed_request(request_id, expiration_ttl, redis)
1556
+ request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
1557
+ requested_platforms = request_hash['requested'].split(',')
1558
+ requested_platforms.each do |platform|
1559
+ platform_alias, pool, _count = platform.split(':')
1560
+ pools_filled = redis.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1561
+ redis.pipelined do
1562
+ pools_filled&.each do |vm|
1563
+ move_vm_queue(pool, vm, 'running', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
1564
+ end
1565
+ redis.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
1566
+ end
1567
+ end
1568
+ end
1569
+
1216
1570
  def execute!(maxloop = 0, loop_delay = 1)
1217
1571
  $logger.log('d', 'starting vmpooler')
1218
1572
 
1219
- # Clear out the tasks manager, as we don't know about any tasks at this point
1220
- $redis.set('vmpooler__tasks__clone', 0)
1221
- # Clear out vmpooler__migrations since stale entries may be left after a restart
1222
- $redis.del('vmpooler__migration')
1573
+ @redis.with_metrics do |redis|
1574
+ # Clear out the tasks manager, as we don't know about any tasks at this point
1575
+ redis.set('vmpooler__tasks__clone', 0)
1576
+ redis.set('vmpooler__tasks__ondemandclone', 0)
1577
+ # Clear out vmpooler__migrations since stale entries may be left after a restart
1578
+ redis.del('vmpooler__migration')
1579
+ end
1223
1580
 
1224
1581
  # Copy vSphere settings to correct location. This happens with older configuration files
1225
1582
  if !$config[:vsphere].nil? && ($config[:providers].nil? || $config[:providers][:vsphere].nil?)
@@ -1269,7 +1626,7 @@ module Vmpooler
1269
1626
  provider_class = $config[:providers][provider_name.to_sym]['provider_class']
1270
1627
  end
1271
1628
  begin
1272
- $providers[provider_name] = create_provider_object($config, $logger, $metrics, provider_class, provider_name, {}) if $providers[provider_name].nil?
1629
+ $providers[provider_name] = create_provider_object($config, $logger, $metrics, @redis, provider_class, provider_name, {}) if $providers[provider_name].nil?
1273
1630
  rescue StandardError => e
1274
1631
  $logger.log('s', "Error while creating provider for pool #{pool['name']}: #{e}")
1275
1632
  raise
@@ -1303,6 +1660,13 @@ module Vmpooler
1303
1660
  end
1304
1661
  end
1305
1662
 
1663
+ if !$threads['ondemand_provisioner']
1664
+ check_ondemand_requests
1665
+ elsif !$threads['ondemand_provisioner'].alive?
1666
+ $logger.log('d', '[!] [ondemand_provisioner] worker thread died, restarting')
1667
+ check_ondemand_requests(check_loop_delay_min, check_loop_delay_max, check_loop_delay_decay)
1668
+ end
1669
+
1306
1670
  sleep(loop_delay)
1307
1671
 
1308
1672
  unless maxloop == 0