vmpooler 0.11.2 → 0.13.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -15,35 +15,17 @@ module Vmpooler
15
15
  @metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == ''
16
16
  end
17
17
 
18
- if Thread.respond_to?(:handle_interrupt)
19
- # MRI
20
- def with_metrics(options = {})
21
- Thread.handle_interrupt(Exception => :never) do
22
- start = Time.now
23
- conn = checkout(options)
24
- timespan_ms = ((Time.now - start) * 1000).to_i
25
- @metrics&.gauge(@metric_prefix + '.available', @available.length)
26
- @metrics&.timing(@metric_prefix + '.waited', timespan_ms)
27
- begin
28
- Thread.handle_interrupt(Exception => :immediate) do
29
- yield conn
30
- end
31
- ensure
32
- checkin
33
- @metrics&.gauge(@metric_prefix + '.available', @available.length)
34
- end
35
- end
36
- end
37
- else
38
- # jruby 1.7.x
39
- def with_metrics(options = {})
18
+ def with_metrics(options = {})
19
+ Thread.handle_interrupt(Exception => :never) do
40
20
  start = Time.now
41
21
  conn = checkout(options)
42
22
  timespan_ms = ((Time.now - start) * 1000).to_i
43
23
  @metrics&.gauge(@metric_prefix + '.available', @available.length)
44
24
  @metrics&.timing(@metric_prefix + '.waited', timespan_ms)
45
25
  begin
46
- yield conn
26
+ Thread.handle_interrupt(Exception => :immediate) do
27
+ yield conn
28
+ end
47
29
  ensure
48
30
  checkin
49
31
  @metrics&.gauge(@metric_prefix + '.available', @available.length)
@@ -35,6 +35,8 @@ module Vmpooler
35
35
  socket.close
36
36
  end
37
37
  end
38
+ rescue Errno::EADDRNOTAVAIL => e
39
+ warn "Could not assign address to graphite server #{server}: #{e}"
38
40
  rescue StandardError => e
39
41
  warn "Failure logging #{path} to graphite server [#{server}:#{port}]: #{e}"
40
42
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'vmpooler/providers'
4
4
  require 'spicy-proton'
5
+ require 'resolv' # ruby standard lib
5
6
 
6
7
  module Vmpooler
7
8
  class PoolManager
@@ -9,7 +10,7 @@ module Vmpooler
9
10
  CHECK_LOOP_DELAY_MAX_DEFAULT = 60
10
11
  CHECK_LOOP_DELAY_DECAY_DEFAULT = 2.0
11
12
 
12
- def initialize(config, logger, redis, metrics)
13
+ def initialize(config, logger, redis_connection_pool, metrics)
13
14
  $config = config
14
15
 
15
16
  # Load logger library
@@ -18,19 +19,19 @@ module Vmpooler
18
19
  # metrics logging handle
19
20
  $metrics = metrics
20
21
 
21
- # Connect to Redis
22
- $redis = redis
22
+ # Redis connection pool
23
+ @redis = redis_connection_pool
23
24
 
24
25
  # VM Provider objects
25
- $providers = {}
26
+ $providers = Concurrent::Hash.new
26
27
 
27
28
  # Our thread-tracker object
28
- $threads = {}
29
+ $threads = Concurrent::Hash.new
29
30
 
30
31
  # Pool mutex
31
- @reconfigure_pool = {}
32
+ @reconfigure_pool = Concurrent::Hash.new
32
33
 
33
- @vm_mutex = {}
34
+ @vm_mutex = Concurrent::Hash.new
34
35
 
35
36
  # Name generator for generating host names
36
37
  @name_generator = Spicy::Proton.new
@@ -45,24 +46,26 @@ module Vmpooler
45
46
 
46
47
  # Place pool configuration in redis so an API instance can discover running pool configuration
47
48
  def load_pools_to_redis
48
- previously_configured_pools = $redis.smembers('vmpooler__pools')
49
- currently_configured_pools = []
50
- config[:pools].each do |pool|
51
- currently_configured_pools << pool['name']
52
- $redis.sadd('vmpooler__pools', pool['name'])
53
- pool_keys = pool.keys
54
- pool_keys.delete('alias')
55
- to_set = {}
56
- pool_keys.each do |k|
57
- to_set[k] = pool[k]
49
+ @redis.with_metrics do |redis|
50
+ previously_configured_pools = redis.smembers('vmpooler__pools')
51
+ currently_configured_pools = []
52
+ config[:pools].each do |pool|
53
+ currently_configured_pools << pool['name']
54
+ redis.sadd('vmpooler__pools', pool['name'])
55
+ pool_keys = pool.keys
56
+ pool_keys.delete('alias')
57
+ to_set = {}
58
+ pool_keys.each do |k|
59
+ to_set[k] = pool[k]
60
+ end
61
+ to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
62
+ redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
58
63
  end
59
- to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
60
- $redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
61
- end
62
- previously_configured_pools.each do |pool|
63
- unless currently_configured_pools.include? pool
64
- $redis.srem('vmpooler__pools', pool)
65
- $redis.del("vmpooler__pool__#{pool}")
64
+ previously_configured_pools.each do |pool|
65
+ unless currently_configured_pools.include? pool
66
+ redis.srem('vmpooler__pools', pool)
67
+ redis.del("vmpooler__pool__#{pool}")
68
+ end
66
69
  end
67
70
  end
68
71
  nil
@@ -75,7 +78,9 @@ module Vmpooler
75
78
  _check_pending_vm(vm, pool, timeout, provider)
76
79
  rescue StandardError => e
77
80
  $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
78
- fail_pending_vm(vm, pool, timeout)
81
+ @redis.with_metrics do |redis|
82
+ fail_pending_vm(vm, pool, timeout, redis)
83
+ end
79
84
  raise
80
85
  end
81
86
  end
@@ -86,31 +91,38 @@ module Vmpooler
86
91
  return if mutex.locked?
87
92
 
88
93
  mutex.synchronize do
89
- if provider.vm_ready?(pool, vm)
90
- move_pending_vm_to_ready(vm, pool)
91
- else
92
- fail_pending_vm(vm, pool, timeout)
94
+ @redis.with_metrics do |redis|
95
+ request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
96
+ if provider.vm_ready?(pool, vm)
97
+ move_pending_vm_to_ready(vm, pool, redis, request_id)
98
+ else
99
+ fail_pending_vm(vm, pool, timeout, redis)
100
+ end
93
101
  end
94
102
  end
95
103
  end
96
104
 
97
- def remove_nonexistent_vm(vm, pool)
98
- $redis.srem("vmpooler__pending__#{pool}", vm)
105
+ def remove_nonexistent_vm(vm, pool, redis)
106
+ redis.srem("vmpooler__pending__#{pool}", vm)
99
107
  $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
100
108
  end
101
109
 
102
- def fail_pending_vm(vm, pool, timeout, exists = true)
103
- clone_stamp = $redis.hget("vmpooler__vm__#{vm}", 'clone')
104
- return true unless clone_stamp
110
+ def fail_pending_vm(vm, pool, timeout, redis, exists = true)
111
+ clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
105
112
 
106
113
  time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
107
114
  if time_since_clone > timeout
108
115
  if exists
109
- $redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm)
116
+ request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
117
+ pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
118
+ redis.multi
119
+ redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm)
120
+ redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}") if request_id
121
+ redis.exec
110
122
  $metrics.increment("errors.markedasfailed.#{pool}")
111
123
  $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
112
124
  else
113
- remove_nonexistent_vm(vm, pool)
125
+ remove_nonexistent_vm(vm, pool, redis)
114
126
  end
115
127
  end
116
128
  true
@@ -119,28 +131,54 @@ module Vmpooler
119
131
  false
120
132
  end
121
133
 
122
- def move_pending_vm_to_ready(vm, pool)
123
- clone_time = $redis.hget('vmpooler__vm__' + vm, 'clone')
124
- finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time)) if clone_time
134
+ def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
135
+ clone_time = redis.hget('vmpooler__vm__' + vm, 'clone')
136
+ finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
125
137
 
126
- $redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
127
- $redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
128
- $redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
138
+ if request_id
139
+ ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
140
+ if ondemandrequest_hash['status'] == 'failed'
141
+ move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
142
+ return nil
143
+ elsif ondemandrequest_hash['status'] == 'deleted'
144
+ move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' has been deleted")
145
+ return nil
146
+ end
147
+ pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
148
+
149
+ redis.pipelined do
150
+ redis.hset("vmpooler__active__#{pool}", vm, Time.now)
151
+ redis.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
152
+ redis.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token']) if ondemandrequest_hash['token:token']
153
+ redis.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user']) if ondemandrequest_hash['token:user']
154
+ redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
155
+ end
156
+ move_vm_queue(pool, vm, 'pending', 'running', redis)
157
+ check_ondemand_request_ready(request_id, redis)
158
+ else
159
+ redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
160
+ end
129
161
 
130
- # last boot time is displayed in API, and used by alarming script
131
- $redis.hset('vmpooler__lastboot', pool, Time.now)
162
+ redis.pipelined do
163
+ redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
164
+ redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
165
+
166
+ # last boot time is displayed in API, and used by alarming script
167
+ redis.hset('vmpooler__lastboot', pool, Time.now)
168
+ end
132
169
 
133
170
  $metrics.timing("time_to_ready_state.#{pool}", finish)
134
- $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
171
+ $logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") unless request_id
172
+ $logger.log('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'") if request_id
135
173
  end
136
174
 
137
- def vm_still_ready?(pool_name, vm_name, provider)
175
+ def vm_still_ready?(pool_name, vm_name, provider, redis)
138
176
  # Check if the VM is still ready/available
139
177
  return true if provider.vm_ready?(pool_name, vm_name)
140
178
 
141
179
  raise("VM #{vm_name} is not ready")
142
180
  rescue StandardError
143
- move_vm_queue(pool_name, vm_name, 'ready', 'completed', "is unreachable, removed from 'ready' queue")
181
+ move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "is unreachable, removed from 'ready' queue")
144
182
  end
145
183
 
146
184
  def check_ready_vm(vm, pool_name, ttl, provider)
@@ -160,34 +198,35 @@ module Vmpooler
160
198
  return if mutex.locked?
161
199
 
162
200
  mutex.synchronize do
163
- check_stamp = $redis.hget('vmpooler__vm__' + vm, 'check')
164
- return if check_stamp && (((Time.now - Time.parse(check_stamp)) / 60) <= $config[:config]['vm_checktime'])
201
+ @redis.with_metrics do |redis|
202
+ check_stamp = redis.hget('vmpooler__vm__' + vm, 'check')
203
+ last_checked_too_soon = ((Time.now - Time.parse(check_stamp)).to_i < $config[:config]['vm_checktime'] * 60) if check_stamp
204
+ break if check_stamp && last_checked_too_soon
165
205
 
166
- $redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
167
- # Check if the hosts TTL has expired
168
- if ttl > 0
206
+ redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
207
+ # Check if the hosts TTL has expired
169
208
  # if 'boottime' is nil, set bootime to beginning of unix epoch, forces TTL to be assumed expired
170
- boottime = $redis.hget("vmpooler__vm__#{vm}", 'ready')
209
+ boottime = redis.hget("vmpooler__vm__#{vm}", 'ready')
171
210
  if boottime
172
211
  boottime = Time.parse(boottime)
173
212
  else
174
213
  boottime = Time.at(0)
175
214
  end
176
- if ((Time.now - boottime) / 60).to_s[/^\d+\.\d{1}/].to_f > ttl
177
- $redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
215
+ if (Time.now - boottime).to_i > ttl * 60
216
+ redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
178
217
 
179
218
  $logger.log('d', "[!] [#{pool_name}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
180
- return
219
+ return nil
181
220
  end
182
- end
183
221
 
184
- return if mismatched_hostname?(vm, pool_name, provider)
222
+ break if mismatched_hostname?(vm, pool_name, provider, redis)
185
223
 
186
- vm_still_ready?(pool_name, vm, provider)
224
+ vm_still_ready?(pool_name, vm, provider, redis)
225
+ end
187
226
  end
188
227
  end
189
228
 
190
- def mismatched_hostname?(vm, pool_name, provider)
229
+ def mismatched_hostname?(vm, pool_name, provider, redis)
191
230
  pool_config = $config[:pools][$config[:pool_index][pool_name]]
192
231
  check_hostname = pool_config['check_hostname_for_mismatch']
193
232
  check_hostname = $config[:config]['check_ready_vm_hostname_for_mismatch'] if check_hostname.nil?
@@ -196,7 +235,7 @@ module Vmpooler
196
235
  # Wait one minute before checking a VM for hostname mismatch
197
236
  # When checking as soon as the VM passes the ready test the instance
198
237
  # often doesn't report its hostname yet causing the VM to be removed immediately
199
- vm_ready_time = $redis.hget("vmpooler__vm__#{vm}", 'ready')
238
+ vm_ready_time = redis.hget("vmpooler__vm__#{vm}", 'ready')
200
239
  if vm_ready_time
201
240
  wait_before_checking = 60
202
241
  time_since_ready = (Time.now - Time.parse(vm_ready_time)).to_i
@@ -213,7 +252,7 @@ module Vmpooler
213
252
  return if hostname.empty?
214
253
  return if hostname == vm
215
254
 
216
- $redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
255
+ redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
217
256
  $logger.log('d', "[!] [#{pool_name}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
218
257
  true
219
258
  end
@@ -234,49 +273,61 @@ module Vmpooler
234
273
  return if mutex.locked?
235
274
 
236
275
  mutex.synchronize do
237
- # Check that VM is within defined lifetime
238
- checkouttime = $redis.hget('vmpooler__active__' + pool, vm)
239
- if checkouttime
240
- running = (Time.now - Time.parse(checkouttime)) / 60 / 60
241
-
242
- if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
243
- move_vm_queue(pool, vm, 'running', 'completed', "reached end of TTL after #{ttl} hours")
244
- return
245
- end
246
- end
276
+ catch :stop_checking do
277
+ @redis.with_metrics do |redis|
278
+ # Check that VM is within defined lifetime
279
+ checkouttime = redis.hget('vmpooler__active__' + pool, vm)
280
+ if checkouttime
281
+ time_since_checkout = Time.now - Time.parse(checkouttime)
282
+ running = time_since_checkout / 60 / 60
283
+
284
+ if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
285
+ move_vm_queue(pool, vm, 'running', 'completed', redis, "reached end of TTL after #{ttl} hours")
286
+ throw :stop_checking
287
+ end
288
+ end
247
289
 
248
- if provider.vm_ready?(pool, vm)
249
- return
250
- else
251
- host = provider.get_vm(pool, vm)
290
+ if provider.vm_ready?(pool, vm)
291
+ throw :stop_checking
292
+ else
293
+ host = provider.get_vm(pool, vm)
252
294
 
253
- if host
254
- return
255
- else
256
- move_vm_queue(pool, vm, 'running', 'completed', 'is no longer in inventory, removing from running')
295
+ if host
296
+ throw :stop_checking
297
+ else
298
+ move_vm_queue(pool, vm, 'running', 'completed', redis, 'is no longer in inventory, removing from running')
299
+ end
300
+ end
257
301
  end
258
302
  end
259
303
  end
260
304
  end
261
305
 
262
- def move_vm_queue(pool, vm, queue_from, queue_to, msg = nil)
263
- $redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
306
+ def move_vm_queue(pool, vm, queue_from, queue_to, redis, msg = nil)
307
+ redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
264
308
  $logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") if msg
265
309
  end
266
310
 
267
311
  # Clone a VM
268
- def clone_vm(pool_name, provider)
312
+ def clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
269
313
  Thread.new do
270
314
  begin
271
- _clone_vm(pool_name, provider)
315
+ _clone_vm(pool_name, provider, request_id, pool_alias)
272
316
  rescue StandardError => e
273
- $logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
317
+ if request_id
318
+ $logger.log('s', "[!] [#{pool_name}] failed while cloning VM for request #{request_id} with an error: #{e}")
319
+ @redis.with_metrics do |redis|
320
+ redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
321
+ end
322
+ else
323
+ $logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
324
+ end
274
325
  raise
275
326
  end
276
327
  end
277
328
  end
278
329
 
279
- def generate_and_check_hostname(_pool_name)
330
+ def generate_and_check_hostname
280
331
  # Generate a randomized hostname. The total name must no longer than 15
281
332
  # character including the hyphen. The shortest adjective in the corpus is
282
333
  # three characters long. Therefore, we can technically select a noun up to 11
@@ -285,58 +336,104 @@ module Vmpooler
285
336
  # letter adjectives, we actually limit the noun to 10 letters to avoid
286
337
  # inviting more conflicts. We favor selecting a longer noun rather than a
287
338
  # longer adjective because longer adjectives tend to be less fun.
288
- noun = @name_generator.noun(max: 10)
289
- adjective = @name_generator.adjective(max: 14 - noun.length)
290
- random_name = [adjective, noun].join('-')
291
- hostname = $config[:config]['prefix'] + random_name
292
- available = $redis.hlen('vmpooler__vm__' + hostname) == 0
293
-
294
- [hostname, available]
339
+ @redis.with do |redis|
340
+ noun = @name_generator.noun(max: 10)
341
+ adjective = @name_generator.adjective(max: 14 - noun.length)
342
+ random_name = [adjective, noun].join('-')
343
+ hostname = $config[:config]['prefix'] + random_name
344
+ available = redis.hlen('vmpooler__vm__' + hostname) == 0
345
+
346
+ [hostname, available]
347
+ end
295
348
  end
296
349
 
297
350
  def find_unique_hostname(pool_name)
351
+ # generate hostname that is not already in use in vmpooler
352
+ # also check that no dns record already exists
298
353
  hostname_retries = 0
299
354
  max_hostname_retries = 3
300
355
  while hostname_retries < max_hostname_retries
301
- hostname, available = generate_and_check_hostname(pool_name)
302
- break if available
356
+ hostname, hostname_available = generate_and_check_hostname
357
+ domain = $config[:config]['domain']
358
+ dns_ip, dns_available = check_dns_available(hostname, domain)
359
+ break if hostname_available && dns_available
303
360
 
304
361
  hostname_retries += 1
305
- $metrics.increment("errors.duplicatehostname.#{pool_name}")
306
- $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
362
+
363
+ if !hostname_available
364
+ $metrics.increment("errors.duplicatehostname.#{pool_name}")
365
+ $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
366
+ elsif !dns_available
367
+ $metrics.increment("errors.staledns.#{hostname}")
368
+ $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} already exists in DNS records (#{dns_ip}), stale DNS")
369
+ end
307
370
  end
308
371
 
309
- raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless available
372
+ raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless hostname_available && dns_available
310
373
 
311
374
  hostname
312
375
  end
313
376
 
314
- def _clone_vm(pool_name, provider)
315
- new_vmname = find_unique_hostname(pool_name)
316
-
317
- # Add VM to Redis inventory ('pending' pool)
318
- $redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
319
- $redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
320
- $redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name)
321
-
377
+ def check_dns_available(vm_name, domain = nil)
378
+ # Query the DNS for the name we want to create and if it already exists, mark it unavailable
379
+ # This protects against stale DNS records
380
+ vm_name = "#{vm_name}.#{domain}" if domain
322
381
  begin
323
- $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
324
- start = Time.now
325
- provider.create_vm(pool_name, new_vmname)
326
- finish = format('%<time>.2f', time: Time.now - start)
382
+ dns_ip = Resolv.getaddress(vm_name)
383
+ rescue Resolv::ResolvError
384
+ # this is the expected case, swallow the error
385
+ # eg "no address for blah-daisy"
386
+ return ['', true]
387
+ end
388
+ [dns_ip, false]
389
+ end
327
390
 
328
- $redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
329
- $redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
330
- $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
391
+ def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
392
+ new_vmname = find_unique_hostname(pool_name)
393
+ mutex = vm_mutex(new_vmname)
394
+ mutex.synchronize do
395
+ @redis.with_metrics do |redis|
396
+ # Add VM to Redis inventory ('pending' pool)
397
+ redis.multi
398
+ redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
399
+ redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
400
+ redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name) # This value is used to represent the pool.
401
+ redis.hset('vmpooler__vm__' + new_vmname, 'pool', pool_name)
402
+ redis.hset('vmpooler__vm__' + new_vmname, 'request_id', request_id) if request_id
403
+ redis.hset('vmpooler__vm__' + new_vmname, 'pool_alias', pool_alias) if pool_alias
404
+ redis.exec
405
+ end
331
406
 
332
- $metrics.timing("clone.#{pool_name}", finish)
333
- rescue StandardError
334
- $redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
335
- expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
336
- $redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
337
- raise
338
- ensure
339
- $redis.decr('vmpooler__tasks__clone')
407
+ begin
408
+ $logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
409
+ start = Time.now
410
+ provider.create_vm(pool_name, new_vmname)
411
+ finish = format('%<time>.2f', time: Time.now - start)
412
+
413
+ @redis.with_metrics do |redis|
414
+ redis.pipelined do
415
+ redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
416
+ redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
417
+ end
418
+ end
419
+ $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
420
+
421
+ $metrics.timing("clone.#{pool_name}", finish)
422
+ rescue StandardError
423
+ @redis.with_metrics do |redis|
424
+ redis.pipelined do
425
+ redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
426
+ expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
427
+ redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
428
+ end
429
+ end
430
+ raise
431
+ ensure
432
+ @redis.with_metrics do |redis|
433
+ redis.decr('vmpooler__tasks__ondemandclone') if request_id
434
+ redis.decr('vmpooler__tasks__clone') unless request_id
435
+ end
436
+ end
340
437
  end
341
438
  end
342
439
 
@@ -357,36 +454,42 @@ module Vmpooler
357
454
  return if mutex.locked?
358
455
 
359
456
  mutex.synchronize do
360
- $redis.hdel('vmpooler__active__' + pool, vm)
361
- $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
457
+ @redis.with_metrics do |redis|
458
+ redis.pipelined do
459
+ redis.hdel('vmpooler__active__' + pool, vm)
460
+ redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
362
461
 
363
- # Auto-expire metadata key
364
- $redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
462
+ # Auto-expire metadata key
463
+ redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
464
+ end
365
465
 
366
- start = Time.now
466
+ start = Time.now
367
467
 
368
- provider.destroy_vm(pool, vm)
468
+ provider.destroy_vm(pool, vm)
369
469
 
370
- $redis.srem('vmpooler__completed__' + pool, vm)
470
+ redis.srem('vmpooler__completed__' + pool, vm)
371
471
 
372
- finish = format('%<time>.2f', time: Time.now - start)
373
- $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
374
- $metrics.timing("destroy.#{pool}", finish)
375
- get_vm_usage_labels(vm)
472
+ finish = format('%<time>.2f', time: Time.now - start)
473
+ $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
474
+ $metrics.timing("destroy.#{pool}", finish)
475
+ get_vm_usage_labels(vm, redis)
476
+ end
376
477
  end
377
478
  dereference_mutex(vm)
378
479
  end
379
480
 
380
- def get_vm_usage_labels(vm)
481
+ def get_vm_usage_labels(vm, redis)
381
482
  return unless $config[:config]['usage_stats']
382
483
 
383
- checkout = $redis.hget("vmpooler__vm__#{vm}", 'checkout')
484
+ redis.multi
485
+ redis.hget("vmpooler__vm__#{vm}", 'checkout')
486
+ redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
487
+ redis.hget("vmpooler__vm__#{vm}", 'token:user')
488
+ redis.hget("vmpooler__vm__#{vm}", 'template')
489
+ checkout, jenkins_build_url, user, poolname = redis.exec
384
490
  return if checkout.nil?
385
491
 
386
- jenkins_build_url = $redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
387
- user = $redis.hget("vmpooler__vm__#{vm}", 'token:user') || 'unauthenticated'
388
- poolname = $redis.hget("vmpooler__vm__#{vm}", 'template')
389
-
492
+ user ||= 'unauthenticated'
390
493
  unless jenkins_build_url
391
494
  user = user.gsub('.', '_')
392
495
  $metrics.increment("usage.#{user}.#{poolname}")
@@ -420,7 +523,8 @@ module Vmpooler
420
523
 
421
524
  $metrics.increment(metric_parts.join('.'))
422
525
  rescue StandardError => e
423
- logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
526
+ $logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
527
+ raise
424
528
  end
425
529
 
426
530
  def component_to_test(match, labels_string)
@@ -444,7 +548,7 @@ module Vmpooler
444
548
  if provider_purge
445
549
  Thread.new do
446
550
  begin
447
- purge_vms_and_folders(provider.to_s)
551
+ purge_vms_and_folders($providers[provider.to_s])
448
552
  rescue StandardError => e
449
553
  $logger.log('s', "[!] failed while purging provider #{provider} VMs and folders with an error: #{e}")
450
554
  end
@@ -455,13 +559,14 @@ module Vmpooler
455
559
  end
456
560
 
457
561
  # Return a list of pool folders
458
- def pool_folders(provider_name)
562
+ def pool_folders(provider)
563
+ provider_name = provider.name
459
564
  folders = {}
460
565
  $config[:pools].each do |pool|
461
566
  next unless pool['provider'] == provider_name
462
567
 
463
568
  folder_parts = pool['folder'].split('/')
464
- datacenter = $providers[provider_name].get_target_datacenter_from_config(pool['name'])
569
+ datacenter = provider.get_target_datacenter_from_config(pool['name'])
465
570
  folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
466
571
  end
467
572
  folders
@@ -478,8 +583,8 @@ module Vmpooler
478
583
  def purge_vms_and_folders(provider)
479
584
  configured_folders = pool_folders(provider)
480
585
  base_folders = get_base_folders(configured_folders)
481
- whitelist = $providers[provider].provider_config['folder_whitelist']
482
- $providers[provider].purge_unconfigured_folders(base_folders, configured_folders, whitelist)
586
+ whitelist = provider.provider_config['folder_whitelist']
587
+ provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
483
588
  end
484
589
 
485
590
  def create_vm_disk(pool_name, vm, disk_size, provider)
@@ -505,10 +610,12 @@ module Vmpooler
505
610
  finish = format('%<time>.2f', time: Time.now - start)
506
611
 
507
612
  if result
508
- rdisks = $redis.hget('vmpooler__vm__' + vm_name, 'disk')
509
- disks = rdisks ? rdisks.split(':') : []
510
- disks.push("+#{disk_size}gb")
511
- $redis.hset('vmpooler__vm__' + vm_name, 'disk', disks.join(':'))
613
+ @redis.with_metrics do |redis|
614
+ rdisks = redis.hget('vmpooler__vm__' + vm_name, 'disk')
615
+ disks = rdisks ? rdisks.split(':') : []
616
+ disks.push("+#{disk_size}gb")
617
+ redis.hset('vmpooler__vm__' + vm_name, 'disk', disks.join(':'))
618
+ end
512
619
 
513
620
  $logger.log('s', "[+] [disk_manager] '#{vm_name}' attached #{disk_size}gb disk in #{finish} seconds")
514
621
  else
@@ -538,7 +645,9 @@ module Vmpooler
538
645
  finish = format('%<time>.2f', time: Time.now - start)
539
646
 
540
647
  if result
541
- $redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
648
+ @redis.with_metrics do |redis|
649
+ redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
650
+ end
542
651
  $logger.log('s', "[+] [snapshot_manager] '#{vm_name}' snapshot created in #{finish} seconds")
543
652
  else
544
653
  $logger.log('s', "[+] [snapshot_manager] Failed to snapshot '#{vm_name}'")
@@ -594,9 +703,9 @@ module Vmpooler
594
703
  @default_providers ||= %w[vsphere dummy]
595
704
  end
596
705
 
597
- def get_pool_name_for_vm(vm_name)
706
+ def get_pool_name_for_vm(vm_name, redis)
598
707
  # the 'template' is a bad name. Should really be 'poolname'
599
- $redis.hget('vmpooler__vm__' + vm_name, 'template')
708
+ redis.hget('vmpooler__vm__' + vm_name, 'template')
600
709
  end
601
710
 
602
711
  # @param pool_name [String] - the name of the pool
@@ -628,19 +737,21 @@ module Vmpooler
628
737
  end
629
738
 
630
739
  def _check_disk_queue
631
- task_detail = $redis.spop('vmpooler__tasks__disk')
632
- unless task_detail.nil?
633
- begin
634
- vm_name, disk_size = task_detail.split(':')
635
- pool_name = get_pool_name_for_vm(vm_name)
636
- raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
740
+ @redis.with_metrics do |redis|
741
+ task_detail = redis.spop('vmpooler__tasks__disk')
742
+ unless task_detail.nil?
743
+ begin
744
+ vm_name, disk_size = task_detail.split(':')
745
+ pool_name = get_pool_name_for_vm(vm_name, redis)
746
+ raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
637
747
 
638
- provider = get_provider_for_pool(pool_name)
639
- raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
748
+ provider = get_provider_for_pool(pool_name)
749
+ raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
640
750
 
641
- create_vm_disk(pool_name, vm_name, disk_size, provider)
642
- rescue StandardError => e
643
- $logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{e}")
751
+ create_vm_disk(pool_name, vm_name, disk_size, provider)
752
+ rescue StandardError => e
753
+ $logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{e}")
754
+ end
644
755
  end
645
756
  end
646
757
  end
@@ -664,37 +775,39 @@ module Vmpooler
664
775
  end
665
776
 
666
777
  def _check_snapshot_queue
667
- task_detail = $redis.spop('vmpooler__tasks__snapshot')
778
+ @redis.with_metrics do |redis|
779
+ task_detail = redis.spop('vmpooler__tasks__snapshot')
668
780
 
669
- unless task_detail.nil?
670
- begin
671
- vm_name, snapshot_name = task_detail.split(':')
672
- pool_name = get_pool_name_for_vm(vm_name)
673
- raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
781
+ unless task_detail.nil?
782
+ begin
783
+ vm_name, snapshot_name = task_detail.split(':')
784
+ pool_name = get_pool_name_for_vm(vm_name, redis)
785
+ raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
674
786
 
675
- provider = get_provider_for_pool(pool_name)
676
- raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
787
+ provider = get_provider_for_pool(pool_name)
788
+ raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
677
789
 
678
- create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
679
- rescue StandardError => e
680
- $logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{e}")
790
+ create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
791
+ rescue StandardError => e
792
+ $logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{e}")
793
+ end
681
794
  end
682
- end
683
795
 
684
- task_detail = $redis.spop('vmpooler__tasks__snapshot-revert')
796
+ task_detail = redis.spop('vmpooler__tasks__snapshot-revert')
685
797
 
686
- unless task_detail.nil?
687
- begin
688
- vm_name, snapshot_name = task_detail.split(':')
689
- pool_name = get_pool_name_for_vm(vm_name)
690
- raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
798
+ unless task_detail.nil?
799
+ begin
800
+ vm_name, snapshot_name = task_detail.split(':')
801
+ pool_name = get_pool_name_for_vm(vm_name, redis)
802
+ raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
691
803
 
692
- provider = get_provider_for_pool(pool_name)
693
- raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
804
+ provider = get_provider_for_pool(pool_name)
805
+ raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
694
806
 
695
- revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
696
- rescue StandardError => e
697
- $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{e}")
807
+ revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
808
+ rescue StandardError => e
809
+ $logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{e}")
810
+ end
698
811
  end
699
812
  end
700
813
  end
@@ -704,7 +817,9 @@ module Vmpooler
704
817
  begin
705
818
  mutex = vm_mutex(vm_name)
706
819
  mutex.synchronize do
707
- $redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
820
+ @redis.with_metrics do |redis|
821
+ redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
822
+ end
708
823
  provider.migrate_vm(pool_name, vm_name)
709
824
  end
710
825
  rescue StandardError => e
@@ -737,47 +852,65 @@ module Vmpooler
737
852
  wakeup_by = Time.now + wakeup_period
738
853
  return if time_passed?(:exit_by, exit_by)
739
854
 
740
- initial_ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
855
+ @redis.with_metrics do |redis|
856
+ initial_ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
741
857
 
742
- initial_clone_target = $redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
858
+ initial_clone_target = redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
743
859
 
744
- initial_template = $redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
860
+ initial_template = redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
745
861
 
746
- loop do
747
- sleep(1)
748
- break if time_passed?(:exit_by, exit_by)
862
+ loop do
863
+ sleep(1)
864
+ break if time_passed?(:exit_by, exit_by)
749
865
 
750
- # Check for wakeup events
751
- if time_passed?(:wakeup_by, wakeup_by)
752
- wakeup_by = Time.now + wakeup_period
866
+ # Check for wakeup events
867
+ if time_passed?(:wakeup_by, wakeup_by)
868
+ wakeup_by = Time.now + wakeup_period
753
869
 
754
- # Wakeup if the number of ready VMs has changed
755
- if options[:pool_size_change]
756
- ready_size = $redis.scard("vmpooler__ready__#{options[:poolname]}")
757
- break unless ready_size == initial_ready_size
758
- end
870
+ # Wakeup if the number of ready VMs has changed
871
+ if options[:pool_size_change]
872
+ ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}")
873
+ break unless ready_size == initial_ready_size
874
+ end
759
875
 
760
- if options[:clone_target_change]
761
- clone_target = $redis.hget('vmpooler__config__clone_target}', options[:poolname])
762
- if clone_target
763
- break unless clone_target == initial_clone_target
876
+ if options[:clone_target_change]
877
+ clone_target = redis.hget('vmpooler__config__clone_target}', options[:poolname])
878
+ if clone_target
879
+ break unless clone_target == initial_clone_target
880
+ end
764
881
  end
765
- end
766
882
 
767
- if options[:pool_template_change]
768
- configured_template = $redis.hget('vmpooler__config__template', options[:poolname])
769
- if configured_template
770
- break unless initial_template == configured_template
883
+ if options[:pool_template_change]
884
+ configured_template = redis.hget('vmpooler__config__template', options[:poolname])
885
+ if configured_template
886
+ break unless initial_template == configured_template
887
+ end
888
+ end
889
+
890
+ if options[:pool_reset]
891
+ pending = redis.sismember('vmpooler__poolreset', options[:poolname])
892
+ break if pending
771
893
  end
772
- end
773
894
 
774
- if options[:pool_reset]
775
- break if $redis.sismember('vmpooler__poolreset', options[:poolname])
895
+ if options[:pending_vm]
896
+ pending_vm_count = redis.scard("vmpooler__pending__#{options[:poolname]}")
897
+ break unless pending_vm_count == 0
898
+ end
899
+
900
+ if options[:ondemand_request]
901
+ redis.multi
902
+ redis.zcard('vmpooler__provisioning__request')
903
+ redis.zcard('vmpooler__provisioning__processing')
904
+ redis.zcard('vmpooler__odcreate__task')
905
+ od_request, od_processing, od_createtask = redis.exec
906
+ break unless od_request == 0
907
+ break unless od_processing == 0
908
+ break unless od_createtask == 0
909
+ end
776
910
  end
777
911
 
912
+ break if time_passed?(:exit_by, exit_by)
778
913
  end
779
-
780
- break if time_passed?(:exit_by, exit_by)
781
914
  end
782
915
  end
783
916
 
@@ -813,7 +946,7 @@ module Vmpooler
813
946
  loop_delay = (loop_delay * loop_delay_decay).to_i
814
947
  loop_delay = loop_delay_max if loop_delay > loop_delay_max
815
948
  end
816
- sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pool_reset: true)
949
+ sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pending_vm: true, pool_reset: true)
817
950
 
818
951
  unless maxloop == 0
819
952
  break if loop_count >= maxloop
@@ -843,77 +976,84 @@ module Vmpooler
843
976
  end
844
977
 
845
978
  def sync_pool_template(pool)
846
- pool_template = $redis.hget('vmpooler__config__template', pool['name'])
847
- if pool_template
848
- pool['template'] = pool_template unless pool['template'] == pool_template
979
+ @redis.with_metrics do |redis|
980
+ pool_template = redis.hget('vmpooler__config__template', pool['name'])
981
+ if pool_template
982
+ pool['template'] = pool_template unless pool['template'] == pool_template
983
+ end
849
984
  end
850
985
  end
851
986
 
852
- def prepare_template(pool, provider)
987
+ def prepare_template(pool, provider, redis)
853
988
  if $config[:config]['create_template_delta_disks']
854
- unless $redis.sismember('vmpooler__template__deltas', pool['template'])
989
+ unless redis.sismember('vmpooler__template__deltas', pool['template'])
855
990
  begin
856
991
  provider.create_template_delta_disks(pool)
857
- $redis.sadd('vmpooler__template__deltas', pool['template'])
992
+ redis.sadd('vmpooler__template__deltas', pool['template'])
858
993
  rescue StandardError => e
859
994
  $logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{e}")
860
995
  end
861
996
  end
862
997
  end
863
- $redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
998
+ redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
864
999
  end
865
1000
 
866
1001
  def evaluate_template(pool, provider)
867
1002
  mutex = pool_mutex(pool['name'])
868
- prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
869
- configured_template = $redis.hget('vmpooler__config__template', pool['name'])
870
1003
  return if mutex.locked?
871
1004
 
872
- if prepared_template.nil?
873
- mutex.synchronize do
874
- prepare_template(pool, provider)
875
- prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
876
- end
877
- elsif prepared_template != pool['template']
878
- if configured_template.nil?
1005
+ catch :update_not_needed do
1006
+ @redis.with_metrics do |redis|
1007
+ prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
1008
+ configured_template = redis.hget('vmpooler__config__template', pool['name'])
1009
+
1010
+ if prepared_template.nil?
1011
+ mutex.synchronize do
1012
+ prepare_template(pool, provider, redis)
1013
+ prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
1014
+ end
1015
+ elsif prepared_template != pool['template']
1016
+ if configured_template.nil?
1017
+ mutex.synchronize do
1018
+ prepare_template(pool, provider, redis)
1019
+ prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
1020
+ end
1021
+ end
1022
+ end
1023
+ throw :update_not_needed if configured_template.nil?
1024
+ throw :update_not_needed if configured_template == prepared_template
1025
+
879
1026
  mutex.synchronize do
880
- prepare_template(pool, provider)
881
- prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
1027
+ update_pool_template(pool, provider, configured_template, prepared_template, redis)
882
1028
  end
883
1029
  end
884
1030
  end
885
- return if configured_template.nil?
886
- return if configured_template == prepared_template
887
-
888
- mutex.synchronize do
889
- update_pool_template(pool, provider, configured_template, prepared_template)
890
- end
891
1031
  end
892
1032
 
893
- def drain_pool(poolname)
1033
+ def drain_pool(poolname, redis)
894
1034
  # Clear a pool of ready and pending instances
895
- if $redis.scard("vmpooler__ready__#{poolname}") > 0
1035
+ if redis.scard("vmpooler__ready__#{poolname}") > 0
896
1036
  $logger.log('s', "[*] [#{poolname}] removing ready instances")
897
- $redis.smembers("vmpooler__ready__#{poolname}").each do |vm|
898
- move_vm_queue(poolname, vm, 'ready', 'completed')
1037
+ redis.smembers("vmpooler__ready__#{poolname}").each do |vm|
1038
+ move_vm_queue(poolname, vm, 'ready', 'completed', redis)
899
1039
  end
900
1040
  end
901
- if $redis.scard("vmpooler__pending__#{poolname}") > 0
1041
+ if redis.scard("vmpooler__pending__#{poolname}") > 0
902
1042
  $logger.log('s', "[*] [#{poolname}] removing pending instances")
903
- $redis.smembers("vmpooler__pending__#{poolname}").each do |vm|
904
- move_vm_queue(poolname, vm, 'pending', 'completed')
1043
+ redis.smembers("vmpooler__pending__#{poolname}").each do |vm|
1044
+ move_vm_queue(poolname, vm, 'pending', 'completed', redis)
905
1045
  end
906
1046
  end
907
1047
  end
908
1048
 
909
- def update_pool_template(pool, provider, configured_template, prepared_template)
1049
+ def update_pool_template(pool, provider, configured_template, prepared_template, redis)
910
1050
  pool['template'] = configured_template
911
1051
  $logger.log('s', "[*] [#{pool['name']}] template updated from #{prepared_template} to #{configured_template}")
912
1052
  # Remove all ready and pending VMs so new instances are created from the new template
913
- drain_pool(pool['name'])
1053
+ drain_pool(pool['name'], redis)
914
1054
  # Prepare template for deployment
915
1055
  $logger.log('s', "[*] [#{pool['name']}] preparing pool template for deployment")
916
- prepare_template(pool, provider)
1056
+ prepare_template(pool, provider, redis)
917
1057
  $logger.log('s', "[*] [#{pool['name']}] is ready for use")
918
1058
  end
919
1059
 
@@ -921,38 +1061,45 @@ module Vmpooler
921
1061
  mutex = pool_mutex(pool['name'])
922
1062
  return if mutex.locked?
923
1063
 
924
- clone_target = $redis.hget('vmpooler__config__clone_target', pool['name'])
925
- return if clone_target.nil?
926
- return if clone_target == pool['clone_target']
1064
+ @redis.with_metrics do |redis|
1065
+ clone_target = redis.hget('vmpooler__config__clone_target', pool['name'])
1066
+ break if clone_target.nil?
1067
+ break if clone_target == pool['clone_target']
927
1068
 
928
- $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
929
- mutex.synchronize do
930
- pool['clone_target'] = clone_target
931
- # Remove all ready and pending VMs so new instances are created for the new clone_target
932
- drain_pool(pool['name'])
1069
+ $logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
1070
+ mutex.synchronize do
1071
+ pool['clone_target'] = clone_target
1072
+ # Remove all ready and pending VMs so new instances are created for the new clone_target
1073
+ drain_pool(pool['name'], redis)
1074
+ end
1075
+ $logger.log('s', "[*] [#{pool['name']}] is ready for use")
933
1076
  end
934
- $logger.log('s', "[*] [#{pool['name']}] is ready for use")
935
1077
  end
936
1078
 
937
1079
  def remove_excess_vms(pool)
938
- ready = $redis.scard("vmpooler__ready__#{pool['name']}")
939
- total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
940
- return if total.nil?
941
- return if total == 0
1080
+ @redis.with_metrics do |redis|
1081
+ redis.multi
1082
+ redis.scard("vmpooler__ready__#{pool['name']}")
1083
+ redis.scard("vmpooler__pending__#{pool['name']}")
1084
+ ready, pending = redis.exec
1085
+ total = pending.to_i + ready.to_i
1086
+ break if total.nil?
1087
+ break if total == 0
942
1088
 
943
- mutex = pool_mutex(pool['name'])
944
- return if mutex.locked?
945
- return unless ready > pool['size']
1089
+ mutex = pool_mutex(pool['name'])
1090
+ break if mutex.locked?
1091
+ break unless ready.to_i > pool['size']
946
1092
 
947
- mutex.synchronize do
948
- difference = ready - pool['size']
949
- difference.times do
950
- next_vm = $redis.spop("vmpooler__ready__#{pool['name']}")
951
- move_vm_queue(pool['name'], next_vm, 'ready', 'completed')
952
- end
953
- if total > ready
954
- $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
955
- move_vm_queue(pool['name'], vm, 'pending', 'completed')
1093
+ mutex.synchronize do
1094
+ difference = ready.to_i - pool['size']
1095
+ difference.times do
1096
+ next_vm = redis.spop("vmpooler__ready__#{pool['name']}")
1097
+ move_vm_queue(pool['name'], next_vm, 'ready', 'completed', redis)
1098
+ end
1099
+ if total > ready
1100
+ redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
1101
+ move_vm_queue(pool['name'], vm, 'pending', 'completed', redis)
1102
+ end
956
1103
  end
957
1104
  end
958
1105
  end
@@ -962,26 +1109,30 @@ module Vmpooler
962
1109
  mutex = pool_mutex(pool['name'])
963
1110
  return if mutex.locked?
964
1111
 
965
- poolsize = $redis.hget('vmpooler__config__poolsize', pool['name'])
966
- return if poolsize.nil?
1112
+ @redis.with_metrics do |redis|
1113
+ poolsize = redis.hget('vmpooler__config__poolsize', pool['name'])
1114
+ break if poolsize.nil?
967
1115
 
968
- poolsize = Integer(poolsize)
969
- return if poolsize == pool['size']
1116
+ poolsize = Integer(poolsize)
1117
+ break if poolsize == pool['size']
970
1118
 
971
- mutex.synchronize do
972
- pool['size'] = poolsize
1119
+ mutex.synchronize do
1120
+ pool['size'] = poolsize
1121
+ end
973
1122
  end
974
1123
  end
975
1124
 
976
1125
  def reset_pool(pool)
977
1126
  poolname = pool['name']
978
- return unless $redis.sismember('vmpooler__poolreset', poolname)
1127
+ @redis.with_metrics do |redis|
1128
+ break unless redis.sismember('vmpooler__poolreset', poolname)
979
1129
 
980
- $redis.srem('vmpooler__poolreset', poolname)
981
- mutex = pool_mutex(poolname)
982
- mutex.synchronize do
983
- drain_pool(poolname)
984
- $logger.log('s', "[*] [#{poolname}] reset has cleared ready and pending instances")
1130
+ redis.srem('vmpooler__poolreset', poolname)
1131
+ mutex = pool_mutex(poolname)
1132
+ mutex.synchronize do
1133
+ drain_pool(poolname, redis)
1134
+ $logger.log('s', "[*] [#{poolname}] reset has cleared ready and pending instances")
1135
+ end
985
1136
  end
986
1137
  end
987
1138
 
@@ -990,21 +1141,23 @@ module Vmpooler
990
1141
  begin
991
1142
  mutex = pool_mutex(pool['name'])
992
1143
  mutex.synchronize do
993
- provider.vms_in_pool(pool['name']).each do |vm|
994
- if !$redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
995
- !$redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
996
- !$redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) &&
997
- !$redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) &&
998
- !$redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) &&
999
- !$redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])
1000
-
1001
- pool_check_response[:discovered_vms] += 1
1002
- $redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
1003
-
1004
- $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
1144
+ @redis.with_metrics do |redis|
1145
+ provider.vms_in_pool(pool['name']).each do |vm|
1146
+ if !redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
1147
+ !redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
1148
+ !redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) &&
1149
+ !redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) &&
1150
+ !redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) &&
1151
+ !redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])
1152
+
1153
+ pool_check_response[:discovered_vms] += 1
1154
+ redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
1155
+
1156
+ $logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
1157
+ end
1158
+
1159
+ inventory[vm['name']] = 1
1005
1160
  end
1006
-
1007
- inventory[vm['name']] = 1
1008
1161
  end
1009
1162
  end
1010
1163
  rescue StandardError => e
@@ -1015,96 +1168,112 @@ module Vmpooler
1015
1168
  end
1016
1169
 
1017
1170
  def check_running_pool_vms(pool_name, provider, pool_check_response, inventory)
1018
- $redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
1019
- if inventory[vm]
1020
- begin
1021
- vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
1022
- pool_check_response[:checked_running_vms] += 1
1023
- check_running_vm(vm, pool_name, vm_lifetime, provider)
1024
- rescue StandardError => e
1025
- $logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{e}")
1171
+ @redis.with_metrics do |redis|
1172
+ redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
1173
+ if inventory[vm]
1174
+ begin
1175
+ vm_lifetime = redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
1176
+ pool_check_response[:checked_running_vms] += 1
1177
+ check_running_vm(vm, pool_name, vm_lifetime, provider)
1178
+ rescue StandardError => e
1179
+ $logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{e}")
1180
+ end
1181
+ else
1182
+ move_vm_queue(pool_name, vm, 'running', 'completed', redis, 'is a running VM but is missing from inventory. Marking as completed.')
1026
1183
  end
1027
- else
1028
- move_vm_queue(pool_name, vm, 'running', 'completed', 'is a running VM but is missing from inventory. Marking as completed.')
1029
1184
  end
1030
1185
  end
1031
1186
  end
1032
1187
 
1033
- def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl = 0)
1034
- $redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
1035
- if inventory[vm]
1036
- begin
1037
- pool_check_response[:checked_ready_vms] += 1
1038
- check_ready_vm(vm, pool_name, pool_ttl || 0, provider)
1039
- rescue StandardError => e
1040
- $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
1188
+ def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl)
1189
+ @redis.with_metrics do |redis|
1190
+ redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
1191
+ if inventory[vm]
1192
+ begin
1193
+ pool_check_response[:checked_ready_vms] += 1
1194
+ check_ready_vm(vm, pool_name, pool_ttl, provider)
1195
+ rescue StandardError => e
1196
+ $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
1197
+ end
1198
+ else
1199
+ move_vm_queue(pool_name, vm, 'ready', 'completed', redis, 'is a ready VM but is missing from inventory. Marking as completed.')
1041
1200
  end
1042
- else
1043
- move_vm_queue(pool_name, vm, 'ready', 'completed', 'is a ready VM but is missing from inventory. Marking as completed.')
1044
1201
  end
1045
1202
  end
1046
1203
  end
1047
1204
 
1048
- def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout = nil)
1205
+ def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout)
1049
1206
  pool_timeout ||= $config[:config]['timeout'] || 15
1050
- $redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
1051
- if inventory[vm]
1052
- begin
1053
- pool_check_response[:checked_pending_vms] += 1
1054
- check_pending_vm(vm, pool_name, pool_timeout, provider)
1055
- rescue StandardError => e
1056
- $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
1207
+ @redis.with_metrics do |redis|
1208
+ redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
1209
+ if inventory[vm]
1210
+ begin
1211
+ pool_check_response[:checked_pending_vms] += 1
1212
+ check_pending_vm(vm, pool_name, pool_timeout, provider)
1213
+ rescue StandardError => e
1214
+ $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
1215
+ end
1216
+ else
1217
+ fail_pending_vm(vm, pool_name, pool_timeout, redis, false)
1057
1218
  end
1058
- else
1059
- fail_pending_vm(vm, pool_name, pool_timeout, false)
1060
1219
  end
1061
1220
  end
1062
1221
  end
1063
1222
 
1064
1223
  def check_completed_pool_vms(pool_name, provider, pool_check_response, inventory)
1065
- $redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
1066
- if inventory[vm]
1067
- begin
1068
- pool_check_response[:destroyed_vms] += 1
1069
- destroy_vm(vm, pool_name, provider)
1070
- rescue StandardError => e
1071
- $redis.srem("vmpooler__completed__#{pool_name}", vm)
1072
- $redis.hdel("vmpooler__active__#{pool_name}", vm)
1073
- $redis.del("vmpooler__vm__#{vm}")
1074
- $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
1224
+ @redis.with_metrics do |redis|
1225
+ redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
1226
+ if inventory[vm]
1227
+ begin
1228
+ pool_check_response[:destroyed_vms] += 1
1229
+ destroy_vm(vm, pool_name, provider)
1230
+ rescue StandardError => e
1231
+ redis.pipelined do
1232
+ redis.srem("vmpooler__completed__#{pool_name}", vm)
1233
+ redis.hdel("vmpooler__active__#{pool_name}", vm)
1234
+ redis.del("vmpooler__vm__#{vm}")
1235
+ end
1236
+ $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
1237
+ end
1238
+ else
1239
+ $logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
1240
+ redis.pipelined do
1241
+ redis.srem("vmpooler__completed__#{pool_name}", vm)
1242
+ redis.hdel("vmpooler__active__#{pool_name}", vm)
1243
+ redis.del("vmpooler__vm__#{vm}")
1244
+ end
1075
1245
  end
1076
- else
1077
- $logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
1078
- $redis.srem("vmpooler__completed__#{pool_name}", vm)
1079
- $redis.hdel("vmpooler__active__#{pool_name}", vm)
1080
- $redis.del("vmpooler__vm__#{vm}")
1081
1246
  end
1082
1247
  end
1083
1248
  end
1084
1249
 
1085
1250
  def check_discovered_pool_vms(pool_name)
1086
- $redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
1087
- %w[pending ready running completed].each do |queue|
1088
- if $redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
1089
- $logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
1090
- $redis.srem("vmpooler__discovered__#{pool_name}", vm)
1251
+ @redis.with_metrics do |redis|
1252
+ redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
1253
+ %w[pending ready running completed].each do |queue|
1254
+ if redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
1255
+ $logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
1256
+ redis.srem("vmpooler__discovered__#{pool_name}", vm)
1257
+ end
1091
1258
  end
1092
- end
1093
1259
 
1094
- $redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm) if $redis.sismember("vmpooler__discovered__#{pool_name}", vm)
1260
+ redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm) if redis.sismember("vmpooler__discovered__#{pool_name}", vm)
1261
+ end
1095
1262
  end
1096
1263
  rescue StandardError => e
1097
1264
  $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating discovered VMs: #{e}")
1098
1265
  end
1099
1266
 
1100
1267
  def check_migrating_pool_vms(pool_name, provider, pool_check_response, inventory)
1101
- $redis.smembers("vmpooler__migrating__#{pool_name}").reverse.each do |vm|
1102
- if inventory[vm]
1103
- begin
1104
- pool_check_response[:migrated_vms] += 1
1105
- migrate_vm(vm, pool_name, provider)
1106
- rescue StandardError => e
1107
- $logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{e}")
1268
+ @redis.with_metrics do |redis|
1269
+ redis.smembers("vmpooler__migrating__#{pool_name}").reverse.each do |vm|
1270
+ if inventory[vm]
1271
+ begin
1272
+ pool_check_response[:migrated_vms] += 1
1273
+ migrate_vm(vm, pool_name, provider)
1274
+ rescue StandardError => e
1275
+ $logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{e}")
1276
+ end
1108
1277
  end
1109
1278
  end
1110
1279
  end
@@ -1113,29 +1282,37 @@ module Vmpooler
1113
1282
  def repopulate_pool_vms(pool_name, provider, pool_check_response, pool_size)
1114
1283
  return if pool_mutex(pool_name).locked?
1115
1284
 
1116
- ready = $redis.scard("vmpooler__ready__#{pool_name}")
1117
- total = $redis.scard("vmpooler__pending__#{pool_name}") + ready
1118
-
1119
- $metrics.gauge("ready.#{pool_name}", $redis.scard("vmpooler__ready__#{pool_name}"))
1120
- $metrics.gauge("running.#{pool_name}", $redis.scard("vmpooler__running__#{pool_name}"))
1121
-
1122
- if $redis.get("vmpooler__empty__#{pool_name}")
1123
- $redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
1124
- elsif ready == 0
1125
- $redis.set("vmpooler__empty__#{pool_name}", 'true')
1126
- $logger.log('s', "[!] [#{pool_name}] is empty")
1127
- end
1285
+ @redis.with_metrics do |redis|
1286
+ redis.multi
1287
+ redis.scard("vmpooler__ready__#{pool_name}")
1288
+ redis.scard("vmpooler__pending__#{pool_name}")
1289
+ redis.scard("vmpooler__running__#{pool_name}")
1290
+ ready, pending, running = redis.exec
1291
+ total = pending.to_i + ready.to_i
1292
+
1293
+ $metrics.gauge("ready.#{pool_name}", ready)
1294
+ $metrics.gauge("running.#{pool_name}", running)
1295
+
1296
+ unless pool_size == 0
1297
+ if redis.get("vmpooler__empty__#{pool_name}")
1298
+ redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
1299
+ elsif ready == 0
1300
+ redis.set("vmpooler__empty__#{pool_name}", 'true')
1301
+ $logger.log('s', "[!] [#{pool_name}] is empty")
1302
+ end
1303
+ end
1128
1304
 
1129
- (pool_size - total).times do
1130
- if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
1131
- begin
1132
- $redis.incr('vmpooler__tasks__clone')
1133
- pool_check_response[:cloned_vms] += 1
1134
- clone_vm(pool_name, provider)
1135
- rescue StandardError => e
1136
- $logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
1137
- $redis.decr('vmpooler__tasks__clone')
1138
- raise
1305
+ (pool_size - total.to_i).times do
1306
+ if redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
1307
+ begin
1308
+ redis.incr('vmpooler__tasks__clone')
1309
+ pool_check_response[:cloned_vms] += 1
1310
+ clone_vm(pool_name, provider)
1311
+ rescue StandardError => e
1312
+ $logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
1313
+ redis.decr('vmpooler__tasks__clone')
1314
+ raise
1315
+ end
1139
1316
  end
1140
1317
  end
1141
1318
  end
@@ -1160,7 +1337,7 @@ module Vmpooler
1160
1337
 
1161
1338
  check_running_pool_vms(pool['name'], provider, pool_check_response, inventory)
1162
1339
 
1163
- check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'])
1340
+ check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
1164
1341
 
1165
1342
  check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
1166
1343
 
@@ -1203,23 +1380,203 @@ module Vmpooler
1203
1380
  #
1204
1381
  # returns an object Vmpooler::PoolManager::Provider::*
1205
1382
  # or raises an error if the class does not exist
1206
- def create_provider_object(config, logger, metrics, provider_class, provider_name, options)
1383
+ def create_provider_object(config, logger, metrics, redis_connection_pool, provider_class, provider_name, options)
1207
1384
  provider_klass = Vmpooler::PoolManager::Provider
1208
1385
  provider_klass.constants.each do |classname|
1209
1386
  next unless classname.to_s.casecmp(provider_class) == 0
1210
1387
 
1211
- return provider_klass.const_get(classname).new(config, logger, metrics, provider_name, options)
1388
+ return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options)
1212
1389
  end
1213
1390
  raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
1214
1391
  end
1215
1392
 
1393
+ def check_ondemand_requests(maxloop = 0,
1394
+ loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
1395
+ loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
1396
+ loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
1397
+
1398
+ $logger.log('d', '[*] [ondemand_provisioner] starting worker thread')
1399
+
1400
+ $threads['ondemand_provisioner'] = Thread.new do
1401
+ _check_ondemand_requests(maxloop, loop_delay_min, loop_delay_max, loop_delay_decay)
1402
+ end
1403
+ end
1404
+
1405
+ def _check_ondemand_requests(maxloop = 0,
1406
+ loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
1407
+ loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
1408
+ loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
1409
+
1410
+ loop_delay_min = $config[:config]['check_loop_delay_min'] unless $config[:config]['check_loop_delay_min'].nil?
1411
+ loop_delay_max = $config[:config]['check_loop_delay_max'] unless $config[:config]['check_loop_delay_max'].nil?
1412
+ loop_delay_decay = $config[:config]['check_loop_delay_decay'] unless $config[:config]['check_loop_delay_decay'].nil?
1413
+
1414
+ loop_delay_decay = 2.0 if loop_delay_decay <= 1.0
1415
+ loop_delay_max = loop_delay_min if loop_delay_max.nil? || loop_delay_max < loop_delay_min
1416
+
1417
+ loop_count = 1
1418
+ loop_delay = loop_delay_min
1419
+
1420
+ loop do
1421
+ result = process_ondemand_requests
1422
+
1423
+ loop_delay = (loop_delay * loop_delay_decay).to_i
1424
+ loop_delay = loop_delay_min if result > 0
1425
+ loop_delay = loop_delay_max if loop_delay > loop_delay_max
1426
+ sleep_with_wakeup_events(loop_delay, loop_delay_min, ondemand_request: true)
1427
+
1428
+ unless maxloop == 0
1429
+ break if loop_count >= maxloop
1430
+
1431
+ loop_count += 1
1432
+ end
1433
+ end
1434
+ end
1435
+
1436
+ def process_ondemand_requests
1437
+ @redis.with_metrics do |redis|
1438
+ requests = redis.zrange('vmpooler__provisioning__request', 0, -1)
1439
+ requests&.map { |request_id| create_ondemand_vms(request_id, redis) }
1440
+ provisioning_tasks = process_ondemand_vms(redis)
1441
+ requests_ready = check_ondemand_requests_ready(redis)
1442
+ requests.length + provisioning_tasks + requests_ready
1443
+ end
1444
+ end
1445
+
1446
+ def create_ondemand_vms(request_id, redis)
1447
+ requested = redis.hget("vmpooler__odrequest__#{request_id}", 'requested')
1448
+ unless requested
1449
+ $logger.log('s', "Failed to find odrequest for request_id '#{request_id}'")
1450
+ redis.zrem('vmpooler__provisioning__request', request_id)
1451
+ return
1452
+ end
1453
+ score = redis.zscore('vmpooler__provisioning__request', request_id)
1454
+ requested = requested.split(',')
1455
+
1456
+ redis.pipelined do
1457
+ requested.each do |request|
1458
+ redis.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
1459
+ end
1460
+ redis.zrem('vmpooler__provisioning__request', request_id)
1461
+ redis.zadd('vmpooler__provisioning__processing', score, request_id)
1462
+ end
1463
+ end
1464
+
1465
+ def process_ondemand_vms(redis)
1466
+ queue_key = 'vmpooler__odcreate__task'
1467
+ queue = redis.zrange(queue_key, 0, -1, with_scores: true)
1468
+ ondemand_clone_limit = $config[:config]['ondemand_clone_limit']
1469
+ queue.each do |request, score|
1470
+ clone_count = redis.get('vmpooler__tasks__ondemandclone').to_i
1471
+ break unless clone_count < ondemand_clone_limit
1472
+
1473
+ pool_alias, pool, count, request_id = request.split(':')
1474
+ count = count.to_i
1475
+ provider = get_provider_for_pool(pool)
1476
+ slots = ondemand_clone_limit - clone_count
1477
+ break if slots == 0
1478
+
1479
+ if slots >= count
1480
+ count.times do
1481
+ redis.incr('vmpooler__tasks__ondemandclone')
1482
+ clone_vm(pool, provider, request_id, pool_alias)
1483
+ end
1484
+ redis.zrem(queue_key, request)
1485
+ else
1486
+ remaining_count = count - slots
1487
+ slots.times do
1488
+ redis.incr('vmpooler__tasks__ondemandclone')
1489
+ clone_vm(pool, provider, request_id, pool_alias)
1490
+ end
1491
+ redis.pipelined do
1492
+ redis.zrem(queue_key, request)
1493
+ redis.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
1494
+ end
1495
+ end
1496
+ end
1497
+ queue.length
1498
+ end
1499
+
1500
+ def vms_ready?(request_id, redis)
1501
+ catch :request_not_ready do
1502
+ request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
1503
+ requested_platforms = request_hash['requested'].split(',')
1504
+ requested_platforms.each do |platform|
1505
+ platform_alias, pool, count = platform.split(':')
1506
+ pools_filled = redis.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1507
+ throw :request_not_ready unless pools_filled.to_i == count.to_i
1508
+ end
1509
+ return true
1510
+ end
1511
+ false
1512
+ end
1513
+
1514
+ def check_ondemand_requests_ready(redis)
1515
+ in_progress_requests = redis.zrange('vmpooler__provisioning__processing', 0, -1, with_scores: true)
1516
+ in_progress_requests&.each do |request_id, score|
1517
+ check_ondemand_request_ready(request_id, redis, score)
1518
+ end
1519
+ in_progress_requests.length
1520
+ end
1521
+
1522
+ def check_ondemand_request_ready(request_id, redis, score = nil)
1523
+ # default expiration is one month to ensure the data does not stay in redis forever
1524
+ default_expiration = 259_200_0
1525
+ processing_key = 'vmpooler__provisioning__processing'
1526
+ ondemand_hash_key = "vmpooler__odrequest__#{request_id}"
1527
+ score ||= redis.zscore(processing_key, request_id)
1528
+ return if request_expired?(request_id, score, redis)
1529
+
1530
+ return unless vms_ready?(request_id, redis)
1531
+
1532
+ redis.multi
1533
+ redis.hset(ondemand_hash_key, 'status', 'ready')
1534
+ redis.expire(ondemand_hash_key, default_expiration)
1535
+ redis.zrem(processing_key, request_id)
1536
+ redis.exec
1537
+ end
1538
+
1539
+ def request_expired?(request_id, score, redis)
1540
+ delta = Time.now.to_i - score.to_i
1541
+ ondemand_request_ttl = $config[:config]['ondemand_request_ttl']
1542
+ return false unless delta > ondemand_request_ttl * 60
1543
+
1544
+ $logger.log('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
1545
+ expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
1546
+ redis.pipelined do
1547
+ redis.zrem('vmpooler__provisioning__processing', request_id)
1548
+ redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
1549
+ redis.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
1550
+ end
1551
+ remove_vms_for_failed_request(request_id, expiration_ttl, redis)
1552
+ true
1553
+ end
1554
+
1555
+ def remove_vms_for_failed_request(request_id, expiration_ttl, redis)
1556
+ request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
1557
+ requested_platforms = request_hash['requested'].split(',')
1558
+ requested_platforms.each do |platform|
1559
+ platform_alias, pool, _count = platform.split(':')
1560
+ pools_filled = redis.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
1561
+ redis.pipelined do
1562
+ pools_filled&.each do |vm|
1563
+ move_vm_queue(pool, vm, 'running', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
1564
+ end
1565
+ redis.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
1566
+ end
1567
+ end
1568
+ end
1569
+
1216
1570
  def execute!(maxloop = 0, loop_delay = 1)
1217
1571
  $logger.log('d', 'starting vmpooler')
1218
1572
 
1219
- # Clear out the tasks manager, as we don't know about any tasks at this point
1220
- $redis.set('vmpooler__tasks__clone', 0)
1221
- # Clear out vmpooler__migrations since stale entries may be left after a restart
1222
- $redis.del('vmpooler__migration')
1573
+ @redis.with_metrics do |redis|
1574
+ # Clear out the tasks manager, as we don't know about any tasks at this point
1575
+ redis.set('vmpooler__tasks__clone', 0)
1576
+ redis.set('vmpooler__tasks__ondemandclone', 0)
1577
+ # Clear out vmpooler__migrations since stale entries may be left after a restart
1578
+ redis.del('vmpooler__migration')
1579
+ end
1223
1580
 
1224
1581
  # Copy vSphere settings to correct location. This happens with older configuration files
1225
1582
  if !$config[:vsphere].nil? && ($config[:providers].nil? || $config[:providers][:vsphere].nil?)
@@ -1269,7 +1626,7 @@ module Vmpooler
1269
1626
  provider_class = $config[:providers][provider_name.to_sym]['provider_class']
1270
1627
  end
1271
1628
  begin
1272
- $providers[provider_name] = create_provider_object($config, $logger, $metrics, provider_class, provider_name, {}) if $providers[provider_name].nil?
1629
+ $providers[provider_name] = create_provider_object($config, $logger, $metrics, @redis, provider_class, provider_name, {}) if $providers[provider_name].nil?
1273
1630
  rescue StandardError => e
1274
1631
  $logger.log('s', "Error while creating provider for pool #{pool['name']}: #{e}")
1275
1632
  raise
@@ -1303,6 +1660,13 @@ module Vmpooler
1303
1660
  end
1304
1661
  end
1305
1662
 
1663
+ if !$threads['ondemand_provisioner']
1664
+ check_ondemand_requests
1665
+ elsif !$threads['ondemand_provisioner'].alive?
1666
+ $logger.log('d', '[!] [ondemand_provisioner] worker thread died, restarting')
1667
+ check_ondemand_requests(check_loop_delay_min, check_loop_delay_max, check_loop_delay_decay)
1668
+ end
1669
+
1306
1670
  sleep(loop_delay)
1307
1671
 
1308
1672
  unless maxloop == 0