vmpooler 0.11.3 → 0.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/vmpooler +3 -1
- data/lib/vmpooler.rb +23 -1
- data/lib/vmpooler/api/helpers.rb +16 -16
- data/lib/vmpooler/api/v1.rb +285 -17
- data/lib/vmpooler/generic_connection_pool.rb +5 -23
- data/lib/vmpooler/graphite.rb +2 -0
- data/lib/vmpooler/pool_manager.rb +750 -389
- data/lib/vmpooler/providers/base.rb +2 -1
- data/lib/vmpooler/providers/dummy.rb +2 -2
- data/lib/vmpooler/providers/vsphere.rb +50 -30
- data/lib/vmpooler/util/parsing.rb +16 -0
- data/lib/vmpooler/version.rb +1 -1
- metadata +18 -3
|
@@ -15,35 +15,17 @@ module Vmpooler
|
|
|
15
15
|
@metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == ''
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def with_metrics(options = {})
|
|
21
|
-
Thread.handle_interrupt(Exception => :never) do
|
|
22
|
-
start = Time.now
|
|
23
|
-
conn = checkout(options)
|
|
24
|
-
timespan_ms = ((Time.now - start) * 1000).to_i
|
|
25
|
-
@metrics&.gauge(@metric_prefix + '.available', @available.length)
|
|
26
|
-
@metrics&.timing(@metric_prefix + '.waited', timespan_ms)
|
|
27
|
-
begin
|
|
28
|
-
Thread.handle_interrupt(Exception => :immediate) do
|
|
29
|
-
yield conn
|
|
30
|
-
end
|
|
31
|
-
ensure
|
|
32
|
-
checkin
|
|
33
|
-
@metrics&.gauge(@metric_prefix + '.available', @available.length)
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
else
|
|
38
|
-
# jruby 1.7.x
|
|
39
|
-
def with_metrics(options = {})
|
|
18
|
+
def with_metrics(options = {})
|
|
19
|
+
Thread.handle_interrupt(Exception => :never) do
|
|
40
20
|
start = Time.now
|
|
41
21
|
conn = checkout(options)
|
|
42
22
|
timespan_ms = ((Time.now - start) * 1000).to_i
|
|
43
23
|
@metrics&.gauge(@metric_prefix + '.available', @available.length)
|
|
44
24
|
@metrics&.timing(@metric_prefix + '.waited', timespan_ms)
|
|
45
25
|
begin
|
|
46
|
-
|
|
26
|
+
Thread.handle_interrupt(Exception => :immediate) do
|
|
27
|
+
yield conn
|
|
28
|
+
end
|
|
47
29
|
ensure
|
|
48
30
|
checkin
|
|
49
31
|
@metrics&.gauge(@metric_prefix + '.available', @available.length)
|
data/lib/vmpooler/graphite.rb
CHANGED
|
@@ -35,6 +35,8 @@ module Vmpooler
|
|
|
35
35
|
socket.close
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
|
+
rescue Errno::EADDRNOTAVAIL => e
|
|
39
|
+
warn "Could not assign address to graphite server #{server}: #{e}"
|
|
38
40
|
rescue StandardError => e
|
|
39
41
|
warn "Failure logging #{path} to graphite server [#{server}:#{port}]: #{e}"
|
|
40
42
|
end
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'vmpooler/providers'
|
|
4
|
+
require 'vmpooler/util/parsing'
|
|
4
5
|
require 'spicy-proton'
|
|
6
|
+
require 'resolv' # ruby standard lib
|
|
5
7
|
|
|
6
8
|
module Vmpooler
|
|
7
9
|
class PoolManager
|
|
@@ -9,7 +11,7 @@ module Vmpooler
|
|
|
9
11
|
CHECK_LOOP_DELAY_MAX_DEFAULT = 60
|
|
10
12
|
CHECK_LOOP_DELAY_DECAY_DEFAULT = 2.0
|
|
11
13
|
|
|
12
|
-
def initialize(config, logger,
|
|
14
|
+
def initialize(config, logger, redis_connection_pool, metrics)
|
|
13
15
|
$config = config
|
|
14
16
|
|
|
15
17
|
# Load logger library
|
|
@@ -18,19 +20,19 @@ module Vmpooler
|
|
|
18
20
|
# metrics logging handle
|
|
19
21
|
$metrics = metrics
|
|
20
22
|
|
|
21
|
-
#
|
|
22
|
-
|
|
23
|
+
# Redis connection pool
|
|
24
|
+
@redis = redis_connection_pool
|
|
23
25
|
|
|
24
26
|
# VM Provider objects
|
|
25
|
-
$providers =
|
|
27
|
+
$providers = Concurrent::Hash.new
|
|
26
28
|
|
|
27
29
|
# Our thread-tracker object
|
|
28
|
-
$threads =
|
|
30
|
+
$threads = Concurrent::Hash.new
|
|
29
31
|
|
|
30
32
|
# Pool mutex
|
|
31
|
-
@reconfigure_pool =
|
|
33
|
+
@reconfigure_pool = Concurrent::Hash.new
|
|
32
34
|
|
|
33
|
-
@vm_mutex =
|
|
35
|
+
@vm_mutex = Concurrent::Hash.new
|
|
34
36
|
|
|
35
37
|
# Name generator for generating host names
|
|
36
38
|
@name_generator = Spicy::Proton.new
|
|
@@ -45,24 +47,26 @@ module Vmpooler
|
|
|
45
47
|
|
|
46
48
|
# Place pool configuration in redis so an API instance can discover running pool configuration
|
|
47
49
|
def load_pools_to_redis
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
50
|
+
@redis.with_metrics do |redis|
|
|
51
|
+
previously_configured_pools = redis.smembers('vmpooler__pools')
|
|
52
|
+
currently_configured_pools = []
|
|
53
|
+
config[:pools].each do |pool|
|
|
54
|
+
currently_configured_pools << pool['name']
|
|
55
|
+
redis.sadd('vmpooler__pools', pool['name'])
|
|
56
|
+
pool_keys = pool.keys
|
|
57
|
+
pool_keys.delete('alias')
|
|
58
|
+
to_set = {}
|
|
59
|
+
pool_keys.each do |k|
|
|
60
|
+
to_set[k] = pool[k]
|
|
61
|
+
end
|
|
62
|
+
to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
|
|
63
|
+
redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
|
|
58
64
|
end
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
$redis.srem('vmpooler__pools', pool)
|
|
65
|
-
$redis.del("vmpooler__pool__#{pool}")
|
|
65
|
+
previously_configured_pools.each do |pool|
|
|
66
|
+
unless currently_configured_pools.include? pool
|
|
67
|
+
redis.srem('vmpooler__pools', pool)
|
|
68
|
+
redis.del("vmpooler__pool__#{pool}")
|
|
69
|
+
end
|
|
66
70
|
end
|
|
67
71
|
end
|
|
68
72
|
nil
|
|
@@ -75,7 +79,9 @@ module Vmpooler
|
|
|
75
79
|
_check_pending_vm(vm, pool, timeout, provider)
|
|
76
80
|
rescue StandardError => e
|
|
77
81
|
$logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
|
|
78
|
-
|
|
82
|
+
@redis.with_metrics do |redis|
|
|
83
|
+
fail_pending_vm(vm, pool, timeout, redis)
|
|
84
|
+
end
|
|
79
85
|
raise
|
|
80
86
|
end
|
|
81
87
|
end
|
|
@@ -86,31 +92,38 @@ module Vmpooler
|
|
|
86
92
|
return if mutex.locked?
|
|
87
93
|
|
|
88
94
|
mutex.synchronize do
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
95
|
+
@redis.with_metrics do |redis|
|
|
96
|
+
request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
|
|
97
|
+
if provider.vm_ready?(pool, vm)
|
|
98
|
+
move_pending_vm_to_ready(vm, pool, redis, request_id)
|
|
99
|
+
else
|
|
100
|
+
fail_pending_vm(vm, pool, timeout, redis)
|
|
101
|
+
end
|
|
93
102
|
end
|
|
94
103
|
end
|
|
95
104
|
end
|
|
96
105
|
|
|
97
|
-
def remove_nonexistent_vm(vm, pool)
|
|
98
|
-
|
|
106
|
+
def remove_nonexistent_vm(vm, pool, redis)
|
|
107
|
+
redis.srem("vmpooler__pending__#{pool}", vm)
|
|
99
108
|
$logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
|
|
100
109
|
end
|
|
101
110
|
|
|
102
|
-
def fail_pending_vm(vm, pool, timeout, exists = true)
|
|
103
|
-
clone_stamp =
|
|
104
|
-
return true unless clone_stamp
|
|
111
|
+
def fail_pending_vm(vm, pool, timeout, redis, exists = true)
|
|
112
|
+
clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
|
|
105
113
|
|
|
106
114
|
time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
|
|
107
115
|
if time_since_clone > timeout
|
|
108
116
|
if exists
|
|
109
|
-
|
|
117
|
+
request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
|
|
118
|
+
pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
|
|
119
|
+
redis.multi
|
|
120
|
+
redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm)
|
|
121
|
+
redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}") if request_id
|
|
122
|
+
redis.exec
|
|
110
123
|
$metrics.increment("errors.markedasfailed.#{pool}")
|
|
111
124
|
$logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
|
|
112
125
|
else
|
|
113
|
-
remove_nonexistent_vm(vm, pool)
|
|
126
|
+
remove_nonexistent_vm(vm, pool, redis)
|
|
114
127
|
end
|
|
115
128
|
end
|
|
116
129
|
true
|
|
@@ -119,28 +132,54 @@ module Vmpooler
|
|
|
119
132
|
false
|
|
120
133
|
end
|
|
121
134
|
|
|
122
|
-
def move_pending_vm_to_ready(vm, pool)
|
|
123
|
-
clone_time =
|
|
124
|
-
finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
|
|
135
|
+
def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
|
|
136
|
+
clone_time = redis.hget('vmpooler__vm__' + vm, 'clone')
|
|
137
|
+
finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
|
|
125
138
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
139
|
+
if request_id
|
|
140
|
+
ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
|
141
|
+
if ondemandrequest_hash['status'] == 'failed'
|
|
142
|
+
move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
|
|
143
|
+
return nil
|
|
144
|
+
elsif ondemandrequest_hash['status'] == 'deleted'
|
|
145
|
+
move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' has been deleted")
|
|
146
|
+
return nil
|
|
147
|
+
end
|
|
148
|
+
pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
|
|
149
|
+
|
|
150
|
+
redis.pipelined do
|
|
151
|
+
redis.hset("vmpooler__active__#{pool}", vm, Time.now)
|
|
152
|
+
redis.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
|
|
153
|
+
redis.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token']) if ondemandrequest_hash['token:token']
|
|
154
|
+
redis.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user']) if ondemandrequest_hash['token:user']
|
|
155
|
+
redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
|
|
156
|
+
end
|
|
157
|
+
move_vm_queue(pool, vm, 'pending', 'running', redis)
|
|
158
|
+
check_ondemand_request_ready(request_id, redis)
|
|
159
|
+
else
|
|
160
|
+
redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
|
|
161
|
+
end
|
|
129
162
|
|
|
130
|
-
|
|
131
|
-
|
|
163
|
+
redis.pipelined do
|
|
164
|
+
redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
|
|
165
|
+
redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
|
|
166
|
+
|
|
167
|
+
# last boot time is displayed in API, and used by alarming script
|
|
168
|
+
redis.hset('vmpooler__lastboot', pool, Time.now)
|
|
169
|
+
end
|
|
132
170
|
|
|
133
171
|
$metrics.timing("time_to_ready_state.#{pool}", finish)
|
|
134
|
-
$logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
|
|
172
|
+
$logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") unless request_id
|
|
173
|
+
$logger.log('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'") if request_id
|
|
135
174
|
end
|
|
136
175
|
|
|
137
|
-
def vm_still_ready?(pool_name, vm_name, provider)
|
|
176
|
+
def vm_still_ready?(pool_name, vm_name, provider, redis)
|
|
138
177
|
# Check if the VM is still ready/available
|
|
139
178
|
return true if provider.vm_ready?(pool_name, vm_name)
|
|
140
179
|
|
|
141
180
|
raise("VM #{vm_name} is not ready")
|
|
142
181
|
rescue StandardError
|
|
143
|
-
move_vm_queue(pool_name, vm_name, 'ready', 'completed', "is unreachable, removed from 'ready' queue")
|
|
182
|
+
move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "is unreachable, removed from 'ready' queue")
|
|
144
183
|
end
|
|
145
184
|
|
|
146
185
|
def check_ready_vm(vm, pool_name, ttl, provider)
|
|
@@ -160,34 +199,35 @@ module Vmpooler
|
|
|
160
199
|
return if mutex.locked?
|
|
161
200
|
|
|
162
201
|
mutex.synchronize do
|
|
163
|
-
|
|
164
|
-
|
|
202
|
+
@redis.with_metrics do |redis|
|
|
203
|
+
check_stamp = redis.hget('vmpooler__vm__' + vm, 'check')
|
|
204
|
+
last_checked_too_soon = ((Time.now - Time.parse(check_stamp)).to_i < $config[:config]['vm_checktime'] * 60) if check_stamp
|
|
205
|
+
break if check_stamp && last_checked_too_soon
|
|
165
206
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
if ttl > 0
|
|
207
|
+
redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
|
|
208
|
+
# Check if the hosts TTL has expired
|
|
169
209
|
# if 'boottime' is nil, set bootime to beginning of unix epoch, forces TTL to be assumed expired
|
|
170
|
-
boottime =
|
|
210
|
+
boottime = redis.hget("vmpooler__vm__#{vm}", 'ready')
|
|
171
211
|
if boottime
|
|
172
212
|
boottime = Time.parse(boottime)
|
|
173
213
|
else
|
|
174
214
|
boottime = Time.at(0)
|
|
175
215
|
end
|
|
176
|
-
if (
|
|
177
|
-
|
|
216
|
+
if (Time.now - boottime).to_i > ttl * 60
|
|
217
|
+
redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
|
|
178
218
|
|
|
179
219
|
$logger.log('d', "[!] [#{pool_name}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
|
|
180
|
-
return
|
|
220
|
+
return nil
|
|
181
221
|
end
|
|
182
|
-
end
|
|
183
222
|
|
|
184
|
-
|
|
223
|
+
break if mismatched_hostname?(vm, pool_name, provider, redis)
|
|
185
224
|
|
|
186
|
-
|
|
225
|
+
vm_still_ready?(pool_name, vm, provider, redis)
|
|
226
|
+
end
|
|
187
227
|
end
|
|
188
228
|
end
|
|
189
229
|
|
|
190
|
-
def mismatched_hostname?(vm, pool_name, provider)
|
|
230
|
+
def mismatched_hostname?(vm, pool_name, provider, redis)
|
|
191
231
|
pool_config = $config[:pools][$config[:pool_index][pool_name]]
|
|
192
232
|
check_hostname = pool_config['check_hostname_for_mismatch']
|
|
193
233
|
check_hostname = $config[:config]['check_ready_vm_hostname_for_mismatch'] if check_hostname.nil?
|
|
@@ -196,7 +236,7 @@ module Vmpooler
|
|
|
196
236
|
# Wait one minute before checking a VM for hostname mismatch
|
|
197
237
|
# When checking as soon as the VM passes the ready test the instance
|
|
198
238
|
# often doesn't report its hostname yet causing the VM to be removed immediately
|
|
199
|
-
vm_ready_time =
|
|
239
|
+
vm_ready_time = redis.hget("vmpooler__vm__#{vm}", 'ready')
|
|
200
240
|
if vm_ready_time
|
|
201
241
|
wait_before_checking = 60
|
|
202
242
|
time_since_ready = (Time.now - Time.parse(vm_ready_time)).to_i
|
|
@@ -213,7 +253,7 @@ module Vmpooler
|
|
|
213
253
|
return if hostname.empty?
|
|
214
254
|
return if hostname == vm
|
|
215
255
|
|
|
216
|
-
|
|
256
|
+
redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
|
|
217
257
|
$logger.log('d', "[!] [#{pool_name}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
|
|
218
258
|
true
|
|
219
259
|
end
|
|
@@ -234,49 +274,61 @@ module Vmpooler
|
|
|
234
274
|
return if mutex.locked?
|
|
235
275
|
|
|
236
276
|
mutex.synchronize do
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
277
|
+
catch :stop_checking do
|
|
278
|
+
@redis.with_metrics do |redis|
|
|
279
|
+
# Check that VM is within defined lifetime
|
|
280
|
+
checkouttime = redis.hget('vmpooler__active__' + pool, vm)
|
|
281
|
+
if checkouttime
|
|
282
|
+
time_since_checkout = Time.now - Time.parse(checkouttime)
|
|
283
|
+
running = time_since_checkout / 60 / 60
|
|
284
|
+
|
|
285
|
+
if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
|
|
286
|
+
move_vm_queue(pool, vm, 'running', 'completed', redis, "reached end of TTL after #{ttl} hours")
|
|
287
|
+
throw :stop_checking
|
|
288
|
+
end
|
|
289
|
+
end
|
|
247
290
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
291
|
+
if provider.vm_ready?(pool, vm)
|
|
292
|
+
throw :stop_checking
|
|
293
|
+
else
|
|
294
|
+
host = provider.get_vm(pool, vm)
|
|
252
295
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
296
|
+
if host
|
|
297
|
+
throw :stop_checking
|
|
298
|
+
else
|
|
299
|
+
move_vm_queue(pool, vm, 'running', 'completed', redis, 'is no longer in inventory, removing from running')
|
|
300
|
+
end
|
|
301
|
+
end
|
|
257
302
|
end
|
|
258
303
|
end
|
|
259
304
|
end
|
|
260
305
|
end
|
|
261
306
|
|
|
262
|
-
def move_vm_queue(pool, vm, queue_from, queue_to, msg = nil)
|
|
263
|
-
|
|
307
|
+
def move_vm_queue(pool, vm, queue_from, queue_to, redis, msg = nil)
|
|
308
|
+
redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
|
|
264
309
|
$logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") if msg
|
|
265
310
|
end
|
|
266
311
|
|
|
267
312
|
# Clone a VM
|
|
268
|
-
def clone_vm(pool_name, provider)
|
|
313
|
+
def clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
|
|
269
314
|
Thread.new do
|
|
270
315
|
begin
|
|
271
|
-
_clone_vm(pool_name, provider)
|
|
316
|
+
_clone_vm(pool_name, provider, request_id, pool_alias)
|
|
272
317
|
rescue StandardError => e
|
|
273
|
-
|
|
318
|
+
if request_id
|
|
319
|
+
$logger.log('s', "[!] [#{pool_name}] failed while cloning VM for request #{request_id} with an error: #{e}")
|
|
320
|
+
@redis.with_metrics do |redis|
|
|
321
|
+
redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
|
|
322
|
+
end
|
|
323
|
+
else
|
|
324
|
+
$logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
|
|
325
|
+
end
|
|
274
326
|
raise
|
|
275
327
|
end
|
|
276
328
|
end
|
|
277
329
|
end
|
|
278
330
|
|
|
279
|
-
def generate_and_check_hostname
|
|
331
|
+
def generate_and_check_hostname
|
|
280
332
|
# Generate a randomized hostname. The total name must no longer than 15
|
|
281
333
|
# character including the hyphen. The shortest adjective in the corpus is
|
|
282
334
|
# three characters long. Therefore, we can technically select a noun up to 11
|
|
@@ -285,58 +337,104 @@ module Vmpooler
|
|
|
285
337
|
# letter adjectives, we actually limit the noun to 10 letters to avoid
|
|
286
338
|
# inviting more conflicts. We favor selecting a longer noun rather than a
|
|
287
339
|
# longer adjective because longer adjectives tend to be less fun.
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
340
|
+
@redis.with do |redis|
|
|
341
|
+
noun = @name_generator.noun(max: 10)
|
|
342
|
+
adjective = @name_generator.adjective(max: 14 - noun.length)
|
|
343
|
+
random_name = [adjective, noun].join('-')
|
|
344
|
+
hostname = $config[:config]['prefix'] + random_name
|
|
345
|
+
available = redis.hlen('vmpooler__vm__' + hostname) == 0
|
|
346
|
+
|
|
347
|
+
[hostname, available]
|
|
348
|
+
end
|
|
295
349
|
end
|
|
296
350
|
|
|
297
351
|
def find_unique_hostname(pool_name)
|
|
352
|
+
# generate hostname that is not already in use in vmpooler
|
|
353
|
+
# also check that no dns record already exists
|
|
298
354
|
hostname_retries = 0
|
|
299
355
|
max_hostname_retries = 3
|
|
300
356
|
while hostname_retries < max_hostname_retries
|
|
301
|
-
hostname,
|
|
302
|
-
|
|
357
|
+
hostname, hostname_available = generate_and_check_hostname
|
|
358
|
+
domain = $config[:config]['domain']
|
|
359
|
+
dns_ip, dns_available = check_dns_available(hostname, domain)
|
|
360
|
+
break if hostname_available && dns_available
|
|
303
361
|
|
|
304
362
|
hostname_retries += 1
|
|
305
|
-
|
|
306
|
-
|
|
363
|
+
|
|
364
|
+
if !hostname_available
|
|
365
|
+
$metrics.increment("errors.duplicatehostname.#{pool_name}")
|
|
366
|
+
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
|
|
367
|
+
elsif !dns_available
|
|
368
|
+
$metrics.increment("errors.staledns.#{hostname}")
|
|
369
|
+
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} already exists in DNS records (#{dns_ip}), stale DNS")
|
|
370
|
+
end
|
|
307
371
|
end
|
|
308
372
|
|
|
309
|
-
raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless
|
|
373
|
+
raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless hostname_available && dns_available
|
|
310
374
|
|
|
311
375
|
hostname
|
|
312
376
|
end
|
|
313
377
|
|
|
314
|
-
def
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
$redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
|
|
319
|
-
$redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
|
|
320
|
-
$redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name)
|
|
321
|
-
|
|
378
|
+
def check_dns_available(vm_name, domain = nil)
|
|
379
|
+
# Query the DNS for the name we want to create and if it already exists, mark it unavailable
|
|
380
|
+
# This protects against stale DNS records
|
|
381
|
+
vm_name = "#{vm_name}.#{domain}" if domain
|
|
322
382
|
begin
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
383
|
+
dns_ip = Resolv.getaddress(vm_name)
|
|
384
|
+
rescue Resolv::ResolvError
|
|
385
|
+
# this is the expected case, swallow the error
|
|
386
|
+
# eg "no address for blah-daisy"
|
|
387
|
+
return ['', true]
|
|
388
|
+
end
|
|
389
|
+
[dns_ip, false]
|
|
390
|
+
end
|
|
327
391
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
392
|
+
def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
|
|
393
|
+
new_vmname = find_unique_hostname(pool_name)
|
|
394
|
+
mutex = vm_mutex(new_vmname)
|
|
395
|
+
mutex.synchronize do
|
|
396
|
+
@redis.with_metrics do |redis|
|
|
397
|
+
# Add VM to Redis inventory ('pending' pool)
|
|
398
|
+
redis.multi
|
|
399
|
+
redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
|
|
400
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
|
|
401
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name) # This value is used to represent the pool.
|
|
402
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'pool', pool_name)
|
|
403
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'request_id', request_id) if request_id
|
|
404
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'pool_alias', pool_alias) if pool_alias
|
|
405
|
+
redis.exec
|
|
406
|
+
end
|
|
331
407
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
408
|
+
begin
|
|
409
|
+
$logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
|
|
410
|
+
start = Time.now
|
|
411
|
+
provider.create_vm(pool_name, new_vmname)
|
|
412
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
|
413
|
+
|
|
414
|
+
@redis.with_metrics do |redis|
|
|
415
|
+
redis.pipelined do
|
|
416
|
+
redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
|
|
417
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
|
|
418
|
+
end
|
|
419
|
+
end
|
|
420
|
+
$logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
|
|
421
|
+
|
|
422
|
+
$metrics.timing("clone.#{pool_name}", finish)
|
|
423
|
+
rescue StandardError
|
|
424
|
+
@redis.with_metrics do |redis|
|
|
425
|
+
redis.pipelined do
|
|
426
|
+
redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
|
|
427
|
+
expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
|
|
428
|
+
redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
|
|
429
|
+
end
|
|
430
|
+
end
|
|
431
|
+
raise
|
|
432
|
+
ensure
|
|
433
|
+
@redis.with_metrics do |redis|
|
|
434
|
+
redis.decr('vmpooler__tasks__ondemandclone') if request_id
|
|
435
|
+
redis.decr('vmpooler__tasks__clone') unless request_id
|
|
436
|
+
end
|
|
437
|
+
end
|
|
340
438
|
end
|
|
341
439
|
end
|
|
342
440
|
|
|
@@ -357,36 +455,42 @@ module Vmpooler
|
|
|
357
455
|
return if mutex.locked?
|
|
358
456
|
|
|
359
457
|
mutex.synchronize do
|
|
360
|
-
|
|
361
|
-
|
|
458
|
+
@redis.with_metrics do |redis|
|
|
459
|
+
redis.pipelined do
|
|
460
|
+
redis.hdel('vmpooler__active__' + pool, vm)
|
|
461
|
+
redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
|
|
362
462
|
|
|
363
|
-
|
|
364
|
-
|
|
463
|
+
# Auto-expire metadata key
|
|
464
|
+
redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
|
|
465
|
+
end
|
|
365
466
|
|
|
366
|
-
|
|
467
|
+
start = Time.now
|
|
367
468
|
|
|
368
|
-
|
|
469
|
+
provider.destroy_vm(pool, vm)
|
|
369
470
|
|
|
370
|
-
|
|
471
|
+
redis.srem('vmpooler__completed__' + pool, vm)
|
|
371
472
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
473
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
|
474
|
+
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
|
|
475
|
+
$metrics.timing("destroy.#{pool}", finish)
|
|
476
|
+
get_vm_usage_labels(vm, redis)
|
|
477
|
+
end
|
|
376
478
|
end
|
|
377
479
|
dereference_mutex(vm)
|
|
378
480
|
end
|
|
379
481
|
|
|
380
|
-
def get_vm_usage_labels(vm)
|
|
482
|
+
def get_vm_usage_labels(vm, redis)
|
|
381
483
|
return unless $config[:config]['usage_stats']
|
|
382
484
|
|
|
383
|
-
|
|
485
|
+
redis.multi
|
|
486
|
+
redis.hget("vmpooler__vm__#{vm}", 'checkout')
|
|
487
|
+
redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
|
|
488
|
+
redis.hget("vmpooler__vm__#{vm}", 'token:user')
|
|
489
|
+
redis.hget("vmpooler__vm__#{vm}", 'template')
|
|
490
|
+
checkout, jenkins_build_url, user, poolname = redis.exec
|
|
384
491
|
return if checkout.nil?
|
|
385
492
|
|
|
386
|
-
|
|
387
|
-
user = $redis.hget("vmpooler__vm__#{vm}", 'token:user') || 'unauthenticated'
|
|
388
|
-
poolname = $redis.hget("vmpooler__vm__#{vm}", 'template')
|
|
389
|
-
|
|
493
|
+
user ||= 'unauthenticated'
|
|
390
494
|
unless jenkins_build_url
|
|
391
495
|
user = user.gsub('.', '_')
|
|
392
496
|
$metrics.increment("usage.#{user}.#{poolname}")
|
|
@@ -420,7 +524,8 @@ module Vmpooler
|
|
|
420
524
|
|
|
421
525
|
$metrics.increment(metric_parts.join('.'))
|
|
422
526
|
rescue StandardError => e
|
|
423
|
-
logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
|
|
527
|
+
$logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
|
|
528
|
+
raise
|
|
424
529
|
end
|
|
425
530
|
|
|
426
531
|
def component_to_test(match, labels_string)
|
|
@@ -444,7 +549,7 @@ module Vmpooler
|
|
|
444
549
|
if provider_purge
|
|
445
550
|
Thread.new do
|
|
446
551
|
begin
|
|
447
|
-
purge_vms_and_folders(provider.to_s)
|
|
552
|
+
purge_vms_and_folders($providers[provider.to_s])
|
|
448
553
|
rescue StandardError => e
|
|
449
554
|
$logger.log('s', "[!] failed while purging provider #{provider} VMs and folders with an error: #{e}")
|
|
450
555
|
end
|
|
@@ -455,13 +560,14 @@ module Vmpooler
|
|
|
455
560
|
end
|
|
456
561
|
|
|
457
562
|
# Return a list of pool folders
|
|
458
|
-
def pool_folders(
|
|
563
|
+
def pool_folders(provider)
|
|
564
|
+
provider_name = provider.name
|
|
459
565
|
folders = {}
|
|
460
566
|
$config[:pools].each do |pool|
|
|
461
567
|
next unless pool['provider'] == provider_name
|
|
462
568
|
|
|
463
569
|
folder_parts = pool['folder'].split('/')
|
|
464
|
-
datacenter =
|
|
570
|
+
datacenter = provider.get_target_datacenter_from_config(pool['name'])
|
|
465
571
|
folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
|
|
466
572
|
end
|
|
467
573
|
folders
|
|
@@ -478,8 +584,8 @@ module Vmpooler
|
|
|
478
584
|
def purge_vms_and_folders(provider)
|
|
479
585
|
configured_folders = pool_folders(provider)
|
|
480
586
|
base_folders = get_base_folders(configured_folders)
|
|
481
|
-
whitelist =
|
|
482
|
-
|
|
587
|
+
whitelist = provider.provider_config['folder_whitelist']
|
|
588
|
+
provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
|
|
483
589
|
end
|
|
484
590
|
|
|
485
591
|
def create_vm_disk(pool_name, vm, disk_size, provider)
|
|
@@ -505,10 +611,12 @@ module Vmpooler
|
|
|
505
611
|
finish = format('%<time>.2f', time: Time.now - start)
|
|
506
612
|
|
|
507
613
|
if result
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
614
|
+
@redis.with_metrics do |redis|
|
|
615
|
+
rdisks = redis.hget('vmpooler__vm__' + vm_name, 'disk')
|
|
616
|
+
disks = rdisks ? rdisks.split(':') : []
|
|
617
|
+
disks.push("+#{disk_size}gb")
|
|
618
|
+
redis.hset('vmpooler__vm__' + vm_name, 'disk', disks.join(':'))
|
|
619
|
+
end
|
|
512
620
|
|
|
513
621
|
$logger.log('s', "[+] [disk_manager] '#{vm_name}' attached #{disk_size}gb disk in #{finish} seconds")
|
|
514
622
|
else
|
|
@@ -538,7 +646,9 @@ module Vmpooler
|
|
|
538
646
|
finish = format('%<time>.2f', time: Time.now - start)
|
|
539
647
|
|
|
540
648
|
if result
|
|
541
|
-
|
|
649
|
+
@redis.with_metrics do |redis|
|
|
650
|
+
redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
|
|
651
|
+
end
|
|
542
652
|
$logger.log('s', "[+] [snapshot_manager] '#{vm_name}' snapshot created in #{finish} seconds")
|
|
543
653
|
else
|
|
544
654
|
$logger.log('s', "[+] [snapshot_manager] Failed to snapshot '#{vm_name}'")
|
|
@@ -594,9 +704,9 @@ module Vmpooler
|
|
|
594
704
|
@default_providers ||= %w[vsphere dummy]
|
|
595
705
|
end
|
|
596
706
|
|
|
597
|
-
def get_pool_name_for_vm(vm_name)
|
|
707
|
+
def get_pool_name_for_vm(vm_name, redis)
|
|
598
708
|
# the 'template' is a bad name. Should really be 'poolname'
|
|
599
|
-
|
|
709
|
+
redis.hget('vmpooler__vm__' + vm_name, 'template')
|
|
600
710
|
end
|
|
601
711
|
|
|
602
712
|
# @param pool_name [String] - the name of the pool
|
|
@@ -628,19 +738,21 @@ module Vmpooler
|
|
|
628
738
|
end
|
|
629
739
|
|
|
630
740
|
def _check_disk_queue
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
741
|
+
@redis.with_metrics do |redis|
|
|
742
|
+
task_detail = redis.spop('vmpooler__tasks__disk')
|
|
743
|
+
unless task_detail.nil?
|
|
744
|
+
begin
|
|
745
|
+
vm_name, disk_size = task_detail.split(':')
|
|
746
|
+
pool_name = get_pool_name_for_vm(vm_name, redis)
|
|
747
|
+
raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
|
|
637
748
|
|
|
638
|
-
|
|
639
|
-
|
|
749
|
+
provider = get_provider_for_pool(pool_name)
|
|
750
|
+
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
|
640
751
|
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
752
|
+
create_vm_disk(pool_name, vm_name, disk_size, provider)
|
|
753
|
+
rescue StandardError => e
|
|
754
|
+
$logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{e}")
|
|
755
|
+
end
|
|
644
756
|
end
|
|
645
757
|
end
|
|
646
758
|
end
|
|
@@ -664,37 +776,39 @@ module Vmpooler
|
|
|
664
776
|
end
|
|
665
777
|
|
|
666
778
|
def _check_snapshot_queue
|
|
667
|
-
|
|
779
|
+
@redis.with_metrics do |redis|
|
|
780
|
+
task_detail = redis.spop('vmpooler__tasks__snapshot')
|
|
668
781
|
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
782
|
+
unless task_detail.nil?
|
|
783
|
+
begin
|
|
784
|
+
vm_name, snapshot_name = task_detail.split(':')
|
|
785
|
+
pool_name = get_pool_name_for_vm(vm_name, redis)
|
|
786
|
+
raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
|
|
674
787
|
|
|
675
|
-
|
|
676
|
-
|
|
788
|
+
provider = get_provider_for_pool(pool_name)
|
|
789
|
+
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
|
677
790
|
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
791
|
+
create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
|
|
792
|
+
rescue StandardError => e
|
|
793
|
+
$logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{e}")
|
|
794
|
+
end
|
|
681
795
|
end
|
|
682
|
-
end
|
|
683
796
|
|
|
684
|
-
|
|
797
|
+
task_detail = redis.spop('vmpooler__tasks__snapshot-revert')
|
|
685
798
|
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
799
|
+
unless task_detail.nil?
|
|
800
|
+
begin
|
|
801
|
+
vm_name, snapshot_name = task_detail.split(':')
|
|
802
|
+
pool_name = get_pool_name_for_vm(vm_name, redis)
|
|
803
|
+
raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
|
|
691
804
|
|
|
692
|
-
|
|
693
|
-
|
|
805
|
+
provider = get_provider_for_pool(pool_name)
|
|
806
|
+
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
|
694
807
|
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
808
|
+
revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
|
|
809
|
+
rescue StandardError => e
|
|
810
|
+
$logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{e}")
|
|
811
|
+
end
|
|
698
812
|
end
|
|
699
813
|
end
|
|
700
814
|
end
|
|
@@ -704,7 +818,9 @@ module Vmpooler
|
|
|
704
818
|
begin
|
|
705
819
|
mutex = vm_mutex(vm_name)
|
|
706
820
|
mutex.synchronize do
|
|
707
|
-
|
|
821
|
+
@redis.with_metrics do |redis|
|
|
822
|
+
redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
|
|
823
|
+
end
|
|
708
824
|
provider.migrate_vm(pool_name, vm_name)
|
|
709
825
|
end
|
|
710
826
|
rescue StandardError => e
|
|
@@ -737,47 +853,65 @@ module Vmpooler
|
|
|
737
853
|
wakeup_by = Time.now + wakeup_period
|
|
738
854
|
return if time_passed?(:exit_by, exit_by)
|
|
739
855
|
|
|
740
|
-
|
|
856
|
+
@redis.with_metrics do |redis|
|
|
857
|
+
initial_ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
|
|
741
858
|
|
|
742
|
-
|
|
859
|
+
initial_clone_target = redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
|
|
743
860
|
|
|
744
|
-
|
|
861
|
+
initial_template = redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
|
|
745
862
|
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
863
|
+
loop do
|
|
864
|
+
sleep(1)
|
|
865
|
+
break if time_passed?(:exit_by, exit_by)
|
|
749
866
|
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
867
|
+
# Check for wakeup events
|
|
868
|
+
if time_passed?(:wakeup_by, wakeup_by)
|
|
869
|
+
wakeup_by = Time.now + wakeup_period
|
|
753
870
|
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
871
|
+
# Wakeup if the number of ready VMs has changed
|
|
872
|
+
if options[:pool_size_change]
|
|
873
|
+
ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}")
|
|
874
|
+
break unless ready_size == initial_ready_size
|
|
875
|
+
end
|
|
759
876
|
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
877
|
+
if options[:clone_target_change]
|
|
878
|
+
clone_target = redis.hget('vmpooler__config__clone_target}', options[:poolname])
|
|
879
|
+
if clone_target
|
|
880
|
+
break unless clone_target == initial_clone_target
|
|
881
|
+
end
|
|
764
882
|
end
|
|
765
|
-
end
|
|
766
883
|
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
884
|
+
if options[:pool_template_change]
|
|
885
|
+
configured_template = redis.hget('vmpooler__config__template', options[:poolname])
|
|
886
|
+
if configured_template
|
|
887
|
+
break unless initial_template == configured_template
|
|
888
|
+
end
|
|
889
|
+
end
|
|
890
|
+
|
|
891
|
+
if options[:pool_reset]
|
|
892
|
+
pending = redis.sismember('vmpooler__poolreset', options[:poolname])
|
|
893
|
+
break if pending
|
|
771
894
|
end
|
|
772
|
-
end
|
|
773
895
|
|
|
774
|
-
|
|
775
|
-
|
|
896
|
+
if options[:pending_vm]
|
|
897
|
+
pending_vm_count = redis.scard("vmpooler__pending__#{options[:poolname]}")
|
|
898
|
+
break unless pending_vm_count == 0
|
|
899
|
+
end
|
|
900
|
+
|
|
901
|
+
if options[:ondemand_request]
|
|
902
|
+
redis.multi
|
|
903
|
+
redis.zcard('vmpooler__provisioning__request')
|
|
904
|
+
redis.zcard('vmpooler__provisioning__processing')
|
|
905
|
+
redis.zcard('vmpooler__odcreate__task')
|
|
906
|
+
od_request, od_processing, od_createtask = redis.exec
|
|
907
|
+
break unless od_request == 0
|
|
908
|
+
break unless od_processing == 0
|
|
909
|
+
break unless od_createtask == 0
|
|
910
|
+
end
|
|
776
911
|
end
|
|
777
912
|
|
|
913
|
+
break if time_passed?(:exit_by, exit_by)
|
|
778
914
|
end
|
|
779
|
-
|
|
780
|
-
break if time_passed?(:exit_by, exit_by)
|
|
781
915
|
end
|
|
782
916
|
end
|
|
783
917
|
|
|
@@ -813,7 +947,7 @@ module Vmpooler
|
|
|
813
947
|
loop_delay = (loop_delay * loop_delay_decay).to_i
|
|
814
948
|
loop_delay = loop_delay_max if loop_delay > loop_delay_max
|
|
815
949
|
end
|
|
816
|
-
sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pool_reset: true)
|
|
950
|
+
sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pending_vm: true, pool_reset: true)
|
|
817
951
|
|
|
818
952
|
unless maxloop == 0
|
|
819
953
|
break if loop_count >= maxloop
|
|
@@ -843,77 +977,84 @@ module Vmpooler
|
|
|
843
977
|
end
|
|
844
978
|
|
|
845
979
|
def sync_pool_template(pool)
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
980
|
+
@redis.with_metrics do |redis|
|
|
981
|
+
pool_template = redis.hget('vmpooler__config__template', pool['name'])
|
|
982
|
+
if pool_template
|
|
983
|
+
pool['template'] = pool_template unless pool['template'] == pool_template
|
|
984
|
+
end
|
|
849
985
|
end
|
|
850
986
|
end
|
|
851
987
|
|
|
852
|
-
def prepare_template(pool, provider)
|
|
988
|
+
def prepare_template(pool, provider, redis)
|
|
853
989
|
if $config[:config]['create_template_delta_disks']
|
|
854
|
-
unless
|
|
990
|
+
unless redis.sismember('vmpooler__template__deltas', pool['template'])
|
|
855
991
|
begin
|
|
856
992
|
provider.create_template_delta_disks(pool)
|
|
857
|
-
|
|
993
|
+
redis.sadd('vmpooler__template__deltas', pool['template'])
|
|
858
994
|
rescue StandardError => e
|
|
859
995
|
$logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{e}")
|
|
860
996
|
end
|
|
861
997
|
end
|
|
862
998
|
end
|
|
863
|
-
|
|
999
|
+
redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
|
|
864
1000
|
end
|
|
865
1001
|
|
|
866
1002
|
def evaluate_template(pool, provider)
|
|
867
1003
|
mutex = pool_mutex(pool['name'])
|
|
868
|
-
prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
|
|
869
|
-
configured_template = $redis.hget('vmpooler__config__template', pool['name'])
|
|
870
1004
|
return if mutex.locked?
|
|
871
1005
|
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
1006
|
+
catch :update_not_needed do
|
|
1007
|
+
@redis.with_metrics do |redis|
|
|
1008
|
+
prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
|
|
1009
|
+
configured_template = redis.hget('vmpooler__config__template', pool['name'])
|
|
1010
|
+
|
|
1011
|
+
if prepared_template.nil?
|
|
1012
|
+
mutex.synchronize do
|
|
1013
|
+
prepare_template(pool, provider, redis)
|
|
1014
|
+
prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
|
|
1015
|
+
end
|
|
1016
|
+
elsif prepared_template != pool['template']
|
|
1017
|
+
if configured_template.nil?
|
|
1018
|
+
mutex.synchronize do
|
|
1019
|
+
prepare_template(pool, provider, redis)
|
|
1020
|
+
prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
|
|
1021
|
+
end
|
|
1022
|
+
end
|
|
1023
|
+
end
|
|
1024
|
+
throw :update_not_needed if configured_template.nil?
|
|
1025
|
+
throw :update_not_needed if configured_template == prepared_template
|
|
1026
|
+
|
|
879
1027
|
mutex.synchronize do
|
|
880
|
-
|
|
881
|
-
prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
|
|
1028
|
+
update_pool_template(pool, provider, configured_template, prepared_template, redis)
|
|
882
1029
|
end
|
|
883
1030
|
end
|
|
884
1031
|
end
|
|
885
|
-
return if configured_template.nil?
|
|
886
|
-
return if configured_template == prepared_template
|
|
887
|
-
|
|
888
|
-
mutex.synchronize do
|
|
889
|
-
update_pool_template(pool, provider, configured_template, prepared_template)
|
|
890
|
-
end
|
|
891
1032
|
end
|
|
892
1033
|
|
|
893
|
-
def drain_pool(poolname)
|
|
1034
|
+
def drain_pool(poolname, redis)
|
|
894
1035
|
# Clear a pool of ready and pending instances
|
|
895
|
-
if
|
|
1036
|
+
if redis.scard("vmpooler__ready__#{poolname}") > 0
|
|
896
1037
|
$logger.log('s', "[*] [#{poolname}] removing ready instances")
|
|
897
|
-
|
|
898
|
-
move_vm_queue(poolname, vm, 'ready', 'completed')
|
|
1038
|
+
redis.smembers("vmpooler__ready__#{poolname}").each do |vm|
|
|
1039
|
+
move_vm_queue(poolname, vm, 'ready', 'completed', redis)
|
|
899
1040
|
end
|
|
900
1041
|
end
|
|
901
|
-
if
|
|
1042
|
+
if redis.scard("vmpooler__pending__#{poolname}") > 0
|
|
902
1043
|
$logger.log('s', "[*] [#{poolname}] removing pending instances")
|
|
903
|
-
|
|
904
|
-
move_vm_queue(poolname, vm, 'pending', 'completed')
|
|
1044
|
+
redis.smembers("vmpooler__pending__#{poolname}").each do |vm|
|
|
1045
|
+
move_vm_queue(poolname, vm, 'pending', 'completed', redis)
|
|
905
1046
|
end
|
|
906
1047
|
end
|
|
907
1048
|
end
|
|
908
1049
|
|
|
909
|
-
def update_pool_template(pool, provider, configured_template, prepared_template)
|
|
1050
|
+
def update_pool_template(pool, provider, configured_template, prepared_template, redis)
|
|
910
1051
|
pool['template'] = configured_template
|
|
911
1052
|
$logger.log('s', "[*] [#{pool['name']}] template updated from #{prepared_template} to #{configured_template}")
|
|
912
1053
|
# Remove all ready and pending VMs so new instances are created from the new template
|
|
913
|
-
drain_pool(pool['name'])
|
|
1054
|
+
drain_pool(pool['name'], redis)
|
|
914
1055
|
# Prepare template for deployment
|
|
915
1056
|
$logger.log('s', "[*] [#{pool['name']}] preparing pool template for deployment")
|
|
916
|
-
prepare_template(pool, provider)
|
|
1057
|
+
prepare_template(pool, provider, redis)
|
|
917
1058
|
$logger.log('s', "[*] [#{pool['name']}] is ready for use")
|
|
918
1059
|
end
|
|
919
1060
|
|
|
@@ -921,38 +1062,45 @@ module Vmpooler
|
|
|
921
1062
|
mutex = pool_mutex(pool['name'])
|
|
922
1063
|
return if mutex.locked?
|
|
923
1064
|
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
1065
|
+
@redis.with_metrics do |redis|
|
|
1066
|
+
clone_target = redis.hget('vmpooler__config__clone_target', pool['name'])
|
|
1067
|
+
break if clone_target.nil?
|
|
1068
|
+
break if clone_target == pool['clone_target']
|
|
927
1069
|
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
1070
|
+
$logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
|
|
1071
|
+
mutex.synchronize do
|
|
1072
|
+
pool['clone_target'] = clone_target
|
|
1073
|
+
# Remove all ready and pending VMs so new instances are created for the new clone_target
|
|
1074
|
+
drain_pool(pool['name'], redis)
|
|
1075
|
+
end
|
|
1076
|
+
$logger.log('s', "[*] [#{pool['name']}] is ready for use")
|
|
933
1077
|
end
|
|
934
|
-
$logger.log('s', "[*] [#{pool['name']}] is ready for use")
|
|
935
1078
|
end
|
|
936
1079
|
|
|
937
1080
|
def remove_excess_vms(pool)
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
1081
|
+
@redis.with_metrics do |redis|
|
|
1082
|
+
redis.multi
|
|
1083
|
+
redis.scard("vmpooler__ready__#{pool['name']}")
|
|
1084
|
+
redis.scard("vmpooler__pending__#{pool['name']}")
|
|
1085
|
+
ready, pending = redis.exec
|
|
1086
|
+
total = pending.to_i + ready.to_i
|
|
1087
|
+
break if total.nil?
|
|
1088
|
+
break if total == 0
|
|
942
1089
|
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
1090
|
+
mutex = pool_mutex(pool['name'])
|
|
1091
|
+
break if mutex.locked?
|
|
1092
|
+
break unless ready.to_i > pool['size']
|
|
946
1093
|
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
1094
|
+
mutex.synchronize do
|
|
1095
|
+
difference = ready.to_i - pool['size']
|
|
1096
|
+
difference.times do
|
|
1097
|
+
next_vm = redis.spop("vmpooler__ready__#{pool['name']}")
|
|
1098
|
+
move_vm_queue(pool['name'], next_vm, 'ready', 'completed', redis)
|
|
1099
|
+
end
|
|
1100
|
+
if total > ready
|
|
1101
|
+
redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
|
|
1102
|
+
move_vm_queue(pool['name'], vm, 'pending', 'completed', redis)
|
|
1103
|
+
end
|
|
956
1104
|
end
|
|
957
1105
|
end
|
|
958
1106
|
end
|
|
@@ -962,26 +1110,30 @@ module Vmpooler
|
|
|
962
1110
|
mutex = pool_mutex(pool['name'])
|
|
963
1111
|
return if mutex.locked?
|
|
964
1112
|
|
|
965
|
-
|
|
966
|
-
|
|
1113
|
+
@redis.with_metrics do |redis|
|
|
1114
|
+
poolsize = redis.hget('vmpooler__config__poolsize', pool['name'])
|
|
1115
|
+
break if poolsize.nil?
|
|
967
1116
|
|
|
968
|
-
|
|
969
|
-
|
|
1117
|
+
poolsize = Integer(poolsize)
|
|
1118
|
+
break if poolsize == pool['size']
|
|
970
1119
|
|
|
971
|
-
|
|
972
|
-
|
|
1120
|
+
mutex.synchronize do
|
|
1121
|
+
pool['size'] = poolsize
|
|
1122
|
+
end
|
|
973
1123
|
end
|
|
974
1124
|
end
|
|
975
1125
|
|
|
976
1126
|
def reset_pool(pool)
|
|
977
1127
|
poolname = pool['name']
|
|
978
|
-
|
|
1128
|
+
@redis.with_metrics do |redis|
|
|
1129
|
+
break unless redis.sismember('vmpooler__poolreset', poolname)
|
|
979
1130
|
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1131
|
+
redis.srem('vmpooler__poolreset', poolname)
|
|
1132
|
+
mutex = pool_mutex(poolname)
|
|
1133
|
+
mutex.synchronize do
|
|
1134
|
+
drain_pool(poolname, redis)
|
|
1135
|
+
$logger.log('s', "[*] [#{poolname}] reset has cleared ready and pending instances")
|
|
1136
|
+
end
|
|
985
1137
|
end
|
|
986
1138
|
end
|
|
987
1139
|
|
|
@@ -990,21 +1142,23 @@ module Vmpooler
|
|
|
990
1142
|
begin
|
|
991
1143
|
mutex = pool_mutex(pool['name'])
|
|
992
1144
|
mutex.synchronize do
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1145
|
+
@redis.with_metrics do |redis|
|
|
1146
|
+
provider.vms_in_pool(pool['name']).each do |vm|
|
|
1147
|
+
if !redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
|
|
1148
|
+
!redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
|
|
1149
|
+
!redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) &&
|
|
1150
|
+
!redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) &&
|
|
1151
|
+
!redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) &&
|
|
1152
|
+
!redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])
|
|
1153
|
+
|
|
1154
|
+
pool_check_response[:discovered_vms] += 1
|
|
1155
|
+
redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
|
|
1156
|
+
|
|
1157
|
+
$logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
|
|
1158
|
+
end
|
|
1159
|
+
|
|
1160
|
+
inventory[vm['name']] = 1
|
|
1005
1161
|
end
|
|
1006
|
-
|
|
1007
|
-
inventory[vm['name']] = 1
|
|
1008
1162
|
end
|
|
1009
1163
|
end
|
|
1010
1164
|
rescue StandardError => e
|
|
@@ -1015,96 +1169,112 @@ module Vmpooler
|
|
|
1015
1169
|
end
|
|
1016
1170
|
|
|
1017
1171
|
def check_running_pool_vms(pool_name, provider, pool_check_response, inventory)
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1172
|
+
@redis.with_metrics do |redis|
|
|
1173
|
+
redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
|
|
1174
|
+
if inventory[vm]
|
|
1175
|
+
begin
|
|
1176
|
+
vm_lifetime = redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
|
|
1177
|
+
pool_check_response[:checked_running_vms] += 1
|
|
1178
|
+
check_running_vm(vm, pool_name, vm_lifetime, provider)
|
|
1179
|
+
rescue StandardError => e
|
|
1180
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{e}")
|
|
1181
|
+
end
|
|
1182
|
+
else
|
|
1183
|
+
move_vm_queue(pool_name, vm, 'running', 'completed', redis, 'is a running VM but is missing from inventory. Marking as completed.')
|
|
1026
1184
|
end
|
|
1027
|
-
else
|
|
1028
|
-
move_vm_queue(pool_name, vm, 'running', 'completed', 'is a running VM but is missing from inventory. Marking as completed.')
|
|
1029
1185
|
end
|
|
1030
1186
|
end
|
|
1031
1187
|
end
|
|
1032
1188
|
|
|
1033
|
-
def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1189
|
+
def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl)
|
|
1190
|
+
@redis.with_metrics do |redis|
|
|
1191
|
+
redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
|
|
1192
|
+
if inventory[vm]
|
|
1193
|
+
begin
|
|
1194
|
+
pool_check_response[:checked_ready_vms] += 1
|
|
1195
|
+
check_ready_vm(vm, pool_name, pool_ttl, provider)
|
|
1196
|
+
rescue StandardError => e
|
|
1197
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
|
|
1198
|
+
end
|
|
1199
|
+
else
|
|
1200
|
+
move_vm_queue(pool_name, vm, 'ready', 'completed', redis, 'is a ready VM but is missing from inventory. Marking as completed.')
|
|
1041
1201
|
end
|
|
1042
|
-
else
|
|
1043
|
-
move_vm_queue(pool_name, vm, 'ready', 'completed', 'is a ready VM but is missing from inventory. Marking as completed.')
|
|
1044
1202
|
end
|
|
1045
1203
|
end
|
|
1046
1204
|
end
|
|
1047
1205
|
|
|
1048
|
-
def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout
|
|
1206
|
+
def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout)
|
|
1049
1207
|
pool_timeout ||= $config[:config]['timeout'] || 15
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1208
|
+
@redis.with_metrics do |redis|
|
|
1209
|
+
redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
|
|
1210
|
+
if inventory[vm]
|
|
1211
|
+
begin
|
|
1212
|
+
pool_check_response[:checked_pending_vms] += 1
|
|
1213
|
+
check_pending_vm(vm, pool_name, pool_timeout, provider)
|
|
1214
|
+
rescue StandardError => e
|
|
1215
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
|
|
1216
|
+
end
|
|
1217
|
+
else
|
|
1218
|
+
fail_pending_vm(vm, pool_name, pool_timeout, redis, false)
|
|
1057
1219
|
end
|
|
1058
|
-
else
|
|
1059
|
-
fail_pending_vm(vm, pool_name, pool_timeout, false)
|
|
1060
1220
|
end
|
|
1061
1221
|
end
|
|
1062
1222
|
end
|
|
1063
1223
|
|
|
1064
1224
|
def check_completed_pool_vms(pool_name, provider, pool_check_response, inventory)
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1225
|
+
@redis.with_metrics do |redis|
|
|
1226
|
+
redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
|
|
1227
|
+
if inventory[vm]
|
|
1228
|
+
begin
|
|
1229
|
+
pool_check_response[:destroyed_vms] += 1
|
|
1230
|
+
destroy_vm(vm, pool_name, provider)
|
|
1231
|
+
rescue StandardError => e
|
|
1232
|
+
redis.pipelined do
|
|
1233
|
+
redis.srem("vmpooler__completed__#{pool_name}", vm)
|
|
1234
|
+
redis.hdel("vmpooler__active__#{pool_name}", vm)
|
|
1235
|
+
redis.del("vmpooler__vm__#{vm}")
|
|
1236
|
+
end
|
|
1237
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
|
|
1238
|
+
end
|
|
1239
|
+
else
|
|
1240
|
+
$logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
|
1241
|
+
redis.pipelined do
|
|
1242
|
+
redis.srem("vmpooler__completed__#{pool_name}", vm)
|
|
1243
|
+
redis.hdel("vmpooler__active__#{pool_name}", vm)
|
|
1244
|
+
redis.del("vmpooler__vm__#{vm}")
|
|
1245
|
+
end
|
|
1075
1246
|
end
|
|
1076
|
-
else
|
|
1077
|
-
$logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
|
1078
|
-
$redis.srem("vmpooler__completed__#{pool_name}", vm)
|
|
1079
|
-
$redis.hdel("vmpooler__active__#{pool_name}", vm)
|
|
1080
|
-
$redis.del("vmpooler__vm__#{vm}")
|
|
1081
1247
|
end
|
|
1082
1248
|
end
|
|
1083
1249
|
end
|
|
1084
1250
|
|
|
1085
1251
|
def check_discovered_pool_vms(pool_name)
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1252
|
+
@redis.with_metrics do |redis|
|
|
1253
|
+
redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
|
|
1254
|
+
%w[pending ready running completed].each do |queue|
|
|
1255
|
+
if redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
|
|
1256
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
|
|
1257
|
+
redis.srem("vmpooler__discovered__#{pool_name}", vm)
|
|
1258
|
+
end
|
|
1091
1259
|
end
|
|
1092
|
-
end
|
|
1093
1260
|
|
|
1094
|
-
|
|
1261
|
+
redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm) if redis.sismember("vmpooler__discovered__#{pool_name}", vm)
|
|
1262
|
+
end
|
|
1095
1263
|
end
|
|
1096
1264
|
rescue StandardError => e
|
|
1097
1265
|
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating discovered VMs: #{e}")
|
|
1098
1266
|
end
|
|
1099
1267
|
|
|
1100
1268
|
def check_migrating_pool_vms(pool_name, provider, pool_check_response, inventory)
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1269
|
+
@redis.with_metrics do |redis|
|
|
1270
|
+
redis.smembers("vmpooler__migrating__#{pool_name}").reverse.each do |vm|
|
|
1271
|
+
if inventory[vm]
|
|
1272
|
+
begin
|
|
1273
|
+
pool_check_response[:migrated_vms] += 1
|
|
1274
|
+
migrate_vm(vm, pool_name, provider)
|
|
1275
|
+
rescue StandardError => e
|
|
1276
|
+
$logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{e}")
|
|
1277
|
+
end
|
|
1108
1278
|
end
|
|
1109
1279
|
end
|
|
1110
1280
|
end
|
|
@@ -1113,29 +1283,37 @@ module Vmpooler
|
|
|
1113
1283
|
def repopulate_pool_vms(pool_name, provider, pool_check_response, pool_size)
|
|
1114
1284
|
return if pool_mutex(pool_name).locked?
|
|
1115
1285
|
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
$
|
|
1126
|
-
|
|
1127
|
-
|
|
1286
|
+
@redis.with_metrics do |redis|
|
|
1287
|
+
redis.multi
|
|
1288
|
+
redis.scard("vmpooler__ready__#{pool_name}")
|
|
1289
|
+
redis.scard("vmpooler__pending__#{pool_name}")
|
|
1290
|
+
redis.scard("vmpooler__running__#{pool_name}")
|
|
1291
|
+
ready, pending, running = redis.exec
|
|
1292
|
+
total = pending.to_i + ready.to_i
|
|
1293
|
+
|
|
1294
|
+
$metrics.gauge("ready.#{pool_name}", ready)
|
|
1295
|
+
$metrics.gauge("running.#{pool_name}", running)
|
|
1296
|
+
|
|
1297
|
+
unless pool_size == 0
|
|
1298
|
+
if redis.get("vmpooler__empty__#{pool_name}")
|
|
1299
|
+
redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
|
|
1300
|
+
elsif ready == 0
|
|
1301
|
+
redis.set("vmpooler__empty__#{pool_name}", 'true')
|
|
1302
|
+
$logger.log('s', "[!] [#{pool_name}] is empty")
|
|
1303
|
+
end
|
|
1304
|
+
end
|
|
1128
1305
|
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1306
|
+
(pool_size - total.to_i).times do
|
|
1307
|
+
if redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
|
|
1308
|
+
begin
|
|
1309
|
+
redis.incr('vmpooler__tasks__clone')
|
|
1310
|
+
pool_check_response[:cloned_vms] += 1
|
|
1311
|
+
clone_vm(pool_name, provider)
|
|
1312
|
+
rescue StandardError => e
|
|
1313
|
+
$logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
|
|
1314
|
+
redis.decr('vmpooler__tasks__clone')
|
|
1315
|
+
raise
|
|
1316
|
+
end
|
|
1139
1317
|
end
|
|
1140
1318
|
end
|
|
1141
1319
|
end
|
|
@@ -1160,7 +1338,7 @@ module Vmpooler
|
|
|
1160
1338
|
|
|
1161
1339
|
check_running_pool_vms(pool['name'], provider, pool_check_response, inventory)
|
|
1162
1340
|
|
|
1163
|
-
check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'])
|
|
1341
|
+
check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
|
|
1164
1342
|
|
|
1165
1343
|
check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
|
|
1166
1344
|
|
|
@@ -1203,23 +1381,199 @@ module Vmpooler
|
|
|
1203
1381
|
#
|
|
1204
1382
|
# returns an object Vmpooler::PoolManager::Provider::*
|
|
1205
1383
|
# or raises an error if the class does not exist
|
|
1206
|
-
def create_provider_object(config, logger, metrics, provider_class, provider_name, options)
|
|
1384
|
+
def create_provider_object(config, logger, metrics, redis_connection_pool, provider_class, provider_name, options)
|
|
1207
1385
|
provider_klass = Vmpooler::PoolManager::Provider
|
|
1208
1386
|
provider_klass.constants.each do |classname|
|
|
1209
1387
|
next unless classname.to_s.casecmp(provider_class) == 0
|
|
1210
1388
|
|
|
1211
|
-
return provider_klass.const_get(classname).new(config, logger, metrics, provider_name, options)
|
|
1389
|
+
return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options)
|
|
1212
1390
|
end
|
|
1213
1391
|
raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
|
|
1214
1392
|
end
|
|
1215
1393
|
|
|
1394
|
+
def check_ondemand_requests(maxloop = 0,
|
|
1395
|
+
loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
|
|
1396
|
+
loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
|
|
1397
|
+
loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
|
|
1398
|
+
|
|
1399
|
+
$logger.log('d', '[*] [ondemand_provisioner] starting worker thread')
|
|
1400
|
+
|
|
1401
|
+
$threads['ondemand_provisioner'] = Thread.new do
|
|
1402
|
+
_check_ondemand_requests(maxloop, loop_delay_min, loop_delay_max, loop_delay_decay)
|
|
1403
|
+
end
|
|
1404
|
+
end
|
|
1405
|
+
|
|
1406
|
+
def _check_ondemand_requests(maxloop = 0,
|
|
1407
|
+
loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
|
|
1408
|
+
loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
|
|
1409
|
+
loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
|
|
1410
|
+
|
|
1411
|
+
loop_delay_min = $config[:config]['check_loop_delay_min'] unless $config[:config]['check_loop_delay_min'].nil?
|
|
1412
|
+
loop_delay_max = $config[:config]['check_loop_delay_max'] unless $config[:config]['check_loop_delay_max'].nil?
|
|
1413
|
+
loop_delay_decay = $config[:config]['check_loop_delay_decay'] unless $config[:config]['check_loop_delay_decay'].nil?
|
|
1414
|
+
|
|
1415
|
+
loop_delay_decay = 2.0 if loop_delay_decay <= 1.0
|
|
1416
|
+
loop_delay_max = loop_delay_min if loop_delay_max.nil? || loop_delay_max < loop_delay_min
|
|
1417
|
+
|
|
1418
|
+
loop_count = 1
|
|
1419
|
+
loop_delay = loop_delay_min
|
|
1420
|
+
|
|
1421
|
+
loop do
|
|
1422
|
+
result = process_ondemand_requests
|
|
1423
|
+
|
|
1424
|
+
loop_delay = (loop_delay * loop_delay_decay).to_i
|
|
1425
|
+
loop_delay = loop_delay_min if result > 0
|
|
1426
|
+
loop_delay = loop_delay_max if loop_delay > loop_delay_max
|
|
1427
|
+
sleep_with_wakeup_events(loop_delay, loop_delay_min, ondemand_request: true)
|
|
1428
|
+
|
|
1429
|
+
unless maxloop == 0
|
|
1430
|
+
break if loop_count >= maxloop
|
|
1431
|
+
|
|
1432
|
+
loop_count += 1
|
|
1433
|
+
end
|
|
1434
|
+
end
|
|
1435
|
+
end
|
|
1436
|
+
|
|
1437
|
+
def process_ondemand_requests
|
|
1438
|
+
@redis.with_metrics do |redis|
|
|
1439
|
+
requests = redis.zrange('vmpooler__provisioning__request', 0, -1)
|
|
1440
|
+
requests&.map { |request_id| create_ondemand_vms(request_id, redis) }
|
|
1441
|
+
provisioning_tasks = process_ondemand_vms(redis)
|
|
1442
|
+
requests_ready = check_ondemand_requests_ready(redis)
|
|
1443
|
+
requests.length + provisioning_tasks + requests_ready
|
|
1444
|
+
end
|
|
1445
|
+
end
|
|
1446
|
+
|
|
1447
|
+
def create_ondemand_vms(request_id, redis)
|
|
1448
|
+
requested = redis.hget("vmpooler__odrequest__#{request_id}", 'requested')
|
|
1449
|
+
unless requested
|
|
1450
|
+
$logger.log('s', "Failed to find odrequest for request_id '#{request_id}'")
|
|
1451
|
+
redis.zrem('vmpooler__provisioning__request', request_id)
|
|
1452
|
+
return
|
|
1453
|
+
end
|
|
1454
|
+
score = redis.zscore('vmpooler__provisioning__request', request_id)
|
|
1455
|
+
requested = requested.split(',')
|
|
1456
|
+
|
|
1457
|
+
redis.pipelined do
|
|
1458
|
+
requested.each do |request|
|
|
1459
|
+
redis.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
|
|
1460
|
+
end
|
|
1461
|
+
redis.zrem('vmpooler__provisioning__request', request_id)
|
|
1462
|
+
redis.zadd('vmpooler__provisioning__processing', score, request_id)
|
|
1463
|
+
end
|
|
1464
|
+
end
|
|
1465
|
+
|
|
1466
|
+
def process_ondemand_vms(redis)
|
|
1467
|
+
queue_key = 'vmpooler__odcreate__task'
|
|
1468
|
+
queue = redis.zrange(queue_key, 0, -1, with_scores: true)
|
|
1469
|
+
ondemand_clone_limit = $config[:config]['ondemand_clone_limit']
|
|
1470
|
+
queue.each do |request, score|
|
|
1471
|
+
clone_count = redis.get('vmpooler__tasks__ondemandclone').to_i
|
|
1472
|
+
break unless clone_count < ondemand_clone_limit
|
|
1473
|
+
|
|
1474
|
+
pool_alias, pool, count, request_id = request.split(':')
|
|
1475
|
+
count = count.to_i
|
|
1476
|
+
provider = get_provider_for_pool(pool)
|
|
1477
|
+
slots = ondemand_clone_limit - clone_count
|
|
1478
|
+
break if slots == 0
|
|
1479
|
+
|
|
1480
|
+
if slots >= count
|
|
1481
|
+
count.times do
|
|
1482
|
+
redis.incr('vmpooler__tasks__ondemandclone')
|
|
1483
|
+
clone_vm(pool, provider, request_id, pool_alias)
|
|
1484
|
+
end
|
|
1485
|
+
redis.zrem(queue_key, request)
|
|
1486
|
+
else
|
|
1487
|
+
remaining_count = count - slots
|
|
1488
|
+
slots.times do
|
|
1489
|
+
redis.incr('vmpooler__tasks__ondemandclone')
|
|
1490
|
+
clone_vm(pool, provider, request_id, pool_alias)
|
|
1491
|
+
end
|
|
1492
|
+
redis.pipelined do
|
|
1493
|
+
redis.zrem(queue_key, request)
|
|
1494
|
+
redis.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
|
|
1495
|
+
end
|
|
1496
|
+
end
|
|
1497
|
+
end
|
|
1498
|
+
queue.length
|
|
1499
|
+
end
|
|
1500
|
+
|
|
1501
|
+
def vms_ready?(request_id, redis)
|
|
1502
|
+
catch :request_not_ready do
|
|
1503
|
+
request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
|
1504
|
+
Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
|
|
1505
|
+
pools_filled = redis.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
|
1506
|
+
throw :request_not_ready unless pools_filled.to_i == count.to_i
|
|
1507
|
+
end
|
|
1508
|
+
return true
|
|
1509
|
+
end
|
|
1510
|
+
false
|
|
1511
|
+
end
|
|
1512
|
+
|
|
1513
|
+
def check_ondemand_requests_ready(redis)
|
|
1514
|
+
in_progress_requests = redis.zrange('vmpooler__provisioning__processing', 0, -1, with_scores: true)
|
|
1515
|
+
in_progress_requests&.each do |request_id, score|
|
|
1516
|
+
check_ondemand_request_ready(request_id, redis, score)
|
|
1517
|
+
end
|
|
1518
|
+
in_progress_requests.length
|
|
1519
|
+
end
|
|
1520
|
+
|
|
1521
|
+
def check_ondemand_request_ready(request_id, redis, score = nil)
|
|
1522
|
+
# default expiration is one month to ensure the data does not stay in redis forever
|
|
1523
|
+
default_expiration = 259_200_0
|
|
1524
|
+
processing_key = 'vmpooler__provisioning__processing'
|
|
1525
|
+
ondemand_hash_key = "vmpooler__odrequest__#{request_id}"
|
|
1526
|
+
score ||= redis.zscore(processing_key, request_id)
|
|
1527
|
+
return if request_expired?(request_id, score, redis)
|
|
1528
|
+
|
|
1529
|
+
return unless vms_ready?(request_id, redis)
|
|
1530
|
+
|
|
1531
|
+
redis.multi
|
|
1532
|
+
redis.hset(ondemand_hash_key, 'status', 'ready')
|
|
1533
|
+
redis.expire(ondemand_hash_key, default_expiration)
|
|
1534
|
+
redis.zrem(processing_key, request_id)
|
|
1535
|
+
redis.exec
|
|
1536
|
+
end
|
|
1537
|
+
|
|
1538
|
+
def request_expired?(request_id, score, redis)
|
|
1539
|
+
delta = Time.now.to_i - score.to_i
|
|
1540
|
+
ondemand_request_ttl = $config[:config]['ondemand_request_ttl']
|
|
1541
|
+
return false unless delta > ondemand_request_ttl * 60
|
|
1542
|
+
|
|
1543
|
+
$logger.log('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
|
|
1544
|
+
expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
|
|
1545
|
+
redis.pipelined do
|
|
1546
|
+
redis.zrem('vmpooler__provisioning__processing', request_id)
|
|
1547
|
+
redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
|
|
1548
|
+
redis.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
|
|
1549
|
+
end
|
|
1550
|
+
remove_vms_for_failed_request(request_id, expiration_ttl, redis)
|
|
1551
|
+
true
|
|
1552
|
+
end
|
|
1553
|
+
|
|
1554
|
+
def remove_vms_for_failed_request(request_id, expiration_ttl, redis)
|
|
1555
|
+
request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
|
1556
|
+
Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
|
|
1557
|
+
pools_filled = redis.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
|
1558
|
+
redis.pipelined do
|
|
1559
|
+
pools_filled&.each do |vm|
|
|
1560
|
+
move_vm_queue(pool, vm, 'running', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
|
|
1561
|
+
end
|
|
1562
|
+
redis.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
|
|
1563
|
+
end
|
|
1564
|
+
end
|
|
1565
|
+
end
|
|
1566
|
+
|
|
1216
1567
|
def execute!(maxloop = 0, loop_delay = 1)
|
|
1217
1568
|
$logger.log('d', 'starting vmpooler')
|
|
1218
1569
|
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1570
|
+
@redis.with_metrics do |redis|
|
|
1571
|
+
# Clear out the tasks manager, as we don't know about any tasks at this point
|
|
1572
|
+
redis.set('vmpooler__tasks__clone', 0)
|
|
1573
|
+
redis.set('vmpooler__tasks__ondemandclone', 0)
|
|
1574
|
+
# Clear out vmpooler__migrations since stale entries may be left after a restart
|
|
1575
|
+
redis.del('vmpooler__migration')
|
|
1576
|
+
end
|
|
1223
1577
|
|
|
1224
1578
|
# Copy vSphere settings to correct location. This happens with older configuration files
|
|
1225
1579
|
if !$config[:vsphere].nil? && ($config[:providers].nil? || $config[:providers][:vsphere].nil?)
|
|
@@ -1269,7 +1623,7 @@ module Vmpooler
|
|
|
1269
1623
|
provider_class = $config[:providers][provider_name.to_sym]['provider_class']
|
|
1270
1624
|
end
|
|
1271
1625
|
begin
|
|
1272
|
-
$providers[provider_name] = create_provider_object($config, $logger, $metrics, provider_class, provider_name, {}) if $providers[provider_name].nil?
|
|
1626
|
+
$providers[provider_name] = create_provider_object($config, $logger, $metrics, @redis, provider_class, provider_name, {}) if $providers[provider_name].nil?
|
|
1273
1627
|
rescue StandardError => e
|
|
1274
1628
|
$logger.log('s', "Error while creating provider for pool #{pool['name']}: #{e}")
|
|
1275
1629
|
raise
|
|
@@ -1303,6 +1657,13 @@ module Vmpooler
|
|
|
1303
1657
|
end
|
|
1304
1658
|
end
|
|
1305
1659
|
|
|
1660
|
+
if !$threads['ondemand_provisioner']
|
|
1661
|
+
check_ondemand_requests
|
|
1662
|
+
elsif !$threads['ondemand_provisioner'].alive?
|
|
1663
|
+
$logger.log('d', '[!] [ondemand_provisioner] worker thread died, restarting')
|
|
1664
|
+
check_ondemand_requests(check_loop_delay_min, check_loop_delay_max, check_loop_delay_decay)
|
|
1665
|
+
end
|
|
1666
|
+
|
|
1306
1667
|
sleep(loop_delay)
|
|
1307
1668
|
|
|
1308
1669
|
unless maxloop == 0
|