vmpooler 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/vmpooler +3 -1
- data/lib/vmpooler.rb +23 -1
- data/lib/vmpooler/api/helpers.rb +1 -1
- data/lib/vmpooler/api/v1.rb +285 -17
- data/lib/vmpooler/generic_connection_pool.rb +5 -23
- data/lib/vmpooler/pool_manager.rb +753 -389
- data/lib/vmpooler/providers/base.rb +2 -1
- data/lib/vmpooler/providers/dummy.rb +2 -2
- data/lib/vmpooler/providers/vsphere.rb +50 -30
- data/lib/vmpooler/version.rb +1 -1
- metadata +16 -2
|
@@ -15,35 +15,17 @@ module Vmpooler
|
|
|
15
15
|
@metric_prefix = 'connectionpool' if @metric_prefix.nil? || @metric_prefix == ''
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def with_metrics(options = {})
|
|
21
|
-
Thread.handle_interrupt(Exception => :never) do
|
|
22
|
-
start = Time.now
|
|
23
|
-
conn = checkout(options)
|
|
24
|
-
timespan_ms = ((Time.now - start) * 1000).to_i
|
|
25
|
-
@metrics&.gauge(@metric_prefix + '.available', @available.length)
|
|
26
|
-
@metrics&.timing(@metric_prefix + '.waited', timespan_ms)
|
|
27
|
-
begin
|
|
28
|
-
Thread.handle_interrupt(Exception => :immediate) do
|
|
29
|
-
yield conn
|
|
30
|
-
end
|
|
31
|
-
ensure
|
|
32
|
-
checkin
|
|
33
|
-
@metrics&.gauge(@metric_prefix + '.available', @available.length)
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
else
|
|
38
|
-
# jruby 1.7.x
|
|
39
|
-
def with_metrics(options = {})
|
|
18
|
+
def with_metrics(options = {})
|
|
19
|
+
Thread.handle_interrupt(Exception => :never) do
|
|
40
20
|
start = Time.now
|
|
41
21
|
conn = checkout(options)
|
|
42
22
|
timespan_ms = ((Time.now - start) * 1000).to_i
|
|
43
23
|
@metrics&.gauge(@metric_prefix + '.available', @available.length)
|
|
44
24
|
@metrics&.timing(@metric_prefix + '.waited', timespan_ms)
|
|
45
25
|
begin
|
|
46
|
-
|
|
26
|
+
Thread.handle_interrupt(Exception => :immediate) do
|
|
27
|
+
yield conn
|
|
28
|
+
end
|
|
47
29
|
ensure
|
|
48
30
|
checkin
|
|
49
31
|
@metrics&.gauge(@metric_prefix + '.available', @available.length)
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'vmpooler/providers'
|
|
4
4
|
require 'spicy-proton'
|
|
5
|
+
require 'resolv' # ruby standard lib
|
|
5
6
|
|
|
6
7
|
module Vmpooler
|
|
7
8
|
class PoolManager
|
|
@@ -9,7 +10,7 @@ module Vmpooler
|
|
|
9
10
|
CHECK_LOOP_DELAY_MAX_DEFAULT = 60
|
|
10
11
|
CHECK_LOOP_DELAY_DECAY_DEFAULT = 2.0
|
|
11
12
|
|
|
12
|
-
def initialize(config, logger,
|
|
13
|
+
def initialize(config, logger, redis_connection_pool, metrics)
|
|
13
14
|
$config = config
|
|
14
15
|
|
|
15
16
|
# Load logger library
|
|
@@ -18,19 +19,19 @@ module Vmpooler
|
|
|
18
19
|
# metrics logging handle
|
|
19
20
|
$metrics = metrics
|
|
20
21
|
|
|
21
|
-
#
|
|
22
|
-
|
|
22
|
+
# Redis connection pool
|
|
23
|
+
@redis = redis_connection_pool
|
|
23
24
|
|
|
24
25
|
# VM Provider objects
|
|
25
|
-
$providers =
|
|
26
|
+
$providers = Concurrent::Hash.new
|
|
26
27
|
|
|
27
28
|
# Our thread-tracker object
|
|
28
|
-
$threads =
|
|
29
|
+
$threads = Concurrent::Hash.new
|
|
29
30
|
|
|
30
31
|
# Pool mutex
|
|
31
|
-
@reconfigure_pool =
|
|
32
|
+
@reconfigure_pool = Concurrent::Hash.new
|
|
32
33
|
|
|
33
|
-
@vm_mutex =
|
|
34
|
+
@vm_mutex = Concurrent::Hash.new
|
|
34
35
|
|
|
35
36
|
# Name generator for generating host names
|
|
36
37
|
@name_generator = Spicy::Proton.new
|
|
@@ -45,24 +46,26 @@ module Vmpooler
|
|
|
45
46
|
|
|
46
47
|
# Place pool configuration in redis so an API instance can discover running pool configuration
|
|
47
48
|
def load_pools_to_redis
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
49
|
+
@redis.with_metrics do |redis|
|
|
50
|
+
previously_configured_pools = redis.smembers('vmpooler__pools')
|
|
51
|
+
currently_configured_pools = []
|
|
52
|
+
config[:pools].each do |pool|
|
|
53
|
+
currently_configured_pools << pool['name']
|
|
54
|
+
redis.sadd('vmpooler__pools', pool['name'])
|
|
55
|
+
pool_keys = pool.keys
|
|
56
|
+
pool_keys.delete('alias')
|
|
57
|
+
to_set = {}
|
|
58
|
+
pool_keys.each do |k|
|
|
59
|
+
to_set[k] = pool[k]
|
|
60
|
+
end
|
|
61
|
+
to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
|
|
62
|
+
redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
|
|
58
63
|
end
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
$redis.srem('vmpooler__pools', pool)
|
|
65
|
-
$redis.del("vmpooler__pool__#{pool}")
|
|
64
|
+
previously_configured_pools.each do |pool|
|
|
65
|
+
unless currently_configured_pools.include? pool
|
|
66
|
+
redis.srem('vmpooler__pools', pool)
|
|
67
|
+
redis.del("vmpooler__pool__#{pool}")
|
|
68
|
+
end
|
|
66
69
|
end
|
|
67
70
|
end
|
|
68
71
|
nil
|
|
@@ -75,7 +78,9 @@ module Vmpooler
|
|
|
75
78
|
_check_pending_vm(vm, pool, timeout, provider)
|
|
76
79
|
rescue StandardError => e
|
|
77
80
|
$logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
|
|
78
|
-
|
|
81
|
+
@redis.with_metrics do |redis|
|
|
82
|
+
fail_pending_vm(vm, pool, timeout, redis)
|
|
83
|
+
end
|
|
79
84
|
raise
|
|
80
85
|
end
|
|
81
86
|
end
|
|
@@ -86,31 +91,38 @@ module Vmpooler
|
|
|
86
91
|
return if mutex.locked?
|
|
87
92
|
|
|
88
93
|
mutex.synchronize do
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
94
|
+
@redis.with_metrics do |redis|
|
|
95
|
+
request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
|
|
96
|
+
if provider.vm_ready?(pool, vm)
|
|
97
|
+
move_pending_vm_to_ready(vm, pool, redis, request_id)
|
|
98
|
+
else
|
|
99
|
+
fail_pending_vm(vm, pool, timeout, redis)
|
|
100
|
+
end
|
|
93
101
|
end
|
|
94
102
|
end
|
|
95
103
|
end
|
|
96
104
|
|
|
97
|
-
def remove_nonexistent_vm(vm, pool)
|
|
98
|
-
|
|
105
|
+
def remove_nonexistent_vm(vm, pool, redis)
|
|
106
|
+
redis.srem("vmpooler__pending__#{pool}", vm)
|
|
99
107
|
$logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
|
|
100
108
|
end
|
|
101
109
|
|
|
102
|
-
def fail_pending_vm(vm, pool, timeout, exists = true)
|
|
103
|
-
clone_stamp =
|
|
104
|
-
return true unless clone_stamp
|
|
110
|
+
def fail_pending_vm(vm, pool, timeout, redis, exists = true)
|
|
111
|
+
clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
|
|
105
112
|
|
|
106
113
|
time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
|
|
107
114
|
if time_since_clone > timeout
|
|
108
115
|
if exists
|
|
109
|
-
|
|
116
|
+
request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
|
|
117
|
+
pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
|
|
118
|
+
redis.multi
|
|
119
|
+
redis.smove('vmpooler__pending__' + pool, 'vmpooler__completed__' + pool, vm)
|
|
120
|
+
redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}") if request_id
|
|
121
|
+
redis.exec
|
|
110
122
|
$metrics.increment("errors.markedasfailed.#{pool}")
|
|
111
123
|
$logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
|
|
112
124
|
else
|
|
113
|
-
remove_nonexistent_vm(vm, pool)
|
|
125
|
+
remove_nonexistent_vm(vm, pool, redis)
|
|
114
126
|
end
|
|
115
127
|
end
|
|
116
128
|
true
|
|
@@ -119,28 +131,54 @@ module Vmpooler
|
|
|
119
131
|
false
|
|
120
132
|
end
|
|
121
133
|
|
|
122
|
-
def move_pending_vm_to_ready(vm, pool)
|
|
123
|
-
clone_time =
|
|
124
|
-
finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
|
|
134
|
+
def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
|
|
135
|
+
clone_time = redis.hget('vmpooler__vm__' + vm, 'clone')
|
|
136
|
+
finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
|
|
125
137
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
138
|
+
if request_id
|
|
139
|
+
ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
|
140
|
+
if ondemandrequest_hash['status'] == 'failed'
|
|
141
|
+
move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
|
|
142
|
+
return nil
|
|
143
|
+
elsif ondemandrequest_hash['status'] == 'deleted'
|
|
144
|
+
move_vm_queue(pool, vm, 'pending', 'completed', redis, "moved to completed queue. '#{request_id}' has been deleted")
|
|
145
|
+
return nil
|
|
146
|
+
end
|
|
147
|
+
pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
|
|
148
|
+
|
|
149
|
+
redis.pipelined do
|
|
150
|
+
redis.hset("vmpooler__active__#{pool}", vm, Time.now)
|
|
151
|
+
redis.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
|
|
152
|
+
redis.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token']) if ondemandrequest_hash['token:token']
|
|
153
|
+
redis.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user']) if ondemandrequest_hash['token:user']
|
|
154
|
+
redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
|
|
155
|
+
end
|
|
156
|
+
move_vm_queue(pool, vm, 'pending', 'running', redis)
|
|
157
|
+
check_ondemand_request_ready(request_id, redis)
|
|
158
|
+
else
|
|
159
|
+
redis.smove('vmpooler__pending__' + pool, 'vmpooler__ready__' + pool, vm)
|
|
160
|
+
end
|
|
129
161
|
|
|
130
|
-
|
|
131
|
-
|
|
162
|
+
redis.pipelined do
|
|
163
|
+
redis.hset('vmpooler__boot__' + Date.today.to_s, pool + ':' + vm, finish) # maybe remove as this is never used by vmpooler itself?
|
|
164
|
+
redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
|
|
165
|
+
|
|
166
|
+
# last boot time is displayed in API, and used by alarming script
|
|
167
|
+
redis.hset('vmpooler__lastboot', pool, Time.now)
|
|
168
|
+
end
|
|
132
169
|
|
|
133
170
|
$metrics.timing("time_to_ready_state.#{pool}", finish)
|
|
134
|
-
$logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue")
|
|
171
|
+
$logger.log('s', "[>] [#{pool}] '#{vm}' moved from 'pending' to 'ready' queue") unless request_id
|
|
172
|
+
$logger.log('s', "[>] [#{pool}] '#{vm}' is 'ready' for request '#{request_id}'") if request_id
|
|
135
173
|
end
|
|
136
174
|
|
|
137
|
-
def vm_still_ready?(pool_name, vm_name, provider)
|
|
175
|
+
def vm_still_ready?(pool_name, vm_name, provider, redis)
|
|
138
176
|
# Check if the VM is still ready/available
|
|
139
177
|
return true if provider.vm_ready?(pool_name, vm_name)
|
|
140
178
|
|
|
141
179
|
raise("VM #{vm_name} is not ready")
|
|
142
180
|
rescue StandardError
|
|
143
|
-
move_vm_queue(pool_name, vm_name, 'ready', 'completed', "is unreachable, removed from 'ready' queue")
|
|
181
|
+
move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "is unreachable, removed from 'ready' queue")
|
|
144
182
|
end
|
|
145
183
|
|
|
146
184
|
def check_ready_vm(vm, pool_name, ttl, provider)
|
|
@@ -160,34 +198,35 @@ module Vmpooler
|
|
|
160
198
|
return if mutex.locked?
|
|
161
199
|
|
|
162
200
|
mutex.synchronize do
|
|
163
|
-
|
|
164
|
-
|
|
201
|
+
@redis.with_metrics do |redis|
|
|
202
|
+
check_stamp = redis.hget('vmpooler__vm__' + vm, 'check')
|
|
203
|
+
last_checked_too_soon = ((Time.now - Time.parse(check_stamp)).to_i < $config[:config]['vm_checktime'] * 60) if check_stamp
|
|
204
|
+
break if check_stamp && last_checked_too_soon
|
|
165
205
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
if ttl > 0
|
|
206
|
+
redis.hset('vmpooler__vm__' + vm, 'check', Time.now)
|
|
207
|
+
# Check if the hosts TTL has expired
|
|
169
208
|
# if 'boottime' is nil, set bootime to beginning of unix epoch, forces TTL to be assumed expired
|
|
170
|
-
boottime =
|
|
209
|
+
boottime = redis.hget("vmpooler__vm__#{vm}", 'ready')
|
|
171
210
|
if boottime
|
|
172
211
|
boottime = Time.parse(boottime)
|
|
173
212
|
else
|
|
174
213
|
boottime = Time.at(0)
|
|
175
214
|
end
|
|
176
|
-
if (
|
|
177
|
-
|
|
215
|
+
if (Time.now - boottime).to_i > ttl * 60
|
|
216
|
+
redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
|
|
178
217
|
|
|
179
218
|
$logger.log('d', "[!] [#{pool_name}] '#{vm}' reached end of TTL after #{ttl} minutes, removed from 'ready' queue")
|
|
180
|
-
return
|
|
219
|
+
return nil
|
|
181
220
|
end
|
|
182
|
-
end
|
|
183
221
|
|
|
184
|
-
|
|
222
|
+
break if mismatched_hostname?(vm, pool_name, provider, redis)
|
|
185
223
|
|
|
186
|
-
|
|
224
|
+
vm_still_ready?(pool_name, vm, provider, redis)
|
|
225
|
+
end
|
|
187
226
|
end
|
|
188
227
|
end
|
|
189
228
|
|
|
190
|
-
def mismatched_hostname?(vm, pool_name, provider)
|
|
229
|
+
def mismatched_hostname?(vm, pool_name, provider, redis)
|
|
191
230
|
pool_config = $config[:pools][$config[:pool_index][pool_name]]
|
|
192
231
|
check_hostname = pool_config['check_hostname_for_mismatch']
|
|
193
232
|
check_hostname = $config[:config]['check_ready_vm_hostname_for_mismatch'] if check_hostname.nil?
|
|
@@ -196,7 +235,7 @@ module Vmpooler
|
|
|
196
235
|
# Wait one minute before checking a VM for hostname mismatch
|
|
197
236
|
# When checking as soon as the VM passes the ready test the instance
|
|
198
237
|
# often doesn't report its hostname yet causing the VM to be removed immediately
|
|
199
|
-
vm_ready_time =
|
|
238
|
+
vm_ready_time = redis.hget("vmpooler__vm__#{vm}", 'ready')
|
|
200
239
|
if vm_ready_time
|
|
201
240
|
wait_before_checking = 60
|
|
202
241
|
time_since_ready = (Time.now - Time.parse(vm_ready_time)).to_i
|
|
@@ -213,7 +252,7 @@ module Vmpooler
|
|
|
213
252
|
return if hostname.empty?
|
|
214
253
|
return if hostname == vm
|
|
215
254
|
|
|
216
|
-
|
|
255
|
+
redis.smove('vmpooler__ready__' + pool_name, 'vmpooler__completed__' + pool_name, vm)
|
|
217
256
|
$logger.log('d', "[!] [#{pool_name}] '#{vm}' has mismatched hostname #{hostname}, removed from 'ready' queue")
|
|
218
257
|
true
|
|
219
258
|
end
|
|
@@ -234,49 +273,61 @@ module Vmpooler
|
|
|
234
273
|
return if mutex.locked?
|
|
235
274
|
|
|
236
275
|
mutex.synchronize do
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
276
|
+
catch :stop_checking do
|
|
277
|
+
@redis.with_metrics do |redis|
|
|
278
|
+
# Check that VM is within defined lifetime
|
|
279
|
+
checkouttime = redis.hget('vmpooler__active__' + pool, vm)
|
|
280
|
+
if checkouttime
|
|
281
|
+
time_since_checkout = Time.now - Time.parse(checkouttime)
|
|
282
|
+
running = time_since_checkout / 60 / 60
|
|
283
|
+
|
|
284
|
+
if (ttl.to_i > 0) && (running.to_i >= ttl.to_i)
|
|
285
|
+
move_vm_queue(pool, vm, 'running', 'completed', redis, "reached end of TTL after #{ttl} hours")
|
|
286
|
+
throw :stop_checking
|
|
287
|
+
end
|
|
288
|
+
end
|
|
247
289
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
290
|
+
if provider.vm_ready?(pool, vm)
|
|
291
|
+
throw :stop_checking
|
|
292
|
+
else
|
|
293
|
+
host = provider.get_vm(pool, vm)
|
|
252
294
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
295
|
+
if host
|
|
296
|
+
throw :stop_checking
|
|
297
|
+
else
|
|
298
|
+
move_vm_queue(pool, vm, 'running', 'completed', redis, 'is no longer in inventory, removing from running')
|
|
299
|
+
end
|
|
300
|
+
end
|
|
257
301
|
end
|
|
258
302
|
end
|
|
259
303
|
end
|
|
260
304
|
end
|
|
261
305
|
|
|
262
|
-
def move_vm_queue(pool, vm, queue_from, queue_to, msg = nil)
|
|
263
|
-
|
|
306
|
+
def move_vm_queue(pool, vm, queue_from, queue_to, redis, msg = nil)
|
|
307
|
+
redis.smove("vmpooler__#{queue_from}__#{pool}", "vmpooler__#{queue_to}__#{pool}", vm)
|
|
264
308
|
$logger.log('d', "[!] [#{pool}] '#{vm}' #{msg}") if msg
|
|
265
309
|
end
|
|
266
310
|
|
|
267
311
|
# Clone a VM
|
|
268
|
-
def clone_vm(pool_name, provider)
|
|
312
|
+
def clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
|
|
269
313
|
Thread.new do
|
|
270
314
|
begin
|
|
271
|
-
_clone_vm(pool_name, provider)
|
|
315
|
+
_clone_vm(pool_name, provider, request_id, pool_alias)
|
|
272
316
|
rescue StandardError => e
|
|
273
|
-
|
|
317
|
+
if request_id
|
|
318
|
+
$logger.log('s', "[!] [#{pool_name}] failed while cloning VM for request #{request_id} with an error: #{e}")
|
|
319
|
+
@redis.with_metrics do |redis|
|
|
320
|
+
redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool_name}:1:#{request_id}")
|
|
321
|
+
end
|
|
322
|
+
else
|
|
323
|
+
$logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{e}")
|
|
324
|
+
end
|
|
274
325
|
raise
|
|
275
326
|
end
|
|
276
327
|
end
|
|
277
328
|
end
|
|
278
329
|
|
|
279
|
-
def generate_and_check_hostname
|
|
330
|
+
def generate_and_check_hostname
|
|
280
331
|
# Generate a randomized hostname. The total name must no longer than 15
|
|
281
332
|
# character including the hyphen. The shortest adjective in the corpus is
|
|
282
333
|
# three characters long. Therefore, we can technically select a noun up to 11
|
|
@@ -285,58 +336,104 @@ module Vmpooler
|
|
|
285
336
|
# letter adjectives, we actually limit the noun to 10 letters to avoid
|
|
286
337
|
# inviting more conflicts. We favor selecting a longer noun rather than a
|
|
287
338
|
# longer adjective because longer adjectives tend to be less fun.
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
339
|
+
@redis.with do |redis|
|
|
340
|
+
noun = @name_generator.noun(max: 10)
|
|
341
|
+
adjective = @name_generator.adjective(max: 14 - noun.length)
|
|
342
|
+
random_name = [adjective, noun].join('-')
|
|
343
|
+
hostname = $config[:config]['prefix'] + random_name
|
|
344
|
+
available = redis.hlen('vmpooler__vm__' + hostname) == 0
|
|
345
|
+
|
|
346
|
+
[hostname, available]
|
|
347
|
+
end
|
|
295
348
|
end
|
|
296
349
|
|
|
297
350
|
def find_unique_hostname(pool_name)
|
|
351
|
+
# generate hostname that is not already in use in vmpooler
|
|
352
|
+
# also check that no dns record already exists
|
|
298
353
|
hostname_retries = 0
|
|
299
354
|
max_hostname_retries = 3
|
|
300
355
|
while hostname_retries < max_hostname_retries
|
|
301
|
-
hostname,
|
|
302
|
-
|
|
356
|
+
hostname, hostname_available = generate_and_check_hostname(pool_name)
|
|
357
|
+
domain = $config[:config]['domain']
|
|
358
|
+
dns_ip, dns_available = check_dns_available(hostname, domain)
|
|
359
|
+
break if hostname_available && dns_available
|
|
303
360
|
|
|
304
361
|
hostname_retries += 1
|
|
305
|
-
|
|
306
|
-
|
|
362
|
+
|
|
363
|
+
if !hostname_available
|
|
364
|
+
$metrics.increment("errors.duplicatehostname.#{pool_name}")
|
|
365
|
+
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
|
|
366
|
+
elsif !dns_available
|
|
367
|
+
$metrics.increment("errors.staledns.#{hostname}")
|
|
368
|
+
$logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} already exists in DNS records (#{dns_ip}), stale DNS")
|
|
369
|
+
end
|
|
307
370
|
end
|
|
308
371
|
|
|
309
|
-
raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless
|
|
372
|
+
raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless hostname_available && dns_available
|
|
310
373
|
|
|
311
374
|
hostname
|
|
312
375
|
end
|
|
313
376
|
|
|
314
|
-
def
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
$redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
|
|
319
|
-
$redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
|
|
320
|
-
$redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name)
|
|
321
|
-
|
|
377
|
+
def check_dns_available(vm_name, domain = nil)
|
|
378
|
+
# Query the DNS for the name we want to create and if it already exists, mark it unavailable
|
|
379
|
+
# This protects against stale DNS records
|
|
380
|
+
vm_name = "#{vm_name}.#{domain}" if domain
|
|
322
381
|
begin
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
382
|
+
dns_ip = Resolv.getaddress(vm_name)
|
|
383
|
+
rescue Resolv::ResolvError
|
|
384
|
+
# this is the expected case, swallow the error
|
|
385
|
+
# eg "no address for blah-daisy"
|
|
386
|
+
return ['', true]
|
|
387
|
+
end
|
|
388
|
+
[dns_ip, false]
|
|
389
|
+
end
|
|
327
390
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
391
|
+
def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
|
|
392
|
+
new_vmname = find_unique_hostname(pool_name)
|
|
393
|
+
mutex = vm_mutex(new_vmname)
|
|
394
|
+
mutex.synchronize do
|
|
395
|
+
@redis.with_metrics do |redis|
|
|
396
|
+
# Add VM to Redis inventory ('pending' pool)
|
|
397
|
+
redis.multi
|
|
398
|
+
redis.sadd('vmpooler__pending__' + pool_name, new_vmname)
|
|
399
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'clone', Time.now)
|
|
400
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'template', pool_name) # This value is used to represent the pool.
|
|
401
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'pool', pool_name)
|
|
402
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'request_id', request_id) if request_id
|
|
403
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'pool_alias', pool_alias) if pool_alias
|
|
404
|
+
redis.exec
|
|
405
|
+
end
|
|
331
406
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
407
|
+
begin
|
|
408
|
+
$logger.log('d', "[ ] [#{pool_name}] Starting to clone '#{new_vmname}'")
|
|
409
|
+
start = Time.now
|
|
410
|
+
provider.create_vm(pool_name, new_vmname)
|
|
411
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
|
412
|
+
|
|
413
|
+
@redis.with_metrics do |redis|
|
|
414
|
+
redis.pipelined do
|
|
415
|
+
redis.hset('vmpooler__clone__' + Date.today.to_s, pool_name + ':' + new_vmname, finish)
|
|
416
|
+
redis.hset('vmpooler__vm__' + new_vmname, 'clone_time', finish)
|
|
417
|
+
end
|
|
418
|
+
end
|
|
419
|
+
$logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
|
|
420
|
+
|
|
421
|
+
$metrics.timing("clone.#{pool_name}", finish)
|
|
422
|
+
rescue StandardError
|
|
423
|
+
@redis.with_metrics do |redis|
|
|
424
|
+
redis.pipelined do
|
|
425
|
+
redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
|
|
426
|
+
expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
|
|
427
|
+
redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
|
|
428
|
+
end
|
|
429
|
+
end
|
|
430
|
+
raise
|
|
431
|
+
ensure
|
|
432
|
+
@redis.with_metrics do |redis|
|
|
433
|
+
redis.decr('vmpooler__tasks__ondemandclone') if request_id
|
|
434
|
+
redis.decr('vmpooler__tasks__clone') unless request_id
|
|
435
|
+
end
|
|
436
|
+
end
|
|
340
437
|
end
|
|
341
438
|
end
|
|
342
439
|
|
|
@@ -357,36 +454,42 @@ module Vmpooler
|
|
|
357
454
|
return if mutex.locked?
|
|
358
455
|
|
|
359
456
|
mutex.synchronize do
|
|
360
|
-
|
|
361
|
-
|
|
457
|
+
@redis.with_metrics do |redis|
|
|
458
|
+
redis.pipelined do
|
|
459
|
+
redis.hdel('vmpooler__active__' + pool, vm)
|
|
460
|
+
redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
|
|
362
461
|
|
|
363
|
-
|
|
364
|
-
|
|
462
|
+
# Auto-expire metadata key
|
|
463
|
+
redis.expire('vmpooler__vm__' + vm, ($config[:redis]['data_ttl'].to_i * 60 * 60))
|
|
464
|
+
end
|
|
365
465
|
|
|
366
|
-
|
|
466
|
+
start = Time.now
|
|
367
467
|
|
|
368
|
-
|
|
468
|
+
provider.destroy_vm(pool, vm)
|
|
369
469
|
|
|
370
|
-
|
|
470
|
+
redis.srem('vmpooler__completed__' + pool, vm)
|
|
371
471
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
472
|
+
finish = format('%<time>.2f', time: Time.now - start)
|
|
473
|
+
$logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
|
|
474
|
+
$metrics.timing("destroy.#{pool}", finish)
|
|
475
|
+
get_vm_usage_labels(vm, redis)
|
|
476
|
+
end
|
|
376
477
|
end
|
|
377
478
|
dereference_mutex(vm)
|
|
378
479
|
end
|
|
379
480
|
|
|
380
|
-
def get_vm_usage_labels(vm)
|
|
481
|
+
def get_vm_usage_labels(vm, redis)
|
|
381
482
|
return unless $config[:config]['usage_stats']
|
|
382
483
|
|
|
383
|
-
|
|
484
|
+
redis.multi
|
|
485
|
+
redis.hget("vmpooler__vm__#{vm}", 'checkout')
|
|
486
|
+
redis.hget("vmpooler__vm__#{vm}", 'tag:jenkins_build_url')
|
|
487
|
+
redis.hget("vmpooler__vm__#{vm}", 'token:user')
|
|
488
|
+
redis.hget("vmpooler__vm__#{vm}", 'template')
|
|
489
|
+
checkout, jenkins_build_url, user, poolname = redis.exec
|
|
384
490
|
return if checkout.nil?
|
|
385
491
|
|
|
386
|
-
|
|
387
|
-
user = $redis.hget("vmpooler__vm__#{vm}", 'token:user') || 'unauthenticated'
|
|
388
|
-
poolname = $redis.hget("vmpooler__vm__#{vm}", 'template')
|
|
389
|
-
|
|
492
|
+
user ||= 'unauthenticated'
|
|
390
493
|
unless jenkins_build_url
|
|
391
494
|
user = user.gsub('.', '_')
|
|
392
495
|
$metrics.increment("usage.#{user}.#{poolname}")
|
|
@@ -420,7 +523,8 @@ module Vmpooler
|
|
|
420
523
|
|
|
421
524
|
$metrics.increment(metric_parts.join('.'))
|
|
422
525
|
rescue StandardError => e
|
|
423
|
-
logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
|
|
526
|
+
$logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
|
|
527
|
+
raise
|
|
424
528
|
end
|
|
425
529
|
|
|
426
530
|
def component_to_test(match, labels_string)
|
|
@@ -444,7 +548,7 @@ module Vmpooler
|
|
|
444
548
|
if provider_purge
|
|
445
549
|
Thread.new do
|
|
446
550
|
begin
|
|
447
|
-
purge_vms_and_folders(provider.to_s)
|
|
551
|
+
purge_vms_and_folders($providers[provider.to_s])
|
|
448
552
|
rescue StandardError => e
|
|
449
553
|
$logger.log('s', "[!] failed while purging provider #{provider} VMs and folders with an error: #{e}")
|
|
450
554
|
end
|
|
@@ -455,13 +559,14 @@ module Vmpooler
|
|
|
455
559
|
end
|
|
456
560
|
|
|
457
561
|
# Return a list of pool folders
|
|
458
|
-
def pool_folders(
|
|
562
|
+
def pool_folders(provider)
|
|
563
|
+
provider_name = provider.name
|
|
459
564
|
folders = {}
|
|
460
565
|
$config[:pools].each do |pool|
|
|
461
566
|
next unless pool['provider'] == provider_name
|
|
462
567
|
|
|
463
568
|
folder_parts = pool['folder'].split('/')
|
|
464
|
-
datacenter =
|
|
569
|
+
datacenter = provider.get_target_datacenter_from_config(pool['name'])
|
|
465
570
|
folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
|
|
466
571
|
end
|
|
467
572
|
folders
|
|
@@ -478,8 +583,8 @@ module Vmpooler
|
|
|
478
583
|
def purge_vms_and_folders(provider)
|
|
479
584
|
configured_folders = pool_folders(provider)
|
|
480
585
|
base_folders = get_base_folders(configured_folders)
|
|
481
|
-
whitelist =
|
|
482
|
-
|
|
586
|
+
whitelist = provider.provider_config['folder_whitelist']
|
|
587
|
+
provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
|
|
483
588
|
end
|
|
484
589
|
|
|
485
590
|
def create_vm_disk(pool_name, vm, disk_size, provider)
|
|
@@ -505,10 +610,12 @@ module Vmpooler
|
|
|
505
610
|
finish = format('%<time>.2f', time: Time.now - start)
|
|
506
611
|
|
|
507
612
|
if result
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
613
|
+
@redis.with_metrics do |redis|
|
|
614
|
+
rdisks = redis.hget('vmpooler__vm__' + vm_name, 'disk')
|
|
615
|
+
disks = rdisks ? rdisks.split(':') : []
|
|
616
|
+
disks.push("+#{disk_size}gb")
|
|
617
|
+
redis.hset('vmpooler__vm__' + vm_name, 'disk', disks.join(':'))
|
|
618
|
+
end
|
|
512
619
|
|
|
513
620
|
$logger.log('s', "[+] [disk_manager] '#{vm_name}' attached #{disk_size}gb disk in #{finish} seconds")
|
|
514
621
|
else
|
|
@@ -538,7 +645,9 @@ module Vmpooler
|
|
|
538
645
|
finish = format('%<time>.2f', time: Time.now - start)
|
|
539
646
|
|
|
540
647
|
if result
|
|
541
|
-
|
|
648
|
+
@redis.with_metrics do |redis|
|
|
649
|
+
redis.hset('vmpooler__vm__' + vm_name, 'snapshot:' + snapshot_name, Time.now.to_s)
|
|
650
|
+
end
|
|
542
651
|
$logger.log('s', "[+] [snapshot_manager] '#{vm_name}' snapshot created in #{finish} seconds")
|
|
543
652
|
else
|
|
544
653
|
$logger.log('s', "[+] [snapshot_manager] Failed to snapshot '#{vm_name}'")
|
|
@@ -594,9 +703,9 @@ module Vmpooler
|
|
|
594
703
|
@default_providers ||= %w[vsphere dummy]
|
|
595
704
|
end
|
|
596
705
|
|
|
597
|
-
def get_pool_name_for_vm(vm_name)
|
|
706
|
+
def get_pool_name_for_vm(vm_name, redis)
|
|
598
707
|
# the 'template' is a bad name. Should really be 'poolname'
|
|
599
|
-
|
|
708
|
+
redis.hget('vmpooler__vm__' + vm_name, 'template')
|
|
600
709
|
end
|
|
601
710
|
|
|
602
711
|
# @param pool_name [String] - the name of the pool
|
|
@@ -628,19 +737,21 @@ module Vmpooler
|
|
|
628
737
|
end
|
|
629
738
|
|
|
630
739
|
def _check_disk_queue
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
740
|
+
@redis.with_metrics do |redis|
|
|
741
|
+
task_detail = redis.spop('vmpooler__tasks__disk')
|
|
742
|
+
unless task_detail.nil?
|
|
743
|
+
begin
|
|
744
|
+
vm_name, disk_size = task_detail.split(':')
|
|
745
|
+
pool_name = get_pool_name_for_vm(vm_name, redis)
|
|
746
|
+
raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
|
|
637
747
|
|
|
638
|
-
|
|
639
|
-
|
|
748
|
+
provider = get_provider_for_pool(pool_name)
|
|
749
|
+
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
|
640
750
|
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
751
|
+
create_vm_disk(pool_name, vm_name, disk_size, provider)
|
|
752
|
+
rescue StandardError => e
|
|
753
|
+
$logger.log('s', "[!] [disk_manager] disk creation appears to have failed: #{e}")
|
|
754
|
+
end
|
|
644
755
|
end
|
|
645
756
|
end
|
|
646
757
|
end
|
|
@@ -664,37 +775,39 @@ module Vmpooler
|
|
|
664
775
|
end
|
|
665
776
|
|
|
666
777
|
def _check_snapshot_queue
|
|
667
|
-
|
|
778
|
+
@redis.with_metrics do |redis|
|
|
779
|
+
task_detail = redis.spop('vmpooler__tasks__snapshot')
|
|
668
780
|
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
781
|
+
unless task_detail.nil?
|
|
782
|
+
begin
|
|
783
|
+
vm_name, snapshot_name = task_detail.split(':')
|
|
784
|
+
pool_name = get_pool_name_for_vm(vm_name, redis)
|
|
785
|
+
raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
|
|
674
786
|
|
|
675
|
-
|
|
676
|
-
|
|
787
|
+
provider = get_provider_for_pool(pool_name)
|
|
788
|
+
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
|
677
789
|
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
790
|
+
create_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
|
|
791
|
+
rescue StandardError => e
|
|
792
|
+
$logger.log('s', "[!] [snapshot_manager] snapshot create appears to have failed: #{e}")
|
|
793
|
+
end
|
|
681
794
|
end
|
|
682
|
-
end
|
|
683
795
|
|
|
684
|
-
|
|
796
|
+
task_detail = redis.spop('vmpooler__tasks__snapshot-revert')
|
|
685
797
|
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
798
|
+
unless task_detail.nil?
|
|
799
|
+
begin
|
|
800
|
+
vm_name, snapshot_name = task_detail.split(':')
|
|
801
|
+
pool_name = get_pool_name_for_vm(vm_name, redis)
|
|
802
|
+
raise("Unable to determine which pool #{vm_name} is a member of") if pool_name.nil?
|
|
691
803
|
|
|
692
|
-
|
|
693
|
-
|
|
804
|
+
provider = get_provider_for_pool(pool_name)
|
|
805
|
+
raise("Missing Provider for vm #{vm_name} in pool #{pool_name}") if provider.nil?
|
|
694
806
|
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
807
|
+
revert_vm_snapshot(pool_name, vm_name, snapshot_name, provider)
|
|
808
|
+
rescue StandardError => e
|
|
809
|
+
$logger.log('s', "[!] [snapshot_manager] snapshot revert appears to have failed: #{e}")
|
|
810
|
+
end
|
|
698
811
|
end
|
|
699
812
|
end
|
|
700
813
|
end
|
|
@@ -704,7 +817,9 @@ module Vmpooler
|
|
|
704
817
|
begin
|
|
705
818
|
mutex = vm_mutex(vm_name)
|
|
706
819
|
mutex.synchronize do
|
|
707
|
-
|
|
820
|
+
@redis.with_metrics do |redis|
|
|
821
|
+
redis.srem("vmpooler__migrating__#{pool_name}", vm_name)
|
|
822
|
+
end
|
|
708
823
|
provider.migrate_vm(pool_name, vm_name)
|
|
709
824
|
end
|
|
710
825
|
rescue StandardError => e
|
|
@@ -737,47 +852,65 @@ module Vmpooler
|
|
|
737
852
|
wakeup_by = Time.now + wakeup_period
|
|
738
853
|
return if time_passed?(:exit_by, exit_by)
|
|
739
854
|
|
|
740
|
-
|
|
855
|
+
@redis.with_metrics do |redis|
|
|
856
|
+
initial_ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}") if options[:pool_size_change]
|
|
741
857
|
|
|
742
|
-
|
|
858
|
+
initial_clone_target = redis.hget("vmpooler__pool__#{options[:poolname]}", options[:clone_target]) if options[:clone_target_change]
|
|
743
859
|
|
|
744
|
-
|
|
860
|
+
initial_template = redis.hget('vmpooler__template__prepared', options[:poolname]) if options[:pool_template_change]
|
|
745
861
|
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
862
|
+
loop do
|
|
863
|
+
sleep(1)
|
|
864
|
+
break if time_passed?(:exit_by, exit_by)
|
|
749
865
|
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
866
|
+
# Check for wakeup events
|
|
867
|
+
if time_passed?(:wakeup_by, wakeup_by)
|
|
868
|
+
wakeup_by = Time.now + wakeup_period
|
|
753
869
|
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
870
|
+
# Wakeup if the number of ready VMs has changed
|
|
871
|
+
if options[:pool_size_change]
|
|
872
|
+
ready_size = redis.scard("vmpooler__ready__#{options[:poolname]}")
|
|
873
|
+
break unless ready_size == initial_ready_size
|
|
874
|
+
end
|
|
759
875
|
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
876
|
+
if options[:clone_target_change]
|
|
877
|
+
clone_target = redis.hget('vmpooler__config__clone_target}', options[:poolname])
|
|
878
|
+
if clone_target
|
|
879
|
+
break unless clone_target == initial_clone_target
|
|
880
|
+
end
|
|
764
881
|
end
|
|
765
|
-
end
|
|
766
882
|
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
883
|
+
if options[:pool_template_change]
|
|
884
|
+
configured_template = redis.hget('vmpooler__config__template', options[:poolname])
|
|
885
|
+
if configured_template
|
|
886
|
+
break unless initial_template == configured_template
|
|
887
|
+
end
|
|
888
|
+
end
|
|
889
|
+
|
|
890
|
+
if options[:pool_reset]
|
|
891
|
+
pending = redis.sismember('vmpooler__poolreset', options[:poolname])
|
|
892
|
+
break if pending
|
|
771
893
|
end
|
|
772
|
-
end
|
|
773
894
|
|
|
774
|
-
|
|
775
|
-
|
|
895
|
+
if options[:pending_vm]
|
|
896
|
+
pending_vm_count = redis.scard("vmpooler__pending__#{options[:poolname]}")
|
|
897
|
+
break unless pending_vm_count == 0
|
|
898
|
+
end
|
|
899
|
+
|
|
900
|
+
if options[:ondemand_request]
|
|
901
|
+
redis.multi
|
|
902
|
+
redis.zcard('vmpooler__provisioning__request')
|
|
903
|
+
redis.zcard('vmpooler__provisioning__processing')
|
|
904
|
+
redis.zcard('vmpooler__odcreate__task')
|
|
905
|
+
od_request, od_processing, od_createtask = redis.exec
|
|
906
|
+
break unless od_request == 0
|
|
907
|
+
break unless od_processing == 0
|
|
908
|
+
break unless od_createtask == 0
|
|
909
|
+
end
|
|
776
910
|
end
|
|
777
911
|
|
|
912
|
+
break if time_passed?(:exit_by, exit_by)
|
|
778
913
|
end
|
|
779
|
-
|
|
780
|
-
break if time_passed?(:exit_by, exit_by)
|
|
781
914
|
end
|
|
782
915
|
end
|
|
783
916
|
|
|
@@ -813,7 +946,7 @@ module Vmpooler
|
|
|
813
946
|
loop_delay = (loop_delay * loop_delay_decay).to_i
|
|
814
947
|
loop_delay = loop_delay_max if loop_delay > loop_delay_max
|
|
815
948
|
end
|
|
816
|
-
sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pool_reset: true)
|
|
949
|
+
sleep_with_wakeup_events(loop_delay, loop_delay_min, pool_size_change: true, poolname: pool['name'], pool_template_change: true, clone_target_change: true, pending_vm: true, pool_reset: true)
|
|
817
950
|
|
|
818
951
|
unless maxloop == 0
|
|
819
952
|
break if loop_count >= maxloop
|
|
@@ -843,77 +976,84 @@ module Vmpooler
|
|
|
843
976
|
end
|
|
844
977
|
|
|
845
978
|
def sync_pool_template(pool)
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
979
|
+
@redis.with_metrics do |redis|
|
|
980
|
+
pool_template = redis.hget('vmpooler__config__template', pool['name'])
|
|
981
|
+
if pool_template
|
|
982
|
+
pool['template'] = pool_template unless pool['template'] == pool_template
|
|
983
|
+
end
|
|
849
984
|
end
|
|
850
985
|
end
|
|
851
986
|
|
|
852
|
-
def prepare_template(pool, provider)
|
|
987
|
+
def prepare_template(pool, provider, redis)
|
|
853
988
|
if $config[:config]['create_template_delta_disks']
|
|
854
|
-
unless
|
|
989
|
+
unless redis.sismember('vmpooler__template__deltas', pool['template'])
|
|
855
990
|
begin
|
|
856
991
|
provider.create_template_delta_disks(pool)
|
|
857
|
-
|
|
992
|
+
redis.sadd('vmpooler__template__deltas', pool['template'])
|
|
858
993
|
rescue StandardError => e
|
|
859
994
|
$logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{e}")
|
|
860
995
|
end
|
|
861
996
|
end
|
|
862
997
|
end
|
|
863
|
-
|
|
998
|
+
redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
|
|
864
999
|
end
|
|
865
1000
|
|
|
866
1001
|
def evaluate_template(pool, provider)
|
|
867
1002
|
mutex = pool_mutex(pool['name'])
|
|
868
|
-
prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
|
|
869
|
-
configured_template = $redis.hget('vmpooler__config__template', pool['name'])
|
|
870
1003
|
return if mutex.locked?
|
|
871
1004
|
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
1005
|
+
catch :update_not_needed do
|
|
1006
|
+
@redis.with_metrics do |redis|
|
|
1007
|
+
prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
|
|
1008
|
+
configured_template = redis.hget('vmpooler__config__template', pool['name'])
|
|
1009
|
+
|
|
1010
|
+
if prepared_template.nil?
|
|
1011
|
+
mutex.synchronize do
|
|
1012
|
+
prepare_template(pool, provider, redis)
|
|
1013
|
+
prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
|
|
1014
|
+
end
|
|
1015
|
+
elsif prepared_template != pool['template']
|
|
1016
|
+
if configured_template.nil?
|
|
1017
|
+
mutex.synchronize do
|
|
1018
|
+
prepare_template(pool, provider, redis)
|
|
1019
|
+
prepared_template = redis.hget('vmpooler__template__prepared', pool['name'])
|
|
1020
|
+
end
|
|
1021
|
+
end
|
|
1022
|
+
end
|
|
1023
|
+
throw :update_not_needed if configured_template.nil?
|
|
1024
|
+
throw :update_not_needed if configured_template == prepared_template
|
|
1025
|
+
|
|
879
1026
|
mutex.synchronize do
|
|
880
|
-
|
|
881
|
-
prepared_template = $redis.hget('vmpooler__template__prepared', pool['name'])
|
|
1027
|
+
update_pool_template(pool, provider, configured_template, prepared_template, redis)
|
|
882
1028
|
end
|
|
883
1029
|
end
|
|
884
1030
|
end
|
|
885
|
-
return if configured_template.nil?
|
|
886
|
-
return if configured_template == prepared_template
|
|
887
|
-
|
|
888
|
-
mutex.synchronize do
|
|
889
|
-
update_pool_template(pool, provider, configured_template, prepared_template)
|
|
890
|
-
end
|
|
891
1031
|
end
|
|
892
1032
|
|
|
893
|
-
def drain_pool(poolname)
|
|
1033
|
+
def drain_pool(poolname, redis)
|
|
894
1034
|
# Clear a pool of ready and pending instances
|
|
895
|
-
if
|
|
1035
|
+
if redis.scard("vmpooler__ready__#{poolname}") > 0
|
|
896
1036
|
$logger.log('s', "[*] [#{poolname}] removing ready instances")
|
|
897
|
-
|
|
898
|
-
move_vm_queue(poolname, vm, 'ready', 'completed')
|
|
1037
|
+
redis.smembers("vmpooler__ready__#{poolname}").each do |vm|
|
|
1038
|
+
move_vm_queue(poolname, vm, 'ready', 'completed', redis)
|
|
899
1039
|
end
|
|
900
1040
|
end
|
|
901
|
-
if
|
|
1041
|
+
if redis.scard("vmpooler__pending__#{poolname}") > 0
|
|
902
1042
|
$logger.log('s', "[*] [#{poolname}] removing pending instances")
|
|
903
|
-
|
|
904
|
-
move_vm_queue(poolname, vm, 'pending', 'completed')
|
|
1043
|
+
redis.smembers("vmpooler__pending__#{poolname}").each do |vm|
|
|
1044
|
+
move_vm_queue(poolname, vm, 'pending', 'completed', redis)
|
|
905
1045
|
end
|
|
906
1046
|
end
|
|
907
1047
|
end
|
|
908
1048
|
|
|
909
|
-
def update_pool_template(pool, provider, configured_template, prepared_template)
|
|
1049
|
+
def update_pool_template(pool, provider, configured_template, prepared_template, redis)
|
|
910
1050
|
pool['template'] = configured_template
|
|
911
1051
|
$logger.log('s', "[*] [#{pool['name']}] template updated from #{prepared_template} to #{configured_template}")
|
|
912
1052
|
# Remove all ready and pending VMs so new instances are created from the new template
|
|
913
|
-
drain_pool(pool['name'])
|
|
1053
|
+
drain_pool(pool['name'], redis)
|
|
914
1054
|
# Prepare template for deployment
|
|
915
1055
|
$logger.log('s', "[*] [#{pool['name']}] preparing pool template for deployment")
|
|
916
|
-
prepare_template(pool, provider)
|
|
1056
|
+
prepare_template(pool, provider, redis)
|
|
917
1057
|
$logger.log('s', "[*] [#{pool['name']}] is ready for use")
|
|
918
1058
|
end
|
|
919
1059
|
|
|
@@ -921,38 +1061,45 @@ module Vmpooler
|
|
|
921
1061
|
mutex = pool_mutex(pool['name'])
|
|
922
1062
|
return if mutex.locked?
|
|
923
1063
|
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
1064
|
+
@redis.with_metrics do |redis|
|
|
1065
|
+
clone_target = redis.hget('vmpooler__config__clone_target', pool['name'])
|
|
1066
|
+
break if clone_target.nil?
|
|
1067
|
+
break if clone_target == pool['clone_target']
|
|
927
1068
|
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
1069
|
+
$logger.log('s', "[*] [#{pool['name']}] clone updated from #{pool['clone_target']} to #{clone_target}")
|
|
1070
|
+
mutex.synchronize do
|
|
1071
|
+
pool['clone_target'] = clone_target
|
|
1072
|
+
# Remove all ready and pending VMs so new instances are created for the new clone_target
|
|
1073
|
+
drain_pool(pool['name'], redis)
|
|
1074
|
+
end
|
|
1075
|
+
$logger.log('s', "[*] [#{pool['name']}] is ready for use")
|
|
933
1076
|
end
|
|
934
|
-
$logger.log('s', "[*] [#{pool['name']}] is ready for use")
|
|
935
1077
|
end
|
|
936
1078
|
|
|
937
1079
|
def remove_excess_vms(pool)
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
1080
|
+
@redis.with_metrics do |redis|
|
|
1081
|
+
redis.multi
|
|
1082
|
+
redis.scard("vmpooler__ready__#{pool['name']}")
|
|
1083
|
+
redis.scard("vmpooler__pending__#{pool['name']}")
|
|
1084
|
+
ready, pending = redis.exec
|
|
1085
|
+
total = pending.to_i + ready.to_i
|
|
1086
|
+
break if total.nil?
|
|
1087
|
+
break if total == 0
|
|
942
1088
|
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
1089
|
+
mutex = pool_mutex(pool['name'])
|
|
1090
|
+
break if mutex.locked?
|
|
1091
|
+
break unless ready.to_i > pool['size']
|
|
946
1092
|
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
1093
|
+
mutex.synchronize do
|
|
1094
|
+
difference = ready.to_i - pool['size']
|
|
1095
|
+
difference.times do
|
|
1096
|
+
next_vm = redis.spop("vmpooler__ready__#{pool['name']}")
|
|
1097
|
+
move_vm_queue(pool['name'], next_vm, 'ready', 'completed', redis)
|
|
1098
|
+
end
|
|
1099
|
+
if total > ready
|
|
1100
|
+
redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
|
|
1101
|
+
move_vm_queue(pool['name'], vm, 'pending', 'completed', redis)
|
|
1102
|
+
end
|
|
956
1103
|
end
|
|
957
1104
|
end
|
|
958
1105
|
end
|
|
@@ -962,26 +1109,30 @@ module Vmpooler
|
|
|
962
1109
|
mutex = pool_mutex(pool['name'])
|
|
963
1110
|
return if mutex.locked?
|
|
964
1111
|
|
|
965
|
-
|
|
966
|
-
|
|
1112
|
+
@redis.with_metrics do |redis|
|
|
1113
|
+
poolsize = redis.hget('vmpooler__config__poolsize', pool['name'])
|
|
1114
|
+
break if poolsize.nil?
|
|
967
1115
|
|
|
968
|
-
|
|
969
|
-
|
|
1116
|
+
poolsize = Integer(poolsize)
|
|
1117
|
+
break if poolsize == pool['size']
|
|
970
1118
|
|
|
971
|
-
|
|
972
|
-
|
|
1119
|
+
mutex.synchronize do
|
|
1120
|
+
pool['size'] = poolsize
|
|
1121
|
+
end
|
|
973
1122
|
end
|
|
974
1123
|
end
|
|
975
1124
|
|
|
976
1125
|
def reset_pool(pool)
|
|
977
1126
|
poolname = pool['name']
|
|
978
|
-
|
|
1127
|
+
@redis.with_metrics do |redis|
|
|
1128
|
+
break unless redis.sismember('vmpooler__poolreset', poolname)
|
|
979
1129
|
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1130
|
+
redis.srem('vmpooler__poolreset', poolname)
|
|
1131
|
+
mutex = pool_mutex(poolname)
|
|
1132
|
+
mutex.synchronize do
|
|
1133
|
+
drain_pool(poolname, redis)
|
|
1134
|
+
$logger.log('s', "[*] [#{poolname}] reset has cleared ready and pending instances")
|
|
1135
|
+
end
|
|
985
1136
|
end
|
|
986
1137
|
end
|
|
987
1138
|
|
|
@@ -990,21 +1141,23 @@ module Vmpooler
|
|
|
990
1141
|
begin
|
|
991
1142
|
mutex = pool_mutex(pool['name'])
|
|
992
1143
|
mutex.synchronize do
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1144
|
+
@redis.with_metrics do |redis|
|
|
1145
|
+
provider.vms_in_pool(pool['name']).each do |vm|
|
|
1146
|
+
if !redis.sismember('vmpooler__running__' + pool['name'], vm['name']) &&
|
|
1147
|
+
!redis.sismember('vmpooler__ready__' + pool['name'], vm['name']) &&
|
|
1148
|
+
!redis.sismember('vmpooler__pending__' + pool['name'], vm['name']) &&
|
|
1149
|
+
!redis.sismember('vmpooler__completed__' + pool['name'], vm['name']) &&
|
|
1150
|
+
!redis.sismember('vmpooler__discovered__' + pool['name'], vm['name']) &&
|
|
1151
|
+
!redis.sismember('vmpooler__migrating__' + pool['name'], vm['name'])
|
|
1152
|
+
|
|
1153
|
+
pool_check_response[:discovered_vms] += 1
|
|
1154
|
+
redis.sadd('vmpooler__discovered__' + pool['name'], vm['name'])
|
|
1155
|
+
|
|
1156
|
+
$logger.log('s', "[?] [#{pool['name']}] '#{vm['name']}' added to 'discovered' queue")
|
|
1157
|
+
end
|
|
1158
|
+
|
|
1159
|
+
inventory[vm['name']] = 1
|
|
1005
1160
|
end
|
|
1006
|
-
|
|
1007
|
-
inventory[vm['name']] = 1
|
|
1008
1161
|
end
|
|
1009
1162
|
end
|
|
1010
1163
|
rescue StandardError => e
|
|
@@ -1015,96 +1168,112 @@ module Vmpooler
|
|
|
1015
1168
|
end
|
|
1016
1169
|
|
|
1017
1170
|
def check_running_pool_vms(pool_name, provider, pool_check_response, inventory)
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1171
|
+
@redis.with_metrics do |redis|
|
|
1172
|
+
redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
|
|
1173
|
+
if inventory[vm]
|
|
1174
|
+
begin
|
|
1175
|
+
vm_lifetime = redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
|
|
1176
|
+
pool_check_response[:checked_running_vms] += 1
|
|
1177
|
+
check_running_vm(vm, pool_name, vm_lifetime, provider)
|
|
1178
|
+
rescue StandardError => e
|
|
1179
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{e}")
|
|
1180
|
+
end
|
|
1181
|
+
else
|
|
1182
|
+
move_vm_queue(pool_name, vm, 'running', 'completed', redis, 'is a running VM but is missing from inventory. Marking as completed.')
|
|
1026
1183
|
end
|
|
1027
|
-
else
|
|
1028
|
-
move_vm_queue(pool_name, vm, 'running', 'completed', 'is a running VM but is missing from inventory. Marking as completed.')
|
|
1029
1184
|
end
|
|
1030
1185
|
end
|
|
1031
1186
|
end
|
|
1032
1187
|
|
|
1033
|
-
def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1188
|
+
def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl)
|
|
1189
|
+
@redis.with_metrics do |redis|
|
|
1190
|
+
redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
|
|
1191
|
+
if inventory[vm]
|
|
1192
|
+
begin
|
|
1193
|
+
pool_check_response[:checked_ready_vms] += 1
|
|
1194
|
+
check_ready_vm(vm, pool_name, pool_ttl, provider)
|
|
1195
|
+
rescue StandardError => e
|
|
1196
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{e}")
|
|
1197
|
+
end
|
|
1198
|
+
else
|
|
1199
|
+
move_vm_queue(pool_name, vm, 'ready', 'completed', redis, 'is a ready VM but is missing from inventory. Marking as completed.')
|
|
1041
1200
|
end
|
|
1042
|
-
else
|
|
1043
|
-
move_vm_queue(pool_name, vm, 'ready', 'completed', 'is a ready VM but is missing from inventory. Marking as completed.')
|
|
1044
1201
|
end
|
|
1045
1202
|
end
|
|
1046
1203
|
end
|
|
1047
1204
|
|
|
1048
|
-
def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout
|
|
1205
|
+
def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout)
|
|
1049
1206
|
pool_timeout ||= $config[:config]['timeout'] || 15
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1207
|
+
@redis.with_metrics do |redis|
|
|
1208
|
+
redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
|
|
1209
|
+
if inventory[vm]
|
|
1210
|
+
begin
|
|
1211
|
+
pool_check_response[:checked_pending_vms] += 1
|
|
1212
|
+
check_pending_vm(vm, pool_name, pool_timeout, provider)
|
|
1213
|
+
rescue StandardError => e
|
|
1214
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
|
|
1215
|
+
end
|
|
1216
|
+
else
|
|
1217
|
+
fail_pending_vm(vm, pool_name, pool_timeout, redis, false)
|
|
1057
1218
|
end
|
|
1058
|
-
else
|
|
1059
|
-
fail_pending_vm(vm, pool_name, pool_timeout, false)
|
|
1060
1219
|
end
|
|
1061
1220
|
end
|
|
1062
1221
|
end
|
|
1063
1222
|
|
|
1064
1223
|
def check_completed_pool_vms(pool_name, provider, pool_check_response, inventory)
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1224
|
+
@redis.with_metrics do |redis|
|
|
1225
|
+
redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
|
|
1226
|
+
if inventory[vm]
|
|
1227
|
+
begin
|
|
1228
|
+
pool_check_response[:destroyed_vms] += 1
|
|
1229
|
+
destroy_vm(vm, pool_name, provider)
|
|
1230
|
+
rescue StandardError => e
|
|
1231
|
+
redis.pipelined do
|
|
1232
|
+
redis.srem("vmpooler__completed__#{pool_name}", vm)
|
|
1233
|
+
redis.hdel("vmpooler__active__#{pool_name}", vm)
|
|
1234
|
+
redis.del("vmpooler__vm__#{vm}")
|
|
1235
|
+
end
|
|
1236
|
+
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
|
|
1237
|
+
end
|
|
1238
|
+
else
|
|
1239
|
+
$logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
|
1240
|
+
redis.pipelined do
|
|
1241
|
+
redis.srem("vmpooler__completed__#{pool_name}", vm)
|
|
1242
|
+
redis.hdel("vmpooler__active__#{pool_name}", vm)
|
|
1243
|
+
redis.del("vmpooler__vm__#{vm}")
|
|
1244
|
+
end
|
|
1075
1245
|
end
|
|
1076
|
-
else
|
|
1077
|
-
$logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
|
|
1078
|
-
$redis.srem("vmpooler__completed__#{pool_name}", vm)
|
|
1079
|
-
$redis.hdel("vmpooler__active__#{pool_name}", vm)
|
|
1080
|
-
$redis.del("vmpooler__vm__#{vm}")
|
|
1081
1246
|
end
|
|
1082
1247
|
end
|
|
1083
1248
|
end
|
|
1084
1249
|
|
|
1085
1250
|
def check_discovered_pool_vms(pool_name)
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1251
|
+
@redis.with_metrics do |redis|
|
|
1252
|
+
redis.smembers("vmpooler__discovered__#{pool_name}").reverse.each do |vm|
|
|
1253
|
+
%w[pending ready running completed].each do |queue|
|
|
1254
|
+
if redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
|
|
1255
|
+
$logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
|
|
1256
|
+
redis.srem("vmpooler__discovered__#{pool_name}", vm)
|
|
1257
|
+
end
|
|
1091
1258
|
end
|
|
1092
|
-
end
|
|
1093
1259
|
|
|
1094
|
-
|
|
1260
|
+
redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm) if redis.sismember("vmpooler__discovered__#{pool_name}", vm)
|
|
1261
|
+
end
|
|
1095
1262
|
end
|
|
1096
1263
|
rescue StandardError => e
|
|
1097
1264
|
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating discovered VMs: #{e}")
|
|
1098
1265
|
end
|
|
1099
1266
|
|
|
1100
1267
|
def check_migrating_pool_vms(pool_name, provider, pool_check_response, inventory)
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1268
|
+
@redis.with_metrics do |redis|
|
|
1269
|
+
redis.smembers("vmpooler__migrating__#{pool_name}").reverse.each do |vm|
|
|
1270
|
+
if inventory[vm]
|
|
1271
|
+
begin
|
|
1272
|
+
pool_check_response[:migrated_vms] += 1
|
|
1273
|
+
migrate_vm(vm, pool_name, provider)
|
|
1274
|
+
rescue StandardError => e
|
|
1275
|
+
$logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{e}")
|
|
1276
|
+
end
|
|
1108
1277
|
end
|
|
1109
1278
|
end
|
|
1110
1279
|
end
|
|
@@ -1113,29 +1282,37 @@ module Vmpooler
|
|
|
1113
1282
|
def repopulate_pool_vms(pool_name, provider, pool_check_response, pool_size)
|
|
1114
1283
|
return if pool_mutex(pool_name).locked?
|
|
1115
1284
|
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
$
|
|
1126
|
-
|
|
1127
|
-
|
|
1285
|
+
@redis.with_metrics do |redis|
|
|
1286
|
+
redis.multi
|
|
1287
|
+
redis.scard("vmpooler__ready__#{pool_name}")
|
|
1288
|
+
redis.scard("vmpooler__pending__#{pool_name}")
|
|
1289
|
+
redis.scard("vmpooler__running__#{pool_name}")
|
|
1290
|
+
ready, pending, running = redis.exec
|
|
1291
|
+
total = pending.to_i + ready.to_i
|
|
1292
|
+
|
|
1293
|
+
$metrics.gauge("ready.#{pool_name}", ready)
|
|
1294
|
+
$metrics.gauge("running.#{pool_name}", running)
|
|
1295
|
+
|
|
1296
|
+
unless pool_size == 0
|
|
1297
|
+
if redis.get("vmpooler__empty__#{pool_name}")
|
|
1298
|
+
redis.del("vmpooler__empty__#{pool_name}") unless ready == 0
|
|
1299
|
+
elsif ready == 0
|
|
1300
|
+
redis.set("vmpooler__empty__#{pool_name}", 'true')
|
|
1301
|
+
$logger.log('s', "[!] [#{pool_name}] is empty")
|
|
1302
|
+
end
|
|
1303
|
+
end
|
|
1128
1304
|
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1305
|
+
(pool_size - total.to_i).times do
|
|
1306
|
+
if redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
|
|
1307
|
+
begin
|
|
1308
|
+
redis.incr('vmpooler__tasks__clone')
|
|
1309
|
+
pool_check_response[:cloned_vms] += 1
|
|
1310
|
+
clone_vm(pool_name, provider)
|
|
1311
|
+
rescue StandardError => e
|
|
1312
|
+
$logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{e}")
|
|
1313
|
+
redis.decr('vmpooler__tasks__clone')
|
|
1314
|
+
raise
|
|
1315
|
+
end
|
|
1139
1316
|
end
|
|
1140
1317
|
end
|
|
1141
1318
|
end
|
|
@@ -1160,7 +1337,7 @@ module Vmpooler
|
|
|
1160
1337
|
|
|
1161
1338
|
check_running_pool_vms(pool['name'], provider, pool_check_response, inventory)
|
|
1162
1339
|
|
|
1163
|
-
check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'])
|
|
1340
|
+
check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
|
|
1164
1341
|
|
|
1165
1342
|
check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
|
|
1166
1343
|
|
|
@@ -1203,23 +1380,203 @@ module Vmpooler
|
|
|
1203
1380
|
#
|
|
1204
1381
|
# returns an object Vmpooler::PoolManager::Provider::*
|
|
1205
1382
|
# or raises an error if the class does not exist
|
|
1206
|
-
def create_provider_object(config, logger, metrics, provider_class, provider_name, options)
|
|
1383
|
+
def create_provider_object(config, logger, metrics, redis_connection_pool, provider_class, provider_name, options)
|
|
1207
1384
|
provider_klass = Vmpooler::PoolManager::Provider
|
|
1208
1385
|
provider_klass.constants.each do |classname|
|
|
1209
1386
|
next unless classname.to_s.casecmp(provider_class) == 0
|
|
1210
1387
|
|
|
1211
|
-
return provider_klass.const_get(classname).new(config, logger, metrics, provider_name, options)
|
|
1388
|
+
return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options)
|
|
1212
1389
|
end
|
|
1213
1390
|
raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
|
|
1214
1391
|
end
|
|
1215
1392
|
|
|
1393
|
+
def check_ondemand_requests(maxloop = 0,
|
|
1394
|
+
loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
|
|
1395
|
+
loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
|
|
1396
|
+
loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
|
|
1397
|
+
|
|
1398
|
+
$logger.log('d', '[*] [ondemand_provisioner] starting worker thread')
|
|
1399
|
+
|
|
1400
|
+
$threads['ondemand_provisioner'] = Thread.new do
|
|
1401
|
+
_check_ondemand_requests(maxloop, loop_delay_min, loop_delay_max, loop_delay_decay)
|
|
1402
|
+
end
|
|
1403
|
+
end
|
|
1404
|
+
|
|
1405
|
+
def _check_ondemand_requests(maxloop = 0,
|
|
1406
|
+
loop_delay_min = CHECK_LOOP_DELAY_MIN_DEFAULT,
|
|
1407
|
+
loop_delay_max = CHECK_LOOP_DELAY_MAX_DEFAULT,
|
|
1408
|
+
loop_delay_decay = CHECK_LOOP_DELAY_DECAY_DEFAULT)
|
|
1409
|
+
|
|
1410
|
+
loop_delay_min = $config[:config]['check_loop_delay_min'] unless $config[:config]['check_loop_delay_min'].nil?
|
|
1411
|
+
loop_delay_max = $config[:config]['check_loop_delay_max'] unless $config[:config]['check_loop_delay_max'].nil?
|
|
1412
|
+
loop_delay_decay = $config[:config]['check_loop_delay_decay'] unless $config[:config]['check_loop_delay_decay'].nil?
|
|
1413
|
+
|
|
1414
|
+
loop_delay_decay = 2.0 if loop_delay_decay <= 1.0
|
|
1415
|
+
loop_delay_max = loop_delay_min if loop_delay_max.nil? || loop_delay_max < loop_delay_min
|
|
1416
|
+
|
|
1417
|
+
loop_count = 1
|
|
1418
|
+
loop_delay = loop_delay_min
|
|
1419
|
+
|
|
1420
|
+
loop do
|
|
1421
|
+
result = process_ondemand_requests
|
|
1422
|
+
|
|
1423
|
+
loop_delay = (loop_delay * loop_delay_decay).to_i
|
|
1424
|
+
loop_delay = loop_delay_min if result > 0
|
|
1425
|
+
loop_delay = loop_delay_max if loop_delay > loop_delay_max
|
|
1426
|
+
sleep_with_wakeup_events(loop_delay, loop_delay_min, ondemand_request: true)
|
|
1427
|
+
|
|
1428
|
+
unless maxloop == 0
|
|
1429
|
+
break if loop_count >= maxloop
|
|
1430
|
+
|
|
1431
|
+
loop_count += 1
|
|
1432
|
+
end
|
|
1433
|
+
end
|
|
1434
|
+
end
|
|
1435
|
+
|
|
1436
|
+
def process_ondemand_requests
|
|
1437
|
+
@redis.with_metrics do |redis|
|
|
1438
|
+
requests = redis.zrange('vmpooler__provisioning__request', 0, -1)
|
|
1439
|
+
requests&.map { |request_id| create_ondemand_vms(request_id, redis) }
|
|
1440
|
+
provisioning_tasks = process_ondemand_vms(redis)
|
|
1441
|
+
requests_ready = check_ondemand_requests_ready(redis)
|
|
1442
|
+
requests.length + provisioning_tasks + requests_ready
|
|
1443
|
+
end
|
|
1444
|
+
end
|
|
1445
|
+
|
|
1446
|
+
def create_ondemand_vms(request_id, redis)
|
|
1447
|
+
requested = redis.hget("vmpooler__odrequest__#{request_id}", 'requested')
|
|
1448
|
+
unless requested
|
|
1449
|
+
$logger.log('s', "Failed to find odrequest for request_id '#{request_id}'")
|
|
1450
|
+
redis.zrem('vmpooler__provisioning__request', request_id)
|
|
1451
|
+
return
|
|
1452
|
+
end
|
|
1453
|
+
score = redis.zscore('vmpooler__provisioning__request', request_id)
|
|
1454
|
+
requested = requested.split(',')
|
|
1455
|
+
|
|
1456
|
+
redis.pipelined do
|
|
1457
|
+
requested.each do |request|
|
|
1458
|
+
redis.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
|
|
1459
|
+
end
|
|
1460
|
+
redis.zrem('vmpooler__provisioning__request', request_id)
|
|
1461
|
+
redis.zadd('vmpooler__provisioning__processing', score, request_id)
|
|
1462
|
+
end
|
|
1463
|
+
end
|
|
1464
|
+
|
|
1465
|
+
def process_ondemand_vms(redis)
|
|
1466
|
+
queue_key = 'vmpooler__odcreate__task'
|
|
1467
|
+
queue = redis.zrange(queue_key, 0, -1, with_scores: true)
|
|
1468
|
+
ondemand_clone_limit = $config[:config]['ondemand_clone_limit']
|
|
1469
|
+
queue.each do |request, score|
|
|
1470
|
+
clone_count = redis.get('vmpooler__tasks__ondemandclone').to_i
|
|
1471
|
+
break unless clone_count < ondemand_clone_limit
|
|
1472
|
+
|
|
1473
|
+
pool_alias, pool, count, request_id = request.split(':')
|
|
1474
|
+
count = count.to_i
|
|
1475
|
+
provider = get_provider_for_pool(pool)
|
|
1476
|
+
slots = ondemand_clone_limit - clone_count
|
|
1477
|
+
break if slots == 0
|
|
1478
|
+
|
|
1479
|
+
if slots >= count
|
|
1480
|
+
count.times do
|
|
1481
|
+
redis.incr('vmpooler__tasks__ondemandclone')
|
|
1482
|
+
clone_vm(pool, provider, request_id, pool_alias)
|
|
1483
|
+
end
|
|
1484
|
+
redis.zrem(queue_key, request)
|
|
1485
|
+
else
|
|
1486
|
+
remaining_count = count - slots
|
|
1487
|
+
slots.times do
|
|
1488
|
+
redis.incr('vmpooler__tasks__ondemandclone')
|
|
1489
|
+
clone_vm(pool, provider, request_id, pool_alias)
|
|
1490
|
+
end
|
|
1491
|
+
redis.pipelined do
|
|
1492
|
+
redis.zrem(queue_key, request)
|
|
1493
|
+
redis.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
|
|
1494
|
+
end
|
|
1495
|
+
end
|
|
1496
|
+
end
|
|
1497
|
+
queue.length
|
|
1498
|
+
end
|
|
1499
|
+
|
|
1500
|
+
def vms_ready?(request_id, redis)
|
|
1501
|
+
catch :request_not_ready do
|
|
1502
|
+
request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
|
1503
|
+
requested_platforms = request_hash['requested'].split(',')
|
|
1504
|
+
requested_platforms.each do |platform|
|
|
1505
|
+
platform_alias, pool, count = platform.split(':')
|
|
1506
|
+
pools_filled = redis.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
|
1507
|
+
throw :request_not_ready unless pools_filled.to_i == count.to_i
|
|
1508
|
+
end
|
|
1509
|
+
return true
|
|
1510
|
+
end
|
|
1511
|
+
false
|
|
1512
|
+
end
|
|
1513
|
+
|
|
1514
|
+
def check_ondemand_requests_ready(redis)
|
|
1515
|
+
in_progress_requests = redis.zrange('vmpooler__provisioning__processing', 0, -1, with_scores: true)
|
|
1516
|
+
in_progress_requests&.each do |request_id, score|
|
|
1517
|
+
check_ondemand_request_ready(request_id, redis, score)
|
|
1518
|
+
end
|
|
1519
|
+
in_progress_requests.length
|
|
1520
|
+
end
|
|
1521
|
+
|
|
1522
|
+
def check_ondemand_request_ready(request_id, redis, score = nil)
|
|
1523
|
+
# default expiration is one month to ensure the data does not stay in redis forever
|
|
1524
|
+
default_expiration = 259_200_0
|
|
1525
|
+
processing_key = 'vmpooler__provisioning__processing'
|
|
1526
|
+
ondemand_hash_key = "vmpooler__odrequest__#{request_id}"
|
|
1527
|
+
score ||= redis.zscore(processing_key, request_id)
|
|
1528
|
+
return if request_expired?(request_id, score, redis)
|
|
1529
|
+
|
|
1530
|
+
return unless vms_ready?(request_id, redis)
|
|
1531
|
+
|
|
1532
|
+
redis.multi
|
|
1533
|
+
redis.hset(ondemand_hash_key, 'status', 'ready')
|
|
1534
|
+
redis.expire(ondemand_hash_key, default_expiration)
|
|
1535
|
+
redis.zrem(processing_key, request_id)
|
|
1536
|
+
redis.exec
|
|
1537
|
+
end
|
|
1538
|
+
|
|
1539
|
+
def request_expired?(request_id, score, redis)
|
|
1540
|
+
delta = Time.now.to_i - score.to_i
|
|
1541
|
+
ondemand_request_ttl = $config[:config]['ondemand_request_ttl']
|
|
1542
|
+
return false unless delta > ondemand_request_ttl * 60
|
|
1543
|
+
|
|
1544
|
+
$logger.log('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
|
|
1545
|
+
expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
|
|
1546
|
+
redis.pipelined do
|
|
1547
|
+
redis.zrem('vmpooler__provisioning__processing', request_id)
|
|
1548
|
+
redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
|
|
1549
|
+
redis.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
|
|
1550
|
+
end
|
|
1551
|
+
remove_vms_for_failed_request(request_id, expiration_ttl, redis)
|
|
1552
|
+
true
|
|
1553
|
+
end
|
|
1554
|
+
|
|
1555
|
+
def remove_vms_for_failed_request(request_id, expiration_ttl, redis)
|
|
1556
|
+
request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
|
1557
|
+
requested_platforms = request_hash['requested'].split(',')
|
|
1558
|
+
requested_platforms.each do |platform|
|
|
1559
|
+
platform_alias, pool, _count = platform.split(':')
|
|
1560
|
+
pools_filled = redis.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
|
|
1561
|
+
redis.pipelined do
|
|
1562
|
+
pools_filled&.each do |vm|
|
|
1563
|
+
move_vm_queue(pool, vm, 'running', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
|
|
1564
|
+
end
|
|
1565
|
+
redis.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
|
|
1566
|
+
end
|
|
1567
|
+
end
|
|
1568
|
+
end
|
|
1569
|
+
|
|
1216
1570
|
def execute!(maxloop = 0, loop_delay = 1)
|
|
1217
1571
|
$logger.log('d', 'starting vmpooler')
|
|
1218
1572
|
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1573
|
+
@redis.with_metrics do |redis|
|
|
1574
|
+
# Clear out the tasks manager, as we don't know about any tasks at this point
|
|
1575
|
+
redis.set('vmpooler__tasks__clone', 0)
|
|
1576
|
+
redis.set('vmpooler__tasks__ondemandclone', 0)
|
|
1577
|
+
# Clear out vmpooler__migrations since stale entries may be left after a restart
|
|
1578
|
+
redis.del('vmpooler__migration')
|
|
1579
|
+
end
|
|
1223
1580
|
|
|
1224
1581
|
# Copy vSphere settings to correct location. This happens with older configuration files
|
|
1225
1582
|
if !$config[:vsphere].nil? && ($config[:providers].nil? || $config[:providers][:vsphere].nil?)
|
|
@@ -1269,7 +1626,7 @@ module Vmpooler
|
|
|
1269
1626
|
provider_class = $config[:providers][provider_name.to_sym]['provider_class']
|
|
1270
1627
|
end
|
|
1271
1628
|
begin
|
|
1272
|
-
$providers[provider_name] = create_provider_object($config, $logger, $metrics, provider_class, provider_name, {}) if $providers[provider_name].nil?
|
|
1629
|
+
$providers[provider_name] = create_provider_object($config, $logger, $metrics, @redis, provider_class, provider_name, {}) if $providers[provider_name].nil?
|
|
1273
1630
|
rescue StandardError => e
|
|
1274
1631
|
$logger.log('s', "Error while creating provider for pool #{pool['name']}: #{e}")
|
|
1275
1632
|
raise
|
|
@@ -1303,6 +1660,13 @@ module Vmpooler
|
|
|
1303
1660
|
end
|
|
1304
1661
|
end
|
|
1305
1662
|
|
|
1663
|
+
if !$threads['ondemand_provisioner']
|
|
1664
|
+
check_ondemand_requests
|
|
1665
|
+
elsif !$threads['ondemand_provisioner'].alive?
|
|
1666
|
+
$logger.log('d', '[!] [ondemand_provisioner] worker thread died, restarting')
|
|
1667
|
+
check_ondemand_requests(check_loop_delay_min, check_loop_delay_max, check_loop_delay_decay)
|
|
1668
|
+
end
|
|
1669
|
+
|
|
1306
1670
|
sleep(loop_delay)
|
|
1307
1671
|
|
|
1308
1672
|
unless maxloop == 0
|