vmpooler 3.3.0 → 3.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/vmpooler/api/helpers.rb +25 -17
- data/lib/vmpooler/pool_manager.rb +54 -30
- data/lib/vmpooler/version.rb +1 -1
- data/lib/vmpooler.rb +1 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 421d86151ba2c7a9bcc2310e3596b43f27a0280df1b7cb0c042dd12a63c80086
|
4
|
+
data.tar.gz: 7f8e2dea7fc3f4089995fc65022e98e48771f5a40cbbd30b51ddf162fad7b3f7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4722f36453f515e66cf69d2c9c19cd472dd289e012514b2823e6ccf2febbdc525d78969e4a24777779b2fd9dbf5c0f8137421e6fe0bdf95f726effead80cd114
|
7
|
+
data.tar.gz: 979d40b555d547d6ebec64efeb57af5cdb68eb90fbc6f759cf6a1bbfef074e66404b9e238ad410bd938cef611ae006e430aad86c754a50e3901d9c659081c733
|
data/lib/vmpooler/api/helpers.rb
CHANGED
@@ -68,7 +68,7 @@ module Vmpooler
|
|
68
68
|
end
|
69
69
|
end
|
70
70
|
|
71
|
-
def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str)
|
71
|
+
def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str, service_account_hash = nil)
|
72
72
|
tracer.in_span(
|
73
73
|
"Vmpooler::API::Helpers.#{__method__}",
|
74
74
|
attributes: {
|
@@ -79,6 +79,14 @@ module Vmpooler
|
|
79
79
|
},
|
80
80
|
kind: :client
|
81
81
|
) do
|
82
|
+
if service_account_hash
|
83
|
+
username = service_account_hash[:user_dn]
|
84
|
+
password = service_account_hash[:password]
|
85
|
+
else
|
86
|
+
username = "#{user_object}=#{username_str},#{base}"
|
87
|
+
password = password_str
|
88
|
+
end
|
89
|
+
|
82
90
|
ldap = Net::LDAP.new(
|
83
91
|
:host => host,
|
84
92
|
:port => port,
|
@@ -86,12 +94,22 @@ module Vmpooler
|
|
86
94
|
:base => base,
|
87
95
|
:auth => {
|
88
96
|
:method => :simple,
|
89
|
-
:username =>
|
90
|
-
:password =>
|
97
|
+
:username => username,
|
98
|
+
:password => password
|
91
99
|
}
|
92
100
|
)
|
93
101
|
|
94
|
-
|
102
|
+
if service_account_hash
|
103
|
+
return true if ldap.bind_as(
|
104
|
+
:base => base,
|
105
|
+
:filter => "(#{user_object}=#{username_str})",
|
106
|
+
:password => password_str
|
107
|
+
)
|
108
|
+
elsif ldap.bind
|
109
|
+
return true
|
110
|
+
else
|
111
|
+
return false
|
112
|
+
end
|
95
113
|
|
96
114
|
return false
|
97
115
|
end
|
@@ -116,6 +134,7 @@ module Vmpooler
|
|
116
134
|
:method => :start_tls,
|
117
135
|
:tls_options => { :ssl_version => 'TLSv1' }
|
118
136
|
}
|
137
|
+
service_account_hash = auth[:ldap]['service_account_hash']
|
119
138
|
|
120
139
|
unless ldap_base.is_a? Array
|
121
140
|
ldap_base = ldap_base.split
|
@@ -134,7 +153,8 @@ module Vmpooler
|
|
134
153
|
search_user_obj,
|
135
154
|
search_base,
|
136
155
|
username_str,
|
137
|
-
password_str
|
156
|
+
password_str,
|
157
|
+
service_account_hash
|
138
158
|
)
|
139
159
|
return true if result
|
140
160
|
end
|
@@ -551,18 +571,6 @@ module Vmpooler
|
|
551
571
|
end
|
552
572
|
end
|
553
573
|
end
|
554
|
-
|
555
|
-
def vm_ready?(vm_name, domain = nil)
|
556
|
-
tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
|
557
|
-
begin
|
558
|
-
open_socket(vm_name, domain)
|
559
|
-
rescue StandardError => _e
|
560
|
-
return false
|
561
|
-
end
|
562
|
-
|
563
|
-
true
|
564
|
-
end
|
565
|
-
end
|
566
574
|
end
|
567
575
|
end
|
568
576
|
end
|
@@ -82,31 +82,31 @@ module Vmpooler
|
|
82
82
|
end
|
83
83
|
|
84
84
|
# Check the state of a VM
|
85
|
-
def check_pending_vm(vm, pool, timeout, provider)
|
85
|
+
def check_pending_vm(vm, pool, timeout, timeout_notification, provider)
|
86
86
|
Thread.new do
|
87
87
|
begin
|
88
|
-
_check_pending_vm(vm, pool, timeout, provider)
|
88
|
+
_check_pending_vm(vm, pool, timeout, timeout_notification, provider)
|
89
89
|
rescue StandardError => e
|
90
90
|
$logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
|
91
91
|
@redis.with_metrics do |redis|
|
92
|
-
fail_pending_vm(vm, pool, timeout, redis)
|
92
|
+
fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
|
93
93
|
end
|
94
94
|
raise
|
95
95
|
end
|
96
96
|
end
|
97
97
|
end
|
98
98
|
|
99
|
-
def _check_pending_vm(vm, pool, timeout, provider)
|
99
|
+
def _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
|
100
100
|
mutex = vm_mutex(vm)
|
101
101
|
return if mutex.locked?
|
102
102
|
|
103
103
|
mutex.synchronize do
|
104
104
|
@redis.with_metrics do |redis|
|
105
105
|
request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
|
106
|
-
if provider.vm_ready?(pool, vm)
|
106
|
+
if provider.vm_ready?(pool, vm, redis)
|
107
107
|
move_pending_vm_to_ready(vm, pool, redis, request_id)
|
108
108
|
else
|
109
|
-
fail_pending_vm(vm, pool, timeout, redis)
|
109
|
+
fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
|
110
110
|
end
|
111
111
|
end
|
112
112
|
end
|
@@ -122,34 +122,53 @@ module Vmpooler
|
|
122
122
|
$logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
|
123
123
|
end
|
124
124
|
|
125
|
-
def fail_pending_vm(vm, pool, timeout, redis, exists: true)
|
125
|
+
def fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
|
126
126
|
clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
|
127
|
-
|
128
127
|
time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
136
|
-
if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
|
137
|
-
# will retry a VM that did not come up as vm_ready? only if it has not been market failed or deleted
|
138
|
-
redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
|
139
|
-
end
|
140
|
-
end
|
141
|
-
$metrics.increment("errors.markedasfailed.#{pool}")
|
142
|
-
$logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
|
143
|
-
else
|
128
|
+
|
129
|
+
already_timed_out = time_since_clone > timeout
|
130
|
+
timing_out_soon = time_since_clone > timeout_notification && !redis.hget("vmpooler__vm__#{vm}", 'timeout_notification')
|
131
|
+
|
132
|
+
if already_timed_out
|
133
|
+
unless exists
|
144
134
|
remove_nonexistent_vm(vm, pool, redis)
|
135
|
+
return true
|
145
136
|
end
|
137
|
+
open_socket_error = handle_timed_out_vm(vm, pool, redis)
|
146
138
|
end
|
139
|
+
|
140
|
+
redis.hset("vmpooler__vm__#{vm}", 'timeout_notification', 1) if timing_out_soon
|
141
|
+
|
142
|
+
nonexist_warning = if already_timed_out
|
143
|
+
"[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: #{open_socket_error}"
|
144
|
+
elsif timing_out_soon
|
145
|
+
"[!] [#{pool}] '#{vm}' no longer exists when attempting to send notification of impending failure"
|
146
|
+
else
|
147
|
+
"[!] [#{pool}] '#{vm}' This error is wholly unexpected"
|
148
|
+
end
|
149
|
+
$logger.log('d', nonexist_warning)
|
147
150
|
true
|
148
151
|
rescue StandardError => e
|
149
152
|
$logger.log('d', "Fail pending VM failed with an error: #{e}")
|
150
153
|
false
|
151
154
|
end
|
152
155
|
|
156
|
+
def handle_timed_out_vm(vm, pool, redis)
|
157
|
+
request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
|
158
|
+
pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
|
159
|
+
open_socket_error = redis.hget("vmpooler__vm__#{vm}", 'open_socket_error')
|
160
|
+
redis.smove("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
|
161
|
+
if request_id
|
162
|
+
ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
163
|
+
if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
|
164
|
+
# will retry a VM that did not come up as vm_ready? only if it has not been market failed or deleted
|
165
|
+
redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
|
166
|
+
end
|
167
|
+
end
|
168
|
+
$metrics.increment("errors.markedasfailed.#{pool}")
|
169
|
+
open_socket_error
|
170
|
+
end
|
171
|
+
|
153
172
|
def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
|
154
173
|
clone_time = redis.hget("vmpooler__vm__#{vm}", 'clone')
|
155
174
|
finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
|
@@ -197,11 +216,12 @@ module Vmpooler
|
|
197
216
|
|
198
217
|
def vm_still_ready?(pool_name, vm_name, provider, redis)
|
199
218
|
# Check if the VM is still ready/available
|
200
|
-
return true if provider.vm_ready?(pool_name, vm_name)
|
219
|
+
return true if provider.vm_ready?(pool_name, vm_name, redis)
|
201
220
|
|
202
221
|
raise("VM #{vm_name} is not ready")
|
203
222
|
rescue StandardError
|
204
|
-
|
223
|
+
open_socket_error = redis.hget("vmpooler__vm__#{vm_name}", 'open_socket_error')
|
224
|
+
move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "removed from 'ready' queue. vm unreachable with error: #{open_socket_error}")
|
205
225
|
end
|
206
226
|
|
207
227
|
def check_ready_vm(vm, pool_name, ttl, provider)
|
@@ -318,7 +338,7 @@ module Vmpooler
|
|
318
338
|
redis.hset("vmpooler__vm__#{vm}", 'user_tagged', 'true') if success
|
319
339
|
end
|
320
340
|
|
321
|
-
throw :stop_checking if provider.vm_ready?(pool, vm)
|
341
|
+
throw :stop_checking if provider.vm_ready?(pool, vm, redis)
|
322
342
|
|
323
343
|
throw :stop_checking if provider.get_vm(pool, vm)
|
324
344
|
|
@@ -450,6 +470,9 @@ module Vmpooler
|
|
450
470
|
ip_start = Time.now
|
451
471
|
ip = provider.get_vm_ip_address(new_vmname, pool_name)
|
452
472
|
ip_finish = format('%<time>.2f', time: Time.now - ip_start)
|
473
|
+
|
474
|
+
raise StandardError, "failed to obtain IP after #{ip_finish} seconds" if ip.nil?
|
475
|
+
|
453
476
|
$logger.log('s', "[+] [#{pool_name}] Obtained IP for '#{new_vmname}' in #{ip_finish} seconds")
|
454
477
|
|
455
478
|
@redis.with_metrics do |redis|
|
@@ -1248,19 +1271,20 @@ module Vmpooler
|
|
1248
1271
|
end
|
1249
1272
|
end
|
1250
1273
|
|
1251
|
-
def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout)
|
1274
|
+
def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout, pool_timeout_notification)
|
1252
1275
|
pool_timeout ||= $config[:config]['timeout'] || 15
|
1276
|
+
pool_timeout_notification ||= $config[:config]['timeout_notification'] || 5
|
1253
1277
|
@redis.with_metrics do |redis|
|
1254
1278
|
redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
|
1255
1279
|
if inventory[vm]
|
1256
1280
|
begin
|
1257
1281
|
pool_check_response[:checked_pending_vms] += 1
|
1258
|
-
check_pending_vm(vm, pool_name, pool_timeout, provider)
|
1282
|
+
check_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, provider)
|
1259
1283
|
rescue StandardError => e
|
1260
1284
|
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
|
1261
1285
|
end
|
1262
1286
|
else
|
1263
|
-
fail_pending_vm(vm, pool_name, pool_timeout, redis, exists: false)
|
1287
|
+
fail_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, redis, exists: false)
|
1264
1288
|
end
|
1265
1289
|
end
|
1266
1290
|
end
|
@@ -1387,7 +1411,7 @@ module Vmpooler
|
|
1387
1411
|
|
1388
1412
|
check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
|
1389
1413
|
|
1390
|
-
check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
|
1414
|
+
check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'], pool['timeout_notification'])
|
1391
1415
|
|
1392
1416
|
check_completed_pool_vms(pool['name'], provider, pool_check_response, inventory)
|
1393
1417
|
|
data/lib/vmpooler/version.rb
CHANGED
data/lib/vmpooler.rb
CHANGED
@@ -82,6 +82,7 @@ module Vmpooler
|
|
82
82
|
end
|
83
83
|
parsed_config[:config]['clone_target'] = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
|
84
84
|
parsed_config[:config]['timeout'] = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
|
85
|
+
parsed_config[:config]['timeout_notification'] = string_to_int(ENV['TIMEOUT_NOTIFICATION']) if ENV['TIMEOUT_NOTIFICATION']
|
85
86
|
parsed_config[:config]['vm_lifetime_auth'] = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
|
86
87
|
parsed_config[:config]['max_tries'] = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
|
87
88
|
parsed_config[:config]['retry_factor'] = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: vmpooler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Puppet
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-08-
|
11
|
+
date: 2023-08-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|