vmpooler 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/vmpooler/api/helpers.rb +25 -5
- data/lib/vmpooler/pool_manager.rb +49 -27
- data/lib/vmpooler/version.rb +1 -1
- data/lib/vmpooler.rb +1 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 421d86151ba2c7a9bcc2310e3596b43f27a0280df1b7cb0c042dd12a63c80086
|
4
|
+
data.tar.gz: 7f8e2dea7fc3f4089995fc65022e98e48771f5a40cbbd30b51ddf162fad7b3f7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4722f36453f515e66cf69d2c9c19cd472dd289e012514b2823e6ccf2febbdc525d78969e4a24777779b2fd9dbf5c0f8137421e6fe0bdf95f726effead80cd114
|
7
|
+
data.tar.gz: 979d40b555d547d6ebec64efeb57af5cdb68eb90fbc6f759cf6a1bbfef074e66404b9e238ad410bd938cef611ae006e430aad86c754a50e3901d9c659081c733
|
data/lib/vmpooler/api/helpers.rb
CHANGED
@@ -68,7 +68,7 @@ module Vmpooler
|
|
68
68
|
end
|
69
69
|
end
|
70
70
|
|
71
|
-
def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str)
|
71
|
+
def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str, service_account_hash = nil)
|
72
72
|
tracer.in_span(
|
73
73
|
"Vmpooler::API::Helpers.#{__method__}",
|
74
74
|
attributes: {
|
@@ -79,6 +79,14 @@ module Vmpooler
|
|
79
79
|
},
|
80
80
|
kind: :client
|
81
81
|
) do
|
82
|
+
if service_account_hash
|
83
|
+
username = service_account_hash[:user_dn]
|
84
|
+
password = service_account_hash[:password]
|
85
|
+
else
|
86
|
+
username = "#{user_object}=#{username_str},#{base}"
|
87
|
+
password = password_str
|
88
|
+
end
|
89
|
+
|
82
90
|
ldap = Net::LDAP.new(
|
83
91
|
:host => host,
|
84
92
|
:port => port,
|
@@ -86,12 +94,22 @@ module Vmpooler
|
|
86
94
|
:base => base,
|
87
95
|
:auth => {
|
88
96
|
:method => :simple,
|
89
|
-
:username =>
|
90
|
-
:password =>
|
97
|
+
:username => username,
|
98
|
+
:password => password
|
91
99
|
}
|
92
100
|
)
|
93
101
|
|
94
|
-
|
102
|
+
if service_account_hash
|
103
|
+
return true if ldap.bind_as(
|
104
|
+
:base => base,
|
105
|
+
:filter => "(#{user_object}=#{username_str})",
|
106
|
+
:password => password_str
|
107
|
+
)
|
108
|
+
elsif ldap.bind
|
109
|
+
return true
|
110
|
+
else
|
111
|
+
return false
|
112
|
+
end
|
95
113
|
|
96
114
|
return false
|
97
115
|
end
|
@@ -116,6 +134,7 @@ module Vmpooler
|
|
116
134
|
:method => :start_tls,
|
117
135
|
:tls_options => { :ssl_version => 'TLSv1' }
|
118
136
|
}
|
137
|
+
service_account_hash = auth[:ldap]['service_account_hash']
|
119
138
|
|
120
139
|
unless ldap_base.is_a? Array
|
121
140
|
ldap_base = ldap_base.split
|
@@ -134,7 +153,8 @@ module Vmpooler
|
|
134
153
|
search_user_obj,
|
135
154
|
search_base,
|
136
155
|
username_str,
|
137
|
-
password_str
|
156
|
+
password_str,
|
157
|
+
service_account_hash
|
138
158
|
)
|
139
159
|
return true if result
|
140
160
|
end
|
@@ -82,21 +82,21 @@ module Vmpooler
|
|
82
82
|
end
|
83
83
|
|
84
84
|
# Check the state of a VM
|
85
|
-
def check_pending_vm(vm, pool, timeout, provider)
|
85
|
+
def check_pending_vm(vm, pool, timeout, timeout_notification, provider)
|
86
86
|
Thread.new do
|
87
87
|
begin
|
88
|
-
_check_pending_vm(vm, pool, timeout, provider)
|
88
|
+
_check_pending_vm(vm, pool, timeout, timeout_notification, provider)
|
89
89
|
rescue StandardError => e
|
90
90
|
$logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
|
91
91
|
@redis.with_metrics do |redis|
|
92
|
-
fail_pending_vm(vm, pool, timeout, redis)
|
92
|
+
fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
|
93
93
|
end
|
94
94
|
raise
|
95
95
|
end
|
96
96
|
end
|
97
97
|
end
|
98
98
|
|
99
|
-
def _check_pending_vm(vm, pool, timeout, provider)
|
99
|
+
def _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
|
100
100
|
mutex = vm_mutex(vm)
|
101
101
|
return if mutex.locked?
|
102
102
|
|
@@ -106,7 +106,7 @@ module Vmpooler
|
|
106
106
|
if provider.vm_ready?(pool, vm, redis)
|
107
107
|
move_pending_vm_to_ready(vm, pool, redis, request_id)
|
108
108
|
else
|
109
|
-
fail_pending_vm(vm, pool, timeout, redis)
|
109
|
+
fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
|
110
110
|
end
|
111
111
|
end
|
112
112
|
end
|
@@ -122,35 +122,53 @@ module Vmpooler
|
|
122
122
|
$logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
|
123
123
|
end
|
124
124
|
|
125
|
-
def fail_pending_vm(vm, pool, timeout, redis, exists: true)
|
125
|
+
def fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
|
126
126
|
clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
|
127
|
-
|
128
127
|
time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
if request_id
|
136
|
-
ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
137
|
-
if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
|
138
|
-
# will retry a VM that did not come up as vm_ready? only if it has not been market failed or deleted
|
139
|
-
redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
|
140
|
-
end
|
141
|
-
end
|
142
|
-
$metrics.increment("errors.markedasfailed.#{pool}")
|
143
|
-
$logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: #{open_socket_error}")
|
144
|
-
else
|
128
|
+
|
129
|
+
already_timed_out = time_since_clone > timeout
|
130
|
+
timing_out_soon = time_since_clone > timeout_notification && !redis.hget("vmpooler__vm__#{vm}", 'timeout_notification')
|
131
|
+
|
132
|
+
if already_timed_out
|
133
|
+
unless exists
|
145
134
|
remove_nonexistent_vm(vm, pool, redis)
|
135
|
+
return true
|
146
136
|
end
|
137
|
+
open_socket_error = handle_timed_out_vm(vm, pool, redis)
|
147
138
|
end
|
139
|
+
|
140
|
+
redis.hset("vmpooler__vm__#{vm}", 'timeout_notification', 1) if timing_out_soon
|
141
|
+
|
142
|
+
nonexist_warning = if already_timed_out
|
143
|
+
"[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: #{open_socket_error}"
|
144
|
+
elsif timing_out_soon
|
145
|
+
"[!] [#{pool}] '#{vm}' no longer exists when attempting to send notification of impending failure"
|
146
|
+
else
|
147
|
+
"[!] [#{pool}] '#{vm}' This error is wholly unexpected"
|
148
|
+
end
|
149
|
+
$logger.log('d', nonexist_warning)
|
148
150
|
true
|
149
151
|
rescue StandardError => e
|
150
152
|
$logger.log('d', "Fail pending VM failed with an error: #{e}")
|
151
153
|
false
|
152
154
|
end
|
153
155
|
|
156
|
+
def handle_timed_out_vm(vm, pool, redis)
|
157
|
+
request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
|
158
|
+
pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
|
159
|
+
open_socket_error = redis.hget("vmpooler__vm__#{vm}", 'open_socket_error')
|
160
|
+
redis.smove("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
|
161
|
+
if request_id
|
162
|
+
ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
|
163
|
+
if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
|
164
|
+
# will retry a VM that did not come up as vm_ready? only if it has not been market failed or deleted
|
165
|
+
redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
|
166
|
+
end
|
167
|
+
end
|
168
|
+
$metrics.increment("errors.markedasfailed.#{pool}")
|
169
|
+
open_socket_error
|
170
|
+
end
|
171
|
+
|
154
172
|
def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
|
155
173
|
clone_time = redis.hget("vmpooler__vm__#{vm}", 'clone')
|
156
174
|
finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
|
@@ -452,6 +470,9 @@ module Vmpooler
|
|
452
470
|
ip_start = Time.now
|
453
471
|
ip = provider.get_vm_ip_address(new_vmname, pool_name)
|
454
472
|
ip_finish = format('%<time>.2f', time: Time.now - ip_start)
|
473
|
+
|
474
|
+
raise StandardError, "failed to obtain IP after #{ip_finish} seconds" if ip.nil?
|
475
|
+
|
455
476
|
$logger.log('s', "[+] [#{pool_name}] Obtained IP for '#{new_vmname}' in #{ip_finish} seconds")
|
456
477
|
|
457
478
|
@redis.with_metrics do |redis|
|
@@ -1250,19 +1271,20 @@ module Vmpooler
|
|
1250
1271
|
end
|
1251
1272
|
end
|
1252
1273
|
|
1253
|
-
def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout)
|
1274
|
+
def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout, pool_timeout_notification)
|
1254
1275
|
pool_timeout ||= $config[:config]['timeout'] || 15
|
1276
|
+
pool_timeout_notification ||= $config[:config]['timeout_notification'] || 5
|
1255
1277
|
@redis.with_metrics do |redis|
|
1256
1278
|
redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
|
1257
1279
|
if inventory[vm]
|
1258
1280
|
begin
|
1259
1281
|
pool_check_response[:checked_pending_vms] += 1
|
1260
|
-
check_pending_vm(vm, pool_name, pool_timeout, provider)
|
1282
|
+
check_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, provider)
|
1261
1283
|
rescue StandardError => e
|
1262
1284
|
$logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
|
1263
1285
|
end
|
1264
1286
|
else
|
1265
|
-
fail_pending_vm(vm, pool_name, pool_timeout, redis, exists: false)
|
1287
|
+
fail_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, redis, exists: false)
|
1266
1288
|
end
|
1267
1289
|
end
|
1268
1290
|
end
|
@@ -1389,7 +1411,7 @@ module Vmpooler
|
|
1389
1411
|
|
1390
1412
|
check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
|
1391
1413
|
|
1392
|
-
check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
|
1414
|
+
check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'], pool['timeout_notification'])
|
1393
1415
|
|
1394
1416
|
check_completed_pool_vms(pool['name'], provider, pool_check_response, inventory)
|
1395
1417
|
|
data/lib/vmpooler/version.rb
CHANGED
data/lib/vmpooler.rb
CHANGED
@@ -82,6 +82,7 @@ module Vmpooler
|
|
82
82
|
end
|
83
83
|
parsed_config[:config]['clone_target'] = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
|
84
84
|
parsed_config[:config]['timeout'] = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
|
85
|
+
parsed_config[:config]['timeout_notification'] = string_to_int(ENV['TIMEOUT_NOTIFICATION']) if ENV['TIMEOUT_NOTIFICATION']
|
85
86
|
parsed_config[:config]['vm_lifetime_auth'] = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
|
86
87
|
parsed_config[:config]['max_tries'] = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
|
87
88
|
parsed_config[:config]['retry_factor'] = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: vmpooler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Puppet
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-08-
|
11
|
+
date: 2023-08-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|