vmfloaty 1.2.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +7 -9
- data/lib/vmfloaty/abs.rb +50 -51
- data/lib/vmfloaty/http.rb +2 -6
- data/lib/vmfloaty/logger.rb +13 -10
- data/lib/vmfloaty/nonstandard_pooler.rb +3 -2
- data/lib/vmfloaty/pooler.rb +19 -24
- data/lib/vmfloaty/service.rb +3 -3
- data/lib/vmfloaty/utils.rb +68 -64
- data/lib/vmfloaty/version.rb +1 -2
- data/lib/vmfloaty.rb +52 -57
- data/spec/spec_helper.rb +30 -5
- data/spec/vmfloaty/abs/auth_spec.rb +26 -17
- data/spec/vmfloaty/abs_spec.rb +52 -43
- data/spec/vmfloaty/auth_spec.rb +23 -13
- data/spec/vmfloaty/nonstandard_pooler_spec.rb +32 -31
- data/spec/vmfloaty/pooler_spec.rb +29 -26
- data/spec/vmfloaty/service_spec.rb +10 -10
- data/spec/vmfloaty/ssh_spec.rb +3 -3
- data/spec/vmfloaty/utils_spec.rb +178 -161
- metadata +13 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cf8e6d210de6ae23fdd371025ace279b939768d1f197b81b2226f326263bc204
|
4
|
+
data.tar.gz: 7da421b68bb4377e31c4e5784e394ab672adb616ce8be4376e55f0006dd81849
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 846823d38e3b7dc5cfa066efcda72a6f58ad11ce875529a3ac6443551a138fbc3e05e262226e3dd756e93e8d31c9fa5c3bb4639f139158f5903a95c43bfb9b40
|
7
|
+
data.tar.gz: e693187229fb1b845e4a2b24f9bf9b22e7b0e4244887f8c1fc6042e64d6082638266b56421e04a46948b2efc92e4abd3e4d0ccdf28e345e30a89a77909872022
|
data/README.md
CHANGED
@@ -1,11 +1,9 @@
|
|
1
1
|
# vmfloaty
|
2
2
|
|
3
3
|
[](https://badge.fury.io/rb/vmfloaty)
|
4
|
-
[](https://coveralls.io/github/puppetlabs/vmfloaty?branch=master)
|
6
|
-
[](https://dependabot.com)
|
4
|
+
[](https://github.com/puppetlabs/vmfloaty/actions/workflows/ci.yml)
|
7
5
|
|
8
|
-
A CLI helper tool for [Puppet's
|
6
|
+
A CLI helper tool for [Puppet's VMPooler](https://github.com/puppetlabs/vmpooler) to help you stay afloat.
|
9
7
|
|
10
8
|

|
11
9
|
|
@@ -18,7 +16,7 @@ A CLI helper tool for [Puppet's vmpooler](https://github.com/puppetlabs/vmpooler
|
|
18
16
|
- [Using backends besides VMPooler](#using-backends-besides-vmpooler)
|
19
17
|
- [Valid config keys](#valid-config-keys)
|
20
18
|
- [Tab Completion](#tab-completion)
|
21
|
-
- [
|
19
|
+
- [VMPooler API](#vmpooler-api)
|
22
20
|
- [Using the Pooler class](#using-the-pooler-class)
|
23
21
|
- [Example Projects](#example-projects)
|
24
22
|
- [Special thanks](#special-thanks)
|
@@ -41,7 +39,7 @@ $ floaty --help
|
|
41
39
|
|
42
40
|
DESCRIPTION:
|
43
41
|
|
44
|
-
A CLI helper tool for Puppet's
|
42
|
+
A CLI helper tool for Puppet's VMPooler to help you stay afloat
|
45
43
|
|
46
44
|
COMMANDS:
|
47
45
|
|
@@ -151,13 +149,13 @@ There is also tab completion for zsh:
|
|
151
149
|
source $(floaty completion --shell zsh)
|
152
150
|
```
|
153
151
|
|
154
|
-
##
|
152
|
+
## VMPooler API
|
155
153
|
|
156
|
-
This cli tool uses the [
|
154
|
+
This cli tool uses the [VMPooler API](https://github.com/puppetlabs/vmpooler/blob/master/API.md).
|
157
155
|
|
158
156
|
## Using the Pooler class
|
159
157
|
|
160
|
-
vmfloaty providers a `Pooler` class that gives users the ability to make requests to
|
158
|
+
vmfloaty providers a `Pooler` class that gives users the ability to make requests to VMPooler without having to write their own requests. It also provides an `Auth` class for managing VMPooler tokens within your application.
|
161
159
|
|
162
160
|
### Example Projects
|
163
161
|
|
data/lib/vmfloaty/abs.rb
CHANGED
@@ -53,10 +53,10 @@ class ABS
|
|
53
53
|
def self.list_active(verbose, url, _token, user)
|
54
54
|
hosts = []
|
55
55
|
get_active_requests(verbose, url, user).each do |req_hash|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
56
|
+
next unless req_hash.key?('allocated_resources')
|
57
|
+
|
58
|
+
req_hash['allocated_resources'].each do |onehost|
|
59
|
+
hosts.push(onehost['hostname'])
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
@@ -116,7 +116,7 @@ class ABS
|
|
116
116
|
ret_status = {}
|
117
117
|
hosts.each do |host|
|
118
118
|
ret_status[host] = {
|
119
|
-
'ok' => false
|
119
|
+
'ok' => false
|
120
120
|
}
|
121
121
|
end
|
122
122
|
|
@@ -132,7 +132,7 @@ class ABS
|
|
132
132
|
if hosts.include? vm_name['hostname']
|
133
133
|
if all_job_resources_accounted_for(req_hash['allocated_resources'], hosts)
|
134
134
|
ret_status[vm_name['hostname']] = {
|
135
|
-
'ok' => true
|
135
|
+
'ok' => true
|
136
136
|
}
|
137
137
|
jobs_to_delete.push(req_hash)
|
138
138
|
else
|
@@ -147,7 +147,7 @@ class ABS
|
|
147
147
|
jobs_to_delete.each do |job|
|
148
148
|
req_obj = {
|
149
149
|
'job_id' => job['request']['job']['id'],
|
150
|
-
'hosts'
|
150
|
+
'hosts' => job['allocated_resources']
|
151
151
|
}
|
152
152
|
|
153
153
|
FloatyLogger.info "Deleting #{req_obj}" if verbose
|
@@ -172,11 +172,11 @@ class ABS
|
|
172
172
|
res_body = JSON.parse(res.body)
|
173
173
|
if res_body.key?('vmpooler_platforms')
|
174
174
|
os_list << '*** VMPOOLER Pools ***'
|
175
|
-
if res_body['vmpooler_platforms'].is_a?(String)
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
175
|
+
os_list += if res_body['vmpooler_platforms'].is_a?(String)
|
176
|
+
JSON.parse(res_body['vmpooler_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
|
177
|
+
else
|
178
|
+
res_body['vmpooler_platforms']
|
179
|
+
end
|
180
180
|
end
|
181
181
|
end
|
182
182
|
|
@@ -200,11 +200,11 @@ class ABS
|
|
200
200
|
if res_body.key?('nspooler_platforms')
|
201
201
|
os_list << ''
|
202
202
|
os_list << '*** NSPOOLER Pools ***'
|
203
|
-
if res_body['nspooler_platforms'].is_a?(String)
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
203
|
+
os_list += if res_body['nspooler_platforms'].is_a?(String)
|
204
|
+
JSON.parse(res_body['nspooler_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
|
205
|
+
else
|
206
|
+
res_body['nspooler_platforms']
|
207
|
+
end
|
208
208
|
end
|
209
209
|
end
|
210
210
|
|
@@ -214,11 +214,11 @@ class ABS
|
|
214
214
|
if res_body.key?('aws_platforms')
|
215
215
|
os_list << ''
|
216
216
|
os_list << '*** AWS Pools ***'
|
217
|
-
if res_body['aws_platforms'].is_a?(String)
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
217
|
+
os_list += if res_body['aws_platforms'].is_a?(String)
|
218
|
+
JSON.parse(res_body['aws_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
|
219
|
+
else
|
220
|
+
res_body['aws_platforms']
|
221
|
+
end
|
222
222
|
end
|
223
223
|
end
|
224
224
|
|
@@ -248,20 +248,20 @@ class ABS
|
|
248
248
|
conn = Http.get_conn(verbose, supported_abs_url(url))
|
249
249
|
conn.headers['X-AUTH-TOKEN'] = token if token
|
250
250
|
|
251
|
-
if continue.nil?
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
251
|
+
saved_job_id = if continue.nil?
|
252
|
+
"#{user}-#{DateTime.now.strftime('%Q')}"
|
253
|
+
else
|
254
|
+
continue
|
255
|
+
end
|
256
256
|
|
257
257
|
req_obj = {
|
258
|
-
:
|
259
|
-
:
|
260
|
-
:
|
261
|
-
:
|
262
|
-
:
|
263
|
-
}
|
264
|
-
}
|
258
|
+
resources: os_types,
|
259
|
+
job: {
|
260
|
+
id: saved_job_id,
|
261
|
+
tags: {
|
262
|
+
user: user
|
263
|
+
}
|
264
|
+
}
|
265
265
|
}
|
266
266
|
|
267
267
|
if config['vmpooler_fallback'] # optional and not available as cli flag
|
@@ -271,11 +271,12 @@ class ABS
|
|
271
271
|
end
|
272
272
|
|
273
273
|
if config['priority']
|
274
|
-
req_obj[:priority] =
|
274
|
+
req_obj[:priority] = case config['priority']
|
275
|
+
when 'high'
|
275
276
|
1
|
276
|
-
|
277
|
+
when 'medium'
|
277
278
|
2
|
278
|
-
|
279
|
+
when 'low'
|
279
280
|
3
|
280
281
|
else
|
281
282
|
config['priority'].to_i
|
@@ -291,14 +292,12 @@ class ABS
|
|
291
292
|
|
292
293
|
retries = 360
|
293
294
|
|
294
|
-
status = validate_queue_status_response(res.status, res.body,
|
295
|
+
status = validate_queue_status_response(res.status, res.body, 'Initial request', verbose)
|
295
296
|
|
296
297
|
begin
|
297
298
|
(1..retries).each do |i|
|
298
299
|
res_body = check_queue(conn, saved_job_id, req_obj, verbose)
|
299
|
-
|
300
|
-
return translated(res_body, saved_job_id)
|
301
|
-
end
|
300
|
+
return translated(res_body, saved_job_id) if res_body.is_a?(Array) # when we get a response with hostnames
|
302
301
|
|
303
302
|
sleep_seconds = 10 if i >= 10
|
304
303
|
sleep_seconds = i if i < 10
|
@@ -317,10 +316,10 @@ class ABS
|
|
317
316
|
# We should fix the ABS API to be more like the vmpooler or nspooler api, but for now
|
318
317
|
#
|
319
318
|
def self.translated(res_body, job_id)
|
320
|
-
vmpooler_formatted_body = {'job_id' => job_id}
|
319
|
+
vmpooler_formatted_body = { 'job_id' => job_id }
|
321
320
|
|
322
321
|
res_body.each do |host|
|
323
|
-
if vmpooler_formatted_body[host['type']] && vmpooler_formatted_body[host['type']]['hostname'].
|
322
|
+
if vmpooler_formatted_body[host['type']] && vmpooler_formatted_body[host['type']]['hostname'].instance_of?(Array)
|
324
323
|
vmpooler_formatted_body[host['type']]['hostname'] << host['hostname']
|
325
324
|
else
|
326
325
|
vmpooler_formatted_body[host['type']] = { 'hostname' => [host['hostname']] }
|
@@ -331,9 +330,9 @@ class ABS
|
|
331
330
|
vmpooler_formatted_body
|
332
331
|
end
|
333
332
|
|
334
|
-
def self.check_queue(conn,
|
333
|
+
def self.check_queue(conn, _job_id, req_obj, verbose)
|
335
334
|
res = conn.post 'request', req_obj.to_json
|
336
|
-
status = validate_queue_status_response(res.status, res.body,
|
335
|
+
status = validate_queue_status_response(res.status, res.body, 'Check queue request', verbose)
|
337
336
|
unless res.body.empty? || !valid_json?(res.body)
|
338
337
|
res_body = JSON.parse(res.body)
|
339
338
|
return res_body
|
@@ -353,7 +352,7 @@ class ABS
|
|
353
352
|
res.body == 'OK'
|
354
353
|
end
|
355
354
|
|
356
|
-
def self.summary(
|
355
|
+
def self.summary(_verbose, _url)
|
357
356
|
raise NoMethodError, 'summary is not defined for ABS'
|
358
357
|
end
|
359
358
|
|
@@ -405,17 +404,17 @@ class ABS
|
|
405
404
|
|
406
405
|
def self.valid_json?(json)
|
407
406
|
JSON.parse(json)
|
408
|
-
|
407
|
+
true
|
409
408
|
rescue TypeError, JSON::ParserError => e
|
410
|
-
|
409
|
+
false
|
411
410
|
end
|
412
411
|
|
413
412
|
# when missing, adds the required api/v2 in the url
|
414
413
|
def self.supported_abs_url(url)
|
415
|
-
expected_ending =
|
416
|
-
|
414
|
+
expected_ending = 'api/v2'
|
415
|
+
unless url.include?(expected_ending)
|
417
416
|
# add a slash if missing
|
418
|
-
expected_ending = "/#{expected_ending}" if url[-1] !=
|
417
|
+
expected_ending = "/#{expected_ending}" if url[-1] != '/'
|
419
418
|
url = "#{url}#{expected_ending}"
|
420
419
|
end
|
421
420
|
url
|
data/lib/vmfloaty/http.rb
CHANGED
@@ -21,13 +21,11 @@ class Http
|
|
21
21
|
|
22
22
|
url = "https://#{url}" unless url?(url)
|
23
23
|
|
24
|
-
|
24
|
+
Faraday.new(url: url, ssl: { verify: false }) do |faraday|
|
25
25
|
faraday.request :url_encoded
|
26
26
|
faraday.response :logger if verbose
|
27
27
|
faraday.adapter Faraday.default_adapter
|
28
28
|
end
|
29
|
-
|
30
|
-
conn
|
31
29
|
end
|
32
30
|
|
33
31
|
def self.get_conn_with_auth(verbose, url, user, password)
|
@@ -37,13 +35,11 @@ class Http
|
|
37
35
|
|
38
36
|
url = "https://#{url}" unless url?(url)
|
39
37
|
|
40
|
-
|
38
|
+
Faraday.new(url: url, ssl: { verify: false }) do |faraday|
|
41
39
|
faraday.request :url_encoded
|
42
40
|
faraday.request :basic_auth, user, password
|
43
41
|
faraday.response :logger if verbose
|
44
42
|
faraday.adapter Faraday.default_adapter
|
45
43
|
end
|
46
|
-
|
47
|
-
conn
|
48
44
|
end
|
49
45
|
end
|
data/lib/vmfloaty/logger.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'logger'
|
2
4
|
|
3
5
|
class FloatyLogger < ::Logger
|
@@ -19,22 +21,23 @@ class FloatyLogger < ::Logger
|
|
19
21
|
|
20
22
|
def self.setlevel=(level)
|
21
23
|
level = level.downcase
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
24
|
+
case level
|
25
|
+
when 'debug'
|
26
|
+
logger.level = ::Logger::DEBUG
|
27
|
+
when 'info'
|
28
|
+
logger.level = ::Logger::INFO
|
29
|
+
when 'error'
|
30
|
+
logger.level = ::Logger::ERROR
|
28
31
|
else
|
29
|
-
error(
|
32
|
+
error('set loglevel to debug, info or error')
|
30
33
|
end
|
31
34
|
end
|
32
35
|
|
33
36
|
def initialize
|
34
|
-
super(
|
37
|
+
super($stderr)
|
35
38
|
self.level = ::Logger::INFO
|
36
|
-
self.formatter = proc do |
|
37
|
-
|
39
|
+
self.formatter = proc do |_severity, _datetime, _progname, msg|
|
40
|
+
"#{msg}\n"
|
38
41
|
end
|
39
42
|
end
|
40
43
|
end
|
@@ -22,7 +22,7 @@ class NonstandardPooler
|
|
22
22
|
status['reserved_hosts'] || []
|
23
23
|
end
|
24
24
|
|
25
|
-
def self.retrieve(verbose, os_type, token, url, _user, _options,
|
25
|
+
def self.retrieve(verbose, os_type, token, url, _user, _options, _ondemand = nil, _continue = nil)
|
26
26
|
conn = Http.get_conn(verbose, url)
|
27
27
|
conn.headers['X-AUTH-TOKEN'] = token if token
|
28
28
|
|
@@ -46,7 +46,8 @@ class NonstandardPooler
|
|
46
46
|
raise TokenError, 'Token provided was nil; Request cannot be made to modify VM' if token.nil?
|
47
47
|
|
48
48
|
modify_hash.each do |key, _value|
|
49
|
-
raise ModifyError, "Configured service type does not support modification of #{key}" unless %i[reason
|
49
|
+
raise ModifyError, "Configured service type does not support modification of #{key}" unless %i[reason
|
50
|
+
reserved_for_reason].include? key
|
50
51
|
end
|
51
52
|
|
52
53
|
if modify_hash[:reason]
|
data/lib/vmfloaty/pooler.rb
CHANGED
@@ -12,13 +12,11 @@ class Pooler
|
|
12
12
|
response = conn.get 'vm'
|
13
13
|
response_body = JSON.parse(response.body)
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
hosts
|
15
|
+
if os_filter
|
16
|
+
response_body.select { |i| i[/#{os_filter}/] }
|
17
|
+
else
|
18
|
+
response_body
|
19
|
+
end
|
22
20
|
end
|
23
21
|
|
24
22
|
def self.list_active(verbose, url, token, _user)
|
@@ -50,7 +48,10 @@ class Pooler
|
|
50
48
|
elsif response.status == 403
|
51
49
|
raise "HTTP #{response.status}: Failed to obtain VMs from the pooler at #{url}/vm/#{os_string}. Request exceeds the configured per pool maximum. #{res_body}"
|
52
50
|
else
|
53
|
-
|
51
|
+
unless ondemand
|
52
|
+
raise "HTTP #{response.status}: Failed to obtain VMs from the pooler at #{url}/vm/#{os_string}. #{res_body}"
|
53
|
+
end
|
54
|
+
|
54
55
|
raise "HTTP #{response.status}: Failed to obtain VMs from the pooler at #{url}/ondemandvm/#{os_string}. #{res_body}"
|
55
56
|
end
|
56
57
|
end
|
@@ -63,7 +64,7 @@ class Pooler
|
|
63
64
|
FloatyLogger.info "waiting for request #{request_id} to be fulfilled"
|
64
65
|
sleep 5
|
65
66
|
end
|
66
|
-
FloatyLogger.info
|
67
|
+
FloatyLogger.info 'The request has been fulfilled'
|
67
68
|
check_ondemandvm(verbose, request_id, url)
|
68
69
|
end
|
69
70
|
|
@@ -84,8 +85,9 @@ class Pooler
|
|
84
85
|
def self.modify(verbose, url, hostname, token, modify_hash)
|
85
86
|
raise TokenError, 'Token provided was nil. Request cannot be made to modify vm' if token.nil?
|
86
87
|
|
87
|
-
modify_hash.
|
88
|
-
raise ModifyError, "Configured service type does not support modification of #{key}." unless %i[tags lifetime
|
88
|
+
modify_hash.each_key do |key|
|
89
|
+
raise ModifyError, "Configured service type does not support modification of #{key}." unless %i[tags lifetime
|
90
|
+
disk].include? key
|
89
91
|
end
|
90
92
|
|
91
93
|
conn = Http.get_conn(verbose, url)
|
@@ -120,8 +122,7 @@ class Pooler
|
|
120
122
|
|
121
123
|
response = conn.post "vm/#{hostname}/disk/#{disk}"
|
122
124
|
|
123
|
-
|
124
|
-
res_body
|
125
|
+
JSON.parse(response.body)
|
125
126
|
end
|
126
127
|
|
127
128
|
def self.delete(verbose, url, hosts, token, _user)
|
@@ -146,25 +147,21 @@ class Pooler
|
|
146
147
|
conn = Http.get_conn(verbose, url)
|
147
148
|
|
148
149
|
response = conn.get '/status'
|
149
|
-
|
150
|
-
res_body
|
150
|
+
JSON.parse(response.body)
|
151
151
|
end
|
152
152
|
|
153
153
|
def self.summary(verbose, url)
|
154
154
|
conn = Http.get_conn(verbose, url)
|
155
155
|
|
156
156
|
response = conn.get '/summary'
|
157
|
-
|
158
|
-
res_body
|
157
|
+
JSON.parse(response.body)
|
159
158
|
end
|
160
159
|
|
161
160
|
def self.query(verbose, url, hostname)
|
162
161
|
conn = Http.get_conn(verbose, url)
|
163
162
|
|
164
163
|
response = conn.get "vm/#{hostname}"
|
165
|
-
|
166
|
-
|
167
|
-
res_body
|
164
|
+
JSON.parse(response.body)
|
168
165
|
end
|
169
166
|
|
170
167
|
def self.snapshot(verbose, url, hostname, token)
|
@@ -174,8 +171,7 @@ class Pooler
|
|
174
171
|
conn.headers['X-AUTH-TOKEN'] = token
|
175
172
|
|
176
173
|
response = conn.post "vm/#{hostname}/snapshot"
|
177
|
-
|
178
|
-
res_body
|
174
|
+
JSON.parse(response.body)
|
179
175
|
end
|
180
176
|
|
181
177
|
def self.revert(verbose, url, hostname, token, snapshot_sha)
|
@@ -187,7 +183,6 @@ class Pooler
|
|
187
183
|
raise "Snapshot SHA provided was nil, could not revert #{hostname}" if snapshot_sha.nil?
|
188
184
|
|
189
185
|
response = conn.post "vm/#{hostname}/snapshot/#{snapshot_sha}"
|
190
|
-
|
191
|
-
res_body
|
186
|
+
JSON.parse(response.body)
|
192
187
|
end
|
193
188
|
end
|
data/lib/vmfloaty/service.rb
CHANGED
@@ -39,7 +39,7 @@ class Service
|
|
39
39
|
def user
|
40
40
|
unless @config['user']
|
41
41
|
FloatyLogger.info "Enter your #{@config['url']} service username:"
|
42
|
-
@config['user'] =
|
42
|
+
@config['user'] = $stdin.gets.chomp
|
43
43
|
end
|
44
44
|
@config['user']
|
45
45
|
end
|
@@ -140,8 +140,8 @@ class Service
|
|
140
140
|
# some methods do not exist for ABS, and if possible should target the Pooler service
|
141
141
|
def maybe_use_vmpooler
|
142
142
|
if @service_object == ABS # this is not an instance
|
143
|
-
|
144
|
-
FloatyLogger.info
|
143
|
+
unless silent
|
144
|
+
FloatyLogger.info 'The service in use is ABS, but the requested method should run against vmpooler directly, using fallback_vmpooler config from ~/.vmfloaty.yml'
|
145
145
|
self.silent = true
|
146
146
|
end
|
147
147
|
|