vmfloaty 0.11.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/vmfloaty/abs.rb CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'vmfloaty/errors'
4
4
  require 'vmfloaty/http'
5
+ require 'vmfloaty/utils'
5
6
  require 'faraday'
6
7
  require 'json'
7
8
 
@@ -36,39 +37,61 @@ class ABS
36
37
  # }
37
38
  # }
38
39
  #
39
-
40
40
  @active_hostnames = {}
41
41
 
42
- def self.list_active(verbose, url, _token, user)
43
- all_jobs = []
42
+ def self.list_active_job_ids(verbose, url, user)
43
+ all_job_ids = []
44
44
  @active_hostnames = {}
45
+ get_active_requests(verbose, url, user).each do |req_hash|
46
+ @active_hostnames[req_hash['request']['job']['id']] = req_hash # full hash saved for later retrieval
47
+ all_job_ids.push(req_hash['request']['job']['id'])
48
+ end
45
49
 
50
+ all_job_ids
51
+ end
52
+
53
+ def self.list_active(verbose, url, _token, user)
54
+ hosts = []
46
55
  get_active_requests(verbose, url, user).each do |req_hash|
47
- all_jobs.push(req_hash['request']['job']['id'])
48
- @active_hostnames[req_hash['request']['job']['id']] = req_hash
56
+ if req_hash.key?('allocated_resources')
57
+ req_hash['allocated_resources'].each do |onehost|
58
+ hosts.push(onehost['hostname'])
59
+ end
60
+ end
49
61
  end
50
62
 
51
- all_jobs
63
+ hosts
52
64
  end
53
65
 
54
66
  def self.get_active_requests(verbose, url, user)
55
- conn = Http.get_conn(verbose, url)
67
+ conn = Http.get_conn(verbose, supported_abs_url(url))
56
68
  res = conn.get 'status/queue'
57
- requests = JSON.parse(res.body)
69
+ if valid_json?(res.body)
70
+ requests = JSON.parse(res.body)
71
+ else
72
+ FloatyLogger.warn "Warning: couldn't parse body returned from abs/status/queue"
73
+ end
58
74
 
59
75
  ret_val = []
60
76
 
61
77
  requests.each do |req|
62
78
  next if req == 'null'
63
79
 
64
- req_hash = JSON.parse(req)
80
+ if valid_json?(req) # legacy ABS had another JSON string always-be-scheduling/pull/306
81
+ req_hash = JSON.parse(req)
82
+ elsif req.is_a?(Hash)
83
+ req_hash = req
84
+ else
85
+ FloatyLogger.warn "Warning: couldn't parse request returned from abs/status/queue"
86
+ next
87
+ end
65
88
 
66
89
  begin
67
90
  next unless user == req_hash['request']['job']['user']
68
91
 
69
92
  ret_val.push(req_hash)
70
93
  rescue NoMethodError
71
- FloatyLogger.warn "Warning: couldn't parse line returned from abs/status/queue: "
94
+ FloatyLogger.warn "Warning: couldn't parse user returned from abs/status/queue: "
72
95
  end
73
96
  end
74
97
 
@@ -82,7 +105,7 @@ class ABS
82
105
 
83
106
  def self.delete(verbose, url, hosts, token, user)
84
107
  # In ABS terms, this is a "returned" host.
85
- conn = Http.get_conn(verbose, url)
108
+ conn = Http.get_conn(verbose, supported_abs_url(url))
86
109
  conn.headers['X-AUTH-TOKEN'] = token if token
87
110
 
88
111
  FloatyLogger.info "Trying to delete hosts #{hosts}" if verbose
@@ -140,35 +163,64 @@ class ABS
140
163
 
141
164
  # List available VMs in ABS
142
165
  def self.list(verbose, url, os_filter = nil)
143
- conn = Http.get_conn(verbose, url)
166
+ conn = Http.get_conn(verbose, supported_abs_url(url))
144
167
 
145
168
  os_list = []
146
169
 
147
170
  res = conn.get 'status/platforms/vmpooler'
148
-
149
- res_body = JSON.parse(res.body)
150
- os_list << '*** VMPOOLER Pools ***'
151
- os_list += JSON.parse(res_body['vmpooler_platforms'])
171
+ if valid_json?(res.body)
172
+ res_body = JSON.parse(res.body)
173
+ if res_body.key?('vmpooler_platforms')
174
+ os_list << '*** VMPOOLER Pools ***'
175
+ if res_body['vmpooler_platforms'].is_a?(String)
176
+ os_list += JSON.parse(res_body['vmpooler_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
177
+ else
178
+ os_list += res_body['vmpooler_platforms']
179
+ end
180
+ end
181
+ end
152
182
 
153
183
  res = conn.get 'status/platforms/ondemand_vmpooler'
154
- res_body = JSON.parse(res.body)
155
- unless res_body['ondemand_vmpooler_platforms'] == '[]'
156
- os_list << ''
157
- os_list << '*** VMPOOLER ONDEMAND Pools ***'
158
- os_list += JSON.parse(res_body['ondemand_vmpooler_platforms'])
184
+ if valid_json?(res.body)
185
+ res_body = JSON.parse(res.body)
186
+ if res_body.key?('ondemand_vmpooler_platforms') && res_body['ondemand_vmpooler_platforms'] != '[]'
187
+ os_list << ''
188
+ os_list << '*** VMPOOLER ONDEMAND Pools ***'
189
+ if res_body['ondemand_vmpooler_platforms'].is_a?(String)
190
+ os_list += JSON.parse(res_body['ondemand_vmpooler_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
191
+ else
192
+ os_list += res_body['ondemand_vmpooler_platforms']
193
+ end
194
+ end
159
195
  end
160
196
 
161
197
  res = conn.get 'status/platforms/nspooler'
162
- res_body = JSON.parse(res.body)
163
- os_list << ''
164
- os_list << '*** NSPOOLER Pools ***'
165
- os_list += JSON.parse(res_body['nspooler_platforms'])
198
+ if valid_json?(res.body)
199
+ res_body = JSON.parse(res.body)
200
+ if res_body.key?('nspooler_platforms')
201
+ os_list << ''
202
+ os_list << '*** NSPOOLER Pools ***'
203
+ if res_body['nspooler_platforms'].is_a?(String)
204
+ os_list += JSON.parse(res_body['nspooler_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
205
+ else
206
+ os_list += res_body['nspooler_platforms']
207
+ end
208
+ end
209
+ end
166
210
 
167
211
  res = conn.get 'status/platforms/aws'
168
- res_body = JSON.parse(res.body)
169
- os_list << ''
170
- os_list << '*** AWS Pools ***'
171
- os_list += JSON.parse(res_body['aws_platforms'])
212
+ if valid_json?(res.body)
213
+ res_body = JSON.parse(res.body)
214
+ if res_body.key?('aws_platforms')
215
+ os_list << ''
216
+ os_list << '*** AWS Pools ***'
217
+ if res_body['aws_platforms'].is_a?(String)
218
+ os_list += JSON.parse(res_body['aws_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
219
+ else
220
+ os_list += res_body['aws_platforms']
221
+ end
222
+ end
223
+ end
172
224
 
173
225
  os_list.delete 'ok'
174
226
 
@@ -176,7 +228,7 @@ class ABS
176
228
  end
177
229
 
178
230
  # Retrieve an OS from ABS.
179
- def self.retrieve(verbose, os_types, token, url, user, options, _ondemand = nil)
231
+ def self.retrieve(verbose, os_types, token, url, user, config, _ondemand = nil, continue = nil)
180
232
  #
181
233
  # Contents of post must be like:
182
234
  #
@@ -193,10 +245,14 @@ class ABS
193
245
  # }
194
246
  # }
195
247
 
196
- conn = Http.get_conn(verbose, url)
248
+ conn = Http.get_conn(verbose, supported_abs_url(url))
197
249
  conn.headers['X-AUTH-TOKEN'] = token if token
198
250
 
199
- saved_job_id = DateTime.now.strftime('%Q')
251
+ if continue.nil?
252
+ saved_job_id = user + "-" + DateTime.now.strftime('%Q')
253
+ else
254
+ saved_job_id = continue
255
+ end
200
256
 
201
257
  req_obj = {
202
258
  :resources => os_types,
@@ -208,15 +264,21 @@ class ABS
208
264
  },
209
265
  }
210
266
 
211
- if options['priority']
212
- req_obj[:priority] = if options['priority'] == 'high'
267
+ if config['vmpooler_fallback'] # optional and not available as cli flag
268
+ vmpooler_config = Utils.get_vmpooler_service_config(config['vmpooler_fallback'])
269
+ # request with this token, on behalf of this user
270
+ req_obj[:vm_token] = vmpooler_config['token']
271
+ end
272
+
273
+ if config['priority']
274
+ req_obj[:priority] = if config['priority'] == 'high'
213
275
  1
214
- elsif options['priority'] == 'medium'
276
+ elsif config['priority'] == 'medium'
215
277
  2
216
- elsif options['priority'] == 'low'
278
+ elsif config['priority'] == 'low'
217
279
  3
218
280
  else
219
- options['priority'].to_i
281
+ config['priority'].to_i
220
282
  end
221
283
  end
222
284
 
@@ -224,22 +286,29 @@ class ABS
224
286
 
225
287
  # os_string = os_type.map { |os, num| Array(os) * num }.flatten.join('+')
226
288
  # raise MissingParamError, 'No operating systems provided to obtain.' if os_string.empty?
227
- FloatyLogger.info "Requesting VMs with job_id: #{saved_job_id}. Will retry for up to an hour."
289
+ FloatyLogger.info "Requesting VMs with job_id: #{saved_job_id} Will retry for up to an hour."
228
290
  res = conn.post 'request', req_obj.to_json
229
291
 
230
292
  retries = 360
231
293
 
232
- raise AuthError, "HTTP #{res.status}: The token provided could not authenticate to the pooler.\n#{res_body}" if res.status == 401
294
+ status = validate_queue_status_response(res.status, res.body, "Initial request", verbose)
233
295
 
234
- (1..retries).each do |i|
235
- queue_place, res_body = check_queue(conn, saved_job_id, req_obj)
236
- return translated(res_body) if res_body
296
+ begin
297
+ (1..retries).each do |i|
298
+ res_body = check_queue(conn, saved_job_id, req_obj, verbose)
299
+ if res_body && res_body.is_a?(Array) # when we get a response with hostnames
300
+ return translated(res_body, saved_job_id)
301
+ end
237
302
 
238
- sleep_seconds = 10 if i >= 10
239
- sleep_seconds = i if i < 10
240
- FloatyLogger.info "Waiting #{sleep_seconds} seconds to check if ABS request has been filled. Queue Position: #{queue_place}... (x#{i})"
303
+ sleep_seconds = 10 if i >= 10
304
+ sleep_seconds = i if i < 10
305
+ FloatyLogger.info "Waiting #{sleep_seconds}s (x#{i}) #{res_body.strip}"
241
306
 
242
- sleep(sleep_seconds)
307
+ sleep(sleep_seconds)
308
+ end
309
+ rescue SystemExit, Interrupt
310
+ FloatyLogger.info "\n\nFloaty interrupted, you can resume polling with\n1) `floaty get [same arguments] and adding the flag --continue #{saved_job_id}` or query the state of the queue via\n2) `floaty query #{saved_job_id}` or delete it via\n3) `floaty delete #{saved_job_id}`"
311
+ exit 1
243
312
  end
244
313
  nil
245
314
  end
@@ -247,8 +316,8 @@ class ABS
247
316
  #
248
317
  # We should fix the ABS API to be more like the vmpooler or nspooler api, but for now
249
318
  #
250
- def self.translated(res_body)
251
- vmpooler_formatted_body = {}
319
+ def self.translated(res_body, job_id)
320
+ vmpooler_formatted_body = {'job_id' => job_id}
252
321
 
253
322
  res_body.each do |host|
254
323
  if vmpooler_formatted_body[host['type']] && vmpooler_formatted_body[host['type']]['hostname'].class == Array
@@ -262,25 +331,22 @@ class ABS
262
331
  vmpooler_formatted_body
263
332
  end
264
333
 
265
- def self.check_queue(conn, job_id, req_obj)
266
- queue_info_res = conn.get "status/queue/info/#{job_id}"
267
- queue_info = JSON.parse(queue_info_res.body)
268
-
334
+ def self.check_queue(conn, job_id, req_obj, verbose)
269
335
  res = conn.post 'request', req_obj.to_json
270
-
271
- unless res.body.empty?
336
+ status = validate_queue_status_response(res.status, res.body, "Check queue request", verbose)
337
+ unless res.body.empty? || !valid_json?(res.body)
272
338
  res_body = JSON.parse(res.body)
273
- return queue_info['queue_place'], res_body
339
+ return res_body
274
340
  end
275
- [queue_info['queue_place'], nil]
341
+ res.body
276
342
  end
277
343
 
278
344
  def self.snapshot(_verbose, _url, _hostname, _token)
279
- FloatyLogger.info "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
345
+ raise NoMethodError, "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
280
346
  end
281
347
 
282
348
  def self.status(verbose, url)
283
- conn = Http.get_conn(verbose, url)
349
+ conn = Http.get_conn(verbose, supported_abs_url(url))
284
350
 
285
351
  res = conn.get 'status'
286
352
 
@@ -288,20 +354,24 @@ class ABS
288
354
  end
289
355
 
290
356
  def self.summary(verbose, url)
291
- conn = Http.get_conn(verbose, url)
292
-
293
- res = conn.get 'summary'
294
- JSON.parse(res.body)
357
+ raise NoMethodError, 'summary is not defined for ABS'
295
358
  end
296
359
 
297
- def self.query(verbose, url, hostname)
298
- return @active_hostnames if @active_hostnames
360
+ def self.query(verbose, url, job_id)
361
+ # return saved hostnames from the last time list_active was run
362
+ # preventing having to query the API again.
363
+ # This works as long as query is called after list_active
364
+ return @active_hostnames if @active_hostnames && !@active_hostnames.empty?
299
365
 
300
- FloatyLogger.info "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)"
301
- conn = Http.get_conn(verbose, url)
302
-
303
- res = conn.get "host/#{hostname}"
304
- JSON.parse(res.body)
366
+ # If using the cli query job_id
367
+ conn = Http.get_conn(verbose, supported_abs_url(url))
368
+ queue_info_res = conn.get "status/queue/info/#{job_id}"
369
+ if valid_json?(queue_info_res.body)
370
+ queue_info = JSON.parse(queue_info_res.body)
371
+ else
372
+ FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
373
+ end
374
+ queue_info
305
375
  end
306
376
 
307
377
  def self.modify(_verbose, _url, _hostname, _token, _modify_hash)
@@ -315,4 +385,39 @@ class ABS
315
385
  def self.revert(_verbose, _url, _hostname, _token, _snapshot_sha)
316
386
  raise NoMethodError, 'revert is not defined for ABS'
317
387
  end
388
+
389
+ # Validate the http code returned during a queue status request.
390
+ #
391
+ # Return a success message that can be displayed if the status code is
392
+ # success, otherwise raise an error.
393
+ def self.validate_queue_status_response(status_code, body, request_name, verbose)
394
+ case status_code
395
+ when 200
396
+ "#{request_name} returned success (Code 200)" if verbose
397
+ when 202
398
+ "#{request_name} returned accepted, processing (Code 202)" if verbose
399
+ when 401
400
+ raise AuthError, "HTTP #{status_code}: The token provided could not authenticate.\n#{body}"
401
+ else
402
+ raise "HTTP #{status_code}: #{request_name} request to ABS failed!\n#{body}"
403
+ end
404
+ end
405
+
406
+ def self.valid_json?(json)
407
+ JSON.parse(json)
408
+ return true
409
+ rescue TypeError, JSON::ParserError => e
410
+ return false
411
+ end
412
+
413
+ # when missing, adds the required api/v2 in the url
414
+ def self.supported_abs_url(url)
415
+ expected_ending = "api/v2"
416
+ if !url.include?(expected_ending)
417
+ # add a slash if missing
418
+ expected_ending = "/#{expected_ending}" if url[-1] != "/"
419
+ url = "#{url}#{expected_ending}"
420
+ end
421
+ url
422
+ end
318
423
  end
data/lib/vmfloaty/conf.rb CHANGED
@@ -8,7 +8,7 @@ class Conf
8
8
  begin
9
9
  conf = YAML.load_file("#{Dir.home}/.vmfloaty.yml")
10
10
  rescue StandardError
11
- FloatyLogger.warn "WARNING: There was no config file at #{Dir.home}/.vmfloaty.yml"
11
+ # ignore
12
12
  end
13
13
  conf
14
14
  end
@@ -17,6 +17,19 @@ class FloatyLogger < ::Logger
17
17
  FloatyLogger.logger.error msg
18
18
  end
19
19
 
20
+ def self.setlevel=(level)
21
+ level = level.downcase
22
+ if level == "debug"
23
+ self.logger.level = ::Logger::DEBUG
24
+ elsif level == "info"
25
+ self.logger.level = ::Logger::INFO
26
+ elsif level == "error"
27
+ self.logger.level = ::Logger::ERROR
28
+ else
29
+ error("set loglevel to debug, info or error")
30
+ end
31
+ end
32
+
20
33
  def initialize
21
34
  super(STDERR)
22
35
  self.level = ::Logger::INFO
@@ -22,7 +22,7 @@ class NonstandardPooler
22
22
  status['reserved_hosts'] || []
23
23
  end
24
24
 
25
- def self.retrieve(verbose, os_type, token, url, _user, _options)
25
+ def self.retrieve(verbose, os_type, token, url, _user, _options, ondemand = nil, _continue = nil)
26
26
  conn = Http.get_conn(verbose, url)
27
27
  conn.headers['X-AUTH-TOKEN'] = token if token
28
28
 
@@ -28,7 +28,7 @@ class Pooler
28
28
  vms
29
29
  end
30
30
 
31
- def self.retrieve(verbose, os_type, token, url, _user, _options, ondemand = nil)
31
+ def self.retrieve(verbose, os_type, token, url, _user, _options, ondemand = nil, _continue = nil)
32
32
  # NOTE:
33
33
  # Developers can use `Utils.generate_os_hash` to
34
34
  # generate the os_type param.
@@ -7,11 +7,13 @@ require 'vmfloaty/ssh'
7
7
 
8
8
  class Service
9
9
  attr_reader :config
10
+ attr_accessor :silent
10
11
 
11
12
  def initialize(options, config_hash = {})
12
13
  options ||= Commander::Command::Options.new
13
14
  @config = Utils.get_service_config config_hash, options
14
15
  @service_object = Utils.get_service_object @config['type']
16
+ @silent = false
15
17
  end
16
18
 
17
19
  def method_missing(method_name, *args, &block)
@@ -75,10 +77,10 @@ class Service
75
77
  @service_object.list_active verbose, url, token, user
76
78
  end
77
79
 
78
- def retrieve(verbose, os_types, use_token = true, ondemand = nil)
80
+ def retrieve(verbose, os_types, use_token = true, ondemand = nil, continue = nil)
79
81
  FloatyLogger.info 'Requesting a vm without a token...' unless use_token
80
82
  token_value = use_token ? token : nil
81
- @service_object.retrieve verbose, os_types, token_value, url, user, @config, ondemand
83
+ @service_object.retrieve verbose, os_types, token_value, url, user, @config, ondemand, continue
82
84
  end
83
85
 
84
86
  def wait_for_request(verbose, requestid)
@@ -103,6 +105,7 @@ class Service
103
105
  end
104
106
 
105
107
  def modify(verbose, hostname, modify_hash)
108
+ maybe_use_vmpooler
106
109
  @service_object.modify verbose, url, hostname, token, modify_hash
107
110
  end
108
111
 
@@ -115,18 +118,35 @@ class Service
115
118
  end
116
119
 
117
120
  def summary(verbose)
121
+ maybe_use_vmpooler
118
122
  @service_object.summary verbose, url
119
123
  end
120
124
 
121
125
  def snapshot(verbose, hostname)
126
+ maybe_use_vmpooler
122
127
  @service_object.snapshot verbose, url, hostname, token
123
128
  end
124
129
 
125
130
  def revert(verbose, hostname, snapshot_sha)
131
+ maybe_use_vmpooler
126
132
  @service_object.revert verbose, url, hostname, token, snapshot_sha
127
133
  end
128
134
 
129
135
  def disk(verbose, hostname, disk)
136
+ maybe_use_vmpooler
130
137
  @service_object.disk(verbose, url, hostname, token, disk)
131
138
  end
139
+
140
+ # some methods do not exist for ABS, and if possible should target the Pooler service
141
+ def maybe_use_vmpooler
142
+ if @service_object == ABS # this is not an instance
143
+ if !self.silent
144
+ FloatyLogger.info "The service in use is ABS, but the requested method should run against vmpooler directly, using fallback_vmpooler config from ~/.vmfloaty.yml"
145
+ self.silent = true
146
+ end
147
+
148
+ @config = Utils.get_vmpooler_service_config(@config['vmpooler_fallback'])
149
+ @service_object = Pooler
150
+ end
151
+ end
132
152
  end