vmfloaty 0.10.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/vmfloaty/abs.rb CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'vmfloaty/errors'
4
4
  require 'vmfloaty/http'
5
+ require 'vmfloaty/utils'
5
6
  require 'faraday'
6
7
  require 'json'
7
8
 
@@ -36,39 +37,61 @@ class ABS
36
37
  # }
37
38
  # }
38
39
  #
39
-
40
40
  @active_hostnames = {}
41
41
 
42
- def self.list_active(verbose, url, _token, user)
43
- all_jobs = []
42
+ def self.list_active_job_ids(verbose, url, user)
43
+ all_job_ids = []
44
44
  @active_hostnames = {}
45
+ get_active_requests(verbose, url, user).each do |req_hash|
46
+ @active_hostnames[req_hash['request']['job']['id']] = req_hash # full hash saved for later retrieval
47
+ all_job_ids.push(req_hash['request']['job']['id'])
48
+ end
45
49
 
50
+ all_job_ids
51
+ end
52
+
53
+ def self.list_active(verbose, url, _token, user)
54
+ hosts = []
46
55
  get_active_requests(verbose, url, user).each do |req_hash|
47
- all_jobs.push(req_hash['request']['job']['id'])
48
- @active_hostnames[req_hash['request']['job']['id']] = req_hash
56
+ if req_hash.key?('allocated_resources')
57
+ req_hash['allocated_resources'].each do |onehost|
58
+ hosts.push(onehost['hostname'])
59
+ end
60
+ end
49
61
  end
50
62
 
51
- all_jobs
63
+ hosts
52
64
  end
53
65
 
54
66
  def self.get_active_requests(verbose, url, user)
55
- conn = Http.get_conn(verbose, url)
67
+ conn = Http.get_conn(verbose, supported_abs_url(url))
56
68
  res = conn.get 'status/queue'
57
- requests = JSON.parse(res.body)
69
+ if valid_json?(res.body)
70
+ requests = JSON.parse(res.body)
71
+ else
72
+ FloatyLogger.warn "Warning: couldn't parse body returned from abs/status/queue"
73
+ end
58
74
 
59
75
  ret_val = []
60
76
 
61
77
  requests.each do |req|
62
78
  next if req == 'null'
63
79
 
64
- req_hash = JSON.parse(req)
80
+ if valid_json?(req) # legacy ABS had another JSON string always-be-scheduling/pull/306
81
+ req_hash = JSON.parse(req)
82
+ elsif req.is_a?(Hash)
83
+ req_hash = req
84
+ else
85
+ FloatyLogger.warn "Warning: couldn't parse request returned from abs/status/queue"
86
+ next
87
+ end
65
88
 
66
89
  begin
67
90
  next unless user == req_hash['request']['job']['user']
68
91
 
69
92
  ret_val.push(req_hash)
70
93
  rescue NoMethodError
71
- puts "Warning: couldn't parse line returned from abs/status/queue: ".yellow
94
+ FloatyLogger.warn "Warning: couldn't parse user returned from abs/status/queue: "
72
95
  end
73
96
  end
74
97
 
@@ -82,10 +105,10 @@ class ABS
82
105
 
83
106
  def self.delete(verbose, url, hosts, token, user)
84
107
  # In ABS terms, this is a "returned" host.
85
- conn = Http.get_conn(verbose, url)
108
+ conn = Http.get_conn(verbose, supported_abs_url(url))
86
109
  conn.headers['X-AUTH-TOKEN'] = token if token
87
110
 
88
- puts "Trying to delete hosts #{hosts}" if verbose
111
+ FloatyLogger.info "Trying to delete hosts #{hosts}" if verbose
89
112
  requests = get_active_requests(verbose, url, user)
90
113
 
91
114
  jobs_to_delete = []
@@ -113,7 +136,7 @@ class ABS
113
136
  }
114
137
  jobs_to_delete.push(req_hash)
115
138
  else
116
- puts "When using ABS you must delete all vms that you requested at the same time: Can't delete #{req_hash['request']['job']['id']}: #{hosts} does not include all of #{req_hash['allocated_resources']}"
139
+ FloatyLogger.info "When using ABS you must delete all vms that you requested at the same time: Can't delete #{req_hash['request']['job']['id']}: #{hosts} does not include all of #{req_hash['allocated_resources']}"
117
140
  end
118
141
  end
119
142
  end
@@ -127,7 +150,7 @@ class ABS
127
150
  'hosts' => job['allocated_resources'],
128
151
  }
129
152
 
130
- puts "Deleting #{req_obj}" if verbose
153
+ FloatyLogger.info "Deleting #{req_obj}" if verbose
131
154
 
132
155
  return_result = conn.post 'return', req_obj.to_json
133
156
  req_obj['hosts'].each do |host|
@@ -140,35 +163,64 @@ class ABS
140
163
 
141
164
  # List available VMs in ABS
142
165
  def self.list(verbose, url, os_filter = nil)
143
- conn = Http.get_conn(verbose, url)
166
+ conn = Http.get_conn(verbose, supported_abs_url(url))
144
167
 
145
168
  os_list = []
146
169
 
147
170
  res = conn.get 'status/platforms/vmpooler'
148
-
149
- res_body = JSON.parse(res.body)
150
- os_list << '*** VMPOOLER Pools ***'
151
- os_list += JSON.parse(res_body['vmpooler_platforms'])
171
+ if valid_json?(res.body)
172
+ res_body = JSON.parse(res.body)
173
+ if res_body.key?('vmpooler_platforms')
174
+ os_list << '*** VMPOOLER Pools ***'
175
+ if res_body['vmpooler_platforms'].is_a?(String)
176
+ os_list += JSON.parse(res_body['vmpooler_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
177
+ else
178
+ os_list += res_body['vmpooler_platforms']
179
+ end
180
+ end
181
+ end
152
182
 
153
183
  res = conn.get 'status/platforms/ondemand_vmpooler'
154
- res_body = JSON.parse(res.body)
155
- unless res_body['ondemand_vmpooler_platforms'] == '[]'
156
- os_list << ''
157
- os_list << '*** VMPOOLER ONDEMAND Pools ***'
158
- os_list += JSON.parse(res_body['ondemand_vmpooler_platforms'])
184
+ if valid_json?(res.body)
185
+ res_body = JSON.parse(res.body)
186
+ if res_body.key?('ondemand_vmpooler_platforms') && res_body['ondemand_vmpooler_platforms'] != '[]'
187
+ os_list << ''
188
+ os_list << '*** VMPOOLER ONDEMAND Pools ***'
189
+ if res_body['ondemand_vmpooler_platforms'].is_a?(String)
190
+ os_list += JSON.parse(res_body['ondemand_vmpooler_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
191
+ else
192
+ os_list += res_body['ondemand_vmpooler_platforms']
193
+ end
194
+ end
159
195
  end
160
196
 
161
197
  res = conn.get 'status/platforms/nspooler'
162
- res_body = JSON.parse(res.body)
163
- os_list << ''
164
- os_list << '*** NSPOOLER Pools ***'
165
- os_list += JSON.parse(res_body['nspooler_platforms'])
198
+ if valid_json?(res.body)
199
+ res_body = JSON.parse(res.body)
200
+ if res_body.key?('nspooler_platforms')
201
+ os_list << ''
202
+ os_list << '*** NSPOOLER Pools ***'
203
+ if res_body['nspooler_platforms'].is_a?(String)
204
+ os_list += JSON.parse(res_body['nspooler_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
205
+ else
206
+ os_list += res_body['nspooler_platforms']
207
+ end
208
+ end
209
+ end
166
210
 
167
211
  res = conn.get 'status/platforms/aws'
168
- res_body = JSON.parse(res.body)
169
- os_list << ''
170
- os_list << '*** AWS Pools ***'
171
- os_list += JSON.parse(res_body['aws_platforms'])
212
+ if valid_json?(res.body)
213
+ res_body = JSON.parse(res.body)
214
+ if res_body.key?('aws_platforms')
215
+ os_list << ''
216
+ os_list << '*** AWS Pools ***'
217
+ if res_body['aws_platforms'].is_a?(String)
218
+ os_list += JSON.parse(res_body['aws_platforms']) # legacy ABS had another JSON string always-be-scheduling/pull/306
219
+ else
220
+ os_list += res_body['aws_platforms']
221
+ end
222
+ end
223
+ end
172
224
 
173
225
  os_list.delete 'ok'
174
226
 
@@ -176,7 +228,7 @@ class ABS
176
228
  end
177
229
 
178
230
  # Retrieve an OS from ABS.
179
- def self.retrieve(verbose, os_types, token, url, user, options, _ondemand = nil)
231
+ def self.retrieve(verbose, os_types, token, url, user, config, _ondemand = nil, continue = nil)
180
232
  #
181
233
  # Contents of post must be like:
182
234
  #
@@ -193,10 +245,14 @@ class ABS
193
245
  # }
194
246
  # }
195
247
 
196
- conn = Http.get_conn(verbose, url)
248
+ conn = Http.get_conn(verbose, supported_abs_url(url))
197
249
  conn.headers['X-AUTH-TOKEN'] = token if token
198
250
 
199
- saved_job_id = DateTime.now.strftime('%Q')
251
+ if continue.nil?
252
+ saved_job_id = user + "-" + DateTime.now.strftime('%Q')
253
+ else
254
+ saved_job_id = continue
255
+ end
200
256
 
201
257
  req_obj = {
202
258
  :resources => os_types,
@@ -208,38 +264,51 @@ class ABS
208
264
  },
209
265
  }
210
266
 
211
- if options['priority']
212
- req_obj[:priority] = if options['priority'] == 'high'
267
+ if config['vmpooler_fallback'] # optional and not available as cli flag
268
+ vmpooler_config = Utils.get_vmpooler_service_config(config['vmpooler_fallback'])
269
+ # request with this token, on behalf of this user
270
+ req_obj[:vm_token] = vmpooler_config['token']
271
+ end
272
+
273
+ if config['priority']
274
+ req_obj[:priority] = if config['priority'] == 'high'
213
275
  1
214
- elsif options['priority'] == 'medium'
276
+ elsif config['priority'] == 'medium'
215
277
  2
216
- elsif options['priority'] == 'low'
278
+ elsif config['priority'] == 'low'
217
279
  3
218
280
  else
219
- options['priority'].to_i
281
+ config['priority'].to_i
220
282
  end
221
283
  end
222
284
 
223
- puts "Posting to ABS #{req_obj.to_json}" if verbose
285
+ FloatyLogger.info "Posting to ABS #{req_obj.to_json}" if verbose
224
286
 
225
287
  # os_string = os_type.map { |os, num| Array(os) * num }.flatten.join('+')
226
288
  # raise MissingParamError, 'No operating systems provided to obtain.' if os_string.empty?
227
- puts "Requesting VMs with job_id: #{saved_job_id}. Will retry for up to an hour."
289
+ FloatyLogger.info "Requesting VMs with job_id: #{saved_job_id} Will retry for up to an hour."
228
290
  res = conn.post 'request', req_obj.to_json
229
291
 
230
292
  retries = 360
231
293
 
232
- raise AuthError, "HTTP #{res.status}: The token provided could not authenticate to the pooler.\n#{res_body}" if res.status == 401
294
+ status = validate_queue_status_response(res.status, res.body, "Initial request", verbose)
233
295
 
234
- (1..retries).each do |i|
235
- queue_place, res_body = check_queue(conn, saved_job_id, req_obj)
236
- return translated(res_body) if res_body
296
+ begin
297
+ (1..retries).each do |i|
298
+ res_body = check_queue(conn, saved_job_id, req_obj, verbose)
299
+ if res_body && res_body.is_a?(Array) # when we get a response with hostnames
300
+ return translated(res_body, saved_job_id)
301
+ end
237
302
 
238
- sleep_seconds = 10 if i >= 10
239
- sleep_seconds = i if i < 10
240
- puts "Waiting #{sleep_seconds} seconds to check if ABS request has been filled. Queue Position: #{queue_place}... (x#{i})"
303
+ sleep_seconds = 10 if i >= 10
304
+ sleep_seconds = i if i < 10
305
+ FloatyLogger.info "Waiting #{sleep_seconds}s (x#{i}) #{res_body.strip}"
241
306
 
242
- sleep(sleep_seconds)
307
+ sleep(sleep_seconds)
308
+ end
309
+ rescue SystemExit, Interrupt
310
+ FloatyLogger.info "\n\nFloaty interrupted, you can resume polling with\n1) `floaty get [same arguments] and adding the flag --continue #{saved_job_id}` or query the state of the queue via\n2) `floaty query #{saved_job_id}` or delete it via\n3) `floaty delete #{saved_job_id}`"
311
+ exit 1
243
312
  end
244
313
  nil
245
314
  end
@@ -247,8 +316,8 @@ class ABS
247
316
  #
248
317
  # We should fix the ABS API to be more like the vmpooler or nspooler api, but for now
249
318
  #
250
- def self.translated(res_body)
251
- vmpooler_formatted_body = {}
319
+ def self.translated(res_body, job_id)
320
+ vmpooler_formatted_body = {'job_id' => job_id}
252
321
 
253
322
  res_body.each do |host|
254
323
  if vmpooler_formatted_body[host['type']] && vmpooler_formatted_body[host['type']]['hostname'].class == Array
@@ -262,25 +331,22 @@ class ABS
262
331
  vmpooler_formatted_body
263
332
  end
264
333
 
265
- def self.check_queue(conn, job_id, req_obj)
266
- queue_info_res = conn.get "status/queue/info/#{job_id}"
267
- queue_info = JSON.parse(queue_info_res.body)
268
-
334
+ def self.check_queue(conn, job_id, req_obj, verbose)
269
335
  res = conn.post 'request', req_obj.to_json
270
-
271
- unless res.body.empty?
336
+ status = validate_queue_status_response(res.status, res.body, "Check queue request", verbose)
337
+ unless res.body.empty? || !valid_json?(res.body)
272
338
  res_body = JSON.parse(res.body)
273
- return queue_info['queue_place'], res_body
339
+ return res_body
274
340
  end
275
- [queue_info['queue_place'], nil]
341
+ res.body
276
342
  end
277
343
 
278
344
  def self.snapshot(_verbose, _url, _hostname, _token)
279
- puts "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
345
+ raise NoMethodError, "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
280
346
  end
281
347
 
282
348
  def self.status(verbose, url)
283
- conn = Http.get_conn(verbose, url)
349
+ conn = Http.get_conn(verbose, supported_abs_url(url))
284
350
 
285
351
  res = conn.get 'status'
286
352
 
@@ -288,20 +354,24 @@ class ABS
288
354
  end
289
355
 
290
356
  def self.summary(verbose, url)
291
- conn = Http.get_conn(verbose, url)
292
-
293
- res = conn.get 'summary'
294
- JSON.parse(res.body)
357
+ raise NoMethodError, 'summary is not defined for ABS'
295
358
  end
296
359
 
297
- def self.query(verbose, url, hostname)
298
- return @active_hostnames if @active_hostnames
360
+ def self.query(verbose, url, job_id)
361
+ # return saved hostnames from the last time list_active was run
362
+ # preventing having to query the API again.
363
+ # This works as long as query is called after list_active
364
+ return @active_hostnames if @active_hostnames && !@active_hostnames.empty?
299
365
 
300
- puts "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)"
301
- conn = Http.get_conn(verbose, url)
302
-
303
- res = conn.get "host/#{hostname}"
304
- JSON.parse(res.body)
366
+ # If using the cli query job_id
367
+ conn = Http.get_conn(verbose, supported_abs_url(url))
368
+ queue_info_res = conn.get "status/queue/info/#{job_id}"
369
+ if valid_json?(queue_info_res.body)
370
+ queue_info = JSON.parse(queue_info_res.body)
371
+ else
372
+ FloatyLogger.warn "Could not parse the status/queue/info/#{job_id}"
373
+ end
374
+ queue_info
305
375
  end
306
376
 
307
377
  def self.modify(_verbose, _url, _hostname, _token, _modify_hash)
@@ -315,4 +385,39 @@ class ABS
315
385
  def self.revert(_verbose, _url, _hostname, _token, _snapshot_sha)
316
386
  raise NoMethodError, 'revert is not defined for ABS'
317
387
  end
388
+
389
+ # Validate the http code returned during a queue status request.
390
+ #
391
+ # Return a success message that can be displayed if the status code is
392
+ # success, otherwise raise an error.
393
+ def self.validate_queue_status_response(status_code, body, request_name, verbose)
394
+ case status_code
395
+ when 200
396
+ "#{request_name} returned success (Code 200)" if verbose
397
+ when 202
398
+ "#{request_name} returned accepted, processing (Code 202)" if verbose
399
+ when 401
400
+ raise AuthError, "HTTP #{status_code}: The token provided could not authenticate.\n#{body}"
401
+ else
402
+ raise "HTTP #{status_code}: #{request_name} request to ABS failed!\n#{body}"
403
+ end
404
+ end
405
+
406
+ def self.valid_json?(json)
407
+ JSON.parse(json)
408
+ return true
409
+ rescue TypeError, JSON::ParserError => e
410
+ return false
411
+ end
412
+
413
+ # when missing, adds the required api/v2 in the url
414
+ def self.supported_abs_url(url)
415
+ expected_ending = "api/v2"
416
+ if !url.include?(expected_ending)
417
+ # add a slash if missing
418
+ expected_ending = "/#{expected_ending}" if url[-1] != "/"
419
+ url = "#{url}#{expected_ending}"
420
+ end
421
+ url
422
+ end
318
423
  end
data/lib/vmfloaty/conf.rb CHANGED
@@ -8,7 +8,7 @@ class Conf
8
8
  begin
9
9
  conf = YAML.load_file("#{Dir.home}/.vmfloaty.yml")
10
10
  rescue StandardError
11
- STDERR.puts "WARNING: There was no config file at #{Dir.home}/.vmfloaty.yml"
11
+ # ignore
12
12
  end
13
13
  conf
14
14
  end
@@ -0,0 +1,40 @@
1
+ require 'logger'
2
+
3
+ class FloatyLogger < ::Logger
4
+ def self.logger
5
+ @@logger ||= FloatyLogger.new
6
+ end
7
+
8
+ def self.info(msg)
9
+ FloatyLogger.logger.info msg
10
+ end
11
+
12
+ def self.warn(msg)
13
+ FloatyLogger.logger.warn msg
14
+ end
15
+
16
+ def self.error(msg)
17
+ FloatyLogger.logger.error msg
18
+ end
19
+
20
+ def self.setlevel=(level)
21
+ level = level.downcase
22
+ if level == "debug"
23
+ self.logger.level = ::Logger::DEBUG
24
+ elsif level == "info"
25
+ self.logger.level = ::Logger::INFO
26
+ elsif level == "error"
27
+ self.logger.level = ::Logger::ERROR
28
+ else
29
+ error("set loglevel to debug, info or error")
30
+ end
31
+ end
32
+
33
+ def initialize
34
+ super(STDERR)
35
+ self.level = ::Logger::INFO
36
+ self.formatter = proc do |severity, datetime, progname, msg|
37
+ "#{msg}\n"
38
+ end
39
+ end
40
+ end