vmfloaty 0.8.2 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 04b7261aa013260aba635e266b003d845e4a2c58
-  data.tar.gz: 83bbc1c12ace45c6a1ea288121762f13d3b841ae
+SHA256:
+  metadata.gz: 7b08c0be5bb4c8a57ce706db190b37bdf89532e30964aca13bd53dc4b2649d46
+  data.tar.gz: aeaeb37cde837b342ddd438b6e3c389e9e84b12fd6caa36a40fcd87ff8b0427b
 SHA512:
-  metadata.gz: 4e38aedb270708b75d9464744b73b507a6dfdab100567a079287b6b06aa0ac57fabaa03e0e18761c5d1b21e2921f15fbd36ab5beeddcbf3266cac2981d90931c
-  data.tar.gz: 04ddaf46164b38bf84df47ee8d5a0e96d87b6be5d64941ee72a075e5264511c7556b8fdd85a2875fadfea37820cfbc307adc6dd46eb32621c1cb552d70cbec94
+  metadata.gz: d7bb8b68a6dcc391184c910883e4d05c1d7c8580dd7d969f141959fdd46b0e4a9ef4c7ec141a3d4526cab557923d80d0db62e8ae11e8fd76120d2d0e5367fde8
+  data.tar.gz: d0bb6b6ff8fd727a110814ebd3836dcd344b30c0e0fb5791ac4485ed0fafe2d715d00173e7d874ec9a41488eaecd0709d1ce53a8c5162419e3285912a1746b2d

data/README.md

@@ -53,7 +53,7 @@ $ floaty --help
 Grabbing a token for authenticated pooler requests:
 
 ```
-floaty token get --user username --url https://vmpooler.mycompany.net/api/v1
+floaty token get --user username --url https://vmpooler.example.net/api/v1
 ```
 
 This command will then ask you to log in. If successful, it will return a token that you can save either in a dotfile or use with other cli commands.
@@ -61,18 +61,18 @@ This command will then ask you to log in. If successful, it will return a token
 Grabbing vms:
 
 ```
-floaty get centos-7-x86_64=2 debian-7-x86_64 windows-10=3 --token mytokenstring --url https://vmpooler.mycompany.net/api/v1
+floaty get centos-7-x86_64=2 debian-7-x86_64 windows-10=3 --token mytokenstring --url https://vmpooler.example.net/api/v1
 ```
 
 ### vmfloaty dotfile
 
-If you do not wish to continuely specify various config options with the cli, you can have a dotfile in your home directory for some defaults. For example:
+If you do not wish to continually specify various config options with the cli, you can have a dotfile in your home directory for some defaults. For example:
 
 #### Basic configuration
 
 ```yaml
 # file at /Users/me/.vmfloaty.yml
-url: 'https://vmpooler.mycompany.net/api/v1'
+url: 'https://vmpooler.example.net/api/v1'
 user: 'brian'
 token: 'tokenstring'
 ```
@@ -90,10 +90,10 @@ To configure multiple services, you can set up your `~/.vmfloaty.yml` config fil
 user: 'brian'
 services:
   main:
-    url: 'https://vmpooler.mycompany.net/api/v1'
+    url: 'https://vmpooler.example.net/api/v1'
     token: 'tokenstring'
   alternate:
-    url: 'https://vmpooler.alternate.net/api/v1'
+    url: 'https://vmpooler.example.com/api/v1'
     token: 'alternate-tokenstring'
 ```
 
@@ -125,12 +125,17 @@ vmfloaty is capable of working with Puppet's [nonstandard pooler](https://github
 user: 'brian'
 services:
   vm:
-    url: 'https://vmpooler.mycompany.net/api/v1'
+    url: 'https://vmpooler.example.net/api/v1'
     token: 'tokenstring'
   ns:
-    url: 'https://nspooler.mycompany.net/api/v1'
+    url: 'https://nspooler.example.net/api/v1'
     token: 'nspooler-tokenstring'
     type: 'nonstandard'  # <-- 'type' is necessary for any non-vmpooler service
+  abs:
+    url: 'https://abs.example.net/'
+    token: 'abs-tokenstring'
+    type: 'abs'  # <-- 'type' is necessary for any non-vmpooler service
+
 ```
 
 With this configuration, you could list available OS types from nspooler like this:

data/bin/floaty

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
 
 require 'vmfloaty'
 

data/lib/vmfloaty.rb

@@ -1,4 +1,4 @@
-#!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'rubygems'
 require 'commander'
@@ -17,7 +17,7 @@ require 'vmfloaty/ssh'
 class Vmfloaty
   include Commander::Methods
 
-  def run
+  def run # rubocop:disable Metrics/AbcSize
     program :version, Vmfloaty::VERSION
     program :description, 'A CLI helper tool for Puppet Labs VM poolers to help you stay afloat'
 
@@ -33,6 +33,7 @@ class Vmfloaty
       c.option '--user STRING', String, 'User to authenticate with'
       c.option '--url STRING', String, 'URL of pooler service'
       c.option '--token STRING', String, 'Token for pooler service'
+      c.option '--priority STRING', 'Priority for supported backends(ABS) (High(1), Medium(2), Low(3))'
       c.option '--notoken', 'Makes a request without a token'
       c.option '--force', 'Forces vmfloaty to get requested vms'
       c.option '--json', 'Prints retrieved vms in JSON format'
@@ -43,22 +44,22 @@ class Vmfloaty
         force = options.force
 
         if args.empty?
-          STDERR.puts "No operating systems provided to obtain. See `floaty get --help` for more information on how to get VMs."
+          STDERR.puts 'No operating systems provided to obtain. See `floaty get --help` for more information on how to get VMs.'
           exit 1
         end
 
         os_types = Utils.generate_os_hash(args)
 
         max_pool_request = 5
-        large_pool_requests = os_types.select{|_,v| v > max_pool_request}
-        if ! large_pool_requests.empty? and ! force
+        large_pool_requests = os_types.select { |_, v| v > max_pool_request }
+        if !large_pool_requests.empty? && !force
           STDERR.puts "Requesting vms over #{max_pool_request} requires a --force flag."
-          STDERR.puts "Try again with `floaty get --force`"
+          STDERR.puts 'Try again with `floaty get --force`'
           exit 1
         end
 
         if os_types.empty?
-          STDERR.puts "No operating systems provided to obtain. See `floaty get --help` for more information on how to get VMs."
+          STDERR.puts 'No operating systems provided to obtain. See `floaty get --help` for more information on how to get VMs.'
           exit 1
         end
 
@@ -84,6 +85,7 @@ class Vmfloaty
       c.option '--url STRING', String, 'URL of pooler service'
       c.action do |args, options|
         verbose = options.verbose || config['verbose']
+
         service = Service.new(options, config)
         filter = args[0]
 
@@ -143,18 +145,18 @@ class Vmfloaty
         hostname = args[0]
         modify_all = options.all
 
-        if hostname.nil? and !modify_all
-          STDERR.puts "ERROR: Provide a hostname or specify --all."
+        if hostname.nil? && !modify_all
+          STDERR.puts 'ERROR: Provide a hostname or specify --all.'
           exit 1
         end
-        running_vms = modify_all ? service.list_active(verbose) : hostname.split(",")
+        running_vms = modify_all ? service.list_active(verbose) : hostname.split(',')
 
         tags = options.tags ? JSON.parse(options.tags) : nil
         modify_hash = {
-            lifetime: options.lifetime,
-            disk: options.disk,
-            tags: tags,
-            reason: options.reason
+          :lifetime => options.lifetime,
+          :disk     => options.disk,
+          :tags     => tags,
+          :reason   => options.reason,
         }
         modify_hash.delete_if { |_, value| value.nil? }
 
@@ -171,11 +173,11 @@ class Vmfloaty
           end
           if ok
             if modify_all
-              puts "Successfully modified all VMs."
+              puts 'Successfully modified all VMs.'
             else
               puts "Successfully modified VM #{hostname}."
             end
-            puts "Use `floaty list --active` to see the results."
+            puts 'Use `floaty list --active` to see the results.'
           end
         end
       end
@@ -205,15 +207,13 @@ class Vmfloaty
         if delete_all
           running_vms = service.list_active(verbose)
           if running_vms.empty?
-            STDERR.puts "You have no running VMs."
+            STDERR.puts 'You have no running VMs.'
           else
             Utils.pretty_print_hosts(verbose, service, running_vms)
             # Confirm deletion
             puts
             confirmed = true
-            unless force
-              confirmed = agree('Delete all these VMs? [y/N]')
-            end
+            confirmed = agree('Delete all these VMs? [y/N]') unless force
             if confirmed
               response = service.delete(verbose, running_vms)
               response.each do |hostname, result|
@@ -236,7 +236,7 @@ class Vmfloaty
             end
           end
         else
-          STDERR.puts "You did not provide any hosts to delete"
+          STDERR.puts 'You did not provide any hosts to delete'
           exit 1
         end
 
@@ -302,9 +302,7 @@ class Vmfloaty
         hostname = args[0]
         snapshot_sha = args[1] || options.snapshot
 
-        if args[1] && options.snapshot
-          STDERR.puts "Two snapshot arguments were given....using snapshot #{snapshot_sha}"
-        end
+        STDERR.puts "Two snapshot arguments were given....using snapshot #{snapshot_sha}" if args[1] && options.snapshot
 
         begin
           revert_req = service.revert(verbose, hostname, snapshot_sha)
@@ -372,25 +370,23 @@ class Vmfloaty
 
         begin
           case action
-            when 'get'
-              token = service.get_new_token(verbose)
-              puts token
-            when 'delete'
-              result = service.delete_token(verbose, options.token)
-              puts result
-            when 'status'
-              token_value = options.token
-              if token_value.nil?
-                token_value = args[1]
-              end
-              status = service.token_status(verbose, token_value)
-              puts status
-            when nil
-              STDERR.puts 'No action provided'
-              exit 1
-            else
-              STDERR.puts "Unknown action: #{action}"
-              exit 1
+          when 'get'
+            token = service.get_new_token(verbose)
+            puts token
+          when 'delete'
+            result = service.delete_token(verbose, options.token)
+            puts result
+          when 'status'
+            token_value = options.token
+            token_value = args[1] if token_value.nil?
+            status = service.token_status(verbose, token_value)
+            puts status
+          when nil
+            STDERR.puts 'No action provided'
+            exit 1
+          else
+            STDERR.puts "Unknown action: #{action}"
+            exit 1
           end
         rescue TokenError => e
           STDERR.puts e
@@ -417,15 +413,13 @@ class Vmfloaty
         use_token = !options.notoken
 
         if args.empty?
-          STDERR.puts "No operating systems provided to obtain. See `floaty ssh --help` for more information on how to get VMs."
+          STDERR.puts 'No operating systems provided to obtain. See `floaty ssh --help` for more information on how to get VMs.'
           exit 1
         end
 
         host_os = args.first
 
-        if args.length > 1
-          STDERR.puts "Can't ssh to multiple hosts; Using #{host_os} only..."
-        end
+        STDERR.puts "Can't ssh to multiple hosts; Using #{host_os} only..." if args.length > 1
 
         service.ssh(verbose, host_os, use_token)
         exit 0
@@ -435,13 +429,13 @@ class Vmfloaty
     command :completion do |c|
       c.syntax = 'floaty completion [options]'
       c.summary = 'Outputs path to completion script'
-      c.description = Utils.strip_heredoc(<<-EOF)
+      c.description = Utils.strip_heredoc(<<-DESCRIPTION)
         Outputs path to a completion script for the specified shell (or 'bash' if not specified). This makes it easy to add the completion script to your profile:
 
           source $(floaty completion --shell bash)
 
         This subcommand will exit non-zero with an error message if no completion script is available for the requested shell.
-      EOF
+      DESCRIPTION
       c.example 'Gets path to bash tab completion script', 'floaty completion --shell bash'
       c.option '--shell STRING', String, 'Shell to request completion script for'
       c.action do |_, options|

data/lib/vmfloaty/abs.rb

@@ -0,0 +1,293 @@
+# frozen_string_literal: true
+
+require 'vmfloaty/errors'
+require 'vmfloaty/http'
+require 'faraday'
+require 'json'
+
+class ABS
+  # List active VMs in ABS
+  # This is what a job request looks like:
+  # {
+  #   "state":"filled",
+  #   "last_processed":"2019-10-31 20:59:33 +0000",
+  #   "allocated_resources": [
+  #     {
+  #       "hostname":"h3oyntawjm7xdch.delivery.puppetlabs.net",
+  #       "type":"centos-7.2-tmpfs-x86_64",
+  #       "engine":"vmpooler"}
+  #     ],
+  #   "audit_log":{
+  #     "2019-10-30 20:33:12 +0000":"Allocated h3oyntawjm7xdch.delivery.puppetlabs.net for job 1572467589"
+  #   },
+  #   "request":{
+  #     "resources":{
+  #       "centos-7.2-tmpfs-x86_64":1
+  #     },
+  #   "job": {
+  #     "id":1572467589,
+  #     "tags": {
+  #           "user":"mikker",
+  #           "url_string":"floaty://mikker/1572467589"
+  # },
+  #           "user":"mikker",
+  #           "time-received":1572467589
+  #         }
+  #       }
+  #     }
+  #
+
+  @active_hostnames = {}
+
+  def self.list_active(verbose, url, _token, user)
+    all_jobs = []
+    @active_hostnames = {}
+
+    get_active_requests(verbose, url, user).each do |req_hash|
+      all_jobs.push(req_hash['request']['job']['id'])
+      @active_hostnames[req_hash['request']['job']['id']] = req_hash
+    end
+
+    all_jobs
+  end
+
+  def self.get_active_requests(verbose, url, user)
+    conn = Http.get_conn(verbose, url)
+    res = conn.get 'status/queue'
+    requests = JSON.parse(res.body)
+
+    ret_val = []
+
+    requests.each do |req|
+      next if req == 'null'
+
+      req_hash = JSON.parse(req)
+
+      begin
+        next unless user == req_hash['request']['job']['user']
+
+        ret_val.push(req_hash)
+      rescue NoMethodError
+        puts "Warning: couldn't parse line returned from abs/status/queue: ".yellow
+      end
+    end
+
+    ret_val
+  end
+
+  def self.all_job_resources_accounted_for(allocated_resources, hosts)
+    allocated_host_list = allocated_resources.map { |ar| ar['hostname'] }
+    (allocated_host_list - hosts).empty?
+  end
+
+  def self.delete(verbose, url, hosts, token, user)
+    # In ABS terms, this is a "returned" host.
+    conn = Http.get_conn(verbose, url)
+    conn.headers['X-AUTH-TOKEN'] = token if token
+
+    puts "Trying to delete hosts #{hosts}" if verbose
+    requests = get_active_requests(verbose, url, user)
+
+    jobs_to_delete = []
+
+    ret_status = {}
+    hosts.each do |host|
+      ret_status[host] = {
+        'ok' => false,
+      }
+    end
+
+    requests.each do |req_hash|
+      next unless req_hash['state'] == 'allocated' || req_hash['state'] == 'filled'
+
+      req_hash['allocated_resources'].each do |vm_name, _i|
+        if hosts.include? vm_name['hostname']
+          if all_job_resources_accounted_for(req_hash['allocated_resources'], hosts)
+            ret_status[vm_name['hostname']] = {
+              'ok' => true,
+            }
+            jobs_to_delete.push(req_hash)
+          else
+            puts "When using ABS you must delete all vms that you requested at the same time: Can't delete #{req_hash['request']['job']['id']}: #{hosts} does not include all of #{req_hash['allocated_resources']}"
+          end
+        end
+      end
+    end
+
+    response_body = {}
+
+    jobs_to_delete.each do |job|
+      req_obj = {
+        'job_id' => job['request']['job']['id'],
+        'hosts'  => job['allocated_resources'],
+      }
+
+      puts "Deleting #{req_obj}" if verbose
+
+      return_result = conn.post 'return', req_obj.to_json
+      req_obj['hosts'].each do |host|
+        response_body[host['hostname']] = { 'ok' => true } if return_result.body == 'OK'
+      end
+    end
+
+    response_body
+  end
+
+  # List available VMs in ABS
+  def self.list(verbose, url, os_filter = nil)
+    conn = Http.get_conn(verbose, url)
+
+    os_list = []
+
+    res = conn.get 'status/platforms/vmpooler'
+
+    res_body = JSON.parse(res.body)
+    os_list << '*** VMPOOLER Pools ***'
+    os_list += JSON.parse(res_body['vmpooler_platforms'])
+
+    res = conn.get 'status/platforms/nspooler'
+    res_body = JSON.parse(res.body)
+    os_list << ''
+    os_list << '*** NSPOOLER Pools ***'
+    os_list += JSON.parse(res_body['nspooler_platforms'])
+
+    res = conn.get 'status/platforms/aws'
+    res_body = JSON.parse(res.body)
+    os_list << ''
+    os_list << '*** AWS Pools ***'
+    os_list += JSON.parse(res_body['aws_platforms'])
+
+    os_list.delete 'ok'
+
+    os_filter ? os_list.select { |i| i[/#{os_filter}/] } : os_list
+  end
+
+  # Retrieve an OS from ABS.
+  def self.retrieve(verbose, os_types, token, url, user, options)
+    #
+    # Contents of post must be like:
+    #
+    # {
+    #   "resources": {
+    #     "centos-7-i386": 1,
+    #     "ubuntu-1404-x86_64": 2
+    #   },
+    #   "job": {
+    #     "id": "12345",
+    #     "tags": {
+    #       "user": "username",
+    #     }
+    #   }
+    # }
+
+    conn = Http.get_conn(verbose, url)
+    conn.headers['X-AUTH-TOKEN'] = token if token
+
+    saved_job_id = DateTime.now.strftime('%Q')
+
+    req_obj = {
+      :resources => os_types,
+      :job       => {
+        :id   => saved_job_id,
+        :tags => {
+          :user => user,
+        },
+      },
+    }
+
+    if options['priority']
+      req_obj[:priority] = if options['priority'] == 'high'
+                             1
+                           elsif options['priority'] == 'medium'
+                             2
+                           elsif options['priority'] == 'low'
+                             3
+                           else
+                             options['priority'].to_i
+                           end
+    end
+
+    puts "Posting to ABS #{req_obj.to_json}" if verbose
+
+    # os_string = os_type.map { |os, num| Array(os) * num }.flatten.join('+')
+    # raise MissingParamError, 'No operating systems provided to obtain.' if os_string.empty?
+    puts "Requesting VMs with job_id: #{saved_job_id}.  Will retry for up to an hour."
+    res = conn.post 'request', req_obj.to_json
+
+    retries = 360
+
+    raise AuthError, "HTTP #{res.status}: The token provided could not authenticate to the pooler.\n#{res_body}" if res.status == 401
+
+    (1..retries).each do |i|
+      queue_place, res_body = check_queue(conn, saved_job_id, req_obj)
+      return translated(res_body) if res_body
+
+      sleep_seconds = 10 if i >= 10
+      sleep_seconds = i if i < 10
+      puts "Waiting #{sleep_seconds} seconds to check if ABS request has been filled.  Queue Position: #{queue_place}... (x#{i})"
+
+      sleep(sleep_seconds)
+    end
+    nil
+  end
+
+  #
+  # We should fix the ABS API to be more like the vmpooler or nspooler api, but for now
+  #
+  def self.translated(res_body)
+    vmpooler_formatted_body = {}
+
+    res_body.each do |host|
+      if vmpooler_formatted_body[host['type']] && vmpooler_formatted_body[host['type']]['hostname'].class == Array
+        vmpooler_formatted_body[host['type']]['hostname'] << host['hostname']
+      else
+        vmpooler_formatted_body[host['type']] = { 'hostname' => [host['hostname']] }
+      end
+    end
+    vmpooler_formatted_body['ok'] = true
+
+    vmpooler_formatted_body
+  end
+
+  def self.check_queue(conn, job_id, req_obj)
+    queue_info_res = conn.get "status/queue/info/#{job_id}"
+    queue_info = JSON.parse(queue_info_res.body)
+
+    res = conn.post 'request', req_obj.to_json
+
+    unless res.body.empty?
+      res_body = JSON.parse(res.body)
+      return queue_info['queue_place'], res_body
+    end
+    [queue_info['queue_place'], nil]
+  end
+
+  def self.snapshot(_verbose, _url, _hostname, _token)
+    puts "Can't snapshot with ABS, use '--service vmpooler' (even for vms checked out with ABS)"
+  end
+
+  def self.status(verbose, url)
+    conn = Http.get_conn(verbose, url)
+
+    res = conn.get 'status'
+
+    res.body == 'OK'
+  end
+
+  def self.summary(verbose, url)
+    conn = Http.get_conn(verbose, url)
+
+    res = conn.get 'summary'
+    JSON.parse(res.body)
+  end
+
+  def self.query(verbose, url, hostname)
+    return @active_hostnames if @active_hostnames
+
+    puts "For vmpooler/snapshot information, use '--service vmpooler' (even for vms checked out with ABS)"
+    conn = Http.get_conn(verbose, url)
+
+    res = conn.get "host/#{hostname}"
+    JSON.parse(res.body)
+  end
+end