floatyhelper 1.2 → 2.0.4

data/lib/floatyhelper/config.rb

@@ -0,0 +1,75 @@
+# This is for directly manipulating the yaml file, as well as
+# storing configuration for floatyhelper itself.
+require 'yaml'
+require 'etc'
+
+class Config
+  DEFAULT_CONF_FILE = <<~EOT
+    ---
+    config: {}
+    vms: {}
+    snapshots: {}
+  EOT
+                      .freeze
+
+  VALID_SETTINGS = {
+    'increaselife' => {
+      'description' => 'Number of hours beyond the service default to increase the VM lifetime when adding a VM to floatyhelper.',
+      'default' => 0,
+    },
+    'vertical_snapshot_status' => {
+      'description' => 'When true, clear the screen and show the snapshot in progress status vertically. Otherwise, show status on a single line.',
+      'default' => true,
+    },
+  }.freeze
+
+  attr_reader :VALID_SETTINGS
+
+  def self.fhfile
+    "#{Etc.getpwuid.dir}/.floatyhelper.yaml"
+  end
+
+  def self.load_data
+    File.write(fhfile, DEFAULT_CONF_FILE) unless File.exist?(fhfile)
+    data = YAML.load_file(fhfile)
+    # If someone is using an older version, ensure the conf file has
+    # all the correct top-level keys. Only write it back if we modify
+    # the data to add a top-level key.
+    writeback = false
+    ['config', 'vms', 'snapshots'].each do |key|
+      if !data.keys.include?(key)
+        data[key] = {}
+        writeback = true
+      end
+    end
+
+    # Config setting defaults
+    VALID_SETTINGS.each do |setting, info|
+      writeback = true if data['config'][setting].nil?
+      data['config'][setting] ||= info['default']
+    end
+
+    write_data(data) if writeback
+    data
+  end
+
+  # It is up to the calling function to ensure the data object
+  # being passed in is a properly formatted hash. It should only be
+  # used to modify after reading the current state with load_data.
+  def self.write_data(data)
+    File.write(fhfile, data.to_yaml)
+  end
+
+  # NOTE: Anything currently set with 'floatyhelper config set' will return a string,
+  # regardless of its intended type. Need to add typing one of these days.
+  def self.get_config_setting(setting)
+    data = load_data
+    data['config'][setting]
+  end
+
+  def self.set_config_setting(setting, value)
+    data = load_data
+    data['config'][setting] = value
+    write_data(data)
+  end
+end

data/lib/floatyhelper/floaty.rb

@@ -0,0 +1,163 @@
+# This is for talking to floaty and getting data back from it.
+require 'json'
+require 'open3'
+require 'colorize'
+require 'pty'
+
+class Floaty
+  FLOATY_SERVICES = {
+    'abs' => {
+      'type' => 'abs',
+      'url' => 'https://abs-prod.k8s.infracore.puppet.net/api/v2',
+    },
+    'vmpooler' => {
+      'type' => 'vm',
+      'url' => 'http://vmpooler.delivery.puppetlabs.net',
+    },
+    'nspooler' => {
+      'type' => 'nonstandard',
+      'url' => 'https://nspooler-prod.k8s.infracore.puppet.net',
+    },
+  }.freeze
+
+  def self.vmfloatyfile
+    "#{Etc.getpwuid.dir}/.vmfloaty.yml"
+  end
+
+  def self.load_vmfloaty_config
+    if File.exist?(vmfloatyfile)
+      YAML.safe_load(File.read(vmfloatyfile))
+    else
+      {}
+    end
+  end
+
+  # This adds some stdout printing here which I was trying to avoid, but
+  # I think it makes sense to put it here rather than the main floatyhelper
+  # somewhere.
+  def self.check_floaty(data = nil)
+    data ||= load_vmfloaty_config
+
+    user = data['user']
+    unless user
+      puts 'No username found in .vmfloaty.yml.'.yellow
+      user = ask('Username: ').chomp
+      data['user'] = user
+    end
+    data['services'] ||= {}
+    puts 'No services defined in .vmfloaty.yml.' if data['services'].empty?
+
+    need_token = FLOATY_SERVICES.keys.any? { |k| !data['services'].keys.include?(k) } || data['services'].any? { |_svc, val| val['token'].nil? }
+    return unless need_token
+
+    puts 'It appears we need to fetch one or more tokens for the ~/.vmfloaty.yml file. Please enter your Puppet password.'
+    password = ask('Password: ') { |q| q.echo = '*' }
+    FLOATY_SERVICES.each do |service, info|
+      next if data['services'][service] && data['services'][service]['token']
+
+      data['services'][service] ||= {}
+      # Kind of silly to call out to curl here.  Replace this with a Net::HTTP call
+      output, status = Open3.capture2e("curl -s -X POST -u #{user}:#{password} --url #{info['url']}/token")
+      unless status.exitstatus.zero?
+        puts "Bad return status from curl to #{info['url']}: #{status.exitstatus}".red
+        exit status.exitstatus
+      end
+      result = JSON.parse(output)
+      if result['ok']
+        data['services'][service]['type'] = info['type']
+        data['services'][service]['url'] = info['url']
+        data['services'][service]['token'] = result['token']
+        puts "Successfully fetched token for #{service}".green
+      else
+        puts "Could not get a token from #{service}. Please ensure your username and password are correct.".red
+        puts "Result: #{result}".red
+        exit 1
+      end
+    end
+    File.write(vmfloatyfile, data.to_yaml)
+  end
+
+  # Make sure all tokens are valid
+  def self.check_tokens
+    data = load_vmfloaty_config
+    issues = false
+    FLOATY_SERVICES.each do |service, info|
+      if data['services'].nil? || data['services'][service].nil? || data['services'][service]['token'].nil?
+        puts "#{service} service token not found in .vmfloaty.yml".yellow
+        issues = true
+        next
+      end
+      # TODO: See what exitcode is when token is bad vs. some other fatal error
+      output, _status = Open3.capture2e("/usr/bin/env floaty token status --service #{service}")
+      result = parse_floaty_json_output(output)
+      next if result['ok']
+
+      puts "Problem checking #{service} token: #{result['reason']}".red
+      data['services'][service]['token'] = nil
+      if result['reason'].include?('Unparseable')
+        # User might have an old URL. Let's make sure to replace it with the latest.
+        # Should probably actually check the output for a 503/404 rather than make the
+        # assumption here.
+        data['services'][service]['url'] = info['url']
+      end
+      issues = true
+    end
+    if issues
+      check_floaty(data)
+    else
+      puts 'All tokens valid'.green
+    end
+  end
+
+  def self.parse_floaty_json_output(output)
+    # Sometimes there will be extra stuff before the hash output,
+    # so ignore it until we reach the hash.
+    lines = output.split("\n")
+    index = lines.index { |l| l[0] == '{' }
+    return { 'ok' => false, 'reason' => 'Unparseable response from floaty' } if index.nil?
+    output = lines[index..-1].join
+    JSON.parse(output.gsub('=>',':'))
+  end
+
+  def self.floaty_cmd(command, ignore_error: false, use_pty: false)
+    check_floaty
+    # While we could potentially make a Vmfloaty object and send a command
+    # that way, Commander doesn't make it easy.  Parsing floaty's stdout
+    # isn't great, but it works for now.
+    if use_pty
+      output = ''
+      status = nil
+      PTY.spawn("/usr/bin/env floaty #{command}") do |read, write, pid|
+        write.close
+        while status.nil?
+          begin
+            line = read.gets
+            unless line.nil?
+              puts line
+              output += line
+            end
+          rescue EOFError, Errno::EIO
+            # GNU/Linux raises EIO on read operation when pty is closed - see pty.rb docs
+            # Ensure child process finishes and then pass through to ensure below to get status
+            nil
+          rescue IO::WaitReadable, IO::WaitWritable
+            retry
+          ensure
+            status ||= PTY.check(pid)
+          end
+        end
+
+        Process.waitall
+        # Double check we have the status
+        status ||= PTY.check(pid)
+      end
+    else
+      output, status = Open3.capture2e("/usr/bin/env floaty #{command}")
+    end
+    if !status.exitstatus.zero? && !ignore_error
+      puts "Error running 'floaty #{command}': #{output}".red
+      exit status.exitstatus
+    end
+    output
+  end
+end

data/lib/floatyhelper/groups.rb

@@ -1,65 +1,76 @@
-require 'floatyhelper/conf'
+# This is for the management of hosts and snapshots in the yaml config file.
+# Anything for adding, removing, or modifying hosts or snapshots in the
+# floatyhelper.yaml file belongs here. Anything for modifying the VMs themselves
+# does not.
+require 'floatyhelper/config'
 
 class Groups
-  def self.is_tag?(id)
-    data = Conf.load_data
+  def self.get_tags
+    data = Config.load_data
+    data['vms'].keys
+  end
+
+  def self.tag?(id)
+    data = Config.load_data
     data['vms'].keys.include?(id)
   end
 
-  def self.delete_tag(tag)
-    data = Conf.load_data
+  def self.delete_tag(tag, data: nil)
+    data ||= Config.load_data
     data['vms'].delete(tag)
     data['snapshots'].delete(tag) if data['snapshots'].keys.include?(tag)
-    Conf.write_data(data)
+    Config.write_data(data)
   end
 
   def self.delete_all
-    data = Conf.load_data
+    data = Config.load_data
     data['vms'] = {}
     data['snapshots'] = {}
-    Conf.write_data(data)
+    Config.write_data(data)
   end
 
-  def self.addhosts(hosts, tag)
-    data = Conf.load_data
-    data['vms'] ||= {}
-    tag = 'Blank Tag' unless tag #Removed the ability to do this
-    hosts.each do |host|
-      data['vms'][tag] ||= []
-      data['vms'][tag] << host unless data['vms'][tag].include?(host)
-    end
-    Conf.write_data(data)
+  def self.get_hosts(tag)
+    data = Config.load_data
+    data['vms'][tag]
   end
 
-  def self.appendhosts(hosts, tag)
-    data = Conf.load_data
+  def self.addhosts(hosts, tag)
+    data = Config.load_data
+    data['vms'] ||= {}
+    tag ||= 'Blank Tag' # Removed the ability to do this
     hosts.each do |host|
       data['vms'][tag] ||= []
       data['vms'][tag] << host unless data['vms'][tag].include?(host)
     end
-    Conf.write_data(data)
+    Config.write_data(data)
   end
 
   def self.removehosts(hosts, tag)
-    data = Conf.load_data
+    data = Config.load_data
+    return if data['vms'][tag].nil?
+
     hosts.each do |host|
       data['vms'][tag] ||= []
       data['vms'][tag].delete(host)
-      data['vms'].delete(tag) if data['vms'][tag].empty?
+      if data['snapshots'][tag]
+        data['snapshots'][tag].each do |snaptag, _vals|
+          data['snapshots'][tag][snaptag].delete(host)
+        end
+      end
+      delete_tag(tag, data: data) if data['vms'][tag].empty?
     end
-    Conf.write_data(data)
+    Config.write_data(data)
   end
 
-  def self.is_managed_host?(host)
-    data = Conf.load_data
-    data['vms'].any? { |tag, hosts| hosts.include?(host) }
+  def self.managed_host?(host)
+    data = Config.load_data
+    data['vms'].any? { |_tag, hosts| hosts.include?(host) }
   end
 
   def self.host_tag(host)
-    data = Conf.load_data
-    raise 'Host not found' unless is_managed_host?(host)
-    tags = data['vms'].select { |tag, hosts| hosts.include?(host) }.keys
+    data = Config.load_data
+    raise 'Host not found' unless managed_host?(host)
+    tags = data['vms'].select { |_tag, hosts| hosts.include?(host) }.keys
     tags[0]
   end
-  
-end
+end

data/lib/floatyhelper/hosts.rb

@@ -1,14 +1,15 @@
-require 'floatyhelper/conf'
+# This is for finding hosts in various places (sut.log, --hosts flag)
+# and returning an array of hostnames.
+require 'floatyhelper/config'
 require 'floatyhelper/groups'
 
 class Hosts
-
   def self.get_hosts_from_sut_log(file)
     hosts = []
     File.open(file).each do |line|
       items = line.split("\t")
-      hostname, tag = items[4].split(" ")
-      short_host = hostname.split(".")[0]
+      hostname, tag = items[4].split # rubocop:disable Lint/UselessAssignment
+      short_host = hostname.split('.')[0]
       hosts << short_host
     end
     hosts
@@ -20,20 +21,19 @@ class Hosts
   end
 
   def self.get_hosts_from_id(id)
-    data = Conf.load_data
+    data = Config.load_data
     if id == 'all'
       hosts = []
-      data['vms'].each do |tag, hostlist|
+      data['vms'].each do |_tag, hostlist|
         hostlist.each do |host|
           hosts << host unless hosts.include?(host)
         end
       end
-    elsif Groups.is_tag?(id)
+    elsif Groups.tag?(id)
       hosts = data['vms'][id]
     else
       hosts = [id].flatten
     end
     hosts
   end
-
-end
+end

data/lib/floatyhelper/version.rb

@@ -1,3 +1,3 @@
-module Floatyhelper
-  VERSION = "1.2"
+module FloatyhelperVersion
+  VERSION = '2.0.4'.freeze
 end

data/lib/floatyhelper/vm.rb

@@ -1,19 +1,58 @@
+# This is for interfacing with floaty (or vmpooler/ABS directly if needed) to
+# request, query, or modify VMs on the service, and managing snapshots in the
+# yaml file.
 require 'floatyhelper/groups'
 require 'floatyhelper/hosts'
-require 'floatyhelper/conf'
+require 'floatyhelper/config'
+require 'colorize'
+require 'net/http'
+require 'json'
 
 class VM
+  def self.pooled_platforms_default
+    [
+      'centos-7-x86_64',
+      'centos-8-x86_64',
+      'oracle-7-x86_64',
+      'redhat-7-x86_64',
+      'redhat-8-x86_64',
+      'redhat-fips-7-x86_64',
+      'scientific-7-x86_64',
+      'sles-12-x86_64',
+      'ubuntu-1804-x86_64',
+    ]
+  end
 
+  def self.find_pooled_platforms
+    begin # rubocop:disable Style/RedundantBegin
+      uri = URI.parse('https://vmpooler.delivery.puppetlabs.net/status')
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      result = http.get(uri.request_uri)
+      result = JSON.parse(result.body)
+      # Techinally, 'max > 0' tells you if it's a pooled platform, but if
+      # the pool is empty, we'll want to fall back to ABS anyway.
+      result['pools'].select { |_pool, info| info['ready'].positive? }.map { |pool, _info| pool.gsub('-pixa4','') }
+    rescue StandardError
+      # Not a great practice to swallow all errors, but this list is probably
+      # pretty stable, so let's just pass along the default.
+      puts 'Error looking up pooled platforms from vmpooler.delivery.puppetlabs.net. Using default pooled platform list instead.'.yellow
+      pooled_platforms_default
+    end
+  end
+
+  ### VM Management ###
   def self.destroy(id)
     hosts = Hosts.get_hosts_from_id(id)
-    Groups.delete_tag(id) if Groups.is_tag?(id)
+    Groups.delete_tag(id) if Groups.tag?(id)
     Groups.delete_all if id == 'all'
     hosts = hosts.select { |host| alive(host) }
-    puts `floaty delete #{hosts.join(',')}` unless hosts.empty?
+    puts Floaty.floaty_cmd("delete #{hosts.join(',')} --service vmpooler") unless hosts.empty?
   end
 
   def self.query(host)
-    eval(`floaty query #{host}`)
+    output = Floaty.floaty_cmd("query #{host} --service vmpooler")
+    Floaty.parse_floaty_json_output(output)
   end
 
   def self.get_current_lifetime(host)
@@ -21,88 +60,113 @@ class VM
     status['ok'] ? status[host]['lifetime'] : nil
   end
 
-  def self.alive(host, query=nil)
+  def self.alive(host, query = nil)
     query ||= query(host)
     query['ok'] && query[host]['state'] == 'running'
   end
 
-  def self.increaselife(id, amount)
-    amount = 100 if amount.nil?
+  def self.increaselife(id, amount = nil)
+    amount ||= Config.get_config_setting('increaselife').to_i
+    return if amount < 1
     hosts = Hosts.get_hosts_from_id(id)
     hosts.each do |host|
-      if lifetime = get_current_lifetime(host)
+      if (lifetime = get_current_lifetime(host))
         lifetime += amount
-        print "#{host} to #{lifetime} hours: "
-        puts `floaty modify #{host} --lifetime #{lifetime}`.split("\n")[0]
+        output = Floaty.floaty_cmd("modify #{host} --lifetime #{lifetime} --service vmpooler")
+        if output =~ /Successfully modified/
+          puts "#{host} lifetime set to #{lifetime} hours".green
+        else
+          puts "Error setting VM lifetime: #{output}".red
+        end
       else
         puts "Could not query host #{host}".red
       end
     end
   end
 
+  ### Snapshots ###
   def self.snapshot(id, snaptag)
-    snaptag = 'Blank Snaptag' unless snaptag
+    clr = Config.get_config_setting('vertical_snapshot_status')
+    clr = clr.to_s.downcase == 'true'
+    snaptag ||= 'Blank Snaptag'
     hosts = Hosts.get_hosts_from_id(id)
     shas = {}
+    # There's probably a better way to do this...
+    puts `tput clear` if clr
+
     hosts.each do |host|
-      print "#{host}: "
-      result = `floaty snapshot #{host}`
-      message = result.split("\n")[0]
-      answer = eval(result.sub(message,''))
+      result = Floaty.floaty_cmd("snapshot #{host} --service vmpooler")
+      answer = Floaty.parse_floaty_json_output(result)
       sha = answer[host]['snapshot']
-      puts sha
+      puts "#{host}: #{sha}"
       shas[host] = sha
     end
 
+    data = Config.load_data
+    data['snapshots'] ||= {}
+    data['snapshots'][id] ||= {}
+    data['snapshots'][id][snaptag] = shas
+    Config.write_data(data)
+
+    puts
     puts 'Waiting for snapshots to appear in floaty query...'
     alldone = false
-    while !alldone do
+    until alldone
+      puts `tput cup #{hosts.count + 2}` if clr
       alldone = true
-      print "\r"
+      print "\r" unless clr
       hosts.each do |host|
-        answer = eval(`floaty query #{host}`)[host]
+        answer = query(host)[host]
         done = answer.keys.include?('snapshots') && answer['snapshots'].include?(shas[host])
-        status = done ? 'Done' : 'Wait'
-        print "* #{host}: #{status} *"
+        status = done ? 'Done'.green : 'Wait'.yellow
+        if clr
+          puts "* %s #{status} *" % "#{host}:".ljust(16)
+        else
+          print "* %s #{status} *" % "#{host}:".ljust(16)
+        end
         alldone &= done
       end
       sleep(1)
     end
     puts ''
-
-    data = Conf.load_data
-    data['snapshots'] ||= {}
-    data['snapshots'][id] ||= {}
-    data['snapshots'][id][snaptag] = shas
-    Conf.write_data(data)
   end
 
   def self.getsnapshot(id, snaptag)
-    data = Conf.load_data
+    data = Config.load_data
     data['snapshots'][id][snaptag]
   end
 
   def self.snaptag_exists?(id, snaptag)
-    data = Conf.load_data
-    exists = data['snapshots'].keys.include?(id) ? data['snapshots'][id].keys.include?(snaptag) : false
+    data = Config.load_data
+    data['snapshots'].keys.include?(id) ? data['snapshots'][id].keys.include?(snaptag) : false
+  end
+
+  def self.snaplist(tag)
+    data = Config.load_data
+    return [] if !data['snapshots'].keys.include?(tag)
+    data['snapshots'][tag].keys
   end
 
   def self.revert(id, snaptag)
-    snaptag = 'Blank Snaptag' unless snaptag
+    snaptag ||= 'Blank Snaptag'
     shas = VM.getsnapshot(id, snaptag)
     shas.each do |host, sha|
       print "#{host}: #{sha} --- "
-      puts `floaty revert #{host} #{sha}`
+      puts Floaty.floaty_cmd("revert #{host} #{sha} --service vmpooler")
     end
     puts 'Waiting 10 seconds for revert to take effect'
     sleep(10)
   end
 
-  def self.get_vm(platform='centos-7-x86_64')
-    response = `floaty get #{platform} 2>&1`
-    raise "Error obtaining a VM: #{response}" if response.include?('error')
-    return response.split(' ')[1].split('.')[0]
+  ### Get a VM from floaty ###
+  def self.get_vm(platform: 'centos-7-x86_64', force_abs: false)
+    if !find_pooled_platforms.include?(platform.gsub('-pixa4','')) || force_abs
+      response = Floaty.floaty_cmd("get #{platform} --service abs --priority 1", use_pty: true)
+      response.chomp.split('- ')[1].split[0].split('.')[0]
+    else
+      response = Floaty.floaty_cmd("get #{platform} --service vmpooler")
+      raise "Error obtaining a VM: #{response}" if response.include?('error')
+      response.split[1].split('.')[0]
+    end
   end
-
 end
-