floatyhelper 1.0 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,75 @@
1
+ # This is for directly manipulating the yaml file, as well as
2
+ # storing configuration for floatyhelper itself.
3
+ require 'yaml'
4
+ require 'etc'
5
+
6
+ class Config
7
+ DEFAULT_CONF_FILE = <<~EOT
8
+ ---
9
+ config: {}
10
+ vms: {}
11
+ snapshots: {}
12
+ EOT
13
+ .freeze
14
+
15
+ VALID_SETTINGS = {
16
+ 'increaselife' => {
17
+ 'description' => 'Number of hours beyond the service default to increase the VM lifetime when adding a VM to floatyhelper.',
18
+ 'default' => 0,
19
+ },
20
+ 'vertical_snapshot_status' => {
21
+ 'description' => 'When true, clear the screen and show the snapshot in progress status vertically. Otherwise, show status on a single line.',
22
+ 'default' => true,
23
+ },
24
+ }.freeze
25
+
26
+ attr_reader :VALID_SETTINGS
27
+
28
+ def self.fhfile
29
+ "#{Etc.getpwuid.dir}/.floatyhelper.yaml"
30
+ end
31
+
32
+ def self.load_data
33
+ File.write(fhfile, DEFAULT_CONF_FILE) unless File.exist?(fhfile)
34
+ data = YAML.load_file(fhfile)
35
+ # If someone is using an older version, ensure the conf file has
36
+ # all the correct top-level keys. Only write it back if we modify
37
+ # the data to add a top-level key.
38
+ writeback = false
39
+ ['config', 'vms', 'snapshots'].each do |key|
40
+ if !data.keys.include?(key)
41
+ data[key] = {}
42
+ writeback = true
43
+ end
44
+ end
45
+
46
+ # Config setting defaults
47
+ VALID_SETTINGS.each do |setting, info|
48
+ writeback = true if data['config'][setting].nil?
49
+ data['config'][setting] ||= info['default']
50
+ end
51
+
52
+ write_data(data) if writeback
53
+ data
54
+ end
55
+
56
+ # It is up to the calling function to ensure the data object
57
+ # being passed in is a properly formatted hash. It should only be
58
+ # used to modify after reading the current state with load_data.
59
+ def self.write_data(data)
60
+ File.write(fhfile, data.to_yaml)
61
+ end
62
+
63
+ # NOTE: Anything currently set with 'floatyhelper config set' will return a string,
64
+ # regardless of its intended type. Need to add typing one of these days.
65
+ def self.get_config_setting(setting)
66
+ data = load_data
67
+ data['config'][setting]
68
+ end
69
+
70
+ def self.set_config_setting(setting, value)
71
+ data = load_data
72
+ data['config'][setting] = value
73
+ write_data(data)
74
+ end
75
+ end
@@ -0,0 +1,154 @@
1
+ # This is for talking to floaty and getting data back from it.
2
+ require 'json'
3
+ require 'open3'
4
+ require 'colorize'
5
+ require 'pty'
6
+
7
+ class Floaty
8
+ FLOATY_SERVICES = {
9
+ 'abs' => {
10
+ 'type' => 'abs',
11
+ 'url' => 'https://abs-prod.k8s.infracore.puppet.net/api/v2',
12
+ },
13
+ 'vmpooler' => {
14
+ 'type' => 'vm',
15
+ 'url' => 'http://vmpooler.delivery.puppetlabs.net',
16
+ },
17
+ 'nspooler' => {
18
+ 'type' => 'nonstandard',
19
+ 'url' => 'https://nspooler-prod.k8s.infracore.puppet.net',
20
+ },
21
+ }.freeze
22
+
23
+ def self.vmfloatyfile
24
+ "#{Etc.getpwuid.dir}/.vmfloaty.yml"
25
+ end
26
+
27
+ def self.load_vmfloaty_config
28
+ if File.exist?(vmfloatyfile)
29
+ YAML.safe_load(File.read(vmfloatyfile))
30
+ else
31
+ {}
32
+ end
33
+ end
34
+
35
+ # This adds some stdout printing here which I was trying to avoid, but
36
+ # I think it makes sense to put it here rather than the main floatyhelper
37
+ # somewhere.
38
+ def self.check_floaty(data = nil)
39
+ data ||= load_vmfloaty_config
40
+
41
+ user = data['user']
42
+ unless user
43
+ puts 'No username found in .vmfloaty.yml.'.yellow
44
+ user = ask('Username: ').chomp
45
+ data['user'] = user
46
+ end
47
+ data['services'] ||= {}
48
+ puts 'No services defined in .vmfloaty.yml.' if data['services'].empty?
49
+
50
+ need_token = FLOATY_SERVICES.keys.any? { |k| !data['services'].keys.include?(k) } || data['services'].any? { |_svc, val| val['token'].nil? }
51
+ return unless need_token
52
+
53
+ puts 'It appears we need to fetch one or more tokens for the ~/.vmfloaty.yml file. Please enter your Puppet password.'
54
+ password = ask('Password: ') { |q| q.echo = '*' }
55
+ FLOATY_SERVICES.each do |service, info|
56
+ next if data['services'][service] && data['services'][service]['token']
57
+
58
+ data['services'][service] ||= {}
59
+ # Kind of silly to call out to curl here. Replace this with a Net::HTTP call
60
+ output, status = Open3.capture2e("curl -s -X POST -u #{user}:#{password} --url #{info['url']}/token")
61
+ unless status.exitstatus.zero?
62
+ puts "Bad return status from curl to #{info['url']}: #{status.exitstatus}".red
63
+ exit status.exitstatus
64
+ end
65
+ result = JSON.parse(output)
66
+ if result['ok']
67
+ data['services'][service]['type'] = info['type']
68
+ data['services'][service]['url'] = info['url']
69
+ data['services'][service]['token'] = result['token']
70
+ puts "Successfully fetched token for #{service}".green
71
+ else
72
+ puts "Could not get a token from #{service}. Please ensure your username and password are correct.".red
73
+ puts "Result: #{result}".red
74
+ exit 1
75
+ end
76
+ end
77
+ File.write(vmfloatyfile, data.to_yaml)
78
+ end
79
+
80
+ # Make sure all tokens are valid
81
+ def self.check_tokens
82
+ data = load_vmfloaty_config
83
+ issues = false
84
+ FLOATY_SERVICES.each do |service, _info|
85
+ if data['services'].nil? || data['services'][service].nil? || data['services'][service]['token'].nil?
86
+ puts "#{service} service token not found in .vmfloaty.yml".yellow
87
+ issues = true
88
+ next
89
+ end
90
+ # TODO: See what exitcode is when token is bad vs. some other fatal error
91
+ output, _status = Open3.capture2e("/usr/bin/env floaty token status --service #{service}")
92
+ result = parse_floaty_json_output(output)
93
+ next if result['ok']
94
+
95
+ puts "Problem checking #{service} token: #{result['reason']}".red
96
+ data['services']['token'] = nil
97
+ issues = true
98
+ end
99
+ if issues
100
+ check_floaty(data)
101
+ else
102
+ puts 'All tokens valid'.green
103
+ end
104
+ end
105
+
106
+ def self.parse_floaty_json_output(output)
107
+ # Sometimes there will be extra stuff before the hash output,
108
+ # so ignore it until we reach the hash.
109
+ lines = output.split("\n")
110
+ index = lines.index { |l| l[0] == '{' }
111
+ output = lines[index..-1].join
112
+ JSON.parse(output.gsub('=>',':'))
113
+ end
114
+
115
+ def self.floaty_cmd(command, ignore_error: false, use_pty: false)
116
+ check_floaty
117
+ # While we could potentially make a Vmfloaty object and send a command
118
+ # that way, Commander doesn't make it easy. Parsing floaty's stdout
119
+ # isn't great, but it works for now.
120
+ if use_pty
121
+ output = ''
122
+ status = nil
123
+ PTY.spawn("/usr/bin/env floaty #{command}") do |read, write, pid|
124
+ write.close
125
+ while status.nil?
126
+ begin
127
+ line = read.gets
128
+ puts line
129
+ output += line
130
+ rescue EOFError, Errno::EIO
131
+ # GNU/Linux raises EIO on read operation when pty is closed - see pty.rb docs
132
+ # Ensure child process finishes and then pass through to ensure below to get status
133
+ nil
134
+ rescue IO::WaitReadable, IO::WaitWritable
135
+ retry
136
+ ensure
137
+ status ||= PTY.check(pid)
138
+ end
139
+ end
140
+
141
+ Process.waitall
142
+ # Double check we have the status
143
+ status ||= PTY.check(pid)
144
+ end
145
+ else
146
+ output, status = Open3.capture2e("/usr/bin/env floaty #{command}")
147
+ end
148
+ if !status.exitstatus.zero? && !ignore_error
149
+ puts "Error running 'floaty #{command}': #{output}".red
150
+ exit status.exitstatus
151
+ end
152
+ output
153
+ end
154
+ end
@@ -1,65 +1,76 @@
1
- require 'floatyhelper/conf'
1
+ # This is for the management of hosts and snapshots in the yaml config file.
2
+ # Anything for adding, removing, or modifying hosts or snapshots in the
3
+ # floatyhelper.yaml file belongs here. Anything for modifying the VMs themselves
4
+ # does not.
5
+ require 'floatyhelper/config'
2
6
 
3
7
  class Groups
4
- def self.is_tag?(id)
5
- data = Conf.load_data
8
+ def self.get_tags
9
+ data = Config.load_data
10
+ data['vms'].keys
11
+ end
12
+
13
+ def self.tag?(id)
14
+ data = Config.load_data
6
15
  data['vms'].keys.include?(id)
7
16
  end
8
17
 
9
- def self.delete_tag(tag)
10
- data = Conf.load_data
18
+ def self.delete_tag(tag, data: nil)
19
+ data ||= Config.load_data
11
20
  data['vms'].delete(tag)
12
21
  data['snapshots'].delete(tag) if data['snapshots'].keys.include?(tag)
13
- Conf.write_data(data)
22
+ Config.write_data(data)
14
23
  end
15
24
 
16
25
  def self.delete_all
17
- data = Conf.load_data
26
+ data = Config.load_data
18
27
  data['vms'] = {}
19
28
  data['snapshots'] = {}
20
- Conf.write_data(data)
29
+ Config.write_data(data)
21
30
  end
22
31
 
23
- def self.addhosts(hosts, tag)
24
- data = Conf.load_data
25
- data['vms'] ||= {}
26
- tag = 'Blank Tag' unless tag #Removed the ability to do this
27
- hosts.each do |host|
28
- data['vms'][tag] ||= []
29
- data['vms'][tag] << host unless data['vms'][tag].include?(host)
30
- end
31
- Conf.write_data(data)
32
+ def self.get_hosts(tag)
33
+ data = Config.load_data
34
+ data['vms'][tag]
32
35
  end
33
36
 
34
- def self.appendhosts(hosts, tag)
35
- data = Conf.load_data
37
+ def self.addhosts(hosts, tag)
38
+ data = Config.load_data
39
+ data['vms'] ||= {}
40
+ tag ||= 'Blank Tag' # Removed the ability to do this
36
41
  hosts.each do |host|
37
42
  data['vms'][tag] ||= []
38
43
  data['vms'][tag] << host unless data['vms'][tag].include?(host)
39
44
  end
40
- Conf.write_data(data)
45
+ Config.write_data(data)
41
46
  end
42
47
 
43
48
  def self.removehosts(hosts, tag)
44
- data = Conf.load_data
49
+ data = Config.load_data
50
+ return if data['vms'][tag].nil?
51
+
45
52
  hosts.each do |host|
46
53
  data['vms'][tag] ||= []
47
54
  data['vms'][tag].delete(host)
48
- data['vms'].delete(tag) if data['vms'][tag].empty?
55
+ if data['snapshots'][tag]
56
+ data['snapshots'][tag].each do |snaptag, _vals|
57
+ data['snapshots'][tag][snaptag].delete(host)
58
+ end
59
+ end
60
+ delete_tag(tag, data: data) if data['vms'][tag].empty?
49
61
  end
50
- Conf.write_data(data)
62
+ Config.write_data(data)
51
63
  end
52
64
 
53
- def self.is_managed_host?(host)
54
- data = Conf.load_data
55
- data['vms'].any? { |tag, hosts| hosts.include?(host) }
65
+ def self.managed_host?(host)
66
+ data = Config.load_data
67
+ data['vms'].any? { |_tag, hosts| hosts.include?(host) }
56
68
  end
57
69
 
58
70
  def self.host_tag(host)
59
- data = Conf.load_data
60
- raise 'Host not found' unless is_managed_host?(host)
61
- tags = data['vms'].select { |tag, hosts| hosts.include?(host) }.keys
71
+ data = Config.load_data
72
+ raise 'Host not found' unless managed_host?(host)
73
+ tags = data['vms'].select { |_tag, hosts| hosts.include?(host) }.keys
62
74
  tags[0]
63
75
  end
64
-
65
- end
76
+ end
@@ -1,14 +1,15 @@
1
- require 'floatyhelper/conf'
1
+ # This is for finding hosts in various places (sut.log, --hosts flag)
2
+ # and returning an array of hostnames.
3
+ require 'floatyhelper/config'
2
4
  require 'floatyhelper/groups'
3
5
 
4
6
  class Hosts
5
-
6
7
  def self.get_hosts_from_sut_log(file)
7
8
  hosts = []
8
9
  File.open(file).each do |line|
9
10
  items = line.split("\t")
10
- hostname, tag = items[4].split(" ")
11
- short_host = hostname.split(".")[0]
11
+ hostname, tag = items[4].split # rubocop:disable Lint/UselessAssignment
12
+ short_host = hostname.split('.')[0]
12
13
  hosts << short_host
13
14
  end
14
15
  hosts
@@ -20,20 +21,19 @@ class Hosts
20
21
  end
21
22
 
22
23
  def self.get_hosts_from_id(id)
23
- data = Conf.load_data
24
+ data = Config.load_data
24
25
  if id == 'all'
25
26
  hosts = []
26
- data['vms'].each do |tag, hostlist|
27
+ data['vms'].each do |_tag, hostlist|
27
28
  hostlist.each do |host|
28
29
  hosts << host unless hosts.include?(host)
29
30
  end
30
31
  end
31
- elsif Groups.is_tag?(id)
32
+ elsif Groups.tag?(id)
32
33
  hosts = data['vms'][id]
33
34
  else
34
35
  hosts = [id].flatten
35
36
  end
36
37
  hosts
37
38
  end
38
-
39
- end
39
+ end
@@ -1,3 +1,3 @@
1
- module Floatyhelper
2
- VERSION = "1.0"
1
+ module FloatyhelperVersion
2
+ VERSION = '2.0.2'.freeze
3
3
  end
@@ -1,19 +1,55 @@
1
+ # This is for interfacing with floaty (or vmpooler/ABS directly if needed) to
2
+ # request, query, or modify VMs on the service, and managing snapshots in the
3
+ # yaml file.
1
4
  require 'floatyhelper/groups'
2
5
  require 'floatyhelper/hosts'
3
- require 'floatyhelper/conf'
6
+ require 'floatyhelper/config'
7
+ require 'colorize'
8
+ require 'net/http'
9
+ require 'json'
4
10
 
5
11
  class VM
12
+ def self.pooled_platforms_default
13
+ [
14
+ 'centos-7-x86_64',
15
+ 'centos-8-x86_64',
16
+ 'oracle-7-x86_64',
17
+ 'redhat-7-x86_64',
18
+ 'redhat-8-x86_64',
19
+ 'redhat-fips-7-x86_64',
20
+ 'scientific-7-x86_64',
21
+ 'sles-12-x86_64',
22
+ 'ubuntu-1804-x86_64',
23
+ ]
24
+ end
6
25
 
26
+ def self.find_pooled_platforms
27
+ begin # rubocop:disable Style/RedundantBegin
28
+ result = Net::HTTP.get('vmpooler.delivery.puppetlabs.net','/status')
29
+ result = JSON.parse(result)
30
+ # Techinally, 'max > 0' tells you if it's a pooled platform, but if
31
+ # the pool is empty, we'll want to fall back to ABS anyway.
32
+ result['pools'].select { |_pool, info| info['ready'].positive? }.map { |pool, _info| pool.gsub('-pixa4','') }
33
+ rescue StandardError
34
+ # Not a great practice to swallow all errors, but this list is probably
35
+ # pretty stable, so let's just pass along the default.
36
+ puts 'Error looking up pooled platforms from vmpooler.delivery.puppetlabs.net. Using default pooled platform list instead.'.yellow
37
+ pooled_platforms_default
38
+ end
39
+ end
40
+
41
+ ### VM Management ###
7
42
  def self.destroy(id)
8
43
  hosts = Hosts.get_hosts_from_id(id)
9
- Groups.delete_tag(id) if Groups.is_tag?(id)
44
+ Groups.delete_tag(id) if Groups.tag?(id)
10
45
  Groups.delete_all if id == 'all'
11
46
  hosts = hosts.select { |host| alive(host) }
12
- puts `floaty delete #{hosts.join(',')}` unless hosts.empty?
47
+ puts Floaty.floaty_cmd("delete #{hosts.join(',')} --service vmpooler") unless hosts.empty?
13
48
  end
14
49
 
15
50
  def self.query(host)
16
- eval(`floaty query #{host}`)
51
+ output = Floaty.floaty_cmd("query #{host} --service vmpooler")
52
+ Floaty.parse_floaty_json_output(output)
17
53
  end
18
54
 
19
55
  def self.get_current_lifetime(host)
@@ -21,88 +57,113 @@ class VM
21
57
  status['ok'] ? status[host]['lifetime'] : nil
22
58
  end
23
59
 
24
- def self.alive(host, query=nil)
60
+ def self.alive(host, query = nil)
25
61
  query ||= query(host)
26
62
  query['ok'] && query[host]['state'] == 'running'
27
63
  end
28
64
 
29
- def self.increaselife(id, amount)
30
- amount = 100 if amount.nil?
65
+ def self.increaselife(id, amount = nil)
66
+ amount ||= Config.get_config_setting('increaselife').to_i
67
+ return if amount < 1
31
68
  hosts = Hosts.get_hosts_from_id(id)
32
69
  hosts.each do |host|
33
- if lifetime = get_current_lifetime(host)
70
+ if (lifetime = get_current_lifetime(host))
34
71
  lifetime += amount
35
- print "#{host} to #{lifetime} hours: "
36
- puts `floaty modify #{host} --lifetime #{lifetime}`.split("\n")[0]
72
+ output = Floaty.floaty_cmd("modify #{host} --lifetime #{lifetime} --service vmpooler")
73
+ if output =~ /Successfully modified/
74
+ puts "#{host} lifetime set to #{lifetime} hours".green
75
+ else
76
+ puts "Error setting VM lifetime: #{output}".red
77
+ end
37
78
  else
38
79
  puts "Could not query host #{host}".red
39
80
  end
40
81
  end
41
82
  end
42
83
 
84
+ ### Snapshots ###
43
85
  def self.snapshot(id, snaptag)
44
- snaptag = 'Blank Snaptag' unless snaptag
86
+ clr = Config.get_config_setting('vertical_snapshot_status')
87
+ clr = clr.to_s.downcase == 'true'
88
+ snaptag ||= 'Blank Snaptag'
45
89
  hosts = Hosts.get_hosts_from_id(id)
46
90
  shas = {}
91
+ # There's probably a better way to do this...
92
+ puts `tput clear` if clr
93
+
47
94
  hosts.each do |host|
48
- print "#{host}: "
49
- result = `floaty snapshot #{host}`
50
- message = result.split("\n")[0]
51
- answer = eval(result.sub(message,''))
95
+ result = Floaty.floaty_cmd("snapshot #{host} --service vmpooler")
96
+ answer = Floaty.parse_floaty_json_output(result)
52
97
  sha = answer[host]['snapshot']
53
- puts sha
98
+ puts "#{host}: #{sha}"
54
99
  shas[host] = sha
55
100
  end
56
101
 
102
+ data = Config.load_data
103
+ data['snapshots'] ||= {}
104
+ data['snapshots'][id] ||= {}
105
+ data['snapshots'][id][snaptag] = shas
106
+ Config.write_data(data)
107
+
108
+ puts
57
109
  puts 'Waiting for snapshots to appear in floaty query...'
58
110
  alldone = false
59
- while !alldone do
111
+ until alldone
112
+ puts `tput cup #{hosts.count+2}` if clr
60
113
  alldone = true
61
- print "\r"
114
+ print "\r" unless clr
62
115
  hosts.each do |host|
63
- answer = eval(`floaty query #{host}`)[host]
116
+ answer = query(host)[host]
64
117
  done = answer.keys.include?('snapshots') && answer['snapshots'].include?(shas[host])
65
- status = done ? 'Done' : 'Wait'
66
- print "* #{host}: #{status} *"
118
+ status = done ? 'Done'.green : 'Wait'.yellow
119
+ if clr
120
+ puts "* %s #{status} *" % "#{host}:".ljust(16)
121
+ else
122
+ print "* %s #{status} *" % "#{host}:".ljust(16)
123
+ end
67
124
  alldone &= done
68
125
  end
69
126
  sleep(1)
70
127
  end
71
128
  puts ''
72
-
73
- data = Conf.load_data
74
- data['snapshots'] ||= {}
75
- data['snapshots'][id] ||= {}
76
- data['snapshots'][id][snaptag] = shas
77
- Conf.write_data(data)
78
129
  end
79
130
 
80
131
  def self.getsnapshot(id, snaptag)
81
- data = Conf.load_data
132
+ data = Config.load_data
82
133
  data['snapshots'][id][snaptag]
83
134
  end
84
135
 
85
136
  def self.snaptag_exists?(id, snaptag)
86
- data = Conf.load_data
87
- exists = data['snapshots'].keys.include?(id) ? data['snapshots'][id].keys.include?(snaptag) : false
137
+ data = Config.load_data
138
+ data['snapshots'].keys.include?(id) ? data['snapshots'][id].keys.include?(snaptag) : false
139
+ end
140
+
141
+ def self.snaplist(tag)
142
+ data = Config.load_data
143
+ return [] if !data['snapshots'].keys.include?(tag)
144
+ data['snapshots'][tag].keys
88
145
  end
89
146
 
90
147
  def self.revert(id, snaptag)
91
- snaptag = 'Blank Snaptag' unless snaptag
148
+ snaptag ||= 'Blank Snaptag'
92
149
  shas = VM.getsnapshot(id, snaptag)
93
150
  shas.each do |host, sha|
94
151
  print "#{host}: #{sha} --- "
95
- puts `floaty revert #{host} #{sha}`
152
+ puts Floaty.floaty_cmd("revert #{host} #{sha} --service vmpooler")
96
153
  end
97
154
  puts 'Waiting 10 seconds for revert to take effect'
98
155
  sleep(10)
99
156
  end
100
157
 
101
- def self.get_vm(platform='centos-7-x86_64')
102
- response = `floaty get #{platform} 2>&1`
103
- raise "Error obtaining a VM: #{response}" if response.include?('error')
104
- return response.split(' ')[1].split('.')[0]
158
+ ### Get a VM from floaty ###
159
+ def self.get_vm(platform: 'centos-7-x86_64', force_abs: false)
160
+ if !find_pooled_platforms.include?(platform.gsub('-pixa4','')) || force_abs
161
+ response = Floaty.floaty_cmd("get #{platform} --service abs --priority 1", use_pty: true)
162
+ response.chomp.split('- ')[1].split[0].split('.')[0]
163
+ else
164
+ response = Floaty.floaty_cmd("get #{platform} --service vmpooler")
165
+ raise "Error obtaining a VM: #{response}" if response.include?('error')
166
+ response.split[1].split('.')[0]
167
+ end
105
168
  end
106
-
107
169
  end
108
-