floatyhelper 1.2 → 2.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,75 @@
1
+ # This is for directly manipulating the yaml file, as well as
2
+ # storing configuration for floatyhelper itself.
3
+ require 'yaml'
4
+ require 'etc'
5
+
6
+ class Config
7
+ DEFAULT_CONF_FILE = <<~EOT
8
+ ---
9
+ config: {}
10
+ vms: {}
11
+ snapshots: {}
12
+ EOT
13
+ .freeze
14
+
15
+ VALID_SETTINGS = {
16
+ 'increaselife' => {
17
+ 'description' => 'Number of hours beyond the service default to increase the VM lifetime when adding a VM to floatyhelper.',
18
+ 'default' => 0,
19
+ },
20
+ 'vertical_snapshot_status' => {
21
+ 'description' => 'When true, clear the screen and show the snapshot in progress status vertically. Otherwise, show status on a single line.',
22
+ 'default' => true,
23
+ },
24
+ }.freeze
25
+
26
+ attr_reader :VALID_SETTINGS
27
+
28
+ def self.fhfile
29
+ "#{Etc.getpwuid.dir}/.floatyhelper.yaml"
30
+ end
31
+
32
+ def self.load_data
33
+ File.write(fhfile, DEFAULT_CONF_FILE) unless File.exist?(fhfile)
34
+ data = YAML.load_file(fhfile)
35
+ # If someone is using an older version, ensure the conf file has
36
+ # all the correct top-level keys. Only write it back if we modify
37
+ # the data to add a top-level key.
38
+ writeback = false
39
+ ['config', 'vms', 'snapshots'].each do |key|
40
+ if !data.keys.include?(key)
41
+ data[key] = {}
42
+ writeback = true
43
+ end
44
+ end
45
+
46
+ # Config setting defaults
47
+ VALID_SETTINGS.each do |setting, info|
48
+ writeback = true if data['config'][setting].nil?
49
+ data['config'][setting] ||= info['default']
50
+ end
51
+
52
+ write_data(data) if writeback
53
+ data
54
+ end
55
+
56
+ # It is up to the calling function to ensure the data object
57
+ # being passed in is a properly formatted hash. It should only be
58
+ # used to modify after reading the current state with load_data.
59
+ def self.write_data(data)
60
+ File.write(fhfile, data.to_yaml)
61
+ end
62
+
63
+ # NOTE: Anything currently set with 'floatyhelper config set' will return a string,
64
+ # regardless of its intended type. Need to add typing one of these days.
65
+ def self.get_config_setting(setting)
66
+ data = load_data
67
+ data['config'][setting]
68
+ end
69
+
70
+ def self.set_config_setting(setting, value)
71
+ data = load_data
72
+ data['config'][setting] = value
73
+ write_data(data)
74
+ end
75
+ end
@@ -0,0 +1,163 @@
1
+ # This is for talking to floaty and getting data back from it.
2
+ require 'json'
3
+ require 'open3'
4
+ require 'colorize'
5
+ require 'pty'
6
+
7
+ class Floaty
8
+ FLOATY_SERVICES = {
9
+ 'abs' => {
10
+ 'type' => 'abs',
11
+ 'url' => 'https://abs-prod.k8s.infracore.puppet.net/api/v2',
12
+ },
13
+ 'vmpooler' => {
14
+ 'type' => 'vm',
15
+ 'url' => 'http://vmpooler.delivery.puppetlabs.net',
16
+ },
17
+ 'nspooler' => {
18
+ 'type' => 'nonstandard',
19
+ 'url' => 'https://nspooler-prod.k8s.infracore.puppet.net',
20
+ },
21
+ }.freeze
22
+
23
+ def self.vmfloatyfile
24
+ "#{Etc.getpwuid.dir}/.vmfloaty.yml"
25
+ end
26
+
27
+ def self.load_vmfloaty_config
28
+ if File.exist?(vmfloatyfile)
29
+ YAML.safe_load(File.read(vmfloatyfile))
30
+ else
31
+ {}
32
+ end
33
+ end
34
+
35
+ # This adds some stdout printing here which I was trying to avoid, but
36
+ # I think it makes sense to put it here rather than the main floatyhelper
37
+ # somewhere.
38
+ def self.check_floaty(data = nil)
39
+ data ||= load_vmfloaty_config
40
+
41
+ user = data['user']
42
+ unless user
43
+ puts 'No username found in .vmfloaty.yml.'.yellow
44
+ user = ask('Username: ').chomp
45
+ data['user'] = user
46
+ end
47
+ data['services'] ||= {}
48
+ puts 'No services defined in .vmfloaty.yml.' if data['services'].empty?
49
+
50
+ need_token = FLOATY_SERVICES.keys.any? { |k| !data['services'].keys.include?(k) } || data['services'].any? { |_svc, val| val['token'].nil? }
51
+ return unless need_token
52
+
53
+ puts 'It appears we need to fetch one or more tokens for the ~/.vmfloaty.yml file. Please enter your Puppet password.'
54
+ password = ask('Password: ') { |q| q.echo = '*' }
55
+ FLOATY_SERVICES.each do |service, info|
56
+ next if data['services'][service] && data['services'][service]['token']
57
+
58
+ data['services'][service] ||= {}
59
+ # Kind of silly to call out to curl here. Replace this with a Net::HTTP call
60
+ output, status = Open3.capture2e("curl -s -X POST -u #{user}:#{password} --url #{info['url']}/token")
61
+ unless status.exitstatus.zero?
62
+ puts "Bad return status from curl to #{info['url']}: #{status.exitstatus}".red
63
+ exit status.exitstatus
64
+ end
65
+ result = JSON.parse(output)
66
+ if result['ok']
67
+ data['services'][service]['type'] = info['type']
68
+ data['services'][service]['url'] = info['url']
69
+ data['services'][service]['token'] = result['token']
70
+ puts "Successfully fetched token for #{service}".green
71
+ else
72
+ puts "Could not get a token from #{service}. Please ensure your username and password are correct.".red
73
+ puts "Result: #{result}".red
74
+ exit 1
75
+ end
76
+ end
77
+ File.write(vmfloatyfile, data.to_yaml)
78
+ end
79
+
80
+ # Make sure all tokens are valid
81
+ def self.check_tokens
82
+ data = load_vmfloaty_config
83
+ issues = false
84
+ FLOATY_SERVICES.each do |service, info|
85
+ if data['services'].nil? || data['services'][service].nil? || data['services'][service]['token'].nil?
86
+ puts "#{service} service token not found in .vmfloaty.yml".yellow
87
+ issues = true
88
+ next
89
+ end
90
+ # TODO: See what exitcode is when token is bad vs. some other fatal error
91
+ output, _status = Open3.capture2e("/usr/bin/env floaty token status --service #{service}")
92
+ result = parse_floaty_json_output(output)
93
+ next if result['ok']
94
+
95
+ puts "Problem checking #{service} token: #{result['reason']}".red
96
+ data['services'][service]['token'] = nil
97
+ if result['reason'].include?('Unparseable')
98
+ # User might have an old URL. Let's make sure to replace it with the latest.
99
+ # Should probably actually check the output for a 503/404 rather than make the
100
+ # assumption here.
101
+ data['services'][service]['url'] = info['url']
102
+ end
103
+ issues = true
104
+ end
105
+ if issues
106
+ check_floaty(data)
107
+ else
108
+ puts 'All tokens valid'.green
109
+ end
110
+ end
111
+
112
+ def self.parse_floaty_json_output(output)
113
+ # Sometimes there will be extra stuff before the hash output,
114
+ # so ignore it until we reach the hash.
115
+ lines = output.split("\n")
116
+ index = lines.index { |l| l[0] == '{' }
117
+ return { 'ok' => false, 'reason' => 'Unparseable response from floaty' } if index.nil?
118
+ output = lines[index..-1].join
119
+ JSON.parse(output.gsub('=>',':'))
120
+ end
121
+
122
+ def self.floaty_cmd(command, ignore_error: false, use_pty: false)
123
+ check_floaty
124
+ # While we could potentially make a Vmfloaty object and send a command
125
+ # that way, Commander doesn't make it easy. Parsing floaty's stdout
126
+ # isn't great, but it works for now.
127
+ if use_pty
128
+ output = ''
129
+ status = nil
130
+ PTY.spawn("/usr/bin/env floaty #{command}") do |read, write, pid|
131
+ write.close
132
+ while status.nil?
133
+ begin
134
+ line = read.gets
135
+ unless line.nil?
136
+ puts line
137
+ output += line
138
+ end
139
+ rescue EOFError, Errno::EIO
140
+ # GNU/Linux raises EIO on read operation when pty is closed - see pty.rb docs
141
+ # Ensure child process finishes and then pass through to ensure below to get status
142
+ nil
143
+ rescue IO::WaitReadable, IO::WaitWritable
144
+ retry
145
+ ensure
146
+ status ||= PTY.check(pid)
147
+ end
148
+ end
149
+
150
+ Process.waitall
151
+ # Double check we have the status
152
+ status ||= PTY.check(pid)
153
+ end
154
+ else
155
+ output, status = Open3.capture2e("/usr/bin/env floaty #{command}")
156
+ end
157
+ if !status.exitstatus.zero? && !ignore_error
158
+ puts "Error running 'floaty #{command}': #{output}".red
159
+ exit status.exitstatus
160
+ end
161
+ output
162
+ end
163
+ end
@@ -1,65 +1,76 @@
1
- require 'floatyhelper/conf'
1
+ # This is for the management of hosts and snapshots in the yaml config file.
2
+ # Anything for adding, removing, or modifying hosts or snapshots in the
3
+ # floatyhelper.yaml file belongs here. Anything for modifying the VMs themselves
4
+ # does not.
5
+ require 'floatyhelper/config'
2
6
 
3
7
  class Groups
4
- def self.is_tag?(id)
5
- data = Conf.load_data
8
+ def self.get_tags
9
+ data = Config.load_data
10
+ data['vms'].keys
11
+ end
12
+
13
+ def self.tag?(id)
14
+ data = Config.load_data
6
15
  data['vms'].keys.include?(id)
7
16
  end
8
17
 
9
- def self.delete_tag(tag)
10
- data = Conf.load_data
18
+ def self.delete_tag(tag, data: nil)
19
+ data ||= Config.load_data
11
20
  data['vms'].delete(tag)
12
21
  data['snapshots'].delete(tag) if data['snapshots'].keys.include?(tag)
13
- Conf.write_data(data)
22
+ Config.write_data(data)
14
23
  end
15
24
 
16
25
  def self.delete_all
17
- data = Conf.load_data
26
+ data = Config.load_data
18
27
  data['vms'] = {}
19
28
  data['snapshots'] = {}
20
- Conf.write_data(data)
29
+ Config.write_data(data)
21
30
  end
22
31
 
23
- def self.addhosts(hosts, tag)
24
- data = Conf.load_data
25
- data['vms'] ||= {}
26
- tag = 'Blank Tag' unless tag #Removed the ability to do this
27
- hosts.each do |host|
28
- data['vms'][tag] ||= []
29
- data['vms'][tag] << host unless data['vms'][tag].include?(host)
30
- end
31
- Conf.write_data(data)
32
+ def self.get_hosts(tag)
33
+ data = Config.load_data
34
+ data['vms'][tag]
32
35
  end
33
36
 
34
- def self.appendhosts(hosts, tag)
35
- data = Conf.load_data
37
+ def self.addhosts(hosts, tag)
38
+ data = Config.load_data
39
+ data['vms'] ||= {}
40
+ tag ||= 'Blank Tag' # Removed the ability to do this
36
41
  hosts.each do |host|
37
42
  data['vms'][tag] ||= []
38
43
  data['vms'][tag] << host unless data['vms'][tag].include?(host)
39
44
  end
40
- Conf.write_data(data)
45
+ Config.write_data(data)
41
46
  end
42
47
 
43
48
  def self.removehosts(hosts, tag)
44
- data = Conf.load_data
49
+ data = Config.load_data
50
+ return if data['vms'][tag].nil?
51
+
45
52
  hosts.each do |host|
46
53
  data['vms'][tag] ||= []
47
54
  data['vms'][tag].delete(host)
48
- data['vms'].delete(tag) if data['vms'][tag].empty?
55
+ if data['snapshots'][tag]
56
+ data['snapshots'][tag].each do |snaptag, _vals|
57
+ data['snapshots'][tag][snaptag].delete(host)
58
+ end
59
+ end
60
+ delete_tag(tag, data: data) if data['vms'][tag].empty?
49
61
  end
50
- Conf.write_data(data)
62
+ Config.write_data(data)
51
63
  end
52
64
 
53
- def self.is_managed_host?(host)
54
- data = Conf.load_data
55
- data['vms'].any? { |tag, hosts| hosts.include?(host) }
65
+ def self.managed_host?(host)
66
+ data = Config.load_data
67
+ data['vms'].any? { |_tag, hosts| hosts.include?(host) }
56
68
  end
57
69
 
58
70
  def self.host_tag(host)
59
- data = Conf.load_data
60
- raise 'Host not found' unless is_managed_host?(host)
61
- tags = data['vms'].select { |tag, hosts| hosts.include?(host) }.keys
71
+ data = Config.load_data
72
+ raise 'Host not found' unless managed_host?(host)
73
+ tags = data['vms'].select { |_tag, hosts| hosts.include?(host) }.keys
62
74
  tags[0]
63
75
  end
64
-
65
- end
76
+ end
@@ -1,14 +1,15 @@
1
- require 'floatyhelper/conf'
1
+ # This is for finding hosts in various places (sut.log, --hosts flag)
2
+ # and returning an array of hostnames.
3
+ require 'floatyhelper/config'
2
4
  require 'floatyhelper/groups'
3
5
 
4
6
  class Hosts
5
-
6
7
  def self.get_hosts_from_sut_log(file)
7
8
  hosts = []
8
9
  File.open(file).each do |line|
9
10
  items = line.split("\t")
10
- hostname, tag = items[4].split(" ")
11
- short_host = hostname.split(".")[0]
11
+ hostname, tag = items[4].split # rubocop:disable Lint/UselessAssignment
12
+ short_host = hostname.split('.')[0]
12
13
  hosts << short_host
13
14
  end
14
15
  hosts
@@ -20,20 +21,19 @@ class Hosts
20
21
  end
21
22
 
22
23
  def self.get_hosts_from_id(id)
23
- data = Conf.load_data
24
+ data = Config.load_data
24
25
  if id == 'all'
25
26
  hosts = []
26
- data['vms'].each do |tag, hostlist|
27
+ data['vms'].each do |_tag, hostlist|
27
28
  hostlist.each do |host|
28
29
  hosts << host unless hosts.include?(host)
29
30
  end
30
31
  end
31
- elsif Groups.is_tag?(id)
32
+ elsif Groups.tag?(id)
32
33
  hosts = data['vms'][id]
33
34
  else
34
35
  hosts = [id].flatten
35
36
  end
36
37
  hosts
37
38
  end
38
-
39
- end
39
+ end
@@ -1,3 +1,3 @@
1
- module Floatyhelper
2
- VERSION = "1.2"
1
+ module FloatyhelperVersion
2
+ VERSION = '2.0.4'.freeze
3
3
  end
@@ -1,19 +1,58 @@
1
+ # This is for interfacing with floaty (or vmpooler/ABS directly if needed) to
2
+ # request, query, or modify VMs on the service, and managing snapshots in the
3
+ # yaml file.
1
4
  require 'floatyhelper/groups'
2
5
  require 'floatyhelper/hosts'
3
- require 'floatyhelper/conf'
6
+ require 'floatyhelper/config'
7
+ require 'colorize'
8
+ require 'net/http'
9
+ require 'json'
4
10
 
5
11
  class VM
12
+ def self.pooled_platforms_default
13
+ [
14
+ 'centos-7-x86_64',
15
+ 'centos-8-x86_64',
16
+ 'oracle-7-x86_64',
17
+ 'redhat-7-x86_64',
18
+ 'redhat-8-x86_64',
19
+ 'redhat-fips-7-x86_64',
20
+ 'scientific-7-x86_64',
21
+ 'sles-12-x86_64',
22
+ 'ubuntu-1804-x86_64',
23
+ ]
24
+ end
6
25
 
26
+ def self.find_pooled_platforms
27
+ begin # rubocop:disable Style/RedundantBegin
28
+ uri = URI.parse('https://vmpooler.delivery.puppetlabs.net/status')
29
+ http = Net::HTTP.new(uri.host, uri.port)
30
+ http.use_ssl = true
31
+ result = http.get(uri.request_uri)
32
+ result = JSON.parse(result.body)
33
+ # Techinally, 'max > 0' tells you if it's a pooled platform, but if
34
+ # the pool is empty, we'll want to fall back to ABS anyway.
35
+ result['pools'].select { |_pool, info| info['ready'].positive? }.map { |pool, _info| pool.gsub('-pixa4','') }
36
+ rescue StandardError
37
+ # Not a great practice to swallow all errors, but this list is probably
38
+ # pretty stable, so let's just pass along the default.
39
+ puts 'Error looking up pooled platforms from vmpooler.delivery.puppetlabs.net. Using default pooled platform list instead.'.yellow
40
+ pooled_platforms_default
41
+ end
42
+ end
43
+
44
+ ### VM Management ###
7
45
  def self.destroy(id)
8
46
  hosts = Hosts.get_hosts_from_id(id)
9
- Groups.delete_tag(id) if Groups.is_tag?(id)
47
+ Groups.delete_tag(id) if Groups.tag?(id)
10
48
  Groups.delete_all if id == 'all'
11
49
  hosts = hosts.select { |host| alive(host) }
12
- puts `floaty delete #{hosts.join(',')}` unless hosts.empty?
50
+ puts Floaty.floaty_cmd("delete #{hosts.join(',')} --service vmpooler") unless hosts.empty?
13
51
  end
14
52
 
15
53
  def self.query(host)
16
- eval(`floaty query #{host}`)
54
+ output = Floaty.floaty_cmd("query #{host} --service vmpooler")
55
+ Floaty.parse_floaty_json_output(output)
17
56
  end
18
57
 
19
58
  def self.get_current_lifetime(host)
@@ -21,88 +60,113 @@ class VM
21
60
  status['ok'] ? status[host]['lifetime'] : nil
22
61
  end
23
62
 
24
- def self.alive(host, query=nil)
63
+ def self.alive(host, query = nil)
25
64
  query ||= query(host)
26
65
  query['ok'] && query[host]['state'] == 'running'
27
66
  end
28
67
 
29
- def self.increaselife(id, amount)
30
- amount = 100 if amount.nil?
68
+ def self.increaselife(id, amount = nil)
69
+ amount ||= Config.get_config_setting('increaselife').to_i
70
+ return if amount < 1
31
71
  hosts = Hosts.get_hosts_from_id(id)
32
72
  hosts.each do |host|
33
- if lifetime = get_current_lifetime(host)
73
+ if (lifetime = get_current_lifetime(host))
34
74
  lifetime += amount
35
- print "#{host} to #{lifetime} hours: "
36
- puts `floaty modify #{host} --lifetime #{lifetime}`.split("\n")[0]
75
+ output = Floaty.floaty_cmd("modify #{host} --lifetime #{lifetime} --service vmpooler")
76
+ if output =~ /Successfully modified/
77
+ puts "#{host} lifetime set to #{lifetime} hours".green
78
+ else
79
+ puts "Error setting VM lifetime: #{output}".red
80
+ end
37
81
  else
38
82
  puts "Could not query host #{host}".red
39
83
  end
40
84
  end
41
85
  end
42
86
 
87
+ ### Snapshots ###
43
88
  def self.snapshot(id, snaptag)
44
- snaptag = 'Blank Snaptag' unless snaptag
89
+ clr = Config.get_config_setting('vertical_snapshot_status')
90
+ clr = clr.to_s.downcase == 'true'
91
+ snaptag ||= 'Blank Snaptag'
45
92
  hosts = Hosts.get_hosts_from_id(id)
46
93
  shas = {}
94
+ # There's probably a better way to do this...
95
+ puts `tput clear` if clr
96
+
47
97
  hosts.each do |host|
48
- print "#{host}: "
49
- result = `floaty snapshot #{host}`
50
- message = result.split("\n")[0]
51
- answer = eval(result.sub(message,''))
98
+ result = Floaty.floaty_cmd("snapshot #{host} --service vmpooler")
99
+ answer = Floaty.parse_floaty_json_output(result)
52
100
  sha = answer[host]['snapshot']
53
- puts sha
101
+ puts "#{host}: #{sha}"
54
102
  shas[host] = sha
55
103
  end
56
104
 
105
+ data = Config.load_data
106
+ data['snapshots'] ||= {}
107
+ data['snapshots'][id] ||= {}
108
+ data['snapshots'][id][snaptag] = shas
109
+ Config.write_data(data)
110
+
111
+ puts
57
112
  puts 'Waiting for snapshots to appear in floaty query...'
58
113
  alldone = false
59
- while !alldone do
114
+ until alldone
115
+ puts `tput cup #{hosts.count + 2}` if clr
60
116
  alldone = true
61
- print "\r"
117
+ print "\r" unless clr
62
118
  hosts.each do |host|
63
- answer = eval(`floaty query #{host}`)[host]
119
+ answer = query(host)[host]
64
120
  done = answer.keys.include?('snapshots') && answer['snapshots'].include?(shas[host])
65
- status = done ? 'Done' : 'Wait'
66
- print "* #{host}: #{status} *"
121
+ status = done ? 'Done'.green : 'Wait'.yellow
122
+ if clr
123
+ puts "* %s #{status} *" % "#{host}:".ljust(16)
124
+ else
125
+ print "* %s #{status} *" % "#{host}:".ljust(16)
126
+ end
67
127
  alldone &= done
68
128
  end
69
129
  sleep(1)
70
130
  end
71
131
  puts ''
72
-
73
- data = Conf.load_data
74
- data['snapshots'] ||= {}
75
- data['snapshots'][id] ||= {}
76
- data['snapshots'][id][snaptag] = shas
77
- Conf.write_data(data)
78
132
  end
79
133
 
80
134
  def self.getsnapshot(id, snaptag)
81
- data = Conf.load_data
135
+ data = Config.load_data
82
136
  data['snapshots'][id][snaptag]
83
137
  end
84
138
 
85
139
  def self.snaptag_exists?(id, snaptag)
86
- data = Conf.load_data
87
- exists = data['snapshots'].keys.include?(id) ? data['snapshots'][id].keys.include?(snaptag) : false
140
+ data = Config.load_data
141
+ data['snapshots'].keys.include?(id) ? data['snapshots'][id].keys.include?(snaptag) : false
142
+ end
143
+
144
+ def self.snaplist(tag)
145
+ data = Config.load_data
146
+ return [] if !data['snapshots'].keys.include?(tag)
147
+ data['snapshots'][tag].keys
88
148
  end
89
149
 
90
150
  def self.revert(id, snaptag)
91
- snaptag = 'Blank Snaptag' unless snaptag
151
+ snaptag ||= 'Blank Snaptag'
92
152
  shas = VM.getsnapshot(id, snaptag)
93
153
  shas.each do |host, sha|
94
154
  print "#{host}: #{sha} --- "
95
- puts `floaty revert #{host} #{sha}`
155
+ puts Floaty.floaty_cmd("revert #{host} #{sha} --service vmpooler")
96
156
  end
97
157
  puts 'Waiting 10 seconds for revert to take effect'
98
158
  sleep(10)
99
159
  end
100
160
 
101
- def self.get_vm(platform='centos-7-x86_64')
102
- response = `floaty get #{platform} 2>&1`
103
- raise "Error obtaining a VM: #{response}" if response.include?('error')
104
- return response.split(' ')[1].split('.')[0]
161
+ ### Get a VM from floaty ###
162
+ def self.get_vm(platform: 'centos-7-x86_64', force_abs: false)
163
+ if !find_pooled_platforms.include?(platform.gsub('-pixa4','')) || force_abs
164
+ response = Floaty.floaty_cmd("get #{platform} --service abs --priority 1", use_pty: true)
165
+ response.chomp.split('- ')[1].split[0].split('.')[0]
166
+ else
167
+ response = Floaty.floaty_cmd("get #{platform} --service vmpooler")
168
+ raise "Error obtaining a VM: #{response}" if response.include?('error')
169
+ response.split[1].split('.')[0]
170
+ end
105
171
  end
106
-
107
172
  end
108
-