foreman-architect 0.1.0

data/lib/architect/version.rb

@@ -0,0 +1,3 @@
+module ForemanArchitect
+  VERSION = "0.1.0"
+end

data/lib/foreman_vm.rb

@@ -0,0 +1,409 @@
+#
+# Manage a cluster of KVM hosts using Foreman
+#
+# Author:	Mark Heily <mark.heily@bronto.com>
+# 
+class ForemanVM
+
+require 'foreman_vm/allocator'
+require 'foreman_vm/buildspec'
+require 'foreman_vm/cluster'
+require 'foreman_vm/console'
+require 'foreman_vm/config'
+require 'foreman_vm/domain'
+require 'foreman_vm/foreman_api'
+require 'foreman_vm/getopt'
+require 'foreman_vm/hypervisor'
+require 'foreman_vm/workqueue'
+require 'foreman_vm/util'
+
+
+require 'bundler/setup'
+require 'libvirt'
+require 'logger'
+require 'resolv'
+
+attr_reader :action, :config
+
+attr_accessor :log, :cluster, :workqueue, :console
+
+# Check if a VM with a given +hostname+ exists.
+def vm_exists?(hostname)
+  @cluster.find(hostname).nil? ? false : true
+end
+   
+def compute_resource=(txt)
+  # KLUDGE: use shortnames because Foreman does
+  @buildspec['compute_resource'] = txt.gsub(/\..*/, '')
+end
+
+# Get the FQDN of the VM
+# 
+def fqdn
+  @buildspec['name'] + '.' + @buildspec['domain']
+end
+
+# Get the hypervisor that hosts the the VM
+# 
+def hypervisor
+  @buildspec['compute_resource'] ||= @cluster.find(fqdn)
+end
+
+# Set the VM hostname
+# 
+def name=(arg)
+  if arg =~ /(.*?)\.(.*)/
+    @hostname = $1
+    @buildspec['name'] = $1
+    @buildspec['domain'] = $2
+  else
+    @hostname = arg
+    @buildspec['name'] = arg
+  end
+end
+
+# Update the build specification
+# 
+def buildspec=(spec)
+  @buildspec.merge! spec
+end
+
+# Ask the user for their Foreman password
+#
+def ask_password
+  printf 'Enter your Foreman password: '
+  system "stty -echo"
+  @password = STDIN.gets.chomp
+  system "stty echo"
+end
+
+# Run a virsh command
+# 
+def virsh(command,xml=nil)
+  
+  # KLUDGE: virsh requires the FQDN of the hypervisor
+  #         while foreman uses shortname
+  hypervisor_fqdn = hypervisor
+  unless hypervisor_fqdn =~ /\./
+    hypervisor_fqdn += '.' + `dnsdomainname`.chomp
+  end
+
+  ENV['LIBVIRT_AUTH_FILE'] = File.dirname(__FILE__) + '/../conf/auth.conf'
+  buf = "virsh -c qemu+tcp://#{hypervisor_fqdn}/system " + command
+  @log.info "running virsh #{command} on #{hypervisor_fqdn}"
+  if xml.nil?
+    res = `#{buf}`
+    raise "virsh command returned #{$?}: #{buf}" if $? != 0;
+  else
+    f = IO.popen(buf, 'w')
+    f.puts xml
+    f.close
+    #XXX-FIXME error check
+    res = '(FIXME -- NEED TO CAPTURE STDOUT)'
+  end
+  return res
+end
+
+
+# Destroy a virtual machine
+#
+def delete
+  # Check if it uses libgfapi. If so, we need to disable it
+#  if self.dumpxml =~ /protocol='gluster'/
+#    self.stop
+#    self.disable_libgfapi
+#  end
+
+  # Call 'virsh destroy' to kill the VM
+  begin
+    @foreman_api.request(:delete, "/hosts/#{self.fqdn}", {'id' => self.fqdn})
+  rescue
+    # Try again, to workaround a bug where the first deletion fails..
+    @foreman_api.request(:delete, "/hosts/#{self.fqdn}", {'id' => self.fqdn})
+
+    # When the bug hits, the volume is left behind. force it's deletion 
+    #
+    # Horrible kludge: hardcoded something that will allow the delete to work
+    gvol = @cluster.member(@cluster.members[0]).storage_pool('gvol')
+    gvol.refresh
+    gvol.volume("#{self.fqdn}-disk1").delete
+  end
+end
+
+# View the list of deferred jobs
+# (TODO: stop leaking Beanstalkd details)
+def job_status
+  'job status: ' + @workqueue.jobs
+end
+
+# Submit a deferred job
+#
+def defer(action)
+  @workqueue.enqueue({
+    'user' => @user,
+    'action' => action,
+    'buildspec' => @buildspec,
+    'api_version' => 1,
+  })
+end
+
+# Rebuild a virtual machine
+#
+def rebuild
+    # Determine the hypervisor 
+    @buildspec['compute_resource'] = @cluster.find(fqdn)
+
+    refresh_storage_pool(@buildspec)
+
+    # Destroy the puppet certificate
+    #  XXX-KLUDGE
+    system "curl -X DELETE http://util-stage-001.brontolabs.local:8443/puppet/ca/#{fqdn}"
+
+# TODO: if spec['_copy']...
+  if @buildspec['_clone']
+    virsh "destroy #{self.fqdn}" if domstate == 'running'
+    @buildspec['disk_format'] = 'qcow2'
+    enable_libgfapi @config.glusterfs_server
+    clone_volume
+    start()
+  else
+    # Build via Kickstart:
+    #
+    
+    # Destroy the puppet certificate, and enable build at the next boot
+    @foreman_api.request(:put, "/hosts/#{fqdn}", { 'host' => {'build' => '1' }})
+
+    # Call 'virsh destroy' to kill the VM, then power it back on
+    stop
+    sleep 3
+    start
+  end
+end
+
+# Create storage and attach it to the virtual machine
+#
+def create_storage
+  pool = @config.storage_pool
+  host = @cluster.member(@cluster.find(fqdn))
+  disk_number = 0
+  @buildspec['disk_capacity'].split(',').each do |disk_size|
+    disk_number += 1
+    capacity = normalize_memory(disk_size)
+    puts "create #{fqdn} - #{capacity}"
+    basedir = '/gvol/images' #XXX-HARDCODED
+    path = basedir + '/' + fqdn + '-disk' + disk_number.to_s
+    @cluster.guest(fqdn).add_libgfapi_volume(
+      "gvol/images/#{fqdn}-disk#{disk_number.to_s}",
+      @config.glusterfs_server,
+      disk_number 
+    )
+    host.storage_pool(pool).create_volume(path, capacity)
+  end
+end
+
+# Build a new virtual machine
+#
+def create
+  spec = @buildspec
+
+  # If no compute resource is given, select the one with the most
+  # available memory.
+  spec['compute_resource'] ||= @cluster.best_fit(spec['name'], normalize_memory(spec['memory'])).gsub(/\..*/, '')
+
+  if spec['_clone'] == true
+   #FIXME: does not belong here
+   spec['disk_format'] = 'qcow2'
+   spec['storage_pool'] = 'gvol'
+  end
+
+  refresh_storage_pool(spec)
+
+  rec = {
+      'domain_id' => @foreman_api.get_id('domains', spec['domain']),
+      'subnet_id' => @foreman_api.get_id('subnets', spec['subnet']),
+      'name' => spec['name'],
+      'build' => "true",
+      'enabled' => "true",
+
+      # XXX-FIXME: hardcoded, should not use this..
+#'compute_profile_id' => '5',
+
+      'compute_resource_id' => @foreman_api.get_id('compute_resources', spec['compute_resource']) ,
+      'environment_id' => @foreman_api.get_id('environments', spec['environment']),
+      'managed' => true, 
+      'hostgroup_id' => @foreman_api.get_id('hostgroups', spec['hostgroup'], 'title'),
+      'provision_method' => spec['provision_method'],
+      'compute_attributes' => {
+        'memory' => normalize_memory(spec['memory']), 
+        'image_id' => spec['image_id'],
+        'nics_attributes' => {
+        '0' => {
+        'bridge' => spec['network_interface'],
+        'model' => 'virtio', 
+        'type' => 'bridge',
+      }
+      }, 
+      'interfaces_attributes' => {
+        '0' => {
+          'bridge' => spec['network_interface'],
+          'model' => 'virtio',
+          'type' => 'bridge'
+        },
+      },
+      'cpus' => spec['cpus'], 
+      'start' => '0', 
+      'volumes_attributes' => {
+        '0' => {
+          'capacity' => spec['disk_capacity'], 
+          'pool_name' => spec['storage_pool'],
+          'format_type' => spec['disk_format'],
+        }
+      }
+      } 
+  }
+  if spec['organization']
+    rec['organization_id'] = @foreman_api.get_id('organizations', spec['organization'], 'title')
+  end
+  if spec['owner']
+    rec['owner_id'] = @foreman_api.get_id('users', spec['owner'], 'login')
+  end
+  if spec['provision_method'] == 'image'
+    rec['image_id'] = 3
+    rec['image_name'] = 'centos6-generic'
+    rec['compute_attributes']['image_id'] = spec['image_id']
+  end
+  if spec['_clone'] or spec['_copy']
+    rec['build'] = false
+  end
+
+  # configure the volumes
+  # TODO: use a BuildSpec object for everything.
+  spec2 = ForemanAP::BuildSpec.new
+  spec2.disk_capacity = spec['disk_capacity']
+  spec2.storage_pool = spec['storage_pool']
+  spec2.disk_format = spec['disk_format']
+  ### XXX-TESTING:
+    rec['compute_attributes']['volumes_attributes'] = {}
+    ###rec['compute_attributes']['volumes_attributes'] = spec2.to_foreman_api['compute_attributes']['volumes_attributes']
+#pp rec
+  #raise 'FIXME'
+
+  @foreman_api.request(:post, "/hosts", rec)
+
+  raise 'FIXME - not implemented' if spec['_clone'] == true or ['spec_copy'] == true
+
+  # Create volumes and attach them to the VM
+  create_storage
+
+  #DEADWOOD:
+  #####if spec['_clone'] == true
+  #####  clone_volume
+  #####elsif spec['_copy'] == true
+  #####  copy_volume
+  #####else
+  #####  # crude way to fix the permissions
+  #####  wipe_volume
+  #####end
+  #####enable_libgfapi if spec['_libgfapi']
+
+  #FIXME: implement this
+  #raise 'Duplicate IP address' if ip_address_in_use? $GET_THE_ADDRESS_HERE
+
+  @cluster.guest(fqdn).start
+
+  # Attach to the console
+  if spec['console']
+    guest = spec['name'] + '.' + spec['domain']
+    host = spec['compute_resource'] + '.' + spec['domain']
+    console_attach(host, guest)
+  end
+end
+
+  def initialize
+  @log = Logger.new(STDERR)
+  @action = nil
+  @config = ForemanAP::Config.new
+  @cluster = ForemanAP::Cluster.new(
+		  @config.hypervisors, 
+		  @config.libvirt_user, 
+		  @config.libvirt_password)
+#FIXME: reenable this: @workqueue = ForemanAP::Workqueue.new('foreman-vm')
+  @console = ForemanAP::ConsoleViewer.new(@cluster)
+
+
+  # TODO: transition to using @config.foreman_user everywhere
+  #          instead of @user/@password
+  #          
+  if @config.foreman_user
+    @user = @config.foreman_user
+    @password = @config.foreman_password
+  else
+    @user = ENV['USER']
+    @password = nil
+  end
+  @foreman_api = ForemanAP::ForemanAPI.new(@config.foreman_uri, @user, @password)
+
+  # Build specifications
+  @buildspec = {
+    'cpus' => '1',            # Number of vCPUs
+    'memory' => '536870912',  # Memory, in bytes (default: 512MB)
+    'disk_capacity' => '20G',
+    'disk_format' => 'raw',
+    'storage_pool' => 'vm-corp-004',
+    'domain' => `dnsdomainname`.chomp,
+    'network_interface' => 'vnet0.201',
+    'provision_method' => 'build',
+    'owner' => 'nil',
+  #  'image_id' => '/srv/images/centos6-generic-template.qcow2',
+    'console' => false,
+
+    #Hidden for testing purposes
+    '_clone' => false,
+    '_copy' => false,
+    '_libgfapi' => true,
+    '_disk_backing_file' => '/var/lib/libvirt/images/centos6-dude-template.qcow2',
+  }
+end
+
+private
+
+# Allow memory to be specified with a G/M/K suffix
+# Returns the number of bytes.
+def normalize_memory(s)
+  if s =~ /G$/
+    ($`.to_i * (1024**3)).to_s
+  elsif s =~ /M$/
+    ($`.to_i * (1024**2)).to_s
+  elsif s =~ /K$/
+    ($`.to_i * 1024).to_s
+  else
+    s
+  end
+end
+
+# Refresh a storage pool to detect changes made by other hypervisors
+def refresh_storage_pool(spec)
+  fqdn = spec['compute_resource']
+  fqdn += '.' + spec['domain'] unless fqdn =~ /\./
+  @log.debug "refreshing the #{@config.storage_pool} pool on #{fqdn}"
+  @cluster.member(fqdn).storage_pool(@config.storage_pool).refresh
+end
+
+# Returns true if an IP address is already in use.
+# Verify that Foreman did not allocate an IP address that is currently in use
+# This is some extra sanity checking that should be handled in Foreman, but
+# it has failed to detect conflicts in the past.
+def ip_address_in_use?(ipaddr)
+  raise ArgumentError if ipaddr =~ /[^0-9.]/
+  has_ping = system "ping -c 3 -W5 #{ipaddr} > /dev/null"
+  begin
+    name = Resolv.new.getname ipaddr
+    has_dns = true
+  rescue Resolv::ResolvError
+    has_dns = false
+  end
+  #puts "has_ping=#{has_ping.to_s} has_dns=#{has_dns.to_s}"
+  return (has_ping or has_dns)
+end
+
+end

data/lib/foreman_vm/allocator.rb

@@ -0,0 +1,49 @@
+
+module ForemanAP
+  # Adds guests to correct hypervisor.
+  class Allocator
+    def initialize
+      @host = []
+    end
+
+    # Add information about a hypervisor
+    #
+    # [+name+] the name of the host
+    # [+free_memory+] how much free memory, in bytes
+    # [+guests+] a list of the names of each VM on the host
+    #
+    def add_host(name,free_memory,guests)
+      @host.push({
+        :name => name,
+        :free_memory => free_memory,
+        :guests => guests
+      })
+    end
+
+    # Find the best hypervisor that meets the allocation policy
+    #
+    # [+name+] the name of the guest
+    # [+memory+] the amount of memory the guest needs, in bytes
+    #
+    # Returns the name of the most suitable hypervisor.
+    # If no hypervisor is suitable, it returns nil.
+    def add_guest(name,memory)
+      # Sort by most free memory
+      host_tmp = @host.sort_by { |x| -x[:free_memory] }
+      # Delete from list if not enough memory for guest
+      host_tmp.delete_if { |x| x[:free_memory] < memory.to_i }
+      # Check if guest already exists and returns nil if so
+      @host.each { |x| return nil if x[:guests].include?(name) }
+      # Delete from list if vm type exists, unless it deletes all then return best host
+      pre = name.gsub(/[0-9]/, '')
+      suitable = host_tmp.dup.delete_if { |x| x[:guests].grep(/^#{pre}/).any? }
+      if suitable.any? 
+        return suitable[0][:name]
+      elsif host_tmp.any? 
+        return host_tmp[0][:name]
+      else
+        return nil 
+      end
+    end
+  end
+end