beaker-openstack 2.0.0 → 3.0.0

data/lib/beaker/hypervisor/openstack.rb

@@ -1,451 +1,445 @@
 module Beaker
-  #Beaker support for OpenStack
-  #This code is EXPERIMENTAL!
-  #Please file any issues/concerns at https://github.com/puppetlabs/beaker/issues
+  # Beaker support for OpenStack
+  # Please file any issues/concerns at https://github.com/voxpupuli/beaker-openstack/issues
+
+  # Additional volumes created via openstack_volume_support are preserved and not deleted by cleanup
   class Openstack < Beaker::Hypervisor
 
-    SLEEPWAIT = 5
-
-    #Create a new instance of the OpenStack hypervisor object
-    #@param [<Host>] openstack_hosts The array of OpenStack hosts to provision
-    #@param [Hash{Symbol=>String}] options The options hash containing configuration values
-    #@option options [String] :openstack_api_key The key to access the OpenStack instance with (required)
-    #@option options [String] :openstack_username The username to access the OpenStack instance with (required)
-    #@option options [String] :openstack_auth_url The URL to access the OpenStack instance with (required)
-    #@option options [String] :openstack_tenant The tenant to access the OpenStack instance with (either this or openstack_project_name is required)
-    #@option options [String] :openstack_project_name The project name to access the OpenStack instance with (either this or openstack_tenant is required)
-    #@option options [String] :openstack_project_id The project id to access the OpenStack instance with (alternative to openstack_project_name)
-    #@option options [String] :openstack_user_domain The user domain name to access the OpenStack instance with
-    #@option options [String] :openstack_user_domain_id The user domain id to access the OpenStack instance with (alternative to openstack_user_domain)
-    #@option options [String] :openstack_project_domain The project domain to access the OpenStack instance with
-    #@option options [String] :openstack_project_domain_id The project domain id to access the OpenStack instance with (alternative to openstack_project_domain)
-    #@option options [String] :openstack_region The region that each OpenStack instance should be provisioned on (optional)
-    #@option options [String] :openstack_network The network that each OpenStack instance should be contacted through (required)
-    #@option options [Bool] :openstack_floating_ip Whether a floating IP should be allocated (required)
-    #@option options [String] :openstack_keyname The name of an existing key pair that should be auto-loaded onto each
-    #@option options [Hash] :security_group An array of security groups to associate with the instance
-    #                                            OpenStack instance (optional)
-    #@option options [String] :jenkins_build_url Added as metadata to each OpenStack instance
-    #@option options [String] :department Added as metadata to each OpenStack instance
-    #@option options [String] :project Added as metadata to each OpenStack instance
-    #@option options [Integer] :timeout The amount of time to attempt execution before quiting and exiting with failure
+    SLEEPWAIT = 5  # Seconds to wait between retry attempts for VM boot
+
+    # Create a new OpenStack hypervisor object
+    #
+    # @param [Array<Host>] openstack_hosts Array of hosts to provision
+    # @param [Hash{Symbol=>Object}] options Configuration options:
+    # @option options [String] :openstack_api_key Required API key
+    # @option options [String] :openstack_username Required username
+    # @option options [String] :openstack_auth_url Required auth URL
+    # @option options [String] :openstack_project_name Project name (v3) or nil
+    # @option options [String] :openstack_project_id Project ID (v3) or nil
+    # @option options [String] :openstack_user_domain Optional user domain for v3
+    # @option options [String] :openstack_user_domain_id Optional user domain ID for v3
+    # @option options [String] :openstack_project_domain Optional project domain for v3
+    # @option options [String] :openstack_project_domain_id Optional project domain ID for v3
+    # @option options [String] :openstack_region The region that each OpenStack instance should be provisioned on (optional)
+    # @option options [String] :openstack_network Required network for VM
+    # @option options [Bool] :openstack_floating_ip Whether to assign a floating IP
+    # @option options [String] :floating_ip_pool Required floating network ID when floating IPs are enabled
+    # @option options [Bool] :openstack_volume_support Whether to provision additional volumes
+    # @option options [String] :openstack_keyname Optional pre-existing keypair name
+    # @option options [Array<String>] :security_group Optional security groups
+    # @option options [Integer] :timeout Timeout in seconds for VM boot
+    # @option options [String] :jenkins_build_url Optional metadata
+    # @option options [String] :department Optional metadata
+    # @option options [String] :project Optional metadata
     def initialize(openstack_hosts, options)
       require 'fog/openstack'
+
       @options = options
-      @logger = options[:logger]
-      @hosts = openstack_hosts
+      @logger  = options[:logger]
+      @hosts   = openstack_hosts
+
+      # Initialize shared resources and mutexes for thread safety
       @vms = []
+      @floating_ips = []
+      @vms_mutex = Mutex.new
+      @fip_mutex = Mutex.new
+      @keypairs_mutex = Mutex.new
+      @cleanup_mutex = Mutex.new
+      @cleanup_ran = false
+
+      # Track keys we create so we only delete what we own
+      @ephemeral_keypairs = []
+
+      # Required options
+      raise 'You must specify :openstack_api_key'      unless @options[:openstack_api_key]
+      raise 'You must specify :openstack_username'     unless @options[:openstack_username]
+      raise 'You must specify :openstack_auth_url'     unless @options[:openstack_auth_url]
+      raise 'You must specify :openstack_network'      unless @options[:openstack_network]
+      raise 'You must specify :openstack_floating_ip (true/false)' if @options[:openstack_floating_ip].nil?
+
+      # Floating IP pool is required only if floating IPs are enabled
+      raise 'You must specify :floating_ip_pool when using floating IPs' if @options[:openstack_floating_ip] && !@options[:floating_ip_pool]
+
+      # Keystone version detection
+      # Matches both /v3 and /v3/ endings to avoid false negatives
+      is_v3 = @options[:openstack_auth_url].match?(%r{/v3/?$})
+
+      # Enforce Keystone v3 only (v2 is no longer supported)
+      raise 'Keystone v2 is no longer supported. Please use a /v3 auth URL.' unless is_v3
+
+      # project_name or project_id required
+      raise 'Specify project_name or project_id' unless @options[:openstack_project_name] || @options[:openstack_project_id]
+
+      # cannot specify both
+      raise 'Do not mix project_name and project_id' if @options[:openstack_project_name] && @options[:openstack_project_id]
+
+      # user_domain XOR user_domain_id
+      raise 'Specify either :openstack_user_domain or :openstack_user_domain_id, not both' if @options[:openstack_user_domain] && @options[:openstack_user_domain_id]
+
+      # project_domain XOR project_domain_id
+      raise 'Specify either :openstack_project_domain or :openstack_project_domain_id, not both' if @options[:openstack_project_domain] && @options[:openstack_project_domain_id]
+
+      # fog-openstack limitation: do not mix _id and non-_id fields
+      if (@options[:openstack_project_name] ||
+          @options[:openstack_user_domain] ||
+          @options[:openstack_project_domain]) &&
+         (@options[:openstack_project_id] ||
+          @options[:openstack_user_domain_id] ||
+          @options[:openstack_project_domain_id])
+        raise 'Do not mix _id and non-_id values for project/user domains due to fog-openstack limitations'
+      end
 
-      raise 'You must specify an Openstack API key (:openstack_api_key) for OpenStack instances!' unless @options[:openstack_api_key]
-      raise 'You must specify an Openstack username (:openstack_username) for OpenStack instances!' unless @options[:openstack_username]
-      raise 'You must specify an Openstack auth URL (:openstack_auth_url) for OpenStack instances!' unless @options[:openstack_auth_url]
-      raise 'You must specify an Openstack network (:openstack_network) for OpenStack instances!' unless @options[:openstack_network]
-      raise 'You must specify whether a floating IP (:openstack_floating_ip) should be used for OpenStack instances!' unless !@options[:openstack_floating_ip].nil?
-
-      is_v3 = @options[:openstack_auth_url].include?('/v3/')
-      raise 'You must specify an Openstack tenant (:openstack_tenant) for OpenStack instances!' if !is_v3 and !@options[:openstack_tenant]
-      raise 'You must specify an Openstack project name (:openstack_project_name) or Openstack project id (:openstack_project_id) for OpenStack instances!' if is_v3 and (!@options[:openstack_project_name] and !@options[:openstack_project_id])
-      raise 'You must specify either Openstack project name (:openstack_project_name) or Openstack project id (:openstack_project_id) not both!' if is_v3 and (@options[:openstack_project_name] and @options[:openstack_project_id])
-      raise 'You may specify either Openstack user domain (:openstack_user_domain) or Openstack user domain id (:openstack_user_domain_id) not both!' if is_v3 and (@options[:openstack_user_domain] and @options[:openstack_user_domain_id])
-      raise 'You may specify either Openstack project domain (:openstack_project_domain) or Openstack project domain id (:openstack_project_domain_id) not both!' if is_v3 and (@options[:openstack_project_domain] and @options[:openstack_project_domain_id])
-      raise 'Invalid option specified: v3 API expects :openstack_project_name or :openstack_project_id, not :openstack_tenant for OpenStack instances!' if is_v3 and @options[:openstack_tenant]
-      raise 'Invalid option specified: v2 API expects :openstack_tenant, not :openstack_project_name or :openstack_project_id for OpenStack instances!' if !is_v3 and (@options[:openstack_project_name] or @options[:openstack_project_id])
-      # Ensure that _id and non _id params are not mixed (due to bug in fog-openstack)
-      raise 'You must not mix _id values non _id (name) values. Please use the same type for (:openstack_project_), (:openstack_user_domain) and (:openstack_project_domain)!' if is_v3 and (@options[:openstack_project_name] or @options[:openstack_user_domain] or @options[:openstack_project_domain]) and (@options[:openstack_project_id] or @options[:openstack_user_domain_id] or @options[:openstack_project_domain_id])
-
-      # Keystone version 3 changed the parameter names
-      if !is_v3
-        extra_credentials = {:openstack_tenant => @options[:openstack_tenant]}
-      else
+      # Build credential scope depending on Keystone version
+      extra_credentials =
         if @options[:openstack_project_id]
-          extra_credentials = {:openstack_project_id => @options[:openstack_project_id]}
+          { openstack_project_id: @options[:openstack_project_id] }
         else
-          extra_credentials = {:openstack_project_name => @options[:openstack_project_name]}
+          { openstack_project_name: @options[:openstack_project_name] }
         end
-      end
 
-      # Common keystone authentication credentials
+      # Base credentials (no duplicate tenant/project fields)
       @credentials = {
-        :provider           => :openstack,
-        :openstack_auth_url => @options[:openstack_auth_url],
-        :openstack_api_key  => @options[:openstack_api_key],
-        :openstack_username => @options[:openstack_username],
-        :openstack_tenant   => @options[:openstack_tenant],
-        :openstack_region   => @options[:openstack_region],
+        provider: :openstack,
+        openstack_auth_url: @options[:openstack_auth_url],
+        openstack_api_key: @options[:openstack_api_key],
+        openstack_username: @options[:openstack_username],
+        openstack_region: @options[:openstack_region]
       }.merge(extra_credentials)
 
-      # Keystone version 3 requires users and projects to be scoped
-      if is_v3
-        if @options[:openstack_user_domain_id]
-          @credentials[:openstack_user_domain_id] = @options[:openstack_user_domain_id]
-        else
-          @credentials[:openstack_user_domain]    = @options[:openstack_user_domain] || 'Default'
-        end
-        if @options[:openstack_project_domain_id]
-          @credentials[:openstack_project_domain_id] = @options[:openstack_project_domain_id]
-        else
-          @credentials[:openstack_project_domain]    = @options[:openstack_project_domain] || 'Default'
-        end        
-      end
+      # Keystone v3 domain scoping
+      @credentials[:openstack_user_domain_id] = @options[:openstack_user_domain_id] if @options[:openstack_user_domain_id]
+      @credentials[:openstack_user_domain] ||= @options[:openstack_user_domain] || 'Default'
 
-      @compute_client ||= Fog::Compute.new(@credentials)
+      @credentials[:openstack_project_domain_id] = @options[:openstack_project_domain_id] if @options[:openstack_project_domain_id]
+      @credentials[:openstack_project_domain] ||= @options[:openstack_project_domain] || 'Default'
 
-      if not @compute_client
-        raise "Unable to create OpenStack Compute instance (api key: #{@options[:openstack_api_key]}, username: #{@options[:openstack_username]}, auth_url: #{@options[:openstack_auth_url]}, tenant: #{@options[:openstack_tenant]}, project_name: #{@options[:openstack_project_name]})"
-      end
+      # Create clients
+      # These are created once during initialization (no memoization needed)
+      @compute_client = Fog::Compute.new(@credentials)
+      raise "Unable to create OpenStack Compute instance" unless @compute_client
 
-      @network_client ||= Fog::Network.new(@credentials)
-
-      if not @network_client
-        raise "Unable to create OpenStack Network instance (api key: #{@options[:openstack_api_key]}, username: #{@options[:openstack_username]}, auth_url: #{@options[:openstack_auth_url]}, tenant: #{@options[:openstack_tenant]}, project_name: #{@options[:openstack_project_name]})"
-      end
+      @network_client = Fog::Network.new(@credentials)
+      raise "Unable to create OpenStack Network instance" unless @network_client
 
-      # Validate openstack_volume_support setting value, reset to boolean if passed via ENV value string
-      @options[:openstack_volume_support] = true  if @options[:openstack_volume_support].to_s.match(/\btrue\b/i)
-      @options[:openstack_volume_support] = false if @options[:openstack_volume_support].to_s.match(/\bfalse\b/i)
-      [true,false].include? @options[:openstack_volume_support] or raise "Invalid openstack_volume_support setting, current: @options[:openstack_volume_support]"
+      # --- openstack_volume_support normalization ---
+      # Accepts string or boolean input, but only true/false are valid outcomes
+      val = @options[:openstack_volume_support].to_s.downcase
+      @options[:openstack_volume_support] = true  if val == "true"
+      @options[:openstack_volume_support] = false if val == "false"
 
+      raise "Invalid openstack_volume_support setting" unless [true, false].include?(@options[:openstack_volume_support])
     end
 
-    #Provided a flavor name return the OpenStack id for that flavor
-    #@param [String] f The flavor name
-    #@return [String] Openstack id for provided flavor name
-    def flavor f
-      @logger.debug "OpenStack: Looking up flavor '#{f}'"
-      @compute_client.flavors.find { |x| x.name == f } || raise("Couldn't find flavor: #{f}")
-    end
+    # @!group Provision Methods
+    # Main provisioning entrypoint
+    def provision
+      if @options[:create_in_parallel]
+        Thread.abort_on_exception = true
+        @logger.notify "Provisioning OpenStack in parallel"
+        provision_parallel
+      else
+        @logger.notify "Provisioning OpenStack sequentially"
+        provision_sequential
+      end
 
-    #Provided an image name return the OpenStack id for that image
-    #@param [String] i The image name
-    #@return [String] Openstack id for provided image name
-    def image i
-      @logger.debug "OpenStack: Looking up image '#{i}'"
-      @compute_client.images.find { |x| x.name == i } || raise("Couldn't find image: #{i}")
+      hack_etc_hosts @hosts, @options
     end
 
-    #Provided a network name return the OpenStack id for that network
-    #@param [String] n The network name
-    #@return [String] Openstack id for provided network name
-    def network n
-      @logger.debug "OpenStack: Looking up network '#{n}'"
-      @network_client.networks.find { |x| x.name == n } || raise("Couldn't find network: #{n}")
+    def provision_parallel
+      threads = @hosts.map { |host| Thread.new { create_instance_resources(host) } }
+      threads.each(&:join)
     end
 
-    #Provided an array of security groups return that array if all
-    #security groups are present
-    #@param [Array] sgs The array of security group names
-    #@return [Array] The array of security group names
-    def security_groups sgs
-      for sg in sgs
-        @logger.debug "Openstack: Looking up security group '#{sg}'"
-        @compute_client.security_groups.find { |x| x.name == sg } || raise("Couldn't find security group: #{sg}")
-        sgs
-      end
+    def provision_sequential
+      @hosts.each { |host| create_instance_resources(host) }
     end
 
-    # Create a volume client on request
-    # @return [Fog::OpenStack::Volume] OpenStack volume client
-    def volume_client_create
-      @volume_client ||= Fog::Volume.new(@credentials)
-      unless @volume_client
-        raise "Unable to create OpenStack Volume instance"\
-          " (api_key: #{@options[:openstack_api_key]},"\
-        " username: #{@options[:openstack_username]},"\
-        " auth_url: #{@options[:openstack_auth_url]},"\
-        " tenant: #{@options[:openstack_tenant]})"
+    # Create all resources required for a single VM
+    def create_instance_resources(host)
+      @logger.notify "Provisioning #{host.name}"
+
+      # Revert to standard Beaker managed style (10 character random string) for VM hostname
+      host[:vmhostname] = ('a'..'z').to_a.sample(10).join
+
+      floating_ip = nil
+      if @options[:openstack_floating_ip]
+        floating_ip = get_floating_ip
+        # Track the Floating IP object immediately for cleanup safety
+        @fip_mutex.synchronize { @floating_ips << floating_ip }
+
+        # Capture the actual IP string for connectivity
+        actual_ip = floating_ip.respond_to?(:floating_ip_address) ? floating_ip.floating_ip_address : (floating_ip.respond_to?(:ip) ? floating_ip.ip : floating_ip.address)
+        host[:ip] = actual_ip
       end
-    end
 
-    # Get a hash of volumes from the host
-    def get_volumes host
-      return host['volumes'] if host['volumes']
-      {}
-    end
+      create_or_associate_keypair(host, host[:vmhostname])
+
+      server_opts = {
+        name: host[:vmhostname],
+        flavor_ref: flavor(host[:flavor]).id,
+        nics: [{ 'net_id' => network(@options[:openstack_network]).id }],
+        key_name: host[:keyname],
+        security_groups: @options[:security_group] ? security_groups(@options[:security_group]) : nil,
+        user_data: host[:user_data] || "#cloud-config\nmanage_etc_hosts: true\n"
+      }
 
-    # Get the API version
-    def get_volume_api_version
-      case @volume_client
-      when Fog::Volume::OpenStack::V1
-        1
+      if boot_from_volume?(host)
+        server_opts[:block_device_mapping_v2] = [{
+          uuid: image(host[:image]).id,
+          source_type: "image",
+          destination_type: "volume",
+          volume_size: host['root_volume']['size'].to_i,
+          delete_on_termination: host['root_volume'].key?('delete_on_termination') ? !!host['root_volume']['delete_on_termination'] : true,
+          boot_index: 0
+        }]
       else
-        -1
+        server_opts[:image_ref] = image(host[:image]).id
       end
-    end
 
-    # Create and attach dynamic volumes
-    #
-    # Creates an array of volumes and attaches them to the current host.
-    # The host bus type is determined by the image type, so by default
-    # devices appear as /dev/vdb, /dev/vdc etc.  Setting the glance
-    # properties hw_disk_bus=scsi, hw_scsi_model=virtio-scsi will present
-    # them as /dev/sdb, /dev/sdc (or 2:0:0:1, 2:0:0:2 in SCSI addresses)
-    #
-    # @param host [Hash] thet current host defined in the nodeset
-    # @param vm [Fog::Compute::OpenStack::Server] the server to attach to
-    def provision_storage host, vm
-      volumes = get_volumes(host)
-      if !volumes.empty?
-        # Lazily create the volume client if needed
-        volume_client_create
-        volumes.keys.each_with_index do |volume, index|
-          @logger.debug "Creating volume #{volume} for OpenStack host #{host.name}"
-
-          # The node defintion file defines volume sizes in MB (due to precedent
-          # with the vagrant virtualbox implementation) however OpenStack requires
-          # this translating into GB
-          openstack_size = volumes[volume]['size'].to_i / 1000
-
-          # Set up the volume creation arguments
-          args = {
-            :size        => openstack_size,
-            :description => "Beaker volume: host=#{host.name} volume=#{volume}",
-          }
-
-          # Between version 1 and subsequent versions the API was updated to
-          # rename 'display_name' to just 'name' for better consistency
-          if get_volume_api_version == 1
-            args[:display_name] = volume
-          else
-            args[:name] = volume
-          end
+      vm = @compute_client.servers.create(server_opts)
+      vm.wait_for(@options[:timeout] || 600) { respond_to?(:ready?) ? ready? : state == 'ACTIVE' }
 
-          # Create the volume and wait for it to become available
-          vol = @volume_client.volumes.create(**args)
-          vol.wait_for { ready? }
+      # Register VM for cleanup
+      @vms_mutex.synchronize { @vms << vm }
 
-          # Fog needs a device name to attach as, so invent one.  The guest
-          # doesn't pay any attention to this
-          device = "/dev/vd#{('b'.ord + index).chr}"
-          vm.attach_volume(vol.id, device)
+      if @options[:openstack_floating_ip] && floating_ip
+        # Associate the IP via Compute client to ensure compatibility with Neutron objects
+        @compute_client.associate_address(vm.id, host[:ip])
+      else
+        # Prefer IPv4 address if multiple networks exist
+        addr = vm.addresses.values.flatten.find { |a| a['version'] == 4 }
+        host[:ip] = addr && addr['addr']
+
+        if host[:ip].nil?
+          @logger.warn "[#{host.name}] No IPv4 address found; VM may be IPv6-only"
         end
       end
-    end
 
-    # Detach and delete guest volumes
-    # @param vm [Fog::Compute::OpenStack::Server] the server to detach from
-    def cleanup_storage vm
-      vm.volumes.each do |vol|
-        @logger.debug "Deleting volume #{vol.name} for OpenStack host #{vm.name}"
-        vm.detach_volume(vol.id)
-        vol.wait_for { ready? }
-        vol.destroy
-      end
-    end
+      @logger.debug "[#{host.name}] Assigned IP #{host[:ip]}"
 
-    # Get a floating IP address to associate with the instance, try
-    # to allocate a new one from the specified pool if none are available
-    #
-    # TODO(GiedriusS): convert to use @network_client. This API will be turned off
-    # completely very soon.
-    def get_floating_ip
+      # Metadata is best-effort (some clouds disable it)
       begin
-        @logger.debug "Creating IP"
-        ip = @compute_client.addresses.create
-      rescue Fog::OpenStack::Compute::NotFound
-        # If there are no more floating IP addresses, allocate a
-        # new one and try again.
-        @compute_client.allocate_address(@options[:floating_ip_pool])
-        ip = @compute_client.addresses.find { |ip| ip.instance_id.nil? }
+        vm.metadata.update(
+          jenkins_build_url: @options[:jenkins_build_url].to_s,
+          department: @options[:department].to_s,
+          project: @options[:project].to_s
+        )
+      rescue => e
+        @logger.debug("[#{host.name}] Metadata update failed: #{e.message}")
       end
-      ip
-    end
 
-    # Create new instances in OpenStack, depending on if create_in_parallel is true or not
-    def provision
-      if @options[:create_in_parallel]
-        # Enable abort on exception for threads
-        Thread.abort_on_exception = true
-        @logger.notify "Provisioning OpenStack in parallel"
-        provision_parallel
-      else
-        @logger.notify "Provisioning OpenStack sequentially"
-        provision_sequential
-      end
-      hack_etc_hosts @hosts, @options
-    end
+      host.wait_for_port(22)
+      enable_root(host)
 
-    # Parallel creation wrapper
-    def provision_parallel
-      # Array to store threads
-      threads = @hosts.map do |host|
-        Thread.new do
-          create_instance_resources(host)
+      provision_storage(host, vm) if @options[:openstack_volume_support]
+
+    rescue => e
+      @logger.error "Provision failed: #{e.message}"
+
+      @cleanup_mutex.synchronize do
+        unless @cleanup_ran
+          @cleanup_ran = true
+          cleanup
         end
       end
-      # Wait for all threads to finish
-      threads.each(&:join)
-    end
 
-    # Sequential creation wrapper
-    def provision_sequential
-      @hosts.each do |host|
-        create_instance_resources(host)
-      end
+      raise e
     end
 
-    # Create the actual instance resources
-    def create_instance_resources(host)
-      @logger.notify "Provisioning OpenStack"
-      if @options[:openstack_floating_ip]
-        ip = get_floating_ip
-        hostname = ip.ip.gsub('.', '-')
-        host[:vmhostname] = hostname + '.rfc1918.puppetlabs.net'
+    # Get key_name from options or generate a new RSA key and add it to OpenStack keypairs
+    def create_or_associate_keypair(host, keyname)
+      if @options[:openstack_keyname]
+        host[:keyname] = @options[:openstack_keyname]
+        @logger.debug "Using existing keypair #{@options[:openstack_keyname]}"
       else
-        hostname = ('a'..'z').to_a.shuffle[0, 10].join
-        host[:vmhostname] = hostname
+        # Remove any existing ephemeral key with this name to avoid collisions
+        @compute_client.key_pairs.get(keyname)&.destroy
+
+        # Generate new RSA keypair
+        key = OpenSSL::PKey::RSA.new(2048)
+        type = key.ssh_type
+        data = [key.to_blob].pack('m0')
+        @compute_client.create_key_pair keyname, "#{type} #{data}"
+
+        # Track ephemeral keypairs for cleanup
+        @keypairs_mutex.synchronize { @ephemeral_keypairs << keyname }
+
+        # Inject private key into Beaker host
+        host['ssh'][:key_data] = [key.to_pem]
+        host[:keyname] = keyname
       end
+    end
 
-      create_or_associate_keypair(host, hostname)
-      @logger.debug "Provisioning #{host.name} (#{host[:vmhostname]})"
-      options = {
-        :flavor_ref => flavor(host[:flavor]).id,
-        :image_ref  => image(host[:image]).id,
-        :nics       => [{'net_id' => network(@options[:openstack_network]).id}],
-        :name       => host[:vmhostname],
-        :hostname   => host[:vmhostname],
-        :user_data  => host[:user_data] || "#cloud-config\nmanage_etc_hosts: true\n",
-        :key_name   => host[:keyname],
-      }
-      options[:security_groups] = security_groups(@options[:security_group]) unless @options[:security_group].nil?
-      vm = @compute_client.servers.create(options)
+    # Get a floating IP address from the configured pool
+    # supports both network name and network UUID
+    def get_floating_ip
+      # Check if floating_ip_pool is a UUID; if not, look up the network ID by name
+      pool_id = if @options[:floating_ip_pool] =~ /^[0-9a-f-]{36}$/i
+                  @options[:floating_ip_pool]
+                else
+                  @logger.debug "Looking up floating IP pool network by name: #{@options[:floating_ip_pool]}"
+                  network(@options[:floating_ip_pool]).id
+                end
+
+      @network_client.floating_ips.create(
+        floating_network_id: pool_id
+      )
+    end
 
-      # Wait for the new instance to start up
-      try = 1
-      attempts = @options[:timeout].to_i / SLEEPWAIT
+    # Provision additional volumes (always preserved, never deleted automatically)
+    def provision_storage(host, vm)
+      return unless @options[:openstack_volume_support]
 
-      while try <= attempts
+      volumes = get_volumes(host)
+      return if volumes.empty?
+
+      volume_client_create
+      device_index = 0
+
+      volumes.each do |vol_name, vol_def|
+        # Skip root volume if already handled via boot_from_volume
+        next if vol_name == 'root' && boot_from_volume?(host)
+
+        @logger.debug "Creating volume #{vol_name} for #{host.name}"
+
+        vol = @volume_client.volumes.create(
+          name: vol_name,
+          size: vol_def['size'].to_i,
+          description: vol_def['description'] || "Beaker volume: #{host.name}:#{vol_name}"
+        )
+
+        # Wait for Cinder to fully provision the volume
+        vol.wait_for(300) do
+          vol.reload
+          raise "Volume #{vol.name} entered error state: #{vol.status}" if vol.status =~ /error/i
+          vol.status == 'available'
+        end
+
+        # Device naming starts at /dev/vdc to avoid conflicts with root/ephemeral disks
+        device_letter = ('c'.ord + device_index)
+        raise "Too many volumes, cannot allocate device name" if device_letter > 'z'.ord
+        device = "/dev/vd#{device_letter.chr}"
+        device_index += 1
+
+        # Attach with retry (Nova attach sometimes races)
+        attempts = 0
         begin
-          vm.wait_for(5) { ready? }
-          break
-        rescue Fog::Errors::TimeoutError => e
-          if try >= attempts
-            @logger.debug "Failed to connect to new OpenStack instance #{host.name} (#{host[:vmhostname]})"
-            raise e
+          vm.attach_volume(vol.id, device)
+        rescue => e
+          attempts += 1
+          if attempts < 3
+            @logger.debug "Attach failed, retrying in 2s... (#{e.message})"
+            sleep 2
+            retry
           end
-          @logger.debug "Timeout connecting to instance #{host.name} (#{host[:vmhostname]}), trying again..."
+          raise "Failed to attach volume #{vol_name} after 3 attempts: #{e}"
         end
-        sleep SLEEPWAIT
-        try += 1
-      end
 
-      if @options[:openstack_floating_ip]
-        # Associate a public IP to the VM
-        ip.server = vm
-        host[:ip] = ip.ip
-      else
-        # Get the first address of the VM that was just created just like in the
-        # OpenStack UI
-        host[:ip] = vm.addresses.first[1][0]["addr"]
+        # Wait for Nova to complete attachment
+        vol.wait_for(120) do
+          vol.reload
+          vol.status == 'in-use'
+        end
       end
 
-      @logger.debug "OpenStack host #{host.name} (#{host[:vmhostname]}) assigned ip: #{host[:ip]}"
-    
-      # Set metadata
-      vm.metadata.update({:jenkins_build_url => @options[:jenkins_build_url].to_s,
-                          :department        => @options[:department].to_s,
-                          :project           => @options[:project].to_s })
-      @vms << vm
-
-      # Wait for the host to accept SSH logins
-      host.wait_for_port(22)
-
-      # Enable root if the user is not root
-      enable_root(host)
-
-      provision_storage(host, vm) if @options[:openstack_volume_support]
-      @logger.notify "OpenStack Volume Support Disabled, can't provision volumes" if not @options[:openstack_volume_support]
+      # Ensure host['ssh'] hash exists for key injection if needed later
+      host['ssh'] ||= {}
+    end
 
-    # Handle exceptions in the thread
-    rescue => e
-      @logger.error "Thread #{host} failed with error: #{e.message}"
-      # Call cleanup function to delete orphaned hosts
-      cleanup
-      # Pass the error to the main thread to terminate all threads
-      Thread.main.raise(e)
-      # Terminate the current thread (to prevent hack_etc_hosts trying to run after error raised)
-      Thread.kill(Thread.current)
+    # Enables root access for a single host when its current user is not 'root'
+    def enable_root(host)
+      return if host['user'] == 'root'
+      copy_ssh_to_root(host, @options)
+      enable_root_login(host, @options)
+      host['user'] = 'root'
+      host.close
     end
 
-    # Destroy any OpenStack instances
+    # Cleanup all resources
+    # Ephemeral keypairs and VMs are destroyed; allocated Floating IPs are released; additional volumes are preserved
     def cleanup
       @logger.notify "Cleaning up OpenStack"
-      @vms.each do |vm|
-        cleanup_storage(vm) if @options[:openstack_volume_support]
-        @logger.debug "Release floating IPs for OpenStack host #{vm.name}"
-        floating_ips = vm.all_addresses # fetch and release its floating IPs
-        floating_ips.each do |address|
-          @compute_client.disassociate_address(vm.id, address['ip'])
-          @compute_client.release_address(address['id'])
+
+      @vms_mutex.synchronize do
+        @vms.each do |vm|
+          begin
+            @logger.debug "Destroying #{vm.name}"
+            vm.destroy rescue nil
+          rescue => e
+            @logger.error "Cleanup error (VM): #{e.message}"
+          end
         end
-        @logger.debug "Destroying OpenStack host #{vm.name}"
-        vm.destroy
-        if @options[:openstack_keyname].nil?
-          @logger.debug "Deleting random keypair"
-          @compute_client.delete_key_pair vm.key_name
+        @vms.clear
+      end
+
+      @fip_mutex.synchronize do
+        @floating_ips.each do |fip|
+          begin
+            ip_addr = fip.respond_to?(:floating_ip_address) ? fip.floating_ip_address : (fip.respond_to?(:ip) ? fip.ip : 'unknown')
+            @logger.debug "Releasing Floating IP #{ip_addr}"
+            fip.destroy rescue nil
+          rescue => e
+            @logger.error "Cleanup error (FIP): #{e.message}"
+          end
         end
+        @floating_ips.clear
       end
-    end
 
-    # Enables root access for a host when username is not root
-    # This method ripped from the aws_sdk implementation and is probably wrong
-    # because it iterates on a collection when there's no guarantee the collection
-    # has all been brought up in openstack yet and will thus explode
-    # @return [void]
-    # @api private
-    def enable_root_on_hosts
-      @hosts.each do |host|
-        enable_root(host)
+      @keypairs_mutex.synchronize do
+        @ephemeral_keypairs.each do |keyname|
+          begin
+            @compute_client.key_pairs.get(keyname)&.destroy
+          rescue => e
+            @logger.error "Failed to delete ephemeral keypair #{keyname}: #{e.message}"
+          end
+        end
+        @ephemeral_keypairs.clear
       end
     end
 
-    # Enable root on a single host (the current one presumably) but only
-    # if the username isn't 'root'
-    def enable_root(host)
-      if host['user'] != 'root'
-        copy_ssh_to_root(host, @options)
-        enable_root_login(host, @options)
-        host['user'] = 'root'
-        host.close
-      end
+    # @!group Lookup Methods
+    # Lookup flavor by name
+    def flavor(f)
+      @logger.debug "Looking up flavor '#{f}'"
+      @compute_client.flavors.find { |x| x.name == f } || raise("Couldn't find flavor: #{f}")
     end
 
-    #Get key_name from options or generate a new rsa key and add it to
-    #OpenStack keypairs
-    #
-    #@param [Host] host The OpenStack host to provision
-    #@api private
-    def create_or_associate_keypair(host, keyname)
-      if @options[:openstack_keyname]
-        @logger.debug "Adding optional key_name #{@options[:openstack_keyname]} to #{host.name} (#{host[:vmhostname]})"
-        keyname = @options[:openstack_keyname]
-      else
-        @logger.debug "Generate a new rsa key"
+    # Lookup image by name
+    def image(i)
+      @logger.debug "Looking up image '#{i}'"
+      @compute_client.images.find { |x| x.name == i } || raise("Couldn't find image: #{i}")
+    end
 
-        # There is apparently an error that can occur when generating RSA keys, probably
-        # due to some timing issue, probably similar to the issue described here:
-        # https://github.com/negativecode/vines/issues/34
-        # In order to mitigate this error, we will simply try again up to three times, and
-        # then fail if we continue to error out.
-        begin
-          retries ||= 0
-          key = OpenSSL::PKey::RSA.new 2048
-        rescue OpenSSL::PKey::RSAError => e
-          retries += 1
-          if retries > 2
-            @logger.notify "error generating RSA key #{retries} times, exiting"
-            raise e
-          end
-          retry
-        end
+    # Lookup network by name
+    def network(n)
+      @logger.debug "Looking up network '#{n}'"
+      @network_client.networks.find { |x| x.name == n } || raise("Couldn't find network: #{n}")
+    end
 
-        type = key.ssh_type
-        data = [ key.to_blob ].pack('m0')
-        @logger.debug "Creating Openstack keypair '#{keyname}' for public key '#{type} #{data}'"
-        @compute_client.create_key_pair keyname, "#{type} #{data}"
-        host['ssh'][:key_data] = [ key.to_pem ]
+    # Validate security groups exist and return them in Fog-compatible format
+    def security_groups(sgs)
+      sgs.each do |sg|
+        @logger.debug "Openstack: Looking up security group '#{sg}'"
+        @compute_client.security_groups.find { |x| x.name == sg } || raise("Couldn't find security group: #{sg}")
       end
 
-      host[:keyname] = keyname
+      # Return an array of strings, not hashes
+      sgs
+    end
+
+    # Determine if host should boot from volume
+    def boot_from_volume?(host)
+      host['root_volume'] && host['root_volume']['size']
+    end
+
+    # Lazy-init volume client
+    def volume_client_create
+      @volume_client ||= Fog::Volume.new(@credentials)
+    end
+
+    # Retrieve additional volumes definition from host
+    def get_volumes(host)
+      host['volumes'] || {}
     end
   end
 end