beaker 2.10.0 → 2.11.0

data/lib/beaker/hypervisor/aws_sdk.rb

@@ -46,9 +46,6 @@ module Beaker
       # Perform the main launch work
       launch_all_nodes()
 
-      # Wait for each node to reach status :running
-      wait_for_status(:running)
-
       # Add metadata tags to each instance
       add_tags()
 
@@ -69,26 +66,30 @@ module Beaker
       nil #void
     end
 
-    # Cleanup all earlier provisioned hosts on EC2 using the AWS::EC2 library
-    #
-    # It goes without saying, but a #cleanup does nothing without a #provision
-    # method call first.
+    # Kill all instances.
     #
+    # @param instances [Enumerable<EC2::Instance>]
     # @return [void]
-    def cleanup
-      @logger.notify("aws-sdk: Cleanup, iterating across all hosts and terminating them")
-      @hosts.each do |host|
-        # This was set previously during provisioning
-        instance = host['instance']
-
-        # Only attempt a terminate if the instance actually is set by provision
-        # and the instance actually 'exists'.
+    def kill_instances(instances)
+      instances.each do |instance|
         if !instance.nil? and instance.exists?
+          @logger.notify("aws-sdk: killing EC2 instance #{instance.id}")
           instance.terminate
         end
       end
+      nil
+    end
 
-      nil #void
+    # Cleanup all earlier provisioned hosts on EC2 using the AWS::EC2 library
+    #
+    # It goes without saying, but a #cleanup does nothing without a #provision
+    # method call first.
+    #
+    # @return [void]
+    def cleanup
+      # Provisioning should have set the host 'instance' values.
+      kill_instances(@hosts.map{|h| h['instance']}.select{|x| !x.nil?})
+      nil
     end
 
     # Print instances to the logger. Instances will be from all regions
@@ -220,125 +221,209 @@ module Beaker
 
     end
 
-    # Launch all nodes
-    #
-    # This is where the main launching work occurs for each node. Here we take
-    # care of feeding the information from the image required into the config
-    # for the new host, we perform the launch operation and ensure that the
-    # instance is properly tagged for identification.
+    # Create an EC2 instance for host, tag it, and return it.
     #
-    # @return [void]
+    # @return [AWS::EC2::Instance)]
     # @api private
-    def launch_all_nodes
-      # Load the ec2_yaml spec file
-      ami_spec = YAML.load_file(@options[:ec2_yaml])["AMI"]
-
-      # Iterate across all hosts and launch them, adding tags along the way
-      @logger.notify("aws-sdk: Iterate across all hosts in configuration and launch them")
-      @hosts.each do |host|
-        amitype = host['vmname'] || host['platform']
-        amisize = host['amisize'] || 'm1.small'
-        subnet_id = host['subnet_id'] || @options['subnet_id'] || nil
-        vpc_id = host['vpc_id'] || @options['vpc_id'] || nil
+    def create_instance(host, ami_spec, subnet_id)
+      amitype = host['vmname'] || host['platform']
+      amisize = host['amisize'] || 'm1.small'
+      vpc_id = host['vpc_id'] || @options['vpc_id'] || nil
 
-        if vpc_id and !subnet_id
-          raise RuntimeError, "A subnet_id must be provided with a vpc_id"
-        end
+      if vpc_id and !subnet_id
+        raise RuntimeError, "A subnet_id must be provided with a vpc_id"
+      end
 
-        # Use snapshot provided for this host
-        image_type = host['snapshot']
-        if not image_type
-          raise RuntimeError, "No snapshot/image_type provided for EC2 provisioning"
-        end
-        ami = ami_spec[amitype]
-        ami_region = ami[:region]
-
-        # Main region object for ec2 operations
-        region = @ec2.regions[ami_region]
-
-        # If we haven't defined a vpc_id then we use the default vpc for the provided region
-        if !vpc_id
-          @logger.notify("aws-sdk: filtering available vpcs in region by 'isDefault")
-          filtered_vpcs = region.client.describe_vpcs(:filters => [{:name => 'isDefault', :values => ['true']}])
-          if !filtered_vpcs[:vpc_set].empty?
-            vpc_id = filtered_vpcs[:vpc_set].first[:vpc_id]
-          else #there's no default vpc, use nil
-            vpc_id = nil
-          end
+      # Use snapshot provided for this host
+      image_type = host['snapshot']
+      if not image_type
+        raise RuntimeError, "No snapshot/image_type provided for EC2 provisioning"
+      end
+      ami = ami_spec[amitype]
+      ami_region = ami[:region]
+
+      # Main region object for ec2 operations
+      region = @ec2.regions[ami_region]
+
+      # If we haven't defined a vpc_id then we use the default vpc for the provided region
+      if !vpc_id
+        @logger.notify("aws-sdk: filtering available vpcs in region by 'isDefault")
+        filtered_vpcs = region.client.describe_vpcs(:filters => [{:name => 'isDefault', :values => ['true']}])
+        if !filtered_vpcs[:vpc_set].empty?
+          vpc_id = filtered_vpcs[:vpc_set].first[:vpc_id]
+        else #there's no default vpc, use nil
+          vpc_id = nil
         end
+      end
 
-        # Grab the vpc object based upon provided id
-        vpc = vpc_id ? region.vpcs[vpc_id] : nil
+      # Grab the vpc object based upon provided id
+      vpc = vpc_id ? region.vpcs[vpc_id] : nil
 
-        # Grab image object
-        image_id = ami[:image][image_type.to_sym]
-        @logger.notify("aws-sdk: Checking image #{image_id} exists and getting its root device")
-        image = region.images[image_id]
-        if image.nil? and not image.exists?
-          raise RuntimeError, "Image not found: #{image_id}"
-        end
+      # Grab image object
+      image_id = ami[:image][image_type.to_sym]
+      @logger.notify("aws-sdk: Checking image #{image_id} exists and getting its root device")
+      image = region.images[image_id]
+      if image.nil? and not image.exists?
+        raise RuntimeError, "Image not found: #{image_id}"
+      end
 
-        # Transform the images block_device_mappings output into a format
-        # ready for a create.
-        orig_bdm = image.block_device_mappings()
-        @logger.notify("aws-sdk: Image block_device_mappings: #{orig_bdm.to_hash}")
-        block_device_mappings = []
-        orig_bdm.each do |device_name, rest|
-          block_device_mappings << {
-            :device_name => device_name,
-            :ebs => {
-              # Change the default size of the root volume.
-              :volume_size => host['volume_size'] || rest[:volume_size],
-              # This is required to override the images default for
-              # delete_on_termination, forcing all volumes to be deleted once the
-              # instance is terminated.
-              :delete_on_termination => true,
-            }
+      # Transform the images block_device_mappings output into a format
+      # ready for a create.
+      orig_bdm = image.block_device_mappings()
+      @logger.notify("aws-sdk: Image block_device_mappings: #{orig_bdm.to_hash}")
+      block_device_mappings = []
+      orig_bdm.each do |device_name, rest|
+        block_device_mappings << {
+          :device_name => device_name,
+          :ebs => {
+            # Change the default size of the root volume.
+            :volume_size => host['volume_size'] || rest[:volume_size],
+            # This is required to override the images default for
+            # delete_on_termination, forcing all volumes to be deleted once the
+            # instance is terminated.
+            :delete_on_termination => true,
           }
-        end
-
-        security_group = ensure_group(vpc || region, Beaker::EC2Helper.amiports(host))
-
-        # Launch the node, filling in the blanks from previous work.
-        @logger.notify("aws-sdk: Launch instance")
-        config = {
-          :count => 1,
-          :image_id => image_id,
-          :monitoring_enabled => true,
-          :key_pair => ensure_key_pair(region),
-          :security_groups => [security_group],
-          :instance_type => amisize,
-          :disable_api_termination => false,
-          :instance_initiated_shutdown_behavior => "terminate",
-          :block_device_mappings => block_device_mappings,
-          :subnet => subnet_id,
         }
-        instance = region.instances.create(config)
+      end
 
-        # Persist the instance object for this host, so later it can be
-        # manipulated by 'cleanup' for example.
-        host['instance'] = instance
+      security_group = ensure_group(vpc || region, Beaker::EC2Helper.amiports(host))
 
-        @logger.notify("aws-sdk: Launched #{host.name} (#{amitype}:#{amisize}) using snapshot/image_type #{image_type}")
+      msg = "aws-sdk: launching %p on %p using %p/%p%s" %
+            [host.name, amitype, amisize, image_type,
+             subnet_id ? ("in %p" % subnet_id) : '']
+      @logger.notify(msg)
+      config = {
+        :count => 1,
+        :image_id => image_id,
+        :monitoring_enabled => true,
+        :key_pair => ensure_key_pair(region),
+        :security_groups => [security_group],
+        :instance_type => amisize,
+        :disable_api_termination => false,
+        :instance_initiated_shutdown_behavior => "terminate",
+        :block_device_mappings => block_device_mappings,
+        :subnet => subnet_id,
+      }
+      region.instances.create(config)
+    end
+
+    # For each host, create an EC2 instance in one of the specified
+    # subnets and push it onto instances_created.  Each subnet will be
+    # tried at most once for each host, and more than one subnet may
+    # be tried if capacity constraints are encountered.  Each Hash in
+    # instances_created will contain an :instance and :host value.
+    #
+    # @param hosts [Enumerable<Host>]
+    # @param subnets [Enumerable<String>]
+    # @param ami_spec [Hash]
+    # @param instances_created Enumerable<Hash{Symbol=>EC2::Instance,Host}>
+    # @return [void]
+    # @api private
+    def launch_nodes_on_some_subnet(hosts, subnets, ami_spec, instances_created)
+      # Shuffle the subnets so we don't always hit the same one
+      # first, and cycle though the subnets independently of the
+      # host, so we stick with one that's working.  Try each subnet
+      # once per-host.
+      if subnets.nil? or subnets.empty?
+        return
       end
+      subnet_i = 0
+      shuffnets = subnets.shuffle
+      hosts.each do |host|
+        instance = nil
+        shuffnets.length.times do
+          begin
+            subnet_id = shuffnets[subnet_i]
+            instance = create_instance(host, ami_spec, subnet_id)
+            instances_created.push({:instance => instance, :host => host})
+            break
+          rescue AWS::EC2::Errors::InsufficientInstanceCapacity => ex
+            @logger.notify("aws-sdk: hit #{subnet_id} capacity limit; moving on")
+            subnet_i = (subnet_i + 1) % shuffnets.length
+          end
+        end
+        if instance.nil?
+          raise RuntimeError, "unable to launch host in any requested subnet"
+        end
+      end
+    end
 
+    # Create EC2 instances for all hosts, tag them, and wait until
+    # they're running.  When a host provides a subnet_id, create the
+    # instance in that subnet, otherwise prefer a CONFIG subnet_id.
+    # If neither are set but there is a CONFIG subnet_ids list,
+    # attempt to create the host in each specified subnet, which might
+    # fail due to capacity constraints, for example.  Specifying both
+    # a CONFIG subnet_id and subnet_ids will provoke an error.
+    #
+    # @return [void]
+    # @api private
+    def launch_all_nodes
+      @logger.notify("aws-sdk: launch all hosts in configuration")
+      ami_spec = YAML.load_file(@options[:ec2_yaml])["AMI"]
+      global_subnet_id = @options['subnet_id']
+      global_subnets = @options['subnet_ids']
+      if global_subnet_id and global_subnets
+        raise RuntimeError, 'Config specifies both subnet_id and subnet_ids'
+      end
+      no_subnet_hosts = []
+      specific_subnet_hosts = []
+      some_subnet_hosts = []
+      @hosts.each do |host|
+        if global_subnet_id or host['subnet_id']
+          specific_subnet_hosts.push(host)
+        elsif global_subnets
+          some_subnet_hosts.push(host)
+        else
+          no_subnet_hosts.push(host)
+        end
+      end
+      instances = [] # Each element is {:instance => i, :host => h}
+      begin
+        @logger.notify("aws-sdk: launch instances not particular about subnet")
+        launch_nodes_on_some_subnet(some_subnet_hosts, global_subnets, ami_spec,
+                                    instances)
+        @logger.notify("aws-sdk: launch instances requiring a specific subnet")
+        specific_subnet_hosts.each do |host|
+          subnet_id = host['subnet_id'] || global_subnet_id
+          instance = create_instance(host, ami_spec, subnet_id)
+          instances.push({:instance => instance, :host => host})
+        end
+        @logger.notify("aws-sdk: launch instances requiring no subnet")
+        no_subnet_hosts.each do |host|
+          instance = create_instance(host, ami_spec, nil)
+          instances.push({:instance => instance, :host => host})
+        end
+        wait_for_status(:running, instances)
+      rescue Exception => ex
+        @logger.notify("aws-sdk: exception - #{ex}")
+        kill_instances(instances.map{|x| x[:instance]})
+        raise ex
+      end
+      # At this point, all instances should be running since wait
+      # either returns on success or throws an exception.
+      if instances.empty?
+        raise RuntimeError, "Didn't manage to launch any EC2 instances"
+      end
+      # Assign the now known running instances to their hosts.
+      instances.each {|x| x[:host]['instance'] = x[:instance]}
       nil
     end
 
-    # Waits until all boxes reach the desired status
+    # Wait until all instances reach the desired state.  Each Hash in
+    # instances must contain an :instance and :host value.
     #
     # @param status [Symbol] EC2 state to wait for, :running :stopped etc.
+    # @param instances Enumerable<Hash{Symbol=>EC2::Instance,Host}>
     # @return [void]
     # @api private
-    def wait_for_status(status)
+    def wait_for_status(status, instances)
       # Wait for each node to reach status :running
-      @logger.notify("aws-sdk: Now wait for all hosts to reach state #{status}")
-      @hosts.each do |host|
-        instance = host['instance']
-        name = host.name
-
-        @logger.notify("aws-sdk: Wait for status #{status} for node #{name}")
-
+      @logger.notify("aws-sdk: Waiting for all hosts to be #{status}")
+      instances.each do |x|
+        name = x[:name]
+        instance = x[:instance]
+        @logger.notify("aws-sdk: Wait for node #{name} to be #{status}")
         # Here we keep waiting for the machine state to reach ':running' with an
         # exponential backoff for each poll.
         # TODO: should probably be a in a shared method somewhere
@@ -356,7 +441,6 @@ module Beaker
           end
           backoff_sleep(tries)
         end
-
       end
     end
 

data/lib/beaker/hypervisor/openstack.rb

@@ -13,8 +13,9 @@ module Beaker
     #@option options [String] :openstack_username The username to access the OpenStack instance with (required)
     #@option options [String] :openstack_auth_url The URL to access the OpenStack instance with (required)
     #@option options [String] :openstack_tenant The tenant to access the OpenStack instance with (required)
+    #@option options [String] :openstack_region The region that each OpenStack instance should be provisioned on (optional)
     #@option options [String] :openstack_network The network that each OpenStack instance should be contacted through (required)
-    #@option options [String] :openstack_keyname The name of an existing key pair that should be auto-loaded onto each 
+    #@option options [String] :openstack_keyname The name of an existing key pair that should be auto-loaded onto each
     #                                            OpenStack instance (optional)
     #@option options [String] :jenkins_build_url Added as metadata to each OpenStack instance
     #@option options [String] :department Added as metadata to each OpenStack instance
@@ -27,25 +28,36 @@ module Beaker
       @hosts = openstack_hosts
       @vms = []
 
-      raise 'You must specify an Openstack API key (:oopenstack_api_key) for OpenStack instances!' unless @options[:openstack_api_key]
+      raise 'You must specify an Openstack API key (:openstack_api_key) for OpenStack instances!' unless @options[:openstack_api_key]
       raise 'You must specify an Openstack username (:openstack_username) for OpenStack instances!' unless @options[:openstack_username]
       raise 'You must specify an Openstack auth URL (:openstack_auth_url) for OpenStack instances!' unless @options[:openstack_auth_url]
       raise 'You must specify an Openstack tenant (:openstack_tenant) for OpenStack instances!' unless @options[:openstack_tenant]
       raise 'You must specify an Openstack network (:openstack_network) for OpenStack instances!' unless @options[:openstack_network]
-      @compute_client ||= Fog::Compute.new(:provider => :openstack,
-                                           :openstack_api_key => @options[:openstack_api_key],
-                                           :openstack_username => @options[:openstack_username],
-                                           :openstack_auth_url => @options[:openstack_auth_url],
-                                           :openstack_tenant => @options[:openstack_tenant])
+
+      optionhash = {}
+      optionhash[:provider]           = :openstack
+      optionhash[:openstack_api_key]  = @options[:openstack_api_key]
+      optionhash[:openstack_username] = @options[:openstack_username]
+      optionhash[:openstack_auth_url] = @options[:openstack_auth_url]
+      optionhash[:openstack_tenant]   = @options[:openstack_tenant]
+      optionhash[:openstack_region]   = @options[:openstack_region] if @options[:openstack_region]
+
+      @compute_client ||= Fog::Compute.new(optionhash)
+
       if not @compute_client
         raise "Unable to create OpenStack Compute instance (api key: #{@options[:openstack_api_key]}, username: #{@options[:openstack_username]}, auth_url: #{@options[:openstack_auth_url]}, tenant: #{@options[:openstack_tenant]})"
       end
-      @network_client ||= Fog::Network.new(
-        :provider => :openstack,
-        :openstack_api_key => @options[:openstack_api_key],
-        :openstack_username => @options[:openstack_username],
-        :openstack_auth_url => @options[:openstack_auth_url],
-        :openstack_tenant => @options[:openstack_tenant])
+
+      networkoptionhash = {}
+      networkoptionhash[:provider]           = :openstack
+      networkoptionhash[:openstack_api_key]  = @options[:openstack_api_key]
+      networkoptionhash[:openstack_username] = @options[:openstack_username]
+      networkoptionhash[:openstack_auth_url] = @options[:openstack_auth_url]
+      networkoptionhash[:openstack_tenant]   = @options[:openstack_tenant]
+      networkoptionhash[:openstack_region]   = @options[:openstack_region] if @options[:openstack_region]
+
+      @network_client ||= Fog::Network.new(networkoptionhash)
+
       if not @network_client
 
         raise "Unable to create OpenStack Network instance (api_key: #{@options[:openstack_api_key]}, username: #{@options[:openstack_username]}, auth_url: #{@options[:openstack_auth_url]}, tenant: #{@options[:openstack_tenant]})"
@@ -85,9 +97,9 @@ module Beaker
         @logger.debug "Provisioning #{host.name} (#{host[:vmhostname]})"
         options = {
           :flavor_ref => flavor(host[:flavor]).id,
-          :image_ref => image(host[:image]).id,
-          :nics => [ {'net_id' => network(@options[:openstack_network]).id } ],
-          :name => host[:vmhostname],
+          :image_ref  => image(host[:image]).id,
+          :nics       => [ {'net_id' => network(@options[:openstack_network]).id } ],
+          :name       => host[:vmhostname],
         }
         options[:key_name] = key_name(host)
         vm = @compute_client.servers.create(options)
@@ -117,24 +129,35 @@ module Beaker
         #
         # Do we already have an address?
         @logger.debug vm.addresses
-
+        address=nil
         begin
-          if vm.addresses[@options[:openstack_network]]
-            address = vm.addresses[@options[:openstack_network]].map{ |network| network['addr'] }.first
-          end
-        rescue NoMethodError
-          @logger.debug "No current address retrievable from OpenStack data"
-        end
-        unless address
+          # Here we try and assign an address from a floating IP pool
+          # This seems to fail on some implementations (FloatingIpPoolNotFound)
           ip = @compute_client.addresses.find { |ip| ip.instance_id.nil? }
-
           if ip.nil?
             @logger.debug "Creating IP for #{host.name} (#{host[:vmhostname]})"
             ip = @compute_client.addresses.create
           end
           ip.server = vm
           address = ip.ip
+
+        rescue Fog::Compute::OpenStack::NotFound
+          # Here, we fail to just trying to use an address that's already assigned if there is one
+          # There may be better logic, but this worked in the original implementation
+          # There might be an argument for checking whether an address is reachable a la
+          # port_open? logic in host.rb but maybe race conditions
+
+          begin
+            if vm.addresses[@options[:openstack_network]]
+              address = vm.addresses[@options[:openstack_network]].map{ |network| network['addr'] }.first
+            end
+          rescue NoMethodError
+            @logger.debug "No current address retrievable from OpenStack data"
+          end
+
         end
+
+        raise 'Could not find or assign an address to the instance' unless address
         host[:ip] = address
 
         @logger.debug "OpenStack host #{host.name} (#{host[:vmhostname]}) assigned ip: #{host[:ip]}"