kitchen-ec2 3.2.0 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 828a479cf1ebe8de5423fad1926bc74f1cb66ddc1a1b4f853977424e65dd4709
-  data.tar.gz: 67851982c4d376be9ca701040e38d8e7b5cb308a7cf52cd0056da8f898558dcd
+  metadata.gz: 0d6429e51c08d43c0e1084b9acc55b76feacae05bbe42b7cf3a1e35e0de70f87
+  data.tar.gz: 9f2a0adbe2866ad6a267ab8adc14d24a890b22b71b0abe272808a2a52def3933
 SHA512:
-  metadata.gz: 1165fa8225401552970c19983091d8cb38f234a08f9f84a68ebb45cdf05b5670271e92b780b9e8fa0fdecc1c18083e3702f3ddace3561a76628226d09f67ce26
-  data.tar.gz: abe70ab29dacefda005ec11ffa0710841532e7f8407c203daf6bd782acb1c1da59d95c1c188bfca81cc20bd010d2ce7df5c6e78371eff0960f46dcbf3c885245
+  metadata.gz: aa308480b37d109ec173851aefe0098523d2a59c06481e88004bf76ccb8d87dfbd865300d2383157af13b38af47588ec2d1adf0541734f7279669a8dbcb0b673
+  data.tar.gz: 48f6f1a3dc22caf5d04079ef1579c19850e7e72558708e633537adb752969c3aac1b79702804864fac3fad9140e8d9053347c3ecde47e578e0545366d0b0b619

data/lib/kitchen/driver/aws/client.rb

@@ -49,14 +49,24 @@ module Kitchen
           ::Aws.config.update(retry_limit: retry_limit) unless retry_limit.nil?
         end
 
+        # create a new AWS EC2 instance
+        # @param options [Hash] has of instance options
+        # @see https://docs.aws.amazon.com/sdkforruby/api/Aws/EC2/Resource.html#create_instances-instance_method
+        # @return [Aws::EC2::Instance]
         def create_instance(options)
           resource.create_instances(options).first
         end
 
+        # get an instance object given an id
+        # @param id [String] aws instance id
+        # @return [Aws::EC2::Instance]
         def get_instance(id)
           resource.instance(id)
         end
 
+        # get an instance object given a spot request ID
+        # @param request_id [String] aws spot instance id
+        # @return [Aws::EC2::Instance]
         def get_instance_from_spot_request(request_id)
           resource.instances(
             filters: [{

data/lib/kitchen/driver/aws/instance_generator.rb

@@ -39,29 +39,34 @@ module Kitchen
           @logger = logger
         end
 
-        # Transform the provided config into the hash to send to AWS.  Some fields
+        # Transform the provided kitchen config into the hash we'll use to create the aws instance
         # can be passed in null, others need to be ommitted if they are null
+        # Some fields can be passed in null, others need to be ommitted if they are null
+        # @return [Hash]
         def ec2_instance_data # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
           # Support for looking up security group id and subnet id using tags.
-
+          vpc_id = nil
+          client = ::Aws::EC2::Client.new(region: config[:region])
           if config[:subnet_id].nil? && config[:subnet_filter]
-            config[:subnet_id] = ::Aws::EC2::Client
-              .new(region: config[:region]).describe_subnets(
-                filters: [
-                  {
-                    name: "tag:#{config[:subnet_filter][:tag]}",
-                    values: [config[:subnet_filter][:value]],
-                  },
-                ]
-              )[0][0].subnet_id
-
-            if config[:subnet_id].nil?
-              raise "The subnet tagged '#{config[:subnet_filter][:tag]}\
-              #{config[:subnet_filter][:value]}' does not exist!"
-            end
+            subnets = client.describe_subnets(
+              filters: [
+                {
+                  name: "tag:#{config[:subnet_filter][:tag]}",
+                  values: [config[:subnet_filter][:value]],
+                },
+              ]
+            ).subnets
+            raise "The subnet tagged '#{config[:subnet_filter][:tag]}:#{config[:subnet_filter][:value]}' does not exist!" unless subnets.any?
+
+            # => Select the least-populated subnet if we have multiple matches
+            subnet = subnets.sort_by { |s| s[:available_ip_address_count] }.last
+            vpc_id = subnet.vpc_id
+            config[:subnet_id] = subnet.subnet_id
           end
 
           if config[:security_group_ids].nil? && config[:security_group_filter]
+            # => Grab the VPC in the case a Subnet ID rather than Filter was set
+            vpc_id ||= client.describe_subnets(subnet_ids: [config[:subnet_id]]).subnets[0].vpc_id
             security_groups = []
             filters = [config[:security_group_filter]].flatten
             filters.each do |sg_filter|
@@ -72,6 +77,10 @@ module Kitchen
                     name: "group-name",
                     values: [sg_filter[:name]],
                   },
+                  {
+                    name: "vpc-id",
+                    values: [vpc_id],
+                  },
                 ]
               end
 
@@ -81,13 +90,17 @@ module Kitchen
                     name: "tag:#{sg_filter[:tag]}",
                     values: [sg_filter[:value]],
                   },
+                  {
+                    name: "vpc-id",
+                    values: [vpc_id],
+                  },
                 ]
               end
 
-              security_group = ::Aws::EC2::Client.new(region: config[:region]).describe_security_groups(r)[0][0]
+              security_group = client.describe_security_groups(r).security_groups
 
-              if security_group
-                security_groups.push(security_group.group_id)
+              if security_group.any?
+                security_group.each { |sg| security_groups.push(sg.group_id) }
               else
                 raise "A Security Group matching the following filter could not be found:\n#{sg_filter}"
               end
@@ -102,8 +115,22 @@ module Kitchen
             key_name: config[:aws_ssh_key_id],
             subnet_id: config[:subnet_id],
             private_ip_address: config[:private_ip_address],
+            min_count: 1,
+            max_count: 1,
           }
 
+          if config[:tags] && !config[:tags].empty?
+            tags = config[:tags].map do |k, v|
+              # we convert the value to a string because
+              # nils should be passed as an empty String
+              # and Integers need to be represented as Strings
+              { key: k, value: v.to_s }
+            end
+            instance_tag_spec = { resource_type: "instance", tags: tags }
+            volume_tag_spec = { resource_type: "volume", tags: tags }
+            i[:tag_specifications] = [instance_tag_spec, volume_tag_spec]
+          end
+
           availability_zone = config[:availability_zone]
           if availability_zone
             if availability_zone =~ /^[a-z]$/i

data/lib/kitchen/driver/aws/standard_platform/amazon.rb

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "kitchen/driver/aws/standard_platform"
+require_relative "../standard_platform"
 
 module Kitchen
   module Driver
@@ -23,6 +23,8 @@ module Kitchen
         class Amazon < StandardPlatform
           StandardPlatform.platforms["amazon"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "ec2-user"
           end

data/lib/kitchen/driver/aws/standard_platform/amazon2.rb

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "kitchen/driver/aws/standard_platform"
+require_relative "../standard_platform"
 
 module Kitchen
   module Driver
@@ -23,6 +23,8 @@ module Kitchen
         class Amazon2 < StandardPlatform
           StandardPlatform.platforms["amazon2"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "ec2-user"
           end

data/lib/kitchen/driver/aws/standard_platform/centos.rb

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "kitchen/driver/aws/standard_platform"
+require_relative "../standard_platform"
 
 module Kitchen
   module Driver
@@ -23,6 +23,8 @@ module Kitchen
         class Centos < StandardPlatform
           StandardPlatform.platforms["centos"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             # Centos 6.x images use root as the username (but the "centos 6"
             # updateable image uses "centos")

data/lib/kitchen/driver/aws/standard_platform/debian.rb

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "kitchen/driver/aws/standard_platform"
+require_relative "../standard_platform"
 
 module Kitchen
   module Driver
@@ -26,14 +26,17 @@ module Kitchen
           # 10/11 are listed last since we default to the first item in the hash
           # and 10/11 are not released yet. When they're released move them up
           DEBIAN_CODENAMES = {
+            10 => "buster",
             9 => "stretch",
             8 => "jessie",
             7 => "wheezy",
             6 => "squeeze",
             11 => "bullseye",
-            10 => "buster",
+            12 => "bookworm",
           }.freeze
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "admin"
           end

data/lib/kitchen/driver/aws/standard_platform/fedora.rb

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "kitchen/driver/aws/standard_platform"
+require_relative "../standard_platform"
 
 module Kitchen
   module Driver
@@ -23,6 +23,8 @@ module Kitchen
         class Fedora < StandardPlatform
           StandardPlatform.platforms["fedora"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "fedora"
           end

data/lib/kitchen/driver/aws/standard_platform/freebsd.rb

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "kitchen/driver/aws/standard_platform"
+require_relative "../standard_platform"
 
 module Kitchen
   module Driver
@@ -23,6 +23,8 @@ module Kitchen
         class Freebsd < StandardPlatform
           StandardPlatform.platforms["freebsd"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "ec2-user"
           end

data/lib/kitchen/driver/aws/standard_platform/rhel.rb

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "kitchen/driver/aws/standard_platform"
+require_relative "../standard_platform"
 
 module Kitchen
   module Driver
@@ -29,6 +29,8 @@ module Kitchen
             super(driver, "rhel", version, architecture)
           end
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             (version && version.to_f < 6.4) ? "root" : "ec2-user"
           end

data/lib/kitchen/driver/aws/standard_platform/ubuntu.rb

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "kitchen/driver/aws/standard_platform"
+require_relative "../standard_platform"
 
 module Kitchen
   module Driver
@@ -23,6 +23,8 @@ module Kitchen
         class Ubuntu < StandardPlatform
           StandardPlatform.platforms["ubuntu"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "ubuntu"
           end

data/lib/kitchen/driver/aws/standard_platform/windows.rb

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "kitchen/driver/aws/standard_platform"
+require_relative "../standard_platform"
 
 module Kitchen
   module Driver
@@ -23,6 +23,8 @@ module Kitchen
         class Windows < StandardPlatform
           StandardPlatform.platforms["windows"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "administrator"
           end

data/lib/kitchen/driver/ec2.rb

@@ -227,7 +227,7 @@ module Kitchen
 
         if config[:spot_price]
           # Spot instance when a price is set
-          server = with_request_limit_backoff(state) { submit_spots(state) }
+          server = with_request_limit_backoff(state) { submit_spots }
         else
           # On-demand instance
           server = with_request_limit_backoff(state) { submit_server }
@@ -238,32 +238,16 @@ module Kitchen
           server.wait_until_exists(before_attempt: logging_proc)
         end
 
+        state[:server_id] = server.id
+        info("EC2 instance <#{state[:server_id]}> created.")
+
         # See https://github.com/aws/aws-sdk-ruby/issues/859
-        # Tagging can fail with a NotFound error even though we waited until the server exists
-        # Waiting can also fail, so we have to also retry on that. If it means we re-tag the
-        # instance, so be it.
-        # Tagging an instance is possible before volumes are attached. Tagging the volumes after
-        # instance creation is consistent.
+        # Waiting can fail, so we have to retry on that.
         Retryable.retryable(
           tries: 10,
           sleep: lambda { |n| [2**n, 30].min },
           on: ::Aws::EC2::Errors::InvalidInstanceIDNotFound
         ) do |r, _|
-          info("Attempting to tag the instance, #{r} retries")
-          tag_server(server)
-
-          # Get information about the AMI (image) used to create the image.
-          image_data = ec2.client.describe_images({ image_ids: [server.image_id] })[0][0]
-
-          state[:server_id] = server.id
-          info("EC2 instance <#{state[:server_id]}> created.")
-
-          # instance-store backed images do not have attached volumes, so only
-          # wait for the volumes to be ready if the instance EBS-backed.
-          if image_data.root_device_type == "ebs"
-            wait_until_volumes_ready(server, state)
-            tag_volumes(server)
-          end
           wait_until_ready(server, state)
         end
 
@@ -277,7 +261,7 @@ module Kitchen
 
         info("EC2 instance <#{state[:server_id]}> ready (hostname: #{state[:hostname]}).")
         instance.transport.connection(state).wait_until_ready
-        create_ec2_json(state) if instance.provisioner.name =~ /chef/
+        create_ec2_json(state) if instance.provisioner.name =~ /chef/i
         debug("ec2:create '#{state[:hostname]}'")
       rescue Exception
         # Clean up any auto-created security groups or keys on the way out.
@@ -291,14 +275,11 @@ module Kitchen
           server = ec2.get_instance(state[:server_id])
           unless server.nil?
             instance.transport.connection(state).close
-            server.terminate
-          end
-          if state[:spot_request_id]
-            debug("Deleting spot request <#{state[:server_id]}>")
-            ec2.client.cancel_spot_instance_requests(
-              spot_instance_request_ids: [state[:spot_request_id]]
-            )
-            state.delete(:spot_request_id)
+            begin
+              server.terminate
+            rescue ::Aws::EC2::Errors::InvalidInstanceIDNotFound => e
+              warn("Received #{e}, instance was probably already destroyed. Ignoring")
+            end
           end
           # If we are going to clean up an automatic security group, we need
           # to wait for the instance to shut down. This slightly breaks the
@@ -405,15 +386,14 @@ module Kitchen
         @instance_generator = Aws::InstanceGenerator.new(config, ec2, instance.logger)
       end
 
-      # Fog AWS helper for creating the instance
+      # AWS helper for creating the instance
       def submit_server
         instance_data = instance_generator.ec2_instance_data
         debug("Creating EC2 instance in region #{config[:region]} with properties:")
         instance_data.each do |key, value|
           debug("- #{key} = #{value.inspect}")
         end
-        instance_data[:min_count] = 1
-        instance_data[:max_count] = 1
+
         ec2.create_instance(instance_data)
       end
 
@@ -441,7 +421,7 @@ module Kitchen
         configs
       end
 
-      def submit_spots(state)
+      def submit_spots
         configs = [config]
         expanded = []
         keys = %i{instance_type subnet_id}
@@ -454,40 +434,22 @@ module Kitchen
           expanded = []
         end
 
+        errs = []
         configs.each do |conf|
           begin
             @config = conf
-            return submit_spot(state)
-          rescue
+            return submit_spot
+          rescue => e
+            errs.append(e)
           end
         end
-
-        raise "Could not create a spot"
+        raise ["Could not create a spot instance:", errs].flatten.join("\n")
       end
 
-      def submit_spot(state)
+      def submit_spot
         debug("Creating EC2 Spot Instance..")
+        instance_data = instance_generator.ec2_instance_data
 
-        spot_request_id = create_spot_request
-        # deleting the instance cancels the request, but deleting the request
-        # does not affect the instance
-        state[:spot_request_id] = spot_request_id
-        ec2.client.wait_until(
-          :spot_instance_request_fulfilled,
-          spot_instance_request_ids: [spot_request_id]
-        ) do |w|
-          w.max_attempts = config[:spot_wait] / config[:retryable_sleep]
-          w.delay = config[:retryable_sleep]
-          w.before_attempt do |attempts|
-            c = attempts * config[:retryable_sleep]
-            t = config[:spot_wait]
-            info "Waited #{c}/#{t}s for spot request <#{spot_request_id}> to become fulfilled."
-          end
-        end
-        ec2.get_instance_from_spot_request(spot_request_id)
-      end
-
-      def create_spot_request
         request_duration = config[:spot_wait]
         config_spot_price = config[:spot_price].to_s
         if %w{ondemand on-demand}.include?(config_spot_price)
@@ -495,56 +457,36 @@ module Kitchen
         else
           spot_price = config_spot_price
         end
-        request_data = {
-          spot_price: spot_price,
-          launch_specification: instance_generator.ec2_instance_data,
+        spot_options = {
+          spot_instance_type: "persistent", # Cannot use one-time with valid_until
           valid_until: Time.now + request_duration,
+          instance_interruption_behavior: "stop",
         }
         if config[:block_duration_minutes]
-          request_data[:block_duration_minutes] = config[:block_duration_minutes]
+          spot_options[:block_duration_minutes] = config[:block_duration_minutes]
         end
-
-        response = ec2.client.request_spot_instances(request_data)
-        response[:spot_instance_requests][0][:spot_instance_request_id]
-      end
-
-      def tag_server(server)
-        if config[:tags] && !config[:tags].empty?
-          tags = config[:tags].map do |k, v|
-            # we convert the value to a string because
-            # nils should be passed as an empty String
-            # and Integers need to be represented as Strings
-            { key: k.to_s, value: v.to_s }
-          end
-          server.create_tags(tags: tags)
+        unless spot_price == "" # i.e. on-demand
+          spot_options[:max_price] = spot_price
         end
-      end
 
-      def tag_volumes(server)
-        if config[:tags] && !config[:tags].empty?
-          tags = config[:tags].map do |k, v|
-            { key: k.to_s, value: v.to_s }
-          end
-          server.volumes.each do |volume|
-            volume.create_tags(tags: tags)
-          end
-        end
-      end
+        instance_data[:instance_market_options] = {
+          market_type: "spot",
+          spot_options: spot_options,
+        }
 
-      # Compares the requested volume count vs what has actually been set to be
-      # attached to the instance. The information requested through
-      # ec2.client.described_volumes is updated before the instance volume
-      # information.
-      def wait_until_volumes_ready(server, state)
-        wait_with_destroy(server, state, "volumes to be ready") do |aws_instance|
-          described_volume_count = 0
-          ready_volume_count = 0
-          if aws_instance.exists?
-            described_volume_count = ec2.client.describe_volumes(filters: [
-              { name: "attachment.instance-id", values: ["#{state[:server_id]}"] }]).volumes.length
-            aws_instance.volumes.each { ready_volume_count += 1 }
-          end
-          (described_volume_count > 0) && (described_volume_count == ready_volume_count)
+        # The preferred way to create a spot instance is via request_spot_instances()
+        # However, it does not allow for tagging to occur at creation time.
+        # create_instances() allows creation of tagged spot instances, but does
+        # not retry if the price could not be satisfied immediately.
+        Retryable.retryable(
+          tries: config[:spot_wait] / config[:retryable_sleep],
+          sleep: lambda { |_n| config[:retryable_sleep] },
+          on: ::Aws::EC2::Errors::SpotMaxPriceTooLow
+        ) do |retries|
+          c = retries * config[:retryable_sleep]
+          t = config[:spot_wait]
+          info "Waited #{c}/#{t}s for spot request to become fulfilled."
+          ec2.create_instance(instance_data)
         end
       end
 
@@ -788,6 +730,7 @@ module Kitchen
                  elsif config[:subnet_filter]
                    subnets = ec2.client.describe_subnets(filters: [{ name: "tag:#{config[:subnet_filter][:tag]}", values: [config[:subnet_filter][:value]] }]).subnets
                    raise "Subnets with tag '#{config[:subnet_filter][:tag]}=#{config[:subnet_filter][:value]}' not found during security group creation" if subnets.empty?
+
                    subnets.first.vpc_id
                  else
                    # Try to check for a default VPC.

data/lib/kitchen/driver/ec2_version.rb

@@ -2,7 +2,7 @@
 #
 # Author:: Fletcher Nichol (<fnichol@nichol.ca>)
 #
-# Copyright:: 2016-2018, Chef Software, Inc.
+# Copyright:: Chef Software, Inc.
 # Copyright:: 2012-2018, Fletcher Nichol
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,6 +22,6 @@ module Kitchen
   module Driver
 
     # Version string for EC2 Test Kitchen driver
-    EC2_VERSION = "3.2.0".freeze
+    EC2_VERSION = "3.7.0".freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kitchen-ec2
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.7.0
 platform: ruby
 authors:
 - Fletcher Nichol
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-17 00:00:00.000000000 Z
+date: 2020-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-kitchen
@@ -168,14 +168,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.13.3
+        version: 1.1.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.13.3
+        version: 1.1.2
 - !ruby/object:Gem::Dependency
   name: climate_control
   requirement: !ruby/object:Gem::Requirement
@@ -216,7 +216,7 @@ homepage: https://github.com/test-kitchen/kitchen-ec2
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -224,15 +224,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: A Test Kitchen Driver for Amazon EC2
 test_files: []