kitchen-ec2 3.3.0 → 3.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1132f5a8bcb58e6a8c4d38ad342ac2e38afeabbd51abf63b6980a6b4c9474339
-  data.tar.gz: 876fb813038936051338a3be5682fc6633b420aa6faa201b3156879bc9f29430
+  metadata.gz: effb6d0cc14377657291cd970e2a6d50680d91df937b6799f0a942a2fba59994
+  data.tar.gz: a2a0b678c22cf3da32be2ee90cb542cc2c2dff015fb3563ea7783b85ddd57649
 SHA512:
-  metadata.gz: ae1058987c9efd9e8b7e0f2e0fa152b0a7cefb297d82fb6ab50cc77e1ee84f59e7b9a7fb54b391e9a3b35ae3ee7f3ef704ab6edd7e8fb908c1abe90870cc2c3d
-  data.tar.gz: 5fd929e0e1752d3b7cd1ff5b1e83383be321c95569bc00fee8ab9ad19539e815a3a1d9f692c516550cf5d4da5f667e500bc2ea8d64704a0fc337152fe34744df
+  metadata.gz: 2761f62c4647000ee4b12ac11a0080f49bb5492c296e08a22c598d1bfb0a0ea993b7d0409c04003d7819d7f287dd9fd65b611f2135c8edf69a5e36421549ea2c
+  data.tar.gz: 96b9d6c0062d85baafa9febca4fe0567f2659dc505c3e89da85caa3b6f82590f5a981d8df2907ee1f15709374e1e31f833f4cbb940a87b83283086e11084adab

data/lib/kitchen/driver/aws/client.rb

@@ -49,14 +49,24 @@ module Kitchen
           ::Aws.config.update(retry_limit: retry_limit) unless retry_limit.nil?
         end
 
+        # create a new AWS EC2 instance
+        # @param options [Hash] has of instance options
+        # @see https://docs.aws.amazon.com/sdkforruby/api/Aws/EC2/Resource.html#create_instances-instance_method
+        # @return [Aws::EC2::Instance]
         def create_instance(options)
           resource.create_instances(options).first
         end
 
+        # get an instance object given an id
+        # @param id [String] aws instance id
+        # @return [Aws::EC2::Instance]
         def get_instance(id)
           resource.instance(id)
         end
 
+        # get an instance object given a spot request ID
+        # @param request_id [String] aws spot instance id
+        # @return [Aws::EC2::Instance]
         def get_instance_from_spot_request(request_id)
           resource.instances(
             filters: [{

data/lib/kitchen/driver/aws/instance_generator.rb

@@ -39,8 +39,10 @@ module Kitchen
           @logger = logger
         end
 
-        # Transform the provided config into the hash to send to AWS.  Some fields
+        # Transform the provided kitchen config into the hash we'll use to create the aws instance
         # can be passed in null, others need to be ommitted if they are null
+        # Some fields can be passed in null, others need to be ommitted if they are null
+        # @return [Hash]
         def ec2_instance_data # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
           # Support for looking up security group id and subnet id using tags.
           vpc_id = nil
@@ -56,8 +58,10 @@ module Kitchen
             ).subnets
             raise "The subnet tagged '#{config[:subnet_filter][:tag]}:#{config[:subnet_filter][:value]}' does not exist!" unless subnets.any?
 
-            vpc_id = subnets[0].vpc_id
-            config[:subnet_id] = subnets[0].subnet_id
+            # => Select the least-populated subnet if we have multiple matches
+            subnet = subnets.sort_by { |s| s[:available_ip_address_count] }.last
+            vpc_id = subnet.vpc_id
+            config[:subnet_id] = subnet.subnet_id
           end
 
           if config[:security_group_ids].nil? && config[:security_group_filter]
@@ -111,8 +115,22 @@ module Kitchen
             key_name: config[:aws_ssh_key_id],
             subnet_id: config[:subnet_id],
             private_ip_address: config[:private_ip_address],
+            min_count: 1,
+            max_count: 1,
           }
 
+          if config[:tags] && !config[:tags].empty?
+            tags = config[:tags].map do |k, v|
+              # we convert the value to a string because
+              # nils should be passed as an empty String
+              # and Integers need to be represented as Strings
+              { key: k, value: v.to_s }
+            end
+            instance_tag_spec = { resource_type: "instance", tags: tags }
+            volume_tag_spec = { resource_type: "volume", tags: tags }
+            i[:tag_specifications] = [instance_tag_spec, volume_tag_spec]
+          end
+
           availability_zone = config[:availability_zone]
           if availability_zone
             if availability_zone =~ /^[a-z]$/i

data/lib/kitchen/driver/aws/standard_platform/amazon.rb

@@ -23,6 +23,8 @@ module Kitchen
         class Amazon < StandardPlatform
           StandardPlatform.platforms["amazon"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "ec2-user"
           end

data/lib/kitchen/driver/aws/standard_platform/amazon2.rb

@@ -23,6 +23,8 @@ module Kitchen
         class Amazon2 < StandardPlatform
           StandardPlatform.platforms["amazon2"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "ec2-user"
           end

data/lib/kitchen/driver/aws/standard_platform/centos.rb

@@ -23,6 +23,16 @@ module Kitchen
         class Centos < StandardPlatform
           StandardPlatform.platforms["centos"] = self
 
+          CENTOS_OWNER_ID = "125523088429".freeze
+          PRODUCT_CODES = {
+            "6" => "6x5jmcajty9edm3f211pqjfn2",
+            "7" => "aw0evgkw8e5c1q413zgy5pjce",
+            # It appears that v8 is not published to the
+            # AWS marketplace and hence does not have a product code
+          }.freeze
+
+          # default username for this platform's ami
+          # @return [String]
           def username
             # Centos 6.x images use root as the username (but the "centos 6"
             # updateable image uses "centos")
@@ -32,10 +42,25 @@ module Kitchen
           end
 
           def image_search
+            # Version 8+ are published directly, not to the AWS marketplace. Use OWNER ID.
             search = {
-              "owner-alias" => "aws-marketplace",
-              "name" => ["CentOS Linux #{version}*", "CentOS-#{version}*-GA-*"],
+              "owner-id" => CENTOS_OWNER_ID,
+              "name" => ["CentOS #{version}*", "CentOS-#{version}*-GA-*"],
             }
+
+            if version && version.split(".").first.to_i < 8
+              # Versions <8 are published to the AWS marketplace and use a different naming convention
+              search = {
+                "owner-alias" => "aws-marketplace",
+                "name" => ["CentOS Linux #{version}*", "CentOS-#{version}*-GA-*"],
+              }
+              # For versions published to aws-marketplace, additionally filter on product code to
+              # avoid non-official AMIs. Can't use CentOS owner ID here, as the owner ID is that of aws marketplace.
+              # https://github.com/test-kitchen/kitchen-ec2/issues/456
+              PRODUCT_CODES.keys.each do |major_version|
+                search["product-code"] = PRODUCT_CODES[major_version] if version.start_with?(major_version)
+              end
+            end
             search["architecture"] = architecture if architecture
             search
           end

data/lib/kitchen/driver/aws/standard_platform/debian.rb

@@ -26,14 +26,17 @@ module Kitchen
           # 10/11 are listed last since we default to the first item in the hash
           # and 10/11 are not released yet. When they're released move them up
           DEBIAN_CODENAMES = {
+            10 => "buster",
             9 => "stretch",
             8 => "jessie",
             7 => "wheezy",
             6 => "squeeze",
             11 => "bullseye",
-            10 => "buster",
+            12 => "bookworm",
           }.freeze
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "admin"
           end

data/lib/kitchen/driver/aws/standard_platform/fedora.rb

@@ -23,6 +23,8 @@ module Kitchen
         class Fedora < StandardPlatform
           StandardPlatform.platforms["fedora"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "fedora"
           end

data/lib/kitchen/driver/aws/standard_platform/freebsd.rb

@@ -23,6 +23,8 @@ module Kitchen
         class Freebsd < StandardPlatform
           StandardPlatform.platforms["freebsd"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "ec2-user"
           end

data/lib/kitchen/driver/aws/standard_platform/rhel.rb

@@ -29,6 +29,8 @@ module Kitchen
             super(driver, "rhel", version, architecture)
           end
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             (version && version.to_f < 6.4) ? "root" : "ec2-user"
           end

data/lib/kitchen/driver/aws/standard_platform/ubuntu.rb

@@ -23,6 +23,8 @@ module Kitchen
         class Ubuntu < StandardPlatform
           StandardPlatform.platforms["ubuntu"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "ubuntu"
           end

data/lib/kitchen/driver/aws/standard_platform/windows.rb

@@ -23,6 +23,8 @@ module Kitchen
         class Windows < StandardPlatform
           StandardPlatform.platforms["windows"] = self
 
+          # default username for this platform's ami
+          # @return [String]
           def username
             "administrator"
           end

data/lib/kitchen/driver/ec2.rb

@@ -227,7 +227,7 @@ module Kitchen
 
         if config[:spot_price]
           # Spot instance when a price is set
-          server = with_request_limit_backoff(state) { submit_spots(state) }
+          server = with_request_limit_backoff(state) { submit_spots }
         else
           # On-demand instance
           server = with_request_limit_backoff(state) { submit_server }
@@ -238,32 +238,16 @@ module Kitchen
           server.wait_until_exists(before_attempt: logging_proc)
         end
 
+        state[:server_id] = server.id
+        info("EC2 instance <#{state[:server_id]}> created.")
+
         # See https://github.com/aws/aws-sdk-ruby/issues/859
-        # Tagging can fail with a NotFound error even though we waited until the server exists
-        # Waiting can also fail, so we have to also retry on that. If it means we re-tag the
-        # instance, so be it.
-        # Tagging an instance is possible before volumes are attached. Tagging the volumes after
-        # instance creation is consistent.
+        # Waiting can fail, so we have to retry on that.
         Retryable.retryable(
           tries: 10,
           sleep: lambda { |n| [2**n, 30].min },
           on: ::Aws::EC2::Errors::InvalidInstanceIDNotFound
         ) do |r, _|
-          info("Attempting to tag the instance, #{r} retries")
-          tag_server(server)
-
-          # Get information about the AMI (image) used to create the image.
-          image_data = ec2.client.describe_images({ image_ids: [server.image_id] })[0][0]
-
-          state[:server_id] = server.id
-          info("EC2 instance <#{state[:server_id]}> created.")
-
-          # instance-store backed images do not have attached volumes, so only
-          # wait for the volumes to be ready if the instance EBS-backed.
-          if image_data.root_device_type == "ebs"
-            wait_until_volumes_ready(server, state)
-            tag_volumes(server)
-          end
           wait_until_ready(server, state)
         end
 
@@ -291,14 +275,11 @@ module Kitchen
           server = ec2.get_instance(state[:server_id])
           unless server.nil?
             instance.transport.connection(state).close
-            server.terminate
-          end
-          if state[:spot_request_id]
-            debug("Deleting spot request <#{state[:server_id]}>")
-            ec2.client.cancel_spot_instance_requests(
-              spot_instance_request_ids: [state[:spot_request_id]]
-            )
-            state.delete(:spot_request_id)
+            begin
+              server.terminate
+            rescue ::Aws::EC2::Errors::InvalidInstanceIDNotFound => e
+              warn("Received #{e}, instance was probably already destroyed. Ignoring")
+            end
           end
           # If we are going to clean up an automatic security group, we need
           # to wait for the instance to shut down. This slightly breaks the
@@ -405,15 +386,14 @@ module Kitchen
         @instance_generator = Aws::InstanceGenerator.new(config, ec2, instance.logger)
       end
 
-      # Fog AWS helper for creating the instance
+      # AWS helper for creating the instance
       def submit_server
         instance_data = instance_generator.ec2_instance_data
         debug("Creating EC2 instance in region #{config[:region]} with properties:")
         instance_data.each do |key, value|
           debug("- #{key} = #{value.inspect}")
         end
-        instance_data[:min_count] = 1
-        instance_data[:max_count] = 1
+
         ec2.create_instance(instance_data)
       end
 
@@ -441,10 +421,34 @@ module Kitchen
         configs
       end
 
-      def submit_spots(state)
+      def submit_spots
         configs = [config]
         expanded = []
-        keys = %i{instance_type subnet_id}
+        keys = %i{instance_type}
+
+        unless config[:subnet_filter]
+          # => Use explicitly specified subnets
+          keys << :subnet_id
+        else
+          # => Enable cascading through matching subnets
+          client = ::Aws::EC2::Client.new(region: config[:region])
+          subnets = client.describe_subnets(
+            filters: [
+              {
+                name: "tag:#{config[:subnet_filter][:tag]}",
+                values: [config[:subnet_filter][:value]],
+              },
+            ]
+          ).subnets
+          raise "A subnet matching '#{config[:subnet_filter][:tag]}:#{config[:subnet_filter][:value]}' does not exist!" unless subnets.any?
+
+          configs = subnets.map do |subnet|
+            new_config = config.clone
+            new_config[:subnet_id] = subnet.subnet_id
+            new_config[:subnet_filter] = nil
+            new_config
+          end
+        end
 
         keys.each do |key|
           configs.each do |conf|
@@ -458,7 +462,7 @@ module Kitchen
         configs.each do |conf|
           begin
             @config = conf
-            return submit_spot(state)
+            return submit_spot
           rescue => e
             errs.append(e)
           end
@@ -466,29 +470,10 @@ module Kitchen
         raise ["Could not create a spot instance:", errs].flatten.join("\n")
       end
 
-      def submit_spot(state)
+      def submit_spot
         debug("Creating EC2 Spot Instance..")
+        instance_data = instance_generator.ec2_instance_data
 
-        spot_request_id = create_spot_request
-        # deleting the instance cancels the request, but deleting the request
-        # does not affect the instance
-        state[:spot_request_id] = spot_request_id
-        ec2.client.wait_until(
-          :spot_instance_request_fulfilled,
-          spot_instance_request_ids: [spot_request_id]
-        ) do |w|
-          w.max_attempts = config[:spot_wait] / config[:retryable_sleep]
-          w.delay = config[:retryable_sleep]
-          w.before_attempt do |attempts|
-            c = attempts * config[:retryable_sleep]
-            t = config[:spot_wait]
-            info "Waited #{c}/#{t}s for spot request <#{spot_request_id}> to become fulfilled."
-          end
-        end
-        ec2.get_instance_from_spot_request(spot_request_id)
-      end
-
-      def create_spot_request
         request_duration = config[:spot_wait]
         config_spot_price = config[:spot_price].to_s
         if %w{ondemand on-demand}.include?(config_spot_price)
@@ -496,56 +481,36 @@ module Kitchen
         else
           spot_price = config_spot_price
         end
-        request_data = {
-          spot_price: spot_price,
-          launch_specification: instance_generator.ec2_instance_data,
+        spot_options = {
+          spot_instance_type: "persistent", # Cannot use one-time with valid_until
           valid_until: Time.now + request_duration,
+          instance_interruption_behavior: "stop",
         }
         if config[:block_duration_minutes]
-          request_data[:block_duration_minutes] = config[:block_duration_minutes]
+          spot_options[:block_duration_minutes] = config[:block_duration_minutes]
         end
-
-        response = ec2.client.request_spot_instances(request_data)
-        response[:spot_instance_requests][0][:spot_instance_request_id]
-      end
-
-      def tag_server(server)
-        if config[:tags] && !config[:tags].empty?
-          tags = config[:tags].map do |k, v|
-            # we convert the value to a string because
-            # nils should be passed as an empty String
-            # and Integers need to be represented as Strings
-            { key: k.to_s, value: v.to_s }
-          end
-          server.create_tags(tags: tags)
+        unless spot_price == "" # i.e. on-demand
+          spot_options[:max_price] = spot_price
         end
-      end
 
-      def tag_volumes(server)
-        if config[:tags] && !config[:tags].empty?
-          tags = config[:tags].map do |k, v|
-            { key: k.to_s, value: v.to_s }
-          end
-          server.volumes.each do |volume|
-            volume.create_tags(tags: tags)
-          end
-        end
-      end
+        instance_data[:instance_market_options] = {
+          market_type: "spot",
+          spot_options: spot_options,
+        }
 
-      # Compares the requested volume count vs what has actually been set to be
-      # attached to the instance. The information requested through
-      # ec2.client.described_volumes is updated before the instance volume
-      # information.
-      def wait_until_volumes_ready(server, state)
-        wait_with_destroy(server, state, "volumes to be ready") do |aws_instance|
-          described_volume_count = 0
-          ready_volume_count = 0
-          if aws_instance.exists?
-            described_volume_count = ec2.client.describe_volumes(filters: [
-              { name: "attachment.instance-id", values: ["#{state[:server_id]}"] }]).volumes.length
-            aws_instance.volumes.each { ready_volume_count += 1 }
-          end
-          (described_volume_count > 0) && (described_volume_count == ready_volume_count)
+        # The preferred way to create a spot instance is via request_spot_instances()
+        # However, it does not allow for tagging to occur at creation time.
+        # create_instances() allows creation of tagged spot instances, but does
+        # not retry if the price could not be satisfied immediately.
+        Retryable.retryable(
+          tries: config[:spot_wait] / config[:retryable_sleep],
+          sleep: lambda { |_n| config[:retryable_sleep] },
+          on: ::Aws::EC2::Errors::SpotMaxPriceTooLow
+        ) do |retries|
+          c = retries * config[:retryable_sleep]
+          t = config[:spot_wait]
+          info "Waited #{c}/#{t}s for spot request to become fulfilled."
+          ec2.create_instance(instance_data)
         end
       end
 

data/lib/kitchen/driver/ec2_version.rb

@@ -2,7 +2,7 @@
 #
 # Author:: Fletcher Nichol (<fnichol@nichol.ca>)
 #
-# Copyright:: 2016-2020, Chef Software, Inc.
+# Copyright:: Chef Software, Inc.
 # Copyright:: 2012-2018, Fletcher Nichol
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,6 +22,6 @@ module Kitchen
   module Driver
 
     # Version string for EC2 Test Kitchen driver
-    EC2_VERSION = "3.3.0".freeze
+    EC2_VERSION = "3.7.1".freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kitchen-ec2
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.7.1
 platform: ruby
 authors:
 - Fletcher Nichol
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-15 00:00:00.000000000 Z
+date: 2020-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-kitchen
@@ -30,34 +30,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
-- !ruby/object:Gem::Dependency
-  name: excon
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: multi_json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ec2
   requirement: !ruby/object:Gem::Requirement
@@ -134,20 +106,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.7'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -168,14 +126,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 1.1.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 1.1.2
 - !ruby/object:Gem::Dependency
   name: climate_control
   requirement: !ruby/object:Gem::Requirement
@@ -224,14 +182,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A Test Kitchen Driver for Amazon EC2