kitchen-ec2 0.10.0 → 1.0.0.beta.1

data/kitchen-ec2.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "excon"
   gem.add_dependency "multi_json"
   gem.add_dependency "aws-sdk", "~> 2"
+  gem.add_dependency "retryable", "~> 2.0"
 
   gem.add_development_dependency "rspec",     "~> 3.2"
   gem.add_development_dependency "countloc",  "~> 0.4"

data/lib/kitchen/driver/aws/client.rb

@@ -38,7 +38,8 @@ module Kitchen
           access_key_id = nil,
           secret_access_key = nil,
           session_token = nil,
-          http_proxy = nil
+          http_proxy = nil,
+          retry_limit = nil
         )
           creds = self.class.get_credentials(
             profile_name, access_key_id, secret_access_key, session_token
@@ -48,6 +49,7 @@ module Kitchen
             :credentials => creds,
             :http_proxy => http_proxy
           )
+          ::Aws.config.update(:retry_limit => retry_limit) unless retry_limit.nil?
         end
 
         # Try and get the credentials from an ordered list of locations
@@ -57,13 +59,6 @@ module Kitchen
           shared_creds = ::Aws::SharedCredentials.new(:profile_name => profile_name)
           if access_key_id && secret_access_key
             ::Aws::Credentials.new(access_key_id, secret_access_key, session_token)
-          # TODO: these are deprecated, remove them in the next major version
-          elsif ENV["AWS_ACCESS_KEY"] && ENV["AWS_SECRET_KEY"]
-            ::Aws::Credentials.new(
-              ENV["AWS_ACCESS_KEY"],
-              ENV["AWS_SECRET_KEY"],
-              ENV["AWS_TOKEN"]
-            )
           elsif ENV["AWS_ACCESS_KEY_ID"] && ENV["AWS_SECRET_ACCESS_KEY"]
             ::Aws::Credentials.new(
               ENV["AWS_ACCESS_KEY_ID"],

data/lib/kitchen/driver/aws/instance_generator.rb

@@ -41,9 +41,6 @@ module Kitchen
         # can be passed in null, others need to be ommitted if they are null
         def ec2_instance_data # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
           i = {
-            :placement => {
-              :availability_zone          => config[:availability_zone]
-            },
             :instance_type                => config[:instance_type],
             :ebs_optimized                => config[:ebs_optimized],
             :image_id                     => config[:image_id],
@@ -51,7 +48,17 @@ module Kitchen
             :subnet_id                    => config[:subnet_id],
             :private_ip_address           => config[:private_ip_address]
           }
-          i[:block_device_mappings] = block_device_mappings unless block_device_mappings.empty?
+
+          availability_zone = config[:availability_zone]
+          if availability_zone
+            if availability_zone =~ /^[a-z]$/i
+              availability_zone = "#{config[:region]}#{availability_zone}"
+            end
+            i[:placement] = { :availability_zone => availability_zone.downcase }
+          end
+          unless config[:block_device_mappings].nil? || config[:block_device_mappings].empty?
+            i[:block_device_mappings] = config[:block_device_mappings]
+          end
           i[:security_group_ids] = Array(config[:security_group_ids]) if config[:security_group_ids]
           i[:user_data] = prepared_user_data if prepared_user_data
           if config[:iam_profile_name]
@@ -80,67 +87,6 @@ module Kitchen
           i
         end
 
-        # Transforms the provided config into the appropriate hash for creating a BDM
-        # in AWS
-        def block_device_mappings # rubocop:disable all
-          return @bdms if @bdms
-          bdms = config[:block_device_mappings] || []
-          if bdms.empty?
-            if config[:ebs_volume_size] || config.fetch(:ebs_delete_on_termination, nil) ||
-                config[:ebs_device_name] || config[:ebs_volume_type]
-              # If the user didn't supply block_device_mappings but did supply
-              # the old configs, copy them into the block_device_mappings array correctly
-              # TODO: remove this logic when we remove the deprecated values
-              bdms << {
-                :ebs_volume_size => config[:ebs_volume_size] || 8,
-                :ebs_delete_on_termination => config.fetch(:ebs_delete_on_termination, true),
-                :ebs_device_name => config[:ebs_device_name] || "/dev/sda1",
-                :ebs_volume_type => config[:ebs_volume_type] || "standard"
-              }
-            end
-          end
-
-          # Convert the provided keys to what AWS expects
-          bdms = bdms.map do |bdm|
-            b = {
-              :ebs => {
-                :volume_size           => bdm[:ebs_volume_size],
-                :delete_on_termination => bdm[:ebs_delete_on_termination]
-              },
-              :device_name             => bdm[:ebs_device_name]
-            }
-            b[:ebs][:volume_type] = bdm[:ebs_volume_type] if bdm[:ebs_volume_type]
-            b[:ebs][:iops] = bdm[:ebs_iops] if bdm[:ebs_iops]
-            b[:ebs][:snapshot_id] = bdm[:ebs_snapshot_id] if bdm[:ebs_snapshot_id]
-            b[:virtual_name] = bdm[:ebs_virtual_name] if bdm[:ebs_virtual_name]
-            b
-          end
-
-          debug_if_root_device(bdms)
-
-          @bdms = bdms
-        end
-
-        # If the provided bdms match the root device in the AMI, emit log that
-        # states this
-        def debug_if_root_device(bdms)
-          return if bdms.nil? || bdms.empty?
-          image_id = config[:image_id]
-          image = ec2.resource.image(image_id)
-          begin
-            root_device_name = image.root_device_name
-          rescue ::Aws::EC2::Errors::InvalidAMIIDNotFound
-            # Not raising here because AWS will give a more meaningful message
-            # when we try to create the instance
-            return
-          end
-          bdms.find { |bdm|
-            if bdm[:device_name] == root_device_name
-              logger.info("Overriding root device [#{root_device_name}] from image [#{image_id}]")
-            end
-          }
-        end
-
         def prepared_user_data
           # If user_data is a file reference, lets read it as such
           return nil if config[:user_data].nil?

data/lib/kitchen/driver/aws/standard_platform.rb

@@ -0,0 +1,229 @@
+module Kitchen
+  module Driver
+    class Aws
+      #
+      # Lets you grab StandardPlatform objects that help search for official
+      # AMIs in your region and tell you useful tidbits like usernames.
+      #
+      # To use these, set your platform name to a supported platform name like:
+      #
+      # centos
+      # rhel
+      # fedora
+      # freebsd
+      # ubuntu
+      # windows
+      #
+      # The implementation will select the latest matching version and AMI.
+      #
+      # You can specify a version and optional architecture as well:
+      #
+      # windows-2012r2-i386
+      # centos-7
+      #
+      # Useful reference for platform AMIs:
+      # https://alestic.com/2014/01/ec2-ssh-username/
+      class StandardPlatform
+        #
+        # Create a new StandardPlatform object.
+        #
+        # @param driver [Kitchen::Driver::Ec2] The driver.
+        # @param name [String] The name of the platform (rhel, centos, etc.)
+        # @param version [String] The version of the platform (7.1, 2008sp1, etc.)
+        # @param architecture [String] The architecture (i386, x86_64)
+        #
+        def initialize(driver, name, version, architecture)
+          @driver = driver
+          @name = name
+          @version = version
+          @architecture = architecture
+        end
+
+        #
+        # The driver.
+        #
+        # @return [Kitchen::Driver::Ec2]
+        #
+        attr_reader :driver
+
+        #
+        # The name of the platform (e.g. rhel, centos, etc.)
+        #
+        # @return [String]
+        #
+        attr_reader :name
+
+        #
+        # The version of the platform (e.g. 7.1, 2008sp1, etc.)
+        #
+        # @return [String]
+        #
+        attr_reader :version
+
+        #
+        # The architecture of the platform, e.g. i386, x86_64
+        #
+        # @return [String]
+        #
+        # @see ARCHITECTURES
+        #
+        attr_reader :architecture
+
+        #
+        # Find the best matching image for the given image search.
+        #
+        # @return [String] The image ID (e.g. ami-213984723)
+        def find_image(image_search)
+          driver.debug("Searching for images matching #{image_search} ...")
+          # Convert to ec2 search format (pairs of name+values)
+          filters = image_search.map do |key, value|
+            { :name => key.to_s, :values => Array(value).map(&:to_s) }
+          end
+
+          # We prefer most recent first
+          images = driver.ec2.resource.images(:filters => filters)
+          images = sort_images(images)
+          show_returned_images(images)
+
+          # Grab the best match
+          images.first && images.first.id
+        end
+
+        #
+        # The list of StandardPlatform objects. StandardPlatforms register
+        # themselves with this.
+        #
+        # @return Array[Kitchen::Driver::Aws::StandardPlatform]
+        #
+        def self.platforms
+          @platforms ||= {}
+        end
+
+        def to_s
+          "#{name}#{version ? " #{version}" : ""}#{architecture ? " #{architecture}" : ""}"
+        end
+
+        #
+        # Instantiate a platform from a platform name.
+        #
+        # @param driver [Kitchen::Driver::Ec2] The driver.
+        # @param platform_string [String] The platform string, e.g. "windows",
+        #        "ubuntu-7.1", "centos-7-i386"
+        #
+        # @return [Kitchen::Driver::Aws::StandardPlatform]
+        #
+        def self.from_platform_string(driver, platform_string)
+          platform, version, architecture = parse_platform_string(platform_string)
+          if platform && platforms[platform]
+            platforms[platform].new(driver, platform, version, architecture)
+          end
+        end
+
+        #
+        # Detect platform from an image.
+        #
+        # @param driver [Kitchen::Driver::Ec2] The driver.
+        # @param image [Aws::Ec2::Image] The EC2 Image object.
+        #
+        # @return [Kitchen::Driver::Aws::StandardPlatform]
+        #
+        def self.from_image(driver, image)
+          platforms.each_value do |platform|
+            result = platform.from_image(driver, image)
+            return result if result
+          end
+          nil
+        end
+
+        #
+        # The list of supported architectures
+        #
+        ARCHITECTURE = %w[x86_64 i386 i86pc sun4v powerpc]
+
+        protected
+
+        #
+        # Sort a list of images by their versions, from greatest to least.
+        #
+        # This MUST perform a stable sort. (Note that `sort` and `sort_by` are
+        # not, by default, stable sorts in Ruby.)
+        #
+        # Used by the default find_image. The default version calls platform_from_image()
+        # on each image, and interprets the versions as floats (7 < 7.1 < 8).
+        #
+        # @param images [Array[Aws::Ec2::Image]] The list of images to sort
+        #
+        # @return [Array[Aws::Ec2::Image]] A sorted list.
+        #
+        def sort_by_version(images)
+          # 7.1 -> [ img1, img2, img3 ]
+          # 6 -> [ img4, img5 ]
+          # ...
+          images.group_by do |image|
+            platform = self.class.from_image(driver, image)
+            platform ? platform.version : nil
+          end.sort_by { |k, _v| k ? k.to_f : nil }.reverse.map { |_k, v| v }.flatten(1)
+        end
+
+        # Not supported yet: aix mac_os_x nexus solaris
+
+        def prefer(images, &block)
+          # Put the matching ones *before* the non-matching ones.
+          matching, non_matching = images.partition(&block)
+          matching + non_matching
+        end
+
+        private
+
+        def self.parse_platform_string(platform_string)
+          platform, version = platform_string.split("-", 2)
+
+          # If the right side is a valid architecture, use it as such
+          # i.e. debian-i386 or windows-server-2012r2-i386
+          if version && ARCHITECTURE.include?(version.split("-")[-1])
+            # server-2012r2-i386 -> server-2012r2, -, i386
+            version, _dash, architecture = version.rpartition("-")
+            version = nil if version == ""
+          end
+
+          [platform, version, architecture]
+        end
+
+        def sort_images(images)
+          # P6: We prefer more recent images over older ones
+          images = images.sort_by(&:creation_date).reverse
+          # P5: We prefer x86_64 over i386 (if available)
+          images = prefer(images) { |image| image.architecture == :x86_64 }
+          # P4: We prefer gp2 (SSD) (if available)
+          images = prefer(images) do |image|
+            image.block_device_mappings.any? do |b|
+              b.device_name == image.root_device_name && b.ebs && b.ebs.volume_type == "gp2"
+            end
+          end
+          # P3: We prefer ebs over instance_store (if available)
+          images = prefer(images) { |image| image.root_device_type == "ebs" }
+          # P2: We prefer hvm (the modern standard)
+          images = prefer(images) { |image| image.virtualization_type == "hvm" }
+          # P1: We prefer the latest version over anything else
+          sort_by_version(images)
+        end
+
+        def show_returned_images(images)
+          if images.empty?
+            driver.error("Search returned 0 images.")
+          else
+            driver.debug("Search returned #{images.size} images:")
+            images.each do |image|
+              platform = self.class.from_image(driver, image)
+              if platform
+                driver.debug("- #{image.name}: Detected #{platform}. #{driver.image_info(image)}")
+              else
+                driver.debug("- #{image.name}: No platform detected. #{driver.image_info(image)}")
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kitchen/driver/aws/standard_platform/centos.rb

@@ -0,0 +1,46 @@
+require "kitchen/driver/aws/standard_platform"
+
+module Kitchen
+  module Driver
+    class Aws
+      class StandardPlatform
+        # https://wiki.centos.org/Cloud/AWS
+        class Centos < StandardPlatform
+          StandardPlatform.platforms["centos"] = self
+
+          def username
+            # Centos 6.x images use root as the username (but the "centos 6"
+            # updateable image uses "centos")
+            return "root" if version && version.start_with?("6.")
+            "centos"
+          end
+
+          def image_search
+            search = {
+              "owner-alias" => "aws-marketplace",
+              "name" => ["CentOS Linux #{version}*", "CentOS-#{version}*-GA-*"]
+            }
+            search["architecture"] = architecture if architecture
+            search
+          end
+
+          def sort_by_version(images)
+            # 7.1 -> [ img1, img2, img3 ]
+            # 6 -> [ img4, img5 ]
+            # ...
+            images.group_by { |image| self.class.from_image(driver, image).version }.
+              sort_by { |k, _v| (k && k.include?(".") ? k.to_f : "#{k}.999".to_f) }.
+              reverse.map { |_k, v| v }.flatten(1)
+          end
+
+          def self.from_image(driver, image)
+            if image.name =~ /centos/i
+              image.name =~ /\b(\d+(\.\d+)?)\b/i
+              new(driver, "centos", (Regexp.last_match || [])[1], image.architecture)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kitchen/driver/aws/standard_platform/debian.rb

@@ -0,0 +1,50 @@
+require "kitchen/driver/aws/standard_platform"
+
+module Kitchen
+  module Driver
+    class Aws
+      class StandardPlatform
+        # https://wiki.debian.org/Cloud/AmazonEC2Image
+        class Debian < StandardPlatform
+          StandardPlatform.platforms["debian"] = self
+
+          DEBIAN_CODENAMES = {
+            "8" => "jessie",
+            "7" => "wheezy",
+            "6" => "squeeze"
+          }
+
+          def username
+            "admin"
+          end
+
+          def codename
+            version ? DEBIAN_CODENAMES[version] : DEBIAN_CODENAMES.values.first
+          end
+
+          def image_search
+            search = {
+              "owner-id" => "379101102735",
+              "name" => "debian-#{codename}-*"
+            }
+            search["architecture"] = architecture if architecture
+            search
+          end
+
+          def self.from_image(driver, image)
+            if image.name =~ /debian/i
+              image.name =~ /\b(\d+|#{DEBIAN_CODENAMES.values.join("|")})\b/i
+              version = (Regexp.last_match || [])[1]
+              if version && version.to_i == 0
+                version = DEBIAN_CODENAMES.find do |_v, codename|
+                  codename == version.downcase
+                end.first
+              end
+              new(driver, "debian", version, image.architecture)
+            end
+          end
+        end
+      end
+    end
+  end
+end