ec2_amitools 1.0.2

data/lib/ec2/amitools/mapids.rb

@@ -0,0 +1,148 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/minimalec2'
+
+class KernelMappings
+  ENDPOINT = "http://ec2.amazonaws.com"
+
+  class MappingError < StandardError; end
+
+  attr_reader :source_image_info
+  attr_reader :target_image_info
+
+  #----------------------------------------------------------------------------#
+
+  def get_ec2_client(akid, secretkey, endpoint=nil)
+    EC2::EC2Client.new(akid, secretkey, endpoint)
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def add_endpoint_from_env(endpoints)
+    str = ENV['AMITOOLS_EC2_REGION_ENDPOINT']
+    if not str
+      return
+    end
+
+    components = str.split('@').collect{|s| s.strip()}
+    if components.size != 2
+      return
+    end
+    
+    endpoints[components[0]] = components[1]
+  end
+
+  def get_endpoints(akid, secretkey, endpoint=nil)
+    endpoint ||= ENDPOINT
+    endpoints = {}
+    ec2conn = get_ec2_client(akid, secretkey, endpoint)
+    resp_doc = ec2conn.describe_regions()
+    REXML::XPath.each(resp_doc.root, '/DescribeRegionsResponse/regionInfo/item') do |region|
+      region_name = REXML::XPath.first(region, 'regionName').text
+      region_url = REXML::XPath.first(region, 'regionEndpoint').text
+      region_url = 'https://'+region_url unless region_url =~ %r{^https?://}
+      endpoints[region_name] = region_url
+    end
+    add_endpoint_from_env(endpoints)
+    endpoints
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def create_ec2_connections(akid, secretkey, endpoint=nil)
+    @ec2conn = {}
+    @endpoints = get_endpoints(akid, secretkey, endpoint)
+    @endpoints.each do |region, endpoint|
+      @ec2conn[region] = get_ec2_client(akid, secretkey, endpoint)
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def get_source_images_info()
+    @ec2conn.each do |region, connection|
+      $stderr.puts "Getting source data from #{region} #{@source_images.inspect}..." if @verbose
+      resp_doc = connection.describe_images(@source_images)
+      @source_images_info = parse_imageset(resp_doc)
+      $stderr.puts "Found #{@source_images_info.size} images in #{region}." if @verbose
+      # We assume all the images we're trying to map are in the same region.
+      return if @source_images_info.size > 0
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def get_target_candidates()
+    owners = @source_images_info.map { |ii| ii['imageOwnerId'] }
+    $stderr.puts "Getting target data from #{region} #{owners.inspect}..." if @verbose
+    resp_doc = @ec2conn[@target_region].describe_images([], :ownerIds => owners)
+    @target_images_info = parse_imageset(resp_doc)
+    $stderr.puts "Found #{@target_images_info.size} images in #{region}." if @verbose
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def parse_imageset(resp_doc)
+    images_info = []
+    resp_doc.each_element('DescribeImagesResponse/imagesSet/item') do |image|
+      image_data = {}
+      image.each_element() { |elem| image_data[elem.name] = elem.text }
+      images_info << image_data
+    end
+    images_info
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def initialize(akid, secretkey, source_images, target_region, endpoint=nil)
+    @target_region = target_region
+    @source_images = source_images
+    create_ec2_connections(akid, secretkey, endpoint)
+    unless @ec2conn.has_key?(target_region)
+      raise MappingError.new("Invalid region: #{target_region}")
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def matchable_image(image)
+    summary = {}
+    ['imageOwnerId', 'imageType'].each { |key| summary[key] = image[key] }
+    summary[:s3key] = image['imageLocation'].sub(%r{^/*[^/]+/},'')
+    summary
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def find_missing_targets(targets)
+    @target_images_info.each { |ii| targets.delete(ii['imageId']) } unless @target_images_info.nil?
+    return nil if targets.empty?
+    lookups = parse_imageset(@ec2conn[@target_region].describe_images(targets))
+    lookups.each { |ii| targets.delete(ii['imageId']) }
+    return nil if targets.empty?
+    targets
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def [](identifier)
+    if @target_images_info.nil?
+      get_source_images_info()
+      get_target_candidates()
+    end
+    source_info = @source_images_info.find { |ii| ii['imageId'] == identifier }
+    raise MappingError.new("'#{identifier}' not found.") if source_info.nil?
+    source_match = matchable_image(source_info)
+    target_info = @target_images_info.find { |ii| source_match == matchable_image(ii) }
+    raise MappingError.new("Mapping for '#{identifier}' not found.") if target_info.nil?
+    target_info['imageId']
+  end
+end

data/lib/ec2/amitools/migratebundle.rb

@@ -0,0 +1,222 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/common/s3support'
+require 'ec2/amitools/region'
+require 'ec2/amitools/migratebundleparameters'
+require 'ec2/amitools/migratemanifest'
+require 'ec2/amitools/downloadbundle'
+require 'ec2/amitools/uploadbundle'
+require 'fileutils'
+require 'ec2/amitools/tool_base'
+
+MIGRATE_BUNDLE_NAME = 'ec2-migrate-bundle'
+
+#------------------------------------------------------------------------------#
+
+MIGRATE_BUNDLE_MANUAL =<<TEXT
+#{MIGRATE_BUNDLE_NAME} is a command line tool to assist with migrating AMIs to new regions.
+
+*** This tool is deprecated. ***
+Please consider using ec2-migrate-image from the EC2 API tools.
+You can download the EC2 API tools here:
+http://aws.amazon.com/developertools/351
+
+#{MIGRATE_BUNDLE_NAME} will:
+- download the manifest of the specified AMI
+- attempt to automatically find replacement kernels and ramdisks
+- optionally replace kernels and ramdisks with user-specified replacements
+- copy AMI into new bucket
+- upload new migrated manifest into new bucket
+
+This tool will not register the AMI in the new region.
+
+TEXT
+
+class BundleMigrator < AMITool
+  
+  def with_temp_dir(manifest_name)
+    # Set up temporary dir
+    tempdir = File::join(Dir::tmpdir, "ami-migration-#{manifest_name}")
+    if File::exists?(tempdir)
+      raise EC2FatalError.new(2, "Temporary directory '#{tempdir}' already exists. Please delete or rename it and try again.")
+    end
+    Dir::mkdir(tempdir)
+
+    # Let the caller use it
+    begin
+      result = yield tempdir
+    rescue Exception => e
+      # Nuke it
+      FileUtils::rm_rf(tempdir)
+      raise e
+    end
+    # Nuke it
+    FileUtils::rm_rf(tempdir)
+    result
+  end
+
+  #------------------------------------------------------------------------------#
+
+  def uri2string(uri)
+    s = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
+    # Remove the trailing '/'.
+    return (s[-1..-1] == "/" ? s[0..-2] : s)
+  end
+
+  #------------------------------------------------------------------------------#
+
+  def make_s3_connection(s3_url, user, pass, bucket)
+    s3_uri = URI.parse(s3_url)
+    s3_url = uri2string(s3_uri)
+    v2_bucket = EC2::Common::S3Support::bucket_name_s3_v2_safe?(bucket)
+    EC2::Common::S3Support.new(s3_url, user, pass, (v2_bucket ? nil : :path))
+  end
+
+  #------------------------------------------------------------------------------#
+
+  def download_manifest(s3_conn, bucket, manifest_name, manifest_path, user_pk_path, retry_stuff)
+    BundleDownloader.new().download_manifest(s3_conn,
+                                             bucket,
+                                             manifest_name,
+                                             manifest_path,
+                                             user_pk_path,
+                                             retry_stuff)
+  end
+
+  #------------------------------------------------------------------------------#
+
+  def get_part_filenames(manifest_path, user_cert_path)
+    manifest = ManifestMigrator.new().get_manifest(manifest_path, user_cert_path)
+    manifest.parts.collect { |part| part.filename }.sort
+  end
+
+  #------------------------------------------------------------------------------#
+
+  def copy_part(s3_conn, bucket, keyprefix, dest_bucket, dest_keyprefix, part, acl, retry_copy)
+    source = "/#{bucket}/#{keyprefix}#{part}"
+    retry_s3(retry_copy) do
+      begin
+        s3_conn.copy(dest_bucket, dest_keyprefix+part, source, {"x-amz-acl"=>acl})
+        return
+      rescue => e
+        raise TryFailed.new("Failed to copy \"#{part}\": #{e.message}")
+      end
+    end
+  end
+
+  #------------------------------------------------------------------------------#
+
+  def upload_manifest(s3_conn, key, manifest_path, bucket, tempdir, acl, retry_stuff)
+    BundleUploader.new().upload(s3_conn, bucket, key, manifest_path, acl, retry_stuff)
+  end
+
+  #------------------------------------------------------------------------------#
+
+  def migrate_bundle(s3_url,
+                     bucket,
+                     keyprefix,
+                     dest_bucket,
+                     dest_keyprefix,
+                     manifest_name,
+                     user_pk_path,
+                     user_cert_path,
+                     user,
+                     pass,
+                     location,
+                     kernel_id=nil,
+                     ramdisk_id=nil,
+                     acl='aws-exec-read',
+                     retry_stuff=nil,
+                     use_mapping=true,
+                     region=nil)
+
+    src_s3_conn = make_s3_connection(s3_url, user, pass, bucket)
+    dest_s3_conn = make_s3_connection(s3_url, user, pass, dest_bucket)
+    
+    # Locate destination bucket and create it if necessary.
+    bu = BundleUploader.new()
+    bu.check_bucket_name(dest_bucket)
+    bucket_location = bu.get_bucket_location(dest_s3_conn, dest_bucket)
+    bu.create_bucket(dest_s3_conn, dest_bucket, bucket_location, location, retry_stuff)
+    
+    # Region/location hack:
+    if region.nil?
+      location ||= bucket_location
+      region = AwsRegion.guess_region_from_s3_bucket(location)
+      puts "Region not provided, guessing from S3 location: #{region}"
+    end
+    
+    with_temp_dir(manifest_name) do |tempdir|
+      manifest_path = File::join(tempdir, "temp-migration.manifest.xml")
+      download_manifest(src_s3_conn, bucket, keyprefix+manifest_name, manifest_path, user_pk_path, retry_stuff)
+      
+      ManifestMigrator.new().migrate_manifest(manifest_path,
+                                              user_pk_path,
+                                              user_cert_path,
+                                              user,
+                                              pass,
+                                              use_mapping,
+                                              kernel_id,
+                                              ramdisk_id,
+                                              region,
+                                              true)
+      
+      get_part_filenames(manifest_path, user_cert_path).each do |part|
+        $stdout.puts("Copying '#{part}'...")
+        copy_part(dest_s3_conn, bucket, keyprefix, dest_bucket, dest_keyprefix, part, acl, retry_stuff)
+      end
+      upload_manifest(dest_s3_conn, dest_keyprefix+manifest_name, manifest_path, dest_bucket, tempdir, acl, retry_stuff)
+    end
+  end
+
+  #------------------------------------------------------------------------------#
+  # Overrides
+  #------------------------------------------------------------------------------#
+
+  def get_manual()
+    MIGRATE_BUNDLE_MANUAL
+  end
+
+  def get_name()
+    MIGRATE_BUNDLE_NAME
+  end
+
+  def main(p)
+    migrate_bundle(p.s3_url,
+                   p.bucket,
+                   p.keyprefix,
+                   p.dest_bucket,
+                   p.dest_keyprefix,
+                   p.manifest_name,
+                   p.user_pk_path,
+                   p.user_cert_path,
+                   p.user,
+                   p.pass,
+                   p.location,
+                   p.kernel_id,
+                   p.ramdisk_id,
+                   p.acl,
+                   p.retry,
+                   p.use_mapping,
+                   p.region)
+    
+    $stdout.puts("\nYour new bundle is in S3 at the following location:")
+    $stdout.puts("#{p.dest_bucket}/#{p.dest_keyprefix}#{p.manifest_name}")
+    $stdout.puts("Please register it using your favorite EC2 client.")
+  end
+
+end
+
+#------------------------------------------------------------------------------#
+# Script entry point. Execute only if this file is being executed.
+if __FILE__ == $0 || $0.match(/bin\/ec2-migrate-bundle/)
+  BundleMigrator.new().run(MigrateBundleParameters)
+end

data/lib/ec2/amitools/migratebundleparameters.rb

@@ -0,0 +1,173 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/parameters_base'
+require 'ec2/platform/current'
+require 'ec2/amitools/region'
+
+#------------------------------------------------------------------------------#
+
+class MigrateBundleParameters < ParametersBase
+  include EC2::Platform::Current::Constants
+
+  MANIFEST_DESCRIPTION = "The name the manifest file."
+  DIRECTORY_DESCRIPTION = ["The directory containing the bundled AMI parts to upload.",
+                           "Defaults to the directory containing the manifest."]
+  USER_CERT_PATH_DESCRIPTION = "The path to the user's PEM encoded RSA public key certificate file."
+  USER_PK_PATH_DESCRIPTION = "The path to the user's PEM encoded RSA private key file."
+  EC2_CERT_PATH_DESCRIPTION = ['The path to the EC2 X509 public key certificate bundled into the AMI.',
+                               "Defaults to '#{Bundling::EC2_X509_CERT}'."]
+  KERNEL_DESCRIPTION = "Kernel id to bundle into the AMI."
+  RAMDISK_DESCRIPTION = "Ramdisk id to bundle into the AMI."
+  DEST_BUCKET_DESCRIPTION = "The bucket to copy bundle to. Created if nonexistent."
+  BUCKET_DESCRIPTION = "The bucket containing the AMI to be migrated."
+  USER_DESCRIPTION = "The user's AWS access key ID."
+  PASS_DESCRIPTION = "The user's AWS secret access key."
+  ACL_DESCRIPTION = ["The access control list policy [\"public-read\" | \"aws-exec-read\"].",
+                     "Defaults to \"aws-exec-read\"."]
+  URL_DESCRIPTION = "The S3 service URL. Defaults to https://s3.amazonaws.com."
+  RETRY_DESCRIPTION = "Automatically retry failed uploads. Use with caution."
+  LOCATION_DESCRIPTION = "The location of the bucket to upload to [#{AwsRegion.regions.join(',')}]."
+  NO_MAPPING_DESCRIPTION = "Do not perform automatic mappings."
+  REGION_DESCRIPTION = "Region to look up in the mapping file."
+
+  attr_accessor :user_pk_path,
+                :user_cert_path,
+                :ec2_cert_path,
+                :user,
+                :pass,
+                :kernel_id,
+                :ramdisk_id,
+                :s3_url,
+                :bucket,
+                :keyprefix,
+                :dest_bucket,
+                :dest_keyprefix,
+                :manifest_name,
+                :location,
+                :acl,
+                :retry,
+                :use_mapping,
+                :region
+  
+  def split_container(container)
+    splitbits = container.sub(%r{^/*},'').sub(%r{/*$},'').split("/")
+    bucket = splitbits.shift
+    keyprefix = splitbits.join("/")
+    keyprefix += "/" unless keyprefix.empty?
+    [bucket, keyprefix]
+  end
+
+  def mandatory_params()
+    on('-c', '--cert PATH', String, USER_CERT_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--cert')
+      @user_cert_path = path
+    end
+    
+    on('-k', '--privatekey PATH', String, USER_PK_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--privatekey')
+      @user_pk_path = path
+    end
+    
+    on('-m', '--manifest NAME', String, MANIFEST_DESCRIPTION) do |manifest|
+      raise InvalidValue.new("--manifest", manifest) unless manifest =~ /\.manifest\.xml$/
+      assert_good_key(manifest, '--manifest')
+      @manifest_name = manifest
+    end
+    
+    on('-b', '--bucket BUCKET', String, BUCKET_DESCRIPTION) do |container|
+      @container = container
+      @bucket, @keyprefix = split_container(@container)
+    end
+    
+    on('-d', '--destination-bucket BUCKET', String, DEST_BUCKET_DESCRIPTION) do |dest_container|
+      @dest_container = dest_container
+      @dest_bucket, @dest_keyprefix = split_container(@dest_container)
+    end
+    
+    on('-a', '--access-key USER', String, USER_DESCRIPTION) do |user|
+      @user = user
+    end
+    
+    on('-s', '--secret-key PASSWORD', String, PASS_DESCRIPTION) do |pass|
+      @pass = pass
+    end
+  end
+  
+  def optional_params()
+    on('--ec2cert PATH', String, *EC2_CERT_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--ec2cert')
+      @ec2_cert_path = path
+    end
+    
+    on('--acl ACL', String, *ACL_DESCRIPTION) do |acl|
+      raise InvalidValue.new('--acl', acl) unless ['public-read', 'aws-exec-read'].include?(acl)
+      @acl = acl
+    end
+
+    on('--url URL', String, URL_DESCRIPTION) do |url|
+      @s3_url = url
+    end
+
+    on('--retry', RETRY_DESCRIPTION) do
+      @retry = true
+    end
+    
+    on('--location LOCATION', LOCATION_DESCRIPTION) do |location|
+      assert_option_in(location, AwsRegion.s3_locations, '--location')
+      @location = case location
+        when "eu-west-1" then "EU"
+        when "US" then :unconstrained
+        else location
+      end
+    end
+    
+    on('--kernel KERNEL_ID', String, KERNEL_DESCRIPTION) do |kernel_id|
+      @kernel_id = kernel_id
+    end
+    
+    on('--ramdisk RAMDISK_ID', String, RAMDISK_DESCRIPTION) do |ramdisk_id|
+      @ramdisk_id = ramdisk_id
+    end
+    
+    on('--no-mapping', String, NO_MAPPING_DESCRIPTION) do
+      @use_mapping = false
+    end
+
+    on('--region REGION', String, REGION_DESCRIPTION) do |region|
+      @region = region
+    end
+  end
+
+  def validate_params()
+    raise MissingMandatory.new('--manifest') unless @manifest_name
+    raise MissingMandatory.new('--cert') unless @user_cert_path
+    raise MissingMandatory.new('--privatekey') unless @user_pk_path
+    raise MissingMandatory.new('--bucket') unless @container
+    raise MissingMandatory.new('--destination-bucket') unless @dest_container
+    raise MissingMandatory.new('--access-key') unless @user
+    raise MissingMandatory.new('--secret-key') unless @pass
+  end
+
+  def set_defaults()
+    @acl ||= 'aws-exec-read'
+    @s3_url ||= 'https://s3.amazonaws.com'
+    @ec2_cert_path ||= Bundling::EC2_X509_CERT
+    @use_mapping = true if @use_mapping.nil? # False is different.
+  end
+
+  def help()
+    s = "*** This tool is deprecated. ***\n"
+    s += "Please consider using ec2-migrate-image from the EC2 API tools.\n"
+    s += "You can download the EC2 API tools here:\n"
+    s += "http://aws.amazon.com/developertools/351\n"
+    s += super()
+  end
+end