ec2_amitools 1.0.2

data/lib/ec2/amitools/migratemanifest.rb

@@ -0,0 +1,225 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/migratemanifestparameters'
+require 'ec2/amitools/manifest_wrapper'
+require 'fileutils'
+require 'csv'
+require 'net/http'
+require 'ec2/amitools/tool_base'
+require 'ec2/amitools/mapids'
+
+MIGRATE_MANIFEST_NAME = "ec2-migrate-manifest"
+
+MIGRATE_MANIFEST_MANUAL =<<TEXT
+#{MIGRATE_MANIFEST_NAME} is a command line tool to assist with migrating AMIs to new regions.
+
+#{MIGRATE_MANIFEST_NAME} will:
+- automatically replace kernels and ramdisks with replacements suitable for a
+  particular target region
+- optionally replace kernels and ramdisks with user-specified replacements
+
+TEXT
+
+class BadManifestError < RuntimeError
+  def initialize(manifest, msg)
+    super("Bad manifest '#{manifest}': #{msg}")
+  end
+end
+
+class ManifestMigrator < AMITool
+  include EC2::Platform::Current::Constants
+  
+  def get_manifest(manifest_path, user_cert_path)
+    unless File::exists?(manifest_path)
+      raise BadManifestError.new(manifest_path, "File not found.")
+    end
+    begin
+      manifest = ManifestWrapper.new(File.open(manifest_path).read())
+    rescue ManifestWrapper::InvalidManifest => e
+      raise BadManifestError.new(manifest_path, e.message)
+    end
+    unless manifest.authenticate(File.open(user_cert_path))
+      raise BadManifestError.new(manifest_path, "Manifest fails authentication.")
+    end
+    manifest
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def get_mappings(*args)
+    KernelMappings.new(*args)
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def map_identifiers(manifest, user, pass, region, kernel_id=nil, ramdisk_id=nil)
+    if region.nil?
+      raise EC2FatalError.new(1, "No region provided, cannot map automatically.")
+    end
+    if manifest.kernel_id.nil? and manifest.ramdisk_id.nil?
+      # Exit early if we have nothing to do
+      return [kernel_id, ramdisk_id]
+    end
+
+    begin
+      mappings = get_mappings(user, pass, [manifest.kernel_id, manifest.ramdisk_id].compact, region)
+    rescue KernelMappings::MappingError => e
+      raise EC2FatalError.new(7, e.message)
+    end
+
+    begin
+      if manifest.kernel_id
+        kernel_id ||= mappings[manifest.kernel_id]
+      end
+      if manifest.ramdisk_id
+        ramdisk_id ||= mappings[manifest.ramdisk_id]
+      end
+      warn_about_mappings(mappings.find_missing_targets([kernel_id, ramdisk_id].compact))
+    rescue KernelMappings::MappingError => e
+      raise EC2FatalError.new(6, e.message)
+    end
+    [kernel_id, ramdisk_id]
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def backup_manifest(manifest_path, quiet=false)
+    backup_manifest = "#{manifest_path}.bak"
+    if File::exists?(backup_manifest)
+      raise EC2FatalError.new(2, "Backup file '#{backup_manifest}' already exists. Please delete or rename it and try again.")
+    end
+    $stdout.puts("Backing up manifest...") unless quiet
+    $stdout.puts("Backup manifest at #{backup_manifest}") if @debug
+    FileUtils::copy(manifest_path, backup_manifest)
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def build_migrated_manifest(manifest, user_pk_path, kernel_id=nil, ramdisk_id=nil)
+    new_manifest = ManifestV20071010.new()
+    manifest_params = {
+      :name => manifest.name,
+      :user => manifest.user,
+      :image_type => manifest.image_type,
+      :arch => manifest.arch,
+      :reserved => nil,
+      :parts => manifest.parts.map { |part| [part.filename, part.digest] },
+      :size => manifest.size,
+      :bundled_size => manifest.bundled_size,
+      :user_encrypted_key => manifest.user_encrypted_key,
+      :ec2_encrypted_key => manifest.ec2_encrypted_key,
+      :cipher_algorithm => manifest.cipher_algorithm,
+      :user_encrypted_iv => manifest.user_encrypted_iv,
+      :ec2_encrypted_iv => manifest.ec2_encrypted_iv,
+      :digest => manifest.digest,
+      :digest_algorithm => manifest.digest_algorithm,
+      :privkey_filename => user_pk_path,
+      :kernel_id => kernel_id,
+      :ramdisk_id => ramdisk_id,
+      :product_codes => manifest.product_codes,
+      :ancestor_ami_ids => manifest.ancestor_ami_ids,
+      :block_device_mapping => manifest.block_device_mapping,
+      :bundler_name => manifest.bundler_name,
+      :bundler_version => manifest.bundler_version,
+      :bundler_release => manifest.bundler_release,
+      :kernel_name => manifest.kernel_name,
+    }
+    new_manifest.init(manifest_params)
+    new_manifest
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def check_and_warn(manifest, kernel_id, ramdisk_id)
+    if (manifest.kernel_id and kernel_id.nil?) or (manifest.ramdisk_id and ramdisk_id.nil?)
+      message = ["This operation will remove the kernel and/or ramdisk associated with",
+                 "the AMI. This may cause the AMI to fail to launch unless you specify",
+                 "an appropriate kernel and ramdisk at launch time.",
+                ].join("\n")
+      unless warn_confirm(message)
+        raise EC2StopExecution.new()
+      end
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def warn_about_mappings(problems)
+    return if problems.nil?
+    message = ["The following identifiers do not exist in the target region:",
+               "  " + problems.inspect.gsub(/['"]/, '')
+              ].join("\n")
+    unless warn_confirm(message)
+      raise EC2StopExecution.new()
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def migrate_manifest(manifest_path,
+                       user_pk_path,
+                       user_cert_path,
+                       user=nil,
+                       pass=nil,
+                       use_mapping=true,
+                       kernel_id=nil,
+                       ramdisk_id=nil,
+                       region=nil,
+                       quiet=false)
+    manifest = get_manifest(manifest_path, user_cert_path)
+    backup_manifest(manifest_path, quiet)
+    if use_mapping
+      kernel_id, ramdisk_id = map_identifiers(manifest,
+                                              user,
+                                              pass,
+                                              region,
+                                              kernel_id,
+                                              ramdisk_id)
+    end
+    check_and_warn(manifest, kernel_id, ramdisk_id)
+    new_manifest = build_migrated_manifest(manifest, user_pk_path, kernel_id, ramdisk_id)
+    File.open(manifest_path, 'w') { |f| f.write(new_manifest.to_s) }
+    $stdout.puts("Successfully migrated #{manifest_path}") unless quiet
+    $stdout.puts("It is now suitable for use in #{region}.") unless quiet
+  end
+
+  #------------------------------------------------------------------------------#
+  # Overrides
+  #------------------------------------------------------------------------------#
+
+  def get_manual()
+    MIGRATE_MANIFEST_MANUAL
+  end
+
+  def get_name()
+    MIGRATE_MANIFEST_NAME
+  end
+
+  def main(p)
+    migrate_manifest(p.manifest_path,
+                     p.user_pk_path,
+                     p.user_cert_path,
+                     p.user,
+                     p.pass,
+                     p.use_mapping,
+                     p.kernel_id,
+                     p.ramdisk_id,
+                     p.region)
+  end
+
+end
+
+#------------------------------------------------------------------------------#
+# Script entry point. Execute only if this file is being executed.
+
+if __FILE__ == $0 || $0.match(/bin\/ec2-migrate-manifest/)
+  ManifestMigrator.new().run(MigrateManifestParameters)
+end

data/lib/ec2/amitools/migratemanifestparameters.rb

@@ -0,0 +1,118 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/parameters_base'
+require 'ec2/amitools/region'
+require 'ec2/platform/current'
+
+#------------------------------------------------------------------------------#
+
+class MigrateManifestParameters < ParametersBase
+  include EC2::Platform::Current::Constants
+
+  MANIFEST_DESCRIPTION = "The path to the manifest file."
+  DIRECTORY_DESCRIPTION = ["The directory containing the bundled AMI parts to upload.",
+                      "Defaults to the directory containing the manifest."]
+  EC2_CERT_PATH_DESCRIPTION = ['The path to the EC2 X509 public key certificate bundled into the AMI.',
+                               "Defaults to the certificate of the region (usually '#{Bundling::EC2_X509_CERT}')."]
+  KERNEL_DESCRIPTION = "Kernel id to bundle into the AMI."
+  RAMDISK_DESCRIPTION = "Ramdisk id to bundle into the AMI."
+  USER_DESCRIPTION = "The user's AWS access key ID."
+  PASS_DESCRIPTION = "The user's AWS secret access key."
+  NO_MAPPING_DESCRIPTION = "Do not perform automatic mappings."
+  REGION_DESCRIPTION = "Region to look up in the mapping file."
+  
+  attr_accessor :user_pk_path,
+                :user_cert_path,
+                :user,
+                :pass,
+                :ec2_cert_path,
+                :manifest_path,
+                :kernel_id,
+                :ramdisk_id,
+                :use_mapping,
+                :region
+    
+  #----------------------------------------------------------------------------#
+
+  def mandatory_params()
+    on('-c', '--cert PATH', String, USER_CERT_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--cert')
+      @user_cert_path = path
+    end
+    
+    on('-k', '--privatekey PATH', String, USER_PK_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--privatekey')
+      @user_pk_path = path
+    end
+
+    on('-m', '--manifest PATH', String, MANIFEST_DESCRIPTION) do |manifest|
+      assert_file_exists(manifest, '--manifest')
+      @manifest_path = manifest
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def optional_params()
+    on('-a', '--access-key USER', String, USER_DESCRIPTION) do |user|
+      @user = user
+    end
+    
+    on('-s', '--secret-key PASSWORD', String, PASS_DESCRIPTION) do |pass|
+      @pass = pass
+    end
+    
+    on('--ec2cert PATH', String, *EC2_CERT_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--ec2cert')
+      @ec2_cert_path = path
+    end
+    
+    on('--kernel KERNEL_ID', String, KERNEL_DESCRIPTION) do |kernel_id|
+      @kernel_id = kernel_id
+    end
+    
+    on('--ramdisk RAMDISK_ID', String, RAMDISK_DESCRIPTION) do |ramdisk_id|
+      @ramdisk_id = ramdisk_id
+    end
+    
+    on('--no-mapping', String, NO_MAPPING_DESCRIPTION) do
+      @use_mapping = false
+    end
+
+    on('--region REGION', String, REGION_DESCRIPTION) do |region|
+      @region = region
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def validate_params()
+    raise MissingMandatory.new('--manifest') unless @manifest_path
+    raise MissingMandatory.new('--cert') unless @user_cert_path
+    raise MissingMandatory.new('--privatekey') unless @user_pk_path
+    @use_mapping = true if @use_mapping.nil? # False is different.
+    if @use_mapping
+      raise ParameterExceptions::Error.new('If using automatic mapping, --region must be provided.') unless @region
+      raise ParameterExceptions::Error.new('If using automatic mapping, --access-key must be provided.') unless @user
+      raise ParameterExceptions::Error.new('If using automatic mapping, --secret-key must be provided.') unless @pass
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def set_defaults()
+    @ec2_cert_path ||= case
+      when (@region=="us-gov-west-1") then Bundling::EC2_X509_GOV_CERT
+      when (@region=="cn-north-1") then Bundling::EC2_X509_CN_NORTH_1_CERT
+      else Bundling::EC2_X509_CERT
+    end
+  end
+end

data/lib/ec2/amitools/minimalec2.rb

@@ -0,0 +1,116 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'base64'
+require 'cgi'
+require 'openssl'
+require 'digest/sha1'
+require 'net/https'
+require 'rexml/document'
+require 'time'
+require 'ec2/amitools/version'
+
+module EC2
+  class EC2Client
+    attr_accessor :verbose
+    attr_accessor :aws_access_key_id
+    attr_accessor :aws_secret_access_key
+    attr_accessor :http
+
+    def parse_url(url)
+      bits = url.split(":")
+      secure = {"https"=>true, "http"=>false}[bits[0]]
+      port = secure ? 443 : 80
+      port = Integer(bits[2]) if bits.size > 2
+      server = bits[1][2..-1]
+      [server, port, secure]
+    end
+    
+    def initialize(akid, secretkey, url)
+      @aws_access_key_id = akid
+      @aws_secret_access_key = secretkey
+      server, port, is_secure = parse_url(url)
+      @http = Net::HTTP.new(server, port)
+      @http.use_ssl = is_secure
+      @verbose = false
+    end
+
+    def pathlist(key, arr)
+      params = {}
+      arr.each_with_index do |value, i|
+        params["#{key}.#{i+1}"] = value
+      end
+      params
+    end
+    
+    def describe_regions(regionNames=[])
+      params = pathlist("regionName", regionNames)
+      make_request("DescribeRegions", params)
+    end
+
+    def describe_images(imageIds=[], kwargs={})
+      params = pathlist("ImageId", imageIds)
+      params.merge!(pathlist("Owner", kwargs[:owners])) if kwargs[:owners]
+      params.merge!(pathlist("ExecutableBy", kwargs[:executableBy])) if kwargs[:executableBy]
+      make_request("DescribeImages", params)
+    end
+
+    def make_request(action, params, data='')
+      resp = nil
+      @http.start do
+        params.merge!({ "Action"=>action,
+                        "SignatureVersion"=>"1",
+                        "AWSAccessKeyId"=>@aws_access_key_id,
+                        "Version"=> "2008-12-01",
+                        "Timestamp"=>Time.now.getutc.iso8601,
+                      })
+        p params if @verbose
+        
+        canonical_string = params.sort_by { |param| param[0].downcase }.map { |param| param.join }.join
+        puts canonical_string if @verbose
+        sig = encode(@aws_secret_access_key, canonical_string)
+        
+        path = "?" + params.sort.collect do |param|
+          CGI::escape(param[0]) + "=" + CGI::escape(param[1])
+        end.join("&") + "&Signature=" + sig
+        
+        puts path if @verbose
+        
+        req = Net::HTTP::Get.new("/#{path}")
+        
+        # ruby will automatically add a random content-type on some verbs, so
+        # here we add a dummy one to 'supress' it.  change this logic if having
+        # an empty content-type header becomes semantically meaningful for any
+        # other verb.
+        req['Content-Type'] ||= ''
+        req['User-Agent'] = 'ec2-migrate-manifest #{PKG_VERSION}-#{PKG_RELEASE}'
+
+        data = nil unless req.request_body_permitted?
+        resp = @http.request(req, data)
+
+      end
+      REXML::Document.new(resp.body)
+    end
+
+    # Encodes the given string with the aws_secret_access_key, by taking the
+    # hmac-sha1 sum, and then base64 encoding it.  Optionally, it will also
+    # url encode the result of that to protect the string if it's going to
+    # be used as a query string parameter.
+    def encode(aws_secret_access_key, str, urlencode=true)
+      digest = OpenSSL::Digest::Digest.new('sha1')
+      b64_hmac = Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, str)).strip
+      if urlencode
+        return CGI::escape(b64_hmac)
+      else
+        return b64_hmac
+      end
+    end
+  end
+end