ec2_amitools 1.0.2

data/lib/ec2/amitools/bundle_base.rb

@@ -0,0 +1,58 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/tool_base'
+require 'ec2/amitools/bundleparameters'
+
+class BundleTool < AMITool
+
+  def user_override(name, value)
+    if interactive?
+      instr = interactive_prompt("Please specify a value for #{name} [#{value}]: ", name)
+      return instr.strip unless instr.nil? or instr.strip.empty?
+    end
+    value
+  end
+
+  def notify(msg)
+    $stdout.puts msg
+    if interactive?
+      print "Hit enter to continue anyway or Control-C to quit."
+      gets
+    end
+  end
+
+  def get_parameters(params_class)
+    params = super(params_class)
+    
+    if params.arch.nil?
+      params.arch = SysChecks::get_system_arch()
+      raise "missing or bad uname" if params.arch.nil?
+      params.arch = user_override("arch", params.arch)
+    end
+    
+    unless BundleParameters::SUPPORTED_ARCHITECTURES.include?(params.arch)
+      unless warn_confirm("Unsupported architecture [#{params.arch}].")
+        raise EC2StopExecution.new()
+      end
+    end
+    
+    tarcheck = SysChecks::good_tar_version?
+    raise "missing or bad tar" if tarcheck.nil?
+    unless tarcheck
+      unless warn_confirm("Possibly broken tar version found. Please use tar version 1.15 or later.")
+        raise EC2StopExecution.new()
+      end
+    end
+    
+    params
+  end
+
+end

data/lib/ec2/amitools/bundleimage.rb

@@ -0,0 +1,94 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/bundle'
+require 'ec2/amitools/bundleimageparameters'
+require 'ec2/amitools/bundle_base'
+
+MAX_SIZE = 10 * 1024 * 1024 * 1024 # 10 GB in bytes.
+BUNDLE_IMAGE_NAME = 'ec2-bundle-image'
+
+# The manual.
+BUNDLE_IMAGE_MANUAL=<<TEXT
+#{BUNDLE_IMAGE_NAME} is a command line tool that creates a bundled Amazon Machine \
+Image (AMI) from a specified loopback filesystem image.
+
+#{BUNDLE_IMAGE_NAME} will:
+- tar -S the AMI to preserve sparseness of the image file
+- gzip the result
+- encrypt it
+- split it into parts
+- generate a manifest file describing the bundled AMI
+
+#{BUNDLE_IMAGE_NAME} will bundle AMIs of up to 10GB.
+TEXT
+
+class ImageBundler < BundleTool
+
+  def bundle_image(p)
+    if p.size_checks
+      file_size = File.size(p.image_path)
+      if file_size <= 0
+        raise "the specified image #{p.image_path} is zero sized"
+      elsif file_size > MAX_SIZE
+        raise "the specified image #{p.image_path} is too large"
+      end
+    else
+      $stderr.puts 'Warning: disabling size-checks can result in unbootable image'
+    end
+    
+    optional_args = {
+      :kernel_id => p.kernel_id,
+      :ramdisk_id => p.ramdisk_id,
+      :product_codes => p.product_codes,
+      :ancestor_ami_ids => p.ancestor_ami_ids,
+      :block_device_mapping => p.block_device_mapping,
+    }
+    $stdout.puts 'Bundling image file...'
+    
+    Bundle.bundle_image(File::expand_path(p.image_path),
+                        p.user,
+                        p.arch,
+                        Bundle::ImageType::MACHINE,
+                        p.destination,
+                        p.user_pk_path,
+                        p.user_cert_path,
+                        p.ec2_cert_path,
+                        p.prefix,
+                        optional_args,
+                        @debug,
+                        false)
+    
+    $stdout.puts( "#{BUNDLE_IMAGE_NAME} complete." )
+  end
+
+  #------------------------------------------------------------------------------#
+  # Overrides
+  #------------------------------------------------------------------------------#
+
+  def get_manual()
+    BUNDLE_IMAGE_MANUAL
+  end
+
+  def get_name()
+    BUNDLE_IMAGE_NAME
+  end
+
+  def main(p)
+    bundle_image(p)
+  end
+
+end
+
+#------------------------------------------------------------------------------#
+# Script entry point. Execute only if this file is being executed.
+if __FILE__ == $0 || $0.match(/bin\/ec2-bundle-image/)
+  ImageBundler.new().run(BundleImageParameters)
+end

data/lib/ec2/amitools/bundleimageparameters.rb

@@ -0,0 +1,42 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/bundlemachineparameters'
+
+# The Bundle Image command line parameters.
+class BundleImageParameters < BundleMachineParameters
+
+  IMAGE_PATH_DESCRIPTION = "The path to the file system image to bundle."
+  PREFIX_DESCRIPTION = "The filename prefix for bundled AMI files. Defaults to image name."
+
+  attr_reader :image_path,
+              :prefix
+                
+  def mandatory_params()
+    super()
+    on('-i', '--image PATH', String, IMAGE_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--image')
+      @image_path = path
+    end
+  end
+
+  def optional_params()
+    super()
+    on('-p', '--prefix PREFIX', String, PREFIX_DESCRIPTION) do |prefix|
+      assert_good_key(prefix, '--prefix')
+      @prefix = prefix
+    end
+  end
+
+  def validate_params()
+    raise MissingMandatory.new('--image') unless @image_path
+    super()
+  end
+end

data/lib/ec2/amitools/bundlemachineparameters.rb

@@ -0,0 +1,60 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/bundleparameters'
+
+# The Bundle command line parameters.
+class BundleMachineParameters < BundleParameters
+
+  KERNEL_DESCRIPTION            = "Id of the default kernel to launch the AMI with."
+  RAMDISK_DESCRIPTION           = "Id of the default ramdisk to launch the AMI with."
+  ANCESTOR_AMI_IDS_DESCRIPTION  = "Lineage of this image. Comma separated list of AMI ids."
+  BDM_DESCRIPTION               = ['Default block-device-mapping scheme to launch the AMI with. This scheme',
+                                   'defines how block devices may be exposed to an EC2 instance of this AMI',
+                                   'if the instance-type of the instance is entitled to the specified device.',
+                                   'The scheme is a comma-separated list of key=value pairs, where each key',
+                                   'is a "virtual-name" and each value, the corresponding native device name',
+                                   'desired. Possible virtual-names are:',
+                                   ' - "ami": denotes the root file system device, as seen by the instance.',
+                                   ' - "root": denotes the root file system device, as seen by the kernel.',
+                                   ' - "swap": denotes the swap device, if present.',
+                                   ' - "ephemeralN": denotes Nth ephemeral store; N is a non-negative integer.',
+                                   'Note that the contents of the AMI form the root file system. Samples of',
+                                   'block-device-mappings are:',
+                                   ' - "ami=sda1,root=/dev/sda1,ephemeral0=sda2,swap=sda3"',
+                                   ' - "ami=0,root=/dev/dsk/c0d0s0,ephemeral0=1"'
+                                  ]
+
+  attr_accessor :kernel_id,
+                :ramdisk_id,
+                :ancestor_ami_ids,
+                :block_device_mapping
+
+  def optional_params()
+    super()
+    on( '--kernel ID', KERNEL_DESCRIPTION ) do |id|
+      @kernel_id = id
+    end
+    
+    on( '--ramdisk ID', RAMDISK_DESCRIPTION ) do |id|
+      @ramdisk_id = id
+    end
+
+    on( '-B', '--block-device-mapping MAPS', String, *BDM_DESCRIPTION ) do |bdm|
+      @block_device_mapping ||= {}
+      raise InvalidValue.new('--block-device-mapping', bdm) if bdm.to_s.empty?
+      bdm.split(',').each do |mapping|
+        raise InvalidValue.new('--block-device-mapping', bdm) unless mapping =~ /^\s*(\S)+\s*=\s*(\S)+\s*$/
+        virtual, device = mapping.split(/=/)
+        @block_device_mapping[virtual.strip] = device.strip
+      end
+    end
+  end
+end

data/lib/ec2/amitools/bundleparameters.rb

@@ -0,0 +1,120 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/parameters_base'
+require 'timeout'
+require 'ec2/platform/current'
+require 'ec2/amitools/syschecks'
+
+# The Bundle command line parameters.
+class BundleParameters < ParametersBase
+  include EC2::Platform::Current::Constants
+
+  SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
+
+  USER_DESCRIPTION = "The user's EC2 user ID (Note: AWS account number, NOT Access Key ID)."
+  HELP_DESCRIPTION = "Display this help message and exit."
+  MANUAL_DESCRIPTION = "Display the user manual and exit."
+  DESTINATION_DESCRIPTION = "The directory to create the bundle in. Defaults to '#{Bundling::DESTINATION}'."
+  DEBUG_DESCRIPTION = "Display debug messages."
+  EC2_CERT_PATH_DESCRIPTION = ['The path to the EC2 X509 public key certificate bundled into the AMI.',
+                               "Defaults to '#{Bundling::EC2_X509_CERT}'."]
+  ARCHITECTURE_DESCRIPTION = "Specify target architecture. One of #{SUPPORTED_ARCHITECTURES.inspect}"
+  BATCH_DESCRIPTION = "Run in batch mode. No interactive prompts."
+  PRODUCT_CODES_DESCRIPTION = ['Default product codes attached to the image at registration time.',
+                               'Comma separated list of product codes.']
+  SIZE_CHECKS_DESCRIPTION = 'If set, disables size checks on bundled artifacts.'
+  VERSION_DESCRIPTION = "Display the version and copyright notice and then exit."
+
+  attr_accessor :user_pk_path,
+                :user_cert_path,
+                :user,
+                :destination,
+                :ec2_cert_path,
+                :debug,
+                :show_help,
+                :manual,
+                :arch,
+                :batch_mode,
+                :size_checks,
+                :product_codes
+
+  PROMPT_TIMEOUT = 30
+
+  #----------------------------------------------------------------------------#
+
+  def mandatory_params()
+    on('-c', '--cert PATH', String, USER_CERT_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--cert')
+      @user_cert_path = path
+    end
+
+    on('-k', '--privatekey PATH', String, USER_PK_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--privatekey')
+      @user_pk_path = path
+    end
+
+    on('-u', '--user USER', String, USER_ACCOUNT_DESCRIPTION) do |user|
+      # Remove hyphens from the Account ID as presented in AWS portal.
+      @user = user.gsub("-", "")
+      # Validate the account ID looks correct (users often provide us with their akid or secret key)
+      unless (@user =~ /\d{12}/)
+        raise InvalidValue.new('--user', @user,
+                               "the user ID should consist of 12 digits (optionally hyphenated); this should not be your Access Key ID")
+      end
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def optional_params()
+    on('-d', '--destination PATH', String, DESTINATION_DESCRIPTION) do |path|
+      assert_directory_exists(path, '--destination')
+      @destination = path
+    end
+
+    on('--ec2cert PATH', String, *BundleParameters::EC2_CERT_PATH_DESCRIPTION) do |path|
+      assert_file_exists(path, '--ec2cert')
+      @ec2_cert_path = path
+    end
+
+    on('-r', '--arch ARCHITECTURE', String, ARCHITECTURE_DESCRIPTION) do |arch|
+      @arch = arch
+    end
+
+    on('--productcodes PRODUCT_CODES', String, *PRODUCT_CODES_DESCRIPTION) do |pc|
+      @product_codes = pc
+    end
+
+    on('--no-size-checks', SIZE_CHECKS_DESCRIPTION ) do |o|
+      @size_checks = o
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def validate_params()
+    unless @clone_only
+      raise MissingMandatory.new('--cert') unless @user_cert_path
+      raise MissingMandatory.new('--privatekey') unless @user_pk_path
+      raise MissingMandatory.new('--user') unless @user
+    end
+  end
+
+  #----------------------------------------------------------------------------#
+
+  def set_defaults()
+    @destination ||= Bundling::DESTINATION
+    @ec2_cert_path ||= Bundling::EC2_X509_CERT
+    @exclude ||= []
+    @size_checks = true
+  end
+
+end

data/lib/ec2/amitools/bundlevol.rb

@@ -0,0 +1,240 @@
+# Copyright 2008-2014 Amazon.com, Inc. or its affiliates.  All Rights
+# Reserved.  Licensed under the Amazon Software License (the
+# "License").  You may not use this file except in compliance with the
+# License. A copy of the License is located at
+# http://aws.amazon.com/asl or in the "license" file accompanying this
+# file.  This file is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
+# the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'ec2/amitools/bundle'
+require 'ec2/amitools/bundlevolparameters'
+require 'ec2/platform/current'
+require 'ec2/amitools/syschecks'
+require 'ec2/amitools/bundle_base'
+
+BUNDLE_VOL_NAME = 'ec2-bundle-vol'
+
+BUNDLE_VOL_MANUAL=<<TEXT
+#{BUNDLE_VOL_NAME} is a command line tool that creates an Amazon Machine Image (AMI)
+from an existing Fedora Core installation and bundles it. Its output is a
+bundled AMI consisting of AMI parts and a manifest. Use the '--help'
+option to display help on Bundle Volume parameters.
+
+#{BUNDLE_VOL_NAME} can be run from within an installation, provided there is sufficient space
+for the AMI in the destination directory. Note that running Bundle Volume from
+within a running installation may be problematic because partially written files
+may get copied into the AMI. To minimize this risk it is recommended that the
+system is brought down to runlevel 1. Note that this will stop networking, so
+only do this if you have access to the console.
+
+If #{BUNDLE_VOL_NAME} is not run from within the installation, the volume belonging to the
+installation must be mounted so that it is accessible to #{BUNDLE_VOL_NAME}.
+
+#{BUNDLE_VOL_NAME} will:
+- create a sparse filesystem image
+- recursively copy the specified volume into the image
+- tar -S the image to preserve the sparseness of the image file
+- compress the image
+- encrypt it
+- split it into parts
+- generate a manifest file describing the bundled AMI
+
+Recursive Copying
+
+The recursive copying process copies directories from the volume into the image.
+The special directories:
+
+- '/dev'
+- '/media'
+- '/mnt'
+- '/proc'
+- '/sys'
+
+are always excluded.
+
+Files matching the following regular expressions:
+
+- '*/#*#'
+- '*/.#*'
+- '*.sw'
+- '*.swo'
+- '*.swp'
+- '*~'
+- '*.pem'
+- '*.priv'
+- '*id_rsa*'
+- '*id_dsa*'
+- '*.gpg'
+- '*.jks'
+- '*/.ssh/authorized_keys'
+- '*/.bash_history'
+
+are excluded by default on Linux. Individual files that match these patterns may be
+included by specifying the '--include' option.
+
+Local directories, which are copied by default, are defined to be those on
+filesystems of the following types:
+
+- ext2
+- ext3
+- xfs
+- jfs
+- reiserfs
+
+Directories on filesystems that are not of one of the types listed above, such as
+remotely mounted NFS filesystems, are excluded by default, but can be copied
+by using the '--all' option.
+
+Symbolic links are preserved by the copying process, provided the link target is
+copied.
+
+Mounted File Systems
+
+#{BUNDLE_VOL_NAME} will default to bundling the existing /etc/fstab file.
+
+#{BUNDLE_VOL_NAME} will create and bundle AMIs of up to 10GB.
+
+Note:
+  Creating the filesystem on the image may fail in the presence of selinux.
+  If you are using selinux, you should disable it before using #{BUNDLE_VOL_NAME}.
+TEXT
+
+MAX_SIZE_MB = 10 * 1024  # 10 GB in MB
+
+class VolBundler < BundleTool
+
+  def bundle_vol(p)
+    name = p.prefix
+    image_file = File::join( p.destination, name )
+    volume = File::join( p.volume, "" ) # Add a trailing "/" if not present.
+
+    #
+    # We can't bundle unless we're root.
+    #
+    raise "You need to be root to run #{$0}" unless SysChecks::root_user?
+
+    #
+    # Extra parameter verification.
+    #
+    raise "the specified size #{p.size}MB is too large" unless p.size <= MAX_SIZE_MB
+    raise "the specified image file #{image_file} already exists" if File::exist?( image_file )
+
+    #
+    # Create a list of files to be included in the image. This list will override any
+    # files that are excluded by the security filtered files list. Files are only added if
+    # they are under the volume root.
+    #
+    includes = []
+    p.includes.each do |file|
+      includes << file
+    end
+    #
+    # Create list of directories to exclude from the image. Always exclude special
+    # directories, directories specified by the user and the image file itself.
+    #
+    exclude = []
+    unless p.all
+      #
+      # Exclude mounted non-local filesystems if they are under the volume root.
+      #
+      EC2::Platform::Current::Mtab.load.entries.values.each do |entry|
+        unless EC2::Platform::Current::LOCAL_FS_TYPES.include? entry.fstype
+          exclude << entry.mpoint if entry.mpoint.index(volume) == 0
+        end
+      end
+    end
+    EC2::Platform::Current::Image::EXCLUDES.each { |dir| exclude << dir }
+
+    #
+    # Exclude user specified excluded directories if they are under the volume root.
+    #
+    p.exclude.each do |dir|
+      exclude << dir
+    end
+
+    #
+    # Exclude the image file if it is under the volume root.
+    #
+    if image_file.index( volume ) == 0
+      exclude << image_file
+    end
+
+    # If we are inheriting instance data but can't access it we want to fail early
+    if p.inherit && !EC2::InstanceData.new.instance_data_accessible
+      raise EC2FatalError.new(12, "Can't access instance metadata. If you are not bundling on an EC2 instance use --no-inherit.")
+    end
+
+    #
+    # Create image from volume.
+    #
+    image = EC2::Platform::Current::Image.new(volume,
+                                              image_file,
+                                              p.size,
+                                              exclude,
+                                              includes,
+                                              p.filter,
+                                              p.fstab,
+                                              p.part_type,
+                                              p.arch,
+                                              p.script,
+                                              @debug,
+                                              p.grub_config)
+    image.make
+
+    $stdout.puts 'Image file created: %s' % [image_file]
+    $stdout.puts 'Volume cloning done.'
+    if p.clone_only
+      $stdout.puts 'Not bundling image file since "--clone-only" flag was specified.'
+    else
+      #
+      # Bundle the created image file.
+      #
+      $stdout.puts 'Bundling image file...'
+      optional_args = {
+        :kernel_id => p.kernel_id,
+        :ramdisk_id => p.ramdisk_id,
+        :product_codes => p.product_codes,
+        :ancestor_ami_ids => p.ancestor_ami_ids,
+        :block_device_mapping => p.block_device_mapping
+      }
+      Bundle.bundle_image(image_file,
+                          p.user,
+                          p.arch,
+                          Bundle::ImageType::VOLUME,
+                          p.destination,
+                          p.user_pk_path,
+                          p.user_cert_path,
+                          p.ec2_cert_path,
+                          nil, # prefix
+                          optional_args,
+                          @debug,
+                          p.inherit)
+    end
+    $stdout.puts("#{BUNDLE_VOL_NAME} complete.")
+  end
+
+  #------------------------------------------------------------------------------#
+  # Overrides
+  #------------------------------------------------------------------------------#
+
+  def get_manual()
+    BUNDLE_VOL_MANUAL
+  end
+
+  def get_name()
+    BUNDLE_VOL_NAME
+  end
+
+  def main(p)
+    bundle_vol(p)
+  end
+
+end
+
+#------------------------------------------------------------------------------#
+# Script entry point. Execute only if this file is being executed.
+if __FILE__ == $0 || $0.match(/bin\/ec2-bundle-vol/)
+  VolBundler.new().run(BundleVolParameters)
+end