image_optimize 1.0.0 → 1.0.1

data/lib/image_bundle.rb

@@ -0,0 +1,19 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'image_bundle/image_bundle_base'
+require 'image_bundle/image_bundle_ec2_ebs'
+require 'image_bundle/image_bundle_ec2_s3'

data/lib/image_bundle/image_bundle_base.rb

@@ -0,0 +1,116 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'mixins/common'
+require 'mixins/command'
+
+# Base class for image utils
+#
+module ImageOptimize
+
+  class ImageBundleBase
+
+    include ImageOptimize::Common
+    include ImageOptimize::Command
+
+    def initialize(instance_api_client, full_api_client)
+
+      @dry_run = false
+
+      @log ||= Logger.new(STDOUT)
+      @instance_client = instance_api_client
+      unless @instance_client
+        @log.error"ERROR: you must pass an instance_api_client parameter."
+      end
+
+      @api_client = full_api_client
+      unless @api_client
+        @log.error"ERROR: you must pass a full_api_client parameter."
+      end
+    end
+
+    # Captures a "snapshot" of the current VM configuration
+    def snapshot_instance(name, description)
+      not_implemented
+    end
+
+    # Turn the VM "snapshot" into an launchable image
+    def register_image(name, description)
+      not_implemented
+    end
+
+    def add_image_to_next_instance
+      # get next instance
+      @log.info "Lookup next_instance for the server."
+      next_instance = get_instance_parent.show.next_instance
+
+      unless @dry_run
+        raise "ERROR: image cannot be nil. Be sure to run register_image method first." unless @image
+        # set next instance to use our cached image
+        @log.info "Update next instance to launch with our new image."
+        next_instance.update(:instance => {:image_href => @image.show.href})
+      end
+    end
+
+    def log(logger)
+      @log = logger
+    end
+
+    protected
+
+    # Get the resources associated with this instance
+    def instance
+      @instance ||= @instance_client.get_instance
+    end
+
+    def get_id(resource)
+      # the id is the last part of thr href path
+      resource.href.split('/').last
+    end
+
+    def fail(message="unspecified error occured", exception=nil)
+      if exception
+        @log.debug "Exception: #{exception.message}"
+        @log.debug exception.backtrace.inspect
+      end
+      @log.error "FATAL: #{message}"
+      exit 1
+    end
+
+    # Get the parent for this instance
+    #
+    # The parent may be a Server or ServerArray reosurce.
+    #
+    # XXX: we can't get to the server resource from the instance facing API
+    #      the server resource holds the reference to the next instance.
+    #      To deal with this we create an instance resource using the user
+    #      authenticated @api_client, then grab it's "parent".
+    #
+    # == Returns:
+    # @return [RightApi::Resource] server a server API resource
+    def get_instance_parent
+      @log.info "Lookup RightScale server for this instance."
+      api_cloud = @api_client.clouds(:id => get_id(instance.cloud))
+      api_instance = api_cloud.show.instances.index(:id => get_id(instance))
+      api_instance.show.parent
+    end
+
+    def debug_mode?
+      @log.level == Logger::DEBUG
+    end
+
+  end
+
+end

data/lib/image_bundle/image_bundle_ec2.rb

@@ -0,0 +1,131 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module ImageOptimize
+
+  # Base class for snapshoting and registration of EC2 images
+  #
+  class ImageBundleEc2Base < ImageBundleBase
+
+    RETRY_TIMEOUT_SEC = 2 * 60 * 60 # 2 hr
+
+    def initialize(instance_api_client, full_api_client, aws_access_key, aws_secret_key, kernel_id_override = nil)
+      load_meta_data
+      @aws_access_key = aws_access_key
+      @aws_secret_key = aws_secret_key
+      @kernel_override = kernel_id_override
+      super(instance_api_client, full_api_client)
+    end
+
+    def register_image(name=nil, description=nil)
+      cmd = register_command(name, description)
+      @log.info "Running register image command..."
+      unless @dry_run
+        status, cmd_output = execute(cmd)
+        raise "FATAL: unable to register new image" unless status.exitstatus == 0
+        image_uuid = parse_ami(cmd_output)
+        @log.info "New image: #{image_uuid}"
+
+        # wait for image to be available
+        wait_for_image(image_uuid)
+      end
+    end
+
+    protected
+
+    def parse_ami(command_output)
+      /(ami-[a-z0-9]+)/.match(command_output).captures[0]
+    end
+
+    def register_command(name, description)
+      raise "Not implemented"
+    end
+
+    def wait_for_image(image_uuid)
+      # wait for image to be available
+      @log.info "Query for the new image"
+      start = Time.now
+      @image = get_image(image_uuid)
+      delay_sec = 10
+      Timeout::timeout(RETRY_TIMEOUT_SEC) do
+        while @image == nil do
+          @log.info "  Image #{image_uuid} not yet available. Checking again in #{delay_sec} seconds..."
+          sleep delay_sec
+          @image = get_image(image_uuid)
+        end
+      end
+      total = Time.new - start
+      @log.info "Total time waited: #{total}"
+    end
+
+    def get_image(uuid)
+      # use full api client to look for image
+      image = get_cloud.show.images.index(:filter => [ "resource_uid==#{uuid}" ]).first
+      @log.info "Found image #{uuid} named '#{image.name}'" if image
+      image
+    end
+
+    # Get the cloud for this instance
+    #
+    # XXX: we cannot query cloud image via instance API
+    #
+    # == Returns:
+    # @return [RightApi::Resource] server a server API resource
+    def get_cloud
+      cloud_href = instance.cloud.href
+      cloud_id = cloud_href.split('/').last
+      cloud = @api_client.clouds(:id => cloud_id)
+    end
+
+    # detect EC2 region from EC2 metadata
+    #
+    def region
+      ec2_zone = ENV['EC2_PLACEMENT_AVAILABILITY_ZONE'] # ec2_zone = `curl http://169.254.169.254/latest/meta-data/placement/availability-zone/`
+      unless ec2_zone
+        fail("The EC2_PLACEMENT_AVAILABILITY_ZONE environment variable is not defined. Did you load EC2 metadata?")
+      end
+      region =  /(.*\-.*\-[1-9]*)/.match(ec2_zone).captures.first
+      @log.info "detected current region is #{region}"
+      region
+    end
+
+    # detect kernel from EC2 metadata
+    #
+    def kernel_aki
+      kernel = @kernel_override         # allow user to override
+      kernel ||= ENV['EC2_KERNEL_ID']   # use same kernel as VM is running
+    end
+
+    private
+
+    def load_meta_data
+      begin
+        require '/var/spool/cloud/meta-data-cache.rb'
+      rescue LoadError => e
+        puts "FATAL: not a RightScale managed VM, or you already cleaned up your meta-data file."
+        exit 1
+      end
+    end
+
+    def common_options
+      options = ""
+      options << "--debug " if debug_mode?
+    end
+
+
+  end
+
+end

data/lib/image_bundle/image_bundle_ec2_ebs.rb

@@ -0,0 +1,77 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'image_bundle/image_bundle_ec2'
+
+module ImageOptimize
+
+  # Handles the snapshoting and registration of EBS based images
+  #
+  class ImageBundleEc2EBS < ImageBundleEc2Base
+
+    def snapshot_instance(name=nil, description=nil)
+
+      # find root volume
+      #
+      root_volume = nil
+      begin
+        device_name = "/dev/sda"
+        @log.info "Locating root volume attached to #{device_name}..."
+        attachment = instance.volume_attachments.index.select { |va| va.device =~ /#{device_name}/ }.first
+        root_volume = attachment.volume
+        @attachment_device = attachment.device
+        @log.info "Found volume #{root_volume.show.name} attached at #{@attachment_device}"
+      rescue Exception => e
+        fail("FATAL: cannot find root volume. Check device_name which can vary depending on hypervisor and/or kernel. Also instance-store images not currently supported.", e)
+      end
+
+      # create snapshot
+      @log.info "Creating snapshot of server."
+      options = { :parent_volume_href => root_volume.href }
+      options.merge!(:name => name) if name
+      options.merge!(:description => description) if description
+      snapshot = @instance_client.volume_snapshots.create(:volume_snapshot => options)
+      @log.info "Snapshot name '#{name}'" if name
+
+      # wait for snapshot to complete
+      @log.info "Waiting for snapshot to become available"
+      current_state = snapshot.show.state
+      delay_sec = 10
+      Timeout::timeout(RETRY_TIMEOUT_SEC) do
+        while current_state != "available" do
+          @log.info "  snapshot state: #{current_state}. try again in #{delay_sec} seconds..."
+          sleep delay_sec
+          current_state = snapshot.show.state
+        end
+      end
+      @log.info "Snapshot is now available"
+      @snapshot_id = snapshot.show.resource_uid
+    end
+
+    def register_command(name=nil, description=nil)
+      unless @snapshot_id
+        fail("@snapshot_id cannot be nil. Be sure to run snapshot_instance first.")
+      end
+
+      # use ec2 tools to register snapshot as an image
+      # TODO: this command only maps in 4 ephemeral devices to the new image. Use metadata to get actual count.
+      cmd = "ec2-register --region #{region} --snapshot #{@snapshot_id} --description '#{@image_description}' --block-device-mapping '/dev/sdb=ephemeral0' --block-device-mapping '/dev/sdc=ephemeral1' --block-device-mapping '/dev/sdd=ephemeral2' --block-device-mapping '/dev/sde=ephemeral3' --kernel #{kernel_aki} --root-device-name #{@attachment_device} --architecture x86_64 --name '#{name}' #{common_options}"
+      cmd
+    end
+
+  end
+
+end

data/lib/image_bundle/image_bundle_ec2_s3.rb

@@ -0,0 +1,114 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'image_bundle/image_bundle_ec2'
+
+module ImageOptimize
+
+  # Handles the snapshoting and registration of EBS based images
+  #
+  class ImageBundleEc2S3 < ImageBundleEc2Base
+
+    BUNDLE_DIR = "/mnt/ephemeral/bundle"
+
+    # x509_key_file : [String] Absolute path to file holding AWS x.509 private key file
+    # x509_cert_file : [String] Absolute path to file holding AWS x.509 certificate file
+    # s3_bucket : [String] place to store image in S3
+    #
+    def initialize(instance_api_client, full_api_client, aws_access_key, aws_secret_key, aws_account_number, x509_key_file, x509_cert_file,  s3_bucket, bundle_dir=nil, no_filter=nil, kernel_id = nil)
+      super(instance_api_client, full_api_client, aws_access_key, aws_secret_key, kernel_id)
+      @aws_account_number = aws_account_number
+      raise "no aws account number specified." unless @aws_account_number
+      @key_file = x509_key_file
+      raise "no path to x.509 private key file specified." unless @key_file
+      @cert_file = x509_cert_file
+      raise "no path to x.509 certificate file specified." unless @cert_file
+
+      @bucket = s3_bucket
+
+      # Where to place temporary image bundle before upload
+      @bundle_dir = bundle_dir
+      @bundle_dir ||= BUNDLE_DIR
+
+      # --no-filter
+      @no_filter = true if no_filter
+    end
+
+    # Captures a "snapshot" of the current VM configuration
+    def snapshot_instance(name, description)
+      bundle_image(name)
+      upload_bundle(name)
+      cleanup_bundle
+    end
+
+    # Turn the VM "snapshot" into an launchable image
+    def register_command(name, description)
+      #ec2-register my-s3-bucket/image_bundles/name/image.manifest.xml -n name -O your_access_key_id -W your_secret_access_key
+      cmd = "ec2-register #{@bucket}/image_bundles/#{name}/image.manifest.xml -n #{name} -O #{@aws_access_key} -W #{@aws_secret_key} --region #{region} --architecture x86_64 --description '#{description}' #{common_options}"
+      cmd
+    end
+
+    private
+
+    def upload_bundle(name)
+       @log.info "Running bundle upload command..."
+       cmd = "ec2-upload-bundle -b #{@bucket}/image_bundles/#{name} -m '#{@bundle_dir}/image.manifest.xml' -a #{@aws_access_key} -s #{@aws_secret_key} --retry --batch --region #{region} #{common_options}"
+       unless @dry_run
+         status, cmd_output = execute(cmd)
+         fail "FATAL: unable to upload S3 image bundle" unless status.exitstatus == 0
+       end
+    end
+
+    def bundle_image(name)
+      @log.info "Using #{@bundle_dir} locally to store image bundle."
+      @log.info "Running EC2 Bundle command..."
+      FileUtils.rm_rf(@bundle_dir)
+      FileUtils.mkdir_p(@bundle_dir)
+      cmd="ec2-bundle-vol --privatekey #{@key_file} --cert #{@cert_file} --user #{@aws_account_number} --destination #{@bundle_dir} --arch x86_64 --kernel #{kernel_aki} -B 'ami=sda,root=/dev/sda,ephemeral0=sdb,swap=sda3' #{excludes} --no-inherit #{common_options}"
+      cmd << " --no-filter" if @no_filter
+      unless @dry_run
+        status, cmd_output = execute(cmd)
+        fail "FATAL: unable to bundle new image" unless status.exitstatus == 0
+      end
+    end
+
+    # What directories should be excluded from bundle
+    # NOTE: be sure to exclude the directory containing the key_file and cert_file x.509 creds
+    #
+    def excludes
+      excludes = ""
+      [ File.dirname(@key_file), File.dirname(@cert_file), "/mnt", @bundle_dir ].uniq.each do |dir|
+        excludes += "--exclude #{dir} "
+      end
+      excludes
+    end
+
+    # remove bundle directory
+    #
+    def cleanup_bundle
+      FileUtils.rm_rf(@bundle_dir)
+    end
+
+    def s3_url(region)
+      if region == "us-east-1"
+        "https://s3.amazonaws.com"
+      else
+        "https://s3-#{region}.amazonaws.com"
+      end
+    end
+
+  end
+end