image_optimize 1.0.0 → 1.0.1

data/lib/mixins/command.rb

@@ -0,0 +1,40 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module ImageOptimize
+  module Command
+
+    # Returns Process::Status and output
+    def execute(command, quiet=false)
+      debug "COMMAND: #{command}"
+      output = ""
+      IO.popen("#{command} 2>&1") do |data|
+        while line = data.gets
+          puts line if !quiet
+          output << line
+        end
+      end
+      process_status = $?
+      debug "OUTPUT: #{output}"
+      return process_status, output
+    end
+
+    def debug(message)
+      @log.debug message if @log.respond_to?(:debug)
+    end
+
+  end
+end

data/lib/mixins/common.rb

@@ -0,0 +1,23 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module ImageOptimize
+  module Common
+    def not_implemented
+      raise "not implemented"
+    end
+  end
+end

data/rightscale/rightscript.sh

@@ -0,0 +1,189 @@
+#!/bin/bash -ex
+#
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# $API_USER_EMAIL - RightScale Dashboard User email
+# $API_USER_PASSWORD - Rightscale Dashboard Password
+#
+# $AWS_ACCESS_KEY - EC2 Account Access Key
+# $AWS_SECRET_KEY - EC2 Account Secret
+# $AWS_ACCOUNT_NUMBER - EC2 Account ID
+#
+# $CACHED_IMAGE_PREFIX - Prefix to generated images name to help search for cached images
+# $CACHED_IMAGE_DESCRIPTION - Description to add to cached images
+#
+# $KERNEL_ID_OVERRIDE - Kernel to use instead of what the VM is running. i.e. "aki-fc8f11cc"
+#                           For a current list of IDs see http://goo.gl/dOS0mB
+#
+# $AWS_S3_BUNDLE_DIRECTORY - The local directory where the image bundle will be stored before uploading to S3.
+#                            NOTE: this must have enough free space to hold the image bundle.
+#                            Default: "/mnt/ephemeral/bundle"
+#
+# $AWS_S3_BUNDLE_NO_FILTER - If set, will disable the default filtering used by the ec2-bundle-vol command.
+#                            WARNING: setting this option could leave ssh keys or other secrets on your image.
+#
+
+# source the profile
+source /etc/profile
+
+# update CA certs
+# fixes intermediate "SSL certificate problem" errors from curl
+# See https://forums.aws.amazon.com/thread.jspa?messageID=341463&#341463
+#
+update-ca-certificates
+
+
+# install right_api_client gem (if not installed) into RightLink Sandbox
+#
+sandbox_bin=/opt/rightscale/sandbox/bin
+gem_bin=$sandbox_bin/gem
+if ! $gem_bin list right_api_client --installed ; then
+  echo "Installing right_api_client gem..."
+  $gem_bin install right_api_client --no-rdoc --no-ri
+fi
+
+if ! $gem_bin list trollop --installed ; then
+  echo "Installing trollop gem..."
+  $gem_bin install trollop --no-rdoc --no-ri
+fi
+
+# install EC2 tools
+#
+tools_dir=/tmp/ec2tools
+if [ ! -d $tools_dir ]; then
+  mkdir -p $tools_dir
+
+  # install EC2 API tools
+  #
+  if [ ! -d $tools_dir/ec2-api-tools-1.6.13.0 ]; then
+    unzip $ATTACH_DIR/ec2-api-tools-1.6.13.0.zip -d $tools_dir
+  fi
+
+  # install EC2 AMI tools
+  #
+  if [ ! -d $tools_dir/ec2-ami-tools-1.5.0.0 ]; then
+    unzip $ATTACH_DIR/ec2-ami-tools-1.5.0.0.zip -d $tools_dir
+  fi
+
+  # install ec2 tools package dependencies
+  apt-get -y install grub kpartx gdisk
+fi
+
+# setup ec2 tools env
+#
+export EC2_HOME=$tools_dir/ec2-api-tools-1.6.13.0
+export PATH=$tools_dir/ec2-api-tools-1.6.13.0/bin:${PATH}
+export EC2_AMITOOL_HOME=$tools_dir/ec2-ami-tools-1.5.0.0
+export PATH=$tools_dir/ec2-ami-tools-1.5.0.0/bin:${PATH}
+
+# unpack attached tarred gem
+#
+if [ -d "$ATTACH_DIR" ]; then
+  echo "Unpacking attached tarred gem..."
+  cd $ATTACH_DIR
+  tar xvf image_optimize*.tar
+  chmod +x bin/image_optimize
+fi
+
+# setup S3 bundle creds (unless EBS)
+#
+opts="--aws-image-type $IMAGE_TYPE"
+if [ "$IMAGE_TYPE" != "EBS" ]; then
+  echo "Setting up S3 bundle creds..."
+  export certs_dir=/tmp/certs
+
+  # set default bucket name for cached images (must be url safe)
+  # (TODO: pass this as command-line options)
+  if [ -n "$AWS_S3_IMAGE_BUCKET" ]; then
+     opts=$opts" --aws-s3-image-bucket $AWS_S3_IMAGE_BUCKET"
+  fi
+
+  # set required S3 environment variables
+  key_path=$certs_dir/x509.key
+  cert_path=$certs_dir/x509.cert
+
+  opts="$opts --aws-s3-key-path $key_path --aws-s3-cert-path $cert_path"
+
+  # write x.509 creds to disk
+  mkdir -p $certs_dir
+  set +x # don't echo these commands!
+  cat <<EOF>$key_path
+$AWS_X509_KEY
+EOF
+  cat <<EOF>$cert_path
+$AWS_X509_CERT
+EOF
+
+  # The local directory where the image bundle will be stored before uploading to S3.
+  # NOTE: this must have enough free space to hold the image bundle.
+  # Default: "/mnt/ephemeral/bundle"
+  if [ -n "$AWS_S3_BUNDLE_DIRECTORY" ]; then
+    opts=$opts" --aws-s3-bundle-directory $AWS_S3_BUNDLE_DIRECTORY"
+  fi
+
+  # If set, will disable the default filtering used by the ec2-bundle-vol command.
+  # WARNING: setting this option could leave ssh keys or other secrets on your image.
+  if [ -n "$AWS_S3_BUNDLE_NO_FILTER" ]; then
+    opts=$opts" --aws-s3-bundle-no-filter"
+  fi
+
+fi
+
+# enable debugging (if specified)
+# WARNING: will write ec2 creds to log if set!!
+if [ -n "$IMAGE_OPTIMIZE_DEBUG" ]; then
+  opts=$opts" --verbose"
+fi
+
+# If set, don't clean up some state files.  This will allow the script to be
+# rerun on the same instance multiple times which is useful for debugging.
+# NOTE: Boot issues may occur if state is not properly cleaned up.
+if [ -n "$DISABLE_CLEANUP" ]; then
+  opts=$opts" --no-cleanup"
+fi
+
+# Prefix to generated images name to help search for cached images
+if [ -n "$CACHED_IMAGE_PREFIX" ]; then
+  opts=$opts" --prefix $CACHED_IMAGE_PREFIX"
+fi
+
+# Description to add to cached images
+if [ -n "$CACHED_IMAGE_DESCRIPTION" ]; then
+  opts=$opts" --description $CACHED_IMAGE_DESCRIPTION"
+fi
+
+# Kernel to use instead of what the VM is running. i.e. "aki-fc8f11cc"
+# For a current list of IDs see http://goo.gl/dOS0mB
+if [ -n "$KERNEL_ID_OVERRIDE" ]; then
+  opts=$opts" --aws-kernel-id $KERNEL_ID_OVERRIDE"
+fi
+
+# run optimizer
+#
+echo "Running image_optimize utility..."
+cmd="$sandbox_bin/ruby bin/image_optimize $opts"
+echo "COMMAND: $cmd"
+eval $cmd
+
+# clean up creds after S3 bundling
+#
+if [ "$IMAGE_TYPE" !=  "EBS" ]; then
+  echo "clean up creds for S3 bundling"
+  rm -rf $certs_dir
+fi
+
+exit 0

data/setup.sh

@@ -0,0 +1,40 @@
+#!/bin/bash -e x
+
+apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
+
+# This only works for Ubuntu rightnow
+# Anyone want to write a Chef recipe?
+apt-get update
+
+# install ec2 tools package dependencies
+apt-get -y install grub kpartx gdisk
+
+# update CA certs
+# fixes intermediate "SSL certificate problem" errors from curl
+# See https://forums.aws.amazon.com/thread.jspa?messageID=341463&#341463
+#
+update-ca-certificates
+
+# setup ruby env
+apt-get -y install ruby1.9.1
+apt-get -y install ruby1.9.1-dev
+update-alternatives --set ruby /usr/bin/ruby1.9.1
+update-alternatives --set gem /usr/bin/gem1.9.1
+gem install bundler --no-rdoc --no-ri
+bundle
+
+export TOOLS_DIR=/tmp/ec2tools
+mkdir -p $TOOLS_DIR
+
+# install EC2 tools v1.5 or greater
+wget http://s3.amazonaws.com/ec2-downloads/ec2-api-tools-1.6.13.0.zip
+unzip ec2-api-tools-1.6.13.0.zip -d $TOOLS_DIR
+wget http://s3.amazonaws.com/ec2-downloads/ec2-ami-tools-1.5.0.0.zip
+unzip ec2-ami-tools-1.5.0.0.zip -d $TOOLS_DIR
+
+# setup ec2 tools env
+#
+export EC2_HOME=$TOOLS_DIR/ec2-api-tools-1.6.13.0
+export PATH=$TOOLS_DIR/ec2-api-tools-1.6.13.0/bin:${PATH}
+export EC2_AMITOOL_HOME=$TOOLS_DIR/ec2-ami-tools-1.5.0.0
+export PATH=$TOOLS_DIR/ec2-ami-tools-1.5.0.0/bin:${PATH}

data/spec/api_client_spec.rb

@@ -0,0 +1,39 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'api_client_15'))
+
+describe "API 1.5 Client Creator" do
+
+  before(:each) do
+    RightScale::ApiClient15.any_instance.stub(:parse_user_data)
+    apiStub = double("RightApi::Client", :log => true)
+    RightApi::Client.should_receive(:new).and_return(apiStub)
+  end
+
+  it "creates an instance facing api connection" do
+    helper = RightScale::ApiClient15.new
+    helper.get_client(:instance)
+  end
+
+  it "creates an user api connection" do
+    ENV['API_USER_EMAIL'] = "somebody"
+    ENV['API_USER_PASSWORD'] = "somepassword"
+    helper = RightScale::ApiClient15.new
+    helper.get_client(:user)
+  end
+
+end

data/spec/command_spec.rb

@@ -0,0 +1,43 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'mixins', 'command'))
+
+describe ImageOptimize::Command do
+
+  class TestObj
+    include ImageOptimize::Command
+    def initialize
+      @log = Logger.new(STDOUT)
+    end
+  end
+
+  before(:each) do
+    Logger.any_instance.stub(:debug)
+  end
+
+  it "runs a child process" do
+    return_val, output = TestObj.new.execute("ls -l", true)
+    return_val.exitstatus.should == 0
+  end
+
+  it "logs debug message" do
+    Logger.any_instance.stub(:debug)
+    obj = TestObj.new
+    obj.debug("hey now")
+  end
+
+end

data/spec/image_bundle_ec2_spec.rb

@@ -0,0 +1,87 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'image_bundle'))
+
+describe ImageOptimize::ImageBundleEc2Base do
+
+  before(:each) do
+    # dont log stuff
+    Logger.any_instance.stub(:info)
+    Logger.any_instance.stub(:debug)
+
+    ImageOptimize::ImageBundleEc2Base.any_instance.stub(:load_meta_data)
+    @image_bundler = ImageOptimize::ImageBundleEc2Base.new(
+      "instance_client",
+      "api_client",
+      "aws_access_key",
+      "aws_secret_key"
+    )
+  end
+
+  # skip this test on a cloud instance -- since there is a metadata file
+  it "fails if metadata not found" do
+    unless File.exists?('/var/spool/cloud/meta-data-cache.rb')
+      lambda{ImageOptimize::ImageBundleEc2Base.new("instance_client", "api_client")}.should raise_error
+    end
+  end
+
+  it "registers image" do
+    @image_bundler.should_receive(:register_command)
+    status = double("ProcessStatus", :exitstatus => 0)
+    @image_bundler.should_receive(:execute).and_return([status, "Using AWS acces key \n IMAGE ami-64b5c054 "])
+    @image_bundler.should_receive(:wait_for_image)
+    @image_bundler.register_image("name", "description")
+  end
+
+  it "parsed ami" do
+    ami = "ami-64b5c054"
+    fake_output = "Using AWS acces key \n IMAGE #{ami} "
+    @image_bundler.send(:parse_ami, fake_output).should == ami
+  end
+
+  it "raise excaption on failure" do
+    @image_bundler.should_receive(:register_command)
+    status = double("ProcessStatus", :exitstatus => 1)
+    lambda{@image_bundler.register_image("name", "description")}.should raise_error
+  end
+
+  it "returns true if using a debug log level" do
+    @image_bundler.send(:debug_mode?).should == true
+  end
+
+  it "returns false if not using a debug log level" do
+    log = Logger.new(STDOUT)
+    log.level = Logger::INFO
+    @image_bundler.log(log)
+    @image_bundler.send(:debug_mode?).should == false
+  end
+
+   # def register_image(name=nil, description=nil)
+   #    cmd = register_command(name, description)
+   #    @log.info "Running register image command..."
+   #    unless @dry_run
+   #      status, cmd_output = execute(cmd)
+   #      raise "FATAL: unable to register new image" unless status.exitstatus == 0
+   #      image_uuid = cmd_output.sub("IMAGE","").strip
+   #      @log.info "New image: #{image_uuid}"
+   #
+   #      # wait for image to be available
+   #      wait_for_image(image_uuid)
+   #    end
+   #  end
+
+end