image_optimize 1.0.0 → 1.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+creds
+.secrets
+pkg
+textmate.tmproj

data/.rspec

@@ -0,0 +1,2 @@
+--color
+-fs

data/.ruby-version

@@ -0,0 +1 @@
+1.9.3-p448

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+CHANGELOG
+=========
+
+1.0.1 - initial release for Ubuntu 12.04 on EC2

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem 'right_api_provision'
+end

data/Gemfile.lock

@@ -0,0 +1,48 @@
+PATH
+  remote: .
+  specs:
+    image_optimize (1.0.1)
+      mime-types (= 2.0)
+      right_api_client (= 1.5.15)
+      trollop (= 2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    json (1.8.1)
+    method_source (0.8.2)
+    mime-types (2.0)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rake (10.3.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    right_api_client (1.5.15)
+      json
+      rest-client (~> 1.6)
+    right_api_provision (0.2.1)
+      right_api_client (= 1.5.15)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.8)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.6)
+    slop (3.5.0)
+    trollop (2.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  image_optimize!
+  pry
+  rake
+  right_api_provision
+  rspec (~> 2.14)

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2014 RightScale, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,128 @@
+Image Optimizer
+===============
+
+Bundle a running VM into a new image for use on next launch.
+
+Useful for drastically reducing boot times in your auto-scaling tier.
+
+Basic Workflow
+--------------
+1. Launch a EC2 instance from a base OS image.
+2. Converge system into whatever you want it to be (i.e. application server)
+3. Prepare the system for snapshotting. (see [clean_for_rebundle](https://github.com/caryp/clean_for_rebundle) project)
+3. Install and run image_optimize
+4. Next launch will be optimized!!
+
+Requirements
+------------
+ * EC2 Cloud only
+ * VMs must be launched via the [RightScale Platform](www.rightscale.com)
+
+Usage
+-----
+
+    image_optimize [options]
+
+    where [options] are:
+
+                      --verbose, -v:   If set will enable debug logging.  WARNING: will write ec2 creds to log if set!!
+                   --prefix, -r <s>:   Prefix to add to the optimized image name. Helps when searching for your optimized images. (Default: optimized-image)
+              --description, -d <s>:   Description to add to optimized images (default: Cached image)
+                 --api-user, -u <s>:   RightScale Dashboard User email. Not needed if API_USER_EMAIL environment variable is set.
+             --api-password, -p <s>:   RightScale Dashboard User email. Not needed if API_USER_PASSWORD environment variable is set.
+        --cleanup, --no-cleanup, -c:   Don't do any cleanup on VM before snapshotting. Useful for debugging. (Default: true)
+               --aws-access-key, -k:   EC2 Account Access Key. Not needed if AWS_ACCESS_KEY environment variable is set.
+               --aws-secret-key, -s:   EC2 Account Secret. Not needed if AWS_SECRET_KEY environment variable is set.
+           --aws-account-number, -a:   EC2 Account ID. Not needed if AWS_ACCOUNT_NUMBER environment variable is set. Required for S3 images only.
+           --aws-image-type, -w <s>:   The type of image to create from this VM. Must be either 'EBS' or 'S3'.  (default: EBS)
+            --aws-kernel-id, -e <s>:   Kernel to use instead of what the VM is running. i.e. 'aki-fc8f11cc'. For a current list of IDs see http://goo.gl/dOS0mB.
+          --aws-s3-key-path, -y <s>:   location to file containing EC2 account key. S3 images only. (Default: /tmp/certs/x509.key)
+         --aws-s3-cert-path, -t <s>:   location to file containing EC2 account cert. S3 images only. (Default: /tmp/certs/x509.cert)
+      --aws-s3-image-bucket, -i <s>:   The bucket name for optimized S3 images (must be url safe). S3 images only (default: optimized-images)
+    --aws-s3-bundle-directory, -b <s>:   The local directory where the image bundle will be stored before uploading to S3. NOTE: this must have enough free space to hold
+                                       the image bundle. (Default: /mnt/ephemeral/bundle)
+      --aws-s3-bundle-no-filter, -n:   If set, will disable the default filtering used by the ec2-bundle-vol command. WARNING: setting this option could leave ssh keys or
+                                       other secrets on your
+                      --version, -o:   Print version and exit
+                         --help, -h:   Show this message
+
+
+Known Limitations
+------------------
+ * When creating S3 backed (instance store) images on AWS, the `--aws-s3-image-bucket` will not be created -- make sure it exists.
+ * Currently will only map up to 4 ephemeral drives to EBS image. Some EC2 instance-types support up to 24.
+ * Only tested on Ubuntu 12.04"
+
+
+Development
+-----------
+To run this locally, setup the required environment prerequisites:
+
+    > ./setup.sh
+
+Run the spec tests:
+
+    > bundle exec rake spec
+
+Setup a `.secrets/creds` file in the root of the project that defines:
+
+  	export API_USER_EMAIL=you@example.com
+  	export API_USER_PASSWORD=your_rightscale_password
+
+  	export AWS_SECRET_KEY=yoursecretkey
+  	export AWS_ACCESS_KEY=youraccesskey
+
+  	# needed for S3 (instance store) images only
+  	export AWS_ACCOUNT_NUMBER=your_account_number
+  	export AWS_S3_IMAGE_BUCKET=my_image_bucket
+
+Then add your credentials to the environment and run the script using the system's ruby interpreter:
+
+    > source .secrets/creds
+    > bundle exec ruby bin/image_optimize
+
+
+Integration Testing
+-------------------
+Follow the "development" directions (above), then rsync this project to VM:
+
+    > rsync -avz --exclude=.git . root@<ip_addr>:/usr/local/src/image_optimize
+
+Where `<ip_addr>` is the IP address of the VM you plan to test on.
+SSH into the VM and run the setup script to install package prerequisites:
+
+    > sudo -i
+    > cd /usr/local/src/image_optimize
+    > source ./setup
+
+On the VM, run the tests
+
+    > export IMAGE_TYPE=<type> ; bundle exec rake test
+
+Where `<type>` is either `EBS` or `S3` images.
+
+
+Future
+------
+ * automate integration testing on EBS and S3 instances
+ * break out rightscale specific calls into 'manager' interface to support different management platforms
+ * break out system command calls into 'platform' interface to enable multi-distro support
+
+
+License
+-------
+Author: cary@rightscale.com
+Copyright 2014 RightScale, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+

data/Rakefile

@@ -0,0 +1,45 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "bundler/gem_tasks"
+
+begin
+  require 'rspec/core/rake_task'
+
+  task :default => :spec
+
+  desc "Run all unit tests in spec directory"
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.pattern = 'spec/**/*_spec.rb'
+  end
+
+  desc "Run all integration tests in spec directory"
+  RSpec::Core::RakeTask.new(:test) do |t|
+    t.pattern = 'test/**/*_test.rb'
+  end
+
+rescue LoadError
+  STDERR.puts "\n*** RSpec not available. (sudo) gem install rspec to run unit tests. ***\n\n"
+end
+
+desc "Create a tarball for non-gem distribution"
+task :tarball do |t|
+  version = File.open("VERSION", "r") { |f| f.read }
+  filename = "image_optimize_#{version}.tar"
+  puts "Creating tarball #{filename}"
+  puts  `mkdir ./pkg ; tar cvf pkg/image_optimize-#{version}.tar bin/ lib/ VERSION Gemfile Gemfile.lock`
+  puts "success!"
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.1

data/bin/image_optimize

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+#
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'rubygems'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'app', 'image_optimizer'))
+
+optimizer = ImageOptimize::Optimizer.new.run

data/image_optimize.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+gem = Gem::Specification.new do |gem|
+  gem.name          = 'image_optimize'
+  gem.version       = File.open("VERSION", "r") { |f| f.read }
+  gem.authors       = ['caryp']
+  gem.email         = ['cary@rightscale.com']
+  gem.description   = %q{Bundle a running server into a new image that will be used on next launch}
+  gem.summary       = %q{Bundle a running server into a new image that will be used on next launch}
+  gem.homepage      = ''
+
+  gem.files         = `git ls-files -z`.split("\x0")
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+  gem.add_dependency 'mime-types', '2.0'
+  gem.add_dependency 'right_api_client', '1.5.15'
+  gem.add_dependency 'trollop', '2.0'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec', '~>2.14'
+  gem.add_development_dependency 'pry'
+end

data/lib/api_client_15.rb

@@ -0,0 +1,86 @@
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'right_api_client'
+require 'logger'
+
+module RightScale
+
+  # Create an instance facing or user base API client
+  class ApiClient15
+
+    def initialize
+      @log ||= Logger.new(STDOUT)
+      parse_user_data
+    end
+
+    def log(logger)
+      @log = logger
+    end
+
+    def get_client(type, user=nil, password=nil)
+      case type
+      when :instance
+        configure_instance_api_client()
+      when :user
+        configure_full_api_client(user, password)
+      else
+        raise "FATAL: ApiClient of #{type} not supported. You must specify either :user or :instance."
+      end
+    end
+
+    # make connection to API v1.5 with 'instance' role
+    def configure_instance_api_client
+      options = {
+        :account_id => @account_id,
+        :instance_token => @instance_token,
+        :api_url => @api_url
+      }
+      instance_client = create_client(options)
+      instance_client.log(@log)
+      instance_client
+    end
+
+    # make connection to RightScale API v1.5 with a specific user's roles
+    def configure_full_api_client(user, password)
+      options = {:email => user, :password => password, :account_id => @account_id }
+      api_client = create_client(options)
+      api_client.log(@log)
+      api_client
+    end
+
+    protected
+
+    def parse_user_data
+      begin
+        require '/var/spool/cloud/user-data.rb'
+      rescue LoadError => e
+        puts "FATAL: not a RightScale managed VM, or you already cleaned up your user-data file."
+        exit 1
+      end
+
+      @api_token = ENV['RS_API_TOKEN']
+      raise 'RightScale API token environment variable RS_API_TOKEN is unset' unless @api_token
+      @account_id, @instance_token = @api_token.split /:/
+      @api_url = "https://#{ENV['RS_SERVER']}"
+    end
+
+    def create_client(options)
+      RightApi::Client.new options
+    end
+
+  end
+end

data/lib/app/image_optimizer.rb

@@ -0,0 +1,195 @@
+#
+# Author: cary@rightscale.com
+# Copyright 2014 RightScale, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+libdir = File.join(File.dirname(__FILE__), '..')
+$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
+
+require 'trollop'
+require 'api_client_15'
+require 'image_bundle'
+require 'mixins/command'
+
+module ImageOptimize
+
+  class Optimizer
+
+    include ImageOptimize::Command
+
+    def run
+      parse_args
+      setup_logging
+      configure
+
+      # "cleaning" should wipe user-data and meta-data which will only allow this script
+      # to be run once.  Default to false to so we can run this multiple times to make
+      # debugging more efficient.
+      # XXX: this probably should be done by the manager class
+      prepare_image if @args[:do_cleanup]
+
+      # generate unique image name and description for our image
+      name = unique_name(@args[:image_prefix])
+      description = @args[:image_description]
+
+      # snapshot and register new image
+      @log.info "Snapshot instance..."
+      @image_util.snapshot_instance(name, description)
+      @log.info "Register image..."
+      @image_util.register_image(name, description)
+      @log.info "Add image to next instance..."
+      @image_util.add_image_to_next_instance
+      @log.info "SUCCESS!"
+    end
+
+    def configure
+
+      @log.debug("Config Parameters: #{@args}")
+
+      # check for AWS credentials set
+      raise 'Environment variable AWS_SECRET_KEY must be set' unless @args[:aws_secret_key]
+      raise 'Environment variable AWS_ACCESS_KEY must be set' unless @args[:aws_access_key]
+
+      # XXX: The fact that we're using two API clients should be hidden
+      #      inside a RightScale 'manager' class.
+      api_helper = RightScale::ApiClient15.new
+      api_helper.log(@log)
+      @instance_client = api_helper.get_client(:instance)
+      @api_client = api_helper.get_client(:user, @args[:api_user], @args[:api_password])
+
+      if @args[:aws_image_type] == "EBS"  # XXX: use an image bundle factory instead of if-else
+        @log.debug("Creating ImageBundleEc2EBS object")
+        @image_util =
+          ImageBundleEc2EBS.new(
+            @instance_client,        # XXX: instead of passing two api handles, just pass a manager class
+            @api_client,
+            @args[:aws_access_key],
+            @args[:aws_secret_key],
+            @args[:aws_kernel_id]
+          )
+      else
+        @log.debug("Creating ImageBundleEc2S3 object")
+
+        # check for credentials set in environment
+        raise 'Environment variable AWS_S3_IMAGE_BUCKET must be set' unless @args[:aws_s3_image_bucket]
+        raise 'Environment variable AWS_ACCOUNT_NUMBER must be set' unless @args[:aws_account_number]
+
+        @image_util =
+          ImageBundleEc2S3.new(
+            @instance_client,
+            @api_client,
+            @args[:aws_access_key],
+            @args[:aws_secret_key],
+            @args[:aws_account_number],
+            @args[:aws_s3_key_path],
+            @args[:aws_s3_cert_path],
+            @args[:aws_s3_image_bucket],
+            @args[:aws_s3_bundle_directory],
+            @args[:aws_s3_bundle_no_filter],
+            @args[:aws_kernel_id]
+          )
+      end
+
+      @image_util.log(@log)
+
+    end
+
+    def parse_args
+      version = File.open("VERSION", "r") { |f| f.read }
+
+      @args = Trollop::options do
+        version "image_optimize #{version} (c) 2014 RightScale, Inc."
+        banner <<-EOS
+
+Bundle a running VM into a new image that will be used on next launch
+
+Usage:
+       image_optimize [options]
+where [options] are:
+EOS
+        opt :verbose, "If set will enable debug logging.  WARNING: will write ec2 creds to log if set!!",
+            :long => "--verbose", :default => false
+
+        opt :image_prefix, "Prefix to add to the optimized image name. Helps when searching for your optimized images.",
+             :default => "optimized-image", :long => "--prefix", :type => String
+        opt :image_description, "Description to add to optimized images",
+           :default => "Cached image", :long => "--description", :type => String
+
+        # Mangement platform API Creds
+        opt :api_user, "RightScale Dashboard User email. Not needed if API_USER_EMAIL environment variable is set.",
+           :default => ENV['API_USER_EMAIL'], :long => "--api-user", :short => "-u", :type => String
+        opt :api_password, "RightScale Dashboard User email. Not needed if API_USER_PASSWORD environment variable is set.",
+           :default => ENV['API_USER_PASSWORD'], :long => "--api-password", :short => "-p", :type => String
+        opt :do_cleanup, "Don't do any cleanup on VM before snapshotting. Useful for debugging.", :long => "--cleanup", :default => true
+
+        # EC2 Creds
+        opt :aws_access_key, "EC2 Account Access Key. Not needed if AWS_ACCESS_KEY environment variable is set.",
+           :default => ENV['AWS_ACCESS_KEY'], :long => "--aws-access-key", :short => "-k"
+        opt :aws_secret_key, "EC2 Account Secret. Not needed if AWS_SECRET_KEY environment variable is set.",
+           :default => ENV['AWS_SECRET_KEY'], :long => "--aws-secret-key", :short => "-s"
+        opt :aws_account_number, "EC2 Account ID. Not needed if AWS_ACCOUNT_NUMBER environment variable is set. Required for S3 images only.",
+           :default => ENV['AWS_ACCOUNT_NUMBER'], :long => "--aws-account-number"
+
+        # EC2 Image type
+        opt :aws_image_type, "The type of image to create from this VM. Must be either 'EBS' or 'S3'. ",
+           :default => 'EBS', :long => "--aws-image-type", :type => String
+        opt :aws_kernel_id, "Kernel to use instead of what the VM is running. i.e. 'aki-fc8f11cc'. For a current list of IDs see http://goo.gl/dOS0mB.",
+           :default => nil, :long => "--aws-kernel-id", :type => String
+
+        # EC2 Instance Store Parameters
+        opt :aws_s3_key_path, "location to file containing EC2 account key. S3 images only.",
+           :default => '/tmp/certs/x509.key', :long => "--aws-s3-key-path"
+        opt :aws_s3_cert_path, "location to file containing EC2 account cert. S3 images only.",
+           :default => '/tmp/certs/x509.cert', :long => "--aws-s3-cert-path"
+        opt :aws_s3_image_bucket, "The bucket name for optimized S3 images (must be url safe). S3 images only",
+           :default => "optimized-images", :long => "--aws-s3-image-bucket"
+        opt :aws_s3_bundle_directory, "The local directory where the image bundle will be stored before uploading to S3. NOTE: this must have enough free space to hold the image bundle.",
+           :default => "/mnt/ephemeral/bundle", :long => "--aws-s3-bundle-directory"
+        opt :aws_s3_bundle_no_filter, "If set, will disable the default filtering used by the ec2-bundle-vol command. WARNING: setting this option could leave ssh keys or other secrets on your",
+           :default => false, :long => "--aws-s3-bundle-no-filter"
+      end
+    end
+
+    private
+
+    # create an image name using prefix and VM name
+    #
+    # Returns: [String] image name
+    def unique_name(prefix)
+      image_name = "#{@instance_client.get_instance.name} #{Time.new.to_s}"
+      image_name = "#{prefix} #{image_name}" if prefix
+      image_name.gsub(/[^0-9A-Za-z.\-]/, '-') # sanitize it!
+    end
+
+    # clean image for snapshoting
+    # taken from http://cpenniman.blogspot.com/2012/09/cleaning-up-rightimage-before.html
+    def prepare_image
+      @log.info "Prepare image"
+      cmd = "rm -rf /var/spool/cloud /var/lib/rightscale/right_link/*.js ; rm -f /opt/rightscale/var/lib/monit/monit.state ; rm -f /opt/rightscale/var/run/monit.pid ; sync"
+      execute(cmd)
+    end
+
+    # configure logging
+    def setup_logging
+      @log = Logger.new(STDOUT)
+      @log.level = Logger::INFO
+      @log.level = Logger::DEBUG if @args[:verbose]
+      @log.formatter = proc do |serverity, time, progname, msg|
+        "#{msg}\n"
+      end
+    end
+
+  end
+end