ec2-copy-snapshot 0.1.0

data/.gitignore

@@ -0,0 +1,18 @@
+.rvmrc
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gem "aws-sdk"
+gem "net-ssh"

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Zach Wily
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Ec2::Copy::Snapshot
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'ec2-copy-snapshot'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ec2-copy-snapshot
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/ec2-copy-snapshot

@@ -0,0 +1,221 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'rubygems'
+require 'optparse'
+require 'aws-sdk'
+require 'net/ssh'
+
+opts = {
+  :aws_access_key => ENV["AWS_ACCESS_KEY_ID"],
+  :aws_secret_access_key => ENV["AWS_SECRET_ACCESS_KEY"],
+  :source_aws_region => 'us-east-1',
+  :destination_aws_region => nil,
+  :snapshot_id => nil,
+  :source_hostname => nil,
+  :destination_hostname => nil,
+  :keep_destination_volume => false,
+}
+
+parser = OptionParser.new do |o|
+  o.banner = "Usage: copy_snapshot_between_regions [options] snapshot-id"
+
+  o.on("--aws-access-key ACCESS_KEY", "AWS Access Key (default: AWS_ACCESS_KEY_ID)") do |v|
+    opts[:aws_access_key] = v
+  end
+
+  o.on("--aws-secret-access-key SECRET_KEY", "AWS Secret Access Key (default: AWS_SECRET_ACCESS_KEY)") do |v|
+    opts[:aws_secret_access_key] = v
+  end
+
+  o.on("--from-region REGION", "AWS Region to copy snapshot FROM (default: us-east-1)") do |v|
+    opts[:source_aws_region] = v
+  end
+
+  o.on("--to-region REGION", "AWS Region to copy snapshot TO") do |v|
+    opts[:destination_aws_region] = v
+  end
+
+  o.on("--from-instance HOSTNAME", "Hostname of an instance in the source region to use for the copy") do |v|
+    opts[:source_hostname] = v
+  end
+
+  o.on("--to-instance HOSTNAME", "Hostname of an instance in the target region to use for the copy") do |v|
+    opts[:destination_hostname] = v
+  end
+
+  o.on("--[no-]keep-volume", "Keep the destination volume the new snapshot is made from (default: no)") do |v|
+    opts[:keep_destination_volume] = v
+  end
+end
+parser.parse!
+
+if ARGV.length == 1
+  opts[:snapshot_id] = ARGV.shift
+end
+
+if opts.values.any? {|v| v.nil? }
+  puts parser.help
+  exit 1
+end
+
+STDOUT.sync = true
+
+ec2 = AWS::EC2.new(:access_key_id => opts[:aws_access_key], :secret_access_key => opts[:aws_secret_access_key])
+ec2_source = ec2.regions[opts[:source_aws_region]]
+ec2_destination = ec2.regions[opts[:destination_aws_region]]
+
+def wait_for(interval = 5, &block)
+  while true
+    print "."
+    sleep interval
+    break if yield
+  end
+end
+
+def available_dev(instance)
+  # Returns a device that appears to be unused. There is a race condition here
+  # if several people are using this script with the same instances at the same
+  # time, but worst case the script will fail when we try to attach to a device
+  # that somebody else has started using.
+  device_mappings = instance.block_device_mappings
+  possible_devs = ('f'..'z').to_a.map {|l| "/dev/sd#{l}" } - device_mappings.keys
+  return nil if possible_devs.empty?
+  possible_devs.sample
+end
+
+def wait_for_volume(volume, desc)
+  print "Waiting for #{desc} volume #{volume.id}..."
+  wait_for { volume.status != :creating }
+  puts "done."
+
+  if volume.status != :available
+    raise "Volume is not available (#{volume.status})"
+  end
+end
+
+def attach_volume_to_instance(volume, instance, desc)
+  dev = available_dev(instance)
+  if dev.nil?
+    raise "Can't find a dev on #{instance.id}"
+  end
+
+  puts "Attaching #{desc} volume to instance #{instance.id} at #{dev}..."
+  attachment = volume.attach_to(instance, dev)
+  return attachment
+end
+
+def wait_for_attachment(attachment, desc)
+  print "Waiting for #{desc} volume to be attached..."
+  wait_for { attachment.status != :attaching }
+  puts "done."
+
+  if attachment.status != :attached
+    raise "Volume did not successfully attach (#{attachment.status})"
+  end
+end
+
+def detach_attachment(attachment, desc)
+  puts "Detaching #{desc} #{attachment.volume.id}"
+  attachment.delete(:force => true)
+end
+
+def wait_for_detachment(attachment, desc)
+  print "Waiting for #{desc} volume to be detached..."
+  volume = attachment.volume
+  wait_for { volume.status != :in_use }
+  puts "done."
+
+  if attachment.volume.status != :available
+    raise "Volume did not successfully detach #{attachment.volume.status})"
+  end
+end
+
+def delete_volume(volume, desc)
+  puts "Deleting #{desc} #{volume.id}..."
+  volume.delete
+end
+
+def mangled_dev(dev)
+  # Newer versions of Linux address the sdX volumes as xvdX
+  dev.gsub('/dev/sd', '/dev/xvd')
+end
+
+begin
+  print "Logging into #{opts[:source_hostname]}..."
+  source_ssh = Net::SSH.start(opts[:source_hostname], nil)
+  source_instance_id = source_ssh.exec!("curl -s http://169.254.169.254/latest/meta-data/instance-id")
+  source_instance = ec2_source.instances[source_instance_id]
+  puts " #{source_instance_id}"
+
+  print "Logging into #{opts[:destination_hostname]}..."
+  destination_ssh = Net::SSH.start(opts[:destination_hostname], nil)
+  destination_instance_id = destination_ssh.exec!("curl -s http://169.254.169.254/latest/meta-data/instance-id")
+  destination_instance = ec2_destination.instances[destination_instance_id]
+  puts " #{destination_instance_id}"
+
+  puts "Getting snapshot information..."
+  source_snapshot = ec2_source.snapshots[opts[:snapshot_id]]
+  if source_snapshot.status != :completed
+    raise "Snapshot does not appear complete (#{source_snapshot.status})"
+  end
+
+  print "Creating a volume from the snapshot..."
+  source_volume = source_snapshot.create_volume(source_instance.availability_zone)
+  puts " #{source_volume.id}"
+
+  print "Creating a destination volume..."
+  destination_volume = ec2_destination.volumes.create(
+    :size => source_volume.size,
+    :availability_zone => destination_instance.availability_zone)
+  puts " #{destination_volume.id}"
+
+  wait_for_volume(source_volume, "source")
+  wait_for_volume(destination_volume, "destination")
+
+  source_attachment = attach_volume_to_instance(source_volume, source_instance, "source")
+  destination_attachment = attach_volume_to_instance(destination_volume, destination_instance, "destination")
+
+  wait_for_attachment(source_attachment, "source")
+  wait_for_attachment(destination_attachment, "destination")
+
+  puts "Copying data from source to destination, this may take awhile..."
+  source_ssh.exec!(%Q{
+    catcmd="cat"; if [[ $(which pv) ]]; then catcmd="pv -f"; fi; \
+    sudo $catcmd #{mangled_dev(source_attachment.device)} | dd bs=1M | bzip2 --compress | \
+    ssh root@#{opts[:destination_hostname]} "bzip2 --decompress | dd of=#{mangled_dev(destination_attachment.device)} bs=1M"
+  }) do |channel, stream, data|
+    STDOUT.print data if stream == :stdout
+    STDERR.print data if stream == :stderr
+  end
+
+  print "Creating snapshot of new volume..."
+  destination_snapshot = destination_volume.create_snapshot("created by #{`whoami`} of #{opts[:snapshot_id]} from #{opts[:source_aws_region]}")
+  puts " #{destination_snapshot.id}"
+
+  print "Waiting for snapshot to complete..."
+  wait_for { destination_snapshot.status != :pending }
+  puts "done."
+
+  if destination_snapshot.status != :completed
+    raise "Could not create snapshot (#{destination_snapshot.status})"
+  end
+ensure
+  puts "Cleaning up..."
+
+  # Start detaching the source and dest volumes in parallel, cause they might take awhile
+  detach_attachment(source_attachment, "source") if source_attachment
+  detach_attachment(destination_attachment, "destination") if destination_attachment && !opts[:keep_destination_volume]
+
+  wait_for_detachment(source_attachment, "source") if source_attachment
+  wait_for_detachment(destination_attachment, "destination") if destination_attachment && !opts[:keep_destination_volume]
+
+  delete_volume(source_volume, "source") if source_volume
+  delete_volume(destination_volume, "destination") if destination_volume && !opts[:keep_destination_volume]
+
+  puts "Closing SSH connections..."
+  destination_ssh.close if destination_ssh
+  source_ssh.close if source_ssh
+end
+
+puts "New snapshot created: #{destination_snapshot.id}" if destination_snapshot

data/ec2-copy-snapshot.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "ec2-copy-snapshot"
+  gem.version       = "0.1.0"
+  gem.authors       = ["Zach Wily"]
+  gem.email         = ["zach@zwily.com"]
+  gem.description   = %q{Script to copy an EC2 snapshot between regions}
+  gem.summary       = %q{Script to copy an EC2 snapshot between regions}
+  gem.homepage      = "https://github.com/zwily/ec2-copy-snapshot"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency('aws-sdk')
+  gem.add_dependency('net-ssh')
+end

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification
+name: ec2-copy-snapshot
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Zach Wily
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Script to copy an EC2 snapshot between regions
+email:
+- zach@zwily.com
+executables:
+- ec2-copy-snapshot
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/ec2-copy-snapshot
+- ec2-copy-snapshot.gemspec
+homepage: https://github.com/zwily/ec2-copy-snapshot
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Script to copy an EC2 snapshot between regions
+test_files: []
+has_rdoc: