CloudyScripts 0.0.14 → 1.4.15

data/Rakefile

@@ -12,7 +12,7 @@ require 'rake/testtask'
 
 spec = Gem::Specification.new do |s|
   s.name = 'CloudyScripts'
-  s.version = '0.0.14'
+  s.version = '1.4.15'
   s.has_rdoc = true
   s.extra_rdoc_files = ['README.rdoc', 'LICENSE']
   s.summary = 'Scripts to facilitate programming for infrastructure clouds.'

data/lib/help/ec2_helper.rb

@@ -59,4 +59,41 @@ class Ec2Helper
     end
     return vols['volumeSet']['item'][0][prop.to_s]
   end
+
+  def snapshot_prop(snapshot_id, prop)
+    snaps = @ec2_api.describe_snapshots(:snapshot_id => snapshot_id)
+    begin
+      if snaps['snapshotSet']['item'].size == 0
+        raise Exception.new("snapshot #{snapshot_id} not found")
+      end
+      return snaps['snapshotSet']['item'][0][prop.to_s]
+    rescue
+      raise Exception.new("snapshot #{snapshot_id} not found")
+    end
+  end
+
+  def ami_prop(ami_id, prop)
+    amis = @ec2_api.describe_images(:image_id => ami_id)
+    begin
+      if amis['imagesSet']['item'].size == 0
+        raise Exception.new("image #{ami_id} not found")
+      end
+      return amis['imagesSet']['item'][0][prop.to_s]
+    rescue
+        raise Exception.new("image #{ami_id} not found")
+    end
+  end
+
+  def instance_prop(instance_id, prop)
+    instances = @ec2_api.describe_instances(:instance_id => instance_id)
+    begin
+      if instances['reservationSet']['item'][0]['instancesSet']['item'].size == 0
+        raise Exception.new("instance #{instance_id} not found")
+      end
+      return instances['reservationSet']['item'][0]['instancesSet']['item'][prop.to_s]
+    rescue
+      raise Exception.new("instance #{instance_id} not found")
+    end
+  end
+
 end

data/lib/help/remote_command_handler.rb

@@ -1,5 +1,7 @@
 require 'rubygems'
 require 'net/ssh'
+require 'net/scp'
+require 'timeout'
 
 # Provides methods to be executed via ssh to remote instances.
 class RemoteCommandHandler
@@ -102,14 +104,21 @@ class RemoteCommandHandler
   end
 
   # Copy directory using options -avHx
-  def rsync(source_path, dest_path, exclude_path = nil)
+  def local_rsync(source_path, dest_path, exclude_path = nil)
     exclude = ""
     if exclude_path != nil
       exclude = "--exclude #{exclude_path}"
     end
     e = "rsync -avHx #{exclude} #{source_path} #{dest_path}"
     @logger.debug "going to execute #{e}"
-    remote_exec_helper(e, nil, nil, false)
+    remote_exec_helper(e, nil, nil, false) #TODO: handle output in stderr?
+  end
+
+  # Copy directory via an ssh-tunnel.
+  def remote_rsync(keyfile, source_path, dest_ip, dest_path)
+    e = "rsync -rlpgoDzq -e "+'"'+"ssh -o stricthostkeychecking=no -i #{keyfile}"+'"'+" #{source_path} root@#{dest_ip}:#{dest_path}"
+    @logger.debug "going to execute #{e}"
+    remote_exec_helper(e, nil, nil, false) #TODO: handle output in stderr?
   end
 
   # Zip the complete contents of the source path into the destination file.
@@ -122,6 +131,15 @@ class RemoteCommandHandler
     end
   end
 
+  def echo(data, file)
+    exec = "echo #{data} > #{file}"
+    @logger.debug "going to execute #{exec}"
+    remote_execute(exec, nil, true)
+    if !file_exists?(file)
+      raise Exception.new("file #{file} could not be created")
+    end
+  end
+
   # Executes the specified #exec_string on a remote session specified.
   # When #push_data is specified, the data will be used as input for the
   # command and thus allow to respond in advance to commands that ask the user
@@ -168,6 +186,14 @@ class RemoteCommandHandler
     stdout.join()
   end
 
+  def upload(ip, user, key_data, local_file, destination_file, timeout = 60)
+    Timeout::timeout(timeout) {
+      Net::SCP.start(ip, user, {:key_data => [key_data], :timeout => timeout}) do |scp|
+        scp.upload!(local_file, destination_file)
+      end
+    }
+  end
+
   private
 
   # Executes the specified #exec_string on the opened remote session.

data/lib/help/state_transition_helper.rb

@@ -1,6 +1,8 @@
+require 'net/scp'
+
 # Contains methods that are used by the scripts in the state-machines. Since
 # they are reused by different scripts, they are factored into this module.
-# 
+#
 # Note: it is supposed that a hash named @context exists @context[:script]
 # must be set to a script object to pass information and messages
 # to listeners.
@@ -20,34 +22,39 @@ module StateTransitionHelper
   def connect(dns_name, ssh_keyfile = nil, ssh_keydata = nil)
     post_message("connecting to #{dns_name}...")
     connected = false
-    remaining_trials = 3
+    last_connection_problem = ""
+    remaining_trials = 5
     while !connected && remaining_trials > 0
       remaining_trials -= 1
       if ssh_keyfile != nil
         begin
+          @logger.info("connecting using keyfile")
           remote_handler().connect_with_keyfile(dns_name, ssh_keyfile)
           connected = true
         rescue Exception => e
           @logger.info("connection failed due to #{e}")
-          @logger.debug(e.backtrace.join("\n"))
+          last_connection_problem = "#{e}"
+          @logger.debug(e.backtrace.select(){|line| line.include?("state_transition_helper")}.join("\n"))
         end
       elsif ssh_keydata != nil
         begin
+          @logger.info("connecting using keydata")
           remote_handler().connect(dns_name, "root", ssh_keydata)
           connected = true
         rescue Exception => e
           @logger.info("connection failed due to #{e}")
-          @logger.debug(e.backtrace.join("\n"))
+          last_connection_problem = "#{e}"
+          @logger.debug(e.backtrace.select(){|line| line.include?("state_transition_helper")}.join("\n"))
         end
       else
         raise Exception.new("no key information specified")
       end
       if !connected
-        sleep(5) #try again
+        sleep(20) #try again
       end
     end
     if !connected
-      raise Exception.new("connection attempts stopped")
+      raise Exception.new("connection attempts stopped (#{last_connection_problem})")
     end
     os = remote_handler().retrieve_os()
     post_message("connected to #{dns_name}. OS installed is #{os}")
@@ -55,6 +62,15 @@ module StateTransitionHelper
     return os
   end
 
+  # If a remote command handler is connected, disconnect him silently.
+  def disconnect
+    begin
+      remote_handler().disconnect()
+    rescue
+    end
+    self.remote_handler= nil
+  end
+
   # Launch an instance based on an AMI ID
   # Input Parameters:
   # * ami_id => ID of the AMI to be launched
@@ -67,11 +83,12 @@ module StateTransitionHelper
   # * kernel_id => EC2 Kernel ID of the started instance
   # * ramdisk_id => EC2 Ramdisk ID of the started instance
   # * architecture => architecture (e.g. 386i, 64x) of the started instance
-  def launch_instance(ami_id, key_name, security_group_name)
+  def launch_instance(ami_id, key_name, security_group_name, ec2_handler = nil)
+    ec2_handler = ec2_handler() if ec2_handler == nil
     post_message("starting up instance to execute the script (AMI = #{ami_id}) ...")
     @logger.debug "start up AMI #{ami_id}"
     # find out the image architecture first
-    image_props = ec2_handler().describe_images(:image_id => ami_id)
+    image_props = ec2_handler.describe_images(:image_id => ami_id)
     architecture = image_props['imagesSet']['item'][0]['architecture']
     instance_type = "m1.small"
     if architecture != "i386"
@@ -81,7 +98,7 @@ module StateTransitionHelper
     @logger.info arch_log_msg
     post_message(arch_log_msg)
     # now start it
-    res = ec2_handler().run_instances(:image_id => ami_id,
+    res = ec2_handler.run_instances(:image_id => ami_id,
       :security_group => security_group_name, :key_name => key_name,
       :instance_type => instance_type
     )
@@ -92,7 +109,7 @@ module StateTransitionHelper
     started = false
     while started == false
       sleep(5)
-      res = ec2_handler().describe_instances(:instance_id => instance_id)
+      res = ec2_handler.describe_instances(:instance_id => instance_id)
       state = res['reservationSet']['item'][0]['instancesSet']['item'][0]['instanceState']
       @logger.info "instance is in state #{state['name']} (#{state['code']})"
       if state['code'].to_i == 16
@@ -252,10 +269,11 @@ module StateTransitionHelper
   # * volume_id => EC2 ID for the EBS volume to be snapshotted
   # Returns:
   # * snapshot_id => EC2 ID for the snapshot created
-  def create_snapshot(volume_id)
+  def create_snapshot(volume_id, description = "")
     post_message("going to create a snapshot for volume #{volume_id}...")
     @logger.debug "create snapshot for volume #{volume_id}"
-    res = ec2_handler().create_snapshot(:volume_id => volume_id)
+    res = ec2_handler().create_snapshot(:volume_id => volume_id,
+      :description => description)
     snapshot_id = res['snapshotId']
     @logger.info "snapshot_id = #{snapshot_id}"
     done = false
@@ -358,8 +376,8 @@ module StateTransitionHelper
     post_message("going to start copying files to #{destination_path}. This may take quite a time...")
     @logger.debug "start copying to #{destination_path}"
     start = Time.new.to_i
-    remote_handler().rsync("/", "#{destination_path}", "#{destination_path}")
-    remote_handler().rsync("/dev/", "#{destination_path}/dev/")
+    remote_handler().local_rsync("/", "#{destination_path}", "#{destination_path}")
+    remote_handler().local_rsync("/dev/", "#{destination_path}/dev/")
     endtime = Time.new.to_i
     @logger.info "copy took #{(endtime-start)}s"
     post_message("copying is done (took #{endtime-start})s")
@@ -376,23 +394,52 @@ module StateTransitionHelper
     post_message("EBS volume successfully zipped")
   end
 
-  protected
+  def remote_copy(keyname, source_dir, dest_machine, dest_dir)
+    post_message("going to remote copy all files from volume. This may take some time...")
+    remote_handler().remote_rsync("/root/.ssh/#{keyname}.pem", source_dir, dest_machine, dest_dir)
+    post_message("remote copy operation done")
+  end
 
-  def post_message(msg)
-    if @context[:script] != nil
-      @context[:script].post_message(msg)
-    end
+  def upload_file(ip, user, key_data, file, target_file)
+    post_message("going to upload #{file} to #{ip}:/#{target_file}")
+    remote_handler().upload(ip, user, key_data, file, target_file)
   end
 
+  #setting/retrieving handlers
+
   def remote_handler()
-    if @context[:remote_command_handler] == nil
-      @context[:remote_command_handler] = RemoteCommandHandler.new
+    if @remote_handler == nil
+      if @context[:remote_command_handler] == nil
+        @context[:remote_command_handler] = RemoteCommandHandler.new
+      else
+        @remote_handler = @context[:remote_command_handler]
+      end
     end
-    @context[:remote_command_handler]
+    @remote_handler
+  end
+
+  def remote_handler=(remote_handler)
+    @remote_handler = remote_handler
   end
 
   def ec2_handler()
-    @context[:ec2_api_handler]
+    if @ec2_handler == nil
+      @ec2_handler = @context[:ec2_api_handler]
+    end
+    @ec2_handler
+  end
+
+  def ec2_handler=(ec2_handler)
+    @ec2_handler = ec2_handler
+  end
+
+
+  protected
+
+  def post_message(msg)
+    if @context[:script] != nil
+      @context[:script].post_message(msg)
+    end
   end
 
 end

data/lib/scripts/ec2/copy_snapshot.rb

@@ -0,0 +1,188 @@
+require "help/script_execution_state"
+require "scripts/ec2/ec2_script"
+require "help/remote_command_handler"
+require "help/dm_crypt_helper"
+require "help/ec2_helper"
+require "AWS"
+
+# Copy a given snapshot to another region
+# * start up instance in source-region, create volume from snapshot, attach volume, and mount it
+# * start up instance in destination-region, create empty volume of same size, attache volume, and mount it
+# * copy the destination key to the source instance
+# * perform an rsynch
+#   sync -PHAXaz --rsh "ssh -i /home/${src_user}/.ssh/id_${dst_keypair}" --rsync-path "sudo rsync" ${src_dir}/ ${dst_user}@${dst_public_fqdn}:${dst_dir}/
+# * create a snapshot of the volume
+# * clean-up everything
+
+class CopySnapshot< Ec2Script
+  # context information needed
+  # * the EC2 credentials (see #Ec2Script)
+  # * snapshot_id => The ID of the snapshot to be downloaded
+  # * target_ec2_handler => The EC2 handler connected to the region where the snapshot is being copied to
+  # * source_key_name => Key name of the instance that manages the snaphot-volume in the source region
+  # * source_ssh_key_data => Key information for the security group that starts the AMI [if not set, use ssh_key_files]
+  # * source_ssh_key_files => Key information for the security group that starts the AMI
+  # * target_key_name => Key name of the instance that manages the snaphot-volume in the target region
+  # * target_ssh_key_data => Key information for the security group that starts the AMI [if not set, use ssh_key_files]
+  # * target_ssh_key_files => Key information for the security group that starts the AMI
+  # * source_ami_id => ID of the AMI to start in the source region
+  # * target_ami_id => ID of the AMI to start in the target region
+  
+  def initialize(input_params)
+    super(input_params)
+  end
+
+  def check_input_parameters()
+  end
+
+  # Load the initial state for the script.
+  # Abstract method to be implemented by extending classes.
+  def load_initial_state()
+    CopySnapshotState.load_state(@input_params)
+  end
+
+  private
+
+  # Here begins the state machine implementation
+  class CopySnapshotState < ScriptExecutionState
+
+    def self.load_state(context)
+      InitialState.new(context)
+    end
+
+    def local_region
+      self.ec2_handler=(@context[:ec2_api_handler])
+    end
+
+    def remote_region
+      self.ec2_handler=(@context[:target_ec2_handler])
+    end
+  end
+
+  # Initial state: start up AMI in source region
+  class InitialState < CopySnapshotState
+    def enter()
+      result = launch_instance(@context[:source_ami_id], @context[:source_key_name], "default")
+      @context[:source_instance_id] = result.first
+      @context[:source_dns_name] = result[1]
+      @context[:source_availability_zone] = result[2]
+      SourceInstanceLaunchedState.new(@context)
+    end
+  end
+
+  # Source is started. Create a volume from the snapshot, attach and mount the volume
+  class SourceInstanceLaunchedState < CopySnapshotState
+    def enter()
+      @context[:source_volume_id] = create_volume_from_snapshot(@context[:snapshot_id],
+        @context[:source_availability_zone])
+      device = "/dev/sdj"  #TODO: make device configurable
+      mount_point = "/mnt/tmp_#{@context[:source_volume_id]}"
+      attach_volume(@context[:source_volume_id], @context[:source_instance_id], device)
+      connect(@context[:source_dns_name], nil, @context[:source_ssh_keydata])
+      mount_fs(mount_point, device)
+      disconnect()
+      SourceVolumeReadyState.new(@context)
+    end
+  end
+
+  # Source is ready. Now start instance in the target region
+  class SourceVolumeReadyState < CopySnapshotState
+    def enter()
+      remote_region()
+      result = launch_instance(@context[:target_ami_id], @context[:target_key_name], 
+        "default")
+      @context[:target_instance_id] = result.first
+      @context[:target_dns_name] = result[1]
+      @context[:target_availability_zone] = result[2]
+      TargetInstanceLaunchedState.new(@context)
+    end
+  end
+
+  # Destination instance is started. Now configure storage.
+  class TargetInstanceLaunchedState < CopySnapshotState
+    def enter()
+      local_region()
+      ec2_helper = Ec2Helper.new(@context[:ec2_api_handler])
+      volume_size = ec2_helper.snapshot_prop(@context[:snapshot_id], :volumeSize).to_i
+      #
+      remote_region()
+      @context[:target_volume_id] = create_volume(@context[:target_availability_zone], volume_size)
+      device = "/dev/sdj"  #TODO: make device configurable
+      mount_point = "/mnt/tmp_#{@context[:target_volume_id]}"
+      attach_volume(@context[:target_volume_id], @context[:target_instance_id], device)
+      connect(@context[:target_dns_name], nil, @context[:target_ssh_keydata])
+      create_fs(@context[:target_dns_name], device)
+      mount_fs(mount_point, device)
+      disconnect()
+      TargetVolumeReadyState.new(@context)
+    end
+  end
+
+  # Storages are ready. Only thing missing: the key of the target region
+  # must be available on the instance in the source region to be able to perform
+  # a remote copy.
+  class TargetVolumeReadyState < CopySnapshotState
+    def enter()
+      post_message("upload key of target-instance to source-instance...")
+      upload_file(@context[:source_dns_name], "root", @context[:source_ssh_keydata],
+        @context[:target_ssh_keyfile], "/root/.ssh/#{@context[:target_key_name]}.pem")
+      post_message("credentials are in place to connect source and target.")
+      KeyInPlaceState.new(@context)
+    end
+  end
+
+  # Now we can copy.
+  class KeyInPlaceState < CopySnapshotState
+    def enter()
+      connect(@context[:source_dns_name], nil, @context[:source_ssh_keydata])
+      source_dir = "/mnt/tmp_#{@context[:source_volume_id]}/"
+      dest_dir = "/mnt/tmp_#{@context[:target_volume_id]}"
+      remote_copy(@context[:target_key_name], source_dir, @context[:target_dns_name], dest_dir)
+      disconnect()
+      DataCopiedState.new(@context)
+    end
+  end
+
+  # Data of snapshot now copied to the new volume. Create a snapshot of the
+  # new volume.
+  class DataCopiedState < CopySnapshotState
+    def enter()
+      remote_region()
+      @context[:new_snapshot_id] = create_snapshot(@context[:target_volume_id], "Created by Cloudy_Scripts - copy_snapshot")
+      @context[:result][:snapshot_id] = @context[:new_snapshot_id]
+      SnapshotCreatedState.new(@context)
+    end
+  end
+
+  # Operation done. Now only cleanup is missing, i.e. shut down instances and
+  # remote the volumes that were created. Start with cleaning the ressources
+  # in the local region.
+  class SnapshotCreatedState < CopySnapshotState
+    def enter()
+      local_region()
+      shut_down_instance(@context[:source_instance_id])
+      delete_volume(@context[:source_volume_id])
+      SourceCleanedUpState.new(@context)
+    end
+  end
+
+  # Cleanup the resources in the target region.
+  class SourceCleanedUpState < CopySnapshotState
+    def enter()
+      remote_region()
+      shut_down_instance(@context[:target_instance_id])
+      delete_volume(@context[:target_volume_id])
+      Done.new(@context)
+    end
+  end
+
+end
+
+#Cloudy_Script: copy snapshots between regions
+#start up instance in source-region, create volume from snapshot, attach volume, and mount it
+#start up instance in destination-region, create empty volume of same size, attache volume, and mount it
+#copy the destination key to the source instance
+#perform an rsynch
+#sync -PHAXaz --rsh "ssh -i /home/${src_user}/.ssh/id_${dst_keypair}" --rsync-path "sudo rsync" ${src_dir}/ ${dst_user}@${dst_public_fqdn}:${dst_dir}/
+#create a snapshot of the volume
+#clean-up everything

data/lib/scripts/ec2/download_snapshot.rb

@@ -21,6 +21,7 @@ class DownloadSnapshot < Ec2Script
   # * the EC2 credentials (see #Ec2Script)
   # * ami_id: the ID of the AMI to be started to perform the operations and to run the web-server for download
   # * security_group_name => name of the security group used to start the AMI (should open ports for SSH and HTTP)
+  # * key_name => Name of the key to be used to access the instance providing the download
   # * ssh_key_data => Key information for the security group that starts the AMI [if not set, use ssh_key_files]
   # * ssh_key_files => Key information for the security group that starts the AMI
   # * snapshot_id => The ID of the snapshot to be downloaded

metadata

@@ -3,10 +3,10 @@ name: CloudyScripts
 version: !ruby/object:Gem::Version 
   prerelease: false
   segments: 
-  - 0
-  - 0
-  - 14
-  version: 0.0.14
+  - 1
+  - 4
+  - 15
+  version: 1.4.15
 platform: ruby
 authors: 
 - Matthias Jung
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-25 00:00:00 +01:00
+date: 2010-04-27 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -63,6 +63,7 @@ files:
 - lib/help/state_change_listener.rb
 - lib/help/state_transition_helper.rb
 - lib/scripts/ec2/ami2_ebs_conversion.rb
+- lib/scripts/ec2/copy_snapshot.rb
 - lib/scripts/ec2/dm_encrypt.rb
 - lib/scripts/ec2/download_snapshot.rb
 - lib/scripts/ec2/ec2_script.rb