CloudyScripts 2.14.54 → 2.14.58

data/Rakefile

@@ -12,7 +12,8 @@ require 'rake/testtask'
 
 spec = Gem::Specification.new do |s|
   s.name = 'CloudyScripts'
-  s.version = '2.14.54' #<number cloud-stacks supported>.<number cloud-scripts>.<counting releases>
+  #s.version = '2.14.54' #<number cloud-stacks supported>.<number cloud-scripts>.<counting releases>
+  s.version = '2.14.58' #<number cloud-stacks supported>.<number cloud-scripts>.<counting releases>
   s.has_rdoc = true
   s.extra_rdoc_files = ['README.rdoc', 'LICENSE']
   s.summary = 'Scripts to facilitate programming for infrastructure clouds.'

data/lib/help/remote_command_handler.rb

@@ -60,6 +60,10 @@ class RemoteCommandHandler
     remote_execute("ls #{path}")
   end
 
+  def file_size(file)
+    get_output("ls -lh #{file} | cut -d ' ' -f 5").strip
+  end
+
   # Returns the result of uname -a (Linux)
   def retrieve_os()
     get_output("uname -r").strip
@@ -251,14 +255,23 @@ class RemoteCommandHandler
 
   # dump and compress a device in a file locally
   def local_dump_and_compress(source_device, target_filename)
-    e = "sh -c 'dd if=#{source_device} | gzip > #{target_filename}'"
+    #e = "sh -c 'dd if=#{source_device} | gzip > #{target_filename}'"
+    e = "sh -c 'dd if=#{source_device} bs=512k | gzip -1 > #{target_filename}'"
     @logger.debug "going to execute #{e}" 
     status = remote_exec_helper(e, nil, nil, true)
   end
 
-  # idecompress and a file to a device locally
+  # dump a file to a device locally
   def local_decompress_and_dump(source_filename, target_device)
-    e = "sh -c 'gunzip -c #{source_filename} | dd of=#{target_device}'"
+    #e = "sh -c 'gunzip -c #{source_filename} | dd of=#{target_device}'"
+    e = "sh -c 'gunzip -1 -c #{source_filename} | dd of=#{target_device} bs=512k'"
+    @logger.debug "going to execute #{e}" 
+    status = remote_exec_helper(e, nil, nil, true)
+  end
+
+  # dump a device in a local file or a dump local file to a device
+  def local_dump(source_device, target_filename)
+    e = "sh -c 'dd if=#{source_device} of=#{target_filename} bs=1M'"
     @logger.debug "going to execute #{e}" 
     status = remote_exec_helper(e, nil, nil, true)
   end

data/lib/help/state_transition_helper.rb

@@ -358,6 +358,7 @@ module StateTransitionHelper
   def create_volume_from_snapshot(snapshot_id, availability_zone)
     post_message("going to create a new EBS volume from the specified snapshot...")
     @logger.debug "create volume in zone #{availability_zone}"
+    start_time = Time.new.to_i
     res = ec2_handler().create_volume(:snapshot_id => snapshot_id, :availability_zone => availability_zone)
     volume_id = res['volumeId']
     started = false
@@ -371,7 +372,9 @@ module StateTransitionHelper
         started = true
       end
     end
-    post_message("EBS volume #{volume_id} is ready")
+    end_time = Time.new.to_i
+    @logger.info "create volume from snapshot took #{(end_time-start_time)}s"
+    post_message("EBS volume #{volume_id} is ready (took #{end_time-start_time}s)")
     return volume_id
   end
 
@@ -382,7 +385,7 @@ module StateTransitionHelper
   # * temp_device_name => device name to be used for attaching (e.g. /dev/sdj1)
   def attach_volume(volume_id, instance_id, temp_device_name, timeout = 240)
     post_message("going to attach volume #{volume_id} to instance #{instance_id} on device #{temp_device_name}...")
-    @logger.info "attach volume #{volume_id} to instance #{instance_id} on device #{temp_device_name} (using a timeout of #{timeout})s"
+    @logger.info "attach volume #{volume_id} to instance #{instance_id} on device #{temp_device_name} (using a timeout of #{timeout}s)"
     ec2_handler().attach_volume(:volume_id => volume_id,
       :instance_id => instance_id,
       :device => temp_device_name
@@ -422,7 +425,7 @@ module StateTransitionHelper
   # * instance_id => EC2 ID for the instance to detach from
   def detach_volume(volume_id, instance_id, timeout = 240)
     post_message("going to detach volume #{volume_id} from instance #{instance_id}...")
-    @logger.info "detach volume #{volume_id} from instance #{instance_id} (using a timeout of #{timeout})s"
+    @logger.info "detach volume #{volume_id} from instance #{instance_id} (using a timeout of #{timeout}s)"
     ec2_handler().detach_volume(:volume_id => volume_id,
       :instance_id => instance_id
     )
@@ -469,6 +472,7 @@ module StateTransitionHelper
   def create_snapshot(volume_id, description = "")
     post_message("going to create a snapshot for volume #{volume_id}...")
     @logger.debug "create snapshot for volume #{volume_id}"
+    start_time = Time.new.to_i
     res = ec2_handler().create_snapshot(:volume_id => volume_id,
       :description => description)
     snapshot_id = res['snapshotId']
@@ -483,7 +487,9 @@ module StateTransitionHelper
         done = true
       end
     end
-    post_message("snapshot is done with ID=#{snapshot_id}")
+    end_time = Time.new.to_i
+    @logger.info "create snapshot took #{(end_time-start_time)}s"
+    post_message("snapshot is done with ID=#{snapshot_id} (took #{(end_time-start_time)}s)")
     return snapshot_id
   end
 
@@ -914,7 +920,7 @@ module StateTransitionHelper
   # dump and compress a device using dd and gzip
   def local_dump_and_compress_device_to_file(source_device, target_filename)
     post_message("going to start dumping and compressing source device '#{source_device}' to '#{target_filename}' file. This may take quite a time...")
-    @logger.debug "start dumping and compressing '#{source_device}' to '#{target_filename}'"
+    @logger.info "start dumping and compressing '#{source_device}' to '#{target_filename}'"
     start_time = Time.new.to_i
     if remote_handler().tools_installed?("dd") && remote_handler().tools_installed?("gzip")
       @logger.debug "use dd and gzip command line"
@@ -928,14 +934,37 @@ module StateTransitionHelper
       raise Exception.new("dd and/or gzip tools not installed")
     end
     end_time = Time.new.to_i
-    @logger.info "dump and compress took #{(end_time-start_time)}s"
-    post_message("dumping and compressing done (took #{end_time-start_time})s")
+    filesize = remote_handler().file_size(target_filename)
+    @logger.info "dump and compress to a #{filesize} file took #{(end_time-start_time)}s"
+    post_message("dumping and compressing to a #{filesize} file done (took #{end_time-start_time}s)")
+  end
+
+  # dump a device using dd
+  def local_dump_device_to_file(source_device, target_filename)
+    post_message("going to start dumping source device '#{source_device}' to '#{target_filename}' file. This may take quite a time...")
+    @logger.info "start dumping '#{source_device}' to '#{target_filename}'"
+    start_time = Time.new.to_i
+    if remote_handler().tools_installed?("dd")
+      @logger.debug "use dd command line"
+      status = remote_handler().local_dump(source_device, target_filename)
+      if status == false
+        @logger.error "failed to dump device"
+        raise Exception.new("failed to dump device")
+      end
+    else
+      @logger.error "dd tool not installed"
+      raise Exception.new("dd tool not installed")
+    end
+    end_time = Time.new.to_i
+    filesize = remote_handler().file_size(target_filename)
+    @logger.info "dump to a #{filesize} file took #{(end_time-start_time)}s"
+    post_message("dumping to a #{filesize} file done (took #{end_time-start_time}s)")
   end
 
   # decompress and dump a file using gunzip and dd
   def local_decompress_and_dump_file_to_device(source_filename, target_device)
     post_message("going to start decompressing and dumping file '#{source_filename}' to '#{target_device}' target device. This may take quite a time...")
-    @logger.debug "start decompressing and dumping '#{source_filename}' to '#{target_device}'"
+    @logger.info "start decompressing and dumping '#{source_filename}' to '#{target_device}'"
     start_time = Time.new.to_i
     if remote_handler().tools_installed?("dd") && remote_handler().tools_installed?("gunzip")
       @logger.debug "use dd and gzip command line"
@@ -949,27 +978,52 @@ module StateTransitionHelper
       raise Exception.new("dd and/or gunzip tools not installed")
     end
     end_time = Time.new.to_i
-    @logger.info "decompress and dump filetook #{(end_time-start_time)}s"
-    post_message("decompressing and dumping done (took #{end_time-start_time})s")
+    filesize = remote_handler().file_size(source_filename)
+    @logger.info "decompress and dump file of #{filesize} took #{(end_time-start_time)}s"
+    post_message("decompressing and dumping file of #{filesize} done (took #{end_time-start_time}s)")
+  end
+
+  # dump a file using dd
+  def local_dump_file_to_device(source_filename, target_device)
+    post_message("going to start dumping file '#{source_filename}' to '#{target_device}' target device. This may take quite a time...")
+    @logger.info "start dumping '#{source_filename}' to '#{target_device}'"
+    start_time = Time.new.to_i
+    if remote_handler().tools_installed?("dd")
+      @logger.debug "use dd command line"
+      status = remote_handler().local_dump(source_filename, target_device)
+      if status == false
+        @logger.error "failed to dump file"
+        raise Exception.new("failed to dump file")
+      end
+    else
+      @logger.error "dd tool not installed"
+      raise Exception.new("dd tool not installed")
+    end
+    end_time = Time.new.to_i
+    filesize = remote_handler().file_size(source_filename)
+    @logger.info "dump file of #{filesize} took #{(end_time-start_time)}s"
+    post_message("dumping file of #{filesize} done (took #{end_time-start_time}s)")
   end
 
   def disable_ssh_tty(host)
     post_message("going to disable SSH tty on #{host}...")
-    @logger.debug "disable SSH tty on #{host}"
+    @logger.info "disable SSH tty on #{host}"
     remote_handler().disable_sudoers_requiretty()
     post_message("SSH tty disabled")
   end
 
   def enable_ssh_tty(host)
     post_message("going to enable SSH tty on #{host}...")
-    @logger.debug "enable SSH tty on #{host}"
+    @logger.info "enable SSH tty on #{host}"
     remote_handler().enable_sudoers_requiretty()
     post_message("SSH tty enabled")
   end
 
   def remote_copy(user_name, keyname, source_dir, dest_machine, dest_user, dest_dir)
     post_message("going to remote copy all files from volume. This may take some time...")
+    @logger.info "going to remote copy all files from volume. This may take some time..."
     key_path_candidates = ["/#{user_name}/.ssh/", "/home/#{user_name}/.ssh/"]
+    start_time = Time.new.to_i
     key_path_candidates.each() {|key_path|
       key_file = "#{key_path}#{keyname}.pem"
       if remote_handler().file_exists?(key_path)
@@ -983,7 +1037,9 @@ module StateTransitionHelper
         break
       end
     }
-    post_message("remote copy operation done")
+    end_time = Time.new.to_i
+    @logger.info "remote copy operation took #{(end_time-start_time)}s"
+    post_message("remote copy operation done (took #{end_time-start_time}s)")
   end
 
   def upload_file(ip, user, key_data, file, target_file)
@@ -1078,6 +1134,11 @@ module StateTransitionHelper
                             'aki-8e5ea7e7' => 'pv-grub-hd00_1.02-x86_64',
                             'aki-805ea7e9' => 'pv-grub-hd0_1.02-i386',
                             'aki-825ea7eb' => 'pv-grub-hd0_1.02-x86_64',
+                            'aki-b6aa75df' => 'pv-grub-hd0_1.03-i386',
+                            'aki-88aa75e1' => 'pv-grub-hd0_1.03-x86_64',
+                            'aki-b2aa75db' => 'pv-grub-hd00_1.03-i386',
+                            'aki-b4aa75dd' => 'pv-grub-hd00_1.03-x86_64',
+
                             #RHEL kernel Amazon Kernel ID
                             'aki-36ed075f' => 'aki-rhel-i386',
                             'aki-08ed0761' => 'aki-rhel-x86_64'	
@@ -1090,9 +1151,14 @@ module StateTransitionHelper
                             'aki-81396bc4' => 'pv-grub-hd00_1.02-x86_64',
                             'aki-83396bc6' => 'pv-grub-hd0_1.02-i386',
                             'aki-8d396bc8' => 'pv-grub-hd0_1.02-x86_64',
-                            #RHEL kernel Amazon Kernel ID
-                            'aki-772c7f32' => 'aki-rhel-i386',
-                            'aki-712c7f34' => 'aki-rhel-x86_64'	
+                            'aki-f57e26b0' => 'pv-grub-hd0_1.03-i386',
+                            'aki-f77e26b2' => 'pv-grub-hd0_1.03-x86_64',
+                            'aki-e97e26ac' => 'pv-grub-hd00_1.03-i386',
+                            'aki-eb7e26ae' => 'pv-grub-hd00_1.03-x86_64',
+
+                            #RHEL kernel Amazon Kernel ID:
+                            'aki-772c7f32' => 'aki-rhel-i386',	# RH-pv-grub-hd0-V1.01-i386
+                            'aki-712c7f34' => 'aki-rhel-x86_64'	# RH-pv-grub-hd0-V1.01-x86_64
                            },
             'us-west-2' => {'aki-dee26fee' => 'pv-grub-hd00-V1.01-i386',
                             'aki-90e26fa0' => 'pv-grub-hd00-V1.01-x86_64',
@@ -1102,9 +1168,14 @@ module StateTransitionHelper
                             'aki-94e26fa4' => 'pv-grub-hd00_1.02-x86_64',
                             'aki-c2e26ff2' => 'pv-grub-hd0_1.02-i386',
                             'aki-98e26fa8' => 'pv-grub-hd0_1.02-x86_64',
+                            'aki-fa37baca' => 'pv-grub-hd0_1.03-i386',
+                            'aki-fc37bacc' => 'pv-grub-hd0_1.03-x86_64',
+                            'aki-f637bac6' => 'pv-grub-hd00_1.03-i386',
+                            'aki-f837bac8' => 'pv-grub-hd00_1.03-x86_64',
+
                             #RHEL kernel Amazon Kernel ID
-                            '' => 'aki-rhel-i386',
-                            '' => 'aki-rhel-x86_64'
+                            'aki-2efa771e' => 'aki-rhel-i386',
+                            'aki-10fa7720' => 'aki-rhel-x86_64'
                            },
             'eu-west-1' => {'aki-47eec433' => 'pv-grub-hd00-V1.01-i386',
                             'aki-41eec435' => 'pv-grub-hd00-V1.01-x86_64',
@@ -1114,6 +1185,11 @@ module StateTransitionHelper
                             'aki-60695814' => 'pv-grub-hd00_1.02-x86_64',
                             'aki-64695810' => 'pv-grub-hd0_1.02-i386',
                             'aki-62695816' => 'pv-grub-hd0_1.02-x86_64',
+                            'aki-75665e01' => 'pv-grub-hd0_1.03-i386',
+                            'aki-71665e05' => 'pv-grub-hd0_1.03-x86_64',
+                            'aki-89655dfd' => 'pv-grub-hd00_1.03-i386',
+                            'aki-8b655dff' => 'pv-grub-hd00_1.03-x86_64',
+
                             #RHEL kernel Amazon Kernel ID
                             'aki-af0a3ddb' => 'aki-rhel-i386',
                             'aki-a90a3ddd' => 'aki-rhel-x86_64'	
@@ -1126,6 +1202,11 @@ module StateTransitionHelper
                                  'aki-a6225af4' => 'pv-grub-hd00_1.02-x86_64',
                                  'aki-a4225af6' => 'pv-grub-hd0_1.02-i386',
                                  'aki-aa225af8' => 'pv-grub-hd0_1.02-x86_64',
+                                 'aki-f81354aa' => 'pv-grub-hd0_1.03-i386',
+                                 'aki-fe1354ac' => 'pv-grub-hd0_1.03-x86_64',
+                                 'aki-f41354a6' => 'pv-grub-hd00_1.03-i386',
+                                 'aki-fa1354a8' => 'pv-grub-hd00_1.03-x86_64',
+
                                  #RHEL kernel Amazon Kernel ID
                                  'aki-9c235ace' => 'aki-rhel-i386',
                                  'aki-82235ad0' => 'aki-rhel-x86_64'	
@@ -1138,9 +1219,14 @@ module StateTransitionHelper
                                  'aki-ea5df7eb' => 'pv-grub-hd00_1.02-x86_64',
                                  'aki-ec5df7ed' => 'pv-grub-hd0_1.02-i386',
                                  'aki-ee5df7ef' => 'pv-grub-hd0_1.02-x86_64',
+                                 'aki-42992843' => 'pv-grub-hd0_1.03-i386',
+                                 'aki-44992845' => 'pv-grub-hd0_1.03-x86_64',
+                                 'aki-3e99283f' => 'pv-grub-hd00_1.03-i386',
+                                 'aki-40992841' => 'pv-grub-hd00_1.03-x86_64',
+
                                  #RHEL kernel Amazon Kernel ID
                                  'aki-66c06a67' => 'aki-rhel-i386',
-                                 'aki-68c06a69' => 'aki-rhel-x86_64'	
+                                 'aki-68c06a69' => 'aki-rhel-x86_64'
                            },
             'sa-east-1' => {'aki-803ce39d' => 'pv-grub-hd00-V1.01-i386',
                             'aki-d03ce3cd' => 'pv-grub-hd00-V1.01-x86_64',
@@ -1150,9 +1236,14 @@ module StateTransitionHelper
                             'aki-cc3ce3d1' => 'pv-grub-hd00_1.02-x86_64',
                             'aki-823ce39f' => 'pv-grub-hd0_1.02-i386',
                             'aki-d23ce3cf' => 'pv-grub-hd0_1.02-x86_64',
+                            'aki-ca8f51d7' => 'pv-grub-hd0_1.03-i386',
+                            'aki-c48f51d9' => 'pv-grub-hd0_1.03-x86_64',
+                            'aki-ce8f51d3' => 'pv-grub-hd00_1.03-i386',
+                            'aki-c88f51d5' => 'pv-grub-hd00_1.03-x86_64',
+
                             #RHEL kernel Amazon Kernel ID
-                            '' => 'aki-rhel-i386',
-                            '' => 'aki-rhel-x86_64'	
+                            'aki-1a38e707' => 'aki-rhel-i386',
+                            'aki-1438e709' => 'aki-rhel-x86_64'
                            }
           }
     target_aki = ''

data/lib/scripts/ec2/copy_mswindows_snapshot.rb

@@ -86,6 +86,11 @@ class CopyMsWindowsSnapshot < Ec2Script
     if @input_params[:description] == nil || check_string_alnum(@input_params[:description])
       @input_params[:description] = "Created by Cloudy_Scripts - copy_mswindows_snapshot"
     end
+    if @input_params[:compression] != nil && (@input_params[:compression] =~ /^on$/i)
+      @input_params[:compression] = true
+    else
+      @input_params[:compression] = false
+    end
   end
 
   # Load the initial state for the script.
@@ -135,7 +140,8 @@ class CopyMsWindowsSnapshot < Ec2Script
     def enter()
       local_region()
       post_message("Lunching an Helper instance in source Region...")
-      result = launch_instance(@context[:source_ami_id], @context[:source_key_name], @context[:source_security_groups])
+      result = launch_instance(@context[:source_ami_id], @context[:source_key_name], @context[:source_security_groups], 
+                               nil, @context[:instance_type])
       @context[:source_instance_id] = result.first
       @context[:source_dns_name] = result[1]
       @context[:source_availability_zone] = result[2]
@@ -178,7 +184,11 @@ class CopyMsWindowsSnapshot < Ec2Script
       post_message("Using AWS name source device '#{@context[:device_name]}' and OS name '#{source_device}'")
       @context[:source_device_name] = source_device
       # Step2: create and attach a temp volume of the same size to dump and compress the entire drive
-      @context[:source_temp_volume_id] = create_volume(@context[:source_availability_zone], @context[:volume_size])
+      if @context[:compression]
+        @context[:source_temp_volume_id] = create_volume(@context[:source_availability_zone], @context[:volume_size])
+      else
+        @context[:source_temp_volume_id] = create_volume(@context[:source_availability_zone], @context[:volume_size] + 4)
+      end
       temp_device = @context[:temp_device_name] 
       attach_volume(@context[:source_temp_volume_id], @context[:source_instance_id], temp_device)
       aws_device_letter = temp_device.split('/')[2].gsub('sd', '').gsub('xvd', '').gsub(/[0-9]/, '')
@@ -206,8 +216,13 @@ class CopyMsWindowsSnapshot < Ec2Script
       local_region()
       connect(@context[:source_dns_name], @context[:source_ssh_username], nil, @context[:source_ssh_keydata])
       mount_point = "/mnt/tmp_#{@context[:source_temp_volume_id]}"
-      @context[:source_filename] = "#{mount_point}" + "/" + "#{@context[:snapshot_id]}" + ".gz"
-      local_dump_and_compress_device_to_file(@context[:source_device_name], @context[:source_filename])
+      if @context[:compression]
+        @context[:source_filename] = "#{mount_point}" + "/" + "#{@context[:snapshot_id]}" + ".img.gz"
+        local_dump_and_compress_device_to_file(@context[:source_device_name], @context[:source_filename])
+      else
+        @context[:source_filename] = "#{mount_point}" + "/" + "#{@context[:snapshot_id]}" + ".img"
+        local_dump_device_to_file(@context[:source_device_name], @context[:source_filename]) 
+      end
       disconnect()
 
       BackupedDataState.new(@context)
@@ -220,7 +235,8 @@ class CopyMsWindowsSnapshot < Ec2Script
   class BackupedDataState < CopyMsWindowsSnapshotState 
     def enter()
       remote_region()
-      result = launch_instance(@context[:target_ami_id], @context[:target_key_name], @context[:target_security_groups])
+      result = launch_instance(@context[:target_ami_id], @context[:target_key_name], @context[:target_security_groups], 
+                               nil, @context[:instance_type])
       @context[:target_instance_id] = result.first
       @context[:target_dns_name] = result[1]
       @context[:target_availability_zone] = result[2]
@@ -239,7 +255,11 @@ class CopyMsWindowsSnapshot < Ec2Script
     def enter()
       remote_region()
       # Step1: create and attach a temp volume for receiving archive of the drive
-      @context[:target_temp_volume_id] = create_volume(@context[:target_availability_zone], @context[:volume_size])
+      if @context[:compression]
+        @context[:target_temp_volume_id] = create_volume(@context[:target_availability_zone], @context[:volume_size])
+      else
+        @context[:target_temp_volume_id] = create_volume(@context[:target_availability_zone], @context[:volume_size] + 4)
+      end
       temp_device = @context[:temp_device_name] 
       attach_volume(@context[:target_temp_volume_id], @context[:target_instance_id], temp_device)
       connect(@context[:target_dns_name], @context[:target_ssh_username], nil, @context[:target_ssh_keydata]) 
@@ -335,8 +355,13 @@ class CopyMsWindowsSnapshot < Ec2Script
       remote_region()
       connect(@context[:target_dns_name], @context[:target_ssh_username], nil, @context[:target_ssh_keydata])
       mount_point = "/mnt/tmp_#{@context[:target_temp_volume_id]}"
-      @context[:source_filename] = "#{mount_point}" + "/" + "#{@context[:snapshot_id]}" + ".gz"
-      local_decompress_and_dump_file_to_device(@context[:source_filename], @context[:target_device_name])
+      if @context[:compression]
+        @context[:source_filename] = "#{mount_point}" + "/" + "#{@context[:snapshot_id]}" + ".img.gz"
+        local_decompress_and_dump_file_to_device(@context[:source_filename], @context[:target_device_name])
+      else
+        @context[:source_filename] = "#{mount_point}" + "/" + "#{@context[:snapshot_id]}" + ".img"
+        local_dump_file_to_device(@context[:source_filename], @context[:target_device_name])
+      end
       disconnect()
 
       RestoredDataState.new(@context)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: CloudyScripts
 version: !ruby/object:Gem::Version 
-  hash: 91
+  hash: 67
   prerelease: false
   segments: 
   - 2
   - 14
-  - 54
-  version: 2.14.54
+  - 58
+  version: 2.14.58
 platform: ruby
 authors: 
 - Matthias Jung
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-25 00:00:00 +00:00
+date: 2012-07-13 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency