rightimage_tools 0.2.1 → 0.5.0

data/Gemfile

@@ -1,14 +1,16 @@
 source "http://rubygems.org"
 
 gem "rest_connection"
+gem "right_api_client"
 gem "right_aws"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "rspec", "~> 2.3.0"
-  gem "bundler", "~> 1.0.0"
+#  gem "bundler", "~> 1.1.4"
   gem "jeweler", "~> 1.6.4"
   gem "rcov", ">= 0"
   gem "flexmock"
+  gem "json", "~> 1.7.5"
 end

data/Rakefile

@@ -22,7 +22,10 @@ Jeweler::Tasks.new do |gem|
   gem.license = "MIT"
   gem.summary = %Q{A set of tools to support the building of RightImages}
   gem.files = %w(LICENSE.txt VERSION README.rdoc Rakefile Gemfile) + FileList["{bin,lib,spec}/**/*"].to_a
-  gem.executables = %w(image_mover mci_merge update_s3_index)
+  # Include built ui files while ignoring their sources.
+  # Used to include report_viewer.html.
+  gem.files.include FileList["ui/build/**/*"].to_a
+  gem.executables = %w(image_mover mci_merge report_tool update_s3_index)
   gem.description = gem.summary
   gem.email = "peter.schroeter@rightscale.com"
   gem.authors = ["Peter Schroeter"]
@@ -30,25 +33,3 @@ Jeweler::Tasks.new do |gem|
 end
 Jeweler::RubygemsDotOrgTasks.new
 
-require 'rspec/core'
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
-end
-
-RSpec::Core::RakeTask.new(:rcov) do |spec|
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
-end
-
-task :default => :spec
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "rightimage_tools #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.5.0

data/bin/azure_api

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'ruby-debug'
+require 'right_cloud_api'
+require File.expand_path("../../lib/azure_helper",__FILE__)
+require 'cloud/azure/storage/manager'
+require 'uri'
+require 'yaml'
+
+include AzureHelper
+
+account = ARGV.shift
+cmd = ARGV.shift
+params = {}
+ARGV.each do |p|
+  (k,v) = p.split(/[=:]/)
+  params[k] = v
+end
+
+unless account && cmd
+  puts "USAGE: azure_api <account> <api_command> [param1=value1] [param2=value2]"
+  puts "EXAMPLE: azure_api rightscalewestus ListBlobs Container=rightimage-linux"
+  exit 1
+end
+
+password = lookup_password(account)
+endpoint = "https://#{account}.blob.core.windows.net"
+
+@azure = RightScale::CloudApi::Azure::Storage::Manager.new(account, password, endpoint, :api_version=>'2012-02-12')
+puts "CALLING #{cmd} WITH #{params.inspect}"
+api_dump(@azure,cmd,params)

data/bin/azure_migrate

@@ -0,0 +1,174 @@
+#!/usr/bin/env ruby
+
+require 'ruby-debug'
+require 'right_cloud_api'
+require 'cloud/azure/storage/manager'
+require File.expand_path("../../lib/azure_helper",__FILE__)
+require 'uri'
+require 'yaml'
+
+include AzureHelper
+
+class CopyState
+  attr_accessor :account
+  attr_accessor :api
+  attr_accessor :state
+  def initialize(api, account, state = "pending")
+    @api = api
+    @account = account
+    @state = state
+  end
+end
+
+url_src = ARGV.shift
+url_dst = ARGV.shift
+image_name = ARGV.shift
+unless url_src =~ /^http/ and url_dst =~ /^http|^(linux|rswin|mswin|windows)$/
+puts <<-EOF
+USAGE: azure_migrate URL_TO_SRC_BLOB URL_TO_DEST_BLOB
+USAGE: azure_migrate URL_TO_SRC_BLOB <linux|rswin|mswin> <image name>
+       if URL_TO_DEST_BLOB is 'windows' or 'linux', program will copy
+       image to all linux or windows image accounts. 'rswin' for rightscale 
+       accounts (for testing), and mswin is microsoft (official publish)
+       also acceptable parameters
+       For the first usage you might use this for migrating to non-standard 
+       accounts, such as test or services
+       Second usage is preferred for a publishing type event
+EXAMPLE: azure_migrate https://linuxros.blob.core.windows.net/vm-images/RightImage-CentOS-6.2-x64-v5.8.8.1.vhd https://rightscalewestus.blob.core.windows.net/vm-images/RightImage-CentOS-6.2-x64-v5.8.8.1.vhd
+EXAMPLE: azure_migrate https://linuxros.blob.core.windows.net/vm-images/RightImage-CentOS-6.2-x64-v5.8.8.1.vhd linux RightImage-CentOS-6.2-x64-v5.8.8.1.vhd 
+EXAMPLE: azure_migrate http://ds1p6hm133skyrl.blob.core.windows.net/vhds/1vqtgdr5.xwr20120719072616.vhd rswin RightImage-Windows-2008R2-SP1-x64-sqlsvr2012-v5.8.8.vhd
+EOF
+  exit 1
+end
+
+# This function takes an url to a blob and grants read access to it
+# by signing a request for read access with the account's shared key
+# The request is made using GET parameters described in bits hash
+def generate_shared_access_key(url)
+  (account,password,endpoint,container,blob) = parse_url(url)
+
+  url_new = url.dup
+  t = Time.now.utc
+#  t1 = Time.now.utc + (60*60*24)
+#  fmt = '%Y-%m-%d'
+  t1 = Time.now.utc + (3600*8)
+  fmt = '%Y-%m-%dT%H:%MZ'
+  sig_fields = ["sp","st","se","crsrc","si", "sv"]
+  get_param_fields  = ["sp","st","se","sr", "sv", "sig"]
+  bits = {
+    'sp' => 'r', # grant holder of this url read access
+    'sr' => 'b', # for a blob
+    'crsrc' => "/#{account}/#{container}/#{blob}", # denoted by this resource
+    'st' => t.strftime(fmt), # starting now
+    'se' => t1.strftime(fmt), # and ending in 8 hours
+    'sv' => '2012-02-12' #api version, required
+  }
+  header = sig_fields.map {|b| bits[b] || ""}.join("\x0A")
+  header = header.toutf8 if header.respond_to?(:toutf8)
+  # Now sign the request with the shared key
+  bits["sig"] = Base64.encode64(HMAC::SHA256.new(Base64.decode64(password)).update(URI.decode(header)).digest).chomp
+  get_params = get_param_fields.map do |p|
+    p+"="+URI.escape(bits[p], Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
+  end
+  url_new << "?" << get_params.join("&")
+#  puts "HEADER: ---------------"
+  puts header
+  puts "-----------------------"
+  puts "URL: #{url}"
+  puts "SIGNATURE: #{bits["sig"]}"
+  puts "URL_SIGNED: #{url_new}"
+  return url_new
+end
+
+if url_dst == "linux"
+  account_names = RS_LINUX_ACCOUNTS
+  container = "rightimage-linux"
+  blob = image_name
+elsif url_dst == "windows"
+  account_names = RS_LINUX_ACCOUNTS + RS_WINDOWS_ACCOUNTS
+  container = "rightimage-windows"
+  blob = image_name
+elsif url_dst == "rswin"
+  account_names = RS_LINUX_ACCOUNTS
+  container = "rightimage-windows"
+  blob = image_name
+elsif url_dst == "mswin"
+  account_names = RS_WINDOWS_ACCOUNTS
+  container = "rightimage-windows"
+  blob = image_name
+else
+  (account,password,endpoint,container,blob) = parse_url(url_dst)
+  account_names = [account]
+end
+
+if url_dst =~ /^linux|^rswin|^mswin/
+  raise "Destination vhd name must be supplied when using mass copy" unless image_name and image_name =~ /./
+  raise "Image name must end in .vhd" unless image_name and image_name =~ /.vhd/
+end
+
+transfers = account_names.map do |account|
+  password = lookup_password(account)
+  endpoint = "https://#{account}.blob.core.windows.net"
+  CopyState.new(
+    RightScale::CloudApi::Azure::Storage::Manager.new(account, password, endpoint, :api_version=>'2012-02-12'),
+    account
+  )
+end
+
+
+url_src_signed = generate_shared_access_key(url_src)
+puts "COPYING #{url_src}"
+transfers.each do |t|
+  puts "TO #{t.account}/#{container}/#{blob}"
+  begin
+    t.api.GetBlobProperties('Container' => container, 'Blob' => blob)
+  rescue
+  end
+  if t.api.response.code.to_s =~ /^2/
+    puts "Blob already exists for #{t.account}, polling status"
+  else
+    puts "Running CopyBlob command for #{t.account} then polling status"
+    api_dump(t.api, "CopyBlob","Container"=>container,"Blob"=>blob,"Source"=>url_src_signed)
+    if t.api.response.code.to_s !~ /^2/
+      raise "ERROR account #{t.account} response code for CopyBlob not in 2xx range"
+    end
+  end
+end
+
+240.times do
+  transfers.each_with_index do |t,i|
+    t.api.GetBlobProperties('Container' => container, 'Blob' => blob)
+    status = t.api.response.headers["x-ms-copy-status"].first
+    if t.api.response.code.to_s !~ /^2/
+      raise "ERROR account #{t.account} container #{container} blob #{blob} response code for GetBlobProperties not in 2xx range"
+    end
+    progress = t.api.response.headers["x-ms-copy-progress"].first
+    if progress =~ /[0-9]+\/[0-9]+/
+      (a,b) = progress.split("/")
+      progress = sprintf("%3s%" % [(a.to_i*100/b.to_i).to_s])
+    end
+    if status =~ /success/
+      progress = "DONE"
+    elsif status =~ /fail/
+      progress = "FAIL"
+    end
+    case status
+    when /fail/ then t.state = :failure
+    when /success/ then t.state = :success
+    when /pending/ then t.state = :pending
+    else t.state = :unknown
+    end
+    print "#{t.account.sub("rightscale","")}[#{progress}] "
+    print "\n" if (i % RS_LINUX_ACCOUNTS.size == (RS_LINUX_ACCOUNTS.size - 1))
+  end
+#  print "\n"
+  break unless transfers.find {|t| t.state == :pending }
+  sleep 30
+end
+if transfers.all? { |t| t.state == :success }
+  puts "ALL successful!"
+  exit 0
+else
+  puts "ERROR occurred, or we timed out, maybe run this command again"
+  exit 1
+end

data/bin/azure_publish

@@ -0,0 +1,213 @@
+#!/usr/bin/env ruby
+
+#require 'ruby-debug'
+require 'right_cloud_api'
+require 'cloud/azure/storage/manager'
+require File.expand_path("../../lib/azure_helper",__FILE__)
+require 'uri'
+require 'yaml'
+require 'json'
+require 'pp'
+
+include AzureHelper
+
+def deregister_image(accounts, image_name)
+    images = registered_images(image_name)
+    if images.size == 0
+      puts "Found no images matching name or blob"
+    else
+      puts "Found images to deregister: "
+      images.each_with_index do |image, i|
+        puts (i+1).to_s + ") Name: " + image['Name']
+        puts "Location: " + image['Location']
+        puts "Blob: " + image['MediaLink'].to_s if image['MediaLink']
+        puts #{i['Name']} #{i['MediaLink'].to_s} #{i['Location']}"
+      end
+      print "Proceed? [yes|no] "
+      proceed = STDIN.gets
+      if proceed =~ /^[yY]/
+        images.each_with_index do |image, i|
+          `azure vm image delete #{image['Name']}`
+        end
+      end
+    end
+end
+
+def registered_images(image_name)
+  images = JSON.parse(`azure vm image list --json`.chomp)
+  registered = []
+  image_name = image_name.sub(".vhd","").downcase
+  images.each do |i|
+#    pp i
+    link_name = i["MediaLink"].to_s.dup
+    name = i["Name"].dup
+    link_name = link_name.sub(".vhd","").downcase
+    name = name.sub(".vhd","").downcase
+    AZURE_LINUX_LOCATIONS.values.each do |location|
+      link_name = link_name.sub(location.sub(" ","-"),"")
+      name = name.sub(location.sub(" ","-"),"")
+    end
+#    p "L #{link_name} IM #{image_name} N #{name}"
+    if link_name == image_name or 
+       link_name.split("/").last == image_name or
+       name == image_name
+      registered << i
+    end
+  end
+  return registered
+end
+
+def register_image(accounts, container, image_url)
+  images = JSON.parse(`azure vm image list --json`)
+  accounts.each do |account, api|
+    next if account =~ /win$/
+    image_name = image_url.split("/").last
+    image_name = image_name.sub(".vhd","")
+    regional_image_url = "https://#{account}.blob.core.windows.net/#{container}/#{image_name}.vhd"
+    name_with_region = image_name + "-" + AZURE_LINUX_LOCATIONS[account].gsub(" ","-")
+    image_name_clean = image_name.gsub(/[_-]/," ")
+
+    if registered_images(name_with_region).size > 0
+      puts "Image #{name_with_region} already registered"
+    else
+      begin
+        results = api.GetBlobMetadata('Container' => container, 'Blob'=>"#{image_name}.vhd")
+        is_public = false
+        if api.response.headers.key?("x-ms-meta-rs_public")
+          is_public = true
+        end
+        found_image = true
+      rescue
+        found_image = false
+      end
+      if found_image
+        unless is_public
+          puts "WARNING: Image for region #{account} is not public, registering anyways"
+        end
+        puts "azure vm image create --label \"#{image_name_clean}\" --os #{OS_TYPE} --blob-url \"#{regional_image_url}\" \"#{name_with_region}\""
+        puts `azure vm image create --label \"#{image_name_clean}\" --os #{OS_TYPE} --blob-url \"#{regional_image_url}\" \"#{name_with_region}\"`
+      else
+        puts "Could not find image #{regional_image_url} for account #{account}"
+      end
+    end
+  end
+end
+
+def list_images(accounts, container)
+  accounts.each do |account, api|
+    results = api.ListBlobs('Container' => container)
+    if results['EnumerationResults']['Blobs']
+      blobs = results['EnumerationResults']['Blobs']['Blob']
+    else
+      blobs = []
+    end
+    if blobs.class == Hash
+      blobs = [blobs]
+    end
+    puts "=================== #{account} ====================="
+    blobs.each do |blob|
+      name = blob["Name"] 
+      # no on-the-fly images
+      next if name =~ /os-i-[0-9a-f]{8,12}$/
+
+      ispublic = 'NOT_PUBLISHED'
+      results = api.GetBlobMetadata('Container'=>container,"Blob"=>name)
+      ispublic = "PUBLISHED" if api.response.headers.key?('x-ms-meta-rs_public')
+#      header = header.first if header.respond_to? "first"
+#      puts "RES"+results.response.inspect
+      puts "NAME #{name} ====== #{ispublic}"
+      puts "URL https://#{account}.blob.core.windows.net/#{container}/#{name}"
+    end
+  end
+end
+
+def set_public(accounts, container, image, publicize=true)
+  accounts.each do |account, api|
+    found_image = false
+    begin
+      results = api.GetBlobMetadata('Container' => container, 'Blob'=>image)
+      is_public = false
+      if api.response.headers.key?("x-ms-meta-rs_public")
+        is_public = true
+      end
+      found_image = true
+    rescue
+      puts "#{account} image #{image} not found"
+    end
+    #    api_dump(api, "GetBlobProperties", {'Container' => container, 'Blob' => blob})
+  #      api_dump(api, "GetBlobMetadata", {'Container' => container, 'Blob' => blob})
+    if found_image
+      if publicize && !is_public
+        timestamp = Time.now.gmtime.to_s
+        puts "#{account} ADDING KEY: x-ms-meta-rs_public: #{timestamp}"
+        api.SetBlobMetadata('Container' => container, 'Blob' => image, 'Metadata'=>{"rs_public"=>timestamp})
+    #    api_dump(api, "SetBlobMetadata", 'Container' => container, 'Blob' => image, 'Metadata'=>{"rs_public"=>timestamp})
+      elsif !publicize && is_public
+        puts "#{account} REMOVING KEY: x-ms-meta-rs_public: #{timestamp}"
+        api.SetBlobMetadata('Container' => container, 'Blob' => image, 'Metadata'=>{})
+    #    api_dump(api, "SetBlobMetadata", 'Container' => container, 'Blob' => image, 'Metadata'=>{})
+      elsif publicize && is_public
+        puts "#{account} image #{image} already public"
+      else #!publicize %% !is_public
+        puts "#{account} image #{image} for already NOT public"
+      end
+    end
+  end
+end
+
+cmd       = ARGV.shift
+platform  = ARGV.shift
+options   = ARGV.dup
+#pp options
+unless cmd and cmd != "help"
+  puts <<-EOF
+USAGE: azure_publish COMMAND [COMMAND OPTIONS]
+USAGE: COMMAND set_public <linux|windows> <image_name.vhd>
+USAGE: COMMAND unset_public <linux|windows> <image_name.vhd>
+USAGE: COMMAND list_images <linux|windows>
+USAGE: COMMAND register_image <linux|windows> <image_name.vhd>
+USAGE: COMMAND deregister_image <linux|windows> <image_name.vhd|url_to_blob>
+EXAMPLE: azure_publish list_images linux
+EXAMPLE: azure_publish set_public linux RightImage-CentOS-6.2-x64-v5.8.8.1.vhd
+EXAMPLE: azure_publish register_image linux RightImage-CentOS-6.2-x64-v5.8.8.1
+EOF
+  exit 1
+end
+
+if platform == "linux"
+  account_names = RS_LINUX_ACCOUNTS
+  container = "rightimage-linux"
+  OS_TYPE = "linux"
+#  (a,p,ep,c,blob) = parse_url(url_src)
+elsif platform == "windows"
+  account_names = RS_LINUX_ACCOUNTS + RS_WINDOWS_ACCOUNTS
+  container = "rightimage-windows"
+  OS_TYPE = "windows"
+elsif platform == "rswin"
+  account_names = RS_LINUX_ACCOUNTS
+  container = "rightimage-windows"
+  OS_TYPE = "windows"
+#  (a,p,ep,c,blob) = parse_url(url_src)
+elsif platform == "mswin"
+  account_names = RS_WINDOWS_ACCOUNTS
+  container = "rightimage-windows"
+  OS_TYPE = "windows"
+#  (a,p,ep,c,blob) = parse_url(url_src)
+end
+
+accounts = account_names.map do |account|
+  password = lookup_password(account)
+  endpoint = "https://#{account}.blob.core.windows.net"
+  [account, RightScale::CloudApi::Azure::Storage::Manager.new(account, password, endpoint, :api_version=>'2012-02-12')]
+end
+
+case cmd
+when "list_images"  then list_images(accounts, container)
+when "set_public"   then set_public(accounts, container, options[0], true)
+when "unset_public" then set_public(accounts, container, options[0], false)
+when "deregister_image" then deregister_image(accounts, options[0])
+when "register_image" then register_image(accounts, container, options[0])
+else raise "Unknown cmd #{cmd}"
+end
+
+