cloudflock 0.4.0

data/lib/cloudflock/target/servers/platform.rb

@@ -0,0 +1,133 @@
+require 'cpe'
+require 'cloudflock'
+
+# Public: Serves as a small class to easily map host specifications to Image
+# and Flavor IDs in Rackspace Cloud.
+#
+# Examples
+#
+#   # Build platform data for a given CPE object
+#   platform = Platform.new(cpe)
+#   platform.image_id
+#   # => Fixnum
+class CloudFlock::Target::Servers::Platform
+  # Public: Gets/sets whether the target platform will be managed.
+  attr_accessor :managed
+  # Public: Gets/sets whether the target platform will use Rack Connect.
+  attr_accessor :rack_connect
+
+  # Public: Initialize the Platform object.
+  #
+  # cpe - CPE object from which to generate platform object.
+  #
+  # Raises ArgumentError if anything but a CPE object was given.
+  # Raises KeyError if the CPE object doesn't have a vendor or version defined.
+  def initialize(cpe)
+    raise ArgumentError unless cpe.kind_of?(CPE)
+    raise KeyError if cpe.vendor.nil? or cpe.version.nil?
+
+    @cpe = cpe
+    @distro = cpe.vendor
+    @product = cpe.product
+    @version = cpe.version
+    @managed = false
+    @rack_connect = false
+
+    build_maps
+  end
+
+  # Public: Generate a String of the platform's name/version suitable for
+  # display
+  #
+  # Returns a String describing the Platform
+  def to_s
+    "#{@distro.capitalize} #{@product.gsub(/_/, ' ').capitalize} #{@version}"
+  end
+
+  # Public: Return the Image ID to be used based on whether the account is
+  # managed, and the platform used
+  #
+  # Returns the Image ID corresponding to the Platform object as a String
+  def image
+    [:MANAGED_MAP, :UNMANAGED_MAP].each do |map|
+      unless self.class.const_defined?(map)
+        raise MapUndefined, "Const #{map} is undefined; maps appear unbuilt"
+      end
+    end
+
+    map = @managed ? self.class::MANAGED_MAP : self.class::UNMANAGED_MAP
+    distro = @distro.downcase.to_sym
+
+    unless map[distro].nil?
+      return map[distro][@version] unless map[distro][@version].nil?
+      return map[distro]["*"] unless map[distro]["*"].nil?
+    end
+
+    nil
+  end
+
+  # Public: Iterate through TARGET_LIST until a suitably large flavor_id is
+  # found, then return the appropriate index.  If no entry can be found, raise
+  # ValueError.
+  #
+  # symbol - A Symbol referencing the search target in TARGET_LIST.
+  # value  - A Fixnum containing the amount of memory or disk space required.
+  #
+  # Returns a Fixnum referencing the TARGET_LIST index.
+  # Raises ValueError if no symbol describes an appropriate target.
+  def get_target_by_symbol(symbol, value)
+    unless self.class.const_defined?(:FLAVOR_LIST)
+      raise MapUndefined, "FLAVOR_LIST is undefined; maps appear unbuild."
+    end
+
+    self.class::FLAVOR_LIST.each_with_index do |target, idx|
+      if target[symbol] > value
+        return idx
+      end
+    end
+
+    raise ValueError, "Unable to find a flavor matching #{symbol} #{value}"
+  end
+
+  # Internal: Build image and flavor maps
+  #
+  # Returns nothing
+  def build_maps
+    build_image_maps
+    build_flavor_maps
+  end
+
+  # Public: Give a recommendation for a Flavor ID and an Image ID which should
+  # suffice for a migration target host.
+  #
+  # args - Hash containing information on which to base the recommendation:
+  #        :memory  - Hash containing memory information:
+  #                   :total     - Total amount of memory allocated to the host
+  #                                profiled.
+  #                   :mem_used  - Amount of RAM in use at the time of
+  #                                profiling.
+  #                   :swapping? - Boolean denoting whether the host was
+  #                                swapping at the time of profiling.
+  #        :disk    - Fixnum containing the amount of disk which appears to be
+  #                   in use at the time of profiling.
+  #
+  # Returns a Hash containing the Flavor ID and a String containing the
+  # reasoning for the decision.
+  def build_recommendation(args)
+    recommendation = {}
+    target_mem = get_target_by_symbol(:mem, args[:memory][:mem_used])
+    target_mem += 1 if args[:memory][:swapping?]
+
+    target_disk = get_target_by_symbol(:hdd, args[:disk])
+
+    if target_mem >= target_disk
+      recommendation[:flavor] = target_mem
+      recommendation[:flavor_reason] = "RAM usage"
+    else
+      recommendation[:flavor] = target_disk
+      recommendation[:flavor_reason] = "Disk usage"
+    end
+
+    recommendation
+  end
+end

data/lib/cloudflock/target/servers/profile.rb

@@ -0,0 +1,394 @@
+require 'cloudflock/remote/ssh'
+require 'cpe'
+
+# Public: Provides methods to create a profile for a given host, mapping it to
+# available public cloud offerings, and reference data gathered during this
+# process.
+#
+# Examples
+#
+#   # Generate a profile from a shell already logged in to a remote host
+#   profile = Profile.new(ssh_object)
+#   profile.build
+#
+#   # Only determine the memory and I/O statistics for a host
+#   profile = Profile.new(ssh_object)
+#   profile.determine_memory
+#   profile.determine_io
+class CloudFlock::Target::Servers::Profile
+  # Public: Array containing warnings generated by the info gathering process.
+  attr_reader :warnings
+
+  # Public: Initialize the Profile object.
+  #
+  # shell - An SSH object open to the host to be profiled.
+  #
+  # Raises ArgumentError if passed anything but an SSH object.
+  def initialize(shell)
+    raise ArgumentError unless shell.kind_of? CloudFlock::Remote::SSH
+
+    @shell = shell
+
+    @warnings = []
+    @info = {}
+  end
+
+  # Public: Run all available determinations against the SSH object.
+  #
+  # Returns nothing.
+  def build
+    determine_version
+    determine_arch
+    determine_hostname
+    determine_memory
+    determine_cpu
+    determine_disk
+    determine_ips
+    determine_io
+    determine_web
+    determine_db
+    determine_lib
+    determine_rsync
+    determine_processes
+  end
+
+  # Public: Determine vendor and version of the OS running on the target host,
+  # and create an appropriate CPE object for it, assigning it to @info[:cpe].
+  #
+  # Returns nothing.
+  def determine_version
+    release = @shell.query("CPE", "cat /etc/system-release-cpe")
+
+    begin
+      cpe = CPE.parse(release)
+      cpe.version.gsub!(/[^0-9.]/, '')
+      @info[:cpe] = cpe
+      return
+    rescue ArgumentError
+      cpe = CPE.new(part: CPE::OS, product: "linux")
+    end
+
+    issue = @shell.query("ISSUE", "cat /etc/issue")
+
+    case issue
+    when /Arch/
+      cpe.vendor = "Arch"
+    when /CentOS/
+      cpe.vendor = "CentOS"
+      issue.gsub!(/\.\d.*$/, '')
+    when /Debian/
+      cpe.vendor = "Debian"
+      issue.gsub!(/\.\d.*$/, '')
+    when /This is/
+      cpe.vendor = "Gentoo"
+    when /SUSE/
+      cpe.vendor = "openSUSE"
+    when /Ubuntu/
+      cpe.vendor = "Ubuntu"
+    when /Red/
+      cpe.vendor = "Redhat"
+      issue.gsub!(/\.\d.*$/, '')
+    else
+      cpe.vendor = "Unknown"
+    end
+
+    cpe.version = version_number(issue)
+
+    @info[:cpe] = cpe
+  end
+
+  # Public: Determine the architecture of the target host and assign it to
+  # @info[:arch].
+  #
+  # Returns nothing.
+  def determine_arch
+    uname = @shell.query("UNAME", "uname -m")
+    case uname
+    when /x86_64/
+      @info[:arch] = 'x86_64'
+    when /i\d86/
+      @info[:arch] = 'i386'
+    else
+      @info[:arch] = "Unknown"
+    end
+  end
+
+  # Public: Determine the hostname of the target host and assign it to
+  # @info[:hostname]
+  #
+  # Returns nothing.
+  def determine_hostname
+    @info[:hostname] = @shell.query("HOST", "hostname")
+  end
+
+  # Public: Determine the available and total memory on the target host as
+  # well as historical usage via sar(1), if possible.  Assign these to
+  # @info[:memory] and @info[:memory_hist], respectively.
+  #
+  # Returns nothing.
+  def determine_memory
+    result = {}
+
+    free = %w{free -m |awk '$1 ~ /Mem/ {print $2, $2-$6-$7}; $1 ~ /Swap/
+              {print $3}'|xargs}.join(' ')
+    mem = @shell.query("MEMORY", free)
+    total, used, swap = mem.split(/\s+/)
+
+    result[:total] = total.to_i
+    result[:mem_used] = used.to_i
+    result[:swap_used] = swap.to_i
+    result[:swapping?] = swap.to_i > 0
+    @info[:memory] = result
+
+    # Determine average mem and swap usage
+    result = {}
+    sar_location = @shell.query("SAR", "which sar 2>/dev/null")
+    sar_command = %w{for l in $(find /var/log/ -name 'sa??'); do sar -r -f $l |
+                     grep Average; done | awk '{I+=1; TOT=$2+$3; CACHE+=$5+$6;
+                     FREE+=$2; SWAP+=$9;} END {CACHE=CACHE/I; FREE=FREE/I;
+                     SWAP=SWAP/I; print (TOT-(CACHE+FREE))/TOT*100,
+                     SWAP;}'}.join(' ')
+
+    if sar_location =~ /bin\//
+      sar_usage = @shell.query("HIST_MEM", sar_command)
+
+      if sar_usage =~ /\d \d/
+        hist_mem, hist_swap = sar_usage.split(/ /)
+        result[:mem_used] = hist_mem.to_i
+        result[:swap_used] = hist_swap.to_i
+      end
+    end
+
+    @info[:memory_hist] = result
+  end
+
+  # Public: Determine the number of CPUs present on the target host and the
+  # speed of the processors.  Assign these to @info[:cpu][:count] and
+  # @info[:cpu][:speed] respectively.
+  #
+  # Returns nothing.
+  def determine_cpu
+    count_command = 'cat /proc/cpuinfo|grep "^processor\\s*: [0-9]"|wc -l'
+    speed_command = 'cat /proc/cpuinfo|grep "MHz"|head -1'
+    cpus = {}
+    cpus[:count] = @shell.query("CPU", count_command).to_i
+    cpus[:speed] = @shell.query("MHZ", speed_command).gsub(/.*: /, '').to_i
+    @info[:cpu] = cpus
+  end
+
+  # Public: Determine the amount of disk space in use on the target host and
+  # set that to @info[:disk].
+  #
+  # Returns nothing.
+  def determine_disk
+    # Use a less accurate (tends to inflate) method if du takes too long
+    df_command = "df 2>/dev/null |awk '$1 ~ /\\// {I = I + $3} END {print I}'"
+    disk = @shell.query("DISK_USED_DF", df_command) 
+
+    # Result is returned as KiB used. We need GB used.
+    @info[:disk] = disk.to_f / 1000 ** 2
+  end
+
+  # Public: Determine the number of public and private IP addressess in use on
+  # the target host and assign that to @info[:ip].
+  #
+  # Returns nothing.
+  def determine_ips
+    ips = {:private => [], :public => []}
+
+    ip_command = %w{/sbin/ifconfig | grep 'inet addr' | egrep -v ':127' | sed
+                    -e 's/.*addr:\([0-9.]*\) .*/\1/'}.join(' ')
+    ifconfig = @shell.query("IP_CONFIG", ip_command)
+
+    ifconfig.each_line do |ip|
+      ip.strip!
+      ips[rfc1918?(ip)] << ip
+    end
+
+    @info[:ip] = ips
+  end
+
+  # Public: Determine amount of historical I/O usage via sysstat and set it to
+  # @info[:io].
+  #
+  # Returns nothing.
+  def determine_io
+    io = {}
+
+    iostat = @shell.query("IOSTAT", "iostat -c | sed -n 4p | awk '{print $4}'")
+    io[:wait] = iostat.to_f
+
+    up = @shell.query("UPTIME", "uptime | sed -e 's/.*up\\([^,]*\\),.*/\\1/'")
+    io[:uptime] = up.chomp
+
+    @info[:io] = io
+  end
+
+  # Public: Determine the web server used on the host, and attempt to enumerate
+  # domain names configured on the server for both non-SSL and SSL; set this
+  # information to @info[:web].  Presently only Apache is supported.
+  #
+  # Returns nothing.
+  def determine_web
+    web = {}
+
+    netstat_command = %w{netstat -ntlp | awk '$4 ~ /:80$/ || $4 ~ /:443$/
+                 {sub(/^[^\/]*\//, ""); print $NF}' | head -1}.join(' ')
+    web[:binary] = @shell.query("WEB_NETSTAT", netstat_command)
+
+    unless web[:binary].empty?
+      version_command = "`which #{web[:binary]}` -v | grep version"
+      web[:version] = @shell.query("WEB_VERSION", version_command)
+      web[:version].gsub!(/.*version: /i, '')
+      binary = web[:binary] == "httpd" ? "apachectl" : "apache2ctl"
+      binary << " -S 2>&1"
+
+      web_command = "#{binary} | grep ')' | grep -vi 'default' | wc -l"
+      hosts = @shell.query("WEB_HTTP", web_command)
+      web[:hosts_http] = hosts.to_i
+
+      ssl_command = "#{binary} | grep ':443'|grep -vi 'default' | wc -l"
+      ssl_hosts = @shell.query("WEB_HTTPS", ssl_command)
+      web[:hosts_https] = ssl_hosts.to_i
+    end
+
+    @info[:web] = web
+  end
+
+  # Public: Determine the amount of disk usage attributable to databases, as
+  # well as database count, and set this to @info[:database].  Currently
+  # supports MySQL.
+  #
+  # Returns nothing.
+  def determine_db
+    db = {}
+    mysql_count = %w{find /var/lib/mysql/* -maxdepth 0 -type d 2>/dev/null |
+                       wc -l}.join(' ')
+    db[:count] = @shell.query("DB_MYSQL_COUNT", mysql_count)
+
+    mysql_size = "du -s /var/lib/mysql 2>/dev/null | awk '{print $1}'"
+    db[:size] = @shell.query("DB_MYSQL_SIZE", mysql_size)
+
+    db[:count] = db[:count].to_i
+    db[:size] = db[:size].to_i
+
+    @info[:db] = db
+  end
+
+  # Public: Gather information about the currently installed libc, ruby, perl,
+  # python and php versions on the host, then set these values to @info[:lib].
+  #
+  # Returns nothing.
+  def determine_lib
+    lib = {}
+
+    libc_command = %w{ls -al `find /lib /usr/lib -name 'libc.so*' | head -1` |
+                     sed 's/.*-> //'}.join(' ')
+    lib[:libc] = @shell.query("LIBC", libc_command)
+    lib[:libc].gsub!(/^.*-|\.so$/, '')
+
+    lib[:perl] = @shell.query("PERL", "perl -e 'print $^V;'")
+    lib[:perl] = lib[:perl].gsub(/^v([0-9.]*).*/, '\1')
+
+    python_command = "python -c 'import sys; print sys.version' 2>/dev/null"
+    lib[:python] = @shell.query("PYTHON", python_command)
+    lib[:python] = lib[:python].gsub(/^([0-9.]*).*/m, '\1')
+
+    ruby_command = "ruby -e 'print RUBY_VERSION' 2>/dev/null"
+    lib[:ruby] = @shell.query("RUBY", ruby_command)
+
+    lib[:php] = @shell.query("PHP_VER", "php -v 2>/dev/null | head -1")
+    lib[:php] = lib[:php].gsub(/^PHP ([0-9.]*).*/, '\1')
+
+    @info[:lib] = lib
+  end
+
+  # Public: Check for the existence of rsync(1) on the host.  Set @info[:rsync]
+  # accordingly.
+  #
+  # Returns nothing.
+  def determine_rsync
+    rsync_command = %w{which rsync 2>/dev/null || ([ -f
+                       /root/.rackspace/rsync ] && printf
+                       '/root/.rackspace/rsync') || printf
+                       'NONE'}.join(' ')
+
+    rsync = @shell.query("RSYNC", rsync_command)
+
+    @info[:rsync] = (rsync =~ /NONE/).nil? ? rsync : false
+  end
+
+  # Public: Check the process listing and store it in 
+  #
+  # Returns nothing.
+  def determine_processes
+    ps_command = 'ps aux'
+
+    processes = @shell.query("PS_LIST", ps_command)
+    @info[:processes] = processes.gsub(/\r/, '').split(/\n/)
+
+    unless @info[:processes].grep(/psa/i).empty?
+      @warnings << "Server likely to be running Plesk"
+    end
+    unless @info[:processes].grep(/cpanellogd/i).empty?
+      @warnings << "Server likely to be running cPanel"
+    end
+  end
+
+  # Public: Simplify access to @info.
+  #
+  # key - Key to check in @info.
+  #
+  # Return value contained in @info by key.
+  def [](key)
+    @info[key]
+  end
+
+  # Public: Allow access to the list of keys extant in @info.
+  #
+  # Return an Array of keys present in @info.
+  def keys
+    @info.keys
+  end
+
+  # Public: Return server information and warnings as a Hash.  Useful for
+  # calling Hash#merge.
+  #
+  # Return info Hash.
+  def to_hash
+    @info.merge({warnings: @warnings})
+  end
+
+  # Internal: Deconstruct the version String provided in order to strip out
+  # extraneous data before and after the version number.  Version number
+  # strings are defined as beginning with a digit, ending with a digit, and
+  # containing nothing but digits and at most one decimal point.
+  #
+  # version - A String containing version number of the OS on the target host.
+  #
+  # Returns a String containing the parsed version string.
+  def version_number(version)
+    if version =~ /\d/
+      version.gsub(/^[^\d]*/, '').gsub(/[^\d]*$/, '').gsub(/(\d*\.\d*).*/, '\1')
+    else
+      "-"
+    end
+  end
+
+  # Internal: Determine whether a given IP resides in a block designated as
+  # private by RFC1918.
+  #
+  # ip - A String containing an IP address.
+  #
+  # Returns :private or :public Symbol based on whether the IP is private.
+  def rfc1918?(ip)
+    octets = ip.split /\./
+    if octets[0] == "10" || (octets[0] == "192" && octets[1] == "168")
+      return :private
+    elsif octets[0] == "172" && octets[1].to_i >=16 && octets[1].to_i <= 31
+      return :private
+    end
+
+    :public
+  end
+end

data/lib/cloudflock/target/servers.rb

@@ -0,0 +1,5 @@
+require 'cloudflock'
+require 'cloudflock/target/servers/profile'
+require 'cloudflock/target/servers/migrate'
+require 'cloudflock/target/servers/platform/v1'
+require 'cloudflock/target/servers/platform/v2'

data/lib/cloudflock/version.rb

@@ -0,0 +1,3 @@
+module CloudFlock
+  VERSION = '0.4.0'
+end

data/lib/cloudflock.rb

@@ -0,0 +1,10 @@
+module CloudFlock
+  module Remote; end
+  module Interface; end
+  module Target;
+    module Servers; end
+  end
+end
+
+require 'cloudflock/version'
+require 'cloudflock/error'

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: cloudflock
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+  prerelease: 
+platform: ruby
+authors:
+- Chris Wuest
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: expectr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cpe
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: CloudFlock is a library and toolchain focused on migration
+email: chris@chriswuest.com
+executables:
+- flock
+- flock-profile
+- flock-servers
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/cloudflock.rb
+- lib/cloudflock/error.rb
+- lib/cloudflock/interface/cli.rb
+- lib/cloudflock/interface/cli/app/common/servers.rb
+- lib/cloudflock/interface/cli/app/servers.rb
+- lib/cloudflock/interface/cli/app/servers/migrate.rb
+- lib/cloudflock/interface/cli/app/servers/profile.rb
+- lib/cloudflock/interface/cli/console.rb
+- lib/cloudflock/interface/cli/opts.rb
+- lib/cloudflock/interface/cli/opts/servers.rb
+- lib/cloudflock/patch/fog.rb
+- lib/cloudflock/remote/ssh.rb
+- lib/cloudflock/target/servers.rb
+- lib/cloudflock/target/servers/data/exceptions/base.txt
+- lib/cloudflock/target/servers/data/exceptions/platform/amazon.txt
+- lib/cloudflock/target/servers/data/exceptions/platform/debian.txt
+- lib/cloudflock/target/servers/data/exceptions/platform/redhat.txt
+- lib/cloudflock/target/servers/data/exceptions/platform/suse.txt
+- lib/cloudflock/target/servers/data/post-migration/chroot/base.txt
+- lib/cloudflock/target/servers/data/post-migration/chroot/platform/amazon.txt
+- lib/cloudflock/target/servers/data/post-migration/pre/base.txt
+- lib/cloudflock/target/servers/data/post-migration/pre/platform/amazon.txt
+- lib/cloudflock/target/servers/migrate.rb
+- lib/cloudflock/target/servers/platform.rb
+- lib/cloudflock/target/servers/platform/v1.rb
+- lib/cloudflock/target/servers/platform/v2.rb
+- lib/cloudflock/target/servers/profile.rb
+- lib/cloudflock/version.rb
+- bin/flock
+- bin/flock-profile
+- bin/flock-servers
+- bin/flock.default
+homepage: http://github.com/cwuest/cloudflock
+licenses:
+- Apache 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Server migration automation
+test_files: []