skytap-yf 0.2.3

data/lib/skytap/plugins/vm_upload.rb

@@ -0,0 +1,401 @@
+module Skytap
+  module Commands
+    class Upload < Skytap::Commands::Base
+      MAX_CONCURRENCY = 5
+      CHECK_PERIOD = 15
+      AT_CAPACITY_RETRY_PERIOD = 15.minutes.to_i
+
+      # What was passed in, either a reference to a VM directory or a VM archive file.
+      attr_reader :vms
+      attr_reader :uploaders
+
+      self.parent = Vm
+      self.plugin = true
+      self.ask_interactively = true
+
+      def self.description
+          <<-"EOF"
+Upload one or more VM directories or VM archive files to Skytap
+
+The file must be one of the following archive types:
+  .tgz .tar.gz .tbz .tbz2 .tar.bz2 .tar .ova .ovf .7z .7z.001 .zip
+
+The contents of the archive file must contain a VMware image in either
+VMX/VMDK or OVF/VMDK format.
+
+See the following page for more information:
+https://cloud.skytap.com/docs/index.php/Importing_and_Exporting_Virtual_Machines#How_to_Import_VMs_in_to_Skytap
+          EOF
+      end
+
+      def expected_args
+        ActiveSupport::OrderedHash[
+          'file*', 'One or more directories or VM archive files to upload'
+        ]
+      end
+
+      def parameters
+        # Only display import parameters if a single VM will be uploaded.
+        if args.length == 1
+          Import.subcommands.detect{|s| s.command_name == 'create'}.spec['params']
+        else
+          {}
+        end
+      end
+
+      def run!
+        @vms = args
+        @uploaders = []
+        params = composed_params
+
+        until finished?
+          if vms.present? && slots_available?
+            kick_off_import(vms.shift, params.dup)
+          else
+            sleep CHECK_PERIOD
+            print_status
+            invoker.try(:call, self)
+
+            signal_available if signal_stale? || (concurrency_at_overage && concurrency < concurrency_at_overage)
+          end
+        end
+
+        signal_available
+        print_summary
+        return response
+      end
+
+      def concurrency_at_overage
+        @concurrency_at_overage
+      end
+
+      def kick_off_import(vm, params)
+        begin
+          # Create import job on server in main thread.
+          job = VmImportJob.new(logger, username, api_token, vm, params)
+
+          # If successful, start FTP upload and subsequent steps in new thread.
+          up = Uploader.new(job)
+          uploaders << up
+          log_line(up.status_line)
+        rescue NoSlotsAvailable => ex
+          vms << vm
+          signal_full
+          log_line(("#{vm}: " << no_capacity_message).color(:yellow))
+        rescue Exception => ex
+          up = DeadUploader.new(vm, ex)
+          uploaders << up
+          log_line(up.status_line)
+        end
+      end
+
+      def no_capacity_message
+        m = AT_CAPACITY_RETRY_PERIOD / 60
+        "No import capacity is currently available on Skytap. Will retry in #{m} minutes".tap do |msg|
+          if active_uploaders.present?
+            msg << ' or when another import completes.'
+          else
+            msg << '.'
+          end
+        end
+      end
+
+      def log_line(msg, include_separator=true)
+        line = msg
+        line += "\n---" if include_separator
+        logger.info line
+      end
+
+      def print_status
+        if (stat = status_lines).present?
+          log_line(stat)
+        end
+      end
+
+      def print_summary
+        unless response.error?
+          logger.info "#{'Summary'.bright}\n#{response.payload}"
+        end
+      end
+
+      def finished?
+        vms.empty? && concurrency == 0
+      end
+
+      def slots_available?
+        estimated_available_slots > 0
+      end
+
+      def signal_stale?
+        full? && Time.now - @full_at > AT_CAPACITY_RETRY_PERIOD
+      end
+
+      def seconds_until_retry
+        return unless full?
+        [0, AT_CAPACITY_RETRY_PERIOD - (Time.now - @full_at)].max
+      end
+
+      def estimated_available_slots
+        if full?
+          0
+        else
+          MAX_CONCURRENCY - concurrency
+        end
+      end
+
+      def signal_full
+        @concurrency_at_overage = concurrency
+        @full_at = Time.now
+      end
+
+      def signal_available
+        @concurrency_at_overage = @full_at = nil
+      end
+
+      def full?
+        @full_at
+      end
+
+      def status_lines
+        active_uploaders.collect(&:status_line).join("\n")
+      end
+
+      def active_uploaders
+        uploaders.reject(&:finished?)
+      end
+
+
+      private
+
+      def concurrency
+        uploaders.select(&:alive?).length
+      end
+
+      def response
+        @_response ||= begin
+                         error = !uploaders.any?(&:success?)
+                         Response.build(uploaders.collect(&:status_line).join("\n"), error)
+                       end
+      end
+    end
+
+    class NoSlotsAvailable < RuntimeError
+    end
+
+    class VmImportJob
+      attr_reader :logger, :import_path, :params, :vm, :username, :api_token, :other_credentials
+
+      def initialize(logger, username, api_token, vm, params = {})
+        @logger = logger
+        @username = username
+        @api_token = api_token
+        @vm = File.expand_path(vm)
+        @params = params
+        @vm_filename = File.basename(@vm)
+
+        create_on_server
+      end
+
+      def create_on_server
+        setup
+        begin
+          import
+        rescue Exception => ex
+          if at_capacity?(ex)
+            raise NoSlotsAvailable.new
+          else
+            raise
+          end
+        end
+      end
+
+      def at_capacity?(exception)
+        exception.message.include?('You cannot import a VM because you may not have more than')
+      end
+
+      def setup
+        if File.directory?(vm)
+          @import_path = File.join(vm, 'vm.7z')
+          unless File.exist?(@import_path)
+            raise Skytap::Error.new("Directory provided (#{vm}) but no vm.7z file found inside")
+          end
+
+          metadata_file = File.join(vm, 'vm.yaml')
+          if File.exist?(metadata_file)
+            #TODO:NLA If hostname is present, truncate it to 15 chars. This is the max import hostname length.
+            extra_params = YAML.load_file(metadata_file)
+          end
+
+        elsif File.exist?(vm)
+          @import_path = vm
+        else
+          raise Skytap::Error.new("File does not exist: #{vm}")
+        end
+
+        @params = (extra_params || {}).merge(params)
+        if @params['credentials'].is_a?(Array)
+          cred = @params['credentials'].shift
+          @other_credentials = @params['credentials']
+          @params['credentials'] = cred
+        end
+      end
+
+      def import(force_reload=false)
+        return @import unless @import.nil? || force_reload
+
+        if @import
+          id = @import['id']
+          @import = Skytap.invoke!(username, api_token, "import show #{id}")
+        else
+          @import = Skytap.invoke!(username, api_token, 'import create', {}, :param => params)
+        end
+      end
+    end
+
+    class DeadUploader
+      def initialize(vm, exception)
+        @vm = vm
+        @exception = exception
+      end
+
+      def finished?
+        true
+      end
+
+      def alive?
+        false
+      end
+
+      def success?
+        false
+      end
+
+      def status_line
+        "VM #{@vm}: Error: #{@exception}"
+      end
+    end
+
+    class Uploader < Thread
+      MAX_WAIT = 2.days
+      IMPORT_CHECK_PERIOD = 5
+
+      attr_reader :job, :bytes_transferred, :bytes_total, :result
+      delegate :logger, :import, :import_path, :params, :vm, :username, :api_token, :other_credentials, :to => :job
+
+      def initialize(job)
+        @job = job
+        @bytes_transferred = @bytes_total = 0
+        path, basename = File.split(File.expand_path(import_path))
+        @vm_filename = File.join(File.basename(path), basename) # E.g., "myfolder/vm.7z"
+        @vm_filename << " (#{params['template_name']})" if params['template_name']
+
+        super do
+          begin
+            run
+          rescue Exception => ex
+            @result = Response.build(ex)
+          end
+        end
+      end
+
+      def run
+        ftp_upload
+        Skytap.invoke!(username, api_token, "import update #{id}", {}, :param => {'status' => 'processing'})
+        wait_until_ready
+        add_other_credentials
+        Skytap.invoke!(username, api_token, "import destroy #{id}")
+        @result = Response.build(import['template_url'])
+      end
+
+
+      def finished?
+        !!@finished
+      end
+
+      def success?
+        result && !result.error?
+      end
+
+      def status_line
+        "#{@vm_filename}: #{status}"
+      end
+
+      def status
+        if result.try(:error?)
+          @finished = true
+          "Error: #{result.error_message}".color(:red).bright
+        elsif result
+          @finished = true
+          "Uploaded to: #{result.payload}".color(:green).bright
+        elsif bytes_transferred == 0
+          'Starting'.color(:yellow)
+        elsif bytes_transferred >= bytes_total
+          'Importing'.color(:yellow)
+        else
+          gb_transferred = bytes_transferred / 1.gigabyte.to_f
+          gb_total = bytes_total / 1.gigabyte.to_f
+          percent_done = 100.0 * bytes_transferred / bytes_total
+          "Uploading #{'%0.1f' % percent_done}% (#{'%0.1f' % gb_transferred} / #{'%0.1f' % gb_total} GB)".color(:yellow)
+        end
+      end
+
+
+      private
+
+
+      def ftp_upload
+        local_path = import_path
+        # FTP URL looks like: ftp://67x8meqCr:a3hnalxZLg@ftp.cloud.skytap.com/upload/
+        remote_dir = import['ftp_url'] =~ %r{@.*?(/.+)$} && $1
+
+        ftp = Net::FTP.new(import['ftp_host'])
+        ftp.login(import['ftp_user_name'], import['ftp_password'])
+        ftp.chdir(remote_dir)
+        @bytes_total = File.size(local_path)
+        ftp.putbinaryfile(local_path) do |data|
+          @bytes_transferred += data.size
+        end
+        ftp.close
+      end
+
+      def add_other_credentials
+        template_id = import['template_url'] =~ /\/templates\/(\d+)/ && $1
+        template = Skytap.invoke!(username, api_token, "template show #{template_id}")
+        vm_id = template['vms'].first['id']
+        (other_credentials || []).each do |cred|
+          Skytap.invoke!(username, api_token, "credential create /vms/#{vm_id}", {}, :param => {'vm_id' => vm_id, 'text' => cred})
+        end
+      end
+
+      def id
+        import['id']
+      end
+
+      def wait_until_ready
+        cutoff = MAX_WAIT.from_now
+        finished = nil
+
+        while Time.now < cutoff
+          case import(true)['status']
+          when 'processing'
+          when 'complete'
+            finished = true
+            break
+          else
+            #TODO:NLA Check that this actually is displayed to the user, for both normal `vm upload` and for `vm copytoregion` too.
+            raise Skytap::Error.new "Import job had unexpected state of #{import['status'].inspect}"
+          end
+
+          sleep IMPORT_CHECK_PERIOD 
+        end
+
+        unless finished
+          raise Skytap::Error.new 'Timed out waiting for import job to complete'
+        end
+      end
+    end
+
+    #TODO:NLA Probably should pull this into a method that also sets e.g., Upload.parent = Vm.
+    Vm.subcommands << Upload
+  end
+end

data/lib/skytap/requester.rb

@@ -0,0 +1,134 @@
+module Skytap
+  class Requester
+    #TODO:NLA Move more of the implementation in request() below into this class.
+    class Response
+      attr_reader :logger, :http_response
+      def initialize(logger, http_response, options = {})
+        @logger = logger
+        @http_response = http_response
+
+        logger.debug 'Response:'.color(:cyan).bright
+        each_header do |k, v|
+          logger.debug "#{k}: #{v}".color(:cyan)
+        end
+        logger.debug "\n#{body}".color(:cyan)
+        logger.debug '---'.color(:cyan).bright
+
+        # Raise unless response is 2XX.
+        @http_response.value if options[:raise]
+      end
+
+      def method_missing(*args, &block)
+        http_response.send(*args, &block)
+      end
+
+      def pretty_body
+        return body if body.blank?
+        if content_type.include?('json')
+          begin
+            JSON.pretty_generate(JSON.load(body))
+          rescue
+            body
+          end
+        else
+          body
+        end
+      end
+    end
+
+    attr_reader :logger, :format
+
+    def initialize(logger, username, api_token, base_url, http_format, verify_certs=true)
+      @logger = logger
+      @username = username or raise Skytap::Error.new 'No username provided'
+      @api_token = api_token or raise Skytap::Error.new 'No API token provided'
+      @base_uri = URI.parse(base_url) or raise Skytap::Error.new 'No base URL provided'
+      @format = "application/#{http_format}" or raise Skytap::Error.new 'No HTTP format provided'
+      @verify_certs = verify_certs
+    end
+
+    # Raises on error code unless :raise => false is passed.
+    def request(method, url, options = {})
+      options = options.symbolize_keys
+      options[:raise] = true unless options.has_key?(:raise)
+
+      with_session do |http|
+        begin
+          #TODO:NLA Move this into method
+          if p = options[:params]
+            if p.is_a?(Hash)
+              p = p.collect {|k,v| CGI.escape(k) + '=' + CGI.escape(v)}.join('&')
+            end
+
+            path, params = url.split('?', 2)
+            if params
+              params << '&'
+            else
+              params = ''
+            end
+            params << p
+            url = [path, params].join('?')
+          end
+
+          body = options[:body] || ''
+          headers = base_headers.merge(options[:headers] || {})
+          logger.debug 'Request:'.color(:cyan).bright
+          logger.debug "#{method} #{url}\n#{headers.collect {|k,v| "#{k}:#{v}"}.join("\n")}\n\n#{body}".color(:cyan)
+          logger.debug '---'.color(:cyan).bright
+
+          response = Response.new(logger, http.send_request(method, url, body, headers),
+                                 :raise => options[:raise])
+        rescue OpenSSL::SSL::SSLError => ex
+          $stderr.puts <<-"EOF"
+An SSL error occurred (probably certificate verification failed).
+
+If you are pointed against an internal test environment, set
+"verify-certs: false" in your ~/.skytaprc file or use --no-verify-certs.
+
+If not, then you may be subject to a man-in-the-middle attack, or the web
+site's SSL certificate may have expired.
+
+The error was: #{ex}
+EOF
+          exit(-1)
+        rescue Net::HTTPServerException => ex
+          logger.debug 'Response:'.color(:cyan).bright
+          logger.info "Code: #{ex.response.code} #{ex.response.message}".color(:red).bright
+          logger.info ex.response.body
+          raise
+        end
+      end
+    end
+
+    def base_headers
+      headers = {
+        'Authorization' => auth_header,
+        'Content-Type' => format,
+        'Accept' => format,
+        'User-Agent' => "SkytapCLI/#{Skytap::VERSION}",
+      }
+    end
+
+    def auth_header
+      "Basic #{Base64.encode64(@username + ":" + @api_token)}".gsub("\n", '')
+    end
+
+    def with_session
+      http = Net::HTTP.new(@base_uri.host, @base_uri.port)
+      http.use_ssl = @base_uri.port == 443 || @base_uri.scheme == 'https'
+
+      if http.use_ssl?
+        # Allow cert-checking to be disabled, since internal test environments
+        # have bad certs.
+        if @verify_certs
+          http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+          http.ca_file = File.join(File.dirname(__FILE__), '..', '..', 'ca-bundle.crt')
+        else
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+      end
+
+      yield http
+    end
+  end
+end

data/lib/skytap/response.rb

@@ -0,0 +1,61 @@
+module Skytap
+  class Response
+    def self.build(result, is_error=nil)
+      case result
+      when self
+        result
+      when Requester::Response
+        is_error ||= result.code !~ /^[123]/
+        if is_error && result.payload.is_a?(Hash)
+          err_msg = "Server error (code #{result.code}): " +
+            (result.payload['error'] ||
+             result.payload['errors'].try(:join, ' ') ||
+             result.body)
+        end
+        new(result.payload, is_error, err_msg)
+      when Skytap::Error
+        new(result, true, result.to_s)
+      when Exception
+        log_exception(result)
+        new(result, true, "Internal error: #{result}")
+      else
+        new(result, is_error)
+      end
+    end
+
+    attr_reader :payload
+
+    def initialize(payload, error = nil, error_message = nil)
+      @payload = payload
+      @error = error
+      if @error
+        @error_message = error_message || @payload.to_s
+      end
+    end
+
+    def error?
+      @error
+    end
+
+    def error_message
+      @error_message.color(:red).bright
+    end
+
+
+    private
+
+
+    def self.log_exception(ex)
+      begin
+        message = "#{Time.now}: -- #{Skytap::VERSION} -- " +
+        "#{ex} (#{ex.class.name})\n#{ex.backtrace.join("\n")}\n"
+
+        File.open(File.expand_path(File.join(ENV['HOME'], '.skytap.log')), 'a') do |f|
+          f << message
+        end
+      rescue
+        # No-op
+      end
+    end
+  end
+end

data/lib/skytap/skytaprc.rb

@@ -0,0 +1,28 @@
+module Skytap
+  module SkytapRC
+    extend self
+
+    RC_FILE = File.expand_path(File.join(ENV['HOME'], '.skytaprc'))
+    RECOGNIZED_OPTIONS = [:'verify-certs', :'base-url', :'api-token', :'http-format', :ask, :'log-level', :username, :colorize]
+
+    def exists?
+      File.exist?(RC_FILE)
+    end
+
+    def load
+      @rc_contents ||= (if File.exist?(RC_FILE)
+                         YAML.load_file(RC_FILE).symbolize_keys
+                       else
+                         {}
+                       end).subset(RECOGNIZED_OPTIONS)
+    end
+
+    def write(hash)
+      @rc_contents = nil
+      hash = (hash || {}).subset(RECOGNIZED_OPTIONS)
+      File.open(RC_FILE, 'w') do |f|
+        f << YAML.dump(hash)
+      end
+    end
+  end
+end

data/lib/skytap/subnet.rb

@@ -0,0 +1,92 @@
+require 'skytap/ip_address'
+
+class Subnet
+  class InvalidSubnet < RuntimeError
+  end
+
+  attr_reader :size, :mask, :address
+
+  def initialize(cidr_block)
+    unless cidr_block =~ /^(.*)\/(.*)/
+      raise InvalidSubnet.new 'Not in CIDR block form (XX.XX.XX.XX/YY)'
+    end
+
+    network = $1
+    begin
+      @size = Integer($2)
+    rescue
+      raise InvalidSubnet.new 'Invalid size'
+    end
+
+    @address = IpAddress.new(network)
+    @mask = size_to_mask(@size)
+  end
+
+  def network_portion
+    @network_portion ||= (mask & address)
+  end
+  alias_method :min, :network_portion
+
+  def ==(other)
+    other.is_a?(Subnet) && \
+    (size == other.size) && \
+    network_portion == other.network_portion
+  end
+
+  def overlaps?(other)
+    min <= other.max && max >= other.min
+  end
+
+  def subsumes?(other)
+    min <= other.min && max >= other.max
+  end
+
+  def strictly_subsumes?(other)
+    subsumes?(other) && self != other
+  end
+
+  def max
+    @max ||= (mask.inverse | network_portion)
+  end
+
+  def each_address
+    (min.to_i..max.to_i).each{|i| yield IpAddress.new(i)}
+  end
+
+  def contains?(ip)
+    ip = IpAddress.new(ip) unless ip.is_a?(IpAddress)
+    (ip & mask) == network_portion
+  end
+
+  def num_addresses
+    2 ** (32-size)
+  end
+
+  def min_machine_ip
+    min + 1
+  end
+
+  def normalized?
+    address == network_portion
+  end
+
+  def normalize
+    Subnet.new("#{network_portion}/#{size}")
+  end
+
+  def to_s
+    "#{address}/#{size}"
+  end
+
+  def <=>(other)
+    to_s <=> other.to_s
+  end
+
+  private
+
+  def size_to_mask(numbits)
+    raise InvalidSubnet.new("Subnet size in bits must be between 0 and 32") unless numbits >= 0 and numbits <= 32
+    IpAddress.new((2**numbits - 1) << (32 - numbits))
+  end
+end
+