vm_shepherd 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dd16c5dc4a8f8276b0c59927637e27a635005091
+  data.tar.gz: a16f15c127599cdf280b3fc4fa21cecced662d8d
+SHA512:
+  metadata.gz: 4ef0386504cbf50ec960bccf9e1eead05746441f7e4ff618249e43b0026da0fd6d1c9ffb17166a56ed203cc75b8726bd3a9214d5b0e27cfebc820e84b4e87899
+  data.tar.gz: 58ebe55c5ced17018c1a8666a8e0ff26b502870179c2cba965735ce67117d2bbde06ef17ec286ba1455bb62efea40892d9043daf28a3c7051f630a467f541751

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+2.1.5

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'vsphere_clients', git: 'http://github.com/pivotal-cf-experimental/vsphere_clients'
+
+gemspec

data/README.md

@@ -0,0 +1,31 @@
+# VmShepherd
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'vm_shepherd'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install vm_shepherd
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/pivotal-cf-experimental/vm_shepherd/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+RuboCop::RakeTask.new(:rubocop) do |t|
+  t.options << '--lint'
+  t.options << '--display-cop-names'
+end
+
+task default: [:rubocop, :spec]

data/ci/run_specs.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -ex
+
+echo "-----> Running script: $0"
+
+docker run \
+  --rm=true \
+  --volume=${PWD}:/vm_shepherd \
+  --workdir=/vm_shepherd \
+  ${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME} \
+  /bin/sh -c 'bundle && bundle exec rspec --format documentation'

data/lib/vm_shepherd/ami_manager.rb

@@ -0,0 +1,81 @@
+require 'aws-sdk-v1'
+
+module VmShepherd
+  class AmiManager
+    class RetryLimitExceeded < StandardError
+    end
+
+    AWS_REGION = 'us-east-1'
+    OPS_MANAGER_PRIVATE_IP = '10.0.1.5'
+    OPS_MANAGER_INSTANCE_TYPE = 'm3.medium'
+    RETRY_LIMIT = 60
+    RETRY_INTERVAL = 5
+    DO_NOT_TERMINATE_TAG_KEY = 'do_not_terminate'
+
+    def initialize(aws_options)
+      AWS.config(
+        access_key_id: aws_options.fetch(:aws_access_key),
+        secret_access_key: aws_options.fetch(:aws_secret_key),
+        region: AWS_REGION
+      )
+      @aws_options = aws_options
+    end
+
+    def deploy(ami_file_path)
+      image_id = File.read(ami_file_path).strip
+
+      instance =
+        retry_ignoring_error_until(AWS::EC2::Errors::InvalidIPAddress::InUse) do
+          AWS.ec2.instances.create(
+            image_id: image_id,
+            key_name: aws_options.fetch(:ssh_key_name),
+            security_group_ids: [aws_options.fetch(:security_group_id)],
+            subnet: aws_options.fetch(:public_subnet_id),
+            private_ip_address: OPS_MANAGER_PRIVATE_IP,
+            instance_type: OPS_MANAGER_INSTANCE_TYPE
+          )
+        end
+
+      retry_ignoring_error_until(AWS::EC2::Errors::InvalidInstanceID::NotFound) do
+        instance.status == :running
+      end
+
+      instance.associate_elastic_ip(aws_options.fetch(:elastic_ip_id))
+      instance.add_tag('Name', value: aws_options.fetch(:vm_name))
+    end
+
+    def destroy
+      subnets = [
+        AWS.ec2.subnets[aws_options.fetch(:public_subnet_id)],
+        AWS.ec2.subnets[aws_options.fetch(:private_subnet_id)]
+      ]
+
+      subnets.each do |subnet|
+        subnet.instances.each do |instance|
+          instance.terminate unless instance.tags.to_h.fetch(DO_NOT_TERMINATE_TAG_KEY, false)
+        end
+      end
+    end
+
+    private
+    attr_reader :aws_options
+
+
+    def retry_ignoring_error_until(exception_class, &block)
+      tries = 0
+      condition_reached = false
+      loop do
+        begin
+          tries += 1
+          raise(RetryLimitExceeded) if tries > RETRY_LIMIT
+          condition_reached = block.call
+          sleep RETRY_INTERVAL
+        rescue exception_class
+          retry
+        end
+        break if condition_reached
+      end
+      condition_reached
+    end
+  end
+end

data/lib/vm_shepherd/ova_manager/base.rb

@@ -0,0 +1,31 @@
+require 'rbvmomi'
+
+module VmShepherd
+  module OvaManager
+    class Base
+      attr_reader :vcenter
+
+      def initialize(vcenter)
+        @vcenter = vcenter
+      end
+
+      def find_datacenter(name)
+        match = connection.searchIndex.FindByInventoryPath(inventoryPath: name)
+        return unless match and match.is_a?(RbVmomi::VIM::Datacenter)
+        match
+      end
+
+      private
+
+      def connection
+        @connection ||= RbVmomi::VIM.connect(
+          host: @vcenter.fetch(:host),
+          user: @vcenter.fetch(:user),
+          password: @vcenter.fetch(:password),
+          ssl: true,
+          insecure: true,
+        )
+      end
+    end
+  end
+end

data/lib/vm_shepherd/ova_manager/deployer.rb

@@ -0,0 +1,202 @@
+require 'tmpdir'
+require 'fileutils'
+require 'rbvmomi'
+require 'rbvmomi/utils/deploy'
+require 'vsphere_clients'
+require 'vm_shepherd/ova_manager/base'
+require 'vm_shepherd/ova_manager/open_monkey_patch'
+
+module VmShepherd
+  module OvaManager
+    class Deployer < Base
+      attr_reader :location
+
+      def initialize(vcenter, location)
+        super(vcenter)
+        @location = location
+        raise 'Target folder must be set' unless @location[:folder]
+      end
+
+      def deploy(name_prefix, ova_path, ova_config)
+        ova_path = File.expand_path(ova_path.strip)
+        check_vm_status(ova_config)
+
+        tmp_dir = untar_vbox_ova(ova_path)
+        ovf_path = obtain_ovf_path(tmp_dir)
+
+        deployer = build_deployer(@location)
+        template = deploy_ovf_template(name_prefix, deployer, ovf_path)
+        vm = create_vm_from_template(deployer, template)
+
+        reconfigure_vm(vm, ova_config)
+        power_on_vm(vm)
+      ensure
+        FileUtils.remove_entry_secure(tmp_dir, force: true)
+      end
+
+      private
+
+      def check_vm_status(ova_config)
+        log('Checking for existing VM') do # Bad idea to redeploy VM over existing running VM
+          ip = ova_config[:external_ip] || ova_config[:ip]
+          port = ova_config[:external_port] || 443
+          raise "VM exists at #{ip}" if system("nc -z -w 5 #{ip} #{port}")
+        end
+      end
+
+      def untar_vbox_ova(ova_path)
+        log("Untarring #{ova_path}") do
+          Dir.mktmpdir.tap do |dir|
+            system_or_exit("cd #{dir} && tar xfv '#{ova_path}'")
+          end
+        end
+      end
+
+      def obtain_ovf_path(dir)
+        raise 'Failed to find ovf' unless (file_path = Dir["#{dir}/*.ovf"].first)
+        "file://#{file_path}"
+      end
+
+      def deploy_ovf_template(name_prefix, deployer, ovf_path)
+        log('Uploading template') do
+          deployer.upload_ovf_as_template(
+            ovf_path,
+            Time.new.strftime("#{name_prefix}-%F-%H-%M"),
+            run_without_interruptions: true,
+          )
+        end
+      end
+
+      def create_vm_from_template(deployer, template)
+        log('Cloning template') do
+          deployer.linked_clone(template, "#{template.name}-vm", {
+              :numCPUs => 2,
+              :memoryMB => 2048,
+            })
+        end
+      end
+
+      def reconfigure_vm(vm, ova_config)
+        ip_configuration = {
+          'ip0' => ova_config[:ip],
+          'netmask0' => ova_config[:netmask],
+          'gateway' => ova_config[:gateway],
+          'DNS' => ova_config[:dns],
+          'ntp_servers' => ova_config[:ntp_servers],
+        }
+
+        log("Reconfiguring VM using #{ip_configuration.inspect}") do
+          property_specs = []
+
+          # Order of ip configuration keys must match
+          # order of OVF template properties.
+          ip_configuration.each_with_index do |(key, value), i|
+            property_specs << RbVmomi::VIM::VAppPropertySpec.new.tap do |spec|
+              spec.operation = 'edit'
+              spec.info = RbVmomi::VIM::VAppPropertyInfo.new.tap do |p|
+                p.key = i
+                p.label = key
+                p.value = value
+              end
+            end
+          end
+
+          property_specs << RbVmomi::VIM::VAppPropertySpec.new.tap do |spec|
+            spec.operation = 'edit'
+            spec.info = RbVmomi::VIM::VAppPropertyInfo.new.tap do |p|
+              p.key = ip_configuration.length
+              p.label = 'admin_password'
+              p.value = ova_config[:vm_password]
+            end
+          end
+
+          vm_config_spec = RbVmomi::VIM::VmConfigSpec.new
+          vm_config_spec.ovfEnvironmentTransport = ['com.vmware.guestInfo']
+          vm_config_spec.property = property_specs
+
+          vmachine_spec = RbVmomi::VIM::VirtualMachineConfigSpec.new
+          vmachine_spec.vAppConfig = vm_config_spec
+          vm.ReconfigVM_Task(spec: vmachine_spec).wait_for_completion
+        end
+      end
+
+      def power_on_vm(vm)
+        log('Powering on VM') do
+          vm.PowerOnVM_Task.wait_for_completion
+          wait_for('VM IP') { vm.guest_ip }
+        end
+      end
+
+      def build_deployer(location)
+        unless (datacenter = find_datacenter(location[:datacenter]))
+          raise "Failed to find datacenter '#{location[:datacenter]}'"
+        end
+
+        unless (cluster = datacenter.find_compute_resource(location[:cluster]))
+          raise "Failed to find cluster '#{location[:cluster]}'"
+        end
+
+        unless (datastore = datacenter.find_datastore(location[:datastore]))
+          raise "Failed to find datastore '#{location[:datastore]}'"
+        end
+
+        unless (network = datacenter.networkFolder.traverse(location[:network]))
+          raise "Failed to find network '#{location[:network]}'"
+        end
+
+        resource_pool_name = location[:resource_pool] || location[:resource_pool_name]
+        unless (resource_pool = find_resource_pool(cluster, resource_pool_name))
+          raise "Failed to find resource pool '#{resource_pool_name}'"
+        end
+
+        target_folder = datacenter.vmFolder.traverse(location[:folder], RbVmomi::VIM::Folder, true)
+
+        VsphereClients::CachedOvfDeployer.new(
+          logged_connection,
+          network,
+          cluster,
+          resource_pool,
+          target_folder, # template
+          target_folder, # vm
+          datastore,
+        )
+      end
+
+      def find_resource_pool(cluster, resource_pool_name)
+        if resource_pool_name
+          cluster.resourcePool.resourcePool.find { |rp| rp.name == resource_pool_name }
+        else
+          cluster.resourcePool
+        end
+      end
+
+      def logged_connection
+        log("connecting to #{@vcenter[:user]}@#{@vcenter[:host]}") { connection }
+      end
+
+      def log(title, &blk)
+        puts "--- Running: #{title} @ #{DateTime.now}"
+        blk.call
+      end
+
+      def system_or_exit(*args)
+        puts "--- Running: #{args} @ #{DateTime.now}"
+        system(*args) || fail('FAILED')
+      end
+
+      def wait_for(title, &blk)
+        Timeout.timeout(7*60) do
+          until (value = blk.call)
+            puts '--- Waiting for 30 secs'
+            sleep 30
+          end
+          puts "--- Value obtained for #{title} is #{value}"
+          value
+        end
+      rescue Timeout::Error
+        puts "--- Timed out waiting for #{title}"
+        raise
+      end
+    end
+  end
+end

data/lib/vm_shepherd/ova_manager/destroyer.rb

@@ -0,0 +1,29 @@
+require 'rbvmomi'
+require 'logger'
+require 'vsphere_clients/vm_folder_client'
+require 'vm_shepherd/ova_manager/base'
+
+module VmShepherd
+  module OvaManager
+    class Destroyer < Base
+      def initialize(datacenter_name, vcenter)
+        @datacenter_name = datacenter_name
+        @vcenter = vcenter
+      end
+
+      def clean_folder(folder_name)
+        vm_folder_client.delete_folder(folder_name)
+        vm_folder_client.create_folder(folder_name)
+      end
+
+      private
+
+      def vm_folder_client
+        @vm_folder_client ||= VsphereClients::VmFolderClient.new(
+          find_datacenter(@datacenter_name),
+          Logger.new(STDERR)
+        )
+      end
+    end
+  end
+end

data/lib/vm_shepherd/ova_manager/open_monkey_patch.rb

@@ -0,0 +1,14 @@
+# open is used by RbvMomi when trying to upload OVA file.
+# It does not support local file upload without following patch.
+module Kernel
+  private
+  alias open_without_file open
+  class << self
+    alias open_without_file open
+  end
+
+  def open(name, *rest, &blk)
+    name = name[7..-1] if name.start_with?('file://')
+    open_without_file(name, *rest, &blk)
+  end
+end