vcloud-box-spinner 0.2.0

data/.gitignore

@@ -0,0 +1,8 @@
+*.gem
+.bundle
+pkg/*
+vendor/*
+*.swp
+*~
+\#*
+/Gemfile.lock

data/CHANGELOG

@@ -0,0 +1,8 @@
+0.2.0 2013-08-09
+
+ - Adding delete vApp feature
+ - Changed the syntax for the command line interface
+
+0.1.0 2013-08-07
+
+ - Introducing vcloud-box-spinner gem (Initial Version)

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in vcloud-box-spinner.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2013 singhgarima
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,79 @@
+# vCloud Provisioner
+
+This is a wrapper around the vCloud director API that should allow for easy
+provisioning of VMs.
+
+## Installation
+
+    gem install vcloud-box-spinner
+
+## Usage
+
+You should be able to do `vcloud-box-spinner --help`
+
+    Usage: vcloud-box-spinner [options] <org_config> <machine_config>
+
+    Provision a machine described by the JSON template `machine_config` in the vCloud organisation
+    described in the JSON config file `org_config`
+
+    e.g. vcloud-box-spinner -u username orgs/staging.json machines/frontend-1.json
+
+        -u, --user=USERNAME              vCloud username
+        -p, --password=PASSWORD          vCloud password
+        -F, --ssh-config=FILENAME        SSH config file(s) to use (can be specified multiple times)
+        -s, --setup-script=SETUP-SCRIPT  path to setup script that should run after machine is brought up
+        -d, --debug                      Enable debugging output
+        -v, --verbose                    Enable verbose output
+        -h, --help                       Show usage instructions
+
+To provision a machine you will need to specify at least two JSON files:
+
+  1. A JSON config file which tells the provisioner about the vCloud
+     organisation into which it is to provision a vApp
+  2. A JSON config file which defines the machine-specific setup
+
+Options:
+
+  - `user` is the username on your "vmware vcloud director" page
+    (usually in the top right corner).
+  - `setup-script` allows you to pass a script file path (shell), which
+    would be loaded as guest customization script. The purpose of
+    providing this option, is to let user do some basic bootstraping.
+    The script is not for the purpose of encouraging configuration
+    management and that should be done separately. A particular example
+    of how you can use the script is - You can set ssh configuration for
+    a user(eg ci), which can ssh in the system later and run the config
+    management script/tool.
+    On how to write this script please refer the following links:
+
+      - [Understand Guest OS Customisation](http://pubs.vmware.com/vcd-51/index.jsp?topic=%2Fcom.vmware.vcloud.users.doc_51%2FGUID-BB682E4D-DCD7-4936-A665-0B0FBD6F0EB5.html)
+      - [Example of scripts](http://pubs.vmware.com/vcd-51/index.jsp?topic=%2Fcom.vmware.vcloud.users.doc_51%2FGUID-724EB7B5-5C97-4A2F-897F-B27F1D4226C7.html)
+
+The best way to understand the formats of the json files, read the docs
+[here](/docs/json_formats.md)
+
+Once you have an org and machine config, you can invoke the provisioner as
+follows:
+
+    vcloud-box-spinner -u username -p password org_config.json machine_config.json
+
+## Hacking
+
+refer [here](/docs/hacking.md)
+
+### Testing
+
+You can run the tests with:
+
+    bundle exec rake
+
+## Known Issues
+
+- We are using fog ruby gem, as a wrapper around vCloud API. The fog gem currently hasn't released fix for "case
+  insensitivity for Set-Cookie headers"
+  [here](https://github.com/singhgarima/fog/commit/b7f8db97b15f1dc48541f5f2781f70fb2b743267). The fix till then is,
+  in your Gemfile, add the following lines
+
+        gem 'fog',
+            :git => 'git://github.com/fog/fog.git',
+            :ref => '3f034ccce030cff32e867f57482387d249b4da90'

data/Rakefile

@@ -0,0 +1,16 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+require 'rspec/core/rake_task'
+require 'gem_publisher'
+
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.pattern = FileList['spec/**/*_spec.rb']
+end
+
+task :default => :spec
+
+desc "Publish gem to RubyGems.org"
+task :publish_gem do |t|
+  gem = GemPublisher.publish_if_updated("vcloud-box-spinner.gemspec", :rubygems)
+  puts "Published #{gem}" if gem
+end

data/bin/vcloud-box-spinner

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'provisioner/cli'
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+Excon.defaults[:ssl_verify_peer] = false
+EXCON_DEBUG = true
+
+Provisioner::CLI.new(ARGV).execute

data/docs/hacking.md

@@ -0,0 +1,7 @@
+# Contribute to this project
+
+* To run tests
+
+        bundle exec rake spec
+
+* In case you adding major changes bump up the version file `lib/provisioner/version.rb`

data/docs/json_formats.md

@@ -0,0 +1,65 @@
+# Organisation & Machine JSON files
+
+## Tree structure
+
+    .
+    ├── machines
+    │   └── tester
+    │       ├── machine-1.json
+    │       └── machine-2.json
+    ├── orgs
+        ├── org1.json
+        └── org2.json
+
+## Organisation JSON file format
+
+Each organisation JSON file contains key value pair to represent meta data for
+the same. An example is as follows
+
+    {
+      "default": {
+        "template_name": "<catalog-item-name>",
+        "host": "<api-vendor-endpoint>",
+        "platform": "<platform-name>",
+        "organisation": "<org-name>",
+        "catalog_items": {
+          "<catalog-item-name": "https://api.vcd.portal.skyscapecloud.com/api/catalogItem/<catalog-item-uuid>"
+        }
+      },
+      "<vdc-ref-name/zone>": {
+        "default_vdc": "https://api.vcd.portal.skyscapecloud.com/api/vdc/<vdc-uuid>",
+        "network_name": "<network-name>",
+        "network_uri": "https://api.vcd.portal.skyscapecloud.com/api/network/<network-uuid>",
+        "vdc_id": "<vdc-uuid>"
+      }
+    }
+
+* catalog is a terminology used by vcloud to represent templates used as base
+  images
+* org-name is the name of organisation on which you would be bringing up the
+  machines.
+* vdc-ref-name/zone, is an non vcloud specific term, which we use to map the
+  machines to a particular network. You would see the reference to the value
+  in machine JSON file.
+* default_vdc, is the herf to vdc network.
+* network-name, can be found out from vcloud UI `Adminstration -> Your VDC -> Org VDC Networks -> the network you would use it for`
+
+To find various uuids, please refer [here](/docs/uuids.md)
+
+Note: We would be removing platform from the settings, as it is currently only
+used to set facter variables on the newly set up machine.
+
+## Machine JSON file format
+
+Each machine JSON file contain key value pair to represent machine specific
+meta data
+
+    {
+      "zone":             "<vdc-ref-name/zone>",
+      "vm_name":          "vm-machine-name",
+      "ip":               "<internal-ip-addr>"
+    }
+
+* zone, is the reference to <vdc-ref-name/zone> used in organisation json
+* vm-machine-name, is the name of vApp
+* ip is the internal ip addr that would be used.

data/docs/uuids.md

@@ -0,0 +1,41 @@
+# Finding various UUIDs needed by the JSON files
+
+## Catalog Item UUID
+
+* Authorization:
+
+        curl -v -X POST -d '' -H "Accept: application/*+xml;version=5.1" -u "username@org-name:password" "https://vendor-api-endpoint/api/sessions"
+
+  - The above returns x-vcloud-authorization token
+  - The abpve request returns xml, which contains the reference to the organisation details url, similar to below
+
+            <Link rel="down" type="application/vnd.vmware.vcloud.org+xml" name="org-name" href="https://vendor-api-endpoint/api/org/{org-uuid}"/>
+
+* Get org details:
+
+        curl -v --insecure -H "x-vcloud-authorization: <token>" -H "Accept: application/*+xml;version=5.1" https://vendor-api-endpoint/api/org/{org-uuid}
+
+  - The above returns catalog details. (In vcloud you have a catalog which is a group pf catalogItems i.e templates)
+
+            <Link rel="down" type="application/vnd.vmware.vcloud.catalog+xml" name="Default" href="https://vendor-api-endpoint/api/catalog/{catalog-uuid}"/>
+
+* Get catalog items
+
+        curl -v --insecure -H "x-vcloud-authorization: {token}" -H "Accept: application/*+xml;version=5.1" https://vendor-api-endpoint/api/catalog/{catalog-uuid}
+
+  - The above returns all the catalogItems in the catalog, you can choose the desired one
+
+            <CatalogItem type="application/vnd.vmware.vcloud.catalogItem+xml" name="{catalog-item-name}" href="https://vendor-api-endpoint/api/catalogItem/{catalog-item-uuid}"/>
+
+
+## Network UUID
+
+* Authorization: (same as describe in above steps)
+
+* Get org details:
+
+        curl -v --insecure -H "x-vcloud-authorization: <token>" -H "Accept: application/*+xml;version=5.1" https://vendor-api-endpoint/api/org/{org-uuid}
+
+  - The above returns network details, choose the one you need on basis of name
+
+            <Link rel="down" type="application/vnd.vmware.vcloud.orgNetwork+xml" name="{network-name}" href="https://vendor-api-endpoint/api/network/{network-uuid}"/>

data/jenkins.sh

@@ -0,0 +1,5 @@
+#!/bin/bash -x
+set -e
+bundle install --path "${HOME}/bundles/${JOB_NAME}"
+bundle exec rake spec
+bundle exec rake publish_gem --trace

data/lib/fog/vcloud/compute/server_ready.rb

@@ -0,0 +1,16 @@
+require 'fog/vcloud/models/compute/server'
+module Fog
+  module Vcloud
+    class Compute
+      class Server < Fog::Vcloud::Model
+        def ready?
+          reload_status
+          running_tasks = tasks && tasks.flatten.any? do |task|
+            task.kind_of?(Hash) && (task[:status] == 'running' && task[:Progress] != '100')
+          end
+          status != '0' && !running_tasks
+        end
+      end
+    end
+  end
+end

data/lib/fog/vcloud/compute/shared.rb

@@ -0,0 +1,30 @@
+require 'builder'
+require 'fog/vcloud/compute'
+require 'fog/vcloud/requests/compute/instantiate_vapp_template'
+
+module Fog::Vcloud::Compute::Shared
+  private
+  def generate_instantiate_vapp_template_request(options)
+    xml = ::Builder::XmlMarkup.new
+    xml.InstantiateVAppTemplateParams(xmlns.merge!(:name => options[:name], :"xml:lang" => "en")) {
+      xml.Description(options[:description])
+      xml.InstantiationParams {
+        if options[:network_uri]
+          # TODO - implement properly
+          xml.NetworkConfigSection {
+            xml.ovf :Info
+            xml.NetworkConfig(:networkName => options[:network_name]) {
+              xml.Configuration {
+                xml.ParentNetwork(:href => options[:network_uri])
+                xml.FenceMode 'bridged'
+              }
+            }
+          }
+        end
+      }
+      # The template
+      xml.Source(:href => options[:template_uri])
+      xml.AllEULAsAccepted("true")
+    }
+  end
+end

data/lib/provisioner/blank_provisioner.rb

@@ -0,0 +1,5 @@
+module Provisioner
+  class BlankProvisioner < Provisioner
+    include ComputeNode
+  end
+end

data/lib/provisioner/cli.rb

@@ -0,0 +1,151 @@
+require 'json'
+require 'optparse'
+require 'provisioner/errors'
+require 'highline/import'
+require 'vcloud_box_provisioner'
+
+module Provisioner
+  class CLI
+    def self.defaults
+      {
+        :debug       => false,
+        :log_level   => 5,
+        :memory      => 4096,
+        :num_cores   => 2,
+        :num_servers => 1,
+        :platform    => "production",
+        :ssh_config  => true,         # if not specified, use system defaults
+      }.freeze
+    end
+
+    def self.process(options = {})
+      # The template must specify a zone so we know where to look in the
+      # organisation config
+      begin
+        zone = options[:machine_metadata].fetch(:zone)
+      rescue KeyError
+        raise ConfigurationError, "The machine configuration doesn't specify " +
+          "a zone (Maybe you've put machine metadata and org config in the wrong order?)"
+      end
+
+      # Internal defaults
+      res = self.defaults.dup
+      # vCloud config defaults
+      org_config = options.delete(:org_config)
+      res.merge!(org_config.fetch(:default, {}))
+      # vCloud zone defaults
+      res.merge!(org_config.fetch(zone.to_sym, {}))
+      # Machine metadata options
+      res.merge!(options.delete(:machine_metadata))
+      # Command line options
+      res.merge!(options)
+
+      unless res.include? :catalog_id
+        begin
+          template_name = res.fetch(:template_name)
+          catalog_id = res.fetch(:catalog_items).fetch(template_name.to_sym)
+        rescue KeyError
+          raise ConfigurationError, 'You must specify catalog_id OR (catalog_items AND template_name)'
+        end
+        res[:catalog_id] = catalog_id
+      end
+
+      res
+    end
+
+
+    def initialize( args )
+      @args = args
+    end
+
+    def execute
+      options = {}
+
+      optparser = OptionParser.new do |o|
+
+        o.banner = "Usage: #{File.basename($0)} [options] <action>"
+
+        o.separator ""
+        o.separator "Provision a machine described by the JSON template `machine_metadata` in the vCloud organisation"
+        o.separator "described in the JSON config file `org_config`"
+        o.separator ""
+        o.separator "e.g. vcloud-box-spinner -u johndoe -o orgs/staging.json -m machines/frontend-1.json create"
+        o.separator ""
+        o.separator "[Available actions]:"
+        o.separator "   #{Provisioner::AVAILABLE_ACTIONS.join(', ')}"
+        o.separator ""
+        o.separator "[Available options]:"
+
+        o.on("-u", "--user", "=USERNAME", "vCloud username") do |v|
+          options[:user] = v
+        end
+
+        o.on("-p", "--password", "=PASSWORD", "vCloud password") do |v|
+          options[:password] = v
+        end
+
+        o.on("-F", "--ssh-config", "=FILENAME", "SSH config file(s) to use (can be specified multiple times)") do |v|
+          options[:ssh_config] ||= []
+          options[:ssh_config].push(v)
+        end
+
+        options[:org_config] = {}
+        o.on("-o", "--org-config", "=ORG-CONFIG-JSON",
+             "The organisation configuration json file path") do |v|
+          options[:org_config] = JSON.parse(File.read(v), :symbolize_names => true)
+        end
+
+        options[:machine_metadata] = {}
+        o.on("-m", "--machine-config", "=METADATA",
+             "The machine configuration json file path") do |v|
+          options[:machine_metadata] = JSON.parse(File.read(v), :symbolize_names => true)
+        end
+
+        o.on('-s', '--setup-script', "=SETUP-SCRIPT", "path to setup script that should run after machine is brought up") do |v|
+          options[:setup_script] = v
+        end
+
+        o.on("-d", "--debug", "Enable debugging output") do
+          options[:debug] = true
+        end
+
+        o.on("-v", "--verbose", "Enable verbose output") do
+          options[:log_level] = 0
+        end
+
+        o.on_tail("-h", "--help", "Show usage instructions") do
+          puts o
+          exit
+        end
+      end
+
+      begin
+        optparser.parse!(@args)
+
+        if @args.length != 1
+          raise ConfigurationError, "#{File.basename($0)} takes one argument"
+        end
+
+        action = @args[0]
+
+        if options[:user].nil? then
+          options[:user] = ask("vCloud username: ")
+        end
+
+        if options[:password].nil? then
+          options[:password] = ask("vCloud password: ") { |q| q.echo = false }
+        end
+
+        provisioner_opts = self.class.process(options)
+
+        provisioner = VcloudBoxProvisioner.build provisioner_opts
+        provisioner.execute(action)
+      rescue OptionParser::InvalidArgument, ConfigurationError => e
+        $stderr.puts "Error: #{e}"
+        $stderr.puts
+        $stderr.puts optparser
+        exit 1
+      end
+    end
+  end
+end

data/lib/provisioner/compute_action/create.rb

@@ -0,0 +1,176 @@
+module Provisioner
+  module ComputeAction
+    module Create
+      def user_data_files
+        [File.expand_path(options[:setup_script])]
+      end
+      private :user_data_files
+
+      def user_data
+        user_data_files.map { |f|
+          File.read(f)
+        }.join("\n")
+      end
+      private :user_data
+
+      def provision(name, options)
+        logger.debug "User data for #{name}: #{user_data}"
+        server = compute.servers.create(
+          :vdc_uri            => options[:vdc_id],
+          :catalog_item_uri   => options[:catalog_id],
+          :name               => options[:vm_name],
+          :network_uri        => options[:network_uri],
+          :network_name       => options[:network_name],
+          :user_data          => user_data,
+          :connection_options => {
+            :ssl_verify_peer   => false,
+            :omit_default_port => true
+          }
+        )
+        notify "Waiting for server to come up", name
+        server.wait_for { server.ready? }
+        server
+      end
+      private :provision
+
+      def modify_xml(url, mime_type)
+        connection = compute.servers.service
+        xml = Nokogiri::XML(connection.request(:uri => url).body)
+        yield xml
+        connection.request(:uri => url,
+                           :expects => 202,
+                           :method => 'PUT',
+                           :body => xml.to_s,
+                           :headers => {'Content-Type' => mime_type})
+      end
+      private :modify_xml
+
+      def update_guest_customization_options(server, options)
+        logger.debug "server attributes: #{server.attributes}"
+        customization_options = compute.servers.service.get_customization_options(server.attributes[:children][:href]).body
+        guest_customization_section = customization_options[:GuestCustomizationSection]
+
+        response = modify_xml(guest_customization_section[:href], guest_customization_section[:type]) do |xml|
+          xml.at_css('ComputerName').content = options[:vm_name]
+          if xml.at_css('CustomizationScript').nil?
+            xml.at_css('ComputerName').before("<CustomizationScript>#{CGI.escapeHTML(user_data)}</CustomizationScript>\n")
+          else
+            xml.at_css('CustomizationScript').content = user_data
+          end
+          logger.debug(xml.to_xml)
+        end
+
+        wait_for_task(server, extract_task_uri(response))
+        server.wait_for { server.ready? }
+      end
+      private :update_guest_customization_options
+
+      def update_machine_resources(server, options)
+        update(server, "cpu", options[:num_cores])
+        update(server, "memory", options[:memory])
+
+        server.wait_for { server.ready? }
+      end
+      private :update_machine_resources
+
+      def update(server, resource_type, value)
+        virtual_hardware_section_links = server.attributes[:children]['ovf:VirtualHardwareSection'.to_sym][:Link]
+        edit_link = virtual_hardware_section_links.select { |item|  item[:href].include?(resource_type) && item[:rel] == "edit" }.first
+
+        response = modify_xml(edit_link[:href], edit_link[:type]) do |xml|
+          xml.at_xpath('//rasd:VirtualQuantity').content = value
+        end
+
+        wait_for_task(server, extract_task_uri(response))
+      end
+      private :update
+
+      def extract_task_uri(response)
+        response_xml = Nokogiri::XML(response.body)
+        response_xml.at_css("Task").attributes['href'].value
+      end
+      private :extract_task_uri
+
+      def wait_for_task(server, task_uri)
+        connection = compute.servers.service
+
+        server.wait_for do
+          puts "... "
+          task = Nokogiri::XML(connection.request(:uri => task_uri, :expects => 200).body)
+          task.at_css("Task").attributes['status'].value == 'success'
+        end
+      end
+      private :wait_for_task
+
+      def update_network_connection_options(server, options)
+        network_connection_section = server.attributes[:children][:NetworkConnectionSection][:Link]
+        logger.debug "server attributes: #{server.attributes}"
+        logger.debug "NetworkConnectionSection #{network_connection_section}"
+
+        response = modify_xml(network_connection_section[:href], network_connection_section[:type]) do |xml|
+          if options[:ip]
+            if xml.at_css('IpAddress').nil?
+              xml.at_css('IsConnected').before("<IpAddress>#{options[:ip]}</IpAddress>\n")
+            else
+              xml.at_css('IpAddress').content = options[:ip]
+            end
+            xml.at_css('IpAddressAllocationMode').content = 'MANUAL'
+          else
+            xml.at_css('IpAddressAllocationMode').content = 'POOL'
+          end
+          xml.at_css('NetworkConnection')[:network] = options[:network_name]
+          xml.at_css('IsConnected').content = 'true'
+        end
+
+        wait_for_task(server, extract_task_uri(response))
+        server.wait_for { server.ready? }
+      end
+      private :update_network_connection_options
+
+      def power_on(server)
+        connection = compute.servers.service
+        power_on_uri = server.links.find {|link| link[:rel] == 'power:powerOn' }[:href]
+        connection.request(:uri => power_on_uri, :method => "POST", :expects => 202)
+      end
+      private :power_on
+
+      def wait_for_vms_to_appear(server, options)
+        100.times.each do |x|
+          logger.debug("waiting for vm to spin up...")
+          if server.attributes[:children] && server.attributes[:children][:href]
+            return
+          end
+          sleep 1
+        end
+
+        if !server.attributes[:children] || !server.attributes[:children][:href]
+          abort "vm didn't properly spin up"
+        end
+      end
+      private :wait_for_vms_to_appear
+
+      def launch_server name
+        super
+        server = provision(name, options)
+
+        wait_for_vms_to_appear(server, options)
+
+        update_guest_customization_options(server, options)
+        update_network_connection_options(server, options)
+        update_machine_resources(server, options)
+
+        power_on(server)
+
+        server
+      end
+
+      def launch_servers
+        super
+      end
+
+      def prepare_run
+        super
+      end
+    end
+  end
+end