vcloud-launcher 0.0.1

data/.gitignore

@@ -0,0 +1,16 @@
+*.gem
+*.rbc
+*.swp
+*.un~
+vcloud_env.sh
+replace_variables.sh
+/.bundle/
+/.ruby-version
+/Gemfile.lock
+/bundle/
+/pkg/
+vendor/
+coverage/
+results.html
+fog_integration_test.config
+.idea

data/Gemfile

@@ -0,0 +1,9 @@
+source 'http://rubygems.org'
+
+gemspec
+
+if ENV['VCLOUD_CORE_DEV_MASTER']
+  gem 'vcloud-core', :git => 'git@github.com:alphagov/vcloud-core.git', :branch => 'master'
+elsif ENV['VCLOUD_CORE_DEV_LOCAL']
+  gem 'vcloud-core', :path => '../vcloud-core'
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 HM Government (Government Digital Service)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,127 @@
+vCloud Launcher
+===============
+A tool that takes a YAML or JSON configuration file describing a vDC, and provisions
+the vApps and VMs contained within.
+
+### Supports
+
+    Configuration of multiple vApps/VMs with:
+        multiple NICs
+        custom CPU and memory size
+        multiple additional disks
+        custom VM metadata
+    Basic idempotent operation - vApps that already exist are skipped.
+
+### Limitations
+
+    Source vApp Template must contain a single VM. This is VMware's recommended 'simple' method of vApp creation. Complex multi-VM vApps are not supported.
+    Org vDC Networks must be precreated.
+    IP addresses are assigned manually (recommended) or via DHCP. VM IP pools are not supported.
+    vCloud has some interesting ideas about the size of potential 'guest customisation scripts' (aka preambles). You may need to use an external minify tool to reduce the size, or speak to your provider to up the limit. 2048 bytes seems to be a practical default maximum.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'vcloud-launcher'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install vcloud-launcher
+
+
+## Usage
+
+vCloud Launcher uses [Fog](http://fog.io/).
+
+To use it you need a .fog file in your home directory.
+
+For example:
+
+test:
+  vcloud_director_username: 'username@org_name'
+  vcloud_director_password: 'password'
+  vcloud_director_host: 'host.api.example.com'
+
+Unfortunately current usage of fog requires the password in this file. Multiple sets of credentials can be specified in the fog file, using the following format:
+
+test:
+  vcloud_director_username: 'username@org_name'
+  vcloud_director_password: 'password'
+  vcloud_director_host: 'host.api.example.com'
+
+test2:
+  vcloud_director_username: 'username@org_name'
+  vcloud_director_password: 'password'
+  vcloud_director_host: 'host.api.vendor.net'
+
+You can then pass the FOG_CREDENTIAL environment variable at the start of your command. The value of the FOG_CREDENTIAL environment variable is the name of the credential set in your fog file which you wish to use. For instance:
+
+FOG_CREDENTIAL=test2 vcloud-launch node.yaml
+
+An example configuration file is located in examples/vcloud-launch
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Other settings
+
+vCloud Launcher uses vCloud Core. If you want to use the latest version of vCloud Core, or a local version, you can export some variables. See the Gemfile for details.
+
+## Testing
+
+Default target: `bundle exec rake`
+Runs the unit and feature tests (pretty quick right now)
+
+* Unit tests only: `bundle exec rake spec`
+* Integration tests ('quick' tests): `bundle exec rake integration:quick`
+* Integration tests (all tests - takes 20mins+): `bundle exec rake integration:all`
+
+You need access to a suitable vCloud Director organization to run the
+integration tests. It is not necessarily safe to run them against an existing
+environment, unless care is taken with the entities being tested.
+
+The easiest thing to do is create a local shell script called
+`vcloud_env.sh` and set the contents:
+
+    export FOG\_CREDENTIAL=test
+    export VCLOUD\_VDC\_NAME="Name of the VDC"
+    export VCLOUD\_CATALOG\_NAME="catalog-name"
+    export VCLOUD\_TEMPLATE\_NAME="name-of-template"
+    export VCLOUD\_NETWORK1\_NAME="name-of-primary-network"
+    export VCLOUD\_NETWORK2\_NAME="name-of-secondary-network"
+    export VCLOUD\_NETWORK1\_IP="ip-on-primary-network"
+    export VCLOUD\_NETWORK2\_IP="ip-on-secondary-network"
+    export VCLOUD\_TEST\_STORAGE\_PROFILE="storage-profile-name"
+    export VCLOUD\_EDGE\_GATEWAY="name-of-edge-gateway-in-vdc"
+
+Then run this before you run the integration tests.
+
+### Specific integration tests
+
+#### Storage profile tests
+
+There is an integration test to check storage profile behaviour, but it requires a lot of set-up so it is not called by the rake task. If you wish to run it you need access to an environment that has two VDCs, each one containing a storage profile with the same name. This named storage profile needs to be different from teh default storage profile.
+
+You will need to set the following environment variables:
+
+      export VDC\_NAME\_1="Name of the first vDC"
+      export VDC\_NAME\_2="Name of the second vDC"
+      export VCLOUD\_CATALOG\_NAME="Catalog name" # Can be the same as above settings if appropriate
+      export VCLOUD\_TEMPLATE\_NAME="Template name" # Can be the same as above setting if appropriate
+      export VCLOUD\_STORAGE\_PROFILE\_NAME="Storage profile name" # This needs to exist in both vDCs
+      export VDC\_1\_STORAGE\_PROFILE\_HREF="Href of the named storage profile in vDC 1"
+      export VDC\_2\_STORAGE\_PROFILE\_HREF="Href of the named storage profile in vDC 2"
+      export DEFAULT\_STORAGE\_PROFILE\_NAME="Default storage profile name"
+      export DEFAULT\_STORAGE\_PROFILE\_HREF="Href of default storage profile"
+
+To run this test: `rspec spec/integration/launcher/storage_profile_integration_test.rb`

data/Rakefile

@@ -0,0 +1,47 @@
+require 'rake/clean'
+require 'rake/testtask'
+require 'cucumber/rake/task'
+require 'rspec/core/rake_task'
+
+require 'vcloud/launcher/version'
+
+include Rake::DSL
+
+Bundler::GemHelper.install_tasks
+
+Rake::TestTask.new do |t|
+  t.pattern = 'test/tc_*.rb'
+end
+
+CUKE_RESULTS = 'results.html'
+CLEAN << CUKE_RESULTS
+
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty --no-source -x"
+  t.fork = false
+end
+
+RSpec::Core::RakeTask.new(:spec) do |task|
+  # Set a bogus Fog credential, otherwise it's possible for the unit
+  # tests to accidentially run (and succeed against!) an actual 
+  # environment, if Fog connection is not stubbed correctly.
+  ENV['FOG_CREDENTIAL'] = 'random_nonsense_owiejfoweijf'
+  task.pattern = FileList['spec/vcloud/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new('integration:quick') do |t|
+  t.rspec_opts = %w(--tag ~take_too_long)
+  t.pattern = FileList['spec/integration/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new('integration:all') do |t|
+  t.pattern = FileList['spec/integration/**/*_spec.rb']
+end
+
+task :default => [:spec,:features]
+
+require "gem_publisher"
+task :publish_gem do |t|
+  gem = GemPublisher.publish_if_updated("vcloud-launcher.gemspec", :rubygems)
+  puts "Published #{gem}" if gem
+end

data/bin/vcloud-launch

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'bundler/setup'
+require 'optparse'
+require 'methadone'
+
+require 'vcloud/launcher/launch'
+
+class App
+  include Methadone::Main
+  include Methadone::CLILogging
+  include Vcloud
+
+  main do |org_config_file|
+    Launch.new.run(org_config_file, options)
+  end
+
+  on("-x", "--dont-power-on", "Do not power on vApps (default is to power on)")
+  on("-c", "--continue-on-error", "Continue on error ( default is false) ")
+
+  arg :org_config_file
+
+  examples_dir = File.absolute_path(
+    File.join(
+      File.dirname(__FILE__),
+      "..",
+      "examples",
+      File.basename(__FILE__),
+  ))
+
+  description "
+  vcloud-launch takes a configuration describing a vCloud Org,
+  and tries to make it a reality.
+
+  See https://github.com/alphagov/vcloud-tools for more info
+
+  Example configuration files can be found in:
+    #{examples_dir}
+  "
+
+  version Vcloud::Launcher::VERSION
+
+  #use_log_level_option
+
+  go!
+end

data/examples/.fog-example.fog

@@ -0,0 +1,15 @@
+p1-production:
+  vcloud_director_username: '<username_from_top_right_of_skyscape_flash_ui>@<org_id_from_url_in_skyscape_flash_ui>'
+  vcloud_director_password: '<your_skyscape_password>'
+  vcloud_director_host: 'vcd.portal.skyscapecloud.com'
+
+# You can extract this information by logging into skyscape portal
+performance-platform-production:
+  vcloud_director_username: '<xxx.x.xxxxxx>@<x-x-xx-xxxxxx>'
+  vcloud_director_password: '<your_skyscape_password>'
+  vcloud_director_host: 'vcd.portal.skyscapecloud.com'
+
+carrenza-preview:
+  vcloud_director_username: '<email>@<org_id>'
+  vcloud_director_password: ''
+  vcloud_director_host: 'myvdc.carrenza.net'

data/examples/vcloud-launch/basic_preamble.erb

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Default preamble
+
+(
+echo "in $*"
+echo "message: <%= vars[:message] -%>"
+) >> /PREAMBLE_OUTPUT

data/examples/vcloud-launch/complete_vapp_config.yaml

@@ -0,0 +1,58 @@
+# 'Complete single vapp' vcloud-launch configuration example
+#
+# Each of the vm: subsections (hardware_config, network_connections, etc) 
+# are optional, if not specified the defaults as set in the vAppTemplate 
+# will be used.
+#
+# Specifically in this example, the instantiated VM will have:
+#
+#  * its memory reconfigured to 4096MB
+#  * its cpu count reconfigured to be 2.
+#  * The Primary NIC will be moved to FrontendNetwork, and given a manual ip
+#    of 192.0.2.10
+#  * A Secondary NIC will be created on BackendNetwork and given an IP address
+#    via DHCP (the result of not supplying an ip_address parameter)
+#  * a bootstrap 'guest customization script' is provided, which will be
+#    interpolated via ERB. 
+#  * Two additional extra_disks are supplied, sized at 10GB and 20GB respectively.
+#    These are in addition to the existing system disk in the templated VM.
+#  * 'vars' are supplied, which will be available to ERB when interpolating
+#  * This output will be run through 'script_post_processor' - generally only 
+#    needed if the interpolated  bootstrap script is 'large' (2048 bytes-ish)
+#  * metadata is supplied which will be applied to the VM and the parent vApp
+#    entities in vCloud Director. This is not available inside the VM itself :(
+#  * A storage_profile is provided. The VM will be moved to this named storage
+#    profile (which must be available in its vDC), if it is different to its 
+#    current SP.
+#
+---
+vapps:
+  - name: 'vapp-example-complete'
+    vdc_name: 'Our vDC 1'
+    catalog: 'our-catalog'
+    catalog_item: 'our-template-vapp'
+    vm:
+      hardware_config:
+        memory: 4096
+        cpu: 2
+      network_connections:
+      - name: 'FrontendNetwork'
+        ip_address: '192.0.2.10'
+      - name: 'BackendNetwork'
+      extra_disks:
+      - name: logs-disk
+        size: 10240
+      - name: data-disk
+        size: 20480
+      bootstrap:
+        script_path: 'scripts/bootstrap_preamble.erb'
+        script_post_processor: 'scripts/minify_bash.py'
+        vars:
+          message: hello world
+          role: webserver
+          environment: example
+      metadata:
+        role: webserver
+        environment: example
+      storage_profile: 'our-updated-storage-profile'
+

data/examples/vcloud-launch/minimal_vapp_config.yaml

@@ -0,0 +1,11 @@
+# Minimum configuration required to instantiate a vApp via vcloud-launch
+# 
+# This will instantiate 'our-vapp-template' into 'vapp-example-1', keeping
+# the settings of any contained VM as-is.
+#
+---
+vapps:
+  - name: 'vapp-example-1'
+    vdc_name: 'vDC Example [1]'
+    catalog: 'example-catalog'
+    catalog_item: 'our-vapp-template'

data/examples/vcloud-launch/multiple_vapps_simple.yaml

@@ -0,0 +1,45 @@
+# Multiple vApps can be defined. Running vcloud-launch will only instantiate
+# vApps that do not already exist, so it is possible to add additional vApps
+# to this configuration, and rerun the tool to add the new vApps.
+#
+# There is a lot of repetition however. YAML anchors can be used to 
+# reduce this - see yaml_anchors_example.yaml for more details.
+# 
+---
+vapps:
+
+- name: vapp-example-1
+  vdc_name: "Our VDC1"
+  catalog: our-catalog
+  catalog_item: our-vapp-template
+  vm:
+    hardware_config:
+      memory: '4096'
+      cpu: '2'
+    network_connections:
+    - name: Default
+      ip_address: 192.0.2.11
+
+- name: vapp-example-2
+  vdc_name: "Our VDC1"
+  catalog: our-catalog
+  catalog_item: our-vapp-template
+  vm:
+    hardware_config:
+      memory: '4096'
+      cpu: '2'
+    network_connections:
+    - name: Default
+      ip_address: 192.0.2.12
+
+- name: vapp-example-3
+  vdc_name: "Our VDC1"
+  catalog: our-catalog
+  catalog_item: our-vapp-template
+  vm:
+    hardware_config:
+      memory: '4096'
+      cpu: '2'
+    network_connections:
+    - name: Default
+      ip_address: 192.0.2.13

data/examples/vcloud-launch/yaml_anchors_example.yaml

@@ -0,0 +1,87 @@
+# A pretty complete example of using YAML Anchors to reduce 
+# repetition when creating multiple vApps.
+# 
+# This example defines 4 vapps: 
+#  * frontend-1
+#  * frontend-2
+#  * backend-1
+#  * backend-2
+#
+---
+anchors:
+
+- &VAPP_TEMPLATE our-vapp-template
+
+- &ENVIRONMENT example
+- &VDC_NAME    example-vdc
+
+- &APP_NETWORK example-app-net
+- &MGMT_NETWORK example-mgmt-net
+
+- &BASE_VAPP
+  vdc_name: *VDC_NAME
+  catalog:  our-catalog
+  catalog_item: *VAPP_TEMPLATE
+
+- &BASE_VM
+  metadata:
+    environment:   *ENVIRONMENT
+    vapp_template: *VAPP_TEMPLATE
+  bootstrap: &BASE_VM_BOOTSTRAP
+    script_path: basic_preamble.erb
+    vars: &BASE_VM_VARS
+      environment: *ENVIRONMENT
+      vapp_template: *VAPP_TEMPLATE
+      vdc: *VDC_NAME
+
+- &SMALL_VM
+  <<: *BASE_VM
+  hardware_config: { memory: 4096, cpu: 2 }
+
+- &MEDIUM_VM
+  <<: *BASE_VM
+  hardware_config: { memory: 8192, cpu: 4 }
+
+vapps:
+
+- &frontend_vapp
+  <<: *BASE_VAPP
+  name: frontend-example-1
+  vm: &frontend_vm
+    <<: *SMALL_VM
+    bootstrap:
+      <<: *BASE_VM_BOOTSTRAP
+      vars:
+        <<: *BASE_VM_VARS
+        role: frontend
+    network_connections:
+    - { name: *APP_NETWORK,  ip_address: 192.0.2.11 }
+    - { name: *MGMT_NETWORK, ip_address: 198.51.100.11 }
+- <<: *frontend_vapp
+  name: frontend-example-2
+  vm:
+    <<: *frontend_vm
+    network_connections:
+    - { name: *APP_NETWORK,  ip_address: 192.0.2.12 }
+    - { name: *MGMT_NETWORK, ip_address: 198.51.100.12 }
+
+- &backend_vapp
+  <<: *BASE_VAPP
+  name: backend-example-1
+  vm: &backend_vm
+    <<: *MEDIUM_VM
+    bootstrap:
+      <<: *BASE_VM_BOOTSTRAP
+      vars:
+        <<: *BASE_VM_VARS
+        role: backend
+    network_connections:
+    - { name: *APP_NETWORK,  ip_address: 192.0.2.51 }
+    - { name: *MGMT_NETWORK, ip_address: 198.51.100.51 }
+- <<: *backend_vapp
+  name: backend-example-2
+  vm:
+    <<: *backend_vm
+    network_connections:
+    - { name: *APP_NETWORK,  ip_address: 192.0.2.52 }
+    - { name: *MGMT_NETWORK, ip_address: 198.51.100.52 }