bosh-bootstrap 0.6.0 → 0.7.0

data/.gitignore

@@ -15,4 +15,5 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
-.DS_Store
+.DS_Store
+.rvmrc

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - rbx-19mode
+  # - ruby-head - generates "Cannot find Syck parser for YAML"

data/ChangeLog.md

@@ -1,5 +1,32 @@
 # Change Log
 
+## v0.7
+
+Notable:
+
+* For existing users: please run "deploy --upgrade-deps" as new inception package (runit) added; and jazor/yaml_command CLIs installed
+* Forces microbosh stemcells 0.8.1 which work with public gems (latest public stemcell does not work with public gems)
+
+Added:
+
+* `mosh` command - connect to Inception VM on trains over flaky internet connections (use instead of `ssh` or `tmux` command) [thx @mrdavidlang]
+* `upgrade-inception` command - to perform an upgrade of the Inception VM without triggering a re-deploy of microbosh.
+
+Changes:
+
+* Inception VM now installs rubygems 2.0.0 & bundler 1.3.0
+* Better idempotence for re-deploying microbosh - will delete&deploy after a failure; will deploy after a deletion.
+* Downloads ubuntu 10.04 ISO to speed up custom stemcell builds
+* Using redcard to ensure ruby 1.9 only
+* `manifest.yml` stores the name of the stemcell created from a custom stemcell build; no longer re-creates stemcell each time
+* git color is enabled on inception VM
+
+Work in progress:
+
+* AWS VPC support was begin by the core BOSH team; though work has stopped sadly.
+* Growing number of specs mostly using Fog.mock! mode; tests being run on travis
+
+
 ## v0.6
 
 Highlights:

data/README.md

@@ -7,7 +7,7 @@ The Stark & Wayne Bosh Bootstrapper is the simplest way to get a Micro BOSH runn
 Bootstrap a Micro BOSH universe from one CLI command. Also allows SSH access and the ability to delete created Micro BOSHes.
 
 ```
-$ bosh-bootstrap deploy --latest-stemcell
+$ bosh-bootstrap deploy
 Creating inception VM...
 Creating micro BOSH VM...
 
@@ -22,11 +22,17 @@ It is now very simple to bootstrap a micro BOSH from a single, local CLI. The bo
 
 To be cute about it, the Stark & Wayne Bosh Bootstrapper aims to provide lifecycle management for the BOSH lifecycle manager. Zing! See the "Deep dive into deploy command" section below for greater understanding why the Stark & Wayne Bosh Bootstrapper is very useful.
 
+[![Build Status](https://travis-ci.org/StarkAndWayne/bosh-bootstrap.png?branch=master)](https://travis-ci.org/StarkAndWayne/bosh-bootstrap)
+
+[![Code Climate](https://codeclimate.com/github/StarkAndWayne/bosh-bootstrap.png)](https://codeclimate.com/github/StarkAndWayne/bosh-bootstrap)
+
 ## Installation
 
-This bootstrapper for BOSH is distributed as a RubyGem for Ruby 1.8+.
+This bootstrapper tool is distributed as a RubyGem for Ruby 1.9+.
 
 ```
+$ ruby -v
+ruby 1.9.3p385 ...
 $ gem install bosh-bootstrap
 ```
 

data/Rakefile

@@ -16,7 +16,7 @@ if defined?(RSpec)
     desc "Run Unit Tests"
     unit_rspec_task = RSpec::Core::RakeTask.new(:unit) do |t|
       t.pattern = "spec/unit/**/*_spec.rb"
-      t.rspec_opts = %w(--format progress --color -d)
+      t.rspec_opts = %w(--format progress --color)
     end
 
     desc "Run Integration Tests"
@@ -29,3 +29,5 @@ if defined?(RSpec)
   desc "Install dependencies and run tests"
   task :spec => %w(spec:unit spec:functional)
 end
+
+task :default => :spec

data/bosh-bootstrap.gemspec

@@ -10,26 +10,30 @@ Gem::Specification.new do |gem|
   gem.email         = ["drnicwilliams@gmail.com"]
   gem.description   = %q{Bootstrap a micro BOSH universe from one CLI}
   gem.summary       = <<-EOS
-Now very simple to bootstrap a micro BOSH from a single, local CLI.
-The bootstrapper first creates an inception VM and then uses
-bosh_deployer (bosh micro deploy) to deploy micro BOSH from
-an available stemcell.
+bosh-bootstrap is a command line tool that you can run on your laptop and
+automatically get a microbosh (and an inception VM) deployed on either
+AWS or OpenStack.
 EOS
   gem.homepage      = "https://github.com/StarkAndWayne/bosh-bootstrap"
 
+  gem.required_ruby_version = '>= 1.9'
+
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  
+
   gem.add_dependency "thor"
   gem.add_dependency "highline"
   gem.add_dependency "settingslogic"
   gem.add_dependency "POpen4"
+  gem.add_dependency "net-ssh", "~> 2.2.2"
   gem.add_dependency "net-scp"
   gem.add_dependency "fog", "~>1.8.0"
   gem.add_dependency "escape"
+  gem.add_dependency "redcard"
   gem.add_dependency "bosh_cli"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec"
+  gem.add_development_dependency "activesupport", ">= 3.0.0"
 end

data/lib/bosh/providers/aws.rb

@@ -24,7 +24,7 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
     end
   end
 
-  # @return [Hash] e.g. { :bits => 0, :cores => 2, :disk => 0, 
+  # @return [Hash] e.g. { :bits => 0, :cores => 2, :disk => 0,
   #   :id => 't1.micro', :name => 'Micro Instance', :ram => 613}
   # or nil if +server_flavor_id+ is not a supported flavor ID
   def fog_compute_flavor(server_flavor_id)
@@ -32,7 +32,7 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
   end
 
   # @return [Array] of [Hash] for each supported compute flavor
-  # Example [Hash] { :bits => 0, :cores => 2, :disk => 0, 
+  # Example [Hash] { :bits => 0, :cores => 2, :disk => 0,
   #   :id => 't1.micro', :name => 'Micro Instance', :ram => 613}
   def aws_compute_flavors
     Fog::Compute::AWS::FLAVORS
@@ -42,10 +42,18 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
     aws_compute_flavors.map { |fl| fl[:id] }
   end
 
+  # Provision an EC2 or VPC elastic IP addess.
+  # * VPC - provision_public_ip_address(vpc: true)
+  # * EC2 - provision_public_ip_address
   # @return [String] provisions a new public IP address in target region
   # TODO nil if none available
-  def provision_public_ip_address
-    address = fog_compute.addresses.create
+  def provision_public_ip_address(options={})
+    if options.delete(:vpc)
+      options[:domain] = "vpc"
+    else
+      options[:domain] = options.delete(:domain) || "standard"
+    end
+    address = fog_compute.addresses.create(options)
     address.public_ip
     # TODO catch error and return nil
   end
@@ -55,8 +63,25 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
     address.server = server
   end
 
+  def create_vpc(name, cidr_block)
+    vpc = fog_compute.vpcs.create(name: name, cidr_block: cidr_block)
+    vpc.id
+  end
+
+  # Creates a VPC subnet
+  # @return [String] the subnet_id
+  def create_subnet(vpc_id, cidr_block)
+    subnet = fog_compute.subnets.create(vpc_id: vpc_id, cidr_block: cidr_block)
+    subnet.subnet_id
+  end
+
+  def create_internet_gateway(vpc_id)
+    gateway = fog_compute.internet_gateways.create(vpc_id: vpc_id)
+    gateway.id
+  end
+
   # Creates or reuses an AWS security group and opens ports.
-  # 
+  #
   # +security_group_name+ is the name to be created or reused
   # +ports+ is a hash of name/port for ports to open, for example:
   # {
@@ -64,6 +89,19 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
   #   http: 80,
   #   https: 443
   # }
+  # protocol defaults to TCP
+  # You can also use a more verbose +ports+ using the format:
+  # {
+  #   ssh: 22,
+  #   http: { ports: (80..82) },
+  #   mosh: { protocol: "udp", ports: (60000..60050) }
+  #   mosh: { protocol: "rdp", ports: (3398..3398), ip_ranges: [ { cidrIp: "196.212.12.34/32" } ] }
+  # }
+  # In this example, 
+  #  * TCP 22 will be opened for ssh from any ip_range,
+  #  * TCP ports 80, 81, 82 for http from any ip_range,
+  #  * UDP 60000 -> 60050 for mosh from any ip_range and
+  #  * TCP 3398 for RDP from ip range: 96.212.12.34/32
   def create_security_group(security_group_name, description, ports)
     unless sg = fog_compute.security_groups.get(security_group_name)
       sg = fog_compute.security_groups.create(name: security_group_name, description: description)
@@ -73,10 +111,11 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
     end
     ip_permissions = sg.ip_permissions
     ports_opened = 0
-    ports.each do |name, port|
-      unless port_open?(ip_permissions, port)
-        sg.authorize_port_range(port..port)
-        puts " -> opened #{name} port #{port}"
+    ports.each do |name, port_defn|
+      (protocol, port_range, ip_range) = extract_port_definition(port_defn)
+      unless port_open?(ip_permissions, port_range, protocol, ip_range)
+        sg.authorize_port_range(port_range, {:ip_protocol => protocol, :cidr_ip => ip_range})
+        puts " -> opened #{name} ports #{protocol.upcase} #{port_range.min}..#{port_range.max} from IP range #{ip_range}"
         ports_opened += 1
       end
     end
@@ -84,8 +123,40 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
     true
   end
 
-  def port_open?(ip_permissions, port)
-    ip_permissions && ip_permissions.find {|ip| ip["fromPort"] <= port && ip["toPort"] >= port }
+  # Any of the following +port_defn+ can be used:
+  # {
+  #   ssh: 22,
+  #   http: { ports: (80..82) },
+  #   mosh: { protocol: "udp", ports: (60000..60050) }
+  #   mosh: { protocol: "rdp", ports: (3398..3398), ip_range: "196.212.12.34/32" }
+  # }
+  # In this example, 
+  #  * TCP 22 will be opened for ssh from any ip_range,
+  #  * TCP ports 80, 81, 82 for http from any ip_range,
+  #  * UDP 60000 -> 60050 for mosh from any ip_range and
+  #  * TCP 3398 for RDP from ip range: 96.212.12.34/32
+  def extract_port_definition(port_defn)
+    protocol = "tcp"
+    ip_range = "0.0.0.0/0"
+    if port_defn.is_a? Integer
+      port_range = (port_defn..port_defn)
+    elsif port_defn.is_a? Range
+      port_range = port_defn
+    elsif port_defn.is_a? Hash
+      protocol = port_defn[:protocol] if port_defn[:protocol]
+      port_range = port_defn[:ports]  if port_defn[:ports]
+      ip_range = port_defn[:ip_range] if port_defn[:ip_range]
+    end
+    [protocol, port_range, ip_range]
+  end
+
+  def port_open?(ip_permissions, port_range, protocol, ip_range)
+    ip_permissions && ip_permissions.find do |ip| 
+      ip["ipProtocol"] == protocol \
+      && ip["ipRanges"].detect { |range| range["cidrIp"] == ip_range } \
+      && ip["fromPort"] <= port_range.min \
+      && ip["toPort"] >= port_range.max
+    end
   end
 
   def find_server_device(server, device)
@@ -109,4 +180,43 @@ class Bosh::Providers::AWS < Bosh::Providers::BaseProvider
     # Instead, using:
     volume.server = server
   end
+
+  def bootstrap(new_attributes = {})
+    vpc = new_attributes[:subnet_id]
+
+    server = fog_compute.servers.new(new_attributes)
+
+    unless new_attributes[:key_name]
+      # first or create fog_#{credential} keypair
+      name = Fog.respond_to?(:credential) && Fog.credential || :default
+      unless server.key_pair = fog_compute.key_pairs.get("fog_#{name}")
+        server.key_pair = fog_compute.key_pairs.create(
+          :name => "fog_#{name}",
+          :public_key => server.public_key
+        )
+      end
+    end
+
+    if vpc
+      # TODO setup security group on new server
+    else
+      # make sure port 22 is open in the first security group
+      security_group = fog_compute.security_groups.get(server.groups.first)
+      authorized = security_group.ip_permissions.detect do |ip_permission|
+        ip_permission['ipRanges'].first && ip_permission['ipRanges'].first['cidrIp'] == '0.0.0.0/0' &&
+        ip_permission['fromPort'] == 22 &&
+        ip_permission['ipProtocol'] == 'tcp' &&
+        ip_permission['toPort'] == 22
+      end
+      unless authorized
+        security_group.authorize_port_range(22..22)
+      end
+    end
+
+    server.save
+    server.wait_for { ready? }
+    server.setup(:key_data => [server.private_key])
+    server
+  end
+
 end

data/lib/bosh/providers/openstack.rb

@@ -7,7 +7,7 @@ require "bosh/providers/base_provider"
 class Bosh::Providers::OpenStack < Bosh::Providers::BaseProvider
   # @return [String] provisions a new public IP address in target region
   # TODO nil if none available
-  def provision_public_ip_address
+  def provision_public_ip_address(options={})
     address = fog_compute.addresses.create
     address.ip
     # TODO catch error and return nil

data/lib/bosh-bootstrap/cli.rb

@@ -15,6 +15,7 @@ module Bosh::Bootstrap
     include Thor::Actions
     include Bosh::Bootstrap::Helpers::FogSetup
     include Bosh::Bootstrap::Helpers::Settings
+    include Bosh::Bootstrap::Helpers::SettingsSetter
     include FileUtils
 
     attr_reader :fog_credentials
@@ -39,11 +40,20 @@ module Bosh::Bootstrap
       deploy_stage_6_setup_new_bosh
     end
 
+    desc "upgrade-inception", "Upgrade inception VM with latest packages, gems, security group ports"
+    method_option :"edge-deployer", :type => :boolean, :desc => "Install bosh deployer from git instead of rubygems"
+    def upgrade_inception
+      load_deploy_options # from method_options above
+
+      setup_server
+      upgrade_inception_stage_1_prepare_inception_vm
+    end
+
     # desc "delete", "Delete Micro BOSH"
     # method_option :all, :type => :boolean, :desc => "Delete all micro-boshes and inception VM [coming soon]"
     # def delete
     #   delete_stage_1_target_inception_vm
-    # 
+    #
     #   if options[:all]
     #     error "I'm sorry; the awesome --all flag is not yet implemented"
     #     delete_all_stage_2_delete_micro_boshes
@@ -52,7 +62,7 @@ module Bosh::Bootstrap
     #     delete_one_stage_2_delete_micro_bosh
     #   end
     # end
-    # 
+    #
     desc "ssh [COMMAND]", "Open an ssh session to the inception VM [do nothing if local machine is inception VM]"
     long_desc <<-DESC
       If a command is supplied, it will be run, otherwise a session will be
@@ -64,13 +74,22 @@ module Bosh::Bootstrap
 
     desc "tmux", "Open an ssh (with tmux) session to the inception VM [do nothing if local machine is inception VM]"
     long_desc <<-DESC
-      Opens a connection using ssh and attaches to the most recent tmux session; 
+      Opens a connection using ssh and attaches to the most recent tmux session;
       giving you persistance across disconnects.
     DESC
     def tmux
       run_ssh_command_or_open_tunnel(["-t", "tmux attach || tmux new-session"])
     end
 
+    desc "mosh", "Open an mosh session to the inception VM [do nothing if local machine is inception VM]"
+    long_desc <<-DESC
+      Opens a connection using MOSH (http://mosh.mit.edu/); ideal for those with slow or flakey internet connections.
+      Requires outgoing UDP port 60001 to the Inception VM
+    DESC
+    def mosh
+      open_mosh_session
+    end
+
     no_tasks do
       DEFAULT_INCEPTION_VOLUME_SIZE = 32 # Gb
 
@@ -79,12 +98,25 @@ module Bosh::Bootstrap
         unless settings[:fog_credentials]
           choose_fog_provider
         end
+
+        unless settings[:bosh_cloud_properties]
+          build_cloud_properties
+        end
         confirm "Using infrastructure provider #{settings.fog_credentials.provider}"
 
+        if aws?
+          if ENV['VPC']
+            choose_aws_vpc_or_ec2
+          end
+        end
+
         unless settings[:region_code]
           choose_provider_region
         end
-        if settings[:region_code]
+        if region = settings[:region_code]
+          settings["fog_credentials"]["region"] = region
+          settings["bosh_cloud_properties"][settings["bosh_provider"]]["region"] = region
+          settings["bosh_cloud_properties"][settings["bosh_provider"]]["ec2_endpoint"] = "ec2.#{region}.amazonaws.com"
           confirm "Using #{settings.fog_credentials.provider} region #{settings.region_code}"
         else
           confirm "No specific region/data center for #{settings.fog_credentials.provider}"
@@ -97,7 +129,7 @@ module Bosh::Bootstrap
           confirm "Using #{settings.fog_credentials.provider} network labelled #{settings['network_label']}"
         end
       end
-      
+
       def deploy_stage_2_bosh_configuration
         header "Stage 2: BOSH configuration"
         unless settings[:bosh_name]
@@ -134,9 +166,13 @@ module Bosh::Bootstrap
         end
 
         unless settings[:bosh]["ip_address"]
-          say "Acquiring IP address for micro BOSH..."
-          ip_address = acquire_ip_address
-          settings[:bosh]["ip_address"] = ip_address
+          if vpc?
+            settings[:bosh]["ip_address"] = "10.0.0.6"
+          else
+            say "Acquiring IP address for micro BOSH..."
+            ip_address = acquire_ip_address
+            settings[:bosh]["ip_address"] = ip_address
+          end
         end
         unless settings[:bosh]["ip_address"]
           error "IP address not available/provided currently"
@@ -145,6 +181,10 @@ module Bosh::Bootstrap
         end
         save_settings!
 
+        if aws? && vpc?
+          create_complete_vpc(settings.bosh_name, "10.0.0.0/16", "10.0.0.0/24")
+        end
+
         unless settings[:bosh_security_group]
           security_group_name = settings.bosh_name
           create_security_group(security_group_name)
@@ -169,12 +209,12 @@ module Bosh::Bootstrap
 
       def deploy_stage_3_create_allocate_inception_vm
         header "Stage 3: Create/Allocate the Inception VM"
-        unless settings["inception"] && settings["inception"]["host"]
+        unless settings["inception"]
           hl.choose do |menu|
             menu.prompt = "Create or specify an Inception VM:  "
             if aws? || openstack?
               menu.choice("create new inception VM") do
-                aws? ? boot_aws_inception_vm : boot_openstack_inception_vm
+                settings["inception"] = {"create_new" => true}
               end
             end
             menu.choice("use an existing Ubuntu server") do
@@ -191,18 +231,18 @@ module Bosh::Bootstrap
             end
           end
         end
-        # If successfully validate inception VM, then save those settings.
         save_settings!
 
-        if settings["inception"]["host"]
-          @server = Commander::RemoteServer.new(settings.inception.host, settings.local.private_key_path)
-          confirm "Using inception VM #{settings.inception.username}@#{settings.inception.host}"
-        else
-          @server = Commander::LocalServer.new
-          confirm "Using this server as the inception VM"
+        if settings["inception"]["create_new"] && !settings["inception"]["host"]
+          aws? ? boot_aws_inception_vm : boot_openstack_inception_vm
         end
+        # If successfully validate inception VM, then save those settings.
+        save_settings!
+
+        setup_server
+
         unless settings["inception"]["validated"]
-          unless server.run(Bosh::Bootstrap::Stages::StageValidateInceptionVm.new(settings).commands)
+          unless run_server(Bosh::Bootstrap::Stages::StageValidateInceptionVm.new(settings).commands)
             error "Failed to complete Stage 3: Create/Allocate the Inception VM"
           end
           settings["inception"]["validated"] = true
@@ -255,42 +295,72 @@ module Bosh::Bootstrap
         confirm "You are now targeting and logged in to your BOSH"
       end
 
-      def delete_stage_1_target_inception_vm
-        header "Stage 1: Target inception VM to use to delete micro-bosh"
-        if settings["inception"]["host"]
-          @server = Commander::RemoteServer.new(settings.inception.host, settings.local.private_key_path)
-          confirm "Using inception VM #{settings.inception.username}@#{settings.inception.host}"
+      def upgrade_inception_stage_1_prepare_inception_vm
+        if settings["inception"] && settings["inception"]["prepared"]
+          header "Stage 1: Upgrade Inception VM"
+          unless run_server(Bosh::Bootstrap::Stages::StagePrepareInceptionVm.new(settings).commands)
+            error "Failed to complete Stage 2: Upgrade Inception VM"
+          end
         else
-          @server = Commander::LocalServer.new
-          confirm "Using this server as the inception VM"
+          error "Please deploy an Inception VM first, using 'bosh-bootstrap deploy' command."
         end
       end
 
+      def delete_stage_1_target_inception_vm
+        header "Stage 1: Target inception VM to use to delete micro-bosh"
+        setup_server
+      end
+
       def delete_one_stage_2_delete_micro_bosh
         header "Stage 2: Deleting micro BOSH"
-        unless server.run(Bosh::Bootstrap::Stages::MicroBoshDelete.new(settings).commands)
+        unless run_server(Bosh::Bootstrap::Stages::MicroBoshDelete.new(settings).commands)
           error "Failed to complete Stage 1: Delete micro BOSH"
         end
         save_settings!
       end
 
       def delete_all_stage_2_delete_micro_boshes
-        
+
       end
 
       def delete_all_stage_3_delete_inception_vm
-        
+
       end
 
-      def run_ssh_command_or_open_tunnel(cmd)
-        unless settings[:inception]
-          say "No inception VM being used", :yellow
-          exit 0
+      def setup_server
+        if settings["inception"]["host"]
+          @server = Commander::RemoteServer.new(settings.inception.host, settings.local.private_key_path)
+          confirm "Using inception VM #{settings.inception.username}@#{settings.inception.host}"
+        else
+          @server = Commander::LocalServer.new
+          confirm "Using this server as the inception VM"
         end
-        unless host = settings.inception[:host]
-          exit "Inception VM has not finished launching; run to complete: #{self.class.banner_base} deploy"
+      end
+
+      def create_complete_vpc(name, vpc_range="10.0.0.0/16", subnet_cidr_block="10.0.0.0/24")
+        with_setting "vpc" do |setting|
+          say "Creating VPC '#{name}'..."
+          setting["id"] = provider.create_vpc(name, vpc_range)
+        end
+
+        vpc_id = settings["vpc"]["id"]
+        with_setting "internet_gateway" do |setting|
+          say "Creating internet gateway..."
+          setting["id"] = provider.create_internet_gateway(vpc_id)
         end
+
+        with_setting "subnet" do |setting|
+          say "Creating subnet #{subnet_cidr_block}..."
+          setting["id"] = provider.create_subnet(vpc_id, subnet_cidr_block)
+        end
+      end
+
+      def run_ssh_command_or_open_tunnel(cmd)
+        ensure_inception_vm
+        ensure_inception_vm_has_launched
+
         username = 'vcap'
+        host = settings.inception[:host]
         result = system Escape.shell_command(['ssh', "#{username}@#{host}", cmd].flatten.compact)
         exit result
 
@@ -302,6 +372,59 @@ module Bosh::Bootstrap
         # Warning: Identity file  /Users/drnic/.ssh/id_rsa not accessible: No such file or directory.
       end
 
+      def ensure_inception_vm
+        unless settings[:inception]
+          say "No inception VM being used", :yellow
+          exit 0
+        end
+      end
+      def ensure_inception_vm_has_launched
+        unless settings.inception[:host]
+          exit "Inception VM has not finished launching; run to complete: #{self.class.banner_base} deploy"
+        end
+      end
+
+      def open_mosh_session
+        ensure_mosh_installed
+        ensure_inception_vm
+        ensure_inception_vm_has_launched
+        ensure_security_group_allows_mosh
+
+        username = 'vcap'
+        host = settings.inception[:host]
+        exit system Escape.shell_command(['mosh', "#{username}@#{host}"])
+      end
+
+      def ensure_mosh_installed
+        system 'mosh --version'
+        unless $?.exitstatus == 255 #mosh --version returns exit code 255, rather than 0 as one might expect.  Grrr.
+          say "You must have MOSH installed to use this command.  See http://mosh.mit.edu/#getting", :yellow
+          exit 0
+        end
+      end
+
+      def ensure_security_group_allows_mosh
+        ports = {
+          mosh: {
+            protocol: "udp",
+            ports: (60000..60050)
+          }
+        }
+        inception_server = fog_compute.servers.get(settings["inception"]["server_id"])
+        security_group_name = inception_server.groups.first
+
+        say "Ensuring #{ports[:mosh][:protocol]} ports #{ports[:mosh][:ports].to_s} are open", [:yellow, :bold]
+        say "on Inception VM's security group (#{security_group_name}) ...", [:yellow, :bold]
+
+        #TODO - remove this guard once the other providers have been extended
+        unless settings['bosh_provider'] == 'aws'
+          say "TODO: Non-AWS providers need to be extended to allow creation of UDP ports (60000..60050) in their security groups", :yellow
+          exit 0
+        end
+
+        provider.create_security_group(security_group_name, 'not used', ports)
+      end
+
       # Display header for a new section of the bootstrapper
       def header(title, options={})
         say "" # golden whitespace
@@ -327,6 +450,10 @@ module Bosh::Bootstrap
       def load_deploy_options
         settings["fog_path"] = File.expand_path(options[:fog] || "~/.fog")
 
+        settings["git"] ||= {}
+        settings["git"]["name"] ||= `git config user.name`.strip
+        settings["git"]["email"] ||= `git config user.email`.strip
+
         settings["bosh_git_source"] = options[:"edge-deployer"] # use bosh git repo instead of rubygems
 
         # determine which micro-bosh stemcell to download/create
@@ -459,6 +586,10 @@ module Bosh::Bootstrap
         @fog_credentials.each do |key, value|
           settings[:fog_credentials][key] = value
         end
+        save_settings!
+      end
+
+      def build_cloud_properties
         setup_bosh_cloud_properties
         settings[:bosh_provider] = settings.bosh_cloud_properties.keys.first # aws, vsphere...
         save_settings!
@@ -506,7 +637,7 @@ module Bosh::Bootstrap
           props[:access_key_id] = settings.fog_credentials.aws_access_key_id
           props[:secret_access_key] = settings.fog_credentials.aws_secret_access_key
           # props[:ec2_endpoint] = "ec2.REGION.amazonaws.com" - via +choose_aws_region+
-          # props[:region] = REGION - via +choose_aws_region+            
+          # props[:region] = REGION - via +choose_aws_region+
           # props[:default_key_name] = "microbosh"  - via +create_aws_key_pair+
           # props[:ec2_private_key] = "/home/vcap/.ssh/microbosh.pem" - via +create_aws_key_pair+
           # props[:default_security_groups] = ["microbosh"], - via +create_aws_security_group+
@@ -561,6 +692,16 @@ module Bosh::Bootstrap
         end
       end
 
+      def choose_aws_vpc_or_ec2
+        if settings["use_vpc"].nil?
+          settings["use_vpc"] = begin
+            answer = hl.ask("You want to use VPC, right? ") {|q| q.default="yes"; q.validate = /(yes|no)/i }.match(/y/)
+            !!answer
+          end
+          save_settings!
+        end
+      end
+
       def choose_aws_region
         aws_regions = provider.region_labels
         default_aws_region = provider.default_region_label
@@ -570,9 +711,6 @@ module Bosh::Bootstrap
           aws_regions.each do |region|
             menu.choice(region) do
               settings["region_code"] = region
-              settings["fog_credentials"]["region"] = region
-              settings["bosh_cloud_properties"]["aws"]["region"] = region
-              settings["bosh_cloud_properties"]["aws"]["ec2_endpoint"] = "ec2.#{region}.amazonaws.com"
               save_settings!
             end
             menu.default = default_aws_region
@@ -682,18 +820,32 @@ module Bosh::Bootstrap
       def boot_aws_inception_vm
         say "" # glowing whitespace
 
+        unless settings["inception"]["ip_address"]
+          say "Provisioning IP address for inception VM..."
+          settings["inception"]["ip_address"] = acquire_ip_address
+          save_settings!
+        end
+
         public_key_path, private_key_path = local_ssh_key_paths
         unless settings["inception"] && settings["inception"]["server_id"]
           username = "ubuntu"
           size = "m1.small"
+          ip_address = settings["inception"]["ip_address"]
           say "Provisioning #{size} for inception VM..."
-          server = fog_compute.servers.bootstrap({
+          inception_vm_attributes = {
             :public_key_path => public_key_path,
             :private_key_path => private_key_path,
             :flavor_id => size,
             :bits => 64,
-            :username => "ubuntu"
-          })
+            :username => "ubuntu",
+            :public_ip_address => ip_address
+          }
+          if vpc?
+            raise "must create subnet before creating VPC inception VM" unless settings["subnet"] && settings["subnet"]["id"]
+            inception_vm_attributes[:subnet_id] = settings["subnet"]["id"]
+            inception_vm_attributes[:private_ip_address] = "10.0.0.5"
+          end
+          server = provider.bootstrap(inception_vm_attributes)
           unless server
             error "Something mysteriously cloudy happened and fog could not provision a VM. Please check your limits."
           end
@@ -706,16 +858,6 @@ module Bosh::Bootstrap
 
         server ||= fog_compute.servers.get(settings["inception"]["server_id"])
 
-        unless settings["inception"]["ip_address"]
-          say "Provisioning IP address for inception VM..."
-          ip_address = acquire_ip_address
-          associate_ip_address_with_server(ip_address, server)
-          host = server.dns_name
-
-          settings["inception"]["ip_address"] = ip_address
-          save_settings!
-        end
-
         unless settings["inception"]["disk_size"]
           disk_size = DEFAULT_INCEPTION_VOLUME_SIZE # Gb
           device = "/dev/sdi"
@@ -884,7 +1026,7 @@ module Bosh::Bootstrap
       # For AWS, it will dynamically provision an elastic IP
       # For OpenStack, it will dynamically provision a floating IP
       def acquire_ip_address
-        unless public_ip = provider.provision_public_ip_address
+        unless public_ip = provider.provision_public_ip_address(vpc: vpc?)
           say "Unable to acquire a public IP. Please check your account for capacity or service issues.".red
           exit 1
         end
@@ -908,14 +1050,26 @@ module Bosh::Bootstrap
 
         # Format and mount the volume
         say "Mounting persistent disk as volume on inception VM..."
-        disk_mounted = false
-        until disk_mounted
+        run_ssh_command_until_successful server, "sudo mkfs.ext4 #{device} -F"
+        run_ssh_command_until_successful server, "sudo mkdir -p /var/vcap/store"
+        run_ssh_command_until_successful server, "sudo mount #{device} /var/vcap/store"
+      end
+
+      def run_ssh_command_until_successful(server, cmd)
+        completed = false
+        until completed
           begin
-            # TODO catch Errno::ETIMEDOUT and re-run ssh commands
-            server.ssh(["sudo mkfs.ext4 #{device} -F"]) 
-            server.ssh(["sudo mkdir -p /var/vcap/store"])
-            server.ssh(["sudo mount #{device} /var/vcap/store"])
-            disk_mounted = true
+            say "Running on inception VM: #{cmd}"
+            result = server.ssh([cmd]).first
+            if result.status == 1
+              result.display_stdout
+              result.display_stderr
+              sleep 1
+              say "trying again..."
+              next
+            else
+            end
+            completed = true
           rescue Errno::ETIMEDOUT => e
             say "Timeout error/warning mounting volume, retrying...", yellow
           end
@@ -953,6 +1107,10 @@ module Bosh::Bootstrap
         settings.fog_credentials.provider == "AWS"
       end
 
+      def vpc?
+        settings["use_vpc"]
+      end
+
       def openstack?
         settings.fog_credentials.provider == "OpenStack"
       end
@@ -967,35 +1125,45 @@ module Bosh::Bootstrap
 
       # Returns the latest micro-bosh stemcell
       # for the target provider (aws, vsphere, openstack)
+      # The name includes the version number.
       def micro_bosh_stemcell_name
-        @micro_bosh_stemcell_name ||= begin
-          stemcell_filter_tags = ['micro', provider_name]
-          if settings["micro_bosh_stemcell_type"] == "stable"
-            unless openstack?
-              # FIXME remove this if when openstack has its first stable
-              stemcell_filter_tags << "stable" # latest stable micro-bosh stemcell by default
-            end
+        @micro_bosh_stemcell_name ||= "micro-bosh-stemcell-#{provider_name}-#{known_stable_micro_bosh_stemcell_version}.tgz"
+      end
+
+      def known_stable_micro_bosh_stemcell_version
+        "0.8.1"
+      end
+
+      def latest_micro_bosh_stemcell_name
+        stemcell_filter_tags = ['micro', provider_name]
+        if settings["micro_bosh_stemcell_type"] == "stable"
+          unless openstack?
+            # FIXME remove this if when openstack has its first stable
+            stemcell_filter_tags << "stable" # latest stable micro-bosh stemcell by default
           end
-          tags = stemcell_filter_tags.join(",")
-          bosh_stemcells_cmd = "bosh public stemcells --tags #{tags}"
-          say "Locating micro-bosh stemcell, running '#{bosh_stemcells_cmd}'..."
-          #
-          # The +bosh_stemcells_cmd+ has an output that looks like:
-          # +-----------------------------------+--------------------+
-          # | Name                              | Tags               |
-          # +-----------------------------------+--------------------+
-          # | micro-bosh-stemcell-aws-0.6.4.tgz | aws, micro, stable |
-          # | micro-bosh-stemcell-aws-0.7.0.tgz | aws, micro, test   |
-          # +-----------------------------------+--------------------+
-          #
-          # So to get the latest version for the filter tags,
-          # get the Name field, reverse sort, and return the first item
-          # Effectively:
-          # `#{bosh_stemcells_cmd} | grep micro | awk '{ print $2 }' | sort -r | head -n 1`.strip
-          stemcell_output = `#{bosh_stemcells_cmd}`
-          say stemcell_output
-          stemcell_output.scan(/[\w.-]+\.tgz/).last
         end
+        tags = stemcell_filter_tags.join(",")
+        bosh_stemcells_cmd = "bosh public stemcells --tags #{tags}"
+        say "Locating micro-bosh stemcell, running '#{bosh_stemcells_cmd}'..."
+        #
+        # The +bosh_stemcells_cmd+ has an output that looks like:
+        # +----------------------------------------+--------------------+
+        # | Name                                   | Tags               |
+        # +----------------------------------------+--------------------+
+        # | micro-bosh-stemcell-aws-0.6.4.tgz      | aws, micro, stable |
+        # | micro-bosh-stemcell-aws-0.7.0.tgz      | aws, micro, test   |
+        # | micro-bosh-stemcell-aws-0.8.1.tgz      | aws, micro, test   |
+        # | micro-bosh-stemcell-aws-1.5.0.pre1.tgz | aws, micro         |
+        # | micro-bosh-stemcell-aws-1.5.0.pre2.tgz | aws, micro         |
+        # +----------------------------------------+--------------------+
+        #
+        # So to get the latest version for the filter tags,
+        # get the Name field, reverse sort, and return the first item
+        # Effectively:
+        # `#{bosh_stemcells_cmd} | grep micro | awk '{ print $2 }' | sort -r | head -n 1`.strip
+        stemcell_output = `#{bosh_stemcells_cmd}`
+        say stemcell_output
+        stemcell_output.scan(/[\w.-]+\.tgz/).last
       end
 
       def provider_name
@@ -1007,6 +1175,12 @@ module Bosh::Bootstrap
         @provider ||= Bosh::Providers.for_bosh_provider_name(settings.bosh_provider, fog_compute)
       end
 
+      # The micro_bosh.yml that is uploaded to the Inception VM before deploying the
+      # MicroBOSH
+      def micro_bosh_yml
+        Bosh::Bootstrap::Stages::MicroBoshDeploy.new(settings).micro_bosh_manifest
+      end
+
       def cyan; "\033[36m" end
       def clear; "\033[0m" end
       def bold; "\033[1m" end
@@ -1020,4 +1194,4 @@ module Bosh::Bootstrap
       end
     end
   end
-end
+end