bosh-bootstrap 0.8.2 → 0.9.0

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/install_bosh

@@ -3,51 +3,9 @@
 # Install BOSH CLI
 #
 # Options:
-# * $UPGRADE - re-install or upgrade gems if already installed
-# * $INSTALL_BOSH_FROM_SOURCE - install from bosh git repo
+# * $BOSH_RUBYGEM_SOURCE - URL for where bosh rubygems can be downloaded
 
 set -e # exit immediately if a simple command exits with a non-zero status
 
-if [[ $EUID -ne 0 ]]; then
-  echo "ERROR: This script must be run as root" 1>&2
-  exit 1
-fi
-
-# Install a gem if $UPGRADE exists or if gem not already installed
-function install_gem() {
-  gem_name=$1
-  options=$2
-  if [[ ("${UPGRADE}X" != "X") || "$(gem list $gem_name | grep $gem_name)X" == "X" ]]; then
-    gem install $gem_name $options --no-ri --no-rdoc
-  else
-    echo gem $gem_name already installed
-  fi
-}
-
-install_gem net-ssh '-v 2.2.2'
-install_gem net-scp '-v 1.0.4'
-install_gem fog '-v 1.9.0'
-
-if [[ "${INSTALL_BOSH_FROM_SOURCE}X" != "X" ]]; then
-  mkdir -p /var/vcap/store/repos
-  cd /var/vcap/store/repos
-  if [[ -d bosh ]]; then
-    cd bosh
-    git pull origin master
-  else
-    git clone https://github.com/cloudfoundry/bosh.git
-  fi
-
-  cd /var/vcap/store/repos/bosh
-  bundle --deployment
-  bundle --binstubs=/usr/local/sbin
-
-else
-  install_gem bosh_deployer
-fi
-
-if [[ ! -d /var/tmp/bosh ]]; then
-  echo "Symlinking the expected stemcell creation folder into mounted volume..."
-  mkdir -p /var/vcap/store/tmp/bosh
-  ln -s /var/vcap/store/tmp/bosh /var/tmp/bosh
-fi
+cd /var/vcap/store/inception
+bundle install

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/install_bosh_by_gem_install

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Install BOSH CLI
+#
+# Options:
+# * $UPGRADE - re-install or upgrade gems if already installed
+# * $INSTALL_BOSH_FROM_SOURCE - install from bosh git repo
+# * $BOSH_RUBYGEM_SOURCE - URL for where bosh rubygems can be downloaded
+
+set -e # exit immediately if a simple command exits with a non-zero status
+
+if [[ $EUID -ne 0 ]]; then
+  echo "ERROR: This script must be run as root" 1>&2
+  exit 1
+fi
+
+# Install a gem if $UPGRADE exists or if gem not already installed
+function install_gem() {
+  gem_name=$1
+  options=$2
+  if [[ ("${UPGRADE}X" != "X") || "$(gem list $gem_name | grep $gem_name)X" == "X" ]]; then
+    gem install $gem_name $options --no-ri --no-rdoc
+  else
+    echo gem $gem_name already installed
+  fi
+}
+
+install_gem net-ssh '-v 2.2.2'
+install_gem net-scp '-v 1.0.4'
+install_gem fog '-v 1.9.0'
+
+if [[ "${INSTALL_BOSH_FROM_SOURCE}X" != "X" ]]; then
+  mkdir -p /var/vcap/store/repos
+  cd /var/vcap/store/repos
+  if [[ -d bosh ]]; then
+    cd bosh
+    git pull origin master
+  else
+    git clone https://github.com/cloudfoundry/bosh.git
+  fi
+
+  cd /var/vcap/store/repos/bosh
+  bundle --deployment
+  bundle --binstubs=/usr/local/sbin
+
+else
+  install_gem bosh_deployer
+fi
+
+if [[ ! -d /var/tmp/bosh ]]; then
+  echo "Symlinking the expected stemcell creation folder into mounted volume..."
+  mkdir -p /var/vcap/store/tmp/bosh
+  ln -s /var/vcap/store/tmp/bosh /var/tmp/bosh
+fi

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/install_hub

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+if [[ $EUID -ne 0 ]]; then
+  echo "ERROR: This script must be run as root" 1>&2
+  exit 1
+fi
+
+set -e # exit immediately if a simple command exits with a non-zero status
+
+cd /var/vcap/store/repos
+if [[ ! -d hub ]]; then
+  git clone git://github.com/defunkt/hub.git
+  cd hub
+else
+  cd hub
+  git pull origin master
+fi
+rake install prefix=/usr/local
+chown -R vcap:vcap /var/vcap/store/repos/hub
+
+if [[ $(grep "hub alias -s" /home/vcap/.bashrc) == "" ]]; then
+  echo 'adding hub alias to .bashrc'
+  echo 'eval "$(hub alias -s)"' >> /home/vcap/.bashrc
+fi
+
+

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/install_ruby

@@ -13,23 +13,18 @@ if [[ $EUID -ne 0 ]]; then
   exit 1
 fi
 
-DISTRIB_CODENAME=lucid
+RUBY_VERSION=ruby-1.9.3
 
 if [[ "$(which ruby)" == "" || "${UPGRADE}X" != "X" || ! ("$(ruby -v)" =~ "ruby 1.9.3") ]]; then
-  echo "Adding the Unboxed APT repository..."
-  echo "deb http://apt.unboxedconsulting.com/ $DISTRIB_CODENAME main" > /etc/apt/sources.list.d/unboxed.list
-  
-  echo -e "\n Retrieving the PGP keys for the repository..."
-  wget -O - http://apt.unboxedconsulting.com/release.asc | apt-key add -
-
-  echo "Resynchronizing the package index files from their sources..."
-  apt-get update
+  curl -L https://get.rvm.io | bash -s head --autolibs=3 --ruby=$RUBY_VERSION
+  source /etc/profile.d/rvm.sh
+else
+  echo $RUBY_VERSION already installed
+fi
 
-  echo -e "Installing Ruby 1.9.3...\n"
-  apt-get -y install ubxd-ruby1.9.3
+sudo usermod -a -G rvm vcap
 
-  echo -e "Upgrading to latest Rubygems..."
-  gem update --system
-else
-  echo Ruby 1.9.3 already installed
+if [[ $(grep 'rvm $RUBY_VERSION --default' ~vcap/.profile) == "" ]]; then
+  echo "setting rvm default for ~vcap user"
+  echo "rvm $RUBY_VERSION --default" >> ~vcap/.profile
 fi

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/install_useful_gems

@@ -17,7 +17,7 @@ function install_gem() {
   gem_name=$1
   options=$2
   if [[ ("${UPGRADE}X" != "X") || "$(gem list $gem_name | grep $gem_name)X" == "X" ]]; then
-    gem install $gem_name $options --no-ri --no-rdoc
+    gem install $gem_name $options --no-ri --no-rdoc -f
   else
     echo gem $gem_name already installed
   fi

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/validate_bosh_deployer

@@ -4,11 +4,6 @@
 
 set -e # exit immediately if a simple command exits with a non-zero status
 
-if [[ $EUID -ne 0 ]]; then
-  echo "ERROR: This script must be run as root" 1>&2
-  exit 1
-fi
-
 echo "Testing for 'bosh micro deploy' command..."
 
 bosh_output=$(bosh --no-color help micro)

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm.rb

@@ -11,25 +11,20 @@ module Bosh::Bootstrap::Stages
         # using inception VM username login, create a vcap user with same authorizations
         server.create "vcap user", script("create_vcap_user",
           "ORIGUSER" => settings.inception.username),
-          :user => settings.inception.username
+          ssh_username: settings.inception.username, run_as_root: true
         # install base Ubuntu packages used for bosh micro deployer
-        server.install "base packages", script("install_base_packages")
+        server.install "base packages", script("install_base_packages"), run_as_root: true
         server.configure "git", script("configure_git",
           "GIT_USER_NAME" => settings["git"]["name"],
           "GIT_USER_EMAIL" => settings["git"]["email"])
-        server.install "ruby 1.9.3", script("install_ruby", "UPGRADE" => settings[:upgrade_deps])
-        server.install "useful ruby gems", script("install_useful_gems", "UPGRADE" => settings[:upgrade_deps])
-        server.install "bosh", script("install_bosh",
-          "UPGRADE" => settings[:upgrade_deps],
-          "INSTALL_BOSH_FROM_SOURCE" => settings["bosh_git_source"] || "")
-        server.install "bosh plugins", script("install_bosh_plugins", "UPGRADE" => settings[:upgrade_deps])
-
-        # use inception VM to generate a salted password (local machine may not have mkpasswd)
-        unless settings["bosh"]["salted_password"]
-          server.capture_value "salted password", script("convert_salted_password", "PASSWORD" => settings.bosh.password),
-            :settings => settings, :save_output_to_settings_key => "bosh.salted_password"
-        end
-
+        server.install "ruby 1.9.3", script("install_ruby", "UPGRADE" => settings[:upgrade_deps]),
+          run_as_root: true
+        server.install "useful ruby gems", script("install_useful_gems", "UPGRADE" => settings[:upgrade_deps]),
+          run_as_root: true
+        server.install "hub", script("install_hub"),
+          run_as_root: true
+        server.upload_file "/var/vcap/store/inception/Gemfile", gemfile
+        server.install "bosh", script("install_bosh")
         server.validate "bosh deployer", script("validate_bosh_deployer")
       end
     end
@@ -58,5 +53,16 @@ module Bosh::Bootstrap::Stages
         exit 1
       end
     end
+
+    # returns a Gemfile to be used to install bosh & other gems
+    def gemfile
+      <<-RUBY.gsub(/      /, '')
+      source 'https://rubygems.org'
+      source 'https://s3.amazonaws.com/bosh-jenkins-gems/'
+      
+      gem "bosh_cli", "~> 1.5.0.pre"
+      gem "bosh_deployer", "~> 1.5.0.pre"
+      RUBY
+    end
   end
 end

data/lib/bosh-bootstrap/stages/stage_salted_password.rb

@@ -0,0 +1,51 @@
+require "json" # for inline hashes within YAML
+
+module Bosh::Bootstrap::Stages
+  class SaltedPassword
+    attr_reader :settings
+
+    def initialize(settings)
+      @settings = settings
+    end
+
+    def commands
+      @commands ||= Bosh::Bootstrap::Commander::Commands.new do |server|
+        # use inception VM to generate a salted password (local machine may not have mkpasswd)
+        server.capture_value "salted password", script("convert_salted_password", "PASSWORD" => settings.bosh.password),
+          :settings => settings, :save_output_to_settings_key => "bosh.salted_password"
+      end
+    end
+
+    def stage_name
+      "stage_salted_password"
+    end
+
+    # Loads local script
+    # If +variables+, then injects KEY=VALUE environment
+    # variables into bash scripts.
+    def script(segment_name, variables={})
+      path = File.expand_path("../#{stage_name}/#{segment_name}", __FILE__)
+      if File.exist?(path)
+        script = File.read(path)
+        if variables.keys.size > 0
+          env_variables = variables.reject { |var| var.is_a?(Symbol) }
+
+          # inject variables into script if its bash script
+          inline_variables = "#!/usr/bin/env bash\n\n"
+          env_variables.each { |name, value| inline_variables << "#{name}='#{value}'\n" }
+          script.gsub!("#!/usr/bin/env bash", inline_variables)
+
+          # inject variables into script if its ruby script
+          inline_variables = "#!/usr/bin/env ruby\n\n"
+          env_variables.each { |name, value| inline_variables << "ENV['#{name}'] = '#{value}'\n" }
+          script.gsub!("#!/usr/bin/env ruby", inline_variables)
+        end
+        script
+      else
+        Thor::Base.shell.new.say_status "error", "Missing script lib/bosh-bootstrap/stages/#{stage_name}/#{segment_name}", :red
+        exit 1
+      end
+    end
+
+  end
+end

data/lib/bosh-bootstrap/stages/stage_setup_new_bosh.rb

@@ -15,7 +15,6 @@ module Bosh::Bootstrap::Stages
                       "BOSH_HOST" => settings.bosh.ip_address,
                       "BOSH_USERNAME" => settings.bosh_username,
                       "BOSH_PASSWORD" => settings.bosh_password)
-        server.cleanup "permissions", script("cleanup_permissions")
       end
     end
 

data/lib/bosh-bootstrap/stages/stage_validate_inception_vm.rb

@@ -8,7 +8,7 @@ module Bosh::Bootstrap::Stages
 
     def commands
       @commands ||= Bosh::Bootstrap::Commander::Commands.new do |server|
-        server.validate "ubuntu", script("validate_ubuntu"), :user => settings.inception.username
+        server.validate "ubuntu", script("validate_ubuntu"), ssh_username: settings.inception.username
       end
     end
 

data/lib/bosh-bootstrap/stages.rb

@@ -5,4 +5,5 @@ require "bosh-bootstrap/stages/stage_validate_inception_vm"
 require "bosh-bootstrap/stages/stage_prepare_inception_vm"
 require "bosh-bootstrap/stages/stage_micro_bosh_deploy"
 require "bosh-bootstrap/stages/stage_setup_new_bosh"
+require "bosh-bootstrap/stages/stage_salted_password"
 require "bosh-bootstrap/stages/stage_micro_bosh_delete"

data/lib/bosh-bootstrap/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Bootstrap
-    VERSION = "0.8.2"
+    VERSION = "0.9.0"
   end
 end

data/spec/integration/aws/aws_basic_spec.rb

@@ -0,0 +1,39 @@
+require File.expand_path("../../../spec_helper", __FILE__)
+require File.expand_path("../aws_helpers", __FILE__)
+
+require "active_support/core_ext/hash/keys"
+
+describe "AWS deployment using gems and publish stemcells" do
+  include FileUtils
+  include Bosh::Bootstrap::Helpers::SettingsSetter
+  include AwsHelpers
+
+  attr_reader :bosh_name
+
+  before { prepare_aws("basic", aws_region) }
+  after { destroy_test_constructs(bosh_name) unless keep_after_test? }
+
+  def aws_region
+    ENV['AWS_REGION'] || "us-west-2"
+  end
+
+  it "creates an EC2 inception/microbosh with the associated resources" do
+    create_manifest
+
+    manifest_file = home_file(".bosh_bootstrap", "manifest.yml")
+    File.should be_exists(manifest_file)
+
+    cmd.deploy
+
+    fog.addresses.should have(2).item
+    inception_ip_address = fog.addresses.first
+    inception_ip_address.domain.should == "standard"
+
+    inception_vms = servers_with_sg("#{bosh_name}-inception-vm")
+    inception_vms.size.should == 1
+
+    micrboshes = servers_with_sg(bosh_name)
+    micrboshes.size.should == 1
+  end
+
+end

data/spec/integration/aws/aws_edge_prebuilt_ami_spec.rb

@@ -0,0 +1,46 @@
+require File.expand_path("../../../spec_helper", __FILE__)
+require File.expand_path("../aws_helpers", __FILE__)
+
+require "active_support/core_ext/hash/keys"
+
+describe "AWS deployment using very latest prebuilt gems and AMIs (us-east-1 only)" do
+  include FileUtils
+  include Bosh::Bootstrap::Helpers::SettingsSetter
+  include AwsHelpers
+
+  attr_reader :bosh_name
+
+  before { prepare_aws("edge-prebuilt", aws_region) }
+  # after { destroy_test_constructs(bosh_name) unless keep_after_test? }
+
+  # Jenkins AMIs are produced for us-east-1
+  def aws_region
+    "us-east-1"
+  end
+
+  it "creates an EC2 inception/microbosh with the associated resources" do
+    create_manifest("edge-prebuilt" => true)
+
+    manifest_file = home_file(".bosh_bootstrap", "manifest.yml")
+    File.should be_exists(manifest_file)
+
+    cmd.deploy
+
+    ip_adresses = fog.addresses
+    public_ips = ip_adresses.map(&:public_ip)
+
+    inception_vms = provider.servers_with_sg("#{bosh_name}-inception-vm")
+    inception_vms.size.should == 1
+
+    # TODO inception VM is not getting its IP address bound correctly
+    # https://github.com/StarkAndWayne/bosh-bootstrap/issues/174
+    # public_ips.include?(inception_vms.first.public_ip_address).should be_true
+
+    micrboshes = provider.servers_with_sg(bosh_name)
+    micrboshes.size.should == 1
+    public_ips.include?(micrboshes.first.public_ip_address).should be_true
+
+    # TODO - no files in /var/vcap/store/stemcells (since it used AMI)
+  end
+
+end

data/spec/integration/aws/aws_edge_prebuilt_spec.rb

@@ -0,0 +1,46 @@
+require File.expand_path("../../../spec_helper", __FILE__)
+require File.expand_path("../aws_helpers", __FILE__)
+
+require "active_support/core_ext/hash/keys"
+
+describe "AWS deployment using very latest prebuilt gems and stemcells (outside of us-east-1)" do
+  include FileUtils
+  include Bosh::Bootstrap::Helpers::SettingsSetter
+  include AwsHelpers
+
+  attr_reader :bosh_name
+
+  before { prepare_aws("edge-prebuilt", aws_region) }
+  # after { destroy_test_constructs(bosh_name) unless keep_after_test? }
+
+  # Jenkins AMIs are produced for us-east-1; so pick a different region to test stemcell
+  def aws_region
+    ENV['AWS_REGION'] || "us-west-2"
+  end
+
+  it "creates an EC2 inception/microbosh with the associated resources" do
+    create_manifest("edge-prebuilt" => true)
+
+    manifest_file = home_file(".bosh_bootstrap", "manifest.yml")
+    File.should be_exists(manifest_file)
+
+    cmd.deploy
+
+    ip_adresses = fog.addresses
+    public_ips = ip_adresses.map(&:public_ip)
+
+    inception_vms = provider.servers_with_sg("#{bosh_name}-inception-vm")
+    inception_vms.size.should == 1
+
+    # TODO inception VM is not getting its IP address bound correctly
+    # https://github.com/StarkAndWayne/bosh-bootstrap/issues/174
+    # public_ips.include?(inception_vms.first.public_ip_address).should be_true
+
+    micrboshes = provider.servers_with_sg(bosh_name)
+    micrboshes.size.should == 1
+    public_ips.include?(micrboshes.first.public_ip_address).should be_true
+
+    # TODO - no files in /var/vcap/store/stemcells (since it used AMI)
+  end
+
+end

data/spec/integration/aws/aws_edge_spec.rb

@@ -0,0 +1,45 @@
+require File.expand_path("../../../spec_helper", __FILE__)
+require File.expand_path("../aws_helpers", __FILE__)
+
+require "active_support/core_ext/hash/keys"
+
+describe "AWS deployment using Bosh edge from source" do
+  include FileUtils
+  include Bosh::Bootstrap::Helpers::SettingsSetter
+  include AwsHelpers
+
+  attr_reader :bosh_name
+
+  before { prepare_aws("bosh-edge", aws_region) }
+  # after { destroy_test_constructs(bosh_name) unless keep_after_test? }
+
+  def aws_region
+    ENV['AWS_REGION'] || "us-west-2"
+  end
+
+  it "creates an EC2 inception/microbosh with the associated resources" do
+    create_manifest("edge" => true)
+
+    manifest_file = home_file(".bosh_bootstrap", "manifest.yml")
+    File.should be_exists(manifest_file)
+
+    cmd.deploy
+
+    ip_adresses = fog.addresses
+    public_ips = ip_adresses.map(&:public_ip)
+
+    inception_vms = provider.servers_with_sg("#{bosh_name}-inception-vm")
+    inception_vms.size.should == 1
+
+    # TODO inception VM is not getting its IP address bound correctly
+    # https://github.com/StarkAndWayne/bosh-bootstrap/issues/174
+    # public_ips.include?(inception_vms.first.public_ip_address).should be_true
+
+    micrboshes = provider.servers_with_sg(bosh_name)
+    micrboshes.size.should == 1
+    public_ips.include?(micrboshes.first.public_ip_address).should be_true
+
+    # TODO - microbosh stemcell in /var/vcap/store/stemcells
+  end
+
+end

data/spec/integration/aws/aws_helpers.rb

@@ -0,0 +1,79 @@
+module AwsHelpers
+  def fog_credentials
+    @fog_credentials ||= begin
+      access_key = ENV['AWS_ACCESS_KEY_ID']
+      secret_key = ENV["AWS_SECRET_ACCESS_KEY"]
+      unless access_key && secret_key
+        raise "Please provided $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY"
+      end
+      credentials = {
+        :provider                 => 'AWS',
+        :aws_access_key_id        => access_key,
+        :aws_secret_access_key    => secret_key
+      }
+    end
+  end
+
+  def keep_after_test?
+    ENV['KEEP_AFTER_TEST']
+  end
+
+  def fog
+    @fog ||= connection = Fog::Compute.new(fog_credentials.merge(:region => aws_region))
+  end
+
+  def cmd
+    @cmd ||= Bosh::Bootstrap::Cli.new
+  end
+
+  def provider
+    cmd.provider
+  end
+
+  # used by +SettingsSetter+ to access the settings
+  def settings
+    cmd.settings
+  end
+
+  def prepare_aws(spec_name, aws_region)
+    setup_home_dir
+    @cmd = nil
+    @fog = nil
+    @bosh_name = "aws-#{spec_name}-#{aws_region}-#{unique_number}"
+    create_manifest
+    destroy_test_constructs(bosh_name)
+  end
+
+  def unique_number
+    ENV['UNIQUE_NUMBER'] || Random.rand(100000)
+  end
+
+  def create_manifest(options = {})
+    setting "bosh_provider", "aws"
+    setting "region_code", aws_region
+    setting "bosh_name", bosh_name
+    setting "inception.create_new", true
+    setting "bosh_username", "testuser"
+    setting "bosh_password", "testpass"
+    setting "fog_credentials", fog_credentials.stringify_keys
+    setting "bosh.salted_password", "pepper"
+    setting "bosh.persistent_disk", 16384
+    setting "git.name", "Dr Nic Williams"
+    setting "git.email", "drnicwilliams@gmail.com"
+    options.each { |key, value| setting(key, value) }
+    cmd.save_settings!
+  end
+
+  def destroy_test_constructs(bosh_name)
+    puts "Destroying everything created by previous tests..."
+    # destroy servers using inception-vm SG
+    provider.delete_security_group_and_servers("#{bosh_name}-inception-vm")
+    provider.delete_security_group_and_servers(bosh_name)
+
+    # TODO delete "inception" key pair? Why isn't it named for the bosh/inception VM?
+    provider.delete_key_pair(bosh_name)
+
+    provider.cleanup_unused_ip_addresses
+  end
+
+end

data/spec/spec_helper.rb

@@ -34,6 +34,12 @@ def setup_home_dir
   ENV['HOME'] = home_dir
 end
 
+# returns the file path to a file
+# in the fake $HOME folder
+def home_file(*path)
+  File.join(ENV['HOME'], *path)
+end
+
 RSpec.configure do |c|
   c.before(:each) do
     setup_home_dir