bosh-bootstrap 0.6.0 → 0.7.0

data/lib/bosh-bootstrap/helpers/settings_setter.rb

@@ -7,6 +7,26 @@ module Bosh; module Bootstrap; module Helpers; end; end; end
 #
 # Assumes +settings+ contains the settings
 module Bosh::Bootstrap::Helpers::SettingsSetter
+
+  # Let's you navigate to a nested setting, do something with it,
+  # and then saves any changes to settings.
+  # Usage:
+  #   with_setting "inception" { |s| s["host"] = "1.2.3.4" }
+  #   with_setting "a.b.c" { |s| s["value"] = "1.2.3.4" }
+  def with_setting(nested_key, &block)
+    target_settings_field = settings
+    settings_key_portions = nested_key.split(".")
+    parent_key_portions, final_key = settings_key_portions[0..-2], settings_key_portions[-1]
+    parent_key_portions.each do |key_portion|
+      target_settings_field[key_portion] ||= {}
+      target_settings_field = target_settings_field[key_portion]
+    end
+
+    target_settings_field[final_key] ||= {}
+    yield target_settings_field[final_key]
+    save_settings!
+  end
+
   def setting(nested_key, value)
     target_settings_field = settings
     settings_key_portions = nested_key.split(".")
@@ -17,4 +37,5 @@ module Bosh::Bootstrap::Helpers::SettingsSetter
     end
     target_settings_field[final_key] = value
   end
+
 end

data/lib/bosh-bootstrap/stages/stage_micro_bosh_deploy/bosh_micro_deploy

@@ -20,17 +20,56 @@ cd /var/vcap/store/microboshes/deployments/
 bosh -n micro deployment $BOSH_NAME
 
 if [[ "${MICRO_BOSH_STEMCELL_NAME}" == "custom" ]]; then
-  MICRO_BOSH_STEMCELL_PATH=$(ls -t /var/tmp/bosh/agent-*/work/work/*.tgz | sort | tail -n 1)
+  MICRO_BOSH_STEMCELL_PATH=$(ls -t /var/tmp/bosh/bosh_agent-*/work/work/*.tgz | sort | tail -n 1)
   echo "Custom stemcell path $MICRO_BOSH_STEMCELL_PATH"
 else
   echo "Downloaded stemcell path $MICRO_BOSH_STEMCELL_PATH"
 fi
 
-if [[ -f bosh-deployments.yml && "$(grep $BOSH_NAME bosh-deployments.yml)" != "" ]]; then
-  echo "Performing update deploy to micro BOSH $BOSH_NAME"
-  bosh -n micro deploy $MICRO_BOSH_STEMCELL_PATH --update
+if [[ -f bosh-deployments.yml ]]; then
+  deployment=$(cat bosh-deployments.yml | yaml get instances -j | jazor "find {|inst| inst['name'] == '$BOSH_NAME' }")
 else
+  deployment=""
+fi
+
+# if $BOSH_NAME has never been deployed before, then $deployment will be empty
+# if $BOSH_NAME was previously deployed but deleted then $deployment will look like:
+#   {
+#     "id": 1,
+#     "name": "microbosh-aws-us-east-1",
+#     "uuid": "bm-096bc437-b531-4260-a57f-8ea54db2f8d2",
+#     "stemcell_cid": null,
+#     "stemcell_name": null,
+#     "vm_cid": null,
+#     "disk_cid": null
+#   }
+# if $BOSH_NAME has been deployed but it failed mid-way, then some of the _cid fields above will be set.
+
+echo ""
+echo ""
+
+if [[ "${deployment}X" == "X" ]]; then
   echo "Performing initial deploy of micro BOSH $BOSH_NAME"
-  bosh -n micro deploy $MICRO_BOSH_STEMCELL_PATH
+  bosh -n --color micro deploy $MICRO_BOSH_STEMCELL_PATH
   touch $BOSH_NAME/new_bosh_deployed # to communicate with setup_bosh_user script
+else
+  # determine if re-deploy, update, or delete&deploy
+
+  stemcell_cid=$(echo $deployment | jazor stemcell_cid)
+  vm_cid=$(echo $deployment | jazor vm_id)
+  disk_cid=$(echo $deployment | jazor disk_cid)
+
+  if [[ $stemcell_cid == "" && $vm_cid == "" && $disk_cid == "" ]]; then
+    echo "Deployment was deleted; deploying micro BOSH $BOSH_NAME again"
+    bosh -n --color micro deploy $MICRO_BOSH_STEMCELL_PATH
+    touch $BOSH_NAME/new_bosh_deployed # to communicate with setup_bosh_user script
+  elif [[ $stemcell_cid == "" || $vm_cid == "" || $disk_cid == "" ]]; then
+    echo "Deployment previously failed to complete. Deleting and deploying micro BOSH $BOSH_NAME again."
+    bosh -n --color micro delete
+    bosh -n --color micro deploy $MICRO_BOSH_STEMCELL_PATH
+    touch $BOSH_NAME/new_bosh_deployed # to communicate with setup_bosh_user script
+  else
+    echo "Performing update deploy to micro BOSH $BOSH_NAME"
+    bosh -n --color micro deploy $MICRO_BOSH_STEMCELL_PATH --update
+  fi
 fi

data/lib/bosh-bootstrap/stages/stage_micro_bosh_deploy/download_micro_bosh_stemcell

@@ -8,7 +8,7 @@
 #
 # Optional:
 # * $PROVIDER - required for 'custom' $MICRO_BOSH_STEMCELL_NAME; e.g. aws, openstack
-# * $BOSH_RELEASE_REPO - defaults to https://github.com/cloudfoundry/bosh-release.git
+# * $ISO_NAME - defaults to ubuntu-10.04.4-server-amd64.iso for creating stemcell
 
 set -e # exit immediately if a simple command exits with a non-zero status
 set -u # report the usage of uninitialized variables
@@ -23,76 +23,26 @@ if [[ "${MICRO_BOSH_STEMCELL_NAME}X" == "X" ]]; then
   exit 1
 fi
 
+STEMCELLS_DIR=/var/vcap/store/stemcells
+
 if [[ "${MICRO_BOSH_STEMCELL_NAME}" == "custom" ]]; then
 
-  BOSH_RELEASE_REPO=${BOSH_RELEASE_REPO:-'https://github.com/cloudfoundry/bosh-release.git'}
-  BOSH_RELEASE_DIRNAME=${BOSH_RELEASE_DIRNAME:-'bosh-release'}
-  BOSH_RELEASE_DIR=/var/vcap/store/releases/$BOSH_RELEASE_DIRNAME
+  ISO_NAME=${ISO_NAME:-ubuntu-10.04.4-server-amd64.iso}
 
   echo "Creating custom stemcell..."
 
+  cd ${STEMCELLS_DIR}
+  if [[ ! -f ${ISO_NAME} ]]; then
+    echo "Fetching base stemcell ISO to speed up stemcell creation..."
+    wget http://releases.ubuntu.com/lucid/${ISO_NAME}
+  fi
+  export UBUNTU_ISO=${STEMCELLS_DIR}/${ISO_NAME}
+
   if [[ -d /var/tmp/bosh/ ]]; then
     echo "But first, cleaning out previous stemcell temporary files..."
     rm -rf /var/tmp/bosh
   fi
 
-
-  cd $(dirname $BOSH_RELEASE_DIR)
-  if [[ -d $BOSH_RELEASE_DIRNAME ]]; then
-    echo "Updating BOSH's own bosh-release for custom stemcell"
-    cd $BOSH_RELEASE_DIRNAME
-    git pull origin master
-  else
-    echo "Fetching BOSH's own bosh-release for custom stemcell"
-    git clone $BOSH_RELEASE_REPO
-    cd $BOSH_RELEASE_DIRNAME
-  fi
-
-  echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
-
-  echo "Rewriting all git:// & git@ to https:// to allow access if port 22 is blocked..."
-  # Snippet written by Mike Reeves <swampfoxmr@gmail.com> on bosh-users mailing list
-  # Date 2012-12-06
-  grep -rI "git[@:/]\{0,3\}github.com" * .gitmodules | awk 'BEGIN {FS=":"} { print($1) }' | while read file
-  do
-    echo "changing - $file"
-    sed -i 's/git\:\/\/github.com/https:\/\/github.com/g' $file
-    sed -i 's/git@github.com:/https:\/\/github.com\//g' $file
-  done
-
-  echo "Updating source submodules..."
-  git submodule update --init
-
-  echo "Within submodules, rewriting all git:// & git@ to https:// to allow access if port 22 is blocked..."
-  # Snippet written by Mike Reeves <swampfoxmr@gmail.com> on bosh-users mailing list
-  # Date 2012-12-06
-  grep -rI "git[@:/]\{0,3\}github.com" * .gitmodules | awk 'BEGIN {FS=":"} { print($1) }' | while read file
-  do
-    echo "changing - $file"
-    sed -i 's/git\:\/\/github.com/https:\/\/github.com/g' $file
-    sed -i 's/git@github.com:/https:\/\/github.com\//g' $file
-  done
-
-  echo "Creating bosh release (fetching any new blobs)..."
-  bosh -n create release --force --with-tarball
-
-  # dev_releases/index.yml contains the dev_releases
-  # and its last line contains the latest dev_release name
-  # For example:
-  # --- 
-  # builds: 
-  #   4800ad1226e2b673595a8646e8e11bc968ab6ab6: 
-  #     version: 119.1-dev
-  #   7e0318d711daf0a5fa2e65c0257cd3b6af2ec4e5: 
-  #     version: 119.2-dev
-  #
-  # So to get the latest dev_release name, grab the last line's 2nd token
-  # For example, above this would be "119.2-dev"
-  RELEASE_NAME="bosh-release"
-  LATEST_DEV_RELEASE_NAME=$(tail -n 1 dev_releases/index.yml | awk '{print $2}')
-  LATEST_BOSH_RELEASE_PATH=$BOSH_RELEASE_DIR/dev_releases/$RELEASE_NAME-$LATEST_DEV_RELEASE_NAME.tgz
-  MICRO_BOSH_MANIFEST_PATH=$BOSH_RELEASE_DIR/micro/$PROVIDER.yml
-
   BOSH_DIR=/var/vcap/store/repos/bosh
   cd $(dirname $BOSH_DIR)
   if [[ -d bosh ]]; then
@@ -107,22 +57,20 @@ if [[ "${MICRO_BOSH_STEMCELL_NAME}" == "custom" ]]; then
 
   cd $BOSH_DIR
   echo "Creating custom stemcell..."
-  bundle install
-  bundle exec rake stemcell:micro[$PROVIDER,$MICRO_BOSH_MANIFEST_PATH,$LATEST_BOSH_RELEASE_PATH]
+  rm -rf .bundle
+  bundle install --deployment
+  bundle exec rake stemcell:micro[$PROVIDER]
 
   echo "Copying to stemcells folder..."
-  MICRO_BOSH_STEMCELL_PATH=$(ls -t /var/tmp/bosh/agent-*/work/work/*.tgz | sort | tail -n 1)
-  mv $MICRO_BOSH_STEMCELL_PATH /var/vcap/store/stemcells
+  MICRO_BOSH_STEMCELL_PATH=$(ls -t /var/tmp/bosh/bosh_agent-*/work/work/*.tgz | sort | tail -n 1)
+  mv $MICRO_BOSH_STEMCELL_PATH $STEMCELLS_DIR
   MICRO_BOSH_STEMCELL_NAME=$(basename $MICRO_BOSH_STEMCELL_PATH)
 
-  echo "Cleaning up..."
-  rm $(dirname $LATEST_BOSH_RELEASE_PATH)/*.tgz
-
-elif [[ -f /var/vcap/store/stemcells/$MICRO_BOSH_STEMCELL_NAME ]]; then
+elif [[ -f $STEMCELLS_DIR/$MICRO_BOSH_STEMCELL_NAME ]]; then
   echo "Stemcell $(pwd)/$MICRO_BOSH_STEMCELL_NAME already exists."
 
 else
-  cd /var/vcap/store/stemcells
+  cd $STEMCELLS_DIR
   echo "Downloading public stemcell $MICRO_BOSH_STEMCELL_NAME"
   # set $COLUMNS manually else progressbar runs `tput cols`.to_i which causes grief
   COLUMNS=80 bosh -n download public stemcell $MICRO_BOSH_STEMCELL_NAME

data/lib/bosh-bootstrap/stages/stage_micro_bosh_deploy.rb

@@ -15,8 +15,8 @@ module Bosh::Bootstrap::Stages
       @commands ||= Bosh::Bootstrap::Commander::Commands.new do |server|
         server.download "micro-bosh stemcell", script("download_micro_bosh_stemcell",
                       "MICRO_BOSH_STEMCELL_NAME" => settings.micro_bosh_stemcell_name,
-                      "PROVIDER" => settings.bosh_provider,
-                      :settings => settings, :save_output_to_settings_key => "micro_bosh_stemcell_name")
+                      "PROVIDER" => settings.bosh_provider),
+                      :settings => settings, :save_output_to_settings_key => "micro_bosh_stemcell_name"
         server.upload_file \
                       "/var/vcap/store/microboshes/deployments/#{settings.bosh_name}/micro_bosh.yml",
                       micro_bosh_manifest
@@ -32,7 +32,6 @@ module Bosh::Bootstrap::Stages
       end
     end
 
-    private
     def stage_name
       "stage_micro_bosh_deploy"
     end

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/configure_git

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Setup git configuration for the user
+#
+# Run as the user to configure
+#
+# Requires:
+# * $GIT_USER_NAME
+# * $GIT_USER_EMAIL
+
+if [[ $EUID -ne 0 ]]; then
+  echo "ERROR: This script must be run as root" 1>&2
+  exit 1
+fi
+
+if [[ "${GIT_USER_NAME}X" == "X" ]]; then
+  echo 'ERROR please provide $GIT_USER_NAME'
+  exit 1
+fi
+if [[ "${GIT_USER_EMAIL}X" == "X" ]]; then
+  echo 'ERROR please provide $GIT_USER_EMAIL'
+  exit 1
+fi
+
+cd ~vcap
+chpst -u vcap:vcap git config -f ~vcap/.gitconfig user.name "${GIT_USER_NAME}"
+chpst -u vcap:vcap git config -f ~vcap/.gitconfig user.email "${GIT_USER_EMAIL}"
+chpst -u vcap:vcap git config -f ~vcap/.gitconfig color.ui true

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/install_base_packages

@@ -5,9 +5,14 @@ if [[ $EUID -ne 0 ]]; then
   exit 1
 fi
 
+apt-get install python-software-properties
+add-apt-repository ppa:keithw/mosh
+
 apt-get update
-apt-get install build-essential libsqlite3-dev curl rsync git-core tmux \
+apt-get install build-essential libsqlite3-dev curl rsync git-core \
+  tmux mosh \
   libmysqlclient-dev libxml2-dev libxslt-dev libpq-dev libsqlite3-dev \
+  runit \
   genisoimage mkpasswd \
   debootstrap kpartx qemu-kvm \
   vim -y

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/install_bosh

@@ -31,37 +31,11 @@ if [[ "${INSTALL_BOSH_FROM_SOURCE}X" != "X" ]]; then
     git clone https://github.com/cloudfoundry/bosh.git
   fi
 
-  bosh_dir=/var/vcap/store/repos/bosh
-  projects=(
-    bosh_common 
-    blobstore_client 
-    bosh_cpi
-    ruby_vcloud_sdk
-    bosh_vcloud_cpi 
-    ruby_vim_sdk
-    bosh_vsphere_cpi
-    bosh_aws_cpi 
-    agent_client 
-    bosh_cli 
-    bosh_aws_registry 
-    bosh_openstack_registry
-    bosh_deployer 
-  )
-  for project in "${projects[@]}"; do
-    cd $bosh_dir/$project/
-    rm -rf pkg
-    bundle install --without=development test
-    gem build *.gemspec
-    mkdir -p pkg
-    mv *.gem pkg/
-    gem install pkg/*.gem --no-ri --no-rdoc
-  done
+  cd /var/vcap/store/repos/bosh
+  bundle --deployment
+  bundle exec rake all:install
 
 else
-  install_gem bosh_cli
-  # TODO remove this when bosh_deployer starts installing aws-sdk 1.8.1.1 or higher
-  install_gem httparty
-  # end TODO
   install_gem bosh_deployer
 fi
 

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm/install_useful_gems

@@ -13,8 +13,9 @@ fi
 # Install a gem if $UPGRADE exists or if gem not already installed
 function install_gem() {
   gem_name=$1
+  options=$2
   if [[ ("${UPGRADE}X" != "X") || "$(gem list $gem_name | grep $gem_name)X" == "X" ]]; then
-    gem install $gem_name --no-ri --no-rdoc
+    gem install $gem_name $options --no-ri --no-rdoc
   else
     echo gem $gem_name already installed
   fi
@@ -22,3 +23,5 @@ function install_gem() {
 
 install_gem bundler
 install_gem rake
+install_gem jazor
+install_gem yaml_command

data/lib/bosh-bootstrap/stages/stage_prepare_inception_vm.rb

@@ -9,19 +9,27 @@ module Bosh::Bootstrap::Stages
     def commands
       @commands ||= Bosh::Bootstrap::Commander::Commands.new do |server|
         # using inception VM username login, create a vcap user with same authorizations
-        server.create "vcap user", script("create_vcap_user", "ORIGUSER" => settings.inception.username),
+        server.create "vcap user", script("create_vcap_user",
+          "ORIGUSER" => settings.inception.username),
           :user => settings.inception.username
         # install base Ubuntu packages used for bosh micro deployer
         server.install "base packages", script("install_base_packages")
+        server.configure "git", script("configure_git",
+          "GIT_USER_NAME" => settings["git"]["name"],
+          "GIT_USER_EMAIL" => settings["git"]["email"])
         server.install "ruby 1.9.3", script("install_ruby", "UPGRADE" => settings[:upgrade_deps])
         server.install "useful ruby gems", script("install_useful_gems", "UPGRADE" => settings[:upgrade_deps])
         server.install "bosh", script("install_bosh",
           "UPGRADE" => settings[:upgrade_deps],
           "INSTALL_BOSH_FROM_SOURCE" => settings["bosh_git_source"] || "")
         server.install "bosh plugins", script("install_bosh_plugins", "UPGRADE" => settings[:upgrade_deps])
+
         # use inception VM to generate a salted password (local machine may not have mkpasswd)
-        server.capture_value "salted password", script("convert_salted_password", "PASSWORD" => settings.bosh.password),
-          :settings => settings, :save_output_to_settings_key => "bosh.salted_password"
+        unless settings["bosh"]["salted_password"]
+          server.capture_value "salted password", script("convert_salted_password", "PASSWORD" => settings.bosh.password),
+            :settings => settings, :save_output_to_settings_key => "bosh.salted_password"
+        end
+
         server.validate "bosh deployer", script("validate_bosh_deployer")
       end
     end
@@ -41,7 +49,7 @@ module Bosh::Bootstrap::Stages
         if variables.keys.size > 0
           inline_variables = "#!/usr/bin/env bash\n\n"
           env_variables = variables.reject { |var| var.is_a?(Symbol) }
-          env_variables.each { |name, value| inline_variables << "#{name}=#{value}\n" }
+          env_variables.each { |name, value| inline_variables << "#{name}='#{value}'\n" }
           script.gsub!("#!/usr/bin/env bash", inline_variables)
         end
         script

data/lib/bosh-bootstrap/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Bootstrap
-    VERSION = "0.6.0"
+    VERSION = "0.7.0"
   end
 end

data/lib/bosh-bootstrap.rb

@@ -1,3 +1,7 @@
+# Requires Ruby language 1.9 and MRI or Rubinius
+require "redcard"
+RedCard.verify :mri, :ruby, :rubinius, "1.9"
+
 module Bosh
   module Bootstrap
   end

data/spec/assets/micro_bosh_yml/micro_bosh.aws_ec2.yml

@@ -0,0 +1,35 @@
+---
+name: test-bosh
+env:
+  bosh:
+    password: pepper
+logging:
+  level: DEBUG
+network:
+  type: dynamic
+  vip: $MICROBOSH_IP$
+resources:
+  persistent_disk: 16384
+  cloud_properties:
+    instance_type: m1.medium
+cloud:
+  plugin: aws
+  properties:
+    aws:
+      access_key_id: YYY
+      secret_access_key: XXX
+      region: us-west-2
+      ec2_endpoint: ec2.us-west-2.amazonaws.com
+      default_security_groups:
+      - test-bosh
+      default_key_name: test-bosh
+      ec2_private_key: /home/vcap/.ssh/test-bosh.pem
+apply_spec:
+  agent:
+    blobstore:
+      address: $MICROBOSH_IP$
+    nats:
+      address: $MICROBOSH_IP$
+  properties:
+    aws_registry:
+      address: $MICROBOSH_IP$

data/spec/assets/micro_bosh_yml/micro_bosh.aws_vpc.yml

@@ -0,0 +1,37 @@
+---
+name: test-bosh
+env:
+  bosh:
+    password: pepper
+logging:
+  level: DEBUG
+network:
+  type: dynamic
+  vip: $MICROBOSH_IP$
+  cloud_properties:
+    subnet: $SUBNET_ID$
+resources:
+  persistent_disk: 16384
+  cloud_properties:
+    instance_type: m1.medium
+cloud:
+  plugin: aws
+  properties:
+    aws:
+      access_key_id: YYY
+      secret_access_key: XXX
+      region: us-west-2
+      ec2_endpoint: ec2.us-west-2.amazonaws.com
+      default_security_groups:
+      - test-bosh
+      default_key_name: test-bosh
+      ec2_private_key: /home/vcap/.ssh/test-bosh.pem
+apply_spec:
+  agent:
+    blobstore:
+      address: $MICROBOSH_IP$
+    nats:
+      address: $MICROBOSH_IP$
+  properties:
+    aws_registry:
+      address: $MICROBOSH_IP$