bosh-bootstrap 0.5.0

data/lib/bosh-bootstrap/helpers/settings.rb

@@ -0,0 +1,36 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+require "settingslogic"
+module Bosh; module Bootstrap; module Helpers; end; end; end
+
+# Helper methods for loading/saving settings
+module Bosh::Bootstrap::Helpers::Settings
+  # Previously selected settings are stored in a YAML manifest
+  # Protects the manifest file with user-only priveleges
+  def settings
+    @settings ||= begin
+      FileUtils.mkdir_p(File.dirname(settings_path))
+      unless File.exists?(settings_path)
+        File.open(settings_path, "w") do |file|
+          file << {}.to_yaml
+        end
+      end
+      FileUtils.chmod 0600, settings_path
+      Settingslogic.new(settings_path)
+    end
+  end
+
+  def save_settings!
+    File.open(settings_path, "w") do |file|
+      raw_settings_yaml = settings.to_yaml.gsub(" !ruby/hash:Settingslogic", "")
+      file << raw_settings_yaml
+    end
+    @settings = nil # force to reload & recreate helper methods
+  end
+
+  def settings_path
+    manifest_path = ENV["MANIFEST"] || "~/.bosh_bootstrap/manifest.yml"
+    File.expand_path(manifest_path)
+  end
+
+end

data/lib/bosh-bootstrap/stages.rb

@@ -0,0 +1,8 @@
+module Bosh::Bootstrap::Stages
+end
+
+require "bosh-bootstrap/stages/stage_validate_inception_vm"
+require "bosh-bootstrap/stages/stage_prepare_inception_vm"
+require "bosh-bootstrap/stages/stage_micro_bosh_deploy"
+require "bosh-bootstrap/stages/stage_setup_new_bosh"
+require "bosh-bootstrap/stages/stage_micro_bosh_delete"

data/lib/bosh-bootstrap/stages/stage_micro_bosh_delete.rb

@@ -0,0 +1,90 @@
+require "json" # for inline hashes within YAML
+
+module Bosh::Bootstrap::Stages
+  class MicroBoshDelete
+    attr_reader :settings
+
+    def initialize(settings)
+      @settings = settings
+    end
+
+    def commands
+      @commands ||= Bosh::Bootstrap::Commander::Commands.new do |server|
+        server.delete "micro bosh", script("bosh_micro_delete",
+                      "BOSH_NAME" => settings.bosh_name)
+      end
+    end
+
+    private
+    def stage_name
+      "stage_micro_bosh_delete"
+    end
+
+    # Loads local script
+    # If +variables+, then injects KEY=VALUE environment
+    # variables into bash scripts.
+    def script(segment_name, variables={})
+      path = File.expand_path("../#{stage_name}/#{segment_name}", __FILE__)
+      if File.exist?(path)
+        script = File.read(path)
+        if variables.keys.size > 0
+          # inject variables into script if its bash script
+          inline_variables = "#!/usr/bin/env bash\n\n"
+          variables.each { |name, value| inline_variables << "#{name}='#{value}'\n" }
+          script.gsub!("#!/usr/bin/env bash", inline_variables)
+
+          # inject variables into script if its ruby script
+          inline_variables = "#!/usr/bin/env ruby\n\n"
+          variables.each { |name, value| inline_variables << "ENV['#{name}'] = '#{value}'\n" }
+          script.gsub!("#!/usr/bin/env ruby", inline_variables)
+        end
+        script
+      else
+        Thor::Base.shell.new.say_status "error", "Missing script lib/bosh-bootstrap/stages/#{stage_name}/#{segment_name}", :red
+        exit 1
+      end
+    end
+
+    def micro_bosh_manifest
+      name                       = settings.bosh_name
+      salted_password            = settings.bosh.salted_password
+      ipaddress                  = settings.bosh.ip_address
+      persistent_disk            = settings.bosh.persistent_disk
+      resources_cloud_properties = settings.bosh_resources_cloud_properties
+      cloud_plugin               = settings.bosh_provider
+
+      # aws:
+      #   access_key_id:     #{access_key}
+      #   secret_access_key: #{secret_key}
+      #   ec2_endpoint: ec2.#{region}.amazonaws.com
+      #   default_key_name: #{key_name}
+      #   default_security_groups: ["#{security_group}"]
+      #   ec2_private_key: /home/vcap/.ssh/#{key_name}.pem
+      cloud_properties = settings.bosh_cloud_properties
+
+      {
+        "name" => name,
+        "env" => { "bosh" => {"password" => salted_password}},
+        "logging" => { "level" => "DEBUG" },
+        "network" => { "type" => "dynamic", "vip" => ipaddress },
+        "resources" => {
+          "persistent_disk" => persistent_disk,
+          "cloud_properties" => resources_cloud_properties
+        },
+        "cloud" => {
+          "plugin" => cloud_plugin,
+          "properties" => cloud_properties
+        },
+        "apply_spec" => {
+          "agent" => {
+            "blobstore" => { "address" => ipaddress },
+            "nats" => { "address" => ipaddress }
+          },
+          "properties" => {
+            "#{cloud_plugin.downcase}_registry" => { "address" => ipaddress }
+          }
+        }
+      }.to_yaml.gsub(" !ruby/hash:Settingslogic", "")
+    end
+  end
+end

data/lib/bosh-bootstrap/stages/stage_micro_bosh_delete/bosh_micro_delete

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Deletes a micro BOSH using "bosh micro delete"
+#
+# Required:
+# * $BOSH_NAME - name of bosh deployment
+#   (thus /var/vcap/store/microboshes/deployments/$BOSH_NAME/micro_bosh.yml exists)
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+# TODO in preparation for http://reviews.cloudfoundry.org/#/c/11976/
+export COLUMNS=80 # force a small width for progress bar
+
+cd /var/vcap/store/microboshes/deployments/
+bosh -n micro deployment $BOSH_NAME
+
+echo "Deleting deployment of micro BOSH $BOSH_NAME"
+bosh -n micro delete

data/lib/bosh-bootstrap/stages/stage_micro_bosh_deploy.rb

@@ -0,0 +1,135 @@
+require "json" # for inline hashes within YAML
+
+module Bosh::Bootstrap::Stages
+  class MicroBoshDeploy
+    attr_reader :settings
+
+    def initialize(settings)
+      @settings = settings
+    end
+
+    # TODO "aws_us_east_1" should come from settings.bosh_name
+    def commands
+      settings[:bosh_name] ||= "unnamed_bosh"
+
+      @commands ||= Bosh::Bootstrap::Commander::Commands.new do |server|
+        server.download "micro-bosh stemcell", script("download_micro_bosh_stemcell",
+                      "MICRO_BOSH_STEMCELL_NAME" => settings.micro_bosh_stemcell_name,
+                      "PROVIDER" => settings.bosh_provider,
+                      :settings => settings, :save_output_to_settings_key => "micro_bosh_stemcell_name")
+        server.upload_file \
+                      "/var/vcap/store/microboshes/deployments/#{settings.bosh_name}/micro_bosh.yml",
+                      micro_bosh_manifest
+        server.install "key pair for user", script("install_key_pair_for_user",
+                      "PRIVATE_KEY" => settings.bosh_key_pair.private_key,
+                      "KEY_PAIR_NAME" => settings.bosh_key_pair.name)
+        server.deploy "micro bosh", script("bosh_micro_deploy",
+                      "BOSH_NAME" => settings.bosh_name,
+                      "MICRO_BOSH_STEMCELL_NAME" => settings.micro_bosh_stemcell_name,
+                      "BOSH_HOST" => settings.bosh.ip_address,
+                      "BOSH_USERNAME" => settings.bosh_username,
+                      "BOSH_PASSWORD" => settings.bosh_password)
+      end
+    end
+
+    private
+    def stage_name
+      "stage_micro_bosh_deploy"
+    end
+
+    # Loads local script
+    # If +variables+, then injects KEY=VALUE environment
+    # variables into bash scripts.
+    def script(segment_name, variables={})
+      path = File.expand_path("../#{stage_name}/#{segment_name}", __FILE__)
+      if File.exist?(path)
+        script = File.read(path)
+        if variables.keys.size > 0
+          # inject variables into script if its bash script
+          inline_variables = "#!/usr/bin/env bash\n\n"
+          variables.each { |name, value| inline_variables << "#{name}='#{value}'\n" }
+          script.gsub!("#!/usr/bin/env bash", inline_variables)
+
+          # inject variables into script if its ruby script
+          inline_variables = "#!/usr/bin/env ruby\n\n"
+          variables.each { |name, value| inline_variables << "ENV['#{name}'] = '#{value}'\n" }
+          script.gsub!("#!/usr/bin/env ruby", inline_variables)
+        end
+        script
+      else
+        Thor::Base.shell.new.say_status "error", "Missing script lib/bosh-bootstrap/stages/#{stage_name}/#{segment_name}", :red
+        exit 1
+      end
+    end
+
+    def micro_bosh_manifest
+      name                       = settings.bosh_name
+      salted_password            = settings.bosh.salted_password
+      ipaddress                  = settings.bosh.ip_address
+      persistent_disk            = settings.bosh.persistent_disk
+      resources_cloud_properties = settings.bosh_resources_cloud_properties
+      cloud_plugin               = settings.bosh_provider
+
+      # aws:
+      #   access_key_id:     #{access_key}
+      #   secret_access_key: #{secret_key}
+      #   ec2_endpoint: ec2.#{region}.amazonaws.com
+      #   default_key_name: #{key_name}
+      #   default_security_groups: ["#{security_group}"]
+      #   ec2_private_key: /home/vcap/.ssh/#{key_name}.pem
+      cloud_properties = settings.bosh_cloud_properties
+
+      {
+        "name" => name,
+        "env" => { "bosh" => {"password" => salted_password}},
+        "logging" => { "level" => "DEBUG" },
+        "network" => { "type" => "dynamic", "vip" => ipaddress },
+        "resources" => {
+          "persistent_disk" => persistent_disk,
+          "cloud_properties" => resources_cloud_properties
+        },
+        "cloud" => {
+          "plugin" => cloud_plugin,
+          "properties" => cloud_properties
+        },
+        "apply_spec" => {
+          "agent" => {
+            "blobstore" => { "address" => ipaddress },
+            "nats" => { "address" => ipaddress }
+          },
+          "properties" => {
+            "#{cloud_plugin.downcase}_registry" => { "address" => ipaddress }
+          }
+        }
+      }.to_yaml.gsub(/\s![^ ]+$/, '')
+
+      # /![^ ]+\s/ removes object notation from the YAML which appears to cause problems when being interpretted by the
+      # Ruby running on the inception vm. A before and after example would look like;
+      #
+      #   properties: !map:Settingslogic
+      #     openstack: !map:Settingslogic
+      #       username: admin
+      #       api_key: xxxxxxxxxxxxxxxxxxx
+      #       tenant: CloudFoundry
+      #       auth_url: http://192.168.1.2:5000/v2.0/tokens
+      #       default_security_groups:
+      #       - !str:HighLine::String microbosh-openstack
+      #       default_key_name: !str:HighLine::String microbosh-openstack
+      #       private_key: /home/vcap/.ssh/microbosh-openstack.pem
+      #
+      # The regex strips the !Module::ClassName notation out and the result looks as it should
+      #
+      #   properties:
+      #     openstack:
+      #       username: admin
+      #       api_key: xxxxxxxxxxxxxxxxxxx
+      #       tenant: CloudFoundry
+      #       auth_url: http://192.168.1.2:5000/v2.0/tokens
+      #       default_security_groups:
+      #       - microbosh-openstack
+      #       default_key_name: microbosh-openstack
+      #       private_key: /home/vcap/.ssh/microbosh-openstack.pem
+
+    end
+  end
+end

data/lib/bosh-bootstrap/stages/stage_micro_bosh_deploy/bosh_micro_deploy

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Deploys a micro BOSH using "bosh micro deploy"
+#
+# Required:
+# * $BOSH_NAME - name of bosh deployment
+#   (thus /var/vcap/store/microboshes/deployments/$BOSH_NAME/micro_bosh.yml exists)
+# * $MICRO_BOSH_STEMCELL_NAME - public stemcell name at
+#   /var/vcap/store/stemcells/$MICRO_BOSH_STEMCELL_NAME
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+MICRO_BOSH_STEMCELL_PATH=/var/vcap/store/stemcells/$MICRO_BOSH_STEMCELL_NAME
+
+export COLUMNS=80 # force a small width for progress bar
+export TMPDIR=/var/vcap/store/tmp
+
+cd /var/vcap/store/microboshes/deployments/
+bosh -n micro deployment $BOSH_NAME
+
+if [[ "${MICRO_BOSH_STEMCELL_NAME}" == "custom" ]]; then
+  MICRO_BOSH_STEMCELL_PATH=$(ls -t /var/tmp/bosh/agent-*/work/work/*.tgz | sort | tail -n 1)
+  echo "Custom stemcell path $MICRO_BOSH_STEMCELL_PATH"
+else
+  echo "Downloaded stemcell path $MICRO_BOSH_STEMCELL_PATH"
+fi
+
+if [[ -f bosh-deployments.yml && "$(grep $BOSH_NAME bosh-deployments.yml)" != "" ]]; then
+  echo "Performing update deploy to micro BOSH $BOSH_NAME"
+  bosh -n micro deploy $MICRO_BOSH_STEMCELL_PATH --update
+else
+  echo "Performing initial deploy of micro BOSH $BOSH_NAME"
+  bosh -n micro deploy $MICRO_BOSH_STEMCELL_PATH
+  touch $BOSH_NAME/new_bosh_deployed # to communicate with setup_bosh_user script
+fi

data/lib/bosh-bootstrap/stages/stage_micro_bosh_deploy/download_micro_bosh_stemcell

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+# Downloads a public stemcell for CPI into /var/vcap/store/stemcells/
+#
+# Required:
+# * $MICRO_BOSH_STEMCELL_NAME - public stemcell name to be downloaded
+#                             - if 'custom' the create stemcell from BOSH source
+#
+# Optional:
+# * $PROVIDER - required for 'custom' $MICRO_BOSH_STEMCELL_NAME; e.g. aws, openstack
+# * $BOSH_RELEASE_REPO - defaults to https://github.com/cloudfoundry/bosh-release.git
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+if [[ $EUID -ne 0 ]]; then
+  echo "ERROR: This script must be run as root" 1>&2
+  exit 1
+fi
+
+if [[ "${MICRO_BOSH_STEMCELL_NAME}X" == "X" ]]; then
+  echo 'INTERNAL ERROR: Missing $MICRO_BOSH_STEMCELL_NAME environment variable'
+  exit 1
+fi
+
+if [[ "${MICRO_BOSH_STEMCELL_NAME}" == "custom" ]]; then
+
+  BOSH_RELEASE_REPO=${BOSH_RELEASE_REPO:-'https://github.com/cloudfoundry/bosh-release.git'}
+  BOSH_RELEASE_DIRNAME=${BOSH_RELEASE_DIRNAME:-'bosh-release'}
+  BOSH_RELEASE_DIR=/var/vcap/store/releases/$BOSH_RELEASE_DIRNAME
+
+  echo "Creating custom stemcell..."
+
+  if [[ -d /var/tmp/bosh/ ]]; then
+    echo "But first, cleaning out previous stemcell temporary files..."
+    rm -rf /var/tmp/bosh
+  fi
+
+
+  cd $(dirname $BOSH_RELEASE_DIR)
+  if [[ -d $BOSH_RELEASE_DIRNAME ]]; then
+    echo "Updating BOSH's own bosh-release for custom stemcell"
+    cd $BOSH_RELEASE_DIRNAME
+    git pull origin master
+  else
+    echo "Fetching BOSH's own bosh-release for custom stemcell"
+    git clone $BOSH_RELEASE_REPO
+    cd $BOSH_RELEASE_DIRNAME
+  fi
+
+  echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+
+  echo "Rewriting all git:// & git@ to https:// to allow access if port 22 is blocked..."
+  # Snippet written by Mike Reeves <swampfoxmr@gmail.com> on bosh-users mailing list
+  # Date 2012-12-06
+  grep -rI "git[@:/]\{0,3\}github.com" * .gitmodules | awk 'BEGIN {FS=":"} { print($1) }' | while read file
+  do
+    echo "changing - $file"
+    sed -i 's/git\:\/\/github.com/https:\/\/github.com/g' $file
+    sed -i 's/git@github.com:/https:\/\/github.com\//g' $file
+  done
+
+  echo "Updating source submodules..."
+  git submodule update --init
+
+  echo "Within submodules, rewriting all git:// & git@ to https:// to allow access if port 22 is blocked..."
+  # Snippet written by Mike Reeves <swampfoxmr@gmail.com> on bosh-users mailing list
+  # Date 2012-12-06
+  grep -rI "git[@:/]\{0,3\}github.com" * .gitmodules | awk 'BEGIN {FS=":"} { print($1) }' | while read file
+  do
+    echo "changing - $file"
+    sed -i 's/git\:\/\/github.com/https:\/\/github.com/g' $file
+    sed -i 's/git@github.com:/https:\/\/github.com\//g' $file
+  done
+
+  echo "Creating bosh release (fetching any new blobs)..."
+  bosh -n create release --force --with-tarball
+
+  # dev_releases/index.yml contains the dev_releases
+  # and its last line contains the latest dev_release name
+  # For example:
+  # --- 
+  # builds: 
+  #   4800ad1226e2b673595a8646e8e11bc968ab6ab6: 
+  #     version: 119.1-dev
+  #   7e0318d711daf0a5fa2e65c0257cd3b6af2ec4e5: 
+  #     version: 119.2-dev
+  #
+  # So to get the latest dev_release name, grab the last line's 2nd token
+  # For example, above this would be "119.2-dev"
+  RELEASE_NAME="bosh-release"
+  LATEST_DEV_RELEASE_NAME=$(tail -n 1 dev_releases/index.yml | awk '{print $2}')
+  LATEST_BOSH_RELEASE_PATH=$BOSH_RELEASE_DIR/dev_releases/$RELEASE_NAME-$LATEST_DEV_RELEASE_NAME.tgz
+  MICRO_BOSH_MANIFEST_PATH=$BOSH_RELEASE_DIR/micro/$PROVIDER.yml
+
+  BOSH_DIR=/var/vcap/store/repos/bosh
+  cd $(dirname $BOSH_DIR)
+  if [[ -d bosh ]]; then
+    echo "Updating bosh source for agent/stemcell scripts"
+    cd bosh
+    git pull origin master
+  else
+    echo "Cloning bosh source for agent/stemcell scripts"
+    git clone https://github.com/cloudfoundry/bosh.git
+    cd bosh
+  fi
+
+  cd $BOSH_DIR
+  echo "Creating custom stemcell..."
+  bundle install --without test development
+  bundle exec rake stemcell:micro[$PROVIDER,$MICRO_BOSH_MANIFEST_PATH,$LATEST_BOSH_RELEASE_PATH]
+
+  echo "Copying to stemcells folder..."
+  MICRO_BOSH_STEMCELL_PATH=$(ls -t /var/tmp/bosh/agent-*/work/work/*.tgz | sort | tail -n 1)
+  mv $MICRO_BOSH_STEMCELL_PATH /var/vcap/store/stemcells
+  MICRO_BOSH_STEMCELL_NAME=$(basename $MICRO_BOSH_STEMCELL_PATH)
+
+  echo "Cleaning up..."
+  rm $(dirname $LATEST_BOSH_RELEASE_PATH)/*.tgz
+
+elif [[ -f /var/vcap/store/stemcells/$MICRO_BOSH_STEMCELL_NAME ]]; then
+  echo "Stemcell $(pwd)/$MICRO_BOSH_STEMCELL_NAME already exists."
+
+else
+  cd /var/vcap/store/stemcells
+  echo "Downloading public stemcell $MICRO_BOSH_STEMCELL_NAME"
+  # set $COLUMNS manually else progressbar runs `tput cols`.to_i which causes grief
+  COLUMNS=80 bosh -n download public stemcell $MICRO_BOSH_STEMCELL_NAME
+fi
+
+# print name of stemcell created/downloaded to be stored back in settings
+echo $MICRO_BOSH_STEMCELL_NAME