bosh-bootstrap 0.10.2 → 0.11.0

data/TODO.md

@@ -0,0 +1,55 @@
+## Known issues
+
+* target & create user after deployment
+* upgrade (if its already running successfully; else delete & redeploy)
+* multiple deployments
+  * bosh name (currently fixed at test-bosh)
+  * ~/.microbosh folder is singular
+
+### Display status
+
+```
+$ ~/.microbosh/deployments 
+$ bundle exec bosh micro status
+Stemcell CID   ami-53137a3a
+Stemcell name  ami-53137a3a
+VM CID         i-78046912
+Disk CID       vol-cebe9196
+Micro BOSH CID bm-2b31bf69-b676-4db1-a084-fd1ac49377ac
+Deployment     /Users/drnic/.microbosh/deployments/test-bosh/micro_bosh.yml
+Target         https://107.21.94.132:25555
+```
+### Create user
+
+Collect user/password at start of process; then target & create user at the end
+
+```
+bosh -u admin -p admin target https://107.21.94.132:25555
+bosh -u admin -p admin create user drnic PASSWORD (do not display password)
+```
+
+## Multiple deployments
+
+* bosh name (currently fixed at test-bosh)
+* ~/.microbosh folder is singular
+
+## Bonus
+
+* AWS/us-east-1 - upload light stemcell
+* Others - upload normal base stemcell
+
+## Validations
+
+If using stemcells, then must be Ubuntu with following packages installed:
+
+* libsqlite3-dev
+* genisoimage
+
+* IP address - like key pair, check that its still available else create new one
+
+How can we validate that there is enough disk space to download & prepare the stemcell?
+
+### AWS
+
+* check if this server is in AWS & same region as target region, else only use AMIs (and switch to us-east-1)
+

data/bosh-bootstrap.gemspec

@@ -8,10 +8,9 @@ Gem::Specification.new do |gem|
   gem.version       = Bosh::Bootstrap::VERSION
   gem.authors       = ["Dr Nic Williams"]
   gem.email         = ["drnicwilliams@gmail.com"]
-  gem.description   = %q{Bootstrap a micro BOSH universe from one CLI}
+  gem.description   = %q{Bootstrap a micro bosh universe from one CLI}
   gem.summary       = <<-EOS
-bosh-bootstrap is a command line tool that you can run on your laptop and
-automatically get a microbosh (and an inception VM) deployed on either
+bosh-bootstrap configures and deploys a microbosh deployed on either
 AWS or OpenStack.
 EOS
   gem.homepage      = "https://github.com/StarkAndWayne/bosh-bootstrap"
@@ -23,17 +22,11 @@ EOS
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "thor"
-  gem.add_dependency "highline"
-  gem.add_dependency "settingslogic"
-  gem.add_dependency "POpen4"
-  gem.add_dependency "net-ssh", "~> 2.2.1"
-  gem.add_dependency "net-scp", "~> 1.0.4"
-  gem.add_dependency "fog", "~>1.8.0"
-  gem.add_dependency "escape"
+  gem.add_dependency "cyoi"
+  gem.add_dependency "readwritesettings", "~> 3.0"
   gem.add_dependency "redcard"
   gem.add_dependency "bosh_cli"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec"
-  gem.add_development_dependency "activesupport", ">= 3.0.0"
+  gem.add_development_dependency "fakeweb"
 end

data/lib/bosh/cli/commands/bootstrap.rb

@@ -0,0 +1,42 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+require "bosh-bootstrap"
+
+module Bosh::Cli::Command
+  class Bootstrap < Base
+    usage "bootstrap"
+    desc  "show bootstrap sub-commands"
+    def help
+      say("bosh bootstrap sub-commands:")
+      nl
+      cmds = Bosh::Cli::Config.commands.values.find_all {|c|
+        c.usage =~ /^bootstrap/
+      }
+      Bosh::Cli::Command::Help.list_commands(cmds)
+    end
+
+    usage "bootstrap deploy"
+    desc "Configure and bootstrap a micro bosh; or deploy/upgrade existing Micro Bosh"
+    def deploy
+      require "bosh-bootstrap/cli/commands/deploy"
+      deploy_cmd = Bosh::Bootstrap::Cli::Commands::Deploy.new
+      deploy_cmd.perform
+    end
+
+    usage "bootstrap ssh"
+    desc "SSH into micro bosh"
+    def ssh
+      require "bosh-bootstrap/cli/commands/ssh"
+      cmd = Bosh::Bootstrap::Cli::Commands::SSH.new
+      cmd.perform
+    end
+
+    usage "bootstrap delete"
+    desc "Delete existing Micro Bosh (does not delete any bosh deployments running)"
+    def delete
+      require "bosh-bootstrap/cli/commands/delete"
+      cmd = Bosh::Bootstrap::Cli::Commands::Delete.new
+      cmd.perform
+    end
+  end
+end

data/lib/bosh-bootstrap/cli/commands/delete.rb

@@ -0,0 +1,26 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Bosh; module Bootstrap; module Cli; module Commands; end; end; end; end
+
+require "bosh-bootstrap/cli/helpers"
+
+# Runs SSH to the microbosh server
+class Bosh::Bootstrap::Cli::Commands::Delete
+  include Bosh::Bootstrap::Cli::Helpers
+
+  def perform
+    chdir(deployment_dir) do
+      bundle "exec bosh -n micro deployment #{bosh_name}"
+      bundle "exec bosh -n micro delete"
+    end
+  end
+
+  protected
+  def bosh_name
+    settings.bosh.name
+  end
+
+  def deployment_dir
+    File.join(settings_dir, "deployments")
+  end
+end

data/lib/bosh-bootstrap/cli/commands/deploy.rb

@@ -0,0 +1,89 @@
+module Bosh; module Bootstrap; module Cli; module Commands; end; end; end; end
+
+require "cyoi/cli/provider"
+require "cyoi/cli/address"
+require "cyoi/cli/key_pair"
+require "bosh-bootstrap/cli/helpers"
+require "bosh-bootstrap/microbosh"
+
+class Bosh::Bootstrap::Cli::Commands::Deploy
+  include Bosh::Bootstrap::Cli::Helpers
+
+  # * select_provider
+  # * select_or_provision_public_networking # public_ip or ip/network/gateway
+  # * select_public_image_or_download_stemcell # download if stemcell
+  # * create_microbosh_manifest
+  # * microbosh_deploy
+  def perform
+    settings.set_default("bosh.name", "firstbosh")
+    save_settings!
+
+    select_provider
+    select_or_provision_public_networking
+    setup_keypair
+    select_public_image_or_download_stemcell
+    perform_microbosh_deploy
+  end
+
+  protected
+  def select_provider
+    provider = Cyoi::Cli::Provider.new([settings_dir])
+    provider.execute!
+    reload_settings!
+  end
+
+  def provider_client
+    @provider_client ||= Cyoi::Providers.provider_client(settings.provider)
+  end
+
+  # public_ip or ip/network/gateway
+  def select_or_provision_public_networking
+    address = Cyoi::Cli::Address.new([settings_dir])
+    address.execute!
+    reload_settings!
+
+    # TODO why passing provider_client rather than a Cyoi::Cli::Network object?
+    network = Bosh::Bootstrap::Network.new(settings.provider.name, provider_client)
+    network.deploy
+  end
+
+  def setup_keypair
+    key_pair_name = settings.exists?("key_pair.name") || settings.bosh.name
+    cli = Cyoi::Cli::KeyPair.new([key_pair_name, settings_dir])
+    cli.execute!
+    reload_settings!
+
+    key_pair = Bosh::Bootstrap::KeyPair.new(settings_dir, settings.key_pair.name, settings.key_pair.private_key)
+    key_pair.execute!
+    settings.set("key_pair.path", key_pair.path)
+  end
+
+  # TODO should this go inside Microbosh, like NetworkProvider is to Network?
+  def microbosh_provider
+    @microbosh_provider ||= begin
+      provider_name = settings.provider.name
+      require "bosh-bootstrap/microbosh_providers/#{provider_name}"
+      klass = Bosh::Bootstrap::MicroboshProviders.provider_class(provider_name)
+      klass.new(File.join(settings_dir, "deployments/#{settings.bosh.name}/micro_bosh.yml"), settings)
+    end
+  end
+
+  # download if stemcell
+  def select_public_image_or_download_stemcell
+    print "Determining stemcell image/file to use... "
+    stemcell = microbosh_provider.stemcell
+    while (stemcell || "").size == 0
+      puts "failed. Retrying..."
+      print "Determining stemcell image/file to use... "
+      stemcell = microbosh_provider.stemcell
+    end
+    settings.set("bosh.stemcell", stemcell)
+    puts settings.bosh.stemcell
+  end
+
+  def perform_microbosh_deploy
+    settings.set("bosh.persistent_disk", 16384)
+    @microbosh ||= Bosh::Bootstrap::Microbosh.new(settings_dir, microbosh_provider)
+    @microbosh.deploy(settings)
+  end
+end

data/lib/bosh-bootstrap/cli/commands/ssh.rb

@@ -0,0 +1,32 @@
+# Copyright (c) 2012-2013 Stark & Wayne, LLC
+
+module Bosh; module Bootstrap; module Cli; module Commands; end; end; end; end
+
+require "bosh-bootstrap/cli/helpers"
+
+# for the #sh helper
+require "rake"
+require "rake/file_utils"
+
+# Runs SSH to the microbosh server
+class Bosh::Bootstrap::Cli::Commands::SSH
+  include Bosh::Bootstrap::Cli::Helpers
+  include FileUtils
+
+  def perform
+    sh "ssh -i #{private_key_path} #{user}@#{host}"
+  end
+
+  protected
+  def user
+    "vcap"
+  end
+
+  def host
+    settings.address.ip
+  end
+
+  def private_key_path
+    File.expand_path(settings.key_pair.path)
+  end
+end

data/lib/bosh-bootstrap/cli/helpers/bundle.rb

@@ -0,0 +1,12 @@
+# for the #sh helper
+require "rake"
+require "rake/file_utils"
+
+module Bosh::Bootstrap::Cli::Helpers::Bundle
+  def bundle(*args)
+    Bundler.with_clean_env {
+      ENV.delete 'RUBYOPT'
+      sh "bundle #{args.join(' ')}"
+    }
+  end
+end

data/lib/bosh-bootstrap/cli/helpers/interactions.rb

@@ -0,0 +1,15 @@
+require "highline"
+
+module Bosh::Bootstrap::Cli::Helpers::Interactions
+  def cyan; "\033[36m" end
+  def clear; "\033[0m" end
+  def bold; "\033[1m" end
+  def red; "\033[31m" end
+  def green; "\033[32m" end
+  def yellow; "\033[33m" end
+
+  # Helper to access HighLine for ask & menu prompts
+  def hl
+    @hl ||= HighLine.new(@stdin, @stdout)
+  end
+end

data/lib/bosh-bootstrap/cli/helpers/settings.rb

@@ -0,0 +1,61 @@
+require "readwritesettings"
+require "fileutils"
+
+module Bosh::Bootstrap::Cli::Helpers::Settings
+  include FileUtils
+
+  # The base directory for holding the manifest settings file
+  # and private keys
+  #
+  # Defaults to ~/.bosh_inception; and can be overridden with either:
+  # * $SETTINGS - to a folder (supported method)
+  def settings_dir
+    @settings_dir ||= File.expand_path(ENV["SETTINGS"] || "~/.microbosh")
+  end
+
+  def settings_dir=(settings_dir)
+    @settings_dir = File.expand_path(settings_dir)
+    reload_settings!
+  end
+
+  def settings_ssh_dir
+    File.join(settings_dir, "ssh")
+  end
+
+  def settings_path
+    @settings_path ||= File.join(settings_dir, "settings.yml")
+  end
+
+  def settings
+    @settings ||= begin
+      unless File.exists?(settings_path)
+        mkdir_p(settings_dir)
+        File.open(settings_path, "w") { |file| file << "--- {}" }
+      end
+      chmod(0600, settings_path)
+      chmod(0700, settings_ssh_dir) if File.directory?(settings_ssh_dir)
+      ReadWriteSettings.new(settings_path)
+    end
+  end
+
+  # Set a nested setting with "key1.key2.key3" notation
+  def setting(nested_key, value)
+    settings.set(nested_key, value)
+    save_settings!
+  end
+
+  # Saves current nested ReadWriteSettings into pure Hash-based YAML file
+  # Recreates accessors on ReadWriteSettings object (since something has changed)
+  def save_settings!
+    File.open(settings_path, "w") { |f| f << settings.to_nested_hash.to_yaml }
+    settings.create_accessors!
+  end
+
+  def reload_settings!
+    @settings = nil
+    settings
+  end
+
+  def migrate_old_settings
+  end
+end

data/lib/bosh-bootstrap/cli/helpers.rb

@@ -0,0 +1,11 @@
+module Bosh; module Bootstrap; module Cli; module Helpers; end; end; end; end
+
+require "bosh-bootstrap/cli/helpers/bundle"
+require "bosh-bootstrap/cli/helpers/interactions"
+require "bosh-bootstrap/cli/helpers/settings"
+
+module Bosh::Bootstrap::Cli::Helpers
+  include Bosh::Bootstrap::Cli::Helpers::Bundle
+  include Bosh::Bootstrap::Cli::Helpers::Interactions
+  include Bosh::Bootstrap::Cli::Helpers::Settings
+end

data/lib/bosh-bootstrap/key_pair.rb

@@ -0,0 +1,21 @@
+class Bosh::Bootstrap::KeyPair
+  include FileUtils
+
+  attr_reader :base_path, :keyname, :private_key
+
+  def initialize(base_path, keyname, private_key)
+    @base_path, @keyname, @private_key = base_path, keyname, private_key
+  end
+
+  def execute!
+    mkdir_p(File.dirname(path))
+    chmod(0700, File.dirname(path))
+
+    File.open(path, "w") { |file| file << private_key }
+    chmod(0600, path)
+  end
+
+  def path
+    @path ||= File.join(base_path, "ssh", keyname)
+  end
+end

data/lib/bosh-bootstrap/microbosh.rb

@@ -0,0 +1,74 @@
+require "bosh-bootstrap/microbosh_providers"
+require "bosh-bootstrap/cli/helpers"
+
+# Configures and deploys (or re-deploys) a micro bosh.
+# A "micro bosh" is a single VM containing all necessary parts of bosh
+# and is deployed from the terminal; rather than from another bosh.
+#
+# Usage:
+#   microbosh = Bosh::Bootstrap::Microbosh.new(project_path)
+#   settings = ReadWriteSettings.new({
+#     "provider" => {"name" => "aws", "credentials" => {...}},
+#     "address" => {"ip" => "1.2.3.4"},
+#     "bosh" => {
+#       "name" => "test-bosh",
+#       "stemcell" => "ami-123456",
+#       "salted_password" => "452435hjg2345hjg2435ghk3452"
+#     }
+#   })
+#   microbosh.deploy("aws", settings)
+class Bosh::Bootstrap::Microbosh
+  include FileUtils
+  include Bosh::Bootstrap::Cli::Helpers::Bundle
+
+  attr_reader :base_path
+  attr_reader :provider
+  attr_reader :bosh_name
+  attr_reader :deployments_dir
+  attr_reader :manifest_yml
+
+  def initialize(base_path, provider)
+    @base_path = base_path
+    @provider = provider
+  end
+
+  def deploy(settings)
+    @bosh_name = settings.bosh.name
+    @deployments_dir = File.join(base_path, "deployments")
+    @manifest_yml = File.join(deployments_dir, bosh_name, "micro_bosh.yml")
+    mkdir_p(File.dirname(manifest_yml))
+    chdir(base_path) do
+      setup_base_path
+      create_microbosh_yml(settings)
+      deploy_or_update(settings.bosh.name, settings.bosh.stemcell)
+    end
+  end
+
+  protected
+  def setup_base_path
+    gempath = File.expand_path("../../..", __FILE__)
+    pwd = File.expand_path(".")
+    File.open("Gemfile", "w") do |f|
+      f << <<-RUBY
+source 'https://rubygems.org'
+source 'https://s3.amazonaws.com/bosh-jenkins-gems/'
+
+gem "bosh-bootstrap", path: "#{gempath}"
+gem "bosh_cli_plugin_micro"
+      RUBY
+    end
+    rm_rf "Gemfile.lock"
+    bundle "install"
+  end
+
+  def create_microbosh_yml(settings)
+    provider.create_microbosh_yml(settings)
+  end
+
+  def deploy_or_update(bosh_name, stemcell)
+    chdir("deployments") do
+      bundle "exec bosh micro deployment", bosh_name
+      bundle "exec bosh -n micro deploy", stemcell
+    end
+  end
+end

data/lib/bosh-bootstrap/microbosh_providers/aws.rb

@@ -0,0 +1,104 @@
+require "bosh-bootstrap/microbosh_providers/base"
+
+module Bosh::Bootstrap::MicroboshProviders
+  class AWS < Base
+    # if us-east-1 -> ami
+    # if not running in target aws region -> error "Must either use us-east-1 or run 'bosh bootstrap deploy' within target AWS region"
+    # else download stemcell & return path
+    def stemcell
+      unless settings.exists?("bosh.stemcell")
+        if ami_region?
+          fetch_ami
+        else
+          download_stemcell
+        end
+      end
+    end
+
+    def to_hash
+      super.merge({
+       "network"=>{"type"=>"dynamic", "vip"=>public_ip},
+       "resources"=>
+        {"persistent_disk"=>persistent_disk,
+         "cloud_properties"=>resources_cloud_properties},
+       "cloud"=>
+        {"plugin"=>"aws",
+         "properties"=>
+          {"aws"=>cloud_properties}},
+       "apply_spec"=>
+        {"agent"=>
+          {"blobstore"=>{"address"=>public_ip},
+           "nats"=>{"address"=>public_ip}},
+         "properties"=>{"aws_registry"=>{"address"=>public_ip}}}})
+    end
+
+    def persistent_disk
+      settings.bosh.persistent_disk
+    end
+
+    def resources_cloud_properties
+      {"instance_type"=>"m1.medium"}
+    end
+
+    def cloud_properties
+      {"access_key_id"=>settings.provider.credentials.aws_access_key_id,
+       "secret_access_key"=>settings.provider.credentials.aws_secret_access_key,
+       "region"=>settings.provider.region,
+       "ec2_endpoint"=>"ec2.#{settings.provider.region}.amazonaws.com",
+       "default_security_groups"=>security_groups,
+       "default_key_name"=>microbosh_name,
+       "ec2_private_key"=>private_key_path}
+    end
+
+    def security_groups
+      ["ssh",
+       "bosh_agent_http",
+       "bosh_nats_server",
+       "bosh_blobstore",
+       "bosh_director",
+       "bosh_registry"]
+    end
+
+    def aws_region
+      settings.provider.region
+    end
+
+    # only us-east-1 has AMIs published currently
+    def ami_region?
+      aws_region == "us-east-1"
+    end
+
+    def aws_jenkins_bucket
+      "bosh-jenkins-artifacts"
+    end
+
+    def fetch_ami
+      Net::HTTP.get("#{aws_jenkins_bucket}.s3.amazonaws.com", ami_uri_path(aws_region)).strip
+    end
+
+    def ami_uri_path(region)
+      "/last_successful_micro-bosh-stemcell-aws_ami_#{region}"
+    end
+
+    def stemcell_uri
+      "http://#{aws_jenkins_bucket}.s3.amazonaws.com/last_successful_micro-bosh-stemcell-aws.tgz"
+    end
+
+    # downloads latest stemcell & returns path
+    def download_stemcell
+      mkdir_p(stemcell_dir)
+      chdir(stemcell_dir) do
+        stemcell_path = File.expand_path(File.basename(stemcell_uri))
+        unless File.exists?(stemcell_path)
+          sh "curl -O '#{stemcell_uri}'"
+        end
+        return stemcell_path
+      end
+    end
+
+    def stemcell_dir
+      File.dirname(manifest_path)
+    end
+  end
+end
+Bosh::Bootstrap::MicroboshProviders.register_provider("aws", Bosh::Bootstrap::MicroboshProviders::AWS)

data/lib/bosh-bootstrap/microbosh_providers/base.rb

@@ -0,0 +1,50 @@
+require "bosh-bootstrap/microbosh_providers"
+
+# for the #sh helper
+require "rake"
+require "rake/file_utils"
+
+class Bosh::Bootstrap::MicroboshProviders::Base
+  include FileUtils
+
+  attr_reader :manifest_path
+  attr_reader :settings
+
+  def initialize(manifest_path, settings)
+    @manifest_path = manifest_path
+    @settings = settings.is_a?(Hash) ? ReadWriteSettings.new(settings) : settings
+    raise "@settings must be ReadWriteSettings (or Hash)" unless @settings.is_a?(ReadWriteSettings)
+  end
+
+  def create_microbosh_yml(settings)
+    @settings = settings.is_a?(Hash) ? ReadWriteSettings.new(settings) : settings
+    raise "@settings must be ReadWriteSettings (or Hash)" unless @settings.is_a?(ReadWriteSettings)
+    mkdir_p(File.dirname(manifest_path))
+    File.open(manifest_path, "w") do |f|
+      f << self.to_hash.to_yaml
+    end
+  end
+
+  def to_hash
+    {"name"=>microbosh_name,
+     "logging"=>{"level"=>"DEBUG"}
+    }
+  end
+
+  def microbosh_name
+    settings.bosh.name
+  end
+
+  def salted_password
+    # BCrypt::Password.create(settings.bosh.password).to_s.force_encoding("UTF-8")
+    settings.bosh.salted_password
+  end
+
+  def public_ip
+    settings.address.ip
+  end
+
+  def private_key_path
+    settings.key_pair.path
+  end
+end