bosh_deployer 0.1.4

data/README.rdoc

@@ -0,0 +1,139 @@
+= DESCRIPTION:
+
+BOSH Deployer is used to deploy a "micro BOSH".  The micro BOSH instance can be used standalone or to deploy additional BOSH instances.
+
+Installing the bosh deployer includes an installation of the BOSH cli and a deployer cli plugin called 'micro'.
+
+It is recommend that you install into an empty gemset (or similar).
+
+= REQUIREMENTS:
+
+  % apt-get -y install libsqlite3-dev genisoimage
+
+  % cd bosh/deployer && rake install
+
+= CONFIGURATION:
+
+For a minimal configuration example, see: deployer/spec/assets/test-bootstrap-config.yml
+
+Note that 'disk_path' is 'BOSH_Deployer' rather than 'BOSH_Disks'.
+
+A datastore folder other than 'BOSH_Disks' is required if your vCenter hosts other directors.
+
+The disk_path folder needs to be created manually.
+
+Your configuration must live inside a 'deployments' directory and follow the convention of having a $name subdir containing micro_bosh.yml,
+where $name is your deployment name.  For example:
+
+  % find deployments -name micro_bosh.yml
+  deployments/vcs01/micro_bosh.yml
+  deployments/dev32/micro_bosh.yml
+  deployments/dev33/micro_bosh.yml
+
+The 'bosh micro' commands must also be run within the deployments directory.
+
+Deployment state is persisted to deployments/bosh-deployments.yml
+
+= USAGE:
+
+First, be sure to read the CONFIGURATION section and cd into your deployments/ directory.
+
+== help
+
+The bosh micro plugin help is display after the standard bosh command usage:
+
+  % bosh help
+  ...
+  Micro
+    micro deployment [<name>] Choose micro deployment to work with
+    micro status              Display micro BOSH deployment status
+    micro deployments         Show the list of deployments
+    micro deploy <stemcell>   Deploy a micro BOSH instance to the currently
+                              selected deployment
+                              --update   update existing instance
+    micro delete              Delete micro BOSH instance (including
+                              persistent disk)
+    micro agent <args>        Send agent messages
+    micro apply <spec>        Apply spec
+
+== deployment
+
+Set the micro bosh deployment to work with:
+
+  % bosh micro deployment dev33
+  Deployment set to '/var/vcap/deployments/dev33/micro_bosh.yml'
+
+== deploy
+
+Deploy a new micro BOSH instance and create a new persistent disk.
+
+  % bosh micro deploy ~/cf/stemcells/micro-bosh-stemcell-0.4.2.tgz
+
+== deploy --update
+
+Update an existing micro BOSH instance.  The existing persistent disk will be attached to the new VM.
+
+  % bosh micro deploy ~/cf/stemcells/micro-bosh-stemcell-0.4.5.tgz --update
+
+== delete
+
+The delete command will delete the VM, stemcell and persistent disk.
+
+Example:
+
+  % bosh micro delete
+
+== status
+
+The status command will show the persisted state for a given micro bosh instance.
+
+  % bosh micro status
+  Stemcell CID   sc-f2430bf9-666d-4034-9028-abf9040f0edf
+  Stemcell name  micro-bosh-stemcell-0.4.5
+  VM CID         vm-9cc859a4-2d51-43ca-8dd5-220425518fd8
+  Disk CID       1
+  Deployment     /var/vcap/deployments/dev33/micro_bosh.yml
+  Target         micro (http://172.23.194.100:25555) Ver: 0.3.12 (00000000)
+
+== deployments
+
+Show the list of deployments, this is just a table view of deployments/bosh-deployments.yml.
+
+ % bosh micro deployments
+
+== apply
+
+The micro-bosh-stemcell includes an embedded apply_spec.yml.  This command can be used to apply a different spec to an existing instance.
+The apply_spec.yml properties are merged with your deployment's network.ip and cloud.properties.vcenters properties.
+
+  % bosh micro apply apply_spec.yml
+
+== agent
+
+The cli can send agent messages over HTTP.
+
+Example:
+
+  % bosh micro agent ping
+  "pong"
+
+= BOSH:
+
+Once your micro BOSH instance is deployed, you can target its director:
+
+  $ bosh micro status
+  ...
+  Target         micro (http://172.23.194.100:25555) Ver: 0.3.12 (00000000)
+
+  $ bosh target http://172.23.194.100:25555
+  Target set to 'micro (http://172.23.194.100:25555) Ver: 0.3.12 (00000000)'
+
+  $ bosh status
+  Updating director data... done
+
+  Target         micro (http://172.23.194.100:25555) Ver: 0.3.12 (00000000)
+  UUID           b599c640-7351-4717-b23c-532bb35593f0
+  User           admin
+  Deployment     not set
+
+You can use micro BOSH as-is or to deploy new BOSH instances using micro BOSH: https://github.com/vmware-ac/bosh-release

data/Rakefile

@@ -0,0 +1,51 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+$:.unshift(File.expand_path("../../rake", __FILE__))
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __FILE__)
+
+require "rubygems"
+require "bundler"
+Bundler.setup(:default, :test)
+
+require "rake/dsl_definition"
+require "rake"
+begin
+  require "rspec/core/rake_task"
+rescue LoadError
+end
+
+require "bundler_task"
+require "ci_task"
+
+gem_helper = Bundler::GemHelper.new(Dir.pwd)
+
+desc "Build BOSH Deployer gem into the pkg directory"
+task "build" do
+  gem_helper.build_gem
+end
+
+desc "Build and install BOSH Deployer gem into system gems"
+task "install" do
+  Rake::Task["bundler:install"].invoke
+  gem_helper.install_gem
+end
+
+BundlerTask.new
+
+if defined?(RSpec)
+  namespace :spec do
+    desc "Run Unit Tests"
+    rspec_task = RSpec::Core::RakeTask.new(:unit) do |t|
+      t.pattern = "spec/unit/**/*_spec.rb"
+      t.rspec_opts = %w(--format progress --colour)
+    end
+
+    CiTask.new do |task|
+      task.rspec_task = rspec_task
+    end
+  end
+
+  desc "Install dependencies and run tests"
+  task :spec => %w(bundler:install:test spec:unit)
+end

data/config/aws_defaults.yml

@@ -0,0 +1,45 @@
+---
+name:
+
+logging:
+  level: INFO
+
+dir:
+
+network:
+  type: dynamic
+  cloud_properties: {}
+
+resources:
+  persistent_disk: 4096
+  cloud_properties:
+    instance_type: m1.small
+    availability_zone:
+
+cloud:
+  plugin: aws
+  properties:
+    aws:
+      access_key_id:
+      secret_access_key:
+      ec2_endpoint:
+      max_retries: 2
+      default_key_name: dougm
+      default_security_groups: []
+    registry:
+      endpoint: http://admin:admin@localhost:25777
+      user: admin
+      password: admin
+    stemcell:
+      kernel_id:
+      disk_size: 4096
+    agent:
+      ntp: []
+      blobstore:
+        plugin: local
+        properties:
+          blobstore_path: /var/vcap/micro_bosh/data/cache
+      mbus:
+
+apply_spec:
+  properties: {}

data/config/vsphere_defaults.yml

@@ -0,0 +1,38 @@
+---
+name:
+
+logging:
+  level: INFO
+
+dir:
+
+network:
+  ip:
+  netmask:
+  gateway:
+  dns: []
+  cloud_properties:
+    name:
+
+resources:
+  persistent_disk: 4096
+  cloud_properties:
+    ram: 1024
+    disk: 4096
+    cpu: 1
+
+cloud:
+  plugin: vsphere
+  properties:
+    agent:
+      ntp: []
+      blobstore:
+        plugin: local
+        properties:
+          blobstore_path: /var/vcap/micro_bosh/data/cache
+      mbus:
+    soap_log:
+    vcenters: []
+
+apply_spec:
+  properties: {}

data/lib/bosh/cli/commands/micro.rb

@@ -0,0 +1,393 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require "deployer"
+
+module Bosh::Cli::Command
+  class Micro < Base
+    include Bosh::Cli::DeploymentHelper
+
+    MICRO_DIRECTOR_PORT = 25555
+    DEFAULT_CONFIG_PATH = File.expand_path("~/.bosh_deployer_config")
+
+    command :micro_deployment do
+      usage "micro deployment [<name>]"
+      desc  "Choose micro deployment to work with"
+      route { |args| (args.size > 0) ? [:micro, :set_current] : [:micro, :show_current] }
+    end
+
+    command :micro_status do
+      usage "micro status"
+      desc  "Display micro BOSH deployment status"
+      route :micro, :status
+    end
+
+    command :micro_list_deployments do
+      usage "micro deployments"
+      desc  "Show the list of deployments"
+      route :micro, :list
+    end
+
+    command :micro_deploy do
+      usage  "micro deploy <stemcell>"
+      desc   "Deploy a micro BOSH instance to the currently selected deployment"
+      option "--update", "update existing instance"
+      route  :micro, :perform
+    end
+
+    command :micro_delete do
+      usage  "micro delete"
+      desc   "Delete micro BOSH instance (including persistent disk)"
+      route  :micro, :delete
+    end
+
+    command :micro_agent do
+      usage "micro agent <args>"
+      desc  "Send agent messages"
+      route :micro, :agent
+    end
+
+    command :micro_apply do
+      usage "micro apply <spec>"
+      desc  "Apply spec"
+      route :micro, :apply
+    end
+
+    def initialize(options = {})
+      options[:config] ||= DEFAULT_CONFIG_PATH #hijack Cli::Config
+      super(options)
+    end
+
+    def status
+      stemcell_cid = deployer_state(:stemcell_cid)
+      stemcell_name = deployer_state(:stemcell_name)
+      vm_cid = deployer_state(:vm_cid)
+      disk_cid = deployer_state(:disk_cid)
+      deployment = config.deployment ? config.deployment.green : "not set".red
+
+      say("Stemcell CID".ljust(15) + stemcell_cid)
+      say("Stemcell name".ljust(15) + stemcell_name)
+      say("VM CID".ljust(15) + vm_cid)
+      say("Disk CID".ljust(15) + disk_cid)
+      say("Micro BOSH CID".ljust(15) + Bosh::Deployer::Config.uuid)
+      say("Deployment".ljust(15) + deployment)
+
+      update_target
+
+      target_name = full_target_name ? full_target_name.green : "not set".red
+      say("Target".ljust(15) + target_name)
+    end
+
+    def perform(*options)
+      update = options.delete("--update")
+      tarball_path = options.shift
+
+      if tarball_path.nil?
+        err "No stemcell provided"
+      end
+
+      rel_path = deployment[/#{Regexp.escape File.join(work_dir, '')}(.*)/, 1]
+
+      desc = "`#{rel_path.green}' to `#{target_name.green}'"
+
+      if update
+        unless deployer.exists?
+          err "No existing instance to update"
+        end
+
+        confirmation = "Updating"
+
+        method = :update
+      else
+        if deployer.exists?
+          err "Instance exists.  Did you mean to --update?"
+        end
+
+        confirmation = "Deploying new"
+
+        method = :create
+      end
+
+      confirm_deployment("#{confirmation} micro BOSH instance #{desc}")
+
+      unless File.directory?(tarball_path)
+        stemcell = Bosh::Cli::Stemcell.new(tarball_path, cache)
+
+        say("\nVerifying stemcell...")
+        stemcell.validate
+        say("\n")
+
+        unless stemcell.valid?
+          err("Stemcell is invalid, please fix, verify and upload again")
+        end
+      end
+
+      renderer = DeployerRenderer.new
+      renderer.start
+      deployer.renderer = renderer
+
+      start_time = Time.now
+
+      deployer.send(method, tarball_path)
+
+      renderer.finish("done")
+
+      duration = renderer.duration || (Time.now - start_time)
+
+      update_target
+
+      say("Deployed #{desc}, took #{format_time(duration).green} to complete")
+    end
+
+    def delete
+      unless deployer.exists?
+        err "No existing instance to delete"
+      end
+
+      name = deployer.state.name
+
+      say "\nYou are going to delete micro BOSH deployment `#{name}'.\n\n" \
+      "THIS IS A VERY DESTRUCTIVE OPERATION AND IT CANNOT BE UNDONE!\n".red
+
+      unless confirmed?
+        say "Canceled deleting deployment".green
+        return
+      end
+
+      renderer = DeployerRenderer.new
+      renderer.start
+      deployer.renderer = renderer
+
+      start_time = Time.now
+
+      deployer.destroy
+
+      renderer.finish("done")
+
+      duration = renderer.duration || (Time.now - start_time)
+
+      say("Deleted deployment '#{name}', took #{format_time(duration).green} to complete")
+    end
+
+    def set_current(name)
+      manifest_filename = find_deployment(name)
+
+      if !File.exists?(manifest_filename)
+        err "Missing manifest for #{name} (tried '#{manifest_filename}')"
+      end
+
+      manifest = load_yaml_file(manifest_filename)
+
+      unless manifest.is_a?(Hash)
+        err "Invalid manifest format"
+      end
+
+      if manifest["network"].blank?
+        err "network is not defined in deployment manifest"
+      end
+      ip = deployer.discover_bosh_ip || name
+
+      if target
+        old_director_ip = URI.parse(target).host
+      else
+        old_director_ip = nil
+      end
+
+      if old_director_ip != ip
+        set_target(ip)
+        say "#{"WARNING!".red} Your target has been changed to `#{target.red}'!"
+      end
+
+      say "Deployment set to '#{manifest_filename.green}'"
+      config.set_deployment(manifest_filename)
+      config.save
+    end
+
+    def show_current
+      say(deployment ? "Current deployment is '#{deployment.green}'" : "Deployment not set")
+    end
+
+    def list
+      file = File.join(work_dir, Bosh::Deployer::InstanceManager::DEPLOYMENTS_FILE)
+      if File.exists?(file)
+        deployments = load_yaml_file(file)["instances"]
+      else
+        deployments = []
+      end
+
+      err("No deployments") if deployments.size == 0
+
+      na = "n/a"
+
+      deployments_table = table do |t|
+        t.headings = [ "Name", "VM name", "Stemcell name" ]
+        deployments.each do |r|
+          t << [ r[:name], r[:vm_cid] || na, r[:stemcell_cid] || na  ]
+        end
+      end
+
+      say("\n")
+      say(deployments_table)
+      say("\n")
+      say("Deployments total: %d" % deployments.size)
+    end
+
+    def agent(*args)
+      message = args.shift
+      args = args.map do |arg|
+        if File.exists?(arg)
+          load_yaml_file(arg)
+        else
+          arg
+        end
+      end
+
+      say(deployer.agent.send(message.to_sym, *args).pretty_inspect)
+    end
+
+    def apply(spec)
+      deployer.apply(load_yaml_file(spec))
+    end
+
+    private
+
+    def deployer
+      check_if_deployments_dir
+      deployment_required
+
+      if @deployer.nil?
+        manifest_filename = deployment
+
+        if !File.exists?(manifest_filename)
+          err("Cannot find deployment manifest in `#{manifest_filename}'")
+        end
+
+        manifest = load_yaml_file(manifest_filename)
+
+        manifest["dir"] ||= work_dir
+        manifest["logging"] ||= {}
+        if manifest["logging"]["level"] == "DEBUG"
+          log_dir = File.dirname(manifest_filename)
+          manifest["logging"]["file"] ||= File.join(log_dir, "bosh_micro_deploy.log")
+        end
+
+        @deployer = Bosh::Deployer::InstanceManager.new(manifest)
+
+        $stderr.reopen("/dev/null") #silence ssl warnings
+      end
+
+      @deployer
+    end
+
+    def check_if_deployments_dir
+      #force the issue to maintain central bosh-deployments.yml
+      if File.basename(work_dir) != "deployments"
+        err "Sorry, your current directory doesn't look like deployments directory"
+      end
+    end
+
+    def find_deployment(name)
+      check_if_deployments_dir
+      File.expand_path(File.join(work_dir, "#{name}", "micro_bosh.yml"))
+    end
+
+    def deployment_name
+      File.basename(File.dirname(deployment))
+    end
+
+    def set_target(ip)
+      config.target = "http://#{ip}:#{MICRO_DIRECTOR_PORT}"
+      config.save
+    end
+
+    def update_target
+      if deployer.exists?
+        bosh_ip = deployer.discover_bosh_ip
+        if URI.parse(target).host != bosh_ip
+          set_current(deployment_name)
+        end
+
+        director = Bosh::Cli::Director.new(target)
+
+        if options[:director_checks]
+          begin
+            status = director.get_status
+          rescue Bosh::Cli::AuthError
+            status = {}
+          rescue Bosh::Cli::DirectorError
+            err("Cannot talk to director at '#{target}', please set correct target")
+          end
+        else
+          status = { "name" => "Unknown Director", "version" => "n/a" }
+        end
+      else
+        status = {}
+      end
+
+      config.target_name = status["name"]
+      config.target_version = status["version"]
+      config.target_uuid = status["uuid"]
+
+      config.save
+    end
+
+    def confirm_deployment(msg)
+      unless confirmed?(msg)
+        cancel_deployment
+      end
+    end
+
+    def deployer_state(column)
+      if value = deployer.state.send(column)
+        value.green
+      else
+        "n/a".red
+      end
+    end
+
+    class DeployerRenderer < Bosh::Cli::EventLogRenderer
+      attr_accessor :stage, :total, :index
+
+      def start
+        @thread = Thread.new do
+          loop do
+            refresh
+            sleep(1)
+          end
+        end
+      end
+
+      def finish(state)
+        @thread.kill
+        super(state)
+      end
+
+      def enter_stage(stage, total)
+        @stage = stage
+        @total = total
+        @index = 0
+      end
+
+      def parse_event(event)
+        event
+      end
+
+      def update(state, task)
+        event = {
+          "time"     => Time.now,
+          "stage"    => @stage,
+          "task"     => task,
+          "tags"     => [],
+          "index"    => @index+1,
+          "total"    => @total,
+          "state"    => state.to_s,
+          "progress" => state == :finished ? 100 : 0
+        }
+
+        add_event(event)
+
+        @index += 1 if state == :finished
+      end
+    end
+
+  end
+end