bosh_cli_plugin_micro 1.5.0.pre.1113

data/lib/bosh/cli/commands/micro.rb

@@ -0,0 +1,442 @@
+require 'pp'
+require 'deployer'
+
+module Bosh::Cli::Command
+  class Micro < Base
+    include Bosh::Deployer::Helpers
+
+    MICRO_DIRECTOR_PORT = 25555
+    DEFAULT_CONFIG_PATH = File.expand_path("~/.bosh_deployer_config")
+    MICRO_BOSH_YAML = "micro_bosh.yml"
+
+    def initialize(runner)
+      super(runner)
+      options[:config] ||= DEFAULT_CONFIG_PATH #hijack Cli::Config
+    end
+
+    usage "micro"
+    desc  "show micro bosh sub-commands"
+    def micro_help
+      say("bosh micro sub-commands:")
+      nl
+      cmds = Bosh::Cli::Config.commands.values.find_all {|c|
+        c.usage =~ /^micro/
+      }
+      Bosh::Cli::Command::Help.list_commands(cmds)
+    end
+
+    usage "micro deployment"
+    desc  "Choose micro deployment to work with, or display current deployment"
+    def micro_deployment(name=nil)
+      if name
+        set_current(name)
+      else
+        show_current
+      end
+    end
+
+    def set_current(name)
+      manifest_filename = find_deployment(name)
+
+      if !File.exists?(manifest_filename)
+        err "Missing manifest for #{name} (tried '#{manifest_filename}')"
+      end
+
+      manifest = load_yaml_file(manifest_filename)
+
+      unless manifest.is_a?(Hash)
+        err "Invalid manifest format"
+      end
+
+      if manifest["network"].blank?
+        err "network is not defined in deployment manifest"
+      end
+      ip = deployer(manifest_filename).discover_bosh_ip || name
+
+      if target
+        old_director_ip = URI.parse(target).host
+      else
+        old_director_ip = nil
+      end
+
+      if old_director_ip != ip
+        set_target(ip)
+        say "#{"WARNING!".make_red} Your target has been changed to `#{target.make_red}'!"
+      end
+
+      say "Deployment set to '#{manifest_filename.make_green}'"
+      config.set_deployment(manifest_filename)
+      config.save
+    end
+
+    def show_current
+      say(deployment ? "Current deployment is '#{deployment.make_green}'" : "Deployment not set")
+    end
+
+    usage "micro status"
+    desc  "Display micro BOSH deployment status"
+    def status
+      stemcell_cid = deployer_state(:stemcell_cid)
+      stemcell_name = deployer_state(:stemcell_name)
+      vm_cid = deployer_state(:vm_cid)
+      disk_cid = deployer_state(:disk_cid)
+      deployment = config.deployment ? config.deployment.make_green : "not set".make_red
+
+      say("Stemcell CID".ljust(15) + stemcell_cid)
+      say("Stemcell name".ljust(15) + stemcell_name)
+      say("VM CID".ljust(15) + vm_cid)
+      say("Disk CID".ljust(15) + disk_cid)
+      say("Micro BOSH CID".ljust(15) + Bosh::Deployer::Config.uuid)
+      say("Deployment".ljust(15) + deployment)
+
+      update_target
+
+      target_name = target ? target.make_green : "not set".make_red
+      say("Target".ljust(15) + target_name)
+    end
+
+    usage  "micro deploy"
+    desc   "Deploy a micro BOSH instance to the currently selected deployment"
+    option "--update", "update existing instance"
+    option "--update-if-exists", "create new or update existing instance"
+    def perform(stemcell=nil)
+      update = !!options[:update]
+
+      err "No deployment set" unless deployment
+
+      manifest = load_yaml_file(deployment)
+
+      if stemcell.nil?
+        unless manifest.is_a?(Hash)
+          err("Invalid manifest format")
+        end
+
+        stemcell = dig_hash(manifest, "resources", "cloud_properties", "image_id")
+
+        if stemcell.nil?
+          err "No stemcell provided"
+        end
+      end
+
+      deployer.check_dependencies
+
+      rel_path = strip_relative_path(deployment)
+
+      desc = "`#{rel_path.make_green}' to `#{target_name.make_green}'"
+
+      if deployer.exists?
+        if !options[:update_if_exists] && !update
+          err "Instance exists. Did you mean to --update?"
+        end
+        confirmation = "Updating"
+        method = :update_deployment
+      else
+        prefered_dir = File.dirname(File.dirname(deployment))
+
+        unless prefered_dir == Dir.pwd
+          confirm_deployment("\n#{'No `bosh-deployments.yml` file found in current directory.'.make_red}\n\n" +
+                                 "Conventionally, `bosh-deployments.yml` should be saved in " +
+                                 "#{prefered_dir.make_green}.\n" +
+                                 "Is #{Dir.pwd.make_yellow} a directory where you can save state?")
+        end
+
+        err "No existing instance to update" if update
+        confirmation = "Deploying new micro BOSH instance"
+        method = :create_deployment
+
+        # make sure the user knows a persistent disk is required
+        unless dig_hash(manifest, "resources", "persistent_disk")
+          quit("No persistent disk configured in #{MICRO_BOSH_YAML}".make_red)
+        end
+      end
+
+      confirm_deployment("#{confirmation} #{desc}")
+
+      if is_tgz?(stemcell)
+        stemcell_file = Bosh::Cli::Stemcell.new(stemcell, cache)
+
+        say("\nVerifying stemcell...")
+        stemcell_file.validate
+        say("\n")
+
+        unless stemcell_file.valid?
+          err("Stemcell is invalid, please fix, verify and upload again")
+        end
+      end
+
+      renderer = DeployerRenderer.new
+      renderer.start
+      deployer.renderer = renderer
+
+      start_time = Time.now
+
+      deployer.send(method, stemcell)
+
+      renderer.finish("done")
+
+      duration = renderer.duration || (Time.now - start_time)
+
+      update_target
+
+      say("Deployed #{desc}, took #{format_time(duration).make_green} to complete")
+    end
+
+    usage  "micro delete"
+    desc   "Delete micro BOSH instance (including persistent disk)"
+    def delete
+      unless deployer.exists?
+        err "No existing instance to delete"
+      end
+
+      name = deployer.state.name
+
+      say "\nYou are going to delete micro BOSH deployment `#{name}'.\n\n" \
+      "THIS IS A VERY DESTRUCTIVE OPERATION AND IT CANNOT BE UNDONE!\n".make_red
+
+      unless confirmed?
+        say "Canceled deleting deployment".make_green
+        return
+      end
+
+      renderer = DeployerRenderer.new
+      renderer.start
+      deployer.renderer = renderer
+
+      start_time = Time.now
+
+      deployer.delete_deployment
+
+      renderer.finish("done")
+
+      duration = renderer.duration || (Time.now - start_time)
+
+      say("Deleted deployment '#{name}', took #{format_time(duration).make_green} to complete")
+    end
+
+    usage "micro deployments"
+    desc  "Show the list of deployments"
+    def list
+      file = File.join(work_dir, DEPLOYMENTS_FILE)
+      if File.exists?(file)
+        deployments = load_yaml_file(file)["instances"]
+      else
+        deployments = []
+      end
+
+      err("No deployments") if deployments.size == 0
+
+      na = "n/a"
+
+      deployments_table = table do |t|
+        t.headings = [ "Name", "VM name", "Stemcell name" ]
+        deployments.each do |r|
+          t << [ r[:name], r[:vm_cid] || na, r[:stemcell_cid] || na  ]
+        end
+      end
+
+      say("\n")
+      say(deployments_table)
+      say("\n")
+      say("Deployments total: %d" % deployments.size)
+    end
+
+    usage "micro agent <args>"
+    desc  <<-AGENT_HELP
+Send agent messages
+
+  Message Types:
+
+    start - Start all jobs on MicroBOSH
+
+    stop - Stop all jobs on MicroBOSH
+
+    ping - Check to see if the agent is responding
+
+    drain TYPE SPEC - Tell the agent to begin draining
+      TYPE - One of 'shutdown', 'update' or 'status'.
+      SPEC - The drain spec to use.
+
+    state [full] - Get the state of a system
+      full - Get additional information about system vitals
+
+    list_disk - List disk CIDs mounted on the system
+
+    migrate_disk OLD NEW - Migrate a disk
+      OLD - The CID of the source disk.
+      NEW - The CID of the destination disk.
+
+    mount_disk CID - Mount a disk on the system
+      CID - The cloud ID of the disk to mount.
+
+    unmount_disk CID - Unmount a disk from the system
+      CID - The cloud ID of the disk to unmount.
+
+AGENT_HELP
+    def agent(*args)
+      message = args.shift
+      args = args.map do |arg|
+        if File.exists?(arg)
+          load_yaml_file(arg)
+        else
+          arg
+        end
+      end
+
+      say(deployer.agent.send(message.to_sym, *args).pretty_inspect)
+    end
+
+    usage "micro apply"
+    desc  "Apply spec"
+    def apply(spec)
+      deployer.apply(Bosh::Deployer::Specification.new(load_yaml_file(spec)))
+    end
+
+    private
+
+    def deployer(manifest_filename=nil)
+      deployment_required unless manifest_filename
+
+      if @deployer.nil?
+        manifest_filename ||= deployment
+
+        if !File.exists?(manifest_filename)
+          err("Cannot find deployment manifest in `#{manifest_filename}'")
+        end
+
+        manifest = load_yaml_file(manifest_filename)
+
+        manifest["dir"] ||= work_dir
+        manifest["logging"] ||= {}
+        unless manifest["logging"]["file"]
+          log_file = File.join(File.dirname(manifest_filename),
+                               "bosh_micro_deploy.log")
+          manifest["logging"]["file"] = log_file
+        end
+
+        @deployer = Bosh::Deployer::InstanceManager.create(manifest)
+      end
+
+      @deployer
+    end
+
+    def find_deployment(name)
+      if File.directory?(name)
+        filename = File.join("#{name}", MICRO_BOSH_YAML)
+      else
+        filename = name
+      end
+
+      File.expand_path(filename, Dir.pwd)
+    end
+
+    def deployment_name
+      File.basename(File.dirname(deployment))
+    end
+
+    # set new target and clear out cached values
+    # does not persist the new values (set_current() does this)
+    def set_target(ip)
+      config.target = "https://#{ip}:#{MICRO_DIRECTOR_PORT}"
+      config.target_name = nil
+      config.target_version = nil
+      config.target_uuid = nil
+    end
+
+    def update_target
+      if deployer.exists?
+        bosh_ip = deployer.discover_bosh_ip
+        if URI.parse(target).host != bosh_ip
+          set_current(deployment)
+        end
+
+        director = Bosh::Cli::Client::Director.new(target)
+
+        if options[:director_checks]
+          begin
+            status = director.get_status
+          rescue Bosh::Cli::AuthError
+            status = {}
+          rescue Bosh::Cli::DirectorError
+            err("Cannot talk to director at '#{target}', please set correct target")
+          end
+        else
+          status = { "name" => "Unknown Director", "version" => "n/a" }
+        end
+      else
+        status = {}
+      end
+
+      config.target_name = status["name"]
+      config.target_version = status["version"]
+      config.target_uuid = status["uuid"]
+
+      config.save
+    end
+
+    def confirm_deployment(msg)
+      unless confirmed?(msg)
+        cancel_deployment
+      end
+    end
+
+    def deployer_state(column)
+      if value = deployer.state.send(column)
+        value.make_green
+      else
+        "n/a".make_red
+      end
+    end
+
+    class DeployerRenderer < Bosh::Cli::EventLogRenderer
+      attr_accessor :stage, :total, :index
+
+      DEFAULT_POLL_INTERVAL = 1
+
+      def interval_poll
+        Bosh::Cli::Config.poll_interval || DEFAULT_POLL_INTERVAL
+      end
+
+      def start
+        @thread = Thread.new do
+          loop do
+            refresh
+            sleep(interval_poll)
+          end
+        end
+      end
+
+      def finish(state)
+        @thread.kill
+        super(state)
+      end
+
+      def enter_stage(stage, total)
+        @stage = stage
+        @total = total
+        @index = 0
+      end
+
+      def parse_event(event)
+        event
+      end
+
+      def update(state, task)
+        event = {
+          "time"     => Time.now,
+          "stage"    => @stage,
+          "task"     => task,
+          "tags"     => [],
+          "index"    => @index+1,
+          "total"    => @total,
+          "state"    => state.to_s,
+          "progress" => state == :finished ? 100 : 0
+        }
+
+        add_event(event)
+
+        @index += 1 if state == :finished
+      end
+    end
+
+  end
+end

data/lib/deployer/config.rb

@@ -0,0 +1,157 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh; end
+
+module Bosh::Deployer
+  class Config
+
+    class << self
+
+      include Helpers
+
+      attr_accessor :logger, :db, :uuid, :resources, :cloud_options,
+        :spec_properties, :agent_properties, :bosh_ip, :env, :name, :net_conf
+
+      def configure(config)
+        plugin = cloud_plugin(config)
+
+        config = deep_merge(load_defaults(plugin), config)
+
+        @base_dir = config["dir"]
+        FileUtils.mkdir_p(@base_dir)
+
+        @name = config["name"]
+        @cloud_options = config["cloud"]
+        @net_conf = config["network"]
+        @bosh_ip = @net_conf["ip"]
+        @resources = config["resources"]
+        @env = config["env"]
+
+        @logger = Logger.new(config["logging"]["file"] || STDOUT)
+        @logger.level = Logger.const_get(config["logging"]["level"].upcase)
+        @logger.formatter = ThreadFormatter.new
+
+        apply_spec = config["apply_spec"]
+        @spec_properties = apply_spec["properties"]
+        @agent_properties = apply_spec["agent"]
+
+        @db = Sequel.sqlite
+
+        migrate_cpi
+
+        @db.create_table :instances do
+          primary_key :id
+          column :name, :text, :unique => true, :null => false
+          column :uuid, :text
+          column :stemcell_cid, :text
+          column :stemcell_name, :text
+          column :vm_cid, :text
+          column :disk_cid, :text
+        end
+
+        Sequel::Model.plugin :validation_helpers
+
+        Bosh::Clouds::Config.configure(self)
+
+        require "deployer/models/instance"
+
+        @cloud_options["properties"]["agent"]["mbus"] ||=
+          "https://vcap:b00tstrap@0.0.0.0:6868"
+
+        @disk_model = nil
+        @cloud = nil
+        @networks = nil
+      end
+
+      def cloud
+        if @cloud.nil?
+          @cloud = Bosh::Clouds::Provider.create(@cloud_options["plugin"],
+                                                 @cloud_options["properties"])
+        end
+        @cloud
+      end
+
+      def agent
+        uri = URI.parse(agent_url)
+        user, password = uri.userinfo.split(":", 2)
+        uri.userinfo = nil
+        uri.host = bosh_ip
+        Bosh::Agent::HTTPClient.new(uri.to_s,
+                                    { "user" => user,
+                                      "password" => password,
+                                      "reply_to" => uuid })
+      end
+
+      def agent_url
+        @cloud_options["properties"]["agent"]["mbus"]
+      end
+
+      def networks
+        return @networks if @networks
+
+        @networks = {
+          "bosh" => {
+            "cloud_properties" => @net_conf["cloud_properties"],
+            "netmask"          => @net_conf["netmask"],
+            "gateway"          => @net_conf["gateway"],
+            "ip"               => @net_conf["ip"],
+            "dns"              => @net_conf["dns"],
+            "type"             => @net_conf["type"],
+            "default"          => ["dns", "gateway"]
+          }
+        }
+        if @net_conf["vip"]
+          @networks["vip"] = {
+              "ip" => @net_conf["vip"],
+              "type" => "vip",
+              "cloud_properties" => {}
+          }
+        end
+
+        @networks
+      end
+
+      def task_checkpoint
+        # Bosh::Clouds::Config (bosh_cli >= 0.5.1) delegates task_checkpoint
+        # method to periodically check if director task is cancelled,
+        # so we need to define a void method in Bosh::Deployer::Config to avoid
+        # NoMethodError exceptions.
+      end
+
+      private
+
+      def migrate_cpi
+        cpi = @cloud_options["plugin"]
+        require_path = File.join("cloud", cpi)
+        cpi_path = $LOAD_PATH.find { |p| File.exist?(
+            File.join(p, require_path)) }
+        migrations = File.expand_path("../db/migrations", cpi_path)
+
+        if File.directory?(migrations)
+          Sequel.extension :migration
+          Sequel::TimestampMigrator.new(
+              @db, migrations, :table => "#{cpi}_cpi_schema").run
+        end
+      end
+
+      def deep_merge(src, dst)
+        src.merge(dst) do |key, old, new|
+          if new.respond_to?(:blank) && new.blank?
+            old
+          elsif old.kind_of?(Hash) and new.kind_of?(Hash)
+            deep_merge(old, new)
+          elsif old.kind_of?(Array) and new.kind_of?(Array)
+            old.concat(new).uniq
+          else
+            new
+          end
+        end
+      end
+
+      def load_defaults(provider)
+        file = File.join(File.dirname(File.expand_path(__FILE__)), "../../config/#{provider}_defaults.yml")
+        Psych.load_file(file)
+      end
+    end
+  end
+end

data/lib/deployer/helpers.rb

@@ -0,0 +1,115 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require 'net/ssh'
+
+module Bosh::Deployer
+
+  module Helpers
+
+    DEPLOYMENTS_FILE = "bosh-deployments.yml"
+
+    def is_tgz?(path)
+      File.extname(path) == ".tgz"
+    end
+
+    def cloud_plugin(config)
+      err "No cloud properties defined" if config["cloud"].nil?
+      err "No cloud plugin defined" if config["cloud"]["plugin"].nil?
+
+      config["cloud"]["plugin"]
+    end
+
+    def dig_hash(hash, *path)
+      path.inject(hash) do |location, key|
+        location.respond_to?(:keys) ? location[key] : nil
+      end
+    end
+
+    def process_exists?(pid)
+      begin
+        Process.kill(0, pid)
+      rescue Errno::ESRCH
+        false
+      end
+    end
+
+    def socket_readable?(ip, port)
+      socket = TCPSocket.new(ip, port)
+      if IO.select([socket], nil, nil, 5)
+        logger.debug("tcp socket #{ip}:#{port} is readable")
+        yield
+        true
+      else
+        false
+      end
+    rescue SocketError => e
+      logger.debug("tcp socket #{ip}:#{port} SocketError: #{e.inspect}")
+      sleep 1
+      false
+    rescue SystemCallError => e
+      logger.debug("tcp socket #{ip}:#{port} SystemCallError: #{e.inspect}")
+      sleep 1
+      false
+    ensure
+      socket.close if socket
+    end
+
+    def remote_tunnel(port)
+      @sessions ||= {}
+      return if @sessions[port]
+
+      ip = Config.bosh_ip
+
+      loop until socket_readable?(ip, @ssh_port) do
+        #sshd is up, sleep while host keys are generated
+        sleep @ssh_wait
+      end
+
+      if @sessions[port].nil?
+        logger.info("Starting SSH session for port forwarding to #{@ssh_user}@#{ip}...")
+        loop do
+          begin
+            @sessions[port] = Net::SSH.start(ip, @ssh_user, :keys => [@ssh_key],
+                                             :paranoid => false)
+            logger.debug("ssh #{@ssh_user}@#{ip}: ESTABLISHED")
+            break
+          rescue => e
+            logger.debug("ssh start #{@ssh_user}@#{ip} failed: #{e.inspect}")
+            sleep 1
+          end
+        end
+      end
+
+      lo = "127.0.0.1"
+      @sessions[port].forward.remote(port, lo, port)
+
+      logger.info("SSH forwarding for port #{port} started: OK")
+
+      Thread.new do
+        while @sessions[port]
+          begin
+            @sessions[port].loop { true }
+          rescue IOError => e
+            logger.debug("SSH session #{@sessions[port].inspect} forwarding for port #{port} terminated: #{e.inspect}")
+            @sessions.delete(port)
+          end
+        end
+      end
+
+      at_exit do
+        status = $!.is_a?(::SystemExit) ? $!.status : nil
+        close_ssh_sessions
+        exit status if status
+      end
+    end
+
+    def close_ssh_sessions
+      @sessions.each_value { |s| s.close }
+    end
+
+    def strip_relative_path(path)
+      path[/#{Regexp.escape File.join(Dir.pwd, '')}(.*)/, 1] || path
+    end
+  end
+
+end