bosh_cli 0.19.6 → 1.0.rc1

data/lib/cli/commands/ssh.rb

@@ -3,67 +3,73 @@
 module Bosh::Cli::Command
   class Ssh < Base
     include Bosh::Cli::DeploymentHelper
-    CMD_UPLOAD = "upload"
-    CMD_DOWNLOAD = "download"
-    CMD_EXEC = "exec"
+
     SSH_USER_PREFIX = "bosh_"
-    SSH_DEFAULT_PASSWORD = "bosh"
     SSH_DSA_PUB = File.expand_path("~/.ssh/id_dsa.pub")
     SSH_RSA_PUB = File.expand_path("~/.ssh/id_rsa.pub")
 
-    def parse_options(args)
-      options = {}
+    # bosh ssh
+    usage "ssh"
+    desc "Execute command or start an interactive session"
+    option "--public_key FILE", "Public key"
+    option "--gateway_host HOST", "Gateway host"
+    option "--gateway_user USER", "Gateway user"
+    option "--default_password PASSWORD",
+           "Use default ssh password (NOT RECOMMENDED)"
+    def shell(*args)
+      job, index, command = parse_args(args)
 
-      # Check if index is supplied on the command line
-      begin
-        # Ruby 1.8.7 treats Integer(nil) as 0, hence the if check
-        if args.size > 0
-          options["index"] = Integer(args[0])
-          args.shift
+      if command.empty?
+        if index.nil?
+          err("Can't run interactive shell on more than one instance")
         end
-      rescue ArgumentError, TypeError
-        # No index given
+        setup_interactive_shell(job, index)
+      else
+        say("Executing `#{command.join(" ")}' on #{job}/#{index}")
+        perform_operation(:exec, job, index, command)
+      end
+    end
+
+    # bosh scp
+    usage "scp"
+    desc "upload/download the source file to the given job. " +
+         "Note: for download /path/to/destination is a directory"
+    option "--download", "Download file"
+    option "--upload", "Upload file"
+    option "--public_key FILE", "Public key"
+    option "--gateway_host HOST", "Gateway host"
+    option "--gateway_user USER", "Gateway user"
+    def scp(*args)
+      job, index, args = parse_args(args)
+      upload = options[:upload]
+      download = options[:download]
+      if (upload && download) || (upload.nil? && download.nil?)
+        err("Please specify either --upload or --download")
       end
 
-      %w(public_key gateway_host gateway_user).each do |option|
-        pos = args.index("--#{option}")
-        if pos
-          options[option] = args[pos + 1]
-          args.delete_at(pos + 1)
-          args.delete_at(pos)
-        end
+      if args.size != 2
+        err("Please enter valid source and destination paths")
       end
-      options
+      say("Executing file operations on job #{job}")
+      perform_operation(upload ? :upload : :download, job, index, args)
     end
 
-    def get_public_key(options)
-      # Get public key
-      public_key = nil
-      if options["public_key"]
-        unless File.file?(options["public_key"])
-          err("Please specify a valid public key file")
-        end
-        public_key = File.read(options["public_key"])
-      else
-        # See if ssh-add can be used
-        %x[ssh-add -L 1>/dev/null 2>&1]
-        if $?.exitstatus == 0
-          keys = %x[ssh-add -L]
-          public_key = keys.split("\n")[0]
-        else
-          # Pick up public key from home dir
-          [SSH_DSA_PUB, SSH_RSA_PUB].each do |key_file|
-            if File.file?(key_file)
-              public_key = File.read(key_file)
-              break
-            end
-          end
-        end
+    usage "cleanup ssh"
+    desc "Cleanup SSH artifacts"
+    def cleanup(*args)
+      job, index, args = parse_args(args)
+      if args.size > 0
+        err("SSH cleanup doesn't accept any extra args")
       end
-      err("Please specify a public key file") if public_key.nil?
-      public_key
+
+      manifest_name = prepare_deployment_manifest["name"]
+
+      say("Cleaning up ssh artifacts from #{job}/#{index}")
+      director.cleanup_ssh(manifest_name, job, "^#{SSH_USER_PREFIX}", [index])
     end
 
+    private
+
     def get_salt_charset
       charset = []
       charset.concat(("a".."z").to_a)
@@ -84,189 +90,173 @@ module Bosh::Cli::Command
       plain_text.crypt(salt)
     end
 
-    def setup_ssh(job, index, password, options, &block)
-      # Get public key
-      public_key = get_public_key(options)
-
-      # Generate a random user name
+    # @param [String] job
+    # @param [Integer] index
+    # @param [optional,String] password
+    def setup_ssh(job, index, password = nil)
+      public_key = get_public_key
       user = SSH_USER_PREFIX + rand(36**9).to_s(36)
+      deployment_name = prepare_deployment_manifest["name"]
 
-      # Get deployment name
-      manifest_name = prepare_deployment_manifest["name"]
+      say("Target deployment is `#{deployment_name}'")
+      status, task_id = director.setup_ssh(
+        deployment_name, job, index, user,
+        public_key, encrypt_password(password))
 
-      say "Target deployment is #{manifest_name}"
-      results = director.setup_ssh(manifest_name, job, index, user, public_key,
-                                   encrypt_password(password))
+      unless status == :done
+        err("Failed to set up SSH: see task #{task_id} log for details")
+      end
 
-      unless results && results.kind_of?(Array) && !results.empty?
-        err("Error setting up ssh, #{results.inspect}, " +
-            "check task logs for more details")
+      sessions = JSON.parse(director.get_task_result_log(task_id))
+
+      unless sessions && sessions.kind_of?(Array) && sessions.size > 0
+        err("Error setting up ssh, check task #{task_id} log for more details")
       end
 
-      results.each do |result|
-        unless result.kind_of?(Hash)
-          err("Unexpected results #{results.inspect}, " +
-              "check task logs for more details")
+      sessions.each do |session|
+        unless session.kind_of?(Hash)
+          err("Unexpected SSH session info: #{session.inspect}. " +
+              "Please check task #{task_id} log for more details")
         end
       end
 
-      if block_given?
-        yield results, user
+      if options[:gateway_host]
+        gw_host = options[:gateway_host]
+        gw_user = options[:gateway_user] || ENV["USER"]
+        gateway = Net::SSH::Gateway.new(gw_host, gw_user)
+      else
+        gateway = nil
       end
-    ensure
-      if results
+
+      begin
+        yield sessions, user, gateway
+      ensure
+        nl
         say("Cleaning up ssh artifacts")
-        indexes = results.map {|result| result["index"]}
-        # Cleanup only this one 'user'
-        director.cleanup_ssh(manifest_name, job, "^#{user}$", indexes)
+        indices = sessions.map { |session| session["index"] }
+        director.cleanup_ssh(deployment_name, job, "^#{user}$", indices)
+        gateway.shutdown! if gateway
       end
     end
 
-    def get_free_port
-      socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
-      socket.bind(Addrinfo.tcp("127.0.0.1", 0))
-      port = socket.local_address.ip_port
-      socket.close
-
-      # The port could get used in the interim, but unlikely in real life
-      port
-    end
-
-    def setup_interactive_shell(job, password, options)
-      index = options["index"]
-      err("Please specify a job index") if index.nil?
+    # @param [String] job Job name
+    # @param [Integer] index Job index
+    def setup_interactive_shell(job, index)
+      password = options[:default_password]
 
       if password.nil?
-        password_retries = 0
-        while password.blank? && password_retries < 3
-          password = ask("Enter password " +
-                     "(use it to sudo on remote host): ") { |q| q.echo = "*" }
-          password_retries += 1
-        end
+        password = ask(
+          "Enter password (use it to " +
+          "sudo on remote host): ") { |q| q.echo = "*" }
+
         err("Please provide ssh password") if password.blank?
       end
 
-      setup_ssh(job, index, password, options) do |results, user|
-        result = results.first
-        unless result["status"] && result["status"] == "success" && result["ip"]
-          err("Failed to setup ssh on index #{index} #{results.inspect}")
+      setup_ssh(job, index, password) do |sessions, user, gateway|
+        session = sessions.first
+
+        unless session["status"] == "success" && session["ip"]
+          err("Failed to set up SSH on #{job}/#{index}: #{session.inspect}")
         end
 
-        say("Starting interactive shell on job #{job}, index #{index}")
-        # Start interactive session
-        if options["gateway_host"]
-          local_port = get_free_port
-          say("Connecting to local port #{local_port}")
-          # Create the ssh tunnel
-          fork do
-            gateway = (options["gateway_user"] ?
-                "#{options["gateway_user"]}@" : "") +
-                options["gateway_host"]
-            # Tunnel will close after 30 seconds,
-            # so no need to worry about cleaning it up
-            exec("ssh -f -L#{local_port}:#{result["ip"]}:22 #{gateway} " +
-                 "sleep 30")
+        say("Starting interactive shell on job #{job}/#{index}")
+
+        if gateway
+          port = gateway.open(session["ip"], 22)
+          ssh_session = fork do
+            exec("ssh #{user}@localhost -p #{port}")
           end
-          result["ip"] = "localhost -p #{local_port}"
-          # Wait for tunnel to get established
-          sleep 3
-        end
-        ssh_session = fork do
-          exec("ssh #{user}@#{result["ip"]}")
+          Process.waitpid(ssh_session)
+          gateway.close(port)
+        else
+          ssh_session = fork do
+            exec("ssh #{user}@#{session["ip"]}")
+          end
+          Process.waitpid(ssh_session)
         end
-        Process.waitpid(ssh_session)
       end
     end
 
-    def shell(*args)
-      job = args.shift
-      password = args.delete("--default_password") && SSH_DEFAULT_PASSWORD
-      options = parse_options(args)
+    def perform_operation(operation, job, index, args)
+      setup_ssh(job, index, nil) do |sessions, user, gateway|
+        sessions.each do |session|
+          unless session["status"] == "success" && session["ip"]
+            err("Failed to set up SSH on #{job}/#{index}: #{session.inspect}")
+          end
 
-      if args.size == 0
-        setup_interactive_shell(job, password, options)
-      else
-        say("Executing command '#{args.join(" ")}' on job #{job}")
-        execute_command(CMD_EXEC, job, options, args)
+          with_ssh(user, session["ip"], gateway) do |ssh|
+            case operation
+            when :exec
+              nl
+              say("#{job}/#{session["index"]}")
+              say(ssh.exec!(args.join(" ")))
+            when :upload
+              ssh.scp.upload!(args[0], args[1])
+            when :download
+              file = File.basename(args[0])
+              path = "#{args[1]}/#{file}.#{job}.#{session["index"]}"
+              ssh.scp.download!(args[0], path)
+              say("Downloaded file to #{path}".green)
+            else
+              err("Unknown operation #{operation}")
+            end
+          end
+        end
       end
     end
 
-    def with_ssh(gateway, ip, user, &block)
-      if gateway
-        gateway.ssh(ip, user) do |ssh|
-          yield(ssh)
-        end
-      else
-        Net::SSH.start(ip, user) do |ssh|
-          yield(ssh)
+    # @param [Array] args
+    # @return [Array] job, index, command
+    def parse_args(args)
+      job = args.shift
+      job, index = job.split("/", 2)
+
+      if index
+        if index =~ /^\d+$/
+          index = index.to_i
+        else
+          err("Invalid job index, integer number expected")
         end
+      elsif args[0] =~ /^\d+$/
+        index = args.shift.to_i
       end
-    end
 
-    def with_gateway(host, user, &block)
-      gateway = Net::SSH::Gateway.new(host, user || ENV['USER']) if host
-      yield(gateway ||= nil)
-    ensure
-      gateway.shutdown! if gateway
+      [job, index, args]
     end
 
-    def execute_command(command, job, options, args)
-      setup_ssh(job, options["index"], nil, options) do |results, user|
-        with_gateway(options["gateway_host"],
-                     options["gateway_user"]) do |gateway|
-          results.each do | result|
-            unless result["status"] && result["status"] == "success" &&
-                   result["ip"]
-              err("Failed to setup ssh on index #{options["index"]}, " +
-                  "error: #{result.inspect}")
-            end
-            with_ssh(gateway, result["ip"], user) do |ssh|
-              case command
-              when CMD_EXEC
-                say("\nJob #{job} index #{result["index"]}")
-                puts ssh.exec!(args.join(" "))
-              when CMD_UPLOAD
-                ssh.scp.upload!(args[0], args[1])
-              when CMD_DOWNLOAD
-                file = File.basename(args[0])
-                path = "#{args[1]}/#{file}.#{job}.#{result["index"]}"
-                ssh.scp.download!(args[0], path)
-                say("Downloaded file to #{path}")
-              end
-            end
+    # @return [String] Public key
+    def get_public_key
+      public_key_path = options[:public_key]
+
+      if public_key_path
+        unless File.file?(public_key_path)
+          err("Can't find file `#{public_key_path}'")
+        end
+        return File.read(public_key_path)
+      else
+        %x[ssh-add -L 1>/dev/null 2>&1]
+        if $?.exitstatus == 0
+          return %x[ssh-add -L].split.first
+        else
+          [SSH_DSA_PUB, SSH_RSA_PUB].each do |key_file|
+            return File.read(key_file) if File.file?(key_file)
           end
         end
       end
-    end
 
-    def scp(*args)
-      job = args.shift
-      options = parse_options(args)
-      upload = args.delete("--upload")
-      download = args.delete("--download")
-      if upload.nil? && download.nil?
-        err("Please specify one of --upload or --download")
-      end
-
-      if args.empty? || args.size < 2
-        err("Please enter valid source and destination paths")
-      end
-      say("Executing file operations on job #{job}")
-      execute_command(upload ? CMD_UPLOAD : CMD_DOWNLOAD, job, options, args)
+      err("Please specify a public key file")
     end
 
-    def cleanup(*args)
-      job = args.shift
-      options = parse_options(args)
-      manifest_name = prepare_deployment_manifest["name"]
-      results = nil
-      if options["index"]
-        results = []
-        results << { "index" => options["index"] }
+    # @param [String] user
+    # @param [String] ip
+    # @param [optional, Net::SSH::Gateway] gateway
+    # @yield [Net::SSH]
+    def with_ssh(user, ip, gateway = nil)
+      if gateway
+        gateway.ssh(ip, user) { |ssh| yield ssh }
+      else
+        Net::SSH.start(ip, user) { |ssh| yield ssh }
       end
-      say "Cleaning up ssh artifacts from job #{job}, index #{options["index"]}"
-      director.cleanup_ssh(manifest_name, job, "^#{SSH_USER_PREFIX}",
-                           [options["index"]])
     end
   end
 end

data/lib/cli/commands/stemcell.rb

@@ -8,35 +8,47 @@ module Bosh::Cli::Command
     PUBLIC_STEMCELL_INDEX = "public_stemcells_index.yml"
 
     # The URL of the public stemcell index.
-    PUBLIC_STEMCELL_INDEX_URL = "https://blob.cfblob.com/rest/objects/4e4e78bca2" +
-        "1e121204e4e86ee151bc04f6a19ce46b22?uid=bb6a0c89ef4048a8a0f814e2538" +
-        "5d1c5/user1&expires=1893484800&signature=NJuAr9c8eOid7dKFmOEN7bmzAlI="
+    PUBLIC_STEMCELL_INDEX_URL =
+      "https://blob.cfblob.com/rest/objects/4e4e78bca2" +
+      "1e121204e4e86ee151bc04f6a19ce46b22?uid=bb6a0c89ef4048a8a0f814e2538" +
+      "5d1c5/user1&expires=1893484800&signature=NJuAr9c8eOid7dKFmOEN7bmzAlI="
 
+    DEFAULT_PUB_STEMCELL_TAG = "stable"
+    ALL_STEMCELLS_TAG = "all"
+
+    usage "verify stemcell"
+    desc "Verify stemcell"
     def verify(tarball_path)
       stemcell = Bosh::Cli::Stemcell.new(tarball_path, cache)
 
-      say("\nVerifying stemcell...")
+      nl
+      say("Verifying stemcell...")
       stemcell.validate
-      say("\n")
+      nl
 
       if stemcell.valid?
-        say("'%s' is a valid stemcell" % [tarball_path])
+        say("`#{tarball_path}' is a valid stemcell".green)
       else
-        say("'%s' is not a valid stemcell:" % [tarball_path])
-        for error in stemcell.errors
+        say("Validation errors:".red)
+        stemcell.errors.each do |error|
           say("- %s" % [error])
         end
+        err("`#{tarball_path}' is not a valid stemcell")
       end
     end
 
+    # bosh upload stemcell
+    usage "upload stemcell"
+    desc "Upload stemcell"
     def upload(tarball_path)
       auth_required
 
       stemcell = Bosh::Cli::Stemcell.new(tarball_path, cache)
 
-      say("\nVerifying stemcell...")
+      nl
+      say("Verifying stemcell...")
       stemcell.validate
-      say("\n")
+      nl
 
       unless stemcell.valid?
         err("Stemcell is invalid, please fix, verify and upload again")
@@ -46,24 +58,25 @@ module Bosh::Cli::Command
       name = stemcell.manifest["name"]
       version = stemcell.manifest["version"]
 
-      existing = director.list_stemcells.select do |sc|
-        sc["name"] == name and sc["version"] == version
-      end
-
-      if existing.empty?
-        say("No")
+      if exists?(name, version)
+        err("Stemcell `#{name}/#{version}' already exists, " +
+              "increment the version if it has changed")
       else
-        err("Stemcell \"#{name}\":\"#{version}\" already exists, " +
-            "increment the version if it has changed")
+        say("No")
       end
 
-      say("\nUploading stemcell...\n")
+      nl
+      say("Uploading stemcell...")
+      nl
 
       status, _ = director.upload_stemcell(stemcell.stemcell_file)
 
       task_report(status, "Stemcell uploaded and created")
     end
 
+    # bosh stemcells
+    usage "stemcells"
+    desc "Show the list of available stemcells"
     def list
       auth_required
       stemcells = director.list_stemcells.sort do |sc1, sc2|
@@ -72,7 +85,7 @@ module Bosh::Cli::Command
             sc1["name"] <=> sc2["name"]
       end
 
-      err("No stemcells") if stemcells.size == 0
+      err("No stemcells") if stemcells.empty?
 
       stemcells_table = table do |t|
         t.headings = "Name", "Version", "CID"
@@ -81,58 +94,59 @@ module Bosh::Cli::Command
         end
       end
 
-      say("\n")
+      nl
       say(stemcells_table)
-      say("\n")
+      nl
       say("Stemcells total: %d" % stemcells.size)
     end
 
-    # Grabs the index file for the publicly available stemcells.
-    # @return [Hash] The index file YAML as a hash.
-    def get_public_stemcell_list
-      @http_client = HTTPClient.new
-      response = @http_client.get(PUBLIC_STEMCELL_INDEX_URL)
-      status_code = response.http_header.status_code
-      if status_code != HTTP::Status::OK
-        err("Received HTTP #{status_code} from #{PUBLIC_STEMCELL_INDEX_URL}.")
-      end
-      YAML.load(response.body)
-    end
-
     # Prints out the publicly available stemcells.
-    def list_public(*args)
-      full = args.include?("--full")
+    usage "public stemcells"
+    desc "Show the list of publicly available stemcells for download."
+    option "--full", "show the full download url"
+    option "--tags tag1,tag2...", Array, "filter by tag"
+    def list_public
+      full = !!options[:full]
+      tags = options[:tags] || [DEFAULT_PUB_STEMCELL_TAG]
+
       yaml = get_public_stemcell_list
       stemcells_table = table do |t|
-        t.headings = "Name", "Url"
+        t.headings = full ? ["Name", "Url", "Tags"] : ["Name", "Tags"]
         yaml.keys.sort.each do |key|
           if key != PUBLIC_STEMCELL_INDEX
-            url = full ? yaml[key]["url"] : "#{yaml[key]["url"][0..49]}..."
-            t << [key, url]
+            url = yaml[key]["url"]
+            yaml_tags = yaml[key]["tags"]
+            next if skip_this_tag?(yaml_tags, tags)
+
+            yaml_tags = yaml_tags ? yaml_tags.join(", ") : ""
+            t << (full ? [key, url, yaml_tags] : [key, yaml_tags])
           end
         end
       end
-      puts(stemcells_table)
-      puts("To download use 'bosh download public stemcell <stemcell_name>'." +
+
+      say(stemcells_table)
+
+      say("To download use `bosh download public stemcell <stemcell_name>'. " +
           "For full url use --full.")
     end
 
     # Downloads one of the publicly available stemcells.
+    usage "download public stemcell"
+    desc "Downloads a stemcell from the public blobstore"
     # @param [String] stemcell_name The name of the stemcell, as seen in the
-    #     public stemcell index file.
+    #   public stemcell index file.
     def download_public(stemcell_name)
       yaml = get_public_stemcell_list
       yaml.delete(PUBLIC_STEMCELL_INDEX) if yaml.has_key?(PUBLIC_STEMCELL_INDEX)
 
       unless yaml.has_key?(stemcell_name)
         available_stemcells = yaml.map { |k| k }.join(", ")
-        puts("'#{stemcell_name}' not found in '#{available_stemcells}'.".red)
-        return
+        err("'#{stemcell_name}' not found in '#{available_stemcells}'.")
       end
 
       if File.exists?(stemcell_name) &&
-          !agree("#{stemcell_name} exists locally. Overwrite it? [yn]")
-        return
+         !confirmed?("Overwrite existing file `#{stemcell_name}'?")
+        err("File `#{stemcell_name}' already exists")
       end
 
       url = yaml[stemcell_name]["url"]
@@ -140,25 +154,36 @@ module Bosh::Cli::Command
       sha1 = yaml[stemcell_name]["sha"]
       progress_bar = ProgressBar.new(stemcell_name, size)
       progress_bar.file_transfer_mode
-      File.open("#{stemcell_name}", "w") { |file|
+
+      File.open("#{stemcell_name}", "w") do |file|
         @http_client.get(url) do |chunk|
           file.write(chunk)
           progress_bar.inc(chunk.size)
         end
-      }
+      end
       progress_bar.finish
+
       file_sha1 = Digest::SHA1.file(stemcell_name).hexdigest
       if file_sha1 != sha1
-        err("The downloaded file sha1 '#{file_sha1}' does not match the " +
-            "expected sha1 '#{sha1}'.")
+        err("The downloaded file sha1 `#{file_sha1}' does not match the " +
+            "expected sha1 `#{sha1}'")
       else
-        puts("Download complete.")
+        say("Download complete".green)
       end
     end
 
+    # bosh delete stemcell
+    usage "delete stemcell"
+    desc  "Delete stemcell"
     def delete(name, version)
       auth_required
 
+      say("Checking if stemcell exists...")
+
+      unless exists?(name, version)
+        err("Stemcell `#{name}/#{version}' does not exist")
+      end
+
       say("You are going to delete stemcell `#{name}/#{version}'".red)
 
       unless confirmed?
@@ -170,5 +195,43 @@ module Bosh::Cli::Command
 
       task_report(status, "Deleted stemcell `#{name}/#{version}'")
     end
+
+    private
+
+    def skip_this_tag?(yaml_tags, requested_tags)
+      if requested_tags == [ALL_STEMCELLS_TAG]
+        return false
+      end
+      unless yaml_tags
+        return true
+      end
+      requested_tags.each do |tag|
+        unless yaml_tags.include?(tag)
+          return true
+        end
+      end
+      return false
+    end
+
+    def exists?(name, version)
+      existing = director.list_stemcells.select do |sc|
+        sc["name"] == name && sc["version"] == version
+      end
+
+      !existing.empty?
+    end
+
+    # Grabs the index file for the publicly available stemcells.
+    # @return [Hash] The index file YAML as a hash.
+    def get_public_stemcell_list
+      @http_client = HTTPClient.new
+      response = @http_client.get(PUBLIC_STEMCELL_INDEX_URL)
+      status_code = response.http_header.status_code
+      if status_code != HTTP::Status::OK
+        err("Received HTTP #{status_code} from #{PUBLIC_STEMCELL_INDEX_URL}.")
+      end
+      YAML.load(response.body)
+    end
+
   end
 end