bosh_cli 1.2792.0 → 1.2797.0

data/lib/cli.rb

@@ -46,6 +46,7 @@ require 'common/version/release_version_list'
 require 'common/version/bosh_version'
 require 'common/version/stemcell_version'
 require 'common/version/stemcell_version_list'
+require 'common/thread_pool'
 
 require 'cli/config'
 require 'cli/core_ext'
@@ -62,6 +63,8 @@ require 'cli/director_task'
 
 require 'cli/line_wrap'
 require 'cli/backup_destination_path'
+require 'cli/interactive_progress_renderer'
+require 'cli/non_interactive_progress_renderer'
 
 require 'cli/source_control/git_ignore'
 

data/lib/cli/base_command.rb

@@ -51,8 +51,12 @@ module Bosh::Cli
         @release = Bosh::Cli::Release.new(@work_dir, options[:final])
       end
 
+      def progress_renderer
+        interactive? ? Bosh::Cli::InteractiveProgressRenderer.new : Bosh::Cli::NonInteractiveProgressRenderer.new
+      end
+
       def blob_manager
-        @blob_manager ||= Bosh::Cli::BlobManager.new(release)
+        @blob_manager ||= Bosh::Cli::BlobManager.new(release, config.max_parallel_downloads, progress_renderer)
       end
 
       def blobstore

data/lib/cli/blob_manager.rb

@@ -1,4 +1,5 @@
 # Copyright (c) 2009-2012 VMware, Inc.
+require 'logging'
 
 module Bosh::Cli
   # In order to avoid storing large objects in git repo,
@@ -10,7 +11,11 @@ module Bosh::Cli
     attr_reader :new_blobs, :updated_blobs
 
     # @param [Bosh::Cli::Release] release BOSH Release object
-    def initialize(release)
+    def initialize(release, max_parallel_downloads, progress_renderer)
+      @progress_renderer = progress_renderer
+      max_parallel_downloads = 1 if max_parallel_downloads.nil? || max_parallel_downloads < 1
+      @max_parallel_downloads = max_parallel_downloads
+
       @release = release
       @index_file = File.join(@release.dir, "config", DEFAULT_INDEX_NAME)
 
@@ -205,6 +210,7 @@ module Bosh::Cli
     # establishes symlinks in blobs directory to any files present in index.
     # @return [void]
     def process_index
+      missing_blobs = []
       @index.each_pair do |path, entry|
         if File.exists?(File.join(@src_dir, path))
           err("File `#{path}' is in both blob index and src directory.\n" +
@@ -219,15 +225,24 @@ module Bosh::Cli
           if checksum == entry["sha"]
             need_download = false
           else
-            progress(path, "checksum mismatch, re-downloading...\n".make_red)
+            @progress_renderer.error(path, "checksum mismatch, re-downloading...")
           end
         end
 
         if need_download
-          local_path = download_blob(path)
+          missing_blobs << [path, entry["sha"]]
+        else
+          install_blob(local_path, path, entry["sha"])
         end
+      end
 
-        install_blob(local_path, path, entry["sha"])
+      Bosh::ThreadPool.new(:max_threads => @max_parallel_downloads, :logger => Logging::Logger.new(nil)).wrap do |pool|
+        missing_blobs.each do |path, sha|
+          pool.process do
+            local_path = download_blob(path)
+            install_blob(local_path, path, sha)
+          end
+        end
       end
     end
 
@@ -250,9 +265,9 @@ module Bosh::Cli
 
       checksum = file_checksum(blob_path)
 
-      progress(path, "uploading...")
+      @progress_renderer.start(path, "uploading...")
       object_id = @blobstore.create(File.open(blob_path, "r"))
-      progress(path, "uploaded\n".make_green)
+      @progress_renderer.finish(path, "uploaded")
 
       @index[path] = {
         "object_id" => object_id,
@@ -278,7 +293,8 @@ module Bosh::Cli
 
       blob = @index[path]
       size = blob["size"].to_i
-      tmp_file = File.open(File.join(Dir.mktmpdir, "bosh-blob"), "w")
+      blob_path = path.gsub(File::SEPARATOR, '-')
+      tmp_file = File.open(File.join(Dir.mktmpdir, blob_path), "w")
 
       download_label = "downloading"
       if size > 0
@@ -286,21 +302,22 @@ module Bosh::Cli
       end
 
       progress_bar = Thread.new do
+        @progress_renderer.start(path, "#{download_label}")
         loop do
           break unless size > 0
           if File.exists?(tmp_file.path)
             pct = 100 * File.size(tmp_file.path).to_f / size
-            progress(path, "#{download_label} (#{pct.to_i}%)...")
+            @progress_renderer.progress(path, "#{download_label}", pct.to_i)
           end
           sleep(0.2)
         end
       end
 
-      progress(path, "#{download_label}...")
+      @progress_renderer.progress(path, "#{download_label}", 100)
       @blobstore.get(blob["object_id"], tmp_file)
       tmp_file.close
       progress_bar.kill
-      progress(path, "downloaded\n".make_green)
+      @progress_renderer.finish(path, "downloaded")
 
       if file_checksum(tmp_file.path) != blob["sha"]
         err("Checksum mismatch for downloaded blob `#{path}'")
@@ -311,15 +328,6 @@ module Bosh::Cli
 
     private
 
-    # Renders blob operation progress
-    # @param [String] path Blob path relative to blobs dir
-    # @param [String] label Operation happening to a blob
-    def progress(path, label)
-      say("\r", " " * 80)
-      say("\r#{path.truncate(40).make_yellow} #{label}", "")
-      Bosh::Cli::Config.output.flush # Ruby 1.8 compatibility
-    end
-
     # @param [String] src Path to a file containing the blob
     # @param [String] dst Resulting blob path relative to blobs dir
     # @param [String] checksum Blob checksum
@@ -379,3 +387,4 @@ module Bosh::Cli
     end
   end
 end
+

data/lib/cli/client/director.rb

@@ -693,6 +693,8 @@ module Bosh
         rescue URI::Error,
                SocketError,
                Errno::ECONNREFUSED,
+               Errno::ETIMEDOUT,
+               Errno::ECONNRESET,
                Timeout::Error,
                HTTPClient::TimeoutError,
                HTTPClient::KeepAliveDisconnected,

data/lib/cli/commands/release/list_releases.rb

@@ -55,7 +55,7 @@ module Bosh::Cli::Command
           t.headings = 'Name', 'Versions', 'Commit Hash'
           t.headings << 'Jobs' if show_jobs
           releases.each do |release|
-            versions, commit_hashes = formatted_versions(release).transpose
+            versions, commit_hashes = formatted_versions(release['release_versions']).transpose
             row = [release['name'], versions.join("\n"), commit_hashes.join("\n")]
             if show_jobs
               jobs = formatted_jobs(release).transpose
@@ -66,8 +66,12 @@ module Bosh::Cli::Command
         end
       end
 
-      def formatted_versions(release)
-        sort_versions(release['release_versions']).map { |v| formatted_version_and_commit_hash(v) }
+      def formatted_versions(release_versions)
+        if release_versions.empty?
+          [["unknown", "unknown"]]
+        else
+          sort_versions(release_versions).map { |v| formatted_version_and_commit_hash(v) }
+        end
       end
 
       def sort_versions(versions)

data/lib/cli/config.rb

@@ -19,6 +19,9 @@ module Bosh::Cli
 
       # @return [Integer] CLI polling interval
       attr_accessor :poll_interval
+
+      # @return [Integer] CLI max parallel downloads
+      attr_accessor :max_parallel_downloads
     end
 
     @commands = {}
@@ -166,6 +169,13 @@ module Bosh::Cli
       end
     end
 
+    # Read the max parallel downloads configuration.
+    #
+    # @return [Integer] The maximum number of parallel downloads
+    def max_parallel_downloads
+      self.class.max_parallel_downloads || @config_file.fetch("max_parallel_downloads", 1)
+    end
+
     def read(attr, try_local_first = true)
       attr = attr.to_s
       if try_local_first && @config_file[@work_dir].is_a?(Hash) &&

data/lib/cli/interactive_progress_renderer.rb

@@ -0,0 +1,70 @@
+module Bosh::Cli
+  class InteractiveProgressRenderer
+    def initialize
+      @mutex = Mutex.new
+      @indices = {}
+    end
+
+    def start(path, label)
+      render(path, label)
+    end
+
+    def progress(path, label, percent)
+      render(path, "#{label} (#{percent}%)")
+    end
+
+    def error(path, message)
+      render(path, message.make_red)
+    end
+
+    def finish(path, label)
+      render(path, label.make_green)
+    end
+
+    private
+
+    def render(path, label)
+      @mutex.synchronize do
+        truncated_path = path.truncate(40)
+        truncated_path_length = truncated_path.length + 1
+
+        if !@indices.has_key?(path)
+          say(truncated_path.make_yellow, " \n")
+        end
+
+        @indices[path] ||= @indices.count
+
+        save_cursor_position
+        up(@indices.count - @indices[path])
+
+        right(truncated_path_length)
+        clear_line
+
+        say(label, "")
+
+        restore_cursor_position
+        Bosh::Cli::Config.output.flush # Ruby 1.8 compatibility
+      end
+    end
+
+    def save_cursor_position
+      say("\033[s", "")
+    end
+
+    def restore_cursor_position
+      say("\033[u", "")
+    end
+
+    def up(count = 1)
+      say("\033[#{count}A", "")
+    end
+
+    def right(count = 1)
+      say("\033[#{count}C", "")
+    end
+
+    def clear_line
+      say("\033[K", "")
+    end
+  end
+end

data/lib/cli/non_interactive_progress_renderer.rb

@@ -0,0 +1,32 @@
+module Bosh::Cli
+  class NonInteractiveProgressRenderer
+    def initialize
+      @mutex = Mutex.new
+    end
+
+    def start(path, label)
+      render(path, label)
+    end
+
+    def progress(path, label, percent)
+    end
+
+    def error(path, message)
+      render(path, message)
+    end
+
+    def finish(path, label)
+      render(path, label)
+    end
+
+    private
+
+    def render(path, label)
+      @mutex.synchronize do
+        truncated_path = path.truncate(40)
+        say("#{truncated_path} #{label}")
+        Bosh::Cli::Config.output.flush # Ruby 1.8 compatibility
+      end
+    end
+  end
+end

data/lib/cli/runner.rb

@@ -114,6 +114,10 @@ module Bosh::Cli
         @options[:config] = file
       end
 
+      opts.on("--parallel MAX", "Sets the max number of parallel downloads") do |max|
+        Config.max_parallel_downloads = Integer(max)
+      end
+
       opts.on("--[no-]color", "Toggle colorized output") do |v|
         Config.colorize = v
       end

data/lib/cli/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Cli
-    VERSION = '1.2792.0'
+    VERSION = '1.2797.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bosh_cli
 version: !ruby/object:Gem::Version
-  version: 1.2792.0
+  version: 1.2797.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-16 00:00:00.000000000 Z
+date: 2014-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bosh_common
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2792.0
+        version: 1.2797.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2792.0
+        version: 1.2797.0
 - !ruby/object:Gem::Dependency
   name: bosh-template
   requirement: !ruby/object:Gem::Requirement
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2792.0
+        version: 1.2797.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2792.0
+        version: 1.2797.0
 - !ruby/object:Gem::Dependency
   name: json_pure
   requirement: !ruby/object:Gem::Requirement
@@ -130,7 +130,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2792.0
+        version: 1.2797.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -138,7 +138,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2792.0
+        version: 1.2797.0
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -221,7 +221,7 @@ dependencies:
         version: 0.5.4
 description: ! 'BOSH CLI
 
-  5e7f33'
+  3fe6aa'
 email: support@cloudfoundry.com
 executables:
 - bosh
@@ -282,6 +282,7 @@ files:
 - lib/cli/download_with_progress.rb
 - lib/cli/errors.rb
 - lib/cli/file_with_progress_bar.rb
+- lib/cli/interactive_progress_renderer.rb
 - lib/cli/job_builder.rb
 - lib/cli/job_command_args.rb
 - lib/cli/job_property_collection.rb
@@ -291,6 +292,7 @@ files:
 - lib/cli/logs_downloader.rb
 - lib/cli/manifest_warnings.rb
 - lib/cli/name_version_pair.rb
+- lib/cli/non_interactive_progress_renderer.rb
 - lib/cli/package_builder.rb
 - lib/cli/packaging_helper.rb
 - lib/cli/public_stemcell.rb
@@ -344,7 +346,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1339837986300098617
+      hash: -2793466830286606539
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23