bosh_cli 1.3112.0 → 1.3115.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ba60ab5dc5440c8eed837deee9de75556c618117
-  data.tar.gz: e6b55ec578ccdc53e735707ecb15bf8f7f76eb61
+  metadata.gz: dfd2d53b788b054d99bd8d7708453c81396f7c85
+  data.tar.gz: c3137783a4253dcaa07585072913d86884df6070
 SHA512:
-  metadata.gz: eb36ca9acb95bddcd1af014f85343b1da61c0049d945ac84dcded6b9010a7c7099e65c911f0e4d35cd8241a4fb20a8a91ccd9f8d99ee28ad0508a147ad5fd4ac
-  data.tar.gz: 8cab925642eb35845486f459fb5a9209382ec1864eea040d9b43f59c2286c82bd39a959f0c859ec467231853f1c2cbd83a28a5920085fe441f345066aea10af2
+  metadata.gz: 508628bc1637eb908b4b6b2c0e0150ab4dfaee34d7a5f8ec27a6960a3802b071901caa77339bba42daaca0cb14615e8e36b7f66b177bc56b5a870c102ae6bc23
+  data.tar.gz: a2585a85ad0c31c0f11554d439dd36e0b81557a7eb7414fed3d54ad6ec33e67b5b45344769712d004e2ee04c4ce8b42d7af16ae414279216f06521c6d394c43f

data/lib/cli/commands/release/upload_release.rb

@@ -84,14 +84,10 @@ module Bosh::Cli::Command
         repack = upload_options[:repack]
         rebase = upload_options[:rebase]
 
-        say("\nVerifying release...")
-        tarball.validate(:allow_sparse => true)
+        say("\nVerifying manifest...")
+        tarball.validate_manifest
         nl
 
-        unless tarball.valid?
-          err('Release is invalid, please fix, verify and upload again')
-        end
-
         if should_convert_to_old_format?(tarball.version)
           msg = "You are using CLI > 1.2579.0 with a director that doesn't support " +
             'the new version format you are using. Upgrade your ' +
@@ -112,6 +108,16 @@ module Bosh::Cli::Command
               package_matches = match_remote_packages(tarball.manifest)
             end
 
+            # if the director is an older version that doesn't support optimized unpack we
+            # have to upload everything
+            if tarball.upload_packages?(package_matches) || !director_supports_fast_unpack?
+              tarball.validate(:allow_sparse => true, :validate_manifest => false)
+              err('Release is invalid, please fix, verify and upload again') unless tarball.valid?
+            else
+              tarball.validate_jobs(:allow_sparse => true, :validate_job_packages => false)
+              tarball.print_manifest
+            end
+
             say('Checking if can repack release for faster upload...')
             repacked_path = tarball.repack(package_matches)
 
@@ -216,6 +222,13 @@ module Bosh::Cli::Command
         Bosh::Common::Version::BoshVersion.parse(director_version) >=
           Bosh::Common::Version::BoshVersion.parse(new_format_director_version)
       end
+
+      def director_supports_fast_unpack?
+        director_version = director.get_status['version']
+        new_format_director_version = '1.3100.0'
+        Bosh::Common::Version::BoshVersion.parse(director_version) >
+            Bosh::Common::Version::BoshVersion.parse(new_format_director_version)
+      end
     end
   end
 end

data/lib/cli/commands/ssh.rb

@@ -15,6 +15,8 @@ module Bosh::Cli
              'Use default ssh password (NOT RECOMMENDED)'
       option '--strict_host_key_checking <yes/no>',
              'Can use this flag to skip host key checking (NOT RECOMMENDED)'
+      option '--no_gateway',
+             'Ignore gateway provided by the director'
 
       def shell(*args)
         if args.size > 0
@@ -46,6 +48,8 @@ module Bosh::Cli
       option '--gateway_host HOST', 'Gateway host'
       option '--gateway_user USER', 'Gateway user'
       option '--gateway_identity_file FILE', 'Gateway identity file'
+      option '--no_gateway',
+             'Ignore gateway provided by the director'
 
       def scp(*args)
         job, index, args = JobCommandArgs.new(args).to_a
@@ -138,10 +142,11 @@ module Bosh::Cli
         ssh_session.set_host_session(sessions.first)
 
         begin
-          if options[:gateway_host]
+          if options[:gateway_host] || (!options[:no_gateway] &&
+              sessions.first["gateway_host"])
             require 'net/ssh/gateway'
-            gw_host    = options[:gateway_host]
-            gw_user    = options[:gateway_user] || ENV['USER']
+            gw_host    = options[:gateway_host] || sessions.first["gateway_host"]
+            gw_user    = options[:gateway_user] || sessions.first["gateway_user"] || ENV['USER']
             gw_options = {}
             gw_options[:keys] = [options[:gateway_identity_file]] if options[:gateway_identity_file]
             begin
@@ -167,15 +172,7 @@ module Bosh::Cli
       # @param [String] job Job name
       # @param [Integer] index Job index
       def setup_interactive_shell(deployment_name, job, index)
-        password = options[:default_password]
-
-        if password.nil?
-          password = ask(
-            'Enter password (use it to ' +
-              'sudo on remote host): ') { |q| q.echo = '*' }
-
-          err('Please provide ssh password') if password.blank?
-        end
+        password = options[:default_password] || ''
 
         setup_ssh(deployment_name, job, index, password) do |sessions, gateway, ssh_session|
           session = sessions.first
@@ -206,7 +203,9 @@ module Bosh::Cli
       end
 
       def perform_operation(operation, deployment_name, job, index, args)
-        setup_ssh(deployment_name, job, index, options[:default_password]) do |sessions, gateway, ssh_session|
+        password = options[:default_password] || ''
+
+        setup_ssh(deployment_name, job, index, password) do |sessions, gateway, ssh_session|
           sessions.each do |session|
             unless session['status'] == 'success' && session['ip']
               err("Failed to set up SSH on #{job}/#{index}: #{session.inspect}")

data/lib/cli/release.rb

@@ -73,9 +73,6 @@ module Bosh::Cli
       has_legacy_secret? ||
         has_blobstore_secrets?(bs, "simple", "user", "password") ||
         has_blobstore_secrets?(bs, "dav", "user", "password") ||
-        has_blobstore_secrets?(bs, "swift", "rackspace") ||
-        has_blobstore_secrets?(bs, "swift", "hp") ||
-        has_blobstore_secrets?(bs, "swift", "openstack") ||
         has_blobstore_secrets?(bs, "s3", "access_key_id", "secret_access_key")
     end
 

data/lib/cli/release_tarball.rb

@@ -19,6 +19,38 @@ module Bosh::Cli
       end
     end
 
+    def unpack_manifest
+      return @unpacked_manifest unless @unpacked_manifest.nil?
+      exit_success = fast_unpack('./release.MF')
+      @unpacked_manifest = !!exit_success
+    end
+
+    def unpack_jobs
+      return @unpacked_jobs unless @unpacked_jobs.nil?
+      exit_success = fast_unpack('./jobs/')
+      @unpacked_jobs = !!exit_success
+    end
+
+    def unpack_license
+      return @unpacked_license unless @unpacked_license.nil?
+      exit_success = fast_unpack('./license.tgz')
+      @unpacked_license = !!exit_success
+    end
+
+    def fast_unpack(target)
+      if RUBY_PLATFORM =~ /linux/
+        system("tar", "-C", @unpack_dir, "-xzf", @tarball_path, "--occurrence", "#{target}", out: "/dev/null", err: "/dev/null")
+      elsif RUBY_PLATFORM =~ /darwin/
+        if target[-1, 1] == "/"
+          system("tar", "-C", @unpack_dir, "-xzf", @tarball_path, "#{target}", out: "/dev/null", err: "/dev/null")
+        else
+          system("tar", "-C", @unpack_dir, "--fast-read", "-xzf", @tarball_path, "#{target}", out: "/dev/null", err: "/dev/null")
+        end
+      else
+        system("tar", "-C", @unpack_dir, "-xzf", @tarball_path, "#{target}", out: "/dev/null", err: "/dev/null")
+      end
+    end
+
     # Unpacks tarball to @unpack_dir, returns true if succeeded, false if failed
     def unpack
       return @unpacked unless @unpacked.nil?
@@ -38,13 +70,19 @@ module Bosh::Cli
 
     def manifest
       return nil unless valid?
-      unpack
+      unpack_manifest
       File.read(File.join(@unpack_dir, "release.MF"))
     end
 
+    def manifest_yaml
+      return @manifest_yaml unless @manifest_yaml.nil?
+      unpack_manifest
+      manifest_file = File.expand_path("release.MF", @unpack_dir)
+      @manifest_yaml = load_yaml_file(manifest_file)
+    end
+
     def compiled_release?
-      unpack
-      File.exists?(File.expand_path("compiled_packages", @unpack_dir))
+      manifest_yaml.has_key?('compiled_packages')
     end
 
     def replace_manifest(hash)
@@ -91,11 +129,16 @@ module Bosh::Cli
       end
     end
 
+    def upload_packages?(package_matches = [])
+      return true if package_matches.nil?
+      package_matches.size != manifest_yaml[@packages_folder].size
+    end
+
     # Repacks tarball according to the structure of remote release
     # Return path to repackaged tarball or nil if repack has failed
     def repack(package_matches = [])
       return nil unless valid?
-      unpack
+      unpack if upload_packages?(package_matches)
 
       tmpdir = Dir.mktmpdir
       repacked_path = File.join(tmpdir, "release-repack.tgz")
@@ -116,7 +159,7 @@ module Bosh::Cli
               package_matches.include?(package["fingerprint"]))
             say("SKIP".make_green)
             @skipped += 1
-            FileUtils.rm_rf(File.join("packages", "#{package["name"]}.tgz"))
+            FileUtils.rm_rf(File.join(@packages_folder, "#{package["name"]}.tgz"))
           else
             say("UPLOAD".make_red)
           end
@@ -137,42 +180,50 @@ module Bosh::Cli
     # and packages in place when we do validation. However for jobs and packages
     # that are present we still need to validate checksums
     def perform_validation(options = {})
-      allow_sparse = options.fetch(:allow_sparse, false)
-      should_print_release_info = options.fetch(:print_release_info, true)
-
-      step("File exists and readable",
-           "Cannot find release file #{@tarball_path}", :fatal) do
+      step("File exists and readable", "Cannot find release file #{@tarball_path}", :fatal) do
         exists?
       end
 
-      step("Extract tarball",
-           "Cannot extract tarball #{@tarball_path}", :fatal) do
-        unpack
+      validate_manifest if options.fetch(:validate_manifest, true)
+      validate_packages(options)
+      validate_jobs(options)
+
+      print_manifest if options.fetch(:print_release_info, true)
+    end
+
+    def validate_manifest
+      step("Extract manifest",
+           "Cannot extract manifest #{@tarball_path}", :fatal) do
+        unpack_manifest
       end
 
       manifest_file = File.expand_path("release.MF", @unpack_dir)
-
       step("Manifest exists", "Cannot find release manifest", :fatal) do
         File.exists?(manifest_file)
       end
 
-      manifest = load_yaml_file(manifest_file)
+      @manifest_yaml = nil
 
       step("Release name/version",
            "Manifest doesn't contain release name and/or version") do
-        manifest.is_a?(Hash) &&
-            manifest.has_key?("name") &&
-            manifest.has_key?("version")
+        manifest_yaml.is_a?(Hash) &&
+            manifest_yaml.has_key?("name") &&
+            manifest_yaml.has_key?("version")
       end
 
-      @release_name = manifest["name"]
-      @version = manifest["version"].to_s
+      @release_name = manifest_yaml["name"]
+      @version = manifest_yaml["version"].to_s
+      @validated = true
+    end
 
-      # Check packages
-      total_packages = manifest[@packages_folder].size
-      available_packages = {}
+    def validate_packages(options = {})
+      allow_sparse = options.fetch(:allow_sparse, false)
+      unpack
+
+      total_packages = manifest_yaml[@packages_folder].size
+      @available_packages = {}
 
-      manifest[@packages_folder].each_with_index do |package, i|
+      manifest_yaml[@packages_folder].each_with_index do |package, i|
         @packages << package
         name, version = package['name'], package['version']
 
@@ -185,7 +236,7 @@ module Bosh::Cli
         end
 
         if package_exists
-          available_packages[name] = true
+          @available_packages[name] = true
           step("Package '#{name}' checksum",
                "Incorrect checksum for package '#{name}'") do
             Digest::SHA1.file(package_file).hexdigest == package["sha1"]
@@ -201,8 +252,8 @@ module Bosh::Cli
         step("Package dependencies",
              "Package dependencies couldn't be resolved") do
           begin
-            tsort_packages(manifest[@packages_folder].inject({}) { |h, p|
-              h[p["name"]] = p["dependencies"] || []; h })
+            tsort_packages(manifest_yaml[@packages_folder].inject({}) { |h, p|
+                             h[p["name"]] = p["dependencies"] || []; h })
             true
           rescue Bosh::Cli::CircularDependency,
               Bosh::Cli::MissingDependency => e
@@ -211,26 +262,30 @@ module Bosh::Cli
           end
         end
       end
+    end
 
-      # Check jobs
-      total_jobs = manifest["jobs"].size
+    def validate_jobs(options = {})
+      allow_sparse = options.fetch(:allow_sparse, false)
+      unpack_jobs
+      unpack_license
+
+      total_jobs = manifest_yaml["jobs"].size
 
       step("Checking jobs format",
-           "Jobs are not versioned, please re-create release " +
-               "with current CLI version (or any CLI >= 0.4.4)", :fatal) do
-        total_jobs > 0 && manifest["jobs"][0].is_a?(Hash)
+           "Jobs are not versioned, please re-create release with current CLI version (or any CLI >= 0.4.4)", :fatal) do
+        total_jobs > 0 && manifest_yaml["jobs"][0].is_a?(Hash)
       end
 
-      manifest["jobs"].each_with_index do |job, i|
+      manifest_yaml["jobs"].each_with_index do |job, i|
         @jobs << job
+
         name    = job["name"]
         version = job["version"]
 
         job_file   = File.expand_path(name + ".tgz", @unpack_dir + "/jobs")
         job_exists = File.exists?(job_file)
 
-        step("Read job '%s' (%d of %d), version %s" % [name, i+1, total_jobs,
-                                                       version],
+        step("Read job '%s' (%d of %d), version %s" % [name, i+1, total_jobs, version],
              "Job '#{name}' not found") do
           job_exists || allow_sparse
         end
@@ -265,32 +320,29 @@ module Bosh::Cli
               job_manifest["templates"].each_key do |template|
                 step("Check template '#{template}' for '#{name}'",
                      "No template named '#{template}' for '#{name}'") do
-                  File.exists?(File.expand_path(template,
-                                                job_tmp_dir + "/templates"))
+                  File.exists?(File.expand_path(template, job_tmp_dir + "/templates"))
                 end
               end
             end
 
-            if job_manifest_valid && job_manifest["packages"]
+            validate_job_packages = options.fetch(:validate_job_packages, true)
+
+            if validate_job_packages && job_manifest_valid && job_manifest["packages"]
               job_manifest["packages"].each do |package_name|
                 step("Job '#{name}' needs '#{package_name}' package",
-                     "Job '#{name}' references missing package " +
-                         "'#{package_name}'") do
-                  available_packages[package_name] || allow_sparse
+                     "Job '#{name}' references missing package '#{package_name}'") do
+                  @available_packages[package_name] || allow_sparse
                 end
               end
             end
 
             step("Monit file for '#{name}'",
                  "Monit script missing for job '#{name}'") do
-              File.exists?(File.expand_path("monit", job_tmp_dir)) ||
-                  Dir.glob("#{job_tmp_dir}/*.monit").size > 0
+              File.exists?(File.expand_path("monit", job_tmp_dir)) || Dir.glob("#{job_tmp_dir}/*.monit").size > 0
             end
           end
         end
       end
-
-      print_info(manifest) if should_print_release_info
     end
 
     class TarballArtifact
@@ -330,7 +382,8 @@ module Bosh::Cli
       []
     end
 
-    def print_info(manifest)
+    def print_manifest
+      manifest = manifest_yaml
       say("\nRelease info")
       say("------------")
 
@@ -363,6 +416,7 @@ module Bosh::Cli
       else
         say("  - license (#{manifest["license"]["version"]})")
       end
+      nl
     end
   end
 end

data/lib/cli/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Cli
-    VERSION = '1.3112.0'
+    VERSION = '1.3115.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh_cli
 version: !ruby/object:Gem::Version
-  version: 1.3112.0
+  version: 1.3115.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-21 00:00:00.000000000 Z
+date: 2015-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bosh_common
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3112.0
+        version: 1.3115.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3112.0
+        version: 1.3115.0
 - !ruby/object:Gem::Dependency
   name: bosh-template
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3112.0
+        version: 1.3115.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3112.0
+        version: 1.3115.0
 - !ruby/object:Gem::Dependency
   name: cf-uaa-lib
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3112.0
+        version: 1.3115.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3112.0
+        version: 1.3115.0
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement