bosh_cli 1.2840.0 → 1.2847.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e93ae5c2fe1c781b37b94fbff419995baa1042a
-  data.tar.gz: 073cca5e0b6b366f4fa0654b24625130e55f4039
+  metadata.gz: 4cfc5df8739eb9e7d78d6a9a2c84c0f1f2fd0e1a
+  data.tar.gz: a1c755ce722b3f95391f3ba5e380d13fd11ff40f
 SHA512:
-  metadata.gz: ed49e3f58a0b79aa89d968a9bbe75acaa2c67a12df7d66bb65e4646b3e8af59a0a967a454e0844e8859696ea576dc1167ec9b511588aa6c8c8060fbc27338d76
-  data.tar.gz: 642c36c8276eab9eb80ace81194994d392fa31c6b41fb78dba2630ce20945e1eff609379e85043ee9a36fc9d20bcb2c4fb3c5380704d570384920246f309a8d6
+  metadata.gz: 1dc06fa69265c241c855cc4d93710ddda55b63c82e4cfa4f9a48340e0f97c84763eb1b75ed2ce729f493b41c0cc34507e1660c94a3384f16eeec095b29bf1f0f
+  data.tar.gz: cb3e88d9a6958aa4ec9da5ce26a28ab1588be46b6eedb4120a90b95ebab4b16916c76873b427722e0a240eaec79afc382c5e69f9f9dc3d5df58c462d330e17bd

data/lib/cli.rb

@@ -70,7 +70,7 @@ require 'cli/non_interactive_progress_renderer'
 require 'cli/source_control/git_ignore'
 
 require 'cli/versions/versions_index'
-require 'cli/versions/local_version_storage'
+require 'cli/versions/local_artifact_storage'
 require 'cli/versions/release_versions_index'
 require 'cli/versions/releases_dir_migrator'
 require 'cli/versions/version_file_resolver'

data/lib/cli/archive_builder.rb

@@ -66,7 +66,7 @@ module Bosh::Cli
       end
 
       if artifact.dev_artifact? && final? && !dry_run?
-        artifact = @archive_repository.copy_from_dev_to_final(artifact)
+        @archive_repository.promote_from_dev_to_final(artifact)
       end
 
       artifact

data/lib/cli/archive_repository.rb

@@ -1,20 +1,21 @@
 module Bosh::Cli
   class ArchiveRepository
-    def initialize(archive_dir, blobstore, resource)
+    def initialize(archive_dir, artifacts_dir, blobstore, resource)
       @archive_dir = archive_dir
       @blobstore = blobstore
 
-      dev_builds_dir = Pathname(@archive_dir).join(".dev_builds", resource.plural_type, resource.name).to_s
+      dev_builds_dir = Pathname(@archive_dir).join('.dev_builds', resource.plural_type, resource.name).to_s
       FileUtils.mkdir_p(dev_builds_dir)
       @dev_index = Versions::VersionsIndex.new(dev_builds_dir)
-      @dev_storage = Versions::LocalVersionStorage.new(dev_builds_dir)
 
-      final_builds_dir = Pathname(@archive_dir).join(".final_builds", resource.plural_type, resource.name).to_s
+      final_builds_dir = Pathname(@archive_dir).join('.final_builds', resource.plural_type, resource.name).to_s
       FileUtils.mkdir_p(final_builds_dir)
       @final_index = Versions::VersionsIndex.new(final_builds_dir)
-      @final_storage = Versions::LocalVersionStorage.new(final_builds_dir)
 
-      @final_resolver = Versions::VersionFileResolver.new(@final_storage, @blobstore)
+      @storage = Versions::LocalArtifactStorage.new(artifacts_dir)
+      FileUtils.mkdir_p(artifacts_dir)
+
+      @final_resolver = Versions::VersionFileResolver.new(@storage, @blobstore)
     end
 
     def lookup(resource)
@@ -26,18 +27,18 @@ module Bosh::Cli
         version = artifact_info['version'] || fingerprint
         sha1 = artifact_info['sha1']
 
-        say('Using final version')
-        tarball_path = @final_resolver.find_file(blobstore_id, sha1, version, "#{resource.singular_type} #{resource.name} (#{version})") # todo: 'package' vs 'job'
+        say("Using final version '#{version}'")
+        tarball_path = @final_resolver.find_file(blobstore_id, sha1, "#{resource.singular_type} #{resource.name} (#{version})")
 
         BuildArtifact.new(resource.name, fingerprint, tarball_path, sha1, resource.dependencies, false, false)
       else
         artifact_info = @dev_index[fingerprint]
         if artifact_info
-          version = artifact_info['version'] || fingerprint
-          if @dev_storage.has_file?(version)
-            say('Using dev version')
+          if @storage.has_file?(artifact_info['sha1'])
+            version = artifact_info['version'] || fingerprint
+            say("Using dev version '#{version}'")
 
-            tarball_path = @dev_storage.get_file(version)
+            tarball_path = @storage.get_file(artifact_info['sha1'])
             if file_checksum(tarball_path) != artifact_info['sha1']
               raise CorruptedArchive, "#{resource.singular_type} #{resource.name} (#{version}) archive at #{tarball_path} corrupted"
             end
@@ -75,27 +76,33 @@ module Bosh::Cli
     def install(artifact)
       fingerprint = artifact.fingerprint
       origin_file = artifact.tarball_path
-      new_tarball_path = place_file_and_update_index(fingerprint, origin_file,
-        artifact.dev_artifact? ? @dev_index : @final_index,
-        artifact.dev_artifact? ? @dev_storage : @final_storage)
 
-      BuildArtifact.new(artifact.name, artifact.fingerprint, new_tarball_path, artifact.sha1, artifact.dependencies, artifact.new_version?, artifact.dev_artifact?)
+      tarball_path = @storage.put_file(artifact.sha1, origin_file)
+
+      update_index(
+        tarball_path,
+        fingerprint,
+        artifact.dev_artifact? ? @dev_index : @final_index)
+
+      BuildArtifact.new(artifact.name, artifact.fingerprint, tarball_path, artifact.sha1, artifact.dependencies, artifact.new_version?, artifact.dev_artifact?)
     end
 
-    def copy_from_dev_to_final(artifact)
-      final_tarball_path = place_file_and_update_index(artifact.fingerprint, artifact.tarball_path, @final_index, @final_storage)
-      BuildArtifact.new(artifact.name, artifact.fingerprint, final_tarball_path, artifact.sha1, artifact.dependencies, artifact.new_version?, false)
+    def promote_from_dev_to_final(artifact)
+      update_index(artifact.tarball_path, artifact.fingerprint, @final_index)
+      artifact.promote_to_final
     end
 
     private
 
-    def place_file_and_update_index(fingerprint, origin_file, index, storage)
-      # add version (with its validation) before adding sha1
-      index.add_version(fingerprint, {'version' => fingerprint} )
-      tarball_path = storage.put_file(fingerprint, origin_file)
+    def update_index(tarball_path, fingerprint, index)
+      # In case of corrupted file the new file will be downloaded/re-generated
+      # so sha1 needs to be updated to fix the index
+      unless index[fingerprint]
+        index.add_version(fingerprint, {'version' => fingerprint} )
+      end
+
       sha1 = file_checksum(tarball_path)
       index.update_version(fingerprint, {'version' => fingerprint, 'sha1' => sha1})
-      tarball_path
     end
 
     def file_checksum(path)

data/lib/cli/archive_repository_provider.rb

@@ -1,12 +1,13 @@
 module Bosh::Cli
   class ArchiveRepositoryProvider
-    def initialize(archive_dir, blobstore)
+    def initialize(archive_dir, artifacts_dir, blobstore)
       @archive_dir = archive_dir
+      @artifacts_dir = artifacts_dir
       @blobstore = blobstore
     end
 
     def get(resource)
-      ArchiveRepository.new(@archive_dir, @blobstore, resource)
+      ArchiveRepository.new(@archive_dir, @artifacts_dir, @blobstore, resource)
     end
   end
 end

data/lib/cli/base_command.rb

@@ -115,6 +115,10 @@ module Bosh::Cli
         options[:target] || config.target_name || target_url
       end
 
+      def cache_dir
+        File.join(Dir.home, '.bosh', 'cache')
+      end
+
       protected
 
       # Prints director task completion report. Note that event log usually

data/lib/cli/build_artifact.rb

@@ -13,6 +13,10 @@ module Bosh::Cli
       @is_new_version = is_new_version
     end
 
+    def promote_to_final
+      @is_dev_artifact = false
+    end
+
     def version
       fingerprint
     end

data/lib/cli/commands/release/create_release.rb

@@ -26,7 +26,7 @@ module Bosh::Cli::Command
           end
 
           say('Recreating release from the manifest')
-          Bosh::Cli::ReleaseCompiler.compile(manifest_file, release.blobstore)
+          Bosh::Cli::ReleaseCompiler.compile(manifest_file, cache_dir, release.blobstore)
           release_filename = manifest_file
         else
           version = options[:version]
@@ -84,6 +84,8 @@ module Bosh::Cli::Command
           header('Building DEV release'.make_green)
         end
 
+        say("Release artifact cache: #{cache_dir}")
+
         header('Building packages')
         package_artifacts = build_packages(dry_run, final)
 
@@ -162,7 +164,7 @@ module Bosh::Cli::Command
       end
 
       def archive_repository_provider
-        @archive_repository_provider ||= Bosh::Cli::ArchiveRepositoryProvider.new(work_dir, release.blobstore)
+        @archive_repository_provider ||= Bosh::Cli::ArchiveRepositoryProvider.new(work_dir, cache_dir, release.blobstore)
       end
 
       def build_release(dry_run, final, job_artifacts, manifest_only, package_artifacts, name, version)

data/lib/cli/commands/release/upload_release.rb

@@ -53,7 +53,7 @@ module Bosh::Cli::Command
         blobstore = release.blobstore
         tmpdir = Dir.mktmpdir
 
-        compiler = Bosh::Cli::ReleaseCompiler.new(manifest_path, blobstore, package_matches)
+        compiler = Bosh::Cli::ReleaseCompiler.new(manifest_path, cache_dir, blobstore, package_matches)
         need_repack = true
 
         unless compiler.exists?

data/lib/cli/release_builder.rb

@@ -25,9 +25,9 @@ module Bosh::Cli
       @final_index = Versions::VersionsIndex.new(final_releases_dir)
       @dev_index = Versions::VersionsIndex.new(dev_releases_dir)
       @index = @final ? @final_index : @dev_index
-      @release_storage = Versions::LocalVersionStorage.new(@index.storage_dir, @name)
+      @release_storage = Versions::LocalArtifactStorage.new(@index.storage_dir)
 
-      if @version && @release_storage.has_file?(@version)
+      if @version && @release_storage.has_file?(release_filename)
         raise ReleaseVersionError.new('Release version already exists')
       end
 
@@ -49,6 +49,10 @@ module Bosh::Cli
       @final
     end
 
+    def release_filename
+      "#{@name}-#{@version}.tgz"
+    end
+
     # @return [Array<Bosh::Cli::BuildArtifact>] List of job artifacts
     #   affected by this release compared to the previous one.
     def affected_jobs
@@ -156,7 +160,7 @@ module Bosh::Cli
 
     def generate_tarball
       generate_manifest unless @manifest_generated
-      return if @release_storage.has_file?(@version)
+      return if @release_storage.has_file?(release_filename)
 
       unless @jobs_copied
         header("Copying jobs...")

data/lib/cli/release_compiler.rb

@@ -6,8 +6,8 @@ module Bosh::Cli
 
     attr_writer :tarball_path
 
-    def self.compile(manifest_file, blobstore)
-      new(manifest_file, blobstore).compile
+    def self.compile(manifest_file, artifacts_dir, blobstore)
+      new(manifest_file, artifacts_dir, blobstore).compile
     end
 
     # @param [String] manifest_file Release manifest path
@@ -15,7 +15,7 @@ module Bosh::Cli
     # @param [Array] package_matches List of package checksums that director
     #   can match
     # @param [String] release_source Release directory
-    def initialize(manifest_file, blobstore,
+    def initialize(manifest_file, artifacts_dir, blobstore,
                    package_matches = [], release_source = nil)
 
       @blobstore = blobstore
@@ -23,6 +23,7 @@ module Bosh::Cli
       @manifest_file = File.expand_path(manifest_file, @release_source)
       @tarball_path = nil
 
+      @artifacts_dir = artifacts_dir
       @build_dir = Dir.mktmpdir
       @jobs_dir = File.join(@build_dir, "jobs")
       @packages_dir = File.join(@build_dir, "packages")
@@ -131,14 +132,13 @@ module Bosh::Cli
         err("Cannot find #{build_type} with checksum `#{build.sha1}'")
       end
 
-      version = found_build["version"]
       sha1 = found_build["sha1"]
       blobstore_id = found_build["blobstore_id"]
 
-      storage = Versions::LocalVersionStorage.new(index.storage_dir)
+      storage = Versions::LocalArtifactStorage.new(@artifacts_dir)
 
       resolver = Versions::VersionFileResolver.new(storage, @blobstore)
-      resolver.find_file(blobstore_id, sha1, version, "#{build_type} #{desc}")
+      resolver.find_file(blobstore_id, sha1, "#{build_type} #{desc}")
     rescue Bosh::Blobstore::BlobstoreError => e
       raise BlobstoreError, "Blobstore error: #{e}"
     end

data/lib/cli/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Cli
-    VERSION = '1.2840.0'
+    VERSION = '1.2847.0'
   end
 end

data/lib/cli/versions/{local_version_storage.rb → local_artifact_storage.rb}

@@ -1,40 +1,39 @@
 module Bosh::Cli::Versions
-  class LocalVersionStorage
+  class LocalArtifactStorage
 
     class Sha1MismatchError < StandardError; end
 
     attr_reader :storage_dir
 
-    def initialize(storage_dir, name_prefix=nil)
+    def initialize(storage_dir)
       @storage_dir = storage_dir
-      @name_prefix = name_prefix
     end
 
-    def put_file(version, origin_file_path)
-      destination = file_path(version)
+    def put_file(sha, origin_file_path)
+      destination = file_path(sha)
       unless File.exist?(origin_file_path)
-        raise "Trying to store non-existant file `#{origin_file_path}' for version `#{version}'"
+        raise "Trying to store non-existant file `#{origin_file_path}' with sha `#{sha}'"
       end
+      FileUtils.mkdir_p(File.dirname(destination))
       FileUtils.cp(origin_file_path, destination, :preserve => true)
 
       File.expand_path(destination)
     end
 
-    def get_file(version)
-      destination = file_path(version)
+    def get_file(sha)
+      destination = file_path(sha)
       unless File.exist?(destination)
-        raise "Trying to retrieve non-existant file `#{destination}' for version `#{version}'"
+        raise "Trying to retrieve non-existant file `#{destination}' with sha `#{sha}'"
       end
 
       File.expand_path(destination)
     end
 
-    def has_file?(version)
-      File.exists?(file_path(version))
+    def has_file?(sha)
+      File.exists?(file_path(sha))
     end
 
-    def file_path(version)
-      name = @name_prefix.blank? ? "#{version}.tgz" : "#{@name_prefix}-#{version}.tgz"
+    def file_path(name)
       File.join(@storage_dir, name)
     end
   end

data/lib/cli/versions/version_file_resolver.rb

@@ -7,9 +7,9 @@ module Bosh::Cli::Versions
       @tmpdir = tmpdir
     end
 
-    def find_file(blobstore_id, sha1, version, desc)
-      if @storage.has_file?(version)
-        file_path = @storage.get_file(version)
+    def find_file(blobstore_id, sha1, desc)
+      if @storage.has_file?(sha1)
+        file_path = @storage.get_file(sha1)
         file_sha1 = Digest::SHA1.file(file_path).hexdigest
         if file_sha1 == sha1
           return file_path
@@ -29,7 +29,7 @@ module Bosh::Cli::Versions
           @blobstore.get(blobstore_id, tmp_file, sha1: sha1)
         end
 
-        @storage.put_file(version, tmp_file_path)
+        @storage.put_file(sha1, tmp_file_path)
       ensure
         FileUtils.rm(tmp_file_path, :force => true)
       end

data/lib/cli/versions/versions_index.rb

@@ -31,7 +31,7 @@ module Bosh::Cli::Versions
       @index_file  = File.join(@storage_dir, 'index.yml')
 
       if File.file?(@index_file)
-        init_index(VersionsIndex.load_index_yaml(@index_file))
+        reload
       else
         init_index({})
       end
@@ -82,6 +82,10 @@ module Bosh::Cli::Versions
         raise "Cannot update non-existent entry with key `#{key}'"
       end
 
+      if old_build['blobstore_id']
+        raise "Cannot update entry `#{old_build}' with a blobstore id"
+      end
+
       if new_build['version'] != old_build['version']
         raise "Cannot update entry `#{old_build}' with a different version: `#{new_build}'"
       end
@@ -128,6 +132,10 @@ module Bosh::Cli::Versions
       VersionsIndex.write_index_yaml(@index_file, @data)
     end
 
+    def reload
+      init_index(VersionsIndex.load_index_yaml(@index_file))
+    end
+
     private
 
     def create_directories

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh_cli
 version: !ruby/object:Gem::Version
-  version: 1.2840.0
+  version: 1.2847.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-11 00:00:00.000000000 Z
+date: 2015-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bosh_common
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2840.0
+        version: 1.2847.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2840.0
+        version: 1.2847.0
 - !ruby/object:Gem::Dependency
   name: bosh-template
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2840.0
+        version: 1.2847.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2840.0
+        version: 1.2847.0
 - !ruby/object:Gem::Dependency
   name: json_pure
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2840.0
+        version: 1.2847.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2840.0
+        version: 1.2847.0
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -194,7 +194,7 @@ dependencies:
         version: 0.5.4
 description: |-
   BOSH CLI
-  e0ed0c
+  139553
 email: support@cloudfoundry.com
 executables:
 - bosh
@@ -294,7 +294,7 @@ files:
 - lib/cli/task_tracking/total_duration.rb
 - lib/cli/validation.rb
 - lib/cli/version.rb
-- lib/cli/versions/local_version_storage.rb
+- lib/cli/versions/local_artifact_storage.rb
 - lib/cli/versions/multi_release_support.rb
 - lib/cli/versions/release_versions_index.rb
 - lib/cli/versions/releases_dir_migrator.rb