bosh_cli 1.2831.0 → 1.2839.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9f61a1c4dc97a4d1f4e909b010779a432370856
-  data.tar.gz: c430c18acb4e3d9803e0ea1d9ce521ed60888aee
+  metadata.gz: 3f11dc3119b4347b9e61c770c43d31b09ec3e1aa
+  data.tar.gz: 5d829eee1ce9742ea980476c1b7cb769887ab59e
 SHA512:
-  metadata.gz: 15c4b0d2a9b40972f22184ccce08917a9a0b2196b306e81cb0f53761796f7c35fadcedd4563605091f82b6dba1fa1cd4f6172e17e63b6a77475df61b208a1e85
-  data.tar.gz: 34889ef7e9ab28a490e7f6d51001694dc36e0780781cafc320ac56aa1bebb23deb1d904d01dae88bfe28772f6c4b466194975be062fda1f994333fe22476e5be
+  metadata.gz: bb7ac41c08a80eee35cec3672d9969c91de24cea903e0ae1c765bfbd1c4c86940e7b4ce3953c3cf5717c610b2988c8e0609ec7621520fc882fc67c449914ab29
+  data.tar.gz: a0ba29122f2ff944793adb8f99473b0e29ffd39ac49db87ad368446bf38e9bdeecaebe3c33d3ccedfaa0c92a70894fbaf70ae019eb615865906108adee3f76ec

data/lib/cli.rb

@@ -51,6 +51,7 @@ require 'common/thread_pool'
 require 'cli/config'
 require 'cli/core_ext'
 require 'cli/errors'
+require 'cli/glob_match'
 require 'cli/yaml_helper'
 require 'cli/dependency_helper'
 require 'cli/deployment_manifest'
@@ -75,9 +76,12 @@ require 'cli/versions/releases_dir_migrator'
 require 'cli/versions/version_file_resolver'
 require 'cli/versions/multi_release_support'
 
-require 'cli/packaging_helper'
-require 'cli/package_builder'
-require 'cli/job_builder'
+require 'cli/archive_builder'
+require 'cli/archive_repository_provider'
+require 'cli/archive_repository'
+require 'cli/build_artifact'
+require 'cli/resources/job'
+require 'cli/resources/package'
 require 'cli/changeset_helper'
 require 'cli/deployment_manifest_compiler'
 require 'cli/task_tracking'

data/lib/cli/archive_builder.rb

@@ -0,0 +1,119 @@
+module Bosh::Cli
+  class ArchiveBuilder
+    attr_reader :options
+
+    def initialize(archive_repository_provider, options = {})
+      @archive_repository_provider = archive_repository_provider
+      @options = options
+    end
+
+    def build(resource)
+      @archive_repository = @archive_repository_provider.get(resource)
+      resource.run_script(:prepare)
+
+      artifact = nil
+      with_indent('  ') do
+        artifact = locate_artifact(resource)
+        if artifact.nil?
+          artifact = create_artifact(resource)
+          say("Generated version '#{artifact.fingerprint}'".make_green)
+
+          unless dry_run?
+            artifact = @archive_repository.install(artifact)
+          end
+        end
+
+        if final? && !dry_run?
+          say("Uploading final version '#{artifact.version}'...")
+          artifact, blobstore_id = @archive_repository.upload_to_blobstore(artifact)
+          say("Uploaded, blobstore id '#{blobstore_id}'")
+        end
+      end
+
+      artifact
+    rescue Bosh::Blobstore::BlobstoreError => e
+      raise BlobstoreError, "Blobstore error: #{e}"
+    end
+
+    def dry_run?
+      !!options[:dry_run]
+    end
+
+    def final?
+      !!options[:final]
+    end
+
+    private
+
+    def copy_files(resource)
+      resource.files.each do |src, dest|
+        dest_path = Pathname(staging_dir).join(dest)
+        if File.directory?(src)
+          FileUtils.mkdir_p(dest_path)
+        else
+          FileUtils.mkdir_p(dest_path.parent)
+          FileUtils.cp(src, dest_path, :preserve => true)
+        end
+      end
+    end
+
+    def locate_artifact(resource)
+      artifact = @archive_repository.lookup(resource)
+
+      if artifact.nil?
+        say("No artifact found for #{resource.name}".make_red)
+        return nil
+      end
+
+      if artifact.dev_artifact? && final? && !dry_run?
+        artifact = @archive_repository.copy_from_dev_to_final(artifact)
+      end
+
+      artifact
+    rescue Bosh::Cli::CorruptedArchive => e
+      say "#{"Warning".make_red}: #{e.message}"
+      nil
+    end
+
+    def create_artifact(resource)
+      tarball_path = safe_temp_file(resource.name, '.tgz')
+
+      say('Generating...')
+
+      copy_files(resource)
+      resource.run_script(:pre_packaging, staging_dir)
+
+      in_staging_dir do
+        tar_out = `tar -chzf #{tarball_path} . 2>&1`
+        unless $?.exitstatus == 0
+          raise PackagingError, "Cannot create tarball: #{tar_out}"
+        end
+      end
+
+      fingerprint = BuildArtifact.make_fingerprint(resource)
+
+      sha1 = BuildArtifact.checksum(tarball_path)
+      BuildArtifact.new(resource.name, fingerprint, tarball_path, sha1, resource.dependencies, true, !final?)
+    end
+
+    def file_checksum(path)
+      Digest::SHA1.file(path).hexdigest
+    end
+
+    def staging_dir
+      @staging_dir ||= Dir.mktmpdir
+    end
+
+    def in_staging_dir
+      Dir.chdir(staging_dir) { yield }
+    end
+
+    private
+
+    def safe_temp_file(prefix, suffix, dir = Dir.tmpdir)
+      Dir::Tmpname.create([prefix, suffix], dir) do |tmpname, _, _|
+        File.open(tmpname, File::RDWR|File::CREAT|File::EXCL).close
+      end
+    end
+  end
+end

data/lib/cli/archive_repository.rb

@@ -0,0 +1,108 @@
+module Bosh::Cli
+  class ArchiveRepository
+    def initialize(archive_dir, blobstore, resource)
+      @archive_dir = archive_dir
+      @blobstore = blobstore
+
+      dev_builds_dir = Pathname(@archive_dir).join(".dev_builds", resource.plural_type, resource.name).to_s
+      FileUtils.mkdir_p(dev_builds_dir)
+      @dev_index = Versions::VersionsIndex.new(dev_builds_dir)
+      @dev_storage = Versions::LocalVersionStorage.new(dev_builds_dir)
+
+      final_builds_dir = Pathname(@archive_dir).join(".final_builds", resource.plural_type, resource.name).to_s
+      FileUtils.mkdir_p(final_builds_dir)
+      @final_index = Versions::VersionsIndex.new(final_builds_dir)
+      @final_storage = Versions::LocalVersionStorage.new(final_builds_dir)
+
+      @final_resolver = Versions::VersionFileResolver.new(@final_storage, @blobstore)
+    end
+
+    def lookup(resource)
+      fingerprint = BuildArtifact.make_fingerprint(resource)
+
+      artifact_info = @final_index[fingerprint]
+      if artifact_info && artifact_info['blobstore_id']
+        blobstore_id = artifact_info['blobstore_id']
+        version = artifact_info['version'] || fingerprint
+        sha1 = artifact_info['sha1']
+
+        say('Using final version')
+        tarball_path = @final_resolver.find_file(blobstore_id, sha1, version, "#{resource.singular_type} #{resource.name} (#{version})") # todo: 'package' vs 'job'
+
+        BuildArtifact.new(resource.name, fingerprint, tarball_path, sha1, resource.dependencies, false, false)
+      else
+        artifact_info = @dev_index[fingerprint]
+        if artifact_info
+          version = artifact_info['version'] || fingerprint
+          if @dev_storage.has_file?(version)
+            say('Using dev version')
+
+            tarball_path = @dev_storage.get_file(version)
+            if file_checksum(tarball_path) != artifact_info['sha1']
+              raise CorruptedArchive, "#{resource.singular_type} #{resource.name} (#{version}) archive at #{tarball_path} corrupted"
+            end
+
+            BuildArtifact.new(resource.name, fingerprint, tarball_path, artifact_info['sha1'], resource.dependencies, false, true)
+          end
+        end
+      end
+
+    rescue Bosh::Blobstore::NotFound
+      raise BlobstoreError, "Final version of '#{name}' not found in blobstore"
+    rescue Bosh::Blobstore::BlobstoreError => e
+      raise BlobstoreError, "Blobstore error: #{e}"
+    end
+
+    def upload_to_blobstore(artifact)
+      artifact_info = @final_index[artifact.fingerprint]
+      # todo raise if artifact.dev_artifact?
+      return artifact, artifact_info['blobstore_id'] if artifact_info['blobstore_id']
+
+      blobstore_id = nil
+      File.open(artifact.tarball_path, 'r') do |f|
+        blobstore_id = @blobstore.create(f)
+      end
+
+      @final_index.update_version(artifact.fingerprint, {
+          'version' => artifact.version,
+          'sha1' => artifact.sha1,
+          'blobstore_id' => blobstore_id
+        })
+      artifact = BuildArtifact.new(artifact.name, artifact.fingerprint, artifact.tarball_path, artifact.sha1, artifact.dependencies, artifact.new_version?, false)
+      return artifact, blobstore_id
+    end
+
+    def install(artifact)
+      fingerprint = artifact.fingerprint
+      origin_file = artifact.tarball_path
+      new_tarball_path = place_file_and_update_index(fingerprint, origin_file,
+        artifact.dev_artifact? ? @dev_index : @final_index,
+        artifact.dev_artifact? ? @dev_storage : @final_storage)
+
+      BuildArtifact.new(artifact.name, artifact.fingerprint, new_tarball_path, artifact.sha1, artifact.dependencies, artifact.new_version?, artifact.dev_artifact?)
+    end
+
+    def copy_from_dev_to_final(artifact)
+      final_tarball_path = place_file_and_update_index(artifact.fingerprint, artifact.tarball_path, @final_index, @final_storage)
+      BuildArtifact.new(artifact.name, artifact.fingerprint, final_tarball_path, artifact.sha1, artifact.dependencies, artifact.new_version?, false)
+    end
+
+    private
+
+    def place_file_and_update_index(fingerprint, origin_file, index, storage)
+      # add version (with its validation) before adding sha1
+      index.add_version(fingerprint, {'version' => fingerprint} )
+      tarball_path = storage.put_file(fingerprint, origin_file)
+      sha1 = file_checksum(tarball_path)
+      index.update_version(fingerprint, {'version' => fingerprint, 'sha1' => sha1})
+      tarball_path
+    end
+
+    def file_checksum(path)
+      Digest::SHA1.file(path).hexdigest
+    end
+  end
+
+  class CorruptedArchive < StandardError
+  end
+end

data/lib/cli/archive_repository_provider.rb

@@ -0,0 +1,12 @@
+module Bosh::Cli
+  class ArchiveRepositoryProvider
+    def initialize(archive_dir, blobstore)
+      @archive_dir = archive_dir
+      @blobstore = blobstore
+    end
+
+    def get(resource)
+      ArchiveRepository.new(@archive_dir, @blobstore, resource)
+    end
+  end
+end

data/lib/cli/base_command.rb

@@ -48,7 +48,7 @@ module Bosh::Cli
       def release
         return @release if @release
         check_if_release_dir
-        @release = Bosh::Cli::Release.new(@work_dir, options[:final])
+        @release = Bosh::Cli::Release.new(work_dir, options[:final])
       end
 
       def progress_renderer

data/lib/cli/build_artifact.rb

@@ -0,0 +1,75 @@
+module Bosh::Cli
+  class BuildArtifact
+    attr_reader :name, :fingerprint, :tarball_path, :sha1, :dependencies
+
+    def initialize(name, fingerprint, tarball_path, sha1, dependencies, is_new_version, is_dev_artifact)
+      @name = name
+      @fingerprint = fingerprint
+      @tarball_path = tarball_path
+      @sha1 = sha1
+      @dependencies = dependencies
+      @is_dev_artifact = is_dev_artifact
+      @notes = []
+      @is_new_version = is_new_version
+    end
+
+    def version
+      fingerprint
+    end
+
+    def dev_artifact?
+      @is_dev_artifact
+    end
+
+    def new_version?
+      @is_new_version
+    end
+
+    private
+
+    def self.checksum(tarball_path)
+      if tarball_path && File.exists?(tarball_path)
+        digest_file(tarball_path)
+      else
+        nil
+      end
+    end
+
+    def self.digest_file(filename)
+      File.file?(filename) ? Digest::SHA1.file(filename).hexdigest : ''
+    end
+
+    # Git doesn't really track file permissions, it just looks at executable
+    # bit and uses 0755 if it's set or 0644 if not. We have to mimic that
+    # behavior in the fingerprint calculation to avoid the situation where
+    # seemingly clean working copy would trigger new fingerprints for
+    # artifacts with changed permissions. Also we don't want current
+    # fingerprints to change, hence the exact values below.
+    def self.file_mode(path)
+      if File.directory?(path)
+        '40755'
+      elsif File.executable?(path)
+        '100755'
+      else
+        '100644'
+      end
+    end
+
+    # TODO: be sure we are handling the case in which there was an index, with a pre-defined fingerprint
+    def self.make_fingerprint(resource)
+      scheme = 2
+      contents = "v#{scheme}"
+
+      resource.files.each do |filename, name|
+        contents << resource.format_fingerprint(digest_file(filename), filename, name, file_mode(filename))
+      end
+
+      contents << resource.additional_fingerprints.join(",")
+      Digest::SHA1.hexdigest(contents)
+    end
+
+    def resource
+      raise
+    end
+  end
+end

data/lib/cli/commands/deployment.rb

@@ -154,18 +154,13 @@ module Bosh::Cli::Command
       end
 
       say(" - discovering packages")
-      packages = Bosh::Cli::PackageBuilder.discover(
-        work_dir,
-        :dry_run => true,
-        :final => false
-      )
+      packages = Bosh::Cli::Resources::Package.discover(work_dir)
 
       say(" - discovering jobs")
-      jobs = Bosh::Cli::JobBuilder.discover(
+      jobs = Bosh::Cli::Resources::Job.discover(
         work_dir,
-        :dry_run => true,
-        :final => false,
-        :package_names => packages.map {|package| package.name}
+        # TODO: be sure this is covered in integration
+        packages.map {|package| package['name']}
       )
 
       say(" - validating properties")

data/lib/cli/commands/release/create_release.rb

@@ -85,13 +85,13 @@ module Bosh::Cli::Command
         end
 
         header('Building packages')
-        packages = build_packages(dry_run, final)
+        package_artifacts = build_packages(dry_run, final)
 
         header('Building jobs')
-        jobs = build_jobs(packages.map(&:name), dry_run, final)
+        job_artifacts = build_jobs(package_artifacts.map { |artifact| artifact.name }, dry_run, final)
 
         header('Building release')
-        release_builder = build_release(dry_run, final, jobs, manifest_only, packages, name, version)
+        release_builder = build_release(dry_run, final, job_artifacts, manifest_only, package_artifacts, name, version)
 
         header('Release summary')
         show_summary(release_builder)
@@ -134,22 +134,19 @@ module Bosh::Cli::Command
       end
 
       def build_packages(dry_run, final)
-        packages = Bosh::Cli::PackageBuilder.discover(
-          work_dir,
-          :final => final,
-          :blobstore => release.blobstore,
-          :dry_run => dry_run
-        )
-
-        packages.each do |package|
+        archive_builder = Bosh::Cli::ArchiveBuilder.new(archive_repository_provider,
+          dry_run: dry_run, final: final)
+        packages = Bosh::Cli::Resources::Package.discover(work_dir)
+        artifacts = packages.map do |package|
           say("Building #{package.name.make_green}...")
-          package.build
+          artifact = archive_builder.build(package)
           nl
+          artifact
         end
 
         if packages.size > 0
-          package_index = packages.inject({}) do |index, package|
-            index[package.name] = package.dependencies
+          package_index = artifacts.inject({}) do |index, artifact|
+            index[artifact.name] = artifact.dependencies
             index
           end
           sorted_packages = tsort_packages(package_index)
@@ -161,11 +158,15 @@ module Bosh::Cli::Command
           nl
         end
 
-        packages
+        artifacts
+      end
+
+      def archive_repository_provider
+        @archive_repository_provider ||= Bosh::Cli::ArchiveRepositoryProvider.new(work_dir, release.blobstore)
       end
 
-      def build_release(dry_run, final, jobs, manifest_only, packages, name, version)
-        release_builder = Bosh::Cli::ReleaseBuilder.new(release, packages, jobs, name,
+      def build_release(dry_run, final, job_artifacts, manifest_only, package_artifacts, name, version)
+        release_builder = Bosh::Cli::ReleaseBuilder.new(release, package_artifacts, job_artifacts, name,
           final: final,
           commit_hash: commit_hash,
           version: version,
@@ -183,22 +184,18 @@ module Bosh::Cli::Command
         release_builder
       end
 
-      def build_jobs(built_package_names, dry_run, final)
-        jobs = Bosh::Cli::JobBuilder.discover(
-          work_dir,
-          :final => final,
-          :blobstore => release.blobstore,
-          :dry_run => dry_run,
-          :package_names => built_package_names
-        )
-
-        jobs.each do |job|
+      def build_jobs(packages, dry_run, final)
+        archive_builder = Bosh::Cli::ArchiveBuilder.new(archive_repository_provider,
+          :final => final, :dry_run => dry_run)
+        jobs = Bosh::Cli::Resources::Job.discover(work_dir, packages)
+        artifacts = jobs.map do |job|
           say("Building #{job.name.make_green}...")
-          job.build
+          artifact = archive_builder.build(job)
           nl
+          artifact
         end
 
-        jobs
+        artifacts
       end
 
       def save_final_release_name
@@ -225,15 +222,15 @@ module Bosh::Cli::Command
       def show_summary(builder)
         packages_table = table do |t|
           t.headings = %w(Name Version Notes)
-          builder.packages.each do |package|
-            t << artifact_summary(package)
+          builder.packages.each do |package_artifact|
+            t << artifact_summary(package_artifact)
           end
         end
 
         jobs_table = table do |t|
           t.headings = %w(Name Version Notes)
-          builder.jobs.each do |job|
-            t << artifact_summary(job)
+          builder.jobs.each do |job_artifact|
+            t << artifact_summary(job_artifact)
           end
         end
 
@@ -260,12 +257,12 @@ module Bosh::Cli::Command
         end
       end
 
-      def artifact_summary(artefact)
-        result = []
-        result << artefact.name
-        result << artefact.version
-        result << artefact.notes.join(', ')
-        result
+      def artifact_summary(artifact)
+        [
+          artifact.name,
+          artifact.version,
+          artifact.new_version? ? 'new version' : '',
+        ]
       end
 
       def commit_hash