bosh_cli 1.2831.0 → 1.2839.0

data/lib/cli/package_builder.rb

@@ -1,316 +0,0 @@
-module Bosh::Cli
-  class PackageBuilder
-    include PackagingHelper
-
-    class GlobMatch
-      # Helper class encapsulating the data we know about the glob. We need
-      # both directory and file path, as we match the same path in several
-      # directories (src, src_alt, blobs)
-      attr_reader :dir
-      attr_reader :path
-
-      def initialize(dir, path)
-        @dir = dir
-        @path = path
-      end
-
-      def full_path
-        File.join(dir, path)
-      end
-
-      def <=>(other)
-        @path <=> other.path
-      end
-
-      # GlobMatch will be used as Hash key (as implied by using Set),
-      # hence we need to define both eql? and hash
-      def eql?(other)
-        @path == other.path
-      end
-
-      def hash
-        @path.hash
-      end
-    end
-
-    attr_reader :name, :globs, :version, :dependencies, :tarball_path
-    # We have two ways of getting/storing a package:
-    # development versions of packages, kept in release directory
-    # final versions of packages, kept in blobstore
-    # development packages and their metadata should always be gitignored
-    # final build tarballs should be ignored as well
-    # final builds metadata should be checked in
-
-    # @param [String] directory Release directory
-    # @param [Hash] options Package build options
-    def self.discover(directory, options = {})
-      builders = []
-
-      Dir[File.join(directory, "packages", "*")].each do |package_dir|
-        next unless File.directory?(package_dir)
-        package_dirname = File.basename(package_dir)
-        package_spec = load_yaml_file(File.join(package_dir, "spec"))
-
-        if package_spec["name"] != package_dirname
-          raise InvalidPackage,
-                "Found `#{package_spec["name"]}' package in " +
-                  "`#{package_dirname}' directory, please fix it"
-        end
-
-        is_final = options[:final]
-        blobstore = options[:blobstore]
-        dry_run = options[:dry_run]
-
-        builder = new(package_spec, directory, is_final, blobstore)
-        builder.dry_run = true if dry_run
-
-        builders << builder
-      end
-
-      builders
-    end
-
-    def initialize(spec, release_dir, final, blobstore,
-                   sources_dir = nil, blobs_dir = nil, alt_src_dir = nil)
-      spec = load_yaml_file(spec) if spec.is_a?(String) && File.file?(spec)
-
-      @name = spec["name"]
-      @globs = spec["files"]
-      @excluded_globs = spec["excluded_files"] || []
-      @dependencies = Array(spec["dependencies"])
-
-      @release_dir = release_dir
-      @sources_dir = sources_dir || File.join(@release_dir, "src")
-      @alt_sources_dir = alt_src_dir || File.join(@release_dir, "src_alt")
-      @blobs_dir = blobs_dir || File.join(@release_dir, "blobs")
-
-      @final = final
-      @blobstore = blobstore
-      @artefact_type = "package"
-
-      @metadata_files = %w(packaging pre_packaging)
-
-      if @final && File.exists?(@alt_sources_dir)
-        err("Please remove `#{File.basename(@alt_sources_dir)}' first")
-      end
-
-      if @name.blank?
-        raise InvalidPackage, "Package name is missing"
-      end
-
-      unless @name.bosh_valid_id?
-        raise InvalidPackage, "Package name should be a valid BOSH identifier"
-      end
-
-      unless @globs.is_a?(Array) && @globs.size > 0
-        raise InvalidPackage, "Package '#{@name}' doesn't include any files"
-      end
-
-      @dev_builds_dir = File.join(@release_dir, ".dev_builds",
-                                  "packages", @name)
-      @final_builds_dir = File.join(@release_dir, ".final_builds",
-                                    "packages", @name)
-
-      FileUtils.mkdir_p(package_dir)
-      FileUtils.mkdir_p(@dev_builds_dir)
-      FileUtils.mkdir_p(@final_builds_dir)
-
-      init_indices
-    end
-
-    def reload # Mostly for tests
-      @fingerprint = nil
-      @resolved_globs = nil
-      init_indices
-      self
-    end
-
-    def fingerprint
-      @fingerprint ||= make_fingerprint
-    end
-
-    def glob_matches
-      @resolved_globs ||= resolve_globs
-    end
-
-    def build_dir
-      @build_dir ||= Dir.mktmpdir
-    end
-
-    def package_dir
-      File.join(@release_dir, "packages", name)
-    end
-
-    def copy_files
-      copied = 0
-
-      glob_matches.each do |match|
-        destination = File.join(build_dir, match.path)
-
-        if File.directory?(match.full_path)
-          FileUtils.mkdir_p(destination)
-        else
-          FileUtils.mkdir_p(File.dirname(destination))
-          FileUtils.cp(match.full_path, destination, :preserve => true)
-          copied += 1
-        end
-      end
-
-      in_package_dir do
-        @metadata_files.each do |filename|
-          destination = File.join(build_dir, filename)
-          next unless File.exists?(filename)
-          if File.exists?(destination)
-            raise InvalidPackage, "Package '#{name}' has '#{filename}' file " +
-              "which conflicts with BOSH packaging"
-          end
-          FileUtils.cp(filename, destination, :preserve => true)
-          copied += 1
-        end
-      end
-
-      pre_package
-      copied
-    end
-
-    def pre_package
-      pre_packaging_script = File.join(package_dir, "pre_packaging")
-
-      if File.exists?(pre_packaging_script)
-
-        say("Pre-packaging...")
-        FileUtils.cp(pre_packaging_script, build_dir, :preserve => true)
-
-        old_env = ENV
-
-        begin
-          ENV.delete_if { |key, _| key[0, 7] == "BUNDLE_" }
-          if ENV["RUBYOPT"]
-            ENV["RUBYOPT"] = ENV["RUBYOPT"].sub("-rbundler/setup", "")
-          end
-          ENV["BUILD_DIR"] = build_dir
-          ENV["RELEASE_DIR"] = @release_dir
-          in_build_dir do
-            pre_packaging_out = `bash -x pre_packaging 2>&1`
-            unless $?.exitstatus == 0
-              pre_packaging_out.split("\n").each do |line|
-                say("> #{line}")
-              end
-              raise InvalidPackage, "`#{name}' pre-packaging failed"
-            end
-          end
-
-        ensure
-          ENV.delete("BUILD_DIR")
-          old_env.each { |k, v| ENV[k] = old_env[k] }
-        end
-
-        FileUtils.rm(File.join(build_dir, "pre_packaging"))
-      end
-    end
-
-    private
-
-    def make_fingerprint
-      versioning_scheme = 2
-      contents = "v#{versioning_scheme}"
-
-      signatures = glob_matches.map do |match|
-        file_digest = nil
-
-        unless File.directory?(match.full_path)
-          file_digest = Digest::SHA1.file(match.full_path).hexdigest
-        end
-
-        "%s%s%s" % [match.path, file_digest,
-                    tracked_permissions(match.full_path)]
-      end
-      contents << signatures.join("")
-
-      in_package_dir do
-        @metadata_files.each do |file|
-          if File.file?(file)
-            file_digest = Digest::SHA1.file(file).hexdigest
-            contents << "%s%s" % [file, file_digest]
-          end
-        end
-      end
-
-      contents << @dependencies.sort.join(",")
-
-      Digest::SHA1.hexdigest(contents)
-    end
-
-    # @return Array<GlobMatch>
-    def resolve_globs
-      all_matches = Set.new
-
-      @globs.each do |glob|
-        matches = Set.new
-
-        src_matches = resolve_glob_in_dir(glob, @sources_dir)
-        src_alt_matches = []
-        if File.directory?(@alt_sources_dir)
-          src_alt_matches = resolve_glob_in_dir(glob, @alt_sources_dir)
-        end
-
-        # Glob like core/dea/**/* might not yield anything in alt source even
-        # when `src_alt/core' exists. That's error prone, so we don't lookup
-        # in `src' if `src_alt' contains any part of the glob hierarchy.
-        top_dir = glob.split(File::SEPARATOR)[0]
-        top_dir_in_src_alt_exists = top_dir && File.exists?(File.join(@alt_sources_dir, top_dir))
-
-        if top_dir_in_src_alt_exists && src_alt_matches.empty? && src_matches.any?
-          raise InvalidPackage, "Package `#{name}' has a glob that " +
-            "doesn't match in `#{File.basename(@alt_sources_dir)}' " +
-            "but matches in `#{File.basename(@sources_dir)}'. " +
-            "However `#{File.basename(@alt_sources_dir)}/#{top_dir}' " +
-            "exists, so this might be an error."
-        end
-
-        # First add src_alt matches since src_alt takes priority over src matches
-        matches += src_alt_matches.map { |path| GlobMatch.new(@alt_sources_dir, path) }
-
-        # Only add if top-level-dir does not exist in src_alt. No partial matches.
-        if !top_dir_in_src_alt_exists
-          matches += src_matches.map { |path| GlobMatch.new(@sources_dir, path) }
-        end
-
-        # Blobs directory is a little bit different: whatever matches a blob
-        # will complement already found matches, unless this particular path
-        # has already been matched.
-        if File.directory?(File.join(@blobs_dir))
-          resolve_glob_in_dir(glob, @blobs_dir).each { |path| matches << GlobMatch.new(@blobs_dir, path) }
-        end
-
-        if matches.empty?
-          raise InvalidPackage, "Package `#{name}' has a glob that resolves to an empty file list: #{glob}"
-        end
-
-        all_matches += matches
-      end
-
-      all_matches.reject! do |match|
-        @excluded_globs.detect { |excluded_glob| File.fnmatch(excluded_glob, match.path) }
-      end
-      all_matches.sort
-    end
-
-    def resolve_glob_in_dir(glob, dir)
-      Dir.chdir(dir) do
-        Dir.glob(glob, File::FNM_DOTMATCH).reject do |fn|
-          %w(. ..).include?(File.basename(fn))
-        end
-      end
-    end
-
-    def in_build_dir(&block)
-      Dir.chdir(build_dir) { yield }
-    end
-
-    def in_package_dir(&block)
-      Dir.chdir(package_dir) { yield }
-    end
-
-  end
-end

data/lib/cli/packaging_helper.rb

@@ -1,217 +0,0 @@
-# This relies on having the following instance variables in a host class:
-# @dev_builds_dir, @final_builds_dir, @blobstore,
-# @name, @version, @tarball_path, @final, @artefact_type
-
-module Bosh::Cli
-  module PackagingHelper
-    attr_accessor :dry_run
-
-    def init_indices
-      @dev_index   = Versions::VersionsIndex.new(@dev_builds_dir)
-      @dev_storage = Versions::LocalVersionStorage.new(@dev_builds_dir)
-
-      @final_index = Versions::VersionsIndex.new(@final_builds_dir)
-      @final_storage = Versions::LocalVersionStorage.new(@final_builds_dir)
-
-      @final_resolver = Versions::VersionFileResolver.new(@final_storage, @blobstore)
-    end
-
-    def final?
-      @final
-    end
-
-    def dry_run?
-      @dry_run
-    end
-
-    def new_version?
-      @tarball_generated || @promoted || @will_be_promoted
-    end
-
-    def notes
-      notes = []
-
-      if @will_be_promoted
-        new_final_version = @version
-        notes << "new final version #{new_final_version}"
-      elsif new_version?
-        notes << 'new version'
-      end
-
-      notes
-    end
-
-    def build
-      with_indent('  ') do
-        use_final_version || use_dev_version || generate_tarball
-      end
-      upload_tarball(@tarball_path) if final? && !dry_run?
-      @will_be_promoted = true if final? && dry_run? && @used_dev_version
-    end
-
-    def use_final_version
-      say('Final version:', ' ')
-
-      item = @final_index[fingerprint]
-
-      if item.nil?
-        say('NOT FOUND'.make_red)
-        return nil
-      end
-
-      blobstore_id = item['blobstore_id']
-      version      = item['version'] || fingerprint
-      sha1         = item['sha1']
-
-      if blobstore_id.nil?
-        say('No blobstore id'.make_red)
-        return nil
-      end
-
-      desc = "#{name} (#{version})"
-
-      @tarball_path = @final_resolver.find_file(blobstore_id, sha1, version, "package #{desc}")
-
-      @version = version
-      @used_final_version = true
-      true
-    rescue Bosh::Blobstore::NotFound
-      raise BlobstoreError, "Final version of `#{name}' not found in blobstore"
-    rescue Bosh::Blobstore::BlobstoreError => e
-      raise BlobstoreError, "Blobstore error: #{e}"
-    end
-
-    def use_dev_version
-      say('Dev version:', '   ')
-      item = @dev_index[fingerprint]
-
-      if item.nil?
-        say('NOT FOUND'.make_red)
-        return nil
-      end
-
-      version = @dev_index['version'] || fingerprint
-
-      if !@dev_storage.has_file?(version)
-        say('TARBALL MISSING'.make_red)
-        return nil
-      end
-
-      say('FOUND LOCAL'.make_green)
-      @tarball_path = @dev_storage.get_file(version)
-
-      if file_checksum(@tarball_path) != item['sha1']
-        say("`#{name} (#{version})' tarball corrupted".make_red)
-        return nil
-      end
-
-      if final? && !dry_run?
-        # copy from dev index/storage to final index/storage
-        @final_index.add_version(fingerprint, item)
-        @tarball_path = @final_storage.put_file(version, @tarball_path)
-        item['sha1'] = Digest::SHA1.file(@tarball_path).hexdigest
-        @final_index.update_version(fingerprint, item)
-      end
-
-      @version = version
-      @used_dev_version = true
-    end
-
-    def generate_tarball
-      version = fingerprint
-      tmp_file = Tempfile.new(name)
-
-      say('Generating...')
-
-      copy_files
-
-      in_build_dir do
-        tar_out = `tar -chzf #{tmp_file.path} . 2>&1`
-        unless $?.exitstatus == 0
-          raise PackagingError, "Cannot create tarball: #{tar_out}"
-        end
-      end
-
-      item = {
-        'version' => version
-      }
-
-      if final?
-        # add version (with its validation) before adding sha1
-        @final_index.add_version(fingerprint, item)
-        @tarball_path = @final_storage.put_file(fingerprint, tmp_file.path)
-        item['sha1'] = file_checksum(@tarball_path)
-        @final_index.update_version(fingerprint, item)
-      elsif dry_run?
-      else
-        # add version (with its validation) before adding sha1
-        @dev_index.add_version(fingerprint, item)
-        @tarball_path = @dev_storage.put_file(fingerprint, tmp_file.path)
-        item['sha1'] = file_checksum(@tarball_path)
-        @dev_index.update_version(fingerprint, item)
-      end
-
-      @version = version
-      @tarball_generated = true
-      say("Generated version #{version}".make_green)
-      true
-    end
-
-    def upload_tarball(path)
-      item = @final_index[fingerprint]
-
-      unless item
-        say("Failed to find entry `#{fingerprint}' in index, check local storage")
-        return
-      end
-
-      if item['blobstore_id']
-        return
-      end
-
-      say("Uploading final version `#{version}'...")
-
-      blobstore_id = nil
-      File.open(path, 'r') do |f|
-        blobstore_id = @blobstore.create(f)
-      end
-
-      say("Uploaded, blobstore id `#{blobstore_id}'")
-      item['blobstore_id'] = blobstore_id
-      @final_index.update_version(fingerprint, item)
-      @promoted = true
-      true
-    rescue Bosh::Blobstore::BlobstoreError => e
-      raise BlobstoreError, "Blobstore error: #{e}"
-    end
-
-    def file_checksum(path)
-      Digest::SHA1.file(path).hexdigest
-    end
-
-    def checksum
-      if @tarball_path && File.exists?(@tarball_path)
-        file_checksum(@tarball_path)
-      else
-        raise RuntimeError,
-          'cannot read checksum for not yet generated package/job'
-      end
-    end
-
-    # Git doesn't really track file permissions, it just looks at executable
-    # bit and uses 0755 if it's set or 0644 if not. We have to mimic that
-    # behavior in the fingerprint calculation to avoid the situation where
-    # seemingly clean working copy would trigger new fingerprints for
-    # artifacts with changed permissions. Also we don't want current
-    # fingerprints to change, hence the exact values below.
-    def tracked_permissions(path)
-      if File.directory?(path)
-        '40755'
-      elsif File.executable?(path)
-        '100755'
-      else
-        '100644'
-      end
-    end
-  end
-end