bosh_cli 0.16

data/lib/cli/packaging_helper.rb

@@ -0,0 +1,248 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+# This relies on having the following instance variables in a host class:
+# @dev_builds_dir, @final_builds_dir, @blobstore,
+# @name, @version, @tarball_path, @final, @artefact_type
+
+module Bosh::Cli
+  module PackagingHelper
+    include Bosh::Cli::VersionCalc
+
+    attr_accessor :dry_run
+
+    def init_indices
+      @dev_index   = VersionsIndex.new(@dev_builds_dir)
+      @final_index = VersionsIndex.new(@final_builds_dir)
+    end
+
+    def final?
+      @final
+    end
+
+    def dry_run?
+      @dry_run
+    end
+
+    def new_version?
+      @tarball_generated || @promoted || @will_be_promoted
+    end
+
+    def older_version?
+      if @tarball_generated
+        false
+      elsif @used_final_version
+        version_cmp(@version, @final_index.latest_version) < 0
+      else
+        version_cmp(@version, @dev_index.latest_version) < 0
+      end
+    end
+
+    def notes
+      notes = []
+
+      if @will_be_promoted
+        new_final_version = @final_index.latest_version.to_i + 1
+        notes << "new final version #{new_final_version}"
+      elsif new_version?
+        notes << "new version"
+      end
+
+      notes << "older than latest" if older_version?
+      notes
+    end
+
+    def build
+      with_indent("  ") do
+        use_final_version || use_dev_version || generate_tarball
+      end
+      upload_tarball(@tarball_path) if final? && !dry_run?
+      @will_be_promoted = true if final? && dry_run? && @used_dev_version
+    end
+
+    def use_final_version
+      say("Final version:", " ")
+
+      item = @final_index[fingerprint]
+
+      if item.nil?
+        say("NOT FOUND".red)
+        return nil
+      end
+
+      blobstore_id = item["blobstore_id"]
+      version      = item["version"]
+
+      if blobstore_id.nil?
+        say("No blobstore id".red)
+        return nil
+      end
+
+      filename = @final_index.filename(version)
+      need_fetch = true
+
+      if File.exists?(filename)
+        say("FOUND LOCAL".green)
+        if file_checksum(filename) == item["sha1"]
+          @tarball_path = filename
+          need_fetch = false
+        else
+          say("LOCAL CHECKSUM MISMATCH".red)
+          need_fetch = true
+        end
+      end
+
+      if need_fetch
+        say("Downloading `#{name} (#{version})' (#{blobstore_id})".green)
+        payload = @blobstore.get(blobstore_id)
+        if Digest::SHA1.hexdigest(payload) == item["sha1"]
+          @tarball_path = @final_index.add_version(fingerprint, item, payload)
+        else
+          err("`#{name}' (#{version}) is corrupted in blobstore " +
+                  "(id=#{blobstore_id}), " +
+                  "please remove it manually and re-generate the final release")
+        end
+      end
+
+      @version = version
+      @used_final_version = true
+      true
+    rescue Bosh::Blobstore::NotFound => e
+      raise BlobstoreError, "Final version of `#{name}' not found in blobstore"
+    rescue Bosh::Blobstore::BlobstoreError => e
+      raise BlobstoreError, "Blobstore error: #{e}"
+    end
+
+    def use_dev_version
+      say("Dev version:", "   ")
+      item = @dev_index[fingerprint]
+
+      if item.nil?
+        say("NOT FOUND".red)
+        return nil
+      end
+
+      version = item["version"]
+      filename = @dev_index.filename(version)
+
+      if File.exists?(filename)
+        say("FOUND LOCAL".green)
+      else
+        say("TARBALL MISSING".red)
+        return nil
+      end
+
+      if file_checksum(filename) == item["sha1"]
+        @tarball_path = filename
+        @version = version
+        @used_dev_version = true
+      else
+        say("`#{name} (#{version})' tarball corrupted".red)
+        return nil
+      end
+    end
+
+    def generate_tarball
+      if final?
+        err_message = "No matching build found for " +
+            "`#{@name}' #{@artefact_type}.\n" +
+            "Please consider creating a dev release first.\n" +
+            "The fingerprint is `#{fingerprint}'."
+        err(err_message)
+      end
+
+      current_final = @final_index.latest_version.to_i
+      new_minor = minor_version(@dev_index.latest_version(current_final)) + 1
+
+      version = "#{current_final}.#{new_minor}-dev"
+      tmp_file = Tempfile.new(name)
+
+      say("Generating...")
+
+      copy_files
+
+      in_build_dir do
+        tar_out = `tar -chzf #{tmp_file.path} . 2>&1`
+        unless $?.exitstatus == 0
+          raise PackagingError, "Cannot create tarball: #{tar_out}"
+        end
+      end
+
+      payload = tmp_file.read
+
+      item = {
+        "version" => version
+      }
+
+      unless dry_run?
+        @dev_index.add_version(fingerprint, item, payload)
+        @tarball_path = @dev_index.filename(version)
+      end
+
+      @version = version
+      @tarball_generated = true
+      say("Generated version #{version}".green)
+      true
+    end
+
+    def upload_tarball(path)
+      item = @final_index[fingerprint]
+
+      say("Uploading final version #{version}...")
+
+      if !item.nil?
+        version = item["version"]
+        say("This package has already been uploaded")
+        return
+      end
+
+      version = @final_index.latest_version.to_i + 1
+      payload = File.read(path)
+
+      blobstore_id = @blobstore.create(payload)
+
+      item = {
+        "blobstore_id" => blobstore_id,
+        "version" => version
+      }
+
+      say("Uploaded, blobstore id #{blobstore_id}")
+      @final_index.add_version(fingerprint, item, payload)
+      @tarball_path = @final_index.filename(version)
+      @version = version
+      @promoted = true
+      true
+    rescue Bosh::Blobstore::BlobstoreError => e
+      raise BlobstoreError, "Blobstore error: #{e}"
+    end
+
+    def file_checksum(path)
+      Digest::SHA1.file(path).hexdigest
+    end
+
+    def checksum
+      if @tarball_path && File.exists?(@tarball_path)
+        file_checksum(@tarball_path)
+      else
+        raise RuntimeError,
+              "cannot read checksum for not yet generated package/job"
+      end
+    end
+
+    # Git doesn't really track file permissions, it just looks at executable
+    # bit and uses 0755 if it's set or 0644 if not. We have to mimic that
+    # behavior in the fingerprint calculation to avoid the situation where
+    # seemingly clean working copy would trigger new fingerprints for
+    # artifacts with changed permissions. Also we don't want current
+    # fingerprints to change, hence the exact values below.
+    def tracked_permissions(path)
+      if File.directory?(path)
+        "40755"
+      elsif File.executable?(path)
+        "100755"
+      else
+        "100644"
+      end
+    end
+  end
+end
+

data/lib/cli/release.rb

@@ -0,0 +1,176 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Cli
+  # This class encapsulates the details of handling dev and final releases:
+  # also it partitions release metadata between final config (which is
+  # under version control) and user dev config.
+  class Release
+    attr_reader :dir
+
+    def initialize(dir)
+      @dir = dir
+      config_dir = File.join(dir, "config")
+      @final_config_file = File.join(config_dir, "final.yml")
+      @dev_config_file = File.join(config_dir, "dev.yml")
+
+      @private_config_file = File.join(config_dir, "private.yml")
+
+      unless File.directory?(dir)
+        err("Cannot find release directory `#{dir}'")
+      end
+
+      unless File.directory?(config_dir)
+        err("Cannot find release config directory `#{config_dir}'")
+      end
+
+      @final_config = load_config(@final_config_file)
+      @dev_config = load_config(@dev_config_file)
+      @private_config = load_config(@private_config_file)
+
+      migrate_legacy_configs
+    end
+
+    # Devbox-specific attributes, gitignored
+    [:dev_name, :latest_release_filename].each do |attr|
+      define_method(attr) do
+        @dev_config[attr.to_s]
+      end
+
+      define_method("#{attr}=".to_sym) do |value|
+        @dev_config[attr.to_s] = value
+      end
+    end
+
+    # Shared attributes, present in repo
+    [:final_name, :min_cli_version].each do |attr|
+      define_method(attr) do
+        @final_config[attr.to_s]
+      end
+
+      define_method("#{attr}=".to_sym) do |value|
+        @final_config[attr.to_s] = value
+      end
+    end
+
+    # Check if the blobstore secret is provided in the private config file
+    #
+    # @return [Boolean]
+    def has_blobstore_secret?
+      @private_config.has_key?("blobstore_secret")
+    end
+
+    # Picks blobstore client to use with current release.
+    #
+    # @return [Bosh::Blobstore::Client] blobstore client
+    def blobstore
+      return @blobstore if @blobstore
+      blobstore_config = Marshal.load(Marshal.dump(@final_config["blobstore"]))
+
+      if blobstore_config.nil?
+        err("Missing blobstore configuration, please update your release")
+      end
+
+      provider = blobstore_config["provider"]
+      options  = blobstore_config["options"] || {}
+
+      if has_blobstore_secret?
+        options["secret"] = @private_config["blobstore_secret"]
+      end
+
+      @blobstore = Bosh::Blobstore::Client.create(provider,
+                                                  symbolize_keys(options))
+
+    rescue Bosh::Blobstore::BlobstoreError => e
+      err("Cannot initialize blobstore: #{e}")
+    end
+
+    def save_config
+      # TODO: introduce write_yaml helper
+      File.open(@dev_config_file, "w") do |f|
+        YAML.dump(@dev_config, f)
+      end
+
+      File.open(@final_config_file, "w") do |f|
+        YAML.dump(@final_config, f)
+      end
+    end
+
+    private
+
+    # Upgrade path for legacy clients that kept release metadata
+    # in config/dev.yml and config/final.yml
+    #
+    def migrate_legacy_configs
+      # We're using blobstore_options as old config marker.
+      # Unfortunately old CLI won't tell you to upgrade because it checks
+      # for valid blobstore options first, so instead of removing
+      # blobstore_options we mark it as deprecated, so new CLI proceeds
+      # to migrate while the old one tells you to upgrade.
+      if @dev_config.has_key?("blobstore_options") &&
+          @dev_config["blobstore_options"] != "deprecated"
+        say("Found legacy dev config file `#{@dev_config_file}'".yellow)
+
+        new_dev_config = {
+          "dev_name" => @dev_config["name"],
+          "latest_release_filename" =>
+              @dev_config["latest_release_filename"],
+
+          # Following two options are only needed for older clients
+          # to fail gracefully and never actually read by a new client
+          "blobstore_options" => "deprecated",
+          "min_cli_version" => "0.12"
+        }
+
+        @dev_config = new_dev_config
+
+        File.open(@dev_config_file, "w") do |f|
+          YAML.dump(@dev_config, f)
+        end
+        say("Migrated dev config file format".green)
+      end
+
+      if @final_config.has_key?("blobstore_options") &&
+          @final_config["blobstore_options"] != "deprecated"
+        say("Found legacy config file `#{@final_config_file}'".yellow)
+
+        unless @final_config["blobstore_options"]["provider"] == "atmos" &&
+            @final_config["blobstore_options"].has_key?("atmos_options")
+          err("Please update your release to the version " +
+                  "that uses Atmos blobstore")
+        end
+
+        new_final_config = {
+          "final_name" => @final_config["name"],
+          "min_cli_version" => @final_config["min_cli_version"],
+          "blobstore" => {
+            "provider" => "atmos",
+            "options" => @final_config["blobstore_options"]["atmos_options"]
+          },
+          "blobstore_options" => "deprecated"
+        }
+
+        @final_config = new_final_config
+
+        File.open(@final_config_file, "w") { |f| YAML.dump(@final_config, f) }
+        say("Migrated final config file format".green)
+      end
+    end
+
+    def symbolize_keys(hash)
+      hash.inject({}) do |h, (key, value)|
+        h[key.to_sym] = value
+        h
+      end
+    end
+
+    def load_config(file)
+      if File.exists?(file)
+        load_yaml_file(file)
+      else
+        {}
+      end
+    end
+
+  end
+
+end

data/lib/cli/release_builder.rb

@@ -0,0 +1,215 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Cli
+  class ReleaseBuilder
+    include Bosh::Cli::DependencyHelper
+
+    DEFAULT_RELEASE_NAME = "bosh_release"
+
+    attr_reader :release, :packages, :jobs, :changed_jobs
+
+    def initialize(release, packages, jobs, options = { })
+      @release  = release
+      @final    = options.has_key?(:final) ? !!options[:final] : false
+      @packages = packages
+      @jobs     = jobs
+
+      @index = VersionsIndex.new(releases_dir, release_name)
+      create_release_build_dir
+    end
+
+    def release_name
+      name = @final ? @release.final_name : @release.dev_name
+      name.blank? ? DEFAULT_RELEASE_NAME : name
+    end
+
+    def version
+      @version ||= assign_version
+    end
+
+    def final?
+      @final
+    end
+
+    def affected_jobs
+      result = Set.new(@jobs.select { |job| job.new_version? })
+      return result if @packages.empty?
+
+      new_package_names = @packages.inject([]) do |list, package|
+        list << package.name if package.new_version?
+        list
+      end
+
+      @jobs.each do |job|
+        result << job if (new_package_names & job.packages).size > 0
+      end
+
+      result.to_a
+    end
+
+    def build(options = {})
+      options = { :generate_tarball => true }.merge(options)
+
+      header("Generating manifest...")
+      generate_manifest
+      if options[:generate_tarball]
+        header("Generating tarball...")
+        generate_tarball
+      end
+      @build_complete = true
+    end
+
+    def copy_packages
+      packages.each do |package|
+        say("%-40s %s" % [package.name.green,
+                           pretty_size(package.tarball_path)])
+        FileUtils.cp(package.tarball_path,
+                     File.join(build_dir, "packages", "#{package.name}.tgz"),
+                     :preserve => true)
+      end
+      @packages_copied = true
+    end
+
+    def copy_jobs
+      jobs.each do |job|
+        say("%-40s %s" % [job.name.green, pretty_size(job.tarball_path)])
+        FileUtils.cp(job.tarball_path,
+                     File.join(build_dir, "jobs", "#{job.name}.tgz"),
+                     :preserve => true)
+      end
+      @jobs_copied = true
+    end
+
+    def generate_manifest
+      manifest = {}
+      manifest["packages"] = []
+
+      manifest["packages"] = packages.map do |package|
+        {
+          "name"         => package.name,
+          "version"      => package.version,
+          "sha1"         => package.checksum,
+          "dependencies" => package.dependencies
+        }
+      end
+
+      manifest["jobs"] = jobs.map do |job|
+        {
+          "name"    => job.name,
+          "version" => job.version,
+          "sha1"    => job.checksum,
+        }
+      end
+
+      manifest["name"] = release_name
+
+      unless manifest["name"].bosh_valid_id?
+        raise InvalidRelease, "Release name `#{manifest["name"]}' " +
+                              "is not a valid BOSH identifier"
+      end
+
+      fingerprint = make_fingerprint(manifest)
+
+      if @index[fingerprint]
+        old_version = @index[fingerprint]["version"]
+        say("This version is no different from version #{old_version}")
+        @version = old_version
+      else
+        @version = assign_version
+        @index.add_version(fingerprint, { "version" => @version })
+      end
+
+      manifest["version"] = @version
+      manifest_yaml = YAML.dump(manifest)
+
+      say("Writing manifest...")
+      File.open(File.join(build_dir, "release.MF"), "w") do |f|
+        f.write(manifest_yaml)
+      end
+
+      File.open(manifest_path, "w") do |f|
+        f.write(manifest_yaml)
+      end
+
+      @manifest_generated = true
+    end
+
+    def generate_tarball
+      generate_manifest unless @manifest_generated
+      return if @index.version_exists?(@version)
+
+      unless @jobs_copied
+        header("Copying jobs...")
+        copy_jobs
+        nl
+      end
+      unless @packages_copied
+        header("Copying packages...")
+        copy_packages
+        nl
+      end
+
+      FileUtils.mkdir_p(File.dirname(tarball_path))
+
+      in_build_dir do
+        `tar -czf #{tarball_path} . 2>&1`
+        unless $?.exitstatus == 0
+          raise InvalidRelease, "Cannot create release tarball"
+        end
+        say("Generated #{tarball_path}")
+      end
+    end
+
+    def releases_dir
+       File.join(@release.dir, final? ? "releases" : "dev_releases")
+    end
+
+    def tarball_path
+      File.join(releases_dir, "#{release_name}-#{version}.tgz")
+    end
+
+    def manifest_path
+      File.join(releases_dir, "#{release_name}-#{version}.yml")
+    end
+
+    def make_fingerprint(item)
+      case item
+      when Array
+        source = item.map { |e| make_fingerprint(e) }.sort.join("")
+      when Hash
+        source = item.keys.sort.map{ |k| make_fingerprint(item[k]) }.join("")
+      else
+        source = item.to_s
+      end
+      Digest::SHA1.hexdigest(source)
+    end
+
+    private
+
+    def version=(version)
+      @version = version
+    end
+
+    def assign_version
+      current_version = @index.latest_version.to_i
+      current_version + 1
+    end
+
+    def build_dir
+      @build_dir ||= Dir.mktmpdir
+    end
+
+    def create_release_build_dir
+      in_build_dir do
+        FileUtils.mkdir("packages")
+        FileUtils.mkdir("jobs")
+      end
+    end
+
+    def in_build_dir(&block)
+      Dir.chdir(build_dir) { yield }
+    end
+
+  end
+
+end