bosh-director 1.5.0.pre.1113

data/lib/bosh/director/jobs/snapshot_deployment.rb

@@ -0,0 +1,61 @@
+module Bosh::Director
+  module Jobs
+    class SnapshotDeployment < BaseJob
+      @queue = :normal
+
+      attr_reader :deployment
+
+      def self.job_type
+        :snapshot_deployment
+      end
+
+      def initialize(deployment_name, options = {})
+        @deployment = deployment_manager.find_by_name(deployment_name)
+        @options = options
+        @errors = 0
+      end
+
+      def deployment_manager
+        @deployment_manager ||= Bosh::Director::Api::DeploymentManager.new
+      end
+
+      def perform
+        logger.info("taking snapshot of: #{deployment.name}")
+        deployment.job_instances.each do |instance|
+          snapshot(instance)
+        end
+
+        msg = "snapshots of deployment '#{deployment.name}' created"
+        msg += ", with #{@errors} failure(s)" unless @errors == 0
+        msg
+      end
+
+      def snapshot(instance)
+        logger.info("taking snapshot of: #{instance.job}/#{instance.index} (#{instance.vm.cid})")
+        Bosh::Director::Api::SnapshotManager.take_snapshot(instance, @options)
+      rescue Bosh::Clouds::CloudError => e
+        @errors += 1
+        logger.error("failed to take snapshot of: #{instance.job}/#{instance.index} (#{instance.vm.cid}) - #{e.inspect}")
+        send_alert(instance, e.inspect)
+      end
+
+      ERROR = 3
+
+      def send_alert(instance, message)
+        nats = Bosh::Director::Config.nats
+        payload = Yajl::Encoder.encode(
+            {
+                "id"         => 'director',
+                "severity"   => ERROR,
+                "title"      => "director - snapshot failure",
+                "summary"    => "failed to snapshot #{instance.job}/#{instance.index}: #{message}",
+                "created_at" => Time.now.to_i
+            }
+        )
+
+        nats.publish('hm.director.alert', payload)
+      end
+    end
+  end
+end
+

data/lib/bosh/director/jobs/snapshot_deployments.rb

@@ -0,0 +1,23 @@
+module Bosh::Director
+  module Jobs
+    class SnapshotDeployments < BaseJob
+      @queue = :normal
+
+      def self.job_type
+        :snapshot_deployments
+      end
+
+      def initialize(options={})
+        @snapshot_manager = options.fetch(:snapshot_manager) { Bosh::Director::Api::SnapshotManager.new }
+      end
+
+      def perform
+        tasks = Models::Deployment.all.map do |deployment|
+          @snapshot_manager.create_deployment_snapshot_task('scheduler', deployment)
+        end
+
+        "Enqueued snapshot tasks [#{tasks.map(&:id).join(', ')}]"
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/snapshot_self.rb

@@ -0,0 +1,43 @@
+module Bosh::Director
+  module Jobs
+    class SnapshotSelf < BaseJob
+      @queue = :normal
+
+      def self.job_type
+        :snapshot_self
+      end
+
+      def initialize(options={})
+        @cloud = options.fetch(:cloud) { Config.cloud }
+        @director_uuid = options.fetch(:director_uuid) { Config.uuid }
+        @director_name = options.fetch(:director_name) { Config.name }
+        @enable_snapshots = options.fetch(:enable_snapshots) { Config.enable_snapshots }
+      end
+
+      def perform
+        unless @enable_snapshots
+          logger.info('Snapshots are disabled; skipping')
+          return
+        end
+
+        vm_id = @cloud.current_vm_id
+        disks = @cloud.get_disks(vm_id)
+        metadata = {
+            deployment: 'self',
+            job: 'director',
+            index: 0,
+            director_name: @director_name,
+            director_uuid: @director_uuid,
+            agent_id: 'self',
+            instance_id: vm_id
+        }
+
+        disks.each { |disk| @cloud.snapshot_disk(disk, metadata) }
+
+        "Snapshot director disks [#{disks.join(', ')}]"
+      rescue Bosh::Clouds::NotImplemented
+        logger.info('CPI does not support disk snapshots; skipping')
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/ssh.rb

@@ -0,0 +1,59 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module Jobs
+    class Ssh < BaseJob
+      DEFAULT_SSH_DATA_LIFETIME = 300
+      SSH_TAG = "ssh"
+      @queue = :normal
+
+      def self.job_type
+        :ssh
+      end
+
+      def initialize(deployment_id, options = {})
+        @deployment_id = deployment_id
+        @target = options["target"]
+        @command = options["command"]
+        @params = options["params"]
+        @blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
+        @instance_manager = Api::InstanceManager.new
+      end
+
+      def perform
+        job = @target["job"]
+        indexes = @target["indexes"]
+
+        filter = {
+          :deployment_id => @deployment_id
+        }
+
+        if indexes && indexes.size > 0
+          filter[:index] = indexes
+        end
+
+        if job
+          filter[:job] = job
+        end
+
+        instances = @instance_manager.filter_by(filter)
+
+        ssh_info = instances.map do |instance|
+          agent = @instance_manager.agent_client_for(instance)
+
+          logger.info("ssh #{@command} `#{instance.job}/#{instance.index}'")
+          result = agent.ssh(@command, @params)
+          result["index"] = instance.index
+
+          result
+        end
+
+        result_file.write(Yajl::Encoder.encode(ssh_info))
+        result_file.write("\n")
+
+        # task result
+        nil
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/update_deployment.rb

@@ -0,0 +1,110 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module Jobs
+    class UpdateDeployment < BaseJob
+      include LockHelper
+
+      @queue = :normal
+
+      def self.job_type
+        :update_deployment
+      end
+
+      # @param [String] manifest_file Path to deployment manifest
+      # @param [Hash] options Deployment options
+      def initialize(manifest_file, options = {})
+        logger.info('Reading deployment manifest')
+        @manifest_file = manifest_file
+        @manifest = File.read(@manifest_file)
+        logger.debug("Manifest:\n#{@manifest}")
+
+        logger.info('Creating deployment plan')
+        logger.info("Deployment plan options: #{options.pretty_inspect}")
+
+        plan_options = {
+          'recreate' => !!options['recreate'],
+          'job_states' => options['job_states'] || {},
+          'job_rename' => options['job_rename'] || {}
+        }
+
+        manifest_as_hash = Psych.load(@manifest)
+        @deployment_plan = DeploymentPlan::Planner.new(manifest_as_hash, plan_options)
+        @deployment_plan.parse
+        logger.info('Created deployment plan')
+
+        resource_pools = @deployment_plan.resource_pools
+        @resource_pool_updaters = resource_pools.map do |resource_pool|
+          ResourcePoolUpdater.new(resource_pool)
+        end
+      end
+
+      def prepare
+        @assembler = DeploymentPlan::Assembler.new(@deployment_plan)
+        preparer = DeploymentPlan::Preparer.new(self, @assembler)
+
+        begin_stage('Preparing deployment', 9)
+        preparer.prepare
+
+        logger.info('Compiling and binding packages')
+        PackageCompiler.new(@deployment_plan).compile
+      end
+
+      def update
+        resource_pools = DeploymentPlan::ResourcePools.new(event_log, @resource_pool_updaters)
+
+        updater = DeploymentPlan::Updater.new(self, event_log, resource_pools, @assembler, @deployment_plan)
+        updater.update
+      end
+
+      def update_stemcell_references
+        current_stemcells = Set.new
+        @deployment_plan.resource_pools.each do |resource_pool|
+          current_stemcells << resource_pool.stemcell.model
+        end
+
+        deployment = @deployment_plan.model
+        stemcells = deployment.stemcells
+        stemcells.each do |stemcell|
+          unless current_stemcells.include?(stemcell)
+            stemcell.remove_deployment(deployment)
+          end
+        end
+      end
+
+      def perform
+        with_deployment_lock(@deployment_plan) do
+          logger.info('Preparing deployment')
+          prepare
+          begin
+            deployment = @deployment_plan.model
+            logger.info('Finished preparing deployment')
+            logger.info('Updating deployment')
+            update
+
+            with_release_locks(@deployment_plan) do
+              deployment.db.transaction do
+                deployment.remove_all_release_versions
+                # Now we know that deployment has succeeded and can remove
+                # previous partial deployments release version references
+                # to be able to delete these release versions later.
+                @deployment_plan.releases.each do |release|
+                  deployment.add_release_version(release.model)
+                end
+              end
+            end
+
+            deployment.manifest = @manifest
+            deployment.save
+            logger.info('Finished updating deployment')
+            "/deployments/#{deployment.name}"
+          ensure
+            update_stemcell_references
+          end
+        end
+      ensure
+        FileUtils.rm_rf(@manifest_file)
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/update_release.rb

@@ -0,0 +1,672 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+require 'common/version_number'
+
+module Bosh::Director
+  module Jobs
+    class UpdateRelease < BaseJob
+      include LockHelper
+      include DownloadHelper
+
+      @queue = :normal
+
+      attr_accessor :release_model
+      attr_accessor :tmp_release_dir
+
+      def self.job_type
+        :update_release
+      end
+
+      # @param [String] tmp_release_dir Directory containing release bundle
+      # @param [Hash] options Release update options
+      def initialize(tmp_release_dir, options = {})
+        @tmp_release_dir = tmp_release_dir
+        @release_model = nil
+        @release_version_model = nil
+
+        @rebase = !!options["rebase"]
+        @package_rebase_mapping = {}
+        @job_rebase_mapping = {}
+
+        @manifest = nil
+        @name = nil
+        @version = nil
+
+        @packages_unchanged = false
+        @jobs_unchanged = false
+        
+        @remote_release = options['remote'] || false
+        @remote_release_location = options['location'] if @remote_release 
+      end
+
+      # Extracts release tarball, verifies release manifest and saves release
+      # in DB
+      # @return [void]
+      def perform
+        logger.info("Processing update release")
+        if @rebase
+          logger.info("Release rebase will be performed")
+        end
+
+        single_step_stage("Downloading remote release") { download_remote_release } if @remote_release
+        single_step_stage("Extracting release") { extract_release }
+        single_step_stage("Verifying manifest") { verify_manifest }
+
+        with_release_lock(@name) { process_release }
+
+        if @rebase && @packages_unchanged && @jobs_unchanged
+          raise DirectorError,
+                "Rebase is attempted without any job or package changes"
+        end
+
+        "Created release `#{@name}/#{@version}'"
+      rescue Exception => e
+        remove_release_version_model
+        raise e
+      ensure
+        if @tmp_release_dir && File.exists?(@tmp_release_dir)
+          FileUtils.rm_rf(@tmp_release_dir)
+        end
+      end
+
+      def download_remote_release         
+        release_file = File.join(@tmp_release_dir, Api::ReleaseManager::RELEASE_TGZ)
+        download_remote_file('release', @remote_release_location, release_file)
+      end
+
+      # Extracts release tarball
+      # @return [void]
+      def extract_release
+        release_tgz = File.join(@tmp_release_dir,
+                                Api::ReleaseManager::RELEASE_TGZ)
+
+        result = Bosh::Exec.sh("tar -C #{@tmp_release_dir} -xzf #{release_tgz} 2>&1", :on_error => :return)
+        if result.failed?
+          logger.error("Extracting release archive failed in dir #{@tmp_release_dir}, " +
+                       "tar returned #{result.exit_status}, " +
+                       "output: #{result.output}")
+          raise ReleaseInvalidArchive, "Extracting release archive failed. Check task debug log for details."
+        end
+      ensure
+        if release_tgz && File.exists?(release_tgz)
+          FileUtils.rm(release_tgz)
+        end
+      end
+
+      # @return [void]
+      def verify_manifest
+        manifest_file = File.join(@tmp_release_dir, "release.MF")
+        unless File.file?(manifest_file)
+          raise ReleaseManifestNotFound, "Release manifest not found"
+        end
+
+        @manifest = Psych.load_file(manifest_file)
+        normalize_manifest
+
+        @name = @manifest["name"]
+        @version = @manifest["version"]
+        @commit_hash = @manifest.fetch("commit_hash", nil)
+        @uncommitted_changes = @manifest.fetch("uncommitted_changes", nil)
+      end
+
+      # Processes uploaded release, creates jobs and packages in DB if needed
+      # @return [void]
+      def process_release
+        @release_model = Models::Release.find_or_create(:name => @name)
+        if @rebase
+          @version = next_release_version
+        end
+
+        version_attrs = {
+          :release => @release_model,
+          :version => @version
+        }
+        version_attrs[:uncommitted_changes] = @uncommitted_changes if @uncommitted_changes
+        version_attrs[:commit_hash] = @commit_hash if @commit_hash
+
+        @release_version_model = Models::ReleaseVersion.new(version_attrs)
+        unless @release_version_model.valid?
+          raise ReleaseAlreadyExists,
+                "Release `#{@name}/#{@version}' already exists"
+        end
+
+        @release_version_model.save
+
+        single_step_stage("Resolving package dependencies") do
+          resolve_package_dependencies(@manifest["packages"])
+        end
+
+        @packages = {}
+        process_packages
+        process_jobs
+
+        unless @package_rebase_mapping.empty?
+          event_log.begin_stage(
+            "Rebased packages", @package_rebase_mapping.size)
+          @package_rebase_mapping.each_pair do |name, transition|
+            event_log.track("#{name}: #{transition}") {}
+          end
+        end
+
+        unless @job_rebase_mapping.empty?
+          event_log.begin_stage(
+            "Rebased jobs", @job_rebase_mapping.size)
+          @job_rebase_mapping.each_pair do |name, transition|
+            event_log.track("#{name}: #{transition}") {}
+          end
+        end
+
+        event_log.begin_stage("Release has been created", 1)
+        event_log.track("#{@name}/#{@version}") {}
+      end
+
+      # Normalizes release manifest, so all names, versions, and checksums
+      # are Strings.
+      # @return [void]
+      def normalize_manifest
+        Bosh::Director.hash_string_vals(@manifest, 'name', 'version')
+
+        @manifest['packages'].each { |p| Bosh::Director.hash_string_vals(p, 'name', 'version', 'sha1') }
+        @manifest['jobs'].each { |j| Bosh::Director.hash_string_vals(j, 'name', 'version', 'sha1') }
+      end
+
+      # Resolves package dependencies, makes sure there are no cycles
+      # and all dependencies are present
+      # @return [void]
+      def resolve_package_dependencies(packages)
+        packages_by_name = {}
+        packages.each do |package|
+          packages_by_name[package["name"]] = package
+          package["dependencies"] ||= []
+        end
+        dependency_lookup = lambda do |package_name|
+          packages_by_name[package_name]["dependencies"]
+        end
+        result = CycleHelper.check_for_cycle(packages_by_name.keys,
+                                             :connected_vertices => true,
+                                             &dependency_lookup)
+
+        packages.each do |package|
+          name = package["name"]
+          dependencies = package["dependencies"]
+
+          logger.info("Resolving package dependencies for `#{name}', " +
+                      "found: #{dependencies.pretty_inspect}")
+          package["dependencies"] = result[:connected_vertices][name]
+          logger.info("Resolved package dependencies for `#{name}', " +
+                      "to: #{dependencies.pretty_inspect}")
+        end
+      end
+
+      # Finds all package definitions in the manifest and sorts them into two
+      # buckets: new and existing packages, then creates new packages and points
+      # current release version to the existing packages.
+      # @return [void]
+      def process_packages
+        logger.info("Checking for new packages in release")
+
+        new_packages = []
+        existing_packages = []
+
+        @manifest["packages"].each do |package_meta|
+          filter = {:sha1 => package_meta["sha1"]}
+          if package_meta["fingerprint"]
+            filter[:fingerprint] = package_meta["fingerprint"]
+            filter = filter.sql_or
+          end
+
+          # Checking whether we might have the same bits somewhere
+          packages = Models::Package.where(filter).all
+
+          if packages.empty?
+            new_packages << package_meta
+            next
+          end
+
+          # Rebase is an interesting use case: we don't really care about
+          # preserving the original package/job versions, so if we have a
+          # checksum/fingerprint match, we can just substitute the original
+          # package/job version with an existing one.
+          if @rebase
+            substitutes = packages.select do |package|
+              package.release_id == @release_model.id &&
+              package.name == package_meta["name"] &&
+              package.dependency_set == Set.new(package_meta["dependencies"])
+            end
+
+            substitute = pick_best(substitutes, package_meta["version"])
+
+            if substitute
+              package_meta["version"] = substitute.version
+              package_meta["sha1"] = substitute.sha1
+              existing_packages << [substitute, package_meta]
+              next
+            end
+          end
+
+          # We can reuse an existing package as long as it
+          # belongs to the same release and has the same name and version.
+          existing_package = packages.find do |package|
+            package.release_id == @release_model.id &&
+            package.name == package_meta["name"] &&
+            package.version == package_meta["version"]
+            # NOT checking dependencies here b/c dependency change would
+            # bump the package version anyway.
+          end
+
+          if existing_package
+            existing_packages << [existing_package, package_meta]
+          else
+            # We found a package with the same checksum but different
+            # (release, name, version) tuple, so we need to make a copy
+            # of the package blob and create a new db entry for it
+            package = packages.first
+            package_meta["blobstore_id"] = package.blobstore_id
+            new_packages << package_meta
+          end
+        end
+
+        create_packages(new_packages)
+        use_existing_packages(existing_packages)
+      end
+
+      # Creates packages using provided metadata
+      # @param [Array<Hash>] packages Packages metadata
+      # @return [void]
+      def create_packages(packages)
+        if packages.empty?
+          @packages_unchanged = true
+          return
+        end
+
+        event_log.begin_stage("Creating new packages", packages.size)
+        packages.each do |package_meta|
+          package_desc = "#{package_meta["name"]}/#{package_meta["version"]}"
+          event_log.track(package_desc) do
+            logger.info("Creating new package `#{package_desc}'")
+            package = create_package(package_meta)
+            register_package(package)
+          end
+        end
+      end
+
+      # Points release DB model to existing packages described by given metadata
+      # @param [Array<Array>] packages Existing packages metadata
+      def use_existing_packages(packages)
+        return if packages.empty?
+
+        n_packages = packages.size
+        event_log.begin_stage("Processing #{n_packages} existing " +
+                              "package#{n_packages > 1 ? "s" : ""}", 1)
+
+        event_log.track("Verifying checksums") do
+          packages.each do |package, package_meta|
+            package_desc = "#{package.name}/#{package.version}"
+            logger.info("Package `#{package_desc}' already exists, " +
+                        "verifying checksum")
+
+            expected = package.sha1
+            received = package_meta["sha1"]
+
+            if expected != received
+              raise ReleaseExistingPackageHashMismatch,
+                    "`#{package_desc}' checksum mismatch, " +
+                      "expected #{expected} but received #{received}"
+            end
+            logger.info("Package `#{package_desc}' verified")
+            register_package(package)
+          end
+        end
+      end
+
+      # Creates package in DB according to given metadata
+      # @param [Hash] package_meta Package metadata
+      # @return [void]
+      def create_package(package_meta)
+        name, version = package_meta["name"], package_meta["version"]
+
+        package_attrs = {
+          :release => @release_model,
+          :name => name,
+          :sha1 => package_meta["sha1"],
+          :fingerprint => package_meta["fingerprint"],
+          :version => version
+        }
+
+        if @rebase
+          new_version = next_package_version(name, version)
+          if new_version != version
+            transition = "#{version} -> #{new_version}"
+            logger.info("Package `#{name}' rebased: #{transition}")
+            package_attrs[:version] = new_version
+            version = new_version
+            @package_rebase_mapping[name] = transition
+          end
+        end
+
+        package = Models::Package.new(package_attrs)
+        package.dependency_set = package_meta["dependencies"]
+
+        existing_blob = package_meta["blobstore_id"]
+        desc = "package `#{name}/#{version}'"
+
+        if existing_blob
+          logger.info("Creating #{desc} from existing blob #{existing_blob}")
+          package.blobstore_id = BlobUtil.copy_blob(existing_blob)
+        else
+          logger.info("Creating #{desc} from provided bits")
+
+          package_tgz = File.join(@tmp_release_dir, "packages", "#{name}.tgz")
+          result = Bosh::Exec.sh("tar -tzf #{package_tgz} 2>&1", :on_error => :return)
+          if result.failed?
+            logger.error("Extracting #{desc} archive failed, " +
+                         "tar returned #{result.exit_status}, " +
+                         "output: #{result.output}")
+            raise PackageInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details."
+          end
+
+          package.blobstore_id = BlobUtil.create_blob(package_tgz)
+        end
+
+        package.save
+      end
+
+      # Marks package model as used by release version model
+      # @param [Models::Package] package Package model
+      # @return [void]
+      def register_package(package)
+        @packages[package.name] = package
+        @release_version_model.add_package(package)
+      end
+
+      # Finds job template definitions in release manifest and sorts them into
+      # two buckets: new and existing job templates, then creates new job
+      # template records in the database and points release version to existing
+      # ones.
+      # @return [void]
+      def process_jobs
+        logger.info("Checking for new jobs in release")
+
+        new_jobs = []
+        existing_jobs = []
+
+        @manifest["jobs"].each do |job_meta|
+          filter = {:sha1 => job_meta["sha1"]}
+          if job_meta["fingerprint"]
+            filter[:fingerprint] = job_meta["fingerprint"]
+            filter = filter.sql_or
+          end
+
+          # Checking whether we might have the same bits somewhere
+          jobs = Models::Template.where(filter).all
+
+          if @rebase
+            substitutes = jobs.select do |job|
+              job.release_id == @release_model.id &&
+              job.name == job_meta["name"]
+            end
+
+            substitute = pick_best(substitutes, job_meta["version"])
+
+            if substitute
+              job_meta["version"] = substitute.version
+              job_meta["sha1"] = substitute.sha1
+              existing_jobs << [substitute, job_meta]
+            else
+              new_jobs << job_meta
+            end
+
+            next
+          end
+
+          template = jobs.find do |job|
+            job.release_id == @release_model.id &&
+            job.name == job_meta["name"] &&
+            job.version == job_meta["version"]
+          end
+
+          if template.nil?
+            new_jobs << job_meta
+          else
+            existing_jobs << [template, job_meta]
+          end
+        end
+
+        create_jobs(new_jobs)
+        use_existing_jobs(existing_jobs)
+      end
+
+      def create_jobs(jobs)
+        if jobs.empty?
+          @jobs_unchanged = true
+          return
+        end
+
+        event_log.begin_stage("Creating new jobs", jobs.size)
+        jobs.each do |job_meta|
+          job_desc = "#{job_meta["name"]}/#{job_meta["version"]}"
+          event_log.track(job_desc) do
+            logger.info("Creating new template `#{job_desc}'")
+            template = create_job(job_meta)
+            register_template(template)
+          end
+        end
+      end
+
+      def create_job(job_meta)
+        name, version = job_meta["name"], job_meta["version"]
+
+        template_attrs = {
+          :release => @release_model,
+          :name => name,
+          :sha1 => job_meta["sha1"],
+          :fingerprint => job_meta["fingerprint"],
+          :version => version
+        }
+
+        if @rebase
+          new_version = next_template_version(name, version)
+          if new_version != version
+            transition = "#{version} -> #{new_version}"
+            logger.info("Job `#{name}' rebased: #{transition}")
+            template_attrs[:version] = new_version
+            version = new_version
+            @job_rebase_mapping[name] = transition
+          end
+        end
+
+        logger.info("Creating job template `#{name}/#{version}' " +
+                    "from provided bits")
+        template = Models::Template.new(template_attrs)
+
+        job_tgz = File.join(@tmp_release_dir, "jobs", "#{name}.tgz")
+        job_dir = File.join(@tmp_release_dir, "jobs", "#{name}")
+
+        FileUtils.mkdir_p(job_dir)
+
+        desc = "job `#{name}/#{version}'"
+        result = Bosh::Exec.sh("tar -C #{job_dir} -xzf #{job_tgz} 2>&1", :on_error => :return)
+        if result.failed?
+          logger.error("Extracting #{desc} archive failed in dir #{job_dir}, " +
+                       "tar returned #{result.exit_status}, " +
+                       "output: #{result.output}")
+          raise JobInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details."
+        end
+
+        manifest_file = File.join(job_dir, "job.MF")
+        unless File.file?(manifest_file)
+          raise JobMissingManifest,
+                "Missing job manifest for `#{template.name}'"
+        end
+
+        job_manifest = Psych.load_file(manifest_file)
+
+        if job_manifest["templates"]
+          job_manifest["templates"].each_key do |relative_path|
+            path = File.join(job_dir, "templates", relative_path)
+            unless File.file?(path)
+              raise JobMissingTemplateFile,
+                    "Missing template file `#{relative_path}' " +
+                    "for job `#{template.name}'"
+            end
+          end
+        end
+
+        main_monit_file = File.join(job_dir, "monit")
+        aux_monit_files = Dir.glob(File.join(job_dir, "*.monit"))
+
+        unless File.exists?(main_monit_file) || aux_monit_files.size > 0
+          raise JobMissingMonit, "Job `#{template.name}' is missing monit file"
+        end
+
+        template.blobstore_id = BlobUtil.create_blob(job_tgz)
+
+        package_names = []
+        job_manifest["packages"].each do |package_name|
+          package = @packages[package_name]
+          if package.nil?
+            raise JobMissingPackage,
+                  "Job `#{template.name}' is referencing " +
+                  "a missing package `#{package_name}'"
+          end
+          package_names << package.name
+        end
+        template.package_names = package_names
+
+        if job_manifest["logs"]
+          unless job_manifest["logs"].is_a?(Hash)
+            raise JobInvalidLogSpec,
+                  "Job `#{template.name}' has invalid logs spec format"
+          end
+
+          template.logs = job_manifest["logs"]
+        end
+
+        if job_manifest["properties"]
+          unless job_manifest["properties"].is_a?(Hash)
+            raise JobInvalidPropertySpec,
+                  "Job `#{template.name}' has invalid properties spec format"
+          end
+
+          template.properties = job_manifest["properties"]
+        end
+
+        template.save
+      end
+
+      # @param [Array<Array>] jobs Existing jobs metadata
+      # @return [void]
+      def use_existing_jobs(jobs)
+        return if jobs.empty?
+
+        n_jobs = jobs.size
+        event_log.begin_stage("Processing #{n_jobs} existing " +
+                              "job#{n_jobs > 1 ? "s" : ""}", 1)
+
+        event_log.track("Verifying checksums") do
+          jobs.each do |template, job_meta|
+            job_desc = "#{template.name}/#{template.version}"
+
+            logger.info("Job `#{job_desc}' already exists, " +
+                        "verifying checksum")
+
+            expected = template.sha1
+            received = job_meta["sha1"]
+
+            if expected != received
+              raise ReleaseExistingJobHashMismatch,
+                    "`#{job_desc}' checksum mismatch, " +
+                    "expected #{expected} but received #{received}"
+            end
+
+            logger.info("Job `#{job_desc}' verified")
+            register_template(template)
+          end
+        end
+      end
+
+      # Marks job template model as being used by release version
+      # @param [Models::Template] template Job template model
+      # @return [void]
+      def register_template(template)
+        @release_version_model.add_template(template)
+      end
+
+      private
+
+      # Returns the next release version (to be used for rebased release)
+      # @return [String]
+      def next_release_version
+        attrs = {
+          :release_id => @release_model.id
+        }
+        next_version(Models::ReleaseVersion.filter(attrs).all, @version)
+      end
+
+      # Returns the next package version (to be used for rebased package)
+      # @param [String] name Package name
+      # @param [String] version Package version
+      # @return [String]
+      def next_package_version(name, version)
+        attrs = {
+          :release_id => @release_model.id,
+          :name => name
+        }
+
+        next_version(Models::Package.filter(attrs).all, version)
+      end
+
+      # Returns the next job template version (to be used for rebased template)
+      # @param [String] name Template name
+      # @param [Fixnum] version Template version
+      # @return [String]
+      def next_template_version(name, version)
+        attrs = {
+          :release_id => @release_model.id,
+          :name => name
+        }
+
+        next_version(Models::Template.filter(attrs).all, version)
+      end
+
+      # Takes collection of versioned items and returns the version
+      # that new item should be promoted to if auto-versioning is used
+      # @param [Array<#version>] Collection of items
+      # @param [String] version Current version of item
+      # @return [String] Next version to be used
+      def next_version(collection, version)
+        Bosh::Director::NextRebaseVersion.new(collection).calculate(version)
+      end
+
+      # Removes release version model, along with all packages and templates.
+      # @return [void]
+      def remove_release_version_model
+        return unless @release_version_model && !@release_version_model.new?
+
+        @release_version_model.remove_all_packages
+        @release_version_model.remove_all_templates
+        @release_version_model.destroy
+      end
+
+      # Picks the best matching package/job from collection based on a simple
+      # heuristics: items with same version are preferred, then items
+      # with non-dev versions, then items with most compiled packages,
+      # then everything else.
+      # @param [Array<#name,#version>] collection
+      # @param [String] original_version
+      def pick_best(collection, original_version)
+        collection.sort_by { |item|
+          if item.version == original_version
+            1
+          elsif Bosh::Common::VersionNumber.new(item.version).final?
+            2
+          elsif item.is_a?(Bosh::Director::Models::Package)
+            3000 - [item.compiled_packages.size, 900].min
+          else
+            3000
+          end
+        }.first
+      end
+    end
+  end
+end