bosh-director 1.3022.0 → 1.3048.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 59a2d6077683ae1eaac10b9926fd87139cbf2193
-  data.tar.gz: e417875f5b9b4311ca2ffc19cbea42a4b4e673ed
+  metadata.gz: 99e55b602be5167be9b84464a78041c5062b3a6b
+  data.tar.gz: 50a15495b6c79bba2264981fa4fe07158d6456ed
 SHA512:
-  metadata.gz: fc4627b56956d3e9ce4795286416798815f1475b5542aee9b7de375f1b8c6b64f083b02b93db041e3530ab799f63732bc67bd3e87662e131ffd81c9dcd8216c5
-  data.tar.gz: c0d98a660d38ed280d769427e1ba7adb37e66426a2ccd7d46156c582194d0950eb6628a68999e257db3a92dd79871eeb8b29a954d8a29b01d303a82e744f6cec
+  metadata.gz: 16088e13da325245d18e3f0c45983f1fd288459acb7411475c15a4a7a141b6ae53a62a774ee4b93ccd768e6a987f544e8d16f0956e79884762e484777b53a6a7
+  data.tar.gz: 2ec85bdee426f93026360a03aac14b8b86dca556ce2f60ce6e55f29d3861ab5dbb500c401a4c4e42c193edd366597b9ff2a0f999bbff86efe272d2ccbe1ca0ce

data/lib/bosh/director/agent_client.rb

@@ -52,59 +52,68 @@ module Bosh::Director
     end
 
     def method_missing(method_name, *args)
-      send_message(method_name, *args)
-    end
-
-    # Define methods on this class to make instance_double more useful
-    [
-      :cancel_task,
-      :get_state,
-      :get_task,
-      :list_disk,
-      :prepare_network_change,
-      :prepare_configure_networks,
-      :start,
-    ].each do |message|
-      define_method(message) do |*args|
-        send_message(message, *args)
-      end
+      handle_message_with_retry(method_name, *args)
+    end
+
+    def get_state(*args)
+      send_message(:get_state, *args)
+    end
+
+    def cancel_task(*args)
+      send_message(:cancel_task, *args)
+    end
+
+    def list_disk(*args)
+      send_message(:list_disk, *args)
+    end
+
+    def prepare_configure_networks(*args)
+      send_message(:prepare_configure_networks, *args)
+    end
+
+    def prepare_network_change(*args)
+      send_message(:prepare_network_change, *args)
+    end
+
+    def start(*args)
+      send_message(:start, *args)
     end
 
     def prepare(*args)
-      send_long_running_message(:prepare, *args)
+      send_message(:prepare, *args)
     end
 
     def apply(*args)
-      send_long_running_message(:apply, *args)
+      send_message(:apply, *args)
     end
 
     def compile_package(*args)
-      send_long_running_message(:compile_package, *args)
+      send_message(:compile_package, *args)
     end
 
     def drain(*args)
-      send_long_running_message(:drain, *args)
+      send_message(:drain, *args)
     end
 
     def fetch_logs(*args)
-      send_long_running_message(:fetch_logs, *args)
+      send_message(:fetch_logs, *args)
     end
 
     def migrate_disk(*args)
-      send_long_running_message(:migrate_disk, *args)
+      send_message(:migrate_disk, *args)
     end
 
     def mount_disk(*args)
-      send_long_running_message(:mount_disk, *args)
+      send_message(:mount_disk, *args)
     end
 
     def unmount_disk(*args)
-      send_long_running_message(:unmount_disk, *args)
+      send_message(:unmount_disk, *args)
     end
 
     def update_settings(certs)
       begin
-        send_long_running_message(:update_settings, {"trusted_certs" => certs})
+        send_message(:update_settings, {"trusted_certs" => certs})
       rescue RpcRemoteException => e
         if e.message =~ /unknown message/
           @logger.warn("Ignoring update_settings 'unknown message' error from the agent: #{e.inspect}")
@@ -115,11 +124,11 @@ module Bosh::Director
     end
 
     def stop(*args)
-      send_long_running_message(:stop, *args)
+      send_message(:stop, *args)
     end
 
-    def start_errand(*args)
-      start_long_running_task(:run_errand, *args)
+    def run_errand(*args)
+      start_task(:run_errand, *args)
     end
 
     def wait_for_task(agent_task_id, &blk)
@@ -135,7 +144,7 @@ module Bosh::Director
     end
 
     def configure_networks(*args)
-      send_long_running_message(:configure_networks, *args)
+      send_message(:configure_networks, *args)
     end
 
     def wait_until_ready(deadline = 600)
@@ -196,8 +205,8 @@ module Bosh::Director
           unless timeout > 0
             @nats_rpc.cancel_request(request_id)
             raise RpcTimeout,
-                  "Timed out sending `#{method_name}' to #{@client_id} " +
-                    "after #{@timeout} seconds"
+              "Timed out sending `#{method_name}' to #{@client_id} " +
+                "after #{@timeout} seconds"
           end
           cond.wait(timeout)
         end
@@ -256,7 +265,7 @@ module Bosh::Director
       @resource_manager.delete_resource(blob_id)
     end
 
-    def send_message(message_name, *args)
+    def handle_message_with_retry(message_name, *args)
       retries = @retry_methods[message_name] || 0
       begin
         handle_method(message_name, args)
@@ -269,13 +278,17 @@ module Bosh::Director
       end
     end
 
-    def send_long_running_message(method_name, *args, &blk)
-      task = start_long_running_task(method_name, *args)
-      wait_for_task(task['agent_task_id'], &blk)
+    def send_message(method_name, *args, &blk)
+      task = start_task(method_name, *args)
+      if task['agent_task_id']
+        wait_for_task(task['agent_task_id'], &blk)
+      else
+        task['value']
+      end
     end
 
-    def start_long_running_task(method_name, *args)
-      AgentMessageConverter.convert_old_message_to_new(send_message(method_name, *args))
+    def start_task(method_name, *args)
+      AgentMessageConverter.convert_old_message_to_new(handle_message_with_retry(method_name, *args))
     end
 
     def get_task_status(agent_task_id)

data/lib/bosh/director/api/controllers/deployments_controller.rb

@@ -48,7 +48,7 @@ module Bosh::Director
         redirect "/tasks/#{task.id}"
       end
 
-      # PUT /deployments/foo/jobs/dea/2?state={started,stopped,detached,restart,recreate}
+      # PUT /deployments/foo/jobs/dea/2?state={started,stopped,detached,restart,recreate}&skip_drain=true
       put '/:deployment/jobs/:job/:index', :consumes => :yaml do
         begin
           index = Integer(params[:index])
@@ -61,9 +61,10 @@ module Bosh::Director
             params[:job] => {
               'instance_states' => {
                 index => params['state']
-              }
+              },
             }
-          }
+          },
+          'skip_drain' => params[:job]
         }
 
         deployment = @deployment_manager.find_by_name(params[:deployment])
@@ -269,6 +270,7 @@ module Bosh::Director
       post '/', :consumes => :yaml do
         options = {}
         options['recreate'] = true if params['recreate'] == 'true'
+        options['skip_drain'] = params['skip_drain'] if params['skip_drain']
         latest_cloud_config = Bosh::Director::Api::CloudConfigManager.new.latest
 
         task = @deployment_manager.create_deployment(current_user, request.body, latest_cloud_config, options)
@@ -313,7 +315,6 @@ module Bosh::Director
         planner_factory.planner_without_vm_binding(manifest_hash, cloud_config_model, {})
       end
 
-
       def convert_job_instance_hash(hash)
         hash.reduce([]) do |jobs, kv|
           job, indicies = kv

data/lib/bosh/director/api/controllers/releases_controller.rb

@@ -16,7 +16,6 @@ module Bosh::Director
       post '/', :consumes => :multipart do
         options = {
           rebase: params['rebase'] == 'true',
-          compiled: params['compiled'] == 'true',
         }
 
         task = @release_manager.create_release_from_file_path(current_user, params[:nginx_upload_path], options)
@@ -61,6 +60,11 @@ module Bosh::Director
 
       get '/:name', scope: :read do
         name = params[:name].to_s.strip
+
+        if params['version']
+          return inspect_release(name, params['version'])
+        end
+
         release = @release_manager.find_by_name(name)
 
         result = { }
@@ -101,6 +105,44 @@ module Bosh::Director
         task = @release_manager.delete_release(current_user, release, options)
         redirect "/tasks/#{task.id}"
       end
+
+      private
+
+      def inspect_release(name, version)
+        release = @release_manager.find_by_name(name)
+        release_version = @release_manager.find_version(release, version)
+
+        result = { }
+
+        result['jobs'] = release_version.templates.sort_by { |t| t.name }.map do |template|
+          {
+              'name' => template.name,
+              'blobstore_id' => template.blobstore_id,
+              'sha1' => template.sha1,
+              'fingerprint' => template.fingerprint.to_s,
+          }
+        end
+
+        result['packages'] = release_version.packages.sort_by { |p| p.name }.map do |package|
+          {
+              'name' => package.name,
+              'blobstore_id' => package.blobstore_id,
+              'sha1' => package.sha1,
+              'fingerprint' => package.fingerprint.to_s,
+              'compiled_packages' => package.compiled_packages.sort_by { |cp| cp.stemcell }.map do |compiled|
+                {
+                    'stemcell' => "#{compiled.stemcell.name}/#{compiled.stemcell.version}",
+                    'sha1' => compiled.sha1,
+                    'blobstore_id' => compiled.blobstore_id,
+                }
+              end
+          }
+        end
+
+        content_type(:json)
+        json_encode(result)
+      end
+
     end
   end
 end

data/lib/bosh/director/compiled_release_downloader.rb

@@ -16,20 +16,30 @@ module Bosh::Director
       path = File.join(@download_dir, 'compiled_packages')
       FileUtils.mkpath(path)
 
-      @compiled_packages_group.compiled_packages.each do |compiled_package|
-        blobstore_id = compiled_package.blobstore_id
-        File.open(File.join(path, "#{compiled_package.package.name}.tgz"), 'w') do |f|
-          @blobstore_client.get(blobstore_id, f, sha1: compiled_package.sha1)
+      compiled_packages = @compiled_packages_group.compiled_packages
+      event_log.begin_stage("copying packages", compiled_packages.count)
+
+      compiled_packages.each do |compiled_package|
+        desc = "#{compiled_package.package.name}/#{compiled_package.package.version}"
+        event_log.track(desc) do
+          blobstore_id = compiled_package.blobstore_id
+          File.open(File.join(path, "#{compiled_package.package.name}.tgz"), 'w') do |f|
+            @blobstore_client.get(blobstore_id, f, sha1: compiled_package.sha1)
+          end
         end
       end
 
       path = File.join(@download_dir, 'jobs')
       FileUtils.mkpath(path)
 
+      event_log.begin_stage("copying jobs", @templates.count)
       @templates.each do |template|
-        blobstore_id = template.blobstore_id
-        File.open(File.join(path, "#{template.name}.tgz"), 'w') do |f|
-          @blobstore_client.get(blobstore_id, f, sha1: template.sha1)
+        desc = "#{template.name}/#{template.version}"
+        event_log.track(desc) do
+          blobstore_id = template.blobstore_id
+          File.open(File.join(path, "#{template.name}.tgz"), 'w') do |f|
+            @blobstore_client.get(blobstore_id, f, sha1: template.sha1)
+          end
         end
       end
 
@@ -39,5 +49,10 @@ module Bosh::Director
     def cleanup
       FileUtils.rm_rf(@download_dir)
     end
+
+    def event_log
+      @event_log ||= Config.event_log
+    end
+
   end
 end

data/lib/bosh/director/compiled_release_manifest.rb

@@ -51,7 +51,15 @@ module Bosh::Director
     end
 
     def write(dest_path)
-      File.open(dest_path, 'w') { |f| f.write(YAML.dump(generate_manifest)) }
+      manifest = generate_manifest
+      manifest_yaml = YAML.dump(manifest)
+      logger.debug("release.MF contents of #{manifest['name']}/#{manifest['version']} compiled release tarball:\n " + manifest_yaml)
+      File.open(dest_path, 'w') { |f| f.write(manifest_yaml) }
     end
+
+    def logger
+      @logger ||= Config.logger
+    end
+
   end
 end

data/lib/bosh/director/config.rb

@@ -36,7 +36,11 @@ module Bosh::Director
         :nats_uri,
       )
 
-      attr_reader :db_config, :redis_logger_level
+      attr_reader(
+        :db_config,
+        :redis_logger_level,
+        :ignore_missing_gateway
+      )
 
       def clear
         self.instance_variables.each do |ivar|
@@ -141,6 +145,7 @@ module Bosh::Director
         @enable_snapshots = config.fetch('snapshots', {}).fetch('enabled', false)
 
         @trusted_certs = config['trusted_certs'] || ''
+        @ignore_missing_gateway = config['ignore_missing_gateway']
 
         Bosh::Clouds::Config.configure(self)
 
@@ -171,7 +176,12 @@ module Bosh::Director
         db_config = db_config.merge(connection_options)
 
         db = Sequel.connect(db_config)
-        db.extension :connection_validator
+
+        Bosh::Common.retryable(sleep: 0.5, tries: 20, on: [Exception]) do
+          db.extension :connection_validator
+          true
+        end
+
         db.pool.connection_validation_timeout = -1
         if logger
           db.logger = logger

data/lib/bosh/director/deployment_plan/network_subnet.rb

@@ -43,7 +43,8 @@ module Bosh::Director
         network_id = @range.network(:Objectify => true)
         broadcast = @range.broadcast(:Objectify => true)
 
-        gateway_property = safe_property(subnet_spec, "gateway", class: String)
+        ignore_missing_gateway = Bosh::Director::Config.ignore_missing_gateway
+        gateway_property = safe_property(subnet_spec, "gateway", class: String, optional: ignore_missing_gateway)
         if gateway_property
           @gateway = NetAddr::CIDR.create(gateway_property)
           unless @gateway.size == 1

data/lib/bosh/director/deployment_plan/options/skip_drain.rb

@@ -0,0 +1,20 @@
+module Bosh::Director
+  module DeploymentPlan
+    class SkipDrain
+      def initialize(param)
+        @jobs = []
+        @all = false
+
+        if param == '*'
+          @all = true
+        elsif param.is_a?(String) && !param.empty?
+          @jobs = param.split(',')
+        end
+      end
+
+      def for_job(job_name)
+        @all || @jobs.include?(job_name)
+      end
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/planner.rb

@@ -45,6 +45,9 @@ module Bosh::Director
       # @return [Boolean] Indicates whether VMs should be recreated
       attr_reader :recreate
 
+      # @return [Boolean] Indicates whether VMs should be drained
+      attr_reader :skip_drain
+
       def initialize(attrs, manifest_text, cloud_config, deployment_model, options = {})
         @name = attrs.fetch(:name)
         @properties = attrs.fetch(:properties)
@@ -67,6 +70,7 @@ module Bosh::Director
           :class => Hash, :default => {})
 
         @recreate = !!options['recreate']
+        @skip_drain = SkipDrain.new(options['skip_drain'])
       end
 
       def_delegators :@cloud_planner, :add_network, :networks, :network,
@@ -84,6 +88,10 @@ module Bosh::Director
         @model.vms
       end
 
+      def skip_drain_for_job?(name)
+        @skip_drain.nil? ? false : @skip_drain.for_job(name)
+      end
+
       # Adds a release by name
       # @param [Bosh::Director::DeploymentPlan::ReleaseVersion] release
       def add_release(release)

data/lib/bosh/director/deployment_plan/planner_factory.rb

@@ -56,6 +56,32 @@ module Bosh
           assemble_without_vm_binding(attrs, deployment_manifest, cloud_manifest, deployment_model, cloud_config, options)
         end
 
+        def self.validate_packages(planner, options)
+          release_manager = Bosh::Director::Api::ReleaseManager.new
+          planner.jobs.each { |job|
+            job.templates.each{ |template|
+              release_model = release_manager.find_by_name(template.release.name)
+              template.package_models.each{ |package|
+
+                release_version_model = release_manager.find_version(release_model, template.release.version)
+                packages_list = release_version_model.transitive_dependencies(package)
+                packages_list << package
+
+                packages_list.each { |needed_package|
+                  if needed_package.sha1.nil? || needed_package.blobstore_id.nil?
+                    compiled_packages_list = Bosh::Director::Models::CompiledPackage[:package_id => needed_package.id, :stemcell_id => job.resource_pool.stemcell.model.id]
+                    if compiled_packages_list.nil?
+                      msg = "Can't #{options[:context]} `#{release_version_model.release.name}/#{release_version_model.version}': it is not " +
+                          "compiled for `#{job.resource_pool.stemcell.model.desc}' and no source package is available"
+                      raise PackageMissingSourceCode, msg
+                    end
+                  end
+                }
+              }
+            }
+          }
+        end
+
         private
 
         def deployment_name(manifest_hash)
@@ -66,11 +92,12 @@ module Bosh
         def assemble_without_vm_binding(attrs, deployment_manifest, cloud_manifest, deployment_model, cloud_config, options)
           plan_options = {
             'recreate' => !!options['recreate'],
+            'skip_drain' => options['skip_drain'],
             'job_states' => options['job_states'] || {},
             'job_rename' => options['job_rename'] || {}
           }
           @logger.info('Creating deployment plan')
-          @logger.info("Deployment plan options: #{plan_options.pretty_inspect}")
+          @logger.info("Deployment plan options: #{plan_options}")
 
           deployment = Planner.new(attrs, deployment_manifest, cloud_config, deployment_model, plan_options)
           deployment = CloudManifestParser.new(deployment, @logger).parse(cloud_manifest)
@@ -94,6 +121,8 @@ module Bosh
 
           run_prepare_step(assembler)
 
+          self.class.validate_packages(planner, {:context => 'deploy'})
+
           DeploymentPlan::Steps::PackageCompileStep.new(
             planner,
             cloud,

data/lib/bosh/director/deployment_plan.rb

@@ -30,3 +30,4 @@ require 'bosh/director/deployment_plan/assembler'
 require 'bosh/director/deployment_plan/planner_factory'
 require 'bosh/director/deployment_plan/manifest_migrator'
 require 'bosh/director/deployment_plan/deployment_repo'
+require 'bosh/director/deployment_plan/options/skip_drain'

data/lib/bosh/director/errand/runner.rb

@@ -26,8 +26,8 @@ module Bosh::Director
 
       begin
         event_log_stage.advance_and_track("#{@job.name}/#{instance.index}") do
-          start_errand_result = agent.start_errand
-          @agent_task_id = start_errand_result['agent_task_id']
+          run_errand_result = agent.run_errand
+          @agent_task_id = run_errand_result['agent_task_id']
           agent_task_result = agent.wait_for_task(agent_task_id, &blk)
         end
       rescue TaskCancelled => e

data/lib/bosh/director/errors.rb

@@ -73,6 +73,7 @@ module Bosh::Director
   ReleaseVersionInUse = err(30008)
   ReleaseDeleteFailed = err(30009)
   ReleaseVersionInvalid = err(30010)
+  ReleaseNotMatchingManifest = err(30011)
 
   ValidationInvalidType = err(40000)
   ValidationMissingField = err(40001)
@@ -86,6 +87,7 @@ module Bosh::Director
   StemcellInUse = err(50004)
 
   PackageInvalidArchive = err(60000)
+  PackageMissingSourceCode = err(60001)
 
   # Models
   DeploymentNotFound = err(70000, NOT_FOUND)
@@ -213,6 +215,7 @@ module Bosh::Director
 
   PackageCompilationNetworkNotReserved = err(430001)
   PackageCompilationNotEnoughWorkersForReuse = err(430002)
+  PackageCompilationNotFound = err(430003)
 
   BadManifest = err(440001)
 

data/lib/bosh/director/instance_updater/stopper.rb

@@ -1,14 +1,27 @@
 module Bosh::Director
   class InstanceUpdater::Stopper
-    def initialize(instance, agent_client, target_state, config, logger)
+    def initialize(instance, agent_client, target_state, skip_drain, config, logger)
       @instance = instance
       @agent_client = agent_client
       @target_state = target_state
+      @skip_drain = skip_drain
       @config = config
       @logger = logger
     end
 
     def stop
+      if @skip_drain
+        @logger.info("Skipping drain for '#{@instance}'")
+      else
+        perform_drain
+      end
+
+      @agent_client.stop
+    end
+
+    private
+
+    def perform_drain
       drain_type = shutting_down? ? 'shutdown' : 'update'
 
       # Apply spec might change after shutdown drain (unlike update drain)
@@ -21,12 +34,8 @@ module Bosh::Director
       else
         wait_for_dynamic_drain(drain_time)
       end
-
-      @agent_client.stop
     end
 
-    private
-
     def shutting_down?
       @instance.resource_pool_changed? ||
         @instance.persistent_disk_changed? ||

data/lib/bosh/director/instance_updater.rb

@@ -143,7 +143,8 @@ module Bosh::Director
     end
 
     def stop
-      stopper = Stopper.new(@instance, agent, @target_state, Config, @logger)
+      skip_drain = @deployment_plan.skip_drain_for_job?(@job.name)
+      stopper = Stopper.new(@instance, agent, @target_state, skip_drain, Config, @logger)
       stopper.stop
     end
 

data/lib/bosh/director/jobs/delete_release.rb

@@ -132,12 +132,13 @@ module Bosh::Director
                       "(#{compiled_package.blobstore_id}) " +
                       "#{package.name}/#{package.version} " +
                       "for #{stemcell.name}/#{stemcell.version}")
-          delete_blobstore_id(compiled_package.blobstore_id) do
+
+          if delete_blobstore_id(compiled_package.blobstore_id)
             compiled_package.destroy
           end
         end
 
-        delete_blobstore_id(package.blobstore_id) do
+        if delete_blobstore_id(package.blobstore_id, true)
           package.remove_all_release_versions
           package.destroy
         end
@@ -146,7 +147,7 @@ module Bosh::Director
       def delete_template(template)
         logger.info("Deleting template: #{template.name}/#{template.version}")
 
-        delete_blobstore_id(template.blobstore_id) do
+        if delete_blobstore_id(template.blobstore_id)
           template.remove_all_release_versions
           template.destroy
         end
@@ -201,17 +202,21 @@ module Bosh::Director
         "/release/#{@name}"
       end
 
-      def delete_blobstore_id(blobstore_id)
+      def delete_blobstore_id(blobstore_id, nil_id_allowed = false)
+        if blobstore_id.nil? && nil_id_allowed
+          return true
+        end
+
         deleted = false
         begin
           @blobstore.delete(blobstore_id)
           deleted = true
         rescue Exception => e
-          logger.warn("Could not delete from blobstore: #{e}\n " +
-                      e.backtrace.join("\n"))
+          logger.warn("Could not delete from blobstore: #{e}\n " + e.backtrace.join("\n"))
           @errors << e
         end
-        yield if deleted || @force
+
+        return deleted || @force
       end
 
     end

data/lib/bosh/director/jobs/export_release.rb

@@ -10,10 +10,6 @@ module Bosh::Director
 
       @queue = :normal
 
-      @stemcell
-      @targeted_deployment
-      @release_version
-
       def self.job_type
         :export_release
       end
@@ -29,7 +25,24 @@ module Bosh::Director
       # @return [void]
       def perform
         logger.info("Exporting release: #{@release_name}/#{@release_version} for #{@stemcell_os}/#{@stemcell_version}")
-        validate_and_prepare
+
+        stemcell_manager = Bosh::Director::Api::StemcellManager.new
+        @stemcell = stemcell_manager.find_by_os_and_version(@stemcell_os, @stemcell_version)
+
+        logger.info "Will compile with stemcell: #{@stemcell.desc}"
+
+        deployment_manager = Bosh::Director::Api::DeploymentManager.new
+        @targeted_deployment = deployment_manager.find_by_name(@deployment_name)
+        @deployment_manifest = Psych.load(@targeted_deployment.manifest)
+
+        release_manager = Bosh::Director::Api::ReleaseManager.new
+        release = release_manager.find_by_name(@release_name)
+        @release_version_model = release_manager.find_version(release, @release_version)
+
+        unless deployment_manifest_has_release?
+          raise ReleaseNotMatchingManifest, "Release #{@release_name}/#{@release_version} not found in deployment #{@deployment_name} manifest"
+        end
+
         lock_timeout = 15 * 60 # 15 minutes
 
         with_deployment_lock(@deployment_name, :timeout => lock_timeout) do
@@ -37,6 +50,7 @@ module Bosh::Director
             with_stemcell_lock(@stemcell.name, @stemcell.version, :timeout => lock_timeout) do
 
               planner = create_planner
+              DeploymentPlan::PlannerFactory.validate_packages(planner, {:context => 'export'})
               package_compile_step = DeploymentPlan::Steps::PackageCompileStep.new(
                   planner,
                   Config.cloud, # CPI
@@ -55,27 +69,21 @@ module Bosh::Director
         "Exported release: #{@release_name}/#{@release_version} for #{@stemcell_os}/#{@stemcell_version}"
       end
 
-      def validate_and_prepare
-        stemcell_manager = Bosh::Director::Api::StemcellManager.new
-        @stemcell = stemcell_manager.find_by_os_and_version(@stemcell_os, @stemcell_version)
-
-        logger.info "Will compile with stemcell: #{@stemcell.desc}"
-
-        deployment_manager = Bosh::Director::Api::DeploymentManager.new
-        @targeted_deployment = deployment_manager.find_by_name(@deployment_name)
-
-        release_manager = Bosh::Director::Api::ReleaseManager.new
-        release = release_manager.find_by_name(@release_name)
-        @release_version = release_manager.find_version(release, @release_version)
+      def deployment_manifest_has_release?
+        @deployment_manifest["releases"].each do |release|
+          if (release["name"] == @release_name) && (release["version"].to_s == @release_version.to_s)
+            return true
+          end
+        end
+        false
       end
 
       def create_planner
-        modified_deployment_manifest = Psych.load(@targeted_deployment.manifest)
         cloud_config_model = @targeted_deployment.cloud_config
 
         planner_factory = DeploymentPlan::PlannerFactory.create(Config.event_log, Config.logger)
         planner = planner_factory.planner_without_vm_binding(
-            modified_deployment_manifest,
+            @deployment_manifest,
             cloud_config_model,
             {}
         )
@@ -101,8 +109,8 @@ module Bosh::Director
 
         blobstore_client = Bosh::Director::App.instance.blobstores.blobstore
 
-        compiled_packages_group = CompiledPackageGroup.new(@release_version, @stemcell)
-        templates = @release_version.templates.map
+        compiled_packages_group = CompiledPackageGroup.new(@release_version_model, @stemcell)
+        templates = @release_version_model.templates.map
 
         compiled_release_downloader = CompiledReleaseDownloader.new(compiled_packages_group, templates, blobstore_client)
         download_dir = compiled_release_downloader.download
@@ -132,14 +140,14 @@ module Bosh::Director
             "release" => @release_name,
             "instances" => 1,
             "resource_pool" => fake_resource_pool_manifest['name'],
-            "templates" => @release_version.templates.map do |template|
+            "templates" => @release_version_model.templates.map do |template|
               { "name" => template.name, "release" => @release_name }
             end,
             "networks" => [ "name" => network_name ],
         }
 
         fake_job = DeploymentPlan::Job.parse(planner, fake_job_spec_for_compiling, Config.event_log, Config.logger)
-        @release_version.packages.each { |package| fake_job.packages[package.name] = package }
+        @release_version_model.packages.each { |package| fake_job.packages[package.name] = package }
         fake_job.resource_pool.stemcell.bind_model
         fake_job.release.bind_model
         fake_job.templates.each { |template| template.bind_models }

data/lib/bosh/director/jobs/update_release.rb

@@ -33,13 +33,6 @@ module Bosh::Director
 
         @rebase = !!options['rebase']
         @skip_if_exists = !!options['skip_if_exists']
-        @compiled_release = !!options['compiled']
-
-        if @compiled_release
-          @packages_folder = "compiled_packages"
-        else
-          @packages_folder = "packages"
-        end
 
         @manifest = nil
         @name = nil
@@ -66,10 +59,6 @@ module Bosh::Director
           process_release(release_dir)
         }
 
-        if @rebase && @packages_unchanged && @jobs_unchanged
-          raise DirectorError, "Rebase is attempted without any job or package changes"
-        end
-
         "Created release `#{@name}/#{@version}'"
 
       rescue Exception => e
@@ -95,6 +84,7 @@ module Bosh::Director
           logger.error("Failed to extract release archive '#{@release_path}' into dir '#{release_dir}', " +
                        "tar returned #{result.exit_status}, " +
                        "output: #{result.output}")
+          FileUtils.rm_rf(release_dir)
           raise ReleaseInvalidArchive, "Extracting release archive failed. Check task debug log for details."
         end
 
@@ -110,6 +100,15 @@ module Bosh::Director
         end
 
         @manifest = Psych.load_file(manifest_file)
+
+        #handle compiled_release case
+        @compiled_release = !!@manifest["compiled_packages"]
+        if @compiled_release
+          @packages_folder = "compiled_packages"
+        else
+          @packages_folder = "packages"
+        end
+
         normalize_manifest
 
         @name = @manifest["name"]
@@ -253,69 +252,134 @@ module Bosh::Director
           end
         end
 
-        create_packages(new_packages, release_dir)
-        use_existing_packages(existing_packages)
+        package_stemcell_hashes1 = create_packages(new_packages, release_dir)
+        package_stemcell_hashes2 = use_existing_packages(existing_packages)
+        consolidated_package_stemcell_hashes = Array(package_stemcell_hashes1) | Array(package_stemcell_hashes2)
+
+        create_compiled_packages(consolidated_package_stemcell_hashes, release_dir)
+      end
+
+      # Points release DB model to existing packages described by given metadata
+      # @param [Array<Array>] packages Existing packages metadata
+      def use_existing_packages(packages)
+        return if packages.empty?
+
+        package_stemcell_hashes = []
+
+        single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do
+          packages.each do |package, package_meta|
+            package_desc = "#{package.name}/#{package.version}"
+            logger.info("Using existing package `#{package_desc}'")
+            register_package(package)
+
+            if @compiled_release
+              stemcells = stemcells_used_by_package(package_meta)
+              stemcells.each do |stemcell|
+                hash = { "package" => package, "stemcell" => stemcell}
+                package_stemcell_hashes << hash
+              end
+            end
+          end
+        end
+
+        package_stemcell_hashes
       end
 
       # Creates packages using provided metadata
       # @param [Array<Hash>] packages Packages metadata
       # @param [String] release_dir local path to the unpacked release
-      # @return [void]
-      def create_packages(packages, release_dir)
-        if packages.empty?
+      # @return [Array<Hash>] package & stemcell
+      def create_packages(package_metas, release_dir)
+        if package_metas.empty?
           @packages_unchanged = true
           return
         end
 
-        event_log.begin_stage("Creating new packages", packages.size)
-        packages.each do |package_meta|
+        package_stemcell_hashes = []
+        event_log.begin_stage("Creating new packages", package_metas.size)
+
+        package = package_metas.each do |package_meta|
           package_desc = "#{package_meta["name"]}/#{package_meta["version"]}"
           event_log.track(package_desc) do
             logger.info("Creating new package `#{package_desc}'")
+            package = create_package(package_meta, release_dir)
+            register_package(package)
+            package
+          end
 
-            if @compiled_release
-              package = create_package_for_compiled_release(package_meta, release_dir)
-            else
-              package = create_package(package_meta, release_dir)
+          if @compiled_release
+            stemcells = stemcells_used_by_package(package_meta)
+            stemcells.each do |stemcell|
+              hash = { "package" => package, "stemcell" => stemcell}
+              package_stemcell_hashes << hash
             end
-
-            register_package(package)
           end
         end
+
+        package_stemcell_hashes
       end
 
-      # Points release DB model to existing packages described by given metadata
-      # @param [Array<Array>] packages Existing packages metadata
-      def use_existing_packages(packages)
-        return if packages.empty?
+      def create_compiled_packages(all_compiled_packages, release_dir)
+        if all_compiled_packages.nil?
+          return
+        end
 
-        single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do
-          packages.each do |package, _|
-            package_desc = "#{package.name}/#{package.version}"
-            logger.info("Using existing package `#{package_desc}'")
-            register_package(package)
+        event_log.begin_stage('Creating new compiled packages', all_compiled_packages.size)
+
+        all_compiled_packages.each do |compiled_package_spec|
+          package = compiled_package_spec['package']
+          stemcell = compiled_package_spec['stemcell']
+
+          existing_compiled_package = Models::CompiledPackage.where(
+              :package_id => package.id,
+              :stemcell_id => stemcell.id)
+
+          if existing_compiled_package.empty?
+            package_desc = "#{package.name}/#{package.version} for #{stemcell.name}/#{stemcell.version}"
+            event_log.track(package_desc) do
+              create_compiled_package(package, stemcell, release_dir)
+            end
           end
         end
       end
 
-      def create_package_for_compiled_release(package_meta, release_dir)
-        name, version = package_meta["name"], package_meta["version"]
-        desc = "package `#{name}/#{version}'"
+      def stemcells_used_by_package(package_meta)
+        if package_meta['stemcell'].nil?
+          raise 'stemcell informatiom(operating system/version) should be listed for each package of a compiled tarball'
+        end
 
-        package_attrs = {
-            :release => @release_model,
-            :name => name,
-            :sha1 => nil,
-            :blobstore_id => nil,
-            :fingerprint => package_meta["fingerprint"],
-            :version => version
-        }
+        values = package_meta['stemcell'].split('/', 2)
+        operating_system = values[0]
+        stemcell_version = values[1]
+        unless operating_system && stemcell_version
+          raise 'stemcell informatiom(operating system/version) should be listed for each package of a compiled tarball'
+        end
 
-        package = Models::Package.new(package_attrs)
-        package.dependency_set = package_meta["dependencies"]
+        stemcells = Models::Stemcell.where(:operating_system => operating_system, :version => stemcell_version)
+        if stemcells.empty?
+          raise "No stemcells matching OS #{operating_system} version #{stemcell_version}"
+        end
 
-        logger.info("Creating #{desc} from provided bits")
-        package.save
+        stemcells
+      end
+
+      def create_compiled_package(package, stemcell, release_dir)
+        tgz = File.join(release_dir, 'compiled_packages', "#{package.name}.tgz")
+        validate_tgz(tgz, "#{package.name}.tgz")
+
+        compiled_package = Models::CompiledPackage.new
+
+        compiled_package.blobstore_id = BlobUtil.create_blob(tgz)
+        compiled_package.sha1 = Digest::SHA1.file(tgz).hexdigest
+
+        transitive_dependencies = @release_version_model.transitive_dependencies(package)
+        compiled_package.dependency_key = Models::CompiledPackage.create_dependency_key(transitive_dependencies)
+
+        compiled_package.build = Models::CompiledPackage.generate_build_number(package, stemcell)
+        compiled_package.package_id = package.id
+        compiled_package.stemcell_id = stemcell.id
+
+        compiled_package.save
       end
 
       # Creates package in DB according to given metadata
@@ -323,43 +387,48 @@ module Bosh::Director
       # @param [String] release_dir local path to the unpacked release
       # @return [void]
       def create_package(package_meta, release_dir)
-        name, version = package_meta["name"], package_meta["version"]
+        name, version = package_meta['name'], package_meta['version']
 
         package_attrs = {
-          :release => @release_model,
-          :name => name,
-          :sha1 => package_meta["sha1"],
-          :fingerprint => package_meta["fingerprint"],
-          :version => version
+            :release => @release_model,
+            :name => name,
+            :sha1 => @compiled_release ? nil : package_meta['sha1'],
+            :blobstore_id => nil,
+            :fingerprint => package_meta['fingerprint'],
+            :version => version
         }
 
         package = Models::Package.new(package_attrs)
-        package.dependency_set = package_meta["dependencies"]
+        package.dependency_set = package_meta['dependencies']
 
-        existing_blob = package_meta["blobstore_id"]
-        desc = "package `#{name}/#{version}'"
+        unless @compiled_release
+          existing_blob = package_meta['blobstore_id']
+          desc = "package '#{name}/#{version}'"
 
-        if existing_blob
-          logger.info("Creating #{desc} from existing blob #{existing_blob}")
-          package.blobstore_id = BlobUtil.copy_blob(existing_blob)
-        else
-          logger.info("Creating #{desc} from provided bits")
-
-          package_tgz = File.join(release_dir, "packages", "#{name}.tgz")
-          result = Bosh::Exec.sh("tar -tzf #{package_tgz} 2>&1", :on_error => :return)
-          if result.failed?
-            logger.error("Extracting #{desc} archive failed, " +
-                         "tar returned #{result.exit_status}, " +
-                         "output: #{result.output}")
-            raise PackageInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details."
-          end
+          if existing_blob
+            logger.info("Creating #{desc} from existing blob #{existing_blob}")
+            package.blobstore_id = BlobUtil.copy_blob(existing_blob)
+
+          else
+            logger.info("Creating #{desc} from provided bits")
 
-          package.blobstore_id = BlobUtil.create_blob(package_tgz)
+            package_tgz = File.join(release_dir, 'packages', "#{name}.tgz")
+            validate_tgz(package_tgz, desc)
+            package.blobstore_id = BlobUtil.create_blob(package_tgz)
+          end
         end
 
         package.save
       end
 
+      def validate_tgz(tgz, desc)
+        result = Bosh::Exec.sh("tar -tzf #{tgz} 2>&1", :on_error => :return)
+        if result.failed?
+          logger.error("Extracting #{desc} archive failed, tar returned #{result.exit_status}, output: #{result.output}")
+          raise PackageInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details."
+        end
+      end
+
       # Marks package model as used by release version model
       # @param [Models::Package] package Package model
       # @return [void]

data/lib/bosh/director/models/package.rb

@@ -18,6 +18,10 @@ module Bosh::Director::Models
     end
 
     def validate
+      if !sha1.nil? || !blobstore_id.nil?
+        validates_presence [:sha1, :blobstore_id]
+      end
+
       validates_presence [:release_id, :name, :version]
       validates_unique [:release_id, :name, :version]
       validates_format VALID_ID, [:name, :version]

data/lib/bosh/director/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Director
-    VERSION = '1.3022.0'
+    VERSION = '1.3048.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh-director
 version: !ruby/object:Gem::Version
-  version: 1.3022.0
+  version: 1.3048.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-23 00:00:00.000000000 Z
+date: 2015-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bcrypt-ruby
@@ -30,126 +30,126 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
 - !ruby/object:Gem::Dependency
   name: bosh-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
 - !ruby/object:Gem::Dependency
   name: bosh-director-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
 - !ruby/object:Gem::Dependency
   name: bosh_common
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
 - !ruby/object:Gem::Dependency
   name: bosh-template
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
 - !ruby/object:Gem::Dependency
   name: bosh_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
 - !ruby/object:Gem::Dependency
   name: bosh_openstack_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
 - !ruby/object:Gem::Dependency
   name: bosh_aws_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
 - !ruby/object:Gem::Dependency
   name: bosh_vsphere_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3022.0
+        version: 1.3048.0
 - !ruby/object:Gem::Dependency
   name: bosh_vcloud_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -572,7 +572,7 @@ dependencies:
         version: '0'
 description: |-
   BOSH Director
-  6073a7
+  e2c11a
 email: support@cloudfoundry.com
 executables:
 - bosh-director
@@ -723,6 +723,7 @@ files:
 - lib/bosh/director/deployment_plan/network.rb
 - lib/bosh/director/deployment_plan/network_subnet.rb
 - lib/bosh/director/deployment_plan/notifier.rb
+- lib/bosh/director/deployment_plan/options/skip_drain.rb
 - lib/bosh/director/deployment_plan/planner.rb
 - lib/bosh/director/deployment_plan/planner_factory.rb
 - lib/bosh/director/deployment_plan/release_version.rb