bosh-director 1.2583.0 → 1.2596.0

data/lib/bosh/director.rb

@@ -23,6 +23,7 @@ require 'zlib'
 
 require 'common/exec'
 require 'common/properties'
+require 'common/version/release_version_list'
 
 require 'bcrypt'
 require 'blobstore_client'
@@ -48,7 +49,6 @@ require 'bosh/director/validation_helper'
 require 'bosh/director/download_helper'
 
 require 'bosh/director/version'
-require 'bosh/director/next_rebase_version'
 require 'bosh/director/config'
 require 'bosh/director/event_log'
 require 'bosh/director/task_result_file'
@@ -72,10 +72,12 @@ require 'bosh/director/hash_string_vals'
 require 'bosh/director/instance_deleter'
 require 'bosh/director/instance_updater'
 require 'bosh/director/instance_updater/preparer'
-require 'bosh/director/instance_updater/network_updater'
 require 'bosh/director/instance_updater/stopper'
+require 'bosh/director/instance_updater/network_updater'
+require 'bosh/director/instance_updater/vm_updater'
 require 'bosh/director/job_runner'
 require 'bosh/director/job_updater'
+require 'bosh/director/job_updater_factory'
 require 'bosh/director/job_queue'
 require 'bosh/director/lock'
 require 'bosh/director/nats_rpc'

data/lib/bosh/director/api/controllers/packages_controller.rb

@@ -5,25 +5,20 @@ module Bosh::Director
     class PackagesController < BaseController
       post '/packages/matches', :consumes => :yaml do
         manifest = Psych.load(request.body)
+
         unless manifest.is_a?(Hash) && manifest['packages'].is_a?(Array)
           raise BadManifest, "Manifest doesn't have a usable packages section"
         end
 
-        fp_list = []
-        sha1_list = []
+        fingerprint_list = []
 
         manifest['packages'].each do |package|
-          fp_list << package['fingerprint'] if package['fingerprint']
-          sha1_list << package['sha1'] if package['sha1']
+          fingerprint_list << package['fingerprint'] if package['fingerprint']
         end
 
-        filter = {:fingerprint => fp_list, :sha1 => sha1_list}.sql_or
-
-        result = Models::Package.where(filter).all.map { |package|
-          [package.sha1, package.fingerprint]
-        }.flatten.compact.uniq
+        matching_packages = Models::Package.where(fingerprint: fingerprint_list).all
 
-        json_encode(result)
+        json_encode(matching_packages.map(&:fingerprint).compact.uniq)
       end
     end
   end

data/lib/bosh/director/api/release_manager.rb

@@ -27,19 +27,22 @@ module Bosh::Director
         release_version = dataset.filter(:version => version).first
         if release_version.nil?
           begin
-            # specified version not found, try formatted version
-            formatted_version = Bosh::Common::VersionNumber.parse(version).to_s
-            unless version == formatted_version
-              # only check db if the formatted version is different
-              release_version = dataset.filter(:version => formatted_version).first
-            end
-            if release_version.nil?
-              raise ReleaseVersionNotFound,
-                    "Release version `#{release.name}/#{version}' doesn't exist"
-            end
+            new_formatted_version = Bosh::Common::Version::ReleaseVersion.parse(version)
           rescue SemiSemantic::ParseError
             raise ReleaseVersionInvalid, "Release version invalid: #{version}"
           end
+          if version == new_formatted_version.to_s
+            old_formatted_version = new_formatted_version.to_old_format
+            if old_formatted_version
+              release_version = dataset.filter(:version => old_formatted_version).first
+            end
+          else
+            release_version = dataset.filter(:version => new_formatted_version.to_s).first
+          end
+          if release_version.nil?
+            raise ReleaseVersionNotFound,
+                  "Release version `#{release.name}/#{version}' doesn't exist"
+          end
         end
 
         release_version

data/lib/bosh/director/deployment_plan/assembler.rb

@@ -312,7 +312,7 @@ module Bosh::Director
     # @return [void]
     def bind_configuration
       @deployment_plan.jobs_starting_on_deploy.each do |job|
-        JobRenderer.new(job).render_job_instances(@blobstore)
+        JobRenderer.new(job, @blobstore).render_job_instances
       end
     end
 

data/lib/bosh/director/deployment_plan/instance_vm_binder.rb

@@ -51,6 +51,7 @@ module Bosh::Director
           idle_vm.vm.update(:apply_spec => state)
           instance.model.update(:vm => idle_vm.vm)
         end
+
         instance.current_state = state
       end
     end

data/lib/bosh/director/deployment_plan/multi_job_updater.rb

@@ -1,19 +1,27 @@
 module Bosh::Director
   module DeploymentPlan
     class SerialMultiJobUpdater
+      def initialize(job_updater_factory)
+        @job_updater_factory = job_updater_factory
+      end
+
       def run(base_job, deployment_plan, jobs)
         base_job.logger.info("Updating jobs serially: #{jobs.map(&:name).join(', ')}")
 
         jobs.each do |j|
           base_job.task_checkpoint
           base_job.logger.info("Updating job: #{j.name}")
-          job_updater = JobUpdater.new(deployment_plan, j)
+          job_updater = @job_updater_factory.new_job_updater(deployment_plan, j)
           job_updater.update
         end
       end
     end
 
     class ParallelMultiJobUpdater
+      def initialize(job_updater_factory)
+        @job_updater_factory = job_updater_factory
+      end
+
       def run(base_job, deployment_plan, jobs)
         base_job.logger.info("Updating jobs in parallel: #{jobs.map(&:name).join(', ')}")
         base_job.task_checkpoint
@@ -22,7 +30,7 @@ module Bosh::Director
           jobs.each do |j|
             pool.process do
               base_job.logger.info("Updating job: #{j.name}")
-              job_updater = JobUpdater.new(deployment_plan, j)
+              job_updater = @job_updater_factory.new_job_updater(deployment_plan, j)
               job_updater.update
             end
           end
@@ -31,9 +39,13 @@ module Bosh::Director
     end
 
     class BatchMultiJobUpdater
+      def initialize(job_updater_factory)
+        @job_updater_factory = job_updater_factory
+      end
+
       def run(base_job, deployment_plan, jobs)
-        serial_updater = SerialMultiJobUpdater.new
-        parallel_updater = ParallelMultiJobUpdater.new
+        serial_updater = SerialMultiJobUpdater.new(@job_updater_factory)
+        parallel_updater = ParallelMultiJobUpdater.new(@job_updater_factory)
 
         partition_jobs_by_serial(jobs).each do |jp|
           updater = jp.first.update.serial? ? serial_updater : parallel_updater

data/lib/bosh/director/errand/job_manager.rb

@@ -20,10 +20,10 @@ module Bosh::Director
       instance_vm_binder = DeploymentPlan::InstanceVmBinder.new(@event_log)
       instance_vm_binder.bind_instance_vms(@job.instances)
 
-      job_renderer = JobRenderer.new(@job)
-      job_renderer.render_job_instances(@blobstore)
+      job_renderer = JobRenderer.new(@job, @blobstore)
+      job_renderer.render_job_instances
 
-      job_updater = JobUpdater.new(@deployment, @job)
+      job_updater = JobUpdater.new(@deployment, @job, job_renderer)
       job_updater.update
     end
 

data/lib/bosh/director/instance_updater.rb

@@ -4,38 +4,33 @@ module Bosh::Director
   class InstanceUpdater
     include DnsHelper
 
-    MAX_ATTACH_DISK_TRIES = 3
     UPDATE_STEPS = 7
     WATCH_INTERVALS = 10
 
     attr_reader :current_state
 
     # @params [DeploymentPlan::Instance] instance
-    def initialize(instance, event_log_task)
+    def initialize(instance, event_log_task, job_renderer)
+      @instance = instance
+      @event_log_task = event_log_task
+      @job_renderer = job_renderer
+
       @cloud = Config.cloud
       @logger = Config.logger
-      @event_log_task = event_log_task
       @blobstore = App.instance.blobstores.blobstore
 
-      @instance = instance
       @job = instance.job
-
       @target_state = @instance.state
 
       @deployment_plan = @job.deployment
-      @resource_pool_spec = @job.resource_pool
+      @resource_pool = @job.resource_pool
       @update_config = @job.update
 
       @vm = @instance.model.vm
 
       @current_state = {}
 
-      @network_updater = NetworkUpdater.new(@instance, @vm, agent, self, @cloud, @logger)
-      @stopper = Stopper.new(@instance, agent, @target_state, Config, @logger)
-    end
-
-    def instance_name
-      "#{@job.name}/#{@instance.index}"
+      @agent = AgentClient.with_defaults(@vm.agent_id)
     end
 
     def step
@@ -67,13 +62,11 @@ module Bosh::Director
       step { take_snapshot }
 
       if @target_state == "detached"
-        detach_disk
-        delete_vm
-        @resource_pool_spec.add_idle_vm
+        vm_updater.detach
         return
       end
 
-      step { update_resource_pool }
+      step { update_resource_pool(nil) }
       step { update_networks }
       step { update_dns }
       step { update_persistent_disk }
@@ -89,11 +82,11 @@ module Bosh::Director
       step { wait_until_running }
 
       if @target_state == "started" && current_state["job_state"] != "running"
-        raise AgentJobNotRunning, "`#{instance_name}' is not running after update"
+        raise AgentJobNotRunning, "`#{@instance}' is not running after update"
       end
 
       if @target_state == "stopped" && current_state["job_state"] == "running"
-        raise AgentJobNotStopped, "`#{instance_name}' is still running despite the stop command"
+        raise AgentJobNotStopped, "`#{@instance}' is still running despite the stop command"
       end
     end
 
@@ -103,9 +96,9 @@ module Bosh::Director
     def wait_until_running
       watch_schedule(min_watch_time, max_watch_time).each do |watch_time|
         sleep_time = watch_time.to_f / 1000
-        @logger.info("Waiting for #{sleep_time} seconds to check #{instance_name} status")
+        @logger.info("Waiting for #{sleep_time} seconds to check #{@instance} status")
         sleep(sleep_time)
-        @logger.info("Checking if #{instance_name} has been updated after #{sleep_time} seconds")
+        @logger.info("Checking if #{@instance} has been updated after #{sleep_time} seconds")
 
         @current_state = agent.get_state
 
@@ -141,7 +134,8 @@ module Bosh::Director
     end
 
     def stop
-      @stopper.stop
+      stopper = Stopper.new(@instance, agent, @target_state, Config, @logger)
+      stopper.stop
     end
 
     def take_snapshot
@@ -152,59 +146,6 @@ module Bosh::Director
       Api::SnapshotManager.delete_snapshots(disk.snapshots)
     end
 
-    def detach_disk
-      return unless @instance.disk_currently_attached?
-
-      if @instance.model.persistent_disk_cid.nil?
-        raise AgentUnexpectedDisk,
-              "`#{instance_name}' VM has disk attached " +
-                  "but it's not reflected in director DB"
-      end
-
-      agent.unmount_disk(@instance.model.persistent_disk_cid)
-      @cloud.detach_disk(@vm.cid, @instance.model.persistent_disk_cid)
-    end
-
-    def attach_disk
-      return if @instance.model.persistent_disk_cid.nil?
-
-      @cloud.attach_disk(@vm.cid, @instance.model.persistent_disk_cid)
-      agent.mount_disk(@instance.model.persistent_disk_cid)
-    end
-
-    def delete_vm
-      @cloud.delete_vm(@vm.cid)
-
-      @instance.model.db.transaction do
-        @instance.model.vm = nil
-        @instance.model.save
-        @vm.destroy
-      end
-    end
-
-    def create_vm(new_disk_id)
-      stemcell = @resource_pool_spec.stemcell
-      disks = [@instance.model.persistent_disk_cid, new_disk_id].compact
-
-      @vm = VmCreator.create(@deployment_plan.model, stemcell.model,
-                             @resource_pool_spec.cloud_properties,
-                             @instance.network_settings, disks,
-                             @resource_pool_spec.env)
-
-      begin
-        @instance.model.vm = @vm
-        @instance.model.save
-
-        agent.wait_until_ready
-      rescue => e
-        if @vm
-          @logger.error("error during create_vm(), deleting vm #{@vm.cid}")
-          delete_vm
-        end
-        raise e
-      end
-    end
-
     def apply_state(state)
       @vm.update(:apply_spec => state)
       agent.apply(state)
@@ -225,6 +166,7 @@ module Bosh::Director
 
     def delete_disk(disk, vm_cid)
       disk_cid = disk.disk_cid
+
       # Unmount the disk only if disk is known by the agent
       if agent && disk_info.include?(disk_cid)
         agent.unmount_disk(disk_cid)
@@ -235,7 +177,7 @@ module Bosh::Director
       rescue Bosh::Clouds::DiskNotAttached
         if disk.active
           raise CloudDiskNotAttached,
-                "`#{instance_name}' VM should have persistent disk attached " +
+                "`#{@instance}' VM should have persistent disk attached " +
                     "but it doesn't (according to CPI)"
         end
       end
@@ -266,57 +208,12 @@ module Bosh::Director
       end
     end
 
-    def update_resource_pool(new_disk_cid = nil)
-      return unless @instance.resource_pool_changed? || new_disk_cid
-
-      detach_disk
-      num_retries = 0
-      begin
-        delete_vm
-        create_vm(new_disk_cid)
-        attach_disk
-      rescue Bosh::Clouds::NoDiskSpace => e
-        if e.ok_to_retry && num_retries < MAX_ATTACH_DISK_TRIES
-          num_retries += 1
-          @logger.warn("Retrying attach disk operation #{num_retries}")
-          retry
-        end
-        @logger.warn("Giving up on attach disk operation")
-        e.ok_to_retry = false
-        raise CloudNotEnoughDiskSpace,
-              "Not enough disk space to update `#{instance_name}'"
-      end
-
-      state = {
-          "deployment" => @deployment_plan.name,
-          "networks" => @instance.network_settings,
-          "resource_pool" => @job.resource_pool.spec,
-          "job" => @job.spec,
-          "index" => @instance.index,
-      }
-
-      if @instance.disk_size > 0
-        state["persistent_disk"] = @instance.disk_size
-      end
-
-      # if we have a failure above the new VM doesn't get any state,
-      # which makes it impossible to recreate it
-      apply_state(state)
-      @instance.current_state = agent.get_state
-    end
-
-    def attach_missing_disk
-      if @instance.model.persistent_disk_cid &&
-          !@instance.disk_currently_attached?
-        attach_disk
-      end
-    rescue Bosh::Clouds::NoDiskSpace => e
-      update_resource_pool(@instance.model.persistent_disk_cid)
+    def update_resource_pool(new_disk_cid)
+      @vm, @agent = vm_updater.update(new_disk_cid)
     end
 
     # Synchronizes persistent_disks with the agent.
-    #
-    # NOTE: Currently assumes that we only have 1 persistent disk.
+    # (Currently assumes that we only have 1 persistent disk.)
     # @return [void]
     def check_persistent_disk
       return if @instance.model.persistent_disks.empty?
@@ -324,23 +221,20 @@ module Bosh::Director
 
       if agent_disk_cid != @instance.model.persistent_disk_cid
         raise AgentDiskOutOfSync,
-              "`#{instance_name}' has invalid disks: agent reports " +
+              "`#{@instance}' has invalid disks: agent reports " +
                   "`#{agent_disk_cid}' while director record shows " +
                   "`#{@instance.model.persistent_disk_cid}'"
       end
 
       @instance.model.persistent_disks.each do |disk|
         unless disk.active
-          @logger.warn("`#{instance_name}' has inactive disk #{disk.disk_cid}")
+          @logger.warn("`#{@instance}' has inactive disk #{disk.disk_cid}")
         end
       end
     end
 
     def update_persistent_disk
-      # CLEANUP FIXME
-      # [olegs] Error cleanup should be performed AFTER logic cleanup, I can't
-      # event comprehend this method.
-      attach_missing_disk
+      vm_updater.attach_missing_disk
       check_persistent_disk
 
       disk_cid = nil
@@ -399,22 +293,8 @@ module Bosh::Director
     end
 
     def update_networks
-      @network_updater.update
-    end
-
-    def agent
-      if @agent && @agent.id == @vm.agent_id
-        @agent
-      else
-        if @vm.agent_id.nil?
-          raise VmAgentIdMissing, "VM #{@vm.id} is missing agent id"
-        end
-        @agent = AgentClient.with_defaults(@vm.agent_id)
-      end
-    end
-
-    def generate_agent_id
-      SecureRandom.uuid
+      network_updater = NetworkUpdater.new(@instance, @vm, agent, vm_updater, @cloud, @logger)
+      @vm, @agent = network_updater.update
     end
 
     # Returns an array of wait times distributed
@@ -446,5 +326,13 @@ module Bosh::Director
     def canary?
       @canary
     end
+
+    attr_reader :agent
+
+    def vm_updater
+      # Do not memoize to avoid caching same VM and agent
+      # which could be replaced after updating a VM
+      VmUpdater.new(@instance, @vm, agent, @job_renderer, @cloud, 3, @logger)
+    end
   end
 end