bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/jobs/scheduled_orphan_cleanup.rb

@@ -0,0 +1,46 @@
+module Bosh::Director
+  module Jobs
+    class ScheduledOrphanCleanup < BaseJob
+      @queue = :normal
+
+      def self.job_type
+        :scheduled_orphan_cleanup
+      end
+
+      def self.has_work(params)
+        time = time_days_ago(params.first['max_orphaned_age_in_days'])
+        Models::OrphanDisk.where('created_at < ?', time).any?
+      end
+
+      def self.time_days_ago(days)
+        Time.now - (days * 24 * 60 * 60)
+      end
+
+      def self.schedule_message
+        "clean up orphan disks"
+      end
+
+      def initialize(params = {})
+        logger.debug("ScheduledOrphanCleanup initialized with params: #{params.inspect}")
+        @max_orphaned_age_in_days = params['max_orphaned_age_in_days']
+        cloud = params.fetch(:cloud) { Config.cloud }
+        @disk_manager = DiskManager.new(cloud, logger)
+      end
+
+      def perform
+        time = self.class.time_days_ago(@max_orphaned_age_in_days)
+        logger.info("Started cleanup of orphan disks and orphan snapshots older than #{time}")
+
+        old_orphans = Models::OrphanDisk.where('created_at < ?', time)
+        old_orphans_count = old_orphans.count
+        stage = event_log.begin_stage('Deleting orphan disks', old_orphans_count)
+        old_orphans.each do |old_orphan|
+          stage.advance_and_track("#{old_orphan.disk_cid}") do
+            @disk_manager.delete_orphan_disk(old_orphan)
+          end
+        end
+        "Deleted #{old_orphans_count} orphaned disk(s) older than #{time}"
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/ssh.rb

@@ -1,5 +1,3 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Bosh::Director
   module Jobs
     class Ssh < BaseJob
@@ -13,7 +11,7 @@ module Bosh::Director
 
       def initialize(deployment_id, options = {})
         @deployment_id = deployment_id
-        @target = options["target"]
+        @target_payload = options["target"]
         @command = options["command"]
         @params = options["params"]
         @blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
@@ -21,29 +19,25 @@ module Bosh::Director
       end
 
       def perform
-        job = @target["job"]
-        indexes = @target["indexes"]
-
-        filter = {
-          :deployment_id => @deployment_id
-        }
+        target = Target.new(@target_payload)
 
-        if indexes && indexes.size > 0
-          filter[:index] = indexes
-        end
+        filter = { deployment_id: @deployment_id }
+        filter[:job] = target.job if target.job
 
-        if job
-          filter[:job] = job
-        end
+        filter.merge!(target.id_filter)
 
         instances = @instance_manager.filter_by(filter)
 
         ssh_info = instances.map do |instance|
           agent = @instance_manager.agent_client_for(instance)
 
-          logger.info("ssh #{@command} `#{instance.job}/#{instance.index}'")
+          logger.info("ssh #{@command} `#{instance.job}/#{instance.uuid}'")
           result = agent.ssh(@command, @params)
-          result["index"] = instance.index
+          if target.ids_provided?
+            result["id"] = instance.uuid
+          else
+            result["index"] = instance.index
+          end
 
           if Config.default_ssh_options
             result["gateway_host"] = Config.default_ssh_options["gateway_host"]
@@ -59,6 +53,45 @@ module Bosh::Director
         # task result
         nil
       end
+
+      private
+
+      class Target
+        attr_reader :job, :indexes, :ids
+
+        def initialize(target_payload)
+          @job = target_payload['job']
+          @ids = target_payload['ids']
+          @indexes = target_payload['indexes']
+        end
+
+        def ids_provided?
+          @ids && @ids.size > 0
+        end
+
+        def indexes_provided?
+          @indexes && @indexes.size > 0
+        end
+
+        def id_filter
+          if !ids_provided? && indexes_provided?
+            # for backwards compatibility with old cli
+            return {index: @indexes}
+          end
+
+          filter = Hash.new { |h,k| h[k] = [] }
+
+          @ids.each do |id|
+            if id.to_s =~ /^\d+$/
+              filter[:index] << id.to_i
+            else
+              filter[:uuid] << id
+            end
+          end
+
+          filter
+        end
+      end
     end
   end
 end

data/lib/bosh/director/jobs/update_deployment.rb

@@ -18,18 +18,35 @@ module Bosh::Director
 
       def perform
         logger.info('Reading deployment manifest')
+
         manifest_text = File.read(@manifest_file_path)
         logger.debug("Manifest:\n#{manifest_text}")
         deployment_manifest_hash = Psych.load(manifest_text)
+
+        cloud_config_model = Bosh::Director::Models::CloudConfig[@cloud_config_id]
+        if cloud_config_model.nil?
+          logger.debug("No cloud config uploaded yet.")
+        else
+          logger.debug("Cloud config:\n#{cloud_config_model.manifest}")
+        end
+
         deployment_name = deployment_manifest_hash['name']
         with_deployment_lock(deployment_name) do
           @notifier = DeploymentPlan::Notifier.new(deployment_name, Config.nats_rpc, logger)
           @notifier.send_start_event
 
-          cloud_config_model = Bosh::Director::Models::CloudConfig[@cloud_config_id]
-          planner_factory = DeploymentPlan::PlannerFactory.create(event_log, logger)
-          deployment_plan = planner_factory.planner(deployment_manifest_hash, cloud_config_model, @options)
+          deployment_plan = nil
+
+          event_log.begin_stage('Preparing deployment', 1)
+          event_log.track('Preparing deployment') do
+            planner_factory = DeploymentPlan::PlannerFactory.create(logger)
+            deployment_plan = planner_factory.create_from_manifest(deployment_manifest_hash, cloud_config_model, @options)
+            deployment_plan.bind_models
+          end
 
+          deployment_plan.compile_packages
+
+          render_job_templates(deployment_plan.jobs_starting_on_deploy)
           update_step(deployment_plan).perform
           @notifier.send_end_event
           logger.info('Finished updating deployment')
@@ -53,18 +70,12 @@ module Bosh::Director
       # Job tasks
 
       def update_step(deployment_plan)
-        resource_pool_updaters = deployment_plan.resource_pools.map do |resource_pool|
-          ResourcePoolUpdater.new(resource_pool)
-        end
-        resource_pools = DeploymentPlan::ResourcePools.new(event_log, resource_pool_updaters)
         DeploymentPlan::Steps::UpdateStep.new(
           self,
           event_log,
-          resource_pools,
           deployment_plan,
           multi_job_updater,
-          Config.cloud,
-          App.instance.blobstores.blobstore
+          Config.cloud
         )
       end
 
@@ -72,7 +83,14 @@ module Bosh::Director
 
       def multi_job_updater
         @multi_job_updater ||= begin
-          DeploymentPlan::BatchMultiJobUpdater.new(JobUpdaterFactory.new(@blobstore))
+          DeploymentPlan::BatchMultiJobUpdater.new(JobUpdaterFactory.new(Config.cloud, logger))
+        end
+      end
+
+      def render_job_templates(jobs)
+        job_renderer = JobRenderer.create
+        jobs.each do |job|
+          job_renderer.render_job_instances(job.needed_instance_plans)
         end
       end
     end

data/lib/bosh/director/jobs/update_release.rb

@@ -383,6 +383,7 @@ module Bosh::Director
         all_compiled_packages.each do |compiled_package_spec|
           package = compiled_package_spec[:package]
           stemcell = compiled_package_spec[:stemcell]
+          compiled_pkg_tgz = File.join(release_dir, 'compiled_packages', "#{package.name}.tgz")
 
           existing_compiled_package = Models::CompiledPackage.where(:package_id => package.id, :stemcell_id => stemcell.id)
           if existing_compiled_package.empty?
@@ -390,9 +391,14 @@ module Bosh::Director
             package_desc = "#{package.name}/#{package.version} for #{stemcell.name}/#{stemcell.version}"
             event_log.track(package_desc) do
               other_compiled_package = get_other_compiled_package(package, packages_existing_from_other_releases, stemcell)
+              if @fix && !other_compiled_package.nil?
+                fix_compiled_package(other_compiled_package, compiled_pkg_tgz)
+              end
               create_compiled_package(package, stemcell, release_dir, other_compiled_package)
               had_effect = true
             end
+          elsif @fix
+            fix_compiled_package(existing_compiled_package.first, compiled_pkg_tgz)
           end
         end
 
@@ -437,7 +443,6 @@ module Bosh::Director
         if other_compiled_package.nil?
           tgz = File.join(release_dir, 'compiled_packages', "#{package.name}.tgz")
           validate_tgz(tgz, "#{package.name}.tgz")
-
           blobstore_id = BlobUtil.create_blob(tgz)
           sha1 = Digest::SHA1.file(tgz).hexdigest
         else
@@ -493,7 +498,7 @@ module Bosh::Director
           package.sha1 = package_meta['sha1']
 
           if package.blobstore_id != nil
-            fix_compiled_packages package
+            delete_compiled_packages package
             validate_tgz(package_tgz, desc)
             fix_package(package, package_tgz)
             return true
@@ -501,7 +506,7 @@ module Bosh::Director
 
           if existing_blob
             pkg = Models::Package.where(blobstore_id: existing_blob).first
-            fix_compiled_packages package
+            delete_compiled_packages package
             fix_package(pkg, package_tgz)
             package.blobstore_id = BlobUtil.copy_blob(pkg.blobstore_id)
             return true
@@ -647,9 +652,10 @@ module Bosh::Director
         package.blobstore_id = BlobUtil.create_blob(package_tgz)
         logger.info("Re-created package '#{package.name}/#{package.version}' \
 with blobstore_id '#{package.blobstore_id}'")
+        package.save
       end
 
-      def fix_compiled_packages(package)
+      def delete_compiled_packages(package)
         package.compiled_packages.each do |compiled_pkg|
           unless BlobUtil.verify_blob(compiled_pkg.blobstore_id, compiled_pkg.sha1)
             logger.info("Deleting compiled package '#{compiled_pkg.name}' \
@@ -665,6 +671,21 @@ for '#{compiled_pkg.stemcell.name}/#{compiled_pkg.stemcell.version}' with blobst
           end
         end
       end
+
+      def fix_compiled_package(compiled_pkg, compiled_pkg_tgz)
+        begin
+          logger.info("Deleting compiled package '#{compiled_pkg.name}/#{compiled_pkg.version}' \
+for '#{compiled_pkg.stemcell.name}/#{compiled_pkg.stemcell.version}' with blobstore_id '#{compiled_pkg.blobstore_id}'")
+          BlobUtil.delete_blob compiled_pkg.blobstore_id
+        rescue Bosh::Blobstore::BlobstoreError => e
+          logger.info("Error deleting compiled package '#{compiled_pkg.name}' \
+with blobstore_id '#{compiled_pkg.blobstore_id}' #{e.inspect}")
+        end
+        compiled_pkg.blobstore_id = BlobUtil.create_blob(compiled_pkg_tgz)
+        logger.info("Re-created compiled package '#{compiled_pkg.name}/#{compiled_pkg.version}' \
+with blobstore_id '#{compiled_pkg.blobstore_id}'")
+        compiled_pkg.save
+      end
     end
   end
 end

data/lib/bosh/director/jobs/vm_state.rb

@@ -1,5 +1,3 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Bosh::Director
   module Jobs
     class VmState < BaseJob
@@ -11,15 +9,12 @@ module Bosh::Director
         :vms
       end
 
-      # @param [Integer] deployment_id Deployment id
       def initialize(deployment_id, format)
         @deployment_id = deployment_id
         @format = format
       end
 
       def perform
-        @domain = Models::Dns::Domain.find(name: Config.dns_domain_name, type: "NATIVE") if Config.dns_enabled?
-
         vms = Models::Vm.filter(:deployment_id => @deployment_id)
         ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
           vms.each do |vm|
@@ -37,62 +32,58 @@ module Bosh::Director
       def process_vm(vm)
         ips = []
         dns_records = []
-        job_name = nil
         job_state = nil
-        resource_pool = nil
         job_vitals = nil
-        job_index = nil
         processes = []
 
         begin
           agent = AgentClient.with_vm(vm, :timeout => TIMEOUT)
           agent_state = agent.get_state(@format)
-          agent_state["networks"].each_value do |network|
-            ips << network["ip"]
+          agent_state['networks'].each_value do |network|
+            ips << network['ip']
           end
 
-          job_index = get_index(agent_state)
-
-          job_name = agent_state["job"]["name"] if agent_state["job"]
-          job_state = agent_state["job_state"]
-          if agent_state["resource_pool"]
-            resource_pool = agent_state["resource_pool"]["name"]
+          job_state = agent_state['job_state']
+          if agent_state['vitals']
+            job_vitals = agent_state['vitals']
           end
-          if agent_state["vitals"]
-            job_vitals = agent_state["vitals"]
-          end
-          processes = agent_state["processes"] if agent_state["processes"]
+          processes = agent_state['processes'] if agent_state['processes']
         rescue Bosh::Director::RpcTimeout
-          job_state = "unresponsive agent"
+          job_state = 'unresponsive agent'
         end
 
-        if @domain
-          ips.each do |ip|
-            records = Models::Dns::Record.filter(domain_id: @domain.id, type: "A", content: ip)
-            dns_records << records.collect { |record| record.name } unless records.empty?
-          end
+        if dns_manager.dns_enabled?
+          dns_records = dns_manager.find_dns_record_names_by_instance(vm.instance)
+          dns_records.sort_by! { |name| -(name.split('.').first.length) }
         end
 
+        vm_apply_spec = vm.instance ? vm.instance.spec : {}
+        vm_type_name = vm_apply_spec && vm_apply_spec['vm_type'] ? vm_apply_spec['vm_type']['name'] : nil
+
         {
           :vm_cid => vm.cid,
           :disk_cid => vm.instance ? vm.instance.persistent_disk_cid : nil,
           :ips => ips,
-          :dns => dns_records.flatten,
+          :dns => dns_records,
           :agent_id => vm.agent_id,
-          :job_name => job_name,
-          :index => job_index,
+          :job_name => vm.instance ? vm.instance.job : nil,
+          :index => vm.instance ? vm.instance.index : nil,
           :job_state => job_state,
-          :resource_pool => resource_pool,
+          :resource_pool => vm_type_name,
+          :vm_type => vm_type_name,
           :vitals => job_vitals,
           :processes => processes,
-          :resurrection_paused => vm.instance ? vm.instance.resurrection_paused  : nil,
+          :resurrection_paused => vm.instance ? vm.instance.resurrection_paused : nil,
+          :az => vm.instance ? vm.instance.availability_zone : nil,
+          :id => vm.instance ? vm.instance.uuid : nil,
+          :bootstrap => vm.instance ? vm.instance.bootstrap : false
         }
       end
 
       private
 
       def get_index(agent_state)
-        index = agent_state["index"]
+        index = agent_state['index']
 
         # Postgres cannot coerce an empty string to integer, and fails on Models::Instance.find
         index = nil if index.is_a?(String) && index.empty?

data/lib/bosh/director/lock.rb

@@ -93,7 +93,7 @@ module Bosh::Director
           end
         end
 
-        raise TimeoutError if Time.now - started > @timeout
+        raise TimeoutError, "Redis lock #{@name} is acquired by another thread" if Time.now - started > @timeout
 
         sleep(0.5)
 
@@ -110,15 +110,20 @@ module Bosh::Director
 
       redis.watch(@name)
       existing_lock = redis.get(@name)
-      lock_id = existing_lock.split(":")[1]
-      if lock_id == @id
-        redis.multi do
-          redis.del(@name)
-        end
-      else
+      if existing_lock.nil?
+        @logger.debug("Lost lock #@name")
         redis.unwatch
+      else
+        lock_id = existing_lock.split(":")[1]
+        if lock_id == @id
+          redis.multi do
+            redis.del(@name)
+          end
+        else
+          redis.unwatch
+        end
+        @logger.debug("Deleted lock: #@name")
       end
-      @logger.debug("Deleted lock: #@name")
     end
 
     def lock_expired?(lock)

data/lib/bosh/director/logs_fetcher.rb

@@ -21,7 +21,7 @@ module Bosh::Director
       agent = @instance_manager.agent_client_for(instance)
       blobstore_id = nil
 
-      stage = @event_log.begin_stage("Fetching logs for #{instance.job}/#{instance.index}", 1)
+      stage = @event_log.begin_stage("Fetching logs for #{instance.job}/#{instance.uuid} (#{instance.index})", 1)
       stage.advance_and_track('Finding and packing log files') do
         fetch_logs_result = agent.fetch_logs(log_type, filters)
         blobstore_id = fetch_logs_result['blobstore_id']