bosh-director 1.3232.24.0 → 1.3262.0

data/lib/bosh/director/jobs/delete_deployment.rb

@@ -23,6 +23,8 @@ module Bosh::Director
         with_deployment_lock(@deployment_name) do
           deployment_model = @deployment_manager.find_by_name(@deployment_name)
 
+          fail_if_ignored_instances_found(deployment_model)
+
           ip_provider = DeploymentPlan::IpProviderFactory.new(true, logger)
 
           dns_manager = DnsManagerProvider.create
@@ -30,7 +32,7 @@ module Bosh::Director
           instance_deleter = InstanceDeleter.new(ip_provider, dns_manager, disk_manager, force: @force)
           deployment_deleter = DeploymentDeleter.new(Config.event_log, logger, dns_manager, Config.max_threads)
 
-          vm_deleter = Bosh::Director::VmDeleter.new(@cloud, logger, force: @force)
+          vm_deleter = Bosh::Director::VmDeleter.new(@cloud, logger, @force, Config.enable_virtual_delete_vms)
           deployment_deleter.delete(deployment_model, instance_deleter, vm_deleter)
           add_event(parent_id)
 
@@ -57,6 +59,16 @@ module Bosh::Director
             })
         event.id
       end
+
+      def fail_if_ignored_instances_found(deployment_model)
+        deployment_model.instances.each do |instance_model|
+          if instance_model.ignore
+            raise DeploymentIgnoredInstancesDeletion, "You are trying to delete deployment '#{deployment_model.name}', which " +
+                'contains ignored instance(s). Operation not allowed.'
+          end
+        end
+      end
+
     end
   end
 end

data/lib/bosh/director/jobs/delete_vm.rb

@@ -0,0 +1,58 @@
+module Bosh::Director
+  module Jobs
+    class DeleteVm < BaseJob
+
+      @queue = :normal
+
+      def self.job_type
+        :delete_vm
+      end
+
+      def initialize(vm_cid)
+        @vm_cid = vm_cid
+        @cloud = Config.cloud
+        @vm_deleter = Bosh::Director::VmDeleter.new(@cloud, logger, false, false)
+        @deployment_name = nil
+        @instance_name = nil
+      end
+
+      def perform
+        logger.info("deleting vm: #{@vm_cid}")
+        begin
+          instance = Bosh::Director::Api::InstanceLookup.new.by_filter(vm_cid: @vm_cid).first
+          @deployment_name = instance.deployment.name
+          @instance_name = instance.name
+          parent_id = add_event
+          @vm_deleter.delete_for_instance(instance, false)
+        rescue InstanceNotFound
+          parent_id = add_event
+          @vm_deleter.delete_vm(@vm_cid)
+        end
+      rescue Bosh::Clouds::VMNotFound
+        logger.info("vm #{@vm_cid} does not exists")
+      rescue Exception => e
+        raise e
+      ensure
+        add_event(parent_id, e)
+        return "vm #{@vm_cid} deleted" unless e
+      end
+
+      private
+      def add_event(parent_id = nil, error = nil)
+        event = Config.current_job.event_manager.create_event(
+            {
+                parent_id: parent_id,
+                user: Config.current_job.username,
+                action: 'delete',
+                object_type: 'vm',
+                object_name: @vm_cid,
+                task: Config.current_job.task_id,
+                deployment: @deployment_name,
+                instance: @instance_name,
+                error: error
+            })
+        event.id
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/export_release.rb

@@ -54,7 +54,7 @@ module Bosh::Director
         release = planner.release(@release_name)
 
         export_release_job = create_job_with_all_the_templates_so_everything_compiles(release_version_model, release, planner, deployment_plan_stemcell)
-        planner.add_job(export_release_job)
+        planner.add_instance_group(export_release_job)
         planner.bind_models(true)
 
         lock_timeout = 15 * 60 # 15 minutes
@@ -105,16 +105,23 @@ module Bosh::Director
 
         oid = blobstore_client.create(tarball_file)
 
+        tarball_hexdigest = Digest::SHA1.file(output_path).hexdigest
+
+        Bosh::Director::Models::EphemeralBlob.new(
+            blobstore_id: oid,
+            sha1: tarball_hexdigest
+        ).save
+
         {
-          :blobstore_id => oid,
-          :sha1 => Digest::SHA1.file(output_path).hexdigest,
+            :blobstore_id => oid,
+            :sha1 => tarball_hexdigest,
         }
       ensure
         compiled_release_downloader.cleanup unless compiled_release_downloader.nil?
       end
 
       def create_job_with_all_the_templates_so_everything_compiles(release_version_model, release, planner, deployment_plan_stemcell)
-        job = DeploymentPlan::Job.new(logger)
+        job = DeploymentPlan::InstanceGroup.new(logger)
 
         job.name = 'dummy-job-for-compilation'
         job.stemcell = deployment_plan_stemcell

data/lib/bosh/director/jobs/helpers.rb

@@ -8,3 +8,5 @@ require 'bosh/director/jobs/helpers/template_deleter'
 require 'bosh/director/jobs/helpers/release_version_deleter'
 require 'bosh/director/jobs/helpers/release_deleter'
 require 'bosh/director/jobs/helpers/name_version_release_deleter'
+require 'bosh/director/jobs/helpers/config_parser'
+require 'bosh/director/jobs/helpers/deep_hash_replacement'

data/lib/bosh/director/jobs/helpers/config_parser.rb

@@ -0,0 +1,66 @@
+require 'net/http'
+
+module Bosh::Director::Jobs
+  module Helpers
+    class ConfigParser
+
+      class << self
+        # Search and Replace at a config placeholders in manifest
+        def parse(manifest)
+          new_manifest = Bosh::Common::DeepCopy.copy(manifest)
+
+          config_map = DeepHashReplacement.replacement_map(new_manifest)
+          parsed_config = apply_replacements(new_manifest, config_map)
+
+          parsed_config
+        end
+
+        private
+
+        def apply_replacements(manifest, config_map)
+          config_keys = config_map.map { |c| c["key"] }.uniq
+
+          config_values, invalid_keys = fetch_config_values(config_keys)
+
+          if invalid_keys.length > 0
+            raise "Failed to find keys in the config server: " + invalid_keys.join(", ")
+          end
+
+          update_manifest!(manifest, config_map, config_values)
+
+          manifest
+        end
+
+        def fetch_config_values(keys)
+          invalid_keys = []
+          config_values = {}
+
+          keys.each do |k|
+            config_server_url = URI.join(Bosh::Director::Config.config_server_url, 'v1/', 'config/', k)
+            response = Net::HTTP.get_response(config_server_url)
+            
+            if response.kind_of? Net::HTTPSuccess
+              config_values[k] = JSON.parse(response.body)['value']
+            else
+              invalid_keys << k
+            end
+          end
+
+          [config_values, invalid_keys]
+        end
+
+        def update_manifest!(manifest, config_map, config_values)
+          config_map.each do |config_loc|
+            config_path = config_loc['path']
+            ret = config_path[0..config_path.length-2].inject(manifest) do |obj, el|
+              obj[el]
+            end
+
+            ret[config_path.last] = config_values[config_loc['key']]
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/bosh/director/jobs/helpers/deep_hash_replacement.rb

@@ -0,0 +1,38 @@
+require 'net/http'
+
+module Bosh::Director::Jobs
+  module Helpers
+    class DeepHashReplacement
+      def self.replacement_map(obj)
+        result = []
+        create_replacement_map(result, obj)
+
+        result
+      end
+
+      private
+
+      def self.create_replacement_map(result, obj, path = nil)
+        if obj.is_a? Array
+          obj.each_with_index do |item, index|
+            new_path = path.nil? ? [] : Bosh::Common::DeepCopy.copy(path)
+            new_path << index
+            create_replacement_map(result, item, new_path)
+          end
+        elsif obj.is_a? Hash
+          obj.each do |key, value|
+            new_path = path.nil? ? [] : Bosh::Common::DeepCopy.copy(path)
+            new_path << key
+            create_replacement_map(result, value, new_path)
+          end
+        else
+          path ||= []
+          if obj.to_s.match(/^\(\(.*\)\)$/)
+            key_name = obj.gsub(/(^\(\(|\)\)$)/, '')
+            result << {'key' => key_name, 'path' => path}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/run_errand.rb

@@ -34,7 +34,7 @@ module Bosh::Director
           planner_factory = DeploymentPlan::PlannerFactory.create(logger)
           deployment = planner_factory.create_from_manifest(deployment_manifest, deployment_model.cloud_config, deployment_model.runtime_config, {})
           deployment.bind_models
-          job = deployment.job(@errand_name)
+          job = deployment.instance_group(@errand_name)
 
           if job.nil?
             raise JobNotFound, "Errand '#{@errand_name}' doesn't exist"

data/lib/bosh/director/jobs/ssh.rb

@@ -3,7 +3,8 @@ module Bosh::Director
     class Ssh < BaseJob
       DEFAULT_SSH_DATA_LIFETIME = 300
       SSH_TAG = "ssh"
-      @queue = :normal
+
+      @queue = :urgent
 
       def self.job_type
         :ssh
@@ -53,7 +54,7 @@ module Bosh::Director
           end
         end
 
-        result_file.write(Yajl::Encoder.encode(ssh_info))
+        result_file.write(JSON.generate(ssh_info))
         result_file.write("\n")
 
         # task result

data/lib/bosh/director/jobs/update_deployment.rb

@@ -2,8 +2,11 @@ module Bosh::Director
   module Jobs
     class UpdateDeployment < BaseJob
       include LockHelper
+      include LegacyDeploymentHelper
 
       @queue = :normal
+      @local_fs = true
+
 
       def self.job_type
         :update_deployment
@@ -15,18 +18,32 @@ module Bosh::Director
         @cloud_config_id = cloud_config_id
         @runtime_config_id = runtime_config_id
         @options = options
+        @event_log = Config.event_log
       end
 
       def perform
         logger.info('Reading deployment manifest')
 
         manifest_text = File.read(@manifest_file_path)
+        manifest_hash = Psych.load(manifest_text)
         logger.debug("Manifest:\n#{manifest_text}")
-        cloud_config_model = Bosh::Director::Models::CloudConfig[@cloud_config_id]
-        if cloud_config_model.nil?
-          logger.debug("No cloud config uploaded yet.")
+
+        if Config.parse_config_values
+          manifest_hash = Bosh::Director::Jobs::Helpers::ConfigParser.parse(manifest_hash)
+        end
+
+        if ignore_cloud_config?(manifest_hash)
+          warning = "Ignoring cloud config. Manifest contains 'networks' section."
+          logger.debug(warning)
+          @event_log.warn_deprecated(warning)
+          cloud_config_model = nil
         else
-          logger.debug("Cloud config:\n#{cloud_config_model.manifest}")
+          cloud_config_model = Bosh::Director::Models::CloudConfig[@cloud_config_id]
+          if cloud_config_model.nil?
+            logger.debug("No cloud config uploaded yet.")
+          else
+            logger.debug("Cloud config:\n#{cloud_config_model.manifest}")
+          end
         end
 
         runtime_config_model = Bosh::Director::Models::RuntimeConfig[@runtime_config_id]
@@ -36,8 +53,12 @@ module Bosh::Director
           logger.debug("Runtime config:\n#{runtime_config_model.manifest}")
         end
 
-        deployment_manifest = Manifest.load_from_text(manifest_text, cloud_config_model, runtime_config_model)
+        deployment_manifest = Manifest.load_from_hash(manifest_hash, cloud_config_model, runtime_config_model)
+
         @deployment_name = deployment_manifest.to_hash['name']
+
+        previous_releases, previous_stemcells = get_stemcells_and_releases
+        context = {}
         parent_id = add_event
 
         with_deployment_lock(@deployment_name) do
@@ -46,13 +67,20 @@ module Bosh::Director
 
           deployment_plan = nil
 
-          event_log_stage = Config.event_log.begin_stage('Preparing deployment', 1)
+          event_log_stage = @event_log.begin_stage('Preparing deployment', 1)
           event_log_stage.advance_and_track('Preparing deployment') do
             planner_factory = DeploymentPlan::PlannerFactory.create(logger)
             deployment_plan = planner_factory.create_from_manifest(deployment_manifest, cloud_config_model, runtime_config_model, @options)
             deployment_plan.bind_models
           end
 
+          if deployment_plan.instance_models.any?(&:ignore)
+            @event_log.warn('You have ignored instances. They will not be changed.')
+          end
+
+          next_releases, next_stemcells  = get_stemcells_and_releases
+          context = event_context(next_releases, previous_releases, next_stemcells, previous_stemcells)
+
           render_job_templates(deployment_plan.jobs_starting_on_deploy)
           deployment_plan.compile_packages
 
@@ -64,7 +92,7 @@ module Bosh::Director
 
           @notifier.send_end_event
           logger.info('Finished updating deployment')
-          add_event(parent_id)
+          add_event(context, parent_id)
 
           "/deployments/#{deployment_plan.name}"
         end
@@ -74,7 +102,7 @@ module Bosh::Director
         rescue Exception => e2
           # log the second error
         ensure
-          add_event(parent_id, e)
+          add_event(context, parent_id, e)
           raise e
         end
       ensure
@@ -83,8 +111,8 @@ module Bosh::Director
 
       private
 
-      def add_event(parent_id = nil, error = nil)
-        action = deployment_new? ? "create" : "update"
+      def add_event(context = {}, parent_id = nil, error = nil)
+        action = @options.fetch('new', false) ? "create" : "update"
         event  = event_manager.create_event(
             {
                 parent_id:   parent_id,
@@ -94,7 +122,8 @@ module Bosh::Director
                 object_name: @deployment_name,
                 deployment:  @deployment_name,
                 task:        task_id,
-                error:       error
+                error:       error,
+                context:     context
             })
         event.id
       end
@@ -102,7 +131,7 @@ module Bosh::Director
       # Job tasks
 
       def check_for_changes(deployment_plan)
-        deployment_plan.jobs.each do |job|
+        deployment_plan.instance_groups.each do |job|
           return true if job.did_change
         end
         false
@@ -147,8 +176,38 @@ module Bosh::Director
         end
       end
 
-      def deployment_new?
-        @deployment_new ||= Models::Deployment.exclude(manifest: nil)[name: @deployment_name].nil?
+      def get_stemcells_and_releases
+        deployment = current_deployment
+        stemcells = []
+        releases = []
+        if deployment
+          releases = deployment.release_versions.map do |rv|
+            "#{rv.release.name}/#{rv.version}"
+          end
+          stemcells = deployment.stemcells.map do |sc|
+            "#{sc.name}/#{sc.version}"
+          end
+        end
+        return releases, stemcells
+      end
+
+      def current_deployment
+        Models::Deployment[name: @deployment_name]
+      end
+
+      def event_context(next_releases, previous_releases, next_stemcells, previous_stemcells)
+        after_objects = {}
+        after_objects['releases'] = next_releases unless next_releases.empty?
+        after_objects['stemcells'] = next_stemcells unless next_stemcells.empty?
+
+        before_objects = {}
+        before_objects['releases'] = previous_releases unless previous_releases.empty?
+        before_objects['stemcells'] = previous_stemcells unless previous_stemcells.empty?
+
+        context = {}
+        context['before'] = before_objects
+        context['after'] = after_objects
+        context
       end
     end
   end

data/lib/bosh/director/jobs/update_release.rb

@@ -8,6 +8,8 @@ module Bosh::Director
       include DownloadHelper
 
       @queue = :normal
+      @local_fs = true
+
       @compiled_release = false
 
       attr_accessor :release_model
@@ -98,7 +100,7 @@ module Bosh::Director
 
         begin
           @version = Bosh::Common::Version::ReleaseVersion.parse(@manifest["version"])
-          logger.info("Formatted version '#{@manifest["version"]}' => '#{@version}'") unless @version == @manifest["version"]
+          logger.info("Formatted version '#{@manifest["version"]}' => '#{@version}'") unless @version.to_s == @manifest["version"]
         rescue SemiSemantic::ParseError
           raise ReleaseVersionInvalid, "Release version invalid: #{@manifest["version"]}"
         end