bosh-director 1.3232.24.0 → 1.3262.0

data/lib/bosh/director/deployment_plan/instance.rb

@@ -207,11 +207,11 @@ module Bosh::Director
       end
 
       def current_job_spec
-        @model.spec['job']
+        @model.spec_p('job')
       end
 
       def current_packages
-        @model.spec['packages']
+        @model.spec_p('packages')
       end
 
       def current_job_state

data/lib/bosh/director/deployment_plan/{job.rb → instance_group.rb}

@@ -1,9 +1,9 @@
-require 'bosh/director/deployment_plan/job_spec_parser'
+require 'bosh/director/deployment_plan/instance_group_spec_parser'
 require 'bosh/template/property_helper'
 
 module Bosh::Director
   module DeploymentPlan
-    class Job
+    class InstanceGroup
       include Bosh::Template::PropertyHelper
 
       VALID_LIFECYCLE_PROFILES = %w(service errand)
@@ -85,9 +85,9 @@ module Bosh::Director
 
       attr_accessor :did_change
 
-      def self.parse(plan, job_spec, event_log, logger)
-        parser = JobSpecParser.new(plan, event_log, logger)
-        parser.parse(job_spec)
+      def self.parse(plan, job_spec, event_log, logger, parse_options = {})
+        parser = InstanceGroupSpecParser.new(plan, event_log, logger)
+        parser.parse(job_spec, parse_options)
       end
 
       def initialize(logger)

data/lib/bosh/director/deployment_plan/{job_spec_parser.rb → instance_group_spec_parser.rb}

@@ -2,7 +2,7 @@ require 'bosh/template/property_helper'
 
 module Bosh::Director
   module DeploymentPlan
-    class JobSpecParser
+    class InstanceGroupSpecParser
       include ValidationHelper
       include Bosh::Template::PropertyHelper
       include IpUtil
@@ -16,9 +16,9 @@ module Bosh::Director
 
       # @param [Hash] job_spec Raw job spec from the deployment manifest
       # @return [DeploymentPlan::Job] Job as build from job_spec
-      def parse(job_spec)
+      def parse(job_spec, options = {})
         @job_spec = job_spec
-        @job = Job.new(@logger)
+        @job = InstanceGroup.new(@logger)
 
         parse_name
         parse_lifecycle
@@ -35,7 +35,10 @@ module Bosh::Director
         parse_resource_pool
         check_remove_dev_tools
 
-        parse_update_config
+        parse_options = {}
+        parse_options['canaries'] = options['canaries'] if options['canaries']
+        parse_options['max_in_flight'] = options['max_in_flight'] if options['max_in_flight']
+        parse_update_config(parse_options)
 
         networks = JobNetworksParser.new(Network::VALID_DEFAULTS).parse(@job_spec, @job.name, @deployment.networks)
         @job.networks = networks
@@ -63,13 +66,13 @@ module Bosh::Director
         lifecycle = safe_property(@job_spec, "lifecycle",
           :class => String,
           :optional => true,
-          :default => Job::DEFAULT_LIFECYCLE_PROFILE,
+          :default => InstanceGroup::DEFAULT_LIFECYCLE_PROFILE,
         )
 
-        unless Job::VALID_LIFECYCLE_PROFILES.include?(lifecycle)
+        unless InstanceGroup::VALID_LIFECYCLE_PROFILES.include?(lifecycle)
           raise JobInvalidLifecycle,
             "Invalid lifecycle '#{lifecycle}' for '#{@job.name}', " +
-            "valid lifecycle profiles are: #{Job::VALID_LIFECYCLE_PROFILES.join(', ')}"
+            "valid lifecycle profiles are: #{InstanceGroup::VALID_LIFECYCLE_PROFILES.join(', ')}"
         end
 
         @job.lifecycle = lifecycle
@@ -201,8 +204,7 @@ module Bosh::Director
         @job.templates.each do |template|
           if all_names.count(template.name) > 1
             raise JobInvalidTemplates,
-                  "Colocated job template '#{template.name}' has the same name in multiple releases. " +
-                  "BOSH cannot currently colocate two job templates with identical names from separate releases."
+                  "Colocated job '#{template.name}' is already added to the instance group '#{@job.name}'"
           end
         end
       end
@@ -325,9 +327,9 @@ module Bosh::Director
         @job.env = Env.new(env_hash)
       end
 
-      def parse_update_config
+      def parse_update_config(parse_options)
         update_spec = safe_property(@job_spec, "update", class: Hash, optional: true)
-        @job.update = UpdateConfig.new(update_spec, @deployment.update)
+        @job.update = UpdateConfig.new((update_spec || {}).merge(parse_options), @deployment.update)
       end
 
       def parse_desired_instances(availability_zones, networks)
@@ -344,17 +346,17 @@ module Bosh::Director
         end
 
         instance_states.each_pair do |index_or_id, state|
-          unless Job::VALID_JOB_STATES.include?(state)
+          unless InstanceGroup::VALID_JOB_STATES.include?(state)
             raise JobInvalidInstanceState,
-              "Invalid state '#{state}' for '#{@job.name}/#{index_or_id}', valid states are: #{Job::VALID_JOB_STATES.join(", ")}"
+              "Invalid state '#{state}' for '#{@job.name}/#{index_or_id}', valid states are: #{InstanceGroup::VALID_JOB_STATES.join(", ")}"
           end
 
           @job.instance_states[index_or_id] = state
         end
 
-        if @job.state && !Job::VALID_JOB_STATES.include?(@job.state)
+        if @job.state && !InstanceGroup::VALID_JOB_STATES.include?(@job.state)
           raise JobInvalidJobState,
-            "Invalid state '#{@job.state}' for '#{@job.name}', valid states are: #{Job::VALID_JOB_STATES.join(", ")}"
+            "Invalid state '#{@job.state}' for '#{@job.name}', valid states are: #{InstanceGroup::VALID_JOB_STATES.join(", ")}"
         end
 
         job_size.times.map { DesiredInstance.new(@job, @deployment) }

data/lib/bosh/director/deployment_plan/instance_plan.rb

@@ -68,6 +68,10 @@ module Bosh
           new? ? instance.model : existing_instance
         end
 
+        def should_be_ignored?
+           !instance_model.nil? && instance_model.ignore
+        end
+
         def needs_restart?
           @instance.virtual_state == 'restart'
         end
@@ -85,7 +89,7 @@ module Bosh
           desired_network_plans = network_plans.select(&:desired?)
           obsolete_network_plans = network_plans.select(&:obsolete?)
 
-          old_network_settings = new? ? {} : @existing_instance.spec['networks']
+          old_network_settings = new? ? {} : @existing_instance.spec_p('networks')
           new_network_settings = network_settings.to_hash
 
           changed = false
@@ -134,8 +138,8 @@ module Bosh
         end
 
         def configuration_changed?
-          changed = instance.configuration_hash != instance_model.spec['configuration_hash']
-          log_changes(__method__, instance_model.spec['configuration_hash'], instance.configuration_hash, instance) if changed
+          changed = instance.configuration_hash != instance_model.spec_p('configuration_hash')
+          log_changes(__method__, instance_model.spec_p('configuration_hash'), instance.configuration_hash, instance) if changed
           changed
         end
 
@@ -231,7 +235,7 @@ module Bosh
           # The agent job spec could be in legacy form.  job_spec cannot be,
           # though, because we got it from the spec function in job.rb which
           # automatically makes it non-legacy.
-          converted_current = Job.convert_from_legacy_spec(@instance.current_job_spec)
+          converted_current = InstanceGroup.convert_from_legacy_spec(@instance.current_job_spec)
           changed = job.spec != converted_current
           log_changes(__method__, converted_current, job.spec, @instance) if changed
           changed
@@ -245,7 +249,7 @@ module Bosh
           changed
         end
 
-        def currently_detached?
+        def already_detached?
           return false if new?
 
           @existing_instance.state == 'detached'
@@ -279,13 +283,13 @@ module Bosh
         end
 
         def stemcell_changed?
-          if @existing_instance && @instance.stemcell.name != @existing_instance.spec['stemcell']['name']
-            log_changes(__method__, @existing_instance.spec['stemcell']['name'], @instance.stemcell.name, @existing_instance)
+          if @existing_instance && @instance.stemcell.name != @existing_instance.spec_p('stemcell.name')
+            log_changes(__method__, @existing_instance.spec_p('stemcell.name'), @instance.stemcell.name, @existing_instance)
             return true
           end
 
-          if @existing_instance && @instance.stemcell.version != @existing_instance.spec['stemcell']['version']
-            log_changes(__method__, "version: #{@existing_instance.spec['stemcell']['version']}", "version: #{@instance.stemcell.version}", @existing_instance)
+          if @existing_instance && @instance.stemcell.version != @existing_instance.spec_p('stemcell.version')
+            log_changes(__method__, "version: #{@existing_instance.spec_p('stemcell.version')}", "version: #{@instance.stemcell.version}", @existing_instance)
             return true
           end
 
@@ -323,7 +327,7 @@ module Bosh
 
       class ResurrectionInstancePlan < InstancePlan
         def network_settings_hash
-          @existing_instance.spec['networks']
+          @existing_instance.spec_p('networks')
         end
 
         def spec

data/lib/bosh/director/deployment_plan/instance_planner.rb

@@ -8,6 +8,10 @@ module Bosh
         end
 
         def plan_job_instances(job, desired_instances, existing_instance_models)
+          if existing_instance_models.count(&:ignore) > 0
+            fail_if_specifically_changing_state_of_ignored_vms(job, existing_instance_models)
+          end
+
           network_planner = NetworkPlanner::Planner.new(@logger)
           placement_plan = PlacementPlanner::Plan.new(@instance_plan_factory, network_planner, @logger)
           vip_networks, non_vip_networks = job.networks.to_a.partition(&:vip?)
@@ -32,6 +36,14 @@ module Bosh
             desired_job_names.include?(existing_instance_model.job) ||
               migrating_job_names.include?(existing_instance_model.job)
           end
+
+          obsolete_existing_instances.each do |instance_model|
+            if instance_model.ignore
+              raise DeploymentIgnoredInstancesDeletion, "You are trying to delete instance group '#{instance_model.job}', which " +
+                  'contains ignored instance(s). Operation not allowed.'
+            end
+          end
+
           obsolete_existing_instances.map do |obsolete_existing_instance|
             @instance_plan_factory.obsolete_instance_plan(obsolete_existing_instance)
           end
@@ -80,6 +92,16 @@ module Bosh
           end
         end
 
+        def fail_if_specifically_changing_state_of_ignored_vms(job, existing_instance_models)
+          ignored_models = existing_instance_models.select(&:ignore)
+          ignored_models.each do |model|
+            unless job.instance_states["#{model.index}"].nil?
+              raise JobInstanceIgnored, "You are trying to change the state of the ignored instance '#{model.job}/#{model.uuid}'. " +
+                  'This operation is not allowed. You need to unignore it first.'
+            end
+          end
+        end
+
         def log_outcome(instance_plans)
           instance_plans.select(&:new?).each do |instance_plan|
             instance = instance_plan.desired_instance

data/lib/bosh/director/deployment_plan/job_migrator.rb

@@ -33,7 +33,7 @@ module Bosh::Director
       migrated_from_instances = []
 
       migrated_from_jobs.each do |migrated_from_job|
-        existing_job = @deployment_plan.job(migrated_from_job.name)
+        existing_job = @deployment_plan.instance_group(migrated_from_job.name)
         if existing_job && existing_job.name != desired_job.name
           raise DeploymentInvalidMigratedFromJob,
             "Failed to migrate instance group '#{migrated_from_job.name}' to '#{desired_job_name}'. " +
@@ -41,7 +41,7 @@ module Bosh::Director
               "Please remove instance group '#{migrated_from_job.name}'."
         end
 
-        other_jobs = @deployment_plan.jobs.reject { |job| job.name == desired_job_name }
+        other_jobs = @deployment_plan.instance_groups.reject { |job| job.name == desired_job_name }
 
         migrate_to_multiple_jobs = other_jobs.any? do |job|
           job.migrated_from.any? do |other_migrated_from_job|

data/lib/bosh/director/deployment_plan/links/link_lookup.rb

@@ -3,7 +3,7 @@ module Bosh::Director
     # tested in link_resolver_spec
 
     class LinkLookupFactory
-      def self.create(consumed_link, link_path, deployment_plan, link_network, consumes_job, consumes_template)
+      def self.create(consumed_link, link_path, deployment_plan, link_network)
         if link_path.deployment == deployment_plan.name
           PlannerLinkLookup.new(consumed_link, link_path, deployment_plan, link_network)
         else
@@ -24,7 +24,7 @@ module Bosh::Director
       def initialize(consumed_link, link_path, deployment_plan, link_network)
         @consumed_link = consumed_link
         @link_path = link_path
-        @jobs = deployment_plan.jobs
+        @jobs = deployment_plan.instance_groups
         @link_network = link_network
       end
 
@@ -71,4 +71,4 @@ module Bosh::Director
       end
     end
   end
-end
+end

data/lib/bosh/director/deployment_plan/links/link_path.rb

@@ -55,7 +55,7 @@ module Bosh::Director
         link_network = link_info["network"]
         found_link_paths = []
 
-        @deployment_plan.jobs.each do |provides_job|
+        @deployment_plan.instance_groups.each do |provides_job|
           if !link_network || provides_job.has_network?(link_network)
             provides_job.templates.each do |provides_template|
               if provides_template.link_infos.has_key?(provides_job.name) && provides_template.link_infos[provides_job.name].has_key?('provides')
@@ -106,7 +106,7 @@ module Bosh::Director
 
       def get_link_path_from_deployment_plan(name, link_network)
         found_link_paths = []
-        @deployment_plan.jobs.each do |job|
+        @deployment_plan.instance_groups.each do |job|
           if !link_network || job.has_network?(link_network)
             job.templates.each do |template|
               if template.link_infos.has_key?(job.name) && template.link_infos[job.name].has_key?('provides')

data/lib/bosh/director/deployment_plan/links/links_resolver.rb

@@ -33,7 +33,7 @@ module Bosh::Director
             job.add_resolved_link(link_name, link_path.manual_spec)
           else
             link_network = template.consumes_link_info(job.name, link_name)['network']
-            link_lookup = LinkLookupFactory.create(consumed_link, link_path, @deployment_plan, link_network, job.name, template.name)
+            link_lookup = LinkLookupFactory.create(consumed_link, link_path, @deployment_plan, link_network)
             link_spec = link_lookup.find_link_spec
 
             unless link_spec

data/lib/bosh/director/deployment_plan/manual_network.rb

@@ -11,7 +11,7 @@ module Bosh::Director
       def self.parse(network_spec, availability_zones, global_network_resolver, logger)
         name = safe_property(network_spec, "name", :class => String)
 
-        reserved_ranges = global_network_resolver.reserved_legacy_ranges(name)
+        reserved_ranges = global_network_resolver.reserved_ranges
         subnet_specs = safe_property(network_spec, 'subnets', :class => Array)
         subnets = []
         subnet_specs.each do |subnet_spec|

data/lib/bosh/director/deployment_plan/manual_network_subnet.rb

@@ -10,7 +10,7 @@ module Bosh::Director
       def self.parse(network_name, subnet_spec, availability_zones, legacy_reserved_ranges)
         @logger = Config.logger
 
-        @logger.debug("reserved ranges #{legacy_reserved_ranges.inspect}")
+        @logger.debug("reserved ranges #{legacy_reserved_ranges.map {|r| r.first == r.last ? "#{r.first}" : "#{r.first}-#{r.last}"}.join(', ')}")
         range_property = safe_property(subnet_spec, "range", :class => String)
         range = NetAddr::CIDR.create(range_property)
 

data/lib/bosh/director/deployment_plan/placement_planner/availability_zone_picker.rb

@@ -12,12 +12,17 @@ module Bosh
           end
 
           def place_and_match_in(desired_instances, existing_instance_models)
+            validate_networks_not_changed_for_ignore_vms(existing_instance_models)
+
             unplaced_existing_instances =  UnplacedExistingInstances.new(existing_instance_models)
             desired_azs_sorted = unplaced_existing_instances.azs_sorted_by_existing_instance_count_descending(@desired_azs)
             @logger.debug("Desired azs: #{desired_azs_sorted.inspect}")
+            validate_desired_azs_contains_all_azs_from_ignored_instances(unplaced_existing_instances, desired_instances)
+
             placed_instances = PlacedDesiredInstances.new(desired_azs_sorted)
 
-            remaining_desired_instances = place_instances_that_have_persistent_disk_in_existing_az(desired_azs_sorted, desired_instances, placed_instances, unplaced_existing_instances)
+            remaining_desired_instances_minus_ignored = place_instances_that_have_ignore_flag_as_true(desired_azs_sorted, desired_instances, placed_instances, unplaced_existing_instances)
+            remaining_desired_instances = place_instances_that_have_persistent_disk_in_existing_az(desired_azs_sorted, remaining_desired_instances_minus_ignored, placed_instances, unplaced_existing_instances)
             balance_across_desired_azs(remaining_desired_instances, placed_instances, unplaced_existing_instances)
 
             obsolete_instance_plans(unplaced_existing_instances.unclaimed) +
@@ -27,6 +32,39 @@ module Bosh
 
           private
 
+          def place_instances_that_have_ignore_flag_as_true(desired_azs, desired_instances, placed_instances, unplaced_existing_instances)
+            return desired_instances if unplaced_existing_instances.ignored_instances.empty?
+
+            desired_instances = desired_instances.dup
+            return desired_instances if desired_azs.nil?
+            unplaced_existing_instances.ignored_instances.each do |existing_instance|
+              az = desired_azs.find { |az| az.name == existing_instance.availability_zone }
+              next if az.nil?
+              desired_instance = desired_instances.pop
+              unplaced_existing_instances.claim_instance(existing_instance)
+              placed_instances.record_placement(az, desired_instance, existing_instance)
+            end
+            desired_instances
+          end
+
+          def validate_desired_azs_contains_all_azs_from_ignored_instances(unplaced_existing_instances, desired_instances)
+            return if unplaced_existing_instances.ignored_instances.empty?
+
+            if unplaced_existing_instances.ignored_instances.count > desired_instances.count
+              @logger.info("Desired instances count, #{desired_instances.count}, is less than existing ignored instances, #{unplaced_existing_instances.ignored_instances.count}")
+              raise DeploymentIgnoredInstancesModification, "Instance Group '#{unplaced_existing_instances.ignored_instances.first.job}' has #{unplaced_existing_instances.ignored_instances.count} ignored instance(s)." +
+                  " #{desired_instances.count} instance(s) of that instance group were requested. Deleting ignored instances is not allowed."
+            end
+
+            ignore_instances_az_names = unplaced_existing_instances.ignored_instances.map(&:availability_zone).compact.uniq
+            desired_az_names = @desired_azs.nil? ? [] : @desired_azs.map(&:name)
+            ignore_instances_az_names_attempted_to_be_deleted = ignore_instances_az_names - desired_az_names
+            unless ignore_instances_az_names_attempted_to_be_deleted.empty?
+              raise DeploymentIgnoredInstancesModification, "Instance Group '#{unplaced_existing_instances.ignored_instances.first.job}' no longer contains AZs " +
+                  "#{ignore_instances_az_names_attempted_to_be_deleted} where ignored instance(s) exist."
+            end
+          end
+
           def place_instances_that_have_persistent_disk_in_existing_az(desired_azs, desired_instances, placed_instances, unplaced_existing_instances)
             desired_instances = desired_instances.dup
             return desired_instances if desired_azs.nil?
@@ -83,6 +121,20 @@ module Bosh
               instance_plan.network_plans << @network_planner.network_plan_with_dynamic_reservation(instance_plan, network)
             end
           end
+
+          def validate_networks_not_changed_for_ignore_vms(existing_instance_models)
+            existing_instance_models.each do |existing_instance_model|
+              next if !existing_instance_model.ignore
+
+              desired_networks_names = @networks.map(&:name).uniq.sort
+              existing_networks_names = existing_instance_model.ip_addresses.map(&:network_name).uniq.sort
+
+              if desired_networks_names != existing_networks_names
+                raise DeploymentIgnoredInstancesModification, "In instance group '#{existing_instance_model.job}', which contains ignored vms,"+
+                    ' an attempt was made to modify the networks. This operation is not allowed.'
+              end
+            end
+          end
         end
       end
     end

data/lib/bosh/director/deployment_plan/placement_planner/static_ips_availability_zone_picker.rb

@@ -18,14 +18,48 @@ module Bosh
           def place_and_match_in(desired_instances, existing_instance_models)
             @networks_to_static_ips.validate_azs_are_declared_in_job_and_subnets(@desired_azs)
             @networks_to_static_ips.validate_ips_are_in_desired_azs(@desired_azs)
+            validate_ignored_instances_networks(existing_instance_models)
+
             desired_instances = desired_instances.dup
 
             instance_plans = place_existing_instance_plans(desired_instances, existing_instance_models)
-            place_new_instance_plans(desired_instances, instance_plans)
+            instance_plans = place_new_instance_plans(desired_instances, instance_plans)
+
+            if ignored_instances_are_obsolete?(instance_plans)
+              raise DeploymentIgnoredInstancesModification, "In instance group '#{@job_name}', an attempt was made to remove a static ip"+
+                  ' that is used by an ignored instance. This operation is not allowed.'
+            end
+
+            instance_plans
           end
 
           private
 
+          def validate_ignored_instances_networks(existing_instance_models)
+            existing_instance_models.each do |existing_instance_model|
+              next if !existing_instance_model.ignore
+
+              # Validate that no networks were added or deleted
+              desired_networks_names = @job_networks.map(&:name).uniq.sort
+              existing_networks_names = existing_instance_model.ip_addresses.map(&:network_name).uniq.sort
+
+              if desired_networks_names != existing_networks_names
+                raise DeploymentIgnoredInstancesModification, "In instance group '#{@job_name}', which contains ignored vms,"+
+                    ' an attempt was made to modify the networks. This operation is not allowed.'
+              end
+
+              # Validate that no ip addresses, that were assigned to an ignored VM, have been removed
+              existing_instance_model.ip_addresses.each do |ip_address|
+                ignored_vm_network = @job_networks.select { |n| n.name == ip_address.network_name }.first
+
+                if !ignored_vm_network.static_ips.include?(ip_address.address)
+                  raise DeploymentIgnoredInstancesModification, "In instance group '#{@job_name}', an attempt was made to remove a static ip"+
+                      ' that is used by an ignored instance. This operation is not allowed.'
+                end
+              end
+            end
+          end
+
           def place_existing_instance_plans(desired_instances, existing_instance_models)
             instance_plans = []
             # create existing instance plans with network plans that use specified static IPs
@@ -231,6 +265,10 @@ module Bosh
           def instance_name(existing_instance_model)
             "#{existing_instance_model.job}/#{existing_instance_model.index}"
           end
+
+          def ignored_instances_are_obsolete?(instance_plans)
+            instance_plans.select{ |i| i.obsolete? && i.should_be_ignored? }.any?
+          end
         end
       end
     end