bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/deployment_plan/network_settings.rb

@@ -0,0 +1,65 @@
+module Bosh::Director::DeploymentPlan
+  class NetworkSettings
+    def initialize(job_name, deployment_name, default_network, desired_reservations, state, availability_zone, instance_index, instance_id, dns_manager)
+      @job_name = job_name
+      @desired_reservations = desired_reservations
+      @default_network = default_network
+      @deployment_name = deployment_name
+      @state = state
+      @availability_zone = availability_zone
+      @instance_index = instance_index
+      @instance_id = instance_id
+      @dns_manager = dns_manager
+    end
+
+    def to_hash
+      default_properties = {}
+      @default_network.each do |key, value|
+        (default_properties[value] ||= []) << key
+      end
+
+      network_settings = {}
+      @desired_reservations.each do |reservation|
+        network_name = reservation.network.name
+        network_settings[network_name] = reservation.network.network_settings(reservation, default_properties[network_name], @availability_zone)
+
+        # Somewhat of a hack: for dynamic networks we might know IP address, Netmask & Gateway
+        # if they're featured in agent state, in that case we put them into network spec to satisfy
+        # ConfigurationHasher in both agent and director.
+        if @state.is_a?(Hash) &&
+          @state['networks'].is_a?(Hash) &&
+          @state['networks'][network_name].is_a?(Hash) &&
+          network_settings[network_name]['type'] == 'dynamic'
+          %w(ip netmask gateway).each do |key|
+            network_settings[network_name][key] = @state['networks'][network_name][key]
+          end
+        end
+      end
+
+      network_settings
+    end
+
+    def dns_record_info
+      dns_record_info = {}
+      to_hash.each do |network_name, network|
+        index_dns_name =  @dns_manager.dns_record_name(@instance_index, @job_name, network_name, @deployment_name)
+        dns_record_info[index_dns_name] = network['ip']
+        id_dns_name =  @dns_manager.dns_record_name(@instance_id, @job_name, network_name, @deployment_name)
+        dns_record_info[id_dns_name] = network['ip']
+      end
+      dns_record_info
+    end
+
+    def network_addresses
+      network_addresses = {}
+      to_hash.each do |network_name, network|
+        network_addresses[network_name] = {
+          'address' => network['type'] == 'dynamic' ?
+            @dns_manager.dns_record_name(@instance_id, @job_name, network_name, @deployment_name) :
+            network['ip']
+        }
+      end
+      network_addresses
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/options/skip_drain.rb

@@ -16,5 +16,12 @@ module Bosh::Director
         @all || @jobs.include?(job_name)
       end
     end
+
+    class AlwaysSkipDrain
+      def for_job(_)
+        true
+      end
+    end
+
   end
 end

data/lib/bosh/director/deployment_plan/package_validator.rb

@@ -0,0 +1,79 @@
+module Bosh::Director
+  module DeploymentPlan
+    class PackageValidator
+      def initialize(logger)
+        @faults = {}
+        @logger = logger
+      end
+
+      def validate(release_version_model, stemcel_model)
+        release_desc = "#{release_version_model.release.name}/#{release_version_model.version}"
+
+        @logger.debug("Validating packages for release '#{release_desc}'")
+        release_version_model.packages.each do |package|
+          packages_list = release_version_model.transitive_dependencies(package)
+          packages_list << package
+          packages_list.each do |needed_package|
+            if needed_package.sha1.nil? || needed_package.blobstore_id.nil?
+              compiled_packages_list = Bosh::Director::Models::CompiledPackage[:package_id => needed_package.id, :stemcell_id => stemcel_model.id]
+              if compiled_packages_list.nil?
+                @faults[release_desc] ||= Set.new
+                @faults[release_desc] << Fault.new(needed_package, stemcel_model)
+              end
+            end
+          end
+        end
+      end
+
+      def handle_faults
+        return if @faults.empty?
+
+        msg = "\n"
+        unique_stemcells = @faults.map { |_, faults| faults.map { |fault| fault.stemcell } }.flatten.to_set
+
+        @faults.each do |release_desc, faults|
+          if unique_stemcells.size > 1
+            msg += message_for_multiple_stemcells(release_desc, faults)
+          else
+            msg += message_for_single_stemcell(release_desc, faults)
+          end
+        end
+
+        raise PackageMissingSourceCode, msg
+      end
+
+      private
+
+      def message_for_multiple_stemcells(release_desc, faults)
+        msg = "Can't use release '#{release_desc}'. It references packages without source code and are not compiled against intended stemcells:\n"
+        sorted_faults = faults.to_a.sort_by { |fault| fault.package.name }
+        sorted_faults.each do |fault|
+          msg += " - '#{fault.package_desc}' against stemcell '#{fault.stemcell.desc}'\n"
+        end
+
+        msg
+      end
+
+      def message_for_single_stemcell(release_desc, faults)
+        sorted_faults = faults.to_a.sort_by { |fault| fault.package.name }
+        stemcell_desc = faults.first.stemcell.desc
+
+        msg = "Can't use release '#{release_desc}'. It references packages without" +
+          " source code and are not compiled against stemcell '#{stemcell_desc}':\n"
+        sorted_faults.each do |fault|
+          msg += " - '#{fault.package_desc}'\n"
+        end
+
+        msg
+      end
+
+      private
+
+      class Fault < Struct.new(:package, :stemcell)
+        def package_desc
+          "#{package.name}/#{package.version}"
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/placement_planner.rb

@@ -0,0 +1,8 @@
+require 'bosh/director/deployment_plan/placement_planner/plan'
+require 'bosh/director/deployment_plan/placement_planner/index_assigner'
+require 'bosh/director/deployment_plan/placement_planner/availability_zone_picker'
+require 'bosh/director/deployment_plan/placement_planner/static_ips_availability_zone_picker'
+require 'bosh/director/deployment_plan/placement_planner/networks_to_static_ips'
+require 'bosh/director/deployment_plan/placement_planner/bruteforce_ip_allocation'
+require 'bosh/director/deployment_plan/placement_planner/placed_desired_instances'
+require 'bosh/director/deployment_plan/placement_planner/unplaced_existing_instances'

data/lib/bosh/director/deployment_plan/placement_planner/availability_zone_picker.rb

@@ -0,0 +1,90 @@
+module Bosh
+  module Director
+    module DeploymentPlan
+      module PlacementPlanner
+        class AvailabilityZonePicker
+          def initialize(instance_plan_factory, network_planner, networks, desired_azs)
+            @instance_plan_factory = instance_plan_factory
+            @network_planner = network_planner
+            @networks = networks
+            @desired_azs = desired_azs
+            @logger = Config.logger
+          end
+
+          def place_and_match_in(desired_instances, existing_instance_models)
+            unplaced_existing_instances =  UnplacedExistingInstances.new(existing_instance_models)
+            desired_azs_sorted = unplaced_existing_instances.azs_sorted_by_existing_instance_count_descending(@desired_azs)
+            @logger.debug("Desired azs: #{desired_azs_sorted.inspect}")
+            placed_instances = PlacedDesiredInstances.new(desired_azs_sorted)
+
+            remaining_desired_instances = place_instances_that_have_persistent_disk_in_existing_az(desired_azs_sorted, desired_instances, placed_instances, unplaced_existing_instances)
+            balance_across_desired_azs(remaining_desired_instances, placed_instances, unplaced_existing_instances)
+
+            obsolete_instance_plans(unplaced_existing_instances.unclaimed) +
+              desired_existing_instance_plans(placed_instances.existing) +
+              desired_new_instance_plans(placed_instances.absent)
+          end
+
+          private
+
+          def place_instances_that_have_persistent_disk_in_existing_az(desired_azs, desired_instances, placed_instances, unplaced_existing_instances)
+            desired_instances = desired_instances.dup
+            return desired_instances if desired_azs.nil?
+            unplaced_existing_instances.instances_with_persistent_disk.each do |existing_instance|
+              break if desired_instances.empty?
+              az = desired_azs.find { |az| az.name == existing_instance.availability_zone }
+              next if az.nil?
+              desired_instance = desired_instances.pop
+              unplaced_existing_instances.claim_instance(existing_instance)
+              placed_instances.record_placement(az, desired_instance, existing_instance)
+            end
+            desired_instances
+          end
+
+          def balance_across_desired_azs(desired_instances, placed_instances, unplaced_existing_instances)
+            desired_instances.each do |desired_instance|
+              azs_with_fewest_placed = placed_instances.azs_with_fewest_instances
+              @logger.debug("azs with fewest placed: #{azs_with_fewest_placed.inspect}")
+              az = unplaced_existing_instances.azs_sorted_by_existing_instance_count_descending(azs_with_fewest_placed).first
+              @logger.debug("az: #{az.inspect}")
+
+              existing_instance = unplaced_existing_instances.claim_instance_for_az(az)
+              placed_instances.record_placement(az, desired_instance, existing_instance)
+            end
+          end
+
+          def obsolete_instance_plans(obsolete_instances)
+            obsolete_instances.map do |existing_instance|
+              @instance_plan_factory.obsolete_instance_plan(existing_instance)
+            end
+          end
+
+          def desired_existing_instance_plans(desired_existing_instances)
+            desired_existing_instances.map do |desired_existing_instance|
+              instance_plan = @instance_plan_factory.desired_existing_instance_plan(
+                desired_existing_instance[:existing_instance_model],
+                desired_existing_instance[:desired_instance]
+              )
+              populate_network_plans(instance_plan)
+              instance_plan
+            end
+          end
+
+          def desired_new_instance_plans(new_desired_instances)
+            new_desired_instances.map do |desired_instance|
+              instance_plan = @instance_plan_factory.desired_new_instance_plan(desired_instance)
+              populate_network_plans(instance_plan)
+              instance_plan
+            end
+          end
+
+          def populate_network_plans(instance_plan)
+            @networks.each do |network|
+              instance_plan.network_plans << @network_planner.network_plan_with_dynamic_reservation(instance_plan, network)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/placement_planner/bruteforce_ip_allocation.rb

@@ -0,0 +1,124 @@
+require 'common/deep_copy'
+
+module Bosh
+  module Director
+    module DeploymentPlan
+      module PlacementPlanner
+        class BruteForceIpAllocation
+          def initialize(networks_to_static_ips)
+            @networks_to_static_ips = networks_to_static_ips
+          end
+
+          def find_best_combination
+            allocated_ips = AllocatedIps.new
+            try_combination(@networks_to_static_ips, allocated_ips)
+          end
+
+          private
+
+          def try_combination(networks_to_static_ips, allocated_ips)
+            if all_ips_belong_to_single_az(networks_to_static_ips)
+              if even_distribution_of_ips?(networks_to_static_ips)
+                return networks_to_static_ips
+              else
+                return nil
+              end
+            end
+
+            previous_network = nil
+            networks_to_static_ips.each do |network, static_ips_to_azs|
+              if previous_network
+                previous_assignment = PreviousAssignment.new(networks_to_static_ips[previous_network])
+                unless previous_assignment.has_same_distribution?(static_ips_to_azs)
+                  return nil
+                end
+              end
+              previous_network = network
+
+              static_ips_to_azs.each_with_index do |static_ip_to_azs|
+                if static_ip_to_azs.az_names.size == 1
+                  allocated_ips.allocate(static_ip_to_azs.az_names.first)
+                  next
+                end
+
+                # prioritize AZs based on least number of allocated IPs
+                sorted_az_names = allocated_ips.sort_by_least_allocated_ips(static_ip_to_azs.az_names)
+                sorted_az_names.each do |az_name|
+                  static_ip_to_azs.az_names = [az_name]
+                  allocated_ips.allocate(az_name)
+                  candidate_networks_to_static_ips = Bosh::Common::DeepCopy.copy(networks_to_static_ips)
+                  result = try_combination(candidate_networks_to_static_ips, AllocatedIps.new)
+                  next if result.nil?
+                  return result
+                end
+              end
+            end
+
+            return nil
+          end
+
+          def even_distribution_of_ips?(networks_to_static_ips)
+            hash = {}
+            networks_to_static_ips.each do |network, static_ips_to_azs|
+              hash[network] ||= {}
+              static_ips_to_azs.each do |static_ip_to_azs|
+                static_ip_to_azs.az_names.each do |az_name|
+                  hash[network][az_name] ||= 0
+                  hash[network][az_name] += 1
+                end
+              end
+
+            end
+
+            hash.values.uniq.size > 1 ? false : true
+          end
+
+          def all_ips_belong_to_single_az(networks_to_static_ips)
+            !networks_to_static_ips.values.any? do |static_ips_to_azs|
+              static_ips_to_azs.any? do |static_ip_to_az|
+                static_ip_to_az.az_names.size > 1
+              end
+            end
+          end
+
+          class AllocatedIps
+            def initialize
+              @allocated_ips = Hash.new {|h,k| h[k] = 0 }
+            end
+
+            def allocate(az_name)
+              @allocated_ips[az_name] += 1
+            end
+
+            def sort_by_least_allocated_ips(az_names)
+              az_names.sort_by do |az_name|
+                @allocated_ips[az_name]
+              end
+            end
+          end
+
+          class PreviousAssignment
+            def initialize(network_to_static_ips)
+              @previous_assignment = Hash.new {|h,k| h[k] = 0 }
+              network_to_static_ips.each do |previous_assignment_ip|
+                @previous_assignment[previous_assignment_ip.az_names.first] += 1
+              end
+            end
+
+            def has_same_distribution?(static_ips_to_azs)
+              @previous_assignment.each do |az_name, required_number_of_ips_in_az|
+                ips_in_az = static_ips_to_azs.select { |static_ip_to_azs| static_ip_to_azs.az_names.include?(az_name) }
+                if ips_in_az.size < required_number_of_ips_in_az
+                  return false
+                end
+              end
+
+              return true
+            end
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/bosh/director/deployment_plan/placement_planner/index_assigner.rb

@@ -0,0 +1,32 @@
+module Bosh
+  module Director
+    module DeploymentPlan
+      module PlacementPlanner
+        class IndexAssigner
+          def initialize(deployment_model)
+            @deployment_model = deployment_model
+          end
+
+          def assign_index(job_name, existing_instance_model=nil)
+            if existing_instance_model && existing_instance_model.job == job_name
+              return existing_instance_model.index
+            end
+
+            sorted_indexes = Models::Instance.filter(job: job_name, deployment: @deployment_model).sort_by(&:index).map(&:index)
+            if sorted_indexes.empty?
+              0
+            else
+              find_unused_index(sorted_indexes)
+            end
+          end
+
+          def find_unused_index(sorted_indexes)
+            sorted_indexes.unshift(-1)
+            next_indexes = sorted_indexes.map { |i| i + 1 }
+            (next_indexes - sorted_indexes).min
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/placement_planner/networks_to_static_ips.rb

@@ -0,0 +1,125 @@
+module Bosh
+  module Director
+    module DeploymentPlan
+      module PlacementPlanner
+        class NetworksToStaticIps
+          extend Bosh::Director::IpUtil
+          include Bosh::Director::IpUtil
+
+          def self.create(job_networks, desired_azs, job_name)
+            networks_to_static_ips = {}
+
+            desired_az_names = desired_azs.nil? ? [nil] : desired_azs.to_a.map(&:name)
+
+            job_networks.each do |job_network|
+              next unless job_network.static?
+              subnets = job_network.deployment_network.subnets
+
+              job_network.static_ips.each do |static_ip|
+                subnet_for_ip = subnets.find { |subnet| subnet.static_ips.include?(static_ip) }
+                if subnet_for_ip.nil?
+                  raise JobNetworkInstanceIpMismatch,
+                    "Job '#{job_name}' with network '#{job_network.name}' declares static ip '#{format_ip(static_ip)}', " +
+                      "which belongs to no subnet"
+                end
+                az_names = subnet_for_ip.availability_zone_names.nil? ? [nil] : subnet_for_ip.availability_zone_names
+                filtered_az_names = az_names.select { |static_ip_az_name| desired_az_names.include?(static_ip_az_name) }.uniq
+                networks_to_static_ips[job_network.name] ||= []
+                networks_to_static_ips[job_network.name] << StaticIpToAzs.new(static_ip, filtered_az_names)
+              end
+            end
+
+            new(networks_to_static_ips, job_name)
+          end
+
+          def initialize(networks_to_static_ips, job_name)
+            @networks_to_static_ips = networks_to_static_ips
+            @job_name = job_name
+          end
+
+          def validate_azs_are_declared_in_job_and_subnets(desired_azs)
+            if desired_azs.nil? &&
+              @networks_to_static_ips.any? do |_, static_ips_to_az|
+                static_ips_to_az.any? { |static_ip_to_az| !static_ip_to_az.az_names.any?(&:nil?) }
+              end
+
+              raise JobInvalidAvailabilityZone,
+                "Job '#{@job_name}' subnets declare availability zones and the job does not"
+            end
+          end
+
+          def validate_ips_are_in_desired_azs(desired_azs)
+            return if desired_azs.to_a.empty?
+
+            desired_az_names = desired_azs.to_a.map(&:name)
+            @networks_to_static_ips.each do |_, static_ips_to_az|
+              non_desired_ip_to_az = static_ips_to_az.find do |static_ip_to_az|
+                !(desired_az_names & static_ip_to_az.az_names).any?
+              end
+
+              if non_desired_ip_to_az
+                raise JobStaticIpsFromInvalidAvailabilityZone,
+                  "Job '#{@job_name}' declares static ip '#{format_ip(non_desired_ip_to_az.ip)}' which does not belong to any of the job's availability zones."
+              end
+            end
+          end
+
+          def azs_to_networks
+            result = {}
+            @networks_to_static_ips.each do |network_name, static_ips_to_azs|
+              static_ips_to_azs.each do |static_ip_to_azs|
+                static_ip_to_azs.az_names.each do |az_name|
+                  result[az_name][network_name] ||= []
+                  result[az_name][network_name] << static_ip_to_azs.ip
+                end
+              end
+            end
+          end
+
+          def distribute_evenly_per_zone
+            best_combination = BruteForceIpAllocation.new(@networks_to_static_ips).find_best_combination
+            if best_combination.nil?
+              raise JobNetworkInstanceIpMismatch, "Failed to evenly distribute static IPs between zones for job '#{@job_name}'"
+            end
+            @networks_to_static_ips = best_combination
+          end
+
+          def next_ip_for_network(network)
+            unclaimed_networks_to_static_ips[network.name].first
+          end
+
+          def claim_in_az(ip, az_name)
+            @networks_to_static_ips.each do |_, static_ips_to_azs|
+              static_ips_to_azs.each do |static_ip_to_azs|
+                if static_ip_to_azs.ip == ip
+                  static_ip_to_azs.claimed = true
+                  static_ip_to_azs.az_names = [az_name]
+                end
+              end
+            end
+          end
+
+          def delete(ip)
+            @networks_to_static_ips.each do |_, static_ip_to_azs|
+              static_ip_to_azs.delete_if { |static_ip_to_az| static_ip_to_az.ip == ip }
+            end
+          end
+
+          def find_by_network_and_ip(network, ip)
+            unclaimed_networks_to_static_ips[network.name].find { |static_ip_to_azs| static_ip_to_azs.ip == ip }
+          end
+
+          def find_by_network_and_az(network, az_name)
+            unclaimed_networks_to_static_ips[network.name].find { |static_ip_to_azs| static_ip_to_azs.az_names.include?(az_name) }
+          end
+
+          def unclaimed_networks_to_static_ips
+            Hash[@networks_to_static_ips.map { |network_name, static_ips_to_azs| [network_name, static_ips_to_azs.reject(&:claimed)] }]
+          end
+
+          class StaticIpToAzs < Struct.new(:ip, :az_names, :claimed); end
+        end
+      end
+    end
+  end
+end