bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/deployment_plan/ip_provider/in_memory_ip_repo.rb

@@ -0,0 +1,81 @@
+module Bosh::Director::DeploymentPlan
+  class InMemoryIpRepo
+    include Bosh::Director::IpUtil
+
+    def initialize(logger)
+      @logger = Bosh::Director::TaggedLogger.new(logger, 'network-configuration')
+      @mutex = Mutex.new
+      @ips = []
+      @recently_released_ips = []
+    end
+
+    def delete(ip, network_name)
+      ip = ip_to_netaddr(ip)
+      entry_to_delete = {ip: ip.to_i, network_name: network_name}
+
+      @logger.debug("Deleting ip '#{ip.ip}' for #{network_name}")
+      @mutex.synchronize do
+        @ips.delete(entry_to_delete)
+        @recently_released_ips << (entry_to_delete)
+      end
+    end
+
+    def add(reservation)
+      ip = ip_to_netaddr(reservation.ip)
+      network_name = reservation.network.name
+      @mutex.synchronize do
+        add_ip(ip, network_name)
+      end
+    end
+
+    def allocate_dynamic_ip(reservation, subnet)
+      item = (0...subnet.range.size).find { |i| available_for_dynamic?(subnet.range[i], subnet) }
+
+      @mutex.synchronize do
+        if item.nil?
+          entry = @recently_released_ips.find do |entry|
+            entry[:network_name] == subnet.network_name && subnet.range.contains?(entry[:ip])
+          end
+
+          ip = ip_to_netaddr(entry[:ip]) unless entry.nil?
+        else
+          ip = subnet.range[item]
+        end
+
+        add_ip(ip, subnet.network_name) unless ip.nil?
+
+        ip
+      end
+    end
+
+    private
+
+    def add_ip(ip, network_name)
+      entry_to_add = {ip: ip.to_i, network_name: network_name}
+
+      if @ips.include?(entry_to_add)
+        message = "Failed to reserve IP '#{ip.ip}' for '#{network_name}': already reserved"
+        @logger.error(message)
+        raise Bosh::Director::NetworkReservationAlreadyInUse, message
+      end
+
+      @logger.debug("Reserving ip '#{ip.ip}' for #{network_name}")
+
+      @ips << entry_to_add
+      @recently_released_ips.delete(entry_to_add)
+    end
+
+    def available_for_dynamic?(ip, subnet)
+      return false unless subnet.range.contains?(ip)
+      return false if subnet.static_ips.include?(ip.to_i)
+      return false if subnet.restricted_ips.include?(ip.to_i)
+
+      @mutex.synchronize do
+        return false if @recently_released_ips.include?({ip: ip.to_i, network_name: subnet.network_name})
+        return false if @ips.include?({ip: ip.to_i, network_name: subnet.network_name})
+      end
+
+      true
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/ip_provider/ip_provider.rb

@@ -0,0 +1,153 @@
+module Bosh::Director
+  module DeploymentPlan
+    class IpProvider
+      include IpUtil
+
+      def initialize(ip_repo, networks, logger)
+        @logger = Bosh::Director::TaggedLogger.new(logger, 'network-configuration')
+        @ip_repo = ip_repo
+        @networks = networks
+      end
+
+      def release(reservation)
+        if reservation.ip.nil?
+          return if reservation.network.is_a?(DynamicNetwork)
+
+          @logger.error("Failed to release IP for manual network '#{reservation.network.name}': IP must be provided")
+          raise Bosh::Director::NetworkReservationIpMissing, "Can't release reservation without an IP"
+        else
+          @ip_repo.delete(reservation.ip, reservation.network.name)
+        end
+      end
+
+      def reserve(reservation)
+        # We should not be calling reserve on reservations that have already been reserved
+        return if reservation.reserved?
+
+        if reservation.network.is_a?(DynamicNetwork)
+          reserve_dynamic(reservation)
+          return
+        end
+
+        if reservation.network.is_a?(VipNetwork)
+          reserve_vip(reservation)
+          return
+        end
+
+        # Reserve IP for Manual Network
+        if reservation.ip.nil?
+          @logger.debug("Allocating dynamic ip for manual network '#{reservation.network.name}'")
+
+          filter_subnet_by_instance_az(reservation).each do |subnet|
+            ip = @ip_repo.allocate_dynamic_ip(reservation, subnet)
+
+            if ip
+              @logger.debug("Reserving dynamic IP '#{format_ip(ip)}' for manual network '#{reservation.network.name}'")
+              reservation.resolve_ip(ip)
+              reservation.resolve_type(:dynamic)
+              reservation.mark_reserved
+              return
+            end
+          end
+
+          raise NetworkReservationNotEnoughCapacity,
+            "Failed to reserve IP for '#{reservation.instance}' for manual network '#{reservation.network.name}': no more available"
+
+        else
+
+          ip_string = format_ip(reservation.ip)
+          @logger.debug("Reserving #{reservation.desc} for manual network '#{reservation.network.name}'")
+
+          subnet = reservation.network.find_subnet_containing(reservation.ip)
+          if subnet
+            if subnet.restricted_ips.include?(reservation.ip.to_i)
+              message = "Failed to reserve IP '#{ip_string}' for network '#{subnet.network_name}': IP belongs to reserved range"
+              @logger.error(message)
+              raise Bosh::Director::NetworkReservationIpReserved, message
+            end
+
+            reserve_manual(reservation, subnet)
+          else
+            raise NetworkReservationIpOutsideSubnet,
+              "Provided static IP '#{ip_string}' does not belong to any subnet in network '#{reservation.network.name}'"
+          end
+        end
+      end
+
+      def reserve_existing_ips(reservation)
+        if reservation.network.is_a?(DynamicNetwork)
+          if reservation.network_type == 'dynamic'
+            # Marking reservation as reserved so that it keeps existing reservation and
+            # does not recreate VM
+            reserve_dynamic(reservation)
+          end
+
+          # If previous network type was not dynamic we should release reservation from DB
+          return
+        end
+
+        if reservation.network.is_a?(VipNetwork)
+          reserve_vip(reservation)
+          return
+        end
+
+        @logger.debug('Reserving existing ips')
+        network, subnet = find_network_and_subnet_containing(reservation.ip)
+        if subnet
+          @logger.debug("Marking existing IP #{format_ip(reservation.ip)} as reserved")
+          reservation.resolve_network(network)
+          reserve_manual(reservation, subnet)
+        end
+      end
+
+      private
+
+      def reserve_manual(reservation, subnet)
+        @ip_repo.add(reservation)
+
+        subnet_az_names = subnet.availability_zone_names.to_a.join(', ')
+        if subnet.static_ips.include?(reservation.ip.to_i)
+          reservation.resolve_type(:static)
+          reservation.mark_reserved
+          @logger.debug("Found subnet with azs '#{subnet_az_names}' for #{format_ip(reservation.ip)}. Reserved as static network reservation.")
+        else
+          reservation.resolve_type(:dynamic)
+          reservation.mark_reserved
+          @logger.debug("Found subnet with azs '#{subnet_az_names}' for #{format_ip(reservation.ip)}. Reserved as dynamic network reservation.")
+        end
+      end
+
+      def reserve_vip(reservation)
+        @logger.debug("Reserving IP '#{format_ip(reservation.ip)}' for vip network '#{reservation.network.name}'")
+        @ip_repo.add(reservation)
+        reservation.resolve_type(:static)
+        reservation.mark_reserved
+      end
+
+      def reserve_dynamic(reservation)
+        reservation.resolve_type(:dynamic)
+        reservation.mark_reserved
+      end
+
+      def filter_subnet_by_instance_az(reservation)
+        instance_az = reservation.instance.availability_zone
+        if instance_az.nil?
+          reservation.network.subnets
+        else
+          reservation.network.subnets.select do |subnet|
+            subnet.availability_zone_names.include?(instance_az.name)
+          end
+        end
+      end
+
+      def find_network_and_subnet_containing(cidr_ip)
+        @networks.values.select(&:manual?).each do |network|
+          subnet = network.subnets.find { |subnet| subnet.is_reservable?(cidr_ip) }
+          return [network, subnet] if subnet
+        end
+
+        return nil
+      end
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/ip_provider/ip_provider_factory.rb

@@ -0,0 +1,22 @@
+module Bosh::Director
+  module DeploymentPlan
+    class IpProviderFactory
+      def initialize(using_global_networking, logger)
+        @using_global_networking = using_global_networking
+        @logger = logger
+      end
+
+      def new_ip_provider(networks)
+        if @using_global_networking
+          @logger.debug('Using database ip repo')
+          ip_repo = DatabaseIpRepo.new(@logger)
+        else
+          @logger.debug('Using in-memory ip repo')
+          ip_repo = InMemoryIpRepo.new(@logger)
+        end
+
+        IpProvider.new(ip_repo, networks, @logger)
+      end
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/job.rb

@@ -25,20 +25,21 @@ module Bosh::Director
       # @return [String] Job canonical name (mostly for DNS)
       attr_accessor :canonical_name
 
-      # @return [DiskPool] Persistent disk pool (or nil)
-      attr_accessor :persistent_disk_pool
-
-      # @return [DeploymentPlan] Current deployment plan
-      attr_accessor :deployment
+      # @return [DiskType] Persistent disk type (or nil)
+      attr_accessor :persistent_disk_type
 
       # @return [DeploymentPlan::ReleaseVersion] Release this job belongs to
       attr_accessor :release
 
-      # @return [DeploymentPlan::ResourcePool] Resource pool this job should
-      #   be run in
-      attr_accessor :resource_pool
+      # @return [DeploymentPlan::Stemcell]
+      attr_accessor :stemcell
+
+      # @return [DeploymentPlan::VmType]
+      attr_accessor :vm_type
+
+      # @return [DeploymentPlan::Env]
+      attr_accessor :env
 
-      # @return [DeploymentPlan::Network] Job default network
       attr_accessor :default_network
 
       # @return [Array<DeploymentPlan::Template] Templates included into the job
@@ -67,21 +68,25 @@ module Bosh::Director
       # @return [Hash<Integer, String>] Individual instance expected states
       attr_accessor :instance_states
 
+      attr_accessor :availability_zones
+
       attr_accessor :all_properties
 
-      # @param [Bosh::Director::DeploymentPlan::Planner] deployment Deployment plan
-      # @param [Hash] job_spec Raw job spec from the deployment manifest
-      # @param [Bosh::Director::EventLog::Log] event_log Event log for recording deprecations
-      # @param [Logger] logger Log for director logging
-      # @return [Bosh::Director::DeploymentPlan::Job]
-      def self.parse(deployment, job_spec, event_log, logger)
-        parser = JobSpecParser.new(deployment, event_log, logger)
+      attr_accessor :networks
+
+      attr_accessor :migrated_from
+
+      attr_accessor :desired_instances
+
+      attr_reader :link_paths
+
+      def self.parse(plan, job_spec, event_log, logger)
+        parser = JobSpecParser.new(plan, event_log, logger)
         parser.parse(job_spec)
       end
 
-      # @param [Bosh::Director::DeploymentPlan] deployment Deployment plan
-      def initialize(deployment)
-        @deployment = deployment
+      def initialize(logger)
+        @logger = logger
 
         @release = nil
         @templates = []
@@ -89,16 +94,33 @@ module Bosh::Director
         @properties = nil # Actual job properties
 
         @instances = []
+        @desired_instances = []
         @unneeded_instances = []
         @instance_states = {}
+        @default_network = {}
 
         @packages = {}
+        @link_paths = {}
+        @resolved_links = {}
+        @migrated_from = []
+        @availability_zones = []
+
+        @instance_plans = []
       end
 
       def self.is_legacy_spec?(job_spec)
         !job_spec.has_key?("templates")
       end
 
+      def add_instance_plans(instance_plans)
+        @instance_plans = instance_plans
+      end
+
+      def sorted_instance_plans
+        @sorted_instance_plans ||= InstancePlanSorter.new(@logger)
+                                   .sort(@instance_plans.reject(&:obsolete?))
+      end
+
       # Takes in a job spec and returns a job spec in the new format, if it
       # needs to be modified.  The new format has "templates" key, which is an
       # array with each template's data.  This is used for job collocation,
@@ -117,6 +139,23 @@ module Bosh::Director
         job_spec["templates"] = [template]
       end
 
+
+      def obsolete_instance_plans
+        @instance_plans.select(&:obsolete?)
+      end
+
+      def instances # to preserve interface for UpdateStep -- switch to instance_plans eventually
+        needed_instance_plans.map(&:instance)
+      end
+
+      def needed_instance_plans
+        sorted_instance_plans
+      end
+
+      def unneeded_instances
+        obsolete_instance_plans.map(&:instance)
+      end
+
       # Returns job spec as a Hash. To be used by all instances of the job to
       # populate agent state.
       # @return [Hash] Hash representation
@@ -164,9 +203,6 @@ module Bosh::Director
         result.select { |name, _| run_time_dependencies.include? name }
       end
 
-      # Returns job instance by index
-      # @param [Integer] index
-      # @return [DeploymentPlan::Instance] index-th instance
       def instance(index)
         @instances[index]
       end
@@ -174,8 +210,8 @@ module Bosh::Director
       # Returns the state state of job instance by its index
       # @param [Integer] index Instance index
       # @return [String, nil] Instance state (nil if not specified)
-      def instance_state(index)
-        @instance_states[index] || @state
+      def state_for_instance(instance_model)
+        @instance_states[instance_model.uuid] || @instance_states[instance_model.index.to_s] || @state
       end
 
       # Registers compiled package with this job.
@@ -196,14 +232,14 @@ module Bosh::Director
 
       def validate_package_names_do_not_collide!
         releases_by_package_names = templates
-          .reduce([]) { |memo, t| memo + t.model.package_names.product([t.release]) }
-          .reduce({}) { |memo, package_name_and_release_version|
-            package_name = package_name_and_release_version.first
-            release_version = package_name_and_release_version.last
-            memo[package_name] ||= Set.new
-            memo[package_name] << release_version
-            memo
-          }
+                                      .reduce([]) { |memo, t| memo + t.model.package_names.product([t.release]) }
+                                      .reduce({}) { |memo, package_name_and_release_version|
+          package_name = package_name_and_release_version.first
+          release_version = package_name_and_release_version.last
+          memo[package_name] ||= Set.new
+          memo[package_name] << release_version
+          memo
+        }
 
         releases_by_package_names.each do |package_name, releases|
           if releases.size > 1
@@ -212,33 +248,35 @@ module Bosh::Director
             offending_template2 = templates.find { |t| t.release == release2 }
 
             raise JobPackageCollision,
-                  "Package name collision detected in job `#{@name}': "\
+              "Package name collision detected in job `#{@name}': "\
                   "template `#{release1.name}/#{offending_template1.name}' depends on package `#{release1.name}/#{package_name}', "\
                   "template `#{release2.name}/#{offending_template2.name}' depends on `#{release2.name}/#{package_name}'. " +
-                  'BOSH cannot currently collocate two packages with identical names from separate releases.'
+                'BOSH cannot currently collocate two packages with identical names from separate releases.'
           end
         end
       end
 
       def bind_unallocated_vms
         instances.each do |instance|
-          instance.bind_unallocated_vm
-
-          # Now that we know every VM has been allocated and
-          # instance models are bound, we can sync the state.
-          instance.sync_state_with_db
+          instance.ensure_vm_allocated
         end
       end
 
-      def bind_instance_networks
-        instances.each do |instance|
-          instance.network_reservations.each do |net_name, reservation|
-            unless reservation.reserved?
-              network = @deployment.network(net_name)
-              network.reserve!(reservation, "`#{name}/#{instance.index}'")
-              instance.vm.use_reservation(reservation) if instance.vm
-            end
-          end
+      def bind_instances(ip_provider)
+        instances.each(&:ensure_model_bound)
+        bind_unallocated_vms
+        bind_instance_networks(ip_provider)
+      end
+
+      #TODO: Job should not be responsible for reserving IPs. Consider moving this somewhere else? Maybe in the consumer?
+      def bind_instance_networks(ip_provider)
+        needed_instance_plans
+          .flat_map(&:network_plans)
+          .reject(&:obsolete?)
+          .reject(&:existing?)
+          .each do |network_plan|
+          reservation = network_plan.reservation
+          ip_provider.reserve(reservation)
         end
       end
 
@@ -252,9 +290,34 @@ module Bosh::Director
 
       # reverse compatibility: translate disk size into a disk pool
       def persistent_disk=(disk_size)
-        disk_pool = DiskPool.new(SecureRandom.uuid)
-        disk_pool.disk_size = disk_size
-        @persistent_disk_pool = disk_pool
+        @persistent_disk_type = DiskType.new(SecureRandom.uuid, disk_size, {})
+      end
+
+      def instance_plans_with_missing_vms
+        needed_instance_plans.reject do |instance_plan|
+          instance_plan.instance.vm_created? || instance_plan.instance.state == 'detached'
+        end
+      end
+
+      def add_resolved_link(link_name, link_spec)
+        @resolved_links[link_name] = link_spec
+      end
+
+      def link_spec
+        @resolved_links
+      end
+
+      def link_path(template_name, link_name)
+        @link_paths.fetch(template_name, {})[link_name]
+      end
+
+      def add_link_path(template_name, link_name, link_path)
+        @link_paths[template_name] ||= {}
+        @link_paths[template_name][link_name] = link_path
+      end
+
+      def compilation?
+        false
       end
 
       private
@@ -276,8 +339,8 @@ module Bosh::Director
 
         raise JobIncompatibleSpecs,
           "Job `#{name}' has specs with conflicting property definition styles between" +
-          " its job spec templates.  This may occur if colocating jobs, one of which has a spec file including" +
-          " `properties' and one which doesn't."
+            " its job spec templates.  This may occur if colocating jobs, one of which has a spec file including" +
+            " `properties' and one which doesn't."
       end
 
       def extract_template_properties(collection)