bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/deployment_plan/vm.rb

@@ -8,112 +8,14 @@ module Bosh::Director
     # (network & storage) required for the VM to match the instance
     # requirements.
     class Vm
-      # @return [DeploymentPlan::ResourcePool] Associated resource pool
-      attr_reader :resource_pool
-
-      # @return [NetworkReservation] VM network reservation
-      attr_accessor :network_reservation
-
       # @return [Models::Vm] Associated DB model
       attr_accessor :model
 
-      # @return [Hash] Current state as provided by the BOSH Agent
-      attr_writer :current_state
-
       # @return [DeploymentPlan::Instance, nil] Instance that reserved this VM
       attr_accessor :bound_instance
 
-      def current_state
-        if @current_state
-          @current_state.delete('release')
-        end
-        @current_state
-      end
-
-      ##
-      # Creates a new idle VM reference for the specific resource pool
-      # @param [DeploymentPlan::ResourcePool] resource_pool Resource pool
-      def initialize(resource_pool)
-        @resource_pool = resource_pool
-        @current_state = nil
-        @bound_instance = nil
-        @network_reservation = nil
-        @model = nil
-      end
-
-      #
-      # @return [Boolean] Does this VM have a network reservation?
-      def has_network_reservation?
-        !@network_reservation.nil?
-      end
-
-      #
-      # Uses provided network reservation
-      # @param [NetworkReservation] reservation Network reservation
-      def use_reservation(reservation)
-        @network_reservation = reservation
-      end
-
-      #
-      # Releases current network reservation (if any)
-      # @return [void]
-      def release_reservation
-        if has_network_reservation?
-          @resource_pool.network.release(@network_reservation)
-          @network_reservation = nil
-        end
-      end
-
-      ##
-      # @return [Hash] BOSH network settings used for Agent apply call
-      def network_settings
-        # use the instance network settings if bound, otherwise use the one
-        # provided by the resource pool
-        if @bound_instance
-          @bound_instance.network_settings
-        else
-          unless @network_reservation
-            raise NetworkReservationMissing,
-                  'Missing network reservation for resource pool VM'
-          end
-
-          network_settings = {}
-          network = @resource_pool.network
-          network_settings[network.name] = network.network_settings(
-              @network_reservation)
-          network_settings
-        end
-      end
-
-      ##
-      # @return [Boolean] returns true if the expected network configuration
-      #   differs from the one provided by the VM
-      def networks_changed?
-        network_settings != @current_state['networks']
-      end
-
-      ##
-      # @return [Boolean] returns true if the expected resource pool
-      #   specification differs from the one provided by the VM
-      def resource_pool_changed?
-        return true if resource_pool.spec != @current_state['resource_pool']
-        return true if resource_pool.deployment_plan.recreate
-        return true if @model && @model.env != resource_pool.env
-
-        false
-      end
-
-      ##
-      # @return [Boolean] returns true if the any of the expected specifications
-      #   differ from the ones provided by the VM
-      def changed?
-        resource_pool_changed? || networks_changed?
-      end
-
-      #TODO: rename 'clean'
-      def clean_vm
+      def clean
         self.model = nil
-        self.current_state = nil
       end
     end
   end

data/lib/bosh/director/deployment_plan/vm_type.rb

@@ -0,0 +1,27 @@
+module Bosh::Director
+  module DeploymentPlan
+    class VmType
+      include ValidationHelper
+
+      attr_reader :name
+
+      attr_reader :cloud_properties
+
+      def initialize(spec)
+
+        @name = safe_property(spec, "name", class: String)
+
+        @cloud_properties =
+          safe_property(spec, "cloud_properties", class: Hash, default: {})
+
+      end
+
+      def spec
+        {
+          "name" => @name,
+          "cloud_properties" => @cloud_properties,
+        }
+      end
+    end
+  end
+end

data/lib/bosh/director/disk_manager.rb

@@ -0,0 +1,268 @@
+module Bosh::Director
+  class DiskManager
+
+    def initialize(cloud, logger)
+      @cloud = cloud
+      @logger = logger
+      @transactor = Transactor.new
+    end
+
+    def update_persistent_disk(instance_plan, vm_recreator)
+      @logger.info('Updating persistent disk')
+      check_persistent_disk(instance_plan)
+
+      return unless instance_plan.persistent_disk_changed?
+
+      instance = instance_plan.instance
+      old_disk = instance.model.persistent_disk
+
+      disk = nil
+      if instance_plan.needs_disk?
+        disk = create_and_attach_disk(instance_plan, vm_recreator)
+        mount_and_migrate_disk(instance, disk, old_disk)
+      end
+
+      @transactor.retryable_transaction(Bosh::Director::Config.db) do
+        old_disk.update(:active => false) if old_disk
+        disk.update(:active => true) if disk
+      end
+
+      delete_mounted_persistent_disk(instance, old_disk) if old_disk
+    end
+
+    def attach_disks_if_needed(instance_plan)
+      unless instance_plan.needs_disk?
+        @logger.warn('Skipping disk attachment, instance no longer needs disk')
+        return
+      end
+
+      instance = instance_plan.instance
+      disk_cid = instance.model.persistent_disk_cid
+      return @logger.info('Skipping disk attaching') if disk_cid.nil?
+      vm_model = instance.vm.model
+      begin
+        @cloud.attach_disk(vm_model.cid, disk_cid)
+        AgentClient.with_vm(vm_model).mount_disk(disk_cid)
+      rescue => e
+        @logger.warn("Failed to attach disk to new VM: #{e.inspect}")
+        raise e
+      end
+    end
+
+    def delete_persistent_disks(instance_model)
+      instance_model.persistent_disks.each do |disk|
+        orphan_disk(disk)
+      end
+    end
+
+    def orphan_disk(disk)
+      @transactor.retryable_transaction(Bosh::Director::Config.db) do
+        orphan_disk = Models::OrphanDisk.create(
+          disk_cid: disk.disk_cid,
+          size: disk.size,
+          availability_zone: disk.instance.availability_zone,
+          deployment_name: disk.instance.deployment.name,
+          instance_name: "#{disk.instance.job}/#{disk.instance.uuid}",
+          cloud_properties: disk.cloud_properties
+        )
+
+        orphan_snapshots(disk.snapshots, orphan_disk)
+        @logger.info("Orphaning disk: '#{disk.disk_cid}', " +
+            "#{disk.active ? "active" : "inactive"}")
+
+        disk.destroy
+      end
+    end
+
+    def list_orphan_disks
+      Models::OrphanDisk.all.map do |disk|
+        {
+          'disk_cid' => disk.disk_cid,
+          'size' => disk.size,
+          'az' => disk.availability_zone,
+          'deployment_name' => disk.deployment_name,
+          'instance_name' => disk.instance_name,
+          'cloud_properties' => disk.cloud_properties,
+          'orphaned_at' => disk.created_at.to_s
+        }
+      end
+    end
+
+    def delete_orphan_disk_by_disk_cid(disk_cid)
+      @logger.info("Deleting orphan disk: #{disk_cid}")
+      orphan_disk = Bosh::Director::Models::OrphanDisk.where(disk_cid: disk_cid).first
+      if orphan_disk
+        delete_orphan_disk(orphan_disk)
+      else
+        @logger.debug("Disk not found: #{disk_cid}")
+      end
+    end
+
+    def unmount_disk_for(instance_plan)
+      disk = instance_plan.instance.model.persistent_disk
+      return if disk.nil?
+      unmount(instance_plan.instance, disk)
+    end
+
+    def delete_orphan_disk(orphan_disk)
+      begin
+        orphan_disk.orphan_snapshots.each do |orphan_snapshot|
+          delete_orphan_snapshot(orphan_snapshot)
+        end
+        @logger.info("Deleting orphan orphan disk: #{orphan_disk.disk_cid}")
+        @cloud.delete_disk(orphan_disk.disk_cid)
+        orphan_disk.destroy
+      rescue Bosh::Clouds::DiskNotFound
+        @logger.debug("Disk not found in IaaS: #{orphan_disk.disk_cid}")
+        orphan_disk.destroy
+      end
+    end
+
+    private
+
+    def delete_orphan_snapshot(orphan_snapshot)
+      begin
+        snapshot_cid = orphan_snapshot.snapshot_cid
+        @logger.info("Deleting orphan snapshot: #{snapshot_cid}")
+        @cloud.delete_snapshot(snapshot_cid)
+        orphan_snapshot.destroy
+      rescue Bosh::Clouds::DiskNotFound
+        @logger.debug("Disk not found in IaaS: #{snapshot_cid}")
+        orphan_snapshot.destroy
+      end
+    end
+
+    def orphan_snapshots(snapshots, orphan_disk)
+      snapshots.each do |snapshot|
+        @logger.info("Orphaning snapshot: '#{snapshot.snapshot_cid}'")
+        Models::OrphanSnapshot.create(
+          orphan_disk: orphan_disk,
+          snapshot_cid: snapshot.snapshot_cid,
+          clean: snapshot.clean,
+          snapshot_created_at: snapshot.created_at
+        )
+        snapshot.delete
+      end
+    end
+
+    def delete_mounted_persistent_disk(instance, disk)
+      unmount(instance, disk)
+
+      disk_cid = disk.disk_cid
+      if disk_cid.nil?
+        @logger.info('Skipping disk detaching, instance does not have a disk')
+        return
+      end
+
+      begin
+        @logger.info("Detaching disk #{disk_cid}")
+        @cloud.detach_disk(instance.model.vm.cid, disk_cid)
+      rescue Bosh::Clouds::DiskNotAttached
+        if disk.active
+          raise CloudDiskNotAttached,
+            "`#{instance}' VM should have persistent disk attached " +
+              "but it doesn't (according to CPI)"
+        end
+      end
+
+      orphan_disk(disk)
+    end
+
+    def unmount(instance, disk)
+      disk_cid = disk.disk_cid
+      if disk_cid.nil?
+        @logger.info('Skipping disk unmounting, instance does not have a disk')
+        return
+      end
+
+      if disks(instance).include?(disk_cid)
+        @logger.info("Stopping instance '#{instance}' before unmount")
+        agent(instance).stop
+        @logger.info("Unmounting disk '#{disk_cid}'")
+        agent(instance).unmount_disk(disk_cid)
+      end
+    end
+
+    # Synchronizes persistent_disks with the agent.
+    # (Currently assumes that we only have 1 persistent disk.)
+    # @return [void]
+    def check_persistent_disk(instance_plan)
+      instance = instance_plan.instance
+      return if instance.model.persistent_disks.empty?
+      agent_disk_cid = disks(instance).first
+
+      if agent_disk_cid.nil? && !instance_plan.needs_disk?
+        @logger.debug('Disk is already detached')
+      elsif agent_disk_cid != instance.model.persistent_disk_cid
+        raise AgentDiskOutOfSync,
+          "`#{instance}' has invalid disks: agent reports " +
+            "`#{agent_disk_cid}' while director record shows " +
+            "`#{instance.model.persistent_disk_cid}'"
+      end
+
+      instance.model.persistent_disks.each do |disk|
+        unless disk.active
+          @logger.warn("`#{instance}' has inactive disk #{disk.disk_cid}")
+        end
+      end
+    end
+
+    def disks(instance)
+      agent(instance).list_disk
+    end
+
+    def agent(instance)
+      AgentClient.with_vm(instance.vm.model)
+    end
+
+    def create_and_attach_disk(instance_plan, vm_recreator)
+      instance = instance_plan.instance
+      disk = create_disk(instance_plan)
+      @cloud.attach_disk(instance.model.vm.cid, disk.disk_cid)
+      return disk
+    rescue Bosh::Clouds::NoDiskSpace => e
+      if e.ok_to_retry
+        @logger.warn('Retrying attach disk operation after persistent disk update failed')
+        # Re-creating the vm may cause it to be re-created in a place with more storage
+        unmount_disk_for(instance_plan)
+        vm_recreator.recreate_vm(instance_plan, disk.disk_cid)
+        begin
+          @cloud.attach_disk(instance.model.vm.cid, disk.disk_cid)
+        rescue
+          orphan_disk(disk)
+          raise
+        end
+      else
+        orphan_disk(disk)
+        raise
+      end
+      return disk
+    end
+
+    def mount_and_migrate_disk(instance, new_disk, old_disk)
+      agent(instance).mount_disk(new_disk.disk_cid)
+      agent(instance).migrate_disk(old_disk.disk_cid, new_disk.disk_cid) if old_disk
+    rescue => e
+      @logger.debug("Failed to migrate disk, deleting new disk. #{e.inspect}")
+      delete_mounted_persistent_disk(instance, new_disk)
+      raise e
+    end
+
+    def create_disk(instance_plan)
+      job = instance_plan.desired_instance.job
+      instance_model = instance_plan.instance.model
+
+      disk_size = job.persistent_disk_type.disk_size
+      cloud_properties = job.persistent_disk_type.cloud_properties
+
+      disk_cid = @cloud.create_disk(disk_size, cloud_properties, instance_model.vm.cid)
+      Models::PersistentDisk.create(
+        disk_cid: disk_cid,
+        active: false,
+        instance_id: instance_model.id,
+        size: disk_size,
+        cloud_properties: cloud_properties,
+      )
+    end
+  end
+end

data/lib/bosh/director/dns/canonicalizer.rb

@@ -0,0 +1,28 @@
+module Bosh::Director
+  class Canonicalizer
+
+    def self.canonicalize(string, opts = {})
+      # a-z, 0-9, -, case insensitive, and must start with a letter
+      string = string.downcase.gsub(/_/, "-")
+      if opts[:allow_dots]
+        string = string.gsub(/[^a-z0-9\-\.]/, "")
+      else
+        string = string.gsub(/[^a-z0-9\-]/, "")
+      end
+
+      validate_dns_name(string)
+    end
+
+    def self.validate_dns_name(string)
+      if string =~ /^(\d|-)/
+        raise DnsInvalidCanonicalName,
+              "Invalid DNS canonical name `#{string}', must begin with a letter"
+      end
+      if string =~ /-$/
+        raise DnsInvalidCanonicalName,
+              "Invalid DNS canonical name `#{string}', can't end with a hyphen"
+      end
+      string
+    end
+  end
+end

data/lib/bosh/director/dns/dns_manager.rb

@@ -0,0 +1,163 @@
+module Bosh::Director
+  class DnsManager
+    attr_reader :dns_domain_name
+
+    def self.create
+      dns_config = Config.dns || {}
+      dns_enabled = !!Config.dns_db # to be consistent with current behavior
+      logger = Config.logger
+      local_dns_repo = LocalDnsRepo.new(logger)
+      dns_domain_name = Canonicalizer.canonicalize(dns_config.fetch('domain_name', 'bosh'), :allow_dots => true)
+      dns_provider = PowerDns.new(dns_domain_name, logger)
+
+      new(dns_domain_name, dns_config, dns_enabled, dns_provider, local_dns_repo, logger)
+    end
+
+    def initialize(dns_domain_name, dns_config, dns_enabled, dns_provider, local_dns_repo, logger)
+      @dns_domain_name = dns_domain_name
+      @dns_provider = dns_provider
+      @dns_enabled = dns_enabled
+      @default_server = dns_config['server']
+      @flush_command = dns_config['flush_command']
+      @ip_address = dns_config['address']
+      @local_dns_repo = local_dns_repo
+      @logger = logger
+    end
+
+    def dns_enabled?
+      @dns_enabled
+    end
+
+    def configure_nameserver
+      return unless dns_enabled?
+
+      @dns_provider.create_or_update_nameserver(@ip_address)
+    end
+
+    def find_dns_record(dns_record_name, ip_address)
+      @dns_provider.find_dns_record(dns_record_name, ip_address)
+    end
+
+    def find_dns_record_names_by_instance(instance_model)
+      instance_model.nil? ? [] : instance_model.dns_record_names.to_a.compact
+    end
+
+    def update_dns_record_for_instance(instance_model, dns_names_to_ip)
+      current_dns_records = @local_dns_repo.find(instance_model)
+      new_dns_records = []
+      dns_names_to_ip.each do |record_name, ip_address|
+        new_dns_records << record_name
+        @logger.info("Updating DNS for: #{record_name} to #{ip_address}")
+        @dns_provider.create_or_update_dns_records(record_name, ip_address)
+      end
+      dns_records = (current_dns_records + new_dns_records).uniq
+      @local_dns_repo.create_or_update(instance_model, dns_records)
+    end
+
+    def migrate_legacy_records(instance_model)
+      return unless dns_enabled?
+
+      return if @local_dns_repo.find(instance_model).any?
+
+      index_pattern_for_all_networks = dns_record_name(
+        instance_model.index,
+        instance_model.job,
+        '%',
+        instance_model.deployment.name
+      )
+      uuid_pattern_for_all_networks = dns_record_name(
+        instance_model.uuid,
+        instance_model.job,
+        '%',
+        instance_model.deployment.name
+      )
+
+      legacy_record_names = [index_pattern_for_all_networks, uuid_pattern_for_all_networks]
+        .map { |pattern| @dns_provider.find_dns_records_by_pattern(pattern) }
+        .flatten
+        .map(&:name)
+
+      @local_dns_repo.create_or_update(instance_model, legacy_record_names)
+    end
+
+    def delete_dns_for_instance(instance_model)
+      return unless dns_enabled?
+
+      current_dns_records = @local_dns_repo.find(instance_model)
+      if current_dns_records.empty?
+        # for backwards compatibility when old instances
+        # did not have records in local repo
+        # we cannot migrate them because powerdns can be different database
+        # those instance only had index-based dns records (before global-net)
+        index_record_pattern = dns_record_name(instance_model.index, instance_model.job, '%', instance_model.deployment.name)
+        @dns_provider.delete(index_record_pattern)
+        return
+      end
+
+      current_dns_records.each do |record_name|
+        @logger.info("Removing DNS for: #{record_name}")
+        @dns_provider.delete(record_name)
+      end
+
+      @local_dns_repo.delete(instance_model)
+    end
+
+    # build a list of dns servers to use
+    def dns_servers(network, dns_spec, add_default_dns = true)
+      servers = nil
+
+      if dns_spec
+        servers = []
+        dns_spec.each do |dns|
+          dns = NetAddr::CIDR.create(dns)
+          unless dns.size == 1
+            raise NetworkInvalidDns,
+              "Invalid DNS for network `#{network}': must be a single IP"
+          end
+
+          servers << dns.ip
+        end
+      end
+
+      return servers unless add_default_dns
+      add_default_dns_server(servers)
+    end
+
+    # Purge cached DNS records
+    def flush_dns_cache
+      if @flush_command && !@flush_command.empty?
+        stdout, stderr, status = Open3.capture3(@flush_command)
+        if status == 0
+          @logger.debug("Flushed #{stdout.chomp} records from DNS cache")
+        else
+          @logger.warn("Failed to flush DNS cache: #{stderr.chomp}")
+        end
+      end
+    end
+
+    def dns_record_name(hostname, job_name, network_name, deployment_name)
+      network_name = Canonicalizer.canonicalize(network_name) unless network_name == '%'
+
+      [ hostname,
+        Canonicalizer.canonicalize(job_name),
+        network_name,
+        Canonicalizer.canonicalize(deployment_name),
+        @dns_domain_name
+      ].join('.')
+    end
+
+    private
+
+    # add default dns server to an array of dns servers
+    def add_default_dns_server(servers)
+      return servers unless dns_enabled?
+
+      unless @default_server.to_s.empty? || @default_server == '127.0.0.1'
+        (servers ||= []) << @default_server
+        servers.uniq!
+      end
+
+      servers
+    end
+  end
+end