bosh-director 1.5.0.pre.1113

data/lib/bosh/director/resource_pool_updater.rb

@@ -0,0 +1,216 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  class ResourcePoolUpdater
+
+    def initialize(resource_pool)
+      @resource_pool = resource_pool
+      @cloud = Config.cloud
+      @logger = Config.logger
+      @event_log = Config.event_log
+    end
+
+    ##
+    # Creates VMs that are considered missing from the deployment
+    #
+    # @param [ThreadPool] thread_pool Thread pool that will be used to
+    #   parallelize the operation
+    # @yield [idle_vm] filter for which missing VMs to create
+    def create_missing_vms(thread_pool)
+      counter = 0
+      vms_to_process = []
+
+      each_idle_vm do |idle_vm|
+        next if idle_vm.vm
+        if !block_given? || yield(idle_vm)
+          counter += 1
+          vms_to_process << idle_vm
+        end
+      end
+
+      @logger.info("Creating #{counter} missing VMs")
+      vms_to_process.each_with_index do |idle_vm, index|
+        thread_pool.process do
+          @event_log.track("#{@resource_pool.name}/#{index}") do
+            with_thread_name("create_missing_vm(#{@resource_pool.name}, #{index}/#{counter})") do
+              @logger.info("Creating missing VM")
+              create_missing_vm(idle_vm)
+            end
+          end
+        end
+      end
+    end
+
+    # Creates missing VMs that have bound instances
+    # (as opposed to missing resource pool VMs)
+    def create_bound_missing_vms(thread_pool)
+      create_missing_vms(thread_pool) { |idle_vm| !idle_vm.bound_instance.nil? }
+    end
+
+    def create_missing_vm(idle_vm)
+      deployment = @resource_pool.deployment_plan.model
+      stemcell = @resource_pool.stemcell.model
+
+      vm = VmCreator.new.create(deployment, stemcell, @resource_pool.cloud_properties,
+                                idle_vm.network_settings, nil, @resource_pool.env)
+
+      agent = AgentClient.new(vm.agent_id)
+      agent.wait_until_ready
+
+      update_state(agent, vm, idle_vm)
+
+      idle_vm.vm = vm
+      idle_vm.current_state = agent.get_state
+    rescue Exception => e
+      @logger.info("Cleaning up the created VM due to an error: #{e}")
+      begin
+        @cloud.delete_vm(vm.cid) if vm && vm.cid
+        vm.destroy if vm && vm.id
+      rescue Exception
+        @logger.info("Could not cleanup VM: #{vm.cid}") if vm
+      end
+      raise e
+    end
+
+    def update_state(agent, vm, idle_vm)
+      state = {
+          "deployment" => @resource_pool.deployment_plan.name,
+          "resource_pool" => @resource_pool.spec,
+          "networks" => idle_vm.network_settings
+      }
+
+      vm.update(:apply_spec => state)
+      agent.apply(state)
+    end
+
+    # Deletes extra VMs in a resource pool
+    # @param thread_pool Thread pool used to parallelize delete operations
+    def delete_extra_vms(thread_pool)
+      count = extra_vm_count
+      @logger.info("Deleting #{count} extra VMs")
+
+      count.times do
+        idle_vm = @resource_pool.idle_vms.shift
+        vm_cid = idle_vm.vm.cid
+
+        thread_pool.process do
+          @event_log.track("#{@resource_pool.name}/#{vm_cid}") do
+            @logger.info("Deleting extra VM: #{vm_cid}")
+            @cloud.delete_vm(vm_cid)
+            idle_vm.vm.destroy
+          end
+        end
+      end
+    end
+
+    def delete_outdated_idle_vms(thread_pool)
+      count = outdated_idle_vm_count
+      index = 0
+      index_lock = Mutex.new
+
+      @logger.info("Deleting #{count} outdated idle VMs")
+
+      @resource_pool.idle_vms.each do |idle_vm|
+        next unless idle_vm.vm && idle_vm.changed?
+        vm_cid = idle_vm.vm.cid
+
+        thread_pool.process do
+          @event_log.track("#{@resource_pool.name}/#{vm_cid}") do
+            index_lock.synchronize { index += 1 }
+
+            with_thread_name("delete_outdated_vm(#{@resource_pool.name}, #{index - 1}/#{count})") do
+              @logger.info("Deleting outdated VM: #{vm_cid}")
+              @cloud.delete_vm(vm_cid)
+              vm = idle_vm.vm
+              idle_vm.vm = nil
+              idle_vm.current_state = nil
+              vm.destroy
+            end
+          end
+        end
+      end
+    end
+
+    # Attempts to allocate a dynamic IP address for all idle VMs
+    # (unless they already have one). This allows us to fail earlier
+    # in case any of resource pools is not big enough to accommodate
+    # those VMs.
+    def reserve_networks
+      network = @resource_pool.network
+
+      each_idle_vm do |idle_vm|
+        unless idle_vm.network_reservation
+          reservation = NetworkReservation.new(
+              :type => NetworkReservation::DYNAMIC)
+          network.reserve(reservation)
+
+          unless reservation.reserved?
+            case reservation.error
+              when NetworkReservation::CAPACITY
+                raise NetworkReservationNotEnoughCapacity,
+                      "`#{name}/#{index}' asked for a dynamic IP " +
+                      "but there were no more available"
+              else
+                raise NetworkReservationError,
+                      "`#{name}/#{index}' failed to reserve " +
+                      "dynamic IP: #{reservation.error}"
+            end
+          end
+
+          idle_vm.network_reservation = reservation
+        end
+      end
+    end
+
+    def generate_agent_id
+      SecureRandom.uuid
+    end
+
+    def each_idle_vm
+      @resource_pool.allocated_vms.each { |idle_vm| yield idle_vm }
+      @resource_pool.idle_vms.each { |idle_vm| yield idle_vm }
+    end
+
+    def each_idle_vm_with_index
+      index = 0
+      each_idle_vm do |idle_vm|
+        yield(idle_vm, index)
+        index += 1
+      end
+    end
+
+    def extra_vm_count
+      @resource_pool.active_vm_count +
+          @resource_pool.idle_vms.size +
+          @resource_pool.allocated_vms.size -
+          @resource_pool.size
+    end
+
+    def outdated_idle_vm_count
+      counter = 0
+      @resource_pool.idle_vms.each do |idle_vm|
+        counter += 1 if idle_vm.vm && idle_vm.changed?
+      end
+      counter
+    end
+
+    def missing_vm_count
+      counter = 0
+      each_idle_vm do |idle_vm|
+        next if idle_vm.vm
+        counter += 1
+      end
+      counter
+    end
+
+    def bound_missing_vm_count
+      counter = 0
+      each_idle_vm do |idle_vm|
+        next if idle_vm.vm
+        next if idle_vm.bound_instance.nil?
+        counter += 1
+      end
+      counter
+    end
+  end
+end

data/lib/bosh/director/scheduler.rb

@@ -0,0 +1,57 @@
+require 'rufus/scheduler'
+
+module Bosh::Director
+  class Scheduler
+    def initialize(scheduled_jobs=[], options={})
+      if scheduled_jobs.nil? || scheduled_jobs.is_a?(Array)
+        @scheduled_jobs = scheduled_jobs
+        @scheduler = options.fetch(:scheduler) { Rufus::Scheduler::PlainScheduler.new }
+        @queue = options.fetch(:queue) { JobQueue.new }
+      else
+        raise 'scheduled_jobs must be an array'
+      end
+    end
+
+    def start!
+      logger.info('starting scheduler')
+      add_jobs unless @added_already
+      @scheduler.start
+      @scheduler.join
+    end
+
+    def stop!
+      logger.info('stopping scheduler')
+      @scheduler.stop
+    end
+
+    def logger
+      @logger ||= Config.logger
+    end
+
+    private
+
+    def add_jobs
+      return if @scheduled_jobs.nil?
+      @added_already = true
+
+      @scheduled_jobs.each do |scheduled_job|
+        begin
+          director_job_class = Bosh::Director::Jobs.const_get(scheduled_job['command'].to_s)
+        rescue NameError
+          raise "unknown job `Bosh::Director::Jobs::#{scheduled_job['command']}'"
+        end
+
+        @scheduler.cron(scheduled_job['schedule']) do |_|
+          logger.info("enqueueing `#{scheduled_job['command']}'")
+
+          @queue.enqueue('scheduler',
+                         director_job_class,
+                         "scheduled #{scheduled_job['command']}",
+                         scheduled_job['params'])
+        end
+
+        logger.info("added scheduled job `#{director_job_class}' with interval '#{scheduled_job['schedule']}'")
+      end
+    end
+  end
+end

data/lib/bosh/director/sequel.rb

@@ -0,0 +1,13 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+Sequel.extension :blank
+
+Sequel::Model.plugin :validation_helpers
+Sequel::Model.raise_on_typecast_failure = false
+
+[:exact_length, :format, :includes, :integer, :length_range, :max_length,
+ :min_length, :not_string, :numeric, :type, :presence, :unique].each do |validation|
+  Sequel::Plugins::ValidationHelpers::DEFAULT_OPTIONS[validation][:message] = validation
+end
+
+Sequel::Plugins::ValidationHelpers::DEFAULT_OPTIONS[:max_length][:nil_message] = :max_length

data/lib/bosh/director/tar_gzipper.rb

@@ -0,0 +1,47 @@
+require 'tmpdir'
+require 'open3'
+
+module Bosh::Director
+  class TarGzipper
+    include Bosh::RunsCommands
+
+    # @param [String] base_dir the directory from which the tar command is run
+    # @param [String, Array] sources the relative paths to include
+    # @param [String] dest the destination filename for the tgz output
+    # @param [Hash] options the options for compress
+    # @option options [Boolean] :copy_first copy the source to a temp dir before archiving
+    def compress(base_dir, sources, dest, options={})
+      sources = [*sources]
+
+      sources.each do |source|
+        raise "Sources must have a path depth of 1 and contain no '#{File::SEPARATOR}'" if source.include?(File::SEPARATOR)
+      end
+
+      base_dir_path = Pathname.new(base_dir)
+
+      unless base_dir_path.exist?
+        raise "The base directory #{base_dir} could not be found."
+      end
+
+      unless base_dir_path.absolute?
+        raise "The base directory #{base_dir} is not an absolute path."
+      end
+
+      if options[:copy_first]
+        Dir.mktmpdir do |tmpdir|
+          FileUtils.cp_r(sources.map { |s| File.join(base_dir, s) }, "#{tmpdir}/")
+          tar(tmpdir, dest, sources)
+        end
+      else
+        tar(base_dir, dest, sources)
+      end
+    end
+
+    private
+    def tar(base_dir, dest, sources)
+      out, err, status = Open3.capture3('tar', '-C', base_dir, '-czf', dest, *sources)
+      raise("tar exited #{status.exitstatus}, output: '#{out}', error: '#{err}'") unless status.success?
+      out
+    end
+  end
+end

data/lib/bosh/director/task_result_file.rb

@@ -0,0 +1,19 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+
+  class TaskResultFile
+    def initialize(file_name)
+      @file = File.open(file_name, "w")
+      @lock = Mutex.new
+    end
+
+    def write(result)
+      @lock.synchronize do
+        @file.write(result)
+        @file.flush
+      end
+    end
+  end
+end
+

data/lib/bosh/director/thread_pool.rb

@@ -0,0 +1,8 @@
+module Bosh::Director
+  class ThreadPool < Bosh::ThreadPool
+    def initialize(options)
+      options[:logger] ||= Config.logger
+      super(options)
+    end
+  end
+end

data/lib/bosh/director/validation_helper.rb

@@ -0,0 +1,55 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module ValidationHelper
+
+    def safe_property(hash, property, options = {})
+      result = nil
+
+      if hash && hash.has_key?(property)
+        result = hash[property]
+
+        if options[:class]
+
+          if options[:class] == :boolean
+            unless result.kind_of?(TrueClass) || result.kind_of?(FalseClass)
+              invalid_type(property, options[:class], result)
+            end
+
+          elsif !result.kind_of?(options[:class])
+            if options[:class] == String && result.kind_of?(Numeric)
+              result = result.to_s
+            else
+              invalid_type(property, options[:class], result)
+            end
+          end
+
+        end
+
+        if options[:min] && result < options[:min]
+          raise ValidationViolatedMin,
+                "`#{property}' value (#{result}) should be greater than #{options[:min]}"
+        end
+
+        if options[:max] && result > options[:max]
+          raise ValidationViolatedMax,
+                "`#{property}' value (#{result}) should be less than #{options[:max]}"
+        end
+
+      elsif options[:default]
+        result = options[:default]
+
+      elsif !options[:optional]
+        raise ValidationMissingField,
+              "Required property `#{property}' was not specified in #{self.class.name}"
+      end
+      result
+    end
+
+    def invalid_type(property, klass, value)
+      raise ValidationInvalidType,
+            "Property `#{property}' (value #{value.inspect}) did not match the required type `#{klass}'"
+    end
+
+  end
+end

data/lib/bosh/director/version.rb

@@ -0,0 +1,7 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh
+  module Director
+    VERSION = '1.5.0.pre.1113'
+  end
+end

data/lib/bosh/director/vm_creator.rb

@@ -0,0 +1,80 @@
+require 'common/deep_copy'
+
+module Bosh::Director
+  # Creates VM model and call out to CPI to create VM in IaaS
+  class VmCreator
+    include EncryptionHelper
+
+    # this is used to retry VM creation on clouds that are unreliable (e.g. AWS)
+    MAX_CREATE_VM_TRIES = 5
+
+    def self.create(*args)
+      new.create(*args)
+    end
+
+    def initialize
+      @cloud = Config.cloud
+      @logger = Config.logger
+    end
+
+    def create(deployment, stemcell, cloud_properties, network_settings,
+               disks=nil, env={})
+      vm = nil
+      vm_cid = nil
+
+      env = Bosh::Common::DeepCopy.copy(env)
+
+      agent_id = self.class.generate_agent_id
+
+      options = {
+          :deployment => deployment,
+          :agent_id => agent_id,
+          :env => env
+      }
+
+      if Config.encryption?
+        credentials = generate_agent_credentials
+        env["bosh"] ||= {}
+        env["bosh"]["credentials"] = credentials
+        options[:credentials] = credentials
+      end
+
+      count = 0
+      begin
+        vm_cid = @cloud.create_vm(agent_id, stemcell.cid, cloud_properties, network_settings, disks, env)
+      rescue Bosh::Clouds::VMCreationFailed => e
+        count += 1
+        logger.error("failed to create VM, retrying (#{count})")
+        retry if e.ok_to_retry && count < MAX_CREATE_VM_TRIES
+        raise e
+      end
+
+      options[:cid] = vm_cid
+      vm = Models::Vm.new(options)
+
+      vm.save
+      VmMetadataUpdater.build.update(vm, {})
+      vm
+    rescue => e
+      logger.error("error creating vm: #{e.message}")
+      delete_vm(vm_cid) if vm_cid
+      vm.destroy if vm
+      raise e
+    end
+
+    def delete_vm(vm_cid)
+      @cloud.delete_vm(vm_cid)
+    rescue => e
+      logger.error("error cleaning up #{vm_cid}: #{e.message}\n#{e.backtrace.join("\n")}")
+    end
+
+    def self.generate_agent_id
+      SecureRandom.uuid
+    end
+
+    def logger
+      Config.logger
+    end
+
+  end
+end