bosh-director 1.5.0.pre.1113

data/lib/bosh/director/blobstores.rb

@@ -0,0 +1,29 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  class Blobstores
+
+    attr_reader :blobstore
+
+    def initialize(config)
+      @blobstore = create_client(config.hash.fetch('blobstore'))
+
+      bd_config = config.hash['backup_destination']
+      @backup_destination = create_client(bd_config) if bd_config
+    end
+
+    def backup_destination
+      raise 'No backup destination configured' unless @backup_destination
+      @backup_destination
+    end
+
+    private
+
+    def create_client(hash)
+      provider = hash.fetch('provider')
+      options = hash.fetch('options')
+
+      Bosh::Blobstore::Client.create(provider, options)
+    end
+  end
+end

data/lib/bosh/director/client.rb

@@ -0,0 +1,156 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  class Client
+
+    attr_accessor :id
+
+    def initialize(service_name, client_id, options = {})
+      @service_name = service_name
+      @client_id = client_id
+      @nats_rpc = Config.nats_rpc
+      @timeout = options[:timeout] || 45
+      @logger = Config.logger
+      @retry_methods = options[:retry_methods] || {}
+
+      if options[:credentials]
+        @encryption_handler =
+          Bosh::Core::EncryptionHandler.new(@client_id, options[:credentials])
+      end
+
+      @resource_manager = Api::ResourceManager.new
+    end
+
+    def method_missing(method_name, *args)
+      retries = @retry_methods[method_name] || 0
+      begin
+        handle_method(method_name, args)
+      rescue RpcTimeout
+        if retries > 0
+          retries -= 1
+          retry
+        end
+        raise
+      end
+    end
+
+    def wait_until_ready(deadline = 600)
+      old_timeout = @timeout
+      @timeout = 1.0
+      @deadline = Time.now.to_i + deadline
+
+      begin
+        ping
+      rescue RpcTimeout
+        retry if @deadline - Time.now.to_i > 0
+        raise RpcTimeout, "Timed out pinging to #{@client_id} after #{deadline} seconds"
+      rescue RpcRemoteException => e
+        retry if e.message =~ /^restarting agent/ && @deadline - Time.now.to_i > 0
+        raise e
+      ensure
+        @timeout = old_timeout
+      end
+    end
+
+    def handle_method(method_name, args)
+      result = {}
+      result.extend(MonitorMixin)
+
+      cond = result.new_cond
+      timeout_time = Time.now.to_f + @timeout
+
+      request = {:method => method_name, :arguments => args}
+
+      if @encryption_handler
+        @logger.info("Request: #{request}")
+        request = {"encrypted_data" => @encryption_handler.encrypt(request)}
+        request["session_id"] = @encryption_handler.session_id
+      end
+
+      recipient = "#{@service_name}.#{@client_id}"
+
+      request_id = @nats_rpc.send_request(recipient, request) do |response|
+        if @encryption_handler
+          begin
+            response = @encryption_handler.decrypt(response["encrypted_data"])
+          rescue Bosh::Core::EncryptionHandler::CryptError => e
+            response["exception"] = "CryptError: #{e.inspect} #{e.backtrace}"
+          end
+          @logger.info("Response: #{response}")
+        end
+
+        result.synchronize do
+          inject_compile_log(response)
+          result.merge!(response)
+          cond.signal
+        end
+      end
+
+      result.synchronize do
+        while result.empty?
+          timeout = timeout_time - Time.now.to_f
+          unless timeout > 0
+            @nats_rpc.cancel_request(request_id)
+            raise RpcTimeout,
+                  "Timed out sending `#{method_name}' to #{@client_id} " +
+                  "after #{@timeout} seconds"
+          end
+          cond.wait(timeout)
+        end
+      end
+
+      if result.has_key?("exception")
+        raise RpcRemoteException, format_exception(result["exception"])
+      end
+
+      result["value"]
+    end
+
+    # Returns formatted exception information
+    # @param [Hash|#to_s] exception Serialized exception
+    # @return [String]
+    def format_exception(exception)
+      return exception.to_s unless exception.is_a?(Hash)
+
+      msg = exception["message"].to_s
+
+      if exception["backtrace"]
+        msg += "\n"
+        msg += Array(exception["backtrace"]).join("\n")
+      end
+
+      if exception["blobstore_id"]
+        blob = download_and_delete_blob(exception["blobstore_id"])
+        msg += "\n"
+        msg += blob.to_s
+      end
+
+      msg
+    end
+
+    private
+
+    # the blob is removed from the blobstore once we have fetched it,
+    # but if there is a crash before it is injected into the response
+    # and then logged, there is a chance that we lose it
+    def inject_compile_log(response)
+      if response["value"] && response["value"].is_a?(Hash) &&
+        response["value"]["result"].is_a?(Hash) &&
+        blob_id = response["value"]["result"]["compile_log_id"]
+        compile_log = download_and_delete_blob(blob_id)
+        response["value"]["result"]["compile_log"] = compile_log
+      end
+    end
+
+    # Downloads blob and ensures it's deleted from the blobstore
+    # @param [String] blob_id Blob id
+    # @return [String] Blob contents
+    def download_and_delete_blob(blob_id)
+      blob = @resource_manager.get_resource(blob_id)
+      blob
+    ensure
+      @resource_manager.delete_resource(blob_id)
+    end
+
+  end
+end

data/lib/bosh/director/cloudcheck_helper.rb

@@ -0,0 +1,204 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module CloudcheckHelper
+    # Helper functions that come in handy for
+    # cloudcheck:
+    # 1. VM/agent interactions
+    # 2. VM lifecycle operations (from cloudcheck POV)
+    # 3. Error handling
+
+    # This timeout has been made pretty short mainly
+    # to avoid long cloudchecks, however 10 seconds should
+    # still be pretty generous interval for agent to respond.
+    DEFAULT_AGENT_TIMEOUT = 10
+
+    def cloud
+      Bosh::Director::Config.cloud
+    end
+
+    def handler_error(message)
+      raise Bosh::Director::ProblemHandlerError, message
+    end
+
+    def instance_name(vm)
+      instance = vm.instance
+      return "Unknown VM" if instance.nil?
+
+      job = instance.job || "unknown job"
+      index = instance.index || "unknown index"
+      "#{job}/#{index}"
+    end
+
+    def agent_client(vm, timeout = DEFAULT_AGENT_TIMEOUT, retries = 0)
+      options = {
+        :timeout => timeout,
+        :retry_methods => { :get_state => retries }
+      }
+      @clients ||= {}
+      @clients[vm.agent_id] ||= AgentClient.new(vm.agent_id, options)
+    end
+
+    def agent_timeout_guard(vm, &block)
+      yield agent_client(vm)
+    rescue Bosh::Director::RpcTimeout
+      handler_error("VM `#{vm.cid}' is not responding")
+    end
+
+    def reboot_vm(vm)
+      cloud.reboot_vm(vm.cid)
+      begin
+        agent_client(vm).wait_until_ready
+      rescue Bosh::Director::RpcTimeout
+        handler_error("Agent still unresponsive after reboot")
+      end
+    end
+
+    def delete_vm(vm)
+      # Paranoia: don't blindly delete VMs with persistent disk
+      disk_list = agent_timeout_guard(vm) { |agent| agent.list_disk }
+      if disk_list.size != 0
+        handler_error("VM has persistent disk attached")
+      end
+
+      cloud.delete_vm(vm.cid)
+      vm.db.transaction do
+        vm.instance.update(:vm => nil) if vm.instance
+        vm.destroy
+      end
+    end
+
+    def delete_vm_reference(vm, options={})
+      if vm.cid && !options[:skip_cid_check]
+        handler_error("VM has a CID")
+      end
+
+      vm.db.transaction do
+        vm.instance.update(:vm => nil) if vm.instance
+        vm.destroy
+      end
+    end
+
+    def recreate_vm(vm)
+      # Best we can do without any feedback from the agent
+      # is to use the spec persisted in the DB at the time
+      # of last apply call.
+      # This method is somewhat similar in its nature to what
+      # InstanceUpdater is doing in case of the stemcell update,
+      # however we don't need to handle some advanced scenarios
+      # such as disk migration.
+
+      spec = validate_spec(vm)
+      env = validate_env(vm)
+
+      resource_pool_spec = spec.fetch("resource_pool", {})
+      stemcell = find_stemcell(resource_pool_spec.fetch("stemcell", {}))
+
+      deployment = vm.deployment
+      handler_error("VM doesn't belong to any deployment") unless deployment
+
+      instance = vm.instance
+      disk_cid = instance ? instance.persistent_disk_cid : nil
+
+      # One situation where this handler is actually useful is when
+      # VM has already been deleted but something failed after that
+      # and it is still referenced in DB. In that case it makes sense
+      # to ignore "VM not found" errors in `delete_vm' and let the method
+      # proceed creating a new VM. Other errors are not forgiven.
+      begin
+        cloud.delete_vm(vm.cid)
+      rescue Bosh::Clouds::VMNotFound => e
+        @logger.warn("VM '#{vm.cid}' might have already been deleted from the cloud")
+      end
+
+      vm.db.transaction do
+        instance.update(:vm => nil) if instance
+        vm.destroy
+      end
+
+      cloud_properties = resource_pool_spec.fetch("cloud_properties", {})
+      networks = spec["networks"]
+      new_vm = VmCreator.create(deployment, stemcell, cloud_properties, networks, Array(disk_cid), env)
+      new_vm.apply_spec = spec
+      new_vm.save
+
+      if instance
+        instance.update(:vm => new_vm)
+
+        # refresh metadata after new instance has been set
+        VmMetadataUpdater.build.update(new_vm, {})
+      end
+
+      agent_client(new_vm).wait_until_ready
+
+      # After this point agent is actually responding to
+      # pings, so if the rest of this handler fails
+      # bcck won't find this type of problem again
+      # but regular deployment will fail with "out-of-sync"
+      # error (as we now have an instance that points to
+      # VM that reports empty state). This problem
+      # should be handled by "out-of-sync VM" problem handler.
+
+      if disk_cid
+        # N.B. attach_disk might fail if disk image is no longer
+        # there or for some other reason. Generally it means
+        # the data has been lost (e.g. someone deleted VM from vCenter
+        # along with the disk.
+        cloud.attach_disk(new_vm.cid, disk_cid)
+        agent_client(new_vm).mount_disk(disk_cid)
+      end
+
+      agent_client(new_vm).apply(spec)
+
+      if instance && instance.state == "started"
+        agent_client(new_vm).start
+      end
+    end
+
+    private
+
+    def validate_spec(vm)
+      handler_error("Unable to look up VM apply spec") unless vm.apply_spec
+
+      spec = vm.apply_spec
+
+      unless spec.kind_of?(Hash)
+        handler_error("Invalid apply spec format")
+      end
+
+      spec
+    end
+
+    def validate_env(vm)
+      handler_error("Unable to look up VM environment") unless vm.env
+
+      env = vm.env
+
+      unless env.kind_of?(Hash)
+        handler_error("Invalid VM environment format")
+      end
+
+      env
+    end
+
+    def find_stemcell(stemcell_spec)
+      stemcell_name = stemcell_spec['name']
+      stemcell_version = stemcell_spec['version']
+
+      unless stemcell_name && stemcell_version
+        handler_error('Unknown stemcell name and/or version')
+      end
+
+      stemcell = Models::Stemcell.find(:name => stemcell_name, :version => stemcell_version)
+
+      handler_error("Unable to find stemcell '#{stemcell_name} #{stemcell_version}'") unless stemcell
+
+      stemcell
+    end
+
+    def generate_agent_id
+      SecureRandom.uuid
+    end
+
+  end
+end

data/lib/bosh/director/compile_task.rb

@@ -0,0 +1,157 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  class CompileTask
+    # @return [Models::Package] What package is being compiled
+    attr_reader :package
+
+    # @return [Models::Stemcell] What stemcell package is compiled for
+    attr_reader :stemcell
+
+    # @return [Array<DeploymentPlan::Job>] Jobs interested in this package
+    attr_reader :jobs
+
+    # @return [Models::CompiledPackage] Compiled package DB model
+    attr_reader :compiled_package
+
+    # @return [String] Dependency key (changing it will trigger recompilation
+    #   even when package bits haven't changed)
+    attr_accessor :dependency_key
+
+    # @return [Array<CompileTask>] Tasks this task depends on
+    attr_reader :dependencies
+
+    # @return [Array<CompileTask>] Tasks depending on this task
+    attr_reader :dependent_tasks
+
+    # @return [String] A unique checksum based on the dependencies in this task
+    attr_reader :cache_key
+
+    # @param [Models::Package] package What package is being compiled
+    #   by this task
+    # @param [Models::Stemcell] stemcell What stemcell package is compiled for
+    # @param [Array<Models::Package>] dependent_packages Package models that this task depends on
+    # @param [DeploymentPlan::Job] initial_job The first job that this task is associate with
+    def initialize(package, stemcell, dependent_packages, initial_job = nil)
+      @package = package
+      @stemcell = stemcell
+
+      @jobs = []
+      add_job(initial_job) if initial_job
+      @dependencies = []
+      @dependent_tasks = []
+
+      @dependency_key = generate_dependency_key(dependent_packages)
+
+      @cache_key = generate_cache_key(dependent_packages)
+    end
+
+    # @return [Boolean] Whether this task is ready to be compiled
+    def ready_to_compile?
+      !compiled? && all_dependencies_compiled?
+    end
+
+    # @return [Boolean]
+    def all_dependencies_compiled?
+      @dependencies.all? { |task| task.compiled? }
+    end
+
+    # @return [Boolean]
+    def compiled?
+      !@compiled_package.nil?
+    end
+
+    # Makes compiled package available to all jobs waiting for it
+    # @param [Models::CompiledPackage] compiled_package Compiled package
+    # @return [void]
+    def use_compiled_package(compiled_package)
+      @compiled_package = compiled_package
+
+      @jobs.each do |job|
+        job.use_compiled_package(@compiled_package)
+      end
+    end
+
+    # Adds job to a list of job requiring this compiled package
+    # @note Cycle detection is done elsewhere
+    # @param [DeploymentPlan::Job] job Job to be added
+    # @return [void]
+    def add_job(job)
+      return if @jobs.include?(job)
+      @jobs << job
+      if @compiled_package
+        # If package is already compiled we can make it available to job
+        # immediately, otherwise it will be done by #use_compiled_package
+        job.use_compiled_package(@compiled_package)
+      end
+    end
+
+    # Adds a compilation task to the list of dependencies
+    # @note Cycle detection performed elsewhere
+    # @param [CompileTask] task Compilation task
+    # @param [Boolean] reciprocate If true, add self as dependent task to other
+    # @return [void]
+    def add_dependency(task, reciprocate=true)
+      @dependencies << task
+      if reciprocate
+        task.add_dependent_task(self, false)
+      end
+    end
+
+    # Adds a compilation task to the list of dependent tasks
+    # @note Cycle detection performed elsewhere
+    # @param [CompileTask] task Compilation task
+    # @param [Boolean] reciprocate If true, add self as dependency to to other
+    # @return [void]
+    def add_dependent_task(task, reciprocate=true)
+      @dependent_tasks << task
+      if reciprocate
+        task.add_dependency(self, false)
+      end
+    end
+
+    # This call only makes sense if all dependencies have already been compiled,
+    # otherwise it raises an exception
+    # @return [Hash] Hash representation of all package dependencies. Agent uses
+    #   that to download package dependencies before compiling the package on a
+    #   compilation VM.
+    def dependency_spec
+      spec = {}
+
+      @dependencies.each do |dep_task|
+        unless dep_task.compiled?
+          raise DirectorError,
+                "Cannot generate package dependency spec " +
+                "for `#{@package.name}', " +
+                "`#{dep_task.package.name}' hasn't been compiled yet"
+        end
+
+        compiled_package = dep_task.compiled_package
+
+        spec[compiled_package.name] = {
+          "name" => compiled_package.name,
+          "version" => "#{compiled_package.version}.#{compiled_package.build}",
+          "sha1" => compiled_package.sha1,
+          "blobstore_id" => compiled_package.blobstore_id
+        }
+      end
+
+      spec
+    end
+
+    private
+    def generate_dependency_key(packages)
+      key = packages.sort { |a, b|
+        a.name <=> b.name
+      }.map { |p| [p.name, p.version]}
+
+      Yajl::Encoder.encode(key)
+    end
+
+    def generate_cache_key(dependent_packages)
+      dependency_fingerprints = dependent_packages.sort_by(&:name).map {|p| p.fingerprint }
+      hash_input = ([package.fingerprint, stemcell.sha1]+dependency_fingerprints).join("")
+      Digest::SHA1.hexdigest(hash_input)
+    end
+  end
+end