bosh-director 1.5.0.pre.1113

data/lib/bosh/director/jobs/update_stemcell.rb

@@ -0,0 +1,109 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module Jobs
+    class UpdateStemcell < BaseJob
+      include ValidationHelper
+      include DownloadHelper
+
+      UPDATE_STEPS = 5
+      
+      @queue = :normal
+
+      def self.job_type
+        :update_stemcell
+      end
+
+      # @param [String] stemcell_file Stemcell tarball path
+      def initialize(stemcell_file, options = {})
+        @stemcell_file = stemcell_file
+        @cloud = Config.cloud
+        @stemcell_manager = Api::StemcellManager.new
+        @remote_stemcell = options['remote'] || false
+      end
+
+      def perform
+        logger.info("Processing update stemcell")
+        event_log.begin_stage("Update stemcell", update_steps)
+
+        if @remote_stemcell
+          downloaded_stemcell_dir = Dir.mktmpdir("downloaded-stemcell")        
+          download_remote_stemcell(downloaded_stemcell_dir)
+        end
+        
+        stemcell_dir = Dir.mktmpdir("stemcell")
+
+        track_and_log("Extracting stemcell archive") do
+          result = Bosh::Exec.sh("tar -C #{stemcell_dir} -xzf #{@stemcell_file} 2>&1", :on_error => :return)
+          if result.failed?
+            logger.error("Extracting stemcell archive failed in dir #{stemcell_dir}, " +
+                         "tar returned #{result.exit_status}, " +
+                         "output: #{result.output}")
+            raise StemcellInvalidArchive, "Extracting stemcell archive failed. Check task debug log for details."
+          end
+        end
+
+        track_and_log("Verifying stemcell manifest") do
+          stemcell_manifest_file = File.join(stemcell_dir, "stemcell.MF")
+          stemcell_manifest = Psych.load_file(stemcell_manifest_file)
+
+          @name = safe_property(stemcell_manifest, "name", :class => String)
+          @version = safe_property(stemcell_manifest, "version", :class => String)
+          @cloud_properties = safe_property(stemcell_manifest, "cloud_properties", :class => Hash, :optional => true)
+          @sha1 = safe_property(stemcell_manifest, "sha1", :class => String)
+
+          logger.info("Found stemcell image `#{@name}/#{@version}', " +
+                      "cloud properties are #{@cloud_properties.inspect}")
+
+          logger.info("Verifying stemcell image")
+          @stemcell_image = File.join(stemcell_dir, "image")
+          unless File.file?(@stemcell_image)
+            raise StemcellImageNotFound, "Stemcell image not found"
+          end
+        end
+
+        track_and_log("Checking if this stemcell already exists") do
+          if @stemcell_manager.stemcell_exists?(@name, @version)
+            raise StemcellAlreadyExists,
+                  "Stemcell `#{@name}/#{@version}' already exists"
+          end
+        end
+
+        stemcell = Models::Stemcell.new
+        stemcell.name = @name
+        stemcell.version = @version
+        stemcell.sha1 = @sha1
+
+        track_and_log("Uploading stemcell #{@name}/#{@version} to the cloud") do
+          stemcell.cid =
+            @cloud.create_stemcell(@stemcell_image, @cloud_properties)
+          logger.info("Cloud created stemcell: #{stemcell.cid}")
+        end
+
+        track_and_log("Save stemcell #{@name}/#{@version} (#{stemcell.cid})") do
+          stemcell.save
+        end
+
+        "/stemcells/#{stemcell.name}/#{stemcell.version}"
+      ensure
+        FileUtils.rm_rf(stemcell_dir) if stemcell_dir
+        FileUtils.rm_rf(downloaded_stemcell_dir) if downloaded_stemcell_dir
+        FileUtils.rm_rf(@stemcell_file) if @stemcell_file
+      end
+      
+      def download_remote_stemcell(downloaded_stemcell_dir)
+        track_and_log("Downloading remote stemcell") do
+          downloaded_stemcell_file = File.join(downloaded_stemcell_dir, "stemcell-#{SecureRandom.uuid}")
+          download_remote_file('stemcell', @stemcell_file, downloaded_stemcell_file)
+          @stemcell_file = downloaded_stemcell_file          
+        end
+      end
+      
+      private
+      
+      def update_steps
+        @remote_stemcell ? UPDATE_STEPS + 1 : UPDATE_STEPS
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/vm_state.rb

@@ -0,0 +1,89 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module Jobs
+    class VmState < BaseJob
+      TIMEOUT = 5
+
+      @queue = :normal
+
+      def self.job_type
+        :vms
+      end
+
+      # @param [Integer] deployment_id Deployment id
+      def initialize(deployment_id, format)
+        @deployment_id = deployment_id
+        @format = format
+      end
+
+      def perform
+        @domain = Models::Dns::Domain.find(name: Config.dns_domain_name, type: "NATIVE") if Config.dns_enabled?
+        
+        vms = Models::Vm.filter(:deployment_id => @deployment_id)
+        ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
+          vms.each do |vm|
+            pool.process do
+              vm_state = process_vm(vm)
+              result_file.write(vm_state.to_json + "\n")
+            end
+          end
+        end
+
+        # task result
+        nil
+      end
+
+      def process_vm(vm)
+        ips = []
+        dns_records = []
+        job_name = nil
+        job_state = nil
+        resource_pool = nil
+        job_vitals = nil
+        index = nil
+
+        begin
+          agent = AgentClient.new(vm.agent_id, :timeout => TIMEOUT)
+          agent_state = agent.get_state(@format)
+          agent_state["networks"].each_value do |network|
+            ips << network["ip"]
+          end
+          index = agent_state["index"]
+          job_name = agent_state["job"]["name"] if agent_state["job"]
+          job_state = agent_state["job_state"]
+          if agent_state["resource_pool"]
+            resource_pool = agent_state["resource_pool"]["name"]
+          end
+          if agent_state["vitals"]
+            job_vitals = agent_state["vitals"]
+          end
+        rescue Bosh::Director::RpcTimeout
+          job_state = "unresponsive agent"
+        end
+
+        instance = Models::Instance.find(deployment_id: @deployment_id, job: job_name, index: index)
+
+        if @domain
+          ips.each do |ip|
+            records = Models::Dns::Record.filter(domain_id: @domain.id, type: "A", content: ip)
+            dns_records << records.collect { |record| record.name } unless records.empty?
+          end
+        end
+
+        {
+          :vm_cid => vm.cid,
+          :ips => ips,
+          :dns => dns_records.flatten,
+          :agent_id => vm.agent_id,
+          :job_name => job_name,
+          :index => index,
+          :job_state => job_state,
+          :resource_pool => resource_pool,
+          :vitals => job_vitals,
+          :resurrection_paused => instance ? instance.resurrection_paused  : nil,
+        }
+      end
+    end
+  end
+end

data/lib/bosh/director/lock.rb

@@ -0,0 +1,133 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+
+  # Distributed lock backed by Redis.
+  class Lock
+
+    # Error returned when Lock could not be acquired.
+    class TimeoutError < StandardError; end
+
+    # Creates new lock with the given name.
+    #
+    # @param name lock name
+    # @option opts [Number] timeout how long to wait before giving up
+    # @option opts [Number] expiration how long to wait before expiring an old
+    #   lock
+    def initialize(name, opts = {})
+      @name = name
+      @id = SecureRandom.uuid
+      @timeout = opts[:timeout] || 1.0
+      @expiration = opts[:expiration] || 10.0
+      @logger = Config.logger
+      @refresh_thread = nil
+    end
+
+    # Acquire a lock.
+    #
+    # @yield [void] optional block to do work before automatically releasing
+    #   the lock.
+    # @return [void]
+    def lock
+      acquire
+
+      @refresh_thread = Thread.new do
+        redis = Config.redis
+        sleep_interval = [1.0, @expiration/2].max
+        begin
+          loop do
+            @logger.debug("Renewing lock: #@name")
+            redis.watch(@name)
+            existing_lock = redis.get(@name)
+            lock_id = existing_lock.split(":")[1]
+            break if lock_id != @id
+            lock_expiration = Time.now.to_f + @expiration + 1
+            redis.multi do
+              redis.set(@name, "#{lock_expiration}:#@id")
+            end
+            sleep(sleep_interval)
+          end
+        ensure
+          @logger.debug("Lock renewal thread exiting")
+          redis.quit
+        end
+      end
+
+      if block_given?
+        begin
+          yield
+        ensure
+          release
+        end
+      end
+    end
+
+    # Release a lock that was not auto released by the lock method.
+    #
+    # @return [void]
+    def release
+      @refresh_thread.exit if @refresh_thread
+      delete
+    end
+
+    private
+
+    def acquire
+      @logger.debug("Acquiring lock: #@name")
+      redis = Config.redis
+      started = Time.now
+
+      lock_expiration = Time.now.to_f + @expiration + 1
+      until redis.setnx(@name, "#{lock_expiration}:#@id")
+        existing_lock = redis.get(@name)
+        @logger.debug("Lock #@name is already locked by someone " +
+                          "else: #{existing_lock}")
+        if lock_expired?(existing_lock)
+          @logger.debug("Lock #@name is already expired, " +
+                            "trying to take it back")
+          replaced_lock = redis.getset(@name, "#{lock_expiration}:#@id")
+          if replaced_lock == existing_lock
+            @logger.debug("Lock #@name was revoked and relocked")
+            break
+          else
+            @logger.debug("Lock #@name was acquired by someone else, " +
+                              "trying again")
+          end
+        end
+
+        raise TimeoutError if Time.now - started > @timeout
+
+        sleep(0.5)
+
+        lock_expiration = Time.now.to_f + @expiration + 1
+      end
+
+      @lock_expiration = lock_expiration
+      @logger.debug("Acquired lock: #@name")
+    end
+
+    def delete
+      @logger.debug("Deleting lock: #@name")
+      redis = Config.redis
+
+      redis.watch(@name)
+      existing_lock = redis.get(@name)
+      lock_id = existing_lock.split(":")[1]
+      if lock_id == @id
+        redis.multi do
+          redis.del(@name)
+        end
+      else
+        redis.unwatch
+      end
+      @logger.debug("Deleted lock: #@name")
+    end
+
+    def lock_expired?(lock)
+      existing_lock_expiration = lock.split(":")[0].to_f
+      lock_time_left = existing_lock_expiration - Time.now.to_f
+      @logger.info("Lock: #{lock} expires in #{lock_time_left} seconds")
+      lock_time_left < 0
+    end
+  end
+end

data/lib/bosh/director/lock_helper.rb

@@ -0,0 +1,92 @@
+module Bosh::Director
+  # Helper for managing BOSH locks.
+  module LockHelper
+
+    # Surround with deployment lock.
+    #
+    # @param [DeploymentPlan|String] deployment plan or name.
+    # @option opts [Number] timeout how long to wait before giving up
+    # @return [void]
+    # @yield [void] block to surround
+    def with_deployment_lock(deployment, opts = {})
+      if deployment.respond_to?(:name)
+        name = deployment.name
+      elsif deployment.kind_of?(String)
+        name = deployment
+      else
+        raise ArgumentError, "invalid deployment: #{deployment}"
+      end
+      timeout = opts[:timeout] || 10
+      Config.logger.info("Acquiring deployment lock on #{name}")
+      Lock.new("lock:deployment:#{name}", :timeout => timeout).lock { yield }
+    end
+
+    # Surround with stemcell lock.
+    #
+    # @param [String] name stemcell name.
+    # @param [String] version stemcell version.
+    # @option opts [Number] timeout how long to wait before giving up
+    # @return [void]
+    # @yield [void] block to surround
+    def with_stemcell_lock(name, version, opts = {})
+      timeout = opts[:timeout] || 10
+      Config.logger.info("Acquiring deployment lock on #{name}:#{version}")
+      Lock.new("lock:stemcells:#{name}:#{version}", :timeout => timeout).
+          lock { yield }
+    end
+
+    # Surround with release lock.
+    #
+    # @param [String] release name.
+    # @option opts [Number] timeout how long to wait before giving up
+    # @return [void]
+    # @yield [void] block to surround
+    def with_release_lock(release, opts = {})
+      timeout = opts[:timeout] || 10
+      Config.logger.info("Acquiring deployment lock on #{release}")
+      Lock.new("lock:release:#{release}", :timeout => timeout).lock { yield }
+    end
+
+    # Surround with deployment releases lock.
+    #
+    # @param [DeploymentPlan] deployment plan.
+    # @option opts [Number] timeout how long to wait before giving up
+    # @return [void]
+    # @yield [void] block to surround
+    def with_release_locks(deployment_plan, opts = {})
+      timeout = opts[:timeout] || 10
+      release_names = deployment_plan.releases.map do |release|
+        release.name
+      end
+
+      # Sorting to enforce lock order to avoid deadlocks
+      locks = release_names.sort.map do |release_name|
+        Config.logger.info("Acquiring release lock: #{release_name}")
+        Lock.new("lock:release:#{release_name}", :timeout => timeout)
+      end
+
+      begin
+        locks.each { |lock| lock.lock }
+        yield
+      ensure
+        locks.reverse_each { |lock| lock.release }
+      end
+    end
+
+    # Surround with compile lock.
+    #
+    # @param [String|Number] package_id package id.
+    # @param [String|Number] stemcell_id stemcell id.
+    # @option opts [Number] timeout how long to wait before giving up
+    # @return [void]
+    # @yield [void] block to surround
+    def with_compile_lock(package_id, stemcell_id, opts = {})
+      timeout = opts[:timeout] || 15 * 60 # 15 minutes
+
+      Config.logger.info("Acquiring compile lock on " +
+                             "#{package_id} #{stemcell_id}")
+      Lock.new("lock:compile:#{package_id}:#{stemcell_id}",
+               :timeout => timeout).lock { yield }
+    end
+  end
+end

data/lib/bosh/director/models.rb

@@ -0,0 +1,29 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+#
+
+require 'bosh/director/models/compiled_package'
+require 'bosh/director/models/deployment'
+require 'bosh/director/models/deployment_problem'
+require 'bosh/director/models/deployment_property'
+require 'bosh/director/models/instance'
+require 'bosh/director/models/log_bundle'
+require 'bosh/director/models/package'
+require 'bosh/director/models/release'
+require 'bosh/director/models/release_version'
+require 'bosh/director/models/stemcell'
+require 'bosh/director/models/snapshot'
+require 'bosh/director/models/task'
+require 'bosh/director/models/template'
+require 'bosh/director/models/user'
+require 'bosh/director/models/vm'
+require 'bosh/director/models/persistent_disk'
+require 'bosh/director/models/director_attribute'
+
+module Bosh::Director
+  module Models
+    VALID_ID = /^[-a-z0-9_.]+$/i
+
+    autoload :Dns, 'bosh/director/models/dns'
+  end
+end
+