bosh-director 1.2641.0 → 1.2652.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 970adfc1d8041fe7624fc4adf1cad86917730920
-  data.tar.gz: 6fb6b4ac83b53114782683c85ba45c9ec667ab8e
+  metadata.gz: e23095b6bfae1e8e3ef95c3ed35d673812f7ff06
+  data.tar.gz: 45d0ef37a600837a982c75cf2c76a012886938a2
 SHA512:
-  metadata.gz: 36949611acd25fdcebab2740ad414ea358633c7e0971d477e43775e58330855d52a39faf04c9669ef34842c1b6e2f2b716d265d09903d52a23602458538e81c7
-  data.tar.gz: fcea68e23b7695ede27826510374db24524032376eeb8ede6e6afdf2a10218777b00c31d9de8250744f9d3a7cd0084e4514290c05c6979d94d594d21f6993d87
+  metadata.gz: c98835abd2840e2d857cf7b24207bc26b765a90846170eb760f028b169977202b394da3aeaa5f94f6c74349fd7d76a1f7174004491fc4a74021316434ca9a7e0
+  data.tar.gz: 42802d742f68e2539451f63e6b55590c91ee2cf3f98fd7e4a54f58131bba993ee17304ad64c6ba2dbe05adfda9d1baf25b28ca001219fa862996910c38b96f93

data/lib/bosh/director/api/api_helper.rb

@@ -78,4 +78,4 @@ module Bosh::Director
       end
     end
   end
-end
+end

data/lib/bosh/director/api/compiled_package_group_manager.rb

@@ -0,0 +1,17 @@
+require 'securerandom'
+
+module Bosh::Director
+  module Api
+    class CompiledPackageGroupManager
+      include ApiHelper
+
+      def create_from_file_path(user, path)
+        unless File.exists?(path)
+          raise DirectorError, "Failed to import compiled packages: file not found - #{path}"
+        end
+
+        JobQueue.new.enqueue(user, Jobs::ImportCompiledPackages, 'import compiled packages', [path])
+      end
+    end
+  end
+end

data/lib/bosh/director/api/controllers/base_controller.rb

@@ -26,14 +26,18 @@ module Bosh::Director
           @logger = Config.logger
         end
 
-        mime_type :tgz, 'application/x-compressed'
+        mime_type :tgz,       'application/x-compressed'
+        mime_type :multipart, 'multipart/form-data'
 
         def self.consumes(*types)
           types = Set.new(types)
           types.map! { |t| mime_type(t) }
 
           condition do
-            types.include?(request.content_type)
+            # Content-Type header may include charset or boundry info
+            content_type = request.content_type || ''
+            mime_type = content_type.split(';')[0]
+            types.include?(mime_type)
           end
         end
 

data/lib/bosh/director/api/controllers/compiled_packages_controller.rb

@@ -1,5 +1,6 @@
 require 'json'
 require 'tempfile'
+require 'bosh/director/api/compiled_package_group_manager'
 require 'bosh/director/api/controllers/base_controller'
 require 'bosh/director/api/stemcell_manager'
 require 'bosh/director/compiled_package_group'
@@ -10,7 +11,12 @@ require 'bosh/director/jobs/import_compiled_packages'
 module Bosh::Director
   module Api::Controllers
     class CompiledPackagesController < BaseController
-      post '/compiled_package_groups/export', consumes: [:json] do
+      def initialize(*args)
+        super
+        @compiled_package_group_manager = Api::CompiledPackageGroupManager.new
+      end
+
+      post '/compiled_package_groups/export', consumes: :json do
         stemcell = find_stemcell_by_name_and_version
         release_version = find_release_version_by_name_and_version
 
@@ -30,20 +36,13 @@ module Bosh::Director
         send_file(output_path, type: :tgz)
       end
 
-      post '/compiled_package_groups/import', consumes: [:tgz] do
-        tempdir = Dir.mktmpdir
-        export_path = File.join(tempdir, 'compiled_packages_export.tgz')
-        # the job is responsible for cleaning this up
-        File.open(export_path, 'w') do |f|
-          while buf = request.body.read(4096)
-            f.write(buf)
-          end
-        end
-
-        task = JobQueue.new.enqueue(@user, Jobs::ImportCompiledPackages, 'import compiled packages', [tempdir])
+      post '/compiled_package_groups/import', consumes: :multipart do
+        task = @compiled_package_group_manager.create_from_file_path(@user, params[:nginx_upload_path])
         redirect "/tasks/#{task.id}"
       end
 
+      private
+
       def find_stemcell_by_name_and_version
         stemcell_manager = Api::StemcellManager.new
         stemcell_manager.find_by_name_and_version(

data/lib/bosh/director/api/controllers/releases_controller.rb

@@ -3,37 +3,35 @@ require 'bosh/director/api/controllers/base_controller'
 module Bosh::Director
   module Api::Controllers
     class ReleasesController < BaseController
-      post '/releases', :consumes => :tgz do
-        options = {}
-        options['remote'] = false
-        options['rebase'] = true if params['rebase'] == 'true'
-
-        task = @release_manager.create_release(@user, request.body, options)
-        redirect "/tasks/#{task.id}"
-      end
-
       post '/releases', :consumes => :json do
-        options = {}
-        options['remote'] = true
-        options['rebase'] = true if params['rebase'] == 'true'
         payload = json_decode(request.body)
+        rebase = params['rebase'] == 'true'
+        task = @release_manager.create_release_from_url(@user, payload['location'], rebase)
+        redirect "/tasks/#{task.id}"
+      end
 
-        task = @release_manager.create_release(@user, payload['location'], options)
+      post '/releases', :consumes => :multipart do
+        rebase = params['rebase'] == 'true'
+        task = @release_manager.create_release_from_file_path(@user, params[:nginx_upload_path], rebase)
         redirect "/tasks/#{task.id}"
       end
 
       get '/releases' do
         releases = Models::Release.order_by(:name.asc).map do |release|
           release_versions = release.versions_dataset.order_by(:version.asc).map do |rv|
-            Hash['version', rv.version.to_s,
-                 'commit_hash', rv.commit_hash,
-                 'uncommitted_changes', rv.uncommitted_changes,
-                 'currently_deployed', !rv.deployments.empty?,
-                 'job_names', rv.templates.map(&:name)]
+            {
+              'version' => rv.version.to_s,
+              'commit_hash' => rv.commit_hash,
+              'uncommitted_changes' => rv.uncommitted_changes,
+              'currently_deployed' => !rv.deployments.empty?,
+              'job_names' => rv.templates.map(&:name),
+            }
           end
 
-          Hash['name', release.name,
-               'release_versions', release_versions]
+          {
+            'name' => release.name,
+            'release_versions' => release_versions,
+          }
         end
 
         json_encode(releases)

data/lib/bosh/director/api/controllers/stemcells_controller.rb

@@ -3,14 +3,14 @@ require 'bosh/director/api/controllers/base_controller'
 module Bosh::Director
   module Api::Controllers
     class StemcellsController < BaseController
-      post '/stemcells', :consumes => :tgz do
-        task = @stemcell_manager.create_stemcell(@user, request.body, :remote => false)
+      post '/stemcells', :consumes => :json do
+        payload = json_decode(request.body)
+        task = @stemcell_manager.create_stemcell_from_url(@user, payload['location'])
         redirect "/tasks/#{task.id}"
       end
 
-      post '/stemcells', :consumes => :json do
-        payload = json_decode(request.body)
-        task = @stemcell_manager.create_stemcell(@user, payload['location'], :remote => true)
+      post '/stemcells', :consumes => :multipart do
+        task = @stemcell_manager.create_stemcell_from_file_path(@user, params[:nginx_upload_path])
         redirect "/tasks/#{task.id}"
       end
 

data/lib/bosh/director/api/release_manager.rb

@@ -3,8 +3,6 @@ module Bosh::Director
     class ReleaseManager
       include ApiHelper
 
-      RELEASE_TGZ = 'release.tgz'
-
       # Finds release by name
       # @param [String] name Release name
       # @return [Models::Release]
@@ -48,21 +46,18 @@ module Bosh::Director
         release_version
       end
 
-      def create_release(user, release_bundle, options = {})
-        release_dir = Dir.mktmpdir('release')
-
-        if options['remote']
-          options['location'] = release_bundle
-        else
-          unless check_available_disk_space(release_dir, release_bundle.size)
-            raise NotEnoughDiskSpace, 'Uploading release archive failed. ' +
-              "Insufficient space on BOSH director in #{release_dir}"
-          end
+      def create_release_from_url(user, release_url, rebase)
+        options = { remote: true, rebase: rebase }
+        JobQueue.new.enqueue(user, Jobs::UpdateRelease, 'create release', [release_url, options])
+      end
 
-          write_file(File.join(release_dir, RELEASE_TGZ), release_bundle)
+      def create_release_from_file_path(user, release_path, rebase)
+        unless File.exists?(release_path)
+          raise DirectorError, "Failed to create release: file not found - #{release_path}"
         end
 
-        JobQueue.new.enqueue(user, Jobs::UpdateRelease, 'create release', [release_dir, options])
+        options = { rebase: rebase }
+        JobQueue.new.enqueue(user, Jobs::UpdateRelease, 'create release', [release_path, options])
       end
 
       def delete_release(user, release, options = {})

data/lib/bosh/director/api/stemcell_manager.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2009-2012 VMware, Inc.
+require 'securerandom'
 
 module Bosh::Director
   module Api
@@ -21,23 +21,16 @@ module Bosh::Director
         false
       end
 
-      def create_stemcell(user, stemcell, options = {})
-        if options[:remote]
-          stemcell_file = stemcell
-        else
-          random_name = "stemcell-#{SecureRandom.uuid}"
-          stemcell_dir = Dir::tmpdir
-          stemcell_file = File.join(stemcell_dir, random_name)
-
-          unless check_available_disk_space(stemcell_dir, stemcell.size)
-            raise NotEnoughDiskSpace, "Uploading stemcell archive failed. " +
-              "Insufficient space on BOSH director in #{stemcell_dir}"
-          end
-          
-          write_file(stemcell_file, stemcell)
+      def create_stemcell_from_url(user, stemcell_url)
+        JobQueue.new.enqueue(user, Jobs::UpdateStemcell, 'create stemcell', [stemcell_url, { remote: true }])
+      end
+
+      def create_stemcell_from_file_path(user, stemcell_path)
+        unless File.exists?(stemcell_path)
+          raise DirectorError, "Failed to create stemcell: file not found - #{stemcell_path}"
         end
 
-        JobQueue.new.enqueue(user, Jobs::UpdateStemcell, 'create stemcell', [stemcell_file, options])
+        JobQueue.new.enqueue(user, Jobs::UpdateStemcell, 'create stemcell', [stemcell_path])
       end
 
       def delete_stemcell(user, stemcell, options={})
@@ -47,4 +40,4 @@ module Bosh::Director
       end
     end
   end
-end
+end

data/lib/bosh/director/api.rb

@@ -19,6 +19,7 @@ require 'bosh/director/api/release_manager'
 require 'bosh/director/api/resource_manager'
 require 'bosh/director/api/snapshot_manager'
 require 'bosh/director/api/stemcell_manager'
+require 'bosh/director/api/compiled_package_group_manager'
 require 'bosh/director/api/task_manager'
 require 'bosh/director/api/user_manager'
 require 'bosh/director/api/vm_state_manager'

data/lib/bosh/director/compiled_package/compiled_package_inserter.rb

@@ -8,33 +8,34 @@ module Bosh::Director::CompiledPackage
 
     def insert(compiled_package, release_version)
       package = Bosh::Director::Models::Package[fingerprint: compiled_package.package_fingerprint]
-      stemcell = Bosh::Director::Models::Stemcell[sha1: compiled_package.stemcell_sha1]
-
       raise ArgumentError, [compiled_package.inspect, release_version.inspect].inspect unless package
 
-      unless Bosh::Director::Models::CompiledPackage[
+      stemcell = Bosh::Director::Models::Stemcell[sha1: compiled_package.stemcell_sha1]
+
+      compiled_package_model = Bosh::Director::Models::CompiledPackage[
         package: package,
         stemcell: stemcell,
         dependency_key: release_version.package_dependency_key(package.name),
       ]
 
-        oid = File.open(compiled_package.blob_path) do |f|
-          @blobstore_client.create(f)
-        end
+      return if compiled_package_model
+
+      oid = File.open(compiled_package.blob_path) do |f|
+        @blobstore_client.create(f)
+      end
 
-        begin
-          Bosh::Director::Models::CompiledPackage.create(
-            blobstore_id: oid,
-            package: package,
-            stemcell: stemcell,
-            sha1: compiled_package.sha1,
-            dependency_key: release_version.package_dependency_key(package.name),
-            build: Bosh::Director::Models::CompiledPackage.generate_build_number(package, stemcell),
-          )
-        rescue
-          @blobstore_client.delete(oid)
-          raise
-        end
+      begin
+        Bosh::Director::Models::CompiledPackage.create(
+          blobstore_id: oid,
+          package: package,
+          stemcell: stemcell,
+          sha1: compiled_package.sha1,
+          dependency_key: release_version.package_dependency_key(package.name),
+          build: Bosh::Director::Models::CompiledPackage.generate_build_number(package, stemcell),
+        )
+      rescue
+        @blobstore_client.delete(oid)
+        raise
       end
     end
   end

data/lib/bosh/director/compiled_package/compiled_packages_export.rb

@@ -8,11 +8,14 @@ module Bosh::Director::CompiledPackage
       @exec = options.fetch(:exec, Bosh::Exec)
     end
 
-    def extract(tempdir=nil)
-      tempdir ||= Dir.mktmpdir
-      exec.sh("tar -C #{tempdir} -xf #{file}")
-      manifest = YAML.load_file("#{tempdir}/compiled_packages.MF")
+    def extract
+      tmp_dir = Dir.mktmpdir
+      
+      @exec.sh("tar -C #{tmp_dir} -xf #{@file}")
+      
+      manifest = YAML.load_file("#{tmp_dir}/compiled_packages.MF")
       packages = []
+      
       manifest['compiled_packages'].each do |p|
         packages << CompiledPackage.new(
           package_name: p['package_name'],
@@ -20,16 +23,13 @@ module Bosh::Director::CompiledPackage
           sha1: p['compiled_package_sha1'],
           stemcell_sha1: p['stemcell_sha1'],
           blobstore_id: p['blobstore_id'],
-          blob_path: File.join(tempdir, 'compiled_packages', 'blobs', p['blobstore_id']))
+          blob_path: File.join(tmp_dir, 'compiled_packages', 'blobs', p['blobstore_id']),
+        )
       end
 
       yield manifest, packages
 
-      FileUtils.rm_rf(tempdir)
+      FileUtils.rm_rf(tmp_dir)
     end
-
-    private
-
-    attr_reader :file, :exec
   end
 end

data/lib/bosh/director/config.rb

@@ -41,7 +41,7 @@ module Bosh::Director
         attr_accessor option
       end
 
-      attr_reader :db_config
+      attr_reader :db_config, :redis_logger_level
 
       def clear
         CONFIG_OPTIONS.each do |option|
@@ -75,14 +75,14 @@ module Bosh::Director
         logging = config.fetch('logging', {})
         @log_device = MonoLogger::LocklessLogDevice.new(logging.fetch('file', STDOUT))
         @logger = MonoLogger.new(@log_device)
-        @logger.level = MonoLogger.const_get(logging.fetch('level', 'debug').upcase)
+        @logger.level = Logger.const_get(logging.fetch('level', 'debug').upcase)
         @logger.formatter = ThreadFormatter.new
 
         # use a separate logger for redis to make it stfu
         redis_logger = MonoLogger.new(@log_device)
         logging = config.fetch('redis', {}).fetch('logging', {})
-        redis_logger_level = logging.fetch('level', 'info').upcase
-        redis_logger.level = MonoLogger.const_get(redis_logger_level)
+        @redis_logger_level = Logger.const_get(logging.fetch('level', 'info').upcase)
+        redis_logger.level = @redis_logger_level
 
         # Event logger supposed to be overridden per task,
         # the default one does nothing
@@ -93,7 +93,7 @@ module Bosh::Director
 
         @max_threads = config.fetch('max_threads', 32).to_i
 
-        self.redis_options= {
+        self.redis_options = {
           :host     => config['redis']['host'],
           :port     => config['redis']['port'],
           :password => config['redis']['password'],
@@ -201,24 +201,20 @@ module Bosh::Director
         Config.cloud_options.fetch('properties', {}).fetch('cpi_log')
       end
 
-      def logger=(logger)
-        @logger = logger
-        redis_options[:logger] = @logger
-        if redis?
-          redis.client.logger = @logger
-        end
-      end
-
       def job_cancelled?
         @current_job.task_checkpoint if @current_job
       end
-      alias_method :task_checkpoint, :job_cancelled?
 
+      alias_method :task_checkpoint, :job_cancelled?
 
       def redis_options
         @redis_options ||= {}
       end
 
+      def redis_logger_level
+        @redis_logger_level || Logger::INFO
+      end
+
       def redis_options=(options)
         @redis_options = options
       end
@@ -252,6 +248,14 @@ module Bosh::Director
         threaded[:redis] ||= Redis.new(redis_options)
       end
 
+      def redis_logger=(logger)
+        if redis?
+          redis.client.logger = logger
+        else
+          redis_options[:logger] = logger
+        end
+      end
+
       def redis?
         !threaded[:redis].nil?
       end
@@ -346,6 +350,5 @@ module Bosh::Director
     def initialize(hash)
       @hash = hash
     end
-
   end
 end

data/lib/bosh/director/download_helper.rb

@@ -1,12 +1,10 @@
+require 'net/http'
+require 'timeout'
+require 'uri'
+
 module Bosh::Director
   module DownloadHelper
-    require 'net/http'
-    require 'timeout'
-    require 'uri'
-
-    ##
     # Downloads a remote file
-    #
     # @param [String] resource Resource name to be logged
     # @param [String] remote_file Remote file to download
     # @param [String] local_file Local file to store the downloaded file
@@ -41,4 +39,4 @@ module Bosh::Director
       raise ResourceError, "Downloading remote #{resource} failed. Check task debug log for details."
     end
   end
-end
+end

data/lib/bosh/director/ext.rb

@@ -1,5 +1,3 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Resque
   def redis
     Bosh::Director::Config.redis
@@ -7,15 +5,15 @@ module Resque
 end
 
 class Object
- def to_openstruct
-   self
- end
+  def to_openstruct
+    self
+  end
 end
 
 class Array
- def to_openstruct
-   map { |el| el.to_openstruct }
- end
+  def to_openstruct
+    map { |el| el.to_openstruct }
+  end
 end
 
 class Hash
@@ -30,35 +28,9 @@ class Hash
     self
   end
 
- def to_openstruct
-   mapped = {}
-   each { |key, value| mapped[key] = value.to_openstruct }
-   OpenStruct.new(mapped)
- end
-end
-
-require "sequel/connection_pool/threaded"
-
-class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
-
-  alias_method :acquire_original, :acquire
-  alias_method :release_original, :release
-
-  def acquire(thread)
-    logger = Bosh::Director::Config.logger
-    result = acquire_original(thread)
-    if logger
-      logger.debug("Acquired connection: #{@allocated[thread].object_id}")
-    end
-    result
-  end
-
-  def release(thread)
-    logger = Bosh::Director::Config.logger
-    if logger
-      logger.debug("Released connection: #{@allocated[thread].object_id}")
-    end
-    release_original(thread)
+  def to_openstruct
+    mapped = {}
+    each { |key, value| mapped[key] = value.to_openstruct }
+    OpenStruct.new(mapped)
   end
-
 end

data/lib/bosh/director/job_runner.rb

@@ -53,7 +53,9 @@ module Bosh::Director
       event_log = File.join(log_dir, 'event')
       result_log = File.join(log_dir, 'result')
 
-      @task_logger = MonoLogger.new(debug_log)
+      debug_log_device = MonoLogger::LocklessLogDevice.new(debug_log)
+
+      @task_logger = MonoLogger.new(debug_log_device)
       @task_logger.level = Config.logger.level
       @task_logger.formatter = ThreadFormatter.new
 
@@ -61,6 +63,11 @@ module Bosh::Director
       Config.result = TaskResultFile.new(result_log)
       Config.logger = @task_logger
 
+      redis_task_logger = MonoLogger.new(debug_log_device)
+      redis_task_logger.level = Config.redis_logger_level
+      redis_task_logger.formatter = ThreadFormatter.new
+      Config.redis_logger = redis_task_logger
+
       Config.db.logger = @task_logger
 
       if Config.dns_enabled?

data/lib/bosh/director/jobs/import_compiled_packages.rb

@@ -10,14 +10,13 @@ module Bosh::Director
         :import_compiled_packages
       end
 
-      def initialize(export_dir)
-        @export_dir = export_dir
-
+      def initialize(compiled_packages_path)
+        @compiled_packages_path = compiled_packages_path
         @blobstore_client = Bosh::Director::App.instance.blobstores.blobstore
       end
 
       def perform
-        export = Bosh::Director::CompiledPackage::CompiledPackagesExport.new(file: export_path)
+        export = Bosh::Director::CompiledPackage::CompiledPackagesExport.new(file: @compiled_packages_path)
 
         export.extract do |manifest, packages|
           release_name = manifest.fetch('release_name')
@@ -38,17 +37,8 @@ module Bosh::Director
           end
         end
       ensure
-        FileUtils.rm_rf(@export_dir)
+        FileUtils.rm_rf(@compiled_packages_path)
       end
-
-      private
-
-      def export_path
-        File.join(@export_dir, 'compiled_packages_export.tgz')
-      end
-
-      attr_reader :blobstore_client
-      attr_reader :export_dir
     end
   end
 end

data/lib/bosh/director/jobs/update_release.rb

@@ -1,5 +1,4 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
+require 'securerandom'
 require 'common/version/release_version'
 
 module Bosh::Director
@@ -11,20 +10,27 @@ module Bosh::Director
       @queue = :normal
 
       attr_accessor :release_model
-      attr_accessor :tmp_release_dir
 
       def self.job_type
         :update_release
       end
 
-      # @param [String] tmp_release_dir Directory containing release bundle
+      # @param [String] release_path local path or remote url of the release archive
       # @param [Hash] options Release update options
-      def initialize(tmp_release_dir, options = {})
-        @tmp_release_dir = tmp_release_dir
+      def initialize(release_path, options = {})
+        if options['remote']
+          # file will be downloaded to the release_path
+          @release_path = File.join(Dir.tmpdir, "release-#{SecureRandom.uuid}")
+          @release_url = release_path
+        else
+          # file already exists at the release_path
+          @release_path = release_path
+        end
+
         @release_model = nil
         @release_version_model = nil
 
-        @rebase = !!options["rebase"]
+        @rebase = !!options['rebase']
 
         @manifest = nil
         @name = nil
@@ -32,68 +38,62 @@ module Bosh::Director
 
         @packages_unchanged = false
         @jobs_unchanged = false
-
-        @remote_release = options['remote'] || false
-        @remote_release_location = options['location'] if @remote_release
       end
 
-      # Extracts release tarball, verifies release manifest and saves release
-      # in DB
+      # Extracts release tarball, verifies release manifest and saves release in DB
       # @return [void]
       def perform
         logger.info("Processing update release")
-        if @rebase
-          logger.info("Release rebase will be performed")
-        end
+        logger.info("Release rebase will be performed") if @rebase
+
+        single_step_stage("Downloading remote release") { download_remote_release } if @release_url
 
-        single_step_stage("Downloading remote release") { download_remote_release } if @remote_release
-        single_step_stage("Extracting release") { extract_release }
-        single_step_stage("Verifying manifest") { verify_manifest }
+        release_dir = nil
+        single_step_stage("Extracting release") { release_dir = extract_release }
 
-        with_release_lock(@name) { process_release }
+        single_step_stage("Verifying manifest") { verify_manifest(release_dir) }
+
+        with_release_lock(@name) { process_release(release_dir) }
 
         if @rebase && @packages_unchanged && @jobs_unchanged
-          raise DirectorError,
-                "Rebase is attempted without any job or package changes"
+          raise DirectorError, "Rebase is attempted without any job or package changes"
         end
 
         "Created release `#{@name}/#{@version}'"
+
       rescue Exception => e
         remove_release_version_model
         raise e
+
       ensure
-        if @tmp_release_dir && File.exists?(@tmp_release_dir)
-          FileUtils.rm_rf(@tmp_release_dir)
-        end
+        FileUtils.rm_rf(release_dir) if release_dir
+        FileUtils.rm_rf(@release_path) if @release_path
       end
 
       def download_remote_release
-        release_file = File.join(@tmp_release_dir, Api::ReleaseManager::RELEASE_TGZ)
-        download_remote_file('release', @remote_release_location, release_file)
+        download_remote_file('release', @release_url, @release_path)
       end
 
       # Extracts release tarball
       # @return [void]
       def extract_release
-        release_tgz = File.join(@tmp_release_dir,
-                                Api::ReleaseManager::RELEASE_TGZ)
+        release_dir = Dir.mktmpdir
 
-        result = Bosh::Exec.sh("tar -C #{@tmp_release_dir} -xzf #{release_tgz} 2>&1", :on_error => :return)
+        result = Bosh::Exec.sh("tar -C #{release_dir} -xzf #{@release_path} 2>&1", :on_error => :return)
         if result.failed?
-          logger.error("Extracting release archive failed in dir #{@tmp_release_dir}, " +
+          logger.error("Failed to extract release archive '#{@release_path}' into dir '#{release_dir}', " +
                        "tar returned #{result.exit_status}, " +
                        "output: #{result.output}")
           raise ReleaseInvalidArchive, "Extracting release archive failed. Check task debug log for details."
         end
-      ensure
-        if release_tgz && File.exists?(release_tgz)
-          FileUtils.rm(release_tgz)
-        end
+
+        release_dir
       end
 
+      # @param [String] release_dir local path to the unpacked release
       # @return [void]
-      def verify_manifest
-        manifest_file = File.join(@tmp_release_dir, "release.MF")
+      def verify_manifest(release_dir)
+        manifest_file = File.join(release_dir, "release.MF")
         unless File.file?(manifest_file)
           raise ReleaseManifestNotFound, "Release manifest not found"
         end
@@ -117,8 +117,9 @@ module Bosh::Director
       end
 
       # Processes uploaded release, creates jobs and packages in DB if needed
+      # @param [String] release_dir local path to the unpacked release
       # @return [void]
-      def process_release
+      def process_release(release_dir)
         @release_model = Models::Release.find_or_create(:name => @name)
 
         if @rebase
@@ -150,8 +151,8 @@ module Bosh::Director
         end
 
         @packages = {}
-        process_packages
-        process_jobs
+        process_packages(release_dir)
+        process_jobs(release_dir)
 
         event_log.begin_stage("Release has been created", 1)
         event_log.track("#{@name}/#{@version}") {}
@@ -197,8 +198,9 @@ module Bosh::Director
       # Finds all package definitions in the manifest and sorts them into two
       # buckets: new and existing packages, then creates new packages and points
       # current release version to the existing packages.
+      # @param [String] release_dir local path to the unpacked release
       # @return [void]
-      def process_packages
+      def process_packages(release_dir)
         logger.info("Checking for new packages in release")
 
         new_packages = []
@@ -232,14 +234,15 @@ module Bosh::Director
           end
         end
 
-        create_packages(new_packages)
+        create_packages(new_packages, release_dir)
         use_existing_packages(existing_packages)
       end
 
       # Creates packages using provided metadata
       # @param [Array<Hash>] packages Packages metadata
+      # @param [String] release_dir local path to the unpacked release
       # @return [void]
-      def create_packages(packages)
+      def create_packages(packages, release_dir)
         if packages.empty?
           @packages_unchanged = true
           return
@@ -250,7 +253,7 @@ module Bosh::Director
           package_desc = "#{package_meta["name"]}/#{package_meta["version"]}"
           event_log.track(package_desc) do
             logger.info("Creating new package `#{package_desc}'")
-            package = create_package(package_meta)
+            package = create_package(package_meta, release_dir)
             register_package(package)
           end
         end
@@ -272,8 +275,9 @@ module Bosh::Director
 
       # Creates package in DB according to given metadata
       # @param [Hash] package_meta Package metadata
+      # @param [String] release_dir local path to the unpacked release
       # @return [void]
-      def create_package(package_meta)
+      def create_package(package_meta, release_dir)
         name, version = package_meta["name"], package_meta["version"]
 
         package_attrs = {
@@ -296,7 +300,7 @@ module Bosh::Director
         else
           logger.info("Creating #{desc} from provided bits")
 
-          package_tgz = File.join(@tmp_release_dir, "packages", "#{name}.tgz")
+          package_tgz = File.join(release_dir, "packages", "#{name}.tgz")
           result = Bosh::Exec.sh("tar -tzf #{package_tgz} 2>&1", :on_error => :return)
           if result.failed?
             logger.error("Extracting #{desc} archive failed, " +
@@ -322,8 +326,9 @@ module Bosh::Director
       # Finds job template definitions in release manifest and sorts them into
       # two buckets: new and existing job templates, then creates new job
       # template records in the database and points release version to existing ones.
+      # @param [String] release_dir local path to the unpacked release
       # @return [void]
-      def process_jobs
+      def process_jobs(release_dir)
         logger.info("Checking for new jobs in release")
 
         new_jobs = []
@@ -346,11 +351,11 @@ module Bosh::Director
           end
         end
 
-        create_jobs(new_jobs)
+        create_jobs(new_jobs, release_dir)
         use_existing_jobs(existing_jobs)
       end
 
-      def create_jobs(jobs)
+      def create_jobs(jobs, release_dir)
         if jobs.empty?
           @jobs_unchanged = true
           return
@@ -361,13 +366,13 @@ module Bosh::Director
           job_desc = "#{job_meta["name"]}/#{job_meta["version"]}"
           event_log.track(job_desc) do
             logger.info("Creating new template `#{job_desc}'")
-            template = create_job(job_meta)
+            template = create_job(job_meta, release_dir)
             register_template(template)
           end
         end
       end
 
-      def create_job(job_meta)
+      def create_job(job_meta, release_dir)
         name, version = job_meta["name"], job_meta["version"]
 
         template_attrs = {
@@ -382,8 +387,8 @@ module Bosh::Director
                     "from provided bits")
         template = Models::Template.new(template_attrs)
 
-        job_tgz = File.join(@tmp_release_dir, "jobs", "#{name}.tgz")
-        job_dir = File.join(@tmp_release_dir, "jobs", "#{name}")
+        job_tgz = File.join(release_dir, "jobs", "#{name}.tgz")
+        job_dir = File.join(release_dir, "jobs", "#{name}")
 
         FileUtils.mkdir_p(job_dir)
 

data/lib/bosh/director/jobs/update_stemcell.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2009-2012 VMware, Inc.
+require 'securerandom'
 
 module Bosh::Director
   module Jobs
@@ -14,27 +14,33 @@ module Bosh::Director
         :update_stemcell
       end
 
-      # @param [String] stemcell_file Stemcell tarball path
-      def initialize(stemcell_file, options = {})
-        @stemcell_file = stemcell_file
+      # @param [String] stemcell_path local path or remote url of the stemcell archive
+      # @param [Hash] options Stemcell update options
+      def initialize(stemcell_path, options = {})
+        if options['remote']
+          # file will be downloaded to the stemcell_path
+          @stemcell_path = File.join(Dir.tmpdir, "stemcell-#{SecureRandom.uuid}")
+          @stemcell_url = stemcell_path
+        else
+          # file already exists at the stemcell_path
+          @stemcell_path = stemcell_path
+        end
+
         @cloud = Config.cloud
         @stemcell_manager = Api::StemcellManager.new
-        @remote_stemcell = options['remote'] || false
       end
 
       def perform
         logger.info("Processing update stemcell")
-        event_log.begin_stage("Update stemcell", update_steps)
 
-        if @remote_stemcell
-          downloaded_stemcell_dir = Dir.mktmpdir("downloaded-stemcell")        
-          download_remote_stemcell(downloaded_stemcell_dir)
-        end
-        
+        event_log.begin_stage("Update stemcell", @stemcell_url ? UPDATE_STEPS + 1 : UPDATE_STEPS)
+
+        track_and_log("Downloading remote stemcell") { download_remote_stemcell } if @stemcell_url
+
         stemcell_dir = Dir.mktmpdir("stemcell")
 
         track_and_log("Extracting stemcell archive") do
-          result = Bosh::Exec.sh("tar -C #{stemcell_dir} -xzf #{@stemcell_file} 2>&1", :on_error => :return)
+          result = Bosh::Exec.sh("tar -C #{stemcell_dir} -xzf #{@stemcell_path} 2>&1", :on_error => :return)
           if result.failed?
             logger.error("Extracting stemcell archive failed in dir #{stemcell_dir}, " +
                          "tar returned #{result.exit_status}, " +
@@ -75,8 +81,7 @@ module Bosh::Director
         stemcell.sha1 = @sha1
 
         track_and_log("Uploading stemcell #{@name}/#{@version} to the cloud") do
-          stemcell.cid =
-            @cloud.create_stemcell(@stemcell_image, @cloud_properties)
+          stemcell.cid = @cloud.create_stemcell(@stemcell_image, @cloud_properties)
           logger.info("Cloud created stemcell: #{stemcell.cid}")
         end
 
@@ -87,22 +92,13 @@ module Bosh::Director
         "/stemcells/#{stemcell.name}/#{stemcell.version}"
       ensure
         FileUtils.rm_rf(stemcell_dir) if stemcell_dir
-        FileUtils.rm_rf(downloaded_stemcell_dir) if downloaded_stemcell_dir
-        FileUtils.rm_rf(@stemcell_file) if @stemcell_file
-      end
-      
-      def download_remote_stemcell(downloaded_stemcell_dir)
-        track_and_log("Downloading remote stemcell") do
-          downloaded_stemcell_file = File.join(downloaded_stemcell_dir, "stemcell-#{SecureRandom.uuid}")
-          download_remote_file('stemcell', @stemcell_file, downloaded_stemcell_file)
-          @stemcell_file = downloaded_stemcell_file          
-        end
+        FileUtils.rm_rf(@stemcell_path) if @stemcell_path
       end
       
       private
-      
-      def update_steps
-        @remote_stemcell ? UPDATE_STEPS + 1 : UPDATE_STEPS
+
+      def download_remote_stemcell
+        download_remote_file('stemcell', @stemcell_url, @stemcell_path)
       end
     end
   end

data/lib/bosh/director/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Director
-    VERSION = '1.2641.0'
+    VERSION = '1.2652.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh-director
 version: !ruby/object:Gem::Version
-  version: 1.2641.0
+  version: 1.2652.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-11 00:00:00.000000000 Z
+date: 2014-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bcrypt-ruby
@@ -30,126 +30,126 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
 - !ruby/object:Gem::Dependency
   name: bosh-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
 - !ruby/object:Gem::Dependency
   name: bosh-director-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
 - !ruby/object:Gem::Dependency
   name: bosh_common
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
 - !ruby/object:Gem::Dependency
   name: bosh_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
 - !ruby/object:Gem::Dependency
   name: bosh_openstack_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
 - !ruby/object:Gem::Dependency
   name: bosh_aws_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
 - !ruby/object:Gem::Dependency
   name: bosh_vsphere_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
 - !ruby/object:Gem::Dependency
   name: bosh_warden_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2641.0
+        version: 1.2652.0
 - !ruby/object:Gem::Dependency
   name: bosh_vcloud_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -446,7 +446,7 @@ dependencies:
         version: '0'
 description: |-
   BOSH Director
-  23fe6f
+  43d2e8
 email: support@cloudfoundry.com
 executables:
 - bosh-director
@@ -502,6 +502,7 @@ files:
 - lib/bosh/director/api.rb
 - lib/bosh/director/api/api_helper.rb
 - lib/bosh/director/api/backup_manager.rb
+- lib/bosh/director/api/compiled_package_group_manager.rb
 - lib/bosh/director/api/controller.rb
 - lib/bosh/director/api/controller_helpers.rb
 - lib/bosh/director/api/controllers/backups_controller.rb