bosh-director 1.3063.0 → 1.3068.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7ac5ff2357caa3c0640c5e26554f1515c241fc9c
-  data.tar.gz: a0a297f8295f7607b2302a8b71586eae974e5100
+  metadata.gz: 9cae55bf1eca66fc2b7350f2d90cf6c771ff6536
+  data.tar.gz: 706c4b982e6e7faef9bee55cf2ee49d0f291dbac
 SHA512:
-  metadata.gz: 88b3761ba7e48100c86310bbf3bec22a00241f1c93e66e6b2092c5c1932913381af25cd37b3c6befc6576856b810266ef449cb70d73c26aedfe6ec51bc9afa20
-  data.tar.gz: 0b9aae13fd73fe0b644edc889edca059b31f288f32e2a5446e15b7e696cf1a11ea557e773a7d3cb07846498e80da07afd8078add4c836af7868185abd57c485a
+  metadata.gz: 94f7c1ee49d3ab4018517437d1f602518ade5803bf712dfab62e83f520108be596fb171aa93b9074380ff35dfea95552313bcc71f4691eb46760dc42d072d367
+  data.tar.gz: 8e3632b636703a18fc8fe46f5987cf952e6726dcdfeffee66dfb4ae800e04b4008d02a40a3f75cb16318599ddbaf81f42ae6fc0f6d53e95f51f702b5c5493260

data/lib/bosh/director/agent_client.rb

@@ -123,12 +123,12 @@ module Bosh::Director
       end
     end
 
-    def run_scripts(scripts_list, options)
+    def run_script(script_name, options)
       begin
-        send_message(:run_scripts, scripts_list, options)
+        send_message(:run_script, script_name, options)
       rescue RpcRemoteException => e
         if e.message =~ /unknown message/
-          @logger.warn("Ignoring run_scripts 'unknown message' error from the agent: #{e.inspect}. Received while trying to run : #{scripts_list}")
+          @logger.warn("Ignoring run_script 'unknown message' error from the agent: #{e.inspect}. Received while trying to run: #{script_name}")
         else
           raise
         end

data/lib/bosh/director/api/controllers/resources_controller.rb

@@ -10,8 +10,15 @@ module Bosh::Director
       end
 
       get '/:id' do
+        @resource_manager.clean_old_tmpfiles
+
         tmp_file = @resource_manager.get_resource_path(params[:id])
-        send_disposable_file(tmp_file, :type => 'application/x-gzip')
+
+        status 200
+        headers \
+          'Content-Type'  => 'application/x-gzip',
+          'X-Accel-Redirect' => File.join('/x_accel_files/', tmp_file)
+        body ''
       end
     end
   end

data/lib/bosh/director/api/resource_manager.rb

@@ -1,5 +1,7 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
+require 'fileutils'
+
 module Bosh::Director
   module Api
     class ResourceManager
@@ -10,14 +12,17 @@ module Bosh::Director
       end
 
       # Retrieves the resource `id` from the blobstore and stores it
-      # locally, and returns the path to the file
+      # locally, and returns the path to the file. It is the caller's
+      # responsibility to delete the resulting file at some point.
+      # An easy option is to call clean_old_tmpfiles before each call
+      # to this method.
       #
       # @param [String] id
       # @return [String] path to the contents of the blobstore id
       def get_resource_path(id)
         blobstore_resource(id) do |blobstore|
           random_name = "resource-#{SecureRandom.uuid}"
-          path = File.join(Dir.tmpdir, random_name)
+          path = File.join(resource_tmpdir, random_name)
 
           File.open(path, "w") do |f|
             blobstore.get(id, f)
@@ -27,6 +32,17 @@ module Bosh::Director
         end
       end
 
+      # Returns the directory where files created by get_resource_path will be written.
+      def resource_tmpdir
+        Dir.tmpdir
+      end
+
+      # Deletes all get_resource_path temporary files that are more than 5 minutes old.
+      def clean_old_tmpfiles
+        Dir.glob("#{resource_tmpdir}/resource-*").
+            select{|f| File.mtime(f) < (Time.now - (60*5)) }.
+            each{|f| FileUtils.rm_f(f) }
+      end
       # Retrieves the resource `id` from the blobstore and returns the
       # contents of it.
       # @param [String] id

data/lib/bosh/director/cloudcheck_helper.rb

@@ -154,6 +154,7 @@ module Bosh::Director
       agent_client(new_vm).apply(spec)
 
       if instance && instance.state == "started"
+        agent_client(new_vm).run_script('pre-start', {})
         agent_client(new_vm).start
       end
     end

data/lib/bosh/director/deployment_plan/cloud_manifest_parser.rb

@@ -1,5 +1,3 @@
-require 'common/deep_copy'
-
 module Bosh::Director
   module DeploymentPlan
     class CloudManifestParser

data/lib/bosh/director/deployment_plan/deployment_spec_parser.rb

@@ -1,5 +1,3 @@
-require 'common/deep_copy'
-
 module Bosh::Director
   module DeploymentPlan
     class DeploymentSpecParser

data/lib/bosh/director/deployment_plan/job_spec_parser.rb

@@ -1,4 +1,3 @@
-require 'common/deep_copy'
 require 'bosh/template/property_helper'
 
 module Bosh::Director

data/lib/bosh/director/deployment_plan/planner.rb

@@ -2,6 +2,7 @@ require 'bosh/director/deployment_plan/deployment_spec_parser'
 require 'bosh/director/deployment_plan/cloud_manifest_parser'
 require 'bosh/director/deployment_plan/disk_pool'
 require 'forwardable'
+require 'common/deep_copy'
 
 module Bosh::Director
   # Encapsulates essential director data structures retrieved
@@ -53,7 +54,7 @@ module Bosh::Director
         @properties = attrs.fetch(:properties)
         @releases = {}
 
-        @manifest_text = manifest_text
+        @manifest_text = Bosh::Common::DeepCopy.copy(manifest_text)
         @cloud_config = cloud_config
         @cloud_planner = CloudPlanner.new(cloud_config)
         @model = deployment_model

data/lib/bosh/director/errors.rb

@@ -75,6 +75,8 @@ module Bosh::Director
   ReleaseVersionInvalid = err(30010)
   ReleaseNotMatchingManifest = err(30011)
   ReleaseInvalidPackage = err(30012)
+  ReleaseExistingJobFingerprintMismatch = err(30013)
+  ReleaseVersionCommitHashMismatch = err(30014)
 
   ValidationInvalidType = err(40000)
   ValidationMissingField = err(40001)

data/lib/bosh/director/instance_updater.rb

@@ -119,17 +119,7 @@ module Bosh::Director
     end
 
     def run_pre_start_scripts
-      event_log.begin_stage('running pre-start scripts', 1)
-
-      pre_start_scripts_list = []
-      @instance.spec['job']['templates'].each do |template|
-        pre_start_scripts_list << "jobs/#{template['name']}/bin/pre-start"
-      end
-
-      event_log.track(@instance.spec['job']['name']) {
-        @agent.run_scripts(pre_start_scripts_list, {})
-      }
-
+      @agent.run_script("pre-start", {})
     end
 
     def start!
@@ -385,9 +375,5 @@ module Bosh::Director
       delete_mounted_disk(new_disk)
       raise
     end
-
-    def event_log
-      @event_log ||= Config.event_log
-    end
   end
 end

data/lib/bosh/director/jobs/update_release.rb

@@ -28,14 +28,9 @@ module Bosh::Director
           @release_path = release_path
         end
 
-        @release_model = nil
-        @release_version_model = nil
+        @release_model, @release_version_model, @manifest, @name, @version = nil, nil, nil, nil, nil
 
         @rebase = !!options['rebase']
-
-        @manifest = nil
-        @name = nil
-        @version = nil
       end
 
       # Extracts release tarball, verifies release manifest and saves release in DB
@@ -51,14 +46,11 @@ module Bosh::Director
 
         single_step_stage("Verifying manifest") { verify_manifest(release_dir) }
 
-        with_release_lock(@name) {
-          process_release(release_dir)
-        }
+        with_release_lock(@name) { process_release(release_dir) }
 
         "Created release `#{@name}/#{@version}'"
 
       rescue Exception => e
-        remove_release_version_model
         raise e
 
       ensure
@@ -103,9 +95,7 @@ module Bosh::Director
 
         begin
           @version = Bosh::Common::Version::ReleaseVersion.parse(@manifest["version"])
-          unless @version == @manifest["version"]
-            logger.info("Formatted version '#{@manifest["version"]}' => '#{@version}'")
-          end
+          logger.info("Formatted version '#{@manifest["version"]}' => '#{@version}'") unless @version == @manifest["version"]
         rescue SemiSemantic::ParseError
           raise ReleaseVersionInvalid, "Release version invalid: #{@manifest["version"]}"
         end
@@ -129,18 +119,22 @@ module Bosh::Director
       def process_release(release_dir)
         @release_model = Models::Release.find_or_create(:name => @name)
 
-        if @rebase
-          @version = next_release_version
-        end
+        @version = next_release_version if @rebase
 
         version_attrs = { :release => @release_model, :version => @version.to_s }
-        version_attrs[:uncommitted_changes] = @uncommitted_changes if @uncommitted_changes
-        version_attrs[:commit_hash] = @commit_hash if @commit_hash
 
-        @release_is_new = false
-        @release_version_model = Models::ReleaseVersion.find_or_create(version_attrs) {
-          @release_is_new = true
-        }
+        release_is_new = false
+        @release_version_model = Models::ReleaseVersion.find_or_create(version_attrs){ release_is_new = true }
+
+        if release_is_new
+          @release_version_model.uncommitted_changes = @uncommitted_changes if @uncommitted_changes
+          @release_version_model.commit_hash = @commit_hash if @commit_hash
+          @release_version_model.save
+        else
+          if @release_version_model.commit_hash != @commit_hash || @release_version_model.uncommitted_changes != @uncommitted_changes
+            raise ReleaseVersionCommitHashMismatch, "release `#{@name}/#{@version}' has already been uploaded with commit_hash as `#{@commit_hash}' and uncommitted_changes as `#{@uncommitted_changes}'"
+          end
+        end
 
         single_step_stage("Resolving package dependencies") do
           resolve_package_dependencies(@manifest[@packages_folder])
@@ -148,7 +142,7 @@ module Bosh::Director
 
         @packages = {}
         process_packages(release_dir)
-        process_jobs(release_dir) if @release_is_new
+        process_jobs(release_dir)
 
         event_log.begin_stage(@compiled_release ? "Compiled Release has been created" : "Release has been created", 1)
         event_log.track("#{@name}/#{@version}") {}
@@ -200,13 +194,16 @@ module Bosh::Director
         registered_packages = []
 
         @manifest[@packages_folder].each do |package_meta|
-          # Checking whether we might have the same bits somewhere
+          # Checking whether we might have the same bits somewhere (in any release, not just the one being uploaded)
+          @release_version_model.packages.select { |pv| pv.name == package_meta['name'] }.each do |package|
+            if package.fingerprint != package_meta['fingerprint']
+              raise ReleaseInvalidPackage, "package `#{package_meta['name']}' had different fingerprint in previously uploaded release `#{@name}/#{@version}'"
+            end
+          end
 
           packages = Models::Package.where(fingerprint: package_meta["fingerprint"]).all
+
           if packages.empty?
-            unless @release_is_new
-              raise ReleaseInvalidPackage, "package #{package_meta['name']}/#{package_meta['version']} not part of previous upload of release #{@name}/#{@version}"
-            end
             new_packages << package_meta
             next
           end
@@ -227,14 +224,25 @@ module Bosh::Director
             end
 
             if existing_package.release_versions.include? @release_version_model
+              if existing_package.blobstore_id.nil?
+                packages.each do |package|
+                  unless package.blobstore_id.nil?
+                    package_meta["blobstore_id"] = package.blobstore_id
+                    package_meta["sha1"] = package.sha1
+                    break
+                  end
+                end
+              end
               registered_packages << [existing_package, package_meta]
             else
               existing_packages << [existing_package, package_meta]
             end
+
           else
             # We found a package with the same fingerprint but different
             # (release, name, version) tuple, so we need to make a copy
             # of the package blob and create a new db entry for it
+
             packages.each do |package|
               unless package.blobstore_id.nil?
                 package_meta["blobstore_id"] = package.blobstore_id
@@ -246,13 +254,9 @@ module Bosh::Director
           end
         end
 
-        did_something = false
-
-        package_stemcell_hashes1, created_package = create_packages(new_packages, release_dir)
-        did_something |= created_package
+        package_stemcell_hashes1 = create_packages(new_packages, release_dir)
 
-        package_stemcell_hashes2, modified_package = use_existing_packages(existing_packages, release_dir)
-        did_something |= modified_package
+        package_stemcell_hashes2 = use_existing_packages(existing_packages, release_dir)
 
         if @compiled_release
           compatible_stemcell_combos = registered_packages.flat_map do |pkg, pkg_meta|
@@ -264,11 +268,10 @@ module Bosh::Director
             end
           end
           consolidated_package_stemcell_hashes = Array(package_stemcell_hashes1) | Array(package_stemcell_hashes2) | compatible_stemcell_combos
-          did_something |= create_compiled_packages(consolidated_package_stemcell_hashes, release_dir)
+          create_compiled_packages(consolidated_package_stemcell_hashes, release_dir)
         else
-          did_something |= backfill_source_for_packages(registered_packages, release_dir)
+          backfill_source_for_packages(registered_packages, release_dir)
         end
-        did_something
       end
 
       # @return [boolean] true if sources were added to at least one package; false if the call had no effect.
@@ -293,7 +296,7 @@ module Bosh::Director
       # @return [Array<Hash>] package & stemcell matching pairs that were registered. empty if no packages were changed.
       def use_existing_packages(packages, release_dir)
         if packages.empty?
-          return [], false
+          return []
         end
 
         package_stemcell_hashes = []
@@ -319,17 +322,16 @@ module Bosh::Director
           end
         end
 
-        return package_stemcell_hashes, true
+        return package_stemcell_hashes
       end
 
       # Creates packages using provided metadata
       # @param [Array<Hash>] packages Packages metadata
       # @param [String] release_dir local path to the unpacked release
       # @return [Array<Hash>, boolean] array of compiled package & stemcell matching pairs that were registered, and a
-      # flag indicating if any changes were made to the database.
       def create_packages(package_metas, release_dir)
         if package_metas.empty?
-          return [], false
+          return []
         end
 
         package_stemcell_hashes = []
@@ -354,7 +356,7 @@ module Bosh::Director
           end
         end
 
-        return package_stemcell_hashes, true
+        return package_stemcell_hashes
       end
 
       # @return [boolean] true if at least one job was created; false if the call had no effect.
@@ -450,8 +452,7 @@ module Bosh::Director
       def save_package_source_blob(package, package_meta, release_dir)
         return false unless package.blobstore_id.nil?
 
-        name, version = package_meta['name'], package_meta['version']
-        existing_blob = package_meta['blobstore_id']
+        name, version, existing_blob = package_meta['name'], package_meta['version'], package_meta['blobstore_id']
         desc = "package '#{name}/#{version}'"
 
         package.sha1 = package_meta['sha1']
@@ -500,6 +501,12 @@ module Bosh::Director
 
         @manifest["jobs"].each do |job_meta|
           # Checking whether we might have the same bits somewhere
+          @release_version_model.templates.select { |t| t.name == job_meta["name"] }.each do |tmpl|
+            if tmpl.fingerprint != job_meta["fingerprint"]
+              raise ReleaseExistingJobFingerprintMismatch, "job `#{job_meta["name"]}' had different fingerprint in previously uploaded release `#{@name}/#{@version}'"
+            end
+          end
+
           jobs = Models::Template.where(fingerprint: job_meta["fingerprint"]).all
 
           template = jobs.find do |job|
@@ -552,13 +559,15 @@ module Bosh::Director
           jobs.each do |template, _|
             job_desc = "#{template.name}/#{template.version}"
             logger.info("Using existing job `#{job_desc}'")
-            register_template(template)
+            register_template(template) unless template.release_versions.include? @release_version_model
           end
         end
 
         true
       end
 
+      private
+
       # Marks job template model as being used by release version
       # @param [Models::Template] template Job template model
       # @return [void]
@@ -566,8 +575,6 @@ module Bosh::Director
         @release_version_model.add_template(template)
       end
 
-      private
-
       # Returns the next release version (to be used for rebased release)
       # @return [String]
       def next_release_version
@@ -577,16 +584,6 @@ module Bosh::Director
         list = Bosh::Common::Version::ReleaseVersionList.parse(strings)
         list.rebase(@version)
       end
-
-      # Removes release version model, along with all packages and templates.
-      # @return [void]
-      def remove_release_version_model
-        return unless @release_version_model && !@release_version_model.new?
-
-        @release_version_model.remove_all_packages
-        @release_version_model.remove_all_templates
-        @release_version_model.destroy
-      end
     end
   end
 end

data/lib/bosh/director/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Director
-    VERSION = '1.3063.0'
+    VERSION = '1.3068.0'
   end
 end

metadata

@@ -1,155 +1,155 @@
 --- !ruby/object:Gem::Specification
 name: bosh-director
 version: !ruby/object:Gem::Version
-  version: 1.3063.0
+  version: 1.3068.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-01 00:00:00.000000000 Z
+date: 2015-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bcrypt-ruby
+  name: bosh_common
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 1.3068.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 1.3068.0
 - !ruby/object:Gem::Dependency
-  name: blobstore_client
+  name: bosh_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
 - !ruby/object:Gem::Dependency
-  name: bosh-core
+  name: bosh-registry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
 - !ruby/object:Gem::Dependency
-  name: bosh-director-core
+  name: blobstore_client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
 - !ruby/object:Gem::Dependency
-  name: bosh_common
+  name: bosh-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
 - !ruby/object:Gem::Dependency
-  name: bosh-template
+  name: bosh-director-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
 - !ruby/object:Gem::Dependency
-  name: bosh_cpi
+  name: bosh-template
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
 - !ruby/object:Gem::Dependency
   name: bosh_openstack_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: bosh_aws_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 1.3068.0
 - !ruby/object:Gem::Dependency
   name: bosh_vsphere_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3063.0
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: bosh_vcloud_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +164,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.11.0
+- !ruby/object:Gem::Dependency
+  name: bcrypt-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.1
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
@@ -570,9 +584,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: |-
-  BOSH Director
-  ac5134
+description: BOSH Director
 email: support@cloudfoundry.com
 executables:
 - bosh-director