bosh-director 1.3016.0 → 1.3022.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13cd05082e949cce8da88e0ce3b104ab3b5db3cb
-  data.tar.gz: ad109214ab949c9b1da47ed09609436dcc902eaa
+  metadata.gz: 59a2d6077683ae1eaac10b9926fd87139cbf2193
+  data.tar.gz: e417875f5b9b4311ca2ffc19cbea42a4b4e673ed
 SHA512:
-  metadata.gz: 087eaba75dfd98fa0a496dc5b1d7acd10b4e55743ab056ac0e054013b74628cc749747bee4e6de73ece1d870bc507dfc3b2aa3f97e7331d7b548106c9a6c6732
-  data.tar.gz: def64e21c02dcc889608d462ce197a0ec746ea05657cc5379685dc82fe6a4bf14cfd21d9d65297b301db7f54f2838d9a297e32a76a3252be9a7e97a883dfde3b
+  metadata.gz: fc4627b56956d3e9ce4795286416798815f1475b5542aee9b7de375f1b8c6b64f083b02b93db041e3530ab799f63732bc67bd3e87662e131ffd81c9dcd8216c5
+  data.tar.gz: c0d98a660d38ed280d769427e1ba7adb37e66426a2ccd7d46156c582194d0950eb6628a68999e257db3a92dd79871eeb8b29a954d8a29b01d303a82e744f6cec

data/db/migrations/director/20150716170926_allow_null_on_blobstore_id_and_sha1_on_package.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    alter_table(:packages) do
+      set_column_allow_null :blobstore_id, true
+      set_column_allow_null :sha1, true
+    end
+  end
+end

data/lib/bosh/director/api/controllers/releases_controller.rb

@@ -16,7 +16,9 @@ module Bosh::Director
       post '/', :consumes => :multipart do
         options = {
           rebase: params['rebase'] == 'true',
+          compiled: params['compiled'] == 'true',
         }
+
         task = @release_manager.create_release_from_file_path(current_user, params[:nginx_upload_path], options)
         redirect "/tasks/#{task.id}"
       end

data/lib/bosh/director/api/release_manager.rb

@@ -67,7 +67,7 @@ module Bosh::Director
         JobQueue.new.enqueue(
             username,
             Jobs::ExportRelease,
-            "export release: #{release_name}/#{release_version} for #{stemcell_os}/#{stemcell_version}",
+            "export release: '#{release_name}/#{release_version}' for '#{stemcell_os}/#{stemcell_version}'",
             [deployment_name, release_name, release_version, stemcell_os, stemcell_version])
       end
     end

data/lib/bosh/director/compiled_release_downloader.rb

@@ -0,0 +1,43 @@
+require 'fileutils'
+require 'tmpdir'
+require 'bosh/director'
+
+module Bosh::Director
+  class CompiledReleaseDownloader
+    def initialize(compiled_packages_group, templates, blobstore_client)
+      @compiled_packages_group = compiled_packages_group
+      @templates = templates
+      @blobstore_client = blobstore_client
+    end
+
+    def download
+      @download_dir = Dir.mktmpdir
+
+      path = File.join(@download_dir, 'compiled_packages')
+      FileUtils.mkpath(path)
+
+      @compiled_packages_group.compiled_packages.each do |compiled_package|
+        blobstore_id = compiled_package.blobstore_id
+        File.open(File.join(path, "#{compiled_package.package.name}.tgz"), 'w') do |f|
+          @blobstore_client.get(blobstore_id, f, sha1: compiled_package.sha1)
+        end
+      end
+
+      path = File.join(@download_dir, 'jobs')
+      FileUtils.mkpath(path)
+
+      @templates.each do |template|
+        blobstore_id = template.blobstore_id
+        File.open(File.join(path, "#{template.name}.tgz"), 'w') do |f|
+          @blobstore_client.get(blobstore_id, f, sha1: template.sha1)
+        end
+      end
+
+      @download_dir
+    end
+
+    def cleanup
+      FileUtils.rm_rf(@download_dir)
+    end
+  end
+end

data/lib/bosh/director/compiled_release_manifest.rb

@@ -0,0 +1,57 @@
+require 'yaml'
+require 'bosh/director'
+
+module Bosh::Director
+  class CompiledReleaseManifest
+    def initialize(compiled_package_group, templates, stemcell)
+      @compiled_packages = compiled_package_group
+      @templates = templates
+      @stemcell = stemcell
+    end
+
+    def generate_manifest
+      manifest = {}
+
+      manifest['compiled_packages'] = @compiled_packages.compiled_packages.map do |compiled_package|
+        {
+          'name' => compiled_package.package.name,
+          'version' => compiled_package.package.version,
+          'fingerprint' => compiled_package.package.fingerprint,
+          'sha1' => compiled_package.sha1,
+          'stemcell' => "#{@stemcell.operating_system}/#{@stemcell.version}",
+          'dependencies' => get_dependencies(compiled_package),
+        }
+      end
+
+      manifest['jobs'] = @templates.map do |template|
+        {
+          'name' => template.name,
+          'version' => template.version,
+          'fingerprint' => template.fingerprint,
+          'sha1' => template.sha1,
+        }
+      end
+
+      manifest['commit_hash'] = @compiled_packages.release_version.commit_hash
+      manifest['uncommitted_changes'] = @compiled_packages.release_version.uncommitted_changes
+      manifest['name'] = @compiled_packages.release_version.release.name
+      manifest['version'] = @compiled_packages.release_version.version
+
+      manifest
+    end
+
+    def get_dependencies(compiled_package)
+      dependencies = []
+      parser = Yajl::Parser.new
+      hash = parser.parse(compiled_package.package.dependency_set_json)
+      hash.each do |name|
+        dependencies << name
+      end
+      dependencies
+    end
+
+    def write(dest_path)
+      File.open(dest_path, 'w') { |f| f.write(YAML.dump(generate_manifest)) }
+    end
+  end
+end

data/lib/bosh/director/jobs/export_release.rb

@@ -1,6 +1,7 @@
 require 'securerandom'
 require 'common/version/release_version'
-require 'pp'
+require 'bosh/director/compiled_release_downloader'
+require 'bosh/director/compiled_release_manifest'
 
 module Bosh::Director
   module Jobs
@@ -44,9 +45,14 @@ module Bosh::Director
                   self
               )
               package_compile_step.perform
+
+              tarball_state = create_tarball
+              result_file.write(tarball_state.to_json + "\n")
+
             end
           end
         end
+        "Exported release: #{@release_name}/#{@release_version} for #{@stemcell_os}/#{@stemcell_version}"
       end
 
       def validate_and_prepare
@@ -66,13 +72,6 @@ module Bosh::Director
       def create_planner
         modified_deployment_manifest = Psych.load(@targeted_deployment.manifest)
         cloud_config_model = @targeted_deployment.cloud_config
-        network_name = cloud_config_model.manifest['networks'][0]['name']
-
-        fake_resource_pool_manifest = {
-            "name" => "just_for_compiling",
-            "network" => network_name,
-            "stemcell" => { "name" => @stemcell.name, "version" => @stemcell.version }
-        }
 
         planner_factory = DeploymentPlan::PlannerFactory.create(Config.event_log, Config.logger)
         planner = planner_factory.planner_without_vm_binding(
@@ -80,6 +79,13 @@ module Bosh::Director
             cloud_config_model,
             {}
         )
+        network_name = planner.networks.first.name
+
+        fake_resource_pool_manifest = {
+            "name" => "just_for_compiling",
+            "network" => network_name,
+            "stemcell" => { "name" => @stemcell.name, "version" => @stemcell.version }
+        }
 
         resource_pool = DeploymentPlan::ResourcePool.new(planner, fake_resource_pool_manifest, Config.logger)
         planner.add_resource_pool(resource_pool)
@@ -91,6 +97,35 @@ module Bosh::Director
         planner
       end
 
+      def create_tarball
+
+        blobstore_client = Bosh::Director::App.instance.blobstores.blobstore
+
+        compiled_packages_group = CompiledPackageGroup.new(@release_version, @stemcell)
+        templates = @release_version.templates.map
+
+        compiled_release_downloader = CompiledReleaseDownloader.new(compiled_packages_group, templates, blobstore_client)
+        download_dir = compiled_release_downloader.download
+
+        manifest = CompiledReleaseManifest.new(compiled_packages_group, templates, @stemcell)
+        manifest.write(File.join(download_dir, 'release.MF'))
+
+        output_path = File.join(download_dir, "compiled_release_#{Time.now.to_f}.tar.gz")
+        archiver = Core::TarGzipper.new
+
+        archiver.compress(download_dir, ['compiled_packages', 'jobs', 'release.MF'], output_path)
+        tarball_file = File.open(output_path, 'r')
+
+        oid = blobstore_client.create(tarball_file)
+
+        {
+            :blobstore_id => oid,
+            :sha1 => Digest::SHA1.file(output_path).hexdigest,
+        }
+      ensure
+        compiled_release_downloader.cleanup unless compiled_release_downloader.nil?
+      end
+
       def create_fake_job(planner, fake_resource_pool_manifest, network_name)
         fake_job_spec_for_compiling = {
             "name" => "dummy-job-for-compilation",
@@ -114,3 +149,4 @@ module Bosh::Director
     end
   end
 end
+

data/lib/bosh/director/jobs/update_release.rb

@@ -8,6 +8,7 @@ module Bosh::Director
       include DownloadHelper
 
       @queue = :normal
+      @compiled_release = false
 
       attr_accessor :release_model
 
@@ -32,6 +33,13 @@ module Bosh::Director
 
         @rebase = !!options['rebase']
         @skip_if_exists = !!options['skip_if_exists']
+        @compiled_release = !!options['compiled']
+
+        if @compiled_release
+          @packages_folder = "compiled_packages"
+        else
+          @packages_folder = "packages"
+        end
 
         @manifest = nil
         @name = nil
@@ -54,7 +62,9 @@ module Bosh::Director
 
         single_step_stage("Verifying manifest") { verify_manifest(release_dir) }
 
-        with_release_lock(@name) { process_release(release_dir) }
+        with_release_lock(@name) {
+          process_release(release_dir)
+        }
 
         if @rebase && @packages_unchanged && @jobs_unchanged
           raise DirectorError, "Rebase is attempted without any job or package changes"
@@ -152,14 +162,14 @@ module Bosh::Director
         @release_version_model.save
 
         single_step_stage("Resolving package dependencies") do
-          resolve_package_dependencies(@manifest["packages"])
+          resolve_package_dependencies(@manifest[@packages_folder])
         end
 
         @packages = {}
         process_packages(release_dir)
         process_jobs(release_dir)
 
-        event_log.begin_stage("Release has been created", 1)
+        event_log.begin_stage(@compiled_release ? "Compiled Release has been created" : "Release has been created", 1)
         event_log.track("#{@name}/#{@version}") {}
       end
 
@@ -168,7 +178,7 @@ module Bosh::Director
       def normalize_manifest
         Bosh::Director.hash_string_vals(@manifest, 'name', 'version')
 
-        @manifest['packages'].each { |p| Bosh::Director.hash_string_vals(p, 'name', 'version', 'sha1') }
+        @manifest[@packages_folder].each { |p| Bosh::Director.hash_string_vals(p, 'name', 'version', 'sha1') }
         @manifest['jobs'].each { |j| Bosh::Director.hash_string_vals(j, 'name', 'version', 'sha1') }
       end
 
@@ -207,7 +217,7 @@ module Bosh::Director
         new_packages = []
         existing_packages = []
 
-        @manifest["packages"].each do |package_meta|
+        @manifest[@packages_folder].each do |package_meta|
           # Checking whether we might have the same bits somewhere
           packages = Models::Package.where(fingerprint: package_meta["fingerprint"]).all
 
@@ -262,7 +272,13 @@ module Bosh::Director
           package_desc = "#{package_meta["name"]}/#{package_meta["version"]}"
           event_log.track(package_desc) do
             logger.info("Creating new package `#{package_desc}'")
-            package = create_package(package_meta, release_dir)
+
+            if @compiled_release
+              package = create_package_for_compiled_release(package_meta, release_dir)
+            else
+              package = create_package(package_meta, release_dir)
+            end
+
             register_package(package)
           end
         end
@@ -282,6 +298,26 @@ module Bosh::Director
         end
       end
 
+      def create_package_for_compiled_release(package_meta, release_dir)
+        name, version = package_meta["name"], package_meta["version"]
+        desc = "package `#{name}/#{version}'"
+
+        package_attrs = {
+            :release => @release_model,
+            :name => name,
+            :sha1 => nil,
+            :blobstore_id => nil,
+            :fingerprint => package_meta["fingerprint"],
+            :version => version
+        }
+
+        package = Models::Package.new(package_attrs)
+        package.dependency_set = package_meta["dependencies"]
+
+        logger.info("Creating #{desc} from provided bits")
+        package.save
+      end
+
       # Creates package in DB according to given metadata
       # @param [Hash] package_meta Package metadata
       # @param [String] release_dir local path to the unpacked release

data/lib/bosh/director/models/package.rb

@@ -18,7 +18,7 @@ module Bosh::Director::Models
     end
 
     def validate
-      validates_presence [:release_id, :name, :version, :blobstore_id, :sha1]
+      validates_presence [:release_id, :name, :version]
       validates_unique [:release_id, :name, :version]
       validates_format VALID_ID, [:name, :version]
     end

data/lib/bosh/director/problem_handlers/missing_disk.rb

@@ -46,7 +46,7 @@ module Bosh::Director
         end
 
         # If VM is present we try to unmount and detach disk from VM
-        if @vm.cid && cloud.has_vm?(@vm.cid)
+        if @vm && @vm.cid && cloud.has_vm?(@vm.cid)
           agent_client = agent_client(@vm)
           disk_list = []
 

data/lib/bosh/director/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Director
-    VERSION = '1.3016.0'
+    VERSION = '1.3022.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh-director
 version: !ruby/object:Gem::Version
-  version: 1.3016.0
+  version: 1.3022.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-14 00:00:00.000000000 Z
+date: 2015-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bcrypt-ruby
@@ -30,126 +30,126 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
 - !ruby/object:Gem::Dependency
   name: bosh-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
 - !ruby/object:Gem::Dependency
   name: bosh-director-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
 - !ruby/object:Gem::Dependency
   name: bosh_common
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
 - !ruby/object:Gem::Dependency
   name: bosh-template
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
 - !ruby/object:Gem::Dependency
   name: bosh_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
 - !ruby/object:Gem::Dependency
   name: bosh_openstack_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
 - !ruby/object:Gem::Dependency
   name: bosh_aws_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
 - !ruby/object:Gem::Dependency
   name: bosh_vsphere_cpi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3016.0
+        version: 1.3022.0
 - !ruby/object:Gem::Dependency
   name: bosh_vcloud_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -572,7 +572,7 @@ dependencies:
         version: '0'
 description: |-
   BOSH Director
-  10557b
+  6073a7
 email: support@cloudfoundry.com
 executables:
 - bosh-director
@@ -629,6 +629,7 @@ files:
 - db/migrations/director/20150401184803_add_cloud_config_to_deployments.rb
 - db/migrations/director/20150611193110_add_trusted_certs_sha1_to_vms.rb
 - db/migrations/director/20150619135210_add_os_name_and_version_to_stemcells.rb
+- db/migrations/director/20150716170926_allow_null_on_blobstore_id_and_sha1_on_package.rb
 - db/migrations/dns/20120123234908_initial.rb
 - lib/bosh/director.rb
 - lib/bosh/director/agent_client.rb
@@ -692,6 +693,8 @@ files:
 - lib/bosh/director/compiled_package_group.rb
 - lib/bosh/director/compiled_package_manifest.rb
 - lib/bosh/director/compiled_packages_exporter.rb
+- lib/bosh/director/compiled_release_downloader.rb
+- lib/bosh/director/compiled_release_manifest.rb
 - lib/bosh/director/config.rb
 - lib/bosh/director/cycle_helper.rb
 - lib/bosh/director/db_backup.rb