bosh-director 1.5.0.pre.1284 → 1.5.0.pre.1302

data/lib/bosh/director/api/controllers/compiled_packages_controller.rb

@@ -1,9 +1,11 @@
 require 'json'
+require 'tempfile'
 require 'bosh/director/api/controllers/base_controller'
 require 'bosh/director/api/stemcell_manager'
 require 'bosh/director/compiled_package_group'
 require 'bosh/director/compiled_packages_exporter'
 require 'bosh/director/stale_file_killer'
+require 'bosh/director/jobs/import_compiled_packages'
 
 module Bosh::Director
   module Api::Controllers
@@ -28,6 +30,20 @@ module Bosh::Director
         send_file(output_path, type: :tgz)
       end
 
+      post '/compiled_package_groups/import', consumes: [:tgz] do
+        tempdir = Dir.mktmpdir
+        export_path = File.join(tempdir, 'compiled_packages_export.tgz')
+        # the job is responsible for cleaning this up
+        File.open(export_path, 'w') do |f|
+          while buf = request.body.read(4096)
+            f.write(buf)
+          end
+        end
+
+        task = JobQueue.new.enqueue(@user, Jobs::ImportCompiledPackages, 'import compiled packages', [tempdir])
+        redirect "/tasks/#{task.id}"
+      end
+
       def find_stemcell_by_name_and_version
         stemcell_manager = Api::StemcellManager.new
         stemcell_manager.find_by_name_and_version(

data/lib/bosh/director/compiled_package/blob_sha_mismatch_error.rb

@@ -0,0 +1,6 @@
+require 'bosh/director/compiled_package'
+
+module Bosh::Director::CompiledPackage
+  class BlobShaMismatchError < StandardError
+  end
+end

data/lib/bosh/director/compiled_package/compiled_package.rb

@@ -0,0 +1,33 @@
+require 'digest/sha1'
+
+require 'bosh/director/compiled_package'
+require 'bosh/director/compiled_package/blob_sha_mismatch_error'
+
+module Bosh::Director::CompiledPackage
+
+  class CompiledPackage
+    attr_reader :package_name, :package_fingerprint, :sha1, :stemcell_sha1, :blobstore_id, :blob_path
+
+    def initialize(options = {})
+      @package_name = options.fetch(:package_name)
+      @package_fingerprint = options.fetch(:package_fingerprint)
+      @sha1 = options.fetch(:sha1)
+      @stemcell_sha1 = options.fetch(:stemcell_sha1)
+      @blobstore_id = options.fetch(:blobstore_id)
+      @blob_path = options.fetch(:blob_path)
+    end
+
+    def check_blob_sha
+      if sha1 != actual_sha
+        raise BlobShaMismatchError, "Blob SHA mismatch in file #{blob_path}: expected: #{sha1}, got #{actual_sha}"
+      end
+    end
+
+    private
+
+    def actual_sha
+      @actual_sha ||= Digest::SHA1.file(blob_path).hexdigest
+    end
+
+  end
+end

data/lib/bosh/director/compiled_package/compiled_package_inserter.rb

@@ -0,0 +1,22 @@
+require 'bosh/director/compiled_package'
+
+module Bosh::Director::CompiledPackage
+
+  class CompiledPackageInserter
+
+    def insert(compiled_package, release_version)
+      package = Bosh::Director::Models::Package[fingerprint: compiled_package.package_fingerprint]
+      stemcell = Bosh::Director::Models::Stemcell[sha1: compiled_package.stemcell_sha1]
+
+      Bosh::Director::Models::CompiledPackage.create(
+        blobstore_id: compiled_package.blobstore_id,
+        package_id: package.id,
+        stemcell_id: stemcell.id,
+        sha1: compiled_package.sha1,
+        dependency_key: release_version.package_dependency_key(package.name),
+        build: Bosh::Director::Models::CompiledPackage.generate_build_number(package, stemcell),
+      )
+    end
+
+  end
+end

data/lib/bosh/director/compiled_package/compiled_packages_export.rb

@@ -0,0 +1,37 @@
+require 'bosh/director/compiled_package'
+require 'bosh/director/compiled_package/compiled_package'
+
+module Bosh::Director::CompiledPackage
+  class CompiledPackagesExport
+
+    def initialize(options={})
+      @file = options.fetch(:file)
+      @exec = options.fetch(:exec, Bosh::Exec)
+    end
+
+    def extract(tempdir=nil)
+      tempdir ||= Dir.mktmpdir
+      exec.sh("tar -C #{tempdir} -xf #{file}")
+      manifest = YAML.load_file("#{tempdir}/compiled_packages.MF")
+      packages = []
+      manifest['compiled_packages'].each do |p|
+        packages << CompiledPackage.new(
+          package_name: p['package_name'],
+          package_fingerprint: p['package_fingerprint'],
+          sha1: p['compiled_package_sha1'],
+          stemcell_sha1: p['stemcell_sha1'],
+          blobstore_id: p['blobstore_id'],
+          blob_path: File.join(tempdir, 'compiled_packages', 'blobs', p['blobstore_id']))
+      end
+
+      yield manifest, packages
+
+      FileUtils.rm_rf(tempdir)
+    end
+
+    private
+
+    attr_reader :file, :exec
+
+  end
+end

data/lib/bosh/director/compiled_package.rb

@@ -0,0 +1,4 @@
+module Bosh::Director
+  module CompiledPackage
+  end
+end

data/lib/bosh/director/compiled_package_group.rb

@@ -18,5 +18,7 @@ module Bosh::Director
     def stemcell_sha1
       @stemcell.sha1
     end
+
+    attr_reader :release_version
   end
 end

data/lib/bosh/director/compiled_package_manifest.rb

@@ -9,10 +9,14 @@ module Bosh::Director
 
     def to_h
       {
+        'release_name' => @compiled_package_group.release_version.release.name,
+        'release_version' => @compiled_package_group.release_version.version,
+        'release_commit_hash' => @compiled_package_group.release_version.commit_hash,
         'compiled_packages' => @compiled_package_group.compiled_packages.map do |compiled_package|
           {
             'package_name' => compiled_package.package.name,
             'package_fingerprint' => compiled_package.package.fingerprint,
+            'compiled_package_sha1' => compiled_package.sha1,
             'stemcell_sha1' => @compiled_package_group.stemcell_sha1,
             'blobstore_id' => compiled_package.blobstore_id,
           }

data/lib/bosh/director/jobs/import_compiled_packages.rb

@@ -0,0 +1,59 @@
+require 'bosh/director/compiled_package/compiled_packages_export'
+require 'bosh/director/compiled_package/compiled_package_inserter'
+
+module Bosh::Director
+  module Jobs
+    class ImportCompiledPackages < BaseJob
+      @queue = :normal
+
+      def self.job_type
+        :import_compiled_packages
+      end
+
+      def initialize(export_dir, options={})
+        @export_dir = export_dir
+
+        @blobstore_client = options.fetch(:blobstore_client) { Bosh::Director::App.instance.blobstores.blobstore }
+      end
+
+      def perform
+        export = Bosh::Director::CompiledPackage::CompiledPackagesExport.new(file: export_path)
+
+        export.extract do |manifest, packages|
+          packages.each { |p| p.check_blob_sha }
+
+          packages.each do |package|
+            File.open(package.blob_path) do |f|
+              blobstore_client.create(f, package.blobstore_id)
+            end
+          end
+
+          release_name = manifest.fetch('release_name')
+          release_version = manifest.fetch('release_version')
+
+          release = Bosh::Director::Models::Release[name: release_name]
+          release_version = Bosh::Director::Models::ReleaseVersion[release_id: release.id, version: release_version]
+
+          inserter = Bosh::Director::CompiledPackage::CompiledPackageInserter.new
+
+          Bosh::Director::Config.db.transaction do
+            packages.each do |package|
+              inserter.insert(package, release_version)
+            end
+          end
+        end
+      ensure
+        FileUtils.rm_rf(@export_dir)
+      end
+
+      private
+
+      def export_path
+        File.join(@export_dir, 'compiled_packages_export.tgz')
+      end
+
+      attr_reader :blobstore_client
+      attr_reader :export_dir
+    end
+  end
+end

data/lib/bosh/director/version.rb

@@ -2,6 +2,6 @@
 
 module Bosh
   module Director
-    VERSION = '1.5.0.pre.1284'
+    VERSION = '1.5.0.pre.1302'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bosh-director
 version: !ruby/object:Gem::Version
-  version: 1.5.0.pre.1284
+  version: 1.5.0.pre.1302
   prerelease: 6
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-08 00:00:00.000000000 Z
+date: 2013-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bcrypt-ruby
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
 - !ruby/object:Gem::Dependency
   name: bosh-core
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
 - !ruby/object:Gem::Dependency
   name: bosh_common
   requirement: !ruby/object:Gem::Requirement
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
 - !ruby/object:Gem::Dependency
   name: bosh_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -82,7 +82,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
 - !ruby/object:Gem::Dependency
   name: bosh_openstack_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -98,7 +98,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
 - !ruby/object:Gem::Dependency
   name: bosh_aws_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -114,7 +114,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
 - !ruby/object:Gem::Dependency
   name: bosh_vsphere_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -130,7 +130,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -138,7 +138,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1284
+        version: 1.5.0.pre.1302
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
@@ -445,7 +445,7 @@ dependencies:
         version: '1.0'
 description: ! 'BOSH Director
 
-  52e0da'
+  64af4e'
 email: support@cloudfoundry.com
 executables:
 - bosh-director
@@ -525,6 +525,11 @@ files:
 - lib/bosh/director/blobstores.rb
 - lib/bosh/director/cloudcheck_helper.rb
 - lib/bosh/director/compile_task.rb
+- lib/bosh/director/compiled_package.rb
+- lib/bosh/director/compiled_package/blob_sha_mismatch_error.rb
+- lib/bosh/director/compiled_package/compiled_package.rb
+- lib/bosh/director/compiled_package/compiled_package_inserter.rb
+- lib/bosh/director/compiled_package/compiled_packages_export.rb
 - lib/bosh/director/compiled_package_downloader.rb
 - lib/bosh/director/compiled_package_group.rb
 - lib/bosh/director/compiled_package_manifest.rb
@@ -586,6 +591,7 @@ files:
 - lib/bosh/director/jobs/delete_snapshots.rb
 - lib/bosh/director/jobs/delete_stemcell.rb
 - lib/bosh/director/jobs/fetch_logs.rb
+- lib/bosh/director/jobs/import_compiled_packages.rb
 - lib/bosh/director/jobs/scheduled_backup.rb
 - lib/bosh/director/jobs/snapshot_deployment.rb
 - lib/bosh/director/jobs/snapshot_deployments.rb