bosh-director 1.5.0.pre.1391 → 1.5.0.pre.1435

data/db/migrations/director/20131125232201_rename_rendered_templates_archives_blob_id_and_checksum_columns.rb

@@ -0,0 +1,17 @@
+Sequel.migration do
+  change do
+    drop_table(:rendered_templates_archives)
+
+    create_table(:rendered_templates_archives) do
+      primary_key :id
+      foreign_key :instance_id, :instances, null: false
+
+      String :blobstore_id, null: false
+      String :sha1,         null: false
+      String :content_sha1, null: false
+      Time   :created_at, null: false
+
+      index :created_at
+    end
+  end
+end

data/lib/bosh/director/compressed_rendered_job_templates.rb

@@ -1,10 +1,28 @@
+require 'tmpdir'
+require 'digest/sha1'
+
 module Bosh::Director
   class CompressedRenderedJobTemplates
-    def initialize(rendered_job_templates)
+    def initialize(path)
+      @path = path
+    end
+
+    def write(rendered_templates)
+      Dir.mktmpdir do |dir|
+        writer = RenderedTemplatesWriter.new
+        writer.write(rendered_templates, dir)
+
+        tar_gzipper = TarGzipper.new
+        tar_gzipper.compress(dir, %w(.), @path)
+      end
     end
 
     def contents
-      'contents of fake-compressed-job-templates.tgz'
+      File.open(@path, 'r')
+    end
+
+    def sha1
+      Digest::SHA1.file(@path).hexdigest
     end
   end
 end

data/lib/bosh/director/deployment_plan/assembler.rb

@@ -1,5 +1,3 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Bosh::Director
   # DeploymentPlan::Assembler is used to populate deployment plan with information
   # about existing deployment and information from director DB
@@ -15,6 +13,7 @@ module Bosh::Director
       @logger = Config.logger
       @event_log = Config.event_log
       @stemcell_manager = Api::StemcellManager.new
+      @blobstore = App.instance.blobstores.blobstore
     end
 
     # Binds deployment DB record to a plan
@@ -322,7 +321,7 @@ module Bosh::Director
     # @return [void]
     def bind_configuration
       @deployment_plan.jobs.each do |job|
-        JobRenderer.new(job).render_job_instances
+        JobRenderer.new(job).render_job_instances(@blobstore)
       end
     end
 
@@ -425,8 +424,7 @@ module Bosh::Director
       unneeded_instances = @deployment_plan.unneeded_instances
       return if unneeded_instances.empty?
 
-      @event_log.begin_stage("Deleting unneeded instances",
-                             unneeded_instances.size)
+      @event_log.begin_stage("Deleting unneeded instances", unneeded_instances.size)
       InstanceDeleter.new(@deployment_plan).delete_instances(unneeded_instances)
       @logger.info("Deleted no longer needed instances")
     end

data/lib/bosh/director/instance_deleter.rb

@@ -1,5 +1,3 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Bosh::Director
   # Coordinates the safe deletion of an instance and all associates resources.
   class InstanceDeleter
@@ -10,6 +8,7 @@ module Bosh::Director
       @cloud = Config.cloud
       @logger = Config.logger
       @event_log = Config.event_log
+      @blobstore = App.instance.blobstores.blobstore
     end
 
     # Deletes a list of instances
@@ -39,6 +38,8 @@ module Bosh::Director
         delete_persistent_disks(instance.persistent_disks)
         delete_dns(instance.job, instance.index)
 
+        RenderedJobTemplatesCleaner.new(instance, @blobstore).clean_all
+
         vm.db.transaction do
           instance.destroy
           vm.destroy

data/lib/bosh/director/instance_updater.rb

@@ -1,4 +1,5 @@
 require 'bosh/director/instance_preparer'
+require 'bosh/director/rendered_job_templates_cleaner'
 
 module Bosh::Director
   class InstanceUpdater
@@ -15,6 +16,7 @@ module Bosh::Director
       @cloud = Config.cloud
       @logger = Config.logger
       @ticker = event_ticker
+      @blobstore = App.instance.blobstores.blobstore
 
       @instance = instance
       @job = instance.job
@@ -78,6 +80,8 @@ module Bosh::Director
 
       step { apply_state(@instance.spec) }
 
+      RenderedJobTemplatesCleaner.new(@instance.model, @blobstore).clean
+
       start! if need_start?
 
       step { wait_until_running }

data/lib/bosh/director/job_instance_renderer.rb

@@ -1,6 +1,5 @@
 module Bosh::Director
   class JobInstanceRenderer
-
     def initialize(job, job_template_loader)
       @job = job
       @job_template_loader = job_template_loader

data/lib/bosh/director/job_renderer.rb

@@ -1,7 +1,7 @@
 require 'bosh/director/job_template_loader'
 require 'bosh/director/job_instance_renderer'
 require 'bosh/director/rendered_job_instance_hasher'
-require 'bosh/director/rendered_templates_uploader'
+require 'bosh/director/rendered_job_templates_persister'
 
 module Bosh::Director
   class JobRenderer
@@ -12,16 +12,16 @@ module Bosh::Director
       @instance_renderer = JobInstanceRenderer.new(@job, job_template_loader)
     end
 
-    def render_job_instances
+    def render_job_instances(blobstore)
       @job.instances.each do |instance|
         rendered_templates = @instance_renderer.render(instance)
 
-        uploader = RenderedTemplatesUploader.new
-        uploader.upload(rendered_templates)
-
         hasher = RenderedJobInstanceHasher.new(rendered_templates)
         instance.configuration_hash = hasher.configuration_hash
         instance.template_hashes = hasher.template_hashes
+
+        persister = RenderedJobTemplatesPersister.new(blobstore)
+        persister.persist(instance, rendered_templates)
       end
     end
   end

data/lib/bosh/director/job_updater.rb

@@ -1,9 +1,5 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Bosh::Director
-
   class JobUpdater
-
     # @param [Bosh::Director::DeploymentPlan] deployment_plan
     # @param [DeploymentPlan::Job] job
     def initialize(deployment_plan, job)
@@ -20,11 +16,9 @@ module Bosh::Director
 
       return if unneeded_instances.empty?
 
-      @event_log.begin_stage("Deleting unneeded instances",
-                             unneeded_instances.size, [@job.name])
-      InstanceDeleter.new(@deployment_plan).
-        delete_instances(unneeded_instances,
-                         :max_threads => @job.update.max_in_flight)
+      @event_log.begin_stage("Deleting unneeded instances", unneeded_instances.size, [@job.name])
+      deleter = InstanceDeleter.new(@deployment_plan)
+      deleter.delete_instances(unneeded_instances, max_threads: @job.update.max_in_flight)
 
       @logger.info("Deleted no longer needed instances")
     end

data/lib/bosh/director/jobs/delete_deployment.rb

@@ -1,5 +1,3 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Bosh::Director
   module Jobs
     class DeleteDeployment < BaseJob
@@ -18,6 +16,7 @@ module Bosh::Director
         @keep_snapshots = options["keep_snapshots"]
         @cloud = Config.cloud
         @deployment_manager = Api::DeploymentManager.new
+        @blobstore = App.instance.blobstores.blobstore
       end
 
       def perform
@@ -144,6 +143,10 @@ module Bosh::Director
             disk.destroy
           end
 
+          ignoring_errors_when_forced do
+            RenderedJobTemplatesCleaner.new(instance, @blobstore).clean_all
+          end
+
           instance.destroy
 
           delete_vm(vm) if vm
@@ -160,7 +163,6 @@ module Bosh::Director
             @cloud.delete_vm(vm.cid)
           end
         end
-
         vm.destroy
       end
 

data/lib/bosh/director/models.rb

@@ -18,6 +18,7 @@ require 'bosh/director/models/user'
 require 'bosh/director/models/vm'
 require 'bosh/director/models/persistent_disk'
 require 'bosh/director/models/director_attribute'
+require 'bosh/director/models/rendered_templates_archive'
 
 module Bosh::Director
   module Models

data/lib/bosh/director/models/instance.rb

@@ -1,5 +1,3 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Bosh::Director::Models
   class Instance < Sequel::Model(Bosh::Director::Config.db)
     many_to_one :deployment
@@ -25,5 +23,18 @@ module Bosh::Director::Models
       return disk.disk_cid if disk
       nil
     end
+
+    def latest_rendered_templates_archive
+      rendered_templates_archives_dataset.order(:created_at).last
+    end
+
+    def stale_rendered_templates_archives
+      stale_archives = rendered_templates_archives_dataset
+      if latest = latest_rendered_templates_archive
+        stale_archives.exclude(id: latest.id)
+      else
+        stale_archives
+      end
+    end
   end
 end

data/lib/bosh/director/rendered_job_templates_cleaner.rb

@@ -0,0 +1,22 @@
+module Bosh::Director
+  class RenderedJobTemplatesCleaner
+    def initialize(instance_model, blobstore)
+      @instance_model = instance_model
+      @blobstore = blobstore
+    end
+
+    def clean
+      @instance_model.stale_rendered_templates_archives.each do |archive|
+        @blobstore.delete(archive.blobstore_id)
+        archive.delete
+      end
+    end
+
+    def clean_all
+      @instance_model.rendered_templates_archives.each do |archive|
+        @blobstore.delete(archive.blobstore_id)
+        archive.delete
+      end
+    end
+  end
+end

data/lib/bosh/director/rendered_job_templates_persister.rb

@@ -0,0 +1,36 @@
+require 'tempfile'
+require 'bosh/director/rendered_templates_writer'
+require 'bosh/director/compressed_rendered_job_templates'
+
+module Bosh::Director
+  class RenderedJobTemplatesPersister
+    def initialize(blobstore)
+      @blobstore = blobstore
+    end
+
+    def persist(instance, rendered_job_templates)
+      current_archive = instance.model.latest_rendered_templates_archive
+      if !current_archive || current_archive.content_sha1 != instance.configuration_hash
+        persist_without_checking(instance, rendered_job_templates)
+      end
+    end
+
+    def persist_without_checking(instance, rendered_job_templates)
+      file = Tempfile.new('compressed-rendered-job-templates')
+
+      compressed_archive = CompressedRenderedJobTemplates.new(file.path)
+      compressed_archive.write(rendered_job_templates)
+
+      blobstore_id = @blobstore.create(compressed_archive.contents)
+
+      instance.model.add_rendered_templates_archive(
+        blobstore_id: blobstore_id,
+        sha1: compressed_archive.sha1,
+        content_sha1: instance.configuration_hash,
+        created_at: Time.now,
+      )
+    ensure
+      file.close!
+    end
+  end
+end

data/lib/bosh/director/version.rb

@@ -2,6 +2,6 @@
 
 module Bosh
   module Director
-    VERSION = '1.5.0.pre.1391'
+    VERSION = '1.5.0.pre.1435'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bosh-director
 version: !ruby/object:Gem::Version
-  version: 1.5.0.pre.1391
+  version: 1.5.0.pre.1435
   prerelease: 6
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-22 00:00:00.000000000 Z
+date: 2013-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bcrypt-ruby
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
 - !ruby/object:Gem::Dependency
   name: bosh-core
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
 - !ruby/object:Gem::Dependency
   name: bosh_common
   requirement: !ruby/object:Gem::Requirement
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
 - !ruby/object:Gem::Dependency
   name: bosh_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -82,7 +82,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
 - !ruby/object:Gem::Dependency
   name: bosh_openstack_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -98,7 +98,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
 - !ruby/object:Gem::Dependency
   name: bosh_aws_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -114,7 +114,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
 - !ruby/object:Gem::Dependency
   name: bosh_vsphere_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -130,7 +130,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -138,7 +138,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.0.pre.1391
+        version: 1.5.0.pre.1435
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
@@ -461,7 +461,7 @@ dependencies:
         version: '0'
 description: ! 'BOSH Director
 
-  a7d75a'
+  496235'
 email: support@cloudfoundry.com
 executables:
 - bosh-director
@@ -500,6 +500,7 @@ files:
 - db/migrations/director/20130530164918_add_paused_flag_to_instance.rb
 - db/migrations/director/20130531172604_add_director_attributes.rb
 - db/migrations/director/20131121182231_add_rendered_templates_archives.rb
+- db/migrations/director/20131125232201_rename_rendered_templates_archives_blob_id_and_checksum_columns.rb
 - db/migrations/dns/20120123234908_initial.rb
 - lib/bosh/director.rb
 - lib/bosh/director/agent_client.rb
@@ -663,7 +664,8 @@ files:
 - lib/bosh/director/problem_scanner.rb
 - lib/bosh/director/rendered_job_instance_hasher.rb
 - lib/bosh/director/rendered_job_template.rb
-- lib/bosh/director/rendered_templates_uploader.rb
+- lib/bosh/director/rendered_job_templates_cleaner.rb
+- lib/bosh/director/rendered_job_templates_persister.rb
 - lib/bosh/director/rendered_templates_writer.rb
 - lib/bosh/director/resource_pool_updater.rb
 - lib/bosh/director/scheduler.rb

data/lib/bosh/director/rendered_templates_uploader.rb

@@ -1,16 +0,0 @@
-require 'bosh/director/rendered_templates_writer'
-require 'bosh/director/compressed_rendered_job_templates'
-require 'blobstore_client/null_blobstore_client'
-
-module Bosh::Director
-  class RenderedTemplatesUploader
-    def initialize(blobstore = Bosh::Blobstore::NullBlobstoreClient.new)
-      @blobstore = blobstore
-    end
-
-    def upload(rendered_job_templates)
-      compressed_archive = CompressedRenderedJobTemplates.new(rendered_job_templates)
-      @blobstore.create(compressed_archive.contents)
-    end
-  end
-end