bosh-director 1.1798.0 → 1.1836.0

data/lib/bosh/director/compile_task_generator.rb

@@ -0,0 +1,51 @@
+require 'bosh/director'
+
+module Bosh::Director
+  class CompileTaskGenerator
+    def initialize(logger, event_log)
+      @logger = logger
+      @event_log = event_log
+    end
+
+    # The compile_tasks hash passed in by the caller will be populated with CompileTasks objects
+    def generate!(compile_tasks, job, template, package, stemcell)
+      # Our assumption here is that package dependency graph
+      # has no cycles: this is being enforced on release upload.
+      # Other than that it's a vanilla DFS.
+
+      @logger.info("Checking whether package `#{package.desc}' needs to be compiled for stemcell `#{stemcell.desc}'")
+      task_key = [package.id, stemcell.id]
+      task = compile_tasks[task_key]
+
+      if task # We already visited this task and its dependencies
+        task.add_job(job) # But we still need to register this job with task
+        return task
+      end
+
+      release_version = template.release.model
+
+      task = CompileTask.new(package,
+                             stemcell,
+                             job,
+                             release_version.package_dependency_key(package.name),
+                             release_version.package_cache_key(package.name, stemcell))
+
+      compiled_package = task.find_compiled_package(@logger, @event_log)
+      if compiled_package
+        task.use_compiled_package(compiled_package)
+      end
+
+        @logger.info("Processing package `#{package.desc}' dependencies")
+      dependencies = release_version.dependencies(package.name)
+      dependencies.each do |dependency|
+        @logger.info("Package `#{package.desc}' depends on package `#{dependency.desc}'")
+        dependency_task = generate!(compile_tasks, job, template, dependency, stemcell)
+        task.add_dependency(dependency_task)
+      end
+
+      compile_tasks[task_key] = task
+      task
+    end
+
+  end
+end

data/lib/bosh/director/compressed_rendered_job_templates.rb

@@ -1,5 +1,7 @@
 require 'tmpdir'
 require 'digest/sha1'
+require 'bosh/director/tar_gzipper'
+require 'bosh/director/rendered_templates_writer'
 
 module Bosh::Director
   class CompressedRenderedJobTemplates

data/lib/bosh/director/config.rb

@@ -53,7 +53,11 @@ module Bosh::Director
         end
 
         @blobstore = nil
+
         @compiled_package_cache = nil
+        @compiled_package_blobstore = nil
+        @compiled_package_cache_options = nil
+
         @nats = nil
         @nats_rpc = nil
         @cloud = nil
@@ -62,7 +66,7 @@ module Bosh::Director
       def configure(config)
         @max_vm_create_tries = Integer(config.fetch('max_vm_create_tries', 5))
 
-        @base_dir = config["dir"]
+        @base_dir = config['dir']
         FileUtils.mkdir_p(@base_dir)
 
         # checkpoint task progress every 30 secs
@@ -85,14 +89,14 @@ module Bosh::Director
         @event_log = EventLog::Log.new
 
         # by default keep only last 500 tasks in disk
-        @max_tasks = config.fetch("max_tasks", 500).to_i
+        @max_tasks = config.fetch('max_tasks', 500).to_i
 
-        @max_threads = config.fetch("max_threads", 32).to_i
+        @max_threads = config.fetch('max_threads', 32).to_i
 
         self.redis_options= {
-          :host     => config["redis"]["host"],
-          :port     => config["redis"]["port"],
-          :password => config["redis"]["password"],
+          :host     => config['redis']['host'],
+          :port     => config['redis']['port'],
+          :password => config['redis']['password'],
           :logger   => redis_logger
         }
 
@@ -101,29 +105,29 @@ module Bosh::Director
         @logger.info("Starting BOSH Director: #{VERSION} (#{@revision})")
 
         @process_uuid = SecureRandom.uuid
-        @nats_uri = config["mbus"]
+        @nats_uri = config['mbus']
 
-        @cloud_options = config["cloud"]
-        @compiled_package_cache_options = config["compiled_package_cache"]
-        @name = config["name"] || ""
+        @cloud_options = config['cloud']
+        @compiled_package_cache_options = config['compiled_package_cache']
+        @name = config['name'] || ''
 
         @compiled_package_cache = nil
 
         @db_config = config['db']
-        @db = configure_db(config["db"])
-        @dns = config["dns"]
-        @dns_domain_name = "bosh"
+        @db = configure_db(config['db'])
+        @dns = config['dns']
+        @dns_domain_name = 'bosh'
         if @dns
-          @dns_db = configure_db(@dns["db"]) if @dns["db"]
-          @dns_domain_name = canonical(@dns["domain_name"]) if @dns["domain_name"]
+          @dns_db = configure_db(@dns['db']) if @dns['db']
+          @dns_domain_name = canonical(@dns['domain_name']) if @dns['domain_name']
         end
 
         @uuid = override_uuid || Bosh::Director::Models::DirectorAttribute.find_or_create_uuid(@logger)
         @logger.info("Director UUID: #{@uuid}")
 
-        @encryption = config["encryption"]
-        @fix_stateful_nodes = config.fetch("scan_and_fix", {})
-          .fetch("auto_fix_stateful_nodes", false)
+        @encryption = config['encryption']
+        @fix_stateful_nodes = config.fetch('scan_and_fix', {})
+          .fetch('auto_fix_stateful_nodes', false)
         @enable_snapshots = config.fetch('snapshots', {}).fetch('enabled', false)
 
         Bosh::Clouds::Config.configure(self)
@@ -140,15 +144,15 @@ module Bosh::Director
       end
 
       def get_revision
-        Dir.chdir(File.expand_path("../../../../../..", __FILE__))
-        revision_command = "(cat REVISION 2> /dev/null || " +
-            "git show-ref --head --hash=8 2> /dev/null || " +
-            "echo 00000000) | head -n1"
+        Dir.chdir(File.expand_path('../../../../../..', __FILE__))
+        revision_command = '(cat REVISION 2> /dev/null || ' +
+            'git show-ref --head --hash=8 2> /dev/null || ' +
+            'echo 00000000) | head -n1'
         `#{revision_command}`.strip
       end
 
       def configure_db(db_config)
-        patch_sqlite if db_config["adapter"] == "sqlite"
+        patch_sqlite if db_config['adapter'] == 'sqlite'
 
         connection_options = db_config.delete('connection_options') {{}}
         db_config.delete_if { |_, v| v.to_s.empty? }
@@ -166,8 +170,8 @@ module Bosh::Director
       def compiled_package_cache_blobstore
         @lock.synchronize do
           if @compiled_package_cache_blobstore.nil? && use_compiled_package_cache?
-            provider = @compiled_package_cache_options["provider"]
-            options = @compiled_package_cache_options["options"]
+            provider = @compiled_package_cache_options['provider']
+            options = @compiled_package_cache_options['options']
             @compiled_package_cache_blobstore = Bosh::Blobstore::Client.safe_create(provider, options)
           end
         end
@@ -175,20 +179,20 @@ module Bosh::Director
       end
 
       def compiled_package_cache_provider
-        use_compiled_package_cache? ? @compiled_package_cache_options["provider"] : nil
+        use_compiled_package_cache? ? @compiled_package_cache_options['provider'] : nil
       end
 
       def cloud_type
         if @cloud_options
-          @cloud_options["plugin"]
+          @cloud_options['plugin']
         end
       end
 
       def cloud
         @lock.synchronize do
           if @cloud.nil?
-            plugin = @cloud_options["plugin"]
-            properties = @cloud_options["properties"]
+            plugin = @cloud_options['plugin']
+            properties = @cloud_options['properties']
             @cloud = Bosh::Clouds::Provider.create(plugin, properties)
           end
         end
@@ -266,8 +270,8 @@ module Bosh::Director
         return if @patched_sqlite
         @patched_sqlite = true
 
-        require "sequel"
-        require "sequel/adapters/sqlite"
+        require 'sequel'
+        require 'sequel/adapters/sqlite'
 
         Sequel::SQLite::Database.class_eval do
           def connect(server)

data/lib/bosh/director/deployment_plan/assembler.rb

@@ -389,7 +389,6 @@ module Bosh::Director
         state = idle_vm.current_state
         state['job'] = instance.job.spec
         state['index'] = instance.index
-        state['release'] = instance.job.release.spec
         agent.apply(state)
 
         # Our assumption here is that director database access

data/lib/bosh/director/deployment_plan/idle_vm.rb

@@ -20,11 +20,18 @@ module Bosh::Director
       attr_accessor :vm
 
       # @return [Hash] Current state as provided by the BOSH Agent
-      attr_accessor :current_state
+      attr_writer :current_state
 
       # @return [DeploymentPlan::Instance, nil] Instance that reserved this VM
       attr_accessor :bound_instance
 
+      def current_state
+        if @current_state
+          @current_state.delete("release")
+        end
+        @current_state
+      end
+
       ##
       # Creates a new idle VM reference for the specific resource pool
       # @param [DeploymentPlan::ResourcePool] resource_pool Resource pool
@@ -106,4 +113,4 @@ module Bosh::Director
       end
     end
   end
-end
+end

data/lib/bosh/director/deployment_plan/instance.rb

@@ -345,7 +345,6 @@ module Bosh::Director
       def spec
         spec = {
           'deployment' => @job.deployment.name,
-          'release' => job.release.spec,
           'job' => job.spec,
           'index' => index,
           'networks' => network_settings,

data/lib/bosh/director/deployment_plan/job.rb

@@ -121,7 +121,6 @@ module Bosh::Director
         first_template = @templates[0]
         result = {
           "name" => @name,
-          "release" => @release.name,
           "templates" => [],
           # --- Legacy ---
           "template" => first_template.name,

data/lib/bosh/director/instance_updater.rb

@@ -322,7 +322,6 @@ module Bosh::Director
           "resource_pool" => @job.resource_pool.spec,
           "job" => @job.spec,
           "index" => @instance.index,
-          "release" => @job.release.spec
       }
 
       if @instance.disk_size > 0

data/lib/bosh/director/package_compiler.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2009-2012 VMware, Inc.
+require 'bosh/director/compile_task_generator'
 
 module Bosh::Director
   class PackageCompiler
@@ -27,6 +27,8 @@ module Bosh::Director
 
       @vm_reuser = VmReuser.new
 
+      @compile_task_generator = CompileTaskGenerator.new(@logger, @event_log)
+
       @compile_tasks = {}
       @ready_tasks = []
       @compilations_performed = 0
@@ -183,59 +185,27 @@ module Bosh::Director
 
       @event_log.track('Finding packages to compile') do
         @deployment_plan.jobs.each do |job|
-          job_desc = "#{job.release.name}/#{job.name}"
           stemcell = job.resource_pool.stemcell
 
-          @logger.info("Job `#{job_desc}' needs to run on stemcell `#{stemcell.model.desc}'")
+          template_descs = job.templates.map do |t|
+            # we purposefully did NOT inline those because
+            # when instance_double blows up,
+            # it's obscure which double is at fault
+            release_name = t.release.name
+            template_name = t.name
+            "`#{release_name}/#{template_name}'"
+          end
+          @logger.info("Job templates #{template_descs.join(', ')} need to run on stemcell `#{stemcell.model.desc}'")
 
           job.templates.each do |template|
             template.package_models.each do |package|
-              generate_compile_task(job, package, stemcell.model)
+              @compile_task_generator.generate!(@compile_tasks, job, template, package, stemcell.model)
             end
           end
         end
       end
     end
 
-    def generate_compile_task(job, package, stemcell)
-      # Our assumption here is that package dependency graph
-      # has no cycles: this is being enforced on release upload.
-      # Other than that it's a vanilla DFS.
-
-      @logger.info("Checking whether package `#{package.desc}' needs to be compiled for stemcell `#{stemcell.desc}'")
-      task_key = [package.id, stemcell.id]
-      task = @compile_tasks[task_key]
-
-      if task # We already visited this task and its dependencies
-        task.add_job(job) # But we still need to register this job with task
-        return task
-      end
-
-      release_version = job.release.model
-
-      task = CompileTask.new(package,
-                             stemcell,
-                             job,
-                             release_version.package_dependency_key(package.name),
-                             release_version.package_cache_key(package.name, stemcell))
-
-      compiled_package = task.find_compiled_package(@logger, @event_log)
-      if compiled_package
-        task.use_compiled_package(compiled_package)
-      end
-
-      @logger.info("Processing package `#{package.desc}' dependencies")
-      dependencies = release_version.dependencies(package.name)
-      dependencies.each do |dependency|
-        @logger.info("Package `#{package.desc}' depends on package `#{dependency.desc}'")
-        dependency_task = generate_compile_task(job, dependency, stemcell)
-        task.add_dependency(dependency_task)
-      end
-
-      @compile_tasks[task_key] = task
-      task
-    end
-
     def tear_down_vm(vm_data)
       vm = vm_data.vm
       if vm.exists?

data/lib/bosh/director/rendered_job_templates_persister.rb

@@ -1,6 +1,6 @@
 require 'tempfile'
-require 'bosh/director/rendered_templates_writer'
 require 'bosh/director/compressed_rendered_job_templates'
+require 'bosh/director/deployment_plan/rendered_templates_archive'
 
 module Bosh::Director
   class RenderedJobTemplatesPersister

data/lib/bosh/director/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Director
-    VERSION = '1.1798.0'
+    VERSION = '1.1836.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bosh-director
 version: !ruby/object:Gem::Version
-  version: 1.1798.0
+  version: 1.1836.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-18 00:00:00.000000000 Z
+date: 2014-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bcrypt-ruby
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
 - !ruby/object:Gem::Dependency
   name: bosh-core
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
 - !ruby/object:Gem::Dependency
   name: bosh_common
   requirement: !ruby/object:Gem::Requirement
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
 - !ruby/object:Gem::Dependency
   name: bosh_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -82,7 +82,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
 - !ruby/object:Gem::Dependency
   name: bosh_openstack_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -98,7 +98,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
 - !ruby/object:Gem::Dependency
   name: bosh_aws_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -114,7 +114,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
 - !ruby/object:Gem::Dependency
   name: bosh_vsphere_cpi
   requirement: !ruby/object:Gem::Requirement
@@ -130,7 +130,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -138,7 +138,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1798.0
+        version: 1.1836.0
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
@@ -461,7 +461,7 @@ dependencies:
         version: '1.0'
 description: ! 'BOSH Director
 
-  4c516b'
+  f49b5d'
 email: support@cloudfoundry.com
 executables:
 - bosh-director
@@ -545,6 +545,7 @@ files:
 - lib/bosh/director/blobstores.rb
 - lib/bosh/director/cloudcheck_helper.rb
 - lib/bosh/director/compile_task.rb
+- lib/bosh/director/compile_task_generator.rb
 - lib/bosh/director/compiled_package.rb
 - lib/bosh/director/compiled_package/blob_sha_mismatch_error.rb
 - lib/bosh/director/compiled_package/compiled_package.rb
@@ -714,7 +715,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1547539602198935562
+      hash: 3345351914595372714
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23