bosh-director 1.3202.0 → 1.3213.0

data/lib/bosh/director/api/instance_manager.rb

@@ -6,26 +6,31 @@ module Bosh::Director
       # @raise [InstanceNotFound]
       def find_instance(instance_id)
         InstanceLookup.new.by_id(instance_id)
-        end
+      end
 
-      # @param [String] deployment_name Deployment name
+      # @param [Models::Deployment] deployment
       # @param [String] job Job name
       # @param [String] index_or_id Job index or id
       # @return [Models::Instance]
-      def find_by_name(deployment_name, job, index_or_id)
+      def find_by_name(deployment, job, index_or_id)
         # This is for backwards compatibility and can be removed when we move to referencing job by instance id only.
         if index_or_id.to_s =~ /^\d+$/
-          InstanceLookup.new.by_attributes(deployment_name, job, index_or_id)
+          InstanceLookup.new.by_attributes(deployment, job, index_or_id)
         else
-          InstanceLookup.new.by_uuid(deployment_name, job, index_or_id)
+          InstanceLookup.new.by_uuid(deployment, job, index_or_id)
         end
       end
 
+      def find_instances_by_deployment(deployment)
+        InstanceLookup.new.by_deployment(deployment)
+      end
+
+      # @param [Models::Deployment] deployment
       # @param [Hash] filter Sequel-style DB record filter
       # @return [Array] List of instances that matched the filter
       # @raise [InstanceNotFound]
-      def filter_by(filter)
-        InstanceLookup.new.by_filter(filter)
+      def filter_by(deployment, filter)
+        InstanceLookup.new.by_filter(filter.merge(deployment_id: deployment.id))
       end
 
       # @param [Models::Instance] instance Instance
@@ -43,25 +48,32 @@ module Bosh::Director
         AgentClient.with_vm_credentials_and_agent_id(instance.credentials, instance.agent_id)
       end
 
-      def fetch_logs(username, deployment_name, job, index_or_id, options = {})
-        if deployment_name.nil? || job.nil? || index_or_id.nil?
+      def fetch_logs(username, deployment, job, index_or_id, options = {})
+        if deployment.nil? || job.nil? || index_or_id.nil?
           raise DirectorError,
-                'deployment, job and index/id parameters are required'
+                'deployment, instance group and index/id parameters are required'
         end
 
         # This is for backwards compatibility and can be removed when we move to referencing job by instance id only.
         if index_or_id.to_s =~ /^\d+$/
-          instance = find_by_name(deployment_name, job, index_or_id)
+          instance = find_by_name(deployment, job, index_or_id)
         else
-          instance = filter_by(uuid: index_or_id).first
+          instance = filter_by(deployment, uuid: index_or_id).first
         end
 
-        JobQueue.new.enqueue(username, Jobs::FetchLogs, 'fetch logs', [instance.id, options])
+        JobQueue.new.enqueue(username, Jobs::FetchLogs, 'fetch logs', [instance.id, options], deployment.name)
+      end
+
+      def fetch_instances(username, deployment, format)
+        JobQueue.new.enqueue(username, Jobs::VmState, 'retrieve vm-stats', [deployment.id, format, true], deployment.name)
+      end
+
+      def fetch_instances_with_vm(username, deployment, format)
+        JobQueue.new.enqueue(username, Jobs::VmState, 'retrieve vm-stats', [deployment.id, format], deployment.name)
       end
 
-      def ssh(username, options)
+      def ssh(username, deployment, options)
         description = "ssh: #{options['command']}:#{options['target']}"
-        deployment = DeploymentLookup.new.by_name(options['deployment_name'])
 
         JobQueue.new.enqueue(username, Jobs::Ssh, description, [deployment.id, options])
       end

data/lib/bosh/director/api/local_identity_provider.rb

@@ -28,14 +28,6 @@ module Bosh
 
           LocalUser.new(*auth.credentials)
         end
-
-        def valid_access?(user, _)
-          @user_manager.authenticate(user.username, user.password)
-        end
-
-        def required_scopes(_)
-          raise NotImplemented
-        end
       end
 
       class LocalUser < Struct.new(:username, :password)

data/lib/bosh/director/api/problem_manager.rb

@@ -1,20 +1,11 @@
 module Bosh::Director
   module Api
     class ProblemManager
-
-      def initialize(deployment_manager)
-        @deployment_manager = deployment_manager
-      end
-
-      def perform_scan(username, deployment_name)
-        deployment = @deployment_manager.find_by_name(deployment_name)
-
-        JobQueue.new.enqueue(username, Jobs::CloudCheck::Scan, 'scan cloud', [deployment.name])
+      def perform_scan(username, deployment)
+        JobQueue.new.enqueue(username, Jobs::CloudCheck::Scan, 'scan cloud', [deployment.name], deployment.name)
       end
 
-      def get_problems(deployment_name)
-        deployment = @deployment_manager.find_by_name(deployment_name)
-
+      def get_problems(deployment)
         filters = {
           :deployment_id => deployment.id,
           :state => 'open'
@@ -23,15 +14,12 @@ module Bosh::Director
         Models::DeploymentProblem.filter(filters).order(:created_at).all
       end
 
-      def apply_resolutions(username, deployment_name, resolutions)
-        deployment = @deployment_manager.find_by_name(deployment_name)
-        JobQueue.new.enqueue(username, Jobs::CloudCheck::ApplyResolutions, 'apply resolutions', [deployment.name, resolutions])
+      def apply_resolutions(username, deployment, resolutions)
+        JobQueue.new.enqueue(username, Jobs::CloudCheck::ApplyResolutions, 'apply resolutions', [deployment.name, resolutions], deployment.name)
       end
 
-      def scan_and_fix(username, deployment_name, jobs)
-        deployment = @deployment_manager.find_by_name(deployment_name)
-
-        JobQueue.new.enqueue(username, Jobs::CloudCheck::ScanAndFix, 'scan and fix', [deployment.name, jobs, Bosh::Director::Config.fix_stateful_nodes])
+      def scan_and_fix(username, deployment, jobs)
+        JobQueue.new.enqueue(username, Jobs::CloudCheck::ScanAndFix, 'scan and fix', [deployment.name, jobs, Bosh::Director::Config.fix_stateful_nodes], deployment.name)
       end
     end
   end

data/lib/bosh/director/api/property_manager.rb

@@ -4,13 +4,9 @@ module Bosh::Director
   module Api
     class PropertyManager
 
-      def initialize(deployment_manager)
-        @deployment_manager = deployment_manager
-      end
-
-      def create_property(deployment_name, property_name, value)
+      def create_property(deployment, property_name, value)
         property = Models::DeploymentProperty.new
-        property.deployment = find_deployment(deployment_name)
+        property.deployment = deployment
         property.name = property_name
         property.value = value
         property.save
@@ -19,13 +15,13 @@ module Bosh::Director
         if e.errors[[:name, :deployment_id]].include?(:unique)
           raise PropertyAlreadyExists,
                 "Property `#{property_name}' already exists " +
-                "for deployment `#{deployment_name}'"
+                "for deployment `#{deployment.name}'"
         end
         invalid_property(e.errors)
       end
 
-      def update_property(deployment_name, property_name, value)
-        property = get_property(deployment_name, property_name)
+      def update_property(deployment, property_name, value)
+        property = get_property(deployment, property_name)
         property.value = value
         property.save
 
@@ -33,24 +29,23 @@ module Bosh::Director
         invalid_property(e.errors)
       end
 
-      def delete_property(deployment_name, property_name)
-        get_property(deployment_name, property_name).destroy
+      def delete_property(deployment, property_name)
+        get_property(deployment, property_name).destroy
       end
 
-      def get_property(deployment_name, property_name)
-        deployment = find_deployment(deployment_name)
+      def get_property(deployment, property_name)
         filters = {:deployment_id => deployment.id, :name => property_name}
         property = Models::DeploymentProperty.find(filters)
         if property.nil?
           raise PropertyNotFound,
                 "Property `#{property_name}' not found " +
-                "for deployment `#{deployment_name}'"
+                "for deployment `#{deployment.name}'"
         end
         property
       end
 
-      def get_properties(deployment_name)
-        filters = {:deployment_id => find_deployment(deployment_name).id}
+      def get_properties(deployment)
+        filters = {:deployment_id => deployment.id}
         Models::DeploymentProperty.filter(filters).all
       end
 
@@ -60,10 +55,6 @@ module Bosh::Director
         raise PropertyInvalid,
               "Property is invalid: #{errors.full_messages.sort.join(", ")}"
       end
-
-      def find_deployment(name)
-        @deployment_manager.find_by_name(name)
-      end
     end
   end
-end
+end

data/lib/bosh/director/api/resurrector_manager.rb

@@ -1,13 +1,13 @@
 module Bosh::Director
   module Api
     class ResurrectorManager
-      def set_pause_for_instance(deployment_name, job_name, index_or_id, desired_state)
+      def set_pause_for_instance(deployment, job_name, index_or_id, desired_state)
 
         # This is for backwards compatibility and can be removed when we move to referencing job by instance id only.
         if index_or_id.to_s =~ /^\d+$/
-          instance = InstanceLookup.new.by_attributes(deployment_name, job_name, index_or_id)
+          instance = InstanceLookup.new.by_attributes(deployment, job_name, index_or_id)
         else
-          instance = InstanceLookup.new.by_uuid(deployment_name, job_name, index_or_id)
+          instance = InstanceLookup.new.by_uuid(deployment, job_name, index_or_id)
         end
 
         instance.resurrection_paused = desired_state
@@ -19,4 +19,4 @@ module Bosh::Director
       end
     end
   end
-end
+end

data/lib/bosh/director/api/route_configuration.rb

@@ -14,6 +14,7 @@ module Bosh
           controllers['/cleanup'] = Bosh::Director::Api::Controllers::CleanupController.new(@config)
           controllers['/restore'] = Bosh::Director::Api::Controllers::RestoreController.new(@config)
           controllers['/cloud_configs'] = Bosh::Director::Api::Controllers::CloudConfigsController.new(@config)
+          controllers['/runtime_configs'] = Bosh::Director::Api::Controllers::RuntimeConfigsController.new(@config)
           controllers['/deployments'] = Bosh::Director::Api::Controllers::DeploymentsController.new(@config)
           controllers['/disks'] = Bosh::Director::Api::Controllers::DisksController.new(@config)
           controllers['/info'] = Bosh::Director::Api::Controllers::InfoController.new(@config)

data/lib/bosh/director/api/runtime_config_manager.rb

@@ -0,0 +1,35 @@
+module Bosh
+  module Director
+    module Api
+      class RuntimeConfigManager
+        def update(runtime_config_yaml)
+          runtime_config = Bosh::Director::Models::RuntimeConfig.new(
+            properties: runtime_config_yaml
+          )
+          validate_manifest!(runtime_config)
+          runtime_config.save
+        end
+
+        def list(limit)
+          Bosh::Director::Models::RuntimeConfig.order(Sequel.desc(:id)).limit(limit).to_a
+        end
+
+        def latest
+          list(1).first
+        end
+
+        def find_by_id(id)
+          Bosh::Director::Models::RuntimeConfig.find(id: id)
+        end
+
+        private
+
+        def validate_manifest!(runtime_config)
+          runtime_manifest = runtime_config.manifest
+
+          Bosh::Director::DeploymentPlan::RuntimeManifestParser.new(@logger).parse(runtime_manifest)
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/api/snapshot_manager.rb

@@ -2,7 +2,7 @@ module Bosh::Director
   module Api
     class SnapshotManager
       def create_deployment_snapshot_task(username, deployment, options = {})
-        JobQueue.new.enqueue(username, Jobs::SnapshotDeployment, 'snapshot deployment', [deployment.name, options])
+        JobQueue.new.enqueue(username, Jobs::SnapshotDeployment, 'snapshot deployment', [deployment.name, options], deployment.name)
       end
 
       def create_snapshot_task(username, instance, options)
@@ -10,7 +10,7 @@ module Bosh::Director
       end
 
       def delete_deployment_snapshots_task(username, deployment)
-        JobQueue.new.enqueue(username, Jobs::DeleteDeploymentSnapshots, 'delete deployment snapshots', [deployment.name])
+        JobQueue.new.enqueue(username, Jobs::DeleteDeploymentSnapshots, 'delete deployment snapshots', [deployment.name], deployment.name)
       end
 
       def delete_snapshots_task(username, snapshot_cids)

data/lib/bosh/director/api/task_helper.rb

@@ -3,11 +3,12 @@ require 'bosh/director/api/task_remover'
 module Bosh::Director
   module Api
     class TaskHelper
-      def create_task(username, type, description)
+      def create_task(username, type, description, deployment_name)
         task = Models::Task.create(:username => username,
                                    :type => type,
                                    :description => description,
                                    :state => :queued,
+                                   :deployment_name => deployment_name,
                                    :timestamp => Time.now,
                                    :checkpoint_time => Time.now)
         log_dir = File.join(Config.base_dir, 'tasks', task.id.to_s)

data/lib/bosh/director/api/task_manager.rb

@@ -25,7 +25,8 @@ module Bosh::Director
           "description" => task.description,
           "timestamp" => task.timestamp.to_i,
           "result" => task.result,
-          "user" => task.username || "admin"
+          "user" => task.username || "admin",
+          "deployment" => task.deployment_name
         }
       end
 
@@ -33,13 +34,6 @@ module Bosh::Director
         # Backward compatibility
         return task.output unless File.directory?(task.output)
 
-        # Backward compatbility from renaming `soap` log to `cpi` log.
-        # Old tasks might have been written to the file `soap` and we should
-        # still return them if log_type = cpi. Same goes for new task logs
-        # written to `cpi` but an old CLI has requested log_type = soap.
-        if %w(soap cpi).include?(log_type)
-          log_type = File.file?(File.join(task.output, "soap")) ? "soap" : "cpi"
-        end
         file = File.join(task.output, log_type)
         file_gz = [file, 'gz'].join('.')
 

data/lib/bosh/director/api/uaa_identity_provider.rb

@@ -9,7 +9,6 @@ module Bosh
         def initialize(options)
           @url = options.fetch('url')
           Config.logger.debug "Initializing UAA Identity provider with url #{@url}"
-          @permission_authorizer = Bosh::Director::PermissionAuthorizer.new
           @token_coder = CF::UAA::TokenCoder.new(skey: options.fetch('symmetric_key', nil), pkey: options.fetch('public_key', nil), scope: [])
         end
 
@@ -44,21 +43,6 @@ module Bosh
         rescue CF::UAA::DecodeError, CF::UAA::AuthError => e
           raise AuthenticationError, e.message
         end
-
-        def valid_access?(user, requested_access)
-          if user.scopes
-            required_scopes = required_scopes(requested_access)
-            return @permission_authorizer.has_team_admin_scope?(user.scopes) ||
-                    @permission_authorizer.has_admin_scope?(user.scopes) ||
-                    @permission_authorizer.contains_requested_scope?(required_scopes, user.scopes)
-          end
-
-          false
-        end
-
-        def required_scopes(requested_access)
-          @permission_authorizer.permissions[requested_access]
-        end
       end
 
       class UaaUser

data/lib/bosh/director/blob_util.rb

@@ -76,9 +76,10 @@ module Bosh::Director
 
       Models::CompiledPackage.create do |p|
         p.package = package
-        p.stemcell = stemcell
+        p.stemcell_os = stemcell.operating_system
+        p.stemcell_version = stemcell.version
         p.sha1 = compiled_package_sha1
-        p.build = Models::CompiledPackage.generate_build_number(package, stemcell)
+        p.build = Models::CompiledPackage.generate_build_number(package, stemcell.operating_system, stemcell.version)
         p.blobstore_id = blobstore_id
         p.dependency_key = dependency_key
       end

data/lib/bosh/director/cloudcheck_helper.rb

@@ -67,7 +67,8 @@ module Bosh::Director
       dns_manager = DnsManagerProvider.create
       dns_names_to_ip = {}
 
-      instance_plan_to_create.existing_instance.spec['networks'].each do |network_name, network|
+      apply_spec = instance_plan_to_create.existing_instance.spec
+      apply_spec['networks'].each do |network_name, network|
         index_dns_name = dns_manager.dns_record_name(instance_model.index, instance_model.job, network_name, instance_model.deployment.name)
         dns_names_to_ip[index_dns_name] = network['ip']
         id_dns_name = dns_manager.dns_record_name(instance_model.uuid, instance_model.job, network_name, instance_model.deployment.name)
@@ -78,8 +79,20 @@ module Bosh::Director
       dns_manager.update_dns_record_for_instance(instance_model, dns_names_to_ip)
       dns_manager.flush_dns_cache
 
-      cleaner = RenderedJobTemplatesCleaner.new(instance_model, App.instance.blobstores.blobstore, @logger)
-      InstanceUpdater::StateApplier.new(instance_plan_to_create, agent_client(instance_model.credentials, instance_model.agent_id), cleaner, @logger).apply
+      InstanceUpdater::InstanceState.with_instance_update(instance_model) do
+        cleaner = RenderedJobTemplatesCleaner.new(instance_model, App.instance.blobstores.blobstore, @logger)
+
+        # for backwards compatibility with instances that don't have update config
+        update_config = apply_spec['update'].nil? ? nil : DeploymentPlan::UpdateConfig.new(apply_spec['update'])
+
+        InstanceUpdater::StateApplier.new(
+          instance_plan_to_create,
+          agent_client(instance_model.credentials, instance_model.agent_id),
+          cleaner,
+          @logger,
+          {}
+        ).apply(update_config)
+      end
     end
 
     private
@@ -96,6 +109,7 @@ module Bosh::Director
         instance_model.index,
         instance_model.state,
         vm_type,
+        nil,
         stemcell,
         env,
         false,

data/lib/bosh/director/compile_task.rb

@@ -134,35 +134,64 @@ module Bosh::Director
     # @param [CompileTask] task
     # @return [Models::CompiledPackage]
     def find_compiled_package(logger, event_log)
-      # Check if this package is already compiled
-      compiled_package = Models::CompiledPackage[
-        :package_id => package.id,
-        :stemcell_id => stemcell.id,
-        :dependency_key => dependency_key
-      ]
-      if compiled_package
-        logger.info("Found compiled version of package `#{package.desc}' " +
-                       "for stemcell `#{stemcell.desc}'")
+      # if `package` has source associated with it (blobstore_id and sha1)
+      #   then we need an exact match in find_compiled_package
+
+      package_already_compiled = !@package.blobstore_id.nil?
+
+      if package_already_compiled
+        compiled_package = Models::CompiledPackage[
+          :package_id => package.id,
+          :stemcell_os => stemcell.os,
+          :stemcell_version => stemcell.version,
+          :dependency_key => dependency_key
+        ]
       else
-        if Config.use_compiled_package_cache?
-          if BlobUtil.exists_in_global_cache?(package, cache_key)
-            event_log.track("Downloading '#{package.desc}' from global cache") do
-              # has side effect of putting CompiledPackage model in db
-              compiled_package = BlobUtil.fetch_from_global_cache(package, stemcell.model, cache_key, dependency_key)
-            end
-          end
-        end
+        compiled_package = find_best_compiled_package_by_version
+      end
 
-        if compiled_package
-          logger.info("Package `Found compiled version of package `#{package.desc}'" +
-                         "for stemcell `#{stemcell.desc}' in global cache")
-        else
-          logger.info("Package `#{package.desc}' " +
-                         "needs to be compiled on `#{stemcell.desc}'")
-        end
+      if compiled_package
+        logger.info("Found compiled version of package '#{package.desc}' for stemcell '#{stemcell.desc}'")
+      else
+        compiled_package = fetch_from_global_cache(logger, event_log)
       end
 
+      logger.info("Package '#{package.desc}' needs to be compiled on '#{stemcell.desc}'") if compiled_package.nil?
+
+      compiled_package
+    end
+
+    private
+
+    def find_best_compiled_package_by_version
+      compiled_package_list = Models::CompiledPackage.where(
+        :package_id => package.id,
+        :stemcell_os => stemcell.os,
+        :dependency_key => dependency_key
+      ).all
+
+      compiled_package = compiled_package_list.select do |compiled_package_model|
+        compiled_package_model.stemcell_version == stemcell.version
+      end.first
+
+      unless compiled_package
+        compiled_package = compiled_package_list.select do |compiled_package_model|
+          Bosh::Common::Version::StemcellVersion.match(compiled_package_model.stemcell_version, stemcell.version)
+        end.sort_by do |compiled_package_model|
+          SemiSemantic::Version.parse(compiled_package_model.stemcell_version).release.components[1] || 0
+        end.last
+      end
       compiled_package
     end
+
+    def fetch_from_global_cache(logger, event_log)
+      if Config.use_compiled_package_cache? && BlobUtil.exists_in_global_cache?(package, cache_key)
+        event_log.track("Downloading '#{package.desc}' from global cache") do
+          # has side effect of putting CompiledPackage model in db
+          logger.info("Found compiled version of package '#{package.desc}' for stemcell '#{stemcell.desc}' in global cache")
+          return BlobUtil.fetch_from_global_cache(package, stemcell.model, cache_key, dependency_key)
+        end
+      end
+    end
   end
 end