bosh-director 1.3215.4.0 → 1.3232.0

data/lib/bosh/director/jobs/release/release_job.rb

@@ -3,13 +3,12 @@ module Bosh::Director
 
     attr_accessor :packages
 
-    def initialize(job_meta, release_model, release_dir, packages, logger)
+    def initialize(job_meta, release_model, release_dir, logger)
       @name = job_meta['name']
       @version = job_meta['version']
       @sha1 = job_meta['sha1']
       @fingerprint = job_meta['fingerprint']
 
-      @packages = packages
       @release_model = release_model
       @release_dir = release_dir
       @logger = logger
@@ -59,12 +58,12 @@ module Bosh::Director
     def unpack
       FileUtils.mkdir_p(job_dir)
 
-      desc = "job `#{@name}/#{@version}'"
+      desc = "job '#{@name}/#{@version}'"
       result = Bosh::Exec.sh("tar -C #{job_dir} -xzf #{job_tgz} 2>&1", :on_error => :return)
       if result.failed?
         @logger.error("Extracting #{desc} archive failed in dir #{job_dir}, " +
-            "tar returned #{result.exit_status}, " +
-            "output: #{result.output}")
+          "tar returned #{result.exit_status}, " +
+          "output: #{result.output}")
         raise JobInvalidArchive, "Extracting #{desc} archive failed. Check task debug log for details."
       end
     end
@@ -81,7 +80,7 @@ module Bosh::Director
       manifest_file = File.join(job_dir, 'job.MF')
       unless File.file?(manifest_file)
         raise JobMissingManifest,
-          "Missing job manifest for `#{@name}'"
+          "Missing job manifest for '#{@name}'"
       end
 
       Psych.load_file(manifest_file)
@@ -93,7 +92,7 @@ module Bosh::Director
           path = File.join(job_dir, 'templates', relative_path)
           unless File.file?(path)
             raise JobMissingTemplateFile,
-              "Missing template file `#{relative_path}' for job `#{@name}'"
+              "Missing template file '#{relative_path}' for job '#{@name}'"
           end
         end
       end
@@ -104,36 +103,22 @@ module Bosh::Director
       aux_monit_files = Dir.glob(File.join(job_dir, '*.monit'))
 
       unless File.exists?(main_monit_file) || aux_monit_files.size > 0
-        raise JobMissingMonit, "Job `#{@name}' is missing monit file"
+        raise JobMissingMonit, "Job '#{@name}' is missing monit file"
       end
     end
 
     def parse_package_names(job_manifest)
-      package_names = []
-      if job_manifest['packages']
-        unless job_manifest['packages'].is_a?(Array)
-          raise JobInvalidPackageSpec,
-            "Job `#{@name}' has invalid package spec format"
-        end
-
-        job_manifest['packages'].each do |package_name|
-          package = @packages[package_name]
-          if package.nil?
-            raise JobMissingPackage,
-              "Job `#{@name}' is referencing " +
-                "a missing package `#{package_name}'"
-          end
-          package_names << package.name
-        end
+      if job_manifest['packages'] && !job_manifest['packages'].is_a?(Array)
+        raise JobInvalidPackageSpec, "Job '#{@name}' has invalid package spec format"
       end
-      package_names
+      job_manifest['packages'] || []
     end
 
     def validate_logs(job_manifest)
       if job_manifest['logs']
         unless job_manifest['logs'].is_a?(Hash)
           raise JobInvalidLogSpec,
-            "Job `#{@name}' has invalid logs spec format"
+            "Job '#{@name}' has invalid logs spec format"
         end
       end
     end
@@ -142,7 +127,7 @@ module Bosh::Director
       if job_manifest['properties']
         unless job_manifest['properties'].is_a?(Hash)
           raise JobInvalidPropertySpec,
-            "Job `#{@name}' has invalid properties spec format"
+            "Job '#{@name}' has invalid properties spec format"
         end
       end
     end
@@ -155,7 +140,7 @@ module Bosh::Director
     def parse_links(links, kind)
       if !links.is_a?(Array)
         raise JobInvalidLinkSpec,
-              "Job '#{@name}' has invalid spec format: '#{kind}' must be an array of hashes with name and type"
+          "Job '#{@name}' has invalid spec format: '#{kind}' must be an array of hashes with name and type"
       end
 
       parsed_links = {}

data/lib/bosh/director/jobs/run_errand.rb

@@ -18,7 +18,7 @@ module Bosh::Director
       @keep_alive = keep_alive
       blobstore = App.instance.blobstores.blobstore
       log_bundles_cleaner = LogBundlesCleaner.new(blobstore, 60 * 60 * 24 * 10, logger) # 10 days
-      @logs_fetcher = LogsFetcher.new(event_log, @instance_manager, log_bundles_cleaner, logger)
+      @logs_fetcher = LogsFetcher.new(@instance_manager, log_bundles_cleaner, logger)
     end
 
     def perform
@@ -29,15 +29,15 @@ module Bosh::Director
         deployment = nil
         job = nil
 
-        event_log.begin_stage('Preparing deployment', 1)
-        event_log.track('Preparing deployment') do
+        event_log_stage = Config.event_log.begin_stage('Preparing deployment', 1)
+        event_log_stage.advance_and_track('Preparing deployment') do
           planner_factory = DeploymentPlan::PlannerFactory.create(logger)
           deployment = planner_factory.create_from_manifest(deployment_manifest, deployment_model.cloud_config, deployment_model.runtime_config, {})
           deployment.bind_models
           job = deployment.job(@errand_name)
 
           if job.nil?
-            raise JobNotFound, "Errand `#{@errand_name}' doesn't exist"
+            raise JobNotFound, "Errand '#{@errand_name}' doesn't exist"
           end
 
           unless job.is_errand?
@@ -58,7 +58,7 @@ module Bosh::Director
 
         deployment.compile_packages
 
-        runner = Errand::Runner.new(job, result_file, @instance_manager, event_log, @logs_fetcher)
+        runner = Errand::Runner.new(job, result_file, @instance_manager, @logs_fetcher)
 
         cancel_blk = lambda {
           begin
@@ -83,7 +83,7 @@ module Bosh::Director
     private
 
     def with_updated_instances(deployment, job, &blk)
-      job_manager = Errand::JobManager.new(deployment, job, Config.cloud, event_log, logger)
+      job_manager = Errand::JobManager.new(deployment, job, Config.cloud, logger)
 
       begin
         update_instances(job_manager)

data/lib/bosh/director/jobs/scheduled_backup.rb

@@ -23,7 +23,7 @@ module Bosh::Director
           @backup_destination.create(f, blobstore_path)
         end
 
-        "Stored `#{blobstore_path}' in backup blobstore"
+        "Stored '#{blobstore_path}' in backup blobstore"
       ensure
         FileUtils.rm_f(@backup_job.backup_file)
       end

data/lib/bosh/director/jobs/scheduled_events_cleanup.rb

@@ -0,0 +1,31 @@
+module Bosh::Director
+  module Jobs
+    class ScheduledEventsCleanup < BaseJob
+      @queue = :normal
+
+      def self.job_type
+        :scheduled_events_cleanup
+      end
+
+      def self.has_work(params = {})
+        max_events = params.first['max_events']
+        Models::Event.count > max_events
+      end
+
+      def self.schedule_message
+        "clean up events"
+      end
+
+      def initialize(params = {})
+        logger.debug("ScheduledEventsCleanup initialized with params: #{params.inspect}")
+        @max_events = params['max_events']
+      end
+
+      def perform
+        logger.info("Started cleanup of events")
+        event_manager.remove_old_events(@max_events)
+        "Old events were deleted"
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/scheduled_orphan_cleanup.rb

@@ -33,7 +33,7 @@ module Bosh::Director
 
         old_orphans = Models::OrphanDisk.where('created_at < ?', time)
         old_orphans_count = old_orphans.count
-        stage = event_log.begin_stage('Deleting orphan disks', old_orphans_count)
+        stage = Config.event_log.begin_stage('Deleting orphan disks', old_orphans_count)
         old_orphans.each do |old_orphan|
           stage.advance_and_track("#{old_orphan.disk_cid}") do
             @disk_manager.delete_orphan_disk(old_orphan)

data/lib/bosh/director/jobs/snapshot_deployment.rb

@@ -31,6 +31,10 @@ module Bosh::Director
       end
 
       def snapshot(instance)
+        if instance.vm_cid.nil?
+          logger.info('No vm attached to this instance, no snapshot; skipping')
+          return
+        end
         logger.info("taking snapshot of: #{instance.job}/#{instance.index} (#{instance.vm_cid})")
         Bosh::Director::Api::SnapshotManager.take_snapshot(instance, @options)
       rescue Bosh::Clouds::CloudError => e
@@ -54,4 +58,3 @@ module Bosh::Director
     end
   end
 end
-

data/lib/bosh/director/jobs/ssh.rb

@@ -29,22 +29,28 @@ module Bosh::Director
         instances = @instance_manager.filter_by(deployment, filter)
 
         ssh_info = instances.map do |instance|
-          agent = @instance_manager.agent_client_for(instance)
-
-          logger.info("ssh #{@command} `#{instance.job}/#{instance.uuid}'")
-          result = agent.ssh(@command, @params)
-          if target.ids_provided?
-            result["id"] = instance.uuid
-          else
-            result["index"] = instance.index
-          end
+          begin
+            agent = @instance_manager.agent_client_for(instance)
 
-          if Config.default_ssh_options
-            result["gateway_host"] = Config.default_ssh_options["gateway_host"]
-            result["gateway_user"] = Config.default_ssh_options["gateway_user"]
-          end
+            logger.info("ssh #{@command} '#{instance.job}/#{instance.uuid}'")
+            result = agent.ssh(@command, @params)
+            if target.ids_provided?
+              result["id"] = instance.uuid
+            else
+              result["index"] = instance.index
+            end
 
-          result
+            if Config.default_ssh_options
+              result["gateway_host"] = Config.default_ssh_options["gateway_host"]
+              result["gateway_user"] = Config.default_ssh_options["gateway_user"]
+            end
+
+            result
+          rescue Exception => e
+            raise e
+          ensure
+            add_event(deployment.name, instance.name, e)
+          end
         end
 
         result_file.write(Yajl::Encoder.encode(ssh_info))
@@ -56,6 +62,22 @@ module Bosh::Director
 
       private
 
+      def add_event(deployment_name, instance_name, error = nil)
+        user =  @params['user_regex'] || @params['user']
+        event_manager.create_event(
+            {
+                user:        username,
+                action:      "#{@command} ssh",
+                object_type: 'instance',
+                object_name: instance_name,
+                task:        task_id,
+                error:       error,
+                deployment:  deployment_name,
+                instance:    instance_name,
+                context:     {user: user}
+            })
+      end
+
       class Target
         attr_reader :job, :indexes, :ids
 

data/lib/bosh/director/jobs/update_deployment.rb

@@ -37,16 +37,17 @@ module Bosh::Director
         end
 
         deployment_manifest = Manifest.load_from_text(manifest_text, cloud_config_model, runtime_config_model)
-        deployment_name = deployment_manifest.to_hash['name']
+        @deployment_name = deployment_manifest.to_hash['name']
+        parent_id = add_event
 
-        with_deployment_lock(deployment_name) do
-          @notifier = DeploymentPlan::Notifier.new(deployment_name, Config.nats_rpc, logger)
+        with_deployment_lock(@deployment_name) do
+          @notifier = DeploymentPlan::Notifier.new(@deployment_name, Config.nats_rpc, logger)
           @notifier.send_start_event
 
           deployment_plan = nil
 
-          event_log.begin_stage('Preparing deployment', 1)
-          event_log.track('Preparing deployment') do
+          event_log_stage = Config.event_log.begin_stage('Preparing deployment', 1)
+          event_log_stage.advance_and_track('Preparing deployment') do
             planner_factory = DeploymentPlan::PlannerFactory.create(logger)
             deployment_plan = planner_factory.create_from_manifest(deployment_manifest, cloud_config_model, runtime_config_model, @options)
             deployment_plan.bind_models
@@ -63,6 +64,7 @@ module Bosh::Director
 
           @notifier.send_end_event
           logger.info('Finished updating deployment')
+          add_event(parent_id)
 
           "/deployments/#{deployment_plan.name}"
         end
@@ -72,6 +74,7 @@ module Bosh::Director
         rescue Exception => e2
           # log the second error
         ensure
+          add_event(parent_id, e)
           raise e
         end
       ensure
@@ -80,6 +83,22 @@ module Bosh::Director
 
       private
 
+      def add_event(parent_id = nil, error = nil)
+        action = deployment_new? ? "create" : "update"
+        event  = event_manager.create_event(
+            {
+                parent_id:   parent_id,
+                user:        username,
+                action:      action,
+                object_type: "deployment",
+                object_name: @deployment_name,
+                deployment:  @deployment_name,
+                task:        task_id,
+                error:       error
+            })
+        event.id
+      end
+
       # Job tasks
 
       def check_for_changes(deployment_plan)
@@ -92,7 +111,6 @@ module Bosh::Director
       def update_step(deployment_plan)
         DeploymentPlan::Steps::UpdateStep.new(
           self,
-          event_log,
           deployment_plan,
           multi_job_updater,
           Config.cloud
@@ -128,6 +146,10 @@ module Bosh::Director
           raise message
         end
       end
+
+      def deployment_new?
+        @deployment_new ||= Models::Deployment.exclude(manifest: nil)[name: @deployment_name].nil?
+      end
     end
   end
 end

data/lib/bosh/director/jobs/update_release.rb

@@ -51,7 +51,7 @@ module Bosh::Director
 
         with_release_lock(@name) { process_release(release_dir) }
 
-        "Created release `#{@name}/#{@version}'"
+        "Created release '#{@name}/#{@version}'"
 
       rescue Exception => e
         raise e
@@ -110,7 +110,7 @@ module Bosh::Director
       def verify_sha1
         release_hash = Digest::SHA1.file(release_path).hexdigest
         if release_hash != sha1
-          raise ReleaseSha1DoesNotMatch, "Release SHA1 `#{release_hash}' does not match the expected SHA1 `#{sha1}'"
+          raise ReleaseSha1DoesNotMatch, "Release SHA1 '#{release_hash}' does not match the expected SHA1 '#{sha1}'"
         end
       end
 
@@ -142,7 +142,7 @@ module Bosh::Director
           @release_version_model.save
         else
           if @release_version_model.commit_hash != @commit_hash || @release_version_model.uncommitted_changes != @uncommitted_changes
-            raise ReleaseVersionCommitHashMismatch, "release `#{@name}/#{@version}' has already been uploaded with commit_hash as `#{@commit_hash}' and uncommitted_changes as `#{@uncommitted_changes}'"
+            raise ReleaseVersionCommitHashMismatch, "release '#{@name}/#{@version}' has already been uploaded with commit_hash as '#{@commit_hash}' and uncommitted_changes as '#{@uncommitted_changes}'"
           end
         end
 
@@ -150,12 +150,11 @@ module Bosh::Director
           resolve_package_dependencies(@manifest[@packages_folder])
         end
 
-        @packages = {}
         process_packages(release_dir)
         process_jobs(release_dir)
 
-        event_log.begin_stage(@compiled_release ? "Compiled Release has been created" : "Release has been created", 1)
-        event_log.track("#{@name}/#{@version}") {}
+        event_log_stage = Config.event_log.begin_stage(@compiled_release ? "Compiled Release has been created" : "Release has been created", 1)
+        event_log_stage.advance_and_track("#{@name}/#{@version}") {}
       end
 
       # Normalizes release manifest, so all names, versions, and checksums are Strings.
@@ -187,7 +186,7 @@ module Bosh::Director
           name = package["name"]
           dependencies = package["dependencies"]
           all_dependencies = result[:connected_vertices][name]
-          logger.info("Resolved package dependencies for `#{name}': #{dependencies.pretty_inspect} => #{all_dependencies.pretty_inspect}")
+          logger.info("Resolved package dependencies for '#{name}': #{dependencies.pretty_inspect} => #{all_dependencies.pretty_inspect}")
         end
       end
 
@@ -202,13 +201,12 @@ module Bosh::Director
         new_packages = []
         existing_packages = []
         registered_packages = []
-        packages_existing_from_other_releases = []
 
         @manifest[@packages_folder].each do |package_meta|
           # Checking whether we might have the same bits somewhere (in any release, not just the one being uploaded)
           @release_version_model.packages.select { |pv| pv.name == package_meta['name'] }.each do |package|
             if package.fingerprint != package_meta['fingerprint']
-              raise ReleaseInvalidPackage, "package `#{package_meta['name']}' had different fingerprint in previously uploaded release `#{@name}/#{@version}'"
+              raise ReleaseInvalidPackage, "package '#{package_meta['name']}' had different fingerprint in previously uploaded release '#{@name}/#{@version}'"
             end
           end
 
@@ -261,7 +259,6 @@ module Bosh::Director
               end
             end
             new_packages << package_meta
-            packages_existing_from_other_releases << package_meta
           end
         end
 
@@ -277,7 +274,7 @@ module Bosh::Director
             }
           end
           all_package_refs = Array(created_package_refs) | Array(existing_package_refs) | registered_package_refs
-          create_compiled_packages(all_package_refs, release_dir, packages_existing_from_other_releases)
+          create_compiled_packages(all_package_refs, release_dir)
         else
           backfill_source_for_packages(registered_packages, release_dir)
         end
@@ -291,10 +288,9 @@ module Bosh::Director
         single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do
           packages.each do |package, package_meta|
             package_desc = "#{package.name}/#{package.version}"
-            logger.info("Adding source for package `#{package_desc}'")
+            logger.info("Adding source for package '#{package_desc}'")
             had_effect |= save_package_source_blob(package, package_meta, release_dir)
             package.save
-            @packages[package.name] = package
           end
         end
 
@@ -314,7 +310,7 @@ module Bosh::Director
         single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do
           packages.each do |package, package_meta|
             package_desc = "#{package.name}/#{package.version}"
-            logger.info("Using existing package `#{package_desc}'")
+            logger.info("Using existing package '#{package_desc}'")
             register_package(package)
 
             if compiled_release
@@ -345,13 +341,13 @@ module Bosh::Director
 
         package_refs = []
 
-        event_log.begin_stage("Creating new packages", package_metas.size)
+        event_log_stage = Config.event_log.begin_stage("Creating new packages", package_metas.size)
 
         package_metas.each do |package_meta|
           package_desc = "#{package_meta["name"]}/#{package_meta["version"]}"
           package = nil
-          event_log.track(package_desc) do
-            logger.info("Creating new package `#{package_desc}'")
+          event_log_stage.advance_and_track(package_desc) do
+            logger.info("Creating new package '#{package_desc}'")
             package = create_package(package_meta, release_dir)
             register_package(package)
           end
@@ -364,14 +360,14 @@ module Bosh::Director
           end
         end
 
-        return package_refs
+        package_refs
       end
 
       # @return [boolean] true if at least one job was created; false if the call had no effect.
-      def create_compiled_packages(all_compiled_packages, release_dir, packages_existing_from_other_releases)
+      def create_compiled_packages(all_compiled_packages, release_dir)
         return false if all_compiled_packages.nil?
 
-        event_log.begin_stage('Creating new compiled packages', all_compiled_packages.size)
+        event_log_stage = Config.event_log.begin_stage('Creating new compiled packages', all_compiled_packages.size)
         had_effect = false
 
         all_compiled_packages.each do |compiled_package_spec|
@@ -386,8 +382,8 @@ module Bosh::Director
 
           if existing_compiled_packages.empty?
             package_desc = "#{package.name}/#{package.version} for #{stemcell_os}/#{stemcell_version}"
-            event_log.track(package_desc) do
-              other_compiled_packages = compiled_packages_matching(package, packages_existing_from_other_releases, stemcell)
+            event_log_stage.advance_and_track(package_desc) do
+              other_compiled_packages = compiled_packages_matching(package, stemcell)
               if @fix
                 other_compiled_packages.each do |other_compiled_package|
                   fix_compiled_package(other_compiled_package, compiled_pkg_tgz)
@@ -398,7 +394,6 @@ module Bosh::Director
             end
           elsif @fix
             existing_compiled_package = existing_compiled_packages.first
-            @packages[existing_compiled_package.name] = existing_compiled_package
             fix_compiled_package(existing_compiled_package, compiled_pkg_tgz)
           end
         end
@@ -406,16 +401,12 @@ module Bosh::Director
         had_effect
       end
 
-      def compiled_packages_matching(package, packages_existing_from_other_releases, stemcell)
+      def compiled_packages_matching(package, stemcell)
         other_compiled_packages = []
         dependency_key = dependency_key(package)
-        packages_existing_from_other_releases.each do |other_package_meta|
-          if other_package_meta["fingerprint"] == package.fingerprint
-            packages = Models::Package.where(fingerprint: other_package_meta["fingerprint"]).all
-            packages.each do |pkg|
-              other_compiled_packages.concat(find_compiled_packages(pkg.id, stemcell[:os], stemcell[:version], dependency_key).all)
-            end
-          end
+        packages = Models::Package.where(fingerprint: package.fingerprint).all
+        packages.each do |pkg|
+          other_compiled_packages.concat(find_compiled_packages(pkg.id, stemcell[:os], stemcell[:version], dependency_key).all)
         end
         other_compiled_packages
       end
@@ -529,7 +520,6 @@ module Bosh::Director
       # @param [Models::Package] package Package model
       # @return [void]
       def register_package(package)
-        @packages[package.name] = package
         @release_version_model.add_package(package)
       end
 
@@ -548,7 +538,7 @@ module Bosh::Director
           # Checking whether we might have the same bits somewhere
           @release_version_model.templates.select { |t| t.name == job_meta["name"] }.each do |tmpl|
             if tmpl.fingerprint != job_meta["fingerprint"]
-              raise ReleaseExistingJobFingerprintMismatch, "job `#{job_meta["name"]}' had different fingerprint in previously uploaded release `#{@name}/#{@version}'"
+              raise ReleaseExistingJobFingerprintMismatch, "job '#{job_meta["name"]}' had different fingerprint in previously uploaded release '#{@name}/#{@version}'"
             end
           end
 
@@ -577,11 +567,11 @@ module Bosh::Director
       def create_jobs(jobs, release_dir)
         return false if jobs.empty?
 
-        event_log.begin_stage("Creating new jobs", jobs.size)
+        event_log_stage = Config.event_log.begin_stage("Creating new jobs", jobs.size)
         jobs.each do |job_meta|
           job_desc = "#{job_meta["name"]}/#{job_meta["version"]}"
-          event_log.track(job_desc) do
-            logger.info("Creating new template `#{job_desc}'")
+          event_log_stage.advance_and_track(job_desc) do
+            logger.info("Creating new template '#{job_desc}'")
             template = create_job(job_meta, release_dir)
             register_template(template)
           end
@@ -591,8 +581,8 @@ module Bosh::Director
       end
 
       def create_job(job_meta, release_dir)
-        release_job = ReleaseJob.new(job_meta, @release_model, release_dir, @packages, logger)
-        logger.info("Creating job template `#{job_meta['name']}/#{job_meta['version']}' " +
+        release_job = ReleaseJob.new(job_meta, @release_model, release_dir, logger)
+        logger.info("Creating job template '#{job_meta['name']}/#{job_meta['version']}' " +
             'from provided bits')
         release_job.update(Models::Template.new())
       end
@@ -607,11 +597,11 @@ module Bosh::Director
             job_desc = "#{template.name}/#{template.version}"
 
             if @fix
-              logger.info("Fixing existing job `#{job_desc}'")
-              release_job = ReleaseJob.new(job_meta, @release_model, release_dir, @packages, logger)
+              logger.info("Fixing existing job '#{job_desc}'")
+              release_job = ReleaseJob.new(job_meta, @release_model, release_dir, logger)
               release_job.update(template)
             else
-              logger.info("Using existing job `#{job_desc}'")
+              logger.info("Using existing job '#{job_desc}'")
             end
 
             register_template(template) unless template.release_versions.include? @release_version_model