bosh-director 1.3215.4.0 → 1.3232.0

data/lib/bosh/director/instance_updater/state_applier.rb

@@ -42,7 +42,7 @@ module Bosh::Director
             process['name'] if process['state'] != 'running'
           end.compact
 
-          error_message = "`#{@instance}' is not running after update."
+          error_message = "'#{@instance}' is not running after update."
           error_message += " Review logs for failed jobs: #{failing_jobs.join(", ")}" if !failing_jobs.empty?
 
           raise AgentJobNotRunning, error_message
@@ -53,7 +53,7 @@ module Bosh::Director
       end
 
       if @instance.state == 'stopped' && current_state['job_state'] == 'running'
-        raise AgentJobNotStopped, "`#{@instance}' is still running despite the stop command"
+        raise AgentJobNotStopped, "'#{@instance}' is still running despite the stop command"
       end
 
       @instance.update_state

data/lib/bosh/director/job_queue.rb

@@ -2,13 +2,15 @@ require 'bosh/director/api/task_helper'
 
 module Bosh::Director
 
-  # Abstracts the resque system.
+  # Abstracts the delayed jobs system.
 
   class JobQueue
     def enqueue(username, job_class, description, params, deployment_name = nil)
       task = Api::TaskHelper.new.create_task(username, job_class.job_type, description, deployment_name)
 
-      Resque.enqueue(job_class, task.id, *params)
+      Delayed::Worker.backend = :sequel
+      db_job = Bosh::Director::Jobs::DBJob.new(job_class, task.id, params)
+      Delayed::Job.enqueue db_job
 
       task
     end

data/lib/bosh/director/job_runner.rb

@@ -8,7 +8,7 @@ module Bosh::Director
     def initialize(job_class, task_id)
       unless job_class.kind_of?(Class) &&
         job_class <= Jobs::BaseJob
-        raise DirectorError, "Invalid director job class `#{job_class}'"
+        raise DirectorError, "Invalid director job class '#{job_class}'"
       end
 
       @task_id = task_id
@@ -60,12 +60,6 @@ module Bosh::Director
       Config.result = TaskResultFile.new(result_log)
       Config.logger = @task_logger
 
-      # use a separate logger with the same appender to avoid multiple file writers
-      redis_task_logger = Logging::Logger.new('DirectorJobRunnerRedis')
-      redis_task_logger.add_appenders(shared_appender)
-      redis_task_logger.level = Config.redis_logger_level
-      Config.redis_logger = redis_task_logger
-
       Config.db.logger = @task_logger
       Config.dns_db.logger = @task_logger if Config.dns_db
 
@@ -93,6 +87,7 @@ module Bosh::Director
 
       @task.state = :processing
       @task.timestamp = Time.now
+      @task.started_at = Time.now
       @task.checkpoint_time = Time.now
       @task.save
 
@@ -113,7 +108,7 @@ module Bosh::Director
 
     # Spawns a thread that periodically updates task checkpoint time.
     # There is no need to kill this thread as job execution lifetime is the
-    # same as Resque worker process lifetime.
+    # same as worker process lifetime.
     # @return [Thread] Checkpoint thread
     def run_checkpointing
       # task check pointer is scoped to separate class to avoid

data/lib/bosh/director/jobs/backup.rb

@@ -19,7 +19,7 @@ module Bosh::Director
 
       def perform
         Dir.mktmpdir do |tmp_output_dir|
-          event_log.begin_stage('Backing up director', 4)
+          begin_stage('Backing up director', 4)
           backup_database("#{tmp_output_dir}/director_db.sql")
           @tar_gzipper.compress(tmp_output_dir, 'director_db.sql', @backup_file)
           "Backup created at #{@backup_file}"

data/lib/bosh/director/jobs/base_job.rb

@@ -20,10 +20,6 @@ module Bosh::Director
         @logger ||= Config.logger
       end
 
-      def event_log
-        @event_log ||= Config.event_log
-      end
-
       def result_file
         @result_file ||= Config.result
       end
@@ -46,12 +42,12 @@ module Bosh::Director
       end
 
       def begin_stage(stage_name, n_steps)
-        event_log.begin_stage(stage_name, n_steps)
+        @event_log_stage = Config.event_log.begin_stage(stage_name, n_steps)
         logger.info(stage_name)
       end
 
       def track_and_log(task, log = true)
-        event_log.track(task) do |ticker|
+        @event_log_stage.advance_and_track(task) do |ticker|
           logger.info(task) if log
           yield ticker if block_given?
         end
@@ -62,6 +58,14 @@ module Bosh::Director
         track_and_log(stage_name, false) { yield }
       end
 
+      def username
+        @user ||= task_manager.find_task(task_id).username
+      end
+
+      def event_manager
+        @event_manager ||= Api::EventManager.new(Config.record_events)
+      end
+
       private
 
       def task_manager

data/lib/bosh/director/jobs/cleanup_artifacts.rb

@@ -23,14 +23,14 @@ module Bosh::Director
         cloud = Config.cloud
         blob_deleter = Jobs::Helpers::BlobDeleter.new(blobstore, logger)
         compiled_package_deleter = Jobs::Helpers::CompiledPackageDeleter.new(blob_deleter, logger)
-        @stemcell_deleter = Jobs::Helpers::StemcellDeleter.new(cloud, compiled_package_deleter, logger, event_log)
+        @stemcell_deleter = Jobs::Helpers::StemcellDeleter.new(cloud, compiled_package_deleter, logger)
         @releases_to_delete_picker = Jobs::Helpers::ReleasesToDeletePicker.new(release_manager)
         @stemcells_to_delete_picker = Jobs::Helpers::StemcellsToDeletePicker.new(@stemcell_manager)
         package_deleter = Helpers::PackageDeleter.new(compiled_package_deleter, blob_deleter, logger)
         template_deleter = Helpers::TemplateDeleter.new(blob_deleter, logger)
-        release_deleter = Helpers::ReleaseDeleter.new(package_deleter, template_deleter, event_log, logger)
+        release_deleter = Helpers::ReleaseDeleter.new(package_deleter, template_deleter, Config.event_log, logger)
         release_version_deleter =
-          Helpers::ReleaseVersionDeleter.new(release_deleter, package_deleter, template_deleter, logger, event_log)
+          Helpers::ReleaseVersionDeleter.new(release_deleter, package_deleter, template_deleter, logger, Config.event_log)
         @name_version_release_deleter =
           Helpers::NameVersionReleaseDeleter.new(release_deleter, release_manager, release_version_deleter, logger)
       end
@@ -43,7 +43,7 @@ module Bosh::Director
         end
 
         unused_release_name_and_version = @releases_to_delete_picker.pick(releases_to_keep)
-        release_stage = event_log.begin_stage('Deleting releases', unused_release_name_and_version.count)
+        release_stage = Config.event_log.begin_stage('Deleting releases', unused_release_name_and_version.count)
         ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
           unused_release_name_and_version.each do |name_and_version|
             pool.process do
@@ -59,7 +59,7 @@ module Bosh::Director
         end
 
         stemcells_to_delete = @stemcells_to_delete_picker.pick(stemcells_to_keep)
-        stemcell_stage = event_log.begin_stage('Deleting stemcells', stemcells_to_delete.count)
+        stemcell_stage = Config.event_log.begin_stage('Deleting stemcells', stemcells_to_delete.count)
         ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
           stemcells_to_delete.each do |stemcell|
             pool.process do
@@ -74,7 +74,7 @@ module Bosh::Director
         orphan_disks = []
         if @config['remove_all']
           orphan_disks = Models::OrphanDisk.all
-          orphan_disk_stage = event_log.begin_stage('Deleting orphaned disks', orphan_disks.count)
+          orphan_disk_stage = Config.event_log.begin_stage('Deleting orphaned disks', orphan_disks.count)
           ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
             orphan_disks.each do |orphan_disk|
               pool.process do

data/lib/bosh/director/jobs/db_job.rb

@@ -0,0 +1,87 @@
+module Bosh::Director
+  module Jobs
+    class DBJob
+      attr_reader :job_class, :task_id
+
+      def initialize(job_class, task_id, args)
+        unless job_class.kind_of?(Class) &&
+            job_class <= Jobs::BaseJob
+          raise DirectorError, "Invalid director job class `#{job_class}'"
+        end
+        raise DirectorError, "Invalid director job class `#{job_class}'. It should have `perform' method."  unless job_class.instance_methods(false).include?(:perform)
+        @job_class = job_class
+        @task_id = task_id
+        @args = args
+        raise DirectorError, "Invalid director job class `#{job_class}'. It should specify queue value." unless queue_name
+      end
+
+      def perform
+        if Models::Task.first(id: @task_id, state: 'queued').nil?
+          raise DirectorError, "Cannot perform job for task #{@task_id} (not in 'queued' state)"
+        end
+
+        process_status = ForkedProcess.run do
+          @job_class.perform(@task_id, *decode(encode(@args)))
+        end
+
+        if process_status.signaled?
+          puts "Task #{@task_id} was terminated, marking as failed"
+          fail_task
+        end
+      end
+
+      def queue_name
+        @job_class.instance_variable_get(:@queue) ||
+            (@job_class.respond_to?(:queue) and @job_class.queue)
+      end
+
+      private
+
+      def fail_task
+        Models::Task.first(id: @task_id).update(state: 'error')
+      end
+
+      def encode(object)
+        if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+          MultiJson.dump object
+        else
+          MultiJson.encode object
+        end
+      end
+
+      # Given a string, returns a Ruby object.
+      def decode(object)
+        return unless object
+
+        begin
+          if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+            MultiJson.load object
+          else
+            MultiJson.decode object
+          end
+        rescue ::MultiJson::DecodeError => e
+          raise DecodeException, e.message, e.backtrace
+        end
+      end
+    end
+  end
+
+  class ForkedProcess
+    def self.run
+      pid = Process.fork do
+        EM.run do
+          EM.defer do
+            begin
+              yield
+            ensure
+              EM.stop
+            end
+          end
+        end
+      end
+      Process.waitpid(pid)
+
+      $?
+    end
+  end
+end

data/lib/bosh/director/jobs/delete_deployment.rb

@@ -19,7 +19,7 @@ module Bosh::Director
 
       def perform
         logger.info("Deleting: #{@deployment_name}")
-
+        parent_id = add_event
         with_deployment_lock(@deployment_name) do
           deployment_model = @deployment_manager.find_by_name(@deployment_name)
 
@@ -28,13 +28,34 @@ module Bosh::Director
           dns_manager = DnsManagerProvider.create
           disk_manager = DiskManager.new(@cloud, logger)
           instance_deleter = InstanceDeleter.new(ip_provider, dns_manager, disk_manager, force: @force)
-          deployment_deleter = DeploymentDeleter.new(event_log, logger, dns_manager, Config.max_threads)
+          deployment_deleter = DeploymentDeleter.new(Config.event_log, logger, dns_manager, Config.max_threads)
 
           vm_deleter = Bosh::Director::VmDeleter.new(@cloud, logger, force: @force)
           deployment_deleter.delete(deployment_model, instance_deleter, vm_deleter)
+          add_event(parent_id)
 
           "/deployments/#{@deployment_name}"
         end
+      rescue Exception => e
+        add_event(parent_id, e)
+        raise e
+      end
+
+      private
+
+      def add_event(parent_id = nil, error = nil)
+        event  = event_manager.create_event(
+            {
+                parent_id:   parent_id,
+                user:        username,
+                action:      "delete",
+                object_type: "deployment",
+                object_name: @deployment_name,
+                deployment:  @deployment_name,
+                task:        task_id,
+                error:       error
+            })
+        event.id
       end
     end
   end

data/lib/bosh/director/jobs/delete_deployment_snapshots.rb

@@ -27,7 +27,7 @@ module Bosh::Director
           end
         end
 
-        "snapshots of deployment `#{deployment.name}' deleted"
+        "snapshots of deployment '#{deployment.name}' deleted"
       end
     end
   end

data/lib/bosh/director/jobs/delete_orphan_disks.rb

@@ -25,12 +25,12 @@ module Bosh::Director
       end
 
       def perform
-        event_log.begin_stage('Deleting orphaned disks', @orphan_disk_cids.count)
+        event_log_stage = Config.event_log.begin_stage('Deleting orphaned disks', @orphan_disk_cids.count)
 
         ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
           @orphan_disk_cids.each do |orphan_disk_cid|
             pool.process do
-              event_log.track("Deleting orphaned disk #{orphan_disk_cid}") do
+              event_log_stage.advance_and_track("Deleting orphaned disk #{orphan_disk_cid}") do
                 @disk_manager.delete_orphan_disk_by_disk_cid(orphan_disk_cid)
               end
             end

data/lib/bosh/director/jobs/delete_release.rb

@@ -19,9 +19,9 @@ module Bosh::Director
         compiled_package_deleter = Helpers::CompiledPackageDeleter.new(blob_deleter, logger)
         package_deleter = Helpers::PackageDeleter.new(compiled_package_deleter, blob_deleter, logger)
         template_deleter = Helpers::TemplateDeleter.new(blob_deleter, logger)
-        release_deleter = Helpers::ReleaseDeleter.new(package_deleter, template_deleter, event_log, logger)
+        release_deleter = Helpers::ReleaseDeleter.new(package_deleter, template_deleter, Config.event_log, logger)
         release_version_deleter =
-          Helpers::ReleaseVersionDeleter.new(release_deleter, package_deleter, template_deleter, logger, event_log)
+          Helpers::ReleaseVersionDeleter.new(release_deleter, package_deleter, template_deleter, logger, Config.event_log)
         release_manager = Api::ReleaseManager.new
         @name_version_release_deleter = Helpers::NameVersionReleaseDeleter.new(release_deleter, release_manager, release_version_deleter, logger)
       end

data/lib/bosh/director/jobs/delete_stemcell.rb

@@ -17,7 +17,7 @@ module Bosh::Director
         blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
         blob_deleter = Helpers::BlobDeleter.new(blobstore, logger)
         compiled_package_deleter = Helpers::CompiledPackageDeleter.new(blob_deleter, logger)
-        @stemcell_deleter = Helpers::StemcellDeleter.new(@cloud, compiled_package_deleter, logger, event_log)
+        @stemcell_deleter = Helpers::StemcellDeleter.new(@cloud, compiled_package_deleter, logger)
       end
 
       def perform

data/lib/bosh/director/jobs/export_release.rb

@@ -1,4 +1,5 @@
 require 'securerandom'
+require 'common/release/release_directory'
 require 'common/version/release_version'
 require 'bosh/director/compiled_release_downloader'
 require 'bosh/director/compiled_release_manifest'
@@ -40,7 +41,7 @@ module Bosh::Director
         release_version_model = release_manager.find_version(release, @release_version)
 
         unless deployment_manifest_has_release?(targeted_deployment.manifest)
-          raise ReleaseNotMatchingManifest, "Release version `#{@release_name}/#{@release_version}' not found in deployment `#{@deployment_name}' manifest"
+          raise ReleaseNotMatchingManifest, "Release version '#{@release_name}/#{@release_version}' not found in deployment '#{@deployment_name}' manifest"
         end
 
         planner_factory = DeploymentPlan::PlannerFactory.create(logger)
@@ -98,7 +99,8 @@ module Bosh::Director
         output_path = File.join(download_dir, "compiled_release_#{Time.now.to_f}.tar.gz")
         archiver = Core::TarGzipper.new
 
-        archiver.compress(download_dir, ['compiled_packages', 'jobs', 'release.MF'], output_path)
+        release_directory = Bosh::Common::Release::ReleaseDirectory.new(download_dir)
+        archiver.compress(download_dir, release_directory.ordered_release_files, output_path)
         tarball_file = File.open(output_path, 'r')
 
         oid = blobstore_client.create(tarball_file)

data/lib/bosh/director/jobs/fetch_logs.rb

@@ -17,7 +17,7 @@ module Bosh::Director
 
         blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
         log_bundles_cleaner = LogBundlesCleaner.new(blobstore, 60 * 60 * 24 * 10, logger) # 10 days
-        @logs_fetcher = LogsFetcher.new(event_log, @instance_manager, log_bundles_cleaner, logger)
+        @logs_fetcher = LogsFetcher.new(@instance_manager, log_bundles_cleaner, logger)
       end
 
       def perform

data/lib/bosh/director/jobs/helpers/blob_deleter.rb

@@ -12,7 +12,7 @@ module Bosh::Director::Jobs
           @blobstore.delete(blobstore_id)
           return true
         rescue Exception => e
-          @logger.warn("Could not delete blob with id `#{blobstore_id}`from blobstore: #{e}\n " + e.backtrace.join("\n"))
+          @logger.warn("Could not delete blob with id '#{blobstore_id}' from blobstore: #{e}\n " + e.backtrace.join("\n"))
           errors << e
         end
 

data/lib/bosh/director/jobs/helpers/name_version_release_deleter.rb

@@ -31,16 +31,16 @@ module Bosh::Director::Jobs
         unless deployments.empty?
           names = deployments.map { |d| d.name }.join(', ')
           raise Bosh::Director::ReleaseInUse,
-            "Release `#{release.name}' is still in use by: #{names}"
+            "Release '#{release.name}' is still in use by: #{names}"
         end
         @release_deleter.delete(release, force)
       end
 
       def delete_release_version(release, version, force)
-        @logger.info("Looking up release version `#{release.name}/#{version}'")
+        @logger.info("Looking up release version '#{release.name}/#{version}'")
         release_version = @release_manager.find_version(release, version)
         # found version may be different than the requested version, due to version formatting
-        @logger.info("Found release version: `#{release.name}/#{release_version.version}'")
+        @logger.info("Found release version: '#{release.name}/#{release_version.version}'")
         @release_version_deleter.delete(release_version, release, force)
       end
     end

data/lib/bosh/director/jobs/helpers/release_version_deleter.rb

@@ -19,7 +19,7 @@ module Bosh::Director::Jobs
         unless deployments.empty?
           names = deployments.map { |d| d.name }.join(', ')
           raise Bosh::Director::ReleaseVersionInUse,
-            "ReleaseVersion `#{release.name}/#{release_version.version}' is still in use by: #{names}"
+            "ReleaseVersion '#{release.name}/#{release_version.version}' is still in use by: #{names}"
         end
 
         delete_release_version(release_version, force)

data/lib/bosh/director/jobs/helpers/stemcell_deleter.rb

@@ -3,11 +3,10 @@ module Bosh::Director::Jobs
     class StemcellDeleter
       include Bosh::Director::LockHelper
 
-      def initialize(cloud, compiled_package_deleter, logger, event_log)
+      def initialize(cloud, compiled_package_deleter, logger)
         @cloud = cloud
         @compiled_package_deleter = compiled_package_deleter
         @logger = logger
-        @event_log = event_log
       end
 
       def delete(stemcell, options = {})
@@ -17,7 +16,7 @@ module Bosh::Director::Jobs
           unless deployments.empty?
             names = deployments.map { |d| d.name }.join(', ')
             raise Bosh::Director::StemcellInUse,
-              "Stemcell `#{stemcell.name}/#{stemcell.version}' is still in use by: #{names}"
+              "Stemcell '#{stemcell.name}/#{stemcell.version}' is still in use by: #{names}"
           end
 
           begin
@@ -31,15 +30,6 @@ module Bosh::Director::Jobs
           stemcell.destroy
         end
       end
-
-      private
-
-      def track_and_log(stage, task, log = true)
-        stage.advance_and_track(task) do |ticker|
-          @logger.info(task) if log
-          yield ticker if block_given?
-        end
-      end
     end
   end
 end