bosh-director 1.2682.1.0 → 1.2685.0

data/lib/bosh/director/api/controllers/resources_controller.rb

@@ -3,7 +3,7 @@ require 'bosh/director/api/controllers/base_controller'
 module Bosh::Director
   module Api::Controllers
     class ResourcesController < BaseController
-      get '/resources/:id' do
+      get '/:id' do
         tmp_file = @resource_manager.get_resource_path(params[:id])
         send_disposable_file(tmp_file, :type => 'application/x-gzip')
       end

data/lib/bosh/director/api/controllers/resurrection_controller.rb

@@ -3,7 +3,7 @@ require 'bosh/director/api/controllers/base_controller'
 module Bosh::Director
   module Api::Controllers
     class ResurrectionController < BaseController
-      put '/resurrection', consumes: :json do
+      put '/', consumes: :json do
         payload = json_decode(request.body)
 
         @resurrector_manager.set_pause_for_all(payload['resurrection_paused'])

data/lib/bosh/director/api/controllers/stemcells_controller.rb

@@ -3,18 +3,18 @@ require 'bosh/director/api/controllers/base_controller'
 module Bosh::Director
   module Api::Controllers
     class StemcellsController < BaseController
-      post '/stemcells', :consumes => :json do
+      post '/', :consumes => :json do
         payload = json_decode(request.body)
         task = @stemcell_manager.create_stemcell_from_url(@user, payload['location'])
         redirect "/tasks/#{task.id}"
       end
 
-      post '/stemcells', :consumes => :multipart do
+      post '/', :consumes => :multipart do
         task = @stemcell_manager.create_stemcell_from_file_path(@user, params[:nginx_upload_path])
         redirect "/tasks/#{task.id}"
       end
 
-      get '/stemcells' do
+      get '/' do
         stemcells = Models::Stemcell.order_by(:name.asc).map do |stemcell|
           {
             'name' => stemcell.name,
@@ -26,7 +26,7 @@ module Bosh::Director
         json_encode(stemcells)
       end
 
-      delete '/stemcells/:name/:version' do
+      delete '/:name/:version' do
         name, version = params[:name], params[:version]
         options = {}
         options['force'] = true if params['force'] == 'true'

data/lib/bosh/director/api/controllers/task_controller.rb

@@ -0,0 +1,20 @@
+require 'bosh/director/api/controllers/base_controller'
+
+module Bosh::Director
+  module Api::Controllers
+    class TaskController < BaseController
+      delete '/:id' do
+        task_id = params[:id]
+        task = @task_manager.find_task(task_id)
+        if task.state != 'processing' && task.state != 'queued'
+          status(400)
+          body("Cannot cancel task #{task_id}: invalid state (#{task.state})")
+        else
+          task.state = :cancelling
+          task.save
+          status(204)
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/api/controllers/tasks_controller.rb

@@ -3,7 +3,7 @@ require 'bosh/director/api/controllers/base_controller'
 module Bosh::Director
   module Api::Controllers
     class TasksController < BaseController
-      get '/tasks' do
+      get '/' do
         dataset = Models::Task.dataset
         limit = params['limit']
         if limit
@@ -45,7 +45,7 @@ module Bosh::Director
         json_encode(tasks)
       end
 
-      get '/tasks/:id' do
+      get '/:id' do
         task = @task_manager.find_task(params[:id])
         if task_timeout?(task)
           task.state = :timeout
@@ -59,7 +59,7 @@ module Bosh::Director
       # Sends back output of given task id and params[:type]
       # Example: `get /tasks/5/output?type=event` will send back the file
       # at /var/vcap/store/director/tasks/5/event
-      get '/tasks/:id/output' do
+      get '/:id/output' do
         log_type = params[:type] || 'debug'
         task = @task_manager.find_task(params[:id])
 
@@ -75,21 +75,6 @@ module Bosh::Director
           status(204)
         end
       end
-
-      delete '/task/:id' do
-        task_id = params[:id]
-        task = @task_manager.find_task(task_id)
-
-        if task.state != 'processing' && task.state != 'queued'
-          status(400)
-          body("Cannot cancel task #{task_id}: invalid state (#{task.state})")
-        else
-          task.state = :cancelling
-          task.save
-          status(204)
-          body("Cancelling task #{task_id}")
-        end
-      end
     end
   end
 end

data/lib/bosh/director/api/controllers/users_controller.rb

@@ -3,14 +3,14 @@ require 'bosh/director/api/controllers/base_controller'
 module Bosh::Director
   module Api::Controllers
     class UsersController < BaseController
-      post '/users', :consumes => [:json] do
+      post '/', :consumes => [:json] do
         user = @user_manager.get_user_from_request(request)
         @user_manager.create_user(user)
         status(204)
         nil
       end
 
-      put '/users/:username', :consumes => [:json] do
+      put '/:username', :consumes => [:json] do
         user = @user_manager.get_user_from_request(request)
         if user.username != params[:username]
           raise UserImmutableUsername, 'The username is immutable'
@@ -20,7 +20,7 @@ module Bosh::Director
         nil
       end
 
-      delete '/users/:username' do
+      delete '/:username' do
         @user_manager.delete_user(params[:username])
         status(204)
         nil

data/lib/bosh/director/api/deployment_lookup.rb

@@ -10,4 +10,4 @@ module Bosh::Director
       end
     end
   end
-end
+end

data/lib/bosh/director/api/resource_manager.rb

@@ -66,4 +66,4 @@ module Bosh::Director
       end
     end
   end
-end
+end

data/lib/bosh/director/instance_updater/vm_updater.rb

@@ -168,7 +168,8 @@ module Bosh::Director
       end
 
       def detach
-        unless @instance.disk_currently_attached?
+        disk_list = @agent_client.list_disk
+        if disk_list.empty?
           @logger.info('Skipping disk detaching')
           return
         end

data/lib/bosh/director/jobs/cloud_check/scan.rb

@@ -21,7 +21,7 @@ module Bosh::Director
         def perform
           begin
             with_deployment_lock(@deployment, :timeout => 0) do
-              scanner = ProblemScanner.new(@deployment)
+              scanner = ProblemScanner::Scanner.new(@deployment)
               scanner.reset
               scanner.scan_vms
               scanner.scan_disks

data/lib/bosh/director/jobs/cloud_check/scan_and_fix.rb

@@ -29,7 +29,7 @@ module Bosh::Director
           begin
             with_deployment_lock(@deployment, :timeout => 0) do
 
-              scanner = ProblemScanner.new(@deployment)
+              scanner = ProblemScanner::Scanner.new(@deployment)
               scanner.reset(jobs)
               scanner.scan_vms(jobs)
 

data/lib/bosh/director/problem_handlers/missing_disk.rb

@@ -0,0 +1,74 @@
+module Bosh::Director
+  module ProblemHandlers
+    class MissingDisk < Base
+
+      register_as :missing_disk
+      auto_resolution :ignore
+
+      def initialize(disk_id, data)
+        super
+        @disk_id = disk_id
+        @data = data
+        @disk = Models::PersistentDisk[@disk_id]
+
+        if @disk.nil?
+          handler_error("Disk `#{@disk_id}' is no longer in the database")
+        end
+
+        @instance = @disk.instance
+        if @instance.nil?
+          handler_error("Cannot find instance for disk `#{@disk.disk_cid}'")
+        end
+
+        @vm = @instance.vm
+      end
+
+      def description
+        job = @instance.job || "unknown job"
+        index = @instance.index || "unknown index"
+        disk_label = "`#{@disk.disk_cid}' (#{job}/#{index}, #{@disk.size.to_i}M)"
+        "Disk #{disk_label} is missing"
+      end
+
+      resolution :ignore do
+        plan { 'Ignore problem' }
+        action { }
+      end
+
+      resolution :delete_disk_reference do
+        plan { 'Delete disk reference (DANGEROUS!)' }
+        action { delete_disk_reference }
+      end
+
+      def delete_disk_reference
+        @disk.db.transaction do
+          @disk.update(active: false)
+        end
+
+        agent_client = agent_client(@vm)
+        disk_list = []
+
+        begin
+          disk_list = agent_client.list_disk
+          if disk_list.include?(@disk.disk_cid)
+            agent_client.unmount_disk(@disk.disk_cid)
+          end
+
+        rescue Bosh::Director::RpcTimeout
+          handler_error('Cannot unmount disk, agent is not responding')
+        rescue Bosh::Director::RpcRemoteException => e
+          handler_error("Cannot unmount disk, #{e.message}")
+        end
+
+        begin
+          cloud.detach_disk(@vm.cid, @disk.disk_cid) if @vm.cid
+        rescue Bosh::Clouds::DiskNotAttached
+        end
+
+        Api::SnapshotManager.delete_snapshots(@disk.snapshots)
+
+        @disk.destroy
+      end
+    end
+  end
+end

data/lib/bosh/director/problem_scanner/disk_scan_stage.rb

@@ -0,0 +1,80 @@
+module Bosh::Director::ProblemScanner
+  class DiskScanStage
+    def initialize(disk_owners, problem_register, cloud, deployment_id, event_logger, logger)
+      @disk_owners = disk_owners
+      @problem_register = problem_register
+      @cloud = cloud
+      @deployment_id = deployment_id
+      @event_logger = event_logger
+      @logger = logger
+    end
+
+    def scan
+      disks = Bosh::Director::Models::PersistentDisk.eager(:instance).all.select do |disk|
+        disk.instance && disk.instance.deployment_id == @deployment_id
+      end
+
+      results = Hash.new(0)
+
+      @event_logger.begin_stage("Scanning #{disks.size} persistent disks", 2)
+
+      @event_logger.track_and_log('Looking for inactive disks') do
+        disks.each do |disk|
+          scan_result = scan_disk(disk)
+          results[scan_result] += 1
+        end
+      end
+
+      @event_logger.track_and_log("#{results[:ok]} OK, " +
+        "#{results[:missing]} missing, " +
+        "#{results[:inactive]} inactive, " +
+        "#{results[:mount_info_mismatch]} mount-info mismatch")
+    end
+
+    private
+
+    def scan_disk(disk)
+      begin
+        unless @cloud.has_disk?(disk.disk_cid)
+          @logger.info("Found missing disk: #{disk.id}")
+          @problem_register.problem_found(:missing_disk, disk)
+          return :missing
+        end
+      rescue Bosh::Clouds::NotImplemented
+        @logger.info('Ignored check for disk presence, CPI does not implement has_disk? method')
+      end
+
+      # inactive disks
+      unless disk.active
+        @logger.info("Found inactive disk: #{disk.id}")
+        @problem_register.problem_found(:inactive_disk, disk)
+        return :inactive
+      end
+
+      disk_cid = disk.disk_cid
+      vm_cid = nil
+      if disk.instance && disk.instance.vm
+        vm_cid = disk.instance.vm.cid
+      end
+
+      if vm_cid.nil?
+        # With the db dependencies this should not happen.
+        @logger.warn("Disk #{disk_cid} is not associated to any VM. " +
+          "Skipping scan")
+        return :ok
+      end
+
+      owner_vms = @disk_owners[disk_cid] || []
+      # active disk is not mounted or mounted more than once -or-
+      # the disk is mounted on a vm that is different form the record.
+      if owner_vms.size != 1 || owner_vms.first != vm_cid
+        @logger.info("Found problem in mount info: " +
+          "active disk #{disk_cid} mounted on " +
+          "#{owner_vms.join(', ')}")
+        @problem_register.problem_found(:mount_info_mismatch, disk, owner_vms: owner_vms)
+        return :mount_info_mismatch
+      end
+      :ok
+    end
+  end
+end

data/lib/bosh/director/problem_scanner/problem_register.rb

@@ -0,0 +1,55 @@
+module Bosh::Director::ProblemScanner
+  class ProblemRegister
+    def initialize(deployment, logger)
+      @deployment = deployment
+      @logger = logger
+
+      @problem_lock = Mutex.new
+    end
+
+    def problem_found(type, resource, data = {})
+      @problem_lock.synchronize do
+        similar_open_problems = Bosh::Director::Models::DeploymentProblem.
+          filter(deployment_id: @deployment.id, type: type.to_s,
+          resource_id: resource.id, state: 'open').all
+
+        if similar_open_problems.size > 1
+          raise Bosh::Director::CloudcheckTooManySimilarProblems,
+            "More than one problem of type `#{type}' " +
+              "exists for resource #{type} #{resource.id}"
+        end
+
+        if similar_open_problems.empty?
+          problem = Bosh::Director::Models::DeploymentProblem.
+            create(type: type.to_s, resource_id: resource.id,
+            state: 'open', deployment_id: @deployment.id,
+            data: data, counter: 1)
+
+          @logger.info("Created problem #{problem.id} (#{problem.type})")
+        else
+          # This assumes we are running with deployment lock acquired,
+          # so there is no possible update conflict
+          problem = similar_open_problems[0]
+          problem.data = data
+          problem.last_seen_at = Time.now
+          problem.counter += 1
+          problem.save
+          @logger.info("Updated problem #{problem.id} (#{problem.type}), " +
+            "count is now #{problem.counter}")
+        end
+      end
+    end
+
+    def get_vm_instance_and_disk(vm)
+      instance = nil
+      mounted_disk_cid = nil
+
+      @problem_lock.synchronize do
+        instance = vm.instance
+        mounted_disk_cid = instance.persistent_disk_cid if instance
+      end
+
+      [instance, mounted_disk_cid]
+    end
+  end
+end

data/lib/bosh/director/problem_scanner/scanner.rb

@@ -0,0 +1,86 @@
+require 'bosh/director/problem_scanner/problem_register'
+require 'bosh/director/problem_scanner/disk_scan_stage'
+require 'bosh/director/problem_scanner/vm_scan_stage'
+
+module Bosh::Director
+  module ProblemScanner
+    class Scanner
+      def initialize(deployment)
+        @deployment = deployment
+        @agent_disks = {}
+
+        @instance_manager = Api::InstanceManager.new
+
+        @logger = Config.logger
+        @event_logger = EventLogger.new(Config.event_log, @logger)
+
+        @problem_register = ProblemRegister.new(deployment, @logger)
+      end
+
+      def reset(vms=nil)
+        if vms
+          vms.each do |job, index|
+            instance = @instance_manager.find_by_name(@deployment.name, job, index)
+
+            Models::DeploymentProblem.where(
+              deployment: @deployment,
+              resource_id: instance.vm.id,
+              state: 'open'
+            ).update(state: 'closed')
+
+          end
+        else
+          Models::DeploymentProblem.where(
+            state: 'open',
+            deployment: @deployment
+          ).update(state: 'closed')
+        end
+      end
+
+      def scan_vms(vms=nil)
+        vm_scanner = VmScanStage.new(
+          @instance_manager,
+          @problem_register,
+          Config.cloud,
+          @deployment,
+          @event_logger,
+          @logger
+        )
+        vm_scanner.scan(vms)
+
+        @agent_disks = vm_scanner.agent_disks
+      end
+
+      def scan_disks
+        disk_scanner = DiskScanStage.new(
+          @agent_disks,
+          @problem_register,
+          Config.cloud,
+          @deployment.id,
+          @event_logger,
+          @logger
+        )
+        disk_scanner.scan
+      end
+    end
+
+    class EventLogger
+      def initialize(event_log, logger)
+        @event_log = event_log
+        @logger = logger
+      end
+
+      def begin_stage(stage_name, n_steps)
+        @event_log.begin_stage(stage_name, n_steps)
+        @logger.info(stage_name)
+      end
+
+      def track_and_log(task, log = true)
+        @event_log.track(task) do |ticker|
+          @logger.info(task) if log
+          yield ticker if block_given?
+        end
+      end
+    end
+  end
+end