bosh-director 1.3160.0 → 1.3163.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 806ce703c35b0836378c836b4dbe1813940ef3a3
-  data.tar.gz: b01aa80bb323efcc97e5a4c6317b880b8a398c60
+  metadata.gz: b1587b4eccc78429904cf066ccf184484654ce06
+  data.tar.gz: 7894455133a637d96a5e3e33d01fb82e65d6ad3c
 SHA512:
-  metadata.gz: 2956b476e13c7d6fe7674ef6314c7e936275de6a4e9d5040db6a6fe00fc7b9aa07ebb017ffce1c831678c46488aae31107b0712df17bde0cb1ce738b5274d1c1
-  data.tar.gz: e08341fc184a328b34254afaf0aa987359da7908eb535732b23adc871a4a9182449400a00b030baac94c47e8d4d1c2db3f16276f3a0e086446b87ba3b87207ec
+  metadata.gz: cc9f2d84ad6a67d3497c88ec62d42a9a49f3811c1039dabe5e81fc75f1ef5e3adc9266990ebdbf3b86573c73ab9c2c258494142c73244a990fbdf9dc153e5cb8
+  data.tar.gz: 2f758e5cc31f37714690cdd13114f2999a0779d241307ac2b6479914727187d3338b8d0912ea417f677496de55f81c3544b4201e742338bebd6254964e73b9c1

data/bin/bosh-director-worker

@@ -70,18 +70,6 @@ end
 
 Resque::Failure.backend = Resque::Failure::Backtrace
 
-begin
-  @resque_retries ||= 0
-  worker.log "Starting worker #{worker}. Resque info:\n#{Resque.info.inspect}"
-  worker.work(ENV['INTERVAL'] || 5) # interval, will block
-rescue Redis::TimeoutError => e
-  worker.log "Redis timeout [#{@resque_retries}]. Error : #{e.inspect}"
-  while @resque_retries < 10
-    @resque_retries += 1
-    sleep 0.5
-    retry
-  end
+worker.log "Starting worker #{worker}"
 
-  worker.log "Redis timeout max retries reached. Error: #{e.inspect}"
-  raise e
-end
+worker.work(ENV['INTERVAL'] || 5) # interval, will block

data/db/migrations/director/20150513225143_ip_addresses.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  change do
+    create_table :ip_addresses do
+      primary_key :id
+      String      :network_name
+      Bignum      :address, unique: true
+      Boolean     :static
+      foreign_key :instance_id, :instances
+    end
+  end
+end

data/db/migrations/director/20150702004608_add_links.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    alter_table(:templates) do
+      add_column :requires_json, String
+      add_column :provides_json, String
+    end
+  end
+end

data/db/migrations/director/20150708231924_add_link_spec.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  change do
+    alter_table(:deployments) do
+      add_column :link_spec_json, String, :text => true
+    end
+  end
+end

data/db/migrations/director/20150724183256_add_debugging_to_ip_addresses.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    alter_table(:ip_addresses) do
+      add_column :created_at, Time
+      add_column :task_id, String
+    end
+  end
+end

data/db/migrations/director/20150730225029_add_uuid_to_instances.rb

@@ -0,0 +1,16 @@
+require('securerandom')
+
+Sequel.migration do
+  up do
+    alter_table(:instances) do
+      add_column :uuid, String, unique: true
+    end
+    self[:instances].each { |row| self[:instances].filter(id: row[:id]).update(uuid: SecureRandom.uuid) }
+  end
+
+  down do
+    alter_table(:instances) do
+      drop_column :uuid
+    end
+  end
+end

data/db/migrations/director/20150803215805_add_availabililty_zone_and_cloud_properties_to_instances.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    alter_table(:instances) do
+      add_column :availability_zone, String
+      add_column :cloud_properties, String, :text => true
+    end
+  end
+end

data/db/migrations/director/20150804211419_add_compilation_flag_to_instance.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  change do
+    alter_table(:instances) do
+      add_column :compilation, TrueClass, default: false
+    end
+  end
+end

data/db/migrations/director/20150918003455_add_bootstrap_node_to_instance.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  change do
+    alter_table(:instances) do
+      add_column :bootstrap, TrueClass, default: false
+    end
+  end
+end

data/db/migrations/director/20151008232214_add_dns_records.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  change do
+    alter_table(:instances) do
+      add_column :dns_records, String, text: true
+    end
+  end
+end

data/db/migrations/director/20151015172551_add_orphan_disks_and_snapshots.rb

@@ -0,0 +1,29 @@
+Sequel.migration do
+  up do
+    create_table(:orphan_disks) do
+      primary_key :id
+      String :disk_cid, :null => false, :unique => true
+      Integer :size
+      String :availability_zone
+      String :deployment_name, :null => false
+      String :instance_name, :null => false
+      String :cloud_properties_json, :text => true
+      Time :orphaned_at, :null => false, :index => true
+    end
+
+    create_table(:orphan_snapshots) do
+      primary_key :id
+      foreign_key :orphan_disk_id, :orphan_disks, :null => false
+      String :snapshot_cid, :unique => true, :null => false
+      Boolean :clean, :default => false
+      Time :created_at, :null => false
+      Time :orphaned_at, :null => false, :index => true
+    end
+  end
+
+  down do
+    drop_table(:orphan_snapshots)
+    drop_table(:orphan_disks)
+  end
+end
+

data/db/migrations/director/20151030222853_add_templates_to_instance.rb

@@ -0,0 +1,10 @@
+Sequel.migration do
+  up do
+    create_table(:instances_templates) do
+      primary_key :id
+      foreign_key :instance_id, :instances, :null => false
+      foreign_key :template_id, :templates, :null => false
+      unique [:instance_id, :template_id]
+    end
+  end
+end

data/db/migrations/director/20151031001039_add_spec_to_instance.rb

@@ -0,0 +1,19 @@
+Sequel.migration do
+  change do
+    alter_table(:instances) do
+      add_column :spec_json, String, text: true
+    end
+
+    self[:instances].each do |instance|
+      next unless instance[:vm_id]
+
+      vm = self[:vms].filter(id: instance[:vm_id]).first
+
+      self[:instances].filter(id: instance[:id]).update(spec_json: vm[:apply_spec_json])
+    end
+
+    alter_table(:vms) do
+      drop_column(:apply_spec_json)
+    end
+  end
+end

data/db/migrations/director/20151109190602_rename_orphan_columns.rb

@@ -0,0 +1,13 @@
+Sequel.migration do
+  change do
+    alter_table :orphan_disks do
+      rename_column :orphaned_at, :created_at
+    end
+
+    alter_table :orphan_snapshots do
+      drop_column :created_at
+      rename_column :orphaned_at, :created_at
+      add_column :snapshot_created_at, Time
+    end
+  end
+end

data/lib/bosh/director.rb

@@ -39,13 +39,16 @@ require 'securerandom'
 require 'common/thread_formatter'
 require 'bosh/core/encryption_handler'
 require 'bosh/director/api'
-require 'bosh/director/dns_helper'
+require 'bosh/director/dns/canonicalizer'
+require 'bosh/director/dns/dns_manager'
+require 'bosh/director/dns/local_dns_repo'
 require 'bosh/director/errors'
 require 'bosh/director/ext'
 require 'bosh/director/ip_util'
 require 'bosh/director/lock_helper'
 require 'bosh/director/validation_helper'
 require 'bosh/director/download_helper'
+require 'bosh/director/tagged_logger'
 
 require 'bosh/director/version'
 require 'bosh/director/config'
@@ -60,9 +63,10 @@ require 'bosh/director/job_renderer'
 require 'bosh/director/cycle_helper'
 require 'bosh/director/encryption_helper'
 require 'bosh/director/vm_creator'
+require 'bosh/director/vm_recreator'
+require 'bosh/director/vm_deleter'
 require 'bosh/director/vm_metadata_updater'
-require 'bosh/director/vm_data'
-require 'bosh/director/vm_reuser'
+require 'bosh/director/instance_reuser'
 require 'bosh/director/deployment_plan'
 require 'bosh/director/errand'
 require 'bosh/director/duration'
@@ -70,9 +74,9 @@ require 'bosh/director/hash_string_vals'
 require 'bosh/director/instance_deleter'
 require 'bosh/director/instance_updater'
 require 'bosh/director/instance_updater/preparer'
-require 'bosh/director/instance_updater/stopper'
-require 'bosh/director/instance_updater/network_updater'
-require 'bosh/director/instance_updater/vm_updater'
+require 'bosh/director/instance_updater/state_applier'
+require 'bosh/director/disk_manager'
+require 'bosh/director/stopper'
 require 'bosh/director/job_runner'
 require 'bosh/director/job_updater'
 require 'bosh/director/job_updater_factory'
@@ -82,7 +86,9 @@ require 'bosh/director/nats_rpc'
 require 'bosh/director/network_reservation'
 require 'bosh/director/problem_scanner/scanner'
 require 'bosh/director/problem_resolver'
-require 'bosh/director/resource_pool_updater'
+require 'bosh/director/error_ignorer'
+require 'bosh/director/deployment_deleter'
+require 'bosh/director/transactor'
 require 'bosh/director/sequel'
 require 'common/thread_pool'
 
@@ -94,15 +100,14 @@ require 'bosh/director/problem_handlers/base'
 require 'bosh/director/problem_handlers/invalid_problem'
 require 'bosh/director/problem_handlers/inactive_disk'
 require 'bosh/director/problem_handlers/missing_disk'
-require 'bosh/director/problem_handlers/out_of_sync_vm'
 require 'bosh/director/problem_handlers/unresponsive_agent'
-require 'bosh/director/problem_handlers/unbound_instance_vm'
 require 'bosh/director/problem_handlers/mount_info_mismatch'
 require 'bosh/director/problem_handlers/missing_vm'
 
 require 'bosh/director/jobs/base_job'
 require 'bosh/director/jobs/backup'
 require 'bosh/director/jobs/scheduled_backup'
+require 'bosh/director/jobs/scheduled_orphan_cleanup'
 require 'bosh/director/jobs/create_snapshot'
 require 'bosh/director/jobs/snapshot_deployment'
 require 'bosh/director/jobs/snapshot_deployments'
@@ -111,7 +116,9 @@ require 'bosh/director/jobs/delete_deployment'
 require 'bosh/director/jobs/delete_deployment_snapshots'
 require 'bosh/director/jobs/delete_release'
 require 'bosh/director/jobs/delete_snapshots'
+require 'bosh/director/jobs/delete_orphan_disks'
 require 'bosh/director/jobs/delete_stemcell'
+require 'bosh/director/jobs/cleanup_artifacts'
 require 'bosh/director/jobs/export_release'
 require 'bosh/director/jobs/update_deployment'
 require 'bosh/director/jobs/update_release'
@@ -124,6 +131,7 @@ require 'bosh/director/jobs/cloud_check/scan_and_fix'
 require 'bosh/director/jobs/cloud_check/apply_resolutions'
 require 'bosh/director/jobs/release/release_job'
 require 'bosh/director/jobs/ssh'
+require 'bosh/director/jobs/helpers'
 
 require 'bosh/director/models/helpers/model_helper'
 
@@ -141,7 +149,9 @@ end
 require 'bosh/director/thread_pool'
 require 'bosh/director/api/extensions/scoping'
 require 'bosh/director/api/controllers/backups_controller'
+require 'bosh/director/api/controllers/cleanup_controller'
 require 'bosh/director/api/controllers/deployments_controller'
+require 'bosh/director/api/controllers/disks_controller'
 require 'bosh/director/api/controllers/packages_controller'
 require 'bosh/director/api/controllers/info_controller'
 require 'bosh/director/api/controllers/releases_controller'

data/lib/bosh/director/agent_client.rb

@@ -19,11 +19,6 @@ module Bosh::Director
 
     attr_accessor :id
 
-    def self.with_defaults(id, options = {})
-      vm = Bosh::Director::Models::Vm.find(:agent_id => id)
-      with_vm(vm, options)
-    end
-
     def self.with_vm(vm, options = {})
       defaults = {
         retry_methods: {
@@ -70,14 +65,6 @@ module Bosh::Director
       send_message(:list_disk, *args)
     end
 
-    def prepare_configure_networks(*args)
-      send_message(:prepare_configure_networks, *args)
-    end
-
-    def prepare_network_change(*args)
-      send_message(:prepare_network_change, *args)
-    end
-
     def start(*args)
       send_message(:start, *args)
     end
@@ -158,10 +145,6 @@ module Bosh::Director
       task['value']
     end
 
-    def configure_networks(*args)
-      send_message(:configure_networks, *args)
-    end
-
     def wait_until_ready(deadline = 600)
       old_timeout = @timeout
       @timeout = 1.0

data/lib/bosh/director/api/cloud_config_manager.rb

@@ -6,7 +6,7 @@ module Bosh
           cloud_config = Bosh::Director::Models::CloudConfig.new(
             properties: cloud_config_yaml
           )
-          validate_manifest(cloud_config)
+          validate_manifest!(cloud_config)
           cloud_config.save
         end
 
@@ -20,10 +20,12 @@ module Bosh
 
         private
 
-        def validate_manifest(cloud_config)
-          deployment = Bosh::Director::DeploymentPlan::CloudPlanner.new(cloud_config)
-          parser = Bosh::Director::DeploymentPlan::CloudManifestParser.new(deployment, Config.logger)
-          parser.parse(cloud_config.manifest)
+        def validate_manifest!(cloud_config)
+          # FIXME: we really just need to validate the manifest, we don't care about the subnets being able to reserve IPs here
+          global_network_resolver = Bosh::Director::DeploymentPlan::NullGlobalNetworkResolver.new
+
+          parser = Bosh::Director::DeploymentPlan::CloudManifestParser.new(Config.logger)
+          _ = parser.parse(cloud_config.manifest, global_network_resolver, nil) # valid if this doesn't blow up
         end
       end
     end

data/lib/bosh/director/api/controllers/base_controller.rb

@@ -4,11 +4,11 @@ module Bosh::Director
       class BaseController < Sinatra::Base
         include ApiHelper
         include Http
-        include DnsHelper
 
         def initialize(config)
           super()
           @config = config
+          @logger = Config.logger
           @identity_provider = config.identity_provider
           @deployment_manager = DeploymentManager.new
           @backup_manager = BackupManager.new
@@ -21,7 +21,8 @@ module Bosh::Director
           @stemcell_manager = StemcellManager.new
           @task_manager = TaskManager.new
           @vm_state_manager = VmStateManager.new
-          @logger = Config.logger
+          @dns_manager = DnsManager.create
+          @disk_manager = DiskManager.new(nil, @logger)
         end
 
         register Bosh::Director::Api::Extensions::Scoping

data/lib/bosh/director/api/controllers/cleanup_controller.rb

@@ -0,0 +1,15 @@
+require 'bosh/director/api/controllers/base_controller'
+
+module Bosh::Director
+  module Api::Controllers
+    class CleanupController < BaseController
+      post '/', :consumes => :json do
+        job_queue = JobQueue.new
+        payload = json_decode(request.body)
+        task = Bosh::Director::Jobs::CleanupArtifacts.enqueue(current_user, payload['config'], job_queue)
+
+        redirect "/tasks/#{task.id}"
+      end
+    end
+  end
+end

data/lib/bosh/director/api/controllers/deployments_controller.rb

@@ -3,13 +3,14 @@ require 'bosh/director/api/controllers/base_controller'
 module Bosh::Director
   module Api::Controllers
     class DeploymentsController < BaseController
-      get '/:deployment/jobs/:job/:index' do
-        instance = @instance_manager.find_by_name(params[:deployment], params[:job], params[:index])
+      get '/:deployment/jobs/:job/:index_or_id' do
+        instance = @instance_manager.find_by_name(params[:deployment], params[:job], params[:index_or_id])
 
         response = {
           deployment: params[:deployment],
           job: instance.job,
           index: instance.index,
+          id: instance.uuid,
           state: instance.state,
           disks: instance.persistent_disks.map {|d| d.disk_cid}
         }
@@ -40,21 +41,20 @@ module Bosh::Director
           options['job_rename']['force'] = true if params['force'] == 'true'
         end
 
-        # we get the deployment here even though it isn't used here, to make sure
-        # the call returns a 404 if the deployment doesn't exist
-        @deployment_manager.find_by_name(params[:deployment])
+        deployment = @deployment_manager.find_by_name(params[:deployment])
+        manifest = ((request.content_length.nil?  || request.content_length.to_i == 0) && (params['state'])) ? StringIO.new(deployment.manifest) : request.body
+
         latest_cloud_config = Bosh::Director::Api::CloudConfigManager.new.latest
-        task = @deployment_manager.create_deployment(current_user, request.body, latest_cloud_config, options)
+        task = @deployment_manager.create_deployment(current_user, manifest, latest_cloud_config, options)
         redirect "/tasks/#{task.id}"
       end
 
       # PUT /deployments/foo/jobs/dea/2?state={started,stopped,detached,restart,recreate}&skip_drain=true
-      put '/:deployment/jobs/:job/:index', :consumes => :yaml do
-        begin
-          index = Integer(params[:index])
-        rescue ArgumentError
-          raise InstanceInvalidIndex, "Invalid instance index `#{params[:index]}'"
-        end
+      put '/:deployment/jobs/:job/:index_or_id', :consumes => :yaml do
+        validate_instance_index_or_id(params[:index_or_id])
+
+        instance = @instance_manager.find_by_name(params[:deployment], params[:job], params[:index_or_id])
+        index = instance.index
 
         options = {
           'job_states' => {
@@ -68,24 +68,24 @@ module Bosh::Director
         options['skip_drain'] = params[:job] if params['skip_drain'] == 'true'
 
         deployment = @deployment_manager.find_by_name(params[:deployment])
-        manifest = (request.content_length.nil?  || request.content_length == 0) ? StringIO.new(deployment.manifest) : request.body
+        manifest = (request.content_length.nil?  || request.content_length.to_i == 0) ? StringIO.new(deployment.manifest) : request.body
         latest_cloud_config = Bosh::Director::Api::CloudConfigManager.new.latest
         task = @deployment_manager.create_deployment(current_user, manifest, latest_cloud_config, options)
         redirect "/tasks/#{task.id}"
       end
 
       # GET /deployments/foo/jobs/dea/2/logs
-      get '/:deployment/jobs/:job/:index/logs' do
+      get '/:deployment/jobs/:job/:index_or_id/logs' do
         deployment = params[:deployment]
         job = params[:job]
-        index = params[:index]
+        index_or_id = params[:index_or_id]
 
         options = {
           'type' => params[:type].to_s.strip,
           'filters' => params[:filters].to_s.strip.split(/[\s\,]+/)
         }
 
-        task = @instance_manager.fetch_logs(current_user, deployment, job, index, options)
+        task = @instance_manager.fetch_logs(current_user, deployment, job, index_or_id, options)
         redirect "/tasks/#{task.id}"
       end
 
@@ -108,14 +108,18 @@ module Bosh::Director
         redirect "/tasks/#{task.id}"
       end
 
-      put '/:deployment/jobs/:job/:index/resurrection', consumes: :json do
+      put '/:deployment/jobs/:job/:index_or_id/resurrection', consumes: :json do
         payload = json_decode(request.body)
 
-        @resurrector_manager.set_pause_for_instance(params[:deployment], params[:job], params[:index], payload['resurrection_paused'])
+        @resurrector_manager.set_pause_for_instance(params[:deployment], params[:job], params[:index_or_id], payload['resurrection_paused'])
       end
 
-      post '/:deployment/jobs/:job/:index/snapshots' do
-        instance = @instance_manager.find_by_name(params[:deployment], params[:job], params[:index])
+      post '/:deployment/jobs/:job/:index_or_id/snapshots' do
+        if params[:index_or_id].to_s =~ /^\d+$/
+          instance = @instance_manager.find_by_name(params[:deployment], params[:job], params[:index_or_id])
+        else
+          instance = @instance_manager.filter_by(uuid: params[:index_or_id]).first
+        end
         # until we can tell the agent to flush and wait, all snapshots are considered dirty
         options = {clean: false}
 
@@ -296,7 +300,7 @@ module Bosh::Director
       end
 
       get '/:deployment_name/errands', scope: :read do
-        deployment_plan = load_deployment_plan_without_binding
+        deployment_plan = load_deployment_plan
 
         errands = deployment_plan.jobs.select(&:can_run_as_errand?)
 
@@ -309,13 +313,11 @@ module Bosh::Director
 
       private
 
-      def load_deployment_plan_without_binding
+      def load_deployment_plan
         deployment_model = @deployment_manager.find_by_name(params[:deployment_name])
-        manifest_hash = Psych.load(deployment_model.manifest)
-        cloud_config_model = deployment_model.cloud_config
 
-        planner_factory = Bosh::Director::DeploymentPlan::PlannerFactory.create(Config.event_log, Config.logger)
-        planner_factory.planner_without_vm_binding(manifest_hash, cloud_config_model, {})
+        planner_factory = Bosh::Director::DeploymentPlan::PlannerFactory.create(Config.logger)
+        planner_factory.create_from_model(deployment_model)
       end
 
       def convert_job_instance_hash(hash)
@@ -334,6 +336,16 @@ module Bosh::Director
         instances = @instance_manager.filter_by(filter)
         instances.any?
       end
+
+      def validate_instance_index_or_id(str)
+        begin
+          Integer(str)
+        rescue ArgumentError
+          if str !~ /^[A-Fa-f0-9]{8}-[A-Fa-f0-9-]{27}$/
+            raise InstanceInvalidIndex, "Invalid instance index or id `#{str}'"
+          end
+        end
+      end
     end
   end
 end