bosh-director 1.3160.0 → 1.3163.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (195) hide show
  1. checksums.yaml +4 -4
  2. data/bin/bosh-director-worker +2 -14
  3. data/db/migrations/director/20150513225143_ip_addresses.rb +11 -0
  4. data/db/migrations/director/20150702004608_add_links.rb +8 -0
  5. data/db/migrations/director/20150708231924_add_link_spec.rb +7 -0
  6. data/db/migrations/director/20150724183256_add_debugging_to_ip_addresses.rb +8 -0
  7. data/db/migrations/director/20150730225029_add_uuid_to_instances.rb +16 -0
  8. data/db/migrations/director/20150803215805_add_availabililty_zone_and_cloud_properties_to_instances.rb +8 -0
  9. data/db/migrations/director/20150804211419_add_compilation_flag_to_instance.rb +7 -0
  10. data/db/migrations/director/20150918003455_add_bootstrap_node_to_instance.rb +7 -0
  11. data/db/migrations/director/20151008232214_add_dns_records.rb +7 -0
  12. data/db/migrations/director/20151015172551_add_orphan_disks_and_snapshots.rb +29 -0
  13. data/db/migrations/director/20151030222853_add_templates_to_instance.rb +10 -0
  14. data/db/migrations/director/20151031001039_add_spec_to_instance.rb +19 -0
  15. data/db/migrations/director/20151109190602_rename_orphan_columns.rb +13 -0
  16. data/lib/bosh/director.rb +19 -9
  17. data/lib/bosh/director/agent_client.rb +0 -17
  18. data/lib/bosh/director/api/cloud_config_manager.rb +7 -5
  19. data/lib/bosh/director/api/controllers/base_controller.rb +3 -2
  20. data/lib/bosh/director/api/controllers/cleanup_controller.rb +15 -0
  21. data/lib/bosh/director/api/controllers/deployments_controller.rb +38 -26
  22. data/lib/bosh/director/api/controllers/disks_controller.rb +20 -0
  23. data/lib/bosh/director/api/controllers/info_controller.rb +2 -2
  24. data/lib/bosh/director/api/controllers/releases_controller.rb +1 -16
  25. data/lib/bosh/director/api/controllers/stemcells_controller.rb +1 -9
  26. data/lib/bosh/director/api/deployment_manager.rb +2 -1
  27. data/lib/bosh/director/api/instance_lookup.rb +17 -0
  28. data/lib/bosh/director/api/instance_manager.rb +20 -10
  29. data/lib/bosh/director/api/release_manager.rb +28 -8
  30. data/lib/bosh/director/api/resurrector_manager.rb +9 -2
  31. data/lib/bosh/director/api/route_configuration.rb +2 -0
  32. data/lib/bosh/director/api/snapshot_manager.rb +9 -5
  33. data/lib/bosh/director/api/stemcell_manager.rb +50 -0
  34. data/lib/bosh/director/app.rb +1 -1
  35. data/lib/bosh/director/cloudcheck_helper.rb +119 -132
  36. data/lib/bosh/director/compile_task.rb +1 -1
  37. data/lib/bosh/director/compile_task_generator.rb +2 -2
  38. data/lib/bosh/director/config.rb +21 -12
  39. data/lib/bosh/director/deployment_deleter.rb +69 -0
  40. data/lib/bosh/director/deployment_plan.rb +35 -4
  41. data/lib/bosh/director/deployment_plan/agent_state_migrator.rb +47 -0
  42. data/lib/bosh/director/deployment_plan/assembler.rb +115 -241
  43. data/lib/bosh/director/deployment_plan/availability_zone.rb +27 -0
  44. data/lib/bosh/director/deployment_plan/cloud_manifest_parser.rb +144 -35
  45. data/lib/bosh/director/deployment_plan/compilation_config.rb +21 -19
  46. data/lib/bosh/director/deployment_plan/compilation_instance_pool.rb +169 -0
  47. data/lib/bosh/director/deployment_plan/deployment_repo.rb +4 -8
  48. data/lib/bosh/director/deployment_plan/deployment_spec_parser.rb +13 -1
  49. data/lib/bosh/director/deployment_plan/deployment_validator.rb +17 -0
  50. data/lib/bosh/director/deployment_plan/desired_instance.rb +15 -0
  51. data/lib/bosh/director/deployment_plan/{disk_pool.rb → disk_type.rb} +14 -19
  52. data/lib/bosh/director/deployment_plan/dynamic_network.rb +105 -53
  53. data/lib/bosh/director/deployment_plan/dynamic_network_subnet.rb +13 -0
  54. data/lib/bosh/director/deployment_plan/env.rb +18 -0
  55. data/lib/bosh/director/deployment_plan/global_network_resolver.rb +77 -0
  56. data/lib/bosh/director/deployment_plan/instance.rb +222 -390
  57. data/lib/bosh/director/deployment_plan/instance_network_reservations.rb +71 -0
  58. data/lib/bosh/director/deployment_plan/instance_plan.rb +336 -0
  59. data/lib/bosh/director/deployment_plan/instance_plan_factory.rb +54 -0
  60. data/lib/bosh/director/deployment_plan/instance_plan_sorter.rb +61 -0
  61. data/lib/bosh/director/deployment_plan/instance_planner.rb +101 -0
  62. data/lib/bosh/director/deployment_plan/instance_repository.rb +36 -0
  63. data/lib/bosh/director/deployment_plan/instance_spec.rb +154 -0
  64. data/lib/bosh/director/deployment_plan/ip_provider/database_ip_repo.rb +136 -0
  65. data/lib/bosh/director/deployment_plan/ip_provider/in_memory_ip_repo.rb +81 -0
  66. data/lib/bosh/director/deployment_plan/ip_provider/ip_provider.rb +153 -0
  67. data/lib/bosh/director/deployment_plan/ip_provider/ip_provider_factory.rb +22 -0
  68. data/lib/bosh/director/deployment_plan/job.rb +116 -53
  69. data/lib/bosh/director/deployment_plan/job_availability_zone_parser.rb +49 -0
  70. data/lib/bosh/director/deployment_plan/job_migrator.rb +90 -0
  71. data/lib/bosh/director/deployment_plan/job_network.rb +42 -0
  72. data/lib/bosh/director/deployment_plan/job_network_parser.rb +118 -0
  73. data/lib/bosh/director/deployment_plan/job_spec_parser.rb +123 -126
  74. data/lib/bosh/director/deployment_plan/links/link.rb +30 -0
  75. data/lib/bosh/director/deployment_plan/links/link_lookup.rb +66 -0
  76. data/lib/bosh/director/deployment_plan/links/link_path.rb +27 -0
  77. data/lib/bosh/director/deployment_plan/links/links_resolver.rb +70 -0
  78. data/lib/bosh/director/deployment_plan/links/template_link.rb +21 -0
  79. data/lib/bosh/director/deployment_plan/manifest_migrator.rb +3 -17
  80. data/lib/bosh/director/deployment_plan/manifest_validator.rb +46 -0
  81. data/lib/bosh/director/deployment_plan/manual_network.rb +70 -97
  82. data/lib/bosh/director/deployment_plan/manual_network_subnet.rb +148 -0
  83. data/lib/bosh/director/deployment_plan/network.rb +50 -39
  84. data/lib/bosh/director/deployment_plan/network_planner.rb +4 -0
  85. data/lib/bosh/director/deployment_plan/network_planner/plan.rb +26 -0
  86. data/lib/bosh/director/deployment_plan/network_planner/planner.rb +21 -0
  87. data/lib/bosh/director/deployment_plan/network_planner/reservation_reconciler.rb +81 -0
  88. data/lib/bosh/director/deployment_plan/network_planner/vip_static_ips_planner.rb +50 -0
  89. data/lib/bosh/director/deployment_plan/network_settings.rb +65 -0
  90. data/lib/bosh/director/deployment_plan/options/skip_drain.rb +7 -0
  91. data/lib/bosh/director/deployment_plan/package_validator.rb +79 -0
  92. data/lib/bosh/director/deployment_plan/placement_planner.rb +8 -0
  93. data/lib/bosh/director/deployment_plan/placement_planner/availability_zone_picker.rb +90 -0
  94. data/lib/bosh/director/deployment_plan/placement_planner/bruteforce_ip_allocation.rb +124 -0
  95. data/lib/bosh/director/deployment_plan/placement_planner/index_assigner.rb +32 -0
  96. data/lib/bosh/director/deployment_plan/placement_planner/networks_to_static_ips.rb +125 -0
  97. data/lib/bosh/director/deployment_plan/placement_planner/placed_desired_instances.rb +40 -0
  98. data/lib/bosh/director/deployment_plan/placement_planner/plan.rb +42 -0
  99. data/lib/bosh/director/deployment_plan/placement_planner/static_ips_availability_zone_picker.rb +237 -0
  100. data/lib/bosh/director/deployment_plan/placement_planner/unplaced_existing_instances.rb +53 -0
  101. data/lib/bosh/director/deployment_plan/planner.rb +186 -74
  102. data/lib/bosh/director/deployment_plan/planner_factory.rb +30 -147
  103. data/lib/bosh/director/deployment_plan/release_version.rb +3 -3
  104. data/lib/bosh/director/deployment_plan/resource_pool.rb +2 -174
  105. data/lib/bosh/director/deployment_plan/stemcell.rb +57 -14
  106. data/lib/bosh/director/deployment_plan/steps/package_compile_step.rb +28 -135
  107. data/lib/bosh/director/deployment_plan/steps/update_step.rb +23 -44
  108. data/lib/bosh/director/deployment_plan/template.rb +15 -4
  109. data/lib/bosh/director/deployment_plan/vip_network.rb +14 -42
  110. data/lib/bosh/director/deployment_plan/vm.rb +1 -99
  111. data/lib/bosh/director/deployment_plan/vm_type.rb +27 -0
  112. data/lib/bosh/director/disk_manager.rb +268 -0
  113. data/lib/bosh/director/dns/canonicalizer.rb +28 -0
  114. data/lib/bosh/director/dns/dns_manager.rb +163 -0
  115. data/lib/bosh/director/dns/local_dns_repo.rb +20 -0
  116. data/lib/bosh/director/dns/powerdns.rb +170 -0
  117. data/lib/bosh/director/errand/job_manager.rb +18 -29
  118. data/lib/bosh/director/error_ignorer.rb +16 -0
  119. data/lib/bosh/director/errors.rb +51 -20
  120. data/lib/bosh/director/event_log.rb +6 -0
  121. data/lib/bosh/director/instance_deleter.rb +53 -81
  122. data/lib/bosh/director/instance_reuser.rb +89 -0
  123. data/lib/bosh/director/instance_updater.rb +139 -281
  124. data/lib/bosh/director/instance_updater/preparer.rb +8 -5
  125. data/lib/bosh/director/instance_updater/state_applier.rb +21 -0
  126. data/lib/bosh/director/ip_util.rb +46 -26
  127. data/lib/bosh/director/job_renderer.rb +22 -10
  128. data/lib/bosh/director/job_runner.rb +1 -4
  129. data/lib/bosh/director/job_updater.rb +47 -35
  130. data/lib/bosh/director/job_updater_factory.rb +5 -4
  131. data/lib/bosh/director/jobs/base_job.rb +8 -0
  132. data/lib/bosh/director/jobs/cleanup_artifacts.rb +93 -0
  133. data/lib/bosh/director/jobs/delete_deployment.rb +10 -154
  134. data/lib/bosh/director/jobs/delete_deployment_snapshots.rb +1 -1
  135. data/lib/bosh/director/jobs/delete_orphan_disks.rb +44 -0
  136. data/lib/bosh/director/jobs/delete_release.rb +19 -196
  137. data/lib/bosh/director/jobs/delete_stemcell.rb +10 -76
  138. data/lib/bosh/director/jobs/export_release.rb +41 -121
  139. data/lib/bosh/director/jobs/fetch_logs.rb +0 -6
  140. data/lib/bosh/director/jobs/helpers.rb +10 -0
  141. data/lib/bosh/director/jobs/helpers/blob_deleter.rb +24 -0
  142. data/lib/bosh/director/jobs/helpers/compiled_package_deleter.rb +24 -0
  143. data/lib/bosh/director/jobs/helpers/name_version_release_deleter.rb +48 -0
  144. data/lib/bosh/director/jobs/helpers/package_deleter.rb +33 -0
  145. data/lib/bosh/director/jobs/helpers/release_deleter.rb +52 -0
  146. data/lib/bosh/director/jobs/helpers/release_version_deleter.rb +115 -0
  147. data/lib/bosh/director/jobs/helpers/releases_to_delete_picker.rb +31 -0
  148. data/lib/bosh/director/jobs/helpers/stemcell_deleter.rb +61 -0
  149. data/lib/bosh/director/jobs/helpers/stemcells_to_delete_picker.rb +30 -0
  150. data/lib/bosh/director/jobs/helpers/template_deleter.rb +20 -0
  151. data/lib/bosh/director/jobs/release/release_job.rb +18 -7
  152. data/lib/bosh/director/jobs/run_errand.rb +57 -36
  153. data/lib/bosh/director/jobs/scheduled_orphan_cleanup.rb +46 -0
  154. data/lib/bosh/director/jobs/ssh.rb +50 -17
  155. data/lib/bosh/director/jobs/update_deployment.rb +29 -11
  156. data/lib/bosh/director/jobs/update_release.rb +25 -4
  157. data/lib/bosh/director/jobs/vm_state.rb +23 -32
  158. data/lib/bosh/director/lock.rb +13 -8
  159. data/lib/bosh/director/logs_fetcher.rb +1 -1
  160. data/lib/bosh/director/models.rb +3 -0
  161. data/lib/bosh/director/models/compiled_package.rb +3 -3
  162. data/lib/bosh/director/models/deployment.rb +10 -0
  163. data/lib/bosh/director/models/instance.rb +77 -1
  164. data/lib/bosh/director/models/ip_address.rb +26 -0
  165. data/lib/bosh/director/models/orphan_disk.rb +23 -0
  166. data/lib/bosh/director/models/orphan_snapshot.rb +14 -0
  167. data/lib/bosh/director/models/template.rb +32 -9
  168. data/lib/bosh/director/models/vm.rb +5 -8
  169. data/lib/bosh/director/network_reservation.rb +69 -99
  170. data/lib/bosh/director/problem_handlers/inactive_disk.rb +5 -20
  171. data/lib/bosh/director/problem_handlers/missing_disk.rb +2 -13
  172. data/lib/bosh/director/problem_resolver.rb +2 -2
  173. data/lib/bosh/director/problem_scanner/vm_scan_stage.rb +2 -21
  174. data/lib/bosh/director/scheduler.rb +23 -6
  175. data/lib/bosh/director/{instance_updater/stopper.rb → stopper.rb} +24 -18
  176. data/lib/bosh/director/tagged_logger.rb +30 -0
  177. data/lib/bosh/director/transactor.rb +9 -0
  178. data/lib/bosh/director/version.rb +1 -1
  179. data/lib/bosh/director/vm_creator.rb +91 -19
  180. data/lib/bosh/director/vm_deleter.rb +25 -0
  181. data/lib/bosh/director/vm_recreator.rb +15 -0
  182. data/lib/cloud/dummy.rb +381 -94
  183. metadata +110 -30
  184. data/lib/bosh/director/deployment_plan/dns_binder.rb +0 -45
  185. data/lib/bosh/director/deployment_plan/instance_vm_binder.rb +0 -37
  186. data/lib/bosh/director/deployment_plan/network_subnet.rb +0 -166
  187. data/lib/bosh/director/deployment_plan/resource_pools.rb +0 -68
  188. data/lib/bosh/director/dns_helper.rb +0 -223
  189. data/lib/bosh/director/instance_updater/network_updater.rb +0 -110
  190. data/lib/bosh/director/instance_updater/vm_updater.rb +0 -189
  191. data/lib/bosh/director/problem_handlers/out_of_sync_vm.rb +0 -64
  192. data/lib/bosh/director/problem_handlers/unbound_instance_vm.rb +0 -85
  193. data/lib/bosh/director/resource_pool_updater.rb +0 -174
  194. data/lib/bosh/director/vm_data.rb +0 -63
  195. data/lib/bosh/director/vm_reuser.rb +0 -63
@@ -148,7 +148,7 @@ module Bosh::Director
148
148
  if BlobUtil.exists_in_global_cache?(package, cache_key)
149
149
  event_log.track("Downloading '#{package.desc}' from global cache") do
150
150
  # has side effect of putting CompiledPackage model in db
151
- compiled_package = BlobUtil.fetch_from_global_cache(package, stemcell, cache_key, dependency_key)
151
+ compiled_package = BlobUtil.fetch_from_global_cache(package, stemcell.model, cache_key, dependency_key)
152
152
  end
153
153
  end
154
154
  end
@@ -13,7 +13,7 @@ module Bosh::Director
13
13
  # has no cycles: this is being enforced on release upload.
14
14
  # Other than that it's a vanilla DFS.
15
15
 
16
- @logger.info("Checking whether package `#{package.desc}' needs to be compiled for stemcell `#{stemcell.desc}'")
16
+ @logger.info("Checking whether package `#{package.desc}' needs to be compiled for stemcell `#{stemcell.model.desc}'")
17
17
  task_key = [package.id, stemcell.id]
18
18
  task = compile_tasks[task_key]
19
19
 
@@ -26,7 +26,7 @@ module Bosh::Director
26
26
 
27
27
  transitive_dependencies = release_version.transitive_dependencies(package)
28
28
  package_dependency_key = Models::CompiledPackage.create_dependency_key(transitive_dependencies)
29
- package_cache_key = Models::CompiledPackage.create_cache_key(package, transitive_dependencies, stemcell)
29
+ package_cache_key = Models::CompiledPackage.create_cache_key(package, transitive_dependencies, stemcell.model)
30
30
 
31
31
  task = CompileTask.new(package, stemcell, job, package_dependency_key, package_cache_key)
32
32
 
@@ -1,6 +1,5 @@
1
1
  require 'fileutils'
2
2
  require 'logging'
3
- require 'bosh/director/dns_helper'
4
3
 
5
4
  module Bosh::Director
6
5
 
@@ -8,8 +7,6 @@ module Bosh::Director
8
7
 
9
8
  class Config
10
9
  class << self
11
- include DnsHelper
12
-
13
10
  attr_accessor(
14
11
  :base_dir,
15
12
  :cloud_options,
@@ -34,7 +31,8 @@ module Bosh::Director
34
31
  :enable_snapshots,
35
32
  :max_vm_create_tries,
36
33
  :nats_uri,
37
- :default_ssh_options
34
+ :default_ssh_options,
35
+ :keep_unreachable_vms
38
36
  )
39
37
 
40
38
  attr_reader(
@@ -133,12 +131,25 @@ module Bosh::Director
133
131
  @db_config = config['db']
134
132
  @db = configure_db(config['db'])
135
133
  @dns = config['dns']
136
- @dns_domain_name = 'bosh'
137
- if @dns
138
- @dns_db = configure_db(@dns['db']) if @dns['db']
139
- @dns_domain_name = canonical(@dns['domain_name']) if @dns['domain_name']
134
+ if @dns && @dns['db']
135
+ @dns_db = configure_db(@dns['db'])
136
+ if @dns_db
137
+ # Load these constants early.
138
+ # These constants are not 'require'd, they are 'autoload'ed
139
+ # in models.rb. We're seeing that in 1.9.3 that sometimes
140
+ # the constants loaded from one thread are not visible to other threads,
141
+ # causing failures.
142
+ # These constants cannot be required because they are Sequel model classes
143
+ # that refer to database configuration that is only present when the (optional)
144
+ # powerdns job is present and configured and points to a valid DB.
145
+ # This is an attempt to make sure the constants are loaded
146
+ # before forking off to other threads, hopefully eliminating the errors.
147
+ Bosh::Director::Models::Dns::Record.class
148
+ Bosh::Director::Models::Dns::Domain.class
149
+ end
140
150
  end
141
151
 
152
+ @dns_manager = DnsManager.create
142
153
  @uuid = override_uuid || Bosh::Director::Models::DirectorAttribute.find_or_create_uuid(@logger)
143
154
  @logger.info("Director UUID: #{@uuid}")
144
155
 
@@ -150,6 +161,8 @@ module Bosh::Director
150
161
  @trusted_certs = config['trusted_certs'] || ''
151
162
  @ignore_missing_gateway = config['ignore_missing_gateway']
152
163
 
164
+ @keep_unreachable_vms = config.fetch('keep_unreachable_vms', false)
165
+
153
166
  Bosh::Clouds::Config.configure(self)
154
167
 
155
168
  @lock = Monitor.new
@@ -281,10 +294,6 @@ module Bosh::Director
281
294
  !threaded[:redis].nil?
282
295
  end
283
296
 
284
- def dns_enabled?
285
- !@dns_db.nil?
286
- end
287
-
288
297
  def encryption?
289
298
  @encryption
290
299
  end
@@ -0,0 +1,69 @@
1
+ module Bosh::Director
2
+ class DeploymentDeleter
3
+ def initialize(event_log, logger, dns_manager, max_threads)
4
+ @event_log = event_log
5
+ @logger = logger
6
+ @dns_manager = dns_manager
7
+ @max_threads = max_threads
8
+ end
9
+
10
+ def delete(deployment_model, instance_deleter, vm_deleter)
11
+ instance_plans = deployment_model.instances.map do |instance_model|
12
+ DeploymentPlan::InstancePlan.new(
13
+ existing_instance: instance_model,
14
+ instance: nil,
15
+ desired_instance: nil,
16
+ network_plans: []
17
+ )
18
+ end
19
+ event_log_stage = @event_log.begin_stage('Deleting instances', instance_plans.size)
20
+ instance_deleter.delete_instance_plans(instance_plans, event_log_stage, max_threads: @max_threads)
21
+
22
+ # For backwards compatibility for VMs that did not have instances
23
+ deployment_model.reload
24
+ delete_vms(vm_deleter, deployment_model.vms)
25
+
26
+ @event_log.begin_stage('Removing deployment artifacts', 3)
27
+
28
+ @event_log.track('Detaching stemcells') do
29
+ @logger.info('Detaching stemcells')
30
+ deployment_model.remove_all_stemcells
31
+ end
32
+
33
+ @event_log.track('Detaching releases') do
34
+ @logger.info('Detaching releases')
35
+ deployment_model.remove_all_release_versions
36
+ end
37
+
38
+ @event_log.begin_stage('Deleting properties', deployment_model.properties.count)
39
+ @logger.info('Deleting deployment properties')
40
+ deployment_model.properties.each do |property|
41
+ @event_log.track(property.name) do
42
+ property.destroy
43
+ end
44
+ end
45
+
46
+ @event_log.track('Destroying deployment') do
47
+ @logger.info('Destroying deployment')
48
+ deployment_model.destroy
49
+ end
50
+ end
51
+
52
+ private
53
+
54
+ def delete_vms(vm_deleter, vms)
55
+ ThreadPool.new(max_threads: @max_threads).wrap do |pool|
56
+ @event_log.begin_stage('Deleting idle VMs', vms.count)
57
+
58
+ vms.each do |vm|
59
+ pool.process do
60
+ @event_log.track("#{vm.cid}") do
61
+ @logger.info("Deleting idle vm #{vm.cid}")
62
+ vm_deleter.delete_vm(vm)
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
@@ -3,18 +3,26 @@ module Bosh::Director
3
3
  end
4
4
  end
5
5
 
6
+ require 'bosh/director/deployment_plan/availability_zone'
6
7
  require 'bosh/director/deployment_plan/compilation_config'
7
8
  require 'bosh/director/deployment_plan/vm'
9
+ require 'bosh/director/deployment_plan/desired_instance'
10
+ require 'bosh/director/deployment_plan/deployment_validator'
8
11
  require 'bosh/director/deployment_plan/instance'
12
+ require 'bosh/director/deployment_plan/instance_plan'
13
+ require 'bosh/director/deployment_plan/instance_planner'
9
14
  require 'bosh/director/deployment_plan/job'
10
15
  require 'bosh/director/deployment_plan/network'
11
- require 'bosh/director/deployment_plan/network_subnet'
16
+ require 'bosh/director/deployment_plan/manual_network_subnet'
17
+ require 'bosh/director/deployment_plan/dynamic_network_subnet'
12
18
  require 'bosh/director/deployment_plan/compiled_package'
13
- require 'bosh/director/deployment_plan/resource_pools'
14
- require 'bosh/director/deployment_plan/instance_vm_binder'
19
+ require 'bosh/director/deployment_plan/ip_provider/ip_provider'
20
+ require 'bosh/director/deployment_plan/ip_provider/ip_provider_factory'
15
21
  require 'bosh/director/deployment_plan/multi_job_updater'
16
22
  require 'bosh/director/deployment_plan/release_version'
17
23
  require 'bosh/director/deployment_plan/resource_pool'
24
+ require 'bosh/director/deployment_plan/env'
25
+ require 'bosh/director/deployment_plan/vm_type'
18
26
  require 'bosh/director/deployment_plan/stemcell'
19
27
  require 'bosh/director/deployment_plan/template'
20
28
  require 'bosh/director/deployment_plan/update_config'
@@ -22,12 +30,35 @@ require 'bosh/director/deployment_plan/dynamic_network'
22
30
  require 'bosh/director/deployment_plan/manual_network'
23
31
  require 'bosh/director/deployment_plan/vip_network'
24
32
  require 'bosh/director/deployment_plan/planner'
25
- require 'bosh/director/deployment_plan/dns_binder'
33
+ require 'bosh/director/deployment_plan/package_validator'
26
34
  require 'bosh/director/deployment_plan/notifier'
35
+ require 'bosh/director/deployment_plan/compilation_instance_pool'
27
36
  require 'bosh/director/deployment_plan/steps/update_step'
28
37
  require 'bosh/director/deployment_plan/steps/package_compile_step'
29
38
  require 'bosh/director/deployment_plan/assembler'
39
+ require 'bosh/director/deployment_plan/agent_state_migrator'
30
40
  require 'bosh/director/deployment_plan/planner_factory'
31
41
  require 'bosh/director/deployment_plan/manifest_migrator'
42
+ require 'bosh/director/deployment_plan/manifest_validator'
32
43
  require 'bosh/director/deployment_plan/deployment_repo'
44
+ require 'bosh/director/deployment_plan/global_network_resolver'
45
+ require 'bosh/director/deployment_plan/links/link'
46
+ require 'bosh/director/deployment_plan/links/link_path'
47
+ require 'bosh/director/deployment_plan/links/links_resolver'
48
+ require 'bosh/director/deployment_plan/links/link_lookup'
49
+ require 'bosh/director/deployment_plan/links/template_link'
50
+ require 'bosh/director/deployment_plan/instance_network_reservations'
33
51
  require 'bosh/director/deployment_plan/options/skip_drain'
52
+ require 'bosh/director/deployment_plan/ip_provider/in_memory_ip_repo'
53
+ require 'bosh/director/deployment_plan/ip_provider/database_ip_repo'
54
+ require 'bosh/director/deployment_plan/network_settings'
55
+ require 'bosh/director/deployment_plan/job_migrator'
56
+ require 'bosh/director/deployment_plan/instance_plan_sorter'
57
+ require 'bosh/director/deployment_plan/instance_plan_factory'
58
+ require 'bosh/director/deployment_plan/instance_repository'
59
+ require 'bosh/director/deployment_plan/job_network_parser'
60
+ require 'bosh/director/deployment_plan/job_availability_zone_parser'
61
+ require 'bosh/director/deployment_plan/placement_planner'
62
+ require 'bosh/director/deployment_plan/network_planner'
63
+ require 'bosh/director/deployment_plan/instance_spec.rb'
64
+ require 'bosh/director/dns/powerdns'
@@ -0,0 +1,47 @@
1
+ module Bosh::Director
2
+ module DeploymentPlan
3
+ class AgentStateMigrator
4
+ def initialize(deployment_plan, logger)
5
+ @deployment_plan = deployment_plan
6
+ @logger = logger
7
+ end
8
+
9
+ def get_state(vm_model)
10
+ @logger.debug("Requesting current VM state for: #{vm_model.agent_id}")
11
+ agent = AgentClient.with_vm(vm_model)
12
+ state = agent.get_state
13
+
14
+ @logger.debug("Received VM state: #{state.pretty_inspect}")
15
+ verify_state(vm_model, state)
16
+ @logger.debug('Verified VM state')
17
+
18
+ state.delete('release')
19
+ if state.include?('job')
20
+ state['job'].delete('release')
21
+ end
22
+ state
23
+ end
24
+
25
+ def verify_state(vm_model, state)
26
+ instance = vm_model.instance
27
+
28
+ unless state.kind_of?(Hash)
29
+ @logger.error("Invalid state for `#{vm_model.cid}': #{state.pretty_inspect}")
30
+ raise AgentInvalidStateFormat,
31
+ "VM `#{vm_model.cid}' returns invalid state: " +
32
+ "expected Hash, got #{state.class}"
33
+ end
34
+
35
+ actual_job = state['job'].is_a?(Hash) ? state['job']['name'] : nil
36
+ actual_index = state['index']
37
+
38
+ if instance.nil? && !actual_job.nil?
39
+ raise AgentUnexpectedJob,
40
+ "VM `#{vm_model.cid}' is out of sync: " +
41
+ "it reports itself as `#{actual_job}/#{actual_index}' but " +
42
+ 'there is no instance reference in DB'
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -5,262 +5,95 @@ module Bosh::Director
5
5
  include LockHelper
6
6
  include IpUtil
7
7
 
8
- def initialize(deployment_plan, stemcell_manager, cloud, blobstore, logger, event_log)
8
+ def initialize(deployment_plan, stemcell_manager, dns_manager, cloud, logger)
9
9
  @deployment_plan = deployment_plan
10
10
  @cloud = cloud
11
11
  @logger = logger
12
- @event_log = event_log
13
12
  @stemcell_manager = stemcell_manager
14
- @blobstore = blobstore
13
+ @dns_manager = dns_manager
15
14
  end
16
15
 
17
- # Binds release DB record(s) to a plan
18
- # @return [void]
19
- def bind_releases
20
- releases = @deployment_plan.releases
21
- with_release_locks(releases.map(&:name)) do
22
- releases.each do |release|
23
- release.bind_model
24
- end
25
- end
26
- end
27
-
28
- # Binds information about existing deployment to a plan
29
- # @return [void]
30
- def bind_existing_deployment
31
- lock = Mutex.new
32
- ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
33
- @deployment_plan.vms.each do |vm_model|
34
- pool.process do
35
- with_thread_name("bind_existing_deployment(#{vm_model.agent_id})") do
36
- bind_existing_vm(vm_model, lock)
37
- end
38
- end
39
- end
40
- end
41
- end
42
-
43
- # Queries agent for VM state and updates deployment plan accordingly
44
- # @param [Models::Vm] vm_model VM database model
45
- # @param [Mutex] lock Lock to hold on to while updating deployment plan
46
- def bind_existing_vm(vm_model, lock)
47
- state = get_state(vm_model)
48
- lock.synchronize do
49
- @logger.debug('Processing VM network reservations')
50
- reservations = get_network_reservations(state)
51
-
52
- instance = vm_model.instance
53
- if instance
54
- bind_instance(instance, state, reservations)
55
- else
56
- resource_pool_name = state['resource_pool']['name']
57
- resource_pool = @deployment_plan.resource_pool(resource_pool_name)
58
- if resource_pool
59
- @logger.debug("Binding VM to resource pool '#{resource_pool_name}'")
60
- bind_idle_vm(vm_model, resource_pool, state, reservations)
61
- else
62
- @logger.debug("Resource pool '#{resource_pool_name}' does not exist, marking VM for deletion")
63
- @deployment_plan.delete_vm(vm_model)
64
- end
65
- end
66
- @logger.debug('Finished processing VM network reservations')
67
- end
68
- end
69
-
70
- # Binds idle VM to a resource pool with a proper network reservation
71
- # @param [Models::Vm] vm_model VM DB model
72
- # @param [DeploymentPlan::ResourcePool] resource_pool Resource pool
73
- # @param [Hash] state VM state according to its agent
74
- # @param [Hash] reservations Network reservations
75
- def bind_idle_vm(vm_model, resource_pool, state, reservations)
76
- if reservations.any? { |network_name, reservation| reservation.static? }
77
- @logger.debug("Releasing all network reservations for VM `#{vm_model.cid}'")
78
- reservations.each do |network_name, reservation|
79
- @logger.debug("Releasing #{reservation.type} network reservation `#{network_name}' for VM `#{vm_model.cid}'")
80
- @deployment_plan.network(network_name).release(reservation)
81
- end
16
+ def bind_models
17
+ @logger.info('Binding models')
18
+ bind_releases
82
19
 
83
- @logger.debug("Deleting VM `#{vm_model.cid}' with static network reservation")
84
- @deployment_plan.delete_vm(vm_model)
85
- return
86
- end
20
+ migrate_legacy_dns_records
21
+ bind_job_renames
87
22
 
88
- @logger.debug("Adding VM `#{vm_model.cid}' to resource pool `#{resource_pool.name}'")
89
- idle_vm = resource_pool.add_idle_vm
90
- idle_vm.model = vm_model
91
- idle_vm.current_state = state
23
+ instance_repo = Bosh::Director::DeploymentPlan::InstanceRepository.new(@logger)
24
+ states_by_existing_instance = current_states_by_instance(@deployment_plan.candidate_existing_instances)
25
+ index_assigner = Bosh::Director::DeploymentPlan::PlacementPlanner::IndexAssigner.new(@deployment_plan.model)
26
+ instance_plan_factory = Bosh::Director::DeploymentPlan::InstancePlanFactory.new(instance_repo, states_by_existing_instance, @deployment_plan.skip_drain, index_assigner, {'recreate' => @deployment_plan.recreate})
27
+ instance_planner = Bosh::Director::DeploymentPlan::InstancePlanner.new(instance_plan_factory, @logger)
28
+ desired_jobs = @deployment_plan.jobs
92
29
 
93
- network_name = resource_pool.network.name
94
- reservation = reservations[network_name]
95
- if reservation
96
- @logger.debug("Using existing `#{reservation.type}' " +
97
- "network reservation of `#{reservation.ip}' for VM `#{vm_model.cid}'")
98
- idle_vm.use_reservation(reservation)
99
- else
100
- @logger.debug("No network reservation for VM `#{vm_model.cid}'")
101
- end
102
- end
103
-
104
- # @param [Models::Instance] instance_model Instance model
105
- # @param [Hash] state Instance state according to agent
106
- # @param [Hash] reservations Instance network reservations
107
- def bind_instance(instance_model, state, reservations)
108
- @logger.debug('Binding instance VM')
109
-
110
- # Update instance, if we are renaming a job.
111
- if @deployment_plan.rename_in_progress?
112
- old_name = @deployment_plan.job_rename['old_name']
113
- new_name = @deployment_plan.job_rename['new_name']
30
+ job_migrator = Bosh::Director::DeploymentPlan::JobMigrator.new(@deployment_plan, @logger)
114
31
 
115
- if instance_model.job == old_name
116
- @logger.info("Renaming `#{old_name}' to `#{new_name}'")
117
- instance_model.update(:job => new_name)
118
- end
32
+ desired_jobs.each do |desired_job|
33
+ desired_instances = desired_job.desired_instances
34
+ existing_instances = job_migrator.find_existing_instances(desired_job)
35
+ instance_plans = instance_planner.plan_job_instances(desired_job, desired_instances, existing_instances)
36
+ desired_job.add_instance_plans(instance_plans)
119
37
  end
120
38
 
121
- instance_name = "#{instance_model.job}/#{instance_model.index}"
39
+ instance_plans_for_obsolete_jobs = instance_planner.plan_obsolete_jobs(desired_jobs, @deployment_plan.existing_instances)
40
+ instance_plans_for_obsolete_jobs.map(&:existing_instance).each { |existing_instance| @deployment_plan.mark_instance_for_deletion(existing_instance) }
122
41
 
123
- job = @deployment_plan.job(instance_model.job)
124
- unless job
125
- @logger.debug("Job `#{instance_model.job}' not found, marking for deletion")
126
- @deployment_plan.delete_instance(instance_model)
127
- return
128
- end
129
-
130
- instance = job.instance(instance_model.index)
131
- unless instance
132
- @logger.debug("Job instance `#{instance_name}' not found, marking for deletion")
133
- @deployment_plan.delete_instance(instance_model)
134
- return
135
- end
42
+ mark_unknown_vms_for_deletion
136
43
 
137
- @logger.debug("Found existing job instance `#{instance_name}'")
138
- instance.bind_existing_instance(instance_model, state, reservations)
139
- end
140
-
141
- def get_network_reservations(state)
142
- reservations = {}
143
- state['networks'].each do |name, network_config|
144
- network = @deployment_plan.network(name)
145
- if network
146
- reservation = NetworkReservation.new(:ip => network_config['ip'])
147
- network.reserve(reservation)
148
- reservations[name] = reservation if reservation.reserved?
149
- end
150
- end
151
- reservations
152
- end
153
-
154
- def get_state(vm_model)
155
- @logger.debug("Requesting current VM state for: #{vm_model.agent_id}")
156
- agent = AgentClient.with_defaults(vm_model.agent_id)
157
- state = agent.get_state
158
-
159
- @logger.debug("Received VM state: #{state.pretty_inspect}")
160
- verify_state(vm_model, state)
161
- @logger.debug('Verified VM state')
162
-
163
- migrate_legacy_state(vm_model, state)
164
- state.delete('release')
165
- if state.include?('job')
166
- state['job'].delete('release')
167
- end
168
- state
44
+ bind_stemcells
45
+ bind_templates
46
+ bind_properties
47
+ bind_unallocated_vms
48
+ bind_instance_networks
49
+ bind_dns
50
+ bind_links
169
51
  end
170
52
 
171
- def verify_state(vm_model, state)
172
- instance = vm_model.instance
53
+ private
173
54
 
174
- if instance && instance.deployment_id != vm_model.deployment_id
175
- # Both VM and instance should reference same deployment
176
- raise VmInstanceOutOfSync,
177
- "VM `#{vm_model.cid}' and instance " +
178
- "`#{instance.job}/#{instance.index}' " +
179
- "don't belong to the same deployment"
180
- end
181
-
182
- unless state.kind_of?(Hash)
183
- @logger.error("Invalid state for `#{vm_model.cid}': #{state.pretty_inspect}")
184
- raise AgentInvalidStateFormat,
185
- "VM `#{vm_model.cid}' returns invalid state: " +
186
- "expected Hash, got #{state.class}"
187
- end
188
-
189
- actual_deployment_name = state['deployment']
190
- expected_deployment_name = @deployment_plan.name
191
-
192
- if actual_deployment_name != expected_deployment_name
193
- raise AgentWrongDeployment,
194
- "VM `#{vm_model.cid}' is out of sync: " +
195
- 'expected to be a part of deployment ' +
196
- "`#{expected_deployment_name}' " +
197
- 'but is actually a part of deployment ' +
198
- "`#{actual_deployment_name}'"
199
- end
200
-
201
- actual_job = state['job'].is_a?(Hash) ? state['job']['name'] : nil
202
- actual_index = state['index']
203
-
204
- if instance.nil? && !actual_job.nil?
205
- raise AgentUnexpectedJob,
206
- "VM `#{vm_model.cid}' is out of sync: " +
207
- "it reports itself as `#{actual_job}/#{actual_index}' but " +
208
- 'there is no instance reference in DB'
209
- end
210
-
211
- if instance &&
212
- (instance.job != actual_job || instance.index != actual_index)
213
- # Check if we are resuming a previously unfinished rename
214
- if actual_job == @deployment_plan.job_rename['old_name'] &&
215
- instance.job == @deployment_plan.job_rename['new_name'] &&
216
- instance.index == actual_index
217
-
218
- # Rename already happened in the DB but then something happened
219
- # and agent has never been updated.
220
- unless @deployment_plan.job_rename['force']
221
- raise AgentRenameInProgress,
222
- "Found a job `#{actual_job}' that seems to be " +
223
- "in the middle of a rename to `#{instance.job}'. " +
224
- "Run 'rename' again with '--force' to proceed."
225
- end
226
- else
227
- raise AgentJobMismatch,
228
- "VM `#{vm_model.cid}' is out of sync: " +
229
- "it reports itself as `#{actual_job}/#{actual_index}' but " +
230
- "according to DB it is `#{instance.job}/#{instance.index}'"
55
+ # Binds release DB record(s) to a plan
56
+ # @return [void]
57
+ def bind_releases
58
+ releases = @deployment_plan.releases
59
+ with_release_locks(releases.map(&:name)) do
60
+ releases.each do |release|
61
+ release.bind_model
231
62
  end
232
63
  end
233
64
  end
234
65
 
235
- def migrate_legacy_state(vm_model, state)
236
- # Persisting apply spec for VMs that were introduced before we started
237
- # persisting it on apply itself (this is for cloudcheck purposes only)
238
- if vm_model.apply_spec.nil?
239
- # The assumption is that apply_spec <=> VM state
240
- vm_model.update(:apply_spec => state)
241
- end
242
-
243
- instance = vm_model.instance
244
- if instance
245
- disk_size = state['persistent_disk'].to_i
246
- persistent_disk = instance.persistent_disk
247
-
248
- # This is to support legacy deployments where we did not have
249
- # the disk_size specified.
250
- if disk_size != 0 && persistent_disk && persistent_disk.size == 0
251
- persistent_disk.update(:size => disk_size)
66
+ def current_states_by_instance(existing_instances)
67
+ lock = Mutex.new
68
+ current_states_by_existing_instance = {}
69
+ ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
70
+ existing_instances.each do |existing_instance|
71
+ if existing_instance.vm
72
+ pool.process do
73
+ with_thread_name("binding agent state for (#{existing_instance.job}/#{existing_instance.index})") do
74
+ # getting current state to obtain IP of dynamic networks
75
+ state = DeploymentPlan::AgentStateMigrator.new(@deployment_plan, @logger).get_state(existing_instance.vm)
76
+ lock.synchronize do
77
+ current_states_by_existing_instance.merge!(existing_instance => state)
78
+ end
79
+ end
80
+ end
81
+ end
252
82
  end
253
83
  end
84
+ current_states_by_existing_instance
254
85
  end
255
86
 
256
- # Takes a look at the current state of all resource pools in the deployment
257
- # and schedules adding any new VMs if needed. VMs are NOT created at this
258
- # stage, only data structures are being allocated. {ResourcePoolUpdater}
259
- # will later perform actual changes based on this data.
260
- # @return [void]
261
- def bind_resource_pools
262
- @deployment_plan.resource_pools.each do |resource_pool|
263
- resource_pool.process_idle_vms
87
+ def mark_unknown_vms_for_deletion
88
+ @deployment_plan.vm_models.select { |vm| vm.instance.nil? }.each do |vm_model|
89
+ # VM without an instance should not exist any more. But we still
90
+ # delete those VMs for backwards compatibility in case if it was ever
91
+ # created incorrectly.
92
+ # It also means that it was created before global networking
93
+ # and should not have any network reservations in DB,
94
+ # so we don't worry about releasing its IPs.
95
+ @logger.debug('Marking VM for deletion')
96
+ @deployment_plan.mark_vm_for_deletion(vm_model)
264
97
  end
265
98
  end
266
99
 
@@ -273,7 +106,18 @@ module Bosh::Director
273
106
  end
274
107
 
275
108
  def bind_instance_networks
276
- @deployment_plan.jobs_starting_on_deploy.each(&:bind_instance_networks)
109
+ # CHANGEME: something about instance plan's new network plans
110
+ @deployment_plan.jobs_starting_on_deploy.each do |job|
111
+ job.bind_instance_networks(@deployment_plan.ip_provider)
112
+ end
113
+ end
114
+
115
+ def bind_links
116
+ links_resolver = Bosh::Director::DeploymentPlan::LinksResolver.new(@deployment_plan, @logger)
117
+
118
+ @deployment_plan.jobs.each do |job|
119
+ links_resolver.resolve(job)
120
+ end
277
121
  end
278
122
 
279
123
  # Binds template models for each release spec in the deployment plan
@@ -296,25 +140,55 @@ module Bosh::Director
296
140
  end
297
141
  end
298
142
 
299
- # Binds stemcell model for each stemcell spec in each resource pool in
143
+ # Binds stemcell model for each stemcell spec in
300
144
  # the deployment plan
301
145
  # @return [void]
302
146
  def bind_stemcells
303
- @deployment_plan.resource_pools.each do |resource_pool|
304
- stemcell = resource_pool.stemcell
147
+ if @deployment_plan.resource_pools && @deployment_plan.resource_pools.any?
148
+ @deployment_plan.resource_pools.each do |resource_pool|
149
+ stemcell = resource_pool.stemcell
150
+
151
+ if stemcell.nil?
152
+ raise DirectorError,
153
+ "Stemcell not bound for resource pool `#{resource_pool.name}'"
154
+ end
305
155
 
306
- if stemcell.nil?
307
- raise DirectorError,
308
- "Stemcell not bound for resource pool `#{resource_pool.name}'"
156
+ stemcell.bind_model(@deployment_plan)
309
157
  end
158
+ return
159
+ end
310
160
 
311
- stemcell.bind_model
161
+ @deployment_plan.stemcells.each do |_, stemcell|
162
+ stemcell.bind_model(@deployment_plan)
312
163
  end
313
164
  end
314
165
 
315
166
  def bind_dns
316
- binder = DeploymentPlan::DnsBinder.new(@deployment_plan)
317
- binder.bind_deployment
167
+ @dns_manager.configure_nameserver
168
+ end
169
+
170
+ def bind_job_renames
171
+ @deployment_plan.instance_models.each do |instance_model|
172
+ update_instance_if_rename(instance_model)
173
+ end
174
+ end
175
+
176
+ def migrate_legacy_dns_records
177
+ @deployment_plan.instance_models.each do |instance_model|
178
+ @dns_manager.migrate_legacy_records(instance_model)
179
+ end
180
+ end
181
+
182
+ def update_instance_if_rename(instance_model)
183
+ if @deployment_plan.rename_in_progress?
184
+ old_name = @deployment_plan.job_rename['old_name']
185
+ new_name = @deployment_plan.job_rename['new_name']
186
+
187
+ if instance_model.job == old_name
188
+ @logger.info("Renaming `#{old_name}' to `#{new_name}'")
189
+ instance_model.update(:job => new_name)
190
+ end
191
+ end
318
192
  end
319
193
  end
320
194
  end