bosh-director 1.3160.0 → 1.3163.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (195) hide show
  1. checksums.yaml +4 -4
  2. data/bin/bosh-director-worker +2 -14
  3. data/db/migrations/director/20150513225143_ip_addresses.rb +11 -0
  4. data/db/migrations/director/20150702004608_add_links.rb +8 -0
  5. data/db/migrations/director/20150708231924_add_link_spec.rb +7 -0
  6. data/db/migrations/director/20150724183256_add_debugging_to_ip_addresses.rb +8 -0
  7. data/db/migrations/director/20150730225029_add_uuid_to_instances.rb +16 -0
  8. data/db/migrations/director/20150803215805_add_availabililty_zone_and_cloud_properties_to_instances.rb +8 -0
  9. data/db/migrations/director/20150804211419_add_compilation_flag_to_instance.rb +7 -0
  10. data/db/migrations/director/20150918003455_add_bootstrap_node_to_instance.rb +7 -0
  11. data/db/migrations/director/20151008232214_add_dns_records.rb +7 -0
  12. data/db/migrations/director/20151015172551_add_orphan_disks_and_snapshots.rb +29 -0
  13. data/db/migrations/director/20151030222853_add_templates_to_instance.rb +10 -0
  14. data/db/migrations/director/20151031001039_add_spec_to_instance.rb +19 -0
  15. data/db/migrations/director/20151109190602_rename_orphan_columns.rb +13 -0
  16. data/lib/bosh/director.rb +19 -9
  17. data/lib/bosh/director/agent_client.rb +0 -17
  18. data/lib/bosh/director/api/cloud_config_manager.rb +7 -5
  19. data/lib/bosh/director/api/controllers/base_controller.rb +3 -2
  20. data/lib/bosh/director/api/controllers/cleanup_controller.rb +15 -0
  21. data/lib/bosh/director/api/controllers/deployments_controller.rb +38 -26
  22. data/lib/bosh/director/api/controllers/disks_controller.rb +20 -0
  23. data/lib/bosh/director/api/controllers/info_controller.rb +2 -2
  24. data/lib/bosh/director/api/controllers/releases_controller.rb +1 -16
  25. data/lib/bosh/director/api/controllers/stemcells_controller.rb +1 -9
  26. data/lib/bosh/director/api/deployment_manager.rb +2 -1
  27. data/lib/bosh/director/api/instance_lookup.rb +17 -0
  28. data/lib/bosh/director/api/instance_manager.rb +20 -10
  29. data/lib/bosh/director/api/release_manager.rb +28 -8
  30. data/lib/bosh/director/api/resurrector_manager.rb +9 -2
  31. data/lib/bosh/director/api/route_configuration.rb +2 -0
  32. data/lib/bosh/director/api/snapshot_manager.rb +9 -5
  33. data/lib/bosh/director/api/stemcell_manager.rb +50 -0
  34. data/lib/bosh/director/app.rb +1 -1
  35. data/lib/bosh/director/cloudcheck_helper.rb +119 -132
  36. data/lib/bosh/director/compile_task.rb +1 -1
  37. data/lib/bosh/director/compile_task_generator.rb +2 -2
  38. data/lib/bosh/director/config.rb +21 -12
  39. data/lib/bosh/director/deployment_deleter.rb +69 -0
  40. data/lib/bosh/director/deployment_plan.rb +35 -4
  41. data/lib/bosh/director/deployment_plan/agent_state_migrator.rb +47 -0
  42. data/lib/bosh/director/deployment_plan/assembler.rb +115 -241
  43. data/lib/bosh/director/deployment_plan/availability_zone.rb +27 -0
  44. data/lib/bosh/director/deployment_plan/cloud_manifest_parser.rb +144 -35
  45. data/lib/bosh/director/deployment_plan/compilation_config.rb +21 -19
  46. data/lib/bosh/director/deployment_plan/compilation_instance_pool.rb +169 -0
  47. data/lib/bosh/director/deployment_plan/deployment_repo.rb +4 -8
  48. data/lib/bosh/director/deployment_plan/deployment_spec_parser.rb +13 -1
  49. data/lib/bosh/director/deployment_plan/deployment_validator.rb +17 -0
  50. data/lib/bosh/director/deployment_plan/desired_instance.rb +15 -0
  51. data/lib/bosh/director/deployment_plan/{disk_pool.rb → disk_type.rb} +14 -19
  52. data/lib/bosh/director/deployment_plan/dynamic_network.rb +105 -53
  53. data/lib/bosh/director/deployment_plan/dynamic_network_subnet.rb +13 -0
  54. data/lib/bosh/director/deployment_plan/env.rb +18 -0
  55. data/lib/bosh/director/deployment_plan/global_network_resolver.rb +77 -0
  56. data/lib/bosh/director/deployment_plan/instance.rb +222 -390
  57. data/lib/bosh/director/deployment_plan/instance_network_reservations.rb +71 -0
  58. data/lib/bosh/director/deployment_plan/instance_plan.rb +336 -0
  59. data/lib/bosh/director/deployment_plan/instance_plan_factory.rb +54 -0
  60. data/lib/bosh/director/deployment_plan/instance_plan_sorter.rb +61 -0
  61. data/lib/bosh/director/deployment_plan/instance_planner.rb +101 -0
  62. data/lib/bosh/director/deployment_plan/instance_repository.rb +36 -0
  63. data/lib/bosh/director/deployment_plan/instance_spec.rb +154 -0
  64. data/lib/bosh/director/deployment_plan/ip_provider/database_ip_repo.rb +136 -0
  65. data/lib/bosh/director/deployment_plan/ip_provider/in_memory_ip_repo.rb +81 -0
  66. data/lib/bosh/director/deployment_plan/ip_provider/ip_provider.rb +153 -0
  67. data/lib/bosh/director/deployment_plan/ip_provider/ip_provider_factory.rb +22 -0
  68. data/lib/bosh/director/deployment_plan/job.rb +116 -53
  69. data/lib/bosh/director/deployment_plan/job_availability_zone_parser.rb +49 -0
  70. data/lib/bosh/director/deployment_plan/job_migrator.rb +90 -0
  71. data/lib/bosh/director/deployment_plan/job_network.rb +42 -0
  72. data/lib/bosh/director/deployment_plan/job_network_parser.rb +118 -0
  73. data/lib/bosh/director/deployment_plan/job_spec_parser.rb +123 -126
  74. data/lib/bosh/director/deployment_plan/links/link.rb +30 -0
  75. data/lib/bosh/director/deployment_plan/links/link_lookup.rb +66 -0
  76. data/lib/bosh/director/deployment_plan/links/link_path.rb +27 -0
  77. data/lib/bosh/director/deployment_plan/links/links_resolver.rb +70 -0
  78. data/lib/bosh/director/deployment_plan/links/template_link.rb +21 -0
  79. data/lib/bosh/director/deployment_plan/manifest_migrator.rb +3 -17
  80. data/lib/bosh/director/deployment_plan/manifest_validator.rb +46 -0
  81. data/lib/bosh/director/deployment_plan/manual_network.rb +70 -97
  82. data/lib/bosh/director/deployment_plan/manual_network_subnet.rb +148 -0
  83. data/lib/bosh/director/deployment_plan/network.rb +50 -39
  84. data/lib/bosh/director/deployment_plan/network_planner.rb +4 -0
  85. data/lib/bosh/director/deployment_plan/network_planner/plan.rb +26 -0
  86. data/lib/bosh/director/deployment_plan/network_planner/planner.rb +21 -0
  87. data/lib/bosh/director/deployment_plan/network_planner/reservation_reconciler.rb +81 -0
  88. data/lib/bosh/director/deployment_plan/network_planner/vip_static_ips_planner.rb +50 -0
  89. data/lib/bosh/director/deployment_plan/network_settings.rb +65 -0
  90. data/lib/bosh/director/deployment_plan/options/skip_drain.rb +7 -0
  91. data/lib/bosh/director/deployment_plan/package_validator.rb +79 -0
  92. data/lib/bosh/director/deployment_plan/placement_planner.rb +8 -0
  93. data/lib/bosh/director/deployment_plan/placement_planner/availability_zone_picker.rb +90 -0
  94. data/lib/bosh/director/deployment_plan/placement_planner/bruteforce_ip_allocation.rb +124 -0
  95. data/lib/bosh/director/deployment_plan/placement_planner/index_assigner.rb +32 -0
  96. data/lib/bosh/director/deployment_plan/placement_planner/networks_to_static_ips.rb +125 -0
  97. data/lib/bosh/director/deployment_plan/placement_planner/placed_desired_instances.rb +40 -0
  98. data/lib/bosh/director/deployment_plan/placement_planner/plan.rb +42 -0
  99. data/lib/bosh/director/deployment_plan/placement_planner/static_ips_availability_zone_picker.rb +237 -0
  100. data/lib/bosh/director/deployment_plan/placement_planner/unplaced_existing_instances.rb +53 -0
  101. data/lib/bosh/director/deployment_plan/planner.rb +186 -74
  102. data/lib/bosh/director/deployment_plan/planner_factory.rb +30 -147
  103. data/lib/bosh/director/deployment_plan/release_version.rb +3 -3
  104. data/lib/bosh/director/deployment_plan/resource_pool.rb +2 -174
  105. data/lib/bosh/director/deployment_plan/stemcell.rb +57 -14
  106. data/lib/bosh/director/deployment_plan/steps/package_compile_step.rb +28 -135
  107. data/lib/bosh/director/deployment_plan/steps/update_step.rb +23 -44
  108. data/lib/bosh/director/deployment_plan/template.rb +15 -4
  109. data/lib/bosh/director/deployment_plan/vip_network.rb +14 -42
  110. data/lib/bosh/director/deployment_plan/vm.rb +1 -99
  111. data/lib/bosh/director/deployment_plan/vm_type.rb +27 -0
  112. data/lib/bosh/director/disk_manager.rb +268 -0
  113. data/lib/bosh/director/dns/canonicalizer.rb +28 -0
  114. data/lib/bosh/director/dns/dns_manager.rb +163 -0
  115. data/lib/bosh/director/dns/local_dns_repo.rb +20 -0
  116. data/lib/bosh/director/dns/powerdns.rb +170 -0
  117. data/lib/bosh/director/errand/job_manager.rb +18 -29
  118. data/lib/bosh/director/error_ignorer.rb +16 -0
  119. data/lib/bosh/director/errors.rb +51 -20
  120. data/lib/bosh/director/event_log.rb +6 -0
  121. data/lib/bosh/director/instance_deleter.rb +53 -81
  122. data/lib/bosh/director/instance_reuser.rb +89 -0
  123. data/lib/bosh/director/instance_updater.rb +139 -281
  124. data/lib/bosh/director/instance_updater/preparer.rb +8 -5
  125. data/lib/bosh/director/instance_updater/state_applier.rb +21 -0
  126. data/lib/bosh/director/ip_util.rb +46 -26
  127. data/lib/bosh/director/job_renderer.rb +22 -10
  128. data/lib/bosh/director/job_runner.rb +1 -4
  129. data/lib/bosh/director/job_updater.rb +47 -35
  130. data/lib/bosh/director/job_updater_factory.rb +5 -4
  131. data/lib/bosh/director/jobs/base_job.rb +8 -0
  132. data/lib/bosh/director/jobs/cleanup_artifacts.rb +93 -0
  133. data/lib/bosh/director/jobs/delete_deployment.rb +10 -154
  134. data/lib/bosh/director/jobs/delete_deployment_snapshots.rb +1 -1
  135. data/lib/bosh/director/jobs/delete_orphan_disks.rb +44 -0
  136. data/lib/bosh/director/jobs/delete_release.rb +19 -196
  137. data/lib/bosh/director/jobs/delete_stemcell.rb +10 -76
  138. data/lib/bosh/director/jobs/export_release.rb +41 -121
  139. data/lib/bosh/director/jobs/fetch_logs.rb +0 -6
  140. data/lib/bosh/director/jobs/helpers.rb +10 -0
  141. data/lib/bosh/director/jobs/helpers/blob_deleter.rb +24 -0
  142. data/lib/bosh/director/jobs/helpers/compiled_package_deleter.rb +24 -0
  143. data/lib/bosh/director/jobs/helpers/name_version_release_deleter.rb +48 -0
  144. data/lib/bosh/director/jobs/helpers/package_deleter.rb +33 -0
  145. data/lib/bosh/director/jobs/helpers/release_deleter.rb +52 -0
  146. data/lib/bosh/director/jobs/helpers/release_version_deleter.rb +115 -0
  147. data/lib/bosh/director/jobs/helpers/releases_to_delete_picker.rb +31 -0
  148. data/lib/bosh/director/jobs/helpers/stemcell_deleter.rb +61 -0
  149. data/lib/bosh/director/jobs/helpers/stemcells_to_delete_picker.rb +30 -0
  150. data/lib/bosh/director/jobs/helpers/template_deleter.rb +20 -0
  151. data/lib/bosh/director/jobs/release/release_job.rb +18 -7
  152. data/lib/bosh/director/jobs/run_errand.rb +57 -36
  153. data/lib/bosh/director/jobs/scheduled_orphan_cleanup.rb +46 -0
  154. data/lib/bosh/director/jobs/ssh.rb +50 -17
  155. data/lib/bosh/director/jobs/update_deployment.rb +29 -11
  156. data/lib/bosh/director/jobs/update_release.rb +25 -4
  157. data/lib/bosh/director/jobs/vm_state.rb +23 -32
  158. data/lib/bosh/director/lock.rb +13 -8
  159. data/lib/bosh/director/logs_fetcher.rb +1 -1
  160. data/lib/bosh/director/models.rb +3 -0
  161. data/lib/bosh/director/models/compiled_package.rb +3 -3
  162. data/lib/bosh/director/models/deployment.rb +10 -0
  163. data/lib/bosh/director/models/instance.rb +77 -1
  164. data/lib/bosh/director/models/ip_address.rb +26 -0
  165. data/lib/bosh/director/models/orphan_disk.rb +23 -0
  166. data/lib/bosh/director/models/orphan_snapshot.rb +14 -0
  167. data/lib/bosh/director/models/template.rb +32 -9
  168. data/lib/bosh/director/models/vm.rb +5 -8
  169. data/lib/bosh/director/network_reservation.rb +69 -99
  170. data/lib/bosh/director/problem_handlers/inactive_disk.rb +5 -20
  171. data/lib/bosh/director/problem_handlers/missing_disk.rb +2 -13
  172. data/lib/bosh/director/problem_resolver.rb +2 -2
  173. data/lib/bosh/director/problem_scanner/vm_scan_stage.rb +2 -21
  174. data/lib/bosh/director/scheduler.rb +23 -6
  175. data/lib/bosh/director/{instance_updater/stopper.rb → stopper.rb} +24 -18
  176. data/lib/bosh/director/tagged_logger.rb +30 -0
  177. data/lib/bosh/director/transactor.rb +9 -0
  178. data/lib/bosh/director/version.rb +1 -1
  179. data/lib/bosh/director/vm_creator.rb +91 -19
  180. data/lib/bosh/director/vm_deleter.rb +25 -0
  181. data/lib/bosh/director/vm_recreator.rb +15 -0
  182. data/lib/cloud/dummy.rb +381 -94
  183. metadata +110 -30
  184. data/lib/bosh/director/deployment_plan/dns_binder.rb +0 -45
  185. data/lib/bosh/director/deployment_plan/instance_vm_binder.rb +0 -37
  186. data/lib/bosh/director/deployment_plan/network_subnet.rb +0 -166
  187. data/lib/bosh/director/deployment_plan/resource_pools.rb +0 -68
  188. data/lib/bosh/director/dns_helper.rb +0 -223
  189. data/lib/bosh/director/instance_updater/network_updater.rb +0 -110
  190. data/lib/bosh/director/instance_updater/vm_updater.rb +0 -189
  191. data/lib/bosh/director/problem_handlers/out_of_sync_vm.rb +0 -64
  192. data/lib/bosh/director/problem_handlers/unbound_instance_vm.rb +0 -85
  193. data/lib/bosh/director/resource_pool_updater.rb +0 -174
  194. data/lib/bosh/director/vm_data.rb +0 -63
  195. data/lib/bosh/director/vm_reuser.rb +0 -63
@@ -0,0 +1,40 @@
1
+ module Bosh
2
+ module Director
3
+ module DeploymentPlan
4
+ module PlacementPlanner
5
+ class PlacedDesiredInstances
6
+ attr_reader :absent, :existing
7
+
8
+ def initialize(azs)
9
+ @placed = {}
10
+ (azs || []).each do |az|
11
+ @placed[az] = []
12
+ end
13
+
14
+ @absent = []
15
+ @existing = []
16
+ end
17
+
18
+ def record_placement(az, desired_instance, existing_instance_model)
19
+ desired_instance.az = az
20
+ @placed[az] = @placed.fetch(az, []) << desired_instance
21
+
22
+ if existing_instance_model
23
+ existing << {
24
+ desired_instance: desired_instance,
25
+ existing_instance_model: existing_instance_model
26
+ }
27
+ else
28
+ absent << desired_instance
29
+ end
30
+ end
31
+
32
+ def azs_with_fewest_instances
33
+ az_with_fewest = @placed.keys.min_by { |az|@placed[az].size }
34
+ @placed.keys.select { |az| (@placed[az].size == @placed[az_with_fewest].size) && az }
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,42 @@
1
+ module Bosh
2
+ module Director
3
+ module DeploymentPlan
4
+ module PlacementPlanner
5
+ class Plan
6
+ def initialize(instance_plan_factory, network_planner, logger)
7
+ @instance_plan_factory = instance_plan_factory
8
+ @network_planner = network_planner
9
+ @logger = logger
10
+ end
11
+
12
+ def create_instance_plans(desired, existing, networks, availability_zones, job_name)
13
+ sorted_existing = existing.sort_by(&:index)
14
+ instance_plans = assign_zones(desired, sorted_existing, networks, availability_zones, job_name)
15
+
16
+ instance_plans.reject(&:obsolete?).each do |instance_plan|
17
+ @logger.debug("Assigning az '#{instance_plan.desired_instance.availability_zone}' to instance '#{instance_plan.instance}'")
18
+ instance_plan.instance.assign_availability_zone(instance_plan.desired_instance.az)
19
+ end
20
+ instance_plans
21
+ end
22
+
23
+ private
24
+
25
+ def assign_zones(desired, existing, networks, availability_zones, job_name)
26
+ if has_static_ips?(networks)
27
+ @logger.debug("Job '#{job_name}' has networks with static IPs, placing instances based on static IP distribution")
28
+ StaticIpsAvailabilityZonePicker.new(@instance_plan_factory, @network_planner, networks, job_name, availability_zones, @logger).place_and_match_in(desired, existing)
29
+ else
30
+ @logger.debug("Job '#{job_name}' does not have networks with static IPs, placing instances based on persistent disk allocation")
31
+ AvailabilityZonePicker.new(@instance_plan_factory, @network_planner, networks, availability_zones).place_and_match_in(desired, existing)
32
+ end
33
+ end
34
+
35
+ def has_static_ips?(networks)
36
+ !networks.nil? && networks.any? { |network| !! network.static_ips }
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,237 @@
1
+ module Bosh
2
+ module Director
3
+ module DeploymentPlan
4
+ module PlacementPlanner
5
+ class StaticIpsAvailabilityZonePicker
6
+ include IpUtil
7
+
8
+ def initialize(instance_plan_factory, network_planner, job_networks, job_name, desired_azs, logger)
9
+ @instance_plan_factory = instance_plan_factory
10
+ @network_planner = network_planner
11
+ @job_networks = job_networks
12
+ @job_name = job_name
13
+ @networks_to_static_ips = NetworksToStaticIps.create(@job_networks, desired_azs, job_name)
14
+ @desired_azs = desired_azs
15
+ @logger = logger
16
+ end
17
+
18
+ def place_and_match_in(desired_instances, existing_instance_models)
19
+ @networks_to_static_ips.validate_azs_are_declared_in_job_and_subnets(@desired_azs)
20
+ @networks_to_static_ips.validate_ips_are_in_desired_azs(@desired_azs)
21
+ desired_instances = desired_instances.dup
22
+
23
+ instance_plans = place_existing_instance_plans(desired_instances, existing_instance_models)
24
+ place_new_instance_plans(desired_instances, instance_plans)
25
+ end
26
+
27
+ private
28
+
29
+ def place_existing_instance_plans(desired_instances, existing_instance_models)
30
+ instance_plans = []
31
+ # create existing instance plans with network plans that use specified static IPs
32
+ existing_instance_models.each do |existing_instance_model|
33
+ instance_plan = create_instance_plan_based_on_existing_ips(desired_instances, existing_instance_model)
34
+ instance_plans << instance_plan if instance_plan
35
+ end
36
+
37
+ # create the rest existing instance plans
38
+ existing_instance_models.each do |existing_instance_model|
39
+ unless already_has_instance_plan?(existing_instance_model, instance_plans)
40
+ instance_plans << create_existing_instance_plan_with_az_validation(desired_instances, instance_plans, existing_instance_model)
41
+ end
42
+ end
43
+
44
+ # fulfill missing network plans
45
+ instance_plans.reject(&:obsolete?).each do |instance_plan|
46
+ @job_networks.each do |network|
47
+ unless network.static?
48
+ instance_plan.network_plans << @network_planner.network_plan_with_dynamic_reservation(instance_plan, network)
49
+ next
50
+ end
51
+
52
+ unless instance_plan.network_plan_for_network(network.deployment_network)
53
+ instance_plan.network_plans << create_network_plan_with_az(instance_plan, network, instance_plans)
54
+ end
55
+ end
56
+ end
57
+
58
+ instance_plans
59
+ end
60
+
61
+ def place_new_instance_plans(desired_instances, instance_plans)
62
+ @networks_to_static_ips.distribute_evenly_per_zone
63
+
64
+ desired_instances.each do |desired_instance|
65
+ instance_plan = @instance_plan_factory.desired_new_instance_plan(desired_instance)
66
+ @job_networks.each do |network|
67
+ unless network.static?
68
+ instance_plan.network_plans << @network_planner.network_plan_with_dynamic_reservation(instance_plan, network)
69
+ next
70
+ end
71
+
72
+ instance_plan.network_plans << create_network_plan_with_az(instance_plan, network, instance_plans)
73
+ end
74
+
75
+ instance_plans << instance_plan
76
+ end
77
+
78
+ instance_plans
79
+ end
80
+
81
+ def create_network_plan_with_az(instance_plan, network, instance_plans)
82
+ desired_instance = instance_plan.desired_instance
83
+ instance = instance_plan.instance
84
+ if desired_instance.az.nil?
85
+ static_ip_to_azs = @networks_to_static_ips.next_ip_for_network(network)
86
+ if static_ip_to_azs.az_names.size == 1
87
+ az_name = static_ip_to_azs.az_names.first
88
+ @logger.debug("Assigning az '#{az_name}' to instance '#{instance}'")
89
+ else
90
+ az_name = find_az_name_with_least_number_of_instances(static_ip_to_azs.az_names, instance_plans)
91
+ @logger.debug("Assigning az '#{az_name}' to instance '#{instance}' based on least number of instances")
92
+ end
93
+ desired_instance.az = to_az(az_name)
94
+ else
95
+ static_ip_to_azs = @networks_to_static_ips.find_by_network_and_az(network, desired_instance.availability_zone)
96
+ end
97
+ if static_ip_to_azs.nil?
98
+ raise Bosh::Director::NetworkReservationError,
99
+ 'Failed to distribute static IPs to satisfy existing instance reservations'
100
+ end
101
+
102
+ @logger.debug("Claiming IP '#{format_ip(static_ip_to_azs.ip)}' on network #{network.name} and az '#{desired_instance.availability_zone}' for instance '#{instance}'")
103
+ @networks_to_static_ips.claim_in_az(static_ip_to_azs.ip, desired_instance.availability_zone)
104
+
105
+ @network_planner.network_plan_with_static_reservation(instance_plan, network, static_ip_to_azs.ip)
106
+ end
107
+
108
+ def create_instance_plan_based_on_existing_ips(desired_instances, existing_instance_model)
109
+ instance_plan = nil
110
+ @job_networks.each do |network|
111
+ next unless network.static?
112
+ instance_ips_on_network = find_instance_ips_on_network(existing_instance_model, network)
113
+ network_plan = nil
114
+
115
+ instance_ips_on_network.each do |instance_ip|
116
+ ip_address = instance_ip.address
117
+ # Instance is using IP in static IPs list, we have to use this instance
118
+ @logger.debug("Existing instance '#{instance_name(existing_instance_model)}' is using static IP '#{format_ip(ip_address)}' on network '#{network.name}'")
119
+ if instance_plan.nil?
120
+ desired_instance = desired_instances.shift
121
+ instance_plan = create_existing_instance_plan_with_az(desired_instance, existing_instance_model, network, ip_address)
122
+ instance_plan
123
+ end
124
+
125
+ if network_plan.nil? && instance_plan.desired_instance
126
+ create_network_plan_with_ip(instance_plan, network, ip_address)
127
+ end
128
+
129
+ if instance_plan.desired_instance.nil?
130
+ # delete so that other instances not reusing ips of existing instance
131
+ # obsolete instances should not affect distribution
132
+ @networks_to_static_ips.delete(ip_address)
133
+ else
134
+ # put ip in az where existing instance is so that
135
+ # during distribution it will be taken into account
136
+ @networks_to_static_ips.claim_in_az(ip_address, instance_plan.desired_instance.availability_zone)
137
+ end
138
+ end
139
+ end
140
+
141
+ instance_plan
142
+ end
143
+
144
+ def find_instance_ips_on_network(existing_instance_model, network)
145
+ existing_instance_model.ip_addresses.select { |ip_address| network.static_ips.include?(ip_address.address) }
146
+ end
147
+
148
+ def already_has_instance_plan?(existing_instance_model, instance_plans)
149
+ instance_plans.map(&:existing_instance).include?(existing_instance_model)
150
+ end
151
+
152
+ def create_existing_instance_plan_with_az(desired_instance, existing_instance_model, network, ip_address)
153
+ instance_plan = create_existing_instance_plan(desired_instance, existing_instance_model)
154
+ unless instance_plan.obsolete?
155
+ assign_az_based_on_ip(desired_instance, existing_instance_model, network, ip_address)
156
+ end
157
+ instance_plan
158
+ end
159
+
160
+ def assign_az_based_on_ip(desired_instance, existing_instance_model, network, ip_address)
161
+ ip_az_names = @networks_to_static_ips.find_by_network_and_ip(network, ip_address).az_names
162
+ if ip_az_names.include?(existing_instance_model.availability_zone)
163
+ az_name = existing_instance_model.availability_zone
164
+ @logger.debug("Instance '#{instance_name(existing_instance_model)}' belongs to az '#{az_name}' that is in subnet az list, reusing instance az.")
165
+ else
166
+ raise Bosh::Director::NetworkReservationError,
167
+ "Existing instance '#{instance_name(existing_instance_model)}' is using IP '#{format_ip(ip_address)}' in availability zone '#{existing_instance_model.availability_zone}'"
168
+ end
169
+ desired_instance.az = to_az(az_name)
170
+ end
171
+
172
+ def create_existing_instance_plan_with_az_validation(desired_instances, instance_plans, existing_instance_model)
173
+ if desired_instances.empty?
174
+ @logger.debug("Marking instance '#{instance_name(existing_instance_model)}' as obsolete")
175
+ @instance_plan_factory.obsolete_instance_plan(existing_instance_model)
176
+ else
177
+ # we can only reuse an instance if its AZ contains enough IPs for its networks
178
+
179
+ instance_az_name = existing_instance_model.availability_zone
180
+ @job_networks.each do |network|
181
+ next unless network.static?
182
+ static_ip_to_azs = @networks_to_static_ips.find_by_network_and_az(network, instance_az_name)
183
+ if static_ip_to_azs.nil?
184
+ @logger.debug("Marking instance '#{instance_name(existing_instance_model)}' as obsolete, not enough IPs in instance az")
185
+ return @instance_plan_factory.obsolete_instance_plan(existing_instance_model)
186
+ end
187
+ end
188
+
189
+ # we have enough IPs to fit an instance in its AZ
190
+ @logger.debug("Reusing instance '#{instance_name(existing_instance_model)}' with new IPs")
191
+ desired_instance = desired_instances.shift
192
+ desired_instance.az = to_az(instance_az_name)
193
+ instance_plan = @instance_plan_factory.desired_existing_instance_plan(existing_instance_model, desired_instance)
194
+ @job_networks.each do |network|
195
+ next unless network.static?
196
+ instance_plan.network_plans << create_network_plan_with_az(instance_plan, network, instance_plans)
197
+ end
198
+
199
+ instance_plan
200
+ end
201
+ end
202
+
203
+ def create_existing_instance_plan(desired_instance, existing_instance_model)
204
+ if desired_instance.nil?
205
+ @instance_plan_factory.obsolete_instance_plan(existing_instance_model, desired_instance)
206
+ else
207
+ @instance_plan_factory.desired_existing_instance_plan(existing_instance_model, desired_instance)
208
+ end
209
+ end
210
+
211
+ def create_network_plan_with_ip(instance_plan, network, ip_address)
212
+ instance_az = instance_plan.desired_instance.az
213
+ instance_az_name = instance_az.nil? ? nil : instance_az.name
214
+ ip_az_names = @networks_to_static_ips.find_by_network_and_ip(network, ip_address).az_names
215
+ if ip_az_names.include?(instance_az_name)
216
+ instance_plan.network_plans << @network_planner.network_plan_with_static_reservation(instance_plan, network, ip_address)
217
+ end
218
+ end
219
+
220
+ def find_az_name_with_least_number_of_instances(az_names, instance_plans)
221
+ az_names.sort_by do |az_name|
222
+ instance_plans.select { |instance_plan| instance_plan.desired_instance.availability_zone == az_name }.size
223
+ end.first
224
+ end
225
+
226
+ def to_az(az_name)
227
+ @desired_azs.to_a.find { |az| az.name == az_name }
228
+ end
229
+
230
+ def instance_name(existing_instance_model)
231
+ "#{existing_instance_model.job}/#{existing_instance_model.index}"
232
+ end
233
+ end
234
+ end
235
+ end
236
+ end
237
+ end
@@ -0,0 +1,53 @@
1
+ module Bosh
2
+ module Director
3
+ module DeploymentPlan
4
+ module PlacementPlanner
5
+ class UnplacedExistingInstances
6
+ def initialize(existing_instance_models)
7
+ @instances = existing_instance_models.sort_by { |instance_model| instance_model.index }
8
+ @az_name_to_existing_instances = initialize_azs_to_instances
9
+ end
10
+
11
+ def instances_with_persistent_disk
12
+ @instances.select do |instance_model|
13
+ instance_model.persistent_disks && instance_model.persistent_disks.count > 0
14
+ end
15
+ end
16
+
17
+ def azs_sorted_by_existing_instance_count_descending(azs)
18
+ return nil if azs.nil?
19
+ azs.sort_by { |az| - @az_name_to_existing_instances.fetch(az.name, []).size }
20
+ end
21
+
22
+ def claim_instance(existing_instance)
23
+ @az_name_to_existing_instances[existing_instance.availability_zone].delete(existing_instance)
24
+ end
25
+
26
+ def claim_instance_for_az(az)
27
+ az_name = az.nil? ? nil : az.name
28
+ instances = @az_name_to_existing_instances[az_name]
29
+ unless instances.nil? || instances.empty?
30
+ instances.shift
31
+ end
32
+ end
33
+
34
+ def unclaimed
35
+ @az_name_to_existing_instances.values.flatten
36
+ end
37
+
38
+ private
39
+
40
+ def initialize_azs_to_instances
41
+ az_name_to_existing_instances = {}
42
+ @instances.each do |instance|
43
+ instances = az_name_to_existing_instances.fetch(instance.availability_zone, [])
44
+ instances << instance
45
+ az_name_to_existing_instances[instance.availability_zone] = instances
46
+ end
47
+ az_name_to_existing_instances
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
@@ -1,6 +1,6 @@
1
1
  require 'bosh/director/deployment_plan/deployment_spec_parser'
2
2
  require 'bosh/director/deployment_plan/cloud_manifest_parser'
3
- require 'bosh/director/deployment_plan/disk_pool'
3
+ require 'bosh/director/deployment_plan/disk_type'
4
4
  require 'forwardable'
5
5
  require 'common/deep_copy'
6
6
 
@@ -10,7 +10,6 @@ module Bosh::Director
10
10
  module DeploymentPlan
11
11
  class Planner
12
12
  include LockHelper
13
- include DnsHelper
14
13
  include ValidationHelper
15
14
  extend Forwardable
16
15
 
@@ -25,6 +24,11 @@ module Bosh::Director
25
24
 
26
25
  attr_accessor :properties
27
26
 
27
+ # Hash of resolved links spec provided by deployment
28
+ # in format job_name > template_name > link_name > link_type
29
+ # used by LinksResolver
30
+ attr_accessor :link_spec
31
+
28
32
  # @return [Bosh::Director::DeploymentPlan::UpdateConfig]
29
33
  # Default job update configuration
30
34
  attr_accessor :update
@@ -33,59 +37,134 @@ module Bosh::Director
33
37
  # All jobs in the deployment
34
38
  attr_reader :jobs
35
39
 
40
+ # Stemcells in deployment by alias
41
+ attr_reader :stemcells
42
+
36
43
  # Job instances from the old manifest that are not in the new manifest
37
- attr_accessor :unneeded_instances
44
+ attr_reader :unneeded_instances
38
45
 
39
46
  # VMs from the old manifest that are not in the new manifest
40
47
  attr_accessor :unneeded_vms
41
48
 
42
- attr_accessor :dns_domain
43
-
44
49
  attr_reader :job_rename
45
50
 
46
51
  # @return [Boolean] Indicates whether VMs should be recreated
47
52
  attr_reader :recreate
48
53
 
54
+ attr_writer :cloud_planner
55
+
49
56
  # @return [Boolean] Indicates whether VMs should be drained
50
57
  attr_reader :skip_drain
51
58
 
52
59
  def initialize(attrs, manifest_text, cloud_config, deployment_model, options = {})
60
+ @cloud_config = cloud_config
61
+
53
62
  @name = attrs.fetch(:name)
54
63
  @properties = attrs.fetch(:properties)
55
64
  @releases = {}
56
65
 
57
66
  @manifest_text = Bosh::Common::DeepCopy.copy(manifest_text)
58
67
  @cloud_config = cloud_config
59
- @cloud_planner = CloudPlanner.new(cloud_config)
60
68
  @model = deployment_model
61
69
 
70
+ @stemcells = {}
62
71
  @jobs = []
63
72
  @jobs_name_index = {}
64
73
  @jobs_canonical_name_index = Set.new
65
74
 
66
75
  @unneeded_vms = []
67
76
  @unneeded_instances = []
68
- @dns_domain = nil
69
77
 
70
78
  @job_rename = safe_property(options, 'job_rename',
71
79
  :class => Hash, :default => {})
72
80
 
73
81
  @recreate = !!options['recreate']
82
+
83
+ @link_spec = Hash.new{ |h,k| h[k] = Hash.new(&h.default_proc) }
74
84
  @skip_drain = SkipDrain.new(options['skip_drain'])
85
+
86
+ @logger = Config.logger
75
87
  end
76
88
 
77
- def_delegators :@cloud_planner, :add_network, :networks, :network,
78
- :add_resource_pool, :resource_pools, :resource_pool,
79
- :add_disk_pool, :disk_pools, :disk_pool,
80
- :compilation, :compilation=
89
+ def_delegators :@cloud_planner,
90
+ :networks,
91
+ :network,
92
+ :deleted_network,
93
+ :availability_zone,
94
+ :availability_zones,
95
+ :resource_pools,
96
+ :resource_pool,
97
+ :vm_types,
98
+ :vm_type,
99
+ :add_resource_pool,
100
+ :disk_types,
101
+ :disk_type,
102
+ :compilation,
103
+ :ip_provider
81
104
 
82
105
  def canonical_name
83
- canonical(@name)
106
+ Canonicalizer.canonicalize(@name)
107
+ end
108
+
109
+ def bind_models
110
+ stemcell_manager = Api::StemcellManager.new
111
+ dns_manager = DnsManager.create
112
+ assembler = DeploymentPlan::Assembler.new(
113
+ self,
114
+ stemcell_manager,
115
+ dns_manager,
116
+ Config.cloud,
117
+ @logger
118
+ )
119
+
120
+ assembler.bind_models
121
+ end
122
+
123
+ def compile_packages
124
+ validate_packages
125
+
126
+ cloud = Config.cloud
127
+ vm_deleter = VmDeleter.new(cloud, @logger)
128
+ disk_manager = DiskManager.new(cloud, @logger)
129
+ job_renderer = JobRenderer.create
130
+ vm_creator = Bosh::Director::VmCreator.new(cloud, @logger, vm_deleter, disk_manager, job_renderer)
131
+ dns_manager = DnsManager.create
132
+ instance_deleter = Bosh::Director::InstanceDeleter.new(ip_provider, dns_manager, disk_manager)
133
+ compilation_instance_pool = CompilationInstancePool.new(InstanceReuser.new, vm_creator, self, @logger, instance_deleter)
134
+ package_compile_step = DeploymentPlan::Steps::PackageCompileStep.new(
135
+ jobs,
136
+ compilation,
137
+ compilation_instance_pool,
138
+ @logger,
139
+ Config.event_log,
140
+ nil
141
+ )
142
+ package_compile_step.perform
143
+ end
144
+
145
+ # Returns a list of Instances in the deployment (according to DB)
146
+ # @return [Array<Models::Instance>]
147
+ def instance_models
148
+ @model.instances
149
+ end
150
+
151
+ def existing_instances
152
+ instance_models
153
+ end
154
+
155
+ def candidate_existing_instances
156
+ desired_job_names = jobs.map(&:name)
157
+ migrating_job_names = jobs.map(&:migrated_from).flatten.map(&:name)
158
+
159
+ existing_instances.select do |instance|
160
+ desired_job_names.include?(instance.job) ||
161
+ migrating_job_names.include?(instance.job)
162
+ end
84
163
  end
85
164
 
86
- # Returns a list of VMs in the deployment (according to DB)
165
+ # Returns a list of Vms in the deployment (according to DB)
87
166
  # @return [Array<Models::Vm>]
88
- def vms
167
+ def vm_models
89
168
  @model.vms
90
169
  end
91
170
 
@@ -93,6 +172,14 @@ module Bosh::Director
93
172
  @skip_drain.nil? ? false : @skip_drain.for_job(name)
94
173
  end
95
174
 
175
+ def add_stemcell(stemcell)
176
+ @stemcells[stemcell.alias] = stemcell
177
+ end
178
+
179
+ def stemcell(name)
180
+ @stemcells[name]
181
+ end
182
+
96
183
  # Adds a release by name
97
184
  # @param [Bosh::Director::DeploymentPlan::ReleaseVersion] release
98
185
  def add_release(release)
@@ -117,20 +204,20 @@ module Bosh::Director
117
204
 
118
205
  # Adds a VM to deletion queue
119
206
  # @param [Bosh::Director::Models::Vm] vm VM DB model
120
- def delete_vm(vm)
207
+ def mark_vm_for_deletion(vm)
121
208
  @unneeded_vms << vm
122
209
  end
123
210
 
124
- # Adds instance to deletion queue
125
- # @param [Bosh::Director::Models::Instance] instance Instance DB model
126
- def delete_instance(instance)
127
- if @jobs_name_index.has_key?(instance.job)
128
- @jobs_name_index[instance.job].unneeded_instances << instance
129
- else
130
- @unneeded_instances << instance
211
+ def instance_plans_with_missing_vms
212
+ jobs_starting_on_deploy.collect_concat do |job|
213
+ job.instance_plans_with_missing_vms
131
214
  end
132
215
  end
133
216
 
217
+ def mark_instance_for_deletion(instance)
218
+ @unneeded_instances << instance
219
+ end
220
+
134
221
  # Adds a job by name
135
222
  # @param [Bosh::Director::DeploymentPlan::Job] job
136
223
  def add_job(job)
@@ -157,10 +244,6 @@ module Bosh::Director
157
244
  @jobs_name_index[name]
158
245
  end
159
246
 
160
- def reset_jobs
161
- @jobs = []
162
- end
163
-
164
247
  def jobs_starting_on_deploy
165
248
  @jobs.select(&:starts_on_deploy?)
166
249
  end
@@ -182,96 +265,125 @@ module Bosh::Director
182
265
 
183
266
  model.manifest = Psych.dump(@manifest_text)
184
267
  model.cloud_config = @cloud_config
268
+ model.link_spec = @link_spec
185
269
  model.save
186
270
  end
187
271
 
188
272
  def update_stemcell_references!
189
273
  current_stemcell_models = resource_pools.map { |pool| pool.stemcell.model }
274
+ @stemcells.values.map(&:model).each do |stemcell|
275
+ current_stemcell_models << stemcell
276
+ end
190
277
  model.stemcells.each do |deployment_stemcell|
191
278
  deployment_stemcell.remove_deployment(model) unless current_stemcell_models.include?(deployment_stemcell)
192
279
  end
193
280
  end
281
+
282
+ def using_global_networking?
283
+ !@cloud_config.nil?
284
+ end
285
+
286
+ private
287
+
288
+ def validate_packages
289
+ release_manager = Bosh::Director::Api::ReleaseManager.new
290
+ validator = DeploymentPlan::PackageValidator.new(@logger)
291
+ jobs.each do |job|
292
+ job.templates.each do |template|
293
+ release_model = release_manager.find_by_name(template.release.name)
294
+ release_version_model = release_manager.find_version(release_model, template.release.version)
295
+
296
+ validator.validate(release_version_model, job.stemcell.model)
297
+ end
298
+ end
299
+ validator.handle_faults
300
+ end
194
301
  end
195
302
 
196
303
  class CloudPlanner
197
- # @return [Bosh::Director::DeploymentPlan::CompilationConfig]
198
- # Resource pool and other configuration for compilation workers
199
304
  attr_accessor :compilation
200
305
 
201
- def initialize(cloud_config)
202
- @cloud_config = cloud_config
203
- @networks_canonical_name_index = Set.new
306
+ def initialize(options)
307
+ @networks = self.class.index_by_name(options.fetch(:networks))
308
+ @global_network_resolver = options.fetch(:global_network_resolver)
309
+ @resource_pools = self.class.index_by_name(options.fetch(:resource_pools))
310
+ @vm_types = self.class.index_by_name(options.fetch(:vm_types, {}))
311
+ @disk_types = self.class.index_by_name(options.fetch(:disk_types))
312
+ @availability_zones = options.fetch(:availability_zones_list)
313
+ @compilation = options.fetch(:compilation)
314
+ @ip_provider_factory = options.fetch(:ip_provider_factory)
315
+ @logger = options.fetch(:logger)
316
+ end
204
317
 
205
- @networks = {}
206
- @resource_pools = {}
207
- @disk_pools = {}
318
+ def ip_provider
319
+ @ip_provider ||= @ip_provider_factory.new_ip_provider(@networks)
208
320
  end
209
321
 
210
- # Adds a resource pool by name
211
- # @param [Bosh::Director::DeploymentPlan::ResourcePool] resource_pool
212
- def add_resource_pool(resource_pool)
213
- if @resource_pools[resource_pool.name]
214
- raise DeploymentDuplicateResourcePoolName,
215
- "Duplicate resource pool name `#{resource_pool.name}'"
216
- end
217
- @resource_pools[resource_pool.name] = resource_pool
322
+ def model
323
+ nil
324
+ end
325
+
326
+ def deleted_network(name)
327
+ ManualNetwork.parse(
328
+ {'subnets' => [], 'name' => name},
329
+ [],
330
+ @global_network_resolver,
331
+ @logger
332
+ )
333
+ end
334
+
335
+ def availability_zone(name)
336
+ @availability_zones[name]
337
+ end
338
+
339
+ def availability_zones
340
+ @availability_zones.values
218
341
  end
219
342
 
220
- # Returns all resource pools in a deployment plan
221
- # @return [Array<Bosh::Director::DeploymentPlan::ResourcePool>]
222
343
  def resource_pools
223
344
  @resource_pools.values
224
345
  end
225
346
 
226
- # Returns a named resource pool spec
227
- # @param [String] name Resource pool name
228
- # @return [Bosh::Director::DeploymentPlan::ResourcePool]
229
347
  def resource_pool(name)
230
348
  @resource_pools[name]
231
349
  end
232
350
 
233
- # Adds a network by name
234
- # @param [Bosh::Director::DeploymentPlan::Network] network
235
- def add_network(network)
236
- if @networks_canonical_name_index.include?(network.canonical_name)
237
- raise DeploymentCanonicalNetworkNameTaken,
238
- "Invalid network name `#{network.name}', " +
239
- 'canonical name already taken'
240
- end
351
+ def vm_types
352
+ @vm_types.values
353
+ end
354
+
355
+ def vm_type(name)
356
+ @vm_types[name]
357
+ end
241
358
 
242
- @networks[network.name] = network
243
- @networks_canonical_name_index << network.canonical_name
359
+ def add_resource_pool(resource_pool)
360
+ @resource_pools[resource_pool.name] = resource_pool
244
361
  end
245
362
 
246
- # Returns all networks in a deployment plan
247
- # @return [Array<Bosh::Director::DeploymentPlan::Network>]
248
363
  def networks
249
364
  @networks.values
250
365
  end
251
366
 
252
- # Returns a named network
253
- # @param [String] name
254
- # @return [Bosh::Director::DeploymentPlan::Network]
255
367
  def network(name)
256
368
  @networks[name]
257
369
  end
258
370
 
259
- # Adds a disk pool by name
260
- # @param [Bosh::Director::DeploymentPlan::DiskPool] disk_pool
261
- def add_disk_pool(disk_pool)
262
- if @disk_pools[disk_pool.name]
263
- raise DeploymentDuplicateDiskPoolName,
264
- "Duplicate disk pool name `#{disk_pool.name}'"
265
- end
266
- @disk_pools[disk_pool.name] = disk_pool
371
+ def disk_types
372
+ @disk_types.values
267
373
  end
268
374
 
269
- def disk_pools
270
- @disk_pools.values
375
+ def using_global_networking?
376
+ false
271
377
  end
272
378
 
273
- def disk_pool(name)
274
- @disk_pools[name]
379
+ def disk_type(name)
380
+ @disk_types[name]
381
+ end
382
+
383
+ def self.index_by_name(collection)
384
+ collection.inject({}) do |index, item|
385
+ index.merge(item.name => item)
386
+ end
275
387
  end
276
388
  end
277
389
  end