cloud-mu 3.1.3 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +21 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +4 -4
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +147 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +158 -111
  45. data/modules/mu/adoption.rb +404 -71
  46. data/modules/mu/cleanup.rb +221 -306
  47. data/modules/mu/cloud.rb +129 -1633
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +44 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +926 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +169 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +32 -3
  61. data/modules/mu/config/cache_cluster.rb +2 -2
  62. data/modules/mu/config/cdn.rb +100 -0
  63. data/modules/mu/config/collection.rb +4 -4
  64. data/modules/mu/config/container_cluster.rb +9 -4
  65. data/modules/mu/config/database.rb +84 -105
  66. data/modules/mu/config/database.yml +1 -2
  67. data/modules/mu/config/dnszone.rb +10 -9
  68. data/modules/mu/config/doc_helpers.rb +516 -0
  69. data/modules/mu/config/endpoint.rb +5 -4
  70. data/modules/mu/config/firewall_rule.rb +103 -4
  71. data/modules/mu/config/folder.rb +4 -4
  72. data/modules/mu/config/function.rb +19 -10
  73. data/modules/mu/config/group.rb +4 -4
  74. data/modules/mu/config/habitat.rb +4 -4
  75. data/modules/mu/config/job.rb +89 -0
  76. data/modules/mu/config/loadbalancer.rb +60 -14
  77. data/modules/mu/config/log.rb +4 -4
  78. data/modules/mu/config/msg_queue.rb +4 -4
  79. data/modules/mu/config/nosqldb.rb +4 -4
  80. data/modules/mu/config/notifier.rb +10 -21
  81. data/modules/mu/config/ref.rb +411 -0
  82. data/modules/mu/config/role.rb +4 -4
  83. data/modules/mu/config/schema_helpers.rb +509 -0
  84. data/modules/mu/config/search_domain.rb +4 -4
  85. data/modules/mu/config/server.rb +98 -71
  86. data/modules/mu/config/server.yml +1 -0
  87. data/modules/mu/config/server_pool.rb +5 -9
  88. data/modules/mu/config/storage_pool.rb +1 -1
  89. data/modules/mu/config/tail.rb +200 -0
  90. data/modules/mu/config/user.rb +4 -4
  91. data/modules/mu/config/vpc.rb +71 -27
  92. data/modules/mu/config/vpc.yml +0 -1
  93. data/modules/mu/defaults/AWS.yaml +91 -68
  94. data/modules/mu/defaults/Azure.yaml +1 -0
  95. data/modules/mu/defaults/Google.yaml +3 -2
  96. data/modules/mu/deploy.rb +43 -26
  97. data/modules/mu/groomer.rb +17 -2
  98. data/modules/mu/groomers/ansible.rb +188 -41
  99. data/modules/mu/groomers/chef.rb +116 -55
  100. data/modules/mu/logger.rb +127 -148
  101. data/modules/mu/master.rb +410 -2
  102. data/modules/mu/master/chef.rb +3 -4
  103. data/modules/mu/master/ldap.rb +3 -3
  104. data/modules/mu/master/ssl.rb +12 -3
  105. data/modules/mu/mommacat.rb +218 -2612
  106. data/modules/mu/mommacat/daemon.rb +403 -0
  107. data/modules/mu/mommacat/naming.rb +473 -0
  108. data/modules/mu/mommacat/search.rb +495 -0
  109. data/modules/mu/mommacat/storage.rb +722 -0
  110. data/modules/mu/{clouds → providers}/README.md +1 -1
  111. data/modules/mu/{clouds → providers}/aws.rb +380 -122
  112. data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
  113. data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
  114. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
  115. data/modules/mu/providers/aws/cdn.rb +782 -0
  116. data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
  117. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
  118. data/modules/mu/providers/aws/database.rb +1744 -0
  119. data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
  120. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  121. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
  122. data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
  123. data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
  124. data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
  125. data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
  126. data/modules/mu/providers/aws/job.rb +466 -0
  127. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
  128. data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
  129. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
  130. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
  131. data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
  132. data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
  133. data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
  134. data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
  135. data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
  136. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
  137. data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
  138. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  139. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  140. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  141. data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
  142. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  143. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  144. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  145. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  146. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  147. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  148. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  149. data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
  150. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  151. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  152. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  153. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  154. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  155. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  156. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  160. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  161. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  162. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  163. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  164. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  165. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  166. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  167. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  168. data/modules/mu/{clouds → providers}/google.rb +68 -30
  169. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  170. data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
  171. data/modules/mu/{clouds → providers}/google/database.rb +11 -21
  172. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  173. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  174. data/modules/mu/{clouds → providers}/google/function.rb +140 -168
  175. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  176. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  177. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
  178. data/modules/mu/{clouds → providers}/google/role.rb +94 -58
  179. data/modules/mu/{clouds → providers}/google/server.rb +243 -156
  180. data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
  181. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  182. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  183. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  184. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  185. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  186. data/modules/tests/aws-jobs-functions.yaml +46 -0
  187. data/modules/tests/bucket.yml +4 -0
  188. data/modules/tests/centos6.yaml +15 -0
  189. data/modules/tests/centos7.yaml +15 -0
  190. data/modules/tests/centos8.yaml +12 -0
  191. data/modules/tests/ecs.yaml +23 -0
  192. data/modules/tests/eks.yaml +1 -1
  193. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  194. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  195. data/modules/tests/includes-and-params.yaml +2 -1
  196. data/modules/tests/microservice_app.yaml +288 -0
  197. data/modules/tests/rds.yaml +108 -0
  198. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  199. data/modules/tests/regrooms/bucket.yml +19 -0
  200. data/modules/tests/regrooms/rds.yaml +123 -0
  201. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  202. data/modules/tests/super_complex_bok.yml +2 -2
  203. data/modules/tests/super_simple_bok.yml +3 -5
  204. data/modules/tests/win2k12.yaml +17 -5
  205. data/modules/tests/win2k16.yaml +25 -0
  206. data/modules/tests/win2k19.yaml +25 -0
  207. data/requirements.txt +1 -0
  208. data/spec/mu/clouds/azure_spec.rb +2 -2
  209. metadata +240 -154
  210. data/extras/image-generators/AWS/windows.yaml +0 -18
  211. data/modules/mu/clouds/aws/database.rb +0 -1985
  212. data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -37,7 +37,7 @@ module MU
37
37
  if !@config['use_if_exists']
38
38
  raise MuError, "IAM user #{@mu_name} already exists and use_if_exists is false"
39
39
  end
40
- rescue Aws::IAM::Errors::NoSuchEntity => e
40
+ rescue Aws::IAM::Errors::NoSuchEntity
41
41
  @config['path'] ||= "/"+@deploy.deploy_id+"/"
42
42
  MU.log "Creating IAM user #{@config['path']}/#{@mu_name}"
43
43
  tags = get_tag_params
@@ -109,7 +109,7 @@ module MU
109
109
  # Create these if necessary, then append them to the list of
110
110
  # attachable_policies
111
111
  if @config['raw_policies']
112
- pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
112
+ pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
113
113
  @config['raw_policies'],
114
114
  basename: @deploy.getResourceName(@config['name']),
115
115
  credentials: @credentials
@@ -120,7 +120,7 @@ module MU
120
120
 
121
121
  if @config['attachable_policies']
122
122
  configured_policies = @config['attachable_policies'].map { |p|
123
- id = if p.is_a?(MU::Config::Ref)
123
+ if p.is_a?(MU::Config::Ref)
124
124
  p.cloud_id
125
125
  else
126
126
  p = MU::Config::Ref.get(p)
@@ -135,7 +135,7 @@ module MU
135
135
  attached_policies.each { |a|
136
136
  if !configured_policies.include?(a.policy_arn)
137
137
  MU.log "Removing IAM policy #{a.policy_arn} from user #{@mu_name}", MU::NOTICE
138
- MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
138
+ MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
139
139
  else
140
140
  configured_policies.delete(a.policy_arn)
141
141
  end
@@ -151,7 +151,7 @@ module MU
151
151
  end
152
152
 
153
153
  if @config['inline_policies']
154
- docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
154
+ docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
155
155
  docs.each { |doc|
156
156
  MU.log "Putting user policy #{doc.keys.first} to user #{@cloud_id} "
157
157
  MU::Cloud::AWS.iam(credentials: @credentials).put_user_policy(
@@ -189,17 +189,17 @@ module MU
189
189
  # Remove all users associated with the currently loaded deployment.
190
190
  # @param noop [Boolean]: If true, will only print what would be done
191
191
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
192
- # @param region [String]: The cloud provider region
193
192
  # @return [void]
194
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
193
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
194
+ MU.log "AWS::User.cleanup: need to support flags['known']", MU::DEBUG, details: flags
195
195
 
196
196
  # XXX this doesn't belong here; maybe under roles, maybe as its own stupid first-class resource
197
197
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
198
- path_prefix: "/"+MU.deploy_id+"/"
198
+ path_prefix: "/"+deploy_id+"/"
199
199
  )
200
200
  if resp and resp.policies
201
201
  resp.policies.each { |policy|
202
- MU.log "Deleting policy /#{MU.deploy_id}/#{policy.policy_name}"
202
+ MU.log "Deleting policy /#{deploy_id}/#{policy.policy_name}"
203
203
  if !noop
204
204
  attachments = begin
205
205
  MU::Cloud::AWS.iam(credentials: credentials).list_entities_for_policy(
@@ -257,7 +257,7 @@ module MU
257
257
  }
258
258
  retry
259
259
  rescue ::Aws::IAM::Errors::NoSuchEntity
260
- rescue Exception => e
260
+ rescue StandardError => e
261
261
  MU.log e.inspect, MU::ERR, details: policy
262
262
  end
263
263
  end
@@ -275,14 +275,17 @@ MU.log e.inspect, MU::ERR, details: policy
275
275
  ).tags
276
276
  has_nodelete = false
277
277
  has_ourdeploy = false
278
+ has_ourmaster = false
278
279
  tags.each { |tag|
279
- if tag.key == "MU-ID" and tag.value == MU.deploy_id
280
+ if tag.key == "MU-ID" and tag.value == deploy_id
280
281
  has_ourdeploy = true
282
+ elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
283
+ has_ourmaster = true
281
284
  elsif tag.key == "MU-NO-DELETE" and tag.value == "true"
282
285
  has_nodelete = true
283
286
  end
284
287
  }
285
- if has_ourdeploy and !has_nodelete
288
+ if has_ourdeploy and !has_nodelete and (ignoremaster or has_ourmaster)
286
289
  MU.log "Deleting IAM user #{u.path}#{u.user_name}"
287
290
  if !@noop
288
291
  begin
@@ -296,7 +299,7 @@ MU.log e.inspect, MU::ERR, details: policy
296
299
  group_name: g.group_name
297
300
  )
298
301
  }
299
- profile = MU::Cloud::AWS.iam(credentials: credentials).get_login_profile(
302
+ MU::Cloud::AWS.iam(credentials: credentials).get_login_profile(
300
303
  user_name: u.user_name
301
304
  )
302
305
  MU.log "Deleting IAM login profile for #{u.user_name}"
@@ -381,7 +384,7 @@ MU.log e.inspect, MU::ERR, details: policy
381
384
  # Reverse-map our cloud description into a runnable config hash.
382
385
  # We assume that any values we have in +@config+ are placeholders, and
383
386
  # calculate our own accordingly based on what's live in the cloud.
384
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
387
+ def toKitten(**_args)
385
388
  bok = {
386
389
  "cloud" => "AWS",
387
390
  "credentials" => @credentials,
@@ -428,7 +431,7 @@ MU.log e.inspect, MU::ERR, details: policy
428
431
  resp.policy_names.each { |pol_name|
429
432
  pol = MU::Cloud::AWS.iam(credentials: @credentials).get_user_policy(user_name: @cloud_id, policy_name: pol_name)
430
433
  doc = JSON.parse(URI.decode(pol.policy_document))
431
- bok["inline_policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
434
+ bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
432
435
  }
433
436
  end
434
437
 
@@ -457,12 +460,12 @@ MU.log e.inspect, MU::ERR, details: policy
457
460
  end
458
461
 
459
462
  # Cloud-specific configuration properties.
460
- # @param config [MU::Config]: The calling MU::Config object
463
+ # @param _config [MU::Config]: The calling MU::Config object
461
464
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
462
- def self.schema(config)
465
+ def self.schema(_config)
463
466
  toplevel_required = []
464
467
  polschema = MU::Config::Role.schema["properties"]["policies"]
465
- polschema.deep_merge!(MU::Cloud::AWS::Role.condition_schema)
468
+ polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
466
469
 
467
470
  schema = {
468
471
  "inline_policies" => polschema,
@@ -514,7 +517,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
514
517
  # If we're attaching some managed policies, make sure all of the ones
515
518
  # that should already exist do indeed exist
516
519
  if user['attachable_policies']
517
- ok = false if !MU::Cloud::AWS::Role.validateAttachablePolicies(
520
+ ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
518
521
  user['attachable_policies'],
519
522
  credentials: user['credentials'],
520
523
  region: user['region']
@@ -527,7 +530,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
527
530
  if configurator.haveLitterMate?(group, "groups")
528
531
  need_dependency = true
529
532
  else
530
- found = MU::Cloud::AWS::Group.find(cloud_id: group)
533
+ found = MU::Cloud.resourceClass("AWS", "Group").find(cloud_id: group)
531
534
  if found.nil? or found.empty? or (configurator.updating and
532
535
  found.values.first.group.path == "/"+configurator.updating+"/")
533
536
  groupdesc = {
@@ -539,11 +542,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
539
542
  end
540
543
 
541
544
  if need_dependency
542
- user["dependencies"] ||= []
543
- user["dependencies"] << {
544
- "type" => "group",
545
- "name" => group
546
- }
545
+ MU::Config.addDependency(user, group, "group")
547
546
  end
548
547
  }
549
548
  end
@@ -42,7 +42,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
42
42
  <% if !$mu.skipApplyUpdates %>
43
43
  set +e
44
44
  if [ ! -f /.mu-installer-ran-updates ];then
45
- service ssh stop
45
+ echo "Applying package updates" > /etc/nologin
46
46
  apt-get --fix-missing -y upgrade
47
47
  touch /.mu-installer-ran-updates
48
48
  if [ $? -eq 0 ]
@@ -58,7 +58,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
58
58
  else
59
59
  echo "FAILED PACKAGE UPDATE" >&2
60
60
  fi
61
- service ssh start
61
+ rm -f /etc/nologin
62
62
  fi
63
63
  <% end %>
64
64
  elif [ -x /usr/bin/yum ];then
@@ -94,7 +94,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
94
94
  <% if !$mu.skipApplyUpdates %>
95
95
  set +e
96
96
  if [ ! -f /.mu-installer-ran-updates ];then
97
- service sshd stop
97
+ echo "Applying package updates" > /etc/nologin
98
98
  kernel_update=`yum list updates | grep kernel`
99
99
  yum -y update
100
100
  touch /.mu-installer-ran-updates
@@ -108,7 +108,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
108
108
  else
109
109
  echo "FAILED PACKAGE UPDATE" >&2
110
110
  fi
111
- service sshd start
111
+ rm -f /etc/nologin
112
112
  fi
113
113
  <% end %>
114
114
  fi
@@ -116,6 +116,7 @@ else
116
116
  /bin/logger "***** Unable to verify internet connectivity, skipping package updates from userdata"
117
117
  touch /.mu-installer-ran-updates
118
118
  fi
119
+ rm -f /etc/nologin
119
120
 
120
121
  AWSCLI='command -v aws'
121
122
  PIP='command -v pip'
@@ -23,7 +23,7 @@ function log
23
23
  }
24
24
 
25
25
  function fetchSecret([string]$file){
26
- log "Fetching s3://<%= $mu.adminBucketName %>/$file to $tmp/$file"
26
+ log "aws.cmd --region $region s3 cp s3://<%= $mu.adminBucketName %>/$file $tmp/$file"
27
27
  aws.cmd --region $region s3 cp s3://<%= $mu.adminBucketName %>/$file $tmp/$file
28
28
  }
29
29
 
@@ -245,6 +245,7 @@ if($ingroup -ne $admin_username){
245
245
  net localgroup WinRMRemoteWMIUsers__ /add $admin_username
246
246
  }
247
247
 
248
+ importCert "$myname-winrm.crt" "root"
248
249
  $winrmcert = importCert "$myname-winrm.crt" "TrustedPeople"
249
250
  Set-Item -Path WSMan:\localhost\Service\Auth\Certificate -Value $true
250
251
  Set-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System" -Name LocalAccountTokenFilterPolicy -Value 1
@@ -18,6 +18,7 @@ module MU
18
18
 
19
19
  # Creation of Virtual Private Clouds and associated artifacts (routes, subnets, etc).
20
20
  class VPC < MU::Cloud::VPC
21
+ require 'mu/providers/aws/vpc_subnet'
21
22
 
22
23
  # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
23
24
  # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
@@ -35,78 +36,40 @@ module MU
35
36
  def create
36
37
  MU.log "Creating VPC #{@mu_name}", details: @config
37
38
  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc(cidr_block: @config['ip_block']).vpc
38
- vpc_id = @config['vpc_id'] = resp.vpc_id
39
+ @cloud_id = resp.vpc_id
40
+ @config['vpc_id'] = @cloud_id
39
41
 
40
- MU::Cloud::AWS.createStandardTags(vpc_id, region: @config['region'], credentials: @config['credentials'])
41
- MU::MommaCat.createTag(vpc_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
42
-
43
- if @config['tags']
44
- @config['tags'].each { |tag|
45
- MU::MommaCat.createTag(vpc_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
46
- }
47
- end
48
-
49
- if @config['optional_tags']
50
- MU::MommaCat.listOptionalTags.each { |key, value|
51
- MU::MommaCat.createTag(vpc_id, key, value, region: @config['region'], credentials: @config['credentials'])
52
- }
53
- end
42
+ tag_me
54
43
 
55
44
  if resp.state != "available"
56
45
  begin
57
- MU.log "Waiting for VPC #{@mu_name} (#{vpc_id}) to be available", MU::NOTICE
46
+ MU.log "Waiting for VPC #{@mu_name} (#{@cloud_id}) to be available", MU::NOTICE
58
47
  sleep 5
59
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpcs(vpc_ids: [vpc_id]).vpcs.first
48
+ resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpcs(vpc_ids: [@cloud_id]).vpcs.first
60
49
  end while resp.state != "available"
61
50
  # There's a default route table that comes with. Let's tag it.
62
51
  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
63
52
  filters: [
64
53
  {
65
54
  name: "vpc-id",
66
- values: [vpc_id]
55
+ values: [@cloud_id]
67
56
  }
68
57
  ]
69
58
  )
70
59
  resp.route_tables.each { |rtb|
71
- MU::MommaCat.createTag(rtb.route_table_id, "Name", @mu_name+"-#DEFAULTPRIV", region: @config['region'], credentials: @config['credentials'])
72
- if @config['tags']
73
- @config['tags'].each { |tag|
74
- MU::MommaCat.createTag(rtb.route_table_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
75
- }
76
- end
77
-
78
- MU::Cloud::AWS.createStandardTags(rtb.route_table_id, region: @config['region'], credentials: @config['credentials'])
79
-
80
- if @config['optional_tags']
81
- MU::MommaCat.listOptionalTags.each { |key, value|
82
- MU::MommaCat.createTag(rtb.route_table_id, key, value, region: @config['region'], credentials: @config['credentials'])
83
- }
84
- end
60
+ tag_me(rtb.route_table_id, @mu_name+"-#DEFAULTPRIV")
85
61
  }
86
62
  end
87
- @config['vpc_id'] = vpc_id
88
- @cloud_id = vpc_id
89
63
 
90
64
  if @config['create_internet_gateway']
91
65
  MU.log "Creating Internet Gateway #{@mu_name}"
92
66
  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_internet_gateway
93
67
  internet_gateway_id = resp.internet_gateway.internet_gateway_id
94
68
  sleep 5
95
- MU::Cloud::AWS.createStandardTags(internet_gateway_id, region: @config['region'], credentials: @config['credentials'])
96
- MU::MommaCat.createTag(internet_gateway_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
97
- if @config['tags']
98
- @config['tags'].each { |tag|
99
- MU::MommaCat.createTag(internet_gateway_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
100
- }
101
- end
102
69
 
103
- if @config['optional_tags']
104
- MU::MommaCat.listOptionalTags.each { |key, value|
105
- MU::MommaCat.createTag(internet_gateway_id, key, value, region: @config['region'], credentials: @config['credentials'])
106
- }
107
- end
70
+ tag_me(internet_gateway_id)
108
71
 
109
- MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_internet_gateway(vpc_id: vpc_id, internet_gateway_id: internet_gateway_id)
72
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_internet_gateway(vpc_id: @cloud_id, internet_gateway_id: internet_gateway_id)
110
73
  @config['internet_gateway_id'] = internet_gateway_id
111
74
  end
112
75
 
@@ -165,237 +128,35 @@ module MU
165
128
  )
166
129
  end
167
130
 
168
- nat_gateways = []
169
- if !@config['subnets'].nil?
170
- allocation_ids = []
171
- subnet_semaphore = Mutex.new
172
- subnetthreads = Array.new
173
- parent_thread_id = Thread.current.object_id
174
- azs = []
175
- @config['subnets'].each { |subnet|
176
- subnet_name = @config['name']+"-"+subnet['name']
177
- MU.log "Creating Subnet #{subnet_name} (#{subnet['ip_block']})", details: subnet
178
- azs = MU::Cloud::AWS.listAZs(region: @config['region'], credentials: @config['credentials']) if azs.size == 0
179
- if !subnet['availability_zone'].nil?
180
- az = subnet['availability_zone']
181
- else
182
- az = azs.pop
183
- end
184
-
185
- subnetthreads << Thread.new {
186
- MU.dupGlobals(parent_thread_id)
187
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_subnet(
188
- vpc_id: vpc_id,
189
- cidr_block: subnet['ip_block'],
190
- availability_zone: az
191
- ).subnet
192
- subnet_id = subnet['subnet_id'] = resp.subnet_id
193
- MU::Cloud::AWS.createStandardTags(subnet_id, region: @config['region'], credentials: @config['credentials'])
194
- MU::MommaCat.createTag(subnet_id, "Name", @mu_name+"-"+subnet['name'], region: @config['region'], credentials: @config['credentials'])
195
- if @config['tags']
196
- @config['tags'].each { |tag|
197
- MU::MommaCat.createTag(subnet_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
198
- }
199
- end
200
-
201
- if @config['optional_tags']
202
- MU::MommaCat.listOptionalTags.each { |key, value|
203
- MU::MommaCat.createTag(subnet_id, key, value, region: @config['region'], credentials: @config['credentials'])
204
- }
205
- end
206
-
207
- retries = 0
208
- begin
209
- if resp.state != "available"
210
- begin
211
- MU.log "Waiting for Subnet #{subnet_name} (#{subnet_id}) to be available", MU::NOTICE if retries > 0 and (retries % 3) == 0
212
- sleep 5
213
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
214
- rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
215
- sleep 10
216
- retry
217
- end while resp.state != "available"
218
- end
219
- rescue NoMethodError => e
220
- if retries <= 3
221
- MU.log "Got bogus Aws::EmptyResponse error on #{subnet_id} (retries used: #{retries}/3)", MU::WARN
222
- retries = retries + 1
223
- sleep 5
224
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
225
- retry
226
- else
227
- raise e
228
- end
229
- end
230
-
231
- if !subnet['route_table'].nil?
232
- routes = {}
233
- @config['route_tables'].each { |tbl|
234
- routes[tbl['name']] = tbl
235
- }
236
- if routes.nil? or routes[subnet['route_table']].nil?
237
- MU.log "Subnet #{subnet_name} references non-existent route #{subnet['route_table']}", MU::ERR, details: @deploy.deployment['vpcs']
238
- raise MuError, "deploy failure"
239
- end
240
- MU.log "Associating Route Table '#{subnet['route_table']}' (#{routes[subnet['route_table']]['route_table_id']}) with #{subnet_name}"
241
- retries = 0
242
- begin
243
- MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_route_table(
244
- route_table_id: routes[subnet['route_table']]['route_table_id'],
245
- subnet_id: subnet_id
246
- )
247
- rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound => e
248
- retries = retries + 1
249
- if retries < 10
250
- sleep 10
251
- retry
252
- else
253
- raise MuError, e.inspect
254
- end
255
- end
256
- end
257
- retries = 0
258
- begin
259
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
260
- rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
261
- if retries < 10
262
- MU.log "Got #{e.inspect}, waiting and retrying", MU::WARN
263
- sleep 10
264
- retries = retries + 1
265
- retry
266
- end
267
- raise MuError, e.inspect, e.backtrace
268
- end
269
-
270
- if subnet['is_public'] && subnet['create_nat_gateway']
271
- MU::MommaCat.lock("nat-gateway-eipalloc")
272
- filters = [{name: "domain", values: ["vpc"]}]
273
- eips = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(filters: filters).addresses
274
- allocation_id = nil
275
- eips.each { |eip|
276
- next if !eip.association_id.nil? and !eip.association_id.empty?
277
- if (eip.private_ip_address.nil? || eip.private_ip_address.empty?) and MU::MommaCat.lock(eip.allocation_id, true, true)
278
- if !allocation_ids.include?(eip.allocation_id)
279
- allocation_id = eip.allocation_id
280
- break
281
- end
282
- end
283
- }
131
+ nat_gateways = create_subnets
284
132
 
285
- if allocation_id.nil?
286
- allocation_id = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).allocate_address(domain: "vpc").allocation_id
287
- MU::MommaCat.lock(allocation_id, false, true)
288
- end
289
-
290
- allocation_ids << allocation_id
291
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_nat_gateway(
292
- subnet_id: subnet['subnet_id'],
293
- allocation_id: allocation_id,
294
- ).nat_gateway
295
-
296
- nat_gateway_id = resp.nat_gateway_id
297
- attempts = 0
298
- MU::MommaCat.unlock("nat-gateway-eipalloc")
299
- while resp.class.name != "Aws::EC2::Types::NatGateway" or resp.state == "pending"
300
- MU.log "Waiting for nat gateway #{nat_gateway_id} () to become available (EIP allocation: #{allocation_id})" if attempts % 5 == 0
301
- sleep 30
302
- begin
303
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
304
- rescue Aws::EmptyStructure, NoMethodError
305
- sleep 5
306
- retry
307
- end
308
- if attempts > 30
309
- MU::MommaCat.unlock(allocation_id, true)
310
- raise MuError, "Timed out while waiting for NAT Gateway #{nat_gateway_id}: #{resp}"
311
- end
312
- attempts += 1
313
- end
314
- MU::MommaCat.unlock(allocation_id, true)
315
-
316
- raise MuError, "NAT Gateway failed #{nat_gateway_id}: #{resp}" if resp.state == "failed"
317
- nat_gateways << {'id' => nat_gateway_id, 'availability_zone' => subnet['availability_zone']}
318
- @tags.each_pair { |k, v|
319
- MU::MommaCat.createTag(nat_gateway_id, k, v, region: @config['region'], credentials: @config['credentials'])
320
- }
321
- end
322
-
323
- if subnet.has_key?("map_public_ips")
324
- retries = 0
325
- begin
326
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_subnet_attribute(
327
- subnet_id: subnet_id,
328
- map_public_ip_on_launch: {
329
- value: subnet['map_public_ips'],
330
- }
331
- )
332
- rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
333
- if retries < 10
334
- MU.log "Got #{e.inspect} while trying to enable map_public_ips on subnet, waiting and retrying", MU::WARN
335
- sleep 10
336
- retries += 1
337
- retry
338
- end
339
- raise MuError, "Got #{e.inspect}, #{e.backtrace} while trying to enable map_public_ips on subnet"
340
- end
341
- end
342
-
343
- if subnet["enable_traffic_logging"]
344
- loggroup = @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs")
345
- logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
346
- MU.log "Enabling traffic logging on Subnet #{subnet_name} in VPC #{@mu_name} to log group #{loggroup.mu_name}"
347
- MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_flow_logs(
348
- resource_ids: [subnet_id],
349
- resource_type: "Subnet",
350
- traffic_type: subnet["traffic_type_to_log"],
351
- log_group_name: loggroup.mu_name,
352
- deliver_logs_permission_arn: logrole.cloudobj.arn
353
- )
354
- end
355
- }
356
- }
357
-
358
- subnetthreads.each { |t|
359
- t.join
360
- }
361
-
362
- notify
363
- end
133
+ notify
364
134
 
365
135
  if !nat_gateways.empty?
366
136
  nat_gateways.each { |gateway|
367
137
  @config['subnets'].each { |subnet|
368
- if subnet['is_public'] == false && subnet['availability_zone'] == gateway['availability_zone']
369
- @config['route_tables'].each { |rtb|
370
- if rtb['name'] == subnet['route_table']
371
- rtb['routes'].each { |route|
372
- if route['gateway'] == '#NAT'
373
- route_config = {
374
- :route_table_id => rtb['route_table_id'],
375
- :destination_cidr_block => route['destination_network'],
376
- :nat_gateway_id => gateway['id']
377
- }
378
-
379
- MU.log "Creating route for #{route['destination_network']} through NAT gatway #{gateway['id']}", details: route_config
380
- nat_retries = 0
381
- begin
382
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
383
- rescue Aws::EC2::Errors::InvalidNatGatewayIDNotFound => e
384
- if nat_retries < 5
385
- nat_retries += 1
386
- sleep 10
387
- retry
388
- else
389
- raise e
390
- end
391
- rescue Aws::EC2::Errors::RouteAlreadyExists => e
392
- MU.log "Attempt to create duplicate route to #{route['destination_network']} for #{gateway['id']} in #{rtb['route_table_id']}", MU::WARN
393
- end
394
- end
395
- }
396
- end
138
+ next if subnet['is_public'] != false or subnet['availability_zone'] != gateway['availability_zone']
139
+
140
+ @config['route_tables'].each { |rtb|
141
+ next if rtb['name'] != subnet['route_table']
142
+ rtb['routes'].each { |route|
143
+ next if route['gateway'] != '#NAT'
144
+ route_config = {
145
+ :route_table_id => rtb['route_table_id'],
146
+ :destination_cidr_block => route['destination_network'],
147
+ :nat_gateway_id => gateway['id']
148
+ }
149
+
150
+ MU.log "Creating route for #{route['destination_network']} through NAT gatway #{gateway['id']}", details: route_config
151
+ MU.retrier([Aws::EC2::Errors::InvalidNatGatewayIDNotFound], wait: 10, max: 5) {
152
+ begin
153
+ resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
154
+ rescue Aws::EC2::Errors::RouteAlreadyExists
155
+ MU.log "Attempt to create duplicate route to #{route['destination_network']} for #{gateway['id']} in #{rtb['route_table_id']}", MU::WARN
156
+ end
157
+ }
397
158
  }
398
- end
159
+ }
399
160
  }
400
161
  }
401
162
  end
@@ -403,14 +164,14 @@ module MU
403
164
  if @config['enable_dns_support']
404
165
  MU.log "Enabling DNS support in #{@mu_name}"
405
166
  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
406
- vpc_id: vpc_id,
167
+ vpc_id: @cloud_id,
407
168
  enable_dns_support: {value: @config['enable_dns_support']}
408
169
  )
409
170
  end
410
171
  if @config['enable_dns_hostnames']
411
172
  MU.log "Enabling DNS hostnames in #{@mu_name}"
412
173
  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
413
- vpc_id: vpc_id,
174
+ vpc_id: @cloud_id,
414
175
  enable_dns_hostnames: {value: @config['enable_dns_hostnames']}
415
176
  )
416
177
  end
@@ -439,29 +200,16 @@ module MU
439
200
  dhcp_configurations: dhcpopts
440
201
  )
441
202
  dhcpopt_id = resp.dhcp_options.dhcp_options_id
442
- MU::Cloud::AWS.createStandardTags(dhcpopt_id, region: @config['region'], credentials: @config['credentials'])
443
- MU::MommaCat.createTag(dhcpopt_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
444
-
445
- if @config['tags']
446
- @config['tags'].each { |tag|
447
- MU::MommaCat.createTag(dhcpopt_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
448
- }
449
- end
203
+ tag_me(dhcpopt_id)
450
204
 
451
- if @config['optional_tags']
452
- MU::MommaCat.listOptionalTags.each { |key, value|
453
- MU::MommaCat.createTag(dhcpopt_id, key, value, region: @config['region'], credentials: @config['credentials'])
454
- }
455
- end
456
-
457
- MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id: vpc_id)
205
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id: @cloud_id)
458
206
  end
459
207
  notify
460
208
 
461
209
  if !MU::Cloud::AWS.isGovCloud?(@config['region'])
462
210
  mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: @config['credentials']).values.first
463
211
  if !mu_zone.nil?
464
- MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: vpc_id, region: @config['region'], credentials: @config['credentials'])
212
+ MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: @cloud_id, region: @config['region'], credentials: @config['credentials'])
465
213
  end
466
214
  end
467
215
  loadSubnets
@@ -472,9 +220,6 @@ module MU
472
220
  # Canonical Amazon Resource Number for this resource
473
221
  # @return [String]
474
222
  def arn
475
- puts @config['region']
476
- puts MU::Cloud::AWS.credToAcct(@config['credentials'])
477
- puts @cloud_id
478
223
  "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":vpc/"+@cloud_id
479
224
  end
480
225
 
@@ -491,202 +236,7 @@ module MU
491
236
  # Generate peering connections
492
237
  if !@config['peers'].nil? and @config['peers'].size > 0
493
238
  @config['peers'].each { |peer|
494
- peer_obj = nil
495
- peer_id = nil
496
- peer['name'] ||= peer['vpc_name']
497
- peer['id'] ||= peer['vpc_id']
498
-
499
- # If we know this to be a sibling VPC elsewhere in our stack,
500
- # go fetch it, and fix it if we've been misconfigured with a
501
- # duplicate peering connection
502
- if peer['vpc']['name'] and !peer['account']
503
- peer_obj = @deploy.findLitterMate(name: peer['vpc']['name'], type: "vpcs")
504
- if peer_obj
505
- if peer_obj.config['peers']
506
- skipme = false
507
- peer_obj.config['peers'].each { |peerpeer|
508
- if peerpeer['vpc']['name'] == @config['name'] and
509
- (peer['vpc']['name'] <=> @config['name']) == -1
510
- skipme = true
511
- MU.log "VPCs #{peer['vpc']['name']} and #{@config['name']} both declare mutual peering connection, ignoring #{@config['name']}'s redundant declaration", MU::DEBUG
512
- # XXX and if deploy_id matches or is unset
513
- end
514
- }
515
- end
516
- next if skipme
517
- peer['account'] = MU::Cloud::AWS.credToAcct(peer_obj.credentials)
518
- peer['vpc']['id'] = peer_obj.cloud_id
519
- peer['vpc']['region'] ||= peer_obj.config['region']
520
- end
521
- end
522
-
523
- # If we still don't know our peer's vpc identifier, go fishing
524
- if !peer_obj
525
- tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
526
- if peer['vpc']['deploy_id'].nil? and peer['vpc']['id'].nil? and tag_key.nil?
527
- peer['vpc']['deploy_id'] = @deploy.deploy_id
528
- end
529
- peer_obj = MU::MommaCat.findStray(
530
- "AWS",
531
- "vpcs",
532
- deploy_id: peer['vpc']['deploy_id'],
533
- cloud_id: peer['vpc']['id'],
534
- # XXX we need a credentials argument here... maybe
535
- name: peer['vpc']['name'],
536
- tag_key: tag_key,
537
- tag_value: tag_value,
538
- dummy_ok: true,
539
- region: peer['vpc']['region']
540
- )
541
- MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
542
- raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil? or peer_obj.first.nil?
543
- peer_obj = peer_obj.first
544
- peer['account'] ||= MU::Cloud::AWS.credToAcct(peer_obj.credentials)
545
- peer['vpc']['id'] ||= peer_obj.cloud_id
546
- peer['vpc']['region'] ||= peer_obj.config['region']
547
- end
548
-
549
- peer_id = peer['vpc']['id']
550
- peer['account'] ||= MU::Cloud::AWS.account_number
551
-
552
- # See if the peering connection exists before we bother
553
- # creating it.
554
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
555
- filters: [
556
- {
557
- name: "requester-vpc-info.vpc-id",
558
- values: [@cloud_id]
559
- },
560
- {
561
- name: "accepter-vpc-info.vpc-id",
562
- values: [peer_id.to_s]
563
- }
564
- ]
565
- )
566
-
567
- peering_id = if !resp or !resp.vpc_peering_connections or
568
- resp.vpc_peering_connections.empty?
569
-
570
- MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id} in account #{MU::Cloud::AWS.credToAcct(@config['credentials'])}) to #{peer_id} in account #{peer['account']}", details: peer
571
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_peering_connection(
572
- vpc_id: @cloud_id,
573
- peer_vpc_id: peer_id,
574
- peer_owner_id: peer['account'],
575
- peer_region: peer['vpc']['region']
576
- )
577
- resp.vpc_peering_connection.vpc_peering_connection_id
578
- else
579
- resp.vpc_peering_connections.first.vpc_peering_connection_id
580
- end
581
-
582
- peering_name = @deploy.getResourceName(@config['name']+"-PEER-"+peer['vpc']['id'])
583
-
584
- MU::Cloud::AWS.createStandardTags(peering_id, region: @config['region'], credentials: @config['credentials'])
585
- MU::MommaCat.createTag(peering_id, "Name", peering_name, region: @config['region'], credentials: @config['credentials'])
586
-
587
- if @config['optional_tags']
588
- MU::MommaCat.listOptionalTags.each { |key, value|
589
- MU::MommaCat.createTag(peering_id, key, value, region: @config['region'], credentials: @config['credentials'])
590
- }
591
- end
592
-
593
- if @config['tags']
594
- @config['tags'].each { |tag|
595
- MU::MommaCat.createTag(peering_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
596
- }
597
- end
598
-
599
- # Create routes to our new friend.
600
- MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @config['region'], credentials: @config['credentials']).each { |rtb_id|
601
- my_route_config = {
602
- :route_table_id => rtb_id,
603
- :destination_cidr_block => peer_obj.cloud_desc.cidr_block,
604
- :vpc_peering_connection_id => peering_id
605
- }
606
- rtbdesc = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
607
- route_table_ids: [rtb_id]
608
- ).route_tables.first
609
- already_exists = false
610
- rtbdesc.routes.each { |r|
611
- if r.destination_cidr_block == peer_obj.cloud_desc.cidr_block
612
- if r.vpc_peering_connection_id != peering_id
613
- MU.log "Attempt to create duplicate route to #{peer_obj.cloud_desc.cidr_block} from VPC #{@config['name']}", MU::ERR, details: r
614
- raise MuError, "Can't create route via #{peering_id}, a route to #{peer_obj.cloud_desc.cidr_block} already exists"
615
- else
616
- already_exists = true
617
- end
618
- end
619
- }
620
- next if already_exists
621
-
622
- MU.log "Creating peering route to #{peer_obj.cloud_desc.cidr_block} in #{peer['vpc']['region']} from VPC #{@config['name']} in #{@config['region']}"
623
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(my_route_config)
624
- } # MU::Cloud::AWS::VPC.listAllSubnetRouteTables
625
-
626
- begin
627
- cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
628
- vpc_peering_connection_ids: [peering_id]
629
- ).vpc_peering_connections.first
630
-
631
- if cnxn.status.code == "pending-acceptance"
632
- if ((!peer_obj.nil? and !peer_obj.deploydata.nil? and peer_obj.deploydata['auto_accept_peers']) or $MU_CFG['allow_invade_foreign_vpcs'])
633
- MU.log "Auto-accepting peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id}", MU::NOTICE
634
- begin
635
- MU::Cloud::AWS.ec2(region: peer['vpc']['region'], credentials: peer['account']).accept_vpc_peering_connection(
636
- vpc_peering_connection_id: peering_id,
637
- )
638
- if peer['account'] != MU::Cloud::AWS.credToAcct(@config['credentials'])
639
- # this seems to take a while across accounts
640
- sleep 5
641
- end
642
- cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
643
- vpc_peering_connection_ids: [peering_id]
644
- ).vpc_peering_connections.first
645
- rescue Aws::EC2::Errors::VpcPeeringConnectionAlreadyExists => e
646
- MU.log "Attempt to create duplicate peering connection to #{peer_id} from VPC #{@config['name']}", MU::WARN
647
- end
648
-
649
- # Create routes back from our new friend to us.
650
- MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_id, region: peer['vpc']['region'], credentials: peer['account']).uniq.each { |rtb_id|
651
- peer_route_config = {
652
- :route_table_id => rtb_id,
653
- :destination_cidr_block => @config['ip_block'],
654
- :vpc_peering_connection_id => peering_id
655
- }
656
- begin
657
- resp = MU::Cloud::AWS.ec2(region: peer['vpc']['region'], credentials: peer['account']).create_route(peer_route_config)
658
- rescue Aws::EC2::Errors::RouteAlreadyExists => e
659
- rtbdesc = MU::Cloud::AWS.ec2(region: peer['vpc']['region'], credentials: peer['account']).describe_route_tables(
660
- route_table_ids: [rtb_id]
661
- ).route_tables.first
662
- rtbdesc.routes.each { |r|
663
- if r.destination_cidr_block == @config['ip_block']
664
- if r.vpc_peering_connection_id != peering_id
665
- MU.log "Attempt to create duplicate route to VPC #{@config['name']} (#{@config['ip_block']}) from peer VPC #{peer_id}'s route table #{rtb_id}", MU::ERR
666
- end
667
- end
668
- }
669
- end
670
- }
671
- else
672
- MU.log "VPC #{peer_id} is not managed by this Mu server or is not configured to auto-accept peering requests. You must accept the peering request for '#{@config['name']}' (#{@cloud_id}) by hand.", MU::WARN, details: "In the AWS Console, go to VPC => Peering Connections and look in the Actions drop-down. You can also set 'Invade Foreign VPCs' to 'true' using mu-configure to auto-accept all peering connections within this account, regardless of whether this Mu server owns the VPCs. This setting is per-user."
673
- end
674
- end
675
-
676
- if cnxn.status.code == "failed" or cnxn.status.code == "rejected" or cnxn.status.code == "expired" or cnxn.status.code == "deleted"
677
- MU.log "VPC peering connection from VPC #{@config['name']} (#{@cloud_id} in #{@config['region']}) to #{peer_id} in #{peer['vpc']['region']} #{cnxn.status.code}: #{cnxn.status.message}", MU::ERR
678
- begin
679
- MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).delete_vpc_peering_connection(
680
- vpc_peering_connection_id: peering_id
681
- )
682
- rescue Aws::EC2::Errors::InvalidStateTransition => e
683
- # XXX apparently this is normal?
684
- end
685
- raise MuError, "VPC peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id} #{cnxn.status.code}: #{cnxn.status.message}"
686
- end
687
-
688
- end while cnxn.status.code != "active" and !((peer_obj.nil? or peer_obj.deploydata.nil? or !peer_obj.deploydata['auto_accept_peers']) and cnxn.status.code == "pending-acceptance")
689
-
239
+ peerWith(peer)
690
240
  }
691
241
  end
692
242
 
@@ -713,7 +263,7 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
713
263
  route_config[:instance_id] = nat_instance.cloud_id
714
264
 
715
265
  MU.log "Creating route for #{route['destination_network']} through NAT host #{nat_instance.cloud_id}", details: route_config
716
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
266
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
717
267
  end
718
268
  }
719
269
 
@@ -756,7 +306,7 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
756
306
  map[vpc.vpc_id] = vpc
757
307
  }
758
308
  return map
759
- rescue Aws::EC2::Errors::InvalidVpcIDNotFound => e
309
+ rescue Aws::EC2::Errors::InvalidVpcIDNotFound
760
310
  end
761
311
  else
762
312
  resp = MU::Cloud::AWS.ec2(region: args[:region], credentials: args[:credentials]).describe_vpcs
@@ -774,7 +324,7 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
774
324
  # Reverse-map our cloud description into a runnable config hash.
775
325
  # We assume that any values we have in +@config+ are placeholders, and
776
326
  # calculate our own accordingly based on what's live in the cloud.
777
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
327
+ def toKitten(**_args)
778
328
  bok = {
779
329
  "cloud" => "AWS",
780
330
  "credentials" => @config['credentials'],
@@ -838,12 +388,9 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
838
388
  if rtb_desc.associations
839
389
  rtb_desc.associations.each { |assoc|
840
390
  if assoc.subnet_id
841
- if associations[assoc.subnet_id] and associations[assoc.subnet_id] != rtb['name']
842
- MU.log "wait more than one route table association for #{assoc.subnet_id} what", MU::WARN, details: associations[assoc.subnet_id]+" => "+rtb['name']
843
- end
844
391
  associations[assoc.subnet_id] = rtb['name']
845
392
  elsif assoc.gateway_id
846
- MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_desc
393
+ MU.log " Saw a route table association I don't know how to adopt in #{@cloud_id}", MU::WARN, details: rtb_desc
847
394
  end
848
395
  }
849
396
  end
@@ -903,20 +450,18 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
903
450
  # Describe subnets associated with this VPC. We'll compose identifying
904
451
  # information similar to what MU::Cloud.describe builds for first-class
905
452
  # resources.
906
- # XXX this is weaksauce. Subnets should be objects with their own methods
907
- # that work like first-class objects. How would we enforce that?
908
- # @return [Array<Hash>]: A list of cloud provider identifiers of subnets associated with this VPC.
453
+ # @return [Array<MU::Cloud::AWS::VPC::Subnet>]
909
454
  def loadSubnets
910
- if @cloud_id
911
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(
912
- filters: [
913
- { name: "vpc-id", values: [@cloud_id] }
914
- ]
915
- )
916
- if resp.nil? or resp.subnets.nil? or resp.subnets.size == 0
917
- MU.log "Got empty results when trying to list subnets in #{@cloud_id}", MU::WARN
918
- return []
919
- end
455
+ return [] if !@cloud_id
456
+
457
+ resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(
458
+ filters: [
459
+ { name: "vpc-id", values: [@cloud_id] }
460
+ ]
461
+ )
462
+ if resp.nil? or resp.subnets.nil? or resp.subnets.empty?
463
+ MU.log "Got empty results when trying to list subnets in #{@cloud_id}", MU::WARN
464
+ return []
920
465
  end
921
466
 
922
467
  @subnetcachesemaphore.synchronize {
@@ -927,23 +472,21 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
927
472
  # metadata. Like ya do.
928
473
  if !@config.nil? and @config.has_key?("subnets")
929
474
  @config['subnets'].each { |subnet|
930
- subnet['mu_name'] = @mu_name+"-"+subnet['name'] if !subnet.has_key?("mu_name")
475
+ subnet['mu_name'] ||= @mu_name+"-"+subnet['name']
931
476
  subnet['region'] = @config['region']
932
477
  subnet['credentials'] = @config['credentials']
933
- if !resp.nil? and !resp.data.nil? and !resp.data.subnets.nil?
934
- resp.data.subnets.each { |desc|
935
- if desc.cidr_block == subnet["ip_block"]
936
- subnet["tags"] = MU.structToHash(desc.tags)
937
- subnet["cloud_id"] = desc.subnet_id
938
- break
939
- end
940
- }
941
- end
478
+ resp.subnets.each { |desc|
479
+ if desc.cidr_block == subnet["ip_block"]
480
+ subnet["tags"] = MU.structToHash(desc.tags)
481
+ subnet["cloud_id"] = desc.subnet_id
482
+ break
483
+ end
484
+ }
942
485
 
943
486
  if subnet["cloud_id"] and !ext_ids.include?(subnet["cloud_id"])
944
487
  @subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
945
488
  elsif !subnet["cloud_id"]
946
- resp.data.subnets.each { |desc|
489
+ resp.subnets.each { |desc|
947
490
  if desc.cidr_block == subnet["ip_block"]
948
491
  subnet['cloud_id'] = desc.subnet_id
949
492
  @subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
@@ -956,21 +499,21 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
956
499
 
957
500
  # Of course we might be loading up a dummy subnet object from a
958
501
  # foreign or non-Mu-created VPC and subnet. So make something up.
959
- if !resp.nil? and @subnets.empty?
960
- resp.data.subnets.each { |desc|
961
- subnet = {}
962
- subnet["ip_block"] = desc.cidr_block
963
- subnet["name"] = subnet["ip_block"].gsub(/[\.\/]/, "_")
502
+ if @subnets.empty?
503
+ resp.subnets.each { |desc|
504
+ subnet = {
505
+ "ip_block" => desc.cidr_block,
506
+ "tags" => MU.structToHash(desc.tags),
507
+ "cloud_id" => desc.subnet_id,
508
+ 'region' => @config['region'],
509
+ 'credentials' => @config['credentials'],
510
+ }
511
+ subnet['name'] = subnet["ip_block"].gsub(/[\.\/]/, "_")
964
512
  subnet['mu_name'] = @mu_name+"-"+subnet['name']
965
- subnet["tags"] = MU.structToHash(desc.tags)
966
- subnet["cloud_id"] = desc.subnet_id
967
- subnet['region'] = @config['region']
968
- subnet['credentials'] = @config['credentials']
969
- if !ext_ids.include?(desc.subnet_id)
970
- @subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
971
- end
513
+ @subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
972
514
  }
973
515
  end
516
+
974
517
  return @subnets
975
518
  }
976
519
  end
@@ -983,13 +526,14 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
983
526
  def findNat(nat_cloud_id: nil, nat_filter_key: nil, nat_filter_value: nil, region: MU.curRegion, credentials: nil)
984
527
  # Discard the nat_cloud_id if it's an AWS instance ID
985
528
  nat_cloud_id = nil if nat_cloud_id && nat_cloud_id.start_with?("i-")
529
+ credentials ||= @credentials
986
530
 
987
531
  if @gateways.nil?
988
532
  @gateways =
989
533
  if nat_cloud_id
990
- MU::Cloud::AWS.ec2(region: region, credentials: nil).describe_nat_gateways(nat_gateway_ids: [nat_cloud_id])
534
+ MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_nat_gateways(nat_gateway_ids: [nat_cloud_id])
991
535
  elsif nat_filter_key && nat_filter_value
992
- MU::Cloud::AWS.ec2(region: region, credentials: nil).describe_nat_gateways(
536
+ MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_nat_gateways(
993
537
  filter: [
994
538
  {
995
539
  name: nat_filter_key,
@@ -1014,8 +558,8 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1014
558
 
1015
559
  deploy_id = nil
1016
560
  nat_name = nat_name.to_s if !nat_name.nil? and nat_name.class.to_s == "MU::Config::Tail"
1017
- nat_ip = nat_ip.to_s if !nat_ip.nil? and nat_ip.class.to_s == "MU::Config::Tail"
1018
561
  nat_cloud_id = nat_cloud_id.to_s if !nat_cloud_id.nil? and nat_cloud_id.class.to_s == "MU::Config::Tail"
562
+ nat_ip = nat_ip.to_s if !nat_ip.nil? and nat_ip.class.to_s == "MU::Config::Tail"
1019
563
  nat_tag_key = nat_tag_key.to_s if !nat_tag_key.nil? and nat_tag_key.class.to_s == "MU::Config::Tail"
1020
564
  nat_tag_value = nat_tag_value.to_s if !nat_tag_value.nil? and nat_tag_value.class.to_s == "MU::Config::Tail"
1021
565
 
@@ -1042,8 +586,8 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1042
586
  found.each { |nat|
1043
587
  # Try some AWS-specific criteria
1044
588
  cloud_desc = nat.cloud_desc
1045
- if !nat_host_ip.nil? and
1046
- (cloud_desc.private_ip_address == nat_host_ip or cloud_desc.public_ip_address == nat_host_ip)
589
+ if !nat_ip.nil? and
590
+ (cloud_desc.private_ip_address == nat_ip or cloud_desc.public_ip_address == nat_ip)
1047
591
  return nat
1048
592
  elsif cloud_desc.vpc_id == @cloud_id
1049
593
  # XXX Strictly speaking we could have different NATs in different
@@ -1090,7 +634,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1090
634
  if instance.nil?
1091
635
  begin
1092
636
  instance = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
1093
- rescue NoMethodError, Aws::EC2::Errors::InvalidInstanceIDNotFound => e
637
+ rescue NoMethodError, Aws::EC2::Errors::InvalidInstanceIDNotFound
1094
638
  MU.log "Failed to identify instance #{instance_id} in MU::Cloud::AWS::VPC.getInstanceSubnets", MU::WARN
1095
639
  return []
1096
640
  end
@@ -1133,20 +677,19 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1133
677
  my_subnets = MU::Cloud::AWS::VPC.getInstanceSubnets(instance: MU.myCloudDescriptor)
1134
678
  target_subnets = MU::Cloud::AWS::VPC.getInstanceSubnets(instance: target_instance, region: region, credentials: credentials)
1135
679
 
1136
- resp = nil
1137
680
  my_subnets_key = my_subnets.join(",")
1138
681
  target_subnets_key = target_subnets.join(",")
1139
682
  MU::Cloud::AWS::VPC.update_route_tables_cache(my_subnets_key, region: MU.myRegion)
1140
683
  MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, region: region, credentials: credentials)
1141
684
 
1142
- if MU::Cloud::AWS::VPC.have_route_peered_vpc?(my_subnets_key, target_subnets_key, instance_id)
685
+ if MU::Cloud::AWS::VPC.can_route_to_master_peer?(my_subnets_key, target_subnets_key, instance_id)
1143
686
  return true
1144
687
  else
1145
688
  # The cache can be out of date at times, check again without it
1146
689
  MU::Cloud::AWS::VPC.update_route_tables_cache(my_subnets_key, use_cache: false, region: MU.myRegion)
1147
690
  MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, use_cache: false, region: region, credentials: credentials)
1148
691
 
1149
- return MU::Cloud::AWS::VPC.have_route_peered_vpc?(my_subnets_key, target_subnets_key, instance_id)
692
+ return MU::Cloud::AWS::VPC.can_route_to_master_peer?(my_subnets_key, target_subnets_key, instance_id)
1150
693
  end
1151
694
 
1152
695
  end
@@ -1185,7 +728,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1185
728
  # @param target_subnets_key [String]: The subnet/subnets on the other side of the peered VPC.
1186
729
  # @param instance_id [String]: The instance ID in the target subnet/subnets.
1187
730
  # @return [Boolean]
1188
- def self.have_route_peered_vpc?(source_subnets_key, target_subnets_key, instance_id)
731
+ def self.can_route_to_master_peer?(source_subnets_key, target_subnets_key, instance_id)
1189
732
  my_routes = []
1190
733
  vpc_peer_mapping = {}
1191
734
 
@@ -1279,30 +822,39 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1279
822
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
1280
823
  # @param region [String]: The cloud provider region
1281
824
  # @return [void]
1282
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
825
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
826
+ MU.log "AWS::VPC.cleanup: need to support flags['known']", MU::DEBUG, details: flags
827
+
1283
828
  tagfilters = [
1284
- {name: "tag:MU-ID", values: [MU.deploy_id]}
829
+ {name: "tag:MU-ID", values: [deploy_id]}
1285
830
  ]
1286
831
  if !ignoremaster
1287
832
  tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
1288
833
  end
1289
834
 
1290
835
  vpcs = []
1291
- retries = 0
1292
- begin
836
+ MU.retrier([Aws::EC2::Errors::InvalidVpcIDNotFound], wait: 5) {
1293
837
  resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs(filters: tagfilters, max_results: 1000).vpcs
1294
838
  vpcs = resp if !resp.empty?
1295
- rescue Aws::EC2::Errors::InvalidVpcIDNotFound => e
1296
- if retries < 5
1297
- sleep 5
1298
- retries += 1
1299
- retry
1300
- end
1301
- end
839
+ }
840
+
841
+ # resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
842
+ # filters: [
843
+ # {
844
+ # name: "requester-vpc-info.vpc-id",
845
+ # values: [@cloud_id]
846
+ # },
847
+ # {
848
+ # name: "accepter-vpc-info.vpc-id",
849
+ # values: [peer_id.to_s]
850
+ # }
851
+ # ]
852
+ # )
1302
853
 
1303
854
  if !vpcs.empty?
1304
855
  gwthreads = []
1305
856
  vpcs.each { |vpc|
857
+ purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
1306
858
  # NAT gateways don't have any tags, and we can't assign them a name. Lets find them based on a VPC ID
1307
859
  gwthreads << Thread.new {
1308
860
  purge_nat_gateways(noop, vpc_id: vpc.vpc_id, region: region, credentials: credentials)
@@ -1322,17 +874,17 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1322
874
  purge_vpcs(noop, tagfilters, region: region, credentials: credentials)
1323
875
  purge_dhcpopts(noop, tagfilters, region: region, credentials: credentials)
1324
876
 
1325
- unless noop
1326
- MU::Cloud::AWS.iam.list_roles.roles.each{ |role|
1327
- match_string = "#{MU.deploy_id}.*TRAFFIC-LOG"
1328
- }
1329
- end
877
+ # unless noop
878
+ # MU::Cloud::AWS.iam.list_roles.roles.each{ |role|
879
+ # match_string = "#{deploy_id}.*TRAFFIC-LOG"
880
+ # }
881
+ # end
1330
882
  end
1331
883
 
1332
884
  # Cloud-specific configuration properties.
1333
- # @param config [MU::Config]: The calling MU::Config object
885
+ # @param _config [MU::Config]: The calling MU::Config object
1334
886
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
1335
- def self.schema(config)
887
+ def self.schema(_config)
1336
888
  toplevel_required = []
1337
889
  # Flow Logs can be declared at the VPC level or the subnet level
1338
890
  flowlogs = {
@@ -1378,11 +930,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1378
930
  logdesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
1379
931
  # logdesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
1380
932
  configurator.insertKitten(logdesc, "logs")
1381
- vpc['dependencies'] ||= []
1382
- vpc['dependencies'] << {
1383
- "type" => "log",
1384
- "name" => vpc['name']+"loggroup"
1385
- }
933
+ MU::Config.addDependency(vpc, vpc['name']+"loggroup", "log")
1386
934
 
1387
935
  roledesc = {
1388
936
  "name" => vpc['name']+"logrole",
@@ -1420,15 +968,10 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1420
968
  roledesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
1421
969
  roledesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
1422
970
  configurator.insertKitten(roledesc, "roles")
1423
- vpc['dependencies'] ||= []
1424
- vpc['dependencies'] << {
1425
- "type" => "role",
1426
- "name" => vpc['name']+"logrole"
1427
- }
971
+ MU::Config.addDependency(vpc, vpc['name']+"logrole", "role")
1428
972
  end
1429
973
 
1430
974
  subnet_routes = Hash.new
1431
- public_routes = Array.new
1432
975
 
1433
976
  if vpc['subnets']
1434
977
  vpc['subnets'].each { |subnet|
@@ -1476,10 +1019,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1476
1019
  subnet_routes[table['name']].each { |subnet|
1477
1020
  nat_routes[subnet] = route['nat_host_name']
1478
1021
  }
1479
- vpc['dependencies'] << {
1480
- "type" => "server",
1481
- "name" => route['nat_host_name']
1482
- }
1022
+ MU::Config.addDependency(vpc, route['nat_host_name'], "server", no_create_wait: true)
1483
1023
  elsif route['gateway'] == '#NAT'
1484
1024
  vpc['create_nat_gateway'] = true
1485
1025
  private_rtbs << table['name']
@@ -1492,17 +1032,11 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1492
1032
  if route['gateway'] == '#INTERNET'
1493
1033
  if table['name'] == subnet['route_table']
1494
1034
  subnet['is_public'] = true
1495
- if vpc['create_nat_gateway']
1496
- if vpc['nat_gateway_multi_az']
1497
- subnet['create_nat_gateway'] = true
1498
- else
1499
- if nat_gateway_added
1500
- subnet['create_nat_gateway'] = false
1501
- else
1502
- subnet['create_nat_gateway'] = true
1503
- nat_gateway_added = true
1504
- end
1505
- end
1035
+ if vpc['create_nat_gateway'] and (vpc['nat_gateway_multi_az'] or !nat_gateway_added)
1036
+ subnet['create_nat_gateway'] = true
1037
+ nat_gateway_added = true
1038
+ else
1039
+ subnet['create_nat_gateway'] = false
1506
1040
  end
1507
1041
  else
1508
1042
  subnet['is_public'] = false
@@ -1600,9 +1134,11 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1600
1134
  @my_visible_cidrs[subnets]
1601
1135
  end
1602
1136
 
1603
- private
1604
1137
 
1605
1138
  # List the route tables for each subnet in the given VPC
1139
+ # @param vpc_id [String]:
1140
+ # @param region [String]:
1141
+ # @param credentials [String]:
1606
1142
  def self.listAllSubnetRouteTables(vpc_id, region: MU.curRegion, credentials: nil)
1607
1143
  resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_subnets(
1608
1144
  filters: [
@@ -1645,6 +1181,299 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1645
1181
  return table_ids.uniq
1646
1182
  end
1647
1183
 
1184
+ # Remove all network interfaces associated with the currently loaded deployment.
1185
+ # @param noop [Boolean]: If true, will only print what would be done
1186
+ # @param filters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
1187
+ # @param region [String]: The cloud provider region
1188
+ # @return [void]
1189
+ def self.purge_interfaces(noop = false, filters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
1190
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
1191
+ filters: filters
1192
+ )
1193
+ ifaces = resp.data.network_interfaces
1194
+
1195
+ return if ifaces.nil? or ifaces.size == 0
1196
+
1197
+ ifaces.each { |iface|
1198
+ if iface.vpc_id
1199
+ default_sg = MU::Cloud::AWS::VPC.getDefaultSg(iface.vpc_id, region: region, credentials: credentials)
1200
+ if default_sg and (iface.groups.size > 1 or (iface.groups.size == 1 and iface.groups.first.group_id != default_sg))
1201
+ MU.log "Removing extra security groups from ENI #{iface.network_interface_id}"
1202
+ if !noop
1203
+ begin
1204
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_network_interface_attribute(
1205
+ network_interface_id: iface.network_interface_id,
1206
+ groups: [default_sg]
1207
+ )
1208
+ rescue ::Aws::EC2::Errors::AuthFailure
1209
+ MU.log "Permission denied attempting to trim Security Group list for #{iface.network_interface_id}", MU::WARN, details: iface.groups.map { |g| g.group_name }.join(",")+" => default"
1210
+ end
1211
+ end
1212
+ end
1213
+ end
1214
+ begin
1215
+ if iface.attachment and iface.attachment.status == "attached"
1216
+ MU.log "Detaching Network Interface #{iface.network_interface_id} from #{iface.attachment.instance_owner_id}"
1217
+ tried_lbs = false
1218
+ begin
1219
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).detach_network_interface(attachment_id: iface.attachment.attachment_id) if !noop
1220
+ rescue Aws::EC2::Errors::OperationNotPermitted => e
1221
+ MU.log "Can't detach #{iface.network_interface_id}: #{e.message}", MU::WARN, details: iface.attachment
1222
+ next
1223
+ rescue Aws::EC2::Errors::IncorrectState => e
1224
+ MU.log e.message, MU::WARN
1225
+ sleep 5
1226
+ retry
1227
+ rescue Aws::EC2::Errors::InvalidAttachmentIDNotFound => e
1228
+ # suits me just fine
1229
+ rescue Aws::EC2::Errors::AuthFailure => e
1230
+ if !tried_lbs and iface.attachment.instance_owner_id == "amazon-elb"
1231
+ MU::Cloud.resourceClass("AWS", "LoadBalancer").cleanup(
1232
+ noop: noop,
1233
+ region: region,
1234
+ credentials: credentials,
1235
+ flags: {"vpc_id" => iface.vpc_id}
1236
+ )
1237
+ tried_lbs = true
1238
+ retry
1239
+ end
1240
+ MU.log e.message, MU::ERR, details: iface.attachment
1241
+ end
1242
+ end
1243
+ MU.log "Deleting Network Interface #{iface.network_interface_id}"
1244
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_network_interface(network_interface_id: iface.network_interface_id) if !noop
1245
+ rescue Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound
1246
+ # ok then!
1247
+ rescue Aws::EC2::Errors::InvalidParameterValue => e
1248
+ MU.log e.message, MU::ERR, details: iface
1249
+ end
1250
+ }
1251
+ end
1252
+
1253
+ # Fetch the group id of the +default+ security group for the given VPC
1254
+ # @param vpc_id [String]
1255
+ # @param region [String]
1256
+ # @param credentials [String]
1257
+ # @return [String]
1258
+ def self.getDefaultSg(vpc_id, region: MU.curRegion, credentials: nil)
1259
+ default_sg_resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
1260
+ filters: [
1261
+ { name: "group-name", values: ["default"] },
1262
+ { name: "vpc-id", values: [vpc_id] }
1263
+ ]
1264
+ ).security_groups
1265
+ if default_sg_resp and default_sg_resp.size == 1
1266
+ return default_sg_resp.first.group_id
1267
+ end
1268
+ nil
1269
+ end
1270
+
1271
+ # Try to locate the default VPC for a region, and return a BoK-style
1272
+ # config fragment for something that might want to live in it.
1273
+ def self.defaultVpc(region, credentials)
1274
+ cfg_fragment = nil
1275
+ MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs.vpcs.each { |vpc|
1276
+ if vpc.is_default
1277
+ cfg_fragment = {
1278
+ "id" => vpc.vpc_id,
1279
+ "cloud" => "AWS",
1280
+ "region" => region,
1281
+ "credentials" => credentials
1282
+ }
1283
+ cfg_fragment['subnets'] = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_subnets(
1284
+ filters: [
1285
+ {
1286
+ name: "vpc-id",
1287
+ values: [vpc.vpc_id]
1288
+ }
1289
+ ]
1290
+ ).subnets.map { |s| { "subnet_id" => s.subnet_id } }
1291
+ break
1292
+ end
1293
+ }
1294
+
1295
+ cfg_fragment
1296
+ end
1297
+
1298
+ # Return a {MU::Config::Ref} that indicates this VPC.
1299
+ # @param subnet_ids [Array<String>]: Optional list of subnet ids with which to infer a +subnet_pref+ parameter.
1300
+ # @return [MU::Config::Ref]
1301
+ def getReference(subnet_ids = [])
1302
+ have_private = have_public = false
1303
+ subnets.each { |s|
1304
+ next if subnet_ids and !subnet_ids.empty? and !subnet_ids.include?(s.cloud_id)
1305
+ if s.private?
1306
+ have_private = true
1307
+ else
1308
+ have_public = true
1309
+ end
1310
+ }
1311
+ subnet_pref = if have_private == have_public
1312
+ "any"
1313
+ elsif have_private
1314
+ "all_private"
1315
+ elsif have_public
1316
+ "all_public"
1317
+ end
1318
+ MU::Config::Ref.get(
1319
+ id: @cloud_id,
1320
+ cloud: "AWS",
1321
+ credentials: @credentials,
1322
+ region: @config['region'],
1323
+ type: "vpcs",
1324
+ subnet_pref: subnet_pref
1325
+ )
1326
+ end
1327
+
1328
+ private
1329
+
1330
+ def peerWith(peer)
1331
+ peer_ref = MU::Config::Ref.get(peer['vpc'])
1332
+ peer_obj = peer_ref.kitten
1333
+ peer_id = peer_ref.kitten.cloud_id
1334
+ if peer_id == @cloud_id
1335
+ MU.log "#{@mu_name} attempted to peer with itself (#{@cloud_id})", MU::ERR, details: peer
1336
+ raise "#{@mu_name} attempted to peer with itself (#{@cloud_id})"
1337
+ end
1338
+
1339
+ if peer_obj and peer_obj.config['peers']
1340
+ peer_obj.config['peers'].each { |peerpeer|
1341
+ if peerpeer['vpc']['name'] == @config['name'] and
1342
+ (peer['vpc']['name'] <=> @config['name']) == -1
1343
+ MU.log "VPCs #{peer['vpc']['name']} and #{@config['name']} both declare mutual peering connection, ignoring #{@config['name']}'s redundant declaration", MU::DEBUG
1344
+ return
1345
+ # XXX and if deploy_id matches or is unset
1346
+ end
1347
+ }
1348
+
1349
+ peer['account'] ||= MU::Cloud::AWS.credToAcct(peer_obj.credentials)
1350
+ end
1351
+
1352
+ peer['account'] ||= MU::Cloud::AWS.account_number
1353
+
1354
+ # See if the peering connection exists before we bother
1355
+ # creating it.
1356
+ resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
1357
+ filters: [
1358
+ {
1359
+ name: "requester-vpc-info.vpc-id",
1360
+ values: [@cloud_id]
1361
+ },
1362
+ {
1363
+ name: "accepter-vpc-info.vpc-id",
1364
+ values: [peer_id.to_s]
1365
+ }
1366
+ ]
1367
+ )
1368
+
1369
+ peering_id = if !resp or !resp.vpc_peering_connections or
1370
+ resp.vpc_peering_connections.empty?
1371
+
1372
+ MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id} in account #{MU::Cloud::AWS.credToAcct(@config['credentials'])}) to #{peer_id} in account #{peer['account']}", details: peer
1373
+ resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_peering_connection(
1374
+ vpc_id: @cloud_id,
1375
+ peer_vpc_id: peer_id,
1376
+ peer_owner_id: peer['account'],
1377
+ peer_region: peer_obj.config['region']
1378
+ )
1379
+ resp.vpc_peering_connection.vpc_peering_connection_id
1380
+ else
1381
+ resp.vpc_peering_connections.first.vpc_peering_connection_id
1382
+ end
1383
+
1384
+ peering_name = @deploy.getResourceName(@config['name']+"-PEER-"+peer_id)
1385
+
1386
+ tag_me(peering_id, peering_name)
1387
+
1388
+ # Create routes to our new friend.
1389
+ MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @config['region'], credentials: @config['credentials']).each { |rtb_id|
1390
+ my_route_config = {
1391
+ :route_table_id => rtb_id,
1392
+ :destination_cidr_block => peer_obj.cloud_desc.cidr_block,
1393
+ :vpc_peering_connection_id => peering_id
1394
+ }
1395
+ rtbdesc = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
1396
+ route_table_ids: [rtb_id]
1397
+ ).route_tables.first
1398
+ already_exists = false
1399
+ rtbdesc.routes.each { |r|
1400
+ if r.destination_cidr_block == peer_obj.cloud_desc.cidr_block
1401
+ if r.vpc_peering_connection_id != peering_id
1402
+ MU.log "Attempt to create duplicate route to #{peer_obj.cloud_desc.cidr_block} from VPC #{@config['name']}", MU::ERR, details: r
1403
+ raise MuError, "Can't create route via #{peering_id}, a route to #{peer_obj.cloud_desc.cidr_block} already exists"
1404
+ else
1405
+ already_exists = true
1406
+ end
1407
+ end
1408
+ }
1409
+ next if already_exists
1410
+
1411
+ MU.log "Creating peering route to #{peer_obj.cloud_desc.cidr_block} in #{peer['vpc']['region']} from VPC #{@config['name']} in #{@config['region']}"
1412
+ resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(my_route_config)
1413
+ } # MU::Cloud::AWS::VPC.listAllSubnetRouteTables
1414
+
1415
+ can_auto_accept = ((!peer_obj.nil? and !peer_obj.deploydata.nil? and peer_obj.deploydata['auto_accept_peers']) or $MU_CFG['allow_invade_foreign_vpcs'])
1416
+
1417
+ cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
1418
+ vpc_peering_connection_ids: [peering_id]
1419
+ ).vpc_peering_connections.first
1420
+
1421
+ loop_if = Proc.new {
1422
+ cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
1423
+ vpc_peering_connection_ids: [peering_id]
1424
+ ).vpc_peering_connections.first
1425
+ ((can_auto_accept and cnxn.status.code == "pending-acceptance") or (cnxn.status.code != "active" and cnxn.status.code != "pending-acceptance"))
1426
+ }
1427
+
1428
+ MU.retrier(wait: 5, loop_if: loop_if, ignoreme: [Aws::EC2::Errors::VpcPeeringConnectionAlreadyExists, Aws::EC2::Errors::RouteAlreadyExists]) {
1429
+ if cnxn.status.code == "pending-acceptance"
1430
+ if can_auto_accept
1431
+ MU.log "Auto-accepting peering connection #{peering_id} from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id}", MU::NOTICE
1432
+ MU::Cloud::AWS.ec2(region: peer_obj.config['region'], credentials: peer['account']).accept_vpc_peering_connection(
1433
+ vpc_peering_connection_id: peering_id,
1434
+ )
1435
+
1436
+ # Create routes back from our new friend to us.
1437
+ MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_id, region: peer_obj.config['region'], credentials: peer['account']).uniq.each { |rtb_id|
1438
+ peer_route_config = {
1439
+ :route_table_id => rtb_id,
1440
+ :destination_cidr_block => @config['ip_block'],
1441
+ :vpc_peering_connection_id => peering_id
1442
+ }
1443
+ resp = MU::Cloud::AWS.ec2(region: peer_obj.config['region'], credentials: peer['account']).create_route(peer_route_config)
1444
+ }
1445
+ else
1446
+ MU.log "VPC #{peer_id} is not managed by this Mu server or is not configured to auto-accept peering requests. You must accept the peering request for '#{@config['name']}' (#{@cloud_id}) by hand.", MU::WARN, details: "In the AWS Console, go to VPC => Peering Connections and look in the Actions drop-down. You can also set 'Invade Foreign VPCs' to 'true' using mu-configure to auto-accept all peering connections within this account, regardless of whether this Mu server owns the VPCs. This setting is per-user."
1447
+ end
1448
+ end
1449
+
1450
+ if ["failed", "rejected", "expired", "deleted"].include?(cnxn.status.code)
1451
+ MU.log "VPC peering connection from VPC #{@config['name']} (#{@cloud_id} in #{@config['region']}) to #{peer_id} in #{peer_obj.config['region']} #{cnxn.status.code}: #{cnxn.status.message}", MU::ERR
1452
+ begin
1453
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).delete_vpc_peering_connection(
1454
+ vpc_peering_connection_id: peering_id
1455
+ )
1456
+ rescue Aws::EC2::Errors::InvalidStateTransition
1457
+ # XXX apparently this is normal?
1458
+ end
1459
+ raise MuError, "VPC peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id} #{cnxn.status.code}: #{cnxn.status.message}"
1460
+ end
1461
+
1462
+ }
1463
+
1464
+ end
1465
+
1466
+ def tag_me(resource_id = @cloud_id, name = @mu_name)
1467
+ MU::Cloud::AWS.createStandardTags(
1468
+ resource_id,
1469
+ region: @config['region'],
1470
+ credentials: @config['credentials'],
1471
+ optional: @config['optional_tags'],
1472
+ nametag: name,
1473
+ othertags: @config['tags']
1474
+ )
1475
+ end
1476
+
1648
1477
  # Helper method for manufacturing route tables. Expect to be called from
1649
1478
  # {MU::Cloud::AWS::VPC#create} or {MU::Cloud::AWS::VPC#groom}.
1650
1479
  # @param rtb [Hash]: A route table description parsed through {MU::Config::BasketofKittens::vpcs::route_tables}.
@@ -1656,21 +1485,9 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1656
1485
  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route_table(vpc_id: vpc_id).route_table
1657
1486
  route_table_id = rtb['route_table_id'] = resp.route_table_id
1658
1487
  sleep 5
1659
- MU::MommaCat.createTag(route_table_id, "Name", vpc_name+"-"+rtb['name'].upcase, credentials: @config['credentials'])
1660
-
1661
- if @config['tags']
1662
- @config['tags'].each { |tag|
1663
- MU::MommaCat.createTag(route_table_id, tag['key'], tag['value'], credentials: @config['credentials'])
1664
- }
1665
- end
1666
1488
 
1667
- if @config['optional_tags']
1668
- MU::MommaCat.listOptionalTags.each { |key, value|
1669
- MU::MommaCat.createTag(route_table_id, key, value, region: @config['region'], credentials: @config['credentials'])
1670
- }
1671
- end
1489
+ tag_me(route_table_id, vpc_name+"-"+rtb['name'].upcase)
1672
1490
 
1673
- MU::Cloud::AWS.createStandardTags(route_table_id, credentials: @config['credentials'])
1674
1491
  rtb['routes'].each { |route|
1675
1492
  if route['nat_host_id'].nil? and route['nat_host_name'].nil?
1676
1493
  route_config = {
@@ -1693,7 +1510,6 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1693
1510
  return rtb
1694
1511
  end
1695
1512
 
1696
-
1697
1513
  # Remove all network gateways associated with the currently loaded deployment.
1698
1514
  # @param noop [Boolean]: If true, will only print what would be done
1699
1515
  # @param region [String]: The cloud provider region
@@ -1746,6 +1562,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1746
1562
  }
1747
1563
  return nil
1748
1564
  end
1565
+ private_class_method :purge_gateways
1749
1566
 
1750
1567
  # Remove all NAT gateways associated with the VPC of the currently loaded deployment.
1751
1568
  # @param noop [Boolean]: If true, will only print what would be done
@@ -1763,36 +1580,25 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1763
1580
  ).nat_gateways
1764
1581
 
1765
1582
  threads = []
1766
- parent_thread_id = Thread.current.object_id
1583
+
1767
1584
  if !gateways.empty?
1768
1585
  gateways.each { |gateway|
1586
+ next if noop
1587
+ MU.log "Deleting NAT Gateway #{gateway.nat_gateway_id}"
1769
1588
  threads << Thread.new {
1770
- MU.dupGlobals(parent_thread_id)
1771
- MU.log "Deleting NAT Gateway #{gateway.nat_gateway_id}"
1772
- if !noop
1773
- begin
1774
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_nat_gateway(nat_gateway_id: gateway.nat_gateway_id)
1775
- resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
1589
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_nat_gateway(nat_gateway_id: gateway.nat_gateway_id)
1590
+
1591
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
1592
+
1593
+ loop_if = Proc.new {
1594
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
1595
+ (resp.state != "deleted" and resp.state != "failed")
1596
+ }
1597
+
1598
+ MU.retrier([Aws::EmptyStructure, NoMethodError], ignoreme: [Aws::EC2::Errors::NatGatewayMalformed, Aws::EC2::Errors::NatGatewayNotFound], max: 50, loop_if: loop_if) { |retries, _wait|
1599
+ MU.log "Waiting for nat gateway #{gateway.nat_gateway_id} to delete" if retries % 3 == 0
1600
+ }
1776
1601
 
1777
- attempts = 0
1778
- while resp.state != "deleted" and resp.state != "failed"
1779
- MU.log "Waiting for nat gateway #{gateway.nat_gateway_id} to delete" if attempts % 2 == 0
1780
- sleep 30
1781
- begin
1782
- resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
1783
- rescue Aws::EmptyStructure, NoMethodError
1784
- sleep 5
1785
- retry
1786
- rescue Aws::EC2::Errors::NatGatewayNotFound
1787
- MU.log "NAT gateway #{gateway.nat_gateway_id} already deleted", MU::NOTICE
1788
- end
1789
- MU.log "Timed out while waiting for NAT Gateway to delete #{gateway.nat_gateway_id}: #{resp}", MU::WARN if attempts > 50
1790
- attempts += 1
1791
- end
1792
- rescue Aws::EC2::Errors::NatGatewayMalformed
1793
- MU.log "NAT Gateway #{gateway.nat_gateway_id} was already deleted", MU::NOTICE
1794
- end
1795
- end
1796
1602
  }
1797
1603
  }
1798
1604
  end
@@ -1803,6 +1609,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1803
1609
 
1804
1610
  return nil
1805
1611
  end
1612
+ private_class_method :purge_nat_gateways
1806
1613
 
1807
1614
  # Remove all VPC endpoints associated with the VPC of the currently loaded deployment.
1808
1615
  # @param noop [Boolean]: If true, will only print what would be done
@@ -1820,38 +1627,21 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1820
1627
  ).vpc_endpoints
1821
1628
 
1822
1629
  threads = []
1823
- parent_thread_id = Thread.current.object_id
1630
+
1824
1631
  if !vpc_endpoints.empty?
1825
1632
  vpc_endpoints.each { |endpoint|
1633
+ MU.log "Deleting VPC endpoint #{endpoint.vpc_endpoint_id}"
1634
+ next if noop
1826
1635
  threads << Thread.new {
1827
- MU.dupGlobals(parent_thread_id)
1828
- MU.log "Deleting VPC endpoint #{endpoint.vpc_endpoint_id}"
1829
- if !noop
1830
- begin
1831
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id])
1832
- resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
1833
-
1834
- attempts = 0
1835
- while resp.state != "deleted"
1836
- MU.log "Waiting for VPC endpoint #{endpoint.vpc_endpoint_id} to delete" if attempts % 5 == 0
1837
- sleep 30
1838
- begin
1839
- resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
1840
- rescue Aws::EmptyStructure, NoMethodError
1841
- sleep 5
1842
- retry
1843
- rescue Aws::EC2::Errors::InvalidVpcEndpointIdNotFound
1844
- MU.log "VPC endpoint #{endpoint.vpc_endpoint_id} already deleted", MU::NOTICE
1845
- end
1846
- MU.log "Timed out while waiting for VPC endpoint to delete #{endpoint.vpc_endpoint_id}: #{resp}", MU::WARN if attempts > 50
1847
- attempts += 1
1848
- end
1849
- rescue Aws::EC2::Errors::VpcEndpointIdMalformed
1850
- MU.log "VPC endpoint #{endpoint.vpc_endpoint_id} was already deleted", MU::NOTICE
1851
- rescue Aws::EC2::Errors::InvalidVpcEndpointIdNotFound
1852
- MU.log "VPC endpoint #{endpoint.vpc_endpoint_id} already deleted", MU::NOTICE
1853
- end
1854
- end
1636
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id])
1637
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
1638
+ loop_if = Proc.new {
1639
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
1640
+ resp.state != "deleted"
1641
+ }
1642
+ MU.retrier([Aws::EmptyStructure, NoMethodError], ignoreme: [Aws::EC2::Errors::InvalidVpcEndpointIdNotFound, Aws::EC2::Errors::VpcEndpointIdMalformed], max: 20, wait: 10, loop_if: loop_if) { |retries, _wait|
1643
+ MU.log "Waiting for VPC endpoint #{endpoint.vpc_endpoint_id} to delete" if retries % 5 == 0
1644
+ }
1855
1645
  }
1856
1646
  }
1857
1647
  end
@@ -1862,6 +1652,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1862
1652
 
1863
1653
  return nil
1864
1654
  end
1655
+ private_class_method :purge_endpoints
1865
1656
 
1866
1657
  # Remove all route tables associated with the currently loaded deployment.
1867
1658
  # @param noop [Boolean]: If true, will only print what would be done
@@ -1882,7 +1673,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1882
1673
  MU.log "Deleting Network Interface #{route.network_interface_id}"
1883
1674
  begin
1884
1675
  MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_network_interface(network_interface_id: route.network_interface_id) if !noop
1885
- rescue Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound => e
1676
+ rescue Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound
1886
1677
  MU.log "Network Interface #{route.network_interface_id} has already been deleted", MU::WARN
1887
1678
  end
1888
1679
  end
@@ -1902,9 +1693,9 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1902
1693
  table.associations.each { |assoc|
1903
1694
  begin
1904
1695
  MU::Cloud::AWS.ec2(credentials: credentials, region: region).disassociate_route_table(association_id: assoc.route_table_association_id) if !noop
1905
- rescue Aws::EC2::Errors::InvalidAssociationIDNotFound => e
1696
+ rescue Aws::EC2::Errors::InvalidAssociationIDNotFound
1906
1697
  MU.log "Route table association #{assoc.route_table_association_id} already removed", MU::WARN
1907
- rescue Aws::EC2::Errors::InvalidParameterValue => e
1698
+ rescue Aws::EC2::Errors::InvalidParameterValue
1908
1699
  # normal and ignorable with the default route table
1909
1700
  can_delete = false
1910
1701
  next
@@ -1920,124 +1711,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
1920
1711
  }
1921
1712
  return nil
1922
1713
  end
1923
-
1924
-
1925
- # Remove all network interfaces associated with the currently loaded deployment.
1926
- # @param noop [Boolean]: If true, will only print what would be done
1927
- # @param filters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
1928
- # @param region [String]: The cloud provider region
1929
- # @return [void]
1930
- def self.purge_interfaces(noop = false, filters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
1931
- resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
1932
- filters: filters
1933
- )
1934
- ifaces = resp.data.network_interfaces
1935
-
1936
- return if ifaces.nil? or ifaces.size == 0
1937
-
1938
- ifaces.each { |iface|
1939
- if iface.vpc_id
1940
- default_sg_resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
1941
- filters: [
1942
- { name: "group-name", values: ["default"] },
1943
- { name: "vpc-id", values: [iface.vpc_id] }
1944
- ]
1945
- ).security_groups
1946
- if default_sg_resp and default_sg_resp.size == 1
1947
- default_sg = default_sg_resp.first.group_id
1948
- if iface.groups.size != 1 or
1949
- iface.groups.first.group_id != default_sg
1950
- MU.log "Removing extra security groups from ENI #{iface.network_interface_id}"
1951
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_network_interface_attribute(
1952
- network_interface_id: iface.network_interface_id,
1953
- groups: [default_sg]
1954
- )
1955
- end
1956
- end
1957
- end
1958
- begin
1959
- if iface.attachment and iface.attachment.status == "attached"
1960
- MU.log "Detaching Network Interface #{iface.network_interface_id} from #{iface.attachment.instance_owner_id}"
1961
- tried_lbs = false
1962
- begin
1963
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).detach_network_interface(attachment_id: iface.attachment.attachment_id) if !noop
1964
- rescue Aws::EC2::Errors::OperationNotPermitted => e
1965
- MU.log "Can't detach #{iface.network_interface_id}: #{e.message}", MU::WARN, details: iface.attachment
1966
- next
1967
- rescue Aws::EC2::Errors::InvalidAttachmentIDNotFound => e
1968
- # suits me just fine
1969
- rescue Aws::EC2::Errors::AuthFailure => e
1970
- if !tried_lbs and iface.attachment.instance_owner_id == "amazon-elb"
1971
- MU::Cloud::AWS::LoadBalancer.cleanup(
1972
- noop: noop,
1973
- region: region,
1974
- credentials: credentials,
1975
- flags: {"vpc_id" => iface.vpc_id}
1976
- )
1977
- tried_lbs = true
1978
- retry
1979
- end
1980
- MU.log e.message, MU::ERR, details: iface.attachment
1981
- end
1982
- end
1983
- MU.log "Deleting Network Interface #{iface.network_interface_id}"
1984
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_network_interface(network_interface_id: iface.network_interface_id) if !noop
1985
- rescue Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound => e
1986
- # ok then!
1987
- rescue Aws::EC2::Errors::InvalidParameterValue => e
1988
- MU.log e.message, MU::ERR, details: iface
1989
- end
1990
- }
1991
- end
1992
-
1993
- # Remove all subnets associated with the currently loaded deployment.
1994
- # @param noop [Boolean]: If true, will only print what would be done
1995
- # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
1996
- # @param region [String]: The cloud provider region
1997
- # @return [void]
1998
- def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
1999
- resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_subnets(
2000
- filters: tagfilters
2001
- )
2002
- subnets = resp.data.subnets
2003
-
2004
- return if subnets.nil? or subnets.size == 0
2005
-
2006
- retries = 0
2007
- subnets.each { |subnet|
2008
- MU.log "Deleting Subnet #{subnet.subnet_id}"
2009
- begin
2010
- if subnet.state != "available"
2011
- MU.log "Waiting for #{subnet.subnet_id} to be in a removable state...", MU::NOTICE
2012
- sleep 30
2013
- else
2014
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_subnet(subnet_id: subnet.subnet_id) if !noop
2015
- end
2016
- rescue Aws::EC2::Errors::DependencyViolation => e
2017
- # We're often stuck waiting for an RDS database or something else
2018
- # that takes 5-ever to delete.
2019
- if retries < 19
2020
- loglevel = (retries > 0 and (retries % 3) == 0) ? MU::NOTICE : MU::DEBUG
2021
- MU.log "#{e.message} (retry #{retries.to_s}/20)", loglevel
2022
- if loglevel == MU::NOTICE
2023
- MU::Cloud::AWS::VPC.purge_interfaces(noop, [{name: "subnet-id", values: [subnet.subnet_id]}], region: region, credentials: credentials)
2024
- end
2025
- sleep 30
2026
- retries = retries + 1
2027
- retry
2028
- elsif retries < 20
2029
- MU.log "#{e.message} (final attempt)", MU::WARN
2030
- sleep 60
2031
- retries = retries + 1
2032
- retry
2033
- else
2034
- raise e
2035
- end
2036
- rescue Aws::EC2::Errors::InvalidSubnetIDNotFound
2037
- next
2038
- end while subnet.state != "available"
2039
- }
2040
- end
1714
+ private_class_method :purge_routetables
2041
1715
 
2042
1716
  # Remove all DHCP options sets associated with the currently loaded
2043
1717
  # deployment.
@@ -2056,7 +1730,9 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
2056
1730
  sets.each { |optset|
2057
1731
  begin
2058
1732
  MU.log "Deleting DHCP Option Set #{optset.dhcp_options_id}"
2059
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_dhcp_options(dhcp_options_id: optset.dhcp_options_id)
1733
+ if !noop
1734
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_dhcp_options(dhcp_options_id: optset.dhcp_options_id)
1735
+ end
2060
1736
  rescue Aws::EC2::Errors::DependencyViolation => e
2061
1737
  MU.log e.inspect, MU::ERR
2062
1738
  # rescue Aws::EC2::Errors::InvalidSubnetIDNotFound
@@ -2065,6 +1741,62 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
2065
1741
  end
2066
1742
  }
2067
1743
  end
1744
+ private_class_method :purge_dhcpopts
1745
+
1746
+ def self.purge_peering_connections(noop, vpc_id, region: MU.curRegion, credentials: nil)
1747
+ my_peer_conns = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
1748
+ filters: [
1749
+ {
1750
+ name: "requester-vpc-info.vpc-id",
1751
+ values: [vpc_id]
1752
+ }
1753
+ ]
1754
+ ).vpc_peering_connections
1755
+ my_peer_conns.concat(MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
1756
+ filters: [
1757
+ {
1758
+ name: "accepter-vpc-info.vpc-id",
1759
+ values: [vpc_id]
1760
+ }
1761
+ ]
1762
+ ).vpc_peering_connections)
1763
+
1764
+ my_peer_conns.each { |cnxn|
1765
+ [cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
1766
+ MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
1767
+ begin
1768
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
1769
+ route_table_ids: [rtb_id]
1770
+ )
1771
+ rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
1772
+ next
1773
+ end
1774
+ resp.route_tables.each { |rtb|
1775
+ rtb.routes.each { |route|
1776
+ if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
1777
+ MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
1778
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
1779
+ route_table_id: rtb_id,
1780
+ destination_cidr_block: route.destination_cidr_block
1781
+ ) if !noop
1782
+ end
1783
+ }
1784
+ }
1785
+ }
1786
+ }
1787
+ MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
1788
+ begin
1789
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
1790
+ vpc_peering_connection_id: cnxn.vpc_peering_connection_id
1791
+ ) if !noop
1792
+ rescue Aws::EC2::Errors::InvalidStateTransition
1793
+ MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
1794
+ rescue Aws::EC2::Errors::OperationNotPermitted => e
1795
+ MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
1796
+ end
1797
+ }
1798
+ end
1799
+ private_class_method :purge_peering_connections
2068
1800
 
2069
1801
  # Remove all VPCs associated with the currently loaded deployment.
2070
1802
  # @param noop [Boolean]: If true, will only print what would be done
@@ -2073,177 +1805,39 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
2073
1805
  # @return [void]
2074
1806
  def self.purge_vpcs(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
2075
1807
  resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpcs(
2076
- filters: tagfilters
1808
+ filters: tagfilters
2077
1809
  )
2078
1810
 
2079
1811
  vpcs = resp.data.vpcs
2080
1812
  return if vpcs.nil? or vpcs.size == 0
2081
1813
 
2082
1814
  vpcs.each { |vpc|
2083
- my_peer_conns = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
2084
- filters: [
2085
- {
2086
- name: "requester-vpc-info.vpc-id",
2087
- values: [vpc.vpc_id]
2088
- }
2089
- ]
2090
- ).vpc_peering_connections
2091
- my_peer_conns.concat(MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
2092
- filters: [
2093
- {
2094
- name: "accepter-vpc-info.vpc-id",
2095
- values: [vpc.vpc_id]
2096
- }
2097
- ]
2098
- ).vpc_peering_connections)
2099
- my_peer_conns.each { |cnxn|
2100
-
2101
- [cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
2102
- MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
2103
- begin
2104
- resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
2105
- route_table_ids: [rtb_id]
2106
- )
2107
- rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound => e
2108
- next
2109
- end
2110
- resp.route_tables.each { |rtb|
2111
- rtb.routes.each { |route|
2112
- if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
2113
- MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
2114
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
2115
- route_table_id: rtb_id,
2116
- destination_cidr_block: route.destination_cidr_block
2117
- ) if !noop
2118
- end
2119
- }
2120
- }
2121
- }
2122
- }
2123
- MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
2124
- begin
2125
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
2126
- vpc_peering_connection_id: cnxn.vpc_peering_connection_id
2127
- ) if !noop
2128
- rescue Aws::EC2::Errors::InvalidStateTransition => e
2129
- MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
2130
- rescue Aws::EC2::Errors::OperationNotPermitted => e
2131
- MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
2132
- end
1815
+ purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
1816
+
1817
+ on_retry = Proc.new {
1818
+ MU::Cloud.resourceClass("AWS", "FirewallRule").cleanup(
1819
+ noop: noop,
1820
+ region: region,
1821
+ credentials: credentials,
1822
+ flags: { "vpc_id" => vpc.vpc_id }
1823
+ )
1824
+ purge_gateways(noop, tagfilters, region: region, credentials: credentials)
2133
1825
  }
2134
1826
 
2135
- retries = 0
2136
- begin
1827
+ MU.retrier([Aws::EC2::Errors::DependencyViolation], ignoreme: [Aws::EC2::Errors::InvalidVpcIDNotFound], max: 20, on_retry: on_retry) {
2137
1828
  MU.log "Deleting VPC #{vpc.vpc_id}"
2138
1829
  MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc(vpc_id: vpc.vpc_id) if !noop
2139
- rescue Aws::EC2::Errors::InvalidVpcIDNotFound
2140
- MU.log "VPC #{vpc.vpc_id} has already been deleted", MU::WARN
2141
- rescue Aws::EC2::Errors::DependencyViolation => e
2142
- if retries < 5
2143
- MU.log "#{vpc.vpc_id} in #{region} had hidden dependencies, will try to remove them", MU::NOTICE
2144
- retries += 1
2145
- # fry some common rogue resources
2146
- MU::Cloud::AWS::FirewallRule.cleanup(
2147
- noop: noop,
2148
- region: region,
2149
- credentials: credentials,
2150
- flags: { "vpc_id" => vpc.vpc_id }
2151
- )
2152
- purge_gateways(noop, tagfilters, region: region, credentials: credentials)
2153
- sleep 10
2154
- retry
2155
- else
2156
- MU.log "Failed to remove #{vpc.vpc_id} in #{region}: #{e.message}", MU::ERR
2157
- next
2158
- end
2159
- end
1830
+ }
2160
1831
 
2161
1832
  if !MU::Cloud::AWS.isGovCloud?(region)
2162
1833
  mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", region: region, credentials: credentials).values.first
2163
1834
  if !mu_zone.nil?
2164
- MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true, credentials: credentials)
1835
+ MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true, credentials: credentials)
2165
1836
  end
2166
1837
  end
2167
1838
  }
2168
1839
  end
2169
-
2170
- protected
2171
-
2172
- # Subnets are almost a first-class resource. So let's kinda sorta treat
2173
- # them like one. This should only be invoked on objects that already
2174
- # exists in the cloud layer.
2175
- class Subnet < MU::Cloud::AWS::VPC
2176
-
2177
- attr_reader :cloud_id
2178
- attr_reader :ip_block
2179
- attr_reader :mu_name
2180
- attr_reader :name
2181
- attr_reader :az
2182
- attr_reader :cloud_desc
2183
-
2184
- # @param parent [MU::Cloud::AWS::VPC]: The parent VPC of this subnet.
2185
- # @param config [Hash<String>]:
2186
- def initialize(parent, config)
2187
- @parent = parent
2188
- @config = MU::Config.manxify(config)
2189
- @cloud_id = config['cloud_id']
2190
- @mu_name = config['mu_name']
2191
- @name = config['name']
2192
- @deploydata = config # This is a dummy for the sake of describe()
2193
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [@cloud_id]).subnets.first
2194
- @az = resp.availability_zone
2195
- @ip_block = resp.cidr_block
2196
- @cloud_desc = resp # XXX this really isn't the cloud implementation's business
2197
-
2198
- end
2199
-
2200
- # Return the cloud identifier for the default route of this subnet.
2201
- def defaultRoute
2202
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
2203
- filters: [{name: "association.subnet-id", values: [@cloud_id]}]
2204
- )
2205
- if resp.route_tables.size == 0 # use default route table for the VPC
2206
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
2207
- filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
2208
- )
2209
- end
2210
- resp.route_tables.each { |route_table|
2211
- route_table.routes.each { |route|
2212
- if route.destination_cidr_block =="0.0.0.0/0" and route.state != "blackhole"
2213
- return route.instance_id if !route.instance_id.nil?
2214
- return route.gateway_id if !route.gateway_id.nil?
2215
- return route.vpc_peering_connection_id if !route.vpc_peering_connection_id.nil?
2216
- return route.network_interface_id if !route.network_interface_id.nil?
2217
- end
2218
- }
2219
- }
2220
- return nil
2221
- end
2222
-
2223
- # Is this subnet privately-routable only, or public?
2224
- # @return [Boolean]
2225
- def private?
2226
- return false if @cloud_id.nil?
2227
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
2228
- filters: [{name: "association.subnet-id", values: [@cloud_id]}]
2229
- )
2230
- if resp.route_tables.size == 0 # use default route table for the VPC
2231
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
2232
- filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
2233
- )
2234
- end
2235
- resp.route_tables.each { |route_table|
2236
- route_table.routes.each { |route|
2237
- return false if !route.gateway_id.nil? and route.gateway_id != "local" # you can have an IgW and route it to a subset of IPs instead of 0.0.0.0/0
2238
- if route.destination_cidr_block == "0.0.0.0/0"
2239
- return true if !route.instance_id.nil?
2240
- return true if route.nat_gateway_id
2241
- end
2242
- }
2243
- }
2244
- return true
2245
- end
2246
- end
1840
+ private_class_method :purge_vpcs
2247
1841
 
2248
1842
  end #class
2249
1843
  end #class