cloud-mu 3.1.3 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +21 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +4 -4
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +147 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +158 -111
  45. data/modules/mu/adoption.rb +404 -71
  46. data/modules/mu/cleanup.rb +221 -306
  47. data/modules/mu/cloud.rb +129 -1633
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +44 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +926 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +169 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +32 -3
  61. data/modules/mu/config/cache_cluster.rb +2 -2
  62. data/modules/mu/config/cdn.rb +100 -0
  63. data/modules/mu/config/collection.rb +4 -4
  64. data/modules/mu/config/container_cluster.rb +9 -4
  65. data/modules/mu/config/database.rb +84 -105
  66. data/modules/mu/config/database.yml +1 -2
  67. data/modules/mu/config/dnszone.rb +10 -9
  68. data/modules/mu/config/doc_helpers.rb +516 -0
  69. data/modules/mu/config/endpoint.rb +5 -4
  70. data/modules/mu/config/firewall_rule.rb +103 -4
  71. data/modules/mu/config/folder.rb +4 -4
  72. data/modules/mu/config/function.rb +19 -10
  73. data/modules/mu/config/group.rb +4 -4
  74. data/modules/mu/config/habitat.rb +4 -4
  75. data/modules/mu/config/job.rb +89 -0
  76. data/modules/mu/config/loadbalancer.rb +60 -14
  77. data/modules/mu/config/log.rb +4 -4
  78. data/modules/mu/config/msg_queue.rb +4 -4
  79. data/modules/mu/config/nosqldb.rb +4 -4
  80. data/modules/mu/config/notifier.rb +10 -21
  81. data/modules/mu/config/ref.rb +411 -0
  82. data/modules/mu/config/role.rb +4 -4
  83. data/modules/mu/config/schema_helpers.rb +509 -0
  84. data/modules/mu/config/search_domain.rb +4 -4
  85. data/modules/mu/config/server.rb +98 -71
  86. data/modules/mu/config/server.yml +1 -0
  87. data/modules/mu/config/server_pool.rb +5 -9
  88. data/modules/mu/config/storage_pool.rb +1 -1
  89. data/modules/mu/config/tail.rb +200 -0
  90. data/modules/mu/config/user.rb +4 -4
  91. data/modules/mu/config/vpc.rb +71 -27
  92. data/modules/mu/config/vpc.yml +0 -1
  93. data/modules/mu/defaults/AWS.yaml +91 -68
  94. data/modules/mu/defaults/Azure.yaml +1 -0
  95. data/modules/mu/defaults/Google.yaml +3 -2
  96. data/modules/mu/deploy.rb +43 -26
  97. data/modules/mu/groomer.rb +17 -2
  98. data/modules/mu/groomers/ansible.rb +188 -41
  99. data/modules/mu/groomers/chef.rb +116 -55
  100. data/modules/mu/logger.rb +127 -148
  101. data/modules/mu/master.rb +410 -2
  102. data/modules/mu/master/chef.rb +3 -4
  103. data/modules/mu/master/ldap.rb +3 -3
  104. data/modules/mu/master/ssl.rb +12 -3
  105. data/modules/mu/mommacat.rb +218 -2612
  106. data/modules/mu/mommacat/daemon.rb +403 -0
  107. data/modules/mu/mommacat/naming.rb +473 -0
  108. data/modules/mu/mommacat/search.rb +495 -0
  109. data/modules/mu/mommacat/storage.rb +722 -0
  110. data/modules/mu/{clouds → providers}/README.md +1 -1
  111. data/modules/mu/{clouds → providers}/aws.rb +380 -122
  112. data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
  113. data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
  114. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
  115. data/modules/mu/providers/aws/cdn.rb +782 -0
  116. data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
  117. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
  118. data/modules/mu/providers/aws/database.rb +1744 -0
  119. data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
  120. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  121. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
  122. data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
  123. data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
  124. data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
  125. data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
  126. data/modules/mu/providers/aws/job.rb +466 -0
  127. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
  128. data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
  129. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
  130. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
  131. data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
  132. data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
  133. data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
  134. data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
  135. data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
  136. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
  137. data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
  138. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  139. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  140. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  141. data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
  142. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  143. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  144. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  145. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  146. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  147. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  148. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  149. data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
  150. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  151. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  152. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  153. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  154. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  155. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  156. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  160. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  161. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  162. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  163. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  164. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  165. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  166. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  167. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  168. data/modules/mu/{clouds → providers}/google.rb +68 -30
  169. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  170. data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
  171. data/modules/mu/{clouds → providers}/google/database.rb +11 -21
  172. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  173. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  174. data/modules/mu/{clouds → providers}/google/function.rb +140 -168
  175. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  176. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  177. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
  178. data/modules/mu/{clouds → providers}/google/role.rb +94 -58
  179. data/modules/mu/{clouds → providers}/google/server.rb +243 -156
  180. data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
  181. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  182. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  183. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  184. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  185. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  186. data/modules/tests/aws-jobs-functions.yaml +46 -0
  187. data/modules/tests/bucket.yml +4 -0
  188. data/modules/tests/centos6.yaml +15 -0
  189. data/modules/tests/centos7.yaml +15 -0
  190. data/modules/tests/centos8.yaml +12 -0
  191. data/modules/tests/ecs.yaml +23 -0
  192. data/modules/tests/eks.yaml +1 -1
  193. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  194. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  195. data/modules/tests/includes-and-params.yaml +2 -1
  196. data/modules/tests/microservice_app.yaml +288 -0
  197. data/modules/tests/rds.yaml +108 -0
  198. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  199. data/modules/tests/regrooms/bucket.yml +19 -0
  200. data/modules/tests/regrooms/rds.yaml +123 -0
  201. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  202. data/modules/tests/super_complex_bok.yml +2 -2
  203. data/modules/tests/super_simple_bok.yml +3 -5
  204. data/modules/tests/win2k12.yaml +17 -5
  205. data/modules/tests/win2k16.yaml +25 -0
  206. data/modules/tests/win2k19.yaml +25 -0
  207. data/requirements.txt +1 -0
  208. data/spec/mu/clouds/azure_spec.rb +2 -2
  209. metadata +240 -154
  210. data/extras/image-generators/AWS/windows.yaml +0 -18
  211. data/modules/mu/clouds/aws/database.rb +0 -1985
  212. data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -18,7 +18,7 @@ module MU
18
18
  class AWS
19
19
  # A firewall ruleset as configured in {MU::Config::BasketofKittens::firewall_rules}
20
20
  class FirewallRule < MU::Cloud::FirewallRule
21
- require "mu/clouds/aws/vpc"
21
+ require "mu/providers/aws/vpc"
22
22
 
23
23
  @admin_sgs = Hash.new
24
24
  @admin_sg_semaphore = Mutex.new
@@ -54,13 +54,12 @@ module MU
54
54
 
55
55
  secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_security_group(sg_struct)
56
56
  @cloud_id = secgroup.group_id
57
- rescue Aws::EC2::Errors::InvalidGroupDuplicate => e
57
+ rescue Aws::EC2::Errors::InvalidGroupDuplicate
58
58
  MU.log "EC2 Security Group #{groupname} already exists, using it", MU::NOTICE
59
59
  filters = [{name: "group-name", values: [groupname]}]
60
60
  filters << {name: "vpc-id", values: [vpc_id]} if !vpc_id.nil?
61
61
 
62
62
  secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(filters: filters).security_groups.first
63
- deploy_id = @deploy.deploy_id if !@deploy_id.nil?
64
63
  if secgroup.nil?
65
64
  raise MuError, "Failed to locate security group named #{groupname}, even though EC2 says it already exists", caller
66
65
  end
@@ -69,24 +68,24 @@ module MU
69
68
 
70
69
  begin
71
70
  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(group_ids: [secgroup.group_id])
72
- rescue Aws::EC2::Errors::InvalidGroupNotFound => e
71
+ rescue Aws::EC2::Errors::InvalidGroupNotFound
73
72
  MU.log "#{secgroup.group_id} not yet ready, waiting...", MU::NOTICE
74
73
  sleep 10
75
74
  retry
76
75
  end
77
76
 
78
77
  MU::Cloud::AWS.createStandardTags(secgroup.group_id, region: @config['region'], credentials: @config['credentials'])
79
- MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'], credentials: @config['credentials'])
78
+ MU::Cloud::AWS.createTag(secgroup.group_id, "Name", groupname, region: @config['region'], credentials: @config['credentials'])
80
79
 
81
80
  if @config['optional_tags']
82
81
  MU::MommaCat.listOptionalTags.each { |key, value|
83
- MU::MommaCat.createTag(secgroup.group_id, key, value, region: @config['region'], credentials: @config['credentials'])
82
+ MU::Cloud::AWS.createTag(secgroup.group_id, key, value, region: @config['region'], credentials: @config['credentials'])
84
83
  }
85
84
  end
86
85
 
87
86
  if @config['tags']
88
87
  @config['tags'].each { |tag|
89
- MU::MommaCat.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
88
+ MU::Cloud::AWS.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
90
89
  }
91
90
  end
92
91
 
@@ -180,7 +179,7 @@ module MU
180
179
  ip_permissions: ec2_rule
181
180
  )
182
181
  end
183
- rescue Aws::EC2::Errors::InvalidPermissionDuplicate => e
182
+ rescue Aws::EC2::Errors::InvalidPermissionDuplicate
184
183
  MU.log "Attempt to add duplicate rule to #{@cloud_id}", MU::DEBUG, details: ec2_rule
185
184
  # Ensure that, at least, the description field gets updated on
186
185
  # existing rules
@@ -246,7 +245,7 @@ module MU
246
245
  # Reverse-map our cloud description into a runnable config hash.
247
246
  # We assume that any values we have in +@config+ are placeholders, and
248
247
  # calculate our own accordingly based on what's live in the cloud.
249
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
248
+ def toKitten(**_args)
250
249
  bok = {
251
250
  "cloud" => "AWS",
252
251
  "credentials" => @config['credentials'],
@@ -382,24 +381,24 @@ module MU
382
381
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
383
382
  # @param region [String]: The cloud provider region
384
383
  # @return [void]
385
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
386
- filters = nil
387
- if flags and flags["vpc_id"]
388
- filters = [
384
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
385
+ filters = if flags and flags["vpc_id"]
386
+ [
389
387
  {name: "vpc-id", values: [flags["vpc_id"]]}
390
388
  ]
391
389
  else
392
390
  filters = [
393
- {name: "tag:MU-ID", values: [MU.deploy_id]}
391
+ {name: "tag:MU-ID", values: [deploy_id]}
394
392
  ]
395
393
  if !ignoremaster
396
394
  filters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
397
395
  end
396
+ filters
398
397
  end
399
398
 
400
399
  # Some services create sneaky rogue ENIs which then block removal of
401
400
  # associated security groups. Find them and fry them.
402
- MU::Cloud::AWS::VPC.purge_interfaces(noop, filters, region: region, credentials: credentials)
401
+ MU::Cloud.resourceClass("AWS", "VPC").purge_interfaces(noop, filters, region: region, credentials: credentials)
403
402
 
404
403
  resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
405
404
  filters: filters
@@ -409,75 +408,8 @@ module MU
409
408
  MU.log "Revoking rules in EC2 Security Group #{sg.group_name} (#{sg.group_id})"
410
409
 
411
410
  if !noop
412
- ingress_to_revoke = Array.new
413
- egress_to_revoke = Array.new
414
- sg.ip_permissions.each { |hole|
415
- ingress_to_revoke << MU.structToHash(hole)
416
- ingress_to_revoke.each { |rule|
417
- if !rule[:user_id_group_pairs].nil? and rule[:user_id_group_pairs] .size == 0
418
- rule.delete(:user_id_group_pairs)
419
- elsif !rule[:user_id_group_pairs].nil?
420
- rule[:user_id_group_pairs].each { |group_ref|
421
- group_ref = MU.structToHash(group_ref)
422
- group_ref.delete(:group_name) if group_ref[:group_id]
423
- }
424
- end
425
-
426
- if !rule[:ip_ranges].nil? and rule[:ip_ranges].size == 0
427
- rule.delete(:ip_ranges)
428
- end
429
-
430
- if !rule[:prefix_list_ids].nil? and rule[:prefix_list_ids].size == 0
431
- rule.delete(:prefix_list_ids)
432
- end
433
-
434
- if !rule[:ipv_6_ranges].nil? and rule[:ipv_6_ranges].size == 0
435
- rule.delete(:ipv_6_ranges)
436
- end
437
- }
438
- }
439
- sg.ip_permissions_egress.each { |hole|
440
- egress_to_revoke << MU.structToHash(hole)
441
- egress_to_revoke.each { |rule|
442
- if !rule[:user_id_group_pairs].nil? and rule[:user_id_group_pairs].size == 0
443
- rule.delete(:user_id_group_pairs)
444
- elsif !rule[:user_id_group_pairs].nil?
445
- rule[:user_id_group_pairs].each { |group_ref|
446
- group_ref = MU.structToHash(group_ref)
447
- group_ref.delete(:group_name) if group_ref[:group_id]
448
- }
449
- end
450
-
451
- if !rule[:ip_ranges].nil? and rule[:ip_ranges].size == 0
452
- rule.delete(:ip_ranges)
453
- end
454
-
455
- if !rule[:prefix_list_ids].nil? and rule[:prefix_list_ids].size == 0
456
- rule.delete(:prefix_list_ids)
457
- end
458
-
459
- if !rule[:ipv_6_ranges].nil? and rule[:ipv_6_ranges].size == 0
460
- rule.delete(:ipv_6_ranges)
461
- end
462
- }
463
- }
464
- begin
465
-
466
- if ingress_to_revoke.size > 0
467
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
468
- group_id: sg.group_id,
469
- ip_permissions: ingress_to_revoke
470
- )
471
- end
472
- if egress_to_revoke.size > 0
473
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
474
- group_id: sg.group_id,
475
- ip_permissions: egress_to_revoke
476
- )
477
- end
478
- rescue Aws::EC2::Errors::InvalidPermissionNotFound
479
- MU.log "Rule in #{sg.group_id} disappeared before I could remove it", MU::WARN
480
- end
411
+ revoke_rules(sg, region: region, credentials: credentials)
412
+ revoke_rules(sg, egress: true, region: region, credentials: credentials)
481
413
  end
482
414
  }
483
415
 
@@ -485,61 +417,132 @@ module MU
485
417
  next if sg.group_name == "default"
486
418
  MU.log "Removing EC2 Security Group #{sg.group_name}"
487
419
 
488
- retries = 0
489
- begin
490
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id) if !noop
491
- rescue Aws::EC2::Errors::CannotDelete => e
492
- MU.log e.message, MU::WARN
493
- rescue Aws::EC2::Errors::InvalidGroupNotFound
494
- MU.log "EC2 Security Group #{sg.group_name} disappeared before I could delete it!", MU::WARN
495
- rescue Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse
496
- if retries < 10
497
- MU.log "EC2 Security Group #{sg.group_name} is still in use, waiting...", MU::NOTICE
498
- # try to get out from under loose network interfaces with which
499
- # we're associated
500
- if sg.vpc_id
501
- # get the default SG for this VPC
502
- default_resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
503
- filters: [
504
- { name: "group-name", values: ["default"] },
505
- { name: "vpc-id", values: [sg.vpc_id] }
506
- ]
507
- ).security_groups
508
- if default_resp and default_resp.size == 1
509
- default_sg = default_resp.first.group_id
510
- eni_resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
511
- filters: [ {name: "group-id", values: [sg.group_id]} ]
512
- )
513
- if eni_resp and eni_resp.data and
514
- eni_resp.data.network_interfaces
515
- eni_resp.data.network_interfaces.each { |iface|
516
- iface_groups = iface.groups.map { |sg| sg.group_id }
517
- iface_groups.delete(sg.group_id)
518
- iface_groups << default_sg if iface_groups.empty?
519
- MU.log "Attempting to remove #{sg.group_id} from ENI #{iface.network_interface_id}"
420
+ on_retry = Proc.new {
421
+ # try to get out from under loose network interfaces with which
422
+ # we're associated
423
+ if sg.vpc_id
424
+ default_sg = MU::Cloud.resourceClass("AWS", "VPC").getDefaultSg(sg.vpc_id, region: region, credentials: credentials)
425
+ if default_sg
426
+ eni_resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
427
+ filters: [ {name: "group-id", values: [sg.group_id]} ]
428
+ )
429
+ if eni_resp and eni_resp.data and
430
+ eni_resp.data.network_interfaces
431
+ eni_resp.data.network_interfaces.each { |iface|
432
+ iface_groups = iface.groups.map { |if_sg| if_sg.group_id }
433
+ iface_groups.delete(sg.group_id)
434
+ iface_groups << default_sg if iface_groups.empty?
435
+ MU.log "Attempting to remove #{sg.group_id} (#{sg.group_name}) from ENI #{iface.network_interface_id}"
436
+ begin
520
437
  MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_network_interface_attribute(
521
438
  network_interface_id: iface.network_interface_id,
522
439
  groups: iface_groups
523
440
  )
524
- }
525
- end
441
+ rescue ::Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound
442
+ # fine by me
443
+ rescue ::Aws::EC2::Errors::AuthFailure
444
+ MU.log "Permission denied attempting to trim Security Group list for #{iface.network_interface_id}", MU::WARN, details: iface.groups.map { |g| g.group_name }.join(",")+" => default"
445
+ end
446
+ }
526
447
  end
527
448
  end
449
+ end
450
+ }
451
+
452
+ if !noop
453
+ MU.retrier([Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse], ignoreme: [Aws::EC2::Errors::InvalidGroupNotFound], max: 10, wait: 10, on_retry: on_retry) {
454
+ begin
455
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id)
456
+ rescue Aws::EC2::Errors::CannotDelete => e
457
+ MU.log e.message, MU::WARN
458
+ end
459
+ }
460
+ end
461
+
462
+ }
463
+ end
464
+
465
+ def self.revoke_rules(sg, egress: false, region: MU.myregion, credentials: nil)
466
+ holes = sg.send(egress ? :ip_permissions_egress : :ip_permissions)
467
+
468
+ to_revoke = []
469
+
470
+ holes.each { |hole|
471
+ to_revoke << MU.structToHash(hole)
472
+ to_revoke.each { |rule|
473
+ if !rule[:user_id_group_pairs].nil? and rule[:user_id_group_pairs].size == 0
474
+ rule.delete(:user_id_group_pairs)
475
+ elsif !rule[:user_id_group_pairs].nil?
476
+ rule[:user_id_group_pairs].each { |group_ref|
477
+ group_ref = MU.structToHash(group_ref)
478
+ group_ref.delete(:group_name) if group_ref[:group_id]
479
+ }
480
+ end
481
+
482
+ if !rule[:ip_ranges].nil? and rule[:ip_ranges].size == 0
483
+ rule.delete(:ip_ranges)
484
+ end
485
+
486
+ if !rule[:prefix_list_ids].nil? and rule[:prefix_list_ids].size == 0
487
+ rule.delete(:prefix_list_ids)
488
+ end
489
+
490
+ if !rule[:ipv_6_ranges].nil? and rule[:ipv_6_ranges].size == 0
491
+ rule.delete(:ipv_6_ranges)
492
+ end
493
+ }
494
+ }
528
495
 
529
- sleep 10
530
- retries = retries + 1
531
- retry
496
+ if to_revoke.size > 0
497
+ begin
498
+ if egress
499
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
500
+ group_id: sg.group_id,
501
+ ip_permissions: to_revoke
502
+ )
532
503
  else
533
- MU.log "Failed to delete #{sg.group_name}", MU::ERR
504
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
505
+ group_id: sg.group_id,
506
+ ip_permissions: to_revoke
507
+ )
534
508
  end
509
+ rescue Aws::EC2::Errors::InvalidPermissionNotFound
510
+ MU.log "Rule in #{sg.group_id} disappeared before I could remove it", MU::WARN
535
511
  end
512
+ end
513
+
514
+ end
515
+ private_class_method :revoke_rules
516
+
517
+ # Return an AWS-specific chunk of schema commonly used in the +ingress_rules+ parameter of other resource types.
518
+ # @return [Hash]
519
+ def self.ingressRuleAddtlSchema
520
+ {
521
+ "items" => {
522
+ "properties" => {
523
+ "sgs" => {
524
+ "type" => "array",
525
+ "items" => {
526
+ "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
527
+ "type" => "string"
528
+ }
529
+ },
530
+ "lbs" => {
531
+ "type" => "array",
532
+ "items" => {
533
+ "description" => "AWS Load Balancers which will have this rule applied to their traffic",
534
+ "type" => "string"
535
+ }
536
+ }
537
+ }
538
+ }
536
539
  }
537
540
  end
538
541
 
539
542
  # Cloud-specific configuration properties.
540
- # @param config [MU::Config]: The calling MU::Config object
543
+ # @param _config [MU::Config]: The calling MU::Config object
541
544
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
542
- def self.schema(config)
545
+ def self.schema(_config)
543
546
  toplevel_required = []
544
547
  schema = {
545
548
  "rules" => {
@@ -645,36 +648,16 @@ module MU
645
648
 
646
649
  if rule['firewall_rules']
647
650
  rule['firewall_rules'].each { |sg|
648
- if sg.is_a?(MU::Config::Ref) and sg.name
649
- acl["dependencies"] << {
650
- "type" => "firewall_rule",
651
- "name" => sg.name,
652
- "no_create_wait" => true
653
- }
654
- elsif sg['name'] and !sg['deploy_id']
655
- acl["dependencies"] << {
656
- "type" => "firewall_rule",
657
- "name" => sg['name'],
658
- "no_create_wait" => true
659
- }
651
+ if sg['name'] and !sg['deploy_id']
652
+ MU::Config.addDependency(acl, sg['name'], "firewall_rule", no_create_wait: true)
660
653
  end
661
654
  }
662
655
  end
663
656
 
664
657
  if rule['loadbalancers']
665
658
  rule['loadbalancers'].each { |lb|
666
- if lb.is_a?(MU::Config::Ref) and lb.name
667
- acl["dependencies"] << {
668
- "type" => "loadbalancer",
669
- "name" => lb.name,
670
- "phase" => "groom"
671
- }
672
- elsif lb['name'] and !lb['deploy_id']
673
- acl["dependencies"] << {
674
- "type" => "loadbalancer",
675
- "name" => lb['name'],
676
- "phase" => "groom"
677
- }
659
+ if lb['name'] and !lb['deploy_id']
660
+ MU::Config.addDependency(acl, lb['name'], "loadbalancer", phase: "groom")
678
661
  end
679
662
  }
680
663
  end
@@ -716,32 +699,7 @@ module MU
716
699
 
717
700
  private
718
701
 
719
- #########################################################################
720
- # Manufacture an EC2 security group. The second parameter, rules, is an
721
- # "ingress_rules" structure parsed and validated by MU::Config.
722
- #########################################################################
723
- def setRules(rules, add_to_self: false, ingress: true, egress: false)
724
- describe
725
- # XXX warn about attempt to set rules before we exist
726
- return if rules.nil? or rules.size == 0 or !@cloud_id
727
-
728
- # add_to_self means that this security is a "member" of its own rules
729
- # (which is to say, objects that have this SG are allowed in my these
730
- # rules)
731
- if add_to_self
732
- rules.each { |rule|
733
- if rule['sgs'].nil? or !rule['sgs'].include?(@cloud_id)
734
- new_rule = rule.clone
735
- new_rule.delete('hosts')
736
- rule['sgs'] = Array.new if rule['sgs'].nil?
737
- rule['sgs'] << @cloud_id
738
- end
739
- }
740
- end
741
-
742
- ec2_rules = convertToEc2(rules)
743
- ext_permissions = MU.structToHash(cloud_desc.ip_permissions)
744
-
702
+ def purge_extraneous_rules(ec2_rules, ext_permissions)
745
703
  # Purge any old rules that we're sure we created (check the comment)
746
704
  # but which are no longer configured.
747
705
  ext_permissions.each { |ext_rule|
@@ -778,97 +736,109 @@ module MU
778
736
  ip_permissions: [ext_rule]
779
737
  )
780
738
  end
781
-
782
739
  }
740
+ end
783
741
 
784
- # Creating an empty security group is ok, so don't freak out if we get
785
- # a null rule list.
786
- if !ec2_rules.nil?
787
- ec2_rules.uniq!
788
- retries = 0
789
- ec2_rules.each { |rule|
790
- haverule = nil
791
- different = false
792
- ext_permissions.each { |ext_rule|
793
- if rule[:from_port] == ext_rule[:from_port] and
794
- rule[:to_port] == ext_rule[:to_port] and
795
- rule[:ip_protocol] == ext_rule[:ip_protocol]
796
- haverule = ext_rule
797
- ext_rule.keys.each { |k|
798
- if ext_rule[k].nil? or ext_rule[k] == []
799
- haverule.delete(k)
800
- end
801
- different = true if rule[k] != ext_rule[k]
802
- }
803
- break
804
- end
805
- }
806
- if haverule and !different
807
- MU.log "Security Group rule already up-to-date in #{@mu_name}", MU::DEBUG, details: rule
808
- next
742
+ #########################################################################
743
+ # Manufacture an EC2 security group. The second parameter, rules, is an
744
+ # "ingress_rules" structure parsed and validated by MU::Config.
745
+ #########################################################################
746
+ def setRules(rules, add_to_self: false, ingress: true, egress: false)
747
+ # XXX warn about attempt to set rules before we exist
748
+ return if rules.nil? or rules.size == 0 or !@cloud_id
749
+
750
+ # add_to_self means that this security is a "member" of its own rules
751
+ # (which is to say, objects that have this SG are allowed in my these
752
+ # rules)
753
+ if add_to_self
754
+ rules.each { |rule|
755
+ if rule['sgs'].nil? or !rule['sgs'].include?(@cloud_id)
756
+ new_rule = rule.clone
757
+ new_rule.delete('hosts')
758
+ rule['sgs'] = Array.new if rule['sgs'].nil?
759
+ rule['sgs'] << @cloud_id
809
760
  end
761
+ }
762
+ end
810
763
 
811
- MU.log "Setting #{ingress ? "ingress" : "egress"} rule in Security Group #{@mu_name} (#{@cloud_id})", MU::NOTICE, details: rule
812
- begin
813
-
814
- if ingress
815
- if haverule
816
- begin
817
- MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_ingress(
818
- group_id: @cloud_id,
819
- ip_permissions: [haverule]
820
- )
821
- rescue Aws::EC2::Errors::InvalidPermissionNotFound => e
822
- end
764
+ ec2_rules = convertToEc2(rules)
765
+ return if ec2_rules.nil?
766
+
767
+ ext_permissions = MU.structToHash(cloud_desc(use_cache: false).ip_permissions)
768
+
769
+ purge_extraneous_rules(ec2_rules, ext_permissions)
770
+
771
+ ec2_rules.uniq!
772
+ ec2_rules.each { |rule|
773
+ haverule = nil
774
+ different = false
775
+ ext_permissions.each { |ext_rule|
776
+ if rule[:from_port] == ext_rule[:from_port] and
777
+ rule[:to_port] == ext_rule[:to_port] and
778
+ rule[:ip_protocol] == ext_rule[:ip_protocol]
779
+ haverule = ext_rule
780
+ ext_rule.keys.each { |k|
781
+ if ext_rule[k].nil? or ext_rule[k] == []
782
+ haverule.delete(k)
823
783
  end
784
+ different = true if rule[k] != ext_rule[k]
785
+ }
786
+ break
787
+ end
788
+ }
789
+ if haverule and !different
790
+ MU.log "Security Group rule already up-to-date in #{@mu_name}", MU::DEBUG, details: rule
791
+ next
792
+ end
793
+
794
+ MU.log "Setting #{ingress ? "ingress" : "egress"} rule in Security Group #{@mu_name} (#{@cloud_id})", MU::NOTICE, details: rule
795
+
796
+ MU.retrier([Aws::EC2::Errors::InvalidGroupNotFound], max: 10, wait: 10, ignoreme: [Aws::EC2::Errors::InvalidPermissionDuplicate]) {
797
+ if ingress
798
+ if haverule
824
799
  begin
825
- MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
800
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_ingress(
826
801
  group_id: @cloud_id,
827
- ip_permissions: [rule]
802
+ ip_permissions: [haverule]
828
803
  )
829
- rescue Aws::EC2::Errors::InvalidParameterCombination => e
830
- MU.log "FirewallRule #{@mu_name} had a bogus rule: #{e.message}", MU::ERR, details: rule
831
- raise e
804
+ rescue Aws::EC2::Errors::InvalidPermissionNotFound
832
805
  end
833
806
  end
834
-
835
- if egress
836
- if haverule
837
- begin
838
- MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_egress(
839
- group_id: @cloud_id,
840
- ip_permissions: [haverule]
841
- )
842
- rescue Aws::EC2::Errors::InvalidPermissionNotFound => e
843
- end
844
- end
845
- MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
807
+ begin
808
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
846
809
  group_id: @cloud_id,
847
810
  ip_permissions: [rule]
848
811
  )
812
+ rescue Aws::EC2::Errors::InvalidParameterCombination => e
813
+ MU.log "FirewallRule #{@mu_name} had a bogus rule: #{e.message}", MU::ERR, details: rule
814
+ raise e
849
815
  end
816
+ end
850
817
 
851
- rescue Aws::EC2::Errors::InvalidGroupNotFound => e
852
- MU.log "#{@mu_name} (#{@cloud_id}) does not yet exist", MU::WARN
853
- retries = retries + 1
854
- if retries < 10
855
- sleep 10
856
- retry
857
- else
858
- raise MuError, "#{@mu_name} does not exist", e.backtrace
818
+ if egress
819
+ if haverule
820
+ begin
821
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_egress(
822
+ group_id: @cloud_id,
823
+ ip_permissions: [haverule]
824
+ )
825
+ rescue Aws::EC2::Errors::InvalidPermissionNotFound
826
+ end
859
827
  end
860
- rescue Aws::EC2::Errors::InvalidPermissionDuplicate => e
861
- MU.log "Attempt to add duplicate rule to #{@mu_name}", MU::DEBUG, details: rule
828
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
829
+ group_id: @cloud_id,
830
+ ip_permissions: [rule]
831
+ )
862
832
  end
863
833
  }
864
- end
834
+ }
865
835
 
866
836
  end
867
837
 
868
- #########################################################################
869
- # Convert our config languages description of firewall rules into Amazon's.
870
- # This rule structure is as defined in MU::Config.
871
- #########################################################################
838
+ #######################################################################
839
+ # Convert our config languages description of firewall rules into
840
+ # Amazon's. Our rule structure is as defined in MU::Config.
841
+ #######################################################################
872
842
  def convertToEc2(rules)
873
843
  ec2_rules = []
874
844
  if rules != nil
@@ -890,8 +860,11 @@ module MU
890
860
  p_start = rule['port'].to_i
891
861
  p_end = rule['port'].to_i
892
862
  elsif rule['proto'] != "icmp"
893
- raise MuError, "Can't create a TCP or UDP security group rule without specifying ports: #{rule}"
863
+ MU.log "Can't create a TCP or UDP security group rule without specifying ports, assuming 'all'", MU::WARN, details: rule
864
+ p_start = "0"
865
+ p_end = "65535"
894
866
  end
867
+
895
868
  if rule['proto'] != "icmp"
896
869
  if p_start.nil? or p_end.nil?
897
870
  raise MuError, "Got nil ports out of rule #{rule}"
@@ -992,8 +965,8 @@ module MU
992
965
  ec2_rules << ec2_rule
993
966
  }
994
967
  end
995
- ec2_rules.uniq!
996
- return ec2_rules
968
+
969
+ ec2_rules.uniq
997
970
  end
998
971
 
999
972
  end #class