cloud-mu 3.1.2 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +10 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +2 -3
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +135 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +165 -111
  45. data/modules/mu/adoption.rb +401 -68
  46. data/modules/mu/cleanup.rb +199 -306
  47. data/modules/mu/cloud.rb +100 -1632
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +46 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +920 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +165 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +4 -4
  61. data/modules/mu/config/cache_cluster.rb +1 -1
  62. data/modules/mu/config/collection.rb +4 -4
  63. data/modules/mu/config/container_cluster.rb +9 -4
  64. data/modules/mu/config/database.rb +83 -104
  65. data/modules/mu/config/database.yml +1 -2
  66. data/modules/mu/config/dnszone.rb +6 -6
  67. data/modules/mu/config/doc_helpers.rb +516 -0
  68. data/modules/mu/config/endpoint.rb +4 -4
  69. data/modules/mu/config/firewall_rule.rb +103 -4
  70. data/modules/mu/config/folder.rb +4 -4
  71. data/modules/mu/config/function.rb +3 -3
  72. data/modules/mu/config/group.rb +4 -4
  73. data/modules/mu/config/habitat.rb +4 -4
  74. data/modules/mu/config/loadbalancer.rb +60 -14
  75. data/modules/mu/config/log.rb +4 -4
  76. data/modules/mu/config/msg_queue.rb +4 -4
  77. data/modules/mu/config/nosqldb.rb +4 -4
  78. data/modules/mu/config/notifier.rb +3 -3
  79. data/modules/mu/config/ref.rb +365 -0
  80. data/modules/mu/config/role.rb +4 -4
  81. data/modules/mu/config/schema_helpers.rb +509 -0
  82. data/modules/mu/config/search_domain.rb +4 -4
  83. data/modules/mu/config/server.rb +97 -70
  84. data/modules/mu/config/server.yml +1 -0
  85. data/modules/mu/config/server_pool.rb +5 -9
  86. data/modules/mu/config/storage_pool.rb +1 -1
  87. data/modules/mu/config/tail.rb +200 -0
  88. data/modules/mu/config/user.rb +4 -4
  89. data/modules/mu/config/vpc.rb +70 -27
  90. data/modules/mu/config/vpc.yml +0 -1
  91. data/modules/mu/defaults/AWS.yaml +83 -60
  92. data/modules/mu/defaults/Azure.yaml +1 -0
  93. data/modules/mu/defaults/Google.yaml +3 -2
  94. data/modules/mu/deploy.rb +30 -26
  95. data/modules/mu/groomer.rb +17 -2
  96. data/modules/mu/groomers/ansible.rb +188 -41
  97. data/modules/mu/groomers/chef.rb +116 -55
  98. data/modules/mu/logger.rb +127 -148
  99. data/modules/mu/master.rb +389 -2
  100. data/modules/mu/master/chef.rb +3 -4
  101. data/modules/mu/master/ldap.rb +3 -3
  102. data/modules/mu/master/ssl.rb +12 -3
  103. data/modules/mu/mommacat.rb +217 -2612
  104. data/modules/mu/mommacat/daemon.rb +397 -0
  105. data/modules/mu/mommacat/naming.rb +473 -0
  106. data/modules/mu/mommacat/search.rb +495 -0
  107. data/modules/mu/mommacat/storage.rb +722 -0
  108. data/modules/mu/{clouds → providers}/README.md +1 -1
  109. data/modules/mu/{clouds → providers}/aws.rb +271 -112
  110. data/modules/mu/{clouds → providers}/aws/alarm.rb +5 -3
  111. data/modules/mu/{clouds → providers}/aws/bucket.rb +26 -22
  112. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +33 -67
  113. data/modules/mu/{clouds → providers}/aws/collection.rb +24 -23
  114. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +681 -721
  115. data/modules/mu/providers/aws/database.rb +1744 -0
  116. data/modules/mu/{clouds → providers}/aws/dnszone.rb +64 -63
  117. data/modules/mu/{clouds → providers}/aws/endpoint.rb +22 -27
  118. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +214 -244
  119. data/modules/mu/{clouds → providers}/aws/folder.rb +7 -7
  120. data/modules/mu/{clouds → providers}/aws/function.rb +17 -22
  121. data/modules/mu/{clouds → providers}/aws/group.rb +23 -23
  122. data/modules/mu/{clouds → providers}/aws/habitat.rb +17 -14
  123. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +57 -48
  124. data/modules/mu/{clouds → providers}/aws/log.rb +15 -12
  125. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +17 -16
  126. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +18 -11
  127. data/modules/mu/{clouds → providers}/aws/notifier.rb +11 -6
  128. data/modules/mu/{clouds → providers}/aws/role.rb +112 -86
  129. data/modules/mu/{clouds → providers}/aws/search_domain.rb +39 -33
  130. data/modules/mu/{clouds → providers}/aws/server.rb +835 -1133
  131. data/modules/mu/{clouds → providers}/aws/server_pool.rb +56 -60
  132. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +24 -42
  133. data/modules/mu/{clouds → providers}/aws/user.rb +21 -22
  134. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  135. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
  136. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  137. data/modules/mu/{clouds → providers}/aws/vpc.rb +523 -929
  138. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  139. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  140. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  141. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  142. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  143. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  144. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  145. data/modules/mu/{clouds → providers}/azure/server.rb +95 -48
  146. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  147. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  148. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  149. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  150. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  151. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  152. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  153. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  154. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  155. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  156. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  160. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  161. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  162. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  163. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  164. data/modules/mu/{clouds → providers}/google.rb +67 -30
  165. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  166. data/modules/mu/{clouds → providers}/google/container_cluster.rb +84 -77
  167. data/modules/mu/{clouds → providers}/google/database.rb +10 -20
  168. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  169. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  170. data/modules/mu/{clouds → providers}/google/function.rb +139 -167
  171. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  172. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  173. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +18 -20
  174. data/modules/mu/{clouds → providers}/google/role.rb +92 -58
  175. data/modules/mu/{clouds → providers}/google/server.rb +242 -155
  176. data/modules/mu/{clouds → providers}/google/server_pool.rb +25 -44
  177. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  178. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  179. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  180. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  181. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  182. data/modules/tests/bucket.yml +4 -0
  183. data/modules/tests/centos6.yaml +11 -0
  184. data/modules/tests/centos7.yaml +11 -0
  185. data/modules/tests/centos8.yaml +12 -0
  186. data/modules/tests/ecs.yaml +23 -0
  187. data/modules/tests/includes-and-params.yaml +2 -1
  188. data/modules/tests/rds.yaml +108 -0
  189. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  190. data/modules/tests/regrooms/bucket.yml +19 -0
  191. data/modules/tests/regrooms/rds.yaml +123 -0
  192. data/modules/tests/server-with-scrub-muisms.yaml +1 -0
  193. data/modules/tests/super_simple_bok.yml +1 -3
  194. data/modules/tests/win2k12.yaml +17 -5
  195. data/modules/tests/win2k16.yaml +25 -0
  196. data/modules/tests/win2k19.yaml +25 -0
  197. data/requirements.txt +1 -0
  198. data/spec/mu/clouds/azure_spec.rb +2 -2
  199. metadata +232 -154
  200. data/extras/image-generators/AWS/windows.yaml +0 -18
  201. data/modules/mu/clouds/aws/database.rb +0 -1985
@@ -15,7 +15,7 @@
15
15
  module MU
16
16
  class Cloud
17
17
  class AWS
18
- # A log as configured in {MU::Config::BasketofKittens::logs}
18
+ # A log as configured in {MU::Config::BasketofKittens::folders}
19
19
  class Folder < MU::Cloud::Folder
20
20
 
21
21
  # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
@@ -78,20 +78,20 @@ module MU
78
78
  end
79
79
 
80
80
  # Cloud-specific configuration properties.
81
- # @param config [MU::Config]: The calling MU::Config object
81
+ # @param _config [MU::Config]: The calling MU::Config object
82
82
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
83
- def self.schema(config)
83
+ def self.schema(_config)
84
84
  toplevel_required = []
85
85
  schema = {
86
86
  }
87
87
  [toplevel_required, schema]
88
88
  end
89
89
 
90
- # Cloud-specific pre-processing of {MU::Config::BasketofKittens::logs}, bare and unvalidated.
91
- # @param log [Hash]: The resource to process and validate
92
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
90
+ # Cloud-specific pre-processing of {MU::Config::BasketofKittens::folders}, bare and unvalidated.
91
+ # @param _folder [Hash]: The resource to process and validate
92
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
93
93
  # @return [Boolean]: True if validation succeeded, False otherwise
94
- def self.validateConfig(log, configurator)
94
+ def self.validateConfig(_folder, _configurator)
95
95
  ok = true
96
96
 
97
97
  ok
@@ -29,12 +29,12 @@ module MU
29
29
  def assign_tag(resource_arn, tag_list, region=@config['region'])
30
30
  begin
31
31
  tag_list.each do |each_pair|
32
- tag_resp = MU::Cloud::AWS.lambda(region: region, credentials: @config['credentials']).tag_resource({
32
+ MU::Cloud::AWS.lambda(region: region, credentials: @config['credentials']).tag_resource({
33
33
  resource: resource_arn,
34
34
  tags: each_pair
35
35
  })
36
36
  end
37
- rescue Exception => e
37
+ rescue StandardError => e
38
38
  MU.log e, MU::ERR
39
39
  end
40
40
  end
@@ -153,7 +153,7 @@ module MU
153
153
 
154
154
  MU.log trigger_properties, MU::DEBUG
155
155
  begin
156
- add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
156
+ MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
157
157
  rescue Aws::Lambda::Errors::ResourceConflictException
158
158
  end
159
159
  adjust_trigger(tr['service'], trigger_arn, func_arn, @mu_name)
@@ -176,7 +176,7 @@ module MU
176
176
  begin
177
177
  # XXX There doesn't seem to be an API call to list or view existing
178
178
  # permissions, wtaf. This means we can't intelligently guard this.
179
- add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger)
179
+ MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger)
180
180
  rescue Aws::Lambda::Errors::ResourceConflictException => e
181
181
  if e.message.match(/already exists/)
182
182
  MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).remove_permission(
@@ -220,15 +220,15 @@ module MU
220
220
 
221
221
  when 'sns'
222
222
  # XXX don't do this, use MU::Cloud::AWS::Notification
223
- sns_client = MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials'])
224
- sub_to_what = sns_client.subscribe({
223
+ sns_client = MU::Cloud::AWS.sns(region: region, credentials: @config['credentials'])
224
+ sns_client.subscribe({
225
225
  topic_arn: trig_arn,
226
226
  protocol: protocol,
227
227
  endpoint: func_arn
228
228
  })
229
229
  when 'event','cloudwatch_event', 'events'
230
230
  # XXX don't do this, use MU::Cloud::AWS::Log
231
- client = MU::Cloud::AWS.cloudwatch_events(region: @config['region'], credentials: @config['credentials']).put_targets({
231
+ MU::Cloud::AWS.cloudwatch_events(region: region, credentials: @config['credentials']).put_targets({
232
232
  rule: @config['trigger']['name'],
233
233
  targets: [
234
234
  {
@@ -271,11 +271,13 @@ module MU
271
271
  # @param region [String]: The cloud provider region
272
272
  # @return [void]
273
273
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
274
+ MU.log "AWS::Function.cleanup: need to support flags['known']", MU::DEBUG, details: flags
275
+
274
276
  MU::Cloud::AWS.lambda(credentials: credentials, region: region).list_functions.functions.each { |f|
275
277
  desc = MU::Cloud::AWS.lambda(credentials: credentials, region: region).get_function(
276
278
  function_name: f.function_name
277
279
  )
278
- if desc.tags and desc.tags["MU-ID"] == MU.deploy_id
280
+ if desc.tags and desc.tags["MU-ID"] == MU.deploy_id and (desc.tags["MU-MASTER-IP"] == MU.mu_public_ip or ignoremaster)
279
281
  MU.log "Deleting Lambda function #{f.function_name}"
280
282
  if !noop
281
283
  MU::Cloud::AWS.lambda(credentials: credentials, region: region).delete_function(
@@ -312,7 +314,7 @@ module MU
312
314
  # Reverse-map our cloud description into a runnable config hash.
313
315
  # We assume that any values we have in +@config+ are placeholders, and
314
316
  # calculate our own accordingly based on what's live in the cloud.
315
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
317
+ def toKitten(**_args)
316
318
  bok = {
317
319
  "cloud" => "AWS",
318
320
  "credentials" => @config['credentials'],
@@ -407,9 +409,9 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
407
409
 
408
410
 
409
411
  # Cloud-specific configuration properties.
410
- # @param config [MU::Config]: The calling MU::Config object
412
+ # @param _config [MU::Config]: The calling MU::Config object
411
413
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
412
- def self.schema(config)
414
+ def self.schema(_config)
413
415
  toplevel_required = ["runtime"]
414
416
  schema = {
415
417
  "triggers" => {
@@ -437,6 +439,7 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
437
439
  },
438
440
  "code" => {
439
441
  "type" => "object",
442
+ "description" => "Zipped deployment package to upload to our function.",
440
443
  "properties" => {
441
444
  "s3_bucket" => {
442
445
  "type" => "string",
@@ -502,11 +505,7 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
502
505
  function["add_firewall_rules"] << {"name" => fwname}
503
506
  function["permissions"] ||= []
504
507
  function["permissions"] << "network"
505
- function['dependencies'] ||= []
506
- function['dependencies'] << {
507
- "name" => fwname,
508
- "type" => "firewall_rule"
509
- }
508
+ MU::Config.addDependency(function, fwname, "firewall_rule")
510
509
  end
511
510
 
512
511
  if !function['iam_role']
@@ -538,13 +537,9 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
538
537
  }
539
538
  configurator.insertKitten(roledesc, "roles")
540
539
 
541
- function['dependencies'] ||= []
542
540
  function['iam_role'] = function['name']+"execrole"
543
541
 
544
- function['dependencies'] << {
545
- "type" => "role",
546
- "name" => function['name']+"execrole"
547
- }
542
+ MU::Config.addDependency(function, function['name']+"execrole", "role")
548
543
  end
549
544
 
550
545
  ok
@@ -565,7 +560,7 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
565
560
  role_name: name.to_s
566
561
  })
567
562
  return role['role']['arn']
568
- rescue Exception => e
563
+ rescue StandardError => e
569
564
  MU.log "#{e}", MU::ERR
570
565
  end
571
566
  nil
@@ -39,7 +39,7 @@ module MU
39
39
  if !@config['use_if_exists']
40
40
  raise MuError, "IAM group #{@mu_name} already exists and use_if_exists is false"
41
41
  end
42
- rescue Aws::IAM::Errors::NoSuchEntity => e
42
+ rescue Aws::IAM::Errors::NoSuchEntity
43
43
  @config['path'] ||= "/"+@deploy.deploy_id+"/"
44
44
  MU.log "Creating IAM group #{@config['path']}#{@mu_name}"
45
45
  MU::Cloud::AWS.iam(credentials: @config['credentials']).create_group(
@@ -60,7 +60,7 @@ module MU
60
60
  userid = user
61
61
  userdesc = @deploy.findLitterMate(name: user, type: "users")
62
62
  userid = userdesc.cloud_id if userdesc
63
- found = MU::Cloud::AWS::User.find(cloud_id: userid)
63
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: userid)
64
64
  if found.size == 1
65
65
  userdesc = found.values.first
66
66
  MU.log "Adding IAM user #{userdesc.path}#{userdesc.user_name} to group #{@mu_name}", MU::NOTICE
@@ -88,7 +88,7 @@ module MU
88
88
  # Create these if necessary, then append them to the list of
89
89
  # attachable_policies
90
90
  if @config['raw_policies']
91
- pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
91
+ pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
92
92
  @config['raw_policies'],
93
93
  basename: @deploy.getResourceName(@config['name']),
94
94
  credentials: @credentials
@@ -99,7 +99,7 @@ module MU
99
99
 
100
100
  if @config['attachable_policies']
101
101
  configured_policies = @config['attachable_policies'].map { |p|
102
- id = if p.is_a?(MU::Config::Ref)
102
+ if p.is_a?(MU::Config::Ref)
103
103
  p.cloud_id
104
104
  else
105
105
  p = MU::Config::Ref.get(p)
@@ -114,7 +114,7 @@ module MU
114
114
  attached_policies.each { |a|
115
115
  if !configured_policies.include?(a.policy_arn)
116
116
  MU.log "Removing IAM policy #{a.policy_arn} from group #{@mu_name}", MU::NOTICE
117
- MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
117
+ MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
118
118
  else
119
119
  configured_policies.delete(a.policy_arn)
120
120
  end
@@ -131,7 +131,7 @@ module MU
131
131
  end
132
132
 
133
133
  if @config['inline_policies']
134
- docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
134
+ docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
135
135
  docs.each { |doc|
136
136
  MU.log "Putting user policy #{doc.keys.first} to group #{@cloud_id} "
137
137
  MU::Cloud::AWS.iam(credentials: @credentials).put_group_policy(
@@ -150,13 +150,15 @@ module MU
150
150
  cloud_desc.arn
151
151
  end
152
152
 
153
-
153
+ @cloud_desc_cache = nil
154
154
  # Fetch the AWS API description of this group
155
155
  # return [Struct]
156
- def cloud_desc
157
- MU::Cloud::AWS.iam(credentials: @config['credentials']).get_group(
156
+ def cloud_desc(use_cache: true)
157
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
158
+ @cloud_desc_cache = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_group(
158
159
  group_name: @mu_name
159
160
  )
161
+ @cloud_desc_cache
160
162
  end
161
163
 
162
164
  # Return the metadata for this group configuration
@@ -183,9 +185,11 @@ module MU
183
185
  # Remove all groups associated with the currently loaded deployment.
184
186
  # @param noop [Boolean]: If true, will only print what would be done
185
187
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
186
- # @param region [String]: The cloud provider region
187
188
  # @return [void]
188
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
189
+ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
190
+ MU.log "AWS::Group.cleanup: need to support flags['known']", MU::DEBUG, details: flags
191
+ MU.log "Placeholder: AWS Group artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
192
+
189
193
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_groups(
190
194
  path_prefix: "/"+MU.deploy_id+"/"
191
195
  )
@@ -259,7 +263,7 @@ module MU
259
263
  # Reverse-map our cloud description into a runnable config hash.
260
264
  # We assume that any values we have in +@config+ are placeholders, and
261
265
  # calculate our own accordingly based on what's live in the cloud.
262
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
266
+ def toKitten(**_args)
263
267
  bok = {
264
268
  "cloud" => "AWS",
265
269
  "credentials" => @config['credentials'],
@@ -287,7 +291,7 @@ module MU
287
291
  resp.policy_names.each { |pol_name|
288
292
  pol = MU::Cloud::AWS.iam(credentials: @credentials).get_group_policy(group_name: @cloud_id, policy_name: pol_name)
289
293
  doc = JSON.parse(URI.decode(pol.policy_document))
290
- bok["inline_policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
294
+ bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
291
295
  }
292
296
  end
293
297
 
@@ -315,12 +319,12 @@ module MU
315
319
  end
316
320
 
317
321
  # Cloud-specific configuration properties.
318
- # @param config [MU::Config]: The calling MU::Config object
322
+ # @param _config [MU::Config]: The calling MU::Config object
319
323
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
320
- def self.schema(config)
324
+ def self.schema(_config)
321
325
  toplevel_required = []
322
326
  polschema = MU::Config::Role.schema["properties"]["policies"]
323
- polschema.deep_merge!(MU::Cloud::AWS::Role.condition_schema)
327
+ polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
324
328
 
325
329
  schema = {
326
330
  "inline_policies" => polschema,
@@ -360,7 +364,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
360
364
  # If we're attaching some managed policies, make sure all of the ones
361
365
  # that should already exist do indeed exist
362
366
  if group['attachable_policies']
363
- ok = false if !MU::Cloud::AWS::Role.validateAttachablePolicies(
367
+ ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
364
368
  group['attachable_policies'],
365
369
  credentials: group['credentials'],
366
370
  region: group['region']
@@ -374,13 +378,9 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
374
378
  if group['members']
375
379
  group['members'].each { |user|
376
380
  if configurator.haveLitterMate?(user, "users")
377
- group["dependencies"] ||= []
378
- group["dependencies"] << {
379
- "type" => "user",
380
- "name" => user
381
- }
381
+ MU::Config.addDependency(group, user, "user")
382
382
  else
383
- found = MU::Cloud::AWS::User.find(cloud_id: user)
383
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: user)
384
384
  if found.nil? or found.empty?
385
385
  MU.log "Error in members for group #{group['name']}: No such user #{user}", MU::ERR
386
386
  ok = false
@@ -45,7 +45,6 @@ module MU
45
45
  resp = MU::Cloud::AWS.orgs(credentials: @config['credentials']).describe_create_account_status(
46
46
  create_account_request_id: createid
47
47
  )
48
- createstatus = resp.create_account_status.state
49
48
  if !["SUCCEEDED", "IN_PROGRESS"].include?(resp.create_account_status.state)
50
49
  raise MuError, "Failed to create account #{@mu_name}: #{resp.create_account_status.failure_reason}"
51
50
  end
@@ -59,9 +58,12 @@ module MU
59
58
  MU.log "Creation of account #{@mu_name} (#{resp.create_account_status.account_id}) complete"
60
59
  end
61
60
 
61
+ @cloud_desc_cache = nil
62
62
  # Return the cloud descriptor for the Habitat
63
- def cloud_desc
64
- MU::Cloud::AWS::Habitat.find(cloud_id: @cloud_id).values.first
63
+ def cloud_desc(use_cache: true)
64
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
65
+ @cloud_desc_cache = MU::Cloud::AWS::Habitat.find(cloud_id: @cloud_id).values.first
66
+ @cloud_desc_cache
65
67
  end
66
68
 
67
69
  # Canonical Amazon Resource Number for this resource
@@ -87,10 +89,11 @@ module MU
87
89
  # Remove all AWS accounts associated with the currently loaded deployment. Try to, anyway.
88
90
  # @param noop [Boolean]: If true, will only print what would be done
89
91
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
90
- # @param region [String]: The cloud provider region
91
92
  # @return [void]
92
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
93
+ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
93
94
  return if !orgMasterCreds?(credentials)
95
+ MU.log "AWS::Habitat.cleanup: need to support flags['known']", MU::DEBUG, details: flags
96
+ MU.log "Placeholder: AWS Habitat artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
94
97
 
95
98
  resp = MU::Cloud::AWS.orgs(credentials: credentials).list_accounts
96
99
 
@@ -108,14 +111,14 @@ module MU
108
111
 
109
112
  # Locate an existing account
110
113
  # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching account
111
- def self.find(**args)
114
+ def self.find(**_args)
112
115
  {}
113
116
  end
114
117
 
115
118
  # Cloud-specific configuration properties.
116
- # @param config [MU::Config]: The calling MU::Config object
119
+ # @param _config [MU::Config]: The calling MU::Config object
117
120
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
118
- def self.schema(config)
121
+ def self.schema(_config)
119
122
  toplevel_required = []
120
123
  schema = {
121
124
  "email" => {
@@ -126,9 +129,10 @@ module MU
126
129
  [toplevel_required, schema]
127
130
  end
128
131
 
129
- # @param account_number [String]
132
+ # @param _account_number [String]
133
+ # @param _credentials [String]
130
134
  # @return [Boolean]
131
- def self.isLive?(account_number, credentials = nil)
135
+ def self.isLive?(_account_number, _credentials = nil)
132
136
  true
133
137
  end
134
138
 
@@ -138,18 +142,17 @@ module MU
138
142
  # @param credentials [String]
139
143
  # @return [Boolean]
140
144
  def self.orgMasterCreds?(credentials = nil)
141
- user_list = MU::Cloud::AWS.iam(credentials: credentials).list_users.users
142
145
  acct_num = MU::Cloud::AWS.iam(credentials: credentials).list_users.users.first.arn.split(/:/)[4]
143
146
 
144
- parentorg = MU::Cloud::AWS::Folder.find(credentials: credentials).values.first
147
+ parentorg = MU::Cloud.resourceClass("AWS", "Folder").find(credentials: credentials).values.first
145
148
  acct_num == parentorg.master_account_id
146
149
  end
147
150
 
148
151
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::habitats}, bare and unvalidated.
149
152
  # @param habitat [Hash]: The resource to process and validate
150
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
153
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
151
154
  # @return [Boolean]: True if validation succeeded, False otherwise
152
- def self.validateConfig(habitat, configurator)
155
+ def self.validateConfig(habitat, _configurator)
153
156
  ok = true
154
157
 
155
158
  if !habitat["email"]
@@ -163,7 +163,7 @@ module MU
163
163
  dnsthread = Thread.new {
164
164
  if !MU::Cloud::AWS.isGovCloud?
165
165
  MU.dupGlobals(parent_thread_id)
166
- generic_mu_dns = MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: @mu_name, target: "#{lb.dns_name}.", cloudclass: MU::Cloud::LoadBalancer, sync_wait: @config['dns_sync_wait'])
166
+ generic_mu_dns = MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: @mu_name, target: "#{lb.dns_name}.", cloudclass: MU::Cloud::LoadBalancer, sync_wait: @config['dns_sync_wait'])
167
167
  end
168
168
  }
169
169
 
@@ -239,16 +239,35 @@ module MU
239
239
  end
240
240
  end
241
241
 
242
+ redirect_block = Proc.new { |r|
243
+ {
244
+ :protocol => r['protocol'],
245
+ :port => r['port'].to_s,
246
+ :host => r['host'],
247
+ :path => r['path'],
248
+ :query => r['query'],
249
+ :status_code => "HTTP_"+r['status_code'].to_s
250
+ }
251
+ }
252
+
242
253
  if !@config['classic']
243
254
  @config["listeners"].each { |l|
244
- if !@targetgroups.has_key?(l['targetgroup'])
245
- raise MuError, "Listener in #{@mu_name} configured for target group #{l['targetgroup']}, but I don't have data on a targetgroup by that name"
246
- end
247
- listen_descriptor = {
248
- :default_actions => [{
255
+ action = if l['redirect']
256
+ {
257
+ :type => "redirect",
258
+ :redirect_config => redirect_block.call(l['redirect'])
259
+ }
260
+ else
261
+ if !@targetgroups.has_key?(l['targetgroup'])
262
+ raise MuError, "Listener in #{@mu_name} configured for target group #{l['targetgroup']}, but I don't have data on a targetgroup by that name"
263
+ end
264
+ {
249
265
  :target_group_arn => @targetgroups[l['targetgroup']].target_group_arn,
250
266
  :type => "forward"
251
- }],
267
+ }
268
+ end
269
+ listen_descriptor = {
270
+ :default_actions => [ action ],
252
271
  :load_balancer_arn => lb.load_balancer_arn,
253
272
  :port => l['lb_port'],
254
273
  :protocol => l['lb_protocol']
@@ -276,10 +295,17 @@ module MU
276
295
  :actions => []
277
296
  }
278
297
  rule['actions'].each { |a|
279
- rule_descriptor[:actions] << {
280
- :target_group_arn => @targetgroups[a['targetgroup']].target_group_arn,
281
- :type => a['action']
282
- }
298
+ rule_descriptor[:actions] << if a['action'] == "forward"
299
+ {
300
+ :target_group_arn => @targetgroups[a['targetgroup']].target_group_arn,
301
+ :type => a['action']
302
+ }
303
+ elsif a['action'] == "redirect"
304
+ {
305
+ :redirect_config => redirect_block.call(rule['redirect']),
306
+ :type => a['action']
307
+ }
308
+ end
283
309
  }
284
310
  MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).create_rule(rule_descriptor)
285
311
  }
@@ -288,7 +314,7 @@ module MU
288
314
  else
289
315
  @config["listeners"].each { |l|
290
316
  if l['ssl_certificate_id']
291
- resp = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).set_load_balancer_policies_of_listener(
317
+ MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).set_load_balancer_policies_of_listener(
292
318
  load_balancer_name: @cloud_id,
293
319
  load_balancer_port: l['lb_port'],
294
320
  policy_names: [
@@ -330,7 +356,7 @@ module MU
330
356
  }
331
357
  )
332
358
  else
333
- @targetgroups.each_pair { |tg_name, tg|
359
+ @targetgroups.values.each { |tg|
334
360
  MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).modify_target_group_attributes(
335
361
  target_group_arn: tg.target_group_arn,
336
362
  attributes: [
@@ -400,7 +426,7 @@ module MU
400
426
  timeout = 0
401
427
  MU.log "Disabling connection draining on #{lb.dns_name}"
402
428
  end
403
- @targetgroups.each_pair { |tg_name, tg|
429
+ @targetgroups.values.each { |tg|
404
430
  MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).modify_target_group_attributes(
405
431
  target_group_arn: tg.target_group_arn,
406
432
  attributes: [
@@ -473,7 +499,7 @@ module MU
473
499
  end
474
500
  end
475
501
  else
476
- @targetgroups.each_pair { |tg_name, tg|
502
+ @targetgroups.values.each { |tg|
477
503
  MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).modify_target_group_attributes(
478
504
  target_group_arn: tg.target_group_arn,
479
505
  attributes: [
@@ -536,7 +562,7 @@ module MU
536
562
  }
537
563
  end
538
564
  if !MU::Cloud::AWS.isGovCloud?
539
- MU::Cloud::AWS::DNSZone.createRecordsFromConfig(@config['dns_records'], target: cloud_desc.dns_name)
565
+ MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: cloud_desc.dns_name)
540
566
  end
541
567
  end
542
568
 
@@ -553,15 +579,17 @@ module MU
553
579
  end
554
580
  end
555
581
 
582
+ @cloud_desc_cache = nil
556
583
  # Wrapper for cloud_desc method that deals with elb vs. elb2 resources.
557
- def cloud_desc
584
+ def cloud_desc(use_cache: true)
585
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
558
586
  if @config['classic']
559
- resp = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
587
+ @cloud_desc_cache = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
560
588
  load_balancer_names: [@cloud_id]
561
589
  ).load_balancer_descriptions.first
562
- return resp
590
+ return @cloud_desc_cache
563
591
  else
564
- resp = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
592
+ @cloud_desc_cache = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
565
593
  names: [@cloud_id]
566
594
  ).load_balancers.first
567
595
  if @targetgroups.nil? and !@deploy.nil? and
@@ -573,7 +601,7 @@ module MU
573
601
  }
574
602
  end
575
603
 
576
- return resp
604
+ return @cloud_desc_cache
577
605
  end
578
606
  end
579
607
 
@@ -692,7 +720,6 @@ module MU
692
720
  classic = false
693
721
  end
694
722
  begin
695
- tags = []
696
723
  matched = false
697
724
  if flags and flags['vpc_id']
698
725
  matched = true if lb.vpc_id == flags['vpc_id']
@@ -705,7 +732,7 @@ module MU
705
732
  end
706
733
  if matched
707
734
  if !MU::Cloud::AWS.isGovCloud?
708
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: lb.load_balancer_name, target: lb.dns_name, cloudclass: MU::Cloud::LoadBalancer, delete: true) if !noop
735
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: lb.load_balancer_name, target: lb.dns_name, cloudclass: MU::Cloud::LoadBalancer, delete: true) if !noop
709
736
  end
710
737
  if classic
711
738
  MU.log "Removing Elastic Load Balancer #{lb.load_balancer_name}"
@@ -773,9 +800,9 @@ module MU
773
800
  end
774
801
 
775
802
  # Cloud-specific configuration properties.
776
- # @param config [MU::Config]: The calling MU::Config object
803
+ # @param _config [MU::Config]: The calling MU::Config object
777
804
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
778
- def self.schema(config)
805
+ def self.schema(_config)
779
806
  toplevel_required = []
780
807
  schema = {
781
808
  "targetgroups" => {
@@ -792,35 +819,16 @@ module MU
792
819
  }
793
820
  }
794
821
  },
795
- "ingress_rules" => {
796
- "items" => {
797
- "properties" => {
798
- "sgs" => {
799
- "type" => "array",
800
- "items" => {
801
- "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
802
- "type" => "string"
803
- }
804
- },
805
- "lbs" => {
806
- "type" => "array",
807
- "items" => {
808
- "description" => "AWS Load Balancers which will have this rule applied to their traffic",
809
- "type" => "string"
810
- }
811
- }
812
- }
813
- }
814
- }
822
+ "ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
815
823
  }
816
824
  [toplevel_required, schema]
817
825
  end
818
826
 
819
827
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::loadbalancers}, bare and unvalidated.
820
828
  # @param lb [Hash]: The resource to process and validate
821
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
829
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
822
830
  # @return [Boolean]: True if validation succeeded, False otherwise
823
- def self.validateConfig(lb, configurator)
831
+ def self.validateConfig(lb, _configurator)
824
832
  ok = true
825
833
 
826
834
  # XXX what about raw targetgroup ssl declarations?
@@ -830,7 +838,7 @@ module MU
830
838
  if lb['cloud'] != "CloudFormation" # XXX or maybe do this anyway?
831
839
  begin
832
840
  listener["ssl_certificate_id"] = MU::Cloud::AWS.findSSLCertificate(name: listener["ssl_certificate_name"].to_s, id: listener["ssl_certificate_id"].to_s, region: lb['region'])
833
- rescue MuError => e
841
+ rescue MuError
834
842
  ok = false
835
843
  next
836
844
  end
@@ -922,6 +930,7 @@ module MU
922
930
  return matches
923
931
 
924
932
  end
933
+
925
934
  end
926
935
  end
927
936
  end