cloud-mu 3.1.3 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +21 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +4 -4
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +147 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +158 -111
  45. data/modules/mu/adoption.rb +404 -71
  46. data/modules/mu/cleanup.rb +221 -306
  47. data/modules/mu/cloud.rb +129 -1633
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +44 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +926 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +169 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +32 -3
  61. data/modules/mu/config/cache_cluster.rb +2 -2
  62. data/modules/mu/config/cdn.rb +100 -0
  63. data/modules/mu/config/collection.rb +4 -4
  64. data/modules/mu/config/container_cluster.rb +9 -4
  65. data/modules/mu/config/database.rb +84 -105
  66. data/modules/mu/config/database.yml +1 -2
  67. data/modules/mu/config/dnszone.rb +10 -9
  68. data/modules/mu/config/doc_helpers.rb +516 -0
  69. data/modules/mu/config/endpoint.rb +5 -4
  70. data/modules/mu/config/firewall_rule.rb +103 -4
  71. data/modules/mu/config/folder.rb +4 -4
  72. data/modules/mu/config/function.rb +19 -10
  73. data/modules/mu/config/group.rb +4 -4
  74. data/modules/mu/config/habitat.rb +4 -4
  75. data/modules/mu/config/job.rb +89 -0
  76. data/modules/mu/config/loadbalancer.rb +60 -14
  77. data/modules/mu/config/log.rb +4 -4
  78. data/modules/mu/config/msg_queue.rb +4 -4
  79. data/modules/mu/config/nosqldb.rb +4 -4
  80. data/modules/mu/config/notifier.rb +10 -21
  81. data/modules/mu/config/ref.rb +411 -0
  82. data/modules/mu/config/role.rb +4 -4
  83. data/modules/mu/config/schema_helpers.rb +509 -0
  84. data/modules/mu/config/search_domain.rb +4 -4
  85. data/modules/mu/config/server.rb +98 -71
  86. data/modules/mu/config/server.yml +1 -0
  87. data/modules/mu/config/server_pool.rb +5 -9
  88. data/modules/mu/config/storage_pool.rb +1 -1
  89. data/modules/mu/config/tail.rb +200 -0
  90. data/modules/mu/config/user.rb +4 -4
  91. data/modules/mu/config/vpc.rb +71 -27
  92. data/modules/mu/config/vpc.yml +0 -1
  93. data/modules/mu/defaults/AWS.yaml +91 -68
  94. data/modules/mu/defaults/Azure.yaml +1 -0
  95. data/modules/mu/defaults/Google.yaml +3 -2
  96. data/modules/mu/deploy.rb +43 -26
  97. data/modules/mu/groomer.rb +17 -2
  98. data/modules/mu/groomers/ansible.rb +188 -41
  99. data/modules/mu/groomers/chef.rb +116 -55
  100. data/modules/mu/logger.rb +127 -148
  101. data/modules/mu/master.rb +410 -2
  102. data/modules/mu/master/chef.rb +3 -4
  103. data/modules/mu/master/ldap.rb +3 -3
  104. data/modules/mu/master/ssl.rb +12 -3
  105. data/modules/mu/mommacat.rb +218 -2612
  106. data/modules/mu/mommacat/daemon.rb +403 -0
  107. data/modules/mu/mommacat/naming.rb +473 -0
  108. data/modules/mu/mommacat/search.rb +495 -0
  109. data/modules/mu/mommacat/storage.rb +722 -0
  110. data/modules/mu/{clouds → providers}/README.md +1 -1
  111. data/modules/mu/{clouds → providers}/aws.rb +380 -122
  112. data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
  113. data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
  114. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
  115. data/modules/mu/providers/aws/cdn.rb +782 -0
  116. data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
  117. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
  118. data/modules/mu/providers/aws/database.rb +1744 -0
  119. data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
  120. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  121. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
  122. data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
  123. data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
  124. data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
  125. data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
  126. data/modules/mu/providers/aws/job.rb +466 -0
  127. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
  128. data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
  129. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
  130. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
  131. data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
  132. data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
  133. data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
  134. data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
  135. data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
  136. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
  137. data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
  138. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  139. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  140. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  141. data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
  142. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  143. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  144. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  145. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  146. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  147. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  148. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  149. data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
  150. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  151. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  152. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  153. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  154. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  155. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  156. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  160. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  161. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  162. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  163. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  164. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  165. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  166. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  167. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  168. data/modules/mu/{clouds → providers}/google.rb +68 -30
  169. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  170. data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
  171. data/modules/mu/{clouds → providers}/google/database.rb +11 -21
  172. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  173. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  174. data/modules/mu/{clouds → providers}/google/function.rb +140 -168
  175. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  176. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  177. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
  178. data/modules/mu/{clouds → providers}/google/role.rb +94 -58
  179. data/modules/mu/{clouds → providers}/google/server.rb +243 -156
  180. data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
  181. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  182. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  183. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  184. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  185. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  186. data/modules/tests/aws-jobs-functions.yaml +46 -0
  187. data/modules/tests/bucket.yml +4 -0
  188. data/modules/tests/centos6.yaml +15 -0
  189. data/modules/tests/centos7.yaml +15 -0
  190. data/modules/tests/centos8.yaml +12 -0
  191. data/modules/tests/ecs.yaml +23 -0
  192. data/modules/tests/eks.yaml +1 -1
  193. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  194. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  195. data/modules/tests/includes-and-params.yaml +2 -1
  196. data/modules/tests/microservice_app.yaml +288 -0
  197. data/modules/tests/rds.yaml +108 -0
  198. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  199. data/modules/tests/regrooms/bucket.yml +19 -0
  200. data/modules/tests/regrooms/rds.yaml +123 -0
  201. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  202. data/modules/tests/super_complex_bok.yml +2 -2
  203. data/modules/tests/super_simple_bok.yml +3 -5
  204. data/modules/tests/win2k12.yaml +17 -5
  205. data/modules/tests/win2k16.yaml +25 -0
  206. data/modules/tests/win2k19.yaml +25 -0
  207. data/requirements.txt +1 -0
  208. data/spec/mu/clouds/azure_spec.rb +2 -2
  209. metadata +240 -154
  210. data/extras/image-generators/AWS/windows.yaml +0 -18
  211. data/modules/mu/clouds/aws/database.rb +0 -1985
  212. data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -39,7 +39,7 @@ module MU
39
39
  if !@config['use_if_exists']
40
40
  raise MuError, "IAM group #{@mu_name} already exists and use_if_exists is false"
41
41
  end
42
- rescue Aws::IAM::Errors::NoSuchEntity => e
42
+ rescue Aws::IAM::Errors::NoSuchEntity
43
43
  @config['path'] ||= "/"+@deploy.deploy_id+"/"
44
44
  MU.log "Creating IAM group #{@config['path']}#{@mu_name}"
45
45
  MU::Cloud::AWS.iam(credentials: @config['credentials']).create_group(
@@ -60,7 +60,7 @@ module MU
60
60
  userid = user
61
61
  userdesc = @deploy.findLitterMate(name: user, type: "users")
62
62
  userid = userdesc.cloud_id if userdesc
63
- found = MU::Cloud::AWS::User.find(cloud_id: userid)
63
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: userid)
64
64
  if found.size == 1
65
65
  userdesc = found.values.first
66
66
  MU.log "Adding IAM user #{userdesc.path}#{userdesc.user_name} to group #{@mu_name}", MU::NOTICE
@@ -88,7 +88,7 @@ module MU
88
88
  # Create these if necessary, then append them to the list of
89
89
  # attachable_policies
90
90
  if @config['raw_policies']
91
- pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
91
+ pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
92
92
  @config['raw_policies'],
93
93
  basename: @deploy.getResourceName(@config['name']),
94
94
  credentials: @credentials
@@ -99,7 +99,7 @@ module MU
99
99
 
100
100
  if @config['attachable_policies']
101
101
  configured_policies = @config['attachable_policies'].map { |p|
102
- id = if p.is_a?(MU::Config::Ref)
102
+ if p.is_a?(MU::Config::Ref)
103
103
  p.cloud_id
104
104
  else
105
105
  p = MU::Config::Ref.get(p)
@@ -114,7 +114,7 @@ module MU
114
114
  attached_policies.each { |a|
115
115
  if !configured_policies.include?(a.policy_arn)
116
116
  MU.log "Removing IAM policy #{a.policy_arn} from group #{@mu_name}", MU::NOTICE
117
- MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
117
+ MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
118
118
  else
119
119
  configured_policies.delete(a.policy_arn)
120
120
  end
@@ -131,7 +131,7 @@ module MU
131
131
  end
132
132
 
133
133
  if @config['inline_policies']
134
- docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
134
+ docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
135
135
  docs.each { |doc|
136
136
  MU.log "Putting user policy #{doc.keys.first} to group #{@cloud_id} "
137
137
  MU::Cloud::AWS.iam(credentials: @credentials).put_group_policy(
@@ -150,13 +150,16 @@ module MU
150
150
  cloud_desc.arn
151
151
  end
152
152
 
153
-
153
+ @cloud_desc_cache = nil
154
154
  # Fetch the AWS API description of this group
155
155
  # return [Struct]
156
- def cloud_desc
157
- MU::Cloud::AWS.iam(credentials: @config['credentials']).get_group(
156
+ def cloud_desc(use_cache: true)
157
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
158
+ return nil if !@mu_name
159
+ @cloud_desc_cache = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_group(
158
160
  group_name: @mu_name
159
161
  )
162
+ @cloud_desc_cache
160
163
  end
161
164
 
162
165
  # Return the metadata for this group configuration
@@ -183,11 +186,13 @@ module MU
183
186
  # Remove all groups associated with the currently loaded deployment.
184
187
  # @param noop [Boolean]: If true, will only print what would be done
185
188
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
186
- # @param region [String]: The cloud provider region
187
189
  # @return [void]
188
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
190
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
191
+ MU.log "AWS::Group.cleanup: need to support flags['known']", MU::DEBUG, details: flags
192
+ MU.log "Placeholder: AWS Group artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
193
+
189
194
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_groups(
190
- path_prefix: "/"+MU.deploy_id+"/"
195
+ path_prefix: "/"+deploy_id+"/"
191
196
  )
192
197
  if resp and resp.groups
193
198
  resp.groups.each { |g|
@@ -259,7 +264,7 @@ module MU
259
264
  # Reverse-map our cloud description into a runnable config hash.
260
265
  # We assume that any values we have in +@config+ are placeholders, and
261
266
  # calculate our own accordingly based on what's live in the cloud.
262
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
267
+ def toKitten(**_args)
263
268
  bok = {
264
269
  "cloud" => "AWS",
265
270
  "credentials" => @config['credentials'],
@@ -270,14 +275,15 @@ module MU
270
275
  MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
271
276
  return nil
272
277
  end
273
-
274
- bok["name"] = cloud_desc.group.group_name
275
278
 
276
- if cloud_desc.group.path != "/"
277
- bok["path"] = cloud_desc.group.path
279
+ group_desc = cloud_desc(use_cache: false).respond_to?(:group) ? cloud_desc.group : cloud_desc
280
+ bok["name"] = group_desc.group_name
281
+
282
+ if group_desc.path != "/"
283
+ bok["path"] = group_desc.path
278
284
  end
279
285
 
280
- if cloud_desc.users and cloud_desc.users.size > 0
286
+ if cloud_desc.respond_to?(:users) and cloud_desc.users and cloud_desc.users.size > 0
281
287
  bok["members"] = cloud_desc.users.map { |u| u.user_name }
282
288
  end
283
289
 
@@ -287,7 +293,7 @@ module MU
287
293
  resp.policy_names.each { |pol_name|
288
294
  pol = MU::Cloud::AWS.iam(credentials: @credentials).get_group_policy(group_name: @cloud_id, policy_name: pol_name)
289
295
  doc = JSON.parse(URI.decode(pol.policy_document))
290
- bok["inline_policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
296
+ bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
291
297
  }
292
298
  end
293
299
 
@@ -315,12 +321,12 @@ module MU
315
321
  end
316
322
 
317
323
  # Cloud-specific configuration properties.
318
- # @param config [MU::Config]: The calling MU::Config object
324
+ # @param _config [MU::Config]: The calling MU::Config object
319
325
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
320
- def self.schema(config)
326
+ def self.schema(_config)
321
327
  toplevel_required = []
322
328
  polschema = MU::Config::Role.schema["properties"]["policies"]
323
- polschema.deep_merge!(MU::Cloud::AWS::Role.condition_schema)
329
+ polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
324
330
 
325
331
  schema = {
326
332
  "inline_policies" => polschema,
@@ -360,7 +366,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
360
366
  # If we're attaching some managed policies, make sure all of the ones
361
367
  # that should already exist do indeed exist
362
368
  if group['attachable_policies']
363
- ok = false if !MU::Cloud::AWS::Role.validateAttachablePolicies(
369
+ ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
364
370
  group['attachable_policies'],
365
371
  credentials: group['credentials'],
366
372
  region: group['region']
@@ -374,13 +380,9 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
374
380
  if group['members']
375
381
  group['members'].each { |user|
376
382
  if configurator.haveLitterMate?(user, "users")
377
- group["dependencies"] ||= []
378
- group["dependencies"] << {
379
- "type" => "user",
380
- "name" => user
381
- }
383
+ MU::Config.addDependency(group, user, "user")
382
384
  else
383
- found = MU::Cloud::AWS::User.find(cloud_id: user)
385
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: user)
384
386
  if found.nil? or found.empty?
385
387
  MU.log "Error in members for group #{group['name']}: No such user #{user}", MU::ERR
386
388
  ok = false
@@ -45,7 +45,6 @@ module MU
45
45
  resp = MU::Cloud::AWS.orgs(credentials: @config['credentials']).describe_create_account_status(
46
46
  create_account_request_id: createid
47
47
  )
48
- createstatus = resp.create_account_status.state
49
48
  if !["SUCCEEDED", "IN_PROGRESS"].include?(resp.create_account_status.state)
50
49
  raise MuError, "Failed to create account #{@mu_name}: #{resp.create_account_status.failure_reason}"
51
50
  end
@@ -59,9 +58,12 @@ module MU
59
58
  MU.log "Creation of account #{@mu_name} (#{resp.create_account_status.account_id}) complete"
60
59
  end
61
60
 
61
+ @cloud_desc_cache = nil
62
62
  # Return the cloud descriptor for the Habitat
63
- def cloud_desc
64
- MU::Cloud::AWS::Habitat.find(cloud_id: @cloud_id).values.first
63
+ def cloud_desc(use_cache: true)
64
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
65
+ @cloud_desc_cache = MU::Cloud::AWS::Habitat.find(cloud_id: @cloud_id).values.first
66
+ @cloud_desc_cache
65
67
  end
66
68
 
67
69
  # Canonical Amazon Resource Number for this resource
@@ -87,16 +89,17 @@ module MU
87
89
  # Remove all AWS accounts associated with the currently loaded deployment. Try to, anyway.
88
90
  # @param noop [Boolean]: If true, will only print what would be done
89
91
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
90
- # @param region [String]: The cloud provider region
91
92
  # @return [void]
92
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
93
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
93
94
  return if !orgMasterCreds?(credentials)
95
+ MU.log "AWS::Habitat.cleanup: need to support flags['known']", MU::DEBUG, details: flags
96
+ MU.log "Placeholder: AWS Habitat artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
94
97
 
95
98
  resp = MU::Cloud::AWS.orgs(credentials: credentials).list_accounts
96
99
 
97
100
  if resp and resp.accounts
98
101
  resp.accounts.each { |acct|
99
- if acct.name.match(/^#{Regexp.quote(MU.deploy_id)}/) or acct.name.match(/BUNS/)
102
+ if acct.name.match(/^#{Regexp.quote(deploy_id)}/) or acct.name.match(/BUNS/)
100
103
  if !noop
101
104
  pp acct
102
105
  end
@@ -108,14 +111,14 @@ module MU
108
111
 
109
112
  # Locate an existing account
110
113
  # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching account
111
- def self.find(**args)
114
+ def self.find(**_args)
112
115
  {}
113
116
  end
114
117
 
115
118
  # Cloud-specific configuration properties.
116
- # @param config [MU::Config]: The calling MU::Config object
119
+ # @param _config [MU::Config]: The calling MU::Config object
117
120
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
118
- def self.schema(config)
121
+ def self.schema(_config)
119
122
  toplevel_required = []
120
123
  schema = {
121
124
  "email" => {
@@ -126,9 +129,10 @@ module MU
126
129
  [toplevel_required, schema]
127
130
  end
128
131
 
129
- # @param account_number [String]
132
+ # @param _account_number [String]
133
+ # @param _credentials [String]
130
134
  # @return [Boolean]
131
- def self.isLive?(account_number, credentials = nil)
135
+ def self.isLive?(_account_number, _credentials = nil)
132
136
  true
133
137
  end
134
138
 
@@ -138,18 +142,17 @@ module MU
138
142
  # @param credentials [String]
139
143
  # @return [Boolean]
140
144
  def self.orgMasterCreds?(credentials = nil)
141
- user_list = MU::Cloud::AWS.iam(credentials: credentials).list_users.users
142
145
  acct_num = MU::Cloud::AWS.iam(credentials: credentials).list_users.users.first.arn.split(/:/)[4]
143
146
 
144
- parentorg = MU::Cloud::AWS::Folder.find(credentials: credentials).values.first
147
+ parentorg = MU::Cloud.resourceClass("AWS", "Folder").find(credentials: credentials).values.first
145
148
  acct_num == parentorg.master_account_id
146
149
  end
147
150
 
148
151
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::habitats}, bare and unvalidated.
149
152
  # @param habitat [Hash]: The resource to process and validate
150
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
153
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
151
154
  # @return [Boolean]: True if validation succeeded, False otherwise
152
- def self.validateConfig(habitat, configurator)
155
+ def self.validateConfig(habitat, _configurator)
153
156
  ok = true
154
157
 
155
158
  if !habitat["email"]
@@ -0,0 +1,466 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ class Cloud
17
+ class AWS
18
+ # A scheduled task facility as configured in {MU::Config::BasketofKittens::jobs}
19
+ class Job < MU::Cloud::Job
20
+
21
+ # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
22
+ # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
23
+ def initialize(**args)
24
+ super
25
+ @mu_name ||= @deploy.getResourceName(@config["name"])
26
+ end
27
+
28
+ # Called automatically by {MU::Deploy#createResources}
29
+ def create
30
+ @cloud_id = @mu_name
31
+
32
+ params = get_properties
33
+
34
+ MU.log "Creating CloudWatch Event #{@mu_name}", MU::NOTICE, details: params
35
+
36
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(params)
37
+ end
38
+
39
+ # Called automatically by {MU::Deploy#createResources}
40
+ def groom
41
+ new_props = get_properties
42
+ current = MU.structToHash(cloud_desc(use_cache: false))
43
+ params = {}
44
+ new_props.each_pair { |k, v|
45
+ next if k == :tags # doesn't seem to do anything
46
+ if v != current[k]
47
+ params[k] = v
48
+ end
49
+ }
50
+
51
+ if params.size > 0
52
+ MU.log "Updating CloudWatch Event #{@cloud_id}", MU::NOTICE, details: params
53
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(new_props)
54
+ end
55
+
56
+ if @config['targets']
57
+ target_params = []
58
+ @config['targets'].each { |t|
59
+ MU.retrier([MuNonFatal], max:5, wait: 9) {
60
+ target_ref = MU::Config::Ref.get(t)
61
+ target_obj = target_ref.kitten(cloud: "AWS")
62
+ this_target = if target_ref.is_mu_type? and target_obj and
63
+ !target_obj.arn.nil?
64
+ {
65
+ id: target_obj.cloud_id,
66
+ arn: target_obj.arn
67
+ }
68
+ elsif target_ref.id and target_ref.id.match(/^arn:/)
69
+ {
70
+ id: target_ref.id || target_ref.name,
71
+ arn: target_ref.id
72
+ }
73
+ else
74
+ raise MuNonFatal.new "Failed to retrieve ARN from CLoudWatch Event target descriptor", details: target_ref.to_h
75
+ end
76
+ if t['role']
77
+ role_obj = MU::Config::Ref.get(t['role']).kitten(@deploy, cloud: "AWS")
78
+ raise MuError.new "Failed to fetch object from role reference", details: t['role'].to_h if !role_obj
79
+ params[:role_arn] = role_obj.arn
80
+ end
81
+ [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters, :ecs_parameters].each { |attr|
82
+ if t[attr.to_s]
83
+ this_target[attr] = MU.structToHash(t[attr.to_s])
84
+ end
85
+ }
86
+ target_params << this_target
87
+ }
88
+ }
89
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_targets(
90
+ rule: @cloud_id,
91
+ event_bus_name: cloud_desc.event_bus_name,
92
+ targets: target_params
93
+ )
94
+ end
95
+
96
+ end
97
+
98
+ # Canonical Amazon Resource Number for this resource
99
+ # @return [String]
100
+ def arn
101
+ cloud_desc ? cloud_desc.arn : nil
102
+ end
103
+
104
+ # Return the metadata for this job
105
+ # @return [Hash]
106
+ def notify
107
+ MU.structToHash(cloud_desc, stringify_keys: true)
108
+ end
109
+
110
+ # Does this resource type exist as a global (cloud-wide) artifact, or
111
+ # is it localized to a region/zone?
112
+ # @return [Boolean]
113
+ def self.isGlobal?
114
+ false
115
+ end
116
+
117
+ # Denote whether this resource implementation is experiment, ready for
118
+ # testing, or ready for production use.
119
+ def self.quality
120
+ MU::Cloud::BETA
121
+ end
122
+
123
+ # Remove all jobs associated with the currently loaded deployment.
124
+ # @param noop [Boolean]: If true, will only print what would be done
125
+ # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
126
+ # @param region [String]: The cloud provider region
127
+ # @return [void]
128
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
129
+ found = find(region: region, credentials: credentials)
130
+
131
+ found.each_pair { |id, desc|
132
+ if (desc.description and desc.description == deploy_id) or
133
+ (flags and flags['known'] and flags['known'].include?(id))
134
+ MU.log "Deleting CloudWatch Event #{id}"
135
+ if !noop
136
+ resp = MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).list_targets_by_rule(
137
+ rule: id,
138
+ event_bus_name: desc.event_bus_name,
139
+ )
140
+ if resp and resp.targets and !resp.targets.empty?
141
+ MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).remove_targets(
142
+ rule: id,
143
+ event_bus_name: desc.event_bus_name,
144
+ ids: resp.targets.map { |t| t.id }
145
+ )
146
+ end
147
+
148
+ MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).delete_rule(
149
+ name: id,
150
+ event_bus_name: desc.event_bus_name
151
+ )
152
+ end
153
+ end
154
+ }
155
+ end
156
+
157
+ # Locate an existing event.
158
+ # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching CloudWatch Event
159
+ def self.find(**args)
160
+ found = {}
161
+
162
+ MU::Cloud::AWS.cloudwatchevents(region: args[:region], credentials: args[:credentials]).list_rules.rules.each { |r|
163
+ next if args[:cloud_id] and ![r.name, r.arn].include?(args[:cloud_id])
164
+ found[r.name] = r
165
+ }
166
+
167
+ found
168
+ end
169
+
170
+ # Reverse-map our cloud description into a runnable config hash.
171
+ # We assume that any values we have in +@config+ are placeholders, and
172
+ # calculate our own accordingly based on what's live in the cloud.
173
+ def toKitten(**_args)
174
+ bok = {
175
+ "cloud" => "AWS",
176
+ "credentials" => @config['credentials'],
177
+ "cloud_id" => @cloud_id,
178
+ "region" => @config['region']
179
+ }
180
+
181
+ if !cloud_desc
182
+ MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
183
+ return nil
184
+ end
185
+ bok['name'] = cloud_desc.name
186
+ if cloud_desc.description and !cloud_desc.description.empty?
187
+ bok['description'] = cloud_desc.description
188
+ end
189
+
190
+ bok['disabled'] = true if cloud_desc.state == "DISABLED"
191
+
192
+ # schedule_expression="cron(15 6 * * ? *)"
193
+ if cloud_desc.schedule_expression
194
+ if cloud_desc.schedule_expression.match(/cron\((\S+) (\S+) (\S+) (\S+) (\S+) (\S+)\)/)
195
+ bok['schedule'] = {
196
+ "minute" => Regexp.last_match[1],
197
+ "hour" => Regexp.last_match[2],
198
+ "day_of_month" => Regexp.last_match[3],
199
+ "month" => Regexp.last_match[4],
200
+ "day_of_week" => Regexp.last_match[5],
201
+ "year" => Regexp.last_match[6]
202
+ }
203
+ else
204
+ MU.log "HALP", MU::ERR, details: cloud_desc.schedule_expression
205
+ end
206
+ end
207
+
208
+ if cloud_desc.role_arn
209
+ shortname = cloud_desc.role_arn.sub(/.*?role\/([^\/]+)$/, '\1')
210
+ bok['role'] = MU::Config::Ref.get(
211
+ id: shortname,
212
+ cloud: "AWS",
213
+ type: "roles"
214
+ )
215
+ end
216
+
217
+ targets = MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).list_targets_by_rule(
218
+ rule: @cloud_id,
219
+ event_bus_name: cloud_desc.event_bus_name
220
+ ).targets
221
+ targets.each { |t|
222
+ bok['targets'] ||= []
223
+ _arn, _plat, service, region, account, resource = t.arn.split(/:/, 6)
224
+ target_type = if service == "lambda"
225
+ resource.sub!(/^function:/, '')
226
+ "functions"
227
+ elsif service == "sns"
228
+ "notifiers"
229
+ elsif service == "sqs"
230
+ "msg_queues"
231
+ else
232
+ service
233
+ end
234
+ ref_params = {
235
+ id: resource,
236
+ region: region,
237
+ type: target_type,
238
+ cloud: "AWS",
239
+ credentials: @credentials,
240
+ habitat: MU::Config::Ref.get(
241
+ id: account,
242
+ cloud: "AWS",
243
+ credentials: @credentials
244
+ )
245
+ }
246
+ [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters].each { |attr|
247
+ if t.respond_to?(attr) and !t.send(attr).nil?
248
+ ref_params[attr] = MU.structToHash(t.send(attr), stringify_keys: true)
249
+ end
250
+ }
251
+
252
+ bok['targets'] << MU::Config::Ref.get(ref_params)
253
+ }
254
+
255
+ # XXX cloud_desc.event_pattern - what do we want to do with this?
256
+
257
+ bok
258
+ end
259
+
260
+
261
+ # Cloud-specific configuration properties.
262
+ # @param _config [MU::Config]: The calling MU::Config object
263
+ # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
264
+ def self.schema(_config)
265
+ toplevel_required = []
266
+
267
+ target_schema = MU::Config::Ref.schema(any_type: true, desc: "A resource which will be invoked by this event. Can be a reference to a sibling Mu resource, typically a +Function+ or +MsgQueue+, or to an unadorned external cloud resource.")
268
+ target_params = {
269
+ "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to use when interacting with this target.", omit_fields: ["region", "tag"]),
270
+ "input" => {
271
+ "type" => "string"
272
+ },
273
+ "input_path" => {
274
+ "type" => "string"
275
+ },
276
+ "run_command_parameters" => {
277
+ "type" => "object",
278
+ "description" => "Parameters used when you are using the rule to invoke Amazon EC2 Run Command",
279
+ "required" => ["run_command_targets"],
280
+ "properties" => {
281
+ "run_command_targets" => {
282
+ "type" => "array",
283
+ "items" => {
284
+ "type" => "object",
285
+ "description" => "Currently, AWS supports including only one +run_command_targets+ block, which specifies either an array of InstanceIds or a tag.",
286
+ "required" => ["key", "values"],
287
+ "properties" => {
288
+ "key" => {
289
+ "type" => "string",
290
+ "description" => "Can be either +tag: tag-key+ or +InstanceIds+"
291
+ },
292
+ "values" => {
293
+ "type" => "array",
294
+ "items" => {
295
+ "description" => "If +key+ is +tag: tag-key+, +values+ is a list of tag values; if +key+ is +InstanceIds+, +values+ is a list of Amazon EC2 instance IDs.",
296
+ "type" => "string"
297
+ }
298
+ }
299
+ }
300
+ }
301
+ }
302
+ }
303
+ },
304
+ "input_transformer" => {
305
+ "type" => "object",
306
+ "description" => "Settings to enable you to provide custom input to a target based on certain event data. You can extract one or more key-value pairs from the event and then use that data to send customized input to the target.",
307
+ "required" => ["input_template"],
308
+ "properties" => {
309
+ "input_template" => {
310
+ "type" => "string",
311
+ "description" => "Input template where you specify placeholders that will be filled with the values of the keys from +input_paths_map+ to customize the data sent to the target."
312
+ },
313
+ "input_paths_map" => {
314
+ "type" => "object",
315
+ "description" => "Hash representing JSON paths to be extracted from the event"
316
+ }
317
+ }
318
+ },
319
+ "batch_parameters" => {
320
+ "type" => "object",
321
+ "description" => "If the event target is an AWS Batch job, this contains the job definition, job name, and other parameters. See: https://docs.aws.amazon.com/batch/latest/userguide/jobs.html",
322
+ "required" => ["job_definition", "job_name"],
323
+ "properties" => {
324
+ "job_definition" => {
325
+ "description" => "The ARN or name of the job definition to use if the event target is an AWS Batch job.",
326
+ "type" => "string"
327
+ },
328
+ "job_name" => {
329
+ "description" => "The name to use for this execution of the job, if the target is an AWS Batch job.",
330
+ "type" => "string"
331
+ },
332
+ "array_properties" => {
333
+ "type" => "object",
334
+ "description" => "The array properties for the submitted job, such as the size of the array.",
335
+ "properties" => {
336
+ "size" => {
337
+ "description" => "Size of the submitted array",
338
+ "type" => "integer"
339
+ }
340
+ }
341
+ },
342
+ "retry_strategy" => {
343
+ "type" => "object",
344
+ "description" => "The retry strategy to use for failed jobs, if the target is an AWS Batch job.",
345
+ "properties" => {
346
+ "attempts" => {
347
+ "description" => "Number of retry attempts, valid values from 1-10",
348
+ "type" => "integer"
349
+ }
350
+ }
351
+ }
352
+ }
353
+ },
354
+ "sqs_parameters" => {
355
+ "type" => "object",
356
+ "description" => "Contains the message group ID to use when the target is an SQS FIFO queue.",
357
+ "required" => ["message_group_id"],
358
+ "properties" => {
359
+ "message_group_id" => {
360
+ "type" => "string"
361
+ }
362
+ }
363
+ },
364
+ "kinesis_parameters" => {
365
+ "type" => "object",
366
+ "description" => "The custom parameter you can use to control the shard assignment, when the target is a Kinesis data stream.",
367
+ "required" => ["partition_key_path"],
368
+ "properties" => {
369
+ "partition_key_path" => {
370
+ "type" => "string"
371
+ }
372
+ }
373
+ },
374
+ "http_parameters" => {
375
+ "type" => "object",
376
+ "description" => "Contains the HTTP parameters to use when the target is a API Gateway REST endpoint.",
377
+ "properties" => {
378
+ "path_parameter_values" => {
379
+ "type" => "array",
380
+ "items" => {
381
+ "description" => "The path parameter values to be used to populate API Gateway REST API path wildcards (\"*\").",
382
+ "type" => "string"
383
+ }
384
+ },
385
+ "header_parameters" => {
386
+ "description" => "Key => value pairs to pass as headers",
387
+ "type" => "object"
388
+ },
389
+ "query_string_parameters" => {
390
+ "description" => "Key => value pairs to pass as query strings",
391
+ "type" => "object"
392
+ }
393
+ }
394
+ }
395
+ }
396
+ target_schema["properties"].merge!(target_params)
397
+
398
+ schema = {
399
+ "disabled" => {
400
+ "type" => "boolean",
401
+ "description" => "Leave this job in place but disabled",
402
+ "default" => false
403
+ },
404
+ "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to this CloudWatch Event.", omit_fields: ["region", "tag"]),
405
+ "targets" => {
406
+ "type" => "array",
407
+ "items" => target_schema
408
+ }
409
+ }
410
+ [toplevel_required, schema]
411
+ end
412
+
413
+ # Cloud-specific pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated.
414
+ # @param job [Hash]: The resource to process and validate
415
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
416
+ # @return [Boolean]: True if validation succeeded, False otherwise
417
+ def self.validateConfig(job, _configurator)
418
+ ok = true
419
+
420
+ job['targets'].each { |t|
421
+ target_ref = MU::Config::Ref.get(t)
422
+ if target_ref.is_mu_type? and target_ref.name
423
+ MU::Config.addDependency(job, target_ref.name, target_ref.type)
424
+ end
425
+ }
426
+
427
+ ok
428
+ end
429
+
430
+ private
431
+
432
+ def get_properties
433
+ params = {
434
+ name: @cloud_id,
435
+ state: @config['disabled'] ? "DISABLED" : "ENABLED",
436
+ event_bus_name: "default" # XXX expose, or create a deploy-specific one?
437
+ }
438
+
439
+ params[:description] = if @config['description'] and @config['scrub_mu_isms']
440
+ @config['description']
441
+ else
442
+ @deploy.deploy_id
443
+ end
444
+
445
+ if @tags
446
+ params[:tags] = @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }
447
+ end
448
+
449
+ if @config['role']
450
+ role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS")
451
+ raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj
452
+ params[:role_arn] = role_obj.arn
453
+ end
454
+
455
+ if @config['schedule']
456
+ params[:schedule_expression] = "cron(" + ["minute", "hour", "day_of_month", "month", "day_of_week", "year"].map { |i| @config['schedule'][i] }.join(" ") +")"
457
+ end
458
+
459
+
460
+ params
461
+ end
462
+
463
+ end
464
+ end
465
+ end
466
+ end