cloud-mu 3.1.3 → 3.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +21 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +4 -4
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +147 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +158 -111
  45. data/modules/mu/adoption.rb +404 -71
  46. data/modules/mu/cleanup.rb +221 -306
  47. data/modules/mu/cloud.rb +129 -1633
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +44 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +926 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +169 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +32 -3
  61. data/modules/mu/config/cache_cluster.rb +2 -2
  62. data/modules/mu/config/cdn.rb +100 -0
  63. data/modules/mu/config/collection.rb +4 -4
  64. data/modules/mu/config/container_cluster.rb +9 -4
  65. data/modules/mu/config/database.rb +84 -105
  66. data/modules/mu/config/database.yml +1 -2
  67. data/modules/mu/config/dnszone.rb +10 -9
  68. data/modules/mu/config/doc_helpers.rb +516 -0
  69. data/modules/mu/config/endpoint.rb +5 -4
  70. data/modules/mu/config/firewall_rule.rb +103 -4
  71. data/modules/mu/config/folder.rb +4 -4
  72. data/modules/mu/config/function.rb +19 -10
  73. data/modules/mu/config/group.rb +4 -4
  74. data/modules/mu/config/habitat.rb +4 -4
  75. data/modules/mu/config/job.rb +89 -0
  76. data/modules/mu/config/loadbalancer.rb +60 -14
  77. data/modules/mu/config/log.rb +4 -4
  78. data/modules/mu/config/msg_queue.rb +4 -4
  79. data/modules/mu/config/nosqldb.rb +4 -4
  80. data/modules/mu/config/notifier.rb +10 -21
  81. data/modules/mu/config/ref.rb +411 -0
  82. data/modules/mu/config/role.rb +4 -4
  83. data/modules/mu/config/schema_helpers.rb +509 -0
  84. data/modules/mu/config/search_domain.rb +4 -4
  85. data/modules/mu/config/server.rb +98 -71
  86. data/modules/mu/config/server.yml +1 -0
  87. data/modules/mu/config/server_pool.rb +5 -9
  88. data/modules/mu/config/storage_pool.rb +1 -1
  89. data/modules/mu/config/tail.rb +200 -0
  90. data/modules/mu/config/user.rb +4 -4
  91. data/modules/mu/config/vpc.rb +71 -27
  92. data/modules/mu/config/vpc.yml +0 -1
  93. data/modules/mu/defaults/AWS.yaml +91 -68
  94. data/modules/mu/defaults/Azure.yaml +1 -0
  95. data/modules/mu/defaults/Google.yaml +3 -2
  96. data/modules/mu/deploy.rb +43 -26
  97. data/modules/mu/groomer.rb +17 -2
  98. data/modules/mu/groomers/ansible.rb +188 -41
  99. data/modules/mu/groomers/chef.rb +116 -55
  100. data/modules/mu/logger.rb +127 -148
  101. data/modules/mu/master.rb +410 -2
  102. data/modules/mu/master/chef.rb +3 -4
  103. data/modules/mu/master/ldap.rb +3 -3
  104. data/modules/mu/master/ssl.rb +12 -3
  105. data/modules/mu/mommacat.rb +218 -2612
  106. data/modules/mu/mommacat/daemon.rb +403 -0
  107. data/modules/mu/mommacat/naming.rb +473 -0
  108. data/modules/mu/mommacat/search.rb +495 -0
  109. data/modules/mu/mommacat/storage.rb +722 -0
  110. data/modules/mu/{clouds → providers}/README.md +1 -1
  111. data/modules/mu/{clouds → providers}/aws.rb +380 -122
  112. data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
  113. data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
  114. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
  115. data/modules/mu/providers/aws/cdn.rb +782 -0
  116. data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
  117. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
  118. data/modules/mu/providers/aws/database.rb +1744 -0
  119. data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
  120. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  121. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
  122. data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
  123. data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
  124. data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
  125. data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
  126. data/modules/mu/providers/aws/job.rb +466 -0
  127. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
  128. data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
  129. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
  130. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
  131. data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
  132. data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
  133. data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
  134. data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
  135. data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
  136. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
  137. data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
  138. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  139. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  140. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  141. data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
  142. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  143. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  144. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  145. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  146. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  147. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  148. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  149. data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
  150. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  151. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  152. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  153. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  154. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  155. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  156. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  160. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  161. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  162. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  163. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  164. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  165. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  166. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  167. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  168. data/modules/mu/{clouds → providers}/google.rb +68 -30
  169. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  170. data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
  171. data/modules/mu/{clouds → providers}/google/database.rb +11 -21
  172. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  173. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  174. data/modules/mu/{clouds → providers}/google/function.rb +140 -168
  175. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  176. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  177. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
  178. data/modules/mu/{clouds → providers}/google/role.rb +94 -58
  179. data/modules/mu/{clouds → providers}/google/server.rb +243 -156
  180. data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
  181. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  182. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  183. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  184. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  185. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  186. data/modules/tests/aws-jobs-functions.yaml +46 -0
  187. data/modules/tests/bucket.yml +4 -0
  188. data/modules/tests/centos6.yaml +15 -0
  189. data/modules/tests/centos7.yaml +15 -0
  190. data/modules/tests/centos8.yaml +12 -0
  191. data/modules/tests/ecs.yaml +23 -0
  192. data/modules/tests/eks.yaml +1 -1
  193. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  194. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  195. data/modules/tests/includes-and-params.yaml +2 -1
  196. data/modules/tests/microservice_app.yaml +288 -0
  197. data/modules/tests/rds.yaml +108 -0
  198. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  199. data/modules/tests/regrooms/bucket.yml +19 -0
  200. data/modules/tests/regrooms/rds.yaml +123 -0
  201. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  202. data/modules/tests/super_complex_bok.yml +2 -2
  203. data/modules/tests/super_simple_bok.yml +3 -5
  204. data/modules/tests/win2k12.yaml +17 -5
  205. data/modules/tests/win2k16.yaml +25 -0
  206. data/modules/tests/win2k19.yaml +25 -0
  207. data/requirements.txt +1 -0
  208. data/spec/mu/clouds/azure_spec.rb +2 -2
  209. metadata +240 -154
  210. data/extras/image-generators/AWS/windows.yaml +0 -18
  211. data/modules/mu/clouds/aws/database.rb +0 -1985
  212. data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -39,7 +39,7 @@ module MU
39
39
  if !@config['use_if_exists']
40
40
  raise MuError, "IAM group #{@mu_name} already exists and use_if_exists is false"
41
41
  end
42
- rescue Aws::IAM::Errors::NoSuchEntity => e
42
+ rescue Aws::IAM::Errors::NoSuchEntity
43
43
  @config['path'] ||= "/"+@deploy.deploy_id+"/"
44
44
  MU.log "Creating IAM group #{@config['path']}#{@mu_name}"
45
45
  MU::Cloud::AWS.iam(credentials: @config['credentials']).create_group(
@@ -60,7 +60,7 @@ module MU
60
60
  userid = user
61
61
  userdesc = @deploy.findLitterMate(name: user, type: "users")
62
62
  userid = userdesc.cloud_id if userdesc
63
- found = MU::Cloud::AWS::User.find(cloud_id: userid)
63
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: userid)
64
64
  if found.size == 1
65
65
  userdesc = found.values.first
66
66
  MU.log "Adding IAM user #{userdesc.path}#{userdesc.user_name} to group #{@mu_name}", MU::NOTICE
@@ -88,7 +88,7 @@ module MU
88
88
  # Create these if necessary, then append them to the list of
89
89
  # attachable_policies
90
90
  if @config['raw_policies']
91
- pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
91
+ pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
92
92
  @config['raw_policies'],
93
93
  basename: @deploy.getResourceName(@config['name']),
94
94
  credentials: @credentials
@@ -99,7 +99,7 @@ module MU
99
99
 
100
100
  if @config['attachable_policies']
101
101
  configured_policies = @config['attachable_policies'].map { |p|
102
- id = if p.is_a?(MU::Config::Ref)
102
+ if p.is_a?(MU::Config::Ref)
103
103
  p.cloud_id
104
104
  else
105
105
  p = MU::Config::Ref.get(p)
@@ -114,7 +114,7 @@ module MU
114
114
  attached_policies.each { |a|
115
115
  if !configured_policies.include?(a.policy_arn)
116
116
  MU.log "Removing IAM policy #{a.policy_arn} from group #{@mu_name}", MU::NOTICE
117
- MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
117
+ MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
118
118
  else
119
119
  configured_policies.delete(a.policy_arn)
120
120
  end
@@ -131,7 +131,7 @@ module MU
131
131
  end
132
132
 
133
133
  if @config['inline_policies']
134
- docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
134
+ docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
135
135
  docs.each { |doc|
136
136
  MU.log "Putting user policy #{doc.keys.first} to group #{@cloud_id} "
137
137
  MU::Cloud::AWS.iam(credentials: @credentials).put_group_policy(
@@ -150,13 +150,16 @@ module MU
150
150
  cloud_desc.arn
151
151
  end
152
152
 
153
-
153
+ @cloud_desc_cache = nil
154
154
  # Fetch the AWS API description of this group
155
155
  # return [Struct]
156
- def cloud_desc
157
- MU::Cloud::AWS.iam(credentials: @config['credentials']).get_group(
156
+ def cloud_desc(use_cache: true)
157
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
158
+ return nil if !@mu_name
159
+ @cloud_desc_cache = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_group(
158
160
  group_name: @mu_name
159
161
  )
162
+ @cloud_desc_cache
160
163
  end
161
164
 
162
165
  # Return the metadata for this group configuration
@@ -183,11 +186,13 @@ module MU
183
186
  # Remove all groups associated with the currently loaded deployment.
184
187
  # @param noop [Boolean]: If true, will only print what would be done
185
188
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
186
- # @param region [String]: The cloud provider region
187
189
  # @return [void]
188
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
190
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
191
+ MU.log "AWS::Group.cleanup: need to support flags['known']", MU::DEBUG, details: flags
192
+ MU.log "Placeholder: AWS Group artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
193
+
189
194
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_groups(
190
- path_prefix: "/"+MU.deploy_id+"/"
195
+ path_prefix: "/"+deploy_id+"/"
191
196
  )
192
197
  if resp and resp.groups
193
198
  resp.groups.each { |g|
@@ -259,7 +264,7 @@ module MU
259
264
  # Reverse-map our cloud description into a runnable config hash.
260
265
  # We assume that any values we have in +@config+ are placeholders, and
261
266
  # calculate our own accordingly based on what's live in the cloud.
262
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
267
+ def toKitten(**_args)
263
268
  bok = {
264
269
  "cloud" => "AWS",
265
270
  "credentials" => @config['credentials'],
@@ -270,14 +275,15 @@ module MU
270
275
  MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
271
276
  return nil
272
277
  end
273
-
274
- bok["name"] = cloud_desc.group.group_name
275
278
 
276
- if cloud_desc.group.path != "/"
277
- bok["path"] = cloud_desc.group.path
279
+ group_desc = cloud_desc(use_cache: false).respond_to?(:group) ? cloud_desc.group : cloud_desc
280
+ bok["name"] = group_desc.group_name
281
+
282
+ if group_desc.path != "/"
283
+ bok["path"] = group_desc.path
278
284
  end
279
285
 
280
- if cloud_desc.users and cloud_desc.users.size > 0
286
+ if cloud_desc.respond_to?(:users) and cloud_desc.users and cloud_desc.users.size > 0
281
287
  bok["members"] = cloud_desc.users.map { |u| u.user_name }
282
288
  end
283
289
 
@@ -287,7 +293,7 @@ module MU
287
293
  resp.policy_names.each { |pol_name|
288
294
  pol = MU::Cloud::AWS.iam(credentials: @credentials).get_group_policy(group_name: @cloud_id, policy_name: pol_name)
289
295
  doc = JSON.parse(URI.decode(pol.policy_document))
290
- bok["inline_policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
296
+ bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
291
297
  }
292
298
  end
293
299
 
@@ -315,12 +321,12 @@ module MU
315
321
  end
316
322
 
317
323
  # Cloud-specific configuration properties.
318
- # @param config [MU::Config]: The calling MU::Config object
324
+ # @param _config [MU::Config]: The calling MU::Config object
319
325
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
320
- def self.schema(config)
326
+ def self.schema(_config)
321
327
  toplevel_required = []
322
328
  polschema = MU::Config::Role.schema["properties"]["policies"]
323
- polschema.deep_merge!(MU::Cloud::AWS::Role.condition_schema)
329
+ polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
324
330
 
325
331
  schema = {
326
332
  "inline_policies" => polschema,
@@ -360,7 +366,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
360
366
  # If we're attaching some managed policies, make sure all of the ones
361
367
  # that should already exist do indeed exist
362
368
  if group['attachable_policies']
363
- ok = false if !MU::Cloud::AWS::Role.validateAttachablePolicies(
369
+ ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
364
370
  group['attachable_policies'],
365
371
  credentials: group['credentials'],
366
372
  region: group['region']
@@ -374,13 +380,9 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
374
380
  if group['members']
375
381
  group['members'].each { |user|
376
382
  if configurator.haveLitterMate?(user, "users")
377
- group["dependencies"] ||= []
378
- group["dependencies"] << {
379
- "type" => "user",
380
- "name" => user
381
- }
383
+ MU::Config.addDependency(group, user, "user")
382
384
  else
383
- found = MU::Cloud::AWS::User.find(cloud_id: user)
385
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: user)
384
386
  if found.nil? or found.empty?
385
387
  MU.log "Error in members for group #{group['name']}: No such user #{user}", MU::ERR
386
388
  ok = false
@@ -45,7 +45,6 @@ module MU
45
45
  resp = MU::Cloud::AWS.orgs(credentials: @config['credentials']).describe_create_account_status(
46
46
  create_account_request_id: createid
47
47
  )
48
- createstatus = resp.create_account_status.state
49
48
  if !["SUCCEEDED", "IN_PROGRESS"].include?(resp.create_account_status.state)
50
49
  raise MuError, "Failed to create account #{@mu_name}: #{resp.create_account_status.failure_reason}"
51
50
  end
@@ -59,9 +58,12 @@ module MU
59
58
  MU.log "Creation of account #{@mu_name} (#{resp.create_account_status.account_id}) complete"
60
59
  end
61
60
 
61
+ @cloud_desc_cache = nil
62
62
  # Return the cloud descriptor for the Habitat
63
- def cloud_desc
64
- MU::Cloud::AWS::Habitat.find(cloud_id: @cloud_id).values.first
63
+ def cloud_desc(use_cache: true)
64
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
65
+ @cloud_desc_cache = MU::Cloud::AWS::Habitat.find(cloud_id: @cloud_id).values.first
66
+ @cloud_desc_cache
65
67
  end
66
68
 
67
69
  # Canonical Amazon Resource Number for this resource
@@ -87,16 +89,17 @@ module MU
87
89
  # Remove all AWS accounts associated with the currently loaded deployment. Try to, anyway.
88
90
  # @param noop [Boolean]: If true, will only print what would be done
89
91
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
90
- # @param region [String]: The cloud provider region
91
92
  # @return [void]
92
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
93
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
93
94
  return if !orgMasterCreds?(credentials)
95
+ MU.log "AWS::Habitat.cleanup: need to support flags['known']", MU::DEBUG, details: flags
96
+ MU.log "Placeholder: AWS Habitat artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
94
97
 
95
98
  resp = MU::Cloud::AWS.orgs(credentials: credentials).list_accounts
96
99
 
97
100
  if resp and resp.accounts
98
101
  resp.accounts.each { |acct|
99
- if acct.name.match(/^#{Regexp.quote(MU.deploy_id)}/) or acct.name.match(/BUNS/)
102
+ if acct.name.match(/^#{Regexp.quote(deploy_id)}/) or acct.name.match(/BUNS/)
100
103
  if !noop
101
104
  pp acct
102
105
  end
@@ -108,14 +111,14 @@ module MU
108
111
 
109
112
  # Locate an existing account
110
113
  # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching account
111
- def self.find(**args)
114
+ def self.find(**_args)
112
115
  {}
113
116
  end
114
117
 
115
118
  # Cloud-specific configuration properties.
116
- # @param config [MU::Config]: The calling MU::Config object
119
+ # @param _config [MU::Config]: The calling MU::Config object
117
120
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
118
- def self.schema(config)
121
+ def self.schema(_config)
119
122
  toplevel_required = []
120
123
  schema = {
121
124
  "email" => {
@@ -126,9 +129,10 @@ module MU
126
129
  [toplevel_required, schema]
127
130
  end
128
131
 
129
- # @param account_number [String]
132
+ # @param _account_number [String]
133
+ # @param _credentials [String]
130
134
  # @return [Boolean]
131
- def self.isLive?(account_number, credentials = nil)
135
+ def self.isLive?(_account_number, _credentials = nil)
132
136
  true
133
137
  end
134
138
 
@@ -138,18 +142,17 @@ module MU
138
142
  # @param credentials [String]
139
143
  # @return [Boolean]
140
144
  def self.orgMasterCreds?(credentials = nil)
141
- user_list = MU::Cloud::AWS.iam(credentials: credentials).list_users.users
142
145
  acct_num = MU::Cloud::AWS.iam(credentials: credentials).list_users.users.first.arn.split(/:/)[4]
143
146
 
144
- parentorg = MU::Cloud::AWS::Folder.find(credentials: credentials).values.first
147
+ parentorg = MU::Cloud.resourceClass("AWS", "Folder").find(credentials: credentials).values.first
145
148
  acct_num == parentorg.master_account_id
146
149
  end
147
150
 
148
151
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::habitats}, bare and unvalidated.
149
152
  # @param habitat [Hash]: The resource to process and validate
150
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
153
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
151
154
  # @return [Boolean]: True if validation succeeded, False otherwise
152
- def self.validateConfig(habitat, configurator)
155
+ def self.validateConfig(habitat, _configurator)
153
156
  ok = true
154
157
 
155
158
  if !habitat["email"]
@@ -0,0 +1,466 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ class Cloud
17
+ class AWS
18
+ # A scheduled task facility as configured in {MU::Config::BasketofKittens::jobs}
19
+ class Job < MU::Cloud::Job
20
+
21
+ # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
22
+ # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
23
+ def initialize(**args)
24
+ super
25
+ @mu_name ||= @deploy.getResourceName(@config["name"])
26
+ end
27
+
28
+ # Called automatically by {MU::Deploy#createResources}
29
+ def create
30
+ @cloud_id = @mu_name
31
+
32
+ params = get_properties
33
+
34
+ MU.log "Creating CloudWatch Event #{@mu_name}", MU::NOTICE, details: params
35
+
36
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(params)
37
+ end
38
+
39
+ # Called automatically by {MU::Deploy#createResources}
40
+ def groom
41
+ new_props = get_properties
42
+ current = MU.structToHash(cloud_desc(use_cache: false))
43
+ params = {}
44
+ new_props.each_pair { |k, v|
45
+ next if k == :tags # doesn't seem to do anything
46
+ if v != current[k]
47
+ params[k] = v
48
+ end
49
+ }
50
+
51
+ if params.size > 0
52
+ MU.log "Updating CloudWatch Event #{@cloud_id}", MU::NOTICE, details: params
53
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(new_props)
54
+ end
55
+
56
+ if @config['targets']
57
+ target_params = []
58
+ @config['targets'].each { |t|
59
+ MU.retrier([MuNonFatal], max:5, wait: 9) {
60
+ target_ref = MU::Config::Ref.get(t)
61
+ target_obj = target_ref.kitten(cloud: "AWS")
62
+ this_target = if target_ref.is_mu_type? and target_obj and
63
+ !target_obj.arn.nil?
64
+ {
65
+ id: target_obj.cloud_id,
66
+ arn: target_obj.arn
67
+ }
68
+ elsif target_ref.id and target_ref.id.match(/^arn:/)
69
+ {
70
+ id: target_ref.id || target_ref.name,
71
+ arn: target_ref.id
72
+ }
73
+ else
74
+ raise MuNonFatal.new "Failed to retrieve ARN from CLoudWatch Event target descriptor", details: target_ref.to_h
75
+ end
76
+ if t['role']
77
+ role_obj = MU::Config::Ref.get(t['role']).kitten(@deploy, cloud: "AWS")
78
+ raise MuError.new "Failed to fetch object from role reference", details: t['role'].to_h if !role_obj
79
+ params[:role_arn] = role_obj.arn
80
+ end
81
+ [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters, :ecs_parameters].each { |attr|
82
+ if t[attr.to_s]
83
+ this_target[attr] = MU.structToHash(t[attr.to_s])
84
+ end
85
+ }
86
+ target_params << this_target
87
+ }
88
+ }
89
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_targets(
90
+ rule: @cloud_id,
91
+ event_bus_name: cloud_desc.event_bus_name,
92
+ targets: target_params
93
+ )
94
+ end
95
+
96
+ end
97
+
98
+ # Canonical Amazon Resource Number for this resource
99
+ # @return [String]
100
+ def arn
101
+ cloud_desc ? cloud_desc.arn : nil
102
+ end
103
+
104
+ # Return the metadata for this job
105
+ # @return [Hash]
106
+ def notify
107
+ MU.structToHash(cloud_desc, stringify_keys: true)
108
+ end
109
+
110
+ # Does this resource type exist as a global (cloud-wide) artifact, or
111
+ # is it localized to a region/zone?
112
+ # @return [Boolean]
113
+ def self.isGlobal?
114
+ false
115
+ end
116
+
117
+ # Denote whether this resource implementation is experiment, ready for
118
+ # testing, or ready for production use.
119
+ def self.quality
120
+ MU::Cloud::BETA
121
+ end
122
+
123
+ # Remove all jobs associated with the currently loaded deployment.
124
+ # @param noop [Boolean]: If true, will only print what would be done
125
+ # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
126
+ # @param region [String]: The cloud provider region
127
+ # @return [void]
128
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
129
+ found = find(region: region, credentials: credentials)
130
+
131
+ found.each_pair { |id, desc|
132
+ if (desc.description and desc.description == deploy_id) or
133
+ (flags and flags['known'] and flags['known'].include?(id))
134
+ MU.log "Deleting CloudWatch Event #{id}"
135
+ if !noop
136
+ resp = MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).list_targets_by_rule(
137
+ rule: id,
138
+ event_bus_name: desc.event_bus_name,
139
+ )
140
+ if resp and resp.targets and !resp.targets.empty?
141
+ MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).remove_targets(
142
+ rule: id,
143
+ event_bus_name: desc.event_bus_name,
144
+ ids: resp.targets.map { |t| t.id }
145
+ )
146
+ end
147
+
148
+ MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).delete_rule(
149
+ name: id,
150
+ event_bus_name: desc.event_bus_name
151
+ )
152
+ end
153
+ end
154
+ }
155
+ end
156
+
157
+ # Locate an existing event.
158
+ # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching CloudWatch Event
159
+ def self.find(**args)
160
+ found = {}
161
+
162
+ MU::Cloud::AWS.cloudwatchevents(region: args[:region], credentials: args[:credentials]).list_rules.rules.each { |r|
163
+ next if args[:cloud_id] and ![r.name, r.arn].include?(args[:cloud_id])
164
+ found[r.name] = r
165
+ }
166
+
167
+ found
168
+ end
169
+
170
+ # Reverse-map our cloud description into a runnable config hash.
171
+ # We assume that any values we have in +@config+ are placeholders, and
172
+ # calculate our own accordingly based on what's live in the cloud.
173
+ def toKitten(**_args)
174
+ bok = {
175
+ "cloud" => "AWS",
176
+ "credentials" => @config['credentials'],
177
+ "cloud_id" => @cloud_id,
178
+ "region" => @config['region']
179
+ }
180
+
181
+ if !cloud_desc
182
+ MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
183
+ return nil
184
+ end
185
+ bok['name'] = cloud_desc.name
186
+ if cloud_desc.description and !cloud_desc.description.empty?
187
+ bok['description'] = cloud_desc.description
188
+ end
189
+
190
+ bok['disabled'] = true if cloud_desc.state == "DISABLED"
191
+
192
+ # schedule_expression="cron(15 6 * * ? *)"
193
+ if cloud_desc.schedule_expression
194
+ if cloud_desc.schedule_expression.match(/cron\((\S+) (\S+) (\S+) (\S+) (\S+) (\S+)\)/)
195
+ bok['schedule'] = {
196
+ "minute" => Regexp.last_match[1],
197
+ "hour" => Regexp.last_match[2],
198
+ "day_of_month" => Regexp.last_match[3],
199
+ "month" => Regexp.last_match[4],
200
+ "day_of_week" => Regexp.last_match[5],
201
+ "year" => Regexp.last_match[6]
202
+ }
203
+ else
204
+ MU.log "HALP", MU::ERR, details: cloud_desc.schedule_expression
205
+ end
206
+ end
207
+
208
+ if cloud_desc.role_arn
209
+ shortname = cloud_desc.role_arn.sub(/.*?role\/([^\/]+)$/, '\1')
210
+ bok['role'] = MU::Config::Ref.get(
211
+ id: shortname,
212
+ cloud: "AWS",
213
+ type: "roles"
214
+ )
215
+ end
216
+
217
+ targets = MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).list_targets_by_rule(
218
+ rule: @cloud_id,
219
+ event_bus_name: cloud_desc.event_bus_name
220
+ ).targets
221
+ targets.each { |t|
222
+ bok['targets'] ||= []
223
+ _arn, _plat, service, region, account, resource = t.arn.split(/:/, 6)
224
+ target_type = if service == "lambda"
225
+ resource.sub!(/^function:/, '')
226
+ "functions"
227
+ elsif service == "sns"
228
+ "notifiers"
229
+ elsif service == "sqs"
230
+ "msg_queues"
231
+ else
232
+ service
233
+ end
234
+ ref_params = {
235
+ id: resource,
236
+ region: region,
237
+ type: target_type,
238
+ cloud: "AWS",
239
+ credentials: @credentials,
240
+ habitat: MU::Config::Ref.get(
241
+ id: account,
242
+ cloud: "AWS",
243
+ credentials: @credentials
244
+ )
245
+ }
246
+ [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters].each { |attr|
247
+ if t.respond_to?(attr) and !t.send(attr).nil?
248
+ ref_params[attr] = MU.structToHash(t.send(attr), stringify_keys: true)
249
+ end
250
+ }
251
+
252
+ bok['targets'] << MU::Config::Ref.get(ref_params)
253
+ }
254
+
255
+ # XXX cloud_desc.event_pattern - what do we want to do with this?
256
+
257
+ bok
258
+ end
259
+
260
+
261
+ # Cloud-specific configuration properties.
262
+ # @param _config [MU::Config]: The calling MU::Config object
263
+ # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
264
+ def self.schema(_config)
265
+ toplevel_required = []
266
+
267
+ target_schema = MU::Config::Ref.schema(any_type: true, desc: "A resource which will be invoked by this event. Can be a reference to a sibling Mu resource, typically a +Function+ or +MsgQueue+, or to an unadorned external cloud resource.")
268
+ target_params = {
269
+ "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to use when interacting with this target.", omit_fields: ["region", "tag"]),
270
+ "input" => {
271
+ "type" => "string"
272
+ },
273
+ "input_path" => {
274
+ "type" => "string"
275
+ },
276
+ "run_command_parameters" => {
277
+ "type" => "object",
278
+ "description" => "Parameters used when you are using the rule to invoke Amazon EC2 Run Command",
279
+ "required" => ["run_command_targets"],
280
+ "properties" => {
281
+ "run_command_targets" => {
282
+ "type" => "array",
283
+ "items" => {
284
+ "type" => "object",
285
+ "description" => "Currently, AWS supports including only one +run_command_targets+ block, which specifies either an array of InstanceIds or a tag.",
286
+ "required" => ["key", "values"],
287
+ "properties" => {
288
+ "key" => {
289
+ "type" => "string",
290
+ "description" => "Can be either +tag: tag-key+ or +InstanceIds+"
291
+ },
292
+ "values" => {
293
+ "type" => "array",
294
+ "items" => {
295
+ "description" => "If +key+ is +tag: tag-key+, +values+ is a list of tag values; if +key+ is +InstanceIds+, +values+ is a list of Amazon EC2 instance IDs.",
296
+ "type" => "string"
297
+ }
298
+ }
299
+ }
300
+ }
301
+ }
302
+ }
303
+ },
304
+ "input_transformer" => {
305
+ "type" => "object",
306
+ "description" => "Settings to enable you to provide custom input to a target based on certain event data. You can extract one or more key-value pairs from the event and then use that data to send customized input to the target.",
307
+ "required" => ["input_template"],
308
+ "properties" => {
309
+ "input_template" => {
310
+ "type" => "string",
311
+ "description" => "Input template where you specify placeholders that will be filled with the values of the keys from +input_paths_map+ to customize the data sent to the target."
312
+ },
313
+ "input_paths_map" => {
314
+ "type" => "object",
315
+ "description" => "Hash representing JSON paths to be extracted from the event"
316
+ }
317
+ }
318
+ },
319
+ "batch_parameters" => {
320
+ "type" => "object",
321
+ "description" => "If the event target is an AWS Batch job, this contains the job definition, job name, and other parameters. See: https://docs.aws.amazon.com/batch/latest/userguide/jobs.html",
322
+ "required" => ["job_definition", "job_name"],
323
+ "properties" => {
324
+ "job_definition" => {
325
+ "description" => "The ARN or name of the job definition to use if the event target is an AWS Batch job.",
326
+ "type" => "string"
327
+ },
328
+ "job_name" => {
329
+ "description" => "The name to use for this execution of the job, if the target is an AWS Batch job.",
330
+ "type" => "string"
331
+ },
332
+ "array_properties" => {
333
+ "type" => "object",
334
+ "description" => "The array properties for the submitted job, such as the size of the array.",
335
+ "properties" => {
336
+ "size" => {
337
+ "description" => "Size of the submitted array",
338
+ "type" => "integer"
339
+ }
340
+ }
341
+ },
342
+ "retry_strategy" => {
343
+ "type" => "object",
344
+ "description" => "The retry strategy to use for failed jobs, if the target is an AWS Batch job.",
345
+ "properties" => {
346
+ "attempts" => {
347
+ "description" => "Number of retry attempts, valid values from 1-10",
348
+ "type" => "integer"
349
+ }
350
+ }
351
+ }
352
+ }
353
+ },
354
+ "sqs_parameters" => {
355
+ "type" => "object",
356
+ "description" => "Contains the message group ID to use when the target is an SQS FIFO queue.",
357
+ "required" => ["message_group_id"],
358
+ "properties" => {
359
+ "message_group_id" => {
360
+ "type" => "string"
361
+ }
362
+ }
363
+ },
364
+ "kinesis_parameters" => {
365
+ "type" => "object",
366
+ "description" => "The custom parameter you can use to control the shard assignment, when the target is a Kinesis data stream.",
367
+ "required" => ["partition_key_path"],
368
+ "properties" => {
369
+ "partition_key_path" => {
370
+ "type" => "string"
371
+ }
372
+ }
373
+ },
374
+ "http_parameters" => {
375
+ "type" => "object",
376
+ "description" => "Contains the HTTP parameters to use when the target is a API Gateway REST endpoint.",
377
+ "properties" => {
378
+ "path_parameter_values" => {
379
+ "type" => "array",
380
+ "items" => {
381
+ "description" => "The path parameter values to be used to populate API Gateway REST API path wildcards (\"*\").",
382
+ "type" => "string"
383
+ }
384
+ },
385
+ "header_parameters" => {
386
+ "description" => "Key => value pairs to pass as headers",
387
+ "type" => "object"
388
+ },
389
+ "query_string_parameters" => {
390
+ "description" => "Key => value pairs to pass as query strings",
391
+ "type" => "object"
392
+ }
393
+ }
394
+ }
395
+ }
396
+ target_schema["properties"].merge!(target_params)
397
+
398
+ schema = {
399
+ "disabled" => {
400
+ "type" => "boolean",
401
+ "description" => "Leave this job in place but disabled",
402
+ "default" => false
403
+ },
404
+ "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to this CloudWatch Event.", omit_fields: ["region", "tag"]),
405
+ "targets" => {
406
+ "type" => "array",
407
+ "items" => target_schema
408
+ }
409
+ }
410
+ [toplevel_required, schema]
411
+ end
412
+
413
+ # Cloud-specific pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated.
414
+ # @param job [Hash]: The resource to process and validate
415
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
416
+ # @return [Boolean]: True if validation succeeded, False otherwise
417
+ def self.validateConfig(job, _configurator)
418
+ ok = true
419
+
420
+ job['targets'].each { |t|
421
+ target_ref = MU::Config::Ref.get(t)
422
+ if target_ref.is_mu_type? and target_ref.name
423
+ MU::Config.addDependency(job, target_ref.name, target_ref.type)
424
+ end
425
+ }
426
+
427
+ ok
428
+ end
429
+
430
+ private
431
+
432
+ def get_properties
433
+ params = {
434
+ name: @cloud_id,
435
+ state: @config['disabled'] ? "DISABLED" : "ENABLED",
436
+ event_bus_name: "default" # XXX expose, or create a deploy-specific one?
437
+ }
438
+
439
+ params[:description] = if @config['description'] and @config['scrub_mu_isms']
440
+ @config['description']
441
+ else
442
+ @deploy.deploy_id
443
+ end
444
+
445
+ if @tags
446
+ params[:tags] = @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }
447
+ end
448
+
449
+ if @config['role']
450
+ role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS")
451
+ raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj
452
+ params[:role_arn] = role_obj.arn
453
+ end
454
+
455
+ if @config['schedule']
456
+ params[:schedule_expression] = "cron(" + ["minute", "hour", "day_of_month", "month", "day_of_week", "year"].map { |i| @config['schedule'][i] }.join(" ") +")"
457
+ end
458
+
459
+
460
+ params
461
+ end
462
+
463
+ end
464
+ end
465
+ end
466
+ end