cloud-mu 3.1.6 → 3.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (181) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +1 -1
  3. data/bin/mu-adopt +15 -12
  4. data/bin/mu-azure-tests +57 -0
  5. data/bin/mu-cleanup +2 -4
  6. data/bin/mu-configure +37 -1
  7. data/bin/mu-deploy +3 -3
  8. data/bin/mu-findstray-tests +25 -0
  9. data/bin/mu-gen-docs +2 -4
  10. data/bin/mu-load-config.rb +2 -1
  11. data/bin/mu-run-tests +37 -12
  12. data/cloud-mu.gemspec +4 -4
  13. data/cookbooks/mu-tools/attributes/default.rb +7 -0
  14. data/cookbooks/mu-tools/libraries/helper.rb +87 -3
  15. data/cookbooks/mu-tools/recipes/apply_security.rb +39 -23
  16. data/cookbooks/mu-tools/recipes/aws_api.rb +13 -0
  17. data/cookbooks/mu-tools/recipes/google_api.rb +4 -0
  18. data/cookbooks/mu-tools/recipes/rsyslog.rb +8 -1
  19. data/cookbooks/mu-tools/resources/disk.rb +33 -12
  20. data/cookbooks/mu-tools/resources/mommacat_request.rb +1 -2
  21. data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb +215 -0
  22. data/extras/clean-stock-amis +10 -2
  23. data/extras/generate-stock-images +7 -3
  24. data/extras/image-generators/AWS/centos7.yaml +19 -16
  25. data/extras/image-generators/AWS/{rhel7.yaml → rhel71.yaml} +0 -0
  26. data/extras/image-generators/AWS/{win2k12.yaml → win2k12r2.yaml} +0 -0
  27. data/modules/mommacat.ru +2 -2
  28. data/modules/mu.rb +84 -97
  29. data/modules/mu/adoption.rb +359 -59
  30. data/modules/mu/cleanup.rb +67 -44
  31. data/modules/mu/cloud.rb +108 -1754
  32. data/modules/mu/cloud/database.rb +49 -0
  33. data/modules/mu/cloud/dnszone.rb +44 -0
  34. data/modules/mu/cloud/machine_images.rb +212 -0
  35. data/modules/mu/cloud/providers.rb +81 -0
  36. data/modules/mu/cloud/resource_base.rb +929 -0
  37. data/modules/mu/cloud/server.rb +40 -0
  38. data/modules/mu/cloud/server_pool.rb +1 -0
  39. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  40. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  41. data/modules/mu/cloud/wrappers.rb +178 -0
  42. data/modules/mu/config.rb +122 -80
  43. data/modules/mu/config/alarm.rb +2 -6
  44. data/modules/mu/config/bucket.rb +32 -3
  45. data/modules/mu/config/cache_cluster.rb +2 -2
  46. data/modules/mu/config/cdn.rb +100 -0
  47. data/modules/mu/config/collection.rb +1 -1
  48. data/modules/mu/config/container_cluster.rb +2 -2
  49. data/modules/mu/config/database.rb +84 -105
  50. data/modules/mu/config/database.yml +1 -2
  51. data/modules/mu/config/dnszone.rb +5 -4
  52. data/modules/mu/config/doc_helpers.rb +4 -5
  53. data/modules/mu/config/endpoint.rb +2 -1
  54. data/modules/mu/config/firewall_rule.rb +3 -19
  55. data/modules/mu/config/folder.rb +1 -1
  56. data/modules/mu/config/function.rb +17 -8
  57. data/modules/mu/config/group.rb +1 -1
  58. data/modules/mu/config/habitat.rb +1 -1
  59. data/modules/mu/config/job.rb +89 -0
  60. data/modules/mu/config/loadbalancer.rb +57 -11
  61. data/modules/mu/config/log.rb +1 -1
  62. data/modules/mu/config/msg_queue.rb +1 -1
  63. data/modules/mu/config/nosqldb.rb +1 -1
  64. data/modules/mu/config/notifier.rb +8 -19
  65. data/modules/mu/config/ref.rb +81 -9
  66. data/modules/mu/config/role.rb +1 -1
  67. data/modules/mu/config/schema_helpers.rb +30 -34
  68. data/modules/mu/config/search_domain.rb +1 -1
  69. data/modules/mu/config/server.rb +5 -13
  70. data/modules/mu/config/server_pool.rb +3 -7
  71. data/modules/mu/config/storage_pool.rb +1 -1
  72. data/modules/mu/config/tail.rb +10 -0
  73. data/modules/mu/config/user.rb +1 -1
  74. data/modules/mu/config/vpc.rb +13 -17
  75. data/modules/mu/defaults/AWS.yaml +106 -106
  76. data/modules/mu/defaults/Azure.yaml +1 -0
  77. data/modules/mu/defaults/Google.yaml +1 -0
  78. data/modules/mu/deploy.rb +33 -19
  79. data/modules/mu/groomer.rb +15 -0
  80. data/modules/mu/groomers/chef.rb +3 -0
  81. data/modules/mu/logger.rb +120 -144
  82. data/modules/mu/master.rb +22 -1
  83. data/modules/mu/mommacat.rb +71 -26
  84. data/modules/mu/mommacat/daemon.rb +23 -14
  85. data/modules/mu/mommacat/naming.rb +82 -3
  86. data/modules/mu/mommacat/search.rb +59 -16
  87. data/modules/mu/mommacat/storage.rb +119 -48
  88. data/modules/mu/{clouds → providers}/README.md +1 -1
  89. data/modules/mu/{clouds → providers}/aws.rb +248 -62
  90. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  91. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  92. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  93. data/modules/mu/providers/aws/cdn.rb +782 -0
  94. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  95. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +65 -63
  96. data/modules/mu/providers/aws/database.rb +1747 -0
  97. data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
  98. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  99. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
  100. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  101. data/modules/mu/{clouds → providers}/aws/function.rb +291 -133
  102. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  103. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  104. data/modules/mu/providers/aws/job.rb +469 -0
  105. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
  106. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  107. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  108. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  109. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  110. data/modules/mu/{clouds → providers}/aws/role.rb +112 -78
  111. data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
  112. data/modules/mu/{clouds → providers}/aws/server.rb +120 -145
  113. data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
  114. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  115. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  116. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  117. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  118. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  119. data/modules/mu/{clouds → providers}/aws/vpc.rb +141 -73
  120. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  121. data/modules/mu/{clouds → providers}/azure.rb +4 -1
  122. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  123. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  124. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  125. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  126. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  127. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  128. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  129. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  130. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  131. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  132. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  133. data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
  134. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  135. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  136. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  137. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  138. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  139. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  140. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  141. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  142. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  143. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  144. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  145. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  146. data/modules/mu/{clouds → providers}/google.rb +15 -6
  147. data/modules/mu/{clouds → providers}/google/bucket.rb +2 -2
  148. data/modules/mu/{clouds → providers}/google/container_cluster.rb +29 -14
  149. data/modules/mu/{clouds → providers}/google/database.rb +2 -9
  150. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +3 -3
  151. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  152. data/modules/mu/{clouds → providers}/google/function.rb +4 -4
  153. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  154. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  155. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +2 -2
  156. data/modules/mu/{clouds → providers}/google/role.rb +46 -35
  157. data/modules/mu/{clouds → providers}/google/server.rb +26 -11
  158. data/modules/mu/{clouds → providers}/google/server_pool.rb +11 -11
  159. data/modules/mu/{clouds → providers}/google/user.rb +32 -22
  160. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  161. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  162. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  163. data/modules/mu/{clouds → providers}/google/vpc.rb +38 -3
  164. data/modules/tests/aws-jobs-functions.yaml +46 -0
  165. data/modules/tests/centos6.yaml +15 -0
  166. data/modules/tests/centos7.yaml +15 -0
  167. data/modules/tests/centos8.yaml +12 -0
  168. data/modules/tests/ecs.yaml +2 -2
  169. data/modules/tests/eks.yaml +1 -1
  170. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  171. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  172. data/modules/tests/microservice_app.yaml +288 -0
  173. data/modules/tests/rds.yaml +108 -0
  174. data/modules/tests/regrooms/rds.yaml +123 -0
  175. data/modules/tests/server-with-scrub-muisms.yaml +1 -1
  176. data/modules/tests/super_complex_bok.yml +2 -2
  177. data/modules/tests/super_simple_bok.yml +2 -2
  178. data/spec/mu/clouds/azure_spec.rb +2 -2
  179. metadata +126 -98
  180. data/modules/mu/clouds/aws/database.rb +0 -1974
  181. data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -60,7 +60,7 @@ module MU
60
60
  userid = user
61
61
  userdesc = @deploy.findLitterMate(name: user, type: "users")
62
62
  userid = userdesc.cloud_id if userdesc
63
- found = MU::Cloud::AWS::User.find(cloud_id: userid)
63
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: userid)
64
64
  if found.size == 1
65
65
  userdesc = found.values.first
66
66
  MU.log "Adding IAM user #{userdesc.path}#{userdesc.user_name} to group #{@mu_name}", MU::NOTICE
@@ -88,7 +88,7 @@ module MU
88
88
  # Create these if necessary, then append them to the list of
89
89
  # attachable_policies
90
90
  if @config['raw_policies']
91
- pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
91
+ pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
92
92
  @config['raw_policies'],
93
93
  basename: @deploy.getResourceName(@config['name']),
94
94
  credentials: @credentials
@@ -114,7 +114,7 @@ module MU
114
114
  attached_policies.each { |a|
115
115
  if !configured_policies.include?(a.policy_arn)
116
116
  MU.log "Removing IAM policy #{a.policy_arn} from group #{@mu_name}", MU::NOTICE
117
- MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
117
+ MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
118
118
  else
119
119
  configured_policies.delete(a.policy_arn)
120
120
  end
@@ -131,7 +131,7 @@ module MU
131
131
  end
132
132
 
133
133
  if @config['inline_policies']
134
- docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
134
+ docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
135
135
  docs.each { |doc|
136
136
  MU.log "Putting user policy #{doc.keys.first} to group #{@cloud_id} "
137
137
  MU::Cloud::AWS.iam(credentials: @credentials).put_group_policy(
@@ -155,6 +155,7 @@ module MU
155
155
  # return [Struct]
156
156
  def cloud_desc(use_cache: true)
157
157
  return @cloud_desc_cache if @cloud_desc_cache and use_cache
158
+ return nil if !@mu_name
158
159
  @cloud_desc_cache = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_group(
159
160
  group_name: @mu_name
160
161
  )
@@ -186,12 +187,12 @@ module MU
186
187
  # @param noop [Boolean]: If true, will only print what would be done
187
188
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
188
189
  # @return [void]
189
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
190
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
190
191
  MU.log "AWS::Group.cleanup: need to support flags['known']", MU::DEBUG, details: flags
191
192
  MU.log "Placeholder: AWS Group artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
192
193
 
193
194
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_groups(
194
- path_prefix: "/"+MU.deploy_id+"/"
195
+ path_prefix: "/"+deploy_id+"/"
195
196
  )
196
197
  if resp and resp.groups
197
198
  resp.groups.each { |g|
@@ -274,14 +275,15 @@ module MU
274
275
  MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
275
276
  return nil
276
277
  end
277
-
278
- bok["name"] = cloud_desc.group.group_name
279
278
 
280
- if cloud_desc.group.path != "/"
281
- bok["path"] = cloud_desc.group.path
279
+ group_desc = cloud_desc(use_cache: false).respond_to?(:group) ? cloud_desc.group : cloud_desc
280
+ bok["name"] = group_desc.group_name
281
+
282
+ if group_desc.path != "/"
283
+ bok["path"] = group_desc.path
282
284
  end
283
285
 
284
- if cloud_desc.users and cloud_desc.users.size > 0
286
+ if cloud_desc.respond_to?(:users) and cloud_desc.users and cloud_desc.users.size > 0
285
287
  bok["members"] = cloud_desc.users.map { |u| u.user_name }
286
288
  end
287
289
 
@@ -291,7 +293,7 @@ module MU
291
293
  resp.policy_names.each { |pol_name|
292
294
  pol = MU::Cloud::AWS.iam(credentials: @credentials).get_group_policy(group_name: @cloud_id, policy_name: pol_name)
293
295
  doc = JSON.parse(URI.decode(pol.policy_document))
294
- bok["inline_policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
296
+ bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
295
297
  }
296
298
  end
297
299
 
@@ -324,7 +326,7 @@ module MU
324
326
  def self.schema(_config)
325
327
  toplevel_required = []
326
328
  polschema = MU::Config::Role.schema["properties"]["policies"]
327
- polschema.deep_merge!(MU::Cloud::AWS::Role.condition_schema)
329
+ polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
328
330
 
329
331
  schema = {
330
332
  "inline_policies" => polschema,
@@ -364,7 +366,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
364
366
  # If we're attaching some managed policies, make sure all of the ones
365
367
  # that should already exist do indeed exist
366
368
  if group['attachable_policies']
367
- ok = false if !MU::Cloud::AWS::Role.validateAttachablePolicies(
369
+ ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
368
370
  group['attachable_policies'],
369
371
  credentials: group['credentials'],
370
372
  region: group['region']
@@ -378,13 +380,9 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
378
380
  if group['members']
379
381
  group['members'].each { |user|
380
382
  if configurator.haveLitterMate?(user, "users")
381
- group["dependencies"] ||= []
382
- group["dependencies"] << {
383
- "type" => "user",
384
- "name" => user
385
- }
383
+ MU::Config.addDependency(group, user, "user")
386
384
  else
387
- found = MU::Cloud::AWS::User.find(cloud_id: user)
385
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: user)
388
386
  if found.nil? or found.empty?
389
387
  MU.log "Error in members for group #{group['name']}: No such user #{user}", MU::ERR
390
388
  ok = false
@@ -90,7 +90,7 @@ module MU
90
90
  # @param noop [Boolean]: If true, will only print what would be done
91
91
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
92
92
  # @return [void]
93
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
93
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
94
94
  return if !orgMasterCreds?(credentials)
95
95
  MU.log "AWS::Habitat.cleanup: need to support flags['known']", MU::DEBUG, details: flags
96
96
  MU.log "Placeholder: AWS Habitat artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
@@ -99,7 +99,7 @@ module MU
99
99
 
100
100
  if resp and resp.accounts
101
101
  resp.accounts.each { |acct|
102
- if acct.name.match(/^#{Regexp.quote(MU.deploy_id)}/) or acct.name.match(/BUNS/)
102
+ if acct.name.match(/^#{Regexp.quote(deploy_id)}/) or acct.name.match(/BUNS/)
103
103
  if !noop
104
104
  pp acct
105
105
  end
@@ -144,7 +144,7 @@ module MU
144
144
  def self.orgMasterCreds?(credentials = nil)
145
145
  acct_num = MU::Cloud::AWS.iam(credentials: credentials).list_users.users.first.arn.split(/:/)[4]
146
146
 
147
- parentorg = MU::Cloud::AWS::Folder.find(credentials: credentials).values.first
147
+ parentorg = MU::Cloud.resourceClass("AWS", "Folder").find(credentials: credentials).values.first
148
148
  acct_num == parentorg.master_account_id
149
149
  end
150
150
 
@@ -0,0 +1,469 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ class Cloud
17
+ class AWS
18
+ # A scheduled task facility as configured in {MU::Config::BasketofKittens::jobs}
19
+ class Job < MU::Cloud::Job
20
+
21
+ # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
22
+ # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
23
+ def initialize(**args)
24
+ super
25
+ @mu_name ||= @deploy.getResourceName(@config["name"])
26
+ end
27
+
28
+ # Called automatically by {MU::Deploy#createResources}
29
+ def create
30
+ @cloud_id = @mu_name
31
+
32
+ params = get_properties
33
+
34
+ MU.log "Creating CloudWatch Event #{@mu_name}", MU::NOTICE, details: params
35
+
36
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(params)
37
+ end
38
+
39
+ # Called automatically by {MU::Deploy#createResources}
40
+ def groom
41
+ new_props = get_properties
42
+ current = MU.structToHash(cloud_desc(use_cache: false))
43
+ params = {}
44
+ new_props.each_pair { |k, v|
45
+ next if k == :tags # doesn't seem to do anything
46
+ if v != current[k]
47
+ params[k] = v
48
+ end
49
+ }
50
+
51
+ if params.size > 0
52
+ MU.log "Updating CloudWatch Event #{@cloud_id}", MU::NOTICE, details: params
53
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(new_props)
54
+ end
55
+
56
+ if @config['targets']
57
+ target_params = []
58
+ @config['targets'].each { |t|
59
+ MU.retrier([MuNonFatal], max:5, wait: 9) {
60
+ target_ref = MU::Config::Ref.get(t)
61
+ target_obj = target_ref.kitten(@deploy, cloud: "AWS")
62
+ this_target = if target_ref.is_mu_type? and target_obj and
63
+ !target_obj.arn.nil?
64
+ if target_ref.type == "functions"
65
+ target_obj.addTrigger(arn, "events", @mu_name)
66
+ end
67
+ {
68
+ id: target_obj.cloud_id,
69
+ arn: target_obj.arn
70
+ }
71
+ elsif target_ref.id and target_ref.id.match(/^arn:/)
72
+ {
73
+ id: target_ref.id || target_ref.name,
74
+ arn: target_ref.id
75
+ }
76
+ else
77
+ raise MuNonFatal.new "Failed to retrieve ARN from CLoudWatch Event target descriptor", details: target_ref.to_h
78
+ end
79
+ if t['role']
80
+ role_obj = MU::Config::Ref.get(t['role']).kitten(@deploy, cloud: "AWS")
81
+ raise MuError.new "Failed to fetch object from role reference", details: t['role'].to_h if !role_obj
82
+ params[:role_arn] = role_obj.arn
83
+ end
84
+ [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters, :ecs_parameters].each { |attr|
85
+ if t[attr.to_s]
86
+ this_target[attr] = MU.structToHash(t[attr.to_s])
87
+ end
88
+ }
89
+ target_params << this_target
90
+ }
91
+ }
92
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_targets(
93
+ rule: @cloud_id,
94
+ event_bus_name: cloud_desc.event_bus_name,
95
+ targets: target_params
96
+ )
97
+ end
98
+
99
+ end
100
+
101
+ # Canonical Amazon Resource Number for this resource
102
+ # @return [String]
103
+ def arn
104
+ cloud_desc ? cloud_desc.arn : nil
105
+ end
106
+
107
+ # Return the metadata for this job
108
+ # @return [Hash]
109
+ def notify
110
+ MU.structToHash(cloud_desc, stringify_keys: true)
111
+ end
112
+
113
+ # Does this resource type exist as a global (cloud-wide) artifact, or
114
+ # is it localized to a region/zone?
115
+ # @return [Boolean]
116
+ def self.isGlobal?
117
+ false
118
+ end
119
+
120
+ # Denote whether this resource implementation is experiment, ready for
121
+ # testing, or ready for production use.
122
+ def self.quality
123
+ MU::Cloud::BETA
124
+ end
125
+
126
+ # Remove all jobs associated with the currently loaded deployment.
127
+ # @param noop [Boolean]: If true, will only print what would be done
128
+ # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
129
+ # @param region [String]: The cloud provider region
130
+ # @return [void]
131
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
132
+ found = find(region: region, credentials: credentials)
133
+
134
+ found.each_pair { |id, desc|
135
+ if (desc.description and desc.description == deploy_id) or
136
+ (flags and flags['known'] and flags['known'].include?(id))
137
+ MU.log "Deleting CloudWatch Event #{id}"
138
+ if !noop
139
+ resp = MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).list_targets_by_rule(
140
+ rule: id,
141
+ event_bus_name: desc.event_bus_name,
142
+ )
143
+ if resp and resp.targets and !resp.targets.empty?
144
+ MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).remove_targets(
145
+ rule: id,
146
+ event_bus_name: desc.event_bus_name,
147
+ ids: resp.targets.map { |t| t.id }
148
+ )
149
+ end
150
+
151
+ MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).delete_rule(
152
+ name: id,
153
+ event_bus_name: desc.event_bus_name
154
+ )
155
+ end
156
+ end
157
+ }
158
+ end
159
+
160
+ # Locate an existing event.
161
+ # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching CloudWatch Event
162
+ def self.find(**args)
163
+ found = {}
164
+
165
+ MU::Cloud::AWS.cloudwatchevents(region: args[:region], credentials: args[:credentials]).list_rules.rules.each { |r|
166
+ next if args[:cloud_id] and ![r.name, r.arn].include?(args[:cloud_id])
167
+ found[r.name] = r
168
+ }
169
+
170
+ found
171
+ end
172
+
173
+ # Reverse-map our cloud description into a runnable config hash.
174
+ # We assume that any values we have in +@config+ are placeholders, and
175
+ # calculate our own accordingly based on what's live in the cloud.
176
+ def toKitten(**_args)
177
+ bok = {
178
+ "cloud" => "AWS",
179
+ "credentials" => @config['credentials'],
180
+ "cloud_id" => @cloud_id,
181
+ "region" => @config['region']
182
+ }
183
+
184
+ if !cloud_desc
185
+ MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
186
+ return nil
187
+ end
188
+ bok['name'] = cloud_desc.name
189
+ if cloud_desc.description and !cloud_desc.description.empty?
190
+ bok['description'] = cloud_desc.description
191
+ end
192
+
193
+ bok['disabled'] = true if cloud_desc.state == "DISABLED"
194
+
195
+ # schedule_expression="cron(15 6 * * ? *)"
196
+ if cloud_desc.schedule_expression
197
+ if cloud_desc.schedule_expression.match(/cron\((\S+) (\S+) (\S+) (\S+) (\S+) (\S+)\)/)
198
+ bok['schedule'] = {
199
+ "minute" => Regexp.last_match[1],
200
+ "hour" => Regexp.last_match[2],
201
+ "day_of_month" => Regexp.last_match[3],
202
+ "month" => Regexp.last_match[4],
203
+ "day_of_week" => Regexp.last_match[5],
204
+ "year" => Regexp.last_match[6]
205
+ }
206
+ else
207
+ MU.log "HALP", MU::ERR, details: cloud_desc.schedule_expression
208
+ end
209
+ end
210
+
211
+ if cloud_desc.role_arn
212
+ shortname = cloud_desc.role_arn.sub(/.*?role\/([^\/]+)$/, '\1')
213
+ bok['role'] = MU::Config::Ref.get(
214
+ id: shortname,
215
+ cloud: "AWS",
216
+ type: "roles"
217
+ )
218
+ end
219
+
220
+ targets = MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).list_targets_by_rule(
221
+ rule: @cloud_id,
222
+ event_bus_name: cloud_desc.event_bus_name
223
+ ).targets
224
+ targets.each { |t|
225
+ bok['targets'] ||= []
226
+ _arn, _plat, service, region, account, resource = t.arn.split(/:/, 6)
227
+ target_type = if service == "lambda"
228
+ resource.sub!(/^function:/, '')
229
+ "functions"
230
+ elsif service == "sns"
231
+ "notifiers"
232
+ elsif service == "sqs"
233
+ "msg_queues"
234
+ else
235
+ service
236
+ end
237
+ ref_params = {
238
+ id: resource,
239
+ region: region,
240
+ type: target_type,
241
+ cloud: "AWS",
242
+ credentials: @credentials,
243
+ habitat: MU::Config::Ref.get(
244
+ id: account,
245
+ cloud: "AWS",
246
+ credentials: @credentials
247
+ )
248
+ }
249
+ [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters].each { |attr|
250
+ if t.respond_to?(attr) and !t.send(attr).nil?
251
+ ref_params[attr] = MU.structToHash(t.send(attr), stringify_keys: true)
252
+ end
253
+ }
254
+
255
+ bok['targets'] << MU::Config::Ref.get(ref_params)
256
+ }
257
+
258
+ # XXX cloud_desc.event_pattern - what do we want to do with this?
259
+
260
+ bok
261
+ end
262
+
263
+
264
+ # Cloud-specific configuration properties.
265
+ # @param _config [MU::Config]: The calling MU::Config object
266
+ # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
267
+ def self.schema(_config)
268
+ toplevel_required = []
269
+
270
+ target_schema = MU::Config::Ref.schema(any_type: true, desc: "A resource which will be invoked by this event. Can be a reference to a sibling Mu resource, typically a +Function+ or +MsgQueue+, or to an unadorned external cloud resource.")
271
+ target_params = {
272
+ "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to use when interacting with this target.", omit_fields: ["region", "tag"]),
273
+ "input" => {
274
+ "type" => "string"
275
+ },
276
+ "input_path" => {
277
+ "type" => "string"
278
+ },
279
+ "run_command_parameters" => {
280
+ "type" => "object",
281
+ "description" => "Parameters used when you are using the rule to invoke Amazon EC2 Run Command",
282
+ "required" => ["run_command_targets"],
283
+ "properties" => {
284
+ "run_command_targets" => {
285
+ "type" => "array",
286
+ "items" => {
287
+ "type" => "object",
288
+ "description" => "Currently, AWS supports including only one +run_command_targets+ block, which specifies either an array of InstanceIds or a tag.",
289
+ "required" => ["key", "values"],
290
+ "properties" => {
291
+ "key" => {
292
+ "type" => "string",
293
+ "description" => "Can be either +tag: tag-key+ or +InstanceIds+"
294
+ },
295
+ "values" => {
296
+ "type" => "array",
297
+ "items" => {
298
+ "description" => "If +key+ is +tag: tag-key+, +values+ is a list of tag values; if +key+ is +InstanceIds+, +values+ is a list of Amazon EC2 instance IDs.",
299
+ "type" => "string"
300
+ }
301
+ }
302
+ }
303
+ }
304
+ }
305
+ }
306
+ },
307
+ "input_transformer" => {
308
+ "type" => "object",
309
+ "description" => "Settings to enable you to provide custom input to a target based on certain event data. You can extract one or more key-value pairs from the event and then use that data to send customized input to the target.",
310
+ "required" => ["input_template"],
311
+ "properties" => {
312
+ "input_template" => {
313
+ "type" => "string",
314
+ "description" => "Input template where you specify placeholders that will be filled with the values of the keys from +input_paths_map+ to customize the data sent to the target."
315
+ },
316
+ "input_paths_map" => {
317
+ "type" => "object",
318
+ "description" => "Hash representing JSON paths to be extracted from the event"
319
+ }
320
+ }
321
+ },
322
+ "batch_parameters" => {
323
+ "type" => "object",
324
+ "description" => "If the event target is an AWS Batch job, this contains the job definition, job name, and other parameters. See: https://docs.aws.amazon.com/batch/latest/userguide/jobs.html",
325
+ "required" => ["job_definition", "job_name"],
326
+ "properties" => {
327
+ "job_definition" => {
328
+ "description" => "The ARN or name of the job definition to use if the event target is an AWS Batch job.",
329
+ "type" => "string"
330
+ },
331
+ "job_name" => {
332
+ "description" => "The name to use for this execution of the job, if the target is an AWS Batch job.",
333
+ "type" => "string"
334
+ },
335
+ "array_properties" => {
336
+ "type" => "object",
337
+ "description" => "The array properties for the submitted job, such as the size of the array.",
338
+ "properties" => {
339
+ "size" => {
340
+ "description" => "Size of the submitted array",
341
+ "type" => "integer"
342
+ }
343
+ }
344
+ },
345
+ "retry_strategy" => {
346
+ "type" => "object",
347
+ "description" => "The retry strategy to use for failed jobs, if the target is an AWS Batch job.",
348
+ "properties" => {
349
+ "attempts" => {
350
+ "description" => "Number of retry attempts, valid values from 1-10",
351
+ "type" => "integer"
352
+ }
353
+ }
354
+ }
355
+ }
356
+ },
357
+ "sqs_parameters" => {
358
+ "type" => "object",
359
+ "description" => "Contains the message group ID to use when the target is an SQS FIFO queue.",
360
+ "required" => ["message_group_id"],
361
+ "properties" => {
362
+ "message_group_id" => {
363
+ "type" => "string"
364
+ }
365
+ }
366
+ },
367
+ "kinesis_parameters" => {
368
+ "type" => "object",
369
+ "description" => "The custom parameter you can use to control the shard assignment, when the target is a Kinesis data stream.",
370
+ "required" => ["partition_key_path"],
371
+ "properties" => {
372
+ "partition_key_path" => {
373
+ "type" => "string"
374
+ }
375
+ }
376
+ },
377
+ "http_parameters" => {
378
+ "type" => "object",
379
+ "description" => "Contains the HTTP parameters to use when the target is a API Gateway REST endpoint.",
380
+ "properties" => {
381
+ "path_parameter_values" => {
382
+ "type" => "array",
383
+ "items" => {
384
+ "description" => "The path parameter values to be used to populate API Gateway REST API path wildcards (\"*\").",
385
+ "type" => "string"
386
+ }
387
+ },
388
+ "header_parameters" => {
389
+ "description" => "Key => value pairs to pass as headers",
390
+ "type" => "object"
391
+ },
392
+ "query_string_parameters" => {
393
+ "description" => "Key => value pairs to pass as query strings",
394
+ "type" => "object"
395
+ }
396
+ }
397
+ }
398
+ }
399
+ target_schema["properties"].merge!(target_params)
400
+
401
+ schema = {
402
+ "disabled" => {
403
+ "type" => "boolean",
404
+ "description" => "Leave this job in place but disabled",
405
+ "default" => false
406
+ },
407
+ "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to this CloudWatch Event.", omit_fields: ["region", "tag"]),
408
+ "targets" => {
409
+ "type" => "array",
410
+ "items" => target_schema
411
+ }
412
+ }
413
+ [toplevel_required, schema]
414
+ end
415
+
416
+ # Cloud-specific pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated.
417
+ # @param job [Hash]: The resource to process and validate
418
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
419
+ # @return [Boolean]: True if validation succeeded, False otherwise
420
+ def self.validateConfig(job, _configurator)
421
+ ok = true
422
+
423
+ job['targets'].each { |t|
424
+ target_ref = MU::Config::Ref.get(t)
425
+ if target_ref.is_mu_type? and target_ref.name
426
+ MU::Config.addDependency(job, target_ref.name, target_ref.type)
427
+ end
428
+ }
429
+
430
+ ok
431
+ end
432
+
433
+ private
434
+
435
+ def get_properties
436
+ params = {
437
+ name: @cloud_id,
438
+ state: @config['disabled'] ? "DISABLED" : "ENABLED",
439
+ event_bus_name: "default" # XXX expose, or create a deploy-specific one?
440
+ }
441
+
442
+ params[:description] = if @config['description'] and @config['scrub_mu_isms']
443
+ @config['description']
444
+ else
445
+ @deploy.deploy_id
446
+ end
447
+
448
+ if @tags
449
+ params[:tags] = @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }
450
+ end
451
+
452
+ if @config['role']
453
+ role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS")
454
+ raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj
455
+ params[:role_arn] = role_obj.arn
456
+ end
457
+
458
+ if @config['schedule']
459
+ params[:schedule_expression] = "cron(" + ["minute", "hour", "day_of_month", "month", "day_of_week", "year"].map { |i| @config['schedule'][i] }.join(" ") +")"
460
+ end
461
+
462
+
463
+ params
464
+ end
465
+
466
+ end
467
+ end
468
+ end
469
+ end