cloud-mu 3.1.6 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +1 -1
  3. data/bin/mu-adopt +15 -12
  4. data/bin/mu-azure-tests +57 -0
  5. data/bin/mu-cleanup +2 -4
  6. data/bin/mu-configure +37 -1
  7. data/bin/mu-deploy +3 -3
  8. data/bin/mu-findstray-tests +25 -0
  9. data/bin/mu-gen-docs +2 -4
  10. data/bin/mu-load-config.rb +2 -1
  11. data/bin/mu-run-tests +37 -12
  12. data/cloud-mu.gemspec +4 -4
  13. data/cookbooks/mu-tools/attributes/default.rb +7 -0
  14. data/cookbooks/mu-tools/libraries/helper.rb +87 -3
  15. data/cookbooks/mu-tools/recipes/apply_security.rb +39 -23
  16. data/cookbooks/mu-tools/recipes/aws_api.rb +13 -0
  17. data/cookbooks/mu-tools/recipes/google_api.rb +4 -0
  18. data/cookbooks/mu-tools/recipes/rsyslog.rb +8 -1
  19. data/cookbooks/mu-tools/resources/disk.rb +33 -12
  20. data/cookbooks/mu-tools/resources/mommacat_request.rb +1 -2
  21. data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb +215 -0
  22. data/extras/clean-stock-amis +10 -2
  23. data/extras/generate-stock-images +7 -3
  24. data/extras/image-generators/AWS/centos7.yaml +19 -16
  25. data/extras/image-generators/AWS/{rhel7.yaml → rhel71.yaml} +0 -0
  26. data/extras/image-generators/AWS/{win2k12.yaml → win2k12r2.yaml} +0 -0
  27. data/modules/mommacat.ru +2 -2
  28. data/modules/mu.rb +84 -97
  29. data/modules/mu/adoption.rb +359 -59
  30. data/modules/mu/cleanup.rb +67 -44
  31. data/modules/mu/cloud.rb +108 -1754
  32. data/modules/mu/cloud/database.rb +49 -0
  33. data/modules/mu/cloud/dnszone.rb +44 -0
  34. data/modules/mu/cloud/machine_images.rb +212 -0
  35. data/modules/mu/cloud/providers.rb +81 -0
  36. data/modules/mu/cloud/resource_base.rb +929 -0
  37. data/modules/mu/cloud/server.rb +40 -0
  38. data/modules/mu/cloud/server_pool.rb +1 -0
  39. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  40. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  41. data/modules/mu/cloud/wrappers.rb +178 -0
  42. data/modules/mu/config.rb +122 -80
  43. data/modules/mu/config/alarm.rb +2 -6
  44. data/modules/mu/config/bucket.rb +32 -3
  45. data/modules/mu/config/cache_cluster.rb +2 -2
  46. data/modules/mu/config/cdn.rb +100 -0
  47. data/modules/mu/config/collection.rb +1 -1
  48. data/modules/mu/config/container_cluster.rb +2 -2
  49. data/modules/mu/config/database.rb +84 -105
  50. data/modules/mu/config/database.yml +1 -2
  51. data/modules/mu/config/dnszone.rb +5 -4
  52. data/modules/mu/config/doc_helpers.rb +4 -5
  53. data/modules/mu/config/endpoint.rb +2 -1
  54. data/modules/mu/config/firewall_rule.rb +3 -19
  55. data/modules/mu/config/folder.rb +1 -1
  56. data/modules/mu/config/function.rb +17 -8
  57. data/modules/mu/config/group.rb +1 -1
  58. data/modules/mu/config/habitat.rb +1 -1
  59. data/modules/mu/config/job.rb +89 -0
  60. data/modules/mu/config/loadbalancer.rb +57 -11
  61. data/modules/mu/config/log.rb +1 -1
  62. data/modules/mu/config/msg_queue.rb +1 -1
  63. data/modules/mu/config/nosqldb.rb +1 -1
  64. data/modules/mu/config/notifier.rb +8 -19
  65. data/modules/mu/config/ref.rb +81 -9
  66. data/modules/mu/config/role.rb +1 -1
  67. data/modules/mu/config/schema_helpers.rb +30 -34
  68. data/modules/mu/config/search_domain.rb +1 -1
  69. data/modules/mu/config/server.rb +5 -13
  70. data/modules/mu/config/server_pool.rb +3 -7
  71. data/modules/mu/config/storage_pool.rb +1 -1
  72. data/modules/mu/config/tail.rb +10 -0
  73. data/modules/mu/config/user.rb +1 -1
  74. data/modules/mu/config/vpc.rb +13 -17
  75. data/modules/mu/defaults/AWS.yaml +106 -106
  76. data/modules/mu/defaults/Azure.yaml +1 -0
  77. data/modules/mu/defaults/Google.yaml +1 -0
  78. data/modules/mu/deploy.rb +33 -19
  79. data/modules/mu/groomer.rb +15 -0
  80. data/modules/mu/groomers/chef.rb +3 -0
  81. data/modules/mu/logger.rb +120 -144
  82. data/modules/mu/master.rb +22 -1
  83. data/modules/mu/mommacat.rb +71 -26
  84. data/modules/mu/mommacat/daemon.rb +23 -14
  85. data/modules/mu/mommacat/naming.rb +82 -3
  86. data/modules/mu/mommacat/search.rb +59 -16
  87. data/modules/mu/mommacat/storage.rb +119 -48
  88. data/modules/mu/{clouds → providers}/README.md +1 -1
  89. data/modules/mu/{clouds → providers}/aws.rb +248 -62
  90. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  91. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  92. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  93. data/modules/mu/providers/aws/cdn.rb +782 -0
  94. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  95. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +65 -63
  96. data/modules/mu/providers/aws/database.rb +1747 -0
  97. data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
  98. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  99. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
  100. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  101. data/modules/mu/{clouds → providers}/aws/function.rb +291 -133
  102. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  103. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  104. data/modules/mu/providers/aws/job.rb +469 -0
  105. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
  106. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  107. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  108. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  109. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  110. data/modules/mu/{clouds → providers}/aws/role.rb +112 -78
  111. data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
  112. data/modules/mu/{clouds → providers}/aws/server.rb +120 -145
  113. data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
  114. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  115. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  116. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  117. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  118. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  119. data/modules/mu/{clouds → providers}/aws/vpc.rb +141 -73
  120. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  121. data/modules/mu/{clouds → providers}/azure.rb +4 -1
  122. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  123. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  124. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  125. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  126. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  127. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  128. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  129. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  130. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  131. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  132. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  133. data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
  134. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  135. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  136. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  137. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  138. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  139. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  140. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  141. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  142. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  143. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  144. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  145. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  146. data/modules/mu/{clouds → providers}/google.rb +15 -6
  147. data/modules/mu/{clouds → providers}/google/bucket.rb +2 -2
  148. data/modules/mu/{clouds → providers}/google/container_cluster.rb +29 -14
  149. data/modules/mu/{clouds → providers}/google/database.rb +2 -9
  150. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +3 -3
  151. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  152. data/modules/mu/{clouds → providers}/google/function.rb +4 -4
  153. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  154. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  155. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +2 -2
  156. data/modules/mu/{clouds → providers}/google/role.rb +46 -35
  157. data/modules/mu/{clouds → providers}/google/server.rb +26 -11
  158. data/modules/mu/{clouds → providers}/google/server_pool.rb +11 -11
  159. data/modules/mu/{clouds → providers}/google/user.rb +32 -22
  160. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  161. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  162. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  163. data/modules/mu/{clouds → providers}/google/vpc.rb +38 -3
  164. data/modules/tests/aws-jobs-functions.yaml +46 -0
  165. data/modules/tests/centos6.yaml +15 -0
  166. data/modules/tests/centos7.yaml +15 -0
  167. data/modules/tests/centos8.yaml +12 -0
  168. data/modules/tests/ecs.yaml +2 -2
  169. data/modules/tests/eks.yaml +1 -1
  170. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  171. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  172. data/modules/tests/microservice_app.yaml +288 -0
  173. data/modules/tests/rds.yaml +108 -0
  174. data/modules/tests/regrooms/rds.yaml +123 -0
  175. data/modules/tests/server-with-scrub-muisms.yaml +1 -1
  176. data/modules/tests/super_complex_bok.yml +2 -2
  177. data/modules/tests/super_simple_bok.yml +2 -2
  178. data/spec/mu/clouds/azure_spec.rb +2 -2
  179. metadata +126 -98
  180. data/modules/mu/clouds/aws/database.rb +0 -1974
  181. data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -60,7 +60,7 @@ module MU
60
60
  userid = user
61
61
  userdesc = @deploy.findLitterMate(name: user, type: "users")
62
62
  userid = userdesc.cloud_id if userdesc
63
- found = MU::Cloud::AWS::User.find(cloud_id: userid)
63
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: userid)
64
64
  if found.size == 1
65
65
  userdesc = found.values.first
66
66
  MU.log "Adding IAM user #{userdesc.path}#{userdesc.user_name} to group #{@mu_name}", MU::NOTICE
@@ -88,7 +88,7 @@ module MU
88
88
  # Create these if necessary, then append them to the list of
89
89
  # attachable_policies
90
90
  if @config['raw_policies']
91
- pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
91
+ pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
92
92
  @config['raw_policies'],
93
93
  basename: @deploy.getResourceName(@config['name']),
94
94
  credentials: @credentials
@@ -114,7 +114,7 @@ module MU
114
114
  attached_policies.each { |a|
115
115
  if !configured_policies.include?(a.policy_arn)
116
116
  MU.log "Removing IAM policy #{a.policy_arn} from group #{@mu_name}", MU::NOTICE
117
- MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @credentials)
117
+ MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
118
118
  else
119
119
  configured_policies.delete(a.policy_arn)
120
120
  end
@@ -131,7 +131,7 @@ module MU
131
131
  end
132
132
 
133
133
  if @config['inline_policies']
134
- docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
134
+ docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
135
135
  docs.each { |doc|
136
136
  MU.log "Putting user policy #{doc.keys.first} to group #{@cloud_id} "
137
137
  MU::Cloud::AWS.iam(credentials: @credentials).put_group_policy(
@@ -155,6 +155,7 @@ module MU
155
155
  # return [Struct]
156
156
  def cloud_desc(use_cache: true)
157
157
  return @cloud_desc_cache if @cloud_desc_cache and use_cache
158
+ return nil if !@mu_name
158
159
  @cloud_desc_cache = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_group(
159
160
  group_name: @mu_name
160
161
  )
@@ -186,12 +187,12 @@ module MU
186
187
  # @param noop [Boolean]: If true, will only print what would be done
187
188
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
188
189
  # @return [void]
189
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
190
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
190
191
  MU.log "AWS::Group.cleanup: need to support flags['known']", MU::DEBUG, details: flags
191
192
  MU.log "Placeholder: AWS Group artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
192
193
 
193
194
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_groups(
194
- path_prefix: "/"+MU.deploy_id+"/"
195
+ path_prefix: "/"+deploy_id+"/"
195
196
  )
196
197
  if resp and resp.groups
197
198
  resp.groups.each { |g|
@@ -274,14 +275,15 @@ module MU
274
275
  MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
275
276
  return nil
276
277
  end
277
-
278
- bok["name"] = cloud_desc.group.group_name
279
278
 
280
- if cloud_desc.group.path != "/"
281
- bok["path"] = cloud_desc.group.path
279
+ group_desc = cloud_desc(use_cache: false).respond_to?(:group) ? cloud_desc.group : cloud_desc
280
+ bok["name"] = group_desc.group_name
281
+
282
+ if group_desc.path != "/"
283
+ bok["path"] = group_desc.path
282
284
  end
283
285
 
284
- if cloud_desc.users and cloud_desc.users.size > 0
286
+ if cloud_desc.respond_to?(:users) and cloud_desc.users and cloud_desc.users.size > 0
285
287
  bok["members"] = cloud_desc.users.map { |u| u.user_name }
286
288
  end
287
289
 
@@ -291,7 +293,7 @@ module MU
291
293
  resp.policy_names.each { |pol_name|
292
294
  pol = MU::Cloud::AWS.iam(credentials: @credentials).get_group_policy(group_name: @cloud_id, policy_name: pol_name)
293
295
  doc = JSON.parse(URI.decode(pol.policy_document))
294
- bok["inline_policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
296
+ bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
295
297
  }
296
298
  end
297
299
 
@@ -324,7 +326,7 @@ module MU
324
326
  def self.schema(_config)
325
327
  toplevel_required = []
326
328
  polschema = MU::Config::Role.schema["properties"]["policies"]
327
- polschema.deep_merge!(MU::Cloud::AWS::Role.condition_schema)
329
+ polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
328
330
 
329
331
  schema = {
330
332
  "inline_policies" => polschema,
@@ -364,7 +366,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
364
366
  # If we're attaching some managed policies, make sure all of the ones
365
367
  # that should already exist do indeed exist
366
368
  if group['attachable_policies']
367
- ok = false if !MU::Cloud::AWS::Role.validateAttachablePolicies(
369
+ ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
368
370
  group['attachable_policies'],
369
371
  credentials: group['credentials'],
370
372
  region: group['region']
@@ -378,13 +380,9 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-GROUP-FOO+. This parameter wil
378
380
  if group['members']
379
381
  group['members'].each { |user|
380
382
  if configurator.haveLitterMate?(user, "users")
381
- group["dependencies"] ||= []
382
- group["dependencies"] << {
383
- "type" => "user",
384
- "name" => user
385
- }
383
+ MU::Config.addDependency(group, user, "user")
386
384
  else
387
- found = MU::Cloud::AWS::User.find(cloud_id: user)
385
+ found = MU::Cloud.resourceClass("AWS", "User").find(cloud_id: user)
388
386
  if found.nil? or found.empty?
389
387
  MU.log "Error in members for group #{group['name']}: No such user #{user}", MU::ERR
390
388
  ok = false
@@ -90,7 +90,7 @@ module MU
90
90
  # @param noop [Boolean]: If true, will only print what would be done
91
91
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
92
92
  # @return [void]
93
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
93
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
94
94
  return if !orgMasterCreds?(credentials)
95
95
  MU.log "AWS::Habitat.cleanup: need to support flags['known']", MU::DEBUG, details: flags
96
96
  MU.log "Placeholder: AWS Habitat artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
@@ -99,7 +99,7 @@ module MU
99
99
 
100
100
  if resp and resp.accounts
101
101
  resp.accounts.each { |acct|
102
- if acct.name.match(/^#{Regexp.quote(MU.deploy_id)}/) or acct.name.match(/BUNS/)
102
+ if acct.name.match(/^#{Regexp.quote(deploy_id)}/) or acct.name.match(/BUNS/)
103
103
  if !noop
104
104
  pp acct
105
105
  end
@@ -144,7 +144,7 @@ module MU
144
144
  def self.orgMasterCreds?(credentials = nil)
145
145
  acct_num = MU::Cloud::AWS.iam(credentials: credentials).list_users.users.first.arn.split(/:/)[4]
146
146
 
147
- parentorg = MU::Cloud::AWS::Folder.find(credentials: credentials).values.first
147
+ parentorg = MU::Cloud.resourceClass("AWS", "Folder").find(credentials: credentials).values.first
148
148
  acct_num == parentorg.master_account_id
149
149
  end
150
150
 
@@ -0,0 +1,469 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ class Cloud
17
+ class AWS
18
+ # A scheduled task facility as configured in {MU::Config::BasketofKittens::jobs}
19
+ class Job < MU::Cloud::Job
20
+
21
+ # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
22
+ # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
23
+ def initialize(**args)
24
+ super
25
+ @mu_name ||= @deploy.getResourceName(@config["name"])
26
+ end
27
+
28
+ # Called automatically by {MU::Deploy#createResources}
29
+ def create
30
+ @cloud_id = @mu_name
31
+
32
+ params = get_properties
33
+
34
+ MU.log "Creating CloudWatch Event #{@mu_name}", MU::NOTICE, details: params
35
+
36
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(params)
37
+ end
38
+
39
+ # Called automatically by {MU::Deploy#createResources}
40
+ def groom
41
+ new_props = get_properties
42
+ current = MU.structToHash(cloud_desc(use_cache: false))
43
+ params = {}
44
+ new_props.each_pair { |k, v|
45
+ next if k == :tags # doesn't seem to do anything
46
+ if v != current[k]
47
+ params[k] = v
48
+ end
49
+ }
50
+
51
+ if params.size > 0
52
+ MU.log "Updating CloudWatch Event #{@cloud_id}", MU::NOTICE, details: params
53
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_rule(new_props)
54
+ end
55
+
56
+ if @config['targets']
57
+ target_params = []
58
+ @config['targets'].each { |t|
59
+ MU.retrier([MuNonFatal], max:5, wait: 9) {
60
+ target_ref = MU::Config::Ref.get(t)
61
+ target_obj = target_ref.kitten(@deploy, cloud: "AWS")
62
+ this_target = if target_ref.is_mu_type? and target_obj and
63
+ !target_obj.arn.nil?
64
+ if target_ref.type == "functions"
65
+ target_obj.addTrigger(arn, "events", @mu_name)
66
+ end
67
+ {
68
+ id: target_obj.cloud_id,
69
+ arn: target_obj.arn
70
+ }
71
+ elsif target_ref.id and target_ref.id.match(/^arn:/)
72
+ {
73
+ id: target_ref.id || target_ref.name,
74
+ arn: target_ref.id
75
+ }
76
+ else
77
+ raise MuNonFatal.new "Failed to retrieve ARN from CLoudWatch Event target descriptor", details: target_ref.to_h
78
+ end
79
+ if t['role']
80
+ role_obj = MU::Config::Ref.get(t['role']).kitten(@deploy, cloud: "AWS")
81
+ raise MuError.new "Failed to fetch object from role reference", details: t['role'].to_h if !role_obj
82
+ params[:role_arn] = role_obj.arn
83
+ end
84
+ [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters, :ecs_parameters].each { |attr|
85
+ if t[attr.to_s]
86
+ this_target[attr] = MU.structToHash(t[attr.to_s])
87
+ end
88
+ }
89
+ target_params << this_target
90
+ }
91
+ }
92
+ MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).put_targets(
93
+ rule: @cloud_id,
94
+ event_bus_name: cloud_desc.event_bus_name,
95
+ targets: target_params
96
+ )
97
+ end
98
+
99
+ end
100
+
101
+ # Canonical Amazon Resource Number for this resource
102
+ # @return [String]
103
+ def arn
104
+ cloud_desc ? cloud_desc.arn : nil
105
+ end
106
+
107
+ # Return the metadata for this job
108
+ # @return [Hash]
109
+ def notify
110
+ MU.structToHash(cloud_desc, stringify_keys: true)
111
+ end
112
+
113
+ # Does this resource type exist as a global (cloud-wide) artifact, or
114
+ # is it localized to a region/zone?
115
+ # @return [Boolean]
116
+ def self.isGlobal?
117
+ false
118
+ end
119
+
120
+ # Denote whether this resource implementation is experiment, ready for
121
+ # testing, or ready for production use.
122
+ def self.quality
123
+ MU::Cloud::BETA
124
+ end
125
+
126
+ # Remove all jobs associated with the currently loaded deployment.
127
+ # @param noop [Boolean]: If true, will only print what would be done
128
+ # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
129
+ # @param region [String]: The cloud provider region
130
+ # @return [void]
131
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
132
+ found = find(region: region, credentials: credentials)
133
+
134
+ found.each_pair { |id, desc|
135
+ if (desc.description and desc.description == deploy_id) or
136
+ (flags and flags['known'] and flags['known'].include?(id))
137
+ MU.log "Deleting CloudWatch Event #{id}"
138
+ if !noop
139
+ resp = MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).list_targets_by_rule(
140
+ rule: id,
141
+ event_bus_name: desc.event_bus_name,
142
+ )
143
+ if resp and resp.targets and !resp.targets.empty?
144
+ MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).remove_targets(
145
+ rule: id,
146
+ event_bus_name: desc.event_bus_name,
147
+ ids: resp.targets.map { |t| t.id }
148
+ )
149
+ end
150
+
151
+ MU::Cloud::AWS.cloudwatchevents(region: region, credentials: credentials).delete_rule(
152
+ name: id,
153
+ event_bus_name: desc.event_bus_name
154
+ )
155
+ end
156
+ end
157
+ }
158
+ end
159
+
160
+ # Locate an existing event.
161
+ # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching CloudWatch Event
162
+ def self.find(**args)
163
+ found = {}
164
+
165
+ MU::Cloud::AWS.cloudwatchevents(region: args[:region], credentials: args[:credentials]).list_rules.rules.each { |r|
166
+ next if args[:cloud_id] and ![r.name, r.arn].include?(args[:cloud_id])
167
+ found[r.name] = r
168
+ }
169
+
170
+ found
171
+ end
172
+
173
+ # Reverse-map our cloud description into a runnable config hash.
174
+ # We assume that any values we have in +@config+ are placeholders, and
175
+ # calculate our own accordingly based on what's live in the cloud.
176
+ def toKitten(**_args)
177
+ bok = {
178
+ "cloud" => "AWS",
179
+ "credentials" => @config['credentials'],
180
+ "cloud_id" => @cloud_id,
181
+ "region" => @config['region']
182
+ }
183
+
184
+ if !cloud_desc
185
+ MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
186
+ return nil
187
+ end
188
+ bok['name'] = cloud_desc.name
189
+ if cloud_desc.description and !cloud_desc.description.empty?
190
+ bok['description'] = cloud_desc.description
191
+ end
192
+
193
+ bok['disabled'] = true if cloud_desc.state == "DISABLED"
194
+
195
+ # schedule_expression="cron(15 6 * * ? *)"
196
+ if cloud_desc.schedule_expression
197
+ if cloud_desc.schedule_expression.match(/cron\((\S+) (\S+) (\S+) (\S+) (\S+) (\S+)\)/)
198
+ bok['schedule'] = {
199
+ "minute" => Regexp.last_match[1],
200
+ "hour" => Regexp.last_match[2],
201
+ "day_of_month" => Regexp.last_match[3],
202
+ "month" => Regexp.last_match[4],
203
+ "day_of_week" => Regexp.last_match[5],
204
+ "year" => Regexp.last_match[6]
205
+ }
206
+ else
207
+ MU.log "HALP", MU::ERR, details: cloud_desc.schedule_expression
208
+ end
209
+ end
210
+
211
+ if cloud_desc.role_arn
212
+ shortname = cloud_desc.role_arn.sub(/.*?role\/([^\/]+)$/, '\1')
213
+ bok['role'] = MU::Config::Ref.get(
214
+ id: shortname,
215
+ cloud: "AWS",
216
+ type: "roles"
217
+ )
218
+ end
219
+
220
+ targets = MU::Cloud::AWS.cloudwatchevents(region: @config['region'], credentials: @credentials).list_targets_by_rule(
221
+ rule: @cloud_id,
222
+ event_bus_name: cloud_desc.event_bus_name
223
+ ).targets
224
+ targets.each { |t|
225
+ bok['targets'] ||= []
226
+ _arn, _plat, service, region, account, resource = t.arn.split(/:/, 6)
227
+ target_type = if service == "lambda"
228
+ resource.sub!(/^function:/, '')
229
+ "functions"
230
+ elsif service == "sns"
231
+ "notifiers"
232
+ elsif service == "sqs"
233
+ "msg_queues"
234
+ else
235
+ service
236
+ end
237
+ ref_params = {
238
+ id: resource,
239
+ region: region,
240
+ type: target_type,
241
+ cloud: "AWS",
242
+ credentials: @credentials,
243
+ habitat: MU::Config::Ref.get(
244
+ id: account,
245
+ cloud: "AWS",
246
+ credentials: @credentials
247
+ )
248
+ }
249
+ [:input, :input_path, :input_transformer, :kinesis_parameters, :run_command_parameters, :batch_parameters, :sqs_parameters].each { |attr|
250
+ if t.respond_to?(attr) and !t.send(attr).nil?
251
+ ref_params[attr] = MU.structToHash(t.send(attr), stringify_keys: true)
252
+ end
253
+ }
254
+
255
+ bok['targets'] << MU::Config::Ref.get(ref_params)
256
+ }
257
+
258
+ # XXX cloud_desc.event_pattern - what do we want to do with this?
259
+
260
+ bok
261
+ end
262
+
263
+
264
+ # Cloud-specific configuration properties.
265
+ # @param _config [MU::Config]: The calling MU::Config object
266
+ # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
267
+ def self.schema(_config)
268
+ toplevel_required = []
269
+
270
+ target_schema = MU::Config::Ref.schema(any_type: true, desc: "A resource which will be invoked by this event. Can be a reference to a sibling Mu resource, typically a +Function+ or +MsgQueue+, or to an unadorned external cloud resource.")
271
+ target_params = {
272
+ "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to use when interacting with this target.", omit_fields: ["region", "tag"]),
273
+ "input" => {
274
+ "type" => "string"
275
+ },
276
+ "input_path" => {
277
+ "type" => "string"
278
+ },
279
+ "run_command_parameters" => {
280
+ "type" => "object",
281
+ "description" => "Parameters used when you are using the rule to invoke Amazon EC2 Run Command",
282
+ "required" => ["run_command_targets"],
283
+ "properties" => {
284
+ "run_command_targets" => {
285
+ "type" => "array",
286
+ "items" => {
287
+ "type" => "object",
288
+ "description" => "Currently, AWS supports including only one +run_command_targets+ block, which specifies either an array of InstanceIds or a tag.",
289
+ "required" => ["key", "values"],
290
+ "properties" => {
291
+ "key" => {
292
+ "type" => "string",
293
+ "description" => "Can be either +tag: tag-key+ or +InstanceIds+"
294
+ },
295
+ "values" => {
296
+ "type" => "array",
297
+ "items" => {
298
+ "description" => "If +key+ is +tag: tag-key+, +values+ is a list of tag values; if +key+ is +InstanceIds+, +values+ is a list of Amazon EC2 instance IDs.",
299
+ "type" => "string"
300
+ }
301
+ }
302
+ }
303
+ }
304
+ }
305
+ }
306
+ },
307
+ "input_transformer" => {
308
+ "type" => "object",
309
+ "description" => "Settings to enable you to provide custom input to a target based on certain event data. You can extract one or more key-value pairs from the event and then use that data to send customized input to the target.",
310
+ "required" => ["input_template"],
311
+ "properties" => {
312
+ "input_template" => {
313
+ "type" => "string",
314
+ "description" => "Input template where you specify placeholders that will be filled with the values of the keys from +input_paths_map+ to customize the data sent to the target."
315
+ },
316
+ "input_paths_map" => {
317
+ "type" => "object",
318
+ "description" => "Hash representing JSON paths to be extracted from the event"
319
+ }
320
+ }
321
+ },
322
+ "batch_parameters" => {
323
+ "type" => "object",
324
+ "description" => "If the event target is an AWS Batch job, this contains the job definition, job name, and other parameters. See: https://docs.aws.amazon.com/batch/latest/userguide/jobs.html",
325
+ "required" => ["job_definition", "job_name"],
326
+ "properties" => {
327
+ "job_definition" => {
328
+ "description" => "The ARN or name of the job definition to use if the event target is an AWS Batch job.",
329
+ "type" => "string"
330
+ },
331
+ "job_name" => {
332
+ "description" => "The name to use for this execution of the job, if the target is an AWS Batch job.",
333
+ "type" => "string"
334
+ },
335
+ "array_properties" => {
336
+ "type" => "object",
337
+ "description" => "The array properties for the submitted job, such as the size of the array.",
338
+ "properties" => {
339
+ "size" => {
340
+ "description" => "Size of the submitted array",
341
+ "type" => "integer"
342
+ }
343
+ }
344
+ },
345
+ "retry_strategy" => {
346
+ "type" => "object",
347
+ "description" => "The retry strategy to use for failed jobs, if the target is an AWS Batch job.",
348
+ "properties" => {
349
+ "attempts" => {
350
+ "description" => "Number of retry attempts, valid values from 1-10",
351
+ "type" => "integer"
352
+ }
353
+ }
354
+ }
355
+ }
356
+ },
357
+ "sqs_parameters" => {
358
+ "type" => "object",
359
+ "description" => "Contains the message group ID to use when the target is an SQS FIFO queue.",
360
+ "required" => ["message_group_id"],
361
+ "properties" => {
362
+ "message_group_id" => {
363
+ "type" => "string"
364
+ }
365
+ }
366
+ },
367
+ "kinesis_parameters" => {
368
+ "type" => "object",
369
+ "description" => "The custom parameter you can use to control the shard assignment, when the target is a Kinesis data stream.",
370
+ "required" => ["partition_key_path"],
371
+ "properties" => {
372
+ "partition_key_path" => {
373
+ "type" => "string"
374
+ }
375
+ }
376
+ },
377
+ "http_parameters" => {
378
+ "type" => "object",
379
+ "description" => "Contains the HTTP parameters to use when the target is a API Gateway REST endpoint.",
380
+ "properties" => {
381
+ "path_parameter_values" => {
382
+ "type" => "array",
383
+ "items" => {
384
+ "description" => "The path parameter values to be used to populate API Gateway REST API path wildcards (\"*\").",
385
+ "type" => "string"
386
+ }
387
+ },
388
+ "header_parameters" => {
389
+ "description" => "Key => value pairs to pass as headers",
390
+ "type" => "object"
391
+ },
392
+ "query_string_parameters" => {
393
+ "description" => "Key => value pairs to pass as query strings",
394
+ "type" => "object"
395
+ }
396
+ }
397
+ }
398
+ }
399
+ target_schema["properties"].merge!(target_params)
400
+
401
+ schema = {
402
+ "disabled" => {
403
+ "type" => "boolean",
404
+ "description" => "Leave this job in place but disabled",
405
+ "default" => false
406
+ },
407
+ "role" => MU::Config::Ref.schema(type: "roles", desc: "A sibling {MU::Config::BasketofKittens::roles} entry or the id of an existing IAM role to assign to this CloudWatch Event.", omit_fields: ["region", "tag"]),
408
+ "targets" => {
409
+ "type" => "array",
410
+ "items" => target_schema
411
+ }
412
+ }
413
+ [toplevel_required, schema]
414
+ end
415
+
416
+ # Cloud-specific pre-processing of {MU::Config::BasketofKittens::jobs}, bare and unvalidated.
417
+ # @param job [Hash]: The resource to process and validate
418
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
419
+ # @return [Boolean]: True if validation succeeded, False otherwise
420
+ def self.validateConfig(job, _configurator)
421
+ ok = true
422
+
423
+ job['targets'].each { |t|
424
+ target_ref = MU::Config::Ref.get(t)
425
+ if target_ref.is_mu_type? and target_ref.name
426
+ MU::Config.addDependency(job, target_ref.name, target_ref.type)
427
+ end
428
+ }
429
+
430
+ ok
431
+ end
432
+
433
+ private
434
+
435
+ def get_properties
436
+ params = {
437
+ name: @cloud_id,
438
+ state: @config['disabled'] ? "DISABLED" : "ENABLED",
439
+ event_bus_name: "default" # XXX expose, or create a deploy-specific one?
440
+ }
441
+
442
+ params[:description] = if @config['description'] and @config['scrub_mu_isms']
443
+ @config['description']
444
+ else
445
+ @deploy.deploy_id
446
+ end
447
+
448
+ if @tags
449
+ params[:tags] = @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }
450
+ end
451
+
452
+ if @config['role']
453
+ role_obj = MU::Config::Ref.get(@config['role']).kitten(@deploy, cloud: "AWS")
454
+ raise MuError.new "Failed to fetch object from role reference", details: @config['role'].to_h if !role_obj
455
+ params[:role_arn] = role_obj.arn
456
+ end
457
+
458
+ if @config['schedule']
459
+ params[:schedule_expression] = "cron(" + ["minute", "hour", "day_of_month", "month", "day_of_week", "year"].map { |i| @config['schedule'][i] }.join(" ") +")"
460
+ end
461
+
462
+
463
+ params
464
+ end
465
+
466
+ end
467
+ end
468
+ end
469
+ end