cloud-mu 3.1.5 → 3.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +5 -1
  3. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  4. data/ansible/roles/mu-windows/files/config.xml +76 -0
  5. data/ansible/roles/mu-windows/tasks/main.yml +16 -0
  6. data/bin/mu-adopt +16 -12
  7. data/bin/mu-azure-tests +57 -0
  8. data/bin/mu-cleanup +2 -4
  9. data/bin/mu-configure +52 -0
  10. data/bin/mu-deploy +3 -3
  11. data/bin/mu-findstray-tests +25 -0
  12. data/bin/mu-gen-docs +2 -4
  13. data/bin/mu-load-config.rb +2 -1
  14. data/bin/mu-node-manage +15 -16
  15. data/bin/mu-run-tests +37 -12
  16. data/cloud-mu.gemspec +3 -3
  17. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  18. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  19. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  20. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  21. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  22. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  23. data/cookbooks/mu-tools/recipes/windows-client.rb +25 -22
  24. data/extras/clean-stock-amis +25 -19
  25. data/extras/generate-stock-images +1 -0
  26. data/extras/image-generators/AWS/win2k12.yaml +2 -0
  27. data/extras/image-generators/AWS/win2k16.yaml +2 -0
  28. data/extras/image-generators/AWS/win2k19.yaml +2 -0
  29. data/modules/mommacat.ru +1 -1
  30. data/modules/mu.rb +86 -98
  31. data/modules/mu/adoption.rb +373 -58
  32. data/modules/mu/cleanup.rb +214 -303
  33. data/modules/mu/cloud.rb +128 -1733
  34. data/modules/mu/cloud/database.rb +49 -0
  35. data/modules/mu/cloud/dnszone.rb +44 -0
  36. data/modules/mu/cloud/machine_images.rb +212 -0
  37. data/modules/mu/cloud/providers.rb +81 -0
  38. data/modules/mu/cloud/resource_base.rb +929 -0
  39. data/modules/mu/cloud/server.rb +40 -0
  40. data/modules/mu/cloud/server_pool.rb +1 -0
  41. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  42. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  43. data/modules/mu/cloud/wrappers.rb +169 -0
  44. data/modules/mu/config.rb +123 -81
  45. data/modules/mu/config/alarm.rb +2 -6
  46. data/modules/mu/config/bucket.rb +32 -3
  47. data/modules/mu/config/cache_cluster.rb +2 -2
  48. data/modules/mu/config/cdn.rb +100 -0
  49. data/modules/mu/config/collection.rb +1 -1
  50. data/modules/mu/config/container_cluster.rb +7 -2
  51. data/modules/mu/config/database.rb +84 -105
  52. data/modules/mu/config/database.yml +1 -2
  53. data/modules/mu/config/dnszone.rb +5 -4
  54. data/modules/mu/config/doc_helpers.rb +5 -6
  55. data/modules/mu/config/endpoint.rb +2 -1
  56. data/modules/mu/config/firewall_rule.rb +3 -19
  57. data/modules/mu/config/folder.rb +1 -1
  58. data/modules/mu/config/function.rb +17 -8
  59. data/modules/mu/config/group.rb +1 -1
  60. data/modules/mu/config/habitat.rb +1 -1
  61. data/modules/mu/config/job.rb +89 -0
  62. data/modules/mu/config/loadbalancer.rb +57 -11
  63. data/modules/mu/config/log.rb +1 -1
  64. data/modules/mu/config/msg_queue.rb +1 -1
  65. data/modules/mu/config/nosqldb.rb +1 -1
  66. data/modules/mu/config/notifier.rb +8 -19
  67. data/modules/mu/config/ref.rb +92 -14
  68. data/modules/mu/config/role.rb +1 -1
  69. data/modules/mu/config/schema_helpers.rb +38 -37
  70. data/modules/mu/config/search_domain.rb +1 -1
  71. data/modules/mu/config/server.rb +12 -13
  72. data/modules/mu/config/server_pool.rb +3 -7
  73. data/modules/mu/config/storage_pool.rb +1 -1
  74. data/modules/mu/config/tail.rb +11 -0
  75. data/modules/mu/config/user.rb +1 -1
  76. data/modules/mu/config/vpc.rb +27 -23
  77. data/modules/mu/config/vpc.yml +0 -1
  78. data/modules/mu/defaults/AWS.yaml +90 -90
  79. data/modules/mu/defaults/Azure.yaml +1 -0
  80. data/modules/mu/defaults/Google.yaml +1 -0
  81. data/modules/mu/deploy.rb +34 -20
  82. data/modules/mu/groomer.rb +16 -1
  83. data/modules/mu/groomers/ansible.rb +69 -4
  84. data/modules/mu/groomers/chef.rb +51 -4
  85. data/modules/mu/logger.rb +120 -144
  86. data/modules/mu/master.rb +97 -4
  87. data/modules/mu/mommacat.rb +160 -874
  88. data/modules/mu/mommacat/daemon.rb +23 -14
  89. data/modules/mu/mommacat/naming.rb +110 -3
  90. data/modules/mu/mommacat/search.rb +497 -0
  91. data/modules/mu/mommacat/storage.rb +252 -194
  92. data/modules/mu/{clouds → providers}/README.md +1 -1
  93. data/modules/mu/{clouds → providers}/aws.rb +258 -57
  94. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  95. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  96. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  97. data/modules/mu/providers/aws/cdn.rb +782 -0
  98. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  99. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +95 -84
  100. data/modules/mu/providers/aws/database.rb +1744 -0
  101. data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
  102. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  103. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
  104. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  105. data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
  106. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  107. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  108. data/modules/mu/providers/aws/job.rb +466 -0
  109. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
  110. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  111. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  112. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  113. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  114. data/modules/mu/{clouds → providers}/aws/role.rb +76 -48
  115. data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
  116. data/modules/mu/{clouds → providers}/aws/server.rb +66 -98
  117. data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
  118. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  119. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  120. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  121. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  122. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  123. data/modules/mu/{clouds → providers}/aws/vpc.rb +143 -74
  124. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  125. data/modules/mu/{clouds → providers}/azure.rb +13 -0
  126. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  127. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  128. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  129. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  130. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  131. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  132. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  133. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  134. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  135. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  136. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  137. data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
  138. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  139. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  140. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  141. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  142. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  143. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  144. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  145. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  146. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  147. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  148. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  149. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  150. data/modules/mu/{clouds → providers}/google.rb +29 -6
  151. data/modules/mu/{clouds → providers}/google/bucket.rb +4 -4
  152. data/modules/mu/{clouds → providers}/google/container_cluster.rb +38 -20
  153. data/modules/mu/{clouds → providers}/google/database.rb +5 -12
  154. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
  155. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  156. data/modules/mu/{clouds → providers}/google/function.rb +6 -6
  157. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  158. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  159. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
  160. data/modules/mu/{clouds → providers}/google/role.rb +50 -31
  161. data/modules/mu/{clouds → providers}/google/server.rb +41 -24
  162. data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
  163. data/modules/mu/{clouds → providers}/google/user.rb +34 -24
  164. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  165. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  166. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  167. data/modules/mu/{clouds → providers}/google/vpc.rb +45 -14
  168. data/modules/tests/aws-jobs-functions.yaml +46 -0
  169. data/modules/tests/centos6.yaml +15 -0
  170. data/modules/tests/centos7.yaml +15 -0
  171. data/modules/tests/centos8.yaml +12 -0
  172. data/modules/tests/ecs.yaml +2 -2
  173. data/modules/tests/eks.yaml +1 -1
  174. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  175. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  176. data/modules/tests/microservice_app.yaml +288 -0
  177. data/modules/tests/rds.yaml +108 -0
  178. data/modules/tests/regrooms/rds.yaml +123 -0
  179. data/modules/tests/server-with-scrub-muisms.yaml +1 -1
  180. data/modules/tests/super_complex_bok.yml +2 -2
  181. data/modules/tests/super_simple_bok.yml +3 -5
  182. data/spec/mu/clouds/azure_spec.rb +2 -2
  183. metadata +122 -92
  184. data/modules/mu/clouds/aws/database.rb +0 -1974
  185. data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -27,8 +27,8 @@ module MU
27
27
 
28
28
  # Called automatically by {MU::Deploy#createResources}
29
29
  def create
30
- MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @mu_name)
31
30
  @cloud_id = @mu_name
31
+ MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @cloud_id)
32
32
  MU.log "Created SNS topic #{@mu_name}"
33
33
  end
34
34
 
@@ -36,17 +36,48 @@ module MU
36
36
  def groom
37
37
  if @config['subscriptions']
38
38
  @config['subscriptions'].each { |sub|
39
- MU::Cloud::AWS::Notifier.subscribe(
40
- arn: arn,
41
- endpoint: sub['endpoint'],
42
- region: @config['region'],
43
- credentials: @config['credentials'],
44
- protocol: sub['type']
45
- )
39
+ if sub['resource'] and !sub['endpoint']
40
+ endpoint_obj = nil
41
+ MU.retrier([], max: 5, wait: 9, loop_if: Proc.new { endpoint_obj.nil? }) {
42
+ endpoint_obj = MU::Config::Ref.get(sub['resource']).kitten(@deploy)
43
+ }
44
+ sub['endpoint'] = endpoint_obj.arn
45
+ end
46
+ subscribe(sub['endpoint'], sub['type'])
46
47
  }
47
48
  end
48
49
  end
49
50
 
51
+ # Subscribe something to this SNS topic
52
+ # @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
53
+ # @param protocol [String]: The protocol being subscribed
54
+ def subscribe(endpoint, protocol)
55
+ self.class.subscribe(arn, endpoint, protocol, region: @config['region'], credentials: @credentials)
56
+ end
57
+
58
+ # Subscribe something to an SNS topic
59
+ # @param cloud_id [String]: The short name or ARN of an existing SNS topic
60
+ # @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
61
+ # @param protocol [String]: The protocol being subscribed
62
+ # @param region [String]: The region of the target SNS topic
63
+ # @param credentials [String]:
64
+ def self.subscribe(cloud_id, endpoint, protocol, region: nil, credentials: nil)
65
+ topic = find(cloud_id: cloud_id, region: region, credentials: credentials).values.first
66
+ if !topic
67
+ raise MuError, "Failed to find SNS Topic #{cloud_id} in #{region}"
68
+ end
69
+ arn = topic["TopicArn"]
70
+
71
+ resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
72
+
73
+ resp.each { |subscription|
74
+ return subscription if subscription.protocol == protocol and subscription.endpoint == endpoint
75
+ }
76
+
77
+ MU.log "Subscribing #{endpoint} (#{protocol}) to SNS topic #{arn}", MU::NOTICE
78
+ MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
79
+ end
80
+
50
81
  # Does this resource type exist as a global (cloud-wide) artifact, or
51
82
  # is it localized to a region/zone?
52
83
  # @return [Boolean]
@@ -65,12 +96,12 @@ module MU
65
96
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
66
97
  # @param region [String]: The cloud provider region
67
98
  # @return [void]
68
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
99
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
69
100
  MU.log "AWS::Notifier.cleanup: need to support flags['known']", MU::DEBUG, details: flags
70
101
  MU.log "Placeholder: AWS Notifier artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
71
102
 
72
103
  MU::Cloud::AWS.sns(region: region, credentials: credentials).list_topics.topics.each { |topic|
73
- if topic.topic_arn.match(MU.deploy_id)
104
+ if topic.topic_arn.match(deploy_id)
74
105
  # We don't have a way to tag our SNS topics, so we will delete any topic that has the MU-ID in its ARN.
75
106
  # This may fail to find notifier groups in some cases (eg. cache_cluster) so we might want to delete from each API as well.
76
107
  MU.log "Deleting SNS topic: #{topic.topic_arn}"
@@ -91,6 +122,7 @@ module MU
91
122
  # Return the metadata for this user cofiguration
92
123
  # @return [Hash]
93
124
  def notify
125
+ return nil if !@cloud_id or !cloud_desc(use_cache: false)
94
126
  desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes
95
127
  MU.structToHash(desc)
96
128
  end
@@ -101,9 +133,16 @@ module MU
101
133
  found = {}
102
134
 
103
135
  if args[:cloud_id]
104
- arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id]
105
- desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes
106
- found[args[:cloud_id]] = desc if desc
136
+ arn = if args[:cloud_id].match(/^arn:/)
137
+ args[:cloud_id]
138
+ else
139
+ "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id]
140
+ end
141
+ begin
142
+ desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes
143
+ found[args[:cloud_id]] = desc if desc
144
+ rescue ::Aws::SNS::Errors::NotFound
145
+ end
107
146
  else
108
147
  next_token = nil
109
148
  begin
@@ -120,6 +159,58 @@ module MU
120
159
  found
121
160
  end
122
161
 
162
+ # Reverse-map our cloud description into a runnable config hash.
163
+ # We assume that any values we have in +@config+ are placeholders, and
164
+ # calculate our own accordingly based on what's live in the cloud.
165
+ def toKitten(**_args)
166
+ bok = {
167
+ "cloud" => "AWS",
168
+ "credentials" => @config['credentials'],
169
+ "cloud_id" => @cloud_id,
170
+ "region" => @config['region']
171
+ }
172
+
173
+ if !cloud_desc
174
+ MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
175
+ return nil
176
+ end
177
+
178
+ bok['name'] = cloud_desc["DisplayName"].empty? ? @cloud_id : cloud_desc["DisplayName"]
179
+ svcmap = {
180
+ "lambda" => "functions",
181
+ "sqs" => "msg_queues"
182
+ }
183
+ MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub|
184
+ bok['subscriptions'] ||= []
185
+
186
+ bok['subscriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/)
187
+ _wholestring, service, region, account, id = Regexp.last_match.to_a
188
+ {
189
+ "type" => sub.protocol,
190
+ "resource" => MU::Config::Ref.get(
191
+ type: svcmap[service],
192
+ region: region,
193
+ credentials: @credentials,
194
+ id: id,
195
+ cloud: "AWS",
196
+ habitat: MU::Config::Ref.get(
197
+ id: account,
198
+ cloud: "AWS",
199
+ credentials: @credentials
200
+ )
201
+ )
202
+ }
203
+ else
204
+ {
205
+ "type" => sub.protocol,
206
+ "endpoint" => sub.endpoint
207
+ }
208
+ end
209
+ }
210
+
211
+ bok
212
+ end
213
+
123
214
  # Cloud-specific configuration properties.
124
215
  # @param _config [MU::Config]: The calling MU::Config object
125
216
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
@@ -130,11 +221,10 @@ module MU
130
221
  "type" => "array",
131
222
  "items" => {
132
223
  "type" => "object",
133
- "required" => ["endpoint"],
134
224
  "properties" => {
135
225
  "type" => {
136
226
  "type" => "string",
137
- "description" => "",
227
+ "description" => "Type of endpoint or resource which should receive notifications. If not specified, will attempt to auto-detect.",
138
228
  "enum" => ["http", "https", "email", "email-json", "sms", "sqs", "application", "lambda"]
139
229
  }
140
230
  }
@@ -148,26 +238,42 @@ module MU
148
238
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::notifier}, bare and unvalidated.
149
239
 
150
240
  # @param notifier [Hash]: The resource to process and validate
151
- # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
241
+ # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
152
242
  # @return [Boolean]: True if validation succeeded, False otherwise
153
- def self.validateConfig(notifier, _configurator)
243
+ def self.validateConfig(notifier, configurator)
154
244
  ok = true
155
245
 
156
246
  if notifier['subscriptions']
157
247
  notifier['subscriptions'].each { |sub|
248
+ if sub['resource'] and configurator.haveLitterMate?(sub['resource']['name'], sub['resource']['type'])
249
+ sub['resource']['cloud'] = "AWS"
250
+ MU::Config.addDependency(notifier, sub['resource']['name'], sub['resource']['type'])
251
+ end
158
252
  if !sub["type"]
159
- if sub["endpoint"].match(/^http:/i)
160
- sub["type"] = "http"
161
- elsif sub["endpoint"].match(/^https:/i)
162
- sub["type"] = "https"
163
- elsif sub["endpoint"].match(/^sqs:/i)
164
- sub["type"] = "sqs"
165
- elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
166
- sub["type"] = "sms"
167
- elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
168
- sub["type"] = "email"
169
- else
170
- MU.log "Notifier #{notifier['name']} subscription #{sub['endpoint']} did not specify a type, and I'm unable to guess one", MU::ERR
253
+ sub['type'] = if sub['resource']
254
+ if sub['resource']['type'] == "functions"
255
+ "lambda"
256
+ elsif sub['resource']['type'] == "msg_queues"
257
+ "sqs"
258
+ end
259
+ elsif sub['endpoint']
260
+ if sub["endpoint"].match(/^http:/i)
261
+ "http"
262
+ elsif sub["endpoint"].match(/^https:/i)
263
+ "https"
264
+ elsif sub["endpoint"].match(/:sqs:/i)
265
+ "sqs"
266
+ elsif sub["endpoint"].match(/:lambda:/i)
267
+ "lambda"
268
+ elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
269
+ "sms"
270
+ elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
271
+ "email"
272
+ end
273
+ end
274
+
275
+ if !sub['type']
276
+ MU.log "Notifier #{notifier['name']} subscription did not specify a type, and I'm unable to guess one", MU::ERR, details: sub
171
277
  ok = false
172
278
  end
173
279
  end
@@ -178,40 +284,6 @@ module MU
178
284
  end
179
285
 
180
286
 
181
- # Subscribe to a notifier group. This can either be an email address, SQS queue, application endpoint, etc...
182
- # Will create the subscription only if it doesn't already exist.
183
- # @param arn [String]: The cloud provider's identifier of the notifier group.
184
- # @param protocol [String]: The type of the subscription (eg. email,https, etc..).
185
- # @param endpoint [String]: The endpoint of the subscription. This will depend on the 'protocol' (as an example if protocol is email, endpoint will be the email address) ..
186
- # @param region [String]: The cloud provider region.
187
- def self.subscribe(arn: nil, protocol: nil, endpoint: nil, region: MU.curRegion, credentials: nil)
188
- retries = 0
189
- begin
190
- resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
191
- rescue Aws::SNS::Errors::NotFound
192
- if retries < 5
193
- MU.log "Couldn't find topic #{arn}, retrying several times in case of a lagging resource"
194
- retries += 1
195
- sleep 30
196
- retry
197
- else
198
- raise MuError, "Couldn't find topic #{arn}, giving up"
199
- end
200
- end
201
-
202
- already_subscribed = false
203
- if resp && !resp.empty?
204
- resp.each { |subscription|
205
- already_subscribed = true if subscription.protocol == protocol && subscription.endpoint == endpoint
206
- }
207
- end
208
-
209
- unless already_subscribed
210
- MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
211
- MU.log "Subscribed #{endpoint} to SNS topic #{arn}"
212
- end
213
- end
214
-
215
287
  # Test if a notifier group exists
216
288
  # Create a new notifier group. Will check if the group exists before creating it.
217
289
  # @param topic_name [String]: The cloud provider's name for the notifier group.
@@ -30,7 +30,7 @@ module MU
30
30
  end
31
31
  end
32
32
 
33
- @mu_name ||= @deploy.getResourceName(@config["name"])
33
+ @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 64)
34
34
  end
35
35
 
36
36
  # Called automatically by {MU::Deploy#createResources}
@@ -92,13 +92,14 @@ module MU
92
92
  configured_policies = []
93
93
 
94
94
  if @config['raw_policies']
95
+ MU.log "Attaching #{@config['raw_policies'].size.to_s} raw #{@config['raw_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
95
96
  configured_policies = @config['raw_policies'].map { |p|
96
97
  @mu_name+"-"+p.keys.first.upcase
97
98
  }
98
99
  end
99
100
 
100
101
  if @config['attachable_policies']
101
- MU.log "Attaching #{@config['attachable_policies'].size.to_s} #{@config['attachable_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
102
+ MU.log "Attaching #{@config['attachable_policies'].size.to_s} external #{@config['attachable_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
102
103
  configured_policies.concat(@config['attachable_policies'].map { |p|
103
104
  id = if p.is_a?(MU::Config::Ref)
104
105
  p.cloud_id
@@ -109,17 +110,16 @@ module MU
109
110
  end
110
111
  id.gsub(/.*?\/([^:\/]+)$/, '\1')
111
112
  })
112
- configured_policies.each { |pol|
113
- }
114
113
  end
115
114
 
115
+ # Purge anything that doesn't belong
116
116
  if !@config['bare_policies']
117
117
  attached_policies = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_role_policies(
118
118
  role_name: @mu_name
119
119
  ).attached_policies
120
120
  attached_policies.each { |a|
121
121
  if !configured_policies.include?(a.policy_name)
122
- MU.log "Removing IAM policy #{a.policy_name} from role #{@mu_name}", MU::NOTICE
122
+ MU.log "Removing IAM policy #{a.policy_name} from role #{@mu_name}", MU::NOTICE, details: configured_policies
123
123
  MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @config['credentials'])
124
124
  end
125
125
  }
@@ -153,6 +153,7 @@ module MU
153
153
  policy.values.each { |p|
154
154
  p["Version"] ||= "2012-10-17"
155
155
  }
156
+
156
157
  policy_name = basename+"-"+policy.keys.first.upcase
157
158
 
158
159
  arn = "arn:"+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+":iam::"+MU::Cloud::AWS.credToAcct(credentials)+":policy#{path}/#{policy_name}"
@@ -216,7 +217,22 @@ module MU
216
217
  # populated with one or both depending on what this resource has
217
218
  # defined.
218
219
  def cloud_desc(use_cache: true)
219
- return @cloud_desc_cache if @cloud_desc_cache and use_cache
220
+
221
+ # we might inherit a naive cached description from the base cloud
222
+ # layer; rearrange it to our tastes
223
+ if @cloud_desc_cache.is_a?(::Aws::IAM::Types::Role)
224
+ new_desc = {
225
+ "role" => @cloud_desc_cache
226
+ }
227
+ @cloud_desc_cache = new_desc
228
+ elsif @cloud_desc_cache.is_a?(::Aws::IAM::Types::Policy)
229
+ new_desc = {
230
+ "policies" => [@cloud_desc_cache]
231
+ }
232
+ @cloud_desc_cache = new_desc
233
+ end
234
+
235
+ return @cloud_desc_cache if @cloud_desc_cache and !@cloud_desc_cache.empty? and use_cache
220
236
 
221
237
  @cloud_desc_cache = {}
222
238
  if @config['bare_policies']
@@ -419,14 +435,14 @@ end
419
435
  # @param noop [Boolean]: If true, will only print what would be done
420
436
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
421
437
  # @return [void]
422
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
438
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
423
439
 
424
440
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
425
- path_prefix: "/"+MU.deploy_id+"/"
441
+ path_prefix: "/"+deploy_id+"/"
426
442
  )
427
443
  if resp and resp.policies
428
444
  resp.policies.each { |policy|
429
- MU.log "Deleting IAM policy /#{MU.deploy_id}/#{policy.policy_name}"
445
+ MU.log "Deleting IAM policy /#{deploy_id}/#{policy.policy_name}"
430
446
  if !noop
431
447
  purgePolicy(policy.arn, credentials)
432
448
  end
@@ -437,19 +453,23 @@ end
437
453
  roles = MU::Cloud::AWS::Role.find(credentials: credentials).values
438
454
  roles.each { |r|
439
455
  next if !r.respond_to?(:role_name)
440
- if r.path.match(/^\/#{Regexp.quote(MU.deploy_id)}/)
456
+ if r.path.match(/^\/#{Regexp.quote(deploy_id)}/)
441
457
  deleteme << r
442
458
  next
443
459
  end
444
460
  # For some dumb reason, the list output that .find gets doesn't
445
461
  # include the tags, so we need to fetch each role individually to
446
462
  # check tags. Hardly seems efficient.
447
- desc = MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
463
+ desc = begin
464
+ MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
465
+ rescue Aws::IAM::Errors::NoSuchEntity
466
+ next
467
+ end
448
468
  if desc.role and desc.role.tags and desc.role.tags
449
469
  master_match = false
450
470
  deploy_match = false
451
471
  desc.role.tags.each { |t|
452
- if t.key == "MU-ID" and t.value == MU.deploy_id
472
+ if t.key == "MU-ID" and t.value == deploy_id
453
473
  deploy_match = true
454
474
  elsif t.key == "MU-MASTER-IP" and t.value == MU.mu_public_ip
455
475
  master_match = true
@@ -516,7 +536,7 @@ end
516
536
 
517
537
  begin
518
538
  # managed policies get fetched by ARN, roles by plain name. Ok!
519
- if args[:cloud_id].match(/^arn:/)
539
+ if args[:cloud_id].match(/^arn:.*?:policy\//)
520
540
  resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_policy(
521
541
  policy_arn: args[:cloud_id]
522
542
  )
@@ -525,39 +545,26 @@ end
525
545
  end
526
546
  else
527
547
  resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_role(
528
- role_name: args[:cloud_id]
548
+ role_name: args[:cloud_id].sub(/^arn:.*?\/([^:\/]+)$/, '\1') # XXX if it's an ARN, actually parse it and look in the correct account when applicable
529
549
  )
550
+
530
551
  if resp and resp.role
531
- found[args[:cloud_id]] = resp.role
552
+ found[resp.role.role_name] = resp.role
532
553
  end
533
554
  end
534
555
  rescue ::Aws::IAM::Errors::NoSuchEntity
535
556
  end
536
557
 
537
558
  else
538
- marker = nil
539
- begin
540
- resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles(
541
- marker: marker
542
- )
543
- break if !resp or !resp.roles
544
- resp.roles.each { |role|
545
- found[role.role_name] = role
546
- }
547
- marker = resp.marker
548
- end while marker
559
+ resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles
560
+ resp.roles.each { |role|
561
+ found[role.role_name] = role
562
+ }
549
563
 
550
- begin
551
- resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(
552
- scope: "Local",
553
- marker: marker
554
- )
555
- break if !resp or !resp.policies
556
- resp.policies.each { |pol|
557
- found[pol.arn] = pol
558
- }
559
- marker = resp.marker
560
- end while marker
564
+ resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(scope: "Local")
565
+ resp.policies.each { |pol|
566
+ found[pol.arn] = pol
567
+ }
561
568
  end
562
569
 
563
570
  found
@@ -615,7 +622,6 @@ end
615
622
  )
616
623
  JSON.parse(URI.decode(version.policy_version.document))
617
624
  end
618
-
619
625
  bok["policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["policies"])
620
626
  end
621
627
  }
@@ -695,6 +701,7 @@ end
695
701
  end
696
702
 
697
703
  bok["attachable_policies"].uniq! if bok["attachable_policies"]
704
+ bok["name"].gsub!(/[^a-zA-Z0-9_\-]/, "_")
698
705
 
699
706
  bok
700
707
  end
@@ -707,6 +714,10 @@ end
707
714
  def self.doc2MuPolicies(basename, doc, policies = [])
708
715
  policies ||= []
709
716
 
717
+ if !doc["Statement"].is_a?(Array)
718
+ doc["Statement"] = [doc["Statement"]]
719
+ end
720
+
710
721
  doc["Statement"].each { |s|
711
722
  if !s["Action"]
712
723
  MU.log "Statement in policy document for #{basename} didn't have an Action field", MU::WARN, details: doc
@@ -804,6 +815,19 @@ end
804
815
  }
805
816
  end
806
817
 
818
+ if @config['raw_policies']
819
+ raw_arns = MU::Cloud::AWS::Role.manageRawPolicies(
820
+ @config['raw_policies'],
821
+ basename: @deploy.getResourceName(@config['name']),
822
+ credentials: @credentials
823
+ )
824
+ raw_arns.each { |p_arn|
825
+ mypolicies << MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy(
826
+ policy_arn: p_arn
827
+ ).policy
828
+ }
829
+ end
830
+
807
831
  mypolicies.each { |p|
808
832
  if entitytype == "user"
809
833
  resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_user_policies(
@@ -925,7 +949,7 @@ end
925
949
  toplevel_required = []
926
950
  aws_resource_types = MU::Cloud.resource_types.keys.reject { |t|
927
951
  begin
928
- MU::Cloud.loadCloudType("AWS", t)
952
+ MU::Cloud.resourceClass("AWS", t)
929
953
  false
930
954
  rescue MuCloudResourceNotImplemented
931
955
  true
@@ -1087,11 +1111,7 @@ end
1087
1111
  role['policies'].each { |policy|
1088
1112
  policy['targets'].each { |target|
1089
1113
  if target['type']
1090
- role['dependencies'] ||= []
1091
- role['dependencies'] << {
1092
- "name" => target['identifier'],
1093
- "type" => target['type']
1094
- }
1114
+ MU::Config.addDependency(role, target['identifier'], target['type'], no_create_wait: true)
1095
1115
  end
1096
1116
  }
1097
1117
  }
@@ -1107,13 +1127,14 @@ end
1107
1127
  # @param policies [Array<Hash>]: One or more policy chunks
1108
1128
  # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
1109
1129
  # @return [Array<Hash>]
1110
- def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false)
1130
+ def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil)
1111
1131
  if policies
1112
1132
  name = nil
1113
1133
  doc = {
1114
- "Version" => "2012-10-17",
1134
+ "Version" => version,
1115
1135
  "Statement" => []
1116
1136
  }
1137
+ doc["Id"] = doc_id if doc_id
1117
1138
  policies.each { |policy|
1118
1139
  policy["flag"] ||= "Allow"
1119
1140
  statement = {
@@ -1154,7 +1175,14 @@ end
1154
1175
  raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy"
1155
1176
  end
1156
1177
  else
1157
- bucket_prefix = grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/) ? "Service" : "AWS"
1178
+ bucket_prefix = if grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/)
1179
+ "Service"
1180
+ elsif grantee["identifier"] =~ /^[a-f0-9]+$/
1181
+ "CanonicalUser"
1182
+ else
1183
+ "AWS"
1184
+ end
1185
+
1158
1186
  if bucket_style
1159
1187
  statement["Principal"] << { bucket_prefix => grantee["identifier"] }
1160
1188
  else
@@ -1186,7 +1214,7 @@ end
1186
1214
  statement["Resource"] << id+"/*"
1187
1215
  end
1188
1216
  else
1189
- raise MuError, "Couldn't find a #{target["entity_type"]} named #{target["identifier"]} when generating IAM policy"
1217
+ raise MuError, "Couldn't find a #{target["type"]} named #{target["identifier"]} when generating IAM policy"
1190
1218
  end
1191
1219
  else
1192
1220
  target["identifier"] += target["path"] if target["path"]