cloud-mu 3.1.5 → 3.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +5 -1
  3. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  4. data/ansible/roles/mu-windows/files/config.xml +76 -0
  5. data/ansible/roles/mu-windows/tasks/main.yml +16 -0
  6. data/bin/mu-adopt +16 -12
  7. data/bin/mu-azure-tests +57 -0
  8. data/bin/mu-cleanup +2 -4
  9. data/bin/mu-configure +52 -0
  10. data/bin/mu-deploy +3 -3
  11. data/bin/mu-findstray-tests +25 -0
  12. data/bin/mu-gen-docs +2 -4
  13. data/bin/mu-load-config.rb +2 -1
  14. data/bin/mu-node-manage +15 -16
  15. data/bin/mu-run-tests +37 -12
  16. data/cloud-mu.gemspec +3 -3
  17. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  18. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  19. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  20. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  21. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  22. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  23. data/cookbooks/mu-tools/recipes/windows-client.rb +25 -22
  24. data/extras/clean-stock-amis +25 -19
  25. data/extras/generate-stock-images +1 -0
  26. data/extras/image-generators/AWS/win2k12.yaml +2 -0
  27. data/extras/image-generators/AWS/win2k16.yaml +2 -0
  28. data/extras/image-generators/AWS/win2k19.yaml +2 -0
  29. data/modules/mommacat.ru +1 -1
  30. data/modules/mu.rb +86 -98
  31. data/modules/mu/adoption.rb +373 -58
  32. data/modules/mu/cleanup.rb +214 -303
  33. data/modules/mu/cloud.rb +128 -1733
  34. data/modules/mu/cloud/database.rb +49 -0
  35. data/modules/mu/cloud/dnszone.rb +44 -0
  36. data/modules/mu/cloud/machine_images.rb +212 -0
  37. data/modules/mu/cloud/providers.rb +81 -0
  38. data/modules/mu/cloud/resource_base.rb +929 -0
  39. data/modules/mu/cloud/server.rb +40 -0
  40. data/modules/mu/cloud/server_pool.rb +1 -0
  41. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  42. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  43. data/modules/mu/cloud/wrappers.rb +169 -0
  44. data/modules/mu/config.rb +123 -81
  45. data/modules/mu/config/alarm.rb +2 -6
  46. data/modules/mu/config/bucket.rb +32 -3
  47. data/modules/mu/config/cache_cluster.rb +2 -2
  48. data/modules/mu/config/cdn.rb +100 -0
  49. data/modules/mu/config/collection.rb +1 -1
  50. data/modules/mu/config/container_cluster.rb +7 -2
  51. data/modules/mu/config/database.rb +84 -105
  52. data/modules/mu/config/database.yml +1 -2
  53. data/modules/mu/config/dnszone.rb +5 -4
  54. data/modules/mu/config/doc_helpers.rb +5 -6
  55. data/modules/mu/config/endpoint.rb +2 -1
  56. data/modules/mu/config/firewall_rule.rb +3 -19
  57. data/modules/mu/config/folder.rb +1 -1
  58. data/modules/mu/config/function.rb +17 -8
  59. data/modules/mu/config/group.rb +1 -1
  60. data/modules/mu/config/habitat.rb +1 -1
  61. data/modules/mu/config/job.rb +89 -0
  62. data/modules/mu/config/loadbalancer.rb +57 -11
  63. data/modules/mu/config/log.rb +1 -1
  64. data/modules/mu/config/msg_queue.rb +1 -1
  65. data/modules/mu/config/nosqldb.rb +1 -1
  66. data/modules/mu/config/notifier.rb +8 -19
  67. data/modules/mu/config/ref.rb +92 -14
  68. data/modules/mu/config/role.rb +1 -1
  69. data/modules/mu/config/schema_helpers.rb +38 -37
  70. data/modules/mu/config/search_domain.rb +1 -1
  71. data/modules/mu/config/server.rb +12 -13
  72. data/modules/mu/config/server_pool.rb +3 -7
  73. data/modules/mu/config/storage_pool.rb +1 -1
  74. data/modules/mu/config/tail.rb +11 -0
  75. data/modules/mu/config/user.rb +1 -1
  76. data/modules/mu/config/vpc.rb +27 -23
  77. data/modules/mu/config/vpc.yml +0 -1
  78. data/modules/mu/defaults/AWS.yaml +90 -90
  79. data/modules/mu/defaults/Azure.yaml +1 -0
  80. data/modules/mu/defaults/Google.yaml +1 -0
  81. data/modules/mu/deploy.rb +34 -20
  82. data/modules/mu/groomer.rb +16 -1
  83. data/modules/mu/groomers/ansible.rb +69 -4
  84. data/modules/mu/groomers/chef.rb +51 -4
  85. data/modules/mu/logger.rb +120 -144
  86. data/modules/mu/master.rb +97 -4
  87. data/modules/mu/mommacat.rb +160 -874
  88. data/modules/mu/mommacat/daemon.rb +23 -14
  89. data/modules/mu/mommacat/naming.rb +110 -3
  90. data/modules/mu/mommacat/search.rb +497 -0
  91. data/modules/mu/mommacat/storage.rb +252 -194
  92. data/modules/mu/{clouds → providers}/README.md +1 -1
  93. data/modules/mu/{clouds → providers}/aws.rb +258 -57
  94. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  95. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  96. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  97. data/modules/mu/providers/aws/cdn.rb +782 -0
  98. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  99. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +95 -84
  100. data/modules/mu/providers/aws/database.rb +1744 -0
  101. data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
  102. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  103. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
  104. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  105. data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
  106. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  107. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  108. data/modules/mu/providers/aws/job.rb +466 -0
  109. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
  110. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  111. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  112. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  113. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  114. data/modules/mu/{clouds → providers}/aws/role.rb +76 -48
  115. data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
  116. data/modules/mu/{clouds → providers}/aws/server.rb +66 -98
  117. data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
  118. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  119. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  120. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  121. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  122. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  123. data/modules/mu/{clouds → providers}/aws/vpc.rb +143 -74
  124. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  125. data/modules/mu/{clouds → providers}/azure.rb +13 -0
  126. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  127. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  128. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  129. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  130. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  131. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  132. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  133. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  134. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  135. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  136. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  137. data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
  138. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  139. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  140. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  141. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  142. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  143. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  144. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  145. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  146. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  147. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  148. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  149. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  150. data/modules/mu/{clouds → providers}/google.rb +29 -6
  151. data/modules/mu/{clouds → providers}/google/bucket.rb +4 -4
  152. data/modules/mu/{clouds → providers}/google/container_cluster.rb +38 -20
  153. data/modules/mu/{clouds → providers}/google/database.rb +5 -12
  154. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
  155. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  156. data/modules/mu/{clouds → providers}/google/function.rb +6 -6
  157. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  158. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  159. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
  160. data/modules/mu/{clouds → providers}/google/role.rb +50 -31
  161. data/modules/mu/{clouds → providers}/google/server.rb +41 -24
  162. data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
  163. data/modules/mu/{clouds → providers}/google/user.rb +34 -24
  164. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  165. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  166. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  167. data/modules/mu/{clouds → providers}/google/vpc.rb +45 -14
  168. data/modules/tests/aws-jobs-functions.yaml +46 -0
  169. data/modules/tests/centos6.yaml +15 -0
  170. data/modules/tests/centos7.yaml +15 -0
  171. data/modules/tests/centos8.yaml +12 -0
  172. data/modules/tests/ecs.yaml +2 -2
  173. data/modules/tests/eks.yaml +1 -1
  174. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  175. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  176. data/modules/tests/microservice_app.yaml +288 -0
  177. data/modules/tests/rds.yaml +108 -0
  178. data/modules/tests/regrooms/rds.yaml +123 -0
  179. data/modules/tests/server-with-scrub-muisms.yaml +1 -1
  180. data/modules/tests/super_complex_bok.yml +2 -2
  181. data/modules/tests/super_simple_bok.yml +3 -5
  182. data/spec/mu/clouds/azure_spec.rb +2 -2
  183. metadata +122 -92
  184. data/modules/mu/clouds/aws/database.rb +0 -1974
  185. data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -27,8 +27,8 @@ module MU
27
27
 
28
28
  # Called automatically by {MU::Deploy#createResources}
29
29
  def create
30
- MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @mu_name)
31
30
  @cloud_id = @mu_name
31
+ MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @cloud_id)
32
32
  MU.log "Created SNS topic #{@mu_name}"
33
33
  end
34
34
 
@@ -36,17 +36,48 @@ module MU
36
36
  def groom
37
37
  if @config['subscriptions']
38
38
  @config['subscriptions'].each { |sub|
39
- MU::Cloud::AWS::Notifier.subscribe(
40
- arn: arn,
41
- endpoint: sub['endpoint'],
42
- region: @config['region'],
43
- credentials: @config['credentials'],
44
- protocol: sub['type']
45
- )
39
+ if sub['resource'] and !sub['endpoint']
40
+ endpoint_obj = nil
41
+ MU.retrier([], max: 5, wait: 9, loop_if: Proc.new { endpoint_obj.nil? }) {
42
+ endpoint_obj = MU::Config::Ref.get(sub['resource']).kitten(@deploy)
43
+ }
44
+ sub['endpoint'] = endpoint_obj.arn
45
+ end
46
+ subscribe(sub['endpoint'], sub['type'])
46
47
  }
47
48
  end
48
49
  end
49
50
 
51
+ # Subscribe something to this SNS topic
52
+ # @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
53
+ # @param protocol [String]: The protocol being subscribed
54
+ def subscribe(endpoint, protocol)
55
+ self.class.subscribe(arn, endpoint, protocol, region: @config['region'], credentials: @credentials)
56
+ end
57
+
58
+ # Subscribe something to an SNS topic
59
+ # @param cloud_id [String]: The short name or ARN of an existing SNS topic
60
+ # @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
61
+ # @param protocol [String]: The protocol being subscribed
62
+ # @param region [String]: The region of the target SNS topic
63
+ # @param credentials [String]:
64
+ def self.subscribe(cloud_id, endpoint, protocol, region: nil, credentials: nil)
65
+ topic = find(cloud_id: cloud_id, region: region, credentials: credentials).values.first
66
+ if !topic
67
+ raise MuError, "Failed to find SNS Topic #{cloud_id} in #{region}"
68
+ end
69
+ arn = topic["TopicArn"]
70
+
71
+ resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
72
+
73
+ resp.each { |subscription|
74
+ return subscription if subscription.protocol == protocol and subscription.endpoint == endpoint
75
+ }
76
+
77
+ MU.log "Subscribing #{endpoint} (#{protocol}) to SNS topic #{arn}", MU::NOTICE
78
+ MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
79
+ end
80
+
50
81
  # Does this resource type exist as a global (cloud-wide) artifact, or
51
82
  # is it localized to a region/zone?
52
83
  # @return [Boolean]
@@ -65,12 +96,12 @@ module MU
65
96
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
66
97
  # @param region [String]: The cloud provider region
67
98
  # @return [void]
68
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
99
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
69
100
  MU.log "AWS::Notifier.cleanup: need to support flags['known']", MU::DEBUG, details: flags
70
101
  MU.log "Placeholder: AWS Notifier artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
71
102
 
72
103
  MU::Cloud::AWS.sns(region: region, credentials: credentials).list_topics.topics.each { |topic|
73
- if topic.topic_arn.match(MU.deploy_id)
104
+ if topic.topic_arn.match(deploy_id)
74
105
  # We don't have a way to tag our SNS topics, so we will delete any topic that has the MU-ID in its ARN.
75
106
  # This may fail to find notifier groups in some cases (eg. cache_cluster) so we might want to delete from each API as well.
76
107
  MU.log "Deleting SNS topic: #{topic.topic_arn}"
@@ -91,6 +122,7 @@ module MU
91
122
  # Return the metadata for this user cofiguration
92
123
  # @return [Hash]
93
124
  def notify
125
+ return nil if !@cloud_id or !cloud_desc(use_cache: false)
94
126
  desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes
95
127
  MU.structToHash(desc)
96
128
  end
@@ -101,9 +133,16 @@ module MU
101
133
  found = {}
102
134
 
103
135
  if args[:cloud_id]
104
- arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id]
105
- desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes
106
- found[args[:cloud_id]] = desc if desc
136
+ arn = if args[:cloud_id].match(/^arn:/)
137
+ args[:cloud_id]
138
+ else
139
+ "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id]
140
+ end
141
+ begin
142
+ desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes
143
+ found[args[:cloud_id]] = desc if desc
144
+ rescue ::Aws::SNS::Errors::NotFound
145
+ end
107
146
  else
108
147
  next_token = nil
109
148
  begin
@@ -120,6 +159,58 @@ module MU
120
159
  found
121
160
  end
122
161
 
162
+ # Reverse-map our cloud description into a runnable config hash.
163
+ # We assume that any values we have in +@config+ are placeholders, and
164
+ # calculate our own accordingly based on what's live in the cloud.
165
+ def toKitten(**_args)
166
+ bok = {
167
+ "cloud" => "AWS",
168
+ "credentials" => @config['credentials'],
169
+ "cloud_id" => @cloud_id,
170
+ "region" => @config['region']
171
+ }
172
+
173
+ if !cloud_desc
174
+ MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
175
+ return nil
176
+ end
177
+
178
+ bok['name'] = cloud_desc["DisplayName"].empty? ? @cloud_id : cloud_desc["DisplayName"]
179
+ svcmap = {
180
+ "lambda" => "functions",
181
+ "sqs" => "msg_queues"
182
+ }
183
+ MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub|
184
+ bok['subscriptions'] ||= []
185
+
186
+ bok['subscriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/)
187
+ _wholestring, service, region, account, id = Regexp.last_match.to_a
188
+ {
189
+ "type" => sub.protocol,
190
+ "resource" => MU::Config::Ref.get(
191
+ type: svcmap[service],
192
+ region: region,
193
+ credentials: @credentials,
194
+ id: id,
195
+ cloud: "AWS",
196
+ habitat: MU::Config::Ref.get(
197
+ id: account,
198
+ cloud: "AWS",
199
+ credentials: @credentials
200
+ )
201
+ )
202
+ }
203
+ else
204
+ {
205
+ "type" => sub.protocol,
206
+ "endpoint" => sub.endpoint
207
+ }
208
+ end
209
+ }
210
+
211
+ bok
212
+ end
213
+
123
214
  # Cloud-specific configuration properties.
124
215
  # @param _config [MU::Config]: The calling MU::Config object
125
216
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
@@ -130,11 +221,10 @@ module MU
130
221
  "type" => "array",
131
222
  "items" => {
132
223
  "type" => "object",
133
- "required" => ["endpoint"],
134
224
  "properties" => {
135
225
  "type" => {
136
226
  "type" => "string",
137
- "description" => "",
227
+ "description" => "Type of endpoint or resource which should receive notifications. If not specified, will attempt to auto-detect.",
138
228
  "enum" => ["http", "https", "email", "email-json", "sms", "sqs", "application", "lambda"]
139
229
  }
140
230
  }
@@ -148,26 +238,42 @@ module MU
148
238
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::notifier}, bare and unvalidated.
149
239
 
150
240
  # @param notifier [Hash]: The resource to process and validate
151
- # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
241
+ # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
152
242
  # @return [Boolean]: True if validation succeeded, False otherwise
153
- def self.validateConfig(notifier, _configurator)
243
+ def self.validateConfig(notifier, configurator)
154
244
  ok = true
155
245
 
156
246
  if notifier['subscriptions']
157
247
  notifier['subscriptions'].each { |sub|
248
+ if sub['resource'] and configurator.haveLitterMate?(sub['resource']['name'], sub['resource']['type'])
249
+ sub['resource']['cloud'] = "AWS"
250
+ MU::Config.addDependency(notifier, sub['resource']['name'], sub['resource']['type'])
251
+ end
158
252
  if !sub["type"]
159
- if sub["endpoint"].match(/^http:/i)
160
- sub["type"] = "http"
161
- elsif sub["endpoint"].match(/^https:/i)
162
- sub["type"] = "https"
163
- elsif sub["endpoint"].match(/^sqs:/i)
164
- sub["type"] = "sqs"
165
- elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
166
- sub["type"] = "sms"
167
- elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
168
- sub["type"] = "email"
169
- else
170
- MU.log "Notifier #{notifier['name']} subscription #{sub['endpoint']} did not specify a type, and I'm unable to guess one", MU::ERR
253
+ sub['type'] = if sub['resource']
254
+ if sub['resource']['type'] == "functions"
255
+ "lambda"
256
+ elsif sub['resource']['type'] == "msg_queues"
257
+ "sqs"
258
+ end
259
+ elsif sub['endpoint']
260
+ if sub["endpoint"].match(/^http:/i)
261
+ "http"
262
+ elsif sub["endpoint"].match(/^https:/i)
263
+ "https"
264
+ elsif sub["endpoint"].match(/:sqs:/i)
265
+ "sqs"
266
+ elsif sub["endpoint"].match(/:lambda:/i)
267
+ "lambda"
268
+ elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
269
+ "sms"
270
+ elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
271
+ "email"
272
+ end
273
+ end
274
+
275
+ if !sub['type']
276
+ MU.log "Notifier #{notifier['name']} subscription did not specify a type, and I'm unable to guess one", MU::ERR, details: sub
171
277
  ok = false
172
278
  end
173
279
  end
@@ -178,40 +284,6 @@ module MU
178
284
  end
179
285
 
180
286
 
181
- # Subscribe to a notifier group. This can either be an email address, SQS queue, application endpoint, etc...
182
- # Will create the subscription only if it doesn't already exist.
183
- # @param arn [String]: The cloud provider's identifier of the notifier group.
184
- # @param protocol [String]: The type of the subscription (eg. email,https, etc..).
185
- # @param endpoint [String]: The endpoint of the subscription. This will depend on the 'protocol' (as an example if protocol is email, endpoint will be the email address) ..
186
- # @param region [String]: The cloud provider region.
187
- def self.subscribe(arn: nil, protocol: nil, endpoint: nil, region: MU.curRegion, credentials: nil)
188
- retries = 0
189
- begin
190
- resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
191
- rescue Aws::SNS::Errors::NotFound
192
- if retries < 5
193
- MU.log "Couldn't find topic #{arn}, retrying several times in case of a lagging resource"
194
- retries += 1
195
- sleep 30
196
- retry
197
- else
198
- raise MuError, "Couldn't find topic #{arn}, giving up"
199
- end
200
- end
201
-
202
- already_subscribed = false
203
- if resp && !resp.empty?
204
- resp.each { |subscription|
205
- already_subscribed = true if subscription.protocol == protocol && subscription.endpoint == endpoint
206
- }
207
- end
208
-
209
- unless already_subscribed
210
- MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
211
- MU.log "Subscribed #{endpoint} to SNS topic #{arn}"
212
- end
213
- end
214
-
215
287
  # Test if a notifier group exists
216
288
  # Create a new notifier group. Will check if the group exists before creating it.
217
289
  # @param topic_name [String]: The cloud provider's name for the notifier group.
@@ -30,7 +30,7 @@ module MU
30
30
  end
31
31
  end
32
32
 
33
- @mu_name ||= @deploy.getResourceName(@config["name"])
33
+ @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 64)
34
34
  end
35
35
 
36
36
  # Called automatically by {MU::Deploy#createResources}
@@ -92,13 +92,14 @@ module MU
92
92
  configured_policies = []
93
93
 
94
94
  if @config['raw_policies']
95
+ MU.log "Attaching #{@config['raw_policies'].size.to_s} raw #{@config['raw_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
95
96
  configured_policies = @config['raw_policies'].map { |p|
96
97
  @mu_name+"-"+p.keys.first.upcase
97
98
  }
98
99
  end
99
100
 
100
101
  if @config['attachable_policies']
101
- MU.log "Attaching #{@config['attachable_policies'].size.to_s} #{@config['attachable_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
102
+ MU.log "Attaching #{@config['attachable_policies'].size.to_s} external #{@config['attachable_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
102
103
  configured_policies.concat(@config['attachable_policies'].map { |p|
103
104
  id = if p.is_a?(MU::Config::Ref)
104
105
  p.cloud_id
@@ -109,17 +110,16 @@ module MU
109
110
  end
110
111
  id.gsub(/.*?\/([^:\/]+)$/, '\1')
111
112
  })
112
- configured_policies.each { |pol|
113
- }
114
113
  end
115
114
 
115
+ # Purge anything that doesn't belong
116
116
  if !@config['bare_policies']
117
117
  attached_policies = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_role_policies(
118
118
  role_name: @mu_name
119
119
  ).attached_policies
120
120
  attached_policies.each { |a|
121
121
  if !configured_policies.include?(a.policy_name)
122
- MU.log "Removing IAM policy #{a.policy_name} from role #{@mu_name}", MU::NOTICE
122
+ MU.log "Removing IAM policy #{a.policy_name} from role #{@mu_name}", MU::NOTICE, details: configured_policies
123
123
  MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @config['credentials'])
124
124
  end
125
125
  }
@@ -153,6 +153,7 @@ module MU
153
153
  policy.values.each { |p|
154
154
  p["Version"] ||= "2012-10-17"
155
155
  }
156
+
156
157
  policy_name = basename+"-"+policy.keys.first.upcase
157
158
 
158
159
  arn = "arn:"+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+":iam::"+MU::Cloud::AWS.credToAcct(credentials)+":policy#{path}/#{policy_name}"
@@ -216,7 +217,22 @@ module MU
216
217
  # populated with one or both depending on what this resource has
217
218
  # defined.
218
219
  def cloud_desc(use_cache: true)
219
- return @cloud_desc_cache if @cloud_desc_cache and use_cache
220
+
221
+ # we might inherit a naive cached description from the base cloud
222
+ # layer; rearrange it to our tastes
223
+ if @cloud_desc_cache.is_a?(::Aws::IAM::Types::Role)
224
+ new_desc = {
225
+ "role" => @cloud_desc_cache
226
+ }
227
+ @cloud_desc_cache = new_desc
228
+ elsif @cloud_desc_cache.is_a?(::Aws::IAM::Types::Policy)
229
+ new_desc = {
230
+ "policies" => [@cloud_desc_cache]
231
+ }
232
+ @cloud_desc_cache = new_desc
233
+ end
234
+
235
+ return @cloud_desc_cache if @cloud_desc_cache and !@cloud_desc_cache.empty? and use_cache
220
236
 
221
237
  @cloud_desc_cache = {}
222
238
  if @config['bare_policies']
@@ -419,14 +435,14 @@ end
419
435
  # @param noop [Boolean]: If true, will only print what would be done
420
436
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
421
437
  # @return [void]
422
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
438
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
423
439
 
424
440
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
425
- path_prefix: "/"+MU.deploy_id+"/"
441
+ path_prefix: "/"+deploy_id+"/"
426
442
  )
427
443
  if resp and resp.policies
428
444
  resp.policies.each { |policy|
429
- MU.log "Deleting IAM policy /#{MU.deploy_id}/#{policy.policy_name}"
445
+ MU.log "Deleting IAM policy /#{deploy_id}/#{policy.policy_name}"
430
446
  if !noop
431
447
  purgePolicy(policy.arn, credentials)
432
448
  end
@@ -437,19 +453,23 @@ end
437
453
  roles = MU::Cloud::AWS::Role.find(credentials: credentials).values
438
454
  roles.each { |r|
439
455
  next if !r.respond_to?(:role_name)
440
- if r.path.match(/^\/#{Regexp.quote(MU.deploy_id)}/)
456
+ if r.path.match(/^\/#{Regexp.quote(deploy_id)}/)
441
457
  deleteme << r
442
458
  next
443
459
  end
444
460
  # For some dumb reason, the list output that .find gets doesn't
445
461
  # include the tags, so we need to fetch each role individually to
446
462
  # check tags. Hardly seems efficient.
447
- desc = MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
463
+ desc = begin
464
+ MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
465
+ rescue Aws::IAM::Errors::NoSuchEntity
466
+ next
467
+ end
448
468
  if desc.role and desc.role.tags and desc.role.tags
449
469
  master_match = false
450
470
  deploy_match = false
451
471
  desc.role.tags.each { |t|
452
- if t.key == "MU-ID" and t.value == MU.deploy_id
472
+ if t.key == "MU-ID" and t.value == deploy_id
453
473
  deploy_match = true
454
474
  elsif t.key == "MU-MASTER-IP" and t.value == MU.mu_public_ip
455
475
  master_match = true
@@ -516,7 +536,7 @@ end
516
536
 
517
537
  begin
518
538
  # managed policies get fetched by ARN, roles by plain name. Ok!
519
- if args[:cloud_id].match(/^arn:/)
539
+ if args[:cloud_id].match(/^arn:.*?:policy\//)
520
540
  resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_policy(
521
541
  policy_arn: args[:cloud_id]
522
542
  )
@@ -525,39 +545,26 @@ end
525
545
  end
526
546
  else
527
547
  resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_role(
528
- role_name: args[:cloud_id]
548
+ role_name: args[:cloud_id].sub(/^arn:.*?\/([^:\/]+)$/, '\1') # XXX if it's an ARN, actually parse it and look in the correct account when applicable
529
549
  )
550
+
530
551
  if resp and resp.role
531
- found[args[:cloud_id]] = resp.role
552
+ found[resp.role.role_name] = resp.role
532
553
  end
533
554
  end
534
555
  rescue ::Aws::IAM::Errors::NoSuchEntity
535
556
  end
536
557
 
537
558
  else
538
- marker = nil
539
- begin
540
- resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles(
541
- marker: marker
542
- )
543
- break if !resp or !resp.roles
544
- resp.roles.each { |role|
545
- found[role.role_name] = role
546
- }
547
- marker = resp.marker
548
- end while marker
559
+ resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles
560
+ resp.roles.each { |role|
561
+ found[role.role_name] = role
562
+ }
549
563
 
550
- begin
551
- resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(
552
- scope: "Local",
553
- marker: marker
554
- )
555
- break if !resp or !resp.policies
556
- resp.policies.each { |pol|
557
- found[pol.arn] = pol
558
- }
559
- marker = resp.marker
560
- end while marker
564
+ resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(scope: "Local")
565
+ resp.policies.each { |pol|
566
+ found[pol.arn] = pol
567
+ }
561
568
  end
562
569
 
563
570
  found
@@ -615,7 +622,6 @@ end
615
622
  )
616
623
  JSON.parse(URI.decode(version.policy_version.document))
617
624
  end
618
-
619
625
  bok["policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["policies"])
620
626
  end
621
627
  }
@@ -695,6 +701,7 @@ end
695
701
  end
696
702
 
697
703
  bok["attachable_policies"].uniq! if bok["attachable_policies"]
704
+ bok["name"].gsub!(/[^a-zA-Z0-9_\-]/, "_")
698
705
 
699
706
  bok
700
707
  end
@@ -707,6 +714,10 @@ end
707
714
  def self.doc2MuPolicies(basename, doc, policies = [])
708
715
  policies ||= []
709
716
 
717
+ if !doc["Statement"].is_a?(Array)
718
+ doc["Statement"] = [doc["Statement"]]
719
+ end
720
+
710
721
  doc["Statement"].each { |s|
711
722
  if !s["Action"]
712
723
  MU.log "Statement in policy document for #{basename} didn't have an Action field", MU::WARN, details: doc
@@ -804,6 +815,19 @@ end
804
815
  }
805
816
  end
806
817
 
818
+ if @config['raw_policies']
819
+ raw_arns = MU::Cloud::AWS::Role.manageRawPolicies(
820
+ @config['raw_policies'],
821
+ basename: @deploy.getResourceName(@config['name']),
822
+ credentials: @credentials
823
+ )
824
+ raw_arns.each { |p_arn|
825
+ mypolicies << MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy(
826
+ policy_arn: p_arn
827
+ ).policy
828
+ }
829
+ end
830
+
807
831
  mypolicies.each { |p|
808
832
  if entitytype == "user"
809
833
  resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_user_policies(
@@ -925,7 +949,7 @@ end
925
949
  toplevel_required = []
926
950
  aws_resource_types = MU::Cloud.resource_types.keys.reject { |t|
927
951
  begin
928
- MU::Cloud.loadCloudType("AWS", t)
952
+ MU::Cloud.resourceClass("AWS", t)
929
953
  false
930
954
  rescue MuCloudResourceNotImplemented
931
955
  true
@@ -1087,11 +1111,7 @@ end
1087
1111
  role['policies'].each { |policy|
1088
1112
  policy['targets'].each { |target|
1089
1113
  if target['type']
1090
- role['dependencies'] ||= []
1091
- role['dependencies'] << {
1092
- "name" => target['identifier'],
1093
- "type" => target['type']
1094
- }
1114
+ MU::Config.addDependency(role, target['identifier'], target['type'], no_create_wait: true)
1095
1115
  end
1096
1116
  }
1097
1117
  }
@@ -1107,13 +1127,14 @@ end
1107
1127
  # @param policies [Array<Hash>]: One or more policy chunks
1108
1128
  # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
1109
1129
  # @return [Array<Hash>]
1110
- def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false)
1130
+ def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil)
1111
1131
  if policies
1112
1132
  name = nil
1113
1133
  doc = {
1114
- "Version" => "2012-10-17",
1134
+ "Version" => version,
1115
1135
  "Statement" => []
1116
1136
  }
1137
+ doc["Id"] = doc_id if doc_id
1117
1138
  policies.each { |policy|
1118
1139
  policy["flag"] ||= "Allow"
1119
1140
  statement = {
@@ -1154,7 +1175,14 @@ end
1154
1175
  raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy"
1155
1176
  end
1156
1177
  else
1157
- bucket_prefix = grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/) ? "Service" : "AWS"
1178
+ bucket_prefix = if grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/)
1179
+ "Service"
1180
+ elsif grantee["identifier"] =~ /^[a-f0-9]+$/
1181
+ "CanonicalUser"
1182
+ else
1183
+ "AWS"
1184
+ end
1185
+
1158
1186
  if bucket_style
1159
1187
  statement["Principal"] << { bucket_prefix => grantee["identifier"] }
1160
1188
  else
@@ -1186,7 +1214,7 @@ end
1186
1214
  statement["Resource"] << id+"/*"
1187
1215
  end
1188
1216
  else
1189
- raise MuError, "Couldn't find a #{target["entity_type"]} named #{target["identifier"]} when generating IAM policy"
1217
+ raise MuError, "Couldn't find a #{target["type"]} named #{target["identifier"]} when generating IAM policy"
1190
1218
  end
1191
1219
  else
1192
1220
  target["identifier"] += target["path"] if target["path"]