cloud-mu 3.1.3 → 3.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +21 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +4 -4
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +147 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +158 -111
  45. data/modules/mu/adoption.rb +404 -71
  46. data/modules/mu/cleanup.rb +221 -306
  47. data/modules/mu/cloud.rb +129 -1633
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +44 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +926 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +169 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +32 -3
  61. data/modules/mu/config/cache_cluster.rb +2 -2
  62. data/modules/mu/config/cdn.rb +100 -0
  63. data/modules/mu/config/collection.rb +4 -4
  64. data/modules/mu/config/container_cluster.rb +9 -4
  65. data/modules/mu/config/database.rb +84 -105
  66. data/modules/mu/config/database.yml +1 -2
  67. data/modules/mu/config/dnszone.rb +10 -9
  68. data/modules/mu/config/doc_helpers.rb +516 -0
  69. data/modules/mu/config/endpoint.rb +5 -4
  70. data/modules/mu/config/firewall_rule.rb +103 -4
  71. data/modules/mu/config/folder.rb +4 -4
  72. data/modules/mu/config/function.rb +19 -10
  73. data/modules/mu/config/group.rb +4 -4
  74. data/modules/mu/config/habitat.rb +4 -4
  75. data/modules/mu/config/job.rb +89 -0
  76. data/modules/mu/config/loadbalancer.rb +60 -14
  77. data/modules/mu/config/log.rb +4 -4
  78. data/modules/mu/config/msg_queue.rb +4 -4
  79. data/modules/mu/config/nosqldb.rb +4 -4
  80. data/modules/mu/config/notifier.rb +10 -21
  81. data/modules/mu/config/ref.rb +411 -0
  82. data/modules/mu/config/role.rb +4 -4
  83. data/modules/mu/config/schema_helpers.rb +509 -0
  84. data/modules/mu/config/search_domain.rb +4 -4
  85. data/modules/mu/config/server.rb +98 -71
  86. data/modules/mu/config/server.yml +1 -0
  87. data/modules/mu/config/server_pool.rb +5 -9
  88. data/modules/mu/config/storage_pool.rb +1 -1
  89. data/modules/mu/config/tail.rb +200 -0
  90. data/modules/mu/config/user.rb +4 -4
  91. data/modules/mu/config/vpc.rb +71 -27
  92. data/modules/mu/config/vpc.yml +0 -1
  93. data/modules/mu/defaults/AWS.yaml +91 -68
  94. data/modules/mu/defaults/Azure.yaml +1 -0
  95. data/modules/mu/defaults/Google.yaml +3 -2
  96. data/modules/mu/deploy.rb +43 -26
  97. data/modules/mu/groomer.rb +17 -2
  98. data/modules/mu/groomers/ansible.rb +188 -41
  99. data/modules/mu/groomers/chef.rb +116 -55
  100. data/modules/mu/logger.rb +127 -148
  101. data/modules/mu/master.rb +410 -2
  102. data/modules/mu/master/chef.rb +3 -4
  103. data/modules/mu/master/ldap.rb +3 -3
  104. data/modules/mu/master/ssl.rb +12 -3
  105. data/modules/mu/mommacat.rb +218 -2612
  106. data/modules/mu/mommacat/daemon.rb +403 -0
  107. data/modules/mu/mommacat/naming.rb +473 -0
  108. data/modules/mu/mommacat/search.rb +495 -0
  109. data/modules/mu/mommacat/storage.rb +722 -0
  110. data/modules/mu/{clouds → providers}/README.md +1 -1
  111. data/modules/mu/{clouds → providers}/aws.rb +380 -122
  112. data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
  113. data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
  114. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
  115. data/modules/mu/providers/aws/cdn.rb +782 -0
  116. data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
  117. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
  118. data/modules/mu/providers/aws/database.rb +1744 -0
  119. data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
  120. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  121. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
  122. data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
  123. data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
  124. data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
  125. data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
  126. data/modules/mu/providers/aws/job.rb +466 -0
  127. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
  128. data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
  129. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
  130. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
  131. data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
  132. data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
  133. data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
  134. data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
  135. data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
  136. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
  137. data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
  138. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  139. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  140. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  141. data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
  142. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  143. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  144. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  145. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  146. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  147. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  148. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  149. data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
  150. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  151. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  152. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  153. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  154. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  155. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  156. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  160. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  161. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  162. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  163. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  164. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  165. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  166. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  167. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  168. data/modules/mu/{clouds → providers}/google.rb +68 -30
  169. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  170. data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
  171. data/modules/mu/{clouds → providers}/google/database.rb +11 -21
  172. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  173. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  174. data/modules/mu/{clouds → providers}/google/function.rb +140 -168
  175. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  176. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  177. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
  178. data/modules/mu/{clouds → providers}/google/role.rb +94 -58
  179. data/modules/mu/{clouds → providers}/google/server.rb +243 -156
  180. data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
  181. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  182. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  183. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  184. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  185. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  186. data/modules/tests/aws-jobs-functions.yaml +46 -0
  187. data/modules/tests/bucket.yml +4 -0
  188. data/modules/tests/centos6.yaml +15 -0
  189. data/modules/tests/centos7.yaml +15 -0
  190. data/modules/tests/centos8.yaml +12 -0
  191. data/modules/tests/ecs.yaml +23 -0
  192. data/modules/tests/eks.yaml +1 -1
  193. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  194. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  195. data/modules/tests/includes-and-params.yaml +2 -1
  196. data/modules/tests/microservice_app.yaml +288 -0
  197. data/modules/tests/rds.yaml +108 -0
  198. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  199. data/modules/tests/regrooms/bucket.yml +19 -0
  200. data/modules/tests/regrooms/rds.yaml +123 -0
  201. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  202. data/modules/tests/super_complex_bok.yml +2 -2
  203. data/modules/tests/super_simple_bok.yml +3 -5
  204. data/modules/tests/win2k12.yaml +17 -5
  205. data/modules/tests/win2k16.yaml +25 -0
  206. data/modules/tests/win2k19.yaml +25 -0
  207. data/requirements.txt +1 -0
  208. data/spec/mu/clouds/azure_spec.rb +2 -2
  209. metadata +240 -154
  210. data/extras/image-generators/AWS/windows.yaml +0 -18
  211. data/modules/mu/clouds/aws/database.rb +0 -1985
  212. data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -30,7 +30,7 @@ module MU
30
30
  end
31
31
  end
32
32
 
33
- @mu_name ||= @deploy.getResourceName(@config["name"])
33
+ @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 64)
34
34
  end
35
35
 
36
36
  # Called automatically by {MU::Deploy#createResources}
@@ -43,7 +43,7 @@ module MU
43
43
 
44
44
  policy_name = @mu_name+"-"+policy.keys.first.upcase
45
45
  MU.log "Creating IAM policy #{policy_name}"
46
- resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).create_policy(
46
+ MU::Cloud::AWS.iam(credentials: @config['credentials']).create_policy(
47
47
  policy_name: policy_name,
48
48
  path: "/"+@deploy.deploy_id+"/",
49
49
  policy_document: JSON.generate(policy.values.first),
@@ -56,8 +56,8 @@ module MU
56
56
  MU.log "Creating IAM role #{@mu_name}"
57
57
  @cloud_id = @mu_name
58
58
  path = @config['strip_path'] ? nil : "/"+@deploy.deploy_id+"/"
59
- resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).create_role(
60
- path: nil,
59
+ MU::Cloud::AWS.iam(credentials: @config['credentials']).create_role(
60
+ path: path,
61
61
  role_name: @mu_name,
62
62
  description: "Generated by Mu",
63
63
  assume_role_policy_document: gen_assume_role_policy_doc,
@@ -89,7 +89,6 @@ module MU
89
89
  end
90
90
 
91
91
  if @config['raw_policies'] or @config['attachable_policies']
92
- attached_policies = []
93
92
  configured_policies = []
94
93
 
95
94
  if @config['raw_policies']
@@ -128,7 +127,7 @@ module MU
128
127
 
129
128
  # XXX not sure we're binding these sanely, validate that
130
129
  if @config['raw_policies']
131
- pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
130
+ MU::Cloud::AWS::Role.manageRawPolicies(
132
131
  @config['raw_policies'],
133
132
  basename: @deploy.getResourceName(@config['name']),
134
133
  credentials: @credentials
@@ -183,7 +182,7 @@ module MU
183
182
  desc
184
183
  end
185
184
 
186
- rescue Aws::IAM::Errors::NoSuchEntity => e
185
+ rescue Aws::IAM::Errors::NoSuchEntity
187
186
  MU.log "Creating IAM policy #{policy_name}", details: policy.values.first
188
187
  MU::Cloud::AWS.iam(credentials: credentials).create_policy(
189
188
  policy_name: policy_name,
@@ -202,65 +201,87 @@ module MU
202
201
  def arn
203
202
  desc = cloud_desc
204
203
  if desc["role"]
205
- desc["role"].arn
204
+ if desc['role'].is_a?(Hash)
205
+ desc["role"][:arn] # why though
206
+ else
207
+ desc["role"].arn
208
+ end
206
209
  else
207
210
  nil
208
211
  end
209
212
  end
210
213
 
214
+ @cloud_desc_cache = nil
211
215
  # Return a hash containing a +role+ element and a +policies+ element,
212
216
  # populated with one or both depending on what this resource has
213
217
  # defined.
214
- def cloud_desc
215
- desc = {}
218
+ def cloud_desc(use_cache: true)
219
+
220
+ # we might inherit a naive cached description from the base cloud
221
+ # layer; rearrange it to our tastes
222
+ if @cloud_desc_cache.is_a?(::Aws::IAM::Types::Role)
223
+ new_desc = {
224
+ "role" => @cloud_desc_cache
225
+ }
226
+ @cloud_desc_cache = new_desc
227
+ elsif @cloud_desc_cache.is_a?(::Aws::IAM::Types::Policy)
228
+ new_desc = {
229
+ "policies" => [@cloud_desc_cache]
230
+ }
231
+ @cloud_desc_cache = new_desc
232
+ end
233
+
234
+ return @cloud_desc_cache if @cloud_desc_cache and !@cloud_desc_cache.empty? and use_cache
235
+
236
+ @cloud_desc_cache = {}
216
237
  if @config['bare_policies']
217
238
  if @cloud_id
218
239
  pol_desc = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
219
240
  if pol_desc
220
- desc['policies'] = [pol_desc]
221
- return desc
241
+ @cloud_desc_cache['policies'] = [pol_desc]
242
+ return @cloud_desc_cache
222
243
  end
223
244
  end
224
245
 
225
246
  if @deploy and @deploy.deploy_id
226
- desc["policies"] = MU::Cloud::AWS.iam(credentials: @credentials).list_policies(
247
+ @cloud_desc_cache["policies"] = MU::Cloud::AWS.iam(credentials: @credentials).list_policies(
227
248
  path_prefix: "/"+@deploy.deploy_id+"/"
228
249
  ).policies
229
- desc["policies"].reject! { |p|
250
+ @cloud_desc_cache["policies"].reject! { |p|
230
251
  !p.policy_name.match(/^#{Regexp.quote(@mu_name)}-/)
231
252
  }
232
253
  # this is quasi-wrong because we can be mulitple cloud is, but
233
254
  # we can't really set this type to has_multiples because that's
234
255
  # just how managed policies work not anything else, goddammit
235
256
  # AWS why can't you just bundle everything in roles
236
- if desc["policies"] and desc["policies"].size > 0
237
- @cloud_id ||= desc["policies"].first.arn
257
+ if @cloud_desc_cache["policies"] and @cloud_desc_cache["policies"].size > 0
258
+ @cloud_id ||= @cloud_desc_cache["policies"].first.arn
238
259
  end
239
260
  end
240
261
  else
241
262
  if @cloud_id.match(/^arn:aws(:?-us-gov)?:[^:]*:[^:]*:\d*:policy\//)
242
263
  pol_desc = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
243
264
  if pol_desc
244
- desc['policies'] = [pol_desc]
245
- return desc
265
+ @cloud_desc_cache['policies'] = [pol_desc]
266
+ return @cloud_desc_cache
246
267
  end
247
268
  end
248
269
  begin
249
- desc['role'] = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
250
- desc['role'] ||= MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @mu_name).values.first
270
+ @cloud_desc_cache['role'] = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
271
+ @cloud_desc_cache['role'] ||= MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @mu_name).values.first
251
272
  MU::Cloud::AWS.iam(credentials: @credentials).list_attached_role_policies(
252
273
  role_name: @mu_name
253
274
  ).attached_policies.each { |p|
254
- desc["policies"] ||= []
255
- desc["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
275
+ @cloud_desc_cache["policies"] ||= []
276
+ @cloud_desc_cache["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
256
277
  policy_arn: p.policy_arn
257
278
  ).policy
258
279
  }
259
280
 
260
281
  inline = MU::Cloud::AWS.iam(credentials: @credentials).list_role_policies(role_name: @mu_name).policy_names
261
282
  inline.each { |pol_name|
262
- desc["policies"] ||= []
263
- desc["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_role_policy(
283
+ @cloud_desc_cache["policies"] ||= []
284
+ @cloud_desc_cache["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_role_policy(
264
285
  role_name: @mu_name,
265
286
  policy_name: pol_name
266
287
  )
@@ -270,9 +291,9 @@ rescue ::Aws::IAM::Errors::ValidationError => e
270
291
  MU.log @cloud_id+" "+@mu_name, MU::WARN, details: e.inspect
271
292
  end
272
293
  end
273
- desc['cloud_id'] ||= @cloud_id
294
+ @cloud_desc_cache['cloud_id'] ||= @cloud_id
274
295
 
275
- desc
296
+ @cloud_desc_cache
276
297
  end
277
298
 
278
299
  # Return the metadata for this user cofiguration
@@ -284,26 +305,25 @@ end
284
305
  # Insert a new target entity into an existing policy.
285
306
  # @param policy [String]: The name of the policy to which we're appending, which must already exist as part of this role resource
286
307
  # @param targets [Array<String>]: The target resource. If +target_type+ isn't specified, this should be a fully-resolved ARN.
287
- # @param mu_type [String]: A valid Mu resource type
288
- def injectPolicyTargets(policy, targets, mu_type = nil)
308
+ def injectPolicyTargets(policy, targets)
289
309
  if !policy.match(/^#{@deploy.deploy_id}/)
290
310
  policy = @mu_name+"-"+policy.upcase
291
311
  end
292
-
293
- my_policies = cloud_desc["policies"]
312
+ my_policies = cloud_desc(use_cache: false)["policies"]
294
313
  my_policies ||= []
295
-
314
+
315
+ seen_policy = false
296
316
  my_policies.each { |p|
297
317
  if p.policy_name == policy
318
+ seen_policy = true
298
319
  old = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy_version(
299
320
  policy_arn: p.arn,
300
321
  version_id: p.default_version_id
301
322
  ).policy_version
302
323
 
303
324
  doc = JSON.parse URI.decode_www_form_component old.document
304
-
305
325
  need_update = false
306
-
326
+
307
327
  doc["Statement"].each { |s|
308
328
  targets.each { |target|
309
329
  target_string = target
@@ -332,6 +352,10 @@ end
332
352
  end
333
353
  end
334
354
  }
355
+
356
+ if !seen_policy
357
+ MU.log "Was given new targets for policy #{policy}, but I don't see any such policy attached to role #{@cloud_id}", MU::WARN, details: targets
358
+ end
335
359
  end
336
360
 
337
361
  # Delete an IAM policy, along with attendant versions and attachments.
@@ -409,16 +433,15 @@ end
409
433
  # Remove all roles associated with the currently loaded deployment.
410
434
  # @param noop [Boolean]: If true, will only print what would be done
411
435
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
412
- # @param region [String]: The cloud provider region
413
436
  # @return [void]
414
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
437
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
415
438
 
416
439
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
417
- path_prefix: "/"+MU.deploy_id+"/"
440
+ path_prefix: "/"+deploy_id+"/"
418
441
  )
419
442
  if resp and resp.policies
420
443
  resp.policies.each { |policy|
421
- MU.log "Deleting IAM policy /#{MU.deploy_id}/#{policy.policy_name}"
444
+ MU.log "Deleting IAM policy /#{deploy_id}/#{policy.policy_name}"
422
445
  if !noop
423
446
  purgePolicy(policy.arn, credentials)
424
447
  end
@@ -429,21 +452,31 @@ end
429
452
  roles = MU::Cloud::AWS::Role.find(credentials: credentials).values
430
453
  roles.each { |r|
431
454
  next if !r.respond_to?(:role_name)
432
- if r.path.match(/^\/#{Regexp.quote(MU.deploy_id)}/)
455
+ if r.path.match(/^\/#{Regexp.quote(deploy_id)}/)
433
456
  deleteme << r
434
457
  next
435
458
  end
436
459
  # For some dumb reason, the list output that .find gets doesn't
437
460
  # include the tags, so we need to fetch each role individually to
438
461
  # check tags. Hardly seems efficient.
439
- desc = MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
462
+ desc = begin
463
+ MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
464
+ rescue Aws::IAM::Errors::NoSuchEntity
465
+ next
466
+ end
440
467
  if desc.role and desc.role.tags and desc.role.tags
468
+ master_match = false
469
+ deploy_match = false
441
470
  desc.role.tags.each { |t|
442
- if t.key == "MU-ID" and t.value == MU.deploy_id
443
- deleteme << r
444
- break
471
+ if t.key == "MU-ID" and t.value == deploy_id
472
+ deploy_match = true
473
+ elsif t.key == "MU-MASTER-IP" and t.value == MU.mu_public_ip
474
+ master_match = true
445
475
  end
446
476
  }
477
+ if deploy_match and (master_match or ignoremaster)
478
+ deleteme << r
479
+ end
447
480
  end
448
481
  }
449
482
 
@@ -481,7 +514,7 @@ end
481
514
  MU::Cloud::AWS.iam(credentials: credentials).delete_instance_profile(instance_profile_name: r.role_name)
482
515
  rescue Aws::IAM::Errors::ValidationError => e
483
516
  MU.log "Cleaning up IAM role #{r.role_name}: #{e.inspect}", MU::WARN
484
- rescue Aws::IAM::Errors::NoSuchEntity => e
517
+ rescue Aws::IAM::Errors::NoSuchEntity
485
518
  end
486
519
 
487
520
  MU::Cloud::AWS.iam(credentials: credentials).delete_role(
@@ -502,7 +535,7 @@ end
502
535
 
503
536
  begin
504
537
  # managed policies get fetched by ARN, roles by plain name. Ok!
505
- if args[:cloud_id].match(/^arn:/)
538
+ if args[:cloud_id].match(/^arn:.*?:policy\//)
506
539
  resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_policy(
507
540
  policy_arn: args[:cloud_id]
508
541
  )
@@ -511,39 +544,26 @@ end
511
544
  end
512
545
  else
513
546
  resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_role(
514
- role_name: args[:cloud_id]
547
+ role_name: args[:cloud_id].sub(/^arn:.*?\/([^:\/]+)$/, '\1') # XXX if it's an ARN, actually parse it and look in the correct account when applicable
515
548
  )
549
+
516
550
  if resp and resp.role
517
- found[args[:cloud_id]] = resp.role
551
+ found[resp.role.role_name] = resp.role
518
552
  end
519
553
  end
520
554
  rescue ::Aws::IAM::Errors::NoSuchEntity
521
555
  end
522
-
556
+
523
557
  else
524
- marker = nil
525
- begin
526
- resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles(
527
- marker: marker
528
- )
529
- break if !resp or !resp.roles
530
- resp.roles.each { |role|
531
- found[role.role_name] = role
532
- }
533
- marker = resp.marker
534
- end while marker
558
+ resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles
559
+ resp.roles.each { |role|
560
+ found[role.role_name] = role
561
+ }
535
562
 
536
- begin
537
- resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(
538
- scope: "Local",
539
- marker: marker
540
- )
541
- break if !resp or !resp.policies
542
- resp.policies.each { |pol|
543
- found[pol.arn] = pol
544
- }
545
- marker = resp.marker
546
- end while marker
563
+ resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(scope: "Local")
564
+ resp.policies.each { |pol|
565
+ found[pol.arn] = pol
566
+ }
547
567
  end
548
568
 
549
569
  found
@@ -552,7 +572,7 @@ end
552
572
  # Reverse-map our cloud description into a runnable config hash.
553
573
  # We assume that any values we have in +@config+ are placeholders, and
554
574
  # calculate our own accordingly based on what's live in the cloud.
555
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
575
+ def toKitten(**_args)
556
576
  bok = {
557
577
  "cloud" => "AWS",
558
578
  "credentials" => @config['credentials'],
@@ -601,14 +621,13 @@ end
601
621
  )
602
622
  JSON.parse(URI.decode(version.policy_version.document))
603
623
  end
604
-
605
624
  bok["policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["policies"])
606
625
  end
607
626
  }
608
627
 
609
628
  return bok if @config['bare_policies']
610
629
  end
611
-
630
+
612
631
  if desc.tags and desc.tags.size > 0
613
632
  bok["tags"] = MU.structToHash(desc.tags, stringify_keys: true)
614
633
  end
@@ -681,6 +700,7 @@ end
681
700
  end
682
701
 
683
702
  bok["attachable_policies"].uniq! if bok["attachable_policies"]
703
+ bok["name"].gsub!(/[^a-zA-Z0-9_\-]/, "_")
684
704
 
685
705
  bok
686
706
  end
@@ -693,6 +713,10 @@ end
693
713
  def self.doc2MuPolicies(basename, doc, policies = [])
694
714
  policies ||= []
695
715
 
716
+ if !doc["Statement"].is_a?(Array)
717
+ doc["Statement"] = [doc["Statement"]]
718
+ end
719
+
696
720
  doc["Statement"].each { |s|
697
721
  if !s["Action"]
698
722
  MU.log "Statement in policy document for #{basename} didn't have an Action field", MU::WARN, details: doc
@@ -721,7 +745,7 @@ end
721
745
  "targets" => s["Resource"].map { |r|
722
746
  if r.match(/^arn:aws(-us-gov)?:([^:]+):.*?:([^:]*)$/)
723
747
  # XXX which cases even count for blind references to sibling resources?
724
- type = if Regexp.last_match[1] == "s3"
748
+ if Regexp.last_match[1] == "s3"
725
749
  "bucket"
726
750
  elsif Regexp.last_match[1]
727
751
  MU.log "Service #{Regexp.last_match[1]} to type...", MU::WARN, details: r
@@ -741,7 +765,7 @@ end
741
765
 
742
766
  # Attach this role or group of loose policies to the specified entity.
743
767
  # @param entitytype [String]: The type of entity (user, group or role for policies; instance_profile for roles)
744
- def bindTo(entitytype, entityname, policies = [])
768
+ def bindTo(entitytype, entityname)
745
769
  if entitytype == "instance_profile"
746
770
  begin
747
771
  resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
@@ -754,7 +778,7 @@ end
754
778
  role_name: @mu_name
755
779
  )
756
780
  end
757
- rescue Exception => e
781
+ rescue StandardError => e
758
782
  MU.log "Error binding role #{@mu_name} to instance profile #{entityname}: #{e.message}", MU::ERR
759
783
  raise e
760
784
  end
@@ -783,7 +807,6 @@ end
783
807
  rescue Aws::IAM::Errors::NoSuchEntity => e
784
808
  if subpaths.size > 0
785
809
  p_arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":iam::aws:policy/#{subpaths.shift}/"+policy
786
- retried = true
787
810
  retry
788
811
  end
789
812
  raise e
@@ -833,6 +856,7 @@ end
833
856
  else
834
857
  raise MuError, "Invalid entitytype '#{entitytype}' passed to MU::Cloud::AWS::Role.bindTo. Must be be one of: user, group, role, instance_profile"
835
858
  end
859
+ cloud_desc(use_cache: false)
836
860
  end
837
861
 
838
862
  # Create an instance profile for EC2 instances, named identically and
@@ -847,7 +871,7 @@ end
847
871
  MU::Cloud::AWS.iam(credentials: @config['credentials']).create_instance_profile(
848
872
  instance_profile_name: @mu_name
849
873
  )
850
- rescue Aws::IAM::Errors::EntityAlreadyExists => e
874
+ rescue Aws::IAM::Errors::EntityAlreadyExists
851
875
  MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
852
876
  instance_profile_name: @mu_name
853
877
  )
@@ -905,13 +929,13 @@ end
905
929
  end
906
930
 
907
931
  # Cloud-specific configuration properties.
908
- # @param config [MU::Config]: The calling MU::Config object
932
+ # @param _config [MU::Config]: The calling MU::Config object
909
933
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
910
- def self.schema(config)
934
+ def self.schema(_config)
911
935
  toplevel_required = []
912
936
  aws_resource_types = MU::Cloud.resource_types.keys.reject { |t|
913
937
  begin
914
- MU::Cloud.loadCloudType("AWS", t)
938
+ MU::Cloud.resourceClass("AWS", t)
915
939
  false
916
940
  rescue MuCloudResourceNotImplemented
917
941
  true
@@ -1012,10 +1036,9 @@ end
1012
1036
  subpaths = ["service-role", "aws-service-role", "job-function"]
1013
1037
  begin
1014
1038
  MU::Cloud::AWS.iam(credentials: credentials).get_policy(policy_arn: arn)
1015
- rescue Aws::IAM::Errors::NoSuchEntity => e
1039
+ rescue Aws::IAM::Errors::NoSuchEntity
1016
1040
  if subpaths.size > 0
1017
1041
  arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(region) ? "aws-us-gov" : "aws")+":iam::aws:policy/#{subpaths.shift}/"+ref["id"]
1018
- retried = true
1019
1042
  retry
1020
1043
  end
1021
1044
  MU.log "No such canned AWS IAM policy '#{arn}'", MU::ERR
@@ -1029,9 +1052,9 @@ end
1029
1052
 
1030
1053
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::roles}, bare and unvalidated.
1031
1054
  # @param role [Hash]: The resource to process and validate
1032
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
1055
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
1033
1056
  # @return [Boolean]: True if validation succeeded, False otherwise
1034
- def self.validateConfig(role, configurator)
1057
+ def self.validateConfig(role, _configurator)
1035
1058
  ok = true
1036
1059
 
1037
1060
  # munge things declared with the deprecated import keyword into
@@ -1074,11 +1097,7 @@ end
1074
1097
  role['policies'].each { |policy|
1075
1098
  policy['targets'].each { |target|
1076
1099
  if target['type']
1077
- role['dependencies'] ||= []
1078
- role['dependencies'] << {
1079
- "name" => target['identifier'],
1080
- "type" => target['type']
1081
- }
1100
+ MU::Config.addDependency(role, target['identifier'], target['type'], no_create_wait: true)
1082
1101
  end
1083
1102
  }
1084
1103
  }
@@ -1094,15 +1113,14 @@ end
1094
1113
  # @param policies [Array<Hash>]: One or more policy chunks
1095
1114
  # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
1096
1115
  # @return [Array<Hash>]
1097
- def self.genPolicyDocument(policies, deploy_obj: nil)
1098
- iam_policies = []
1099
-
1116
+ def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil)
1100
1117
  if policies
1101
1118
  name = nil
1102
1119
  doc = {
1103
- "Version" => "2012-10-17",
1120
+ "Version" => version,
1104
1121
  "Statement" => []
1105
1122
  }
1123
+ doc["Id"] = doc_id if doc_id
1106
1124
  policies.each { |policy|
1107
1125
  policy["flag"] ||= "Allow"
1108
1126
  statement = {
@@ -1134,12 +1152,28 @@ end
1134
1152
  )
1135
1153
  if sibling
1136
1154
  id = sibling.cloudobj.arn
1137
- statement["Principal"] << id
1155
+ if bucket_style
1156
+ statement["Principal"] << { "AWS" => id }
1157
+ else
1158
+ statement["Principal"] << id
1159
+ end
1138
1160
  else
1139
1161
  raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy"
1140
1162
  end
1141
1163
  else
1142
- statement["Principal"] << grantee["identifier"]
1164
+ bucket_prefix = if grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/)
1165
+ "Service"
1166
+ elsif grantee["identifier"] =~ /^[a-f0-9]+$/
1167
+ "CanonicalUser"
1168
+ else
1169
+ "AWS"
1170
+ end
1171
+
1172
+ if bucket_style
1173
+ statement["Principal"] << { bucket_prefix => grantee["identifier"] }
1174
+ else
1175
+ statement["Principal"] << grantee["identifier"]
1176
+ end
1143
1177
  end
1144
1178
  }
1145
1179
  if policy["grant_to"].size == 1
@@ -1162,9 +1196,11 @@ end
1162
1196
  stream_id = id.sub(/:([^:]+)$/, ":log-stream:*")
1163
1197
  # "arn:aws:logs:us-east-2:accountID:log-group:log_group_name:log-stream:CloudTrail_log_stream_name_prefix*"
1164
1198
  statement["Resource"] << stream_id
1199
+ elsif id.match(/:s3:/)
1200
+ statement["Resource"] << id+"/*"
1165
1201
  end
1166
1202
  else
1167
- raise MuError, "Couldn't find a #{target["entity_type"]} named #{target["identifier"]} when generating IAM policy"
1203
+ raise MuError, "Couldn't find a #{target["type"]} named #{target["identifier"]} when generating IAM policy"
1168
1204
  end
1169
1205
  else
1170
1206
  target["identifier"] += target["path"] if target["path"]
@@ -1180,6 +1216,32 @@ end
1180
1216
  []
1181
1217
  end
1182
1218
 
1219
+ # Update a policy, handling deletion of old versions as needed
1220
+ # @param arn [String]:
1221
+ # @param doc [Hash]:
1222
+ # @param credentials [String]:
1223
+ def self.update_policy(arn, doc, credentials: nil)
1224
+ # XXX this is just blindly replacing identical versions, when it should check
1225
+ # and guard
1226
+ begin
1227
+ MU::Cloud::AWS.iam(credentials: credentials).create_policy_version(
1228
+ policy_arn: arn,
1229
+ set_as_default: true,
1230
+ policy_document: JSON.generate(doc)
1231
+ )
1232
+ rescue Aws::IAM::Errors::LimitExceeded
1233
+ delete_version = MU::Cloud::AWS.iam(credentials: credentials).list_policy_versions(
1234
+ policy_arn: arn,
1235
+ ).versions.last.version_id
1236
+ MU.log "Purging oldest version (#{delete_version}) of IAM policy #{arn}", MU::NOTICE
1237
+ MU::Cloud::AWS.iam(credentials: credentials).delete_policy_version(
1238
+ policy_arn: arn,
1239
+ version_id: delete_version
1240
+ )
1241
+ retry
1242
+ end
1243
+ end
1244
+
1183
1245
  private
1184
1246
 
1185
1247
  # Convert entries from the cloud-neutral @config['policies'] list into
@@ -1261,28 +1323,6 @@ end
1261
1323
  MU::Cloud::AWS::Role.update_policy(arn, doc, credentials: @credentials)
1262
1324
  end
1263
1325
 
1264
- # Update a policy, handling deletion of old versions as needed
1265
- def self.update_policy(arn, doc, credentials: nil)
1266
- # XXX this is just blindly replacing identical versions, when it should check
1267
- # and guard
1268
- begin
1269
- MU::Cloud::AWS.iam(credentials: credentials).create_policy_version(
1270
- policy_arn: arn,
1271
- set_as_default: true,
1272
- policy_document: JSON.generate(doc)
1273
- )
1274
- rescue Aws::IAM::Errors::LimitExceeded => e
1275
- delete_version = MU::Cloud::AWS.iam(credentials: credentials).list_policy_versions(
1276
- policy_arn: arn,
1277
- ).versions.last.version_id
1278
- MU.log "Purging oldest version (#{delete_version}) of IAM policy #{arn}", MU::NOTICE
1279
- MU::Cloud::AWS.iam(credentials: credentials).delete_policy_version(
1280
- policy_arn: arn,
1281
- version_id: delete_version
1282
- )
1283
- retry
1284
- end
1285
- end
1286
1326
 
1287
1327
  end
1288
1328
  end