cloud-mu 3.1.3 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +21 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +4 -4
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +147 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +158 -111
  45. data/modules/mu/adoption.rb +404 -71
  46. data/modules/mu/cleanup.rb +221 -306
  47. data/modules/mu/cloud.rb +129 -1633
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +44 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +926 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +169 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +32 -3
  61. data/modules/mu/config/cache_cluster.rb +2 -2
  62. data/modules/mu/config/cdn.rb +100 -0
  63. data/modules/mu/config/collection.rb +4 -4
  64. data/modules/mu/config/container_cluster.rb +9 -4
  65. data/modules/mu/config/database.rb +84 -105
  66. data/modules/mu/config/database.yml +1 -2
  67. data/modules/mu/config/dnszone.rb +10 -9
  68. data/modules/mu/config/doc_helpers.rb +516 -0
  69. data/modules/mu/config/endpoint.rb +5 -4
  70. data/modules/mu/config/firewall_rule.rb +103 -4
  71. data/modules/mu/config/folder.rb +4 -4
  72. data/modules/mu/config/function.rb +19 -10
  73. data/modules/mu/config/group.rb +4 -4
  74. data/modules/mu/config/habitat.rb +4 -4
  75. data/modules/mu/config/job.rb +89 -0
  76. data/modules/mu/config/loadbalancer.rb +60 -14
  77. data/modules/mu/config/log.rb +4 -4
  78. data/modules/mu/config/msg_queue.rb +4 -4
  79. data/modules/mu/config/nosqldb.rb +4 -4
  80. data/modules/mu/config/notifier.rb +10 -21
  81. data/modules/mu/config/ref.rb +411 -0
  82. data/modules/mu/config/role.rb +4 -4
  83. data/modules/mu/config/schema_helpers.rb +509 -0
  84. data/modules/mu/config/search_domain.rb +4 -4
  85. data/modules/mu/config/server.rb +98 -71
  86. data/modules/mu/config/server.yml +1 -0
  87. data/modules/mu/config/server_pool.rb +5 -9
  88. data/modules/mu/config/storage_pool.rb +1 -1
  89. data/modules/mu/config/tail.rb +200 -0
  90. data/modules/mu/config/user.rb +4 -4
  91. data/modules/mu/config/vpc.rb +71 -27
  92. data/modules/mu/config/vpc.yml +0 -1
  93. data/modules/mu/defaults/AWS.yaml +91 -68
  94. data/modules/mu/defaults/Azure.yaml +1 -0
  95. data/modules/mu/defaults/Google.yaml +3 -2
  96. data/modules/mu/deploy.rb +43 -26
  97. data/modules/mu/groomer.rb +17 -2
  98. data/modules/mu/groomers/ansible.rb +188 -41
  99. data/modules/mu/groomers/chef.rb +116 -55
  100. data/modules/mu/logger.rb +127 -148
  101. data/modules/mu/master.rb +410 -2
  102. data/modules/mu/master/chef.rb +3 -4
  103. data/modules/mu/master/ldap.rb +3 -3
  104. data/modules/mu/master/ssl.rb +12 -3
  105. data/modules/mu/mommacat.rb +218 -2612
  106. data/modules/mu/mommacat/daemon.rb +403 -0
  107. data/modules/mu/mommacat/naming.rb +473 -0
  108. data/modules/mu/mommacat/search.rb +495 -0
  109. data/modules/mu/mommacat/storage.rb +722 -0
  110. data/modules/mu/{clouds → providers}/README.md +1 -1
  111. data/modules/mu/{clouds → providers}/aws.rb +380 -122
  112. data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
  113. data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
  114. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
  115. data/modules/mu/providers/aws/cdn.rb +782 -0
  116. data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
  117. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
  118. data/modules/mu/providers/aws/database.rb +1744 -0
  119. data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
  120. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  121. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
  122. data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
  123. data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
  124. data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
  125. data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
  126. data/modules/mu/providers/aws/job.rb +466 -0
  127. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
  128. data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
  129. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
  130. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
  131. data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
  132. data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
  133. data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
  134. data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
  135. data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
  136. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
  137. data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
  138. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  139. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  140. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  141. data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
  142. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  143. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  144. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  145. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  146. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  147. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  148. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  149. data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
  150. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  151. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  152. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  153. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  154. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  155. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  156. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  160. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  161. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  162. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  163. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  164. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  165. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  166. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  167. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  168. data/modules/mu/{clouds → providers}/google.rb +68 -30
  169. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  170. data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
  171. data/modules/mu/{clouds → providers}/google/database.rb +11 -21
  172. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  173. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  174. data/modules/mu/{clouds → providers}/google/function.rb +140 -168
  175. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  176. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  177. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
  178. data/modules/mu/{clouds → providers}/google/role.rb +94 -58
  179. data/modules/mu/{clouds → providers}/google/server.rb +243 -156
  180. data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
  181. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  182. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  183. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  184. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  185. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  186. data/modules/tests/aws-jobs-functions.yaml +46 -0
  187. data/modules/tests/bucket.yml +4 -0
  188. data/modules/tests/centos6.yaml +15 -0
  189. data/modules/tests/centos7.yaml +15 -0
  190. data/modules/tests/centos8.yaml +12 -0
  191. data/modules/tests/ecs.yaml +23 -0
  192. data/modules/tests/eks.yaml +1 -1
  193. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  194. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  195. data/modules/tests/includes-and-params.yaml +2 -1
  196. data/modules/tests/microservice_app.yaml +288 -0
  197. data/modules/tests/rds.yaml +108 -0
  198. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  199. data/modules/tests/regrooms/bucket.yml +19 -0
  200. data/modules/tests/regrooms/rds.yaml +123 -0
  201. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  202. data/modules/tests/super_complex_bok.yml +2 -2
  203. data/modules/tests/super_simple_bok.yml +3 -5
  204. data/modules/tests/win2k12.yaml +17 -5
  205. data/modules/tests/win2k16.yaml +25 -0
  206. data/modules/tests/win2k19.yaml +25 -0
  207. data/requirements.txt +1 -0
  208. data/spec/mu/clouds/azure_spec.rb +2 -2
  209. metadata +240 -154
  210. data/extras/image-generators/AWS/windows.yaml +0 -18
  211. data/modules/mu/clouds/aws/database.rb +0 -1985
  212. data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -30,7 +30,7 @@ module MU
30
30
  end
31
31
  end
32
32
 
33
- @mu_name ||= @deploy.getResourceName(@config["name"])
33
+ @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 64)
34
34
  end
35
35
 
36
36
  # Called automatically by {MU::Deploy#createResources}
@@ -43,7 +43,7 @@ module MU
43
43
 
44
44
  policy_name = @mu_name+"-"+policy.keys.first.upcase
45
45
  MU.log "Creating IAM policy #{policy_name}"
46
- resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).create_policy(
46
+ MU::Cloud::AWS.iam(credentials: @config['credentials']).create_policy(
47
47
  policy_name: policy_name,
48
48
  path: "/"+@deploy.deploy_id+"/",
49
49
  policy_document: JSON.generate(policy.values.first),
@@ -56,8 +56,8 @@ module MU
56
56
  MU.log "Creating IAM role #{@mu_name}"
57
57
  @cloud_id = @mu_name
58
58
  path = @config['strip_path'] ? nil : "/"+@deploy.deploy_id+"/"
59
- resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).create_role(
60
- path: nil,
59
+ MU::Cloud::AWS.iam(credentials: @config['credentials']).create_role(
60
+ path: path,
61
61
  role_name: @mu_name,
62
62
  description: "Generated by Mu",
63
63
  assume_role_policy_document: gen_assume_role_policy_doc,
@@ -89,7 +89,6 @@ module MU
89
89
  end
90
90
 
91
91
  if @config['raw_policies'] or @config['attachable_policies']
92
- attached_policies = []
93
92
  configured_policies = []
94
93
 
95
94
  if @config['raw_policies']
@@ -128,7 +127,7 @@ module MU
128
127
 
129
128
  # XXX not sure we're binding these sanely, validate that
130
129
  if @config['raw_policies']
131
- pol_arns = MU::Cloud::AWS::Role.manageRawPolicies(
130
+ MU::Cloud::AWS::Role.manageRawPolicies(
132
131
  @config['raw_policies'],
133
132
  basename: @deploy.getResourceName(@config['name']),
134
133
  credentials: @credentials
@@ -183,7 +182,7 @@ module MU
183
182
  desc
184
183
  end
185
184
 
186
- rescue Aws::IAM::Errors::NoSuchEntity => e
185
+ rescue Aws::IAM::Errors::NoSuchEntity
187
186
  MU.log "Creating IAM policy #{policy_name}", details: policy.values.first
188
187
  MU::Cloud::AWS.iam(credentials: credentials).create_policy(
189
188
  policy_name: policy_name,
@@ -202,65 +201,87 @@ module MU
202
201
  def arn
203
202
  desc = cloud_desc
204
203
  if desc["role"]
205
- desc["role"].arn
204
+ if desc['role'].is_a?(Hash)
205
+ desc["role"][:arn] # why though
206
+ else
207
+ desc["role"].arn
208
+ end
206
209
  else
207
210
  nil
208
211
  end
209
212
  end
210
213
 
214
+ @cloud_desc_cache = nil
211
215
  # Return a hash containing a +role+ element and a +policies+ element,
212
216
  # populated with one or both depending on what this resource has
213
217
  # defined.
214
- def cloud_desc
215
- desc = {}
218
+ def cloud_desc(use_cache: true)
219
+
220
+ # we might inherit a naive cached description from the base cloud
221
+ # layer; rearrange it to our tastes
222
+ if @cloud_desc_cache.is_a?(::Aws::IAM::Types::Role)
223
+ new_desc = {
224
+ "role" => @cloud_desc_cache
225
+ }
226
+ @cloud_desc_cache = new_desc
227
+ elsif @cloud_desc_cache.is_a?(::Aws::IAM::Types::Policy)
228
+ new_desc = {
229
+ "policies" => [@cloud_desc_cache]
230
+ }
231
+ @cloud_desc_cache = new_desc
232
+ end
233
+
234
+ return @cloud_desc_cache if @cloud_desc_cache and !@cloud_desc_cache.empty? and use_cache
235
+
236
+ @cloud_desc_cache = {}
216
237
  if @config['bare_policies']
217
238
  if @cloud_id
218
239
  pol_desc = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
219
240
  if pol_desc
220
- desc['policies'] = [pol_desc]
221
- return desc
241
+ @cloud_desc_cache['policies'] = [pol_desc]
242
+ return @cloud_desc_cache
222
243
  end
223
244
  end
224
245
 
225
246
  if @deploy and @deploy.deploy_id
226
- desc["policies"] = MU::Cloud::AWS.iam(credentials: @credentials).list_policies(
247
+ @cloud_desc_cache["policies"] = MU::Cloud::AWS.iam(credentials: @credentials).list_policies(
227
248
  path_prefix: "/"+@deploy.deploy_id+"/"
228
249
  ).policies
229
- desc["policies"].reject! { |p|
250
+ @cloud_desc_cache["policies"].reject! { |p|
230
251
  !p.policy_name.match(/^#{Regexp.quote(@mu_name)}-/)
231
252
  }
232
253
  # this is quasi-wrong because we can be mulitple cloud is, but
233
254
  # we can't really set this type to has_multiples because that's
234
255
  # just how managed policies work not anything else, goddammit
235
256
  # AWS why can't you just bundle everything in roles
236
- if desc["policies"] and desc["policies"].size > 0
237
- @cloud_id ||= desc["policies"].first.arn
257
+ if @cloud_desc_cache["policies"] and @cloud_desc_cache["policies"].size > 0
258
+ @cloud_id ||= @cloud_desc_cache["policies"].first.arn
238
259
  end
239
260
  end
240
261
  else
241
262
  if @cloud_id.match(/^arn:aws(:?-us-gov)?:[^:]*:[^:]*:\d*:policy\//)
242
263
  pol_desc = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
243
264
  if pol_desc
244
- desc['policies'] = [pol_desc]
245
- return desc
265
+ @cloud_desc_cache['policies'] = [pol_desc]
266
+ return @cloud_desc_cache
246
267
  end
247
268
  end
248
269
  begin
249
- desc['role'] = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
250
- desc['role'] ||= MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @mu_name).values.first
270
+ @cloud_desc_cache['role'] = MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @cloud_id).values.first
271
+ @cloud_desc_cache['role'] ||= MU::Cloud::AWS::Role.find(credentials: @credentials, cloud_id: @mu_name).values.first
251
272
  MU::Cloud::AWS.iam(credentials: @credentials).list_attached_role_policies(
252
273
  role_name: @mu_name
253
274
  ).attached_policies.each { |p|
254
- desc["policies"] ||= []
255
- desc["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
275
+ @cloud_desc_cache["policies"] ||= []
276
+ @cloud_desc_cache["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_policy(
256
277
  policy_arn: p.policy_arn
257
278
  ).policy
258
279
  }
259
280
 
260
281
  inline = MU::Cloud::AWS.iam(credentials: @credentials).list_role_policies(role_name: @mu_name).policy_names
261
282
  inline.each { |pol_name|
262
- desc["policies"] ||= []
263
- desc["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_role_policy(
283
+ @cloud_desc_cache["policies"] ||= []
284
+ @cloud_desc_cache["policies"] << MU::Cloud::AWS.iam(credentials: @credentials).get_role_policy(
264
285
  role_name: @mu_name,
265
286
  policy_name: pol_name
266
287
  )
@@ -270,9 +291,9 @@ rescue ::Aws::IAM::Errors::ValidationError => e
270
291
  MU.log @cloud_id+" "+@mu_name, MU::WARN, details: e.inspect
271
292
  end
272
293
  end
273
- desc['cloud_id'] ||= @cloud_id
294
+ @cloud_desc_cache['cloud_id'] ||= @cloud_id
274
295
 
275
- desc
296
+ @cloud_desc_cache
276
297
  end
277
298
 
278
299
  # Return the metadata for this user cofiguration
@@ -284,26 +305,25 @@ end
284
305
  # Insert a new target entity into an existing policy.
285
306
  # @param policy [String]: The name of the policy to which we're appending, which must already exist as part of this role resource
286
307
  # @param targets [Array<String>]: The target resource. If +target_type+ isn't specified, this should be a fully-resolved ARN.
287
- # @param mu_type [String]: A valid Mu resource type
288
- def injectPolicyTargets(policy, targets, mu_type = nil)
308
+ def injectPolicyTargets(policy, targets)
289
309
  if !policy.match(/^#{@deploy.deploy_id}/)
290
310
  policy = @mu_name+"-"+policy.upcase
291
311
  end
292
-
293
- my_policies = cloud_desc["policies"]
312
+ my_policies = cloud_desc(use_cache: false)["policies"]
294
313
  my_policies ||= []
295
-
314
+
315
+ seen_policy = false
296
316
  my_policies.each { |p|
297
317
  if p.policy_name == policy
318
+ seen_policy = true
298
319
  old = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy_version(
299
320
  policy_arn: p.arn,
300
321
  version_id: p.default_version_id
301
322
  ).policy_version
302
323
 
303
324
  doc = JSON.parse URI.decode_www_form_component old.document
304
-
305
325
  need_update = false
306
-
326
+
307
327
  doc["Statement"].each { |s|
308
328
  targets.each { |target|
309
329
  target_string = target
@@ -332,6 +352,10 @@ end
332
352
  end
333
353
  end
334
354
  }
355
+
356
+ if !seen_policy
357
+ MU.log "Was given new targets for policy #{policy}, but I don't see any such policy attached to role #{@cloud_id}", MU::WARN, details: targets
358
+ end
335
359
  end
336
360
 
337
361
  # Delete an IAM policy, along with attendant versions and attachments.
@@ -409,16 +433,15 @@ end
409
433
  # Remove all roles associated with the currently loaded deployment.
410
434
  # @param noop [Boolean]: If true, will only print what would be done
411
435
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
412
- # @param region [String]: The cloud provider region
413
436
  # @return [void]
414
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
437
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
415
438
 
416
439
  resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
417
- path_prefix: "/"+MU.deploy_id+"/"
440
+ path_prefix: "/"+deploy_id+"/"
418
441
  )
419
442
  if resp and resp.policies
420
443
  resp.policies.each { |policy|
421
- MU.log "Deleting IAM policy /#{MU.deploy_id}/#{policy.policy_name}"
444
+ MU.log "Deleting IAM policy /#{deploy_id}/#{policy.policy_name}"
422
445
  if !noop
423
446
  purgePolicy(policy.arn, credentials)
424
447
  end
@@ -429,21 +452,31 @@ end
429
452
  roles = MU::Cloud::AWS::Role.find(credentials: credentials).values
430
453
  roles.each { |r|
431
454
  next if !r.respond_to?(:role_name)
432
- if r.path.match(/^\/#{Regexp.quote(MU.deploy_id)}/)
455
+ if r.path.match(/^\/#{Regexp.quote(deploy_id)}/)
433
456
  deleteme << r
434
457
  next
435
458
  end
436
459
  # For some dumb reason, the list output that .find gets doesn't
437
460
  # include the tags, so we need to fetch each role individually to
438
461
  # check tags. Hardly seems efficient.
439
- desc = MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
462
+ desc = begin
463
+ MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
464
+ rescue Aws::IAM::Errors::NoSuchEntity
465
+ next
466
+ end
440
467
  if desc.role and desc.role.tags and desc.role.tags
468
+ master_match = false
469
+ deploy_match = false
441
470
  desc.role.tags.each { |t|
442
- if t.key == "MU-ID" and t.value == MU.deploy_id
443
- deleteme << r
444
- break
471
+ if t.key == "MU-ID" and t.value == deploy_id
472
+ deploy_match = true
473
+ elsif t.key == "MU-MASTER-IP" and t.value == MU.mu_public_ip
474
+ master_match = true
445
475
  end
446
476
  }
477
+ if deploy_match and (master_match or ignoremaster)
478
+ deleteme << r
479
+ end
447
480
  end
448
481
  }
449
482
 
@@ -481,7 +514,7 @@ end
481
514
  MU::Cloud::AWS.iam(credentials: credentials).delete_instance_profile(instance_profile_name: r.role_name)
482
515
  rescue Aws::IAM::Errors::ValidationError => e
483
516
  MU.log "Cleaning up IAM role #{r.role_name}: #{e.inspect}", MU::WARN
484
- rescue Aws::IAM::Errors::NoSuchEntity => e
517
+ rescue Aws::IAM::Errors::NoSuchEntity
485
518
  end
486
519
 
487
520
  MU::Cloud::AWS.iam(credentials: credentials).delete_role(
@@ -502,7 +535,7 @@ end
502
535
 
503
536
  begin
504
537
  # managed policies get fetched by ARN, roles by plain name. Ok!
505
- if args[:cloud_id].match(/^arn:/)
538
+ if args[:cloud_id].match(/^arn:.*?:policy\//)
506
539
  resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_policy(
507
540
  policy_arn: args[:cloud_id]
508
541
  )
@@ -511,39 +544,26 @@ end
511
544
  end
512
545
  else
513
546
  resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_role(
514
- role_name: args[:cloud_id]
547
+ role_name: args[:cloud_id].sub(/^arn:.*?\/([^:\/]+)$/, '\1') # XXX if it's an ARN, actually parse it and look in the correct account when applicable
515
548
  )
549
+
516
550
  if resp and resp.role
517
- found[args[:cloud_id]] = resp.role
551
+ found[resp.role.role_name] = resp.role
518
552
  end
519
553
  end
520
554
  rescue ::Aws::IAM::Errors::NoSuchEntity
521
555
  end
522
-
556
+
523
557
  else
524
- marker = nil
525
- begin
526
- resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles(
527
- marker: marker
528
- )
529
- break if !resp or !resp.roles
530
- resp.roles.each { |role|
531
- found[role.role_name] = role
532
- }
533
- marker = resp.marker
534
- end while marker
558
+ resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles
559
+ resp.roles.each { |role|
560
+ found[role.role_name] = role
561
+ }
535
562
 
536
- begin
537
- resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(
538
- scope: "Local",
539
- marker: marker
540
- )
541
- break if !resp or !resp.policies
542
- resp.policies.each { |pol|
543
- found[pol.arn] = pol
544
- }
545
- marker = resp.marker
546
- end while marker
563
+ resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(scope: "Local")
564
+ resp.policies.each { |pol|
565
+ found[pol.arn] = pol
566
+ }
547
567
  end
548
568
 
549
569
  found
@@ -552,7 +572,7 @@ end
552
572
  # Reverse-map our cloud description into a runnable config hash.
553
573
  # We assume that any values we have in +@config+ are placeholders, and
554
574
  # calculate our own accordingly based on what's live in the cloud.
555
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
575
+ def toKitten(**_args)
556
576
  bok = {
557
577
  "cloud" => "AWS",
558
578
  "credentials" => @config['credentials'],
@@ -601,14 +621,13 @@ end
601
621
  )
602
622
  JSON.parse(URI.decode(version.policy_version.document))
603
623
  end
604
-
605
624
  bok["policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["policies"])
606
625
  end
607
626
  }
608
627
 
609
628
  return bok if @config['bare_policies']
610
629
  end
611
-
630
+
612
631
  if desc.tags and desc.tags.size > 0
613
632
  bok["tags"] = MU.structToHash(desc.tags, stringify_keys: true)
614
633
  end
@@ -681,6 +700,7 @@ end
681
700
  end
682
701
 
683
702
  bok["attachable_policies"].uniq! if bok["attachable_policies"]
703
+ bok["name"].gsub!(/[^a-zA-Z0-9_\-]/, "_")
684
704
 
685
705
  bok
686
706
  end
@@ -693,6 +713,10 @@ end
693
713
  def self.doc2MuPolicies(basename, doc, policies = [])
694
714
  policies ||= []
695
715
 
716
+ if !doc["Statement"].is_a?(Array)
717
+ doc["Statement"] = [doc["Statement"]]
718
+ end
719
+
696
720
  doc["Statement"].each { |s|
697
721
  if !s["Action"]
698
722
  MU.log "Statement in policy document for #{basename} didn't have an Action field", MU::WARN, details: doc
@@ -721,7 +745,7 @@ end
721
745
  "targets" => s["Resource"].map { |r|
722
746
  if r.match(/^arn:aws(-us-gov)?:([^:]+):.*?:([^:]*)$/)
723
747
  # XXX which cases even count for blind references to sibling resources?
724
- type = if Regexp.last_match[1] == "s3"
748
+ if Regexp.last_match[1] == "s3"
725
749
  "bucket"
726
750
  elsif Regexp.last_match[1]
727
751
  MU.log "Service #{Regexp.last_match[1]} to type...", MU::WARN, details: r
@@ -741,7 +765,7 @@ end
741
765
 
742
766
  # Attach this role or group of loose policies to the specified entity.
743
767
  # @param entitytype [String]: The type of entity (user, group or role for policies; instance_profile for roles)
744
- def bindTo(entitytype, entityname, policies = [])
768
+ def bindTo(entitytype, entityname)
745
769
  if entitytype == "instance_profile"
746
770
  begin
747
771
  resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
@@ -754,7 +778,7 @@ end
754
778
  role_name: @mu_name
755
779
  )
756
780
  end
757
- rescue Exception => e
781
+ rescue StandardError => e
758
782
  MU.log "Error binding role #{@mu_name} to instance profile #{entityname}: #{e.message}", MU::ERR
759
783
  raise e
760
784
  end
@@ -783,7 +807,6 @@ end
783
807
  rescue Aws::IAM::Errors::NoSuchEntity => e
784
808
  if subpaths.size > 0
785
809
  p_arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":iam::aws:policy/#{subpaths.shift}/"+policy
786
- retried = true
787
810
  retry
788
811
  end
789
812
  raise e
@@ -833,6 +856,7 @@ end
833
856
  else
834
857
  raise MuError, "Invalid entitytype '#{entitytype}' passed to MU::Cloud::AWS::Role.bindTo. Must be be one of: user, group, role, instance_profile"
835
858
  end
859
+ cloud_desc(use_cache: false)
836
860
  end
837
861
 
838
862
  # Create an instance profile for EC2 instances, named identically and
@@ -847,7 +871,7 @@ end
847
871
  MU::Cloud::AWS.iam(credentials: @config['credentials']).create_instance_profile(
848
872
  instance_profile_name: @mu_name
849
873
  )
850
- rescue Aws::IAM::Errors::EntityAlreadyExists => e
874
+ rescue Aws::IAM::Errors::EntityAlreadyExists
851
875
  MU::Cloud::AWS.iam(credentials: @config['credentials']).get_instance_profile(
852
876
  instance_profile_name: @mu_name
853
877
  )
@@ -905,13 +929,13 @@ end
905
929
  end
906
930
 
907
931
  # Cloud-specific configuration properties.
908
- # @param config [MU::Config]: The calling MU::Config object
932
+ # @param _config [MU::Config]: The calling MU::Config object
909
933
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
910
- def self.schema(config)
934
+ def self.schema(_config)
911
935
  toplevel_required = []
912
936
  aws_resource_types = MU::Cloud.resource_types.keys.reject { |t|
913
937
  begin
914
- MU::Cloud.loadCloudType("AWS", t)
938
+ MU::Cloud.resourceClass("AWS", t)
915
939
  false
916
940
  rescue MuCloudResourceNotImplemented
917
941
  true
@@ -1012,10 +1036,9 @@ end
1012
1036
  subpaths = ["service-role", "aws-service-role", "job-function"]
1013
1037
  begin
1014
1038
  MU::Cloud::AWS.iam(credentials: credentials).get_policy(policy_arn: arn)
1015
- rescue Aws::IAM::Errors::NoSuchEntity => e
1039
+ rescue Aws::IAM::Errors::NoSuchEntity
1016
1040
  if subpaths.size > 0
1017
1041
  arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(region) ? "aws-us-gov" : "aws")+":iam::aws:policy/#{subpaths.shift}/"+ref["id"]
1018
- retried = true
1019
1042
  retry
1020
1043
  end
1021
1044
  MU.log "No such canned AWS IAM policy '#{arn}'", MU::ERR
@@ -1029,9 +1052,9 @@ end
1029
1052
 
1030
1053
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::roles}, bare and unvalidated.
1031
1054
  # @param role [Hash]: The resource to process and validate
1032
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
1055
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
1033
1056
  # @return [Boolean]: True if validation succeeded, False otherwise
1034
- def self.validateConfig(role, configurator)
1057
+ def self.validateConfig(role, _configurator)
1035
1058
  ok = true
1036
1059
 
1037
1060
  # munge things declared with the deprecated import keyword into
@@ -1074,11 +1097,7 @@ end
1074
1097
  role['policies'].each { |policy|
1075
1098
  policy['targets'].each { |target|
1076
1099
  if target['type']
1077
- role['dependencies'] ||= []
1078
- role['dependencies'] << {
1079
- "name" => target['identifier'],
1080
- "type" => target['type']
1081
- }
1100
+ MU::Config.addDependency(role, target['identifier'], target['type'], no_create_wait: true)
1082
1101
  end
1083
1102
  }
1084
1103
  }
@@ -1094,15 +1113,14 @@ end
1094
1113
  # @param policies [Array<Hash>]: One or more policy chunks
1095
1114
  # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
1096
1115
  # @return [Array<Hash>]
1097
- def self.genPolicyDocument(policies, deploy_obj: nil)
1098
- iam_policies = []
1099
-
1116
+ def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil)
1100
1117
  if policies
1101
1118
  name = nil
1102
1119
  doc = {
1103
- "Version" => "2012-10-17",
1120
+ "Version" => version,
1104
1121
  "Statement" => []
1105
1122
  }
1123
+ doc["Id"] = doc_id if doc_id
1106
1124
  policies.each { |policy|
1107
1125
  policy["flag"] ||= "Allow"
1108
1126
  statement = {
@@ -1134,12 +1152,28 @@ end
1134
1152
  )
1135
1153
  if sibling
1136
1154
  id = sibling.cloudobj.arn
1137
- statement["Principal"] << id
1155
+ if bucket_style
1156
+ statement["Principal"] << { "AWS" => id }
1157
+ else
1158
+ statement["Principal"] << id
1159
+ end
1138
1160
  else
1139
1161
  raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy"
1140
1162
  end
1141
1163
  else
1142
- statement["Principal"] << grantee["identifier"]
1164
+ bucket_prefix = if grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/)
1165
+ "Service"
1166
+ elsif grantee["identifier"] =~ /^[a-f0-9]+$/
1167
+ "CanonicalUser"
1168
+ else
1169
+ "AWS"
1170
+ end
1171
+
1172
+ if bucket_style
1173
+ statement["Principal"] << { bucket_prefix => grantee["identifier"] }
1174
+ else
1175
+ statement["Principal"] << grantee["identifier"]
1176
+ end
1143
1177
  end
1144
1178
  }
1145
1179
  if policy["grant_to"].size == 1
@@ -1162,9 +1196,11 @@ end
1162
1196
  stream_id = id.sub(/:([^:]+)$/, ":log-stream:*")
1163
1197
  # "arn:aws:logs:us-east-2:accountID:log-group:log_group_name:log-stream:CloudTrail_log_stream_name_prefix*"
1164
1198
  statement["Resource"] << stream_id
1199
+ elsif id.match(/:s3:/)
1200
+ statement["Resource"] << id+"/*"
1165
1201
  end
1166
1202
  else
1167
- raise MuError, "Couldn't find a #{target["entity_type"]} named #{target["identifier"]} when generating IAM policy"
1203
+ raise MuError, "Couldn't find a #{target["type"]} named #{target["identifier"]} when generating IAM policy"
1168
1204
  end
1169
1205
  else
1170
1206
  target["identifier"] += target["path"] if target["path"]
@@ -1180,6 +1216,32 @@ end
1180
1216
  []
1181
1217
  end
1182
1218
 
1219
+ # Update a policy, handling deletion of old versions as needed
1220
+ # @param arn [String]:
1221
+ # @param doc [Hash]:
1222
+ # @param credentials [String]:
1223
+ def self.update_policy(arn, doc, credentials: nil)
1224
+ # XXX this is just blindly replacing identical versions, when it should check
1225
+ # and guard
1226
+ begin
1227
+ MU::Cloud::AWS.iam(credentials: credentials).create_policy_version(
1228
+ policy_arn: arn,
1229
+ set_as_default: true,
1230
+ policy_document: JSON.generate(doc)
1231
+ )
1232
+ rescue Aws::IAM::Errors::LimitExceeded
1233
+ delete_version = MU::Cloud::AWS.iam(credentials: credentials).list_policy_versions(
1234
+ policy_arn: arn,
1235
+ ).versions.last.version_id
1236
+ MU.log "Purging oldest version (#{delete_version}) of IAM policy #{arn}", MU::NOTICE
1237
+ MU::Cloud::AWS.iam(credentials: credentials).delete_policy_version(
1238
+ policy_arn: arn,
1239
+ version_id: delete_version
1240
+ )
1241
+ retry
1242
+ end
1243
+ end
1244
+
1183
1245
  private
1184
1246
 
1185
1247
  # Convert entries from the cloud-neutral @config['policies'] list into
@@ -1261,28 +1323,6 @@ end
1261
1323
  MU::Cloud::AWS::Role.update_policy(arn, doc, credentials: @credentials)
1262
1324
  end
1263
1325
 
1264
- # Update a policy, handling deletion of old versions as needed
1265
- def self.update_policy(arn, doc, credentials: nil)
1266
- # XXX this is just blindly replacing identical versions, when it should check
1267
- # and guard
1268
- begin
1269
- MU::Cloud::AWS.iam(credentials: credentials).create_policy_version(
1270
- policy_arn: arn,
1271
- set_as_default: true,
1272
- policy_document: JSON.generate(doc)
1273
- )
1274
- rescue Aws::IAM::Errors::LimitExceeded => e
1275
- delete_version = MU::Cloud::AWS.iam(credentials: credentials).list_policy_versions(
1276
- policy_arn: arn,
1277
- ).versions.last.version_id
1278
- MU.log "Purging oldest version (#{delete_version}) of IAM policy #{arn}", MU::NOTICE
1279
- MU::Cloud::AWS.iam(credentials: credentials).delete_policy_version(
1280
- policy_arn: arn,
1281
- version_id: delete_version
1282
- )
1283
- retry
1284
- end
1285
- end
1286
1326
 
1287
1327
  end
1288
1328
  end