cloud-mu 3.1.3 → 3.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +21 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +4 -4
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +147 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +158 -111
  45. data/modules/mu/adoption.rb +404 -71
  46. data/modules/mu/cleanup.rb +221 -306
  47. data/modules/mu/cloud.rb +129 -1633
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +44 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +926 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +169 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +32 -3
  61. data/modules/mu/config/cache_cluster.rb +2 -2
  62. data/modules/mu/config/cdn.rb +100 -0
  63. data/modules/mu/config/collection.rb +4 -4
  64. data/modules/mu/config/container_cluster.rb +9 -4
  65. data/modules/mu/config/database.rb +84 -105
  66. data/modules/mu/config/database.yml +1 -2
  67. data/modules/mu/config/dnszone.rb +10 -9
  68. data/modules/mu/config/doc_helpers.rb +516 -0
  69. data/modules/mu/config/endpoint.rb +5 -4
  70. data/modules/mu/config/firewall_rule.rb +103 -4
  71. data/modules/mu/config/folder.rb +4 -4
  72. data/modules/mu/config/function.rb +19 -10
  73. data/modules/mu/config/group.rb +4 -4
  74. data/modules/mu/config/habitat.rb +4 -4
  75. data/modules/mu/config/job.rb +89 -0
  76. data/modules/mu/config/loadbalancer.rb +60 -14
  77. data/modules/mu/config/log.rb +4 -4
  78. data/modules/mu/config/msg_queue.rb +4 -4
  79. data/modules/mu/config/nosqldb.rb +4 -4
  80. data/modules/mu/config/notifier.rb +10 -21
  81. data/modules/mu/config/ref.rb +411 -0
  82. data/modules/mu/config/role.rb +4 -4
  83. data/modules/mu/config/schema_helpers.rb +509 -0
  84. data/modules/mu/config/search_domain.rb +4 -4
  85. data/modules/mu/config/server.rb +98 -71
  86. data/modules/mu/config/server.yml +1 -0
  87. data/modules/mu/config/server_pool.rb +5 -9
  88. data/modules/mu/config/storage_pool.rb +1 -1
  89. data/modules/mu/config/tail.rb +200 -0
  90. data/modules/mu/config/user.rb +4 -4
  91. data/modules/mu/config/vpc.rb +71 -27
  92. data/modules/mu/config/vpc.yml +0 -1
  93. data/modules/mu/defaults/AWS.yaml +91 -68
  94. data/modules/mu/defaults/Azure.yaml +1 -0
  95. data/modules/mu/defaults/Google.yaml +3 -2
  96. data/modules/mu/deploy.rb +43 -26
  97. data/modules/mu/groomer.rb +17 -2
  98. data/modules/mu/groomers/ansible.rb +188 -41
  99. data/modules/mu/groomers/chef.rb +116 -55
  100. data/modules/mu/logger.rb +127 -148
  101. data/modules/mu/master.rb +410 -2
  102. data/modules/mu/master/chef.rb +3 -4
  103. data/modules/mu/master/ldap.rb +3 -3
  104. data/modules/mu/master/ssl.rb +12 -3
  105. data/modules/mu/mommacat.rb +218 -2612
  106. data/modules/mu/mommacat/daemon.rb +403 -0
  107. data/modules/mu/mommacat/naming.rb +473 -0
  108. data/modules/mu/mommacat/search.rb +495 -0
  109. data/modules/mu/mommacat/storage.rb +722 -0
  110. data/modules/mu/{clouds → providers}/README.md +1 -1
  111. data/modules/mu/{clouds → providers}/aws.rb +380 -122
  112. data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
  113. data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
  114. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
  115. data/modules/mu/providers/aws/cdn.rb +782 -0
  116. data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
  117. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
  118. data/modules/mu/providers/aws/database.rb +1744 -0
  119. data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
  120. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  121. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
  122. data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
  123. data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
  124. data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
  125. data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
  126. data/modules/mu/providers/aws/job.rb +466 -0
  127. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
  128. data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
  129. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
  130. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
  131. data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
  132. data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
  133. data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
  134. data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
  135. data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
  136. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
  137. data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
  138. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  139. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  140. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  141. data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
  142. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  143. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  144. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  145. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  146. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  147. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  148. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  149. data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
  150. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  151. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  152. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  153. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  154. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  155. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  156. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  160. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  161. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  162. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  163. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  164. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  165. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  166. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  167. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  168. data/modules/mu/{clouds → providers}/google.rb +68 -30
  169. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  170. data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
  171. data/modules/mu/{clouds → providers}/google/database.rb +11 -21
  172. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  173. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  174. data/modules/mu/{clouds → providers}/google/function.rb +140 -168
  175. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  176. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  177. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
  178. data/modules/mu/{clouds → providers}/google/role.rb +94 -58
  179. data/modules/mu/{clouds → providers}/google/server.rb +243 -156
  180. data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
  181. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  182. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  183. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  184. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  185. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  186. data/modules/tests/aws-jobs-functions.yaml +46 -0
  187. data/modules/tests/bucket.yml +4 -0
  188. data/modules/tests/centos6.yaml +15 -0
  189. data/modules/tests/centos7.yaml +15 -0
  190. data/modules/tests/centos8.yaml +12 -0
  191. data/modules/tests/ecs.yaml +23 -0
  192. data/modules/tests/eks.yaml +1 -1
  193. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  194. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  195. data/modules/tests/includes-and-params.yaml +2 -1
  196. data/modules/tests/microservice_app.yaml +288 -0
  197. data/modules/tests/rds.yaml +108 -0
  198. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  199. data/modules/tests/regrooms/bucket.yml +19 -0
  200. data/modules/tests/regrooms/rds.yaml +123 -0
  201. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  202. data/modules/tests/super_complex_bok.yml +2 -2
  203. data/modules/tests/super_simple_bok.yml +3 -5
  204. data/modules/tests/win2k12.yaml +17 -5
  205. data/modules/tests/win2k16.yaml +25 -0
  206. data/modules/tests/win2k19.yaml +25 -0
  207. data/requirements.txt +1 -0
  208. data/spec/mu/clouds/azure_spec.rb +2 -2
  209. metadata +240 -154
  210. data/extras/image-generators/AWS/windows.yaml +0 -18
  211. data/modules/mu/clouds/aws/database.rb +0 -1985
  212. data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -170,7 +170,7 @@ module MU
170
170
  # config_struct[:preferred_cache_cluster_a_zs] = @config["preferred_cache_cluster_azs"]
171
171
 
172
172
  MU.log "Creating cache replication group #{@config['identifier']}"
173
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_replication_group(config_struct).replication_group
173
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_replication_group(config_struct).replication_group
174
174
 
175
175
  wait_start_time = Time.now
176
176
  retries = 0
@@ -180,7 +180,7 @@ module MU
180
180
  waiter.before_attempt do |attempts|
181
181
  MU.log "Waiting for cache replication group #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
182
182
  end
183
- waiter.before_wait do |attempts, r|
183
+ waiter.before_wait do |_attempts, r|
184
184
  throw :success if r.replication_groups.first.status == "available"
185
185
  throw :failure if Time.now - wait_start_time > 1800
186
186
  end
@@ -199,7 +199,7 @@ module MU
199
199
  addStandardTags(member, "cluster", region: @config['region'])
200
200
  }
201
201
 
202
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(
202
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
203
203
  name: resp.replication_group_id,
204
204
  target: "#{resp.node_groups.first.primary_endpoint.address}.",
205
205
  cloudclass: MU::Cloud::CacheCluster,
@@ -207,7 +207,7 @@ module MU
207
207
  )
208
208
 
209
209
  resp.node_groups.first.node_group_members.each { |member|
210
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(
210
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
211
211
  name: member.cache_cluster_id,
212
212
  target: "#{member.read_endpoint.address}.",
213
213
  cloudclass: MU::Cloud::CacheCluster,
@@ -228,7 +228,7 @@ module MU
228
228
 
229
229
  MU.log "Creating cache cluster #{@config['identifier']}"
230
230
  begin
231
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
231
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
232
232
  rescue ::Aws::ElastiCache::Errors::InvalidParameterValue => e
233
233
  if e.message.match(/security group (sg-[^\s]+)/)
234
234
  bad_sg = Regexp.last_match[1]
@@ -248,7 +248,7 @@ module MU
248
248
  waiter.before_attempt do |attempts|
249
249
  MU.log "Waiting for cache cluster #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
250
250
  end
251
- waiter.before_wait do |attempts, r|
251
+ waiter.before_wait do |_attempts, r|
252
252
  throw :success if r.cache_clusters.first.cache_cluster_status == "available"
253
253
  throw :failure if Time.now - wait_start_time > 1800
254
254
  end
@@ -270,7 +270,7 @@ module MU
270
270
  def createSubnetGroup
271
271
  subnet_ids = []
272
272
  if @config["vpc"] && !@config["vpc"].empty?
273
- raise MuError, "Didn't find the VPC specified in #{@config["vpc"]}" unless @vpc
273
+ raise MuError.new "Didn't find the VPC specified for #{@mu_name}", details: @config["vpc"].to_h unless @vpc
274
274
 
275
275
  vpc_id = @vpc.cloud_id
276
276
 
@@ -283,7 +283,7 @@ module MU
283
283
  else
284
284
  @config["vpc"]["subnets"].each { |subnet|
285
285
  subnet_obj = @vpc.getSubnet(cloud_id: subnet["subnet_id"].to_s, name: subnet["subnet_name"].to_s)
286
- raise MuError, "Couldn't find a live subnet matching #{subnet} in #{@vpc} (#{@vpc.subnets})" if subnet_obj.nil?
286
+ raise MuError.new "Couldn't find a live subnet matching #{subnet} in #{@vpc}", details: @vpc.subnets if subnet_obj.nil?
287
287
  subnet_ids << subnet_obj.cloud_id
288
288
  }
289
289
  end
@@ -317,7 +317,7 @@ module MU
317
317
  "vpc_id" => vpc_id,
318
318
  "subnets" => mu_subnets
319
319
  }
320
- using_default_vpc = true
320
+
321
321
  MU.log "Using default VPC for cache cluster #{@config['identifier']}"
322
322
  end
323
323
  end
@@ -327,30 +327,13 @@ module MU
327
327
  else
328
328
  MU.log "Creating subnet group #{@config["subnet_group_name"]} for cache cluster #{@config['identifier']}"
329
329
 
330
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_subnet_group(
330
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_subnet_group(
331
331
  cache_subnet_group_name: @config["subnet_group_name"],
332
332
  cache_subnet_group_description: @config["subnet_group_name"],
333
333
  subnet_ids: subnet_ids
334
334
  )
335
335
 
336
- # Find NAT and create holes in security groups.
337
- # Adding just for consistency, but do we really need this for cache clusters? I guess Nagios and such..
338
- if @config["vpc"]["nat_host_name"] || @config["vpc"]["nat_host_id"] || @config["vpc"]["nat_host_tag"] || @config["vpc"]["nat_host_ip"]
339
- nat = @nat
340
- if nat.is_a?(Struct) && nat.nat_gateway_id && nat.nat_gateway_id.start_with?("nat-")
341
- MU.log "Using NAT Gateway, not modifying security groups"
342
- else
343
- nat_name, nat_conf, nat_deploydata = @nat.describe
344
- @deploy.kittens['firewall_rules'].each_pair { |name, acl|
345
- # XXX if a user doesn't set up dependencies correctly, this can die horribly on a NAT that's still in mid-creation. Fix this... possibly in the config parser.
346
- if acl.config["admin"]
347
- acl.addRule([nat_deploydata["private_ip_address"]], proto: "tcp")
348
- acl.addRule([nat_deploydata["private_ip_address"]], proto: "udp")
349
- break
350
- end
351
- }
352
- end
353
- end
336
+ allowBastionAccess
354
337
 
355
338
  if @dependencies.has_key?('firewall_rule')
356
339
  @config["security_group_ids"] = []
@@ -364,7 +347,7 @@ module MU
364
347
  # Create a Cache Cluster parameter group.
365
348
  def createParameterGroup
366
349
  MU.log "Creating a cache cluster parameter group #{@config["parameter_group_name"]}"
367
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_parameter_group(
350
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_parameter_group(
368
351
  cache_parameter_group_name: @config["parameter_group_name"],
369
352
  cache_parameter_group_family: @config["parameter_group_family"],
370
353
  description: "Parameter group for #{@config["parameter_group_family"]}"
@@ -404,7 +387,7 @@ module MU
404
387
  def self.getCacheClusterById(cc_id, region: MU.curRegion, credentials: nil)
405
388
  begin
406
389
  MU::Cloud::AWS.elasticache(region: region, credentials: credentials).describe_cache_clusters(cache_cluster_id: cc_id).cache_clusters.first
407
- rescue Aws::ElastiCache::Errors::CacheClusterNotFound => e
390
+ rescue Aws::ElastiCache::Errors::CacheClusterNotFound
408
391
  nil
409
392
  end
410
393
  end
@@ -430,7 +413,7 @@ module MU
430
413
  }
431
414
  end
432
415
  # XXX this should be a call to @deploy.nameKitten
433
- MU::Cloud::AWS::DNSZone.createRecordsFromConfig(@config['dns_records'], target: repl_group.node_groups.first.primary_endpoint.address)
416
+ MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: repl_group.node_groups.first.primary_endpoint.address)
434
417
 
435
418
  deploy_struct = {
436
419
  "identifier" => repl_group.replication_group_id,
@@ -532,7 +515,7 @@ module MU
532
515
 
533
516
  attempts = 0
534
517
  begin
535
- snapshot = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_snapshot(
518
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_snapshot(
536
519
  cache_cluster_id: @config["identifier"],
537
520
  snapshot_name: snap_id
538
521
  )
@@ -586,7 +569,7 @@ module MU
586
569
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server.
587
570
  # @param region [String]: The cloud provider's region in which to operate.
588
571
  # @return [void]
589
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {})
572
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, region: MU.curRegion, flags: {})
590
573
  skipsnapshots = flags["skipsnapshots"]
591
574
  all_clusters = MU::Cloud::AWS.elasticache(credentials: credentials, region: region).describe_cache_clusters
592
575
  our_clusters = []
@@ -594,7 +577,7 @@ module MU
594
577
 
595
578
  # Because we can't run list_tags_for_resource on a cache cluster that isn't in "available" state we're loading the deploy to make sure we have a cache cluster to cleanup.
596
579
  # To ensure we don't miss cache clusters that have been terminated mid creation we'll load the 'original_config'. We might want to find a better approach for this.
597
- deploy = MU::MommaCat.getLitter(MU.deploy_id)
580
+ deploy = MU::MommaCat.getLitter(deploy_id)
598
581
  if deploy.original_config && deploy.original_config.has_key?("cache_clusters") && !deploy.original_config["cache_clusters"].empty?
599
582
 
600
583
  # The ElastiCache API and documentation are a mess, the replication group ARN resource_type is not documented, and is not easily guessable.
@@ -632,14 +615,14 @@ module MU
632
615
  sleep 30
633
616
  retry
634
617
  else
635
- raise MuError, "Failed to get tags for cache cluster #{cluster_id}, MU-ID #{MU.deploy_id}: #{e.inspect}"
618
+ raise MuError, "Failed to get tags for cache cluster #{cluster_id}, MU-ID #{deploy_id}: #{e.inspect}"
636
619
  end
637
620
  end
638
621
 
639
622
  found_muid = false
640
623
  found_master = false
641
624
  tags.each { |tag|
642
- found_muid = true if tag.key == "MU-ID" && tag.value == MU.deploy_id
625
+ found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id
643
626
  found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip
644
627
  }
645
628
  next if !found_muid
@@ -669,7 +652,7 @@ module MU
669
652
  threads << Thread.new(replication_group) { |myrepl_group|
670
653
  MU.dupGlobals(parent_thread_id)
671
654
  Thread.abort_on_exception = true
672
- MU::Cloud::AWS::CacheCluster.terminate_replication_group(myrepl_group, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: myrepl_group.replication_group_id, credentials: credentials)
655
+ terminate_replication_group(myrepl_group, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
673
656
  }
674
657
  }
675
658
  end
@@ -681,7 +664,7 @@ module MU
681
664
  threads << Thread.new(cluster) { |mycluster|
682
665
  MU.dupGlobals(parent_thread_id)
683
666
  Thread.abort_on_exception = true
684
- MU::Cloud::AWS::CacheCluster.terminate_cache_cluster(mycluster, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: mycluster.cache_cluster_id, credentials: credentials)
667
+ terminate_cache_cluster(mycluster, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
685
668
  }
686
669
  }
687
670
  end
@@ -694,35 +677,16 @@ module MU
694
677
  end
695
678
 
696
679
  # Cloud-specific configuration properties.
697
- # @param config [MU::Config]: The calling MU::Config object
680
+ # @param _config [MU::Config]: The calling MU::Config object
698
681
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
699
- def self.schema(config)
682
+ def self.schema(_config)
700
683
  toplevel_required = []
701
684
  schema = {
702
685
  "create_replication_group" => {
703
686
  "type" => "boolean",
704
687
  "description" => "Create a replication group; will be set automatically if +engine+ is +redis+ and +node_count+ is greated than one."
705
688
  },
706
- "ingress_rules" => {
707
- "items" => {
708
- "properties" => {
709
- "sgs" => {
710
- "type" => "array",
711
- "items" => {
712
- "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
713
- "type" => "string"
714
- }
715
- },
716
- "lbs" => {
717
- "type" => "array",
718
- "items" => {
719
- "description" => "AWS Load Balancers which will have this rule applied to their traffic",
720
- "type" => "string"
721
- }
722
- }
723
- }
724
- }
725
- }
689
+ "ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
726
690
  }
727
691
  [toplevel_required, schema]
728
692
  end
@@ -804,9 +768,8 @@ module MU
804
768
  # @param noop [Boolean]: If true, will only print what would be done.
805
769
  # @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
806
770
  # @param region [String]: The cloud provider's region in which to operate.
807
- # @param cloud_id [String]: The cloud provider's identifier for this resource.
808
771
  # @return [void]
809
- def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, cloud_id: nil, credentials: nil)
772
+ def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
810
773
  raise MuError, "terminate_cache_cluster requires a non-nil cache cluster descriptor" if cluster.nil? || cluster.empty?
811
774
 
812
775
  cluster_id = cluster.cache_cluster_id
@@ -824,7 +787,7 @@ module MU
824
787
  end
825
788
 
826
789
  # The API is broken, cluster.cache_nodes is returnning an empty array, and the only URL we can get is the config one with cluster.configuration_endpoint.address.
827
- # MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: cluster_id, target: , cloudclass: MU::Cloud::CacheCluster, delete: true)
790
+ # MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: cluster_id, target: , cloudclass: MU::Cloud::CacheCluster, delete: true)
828
791
 
829
792
  if %w{deleting deleted}.include?(cluster.cache_cluster_status)
830
793
  MU.log "#{cluster_id} has already been terminated", MU::WARN
@@ -874,7 +837,7 @@ module MU
874
837
  waiter.before_attempt do |attempts|
875
838
  MU.log "Waiting for cache cluster #{cluster_id} to delete..", MU::NOTICE if attempts % 10 == 0
876
839
  end
877
- waiter.before_wait do |attempts, resp|
840
+ waiter.before_wait do |_attempts, resp|
878
841
  throw :success if resp.cache_clusters.first.cache_cluster_status == "deleted"
879
842
  throw :failure if Time.now - wait_start_time > 1800
880
843
  end
@@ -893,19 +856,19 @@ module MU
893
856
  MU.log "#{cluster_id} has been terminated"
894
857
 
895
858
  unless noop
896
- MU::Cloud::AWS::CacheCluster.delete_subnet_group(subnet_group, region: region, credentials: credentials) if subnet_group
897
- MU::Cloud::AWS::CacheCluster.delete_parameter_group(parameter_group, region: region, credentials: credentials) if parameter_group && !parameter_group.start_with?("default")
859
+ delete_subnet_group(subnet_group, region: region, credentials: credentials) if subnet_group
860
+ delete_parameter_group(parameter_group, region: region, credentials: credentials) if parameter_group && !parameter_group.start_with?("default")
898
861
  end
899
862
  end
863
+ private_class_method :terminate_cache_cluster
900
864
 
901
865
  # Remove a Cache Cluster Replication Group and associated artifacts
902
866
  # @param repl_group [OpenStruct]: The cloud provider's description of the Cache Cluster artifact.
903
867
  # @param noop [Boolean]: If true, will only print what would be done.
904
868
  # @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
905
869
  # @param region [String]: The cloud provider's region in which to operate.
906
- # @param cloud_id [String]: The cloud provider's identifier for this resource.
907
870
  # @return [void]
908
- def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, cloud_id: nil, credentials: nil)
871
+ def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
909
872
  raise MuError, "terminate_replication_group requires a non-nil cache replication group descriptor" if repl_group.nil? || repl_group.empty?
910
873
 
911
874
  repl_group_id = repl_group.replication_group_id
@@ -926,10 +889,10 @@ module MU
926
889
  end
927
890
 
928
891
  # What's the likelihood of having more than one node group? maybe iterate over node_groups instead of assuming there is only one?
929
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: repl_group_id, target: repl_group.node_groups.first.primary_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
892
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: repl_group_id, target: repl_group.node_groups.first.primary_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
930
893
  # Assuming we also created DNS records for each of our cluster's read endpoint.
931
894
  repl_group.node_groups.first.node_group_members.each { |member|
932
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: member.cache_cluster_id, target: member.read_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
895
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: member.cache_cluster_id, target: member.read_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
933
896
  }
934
897
 
935
898
  if %w{deleting deleted}.include?(repl_group.status)
@@ -983,7 +946,7 @@ module MU
983
946
  waiter.before_attempt do |attempts|
984
947
  MU.log "Waiting for #{repl_group_id} to delete..", MU::NOTICE if attempts % 10 == 0
985
948
  end
986
- waiter.before_wait do |attempts, resp|
949
+ waiter.before_wait do |_attempts, resp|
987
950
  throw :success if resp.replication_groups.first.status == "deleted"
988
951
  throw :failure if Time.now - wait_start_time > 1800
989
952
  end
@@ -1005,6 +968,7 @@ module MU
1005
968
  MU::Cloud::AWS::CacheCluster.delete_parameter_group(parameter_group, region: region) if parameter_group && !parameter_group.start_with?("default")
1006
969
  end
1007
970
  end
971
+ private_class_method :terminate_replication_group
1008
972
 
1009
973
  # Remove a Cache Cluster Subnet Group.
1010
974
  # @param subnet_group_id [string]: The cloud provider's ID of the cache cluster subnet group.
@@ -1026,6 +990,7 @@ module MU
1026
990
  MU.log "Subnet group #{subnet_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1027
991
  end
1028
992
  end
993
+ private_class_method :delete_subnet_group
1029
994
 
1030
995
  # Remove a Cache Cluster Parameter Group.
1031
996
  # @param parameter_group_id [string]: The cloud provider's ID of the cache cluster parameter group.
@@ -1049,6 +1014,7 @@ module MU
1049
1014
  MU.log "Parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1050
1015
  end
1051
1016
  end
1017
+ private_class_method :delete_parameter_group
1052
1018
  end
1053
1019
  end
1054
1020
  end
@@ -0,0 +1,782 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ class Cloud
17
+ class AWS
18
+ # A scheduled task facility as configured in {MU::Config::BasketofKittens::cdns}
19
+ class CDN < MU::Cloud::CDN
20
+
21
+ # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
22
+ # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
23
+ def initialize(**args)
24
+ super
25
+ @mu_name ||= @deploy.getResourceName(@config["name"])
26
+ end
27
+
28
+ # Called automatically by {MU::Deploy#createResources}
29
+ def create
30
+ resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).create_cloud_front_origin_access_identity(
31
+ cloud_front_origin_access_identity_config: {
32
+ caller_reference: @mu_name,
33
+ comment: @mu_name
34
+ }
35
+ )
36
+
37
+ @origin_access_identity = "origin-access-identity/cloudfront/"+resp.cloud_front_origin_access_identity.id
38
+
39
+ params = get_properties
40
+
41
+ begin
42
+ MU.log "Creating CloudFront distribution #{@mu_name}", details: params
43
+ MU.retrier([Aws::CloudFront::Errors::InvalidOrigin], wait: 10, max: 6) {
44
+ resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).create_distribution_with_tags(
45
+ distribution_config_with_tags: {
46
+ distribution_config: params,
47
+ tags: {
48
+ items: @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }
49
+ }
50
+ }
51
+ )
52
+ @cloud_id = resp.distribution.id
53
+ }
54
+ ready?
55
+ rescue ::Aws::CloudFront::Errors::InvalidViewerCertificate => e
56
+ cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate(
57
+ name: @config['certificate']["name"],
58
+ id: @config['certificate']["id"],
59
+ region: @config['certificate']['region'],
60
+ credentials: @config['certificate']['credentials']
61
+ )
62
+ raise MuError.new e.message, details: { "aliases" => @config['aliases'], "certificate domains" => cert_domains }
63
+ rescue ::Aws::CloudFront::Errors::InvalidOrigin => e
64
+ raise MuError.new e.message, details: params[:origins]
65
+ rescue ::Aws::CloudFront::Errors::InvalidArgument => e
66
+ raise MuError.new e.message, details: params
67
+ end
68
+ end
69
+
70
+ # Called automatically by {MU::Deploy#createResources}
71
+ def groom
72
+ params = get_properties
73
+
74
+ if !@config['dns_records'].nil?
75
+ if !MU::Cloud::AWS.isGovCloud?
76
+ MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: cloud_desc.domain_name)
77
+ end
78
+ end
79
+ MU.log "CloudFront Distribution #{@config['name']} at #{cloud_desc.domain_name}", MU::SUMMARY
80
+ if @config['aliases']
81
+ @config['aliases'].each { |a|
82
+ MU.log "Alias for CloudFront Distribution #{@config['name']}: #{a}", MU::SUMMARY
83
+ }
84
+ end
85
+
86
+ # Make sure we show up in the bucket policy of our target bucket,
87
+ # if it's a sibling in this deploy
88
+ cloud_desc(use_cache: false).origins.items.each { |o|
89
+ if o.s3_origin_config
90
+ id = o.s3_origin_config.origin_access_identity.sub(/^origin-access-identity\/cloudfront\//, '')
91
+ bucketref = get_bucketref_from_domain(o.domain_name)
92
+ next if !bucketref or !bucketref.kitten
93
+ resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).get_cloud_front_origin_access_identity(id: id)
94
+ # bucketref.kitten.allowPrincipal("arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity "+id, doc_id: "PolicyForCloudFrontPrivateContent", permissions: ["GetObject"])
95
+ bucketref.kitten.allowPrincipal(resp.cloud_front_origin_access_identity.s3_canonical_user_id, doc_id: "PolicyForCloudFrontPrivateContent", permissions: ["GetObject"], name: @mu_name)
96
+ end
97
+ }
98
+
99
+ end
100
+
101
+ # Canonical Amazon Resource Number for this resource
102
+ # @return [String]
103
+ def arn
104
+ cloud_desc ? cloud_desc.arn : nil
105
+ end
106
+
107
+ # Return the metadata for this cdn
108
+ # @return [Hash]
109
+ def notify
110
+ MU.structToHash(cloud_desc, stringify_keys: true)
111
+ end
112
+
113
+ # Wait until the distribution is ready (status is +Deployed+)
114
+ def ready?
115
+ self.class.ready?(@cloud_id, credentials: @credentials)
116
+ end
117
+
118
+ # Wait until a distribution is ready (status is +Deployed+)
119
+ # @param id [String]
120
+ # @param credentials [String]
121
+ def self.ready?(id, credentials: nil)
122
+ desc = nil
123
+ MU.retrier([], loop_if: Proc.new { !desc or desc.status != "Deployed" }, wait: 30, max:60) {
124
+ desc = MU::Cloud::AWS.cloudfront(credentials: credentials).get_distribution(id: id).distribution
125
+ }
126
+ end
127
+
128
+ # Does this resource type exist as a global (cloud-wide) artifact, or
129
+ # is it localized to a region/zone?
130
+ # @return [Boolean]
131
+ def self.isGlobal?
132
+ true
133
+ end
134
+
135
+ # Denote whether this resource implementation is experiment, ready for
136
+ # testing, or ready for production use.
137
+ def self.quality
138
+ MU::Cloud::ALPHA
139
+ end
140
+
141
+ # Remove all cdns associated with the currently loaded deployment.
142
+ # @param noop [Boolean]: If true, will only print what would be done
143
+ # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
144
+ # @return [void]
145
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
146
+
147
+ resp = MU::Cloud::AWS.cloudfront(credentials: credentials).list_distributions
148
+ if resp and resp.distribution_list and resp.distribution_list.items
149
+ delete_threads = []
150
+ ids = Hash[resp.distribution_list.items.map { |distro| [distro.arn, distro] }]
151
+
152
+ ids.each_key { |arn|
153
+ tags = MU::Cloud::AWS.cloudfront(credentials: credentials).list_tags_for_resource(resource: arn).tags.items
154
+
155
+ found_muid = found_master = false
156
+ name = nil
157
+ tags.each { |tag|
158
+ name = tag.value if tag.key == "Name"
159
+ found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id
160
+ found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip
161
+ }
162
+
163
+ if found_muid and (ignoremaster or found_master)
164
+ delete_threads << Thread.new(arn, name) { |my_arn, my_name|
165
+ current = MU::Cloud::AWS.cloudfront(credentials: credentials).get_distribution_config(id: ids[my_arn].id)
166
+ etag = current.etag
167
+
168
+ if !noop
169
+
170
+ if current.distribution_config.enabled
171
+ newcfg = MU.structToHash(current.distribution_config)
172
+ newcfg[:enabled] = false
173
+ MU.log "Disabling CloudFront distribution #{my_name ? my_name : ids[my_arn].id})", MU::NOTICE
174
+ updated = MU::Cloud::AWS.cloudfront(credentials: credentials).update_distribution(id: ids[my_arn].id, distribution_config: newcfg, if_match: etag)
175
+ etag = updated.etag
176
+ end
177
+
178
+ end
179
+
180
+ MU.log "Deleting CloudFront distribution #{my_name ? my_name : ids[my_arn].id})"
181
+ if !noop
182
+ ready?(ids[my_arn].id, credentials: credentials)
183
+ MU::Cloud::AWS.cloudfront(credentials: credentials).delete_distribution(id: ids[my_arn].id, if_match: etag)
184
+ end
185
+ }
186
+ end
187
+ }
188
+ delete_threads.each { |t| t.join }
189
+ end
190
+
191
+ resp = MU::Cloud::AWS.cloudfront(credentials: credentials).list_cloud_front_origin_access_identities
192
+ if resp and resp.cloud_front_origin_access_identity_list and
193
+ resp.cloud_front_origin_access_identity_list.items.each and
194
+ deploy_id =~ /-\d{10}-[A-Z]{2}/
195
+ resp.cloud_front_origin_access_identity_list.items.each { |ident|
196
+ if ident.comment =~ /^#{Regexp.quote(deploy_id)}-/
197
+ MU.log "Deleting CloudFront origin access identity #{ident.id} (#{ident.comment})"
198
+ if !noop
199
+ getresp = MU::Cloud::AWS.cloudfront(credentials: credentials).get_cloud_front_origin_access_identity(id: ident.id)
200
+ begin
201
+ MU::Cloud::AWS.cloudfront(credentials: credentials).delete_cloud_front_origin_access_identity(id: ident.id, if_match: getresp.etag)
202
+ rescue ::Aws::CloudFront::Errors::CloudFrontOriginAccessIdentityInUse => e
203
+ MU.log "Got #{e.message} deleting #{ident.id}; it likely belongs to a distribution we can't to delete", MU::WARN, details: ident
204
+ end
205
+ end
206
+ end
207
+ }
208
+ end
209
+ end
210
+
211
+ # Locate an existing event.
212
+ # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching CloudWatch Event
213
+ def self.find(**args)
214
+ found = {}
215
+
216
+ MU::Cloud::AWS.cloudfront(credentials: args[:credentials]).list_distributions.distribution_list.items.each { |d|
217
+ next if args[:cloud_id] and ![d.id, d.arn].include?(args[:cloud_id])
218
+ found[d.id] = d
219
+ }
220
+
221
+ found
222
+ end
223
+
224
+ # Reverse-map our cloud description into a runnable config hash.
225
+ # We assume that any values we have in +@config+ are placeholders, and
226
+ # calculate our own accordingly based on what's live in the cloud.
227
+ def toKitten(**_args)
228
+ bok = {
229
+ "cloud" => "AWS",
230
+ "credentials" => @config['credentials'],
231
+ "cloud_id" => @cloud_id
232
+ }
233
+
234
+ if !cloud_desc
235
+ MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
236
+ return nil
237
+ end
238
+
239
+ resp = MU::Cloud::AWS.cloudfront(credentials: @credentials).list_tags_for_resource(resource: arn)
240
+ if resp and resp.tags and resp.tags.items
241
+ tags = MU.structToHash(resp.tags.items, stringify_keys: true)
242
+ bok['name'] = MU::Adoption.tagsToName(tags)
243
+ bok['tags'] = tags if !tags.empty?
244
+ end
245
+
246
+ if !bok['name']
247
+ bok['name'] = if cloud_desc.domain_name !~ /\.cloudfront\.net$/
248
+ cloud_desc.domain_name.sub(/\..*/, '')
249
+ elsif cloud_desc.aliases and !cloud_desc.aliases.items.empty?
250
+ cloud_desc.aliases.items.first.sub(/\..*/, '')
251
+ # XXX maybe try to guess from the name of an origin resource?
252
+ else
253
+ @cloud_id
254
+ end
255
+ end
256
+
257
+ cloud_desc.origins.items.each { |o|
258
+ bok['origins'] ||= []
259
+ origin = {
260
+ "path" => o.origin_path,
261
+ "name" => o.id
262
+ }
263
+ if o.s3_origin_config
264
+ origin["bucket"] = get_bucketref_from_domain(o.domain_name)
265
+ end
266
+ origin["domain_name"] = o.domain_name if !origin["bucket"]
267
+ if o.custom_origin_config
268
+ origin["http_port"] = o.custom_origin_config.http_port
269
+ origin["https_port"] = o.custom_origin_config.https_port
270
+ origin["protocol_policy"] = o.custom_origin_config.origin_protocol_policy
271
+ origin["ssl_protocols"] = o.custom_origin_config.origin_ssl_protocols.items
272
+ end
273
+
274
+ if o.custom_headers and !o.custom_headers.empty?
275
+ end
276
+
277
+ bok['origins'] << origin
278
+ }
279
+
280
+ if cloud_desc.aliases and cloud_desc.aliases.items and
281
+ !cloud_desc.aliases.items.empty?
282
+ bok['aliases'] = cloud_desc.aliases.items
283
+ end
284
+
285
+ bok['disabled'] = true if !cloud_desc.enabled
286
+
287
+ bok['behaviors'] = []
288
+
289
+ add_behavior = Proc.new { |b, default|
290
+ behavior = {}
291
+
292
+ behavior["origin"] = b.target_origin_id
293
+ behavior["path_pattern"] = b.path_pattern if b.respond_to?(:path_pattern)
294
+ behavior["protocol_policy"] = b.viewer_protocol_policy
295
+ if b.lambda_function_associations and !b.lambda_function_associations.items.empty?
296
+ b.lambda_function_associations.items.each { |f|
297
+ behavior['functions'] ||= []
298
+ f.lambda_function_arn.match(/^arn:.*?:lambda:([^:]+?):(\d*):function:([^:]+)/)
299
+ region = Regexp.last_match[1]
300
+ acct = Regexp.last_match[2]
301
+ id = Regexp.last_match[3]
302
+ behavior['functions'] << MU::Config::Ref.get(
303
+ id: id,
304
+ region: region,
305
+ type: "functions",
306
+ event_type: f.event_type,
307
+ include_body: f.include_body,
308
+ cloud: "AWS",
309
+ credentials: @credentials,
310
+ habitat: MU::Config::Ref.get(
311
+ id: acct,
312
+ cloud: "AWS",
313
+ credentials: @credentials
314
+ )
315
+ )
316
+ }
317
+ [:min_ttl, :default_ttl, :max_ttl].each { |ttl|
318
+ behavior[ttl.to_s] = b.send(ttl)
319
+ }
320
+ end
321
+ bok['behaviors'] << behavior
322
+ }
323
+
324
+ add_behavior.call(cloud_desc.default_cache_behavior, true)
325
+
326
+ if cloud_desc.cache_behaviors and
327
+ !cloud_desc.cache_behaviors.items.empty?
328
+ cloud_desc.cache_behaviors.items.each { |b|
329
+ add_behavior.call(b, false)
330
+ }
331
+ end
332
+
333
+ bok
334
+ end
335
+
336
+
337
+ # Cloud-specific configuration properties.
338
+ # @param _config [MU::Config]: The calling MU::Config object
339
+ # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
340
+ def self.schema(_config)
341
+ toplevel_required = []
342
+
343
+ schema = {
344
+ "disabled" => {
345
+ "type" => "boolean",
346
+ "description" => "Flag this CloudFront distribution as disabled",
347
+ "default" => false
348
+ },
349
+ "certificate" => MU::Config::Ref.schema(type: "certificate", desc: "Required if any domains have been specified with +aliases+; parser will attempt to autodetect a valid ACM or IAM certificate if not specified.", omit_fields: ["cloud", "tag", "deploy_id"]),
350
+ "behaviors" => {
351
+ "items" => {
352
+ "properties" => {
353
+ "min_ttl" => {
354
+ "type" => "integer",
355
+ "description" => "The minimum amount of time that you want objects to stay in CloudFront caches before CloudFront forwards another request to your origin to determine whether the object has been updated.",
356
+ "default" => 0
357
+ },
358
+ "default_ttl" => {
359
+ "type" => "integer",
360
+ "description" => "The default amount of time that you want objects to stay in CloudFront caches before CloudFront forwards another request to your origin to determine whether the object has been updated.",
361
+ "default" => 86400
362
+ },
363
+ "max_ttl" => {
364
+ "type" => "integer",
365
+ "description" => "The maximum amount of time that you want objects to stay in CloudFront caches before CloudFront forwards another request to your origin to determine whether the object has been updated.",
366
+ "default" => 31536000
367
+ },
368
+ "protocol_policy" => {
369
+ "type" => "string",
370
+ "enum" => %w{allow-all https-only redirect-to-https},
371
+ "default" => "redirect-to-https"
372
+ },
373
+ "functions" => {
374
+ "type" => "array",
375
+ "items" => MU::Config::Ref.schema(type: "functions", desc: "Add a Lambda function which can be invoked on requests or responses through this distribution.")
376
+ },
377
+ "forwarded_values" => {
378
+ "type" => "object",
379
+ "description" => "HTTP request artifacts to include in requests passed to our back-end +origin+",
380
+ "default" => {
381
+ "query_string" => false
382
+ },
383
+ "properties" => {
384
+ "query_string" => {
385
+ "type" => "boolean",
386
+ "description" => "Indicates whether you want CloudFront to forward query strings to the origin that is associated with this cache behavior and cache based on the query string parameters.",
387
+ "default" => false
388
+ },
389
+ "cookies" => {
390
+ "type" => "object",
391
+ "description" => "A complex type that specifies whether you want CloudFront to forward cookies to the origin and, if so, which ones.",
392
+ "default" => {
393
+ "forward" => "none"
394
+ },
395
+ "properties" => {
396
+ "forward" => {
397
+ "type" => "string",
398
+ "description" => "Specifies which cookies to forward to the origin for this cache behavior: all, none, or the list of cookies specified in +whitelisted_names+",
399
+ "enum" => %w{none whitelist all}
400
+ },
401
+ "whitelisted_names" => {
402
+ "type" => "array",
403
+ "items" => {
404
+ "description" => "Required if you specify whitelist for the value of +forward+",
405
+ "type" => "string"
406
+ }
407
+ },
408
+ }
409
+ },
410
+ "headers" => {
411
+ "type" => "array",
412
+ "items" => {
413
+ "description" => "Specifies the headers, if any, that you want CloudFront to forward to the origin for this cache behavior (whitelisted headers).",
414
+ "type" => "string"
415
+ }
416
+ },
417
+ "query_string_cache_keys" => {
418
+ "type" => "array",
419
+ "items" => {
420
+ "description" => "Indicates whether you want CloudFront to forward query strings to the origin that is associated with this cache behavior and cache based on the query string parameters",
421
+ "type" => "string"
422
+ }
423
+ }
424
+ }
425
+ }
426
+ }
427
+ }
428
+ },
429
+ "origins" => {
430
+ "items" => {
431
+ "properties" => {
432
+ "bucket" => MU::Config::Ref.schema(type: "buckets", desc: "Reference an S3 bucket for use as an origin"),
433
+ "endpoint" => MU::Config::Ref.schema(type: "endpoints", desc: "Reference an API Gateway for use as an origin"),
434
+ "loadbalancer" => MU::Config::Ref.schema(type: "loadbalancers", desc: "Reference a Load Balancer for use as an origin"),
435
+ "connection_attempts" => {
436
+ "type" => "integer",
437
+ "default" => 3
438
+ },
439
+ "connection_timeout" => {
440
+ "type" => "integer",
441
+ "default" => 10
442
+ },
443
+ "protocol_policy" => {
444
+ "type" => "string",
445
+ "enum" => %w{http-only https-only match-viewer},
446
+ "default" => "match-viewer"
447
+ },
448
+ "ssl_protocols" => {
449
+ "type" => "array",
450
+ "default" => ["TLSv1.2"],
451
+ "items" => {
452
+ "type" => "string",
453
+ "enum" => %w{SSLv3 TLSv1 TLSv1.1 TLSv1.2},
454
+ }
455
+ },
456
+ "http_port" => {
457
+ "type" => "integer",
458
+ "default" => 80
459
+ },
460
+ "https_port" => {
461
+ "type" => "integer",
462
+ "default" => 443
463
+ },
464
+ "custom_headers" => {
465
+ "type" => "array",
466
+ "items" => {
467
+ "description" => "A list of HTTP header names and values that CloudFront adds to requests it sends to the origin.",
468
+ "type" => "object",
469
+ "required" => ["key", "value"],
470
+ "properties" => {
471
+ "key" => {
472
+ "type" => "string"
473
+ },
474
+ "value" => {
475
+ "type" => "string"
476
+ },
477
+ }
478
+ }
479
+ }
480
+ }
481
+ }
482
+ }
483
+ }
484
+
485
+ schema["behaviors"]["items"]["properties"]["functions"]["items"]["include_body"] = {
486
+ "type" => "boolean",
487
+ "default" => false
488
+ }
489
+ schema["behaviors"]["items"]["properties"]["functions"]["items"]["event_type"] = {
490
+ "type" => "string",
491
+ "enum" => %w{viewer-request viewer-response origin-request origin-response},
492
+ "default" => "viewer-request"
493
+ }
494
+
495
+ [toplevel_required, schema]
496
+ end
497
+
498
+ # Cloud-specific pre-processing of {MU::Config::BasketofKittens::cdns}, bare and unvalidated.
499
+ # @param cdn [Hash]: The resource to process and validate
500
+ # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
501
+ # @return [Boolean]: True if validation succeeded, False otherwise
502
+ def self.validateConfig(cdn, configurator)
503
+ ok = true
504
+
505
+ cdn['origins'].each { |o|
506
+ count = 0
507
+ ['bucket', 'endpoint', 'loadbalancer'].each { |sib_type|
508
+ if o[sib_type]
509
+ if count > 0
510
+ ok = false
511
+ MU.log "Origin in CloudFront distro #{cdn['name']} may specify at most one of bucket, endpoint, or loadbalancer.", MU::ERR
512
+ end
513
+ target_ref = MU::Config::Ref.get(o[sib_type])
514
+ if target_ref.name
515
+ MU::Config.addDependency(cdn, target_ref.name, sib_type, phase: "groom")
516
+ end
517
+ count += 1
518
+ end
519
+ }
520
+ }
521
+
522
+ cert_domains = nil
523
+
524
+ if cdn['certificate']
525
+ cert_arn, cert_domains = MU::Cloud::AWS.resolveSSLCertificate(cdn['certificate'], region: cdn['region'], credentials: cdn['credentials'])
526
+ if !cert_arn
527
+ MU.log "Failed to find an ACM or IAM certificate specified in CloudFront distribution #{cdn['name']}", MU::ERR, details: cdn['certificate'].to_h
528
+ ok = false
529
+ end
530
+ end
531
+
532
+ if cdn['aliases']
533
+ cdn['aliases'].each { |a|
534
+ if !cdn['certificate']
535
+ foundcert, cert_domains = MU::Cloud::AWS.findSSLCertificate(name: a, region: cdn['region'], credentials: cdn['credentials'], raise_on_missing: false)
536
+ if !foundcert
537
+ foundcert, cert_domains = MU::Cloud::AWS.findSSLCertificate(name: a.sub(/^[^\.]+\./, '*.'), region: cdn['region'], credentials: cdn['credentials'], raise_on_missing: false)
538
+ end
539
+ if !foundcert
540
+ MU.log "Failed to find an ACM or IAM certificate matching #{a} for CloudFront distribution #{cdn['name']}", MU::ERR
541
+ ok = false
542
+ else
543
+ cdn['certificate'] = {
544
+ "id" => foundcert,
545
+ "credentials" => cdn['credentials']
546
+ }
547
+ MU.log "Auto-detected SSL certificate for CloudFront distribution #{cdn['name']} alias #{a}", MU::NOTICE, details: cdn['certificate']['id']
548
+ end
549
+ else
550
+ if !MU::Cloud::AWS.nameMatchesCertificate(a, cdn['certificate']['id'])
551
+ MU.log "Alias #{a} in CloudFront distro #{cdn['name']} does not appear to fit any domains on our SSL certificate", MU::ERR, details: cert_domains
552
+ ok = false
553
+ end
554
+ end
555
+ }
556
+ end
557
+
558
+ if cdn['dns_records'] and cdn['certificate']
559
+ cdn['dns_records'].each { |rec|
560
+ next if !rec['name']
561
+ dnsname = MU::Cloud.resourceClass("AWS", "DNSZone").recordToName(rec)
562
+ if MU::Cloud::AWS.nameMatchesCertificate(dnsname, cdn['certificate']['id'])
563
+ cdn['aliases'] ||= []
564
+ cdn['aliases'] << dnsname if !cdn['aliases'].include?(dnsname)
565
+ end
566
+ }
567
+ end
568
+
569
+ path_patterns = {}
570
+ cdn['behaviors'].each { |b|
571
+ b['path_pattern'] ||= "*"
572
+ path_patterns[b['path_pattern']] ||= 0
573
+ path_patterns[b['path_pattern']] += 1
574
+ }
575
+ path_patterns.each_pair { |pattern, origins|
576
+ if origins > 1
577
+ MU.log "CDN #{cdn['name']} has #{origins.to_s} uses of path_pattern '#{pattern}' in its behavior list (must be unique)", MU::ERR, details: cdn['behaviors']
578
+ ok = false
579
+ end
580
+ }
581
+
582
+ ok
583
+ end
584
+
585
+ private
586
+
587
+ def get_properties
588
+ params = {
589
+ default_root_object: @config['default_object'],
590
+ caller_reference: @mu_name, # eh, probably should be random
591
+ origins: {
592
+ quantity: @config['origins'].size,
593
+ items: []
594
+ },
595
+ comment: @deploy.deploy_id,
596
+ enabled: !(@config['disabled'])
597
+ }
598
+
599
+ if @config['certificate']
600
+ params[:viewer_certificate] = {
601
+ ssl_support_method: "sni-only"
602
+ }
603
+ if @config['certificate']['id'] =~ /^arn:aws(?:-us-gov)?:iam/
604
+ params[:viewer_certificate][:iam_certificate_id] = @config['certificate']['id']
605
+ params[:viewer_certificate][:certificate_source] = "iam"
606
+ elsif @config['certificate']['id'] =~ /^arn:aws(?:-us-gov)?:acm/
607
+ params[:viewer_certificate][:acm_certificate_arn] = @config['certificate']['id']
608
+ params[:viewer_certificate][:certificate_source] = "acm"
609
+ end
610
+
611
+ end
612
+
613
+ @config['origins'].each { |o|
614
+ origin = {
615
+ id: o['name'],
616
+ }
617
+ sib_obj = nil
618
+ ['bucket', 'endpoint', 'loadbalancer'].each { |sib_type|
619
+ if o[sib_type]
620
+ sib_obj = MU::Config::Ref.get(o[sib_type]).kitten(@deploy, cloud: "AWS")
621
+ if !sib_obj
622
+ raise MuError.new "Failed to resolve #{sib_type} referenced in CloudFront distribution #{@config['name']}", details: o[sib_type].to_h
623
+ end
624
+ break
625
+ end
626
+ }
627
+ if o['bucket']
628
+ origin[:domain_name] = sib_obj.cloud_desc["name"]+".s3.amazonaws.com"
629
+ origin[:origin_path] = o['path'] if o['path']
630
+ origin[:s3_origin_config] = {
631
+ origin_access_identity: @origin_access_identity
632
+ }
633
+ elsif o['endpoint']
634
+ origin[:domain_name] = sib_obj.cloud_id+".execute-api."+sib_obj.config['region']+".amazonaws.com"
635
+ origin[:custom_origin_config] = {
636
+ origin_protocol_policy: "https-only"
637
+ }
638
+ if sib_obj.config['deploy_to']
639
+ origin[:origin_path] ||= "/"+sib_obj.config['deploy_to']
640
+ end
641
+ elsif o['loadbalancer']
642
+ origin[:domain_name] = sib_obj.cloud_desc.dns_name
643
+ origin[:origin_path] = o['path'] if o['path']
644
+ else # XXX make sure parser guarantees these are present
645
+ origin[:domain_name] = o['domain_name']
646
+ origin[:origin_path] = o['path']
647
+ end
648
+
649
+ if o['custom_headers']
650
+ origin[:custom_headers] = {
651
+ quantity: o['custom_headers'].size,
652
+ items: o['custom_headers'].map { |h|
653
+ {
654
+ header_name: h['key'],
655
+ header_value: h['value']
656
+ }
657
+ }
658
+ }
659
+ end
660
+
661
+ [:connection_attempts, :connection_timeout].each { |field|
662
+ origin[field] ||= o[field.to_s]
663
+ }
664
+ if !origin[:s3_origin_config]
665
+ maplet = {
666
+ 'protocol_policy' => :origin_protocol_policy,
667
+ 'ssl_protocols' => :origin_ssl_protocols,
668
+ 'http_port' => :http_port,
669
+ 'https_port' => :https_port
670
+ }
671
+ maplet.each_pair { |field, paramfield|
672
+ next if !o[field]
673
+ origin[:custom_origin_config] ||= {}
674
+ origin[:custom_origin_config][paramfield] ||= if o[field.to_s].is_a?(Array)
675
+ {
676
+ quantity: o[field].size,
677
+ items: o[field]
678
+ }
679
+ else
680
+ o[field]
681
+ end
682
+ }
683
+ end
684
+
685
+ params[:origins][:items] << origin
686
+ }
687
+
688
+ # if we have any placeholder DNS records that are intended to be
689
+ # filled out with our runtime @mu_name, do so, and add an alias if
690
+ # applicable
691
+ if @config['dns_records']
692
+ @config['dns_records'].each { |rec|
693
+ if !rec['name']
694
+ rec['name'] = @mu_name.downcase
695
+ dnsname = MU::Cloud.resourceClass("AWS", "DNSZone").recordToName(rec)
696
+ if @config['certificate'] and MU::Cloud::AWS.nameMatchesCertificate(dnsname, @config['certificate']['id'])
697
+ @config['aliases'] ||= []
698
+ @config['aliases'] << dnsname if !@config['aliases'].include?(dnsname)
699
+ end
700
+ end
701
+ }
702
+ end
703
+
704
+ if @config['aliases']
705
+ params[:aliases] = {
706
+ items: @config['aliases'],
707
+ quantity: @config['aliases'].size
708
+ }
709
+ end
710
+
711
+ # XXX config parser should guarantee a default behavior
712
+ @config['behaviors'].each { |b|
713
+ b['origin'] ||= @config['origins'].first['name']
714
+ behavior = {
715
+ target_origin_id: b['origin'],
716
+ viewer_protocol_policy: b['protocol_policy'],
717
+ min_ttl: b['min_ttl'],
718
+ max_ttl: b['max_ttl'],
719
+ default_ttl: b['default_ttl'],
720
+ }
721
+ behavior[:trusted_signers] = {
722
+ enabled: false,
723
+ quantity: 0,
724
+ # items: []
725
+ }
726
+ behavior[:forwarded_values] = {
727
+ query_string: b['forwarded_values']['query_string'],
728
+ cookies: {
729
+ forward: b['forwarded_values']['cookies']['forward']
730
+ }
731
+ }
732
+ if b['forwarded_values']['cookies']['whitelisted_names']
733
+ behavior[:forwarded_values][:cookies][:whitelisted_names] = {
734
+ quantity: b['forwarded_values']['cookies']['whitelisted_names'].size,
735
+ items: b['forwarded_values']['cookies']['whitelisted_names']
736
+ }
737
+ end
738
+ ['headers', 'query_string_cache_keys'].each { |field|
739
+ if b['forwarded_values'][field]
740
+ behavior[:forwarded_values][field.to_sym] = {
741
+ quantity: b['forwarded_values'][field].size,
742
+ items: b['forwarded_values'][field]
743
+ }
744
+ end
745
+ }
746
+
747
+ if @config['behaviors'].size == 1 or b['path_pattern'] == "*"
748
+ params[:default_cache_behavior] = behavior
749
+ else
750
+ behavior[:path_pattern] = b['path_pattern']
751
+ params[:cache_behaviors] ||= {
752
+ quantity: (@config['behaviors'].size-1),
753
+ items: []
754
+ }
755
+ params[:cache_behaviors][:items] << behavior
756
+ end
757
+ }
758
+
759
+ params
760
+ end
761
+
762
+ def get_bucketref_from_domain(domain_name)
763
+ buckets = MU::Cloud.resourceClass("AWS", "Bucket").find(credentials: @credentials, allregions: true, cloud_id: domain_name.sub(/\..*/, ''))
764
+ if buckets and buckets.size == 1
765
+ return MU::Config::Ref.get(
766
+ id: buckets.keys.first,
767
+ type: "buckets",
768
+ region: buckets.values.first["region"],
769
+ credentials: @credentials,
770
+ cloud: "AWS"
771
+ )
772
+ else
773
+ MU.log "Failed to locate or isolate a bucket object from #{domain_name}", MU::WARN, details: buckets.keys
774
+ end
775
+
776
+ nil
777
+ end
778
+
779
+ end
780
+ end
781
+ end
782
+ end