cloud-mu 3.1.6 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +1 -1
  3. data/bin/mu-adopt +15 -12
  4. data/bin/mu-azure-tests +57 -0
  5. data/bin/mu-cleanup +2 -4
  6. data/bin/mu-configure +37 -1
  7. data/bin/mu-deploy +3 -3
  8. data/bin/mu-findstray-tests +25 -0
  9. data/bin/mu-gen-docs +2 -4
  10. data/bin/mu-load-config.rb +2 -1
  11. data/bin/mu-run-tests +37 -12
  12. data/cloud-mu.gemspec +4 -4
  13. data/cookbooks/mu-tools/attributes/default.rb +7 -0
  14. data/cookbooks/mu-tools/libraries/helper.rb +87 -3
  15. data/cookbooks/mu-tools/recipes/apply_security.rb +39 -23
  16. data/cookbooks/mu-tools/recipes/aws_api.rb +13 -0
  17. data/cookbooks/mu-tools/recipes/google_api.rb +4 -0
  18. data/cookbooks/mu-tools/recipes/rsyslog.rb +8 -1
  19. data/cookbooks/mu-tools/resources/disk.rb +33 -12
  20. data/cookbooks/mu-tools/resources/mommacat_request.rb +1 -2
  21. data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb +215 -0
  22. data/extras/clean-stock-amis +10 -2
  23. data/extras/generate-stock-images +7 -3
  24. data/extras/image-generators/AWS/centos7.yaml +19 -16
  25. data/extras/image-generators/AWS/{rhel7.yaml → rhel71.yaml} +0 -0
  26. data/extras/image-generators/AWS/{win2k12.yaml → win2k12r2.yaml} +0 -0
  27. data/modules/mommacat.ru +2 -2
  28. data/modules/mu.rb +84 -97
  29. data/modules/mu/adoption.rb +359 -59
  30. data/modules/mu/cleanup.rb +67 -44
  31. data/modules/mu/cloud.rb +108 -1754
  32. data/modules/mu/cloud/database.rb +49 -0
  33. data/modules/mu/cloud/dnszone.rb +44 -0
  34. data/modules/mu/cloud/machine_images.rb +212 -0
  35. data/modules/mu/cloud/providers.rb +81 -0
  36. data/modules/mu/cloud/resource_base.rb +929 -0
  37. data/modules/mu/cloud/server.rb +40 -0
  38. data/modules/mu/cloud/server_pool.rb +1 -0
  39. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  40. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  41. data/modules/mu/cloud/wrappers.rb +178 -0
  42. data/modules/mu/config.rb +122 -80
  43. data/modules/mu/config/alarm.rb +2 -6
  44. data/modules/mu/config/bucket.rb +32 -3
  45. data/modules/mu/config/cache_cluster.rb +2 -2
  46. data/modules/mu/config/cdn.rb +100 -0
  47. data/modules/mu/config/collection.rb +1 -1
  48. data/modules/mu/config/container_cluster.rb +2 -2
  49. data/modules/mu/config/database.rb +84 -105
  50. data/modules/mu/config/database.yml +1 -2
  51. data/modules/mu/config/dnszone.rb +5 -4
  52. data/modules/mu/config/doc_helpers.rb +4 -5
  53. data/modules/mu/config/endpoint.rb +2 -1
  54. data/modules/mu/config/firewall_rule.rb +3 -19
  55. data/modules/mu/config/folder.rb +1 -1
  56. data/modules/mu/config/function.rb +17 -8
  57. data/modules/mu/config/group.rb +1 -1
  58. data/modules/mu/config/habitat.rb +1 -1
  59. data/modules/mu/config/job.rb +89 -0
  60. data/modules/mu/config/loadbalancer.rb +57 -11
  61. data/modules/mu/config/log.rb +1 -1
  62. data/modules/mu/config/msg_queue.rb +1 -1
  63. data/modules/mu/config/nosqldb.rb +1 -1
  64. data/modules/mu/config/notifier.rb +8 -19
  65. data/modules/mu/config/ref.rb +81 -9
  66. data/modules/mu/config/role.rb +1 -1
  67. data/modules/mu/config/schema_helpers.rb +30 -34
  68. data/modules/mu/config/search_domain.rb +1 -1
  69. data/modules/mu/config/server.rb +5 -13
  70. data/modules/mu/config/server_pool.rb +3 -7
  71. data/modules/mu/config/storage_pool.rb +1 -1
  72. data/modules/mu/config/tail.rb +10 -0
  73. data/modules/mu/config/user.rb +1 -1
  74. data/modules/mu/config/vpc.rb +13 -17
  75. data/modules/mu/defaults/AWS.yaml +106 -106
  76. data/modules/mu/defaults/Azure.yaml +1 -0
  77. data/modules/mu/defaults/Google.yaml +1 -0
  78. data/modules/mu/deploy.rb +33 -19
  79. data/modules/mu/groomer.rb +15 -0
  80. data/modules/mu/groomers/chef.rb +3 -0
  81. data/modules/mu/logger.rb +120 -144
  82. data/modules/mu/master.rb +22 -1
  83. data/modules/mu/mommacat.rb +71 -26
  84. data/modules/mu/mommacat/daemon.rb +23 -14
  85. data/modules/mu/mommacat/naming.rb +82 -3
  86. data/modules/mu/mommacat/search.rb +59 -16
  87. data/modules/mu/mommacat/storage.rb +119 -48
  88. data/modules/mu/{clouds → providers}/README.md +1 -1
  89. data/modules/mu/{clouds → providers}/aws.rb +248 -62
  90. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  91. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  92. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  93. data/modules/mu/providers/aws/cdn.rb +782 -0
  94. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  95. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +65 -63
  96. data/modules/mu/providers/aws/database.rb +1747 -0
  97. data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
  98. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  99. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
  100. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  101. data/modules/mu/{clouds → providers}/aws/function.rb +291 -133
  102. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  103. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  104. data/modules/mu/providers/aws/job.rb +469 -0
  105. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
  106. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  107. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  108. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  109. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  110. data/modules/mu/{clouds → providers}/aws/role.rb +112 -78
  111. data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
  112. data/modules/mu/{clouds → providers}/aws/server.rb +120 -145
  113. data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
  114. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  115. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  116. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  117. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  118. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  119. data/modules/mu/{clouds → providers}/aws/vpc.rb +141 -73
  120. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  121. data/modules/mu/{clouds → providers}/azure.rb +4 -1
  122. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  123. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  124. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  125. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  126. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  127. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  128. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  129. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  130. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  131. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  132. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  133. data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
  134. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  135. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  136. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  137. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  138. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  139. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  140. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  141. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  142. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  143. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  144. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  145. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  146. data/modules/mu/{clouds → providers}/google.rb +15 -6
  147. data/modules/mu/{clouds → providers}/google/bucket.rb +2 -2
  148. data/modules/mu/{clouds → providers}/google/container_cluster.rb +29 -14
  149. data/modules/mu/{clouds → providers}/google/database.rb +2 -9
  150. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +3 -3
  151. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  152. data/modules/mu/{clouds → providers}/google/function.rb +4 -4
  153. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  154. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  155. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +2 -2
  156. data/modules/mu/{clouds → providers}/google/role.rb +46 -35
  157. data/modules/mu/{clouds → providers}/google/server.rb +26 -11
  158. data/modules/mu/{clouds → providers}/google/server_pool.rb +11 -11
  159. data/modules/mu/{clouds → providers}/google/user.rb +32 -22
  160. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  161. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  162. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  163. data/modules/mu/{clouds → providers}/google/vpc.rb +38 -3
  164. data/modules/tests/aws-jobs-functions.yaml +46 -0
  165. data/modules/tests/centos6.yaml +15 -0
  166. data/modules/tests/centos7.yaml +15 -0
  167. data/modules/tests/centos8.yaml +12 -0
  168. data/modules/tests/ecs.yaml +2 -2
  169. data/modules/tests/eks.yaml +1 -1
  170. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  171. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  172. data/modules/tests/microservice_app.yaml +288 -0
  173. data/modules/tests/rds.yaml +108 -0
  174. data/modules/tests/regrooms/rds.yaml +123 -0
  175. data/modules/tests/server-with-scrub-muisms.yaml +1 -1
  176. data/modules/tests/super_complex_bok.yml +2 -2
  177. data/modules/tests/super_simple_bok.yml +2 -2
  178. data/spec/mu/clouds/azure_spec.rb +2 -2
  179. metadata +126 -98
  180. data/modules/mu/clouds/aws/database.rb +0 -1974
  181. data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/user.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/user.rb
18
18
  class User
19
19
 
20
20
  # Base configuration schema for a User
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/vpc.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/vpc.rb
18
18
  class VPC
19
19
 
20
20
  # Base configuration schema for a VPC
@@ -417,9 +417,10 @@ module MU
417
417
  using_default_cidr = false
418
418
  if !vpc['ip_block']
419
419
  if configurator.updating and configurator.existing_deploy and
420
+ configurator.existing_deploy.original_config and
420
421
  configurator.existing_deploy.original_config['vpcs']
421
422
  configurator.existing_deploy.original_config['vpcs'].each { |v|
422
- if v['name'] == vpc['name']
423
+ if v['name'].to_s == vpc['name'].to_s
423
424
  vpc['ip_block'] = v['ip_block']
424
425
  vpc['peers'] ||= []
425
426
  vpc['peers'].concat(v['peers'])
@@ -431,6 +432,10 @@ module MU
431
432
  break
432
433
  end
433
434
  }
435
+ if !vpc['ip_block']
436
+ MU.log "Loading existing deploy but can't find IP block of VPC #{vpc['name']}", MU::ERR
437
+ ok = false
438
+ end
434
439
  else
435
440
  using_default_cidr = true
436
441
  vpc['ip_block'] = "10.0.0.0/16"
@@ -539,7 +544,7 @@ module MU
539
544
  # Clouds that don't have some kind of native NAT gateway can also
540
545
  # leverage this host to honor "gateway" => "#NAT" situations.
541
546
  if !can_peer and !already_peered and have_public and vpc["create_bastion"]
542
- serverclass = Object.const_get("MU").const_get("Cloud").const_get(vpc["cloud"]).const_get("Server")
547
+ serverclass = MU::Cloud.resourceClass(vpc["cloud"], "Server")
543
548
  bastion = serverclass.genericNAT.dup
544
549
  bastion["groomer_variables"] = {
545
550
  "nat_ip_block" => vpc["ip_block"].to_s
@@ -558,10 +563,7 @@ module MU
558
563
  "name" => vpc["name"],
559
564
  "subnet_pref" => "public"
560
565
  }
561
- vpc["dependencies"] << {
562
- "type" => "server",
563
- "name" => bastion['name'],
564
- }
566
+ MU::Config.addDependency(vpc, bastion['name'], "server", no_create_wait: true)
565
567
  vpc["bastion"] = MU::Config::Ref.get(
566
568
  name: bastion['name'],
567
569
  cloud: vpc['cloud'],
@@ -596,7 +598,7 @@ module MU
596
598
  MU.log "Skipping malformed VPC peer in #{vpc['name']}", MU::ERR, details: peer
597
599
  next
598
600
  end
599
- peer["#MU_CLOUDCLASS"] = Object.const_get("MU").const_get("Cloud").const_get("VPC")
601
+ peer["#MU_CLOUDCLASS"] = MU::Cloud.loadBaseType("VPC")
600
602
  # We check for multiple siblings because some implementations
601
603
  # (Google) can split declared VPCs into parts to get the mimic the
602
604
  # routing behaviors we expect.
@@ -613,17 +615,11 @@ module MU
613
615
  append_me = { "vpc" => peer["vpc"].dup }
614
616
  append_me['vpc']['name'] = sib['name']
615
617
  append << append_me
616
- vpc["dependencies"] << {
617
- "type" => "vpc",
618
- "name" => sib['name']
619
- }
618
+ MU::Config.addDependency(vpc, sib['name'], "vpc", phase: "groom", no_create_wait: true)
620
619
  end
621
620
  delete << peer
622
621
  else
623
- vpc["dependencies"] << {
624
- "type" => "vpc",
625
- "name" => peer['vpc']["name"]
626
- }
622
+ MU::Config.addDependency(vpc, peer['vpc']['name'], "vpc", phase: "groom", no_create_wait: true)
627
623
  end
628
624
  delete << peer if sib['name'] == vpc['name']
629
625
  }
@@ -800,7 +796,7 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
800
796
  @@reference_cache[vpc_block] ||= ext_vpc if ok
801
797
  end
802
798
  rescue StandardError => e
803
- raise MuError, e.inspect, [caller, e.backtrace]
799
+ raise MuError.new e.inspect, details: { "my call stack" => caller, "exception call stack" => e.backtrace }
804
800
  ensure
805
801
  if !ext_vpc and vpc_block['cloud'] != "CloudFormation"
806
802
  MU.log "Couldn't resolve VPC reference to a unique live VPC in #{parent_type} #{parent['name']} (called by #{caller[0]})", MU::ERR, details: vpc_block
@@ -1,56 +1,56 @@
1
1
  ---
2
- rhel71: &4
3
- us-east-1: ami-0f05fce24aa75ba9f
4
- ap-northeast-1: ami-0c0ec19eb19055763
5
- ap-northeast-2: ami-0717ac5c67c99f745
6
- ap-south-1: ami-03454a4bef3ec6a9a
7
- ap-southeast-1: ami-0f3aa03320c0f6524
8
- ap-southeast-2: ami-0aa5e6888260cdb3c
9
- ca-central-1: ami-03e72964d7646b689
10
- eu-central-1: ami-02df259ca785eff54
11
- eu-north-1: ami-05253c445bdf7777d
12
- eu-west-1: ami-0c21c559f6d0f2401
13
- eu-west-2: ami-057c8d4259087594f
14
- eu-west-3: ami-05a428dc7a7f4ba46
15
- sa-east-1: ami-0a1d1cf6a89a2db56
16
- us-east-2: ami-02f6682c7816b3cfc
17
- us-west-1: ami-04898e596c06e802b
18
- us-west-2: ami-02db5457189a8a8c2
19
- centos6: &3
20
- us-east-1: ami-06b6c01abc6998348
21
- ap-northeast-1: ami-0c5da73fde2cb6437
22
- ap-northeast-2: ami-0134fce6dc00eb00d
23
- ap-south-1: ami-0e59a612e7c84836b
24
- ap-southeast-1: ami-0e16974f528ae0dae
25
- ap-southeast-2: ami-0e2feddf3dbf4d539
26
- ca-central-1: ami-089236a344dadad5f
27
- eu-central-1: ami-0c4eed3fe046c3917
28
- eu-north-1: ami-05f636e89d0362c14
29
- eu-west-1: ami-00c50b11d713f90d3
30
- eu-west-2: ami-06cc78c32eed7f944
31
- eu-west-3: ami-0ba626236ad786c54
32
- sa-east-1: ami-07c3b2a5a41e92376
33
- us-east-2: ami-01129e636778acfbc
34
- us-west-1: ami-0632e646cd5089ffc
35
- us-west-2: ami-0ce4c9f2e1037de53
2
+ rhel71: &5
3
+ us-east-1: ami-0c834836b3bd45e2f
4
+ ap-northeast-1: ami-036bb589253fe929e
5
+ ap-northeast-2: ami-0e39b4957dbc7e14d
6
+ ap-south-1: ami-0cd0554d9a05dddc9
7
+ ap-southeast-1: ami-0bae2684e9ed09b8b
8
+ ap-southeast-2: ami-0711ccf93abe3989b
9
+ ca-central-1: ami-0d75d90f73e417c25
10
+ eu-central-1: ami-0cae3cb53b9bbd783
11
+ eu-north-1: ami-0cd30fe8547a809f7
12
+ eu-west-1: ami-01441cca97c35eb0e
13
+ eu-west-2: ami-051aaf1b532b3e6bc
14
+ eu-west-3: ami-09a7af6793a3e8d09
15
+ sa-east-1: ami-0c4064cfe711311d5
16
+ us-east-2: ami-0124fd8917f59f8ce
17
+ us-west-1: ami-00457c55541605cb4
18
+ us-west-2: ami-02211d4e254a9e10f
19
+ centos6: &4
20
+ us-east-1: ami-0ccdc671f12147a1d
21
+ us-east-2: ami-00d0e8bc2f05ab949
22
+ ap-northeast-1: ami-0726801ceef87f5f8
23
+ ap-northeast-2: ami-05fa4afc4a0493b0a
24
+ ap-south-1: ami-0d6e4f3b6592b3139
25
+ ap-southeast-1: ami-0c988e3dc80b14653
26
+ ap-southeast-2: ami-02ac856fd094675ef
27
+ ca-central-1: ami-0ce7e343953af2292
28
+ eu-central-1: ami-0ce8317423cea27b8
29
+ eu-north-1: ami-0a923b493d5fc9743
30
+ eu-west-1: ami-06e0f02328921c865
31
+ eu-west-2: ami-07ae118c8814df140
32
+ eu-west-3: ami-03c1017cd1ccc6e9d
33
+ sa-east-1: ami-05212ae133b9c3ba1
34
+ us-west-1: ami-0b05ec54412b9f8b0
35
+ us-west-2: ami-0447e036b102b2ca0
36
36
  centos7:
37
- us-east-1: ami-07e6f661e71ad964b
38
- ap-northeast-1: ami-0988180d74897c639
39
- ap-northeast-2: ami-0e77cd1c7024b8ae0
40
- ap-south-1: ami-02bd479122041000a
41
- ap-southeast-1: ami-017767778ef9db671
42
- ap-southeast-2: ami-05b09a58c3964d67d
43
- ca-central-1: ami-0a59a176d810fcc5f
44
- eu-central-1: ami-0b48a421fb05d96af
45
- eu-north-1: ami-02337601ea5dc4a5d
46
- eu-west-1: ami-0b0a55b7423eeac07
47
- eu-west-2: ami-060518b40b25b9eb4
48
- eu-west-3: ami-060957bb3adacd831
49
- sa-east-1: ami-0c706132b35071de6
50
- us-east-2: ami-0db4c266ed0bb958b
51
- us-west-1: ami-0980f6eb52c998793
52
- us-west-2: ami-07f2ed4755c01c05c
53
- ubuntu16: &2
37
+ us-east-1: ami-0be9d646b29a9f51d
38
+ ap-northeast-1: ami-0ea500fc488406ad8
39
+ ap-northeast-2: ami-062d5536e5ebf04e9
40
+ ap-south-1: ami-0c2bf51384a5dcd92
41
+ ap-southeast-1: ami-0a8022e9bb353022f
42
+ ap-southeast-2: ami-03d47c88f2e01203b
43
+ ca-central-1: ami-0f0dc8a3e18a28544
44
+ eu-central-1: ami-0ea618bc5a1f372a2
45
+ eu-north-1: ami-02b062056e7dd4741
46
+ eu-west-1: ami-03851110db1a143e7
47
+ eu-west-2: ami-03800a8fe524171d2
48
+ eu-west-3: ami-074acd461ca002f0a
49
+ sa-east-1: ami-00c88550221a205f9
50
+ us-east-2: ami-098da5da2eef484e5
51
+ us-west-1: ami-0eea632474ef51860
52
+ us-west-2: ami-075aad0e6e9fc5654
53
+ ubuntu16: &3
54
54
  us-east-1: ami-bcdc16c6
55
55
  us-west-1: ami-1b17257b
56
56
  us-west-2: ami-19e92861
@@ -73,56 +73,56 @@ ubuntu14:
73
73
  ap-southeast-1: ami-2855964b
74
74
  ap-southeast-2: ami-d19fc4b2
75
75
  win2k12r2: &1
76
- us-east-1: ami-003aea65bc2e7136a
77
- us-east-2: ami-0163293e39ba504c2
78
- ca-central-1: ami-055689dd92f29d2aa
79
- us-west-2: ami-0ce87dda2c9244e57
80
- us-west-1: ami-00d9cf64bd2fafa44
81
- eu-west-1: ami-026d7427b9fadad40
82
- eu-west-2: ami-036a22c0780551794
83
- eu-west-3: ami-05e3d9b79bdc10861
84
- eu-north-1: ami-063eb48504c7d73f1
85
- sa-east-1: ami-0a8c1829a5e650bc5
86
- eu-central-1: ami-0ea20cef52335b008
87
- ap-northeast-1: ami-08db2dc67228dbb90
88
- ap-south-1: ami-012241411db3f09c3
89
- ap-northeast-2: ami-0368c224de1d20502
90
- ap-southeast-1: ami-028ef74e1edc3943a
91
- ap-southeast-2: ami-09e03eab1b1bc151b
92
- win2k16: &5
93
- us-east-1: ami-02801a2c8dcbfb883
94
- us-east-2: ami-0ca4f779a2a58a7ea
95
- ca-central-1: ami-05d3854d9d6e9bcc5
96
- us-west-2: ami-091f4a88ce32d28b6
97
- eu-west-1: ami-0b938c9b23ed7d18c
98
- us-west-1: ami-0fd744c3fbe8260f2
99
- eu-west-2: ami-071a89b959c5eda27
100
- eu-west-3: ami-0b206e3dbda9ff9eb
101
- eu-central-1: ami-0dd9bdad31dd0d3ce
102
- sa-east-1: ami-0d69b8d6c0f9a7bae
103
- ap-northeast-1: ami-02eb4a6f519bc3190
104
- ap-south-1: ami-0666fd543ac8b5501
105
- ap-northeast-2: ami-01277c81f9b91cf77
106
- ap-southeast-2: ami-0426a246f9b0ccadd
107
- ap-southeast-1: ami-07ecb0d55c2eb7247
108
- eu-north-1: ami-047811530583b6d08
76
+ us-east-1: ami-0d28b9a40ed446e35
77
+ us-east-2: ami-010d247b7ee850d55
78
+ ca-central-1: ami-0c223858875f62d11
79
+ us-west-2: ami-01d188c5c06078fee
80
+ us-west-1: ami-0dbcc051c49ec24ec
81
+ eu-west-1: ami-080c7b4d6e32bf9f3
82
+ eu-west-2: ami-01b1edb5894a54bcc
83
+ eu-west-3: ami-09445cdc7a2acb1c0
84
+ eu-north-1: ami-0093cc63496e435df
85
+ sa-east-1: ami-04d015c8b371ba7b3
86
+ eu-central-1: ami-04d6144bcbb029141
87
+ ap-northeast-1: ami-07fad72f121aa157c
88
+ ap-south-1: ami-0b3241fd09b1ce87f
89
+ ap-northeast-2: ami-0ba7cd822e36dc0df
90
+ ap-southeast-1: ami-0648522a4cb50953c
91
+ ap-southeast-2: ami-0fc0bd73cd61a970f
92
+ win2k16: &2
93
+ us-east-1: ami-018151f8c8339a093
94
+ us-east-2: ami-09e045936c7d9ecd5
95
+ ca-central-1: ami-0a16abf0f1c35667a
96
+ us-west-2: ami-00d4216f80a82894d
97
+ eu-west-1: ami-07afc1525928ccad7
98
+ us-west-1: ami-0d2f7d4198b79a625
99
+ eu-west-2: ami-04eb62f17efc84a37
100
+ eu-west-3: ami-0b4affcd5848cf50c
101
+ eu-central-1: ami-0a92ae047ebc7a3da
102
+ sa-east-1: ami-0cc9a87c95fb37832
103
+ ap-northeast-1: ami-014c730050acef11d
104
+ ap-south-1: ami-0c3c8739263aa4844
105
+ ap-northeast-2: ami-0018a05eb15503b5e
106
+ ap-southeast-2: ami-0b50bfbb507285a89
107
+ ap-southeast-1: ami-073c677bcf225774a
108
+ eu-north-1: ami-09bb6d618593f2e7f
109
109
  win2k19:
110
- us-east-1: ami-00820419bf212df7e
111
- us-east-2: ami-0a7916b90aa4629d5
112
- ca-central-1: ami-0d704529661e19185
113
- us-west-2: ami-0ee6a198d7ac35eb1
114
- eu-west-2: ami-0f6ac1634bd7add92
115
- us-west-1: ami-039e3816b4cac1e27
116
- eu-west-1: ami-03a771d99091199b7
117
- eu-central-1: ami-03b648d5b45f51a4f
118
- eu-west-3: ami-068839907c18c3a6e
119
- eu-north-1: ami-0db851ee76f7deefb
120
- sa-east-1: ami-0c2cc60c62159f87c
121
- ap-northeast-2: ami-06bdf8ae9ae9add92
122
- ap-northeast-1: ami-02306d959c7f175b9
123
- ap-southeast-1: ami-0d5b4a3d73e0f471f
124
- ap-southeast-2: ami-00fa88caff4f64937
125
- ap-south-1: ami-0b44feae4bb9f497a
110
+ us-east-1: ami-09c18c34c341f2b6a
111
+ us-east-2: ami-030371d5ee8881350
112
+ ca-central-1: ami-018e5bf45c30fa58f
113
+ us-west-2: ami-07d1e5c4f906877e1
114
+ eu-west-2: ami-0b9d95fef44aa7c11
115
+ us-west-1: ami-0e7e082d6fa1769f3
116
+ eu-west-1: ami-05573fafa080144b6
117
+ eu-central-1: ami-0122b027c265988ea
118
+ eu-west-3: ami-0ba7c0a3dc4148b6a
119
+ eu-north-1: ami-03d8c3307f72f9847
120
+ sa-east-1: ami-0d0f66c3e0dfc09ee
121
+ ap-northeast-2: ami-07eff56de9293ab16
122
+ ap-northeast-1: ami-020fb790a3bed4cda
123
+ ap-southeast-1: ami-0b5d036d6d711a4c8
124
+ ap-southeast-2: ami-0fff96935fef7bf60
125
+ ap-south-1: ami-030ec249497f66a33
126
126
  amazon:
127
127
  us-east-1: ami-b73b63a0
128
128
  us-east-2: ami-58277d3d
@@ -137,9 +137,9 @@ amazon:
137
137
  ap-southeast-1: ami-b953f2da
138
138
  ap-southeast-2: ami-db704cb8
139
139
  win2k12: *1
140
- windows: *5
141
- ubuntu: *2
142
- centos: *3
143
- rhel7: *4
144
- rhel: *4
145
- linux: *3
140
+ windows: *2
141
+ ubuntu: *3
142
+ centos: *4
143
+ rhel7: *5
144
+ rhel: *5
145
+ linux: *4
@@ -2,6 +2,7 @@
2
2
  centos6: &centos6 OpenLogic/CentOS/6
3
3
  #centos7: &centos7 westernoceansoftwaresprivatelimited/centos-7-6/centos-7-6-server
4
4
  centos7: &centos7 OpenLogic/CentOS/7
5
+ centos8: &centos7 OpenLogic/CentOS/8
5
6
  rhel8: &rhel8 RedHat/RHEL/8
6
7
  rhel7: &rhel7 RedHat/RHEL/7
7
8
  rhel6: &rhel6 RedHat/RHEL/6
@@ -1,6 +1,7 @@
1
1
  ---
2
2
  centos6: &centos6 egt-labs-admin/mu-centos-6
3
3
  centos7: &centos7 egt-labs-admin/mu-centos-7
4
+ centos8: &centos8 centos-cloud/centos-8
4
5
  rhel71: &rhel71 rhel-cloud/rhel-7
5
6
  rhel6: &rhel6 rhel-cloud/rhel-6
6
7
  debian10: &debian10 debian-cloud/debian-10
@@ -157,6 +157,7 @@ module MU
157
157
  _shortclass, _cfg_name, _cfg_plural, classname = MU::Cloud.getResourceNames(data[:cfg_plural])
158
158
  @main_config[data[:cfg_plural]].each { |resource|
159
159
  resource["#MU_CLOUDCLASS"] = classname
160
+ # resource["#MU_CLOUDCLASS"] = MU::Cloud.resourceClass(resource['cloud'], data[:cfg_plural])
160
161
  }
161
162
  setThreadDependencies(@main_config[data[:cfg_plural]])
162
163
  end
@@ -265,7 +266,7 @@ module MU
265
266
  # Run cloud provider-specific deploy meta-artifact creation (ssh keys,
266
267
  # resource groups, etc)
267
268
  @mommacat.cloudsUsed.each { |cloud|
268
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
269
+ cloudclass = MU::Cloud.cloudClass(cloud)
269
270
  cloudclass.initDeploy(@mommacat)
270
271
  }
271
272
 
@@ -311,6 +312,17 @@ module MU
311
312
 
312
313
  @mommacat.save!
313
314
 
315
+ # XXX Functions have a special behavior where we re-invoke their groom
316
+ # methods one more time at the end, so we can guarantee their
317
+ # environments are fully populated with all sibling resource idents
318
+ # regardless of dependency order. This is, obviously, a disgusting
319
+ # hack, and we should revisit our dependency language in the next big
320
+ # release.
321
+ if !@main_config["functions"].nil? and
322
+ @main_config["functions"].size > 0
323
+ createResources(@main_config["functions"], "groom")
324
+ end
325
+
314
326
  rescue StandardError => e
315
327
  MU.log e.class.name, MU::ERR, details: caller
316
328
 
@@ -425,10 +437,10 @@ module MU
425
437
  MU.log "Failed to generate AWS cost-calculation URL. Skipping.", MU::WARN, details: "Deployment uses a feature not available in CloudFormation layer.", verbosity: MU::Logger::NORMAL
426
438
  ensure
427
439
  MU.setLogging(@verbosity)
428
- MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" complete", details: deployment, verbosity: @verbosity
440
+ MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" #{@updating ? "updated" : "complete"}", details: deployment, verbosity: @verbosity
429
441
  end
430
442
  else
431
- MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" complete", details: deployment, verbosity: @verbosity
443
+ MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" #{@updating ? "updated" : "complete"}", details: deployment, verbosity: @verbosity
432
444
  end
433
445
 
434
446
 
@@ -438,7 +450,7 @@ module MU
438
450
  }
439
451
  end
440
452
 
441
- @mommacat.sendAdminSlack("Deploy completed succesfully", msg: MU.summary.join("\n"))
453
+ @mommacat.sendAdminSlack("Deploy #{MU.deploy_id} \"#{MU.handle}\" #{@updating ? "updated" : "complete"}", msg: MU.summary.join("\n"))
442
454
  end
443
455
 
444
456
  private
@@ -554,16 +566,9 @@ MESSAGE_END
554
566
  @dependency_threads["#{name}_groom"]=["#{name}_create", "mu_groom_container"]
555
567
 
556
568
  MU.log "Setting dependencies for #{name}", MU::DEBUG, details: resource["dependencies"]
557
- if resource["dependencies"] != nil then
569
+ if !resource["dependencies"].nil? then
558
570
  resource["dependencies"].each { |dependency|
559
- parent_class = nil
560
- MU::Cloud.resource_types.each_pair { |res_class, attrs|
561
- if attrs[:cfg_name] == dependency['type'] or
562
- attrs[:cfg_plural] == dependency['type']
563
- parent_class = Object.const_get("MU").const_get("Cloud").const_get(res_class)
564
- break
565
- end
566
- }
571
+ parent_class = MU::Cloud.loadBaseType(dependency['type'])
567
572
 
568
573
  parent_type = parent_class.cfg_name
569
574
 
@@ -572,10 +577,10 @@ MESSAGE_END
572
577
  addDependentThread(parent, "#{name}_groom")
573
578
 
574
579
  # should our creation thread also wait on our parent's create?
575
- if !resource["no_create_wait"] and
580
+ if !dependency["no_create_wait"] and
576
581
  (resource["#MU_CLOUDCLASS"].waits_on_parent_completion or
577
582
  dependency['phase'] == "create" or
578
- (parent_class.deps_wait_on_my_creation and parent_type != res_type))
583
+ parent_class.deps_wait_on_my_creation)
579
584
  addDependentThread(parent, "#{name}_create")
580
585
  end
581
586
 
@@ -584,12 +589,18 @@ MESSAGE_END
584
589
  if (dependency['phase'] == "groom" or resource["#MU_CLOUDCLASS"].waits_on_parent_completion) and parent_class.instance_methods(false).include?(:groom)
585
590
  parent = parent_type+"_"+dependency["name"]+"_groom"
586
591
  addDependentThread(parent, "#{name}_groom")
587
- if (parent_class.deps_wait_on_my_creation and parent_type != res_type) or resource["#MU_CLOUDCLASS"].waits_on_parent_completion or dependency['phase'] == "groom"
592
+ if !dependency["no_create_wait"] and (
593
+ parent_class.deps_wait_on_my_creation or
594
+ resource["#MU_CLOUDCLASS"].waits_on_parent_completion or
595
+ dependency['phase'] == "groom"
596
+ )
588
597
  addDependentThread(parent, "#{name}_create")
589
598
  end
590
599
  end
591
600
  }
592
601
  end
602
+ MU.log "Thread dependencies #{res_type}[#{name}]", MU::DEBUG, details: { "create" => @dependency_threads["#{name}_create"], "groom" => @dependency_threads["#{name}_groom"] }
603
+ @dependency_threads["#{name}_groom"]=["#{name}_create", "mu_groom_container"]
593
604
  }
594
605
  end
595
606
 
@@ -623,7 +634,7 @@ MESSAGE_END
623
634
  begin
624
635
  if myservice['#MUOBJECT'].nil?
625
636
  if @mommacat
626
- ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false)
637
+ ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false, ignore_missing: !@updating)
627
638
  if @updating and ext_obj
628
639
  ext_obj.config!(myservice)
629
640
  end
@@ -651,8 +662,9 @@ MESSAGE_END
651
662
  run_this_method = myservice['#MUOBJECT'].method(mode)
652
663
  rescue StandardError => e
653
664
  MU::MommaCat.unlockAll
654
- @main_thread.raise MuError, "Error invoking #{myservice["#MU_CLOUDCLASS"]}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
655
- raise e
665
+ @main_thread.raise MuError, "Error invoking #{myservice["#MUOBJECT"].class.name}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
666
+ return
667
+ # raise e
656
668
  end
657
669
  begin
658
670
  MU.log "Checking whether to run #{myservice['#MUOBJECT']}.#{mode} (updating: #{@updating})", MU::DEBUG
@@ -732,7 +744,9 @@ MESSAGE_END
732
744
  sleep 10+Random.rand(20)
733
745
  retry
734
746
  end
747
+
735
748
  end
749
+
736
750
  end
737
751
 
738
752
  end #class