cloud-mu 3.1.5 → 3.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +5 -1
  3. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  4. data/ansible/roles/mu-windows/files/config.xml +76 -0
  5. data/ansible/roles/mu-windows/tasks/main.yml +16 -0
  6. data/bin/mu-adopt +16 -12
  7. data/bin/mu-azure-tests +57 -0
  8. data/bin/mu-cleanup +2 -4
  9. data/bin/mu-configure +52 -0
  10. data/bin/mu-deploy +3 -3
  11. data/bin/mu-findstray-tests +25 -0
  12. data/bin/mu-gen-docs +2 -4
  13. data/bin/mu-load-config.rb +2 -1
  14. data/bin/mu-node-manage +15 -16
  15. data/bin/mu-run-tests +37 -12
  16. data/cloud-mu.gemspec +3 -3
  17. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  18. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  19. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  20. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  21. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  22. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  23. data/cookbooks/mu-tools/recipes/windows-client.rb +25 -22
  24. data/extras/clean-stock-amis +25 -19
  25. data/extras/generate-stock-images +1 -0
  26. data/extras/image-generators/AWS/win2k12.yaml +2 -0
  27. data/extras/image-generators/AWS/win2k16.yaml +2 -0
  28. data/extras/image-generators/AWS/win2k19.yaml +2 -0
  29. data/modules/mommacat.ru +1 -1
  30. data/modules/mu.rb +86 -98
  31. data/modules/mu/adoption.rb +373 -58
  32. data/modules/mu/cleanup.rb +214 -303
  33. data/modules/mu/cloud.rb +128 -1733
  34. data/modules/mu/cloud/database.rb +49 -0
  35. data/modules/mu/cloud/dnszone.rb +44 -0
  36. data/modules/mu/cloud/machine_images.rb +212 -0
  37. data/modules/mu/cloud/providers.rb +81 -0
  38. data/modules/mu/cloud/resource_base.rb +929 -0
  39. data/modules/mu/cloud/server.rb +40 -0
  40. data/modules/mu/cloud/server_pool.rb +1 -0
  41. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  42. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  43. data/modules/mu/cloud/wrappers.rb +169 -0
  44. data/modules/mu/config.rb +123 -81
  45. data/modules/mu/config/alarm.rb +2 -6
  46. data/modules/mu/config/bucket.rb +32 -3
  47. data/modules/mu/config/cache_cluster.rb +2 -2
  48. data/modules/mu/config/cdn.rb +100 -0
  49. data/modules/mu/config/collection.rb +1 -1
  50. data/modules/mu/config/container_cluster.rb +7 -2
  51. data/modules/mu/config/database.rb +84 -105
  52. data/modules/mu/config/database.yml +1 -2
  53. data/modules/mu/config/dnszone.rb +5 -4
  54. data/modules/mu/config/doc_helpers.rb +5 -6
  55. data/modules/mu/config/endpoint.rb +2 -1
  56. data/modules/mu/config/firewall_rule.rb +3 -19
  57. data/modules/mu/config/folder.rb +1 -1
  58. data/modules/mu/config/function.rb +17 -8
  59. data/modules/mu/config/group.rb +1 -1
  60. data/modules/mu/config/habitat.rb +1 -1
  61. data/modules/mu/config/job.rb +89 -0
  62. data/modules/mu/config/loadbalancer.rb +57 -11
  63. data/modules/mu/config/log.rb +1 -1
  64. data/modules/mu/config/msg_queue.rb +1 -1
  65. data/modules/mu/config/nosqldb.rb +1 -1
  66. data/modules/mu/config/notifier.rb +8 -19
  67. data/modules/mu/config/ref.rb +92 -14
  68. data/modules/mu/config/role.rb +1 -1
  69. data/modules/mu/config/schema_helpers.rb +38 -37
  70. data/modules/mu/config/search_domain.rb +1 -1
  71. data/modules/mu/config/server.rb +12 -13
  72. data/modules/mu/config/server_pool.rb +3 -7
  73. data/modules/mu/config/storage_pool.rb +1 -1
  74. data/modules/mu/config/tail.rb +11 -0
  75. data/modules/mu/config/user.rb +1 -1
  76. data/modules/mu/config/vpc.rb +27 -23
  77. data/modules/mu/config/vpc.yml +0 -1
  78. data/modules/mu/defaults/AWS.yaml +90 -90
  79. data/modules/mu/defaults/Azure.yaml +1 -0
  80. data/modules/mu/defaults/Google.yaml +1 -0
  81. data/modules/mu/deploy.rb +34 -20
  82. data/modules/mu/groomer.rb +16 -1
  83. data/modules/mu/groomers/ansible.rb +69 -4
  84. data/modules/mu/groomers/chef.rb +51 -4
  85. data/modules/mu/logger.rb +120 -144
  86. data/modules/mu/master.rb +97 -4
  87. data/modules/mu/mommacat.rb +160 -874
  88. data/modules/mu/mommacat/daemon.rb +23 -14
  89. data/modules/mu/mommacat/naming.rb +110 -3
  90. data/modules/mu/mommacat/search.rb +497 -0
  91. data/modules/mu/mommacat/storage.rb +252 -194
  92. data/modules/mu/{clouds → providers}/README.md +1 -1
  93. data/modules/mu/{clouds → providers}/aws.rb +258 -57
  94. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  95. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  96. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  97. data/modules/mu/providers/aws/cdn.rb +782 -0
  98. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  99. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +95 -84
  100. data/modules/mu/providers/aws/database.rb +1744 -0
  101. data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
  102. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  103. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
  104. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  105. data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
  106. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  107. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  108. data/modules/mu/providers/aws/job.rb +466 -0
  109. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
  110. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  111. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  112. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  113. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  114. data/modules/mu/{clouds → providers}/aws/role.rb +76 -48
  115. data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
  116. data/modules/mu/{clouds → providers}/aws/server.rb +66 -98
  117. data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
  118. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  119. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  120. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  121. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  122. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  123. data/modules/mu/{clouds → providers}/aws/vpc.rb +143 -74
  124. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  125. data/modules/mu/{clouds → providers}/azure.rb +13 -0
  126. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  127. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  128. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  129. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  130. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  131. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  132. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  133. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  134. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  135. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  136. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  137. data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
  138. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  139. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  140. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  141. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  142. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  143. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  144. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  145. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  146. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  147. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  148. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  149. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  150. data/modules/mu/{clouds → providers}/google.rb +29 -6
  151. data/modules/mu/{clouds → providers}/google/bucket.rb +4 -4
  152. data/modules/mu/{clouds → providers}/google/container_cluster.rb +38 -20
  153. data/modules/mu/{clouds → providers}/google/database.rb +5 -12
  154. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
  155. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  156. data/modules/mu/{clouds → providers}/google/function.rb +6 -6
  157. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  158. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  159. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
  160. data/modules/mu/{clouds → providers}/google/role.rb +50 -31
  161. data/modules/mu/{clouds → providers}/google/server.rb +41 -24
  162. data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
  163. data/modules/mu/{clouds → providers}/google/user.rb +34 -24
  164. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  165. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  166. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  167. data/modules/mu/{clouds → providers}/google/vpc.rb +45 -14
  168. data/modules/tests/aws-jobs-functions.yaml +46 -0
  169. data/modules/tests/centos6.yaml +15 -0
  170. data/modules/tests/centos7.yaml +15 -0
  171. data/modules/tests/centos8.yaml +12 -0
  172. data/modules/tests/ecs.yaml +2 -2
  173. data/modules/tests/eks.yaml +1 -1
  174. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  175. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  176. data/modules/tests/microservice_app.yaml +288 -0
  177. data/modules/tests/rds.yaml +108 -0
  178. data/modules/tests/regrooms/rds.yaml +123 -0
  179. data/modules/tests/server-with-scrub-muisms.yaml +1 -1
  180. data/modules/tests/super_complex_bok.yml +2 -2
  181. data/modules/tests/super_simple_bok.yml +3 -5
  182. data/spec/mu/clouds/azure_spec.rb +2 -2
  183. metadata +122 -92
  184. data/modules/mu/clouds/aws/database.rb +0 -1974
  185. data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/server_pool.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/server_pool.rb
18
18
  class ServerPool
19
19
 
20
20
  # Base configuration schema for a ServerPool
@@ -186,11 +186,7 @@ module MU
186
186
 
187
187
  if !pool["vpc"].nil?
188
188
  if !pool["vpc"]["subnet_name"].nil? and configurator.nat_routes.has_key?(pool["vpc"]["subnet_name"])
189
- pool["dependencies"] << {
190
- "type" => "pool",
191
- "name" => configurator.nat_routes[pool["vpc"]["subnet_name"]],
192
- "phase" => "groom"
193
- }
189
+ MU::Config.addDependency(pool, configurator.nat_routes[pool["vpc"]["subnet_name"]], "server", phase: "groom", no_create_wait: true)
194
190
  end
195
191
  end
196
192
  # TODO make sure this is handled... somewhere
@@ -203,7 +199,7 @@ module MU
203
199
  # }
204
200
  # end
205
201
  if pool["basis"] and pool["basis"]["server"]
206
- pool["dependencies"] << {"type" => "server", "name" => pool["basis"]["server"]}
202
+ MU::Config.addDependency(pool, pool["basis"]["server"], "server", phase: "groom")
207
203
  end
208
204
  if !pool['static_ip'].nil? and !pool['ip'].nil?
209
205
  ok = false
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/storage_pool.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/storage_pool.rb
18
18
  class StoragePool
19
19
 
20
20
  # Base configuration schema for a StoragePool
@@ -117,6 +117,16 @@ module MU
117
117
  def gsub(*args)
118
118
  to_s.gsub(*args)
119
119
  end
120
+
121
+ # Lets callers access us like a {Hash}
122
+ # @param attribute [String,Symbol]
123
+ def [](attribute)
124
+ if respond_to?(attribute.to_sym)
125
+ send(attribute.to_sym)
126
+ else
127
+ nil
128
+ end
129
+ end
120
130
  end
121
131
 
122
132
  # Wrapper method for creating a {MU::Config::Tail} object as a reference to
@@ -133,6 +143,7 @@ module MU
133
143
  # @param pseudo [<Boolean>]: This is a pseudo-parameter, automatically provided, and not available as user input.
134
144
  # @param runtimecode [<String>]: Actual code to allow the cloud layer to interpret literally in its own idiom, e.g. '"Ref" : "AWS::StackName"' for CloudFormation
135
145
  def getTail(param, value: nil, prettyname: nil, cloudtype: "String", valid_values: [], description: nil, list_of: nil, prefix: "", suffix: "", pseudo: false, runtimecode: nil)
146
+ param = param.gsub(/[^a-z0-9_]/i, "_")
136
147
  if value.nil?
137
148
  if @@parameters.nil? or !@@parameters.has_key?(param)
138
149
  MU.log "Parameter '#{param}' (#{param.class.name}) referenced in config but not provided (#{caller[0]})", MU::DEBUG, details: @@parameters
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/user.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/user.rb
18
18
  class User
19
19
 
20
20
  # Base configuration schema for a User
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/vpc.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/vpc.rb
18
18
  class VPC
19
19
 
20
20
  # Base configuration schema for a VPC
@@ -417,9 +417,10 @@ module MU
417
417
  using_default_cidr = false
418
418
  if !vpc['ip_block']
419
419
  if configurator.updating and configurator.existing_deploy and
420
+ configurator.existing_deploy.original_config and
420
421
  configurator.existing_deploy.original_config['vpcs']
421
422
  configurator.existing_deploy.original_config['vpcs'].each { |v|
422
- if v['name'] == vpc['name']
423
+ if v['name'].to_s == vpc['name'].to_s
423
424
  vpc['ip_block'] = v['ip_block']
424
425
  vpc['peers'] ||= []
425
426
  vpc['peers'].concat(v['peers'])
@@ -431,6 +432,10 @@ module MU
431
432
  break
432
433
  end
433
434
  }
435
+ if !vpc['ip_block']
436
+ MU.log "Loading existing deploy but can't find IP block of VPC #{vpc['name']}", MU::ERR
437
+ ok = false
438
+ end
434
439
  else
435
440
  using_default_cidr = true
436
441
  vpc['ip_block'] = "10.0.0.0/16"
@@ -493,6 +498,7 @@ module MU
493
498
  # See if we'll be able to create peering connections
494
499
  can_peer = false
495
500
  already_peered = false
501
+
496
502
  if MU.myCloud == vpc["cloud"] and MU.myVPCObj
497
503
  if vpc['peers']
498
504
  vpc['peers'].each { |peer|
@@ -538,7 +544,7 @@ module MU
538
544
  # Clouds that don't have some kind of native NAT gateway can also
539
545
  # leverage this host to honor "gateway" => "#NAT" situations.
540
546
  if !can_peer and !already_peered and have_public and vpc["create_bastion"]
541
- serverclass = Object.const_get("MU").const_get("Cloud").const_get(vpc["cloud"]).const_get("Server")
547
+ serverclass = MU::Cloud.resourceClass(vpc["cloud"], "Server")
542
548
  bastion = serverclass.genericNAT.dup
543
549
  bastion["groomer_variables"] = {
544
550
  "nat_ip_block" => vpc["ip_block"].to_s
@@ -557,10 +563,7 @@ module MU
557
563
  "name" => vpc["name"],
558
564
  "subnet_pref" => "public"
559
565
  }
560
- vpc["dependencies"] << {
561
- "type" => "server",
562
- "name" => bastion['name'],
563
- }
566
+ MU::Config.addDependency(vpc, bastion['name'], "server", no_create_wait: true)
564
567
  vpc["bastion"] = MU::Config::Ref.get(
565
568
  name: bastion['name'],
566
569
  cloud: vpc['cloud'],
@@ -595,7 +598,7 @@ module MU
595
598
  MU.log "Skipping malformed VPC peer in #{vpc['name']}", MU::ERR, details: peer
596
599
  next
597
600
  end
598
- peer["#MU_CLOUDCLASS"] = Object.const_get("MU").const_get("Cloud").const_get("VPC")
601
+ peer["#MU_CLOUDCLASS"] = MU::Cloud.loadBaseType("VPC")
599
602
  # We check for multiple siblings because some implementations
600
603
  # (Google) can split declared VPCs into parts to get the mimic the
601
604
  # routing behaviors we expect.
@@ -612,17 +615,11 @@ module MU
612
615
  append_me = { "vpc" => peer["vpc"].dup }
613
616
  append_me['vpc']['name'] = sib['name']
614
617
  append << append_me
615
- vpc["dependencies"] << {
616
- "type" => "vpc",
617
- "name" => sib['name']
618
- }
618
+ MU::Config.addDependency(vpc, sib['name'], "vpc", phase: "groom", no_create_wait: true)
619
619
  end
620
620
  delete << peer
621
621
  else
622
- vpc["dependencies"] << {
623
- "type" => "vpc",
624
- "name" => peer['vpc']["name"]
625
- }
622
+ MU::Config.addDependency(vpc, peer['vpc']['name'], "vpc", phase: "groom", no_create_wait: true)
626
623
  end
627
624
  delete << peer if sib['name'] == vpc['name']
628
625
  }
@@ -636,7 +633,7 @@ module MU
636
633
  MU.log "VPC peering connections to non-local accounts must specify the vpc_id of the peer.", MU::ERR
637
634
  ok = false
638
635
  end
639
- elsif !processReference(peer['vpc'], "vpcs", "vpc '#{vpc['name']}'", configurator, dflt_region: peer["vpc"]['region'])
636
+ elsif !processReference(peer['vpc'], "vpcs", vpc, configurator, dflt_region: peer["vpc"]['region'])
640
637
  ok = false
641
638
  end
642
639
  end
@@ -735,8 +732,8 @@ module MU
735
732
  vpc_block["subnet_pref"] = "all_private" if vpc_block["subnet_pref"] == "private"
736
733
  end
737
734
 
738
- flags = {}
739
- flags["subnet_pref"] = vpc_block["subnet_pref"] if !vpc_block["subnet_pref"].nil?
735
+ # flags = {}
736
+ # flags["subnet_pref"] = vpc_block["subnet_pref"] if !vpc_block["subnet_pref"].nil?
740
737
  hab_arg = if vpc_block['habitat']
741
738
  if vpc_block['habitat'].is_a?(MU::Config::Ref)
742
739
  [vpc_block['habitat'].id] # XXX actually, findStray it
@@ -770,9 +767,9 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
770
767
  tag_key: tag_key,
771
768
  tag_value: tag_value,
772
769
  region: vpc_block["region"],
773
- flags: flags,
774
770
  habitats: hab_arg,
775
- dummy_ok: true
771
+ dummy_ok: true,
772
+ subnet_pref: vpc_block["subnet_pref"]
776
773
  )
777
774
 
778
775
  found.first if found and found.size == 1
@@ -799,7 +796,7 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
799
796
  @@reference_cache[vpc_block] ||= ext_vpc if ok
800
797
  end
801
798
  rescue StandardError => e
802
- raise MuError, e.inspect, e.backtrace
799
+ raise MuError.new e.inspect, details: { "my call stack" => caller, "exception call stack" => e.backtrace }
803
800
  ensure
804
801
  if !ext_vpc and vpc_block['cloud'] != "CloudFormation"
805
802
  MU.log "Couldn't resolve VPC reference to a unique live VPC in #{parent_type} #{parent['name']} (called by #{caller[0]})", MU::ERR, details: vpc_block
@@ -923,7 +920,14 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
923
920
  ext_vpc.subnets.each { |subnet|
924
921
  next if dflt_region and vpc_block["cloud"] == "Google" and subnet.az != dflt_region
925
922
  if subnet.private? and (vpc_block['subnet_pref'] != "all_public" and vpc_block['subnet_pref'] != "public")
926
- private_subnets << { "subnet_id" => configurator.getTail("#{parent['name']} Private Subnet #{priv}", value: subnet.cloud_id, prettyname: "#{parent['name']} Private Subnet #{priv}", cloudtype: "AWS::EC2::Subnet::Id"), "az" => subnet.az }
923
+ private_subnets << {
924
+ "subnet_id" => configurator.getTail(
925
+ "#{parent['name']} Private Subnet #{priv}",
926
+ value: subnet.cloud_id,
927
+ prettyname: "#{parent['name']} Private Subnet #{priv}",
928
+ cloudtype: "AWS::EC2::Subnet::Id"),
929
+ "az" => subnet.az
930
+ }
927
931
  private_subnets_map[subnet.cloud_id] = subnet
928
932
  priv = priv + 1
929
933
  elsif !subnet.private? and vpc_block['subnet_pref'] != "all_private" and vpc_block['subnet_pref'] != "private"
@@ -1,7 +1,6 @@
1
1
  <% if complexity == 'complex' %>
2
2
  name: <%= vpc_name %>
3
3
  create_nat_gateway: true
4
- ip_block: 10.231.0.0/16
5
4
  enable_traffic_logging: true
6
5
  region: us-east-2
7
6
  availability_zones:
@@ -1,5 +1,5 @@
1
1
  ---
2
- rhel71: &4
2
+ rhel71: &5
3
3
  us-east-1: ami-0f05fce24aa75ba9f
4
4
  ap-northeast-1: ami-0c0ec19eb19055763
5
5
  ap-northeast-2: ami-0717ac5c67c99f745
@@ -16,41 +16,41 @@ rhel71: &4
16
16
  us-east-2: ami-02f6682c7816b3cfc
17
17
  us-west-1: ami-04898e596c06e802b
18
18
  us-west-2: ami-02db5457189a8a8c2
19
- centos6: &3
20
- us-east-1: ami-06b6c01abc6998348
21
- ap-northeast-1: ami-0c5da73fde2cb6437
22
- ap-northeast-2: ami-0134fce6dc00eb00d
23
- ap-south-1: ami-0e59a612e7c84836b
24
- ap-southeast-1: ami-0e16974f528ae0dae
25
- ap-southeast-2: ami-0e2feddf3dbf4d539
26
- ca-central-1: ami-089236a344dadad5f
27
- eu-central-1: ami-0c4eed3fe046c3917
28
- eu-north-1: ami-05f636e89d0362c14
29
- eu-west-1: ami-00c50b11d713f90d3
30
- eu-west-2: ami-06cc78c32eed7f944
31
- eu-west-3: ami-0ba626236ad786c54
32
- sa-east-1: ami-07c3b2a5a41e92376
33
- us-east-2: ami-01129e636778acfbc
34
- us-west-1: ami-0632e646cd5089ffc
35
- us-west-2: ami-0ce4c9f2e1037de53
19
+ centos6: &4
20
+ us-east-1: ami-009723c5c7f8fbc75
21
+ us-east-2: ami-0781f11395714cd39
22
+ ap-northeast-1: ami-07fa5a8795da2b6bc
23
+ ap-northeast-2: ami-0219f0a7c979ff63f
24
+ ap-south-1: ami-0f24817242c401740
25
+ ap-southeast-1: ami-042ef2e0643e8e207
26
+ ap-southeast-2: ami-09fc51de648afa168
27
+ ca-central-1: ami-0dc643db74edc5aa5
28
+ eu-central-1: ami-0628759cb297569d5
29
+ eu-north-1: ami-0aed023791f886315
30
+ eu-west-1: ami-0f87f0f252ff03622
31
+ eu-west-2: ami-00abb555d5a460afe
32
+ eu-west-3: ami-0ccd93d454c2418a2
33
+ sa-east-1: ami-01e10ea6ea72534ae
34
+ us-west-1: ami-01fee56b9ee690ffe
35
+ us-west-2: ami-08bcdb944f185e2a8
36
36
  centos7:
37
- us-east-1: ami-07e6f661e71ad964b
38
- ap-northeast-1: ami-0988180d74897c639
39
- ap-northeast-2: ami-0e77cd1c7024b8ae0
40
- ap-south-1: ami-02bd479122041000a
41
- ap-southeast-1: ami-017767778ef9db671
42
- ap-southeast-2: ami-05b09a58c3964d67d
43
- ca-central-1: ami-0a59a176d810fcc5f
44
- eu-central-1: ami-0b48a421fb05d96af
45
- eu-north-1: ami-02337601ea5dc4a5d
46
- eu-west-1: ami-0b0a55b7423eeac07
47
- eu-west-2: ami-060518b40b25b9eb4
48
- eu-west-3: ami-060957bb3adacd831
49
- sa-east-1: ami-0c706132b35071de6
50
- us-east-2: ami-0db4c266ed0bb958b
51
- us-west-1: ami-0980f6eb52c998793
52
- us-west-2: ami-07f2ed4755c01c05c
53
- ubuntu16: &2
37
+ us-east-1: ami-067256ca1497c924d
38
+ ap-northeast-1: ami-07c1e51354fdfd362
39
+ ap-northeast-2: ami-042b761c93d6df2f1
40
+ ap-south-1: ami-02e879f52322e7c98
41
+ ap-southeast-1: ami-0487e9f84d0ffde89
42
+ ap-southeast-2: ami-0e854dab39fd6a427
43
+ ca-central-1: ami-05a27d311b585a70b
44
+ eu-central-1: ami-0e396d00c787b4f47
45
+ eu-north-1: ami-087763a2ba60b2bfe
46
+ eu-west-1: ami-04e3bd9335a14e635
47
+ eu-west-2: ami-0efd34a8d1fc2b104
48
+ eu-west-3: ami-08d0bcbc780448cf8
49
+ sa-east-1: ami-0284f4a0968263cf0
50
+ us-east-2: ami-0292786917d1e3015
51
+ us-west-1: ami-0ba622529dcdff2bb
52
+ us-west-2: ami-079a309ca6261d7f6
53
+ ubuntu16: &3
54
54
  us-east-1: ami-bcdc16c6
55
55
  us-west-1: ami-1b17257b
56
56
  us-west-2: ami-19e92861
@@ -73,56 +73,56 @@ ubuntu14:
73
73
  ap-southeast-1: ami-2855964b
74
74
  ap-southeast-2: ami-d19fc4b2
75
75
  win2k12r2: &1
76
- us-east-1: ami-00f7cf8d57d29a8a7
77
- us-east-2: ami-0c14a2a9b1d88428d
78
- ca-central-1: ami-0210e4efc4186f89d
79
- us-west-2: ami-036681205605cba8c
80
- us-west-1: ami-072d0f2b03f351e5c
81
- eu-west-1: ami-061524b3efcc026da
82
- eu-west-2: ami-0a7aeb2dae7c7154b
83
- eu-west-3: ami-0b16adff6701f08bb
84
- eu-north-1: ami-09bd34c6465aa914b
85
- sa-east-1: ami-078221cae70b179c4
86
- eu-central-1: ami-047d37ec58a8469fb
87
- ap-northeast-1: ami-0ce23ffef990003d2
88
- ap-south-1: ami-0106284f16a19651a
89
- ap-northeast-2: ami-0518e43d0367f1a6d
90
- ap-southeast-1: ami-0858019a829a5169d
91
- ap-southeast-2: ami-0e0d7d3acb6427f53
92
- win2k16: &5
93
- us-east-1: ami-090dd1749dc0be91d
94
- us-east-2: ami-09c9eeb95291e63d7
95
- ca-central-1: ami-0c63a53a15fca4238
96
- us-west-2: ami-0b8540e9a207143eb
97
- eu-west-1: ami-067b5d8d85d3c8cf8
98
- us-west-1: ami-0a2dc7e2cf21ab3e9
99
- eu-west-2: ami-0cece465ae4b18027
100
- eu-west-3: ami-0cc5f29ed6e0e8a67
101
- eu-central-1: ami-02d4da18299373531
102
- sa-east-1: ami-021d5c906c1898430
103
- ap-northeast-1: ami-01129c98f812a3be7
104
- ap-south-1: ami-005c7910458e9541d
105
- ap-northeast-2: ami-024840a881c449d78
106
- ap-southeast-2: ami-070af17644a596301
107
- ap-southeast-1: ami-0bc2e098a07140bc4
108
- eu-north-1: ami-03dcc78d77d2ee027
76
+ us-east-1: ami-003aea65bc2e7136a
77
+ us-east-2: ami-0163293e39ba504c2
78
+ ca-central-1: ami-055689dd92f29d2aa
79
+ us-west-2: ami-0ce87dda2c9244e57
80
+ us-west-1: ami-00d9cf64bd2fafa44
81
+ eu-west-1: ami-026d7427b9fadad40
82
+ eu-west-2: ami-036a22c0780551794
83
+ eu-west-3: ami-05e3d9b79bdc10861
84
+ eu-north-1: ami-063eb48504c7d73f1
85
+ sa-east-1: ami-0a8c1829a5e650bc5
86
+ eu-central-1: ami-0ea20cef52335b008
87
+ ap-northeast-1: ami-08db2dc67228dbb90
88
+ ap-south-1: ami-012241411db3f09c3
89
+ ap-northeast-2: ami-0368c224de1d20502
90
+ ap-southeast-1: ami-028ef74e1edc3943a
91
+ ap-southeast-2: ami-09e03eab1b1bc151b
92
+ win2k16: &2
93
+ us-east-1: ami-02801a2c8dcbfb883
94
+ us-east-2: ami-0ca4f779a2a58a7ea
95
+ ca-central-1: ami-05d3854d9d6e9bcc5
96
+ us-west-2: ami-091f4a88ce32d28b6
97
+ eu-west-1: ami-0b938c9b23ed7d18c
98
+ us-west-1: ami-0fd744c3fbe8260f2
99
+ eu-west-2: ami-071a89b959c5eda27
100
+ eu-west-3: ami-0b206e3dbda9ff9eb
101
+ eu-central-1: ami-0dd9bdad31dd0d3ce
102
+ sa-east-1: ami-0d69b8d6c0f9a7bae
103
+ ap-northeast-1: ami-02eb4a6f519bc3190
104
+ ap-south-1: ami-0666fd543ac8b5501
105
+ ap-northeast-2: ami-01277c81f9b91cf77
106
+ ap-southeast-2: ami-0426a246f9b0ccadd
107
+ ap-southeast-1: ami-07ecb0d55c2eb7247
108
+ eu-north-1: ami-047811530583b6d08
109
109
  win2k19:
110
- us-east-1: ami-09946f18cbcdce65c
111
- us-east-2: ami-02ab72768678bb7d0
112
- ca-central-1: ami-0fcf1b24169d88d7f
113
- us-west-2: ami-025ca67c85e4e147d
114
- eu-west-2: ami-08a0e09c469e6a557
115
- us-west-1: ami-05960eb854f91cbb8
116
- eu-west-1: ami-0f91d02c05561cf5b
117
- eu-central-1: ami-0faafc220143c941c
118
- eu-west-3: ami-0d2a0b6f21c7ce4a2
119
- eu-north-1: ami-0fecae116e9e331bf
120
- sa-east-1: ami-050693ece618acaa5
121
- ap-northeast-2: ami-0e0ebd96765911d72
122
- ap-northeast-1: ami-0729606d0a4051499
123
- ap-southeast-1: ami-06ba249ee50ee9669
124
- ap-southeast-2: ami-03a051a6d0f2b79d5
125
- ap-south-1: ami-02c287a24c5e872f6
110
+ us-east-1: ami-00820419bf212df7e
111
+ us-east-2: ami-0a7916b90aa4629d5
112
+ ca-central-1: ami-0d704529661e19185
113
+ us-west-2: ami-0ee6a198d7ac35eb1
114
+ eu-west-2: ami-0f6ac1634bd7add92
115
+ us-west-1: ami-039e3816b4cac1e27
116
+ eu-west-1: ami-03a771d99091199b7
117
+ eu-central-1: ami-03b648d5b45f51a4f
118
+ eu-west-3: ami-068839907c18c3a6e
119
+ eu-north-1: ami-0db851ee76f7deefb
120
+ sa-east-1: ami-0c2cc60c62159f87c
121
+ ap-northeast-2: ami-06bdf8ae9ae9add92
122
+ ap-northeast-1: ami-02306d959c7f175b9
123
+ ap-southeast-1: ami-0d5b4a3d73e0f471f
124
+ ap-southeast-2: ami-00fa88caff4f64937
125
+ ap-south-1: ami-0b44feae4bb9f497a
126
126
  amazon:
127
127
  us-east-1: ami-b73b63a0
128
128
  us-east-2: ami-58277d3d
@@ -137,9 +137,9 @@ amazon:
137
137
  ap-southeast-1: ami-b953f2da
138
138
  ap-southeast-2: ami-db704cb8
139
139
  win2k12: *1
140
- windows: *5
141
- ubuntu: *2
142
- centos: *3
143
- rhel7: *4
144
- rhel: *4
145
- linux: *3
140
+ windows: *2
141
+ ubuntu: *3
142
+ centos: *4
143
+ rhel7: *5
144
+ rhel: *5
145
+ linux: *4
@@ -2,6 +2,7 @@
2
2
  centos6: &centos6 OpenLogic/CentOS/6
3
3
  #centos7: &centos7 westernoceansoftwaresprivatelimited/centos-7-6/centos-7-6-server
4
4
  centos7: &centos7 OpenLogic/CentOS/7
5
+ centos8: &centos7 OpenLogic/CentOS/8
5
6
  rhel8: &rhel8 RedHat/RHEL/8
6
7
  rhel7: &rhel7 RedHat/RHEL/7
7
8
  rhel6: &rhel6 RedHat/RHEL/6
@@ -1,6 +1,7 @@
1
1
  ---
2
2
  centos6: &centos6 egt-labs-admin/mu-centos-6
3
3
  centos7: &centos7 egt-labs-admin/mu-centos-7
4
+ centos8: &centos8 centos-cloud/centos-8
4
5
  rhel71: &rhel71 rhel-cloud/rhel-7
5
6
  rhel6: &rhel6 rhel-cloud/rhel-6
6
7
  debian10: &debian10 debian-cloud/debian-10
@@ -157,6 +157,7 @@ module MU
157
157
  _shortclass, _cfg_name, _cfg_plural, classname = MU::Cloud.getResourceNames(data[:cfg_plural])
158
158
  @main_config[data[:cfg_plural]].each { |resource|
159
159
  resource["#MU_CLOUDCLASS"] = classname
160
+ # resource["#MU_CLOUDCLASS"] = MU::Cloud.resourceClass(resource['cloud'], data[:cfg_plural])
160
161
  }
161
162
  setThreadDependencies(@main_config[data[:cfg_plural]])
162
163
  end
@@ -265,7 +266,7 @@ module MU
265
266
  # Run cloud provider-specific deploy meta-artifact creation (ssh keys,
266
267
  # resource groups, etc)
267
268
  @mommacat.cloudsUsed.each { |cloud|
268
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
269
+ cloudclass = MU::Cloud.cloudClass(cloud)
269
270
  cloudclass.initDeploy(@mommacat)
270
271
  }
271
272
 
@@ -311,6 +312,17 @@ module MU
311
312
 
312
313
  @mommacat.save!
313
314
 
315
+ # XXX Functions have a special behavior where we re-invoke their groom
316
+ # methods one more time at the end, so we can guarantee their
317
+ # environments are fully populated with all sibling resource idents
318
+ # regardless of dependency order. This is, obviously, a disgusting
319
+ # hack, and we should revisit our dependency language in the next big
320
+ # release.
321
+ if !@main_config["functions"].nil? and
322
+ @main_config["functions"].size > 0
323
+ createResources(@main_config["functions"], "groom")
324
+ end
325
+
314
326
  rescue StandardError => e
315
327
  MU.log e.class.name, MU::ERR, details: caller
316
328
 
@@ -394,7 +406,7 @@ module MU
394
406
  Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :never) {
395
407
  begin
396
408
  Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :immediate) {
397
- MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::WARN, verbosity: MU::Logger::NORMAL
409
+ MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::DEBUG, verbosity: MU::Logger::NORMAL
398
410
  Thread.current.exit
399
411
  }
400
412
  ensure
@@ -425,10 +437,10 @@ module MU
425
437
  MU.log "Failed to generate AWS cost-calculation URL. Skipping.", MU::WARN, details: "Deployment uses a feature not available in CloudFormation layer.", verbosity: MU::Logger::NORMAL
426
438
  ensure
427
439
  MU.setLogging(@verbosity)
428
- MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" complete", details: deployment, verbosity: @verbosity
440
+ MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" #{@updating ? "updated" : "complete"}", details: deployment, verbosity: @verbosity
429
441
  end
430
442
  else
431
- MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" complete", details: deployment, verbosity: @verbosity
443
+ MU.log "Deployment #{MU.deploy_id} \"#{MU.handle}\" #{@updating ? "updated" : "complete"}", details: deployment, verbosity: @verbosity
432
444
  end
433
445
 
434
446
 
@@ -438,7 +450,7 @@ module MU
438
450
  }
439
451
  end
440
452
 
441
- @mommacat.sendAdminSlack("Deploy completed succesfully", msg: MU.summary.join("\n"))
453
+ @mommacat.sendAdminSlack("Deploy #{MU.deploy_id} \"#{MU.handle}\" #{@updating ? "updated" : "complete"}", msg: MU.summary.join("\n"))
442
454
  end
443
455
 
444
456
  private
@@ -554,16 +566,9 @@ MESSAGE_END
554
566
  @dependency_threads["#{name}_groom"]=["#{name}_create", "mu_groom_container"]
555
567
 
556
568
  MU.log "Setting dependencies for #{name}", MU::DEBUG, details: resource["dependencies"]
557
- if resource["dependencies"] != nil then
569
+ if !resource["dependencies"].nil? then
558
570
  resource["dependencies"].each { |dependency|
559
- parent_class = nil
560
- MU::Cloud.resource_types.each_pair { |res_class, attrs|
561
- if attrs[:cfg_name] == dependency['type'] or
562
- attrs[:cfg_plural] == dependency['type']
563
- parent_class = Object.const_get("MU").const_get("Cloud").const_get(res_class)
564
- break
565
- end
566
- }
571
+ parent_class = MU::Cloud.loadBaseType(dependency['type'])
567
572
 
568
573
  parent_type = parent_class.cfg_name
569
574
 
@@ -572,10 +577,10 @@ MESSAGE_END
572
577
  addDependentThread(parent, "#{name}_groom")
573
578
 
574
579
  # should our creation thread also wait on our parent's create?
575
- if !resource["no_create_wait"] and
580
+ if !dependency["no_create_wait"] and
576
581
  (resource["#MU_CLOUDCLASS"].waits_on_parent_completion or
577
582
  dependency['phase'] == "create" or
578
- (parent_class.deps_wait_on_my_creation and parent_type != res_type))
583
+ parent_class.deps_wait_on_my_creation)
579
584
  addDependentThread(parent, "#{name}_create")
580
585
  end
581
586
 
@@ -584,12 +589,18 @@ MESSAGE_END
584
589
  if (dependency['phase'] == "groom" or resource["#MU_CLOUDCLASS"].waits_on_parent_completion) and parent_class.instance_methods(false).include?(:groom)
585
590
  parent = parent_type+"_"+dependency["name"]+"_groom"
586
591
  addDependentThread(parent, "#{name}_groom")
587
- if (parent_class.deps_wait_on_my_creation and parent_type != res_type) or resource["#MU_CLOUDCLASS"].waits_on_parent_completion or dependency['phase'] == "groom"
592
+ if !dependency["no_create_wait"] and (
593
+ parent_class.deps_wait_on_my_creation or
594
+ resource["#MU_CLOUDCLASS"].waits_on_parent_completion or
595
+ dependency['phase'] == "groom"
596
+ )
588
597
  addDependentThread(parent, "#{name}_create")
589
598
  end
590
599
  end
591
600
  }
592
601
  end
602
+ MU.log "Thread dependencies #{res_type}[#{name}]", MU::DEBUG, details: { "create" => @dependency_threads["#{name}_create"], "groom" => @dependency_threads["#{name}_groom"] }
603
+ @dependency_threads["#{name}_groom"]=["#{name}_create", "mu_groom_container"]
593
604
  }
594
605
  end
595
606
 
@@ -623,7 +634,7 @@ MESSAGE_END
623
634
  begin
624
635
  if myservice['#MUOBJECT'].nil?
625
636
  if @mommacat
626
- ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false)
637
+ ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false, ignore_missing: !@updating)
627
638
  if @updating and ext_obj
628
639
  ext_obj.config!(myservice)
629
640
  end
@@ -651,8 +662,9 @@ MESSAGE_END
651
662
  run_this_method = myservice['#MUOBJECT'].method(mode)
652
663
  rescue StandardError => e
653
664
  MU::MommaCat.unlockAll
654
- @main_thread.raise MuError, "Error invoking #{myservice["#MU_CLOUDCLASS"]}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
655
- raise e
665
+ @main_thread.raise MuError, "Error invoking #{myservice["#MUOBJECT"].class.name}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
666
+ return
667
+ # raise e
656
668
  end
657
669
  begin
658
670
  MU.log "Checking whether to run #{myservice['#MUOBJECT']}.#{mode} (updating: #{@updating})", MU::DEBUG
@@ -732,7 +744,9 @@ MESSAGE_END
732
744
  sleep 10+Random.rand(20)
733
745
  retry
734
746
  end
747
+
735
748
  end
749
+
736
750
  end
737
751
 
738
752
  end #class