cloud-mu 3.1.5 → 3.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +5 -1
  3. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  4. data/ansible/roles/mu-windows/files/config.xml +76 -0
  5. data/ansible/roles/mu-windows/tasks/main.yml +16 -0
  6. data/bin/mu-adopt +16 -12
  7. data/bin/mu-azure-tests +57 -0
  8. data/bin/mu-cleanup +2 -4
  9. data/bin/mu-configure +52 -0
  10. data/bin/mu-deploy +3 -3
  11. data/bin/mu-findstray-tests +25 -0
  12. data/bin/mu-gen-docs +2 -4
  13. data/bin/mu-load-config.rb +2 -1
  14. data/bin/mu-node-manage +15 -16
  15. data/bin/mu-run-tests +37 -12
  16. data/cloud-mu.gemspec +3 -3
  17. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  18. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  19. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  20. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  21. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  22. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  23. data/cookbooks/mu-tools/recipes/windows-client.rb +25 -22
  24. data/extras/clean-stock-amis +25 -19
  25. data/extras/generate-stock-images +1 -0
  26. data/extras/image-generators/AWS/win2k12.yaml +2 -0
  27. data/extras/image-generators/AWS/win2k16.yaml +2 -0
  28. data/extras/image-generators/AWS/win2k19.yaml +2 -0
  29. data/modules/mommacat.ru +1 -1
  30. data/modules/mu.rb +86 -98
  31. data/modules/mu/adoption.rb +373 -58
  32. data/modules/mu/cleanup.rb +214 -303
  33. data/modules/mu/cloud.rb +128 -1733
  34. data/modules/mu/cloud/database.rb +49 -0
  35. data/modules/mu/cloud/dnszone.rb +44 -0
  36. data/modules/mu/cloud/machine_images.rb +212 -0
  37. data/modules/mu/cloud/providers.rb +81 -0
  38. data/modules/mu/cloud/resource_base.rb +929 -0
  39. data/modules/mu/cloud/server.rb +40 -0
  40. data/modules/mu/cloud/server_pool.rb +1 -0
  41. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  42. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  43. data/modules/mu/cloud/wrappers.rb +169 -0
  44. data/modules/mu/config.rb +123 -81
  45. data/modules/mu/config/alarm.rb +2 -6
  46. data/modules/mu/config/bucket.rb +32 -3
  47. data/modules/mu/config/cache_cluster.rb +2 -2
  48. data/modules/mu/config/cdn.rb +100 -0
  49. data/modules/mu/config/collection.rb +1 -1
  50. data/modules/mu/config/container_cluster.rb +7 -2
  51. data/modules/mu/config/database.rb +84 -105
  52. data/modules/mu/config/database.yml +1 -2
  53. data/modules/mu/config/dnszone.rb +5 -4
  54. data/modules/mu/config/doc_helpers.rb +5 -6
  55. data/modules/mu/config/endpoint.rb +2 -1
  56. data/modules/mu/config/firewall_rule.rb +3 -19
  57. data/modules/mu/config/folder.rb +1 -1
  58. data/modules/mu/config/function.rb +17 -8
  59. data/modules/mu/config/group.rb +1 -1
  60. data/modules/mu/config/habitat.rb +1 -1
  61. data/modules/mu/config/job.rb +89 -0
  62. data/modules/mu/config/loadbalancer.rb +57 -11
  63. data/modules/mu/config/log.rb +1 -1
  64. data/modules/mu/config/msg_queue.rb +1 -1
  65. data/modules/mu/config/nosqldb.rb +1 -1
  66. data/modules/mu/config/notifier.rb +8 -19
  67. data/modules/mu/config/ref.rb +92 -14
  68. data/modules/mu/config/role.rb +1 -1
  69. data/modules/mu/config/schema_helpers.rb +38 -37
  70. data/modules/mu/config/search_domain.rb +1 -1
  71. data/modules/mu/config/server.rb +12 -13
  72. data/modules/mu/config/server_pool.rb +3 -7
  73. data/modules/mu/config/storage_pool.rb +1 -1
  74. data/modules/mu/config/tail.rb +11 -0
  75. data/modules/mu/config/user.rb +1 -1
  76. data/modules/mu/config/vpc.rb +27 -23
  77. data/modules/mu/config/vpc.yml +0 -1
  78. data/modules/mu/defaults/AWS.yaml +90 -90
  79. data/modules/mu/defaults/Azure.yaml +1 -0
  80. data/modules/mu/defaults/Google.yaml +1 -0
  81. data/modules/mu/deploy.rb +34 -20
  82. data/modules/mu/groomer.rb +16 -1
  83. data/modules/mu/groomers/ansible.rb +69 -4
  84. data/modules/mu/groomers/chef.rb +51 -4
  85. data/modules/mu/logger.rb +120 -144
  86. data/modules/mu/master.rb +97 -4
  87. data/modules/mu/mommacat.rb +160 -874
  88. data/modules/mu/mommacat/daemon.rb +23 -14
  89. data/modules/mu/mommacat/naming.rb +110 -3
  90. data/modules/mu/mommacat/search.rb +497 -0
  91. data/modules/mu/mommacat/storage.rb +252 -194
  92. data/modules/mu/{clouds → providers}/README.md +1 -1
  93. data/modules/mu/{clouds → providers}/aws.rb +258 -57
  94. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  95. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  96. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  97. data/modules/mu/providers/aws/cdn.rb +782 -0
  98. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  99. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +95 -84
  100. data/modules/mu/providers/aws/database.rb +1744 -0
  101. data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
  102. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  103. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
  104. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  105. data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
  106. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  107. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  108. data/modules/mu/providers/aws/job.rb +466 -0
  109. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
  110. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  111. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  112. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  113. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  114. data/modules/mu/{clouds → providers}/aws/role.rb +76 -48
  115. data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
  116. data/modules/mu/{clouds → providers}/aws/server.rb +66 -98
  117. data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
  118. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  119. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  120. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  121. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  122. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  123. data/modules/mu/{clouds → providers}/aws/vpc.rb +143 -74
  124. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  125. data/modules/mu/{clouds → providers}/azure.rb +13 -0
  126. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  127. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  128. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  129. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  130. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  131. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  132. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  133. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  134. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  135. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  136. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  137. data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
  138. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  139. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  140. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  141. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  142. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  143. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  144. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  145. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  146. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  147. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  148. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  149. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  150. data/modules/mu/{clouds → providers}/google.rb +29 -6
  151. data/modules/mu/{clouds → providers}/google/bucket.rb +4 -4
  152. data/modules/mu/{clouds → providers}/google/container_cluster.rb +38 -20
  153. data/modules/mu/{clouds → providers}/google/database.rb +5 -12
  154. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
  155. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  156. data/modules/mu/{clouds → providers}/google/function.rb +6 -6
  157. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  158. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  159. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
  160. data/modules/mu/{clouds → providers}/google/role.rb +50 -31
  161. data/modules/mu/{clouds → providers}/google/server.rb +41 -24
  162. data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
  163. data/modules/mu/{clouds → providers}/google/user.rb +34 -24
  164. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  165. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  166. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  167. data/modules/mu/{clouds → providers}/google/vpc.rb +45 -14
  168. data/modules/tests/aws-jobs-functions.yaml +46 -0
  169. data/modules/tests/centos6.yaml +15 -0
  170. data/modules/tests/centos7.yaml +15 -0
  171. data/modules/tests/centos8.yaml +12 -0
  172. data/modules/tests/ecs.yaml +2 -2
  173. data/modules/tests/eks.yaml +1 -1
  174. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  175. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  176. data/modules/tests/microservice_app.yaml +288 -0
  177. data/modules/tests/rds.yaml +108 -0
  178. data/modules/tests/regrooms/rds.yaml +123 -0
  179. data/modules/tests/server-with-scrub-muisms.yaml +1 -1
  180. data/modules/tests/super_complex_bok.yml +2 -2
  181. data/modules/tests/super_simple_bok.yml +3 -5
  182. data/spec/mu/clouds/azure_spec.rb +2 -2
  183. metadata +122 -92
  184. data/modules/mu/clouds/aws/database.rb +0 -1974
  185. data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -144,10 +144,10 @@ module MU
144
144
  # @param noop [Boolean]: If true, will only print what would be done
145
145
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
146
146
  # @return [void]
147
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
148
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
147
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
148
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
149
149
 
150
- resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['project'])
150
+ resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['habitat'])
151
151
  if resp and resp.items
152
152
  resp.items.each { |bucket|
153
153
  if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or bucket.labels['mu-master-ip'] == MU.mu_public_ip.gsub(/\./, "_"))
@@ -243,7 +243,7 @@ module MU
243
243
  grantees[binding.role] << { "id" => grantee }
244
244
  elsif grantee.match(/^serviceAccount:(.*)/)
245
245
  sa_name = Regexp.last_match[1]
246
- if MU::Cloud::Google::User.cannedServiceAcctName?(sa_name)
246
+ if MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(sa_name)
247
247
  grantees[binding.role] << { "id" => grantee }
248
248
  else
249
249
  grantees[binding.role] << MU::Config::Ref.get(
@@ -474,7 +474,6 @@ module MU
474
474
  MU.log %Q{How to interact with your GKE cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
475
475
  end
476
476
 
477
-
478
477
  # Locate an existing ContainerCluster or ContainerClusters and return an array containing matching GCP resource descriptors for those that match.
479
478
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ContainerClusters
480
479
  def self.find(**args)
@@ -658,7 +657,7 @@ module MU
658
657
  end
659
658
 
660
659
  if bok['service_account']
661
- found = MU::Cloud::Google::User.find(
660
+ found = MU::Cloud.resourceClass("Google", "User").find(
662
661
  credentials: bok['credentials'],
663
662
  project: bok['project'],
664
663
  cloud_id: bok['service_account']
@@ -745,17 +744,17 @@ module MU
745
744
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
746
745
  # @param region [String]: The cloud provider region in which to operate
747
746
  # @return [void]
748
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
747
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
749
748
 
750
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
751
- return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
749
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
750
+ return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
752
751
  clusters = []
753
752
 
754
753
  # Make sure we catch regional *and* zone clusters
755
- found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{region}")
754
+ found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{region}")
756
755
  clusters.concat(found.clusters) if found and found.clusters
757
756
  MU::Cloud::Google.listAZs(region).each { |az|
758
- found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{az}")
757
+ found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{az}")
759
758
  clusters.concat(found.clusters) if found and found.clusters
760
759
  }
761
760
 
@@ -818,10 +817,10 @@ module MU
818
817
  "type" => "integer",
819
818
  "description" => "The number of local SSD disks to be attached to workers. See https://cloud.google.com/compute/docs/disks/local-ssd#local_ssd_limits"
820
819
  },
821
- "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
822
- "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
823
- "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
824
- "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
820
+ "ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
821
+ "metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
822
+ "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
823
+ "scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
825
824
  "private_cluster" => {
826
825
  "description" => "Set a GKE cluster to be private, that is segregated into its own hidden VPC.",
827
826
  "type" => "object",
@@ -1022,6 +1021,25 @@ module MU
1022
1021
  cluster['ip_aliases'] = true
1023
1022
  end
1024
1023
 
1024
+ # try to stake out some nice /21s for our networking config
1025
+ if cluster['ip_aliases'] and cluster["vpc"] and cluster["vpc"]["id"]
1026
+ habarg = if cluster["vpc"]["habitat"] and cluster["vpc"]["habitat"]["id"]
1027
+ cluster["vpc"]["habitat"]["id"]
1028
+ else
1029
+ cluster["project"]
1030
+ end
1031
+ found = MU::MommaCat.findStray("Google", "vpcs", cloud_id: cluster["vpc"]["id"], credentials: cluster["credentials"], habitats: [habarg], dummy_ok: true)
1032
+ if found and found.size == 1
1033
+ myvpc = found.first
1034
+ # XXX this might not make sense with custom_subnet
1035
+ cluster['pod_ip_block'] ||= myvpc.getUnusedAddressBlock(max_bits: 21)
1036
+ cluster['services_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block']], max_bits: 21)
1037
+ if cluster['tpu']
1038
+ cluster['tpu_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block'], cluster['services_ip_block']], max_bits: 21)
1039
+ end
1040
+ end
1041
+ end
1042
+
1025
1043
  if cluster['service_account']
1026
1044
  cluster['service_account']['cloud'] = "Google"
1027
1045
  cluster['service_account']['habitat'] ||= MU::Config::Ref.get(
@@ -1033,11 +1051,7 @@ module MU
1033
1051
  if cluster['service_account']['name'] and
1034
1052
  !cluster['service_account']['id'] and
1035
1053
  !cluster['service_account']['deploy_id']
1036
- cluster['dependencies'] ||= []
1037
- cluster['dependencies'] << {
1038
- "type" => "user",
1039
- "name" => cluster['service_account']['name']
1040
- }
1054
+ MU::Config.addDependency(cluster, cluster['service_account']['name'], "user")
1041
1055
  end
1042
1056
  found = MU::Config::Ref.get(cluster['service_account'])
1043
1057
  # XXX verify that found.kitten fails when it's supposed to
@@ -1046,7 +1060,7 @@ module MU
1046
1060
  ok = false
1047
1061
  end
1048
1062
  else
1049
- cluster = MU::Cloud::Google::User.genericServiceAccount(cluster, configurator)
1063
+ cluster = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(cluster, configurator)
1050
1064
  end
1051
1065
 
1052
1066
  if cluster['dependencies']
@@ -1097,7 +1111,7 @@ module MU
1097
1111
  }
1098
1112
  if !match
1099
1113
  MU.log "No version matching #{cluster['kubernetes']['version']} available, will try floating minor revision", MU::WARN
1100
- cluster['kubernetes']['version'].sub!(/^(\d+\.\d+\.).*/i, '\1')
1114
+ cluster['kubernetes']['version'].sub!(/^(\d+\.\d+)\..*/i, '\1')
1101
1115
  master_versions.each { |v|
1102
1116
  if v.match(/^#{Regexp.quote(cluster['kubernetes']['version'])}/)
1103
1117
  match = true
@@ -1142,9 +1156,13 @@ module MU
1142
1156
  end
1143
1157
  end
1144
1158
 
1145
- cluster['instance_type'] = MU::Cloud::Google::Server.validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
1159
+ cluster['instance_type'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
1146
1160
  ok = false if cluster['instance_type'].nil?
1147
1161
 
1162
+ if !MU::Master.kubectl
1163
+ MU.log "Since I can't find a kubectl executable, you will have to handle all service account, user, and role bindings manually!", MU::WARN
1164
+ end
1165
+
1148
1166
  ok
1149
1167
  end
1150
1168
 
@@ -1236,7 +1254,7 @@ module MU
1236
1254
  # Take this opportunity to ensure that the 'client' service account
1237
1255
  # used by certificate authentication exists and has appropriate
1238
1256
  # privilege
1239
- if @username and @password
1257
+ if @username and @password and MU::Master.kubectl
1240
1258
  File.open(client_binding, "w"){ |k|
1241
1259
  k.puts <<-EOF
1242
1260
  kind: ClusterRoleBinding
@@ -25,14 +25,7 @@ module MU
25
25
  @config["groomer"] = MU::Config.defaultGroomer unless @config["groomer"]
26
26
  @groomclass = MU::Groomer.loadGroomer(@config["groomer"])
27
27
 
28
- @mu_name ||=
29
- if @config and @config['engine'] and @config["engine"].match(/^sqlserver/)
30
- @deploy.getResourceName(@config["name"], max_length: 15)
31
- else
32
- @deploy.getResourceName(@config["name"], max_length: 63)
33
- end
34
-
35
- @mu_name.gsub(/(--|-$)/i, "").gsub(/(_)/, "-").gsub!(/^[^a-z]/i, "")
28
+ @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 63)
36
29
  end
37
30
 
38
31
  # Called automatically by {MU::Deploy#createResources}
@@ -107,14 +100,14 @@ module MU
107
100
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
108
101
  # @param region [String]: The cloud provider region in which to operate
109
102
  # @return [void]
110
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
111
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
103
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
104
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
112
105
 
113
- # instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['project'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"})
106
+ # instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['habitat'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"})
114
107
  # if instances and instances.items
115
108
  # instances.items.each { |instance|
116
109
  # MU.log "Deleting Cloud SQL instance #{instance.name}"
117
- # MU::Cloud::Google.sql(credentials: credentials).delete_instance(flags['project'], instance.name) if !noop
110
+ # MU::Cloud::Google.sql(credentials: credentials).delete_instance(flags['habitat'], instance.name) if !noop
118
111
  # }
119
112
  # end
120
113
  end
@@ -207,9 +207,9 @@ end
207
207
  # @param noop [Boolean]: If true, will only print what would be done
208
208
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
209
209
  # @return [void]
210
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
211
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
212
- return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
210
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
211
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
212
+ return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
213
213
  filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
214
214
  if !ignoremaster and MU.mu_public_ip
215
215
  filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -218,7 +218,7 @@ end
218
218
 
219
219
  MU::Cloud::Google.compute(credentials: credentials).delete(
220
220
  "firewall",
221
- flags["project"],
221
+ flags["habitat"],
222
222
  nil,
223
223
  noop
224
224
  )
@@ -440,7 +440,7 @@ end
440
440
  elsif acl['vpc']['habitat'] and acl['vpc']['habitat']['name']
441
441
  acl['vpc']['project'] = acl['vpc']['habitat']['name']
442
442
  end
443
- correct_vpc = MU::Cloud::Google::VPC.pickVPC(
443
+ correct_vpc = MU::Cloud.resourceClass("Google", "VPC").pickVPC(
444
444
  acl['vpc'],
445
445
  acl,
446
446
  "firewall_rule",
@@ -138,7 +138,7 @@ module MU
138
138
  # Return the metadata for this folders's configuration
139
139
  # @return [Hash]
140
140
  def notify
141
- desc = MU.structToHash(MU::Cloud::Google.folder(credentials: @config['credentials']).get_folder("folders/"+@cloud_id))
141
+ desc = MU.structToHash(cloud_desc)
142
142
  desc["mu_name"] = @mu_name
143
143
  desc["parent"] = @parent
144
144
  desc["cloud_id"] = @cloud_id
@@ -162,7 +162,7 @@ module MU
162
162
  # @param noop [Boolean]: If true, will only print what would be done
163
163
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
164
164
  # @return [void]
165
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
165
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
166
166
  filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
167
167
  if !ignoremaster and MU.mu_public_ip
168
168
  filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -236,10 +236,10 @@ module MU
236
236
  # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
237
237
  def self.find(**args)
238
238
  found = {}
239
-
240
239
  # Recursively search a GCP folder hierarchy for a folder matching our
241
240
  # supplied name or identifier.
242
241
  def self.find_matching_folder(parent, name: nil, id: nil, credentials: nil)
242
+
243
243
  resp = MU::Cloud::Google.folder(credentials: credentials).list_folders(parent: parent)
244
244
  if resp and resp.folders
245
245
  resp.folders.each { |f|
@@ -278,6 +278,7 @@ module MU
278
278
  end
279
279
  else
280
280
  resp = MU::Cloud::Google.folder(credentials: args[:credentials]).list_folders(parent: parent)
281
+
281
282
  if resp and resp.folders
282
283
  resp.folders.each { |folder|
283
284
  next if folder.lifecycle_state == "DELETE_REQUESTED"
@@ -310,7 +311,6 @@ module MU
310
311
  bok['cloud_id'] = cloud_desc.name
311
312
  bok['name'] = cloud_desc.display_name#+bok['cloud_id'] # only way to guarantee uniqueness
312
313
  if cloud_desc.parent.match(/^folders\/(.*)/)
313
- MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_desc.parent
314
314
  bok['parent'] = MU::Config::Ref.get(
315
315
  id: cloud_desc.parent,
316
316
  cloud: "Google",
@@ -355,11 +355,7 @@ MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_d
355
355
  end
356
356
 
357
357
  if folder['parent'] and folder['parent']['name'] and !folder['parent']['deploy_id'] and configurator.haveLitterMate?(folder['parent']['name'], "folders")
358
- folder["dependencies"] ||= []
359
- folder["dependencies"] << {
360
- "type" => "folder",
361
- "name" => folder['parent']['name']
362
- }
358
+ MU::Config.addDependency(folder, folder['parent']['name'], "folder")
363
359
  end
364
360
 
365
361
  ok
@@ -233,11 +233,11 @@ module example.com/cloudfunction
233
233
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
234
234
  # @param region [String]: The cloud provider region
235
235
  # @return [void]
236
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
237
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
238
- return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
236
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
237
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
238
+ return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
239
239
  # Make sure we catch regional *and* zone functions
240
- found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["project"])
240
+ found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["habitat"])
241
241
  found.each_pair { |cloud_id, desc|
242
242
  if (desc.description and desc.description == MU.deploy_id) or
243
243
  (desc.labels and desc.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or desc.labels["mu-master-ip"] == MU.mu_public_ip.gsub(/\./, "_"))) or
@@ -373,7 +373,7 @@ module example.com/cloudfunction
373
373
  }
374
374
  }
375
375
  },
376
- "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
376
+ "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
377
377
  "runtime" => {
378
378
  "type" => "string",
379
379
  "enum" => %w{nodejs go python nodejs8 nodejs10 python37 go111 go113},
@@ -524,7 +524,7 @@ module example.com/cloudfunction
524
524
  ok = false
525
525
  end
526
526
  else
527
- function = MU::Cloud::Google::User.genericServiceAccount(function, configurator)
527
+ function = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(function, configurator)
528
528
  end
529
529
 
530
530
  # siblings = configurator.haveLitterMate?(nil, "vpcs", has_multiple: true)
@@ -44,7 +44,7 @@ module MU
44
44
  resp = MU::Cloud::Google.admin_directory(credentials: @credentials).insert_group(group_obj)
45
45
  @cloud_id = resp.email
46
46
 
47
- MU::Cloud::Google::Role.bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'])
47
+ MU::Cloud.resourceClass("Google", "Role").bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'])
48
48
  else
49
49
  @cloud_id = @config['name'].sub(/@.*/, "")+"@"+@config['domain']
50
50
  end
@@ -52,7 +52,7 @@ module MU
52
52
 
53
53
  # Called automatically by {MU::Deploy#createResources}
54
54
  def groom
55
- MU::Cloud::Google::Role.bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'], debug: true)
55
+ MU::Cloud.resourceClass("Google", "Role").bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'], debug: true)
56
56
 
57
57
  if @config['members']
58
58
  resolved_desired = []
@@ -140,7 +140,7 @@ module MU
140
140
  # @param noop [Boolean]: If true, will only print what would be done
141
141
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
142
142
  # @return [void]
143
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
143
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
144
144
  MU::Cloud::Google.getDomains(credentials)
145
145
  my_org = MU::Cloud::Google.getOrg(credentials)
146
146
 
@@ -166,7 +166,7 @@ module MU
166
166
 
167
167
  if flags['known']
168
168
  flags['known'].each { |group|
169
- MU::Cloud::Google::Role.removeBindings("group", group, credentials: credentials, noop: noop)
169
+ MU::Cloud.resourceClass("Google", "Role").removeBindings("group", group, credentials: credentials, noop: noop)
170
170
  }
171
171
  end
172
172
  end
@@ -222,10 +222,10 @@ module MU
222
222
  # type: "users"
223
223
  # )
224
224
  # }
225
- group_roles = MU::Cloud::Google::Role.getAllBindings(@config['credentials'])["by_entity"]
225
+ group_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
226
226
  if group_roles["group"] and group_roles["group"][bok['cloud_id']] and
227
227
  group_roles["group"][bok['cloud_id']].size > 0
228
- bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(group_roles["group"][bok['cloud_id']], credentials: @config['credentials'])
228
+ bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(group_roles["group"][bok['cloud_id']], credentials: @config['credentials'])
229
229
  end
230
230
 
231
231
  bok
@@ -264,7 +264,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
264
264
 
265
265
  "roles" => {
266
266
  "type" => "array",
267
- "items" => MU::Cloud::Google::Role.ref_schema
267
+ "items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
268
268
  }
269
269
  }
270
270
  [toplevel_required, schema]
@@ -340,11 +340,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
340
340
  if group['members']
341
341
  group['members'].each { |m|
342
342
  if configurator.haveLitterMate?(m, "users")
343
- group['dependencies'] ||= []
344
- group['dependencies'] << {
345
- "name" => m,
346
- "type" => "user"
347
- }
343
+ MU::Config.addDependency(group, m, "user")
348
344
  end
349
345
  }
350
346
  end
@@ -353,11 +349,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
353
349
  group['roles'].each { |r|
354
350
  if r['role'] and r['role']['name'] and
355
351
  (!r['role']['deploy_id'] and !r['role']['id'])
356
- group['dependencies'] ||= []
357
- group['dependencies'] << {
358
- "type" => "role",
359
- "name" => r['role']['name']
360
- }
352
+ MU::Config.addDependency(group, r['role']['name'], "role")
361
353
  end
362
354
  }
363
355
  end
@@ -61,7 +61,7 @@ module MU
61
61
  if @config['parent']['name'] and !@config['parent']['id']
62
62
  @config['parent']['deploy_id'] = @deploy.deploy_id
63
63
  end
64
- parent = MU::Cloud::Google::Folder.resolveParent(@config['parent'], credentials: @config['credentials'])
64
+ parent = MU::Cloud.resourceClass("Google", "Folder").resolveParent(@config['parent'], credentials: @config['credentials'])
65
65
  if !parent
66
66
  MU.log "Unable to resolve parent resource of Google Project #{@config['name']}", MU::ERR, details: @config['parent']
67
67
  raise "Unable to resolve parent resource of Google Project #{@config['name']}"
@@ -222,7 +222,7 @@ module MU
222
222
  # @param noop [Boolean]: If true, will only print what would be done
223
223
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
224
224
  # @return [void]
225
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
225
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
226
226
  resp = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
227
227
 
228
228
  if resp and resp.projects
@@ -285,7 +285,7 @@ module MU
285
285
  next if p.lifecycle_state == "DELETE_REQUESTED"
286
286
  found[p.project_id] = p
287
287
  }
288
- @@list_projects_cache = found
288
+ @@list_projects_cache = found.clone
289
289
  end
290
290
 
291
291
  found
@@ -376,11 +376,7 @@ module MU
376
376
  end
377
377
 
378
378
  if habitat['parent'] and habitat['parent']['name'] and !habitat['parent']['deploy_id'] and configurator.haveLitterMate?(habitat['parent']['name'], "folders")
379
- habitat["dependencies"] ||= []
380
- habitat["dependencies"] << {
381
- "type" => "folder",
382
- "name" => habitat['parent']['name']
383
- }
379
+ MU::Config.addDependency(habitat, habitat['parent']['name'], "folder")
384
380
  end
385
381
 
386
382
  ok
@@ -146,9 +146,9 @@ module MU
146
146
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
147
147
  # @param region [String]: The cloud provider region
148
148
  # @return [void]
149
- def self.cleanup(noop: false, ignoremaster: false, region: nil, credentials: nil, flags: {})
150
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
151
- return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
149
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: nil, credentials: nil, flags: {})
150
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
151
+ return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
152
152
  filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
153
153
  if !ignoremaster and MU.mu_public_ip
154
154
  filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -159,7 +159,7 @@ module MU
159
159
  ["forwarding_rule", "region_backend_service"].each { |type|
160
160
  MU::Cloud::Google.compute(credentials: credentials).delete(
161
161
  type,
162
- flags["project"],
162
+ flags["habitat"],
163
163
  region,
164
164
  noop
165
165
  )
@@ -170,7 +170,7 @@ module MU
170
170
  ["global_forwarding_rule", "target_http_proxy", "target_https_proxy", "url_map", "backend_service", "health_check", "http_health_check", "https_health_check"].each { |type|
171
171
  MU::Cloud::Google.compute(credentials: credentials).delete(
172
172
  type,
173
- flags["project"],
173
+ flags["habitat"],
174
174
  nil,
175
175
  noop
176
176
  )