cloud-mu 3.1.4 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +5 -1
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +16 -12
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +2 -1
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +37 -12
  23. data/cloud-mu.gemspec +5 -3
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  27. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  28. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  29. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  30. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  31. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  32. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  33. data/extras/clean-stock-amis +25 -19
  34. data/extras/generate-stock-images +1 -0
  35. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  36. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  37. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  38. data/modules/mommacat.ru +1 -1
  39. data/modules/mu.rb +158 -107
  40. data/modules/mu/adoption.rb +386 -59
  41. data/modules/mu/cleanup.rb +214 -303
  42. data/modules/mu/cloud.rb +128 -1632
  43. data/modules/mu/cloud/database.rb +49 -0
  44. data/modules/mu/cloud/dnszone.rb +44 -0
  45. data/modules/mu/cloud/machine_images.rb +212 -0
  46. data/modules/mu/cloud/providers.rb +81 -0
  47. data/modules/mu/cloud/resource_base.rb +926 -0
  48. data/modules/mu/cloud/server.rb +40 -0
  49. data/modules/mu/cloud/server_pool.rb +1 -0
  50. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  51. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  52. data/modules/mu/cloud/wrappers.rb +169 -0
  53. data/modules/mu/config.rb +135 -82
  54. data/modules/mu/config/alarm.rb +2 -6
  55. data/modules/mu/config/bucket.rb +32 -3
  56. data/modules/mu/config/cache_cluster.rb +2 -2
  57. data/modules/mu/config/cdn.rb +100 -0
  58. data/modules/mu/config/collection.rb +1 -1
  59. data/modules/mu/config/container_cluster.rb +7 -2
  60. data/modules/mu/config/database.rb +84 -105
  61. data/modules/mu/config/database.yml +1 -2
  62. data/modules/mu/config/dnszone.rb +5 -4
  63. data/modules/mu/config/doc_helpers.rb +5 -6
  64. data/modules/mu/config/endpoint.rb +2 -1
  65. data/modules/mu/config/firewall_rule.rb +3 -19
  66. data/modules/mu/config/folder.rb +1 -1
  67. data/modules/mu/config/function.rb +17 -8
  68. data/modules/mu/config/group.rb +1 -1
  69. data/modules/mu/config/habitat.rb +1 -1
  70. data/modules/mu/config/job.rb +89 -0
  71. data/modules/mu/config/loadbalancer.rb +57 -11
  72. data/modules/mu/config/log.rb +1 -1
  73. data/modules/mu/config/msg_queue.rb +1 -1
  74. data/modules/mu/config/nosqldb.rb +1 -1
  75. data/modules/mu/config/notifier.rb +8 -19
  76. data/modules/mu/config/ref.rb +92 -14
  77. data/modules/mu/config/role.rb +1 -1
  78. data/modules/mu/config/schema_helpers.rb +38 -37
  79. data/modules/mu/config/search_domain.rb +1 -1
  80. data/modules/mu/config/server.rb +12 -13
  81. data/modules/mu/config/server.yml +1 -0
  82. data/modules/mu/config/server_pool.rb +3 -7
  83. data/modules/mu/config/storage_pool.rb +1 -1
  84. data/modules/mu/config/tail.rb +11 -0
  85. data/modules/mu/config/user.rb +1 -1
  86. data/modules/mu/config/vpc.rb +27 -23
  87. data/modules/mu/config/vpc.yml +0 -1
  88. data/modules/mu/defaults/AWS.yaml +91 -68
  89. data/modules/mu/defaults/Azure.yaml +1 -0
  90. data/modules/mu/defaults/Google.yaml +1 -0
  91. data/modules/mu/deploy.rb +33 -19
  92. data/modules/mu/groomer.rb +16 -1
  93. data/modules/mu/groomers/ansible.rb +123 -21
  94. data/modules/mu/groomers/chef.rb +64 -11
  95. data/modules/mu/logger.rb +120 -144
  96. data/modules/mu/master.rb +97 -4
  97. data/modules/mu/master/ssl.rb +0 -1
  98. data/modules/mu/mommacat.rb +154 -867
  99. data/modules/mu/mommacat/daemon.rb +23 -14
  100. data/modules/mu/mommacat/naming.rb +110 -3
  101. data/modules/mu/mommacat/search.rb +495 -0
  102. data/modules/mu/mommacat/storage.rb +225 -192
  103. data/modules/mu/{clouds → providers}/README.md +1 -1
  104. data/modules/mu/{clouds → providers}/aws.rb +281 -64
  105. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  106. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  107. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  108. data/modules/mu/providers/aws/cdn.rb +782 -0
  109. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  110. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +708 -749
  111. data/modules/mu/providers/aws/database.rb +1744 -0
  112. data/modules/mu/{clouds → providers}/aws/dnszone.rb +75 -57
  113. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  114. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +212 -242
  115. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  116. data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
  117. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  118. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  119. data/modules/mu/providers/aws/job.rb +466 -0
  120. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +50 -41
  121. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  122. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  123. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  124. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  125. data/modules/mu/{clouds → providers}/aws/role.rb +94 -57
  126. data/modules/mu/{clouds → providers}/aws/search_domain.rb +173 -42
  127. data/modules/mu/{clouds → providers}/aws/server.rb +782 -1107
  128. data/modules/mu/{clouds → providers}/aws/server_pool.rb +36 -46
  129. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  130. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  131. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  132. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  133. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  134. data/modules/mu/{clouds → providers}/aws/vpc.rb +429 -849
  135. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  136. data/modules/mu/{clouds → providers}/azure.rb +13 -0
  137. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  138. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  139. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  140. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  141. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  142. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  143. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  144. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  145. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  146. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  147. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  148. data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
  149. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  150. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  151. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  152. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  153. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  154. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  155. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  156. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  158. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  159. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  160. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  161. data/modules/mu/{clouds → providers}/google.rb +29 -6
  162. data/modules/mu/{clouds → providers}/google/bucket.rb +5 -5
  163. data/modules/mu/{clouds → providers}/google/container_cluster.rb +59 -37
  164. data/modules/mu/{clouds → providers}/google/database.rb +5 -12
  165. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
  166. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  167. data/modules/mu/{clouds → providers}/google/function.rb +14 -8
  168. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  169. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  170. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
  171. data/modules/mu/{clouds → providers}/google/role.rb +50 -31
  172. data/modules/mu/{clouds → providers}/google/server.rb +142 -55
  173. data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
  174. data/modules/mu/{clouds → providers}/google/user.rb +34 -24
  175. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  176. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  177. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  178. data/modules/mu/{clouds → providers}/google/vpc.rb +46 -15
  179. data/modules/tests/aws-jobs-functions.yaml +46 -0
  180. data/modules/tests/centos6.yaml +15 -0
  181. data/modules/tests/centos7.yaml +15 -0
  182. data/modules/tests/centos8.yaml +12 -0
  183. data/modules/tests/ecs.yaml +23 -0
  184. data/modules/tests/eks.yaml +1 -1
  185. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  186. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  187. data/modules/tests/includes-and-params.yaml +2 -1
  188. data/modules/tests/microservice_app.yaml +288 -0
  189. data/modules/tests/rds.yaml +108 -0
  190. data/modules/tests/regrooms/rds.yaml +123 -0
  191. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  192. data/modules/tests/super_complex_bok.yml +2 -2
  193. data/modules/tests/super_simple_bok.yml +3 -5
  194. data/modules/tests/win2k12.yaml +25 -0
  195. data/modules/tests/win2k16.yaml +25 -0
  196. data/modules/tests/win2k19.yaml +25 -0
  197. data/requirements.txt +1 -0
  198. data/spec/mu/clouds/azure_spec.rb +2 -2
  199. metadata +169 -93
  200. data/extras/image-generators/AWS/windows.yaml +0 -18
  201. data/modules/mu/clouds/aws/database.rb +0 -1974
  202. data/modules/mu/clouds/aws/endpoint.rb +0 -596
  203. data/modules/tests/needwork/win2k12.yaml +0 -13
@@ -144,10 +144,10 @@ module MU
144
144
  # @param noop [Boolean]: If true, will only print what would be done
145
145
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
146
146
  # @return [void]
147
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
148
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
147
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
148
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
149
149
 
150
- resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['project'])
150
+ resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['habitat'])
151
151
  if resp and resp.items
152
152
  resp.items.each { |bucket|
153
153
  if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or bucket.labels['mu-master-ip'] == MU.mu_public_ip.gsub(/\./, "_"))
@@ -243,7 +243,7 @@ module MU
243
243
  grantees[binding.role] << { "id" => grantee }
244
244
  elsif grantee.match(/^serviceAccount:(.*)/)
245
245
  sa_name = Regexp.last_match[1]
246
- if MU::Cloud::Google::User.cannedServiceAcctName?(sa_name)
246
+ if MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(sa_name)
247
247
  grantees[binding.role] << { "id" => grantee }
248
248
  else
249
249
  grantees[binding.role] << MU::Config::Ref.get(
@@ -305,7 +305,7 @@ module MU
305
305
  schema = {
306
306
  "storage_class" => {
307
307
  "type" => "string",
308
- "enum" => ["MULTI_REGIONAL", "REGIONAL", "STANDARD", "NEARLINE", "COLDLINE", "DURABLE_REDUCED_AVAILABILITY"],
308
+ "enum" => ["MULTI_REGIONAL", "REGIONAL", "STANDARD", "NEARLINE", "COLDLINE", "DURABLE_REDUCED_AVAILABILITY", "ARCHIVE"],
309
309
  "default" => "STANDARD"
310
310
  },
311
311
  "bucket_wide_acls" => {
@@ -474,7 +474,6 @@ module MU
474
474
  MU.log %Q{How to interact with your GKE cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
475
475
  end
476
476
 
477
-
478
477
  # Locate an existing ContainerCluster or ContainerClusters and return an array containing matching GCP resource descriptors for those that match.
479
478
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ContainerClusters
480
479
  def self.find(**args)
@@ -567,22 +566,24 @@ module MU
567
566
  bok['kubernetes']['network_policy_addon'] = true
568
567
  end
569
568
 
570
- if cloud_desc.ip_allocation_policy.use_ip_aliases
571
- bok['ip_aliases'] = true
572
- end
573
- if cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
574
- bok['pod_ip_block'] = cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
575
- end
576
- if cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
577
- bok['services_ip_block'] = cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
578
- end
569
+ if cloud_desc.ip_allocation_policy
570
+ if cloud_desc.ip_allocation_policy.use_ip_aliases
571
+ bok['ip_aliases'] = true
572
+ end
573
+ if cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
574
+ bok['pod_ip_block'] = cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
575
+ end
576
+ if cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
577
+ bok['services_ip_block'] = cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
578
+ end
579
579
 
580
- if cloud_desc.ip_allocation_policy.create_subnetwork
581
- bok['custom_subnet'] = {
582
- "name" => (cloud_desc.ip_allocation_policy.subnetwork_name || cloud_desc.subnetwork)
583
- }
584
- if cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
585
- bok['custom_subnet']['node_ip_block'] = cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
580
+ if cloud_desc.ip_allocation_policy.create_subnetwork
581
+ bok['custom_subnet'] = {
582
+ "name" => (cloud_desc.ip_allocation_policy.subnetwork_name || cloud_desc.subnetwork)
583
+ }
584
+ if cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
585
+ bok['custom_subnet']['node_ip_block'] = cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
586
+ end
586
587
  end
587
588
  end
588
589
 
@@ -656,7 +657,7 @@ module MU
656
657
  end
657
658
 
658
659
  if bok['service_account']
659
- found = MU::Cloud::Google::User.find(
660
+ found = MU::Cloud.resourceClass("Google", "User").find(
660
661
  credentials: bok['credentials'],
661
662
  project: bok['project'],
662
663
  cloud_id: bok['service_account']
@@ -743,17 +744,17 @@ module MU
743
744
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
744
745
  # @param region [String]: The cloud provider region in which to operate
745
746
  # @return [void]
746
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
747
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
747
748
 
748
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
749
- return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
749
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
750
+ return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
750
751
  clusters = []
751
752
 
752
753
  # Make sure we catch regional *and* zone clusters
753
- found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{region}")
754
+ found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{region}")
754
755
  clusters.concat(found.clusters) if found and found.clusters
755
756
  MU::Cloud::Google.listAZs(region).each { |az|
756
- found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{az}")
757
+ found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{az}")
757
758
  clusters.concat(found.clusters) if found and found.clusters
758
759
  }
759
760
 
@@ -816,10 +817,10 @@ module MU
816
817
  "type" => "integer",
817
818
  "description" => "The number of local SSD disks to be attached to workers. See https://cloud.google.com/compute/docs/disks/local-ssd#local_ssd_limits"
818
819
  },
819
- "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
820
- "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
821
- "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
822
- "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
820
+ "ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
821
+ "metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
822
+ "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
823
+ "scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
823
824
  "private_cluster" => {
824
825
  "description" => "Set a GKE cluster to be private, that is segregated into its own hidden VPC.",
825
826
  "type" => "object",
@@ -1020,6 +1021,25 @@ module MU
1020
1021
  cluster['ip_aliases'] = true
1021
1022
  end
1022
1023
 
1024
+ # try to stake out some nice /21s for our networking config
1025
+ if cluster['ip_aliases'] and cluster["vpc"] and cluster["vpc"]["id"]
1026
+ habarg = if cluster["vpc"]["habitat"] and cluster["vpc"]["habitat"]["id"]
1027
+ cluster["vpc"]["habitat"]["id"]
1028
+ else
1029
+ cluster["project"]
1030
+ end
1031
+ found = MU::MommaCat.findStray("Google", "vpcs", cloud_id: cluster["vpc"]["id"], credentials: cluster["credentials"], habitats: [habarg], dummy_ok: true)
1032
+ if found and found.size == 1
1033
+ myvpc = found.first
1034
+ # XXX this might not make sense with custom_subnet
1035
+ cluster['pod_ip_block'] ||= myvpc.getUnusedAddressBlock(max_bits: 21)
1036
+ cluster['services_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block']], max_bits: 21)
1037
+ if cluster['tpu']
1038
+ cluster['tpu_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block'], cluster['services_ip_block']], max_bits: 21)
1039
+ end
1040
+ end
1041
+ end
1042
+
1023
1043
  if cluster['service_account']
1024
1044
  cluster['service_account']['cloud'] = "Google"
1025
1045
  cluster['service_account']['habitat'] ||= MU::Config::Ref.get(
@@ -1029,12 +1049,9 @@ module MU
1029
1049
  type: "habitats"
1030
1050
  )
1031
1051
  if cluster['service_account']['name'] and
1032
- !cluster['service_account']['id']
1033
- cluster['dependencies'] ||= []
1034
- cluster['dependencies'] << {
1035
- "type" => "user",
1036
- "name" => cluster['service_account']['name']
1037
- }
1052
+ !cluster['service_account']['id'] and
1053
+ !cluster['service_account']['deploy_id']
1054
+ MU::Config.addDependency(cluster, cluster['service_account']['name'], "user")
1038
1055
  end
1039
1056
  found = MU::Config::Ref.get(cluster['service_account'])
1040
1057
  # XXX verify that found.kitten fails when it's supposed to
@@ -1043,7 +1060,7 @@ module MU
1043
1060
  ok = false
1044
1061
  end
1045
1062
  else
1046
- cluster = MU::Cloud::Google::User.genericServiceAccount(cluster, configurator)
1063
+ cluster = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(cluster, configurator)
1047
1064
  end
1048
1065
 
1049
1066
  if cluster['dependencies']
@@ -1094,7 +1111,7 @@ module MU
1094
1111
  }
1095
1112
  if !match
1096
1113
  MU.log "No version matching #{cluster['kubernetes']['version']} available, will try floating minor revision", MU::WARN
1097
- cluster['kubernetes']['version'].sub!(/^(\d+\.\d+\.).*/i, '\1')
1114
+ cluster['kubernetes']['version'].sub!(/^(\d+\.\d+)\..*/i, '\1')
1098
1115
  master_versions.each { |v|
1099
1116
  if v.match(/^#{Regexp.quote(cluster['kubernetes']['version'])}/)
1100
1117
  match = true
@@ -1139,9 +1156,13 @@ module MU
1139
1156
  end
1140
1157
  end
1141
1158
 
1142
- cluster['instance_type'] = MU::Cloud::Google::Server.validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
1159
+ cluster['instance_type'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
1143
1160
  ok = false if cluster['instance_type'].nil?
1144
1161
 
1162
+ if !MU::Master.kubectl
1163
+ MU.log "Since I can't find a kubectl executable, you will have to handle all service account, user, and role bindings manually!", MU::WARN
1164
+ end
1165
+
1145
1166
  ok
1146
1167
  end
1147
1168
 
@@ -1188,7 +1209,8 @@ module MU
1188
1209
  labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
1189
1210
 
1190
1211
  labelset = MU::Cloud::Google.container(:SetLabelsRequest).new(
1191
- resource_labels: labels
1212
+ resource_labels: labels,
1213
+ label_fingerprint: cloud_desc.label_fingerprint
1192
1214
  )
1193
1215
  MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_resource_labels(@cloud_id, labelset)
1194
1216
  end
@@ -1232,7 +1254,7 @@ module MU
1232
1254
  # Take this opportunity to ensure that the 'client' service account
1233
1255
  # used by certificate authentication exists and has appropriate
1234
1256
  # privilege
1235
- if @username and @password
1257
+ if @username and @password and MU::Master.kubectl
1236
1258
  File.open(client_binding, "w"){ |k|
1237
1259
  k.puts <<-EOF
1238
1260
  kind: ClusterRoleBinding
@@ -25,14 +25,7 @@ module MU
25
25
  @config["groomer"] = MU::Config.defaultGroomer unless @config["groomer"]
26
26
  @groomclass = MU::Groomer.loadGroomer(@config["groomer"])
27
27
 
28
- @mu_name ||=
29
- if @config and @config['engine'] and @config["engine"].match(/^sqlserver/)
30
- @deploy.getResourceName(@config["name"], max_length: 15)
31
- else
32
- @deploy.getResourceName(@config["name"], max_length: 63)
33
- end
34
-
35
- @mu_name.gsub(/(--|-$)/i, "").gsub(/(_)/, "-").gsub!(/^[^a-z]/i, "")
28
+ @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 63)
36
29
  end
37
30
 
38
31
  # Called automatically by {MU::Deploy#createResources}
@@ -107,14 +100,14 @@ module MU
107
100
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
108
101
  # @param region [String]: The cloud provider region in which to operate
109
102
  # @return [void]
110
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
111
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
103
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
104
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
112
105
 
113
- # instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['project'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"})
106
+ # instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['habitat'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"})
114
107
  # if instances and instances.items
115
108
  # instances.items.each { |instance|
116
109
  # MU.log "Deleting Cloud SQL instance #{instance.name}"
117
- # MU::Cloud::Google.sql(credentials: credentials).delete_instance(flags['project'], instance.name) if !noop
110
+ # MU::Cloud::Google.sql(credentials: credentials).delete_instance(flags['habitat'], instance.name) if !noop
118
111
  # }
119
112
  # end
120
113
  end
@@ -207,9 +207,9 @@ end
207
207
  # @param noop [Boolean]: If true, will only print what would be done
208
208
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
209
209
  # @return [void]
210
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
211
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
212
- return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
210
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
211
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
212
+ return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
213
213
  filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
214
214
  if !ignoremaster and MU.mu_public_ip
215
215
  filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -218,7 +218,7 @@ end
218
218
 
219
219
  MU::Cloud::Google.compute(credentials: credentials).delete(
220
220
  "firewall",
221
- flags["project"],
221
+ flags["habitat"],
222
222
  nil,
223
223
  noop
224
224
  )
@@ -440,7 +440,7 @@ end
440
440
  elsif acl['vpc']['habitat'] and acl['vpc']['habitat']['name']
441
441
  acl['vpc']['project'] = acl['vpc']['habitat']['name']
442
442
  end
443
- correct_vpc = MU::Cloud::Google::VPC.pickVPC(
443
+ correct_vpc = MU::Cloud.resourceClass("Google", "VPC").pickVPC(
444
444
  acl['vpc'],
445
445
  acl,
446
446
  "firewall_rule",
@@ -138,7 +138,7 @@ module MU
138
138
  # Return the metadata for this folders's configuration
139
139
  # @return [Hash]
140
140
  def notify
141
- desc = MU.structToHash(MU::Cloud::Google.folder(credentials: @config['credentials']).get_folder("folders/"+@cloud_id))
141
+ desc = MU.structToHash(cloud_desc)
142
142
  desc["mu_name"] = @mu_name
143
143
  desc["parent"] = @parent
144
144
  desc["cloud_id"] = @cloud_id
@@ -162,7 +162,7 @@ module MU
162
162
  # @param noop [Boolean]: If true, will only print what would be done
163
163
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
164
164
  # @return [void]
165
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
165
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
166
166
  filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
167
167
  if !ignoremaster and MU.mu_public_ip
168
168
  filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -236,10 +236,10 @@ module MU
236
236
  # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
237
237
  def self.find(**args)
238
238
  found = {}
239
-
240
239
  # Recursively search a GCP folder hierarchy for a folder matching our
241
240
  # supplied name or identifier.
242
241
  def self.find_matching_folder(parent, name: nil, id: nil, credentials: nil)
242
+
243
243
  resp = MU::Cloud::Google.folder(credentials: credentials).list_folders(parent: parent)
244
244
  if resp and resp.folders
245
245
  resp.folders.each { |f|
@@ -278,6 +278,7 @@ module MU
278
278
  end
279
279
  else
280
280
  resp = MU::Cloud::Google.folder(credentials: args[:credentials]).list_folders(parent: parent)
281
+
281
282
  if resp and resp.folders
282
283
  resp.folders.each { |folder|
283
284
  next if folder.lifecycle_state == "DELETE_REQUESTED"
@@ -310,7 +311,6 @@ module MU
310
311
  bok['cloud_id'] = cloud_desc.name
311
312
  bok['name'] = cloud_desc.display_name#+bok['cloud_id'] # only way to guarantee uniqueness
312
313
  if cloud_desc.parent.match(/^folders\/(.*)/)
313
- MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_desc.parent
314
314
  bok['parent'] = MU::Config::Ref.get(
315
315
  id: cloud_desc.parent,
316
316
  cloud: "Google",
@@ -355,11 +355,7 @@ MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_d
355
355
  end
356
356
 
357
357
  if folder['parent'] and folder['parent']['name'] and !folder['parent']['deploy_id'] and configurator.haveLitterMate?(folder['parent']['name'], "folders")
358
- folder["dependencies"] ||= []
359
- folder["dependencies"] << {
360
- "type" => "folder",
361
- "name" => folder['parent']['name']
362
- }
358
+ MU::Config.addDependency(folder, folder['parent']['name'], "folder")
363
359
  end
364
360
 
365
361
  ok
@@ -233,11 +233,11 @@ module example.com/cloudfunction
233
233
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
234
234
  # @param region [String]: The cloud provider region
235
235
  # @return [void]
236
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
237
- flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
238
- return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
236
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
237
+ flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
238
+ return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
239
239
  # Make sure we catch regional *and* zone functions
240
- found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["project"])
240
+ found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["habitat"])
241
241
  found.each_pair { |cloud_id, desc|
242
242
  if (desc.description and desc.description == MU.deploy_id) or
243
243
  (desc.labels and desc.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or desc.labels["mu-master-ip"] == MU.mu_public_ip.gsub(/\./, "_"))) or
@@ -340,7 +340,7 @@ module example.com/cloudfunction
340
340
  end
341
341
 
342
342
  codefile = bok["project"]+"_"+bok["region"]+"_"+bok["name"]+".zip"
343
- MU::Cloud::Google::Function.downloadPackage(@cloud_id, codefile, credentials: @config['credentials'])
343
+ return nil if !MU::Cloud::Google::Function.downloadPackage(@cloud_id, codefile, credentials: @config['credentials'])
344
344
  bok['code'] = {
345
345
  'zip_file' => codefile
346
346
  }
@@ -373,7 +373,7 @@ module example.com/cloudfunction
373
373
  }
374
374
  }
375
375
  },
376
- "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
376
+ "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
377
377
  "runtime" => {
378
378
  "type" => "string",
379
379
  "enum" => %w{nodejs go python nodejs8 nodejs10 python37 go111 go113},
@@ -417,7 +417,12 @@ module example.com/cloudfunction
417
417
  bucket = Regexp.last_match[1]
418
418
  path = Regexp.last_match[2]
419
419
 
420
- MU::Cloud::Google.storage(credentials: credentials).get_object(bucket, path, download_dest: zipfile)
420
+ begin
421
+ MU::Cloud::Google.storage(credentials: credentials).get_object(bucket, path, download_dest: zipfile)
422
+ rescue ::Google::Apis::ClientError => e
423
+ MU.log "Couldn't retrieve gs://#{bucket}/#{path} for #{function_id}", MU::WARN, details: e.inspect
424
+ return false
425
+ end
421
426
  elsif cloud_desc.source_upload_url
422
427
  resp = MU::Cloud::Google.function(credentials: credentials).generate_function_download_url(
423
428
  function_id
@@ -428,6 +433,7 @@ module example.com/cloudfunction
428
433
  f.close
429
434
  end
430
435
  end
436
+ true
431
437
  end
432
438
 
433
439
  # Upload a zipfile to our admin Cloud Storage bucket, for use by
@@ -518,7 +524,7 @@ module example.com/cloudfunction
518
524
  ok = false
519
525
  end
520
526
  else
521
- function = MU::Cloud::Google::User.genericServiceAccount(function, configurator)
527
+ function = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(function, configurator)
522
528
  end
523
529
 
524
530
  # siblings = configurator.haveLitterMate?(nil, "vpcs", has_multiple: true)
@@ -44,7 +44,7 @@ module MU
44
44
  resp = MU::Cloud::Google.admin_directory(credentials: @credentials).insert_group(group_obj)
45
45
  @cloud_id = resp.email
46
46
 
47
- MU::Cloud::Google::Role.bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'])
47
+ MU::Cloud.resourceClass("Google", "Role").bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'])
48
48
  else
49
49
  @cloud_id = @config['name'].sub(/@.*/, "")+"@"+@config['domain']
50
50
  end
@@ -52,7 +52,7 @@ module MU
52
52
 
53
53
  # Called automatically by {MU::Deploy#createResources}
54
54
  def groom
55
- MU::Cloud::Google::Role.bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'], debug: true)
55
+ MU::Cloud.resourceClass("Google", "Role").bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'], debug: true)
56
56
 
57
57
  if @config['members']
58
58
  resolved_desired = []
@@ -140,7 +140,7 @@ module MU
140
140
  # @param noop [Boolean]: If true, will only print what would be done
141
141
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
142
142
  # @return [void]
143
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
143
+ def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
144
144
  MU::Cloud::Google.getDomains(credentials)
145
145
  my_org = MU::Cloud::Google.getOrg(credentials)
146
146
 
@@ -166,7 +166,7 @@ module MU
166
166
 
167
167
  if flags['known']
168
168
  flags['known'].each { |group|
169
- MU::Cloud::Google::Role.removeBindings("group", group, credentials: credentials, noop: noop)
169
+ MU::Cloud.resourceClass("Google", "Role").removeBindings("group", group, credentials: credentials, noop: noop)
170
170
  }
171
171
  end
172
172
  end
@@ -222,10 +222,10 @@ module MU
222
222
  # type: "users"
223
223
  # )
224
224
  # }
225
- group_roles = MU::Cloud::Google::Role.getAllBindings(@config['credentials'])["by_entity"]
225
+ group_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
226
226
  if group_roles["group"] and group_roles["group"][bok['cloud_id']] and
227
227
  group_roles["group"][bok['cloud_id']].size > 0
228
- bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(group_roles["group"][bok['cloud_id']], credentials: @config['credentials'])
228
+ bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(group_roles["group"][bok['cloud_id']], credentials: @config['credentials'])
229
229
  end
230
230
 
231
231
  bok
@@ -264,7 +264,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
264
264
 
265
265
  "roles" => {
266
266
  "type" => "array",
267
- "items" => MU::Cloud::Google::Role.ref_schema
267
+ "items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
268
268
  }
269
269
  }
270
270
  [toplevel_required, schema]
@@ -340,11 +340,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
340
340
  if group['members']
341
341
  group['members'].each { |m|
342
342
  if configurator.haveLitterMate?(m, "users")
343
- group['dependencies'] ||= []
344
- group['dependencies'] << {
345
- "name" => m,
346
- "type" => "user"
347
- }
343
+ MU::Config.addDependency(group, m, "user")
348
344
  end
349
345
  }
350
346
  end
@@ -353,11 +349,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
353
349
  group['roles'].each { |r|
354
350
  if r['role'] and r['role']['name'] and
355
351
  (!r['role']['deploy_id'] and !r['role']['id'])
356
- group['dependencies'] ||= []
357
- group['dependencies'] << {
358
- "type" => "role",
359
- "name" => r['role']['name']
360
- }
352
+ MU::Config.addDependency(group, r['role']['name'], "role")
361
353
  end
362
354
  }
363
355
  end