cloud-mu 3.1.4 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +5 -1
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +16 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +5 -3
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/modules/mommacat.ru +1 -1
- data/modules/mu.rb +158 -107
- data/modules/mu/adoption.rb +386 -59
- data/modules/mu/cleanup.rb +214 -303
- data/modules/mu/cloud.rb +128 -1632
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +926 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +135 -82
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +7 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +5 -6
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +92 -14
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +38 -37
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +12 -13
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +11 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +27 -23
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +91 -68
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +33 -19
- data/modules/mu/groomer.rb +16 -1
- data/modules/mu/groomers/ansible.rb +123 -21
- data/modules/mu/groomers/chef.rb +64 -11
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +97 -4
- data/modules/mu/master/ssl.rb +0 -1
- data/modules/mu/mommacat.rb +154 -867
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +110 -3
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +225 -192
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +281 -64
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +708 -749
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +75 -57
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +212 -242
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +50 -41
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +94 -57
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +173 -42
- data/modules/mu/{clouds → providers}/aws/server.rb +782 -1107
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +36 -46
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +429 -849
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +13 -0
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +29 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +5 -5
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +59 -37
- data/modules/mu/{clouds → providers}/google/database.rb +5 -12
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +14 -8
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/google/role.rb +50 -31
- data/modules/mu/{clouds → providers}/google/server.rb +142 -55
- data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
- data/modules/mu/{clouds → providers}/google/user.rb +34 -24
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +46 -15
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +2 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/modules/tests/win2k12.yaml +25 -0
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +169 -93
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
- data/modules/tests/needwork/win2k12.yaml +0 -13
|
@@ -144,10 +144,10 @@ module MU
|
|
|
144
144
|
# @param noop [Boolean]: If true, will only print what would be done
|
|
145
145
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
146
146
|
# @return [void]
|
|
147
|
-
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
|
148
|
-
flags["
|
|
147
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
|
148
|
+
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
|
149
149
|
|
|
150
|
-
resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['
|
|
150
|
+
resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['habitat'])
|
|
151
151
|
if resp and resp.items
|
|
152
152
|
resp.items.each { |bucket|
|
|
153
153
|
if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or bucket.labels['mu-master-ip'] == MU.mu_public_ip.gsub(/\./, "_"))
|
|
@@ -243,7 +243,7 @@ module MU
|
|
|
243
243
|
grantees[binding.role] << { "id" => grantee }
|
|
244
244
|
elsif grantee.match(/^serviceAccount:(.*)/)
|
|
245
245
|
sa_name = Regexp.last_match[1]
|
|
246
|
-
if MU::Cloud
|
|
246
|
+
if MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(sa_name)
|
|
247
247
|
grantees[binding.role] << { "id" => grantee }
|
|
248
248
|
else
|
|
249
249
|
grantees[binding.role] << MU::Config::Ref.get(
|
|
@@ -305,7 +305,7 @@ module MU
|
|
|
305
305
|
schema = {
|
|
306
306
|
"storage_class" => {
|
|
307
307
|
"type" => "string",
|
|
308
|
-
"enum" => ["MULTI_REGIONAL", "REGIONAL", "STANDARD", "NEARLINE", "COLDLINE", "DURABLE_REDUCED_AVAILABILITY"],
|
|
308
|
+
"enum" => ["MULTI_REGIONAL", "REGIONAL", "STANDARD", "NEARLINE", "COLDLINE", "DURABLE_REDUCED_AVAILABILITY", "ARCHIVE"],
|
|
309
309
|
"default" => "STANDARD"
|
|
310
310
|
},
|
|
311
311
|
"bucket_wide_acls" => {
|
|
@@ -474,7 +474,6 @@ module MU
|
|
|
474
474
|
MU.log %Q{How to interact with your GKE cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
|
|
475
475
|
end
|
|
476
476
|
|
|
477
|
-
|
|
478
477
|
# Locate an existing ContainerCluster or ContainerClusters and return an array containing matching GCP resource descriptors for those that match.
|
|
479
478
|
# @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ContainerClusters
|
|
480
479
|
def self.find(**args)
|
|
@@ -567,22 +566,24 @@ module MU
|
|
|
567
566
|
bok['kubernetes']['network_policy_addon'] = true
|
|
568
567
|
end
|
|
569
568
|
|
|
570
|
-
if cloud_desc.ip_allocation_policy
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
569
|
+
if cloud_desc.ip_allocation_policy
|
|
570
|
+
if cloud_desc.ip_allocation_policy.use_ip_aliases
|
|
571
|
+
bok['ip_aliases'] = true
|
|
572
|
+
end
|
|
573
|
+
if cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
|
|
574
|
+
bok['pod_ip_block'] = cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
|
|
575
|
+
end
|
|
576
|
+
if cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
|
|
577
|
+
bok['services_ip_block'] = cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
|
|
578
|
+
end
|
|
579
579
|
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
580
|
+
if cloud_desc.ip_allocation_policy.create_subnetwork
|
|
581
|
+
bok['custom_subnet'] = {
|
|
582
|
+
"name" => (cloud_desc.ip_allocation_policy.subnetwork_name || cloud_desc.subnetwork)
|
|
583
|
+
}
|
|
584
|
+
if cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
|
|
585
|
+
bok['custom_subnet']['node_ip_block'] = cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
|
|
586
|
+
end
|
|
586
587
|
end
|
|
587
588
|
end
|
|
588
589
|
|
|
@@ -656,7 +657,7 @@ module MU
|
|
|
656
657
|
end
|
|
657
658
|
|
|
658
659
|
if bok['service_account']
|
|
659
|
-
found = MU::Cloud
|
|
660
|
+
found = MU::Cloud.resourceClass("Google", "User").find(
|
|
660
661
|
credentials: bok['credentials'],
|
|
661
662
|
project: bok['project'],
|
|
662
663
|
cloud_id: bok['service_account']
|
|
@@ -743,17 +744,17 @@ module MU
|
|
|
743
744
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
744
745
|
# @param region [String]: The cloud provider region in which to operate
|
|
745
746
|
# @return [void]
|
|
746
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
747
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
747
748
|
|
|
748
|
-
flags["
|
|
749
|
-
return if !MU::Cloud
|
|
749
|
+
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
|
750
|
+
return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
|
|
750
751
|
clusters = []
|
|
751
752
|
|
|
752
753
|
# Make sure we catch regional *and* zone clusters
|
|
753
|
-
found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['
|
|
754
|
+
found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{region}")
|
|
754
755
|
clusters.concat(found.clusters) if found and found.clusters
|
|
755
756
|
MU::Cloud::Google.listAZs(region).each { |az|
|
|
756
|
-
found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['
|
|
757
|
+
found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{az}")
|
|
757
758
|
clusters.concat(found.clusters) if found and found.clusters
|
|
758
759
|
}
|
|
759
760
|
|
|
@@ -816,10 +817,10 @@ module MU
|
|
|
816
817
|
"type" => "integer",
|
|
817
818
|
"description" => "The number of local SSD disks to be attached to workers. See https://cloud.google.com/compute/docs/disks/local-ssd#local_ssd_limits"
|
|
818
819
|
},
|
|
819
|
-
"ssh_user" => MU::Cloud
|
|
820
|
-
"metadata" => MU::Cloud
|
|
821
|
-
"service_account" => MU::Cloud
|
|
822
|
-
"scopes" => MU::Cloud
|
|
820
|
+
"ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
|
|
821
|
+
"metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
|
|
822
|
+
"service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
|
|
823
|
+
"scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
|
|
823
824
|
"private_cluster" => {
|
|
824
825
|
"description" => "Set a GKE cluster to be private, that is segregated into its own hidden VPC.",
|
|
825
826
|
"type" => "object",
|
|
@@ -1020,6 +1021,25 @@ module MU
|
|
|
1020
1021
|
cluster['ip_aliases'] = true
|
|
1021
1022
|
end
|
|
1022
1023
|
|
|
1024
|
+
# try to stake out some nice /21s for our networking config
|
|
1025
|
+
if cluster['ip_aliases'] and cluster["vpc"] and cluster["vpc"]["id"]
|
|
1026
|
+
habarg = if cluster["vpc"]["habitat"] and cluster["vpc"]["habitat"]["id"]
|
|
1027
|
+
cluster["vpc"]["habitat"]["id"]
|
|
1028
|
+
else
|
|
1029
|
+
cluster["project"]
|
|
1030
|
+
end
|
|
1031
|
+
found = MU::MommaCat.findStray("Google", "vpcs", cloud_id: cluster["vpc"]["id"], credentials: cluster["credentials"], habitats: [habarg], dummy_ok: true)
|
|
1032
|
+
if found and found.size == 1
|
|
1033
|
+
myvpc = found.first
|
|
1034
|
+
# XXX this might not make sense with custom_subnet
|
|
1035
|
+
cluster['pod_ip_block'] ||= myvpc.getUnusedAddressBlock(max_bits: 21)
|
|
1036
|
+
cluster['services_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block']], max_bits: 21)
|
|
1037
|
+
if cluster['tpu']
|
|
1038
|
+
cluster['tpu_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block'], cluster['services_ip_block']], max_bits: 21)
|
|
1039
|
+
end
|
|
1040
|
+
end
|
|
1041
|
+
end
|
|
1042
|
+
|
|
1023
1043
|
if cluster['service_account']
|
|
1024
1044
|
cluster['service_account']['cloud'] = "Google"
|
|
1025
1045
|
cluster['service_account']['habitat'] ||= MU::Config::Ref.get(
|
|
@@ -1029,12 +1049,9 @@ module MU
|
|
|
1029
1049
|
type: "habitats"
|
|
1030
1050
|
)
|
|
1031
1051
|
if cluster['service_account']['name'] and
|
|
1032
|
-
!cluster['service_account']['id']
|
|
1033
|
-
|
|
1034
|
-
cluster['
|
|
1035
|
-
"type" => "user",
|
|
1036
|
-
"name" => cluster['service_account']['name']
|
|
1037
|
-
}
|
|
1052
|
+
!cluster['service_account']['id'] and
|
|
1053
|
+
!cluster['service_account']['deploy_id']
|
|
1054
|
+
MU::Config.addDependency(cluster, cluster['service_account']['name'], "user")
|
|
1038
1055
|
end
|
|
1039
1056
|
found = MU::Config::Ref.get(cluster['service_account'])
|
|
1040
1057
|
# XXX verify that found.kitten fails when it's supposed to
|
|
@@ -1043,7 +1060,7 @@ module MU
|
|
|
1043
1060
|
ok = false
|
|
1044
1061
|
end
|
|
1045
1062
|
else
|
|
1046
|
-
cluster = MU::Cloud
|
|
1063
|
+
cluster = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(cluster, configurator)
|
|
1047
1064
|
end
|
|
1048
1065
|
|
|
1049
1066
|
if cluster['dependencies']
|
|
@@ -1094,7 +1111,7 @@ module MU
|
|
|
1094
1111
|
}
|
|
1095
1112
|
if !match
|
|
1096
1113
|
MU.log "No version matching #{cluster['kubernetes']['version']} available, will try floating minor revision", MU::WARN
|
|
1097
|
-
cluster['kubernetes']['version'].sub!(/^(\d+\.\d
|
|
1114
|
+
cluster['kubernetes']['version'].sub!(/^(\d+\.\d+)\..*/i, '\1')
|
|
1098
1115
|
master_versions.each { |v|
|
|
1099
1116
|
if v.match(/^#{Regexp.quote(cluster['kubernetes']['version'])}/)
|
|
1100
1117
|
match = true
|
|
@@ -1139,9 +1156,13 @@ module MU
|
|
|
1139
1156
|
end
|
|
1140
1157
|
end
|
|
1141
1158
|
|
|
1142
|
-
cluster['instance_type'] = MU::Cloud
|
|
1159
|
+
cluster['instance_type'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
|
|
1143
1160
|
ok = false if cluster['instance_type'].nil?
|
|
1144
1161
|
|
|
1162
|
+
if !MU::Master.kubectl
|
|
1163
|
+
MU.log "Since I can't find a kubectl executable, you will have to handle all service account, user, and role bindings manually!", MU::WARN
|
|
1164
|
+
end
|
|
1165
|
+
|
|
1145
1166
|
ok
|
|
1146
1167
|
end
|
|
1147
1168
|
|
|
@@ -1188,7 +1209,8 @@ module MU
|
|
|
1188
1209
|
labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
|
|
1189
1210
|
|
|
1190
1211
|
labelset = MU::Cloud::Google.container(:SetLabelsRequest).new(
|
|
1191
|
-
resource_labels: labels
|
|
1212
|
+
resource_labels: labels,
|
|
1213
|
+
label_fingerprint: cloud_desc.label_fingerprint
|
|
1192
1214
|
)
|
|
1193
1215
|
MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_resource_labels(@cloud_id, labelset)
|
|
1194
1216
|
end
|
|
@@ -1232,7 +1254,7 @@ module MU
|
|
|
1232
1254
|
# Take this opportunity to ensure that the 'client' service account
|
|
1233
1255
|
# used by certificate authentication exists and has appropriate
|
|
1234
1256
|
# privilege
|
|
1235
|
-
if @username and @password
|
|
1257
|
+
if @username and @password and MU::Master.kubectl
|
|
1236
1258
|
File.open(client_binding, "w"){ |k|
|
|
1237
1259
|
k.puts <<-EOF
|
|
1238
1260
|
kind: ClusterRoleBinding
|
|
@@ -25,14 +25,7 @@ module MU
|
|
|
25
25
|
@config["groomer"] = MU::Config.defaultGroomer unless @config["groomer"]
|
|
26
26
|
@groomclass = MU::Groomer.loadGroomer(@config["groomer"])
|
|
27
27
|
|
|
28
|
-
@mu_name ||=
|
|
29
|
-
if @config and @config['engine'] and @config["engine"].match(/^sqlserver/)
|
|
30
|
-
@deploy.getResourceName(@config["name"], max_length: 15)
|
|
31
|
-
else
|
|
32
|
-
@deploy.getResourceName(@config["name"], max_length: 63)
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
@mu_name.gsub(/(--|-$)/i, "").gsub(/(_)/, "-").gsub!(/^[^a-z]/i, "")
|
|
28
|
+
@mu_name ||= @deploy.getResourceName(@config["name"], max_length: 63)
|
|
36
29
|
end
|
|
37
30
|
|
|
38
31
|
# Called automatically by {MU::Deploy#createResources}
|
|
@@ -107,14 +100,14 @@ module MU
|
|
|
107
100
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
108
101
|
# @param region [String]: The cloud provider region in which to operate
|
|
109
102
|
# @return [void]
|
|
110
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
111
|
-
flags["
|
|
103
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
104
|
+
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
|
112
105
|
|
|
113
|
-
# instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['
|
|
106
|
+
# instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['habitat'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"})
|
|
114
107
|
# if instances and instances.items
|
|
115
108
|
# instances.items.each { |instance|
|
|
116
109
|
# MU.log "Deleting Cloud SQL instance #{instance.name}"
|
|
117
|
-
# MU::Cloud::Google.sql(credentials: credentials).delete_instance(flags['
|
|
110
|
+
# MU::Cloud::Google.sql(credentials: credentials).delete_instance(flags['habitat'], instance.name) if !noop
|
|
118
111
|
# }
|
|
119
112
|
# end
|
|
120
113
|
end
|
|
@@ -207,9 +207,9 @@ end
|
|
|
207
207
|
# @param noop [Boolean]: If true, will only print what would be done
|
|
208
208
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
209
209
|
# @return [void]
|
|
210
|
-
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
|
211
|
-
flags["
|
|
212
|
-
return if !MU::Cloud
|
|
210
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
|
211
|
+
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
|
212
|
+
return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
|
|
213
213
|
filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
|
|
214
214
|
if !ignoremaster and MU.mu_public_ip
|
|
215
215
|
filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
|
|
@@ -218,7 +218,7 @@ end
|
|
|
218
218
|
|
|
219
219
|
MU::Cloud::Google.compute(credentials: credentials).delete(
|
|
220
220
|
"firewall",
|
|
221
|
-
flags["
|
|
221
|
+
flags["habitat"],
|
|
222
222
|
nil,
|
|
223
223
|
noop
|
|
224
224
|
)
|
|
@@ -440,7 +440,7 @@ end
|
|
|
440
440
|
elsif acl['vpc']['habitat'] and acl['vpc']['habitat']['name']
|
|
441
441
|
acl['vpc']['project'] = acl['vpc']['habitat']['name']
|
|
442
442
|
end
|
|
443
|
-
correct_vpc = MU::Cloud
|
|
443
|
+
correct_vpc = MU::Cloud.resourceClass("Google", "VPC").pickVPC(
|
|
444
444
|
acl['vpc'],
|
|
445
445
|
acl,
|
|
446
446
|
"firewall_rule",
|
|
@@ -138,7 +138,7 @@ module MU
|
|
|
138
138
|
# Return the metadata for this folders's configuration
|
|
139
139
|
# @return [Hash]
|
|
140
140
|
def notify
|
|
141
|
-
desc = MU.structToHash(
|
|
141
|
+
desc = MU.structToHash(cloud_desc)
|
|
142
142
|
desc["mu_name"] = @mu_name
|
|
143
143
|
desc["parent"] = @parent
|
|
144
144
|
desc["cloud_id"] = @cloud_id
|
|
@@ -162,7 +162,7 @@ module MU
|
|
|
162
162
|
# @param noop [Boolean]: If true, will only print what would be done
|
|
163
163
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
164
164
|
# @return [void]
|
|
165
|
-
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
|
165
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
|
166
166
|
filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
|
|
167
167
|
if !ignoremaster and MU.mu_public_ip
|
|
168
168
|
filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
|
|
@@ -236,10 +236,10 @@ module MU
|
|
|
236
236
|
# @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
|
|
237
237
|
def self.find(**args)
|
|
238
238
|
found = {}
|
|
239
|
-
|
|
240
239
|
# Recursively search a GCP folder hierarchy for a folder matching our
|
|
241
240
|
# supplied name or identifier.
|
|
242
241
|
def self.find_matching_folder(parent, name: nil, id: nil, credentials: nil)
|
|
242
|
+
|
|
243
243
|
resp = MU::Cloud::Google.folder(credentials: credentials).list_folders(parent: parent)
|
|
244
244
|
if resp and resp.folders
|
|
245
245
|
resp.folders.each { |f|
|
|
@@ -278,6 +278,7 @@ module MU
|
|
|
278
278
|
end
|
|
279
279
|
else
|
|
280
280
|
resp = MU::Cloud::Google.folder(credentials: args[:credentials]).list_folders(parent: parent)
|
|
281
|
+
|
|
281
282
|
if resp and resp.folders
|
|
282
283
|
resp.folders.each { |folder|
|
|
283
284
|
next if folder.lifecycle_state == "DELETE_REQUESTED"
|
|
@@ -310,7 +311,6 @@ module MU
|
|
|
310
311
|
bok['cloud_id'] = cloud_desc.name
|
|
311
312
|
bok['name'] = cloud_desc.display_name#+bok['cloud_id'] # only way to guarantee uniqueness
|
|
312
313
|
if cloud_desc.parent.match(/^folders\/(.*)/)
|
|
313
|
-
MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_desc.parent
|
|
314
314
|
bok['parent'] = MU::Config::Ref.get(
|
|
315
315
|
id: cloud_desc.parent,
|
|
316
316
|
cloud: "Google",
|
|
@@ -355,11 +355,7 @@ MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_d
|
|
|
355
355
|
end
|
|
356
356
|
|
|
357
357
|
if folder['parent'] and folder['parent']['name'] and !folder['parent']['deploy_id'] and configurator.haveLitterMate?(folder['parent']['name'], "folders")
|
|
358
|
-
folder[
|
|
359
|
-
folder["dependencies"] << {
|
|
360
|
-
"type" => "folder",
|
|
361
|
-
"name" => folder['parent']['name']
|
|
362
|
-
}
|
|
358
|
+
MU::Config.addDependency(folder, folder['parent']['name'], "folder")
|
|
363
359
|
end
|
|
364
360
|
|
|
365
361
|
ok
|
|
@@ -233,11 +233,11 @@ module example.com/cloudfunction
|
|
|
233
233
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
234
234
|
# @param region [String]: The cloud provider region
|
|
235
235
|
# @return [void]
|
|
236
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
237
|
-
flags["
|
|
238
|
-
return if !MU::Cloud
|
|
236
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
237
|
+
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
|
238
|
+
return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
|
|
239
239
|
# Make sure we catch regional *and* zone functions
|
|
240
|
-
found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["
|
|
240
|
+
found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["habitat"])
|
|
241
241
|
found.each_pair { |cloud_id, desc|
|
|
242
242
|
if (desc.description and desc.description == MU.deploy_id) or
|
|
243
243
|
(desc.labels and desc.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or desc.labels["mu-master-ip"] == MU.mu_public_ip.gsub(/\./, "_"))) or
|
|
@@ -340,7 +340,7 @@ module example.com/cloudfunction
|
|
|
340
340
|
end
|
|
341
341
|
|
|
342
342
|
codefile = bok["project"]+"_"+bok["region"]+"_"+bok["name"]+".zip"
|
|
343
|
-
MU::Cloud::Google::Function.downloadPackage(@cloud_id, codefile, credentials: @config['credentials'])
|
|
343
|
+
return nil if !MU::Cloud::Google::Function.downloadPackage(@cloud_id, codefile, credentials: @config['credentials'])
|
|
344
344
|
bok['code'] = {
|
|
345
345
|
'zip_file' => codefile
|
|
346
346
|
}
|
|
@@ -373,7 +373,7 @@ module example.com/cloudfunction
|
|
|
373
373
|
}
|
|
374
374
|
}
|
|
375
375
|
},
|
|
376
|
-
"service_account" => MU::Cloud
|
|
376
|
+
"service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
|
|
377
377
|
"runtime" => {
|
|
378
378
|
"type" => "string",
|
|
379
379
|
"enum" => %w{nodejs go python nodejs8 nodejs10 python37 go111 go113},
|
|
@@ -417,7 +417,12 @@ module example.com/cloudfunction
|
|
|
417
417
|
bucket = Regexp.last_match[1]
|
|
418
418
|
path = Regexp.last_match[2]
|
|
419
419
|
|
|
420
|
-
|
|
420
|
+
begin
|
|
421
|
+
MU::Cloud::Google.storage(credentials: credentials).get_object(bucket, path, download_dest: zipfile)
|
|
422
|
+
rescue ::Google::Apis::ClientError => e
|
|
423
|
+
MU.log "Couldn't retrieve gs://#{bucket}/#{path} for #{function_id}", MU::WARN, details: e.inspect
|
|
424
|
+
return false
|
|
425
|
+
end
|
|
421
426
|
elsif cloud_desc.source_upload_url
|
|
422
427
|
resp = MU::Cloud::Google.function(credentials: credentials).generate_function_download_url(
|
|
423
428
|
function_id
|
|
@@ -428,6 +433,7 @@ module example.com/cloudfunction
|
|
|
428
433
|
f.close
|
|
429
434
|
end
|
|
430
435
|
end
|
|
436
|
+
true
|
|
431
437
|
end
|
|
432
438
|
|
|
433
439
|
# Upload a zipfile to our admin Cloud Storage bucket, for use by
|
|
@@ -518,7 +524,7 @@ module example.com/cloudfunction
|
|
|
518
524
|
ok = false
|
|
519
525
|
end
|
|
520
526
|
else
|
|
521
|
-
function = MU::Cloud
|
|
527
|
+
function = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(function, configurator)
|
|
522
528
|
end
|
|
523
529
|
|
|
524
530
|
# siblings = configurator.haveLitterMate?(nil, "vpcs", has_multiple: true)
|
|
@@ -44,7 +44,7 @@ module MU
|
|
|
44
44
|
resp = MU::Cloud::Google.admin_directory(credentials: @credentials).insert_group(group_obj)
|
|
45
45
|
@cloud_id = resp.email
|
|
46
46
|
|
|
47
|
-
MU::Cloud
|
|
47
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'])
|
|
48
48
|
else
|
|
49
49
|
@cloud_id = @config['name'].sub(/@.*/, "")+"@"+@config['domain']
|
|
50
50
|
end
|
|
@@ -52,7 +52,7 @@ module MU
|
|
|
52
52
|
|
|
53
53
|
# Called automatically by {MU::Deploy#createResources}
|
|
54
54
|
def groom
|
|
55
|
-
MU::Cloud
|
|
55
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'], debug: true)
|
|
56
56
|
|
|
57
57
|
if @config['members']
|
|
58
58
|
resolved_desired = []
|
|
@@ -140,7 +140,7 @@ module MU
|
|
|
140
140
|
# @param noop [Boolean]: If true, will only print what would be done
|
|
141
141
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
142
142
|
# @return [void]
|
|
143
|
-
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
|
143
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
|
144
144
|
MU::Cloud::Google.getDomains(credentials)
|
|
145
145
|
my_org = MU::Cloud::Google.getOrg(credentials)
|
|
146
146
|
|
|
@@ -166,7 +166,7 @@ module MU
|
|
|
166
166
|
|
|
167
167
|
if flags['known']
|
|
168
168
|
flags['known'].each { |group|
|
|
169
|
-
MU::Cloud
|
|
169
|
+
MU::Cloud.resourceClass("Google", "Role").removeBindings("group", group, credentials: credentials, noop: noop)
|
|
170
170
|
}
|
|
171
171
|
end
|
|
172
172
|
end
|
|
@@ -222,10 +222,10 @@ module MU
|
|
|
222
222
|
# type: "users"
|
|
223
223
|
# )
|
|
224
224
|
# }
|
|
225
|
-
group_roles = MU::Cloud
|
|
225
|
+
group_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
|
|
226
226
|
if group_roles["group"] and group_roles["group"][bok['cloud_id']] and
|
|
227
227
|
group_roles["group"][bok['cloud_id']].size > 0
|
|
228
|
-
bok['roles'] = MU::Cloud
|
|
228
|
+
bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(group_roles["group"][bok['cloud_id']], credentials: @config['credentials'])
|
|
229
229
|
end
|
|
230
230
|
|
|
231
231
|
bok
|
|
@@ -264,7 +264,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
|
|
|
264
264
|
|
|
265
265
|
"roles" => {
|
|
266
266
|
"type" => "array",
|
|
267
|
-
"items" => MU::Cloud
|
|
267
|
+
"items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
|
|
268
268
|
}
|
|
269
269
|
}
|
|
270
270
|
[toplevel_required, schema]
|
|
@@ -340,11 +340,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
|
|
|
340
340
|
if group['members']
|
|
341
341
|
group['members'].each { |m|
|
|
342
342
|
if configurator.haveLitterMate?(m, "users")
|
|
343
|
-
group
|
|
344
|
-
group['dependencies'] << {
|
|
345
|
-
"name" => m,
|
|
346
|
-
"type" => "user"
|
|
347
|
-
}
|
|
343
|
+
MU::Config.addDependency(group, m, "user")
|
|
348
344
|
end
|
|
349
345
|
}
|
|
350
346
|
end
|
|
@@ -353,11 +349,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
|
|
|
353
349
|
group['roles'].each { |r|
|
|
354
350
|
if r['role'] and r['role']['name'] and
|
|
355
351
|
(!r['role']['deploy_id'] and !r['role']['id'])
|
|
356
|
-
group['
|
|
357
|
-
group['dependencies'] << {
|
|
358
|
-
"type" => "role",
|
|
359
|
-
"name" => r['role']['name']
|
|
360
|
-
}
|
|
352
|
+
MU::Config.addDependency(group, r['role']['name'], "role")
|
|
361
353
|
end
|
|
362
354
|
}
|
|
363
355
|
end
|