cloud-mu 3.1.6 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/mu-adopt +4 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +37 -1
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-run-tests +23 -10
- data/cloud-mu.gemspec +2 -2
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/extras/generate-stock-images +1 -0
- data/modules/mu.rb +82 -95
- data/modules/mu/adoption.rb +356 -56
- data/modules/mu/cleanup.rb +21 -20
- data/modules/mu/cloud.rb +79 -1753
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +46 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +920 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +165 -0
- data/modules/mu/config.rb +122 -80
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +1 -1
- data/modules/mu/config/cache_cluster.rb +1 -1
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +2 -2
- data/modules/mu/config/database.rb +83 -104
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +1 -1
- data/modules/mu/config/doc_helpers.rb +4 -5
- data/modules/mu/config/endpoint.rb +1 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +1 -1
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +1 -1
- data/modules/mu/config/ref.rb +30 -4
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +30 -34
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +4 -12
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +10 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +12 -17
- data/modules/mu/defaults/AWS.yaml +32 -32
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +16 -15
- data/modules/mu/groomer.rb +15 -0
- data/modules/mu/groomers/chef.rb +3 -0
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +1 -1
- data/modules/mu/mommacat.rb +54 -25
- data/modules/mu/mommacat/daemon.rb +10 -7
- data/modules/mu/mommacat/naming.rb +82 -3
- data/modules/mu/mommacat/search.rb +47 -15
- data/modules/mu/mommacat/storage.rb +72 -41
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +114 -47
- data/modules/mu/{clouds → providers}/aws/alarm.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/bucket.rb +2 -2
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +10 -46
- data/modules/mu/{clouds → providers}/aws/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +15 -33
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +2 -5
- data/modules/mu/{clouds → providers}/aws/endpoint.rb +2 -11
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +33 -29
- data/modules/mu/{clouds → providers}/aws/folder.rb +0 -0
- data/modules/mu/{clouds → providers}/aws/function.rb +2 -10
- data/modules/mu/{clouds → providers}/aws/group.rb +9 -13
- data/modules/mu/{clouds → providers}/aws/habitat.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +41 -33
- data/modules/mu/{clouds → providers}/aws/log.rb +2 -2
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +2 -8
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +0 -0
- data/modules/mu/{clouds → providers}/aws/notifier.rb +0 -0
- data/modules/mu/{clouds → providers}/aws/role.rb +7 -7
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +8 -13
- data/modules/mu/{clouds → providers}/aws/server.rb +55 -90
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +10 -33
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +19 -36
- data/modules/mu/{clouds → providers}/aws/user.rb +8 -12
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/vpc.rb +135 -70
- data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
- data/modules/mu/{clouds → providers}/azure.rb +4 -1
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +30 -23
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +14 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +1 -1
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +28 -13
- data/modules/mu/{clouds → providers}/google/database.rb +1 -8
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +2 -2
- data/modules/mu/{clouds → providers}/google/folder.rb +4 -8
- data/modules/mu/{clouds → providers}/google/function.rb +3 -3
- data/modules/mu/{clouds → providers}/google/group.rb +8 -16
- data/modules/mu/{clouds → providers}/google/habitat.rb +3 -7
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +1 -1
- data/modules/mu/{clouds → providers}/google/role.rb +42 -34
- data/modules/mu/{clouds → providers}/google/server.rb +25 -10
- data/modules/mu/{clouds → providers}/google/server_pool.rb +10 -10
- data/modules/mu/{clouds → providers}/google/user.rb +31 -21
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +37 -2
- data/modules/tests/centos6.yaml +11 -0
- data/modules/tests/centos7.yaml +11 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +108 -89
- data/modules/mu/clouds/aws/database.rb +0 -1974
|
@@ -271,13 +271,13 @@ module MU
|
|
|
271
271
|
my_org = MU::Cloud::Google.getOrg(credentials)
|
|
272
272
|
if my_org
|
|
273
273
|
scopes["organizations"] = [my_org.name]
|
|
274
|
-
folders = MU::Cloud
|
|
274
|
+
folders = MU::Cloud.resourceClass("Google", "Folder").find(credentials: credentials)
|
|
275
275
|
if folders and folders.size > 0
|
|
276
276
|
scopes["folders"] = folders.keys
|
|
277
277
|
end
|
|
278
278
|
end
|
|
279
279
|
|
|
280
|
-
projects = MU::Cloud
|
|
280
|
+
projects = MU::Cloud.resourceClass("Google", "Habitat").find(credentials: credentials)
|
|
281
281
|
if projects and projects.size > 0
|
|
282
282
|
scopes["projects"] = projects.keys
|
|
283
283
|
end
|
|
@@ -407,12 +407,12 @@ module MU
|
|
|
407
407
|
# email field (which is the "real" id most of the time)
|
|
408
408
|
real_id = nil
|
|
409
409
|
if entity_type == "group"
|
|
410
|
-
found = MU::Cloud
|
|
410
|
+
found = MU::Cloud.resourceClass("Google", "Group").find(cloud_id: entity_id, credentials: credentials)
|
|
411
411
|
if found[entity_id]
|
|
412
412
|
real_id = found[entity_id].id
|
|
413
413
|
end
|
|
414
414
|
elsif entity_type == "user"
|
|
415
|
-
found = MU::Cloud
|
|
415
|
+
found = MU::Cloud.resourceClass("Google", "User").find(cloud_id: entity_id, credentials: credentials)
|
|
416
416
|
if found[entity_id]
|
|
417
417
|
real_id = found[entity_id].id
|
|
418
418
|
end
|
|
@@ -563,7 +563,7 @@ module MU
|
|
|
563
563
|
if args[:project]
|
|
564
564
|
canned = Hash[MU::Cloud::Google.iam(credentials: args[:credentials]).list_roles.roles.map { |r| [r.name, r] }]
|
|
565
565
|
begin
|
|
566
|
-
MU::Cloud
|
|
566
|
+
MU::Cloud.resourceClass("Google", "Habitat").bindings(args[:project], credentials: args[:credentials]).each { |binding|
|
|
567
567
|
found[binding.role] = canned[binding.role]
|
|
568
568
|
}
|
|
569
569
|
rescue ::Google::Apis::ClientError => e
|
|
@@ -591,10 +591,15 @@ module MU
|
|
|
591
591
|
bindings['by_scope']['projects'][args[:project]]
|
|
592
592
|
bindings['by_scope']['projects'][args[:project]].keys.each { |r|
|
|
593
593
|
if r.match(/^roles\//)
|
|
594
|
-
|
|
595
|
-
|
|
594
|
+
begin
|
|
595
|
+
role = MU::Cloud::Google.iam(credentials: args[:credentials]).get_role(r)
|
|
596
|
+
found[role.name] = role
|
|
597
|
+
rescue ::Google::Apis::ClientError => e
|
|
598
|
+
raise e if !e.message.match(/(?:forbidden|notFound): /)
|
|
599
|
+
MU.log "Failed MU::Cloud::Google.iam(credentials: #{args[:credentials]}).get_role(#{r})", MU::WARN, details: e.message
|
|
600
|
+
end
|
|
596
601
|
elsif !found[r]
|
|
597
|
-
MU.log "NEED TO GET #{r}", MU::WARN
|
|
602
|
+
# MU.log "NEED TO GET #{r}", MU::WARN
|
|
598
603
|
end
|
|
599
604
|
}
|
|
600
605
|
end
|
|
@@ -688,7 +693,7 @@ module MU
|
|
|
688
693
|
ids, _names, _privs = MU::Cloud::Google::Role.privilege_service_to_name(@config['credentials'])
|
|
689
694
|
cloud_desc.role_privileges.each { |priv|
|
|
690
695
|
if !ids[priv.service_id]
|
|
691
|
-
MU.log "Role privilege defined for a service id with no name I can find, writing with raw id", MU::
|
|
696
|
+
MU.log "Role privilege defined for a service id with no name I can find, writing with raw id", MU::DEBUG, details: priv
|
|
692
697
|
bok["import"] << priv.service_id+"/"+priv.privilege_name
|
|
693
698
|
else
|
|
694
699
|
bok["import"] << ids[priv.service_id]+"/"+priv.privilege_name
|
|
@@ -742,23 +747,33 @@ module MU
|
|
|
742
747
|
entity_types.each_pair { |entity_type, entities|
|
|
743
748
|
mu_entitytype = (entity_type == "serviceAccount" ? "user" : entity_type)+"s"
|
|
744
749
|
entities.each { |entity|
|
|
750
|
+
next if entity.nil?
|
|
745
751
|
foreign = if entity_type == "serviceAccount" and entity.match(/@(.*?)\.iam\.gserviceaccount\.com/)
|
|
746
752
|
!MU::Cloud::Google.listHabitats(@credentials).include?(Regexp.last_match[1])
|
|
747
753
|
end
|
|
754
|
+
|
|
748
755
|
entity_ref = if entity_type == "organizations"
|
|
749
756
|
{ "id" => ((org == my_org.name and @config['credentials']) ? @config['credentials'] : org) }
|
|
750
757
|
elsif entity_type == "domain"
|
|
751
758
|
{ "id" => entity }
|
|
752
759
|
else
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
type: mu_entitytype
|
|
760
|
-
)
|
|
760
|
+
shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(mu_entitytype)
|
|
761
|
+
if args[:types].include?(shortclass) and
|
|
762
|
+
!(entity_type == "serviceAccount" and
|
|
763
|
+
MU::Cloud::Google::User.cannedServiceAcctName?(entity))
|
|
764
|
+
MU.log "Role #{@cloud_id}: Skipping #{shortclass} binding for #{entity}; we are adopting that type and will set bindings from that resource", MU::DEBUG
|
|
765
|
+
next
|
|
761
766
|
end
|
|
767
|
+
|
|
768
|
+
MU::Config::Ref.get(
|
|
769
|
+
id: entity,
|
|
770
|
+
cloud: "Google",
|
|
771
|
+
type: mu_entitytype
|
|
772
|
+
)
|
|
773
|
+
end
|
|
774
|
+
if entity_ref.nil?
|
|
775
|
+
MU.log "I somehow ended up with a nil entity reference for #{entity_type} #{entity}", MU::ERR, details: [ bok, bindings ]
|
|
776
|
+
next
|
|
762
777
|
end
|
|
763
778
|
refmap ||= {}
|
|
764
779
|
refmap[entity_ref] ||= {}
|
|
@@ -778,6 +793,7 @@ module MU
|
|
|
778
793
|
}
|
|
779
794
|
}
|
|
780
795
|
}
|
|
796
|
+
|
|
781
797
|
bok["bindings"] ||= []
|
|
782
798
|
refmap.each_pair { |entity, scopes|
|
|
783
799
|
newbinding = { "entity" => entity }
|
|
@@ -900,7 +916,7 @@ module MU
|
|
|
900
916
|
end
|
|
901
917
|
|
|
902
918
|
role = MU::Cloud::Google.admin_directory(credentials: credentials).get_role(MU::Cloud::Google.customerID(credentials), binding.role_id)
|
|
903
|
-
MU.log "Failed to find entity #{binding.assigned_to} referenced in GSuite/Cloud Identity binding to role #{role.role_name}", MU::
|
|
919
|
+
MU.log "Failed to find entity #{binding.assigned_to} referenced in GSuite/Cloud Identity binding to role #{role.role_name}", MU::DEBUG, details: role
|
|
904
920
|
}
|
|
905
921
|
|
|
906
922
|
resp = MU::Cloud::Google.resource_manager(credentials: credentials).get_organization_iam_policy(my_org.name)
|
|
@@ -908,15 +924,15 @@ module MU
|
|
|
908
924
|
insertBinding("organizations", my_org.name, binding)
|
|
909
925
|
}
|
|
910
926
|
|
|
911
|
-
MU::Cloud
|
|
912
|
-
MU::Cloud
|
|
927
|
+
MU::Cloud.resourceClass("Google", "Folder").find(credentials: credentials).keys.each { |folder|
|
|
928
|
+
MU::Cloud.resourceClass("Google", "Folder").bindings(folder, credentials: credentials).each { |binding|
|
|
913
929
|
insertBinding("folders", folder, binding)
|
|
914
930
|
}
|
|
915
931
|
}
|
|
916
932
|
end
|
|
917
|
-
MU::Cloud::Google
|
|
933
|
+
MU::Cloud::Google.listHabitats(credentials).each { |project|
|
|
918
934
|
begin
|
|
919
|
-
MU::Cloud
|
|
935
|
+
MU::Cloud.resourceClass("Google", "Habitat").bindings(project, credentials: credentials).each { |binding|
|
|
920
936
|
insertBinding("projects", project, binding)
|
|
921
937
|
}
|
|
922
938
|
rescue ::Google::Apis::ClientError => e
|
|
@@ -1099,7 +1115,7 @@ If this value is not specified, and the role name matches the name of an existin
|
|
|
1099
1115
|
MU.log "None of the directory service privileges available to credentials #{role['credentials']} map to the ones declared for role #{role['name']}", MU::ERR, details: role['import'].sort
|
|
1100
1116
|
ok = false
|
|
1101
1117
|
elsif missing.size > 0
|
|
1102
|
-
MU.log "Some directory service privileges declared for role #{role['name']} aren't available to credentials #{role['credentials']}, will skip", MU::
|
|
1118
|
+
MU.log "Some directory service privileges declared for role #{role['name']} aren't available to credentials #{role['credentials']}, will skip", MU::DEBUG, details: missing
|
|
1103
1119
|
end
|
|
1104
1120
|
end
|
|
1105
1121
|
end
|
|
@@ -1114,11 +1130,7 @@ If this value is not specified, and the role name matches the name of an existin
|
|
|
1114
1130
|
if role['role_source'] == "project"
|
|
1115
1131
|
role['project'] ||= MU::Cloud::Google.defaultProject(role['credentials'])
|
|
1116
1132
|
if configurator.haveLitterMate?(role['project'], "habitats")
|
|
1117
|
-
role['
|
|
1118
|
-
role['dependencies'] << {
|
|
1119
|
-
"type" => "habitat",
|
|
1120
|
-
"name" => role['project']
|
|
1121
|
-
}
|
|
1133
|
+
MU::Config.addDependency(role, role['project'], "habitat")
|
|
1122
1134
|
end
|
|
1123
1135
|
end
|
|
1124
1136
|
|
|
@@ -1126,14 +1138,10 @@ If this value is not specified, and the role name matches the name of an existin
|
|
|
1126
1138
|
role['bindings'].each { |binding|
|
|
1127
1139
|
if binding['entity'] and binding['entity']['name'] and
|
|
1128
1140
|
configurator.haveLitterMate?(binding['entity']['name'], binding['entity']['type'])
|
|
1129
|
-
role['
|
|
1130
|
-
role['dependencies'] << {
|
|
1131
|
-
"type" => binding['entity']['type'].sub(/s$/, ''),
|
|
1132
|
-
"name" => binding['entity']['name']
|
|
1133
|
-
}
|
|
1134
|
-
|
|
1141
|
+
MU::Config.addDependency(role, binding['entity']['name'], binding['entity']['type'])
|
|
1135
1142
|
end
|
|
1136
1143
|
}
|
|
1144
|
+
role['bindings'].uniq!
|
|
1137
1145
|
end
|
|
1138
1146
|
|
|
1139
1147
|
ok
|
|
@@ -492,7 +492,7 @@ next if !create
|
|
|
492
492
|
return nil if @config.nil? or @deploy.nil?
|
|
493
493
|
|
|
494
494
|
nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
|
|
495
|
-
if !@config["vpc"].nil? and !MU::Cloud
|
|
495
|
+
if !@config["vpc"].nil? and !MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
|
|
496
496
|
|
|
497
497
|
if !@nat.nil?
|
|
498
498
|
if @nat.cloud_desc.nil?
|
|
@@ -623,7 +623,7 @@ next if !create
|
|
|
623
623
|
end
|
|
624
624
|
|
|
625
625
|
_nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
|
|
626
|
-
if !nat_ssh_host and !MU::Cloud
|
|
626
|
+
if !nat_ssh_host and !MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
|
|
627
627
|
# XXX check if canonical_ip is in the private ranges
|
|
628
628
|
# raise MuError, "#{node} has no NAT host configured, and I have no other route to it"
|
|
629
629
|
end
|
|
@@ -992,7 +992,7 @@ next if !create
|
|
|
992
992
|
# Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
|
|
993
993
|
# which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
|
|
994
994
|
# The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
|
|
995
|
-
if MU::Cloud
|
|
995
|
+
if MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
|
|
996
996
|
@config['canonical_ip'] = private_ips.first
|
|
997
997
|
return private_ips.first
|
|
998
998
|
else
|
|
@@ -1001,6 +1001,22 @@ next if !create
|
|
|
1001
1001
|
end
|
|
1002
1002
|
end
|
|
1003
1003
|
|
|
1004
|
+
# Return all of the IP addresses, public and private, from all of our
|
|
1005
|
+
# network interfaces.
|
|
1006
|
+
# @return [Array<String>]
|
|
1007
|
+
def listIPs
|
|
1008
|
+
ips = []
|
|
1009
|
+
cloud_desc.network_interfaces.each { |iface|
|
|
1010
|
+
ips << iface.network_ip
|
|
1011
|
+
if iface.access_configs
|
|
1012
|
+
iface.access_configs.each { |acfg|
|
|
1013
|
+
ips << acfg.nat_ip if acfg.nat_ip
|
|
1014
|
+
}
|
|
1015
|
+
end
|
|
1016
|
+
}
|
|
1017
|
+
ips
|
|
1018
|
+
end
|
|
1019
|
+
|
|
1004
1020
|
# return [String]: A password string.
|
|
1005
1021
|
def getWindowsAdminPassword(use_cache: true)
|
|
1006
1022
|
@config['windows_auth_vault'] ||= {
|
|
@@ -1276,7 +1292,7 @@ next if !create
|
|
|
1276
1292
|
# @return [void]
|
|
1277
1293
|
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
1278
1294
|
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
|
1279
|
-
return if !MU::Cloud
|
|
1295
|
+
return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
|
|
1280
1296
|
|
|
1281
1297
|
# XXX make damn sure MU.deploy_id is set
|
|
1282
1298
|
filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
|
|
@@ -1340,7 +1356,7 @@ next if !create
|
|
|
1340
1356
|
def self.schema(config)
|
|
1341
1357
|
toplevel_required = []
|
|
1342
1358
|
schema = {
|
|
1343
|
-
"roles" => MU::Cloud
|
|
1359
|
+
"roles" => MU::Cloud.resourceClass("Google", "User").schema(config)[1]["roles"],
|
|
1344
1360
|
"windows_admin_username" => {
|
|
1345
1361
|
"type" => "string",
|
|
1346
1362
|
"default" => "muadmin"
|
|
@@ -1451,8 +1467,7 @@ next if !create
|
|
|
1451
1467
|
foundmatch = false
|
|
1452
1468
|
MU::Cloud.availableClouds.each { |cloud|
|
|
1453
1469
|
next if cloud == "Google"
|
|
1454
|
-
|
|
1455
|
-
foreign_types = (cloudbase.listInstanceTypes).values.first
|
|
1470
|
+
foreign_types = (MU::Cloud.cloudClass(cloud).listInstanceTypes).values.first
|
|
1456
1471
|
if foreign_types.size == 1
|
|
1457
1472
|
foreign_types = foreign_types.values.first
|
|
1458
1473
|
end
|
|
@@ -1518,12 +1533,12 @@ next if !create
|
|
|
1518
1533
|
ok = false
|
|
1519
1534
|
end
|
|
1520
1535
|
else
|
|
1521
|
-
server = MU::Cloud
|
|
1536
|
+
server = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(server, configurator)
|
|
1522
1537
|
end
|
|
1523
1538
|
|
|
1524
1539
|
subnets = nil
|
|
1525
1540
|
if !server['vpc']
|
|
1526
|
-
vpcs = MU::Cloud
|
|
1541
|
+
vpcs = MU::Cloud.resourceClass("Google", "VPC").find(credentials: server['credentials'])
|
|
1527
1542
|
if vpcs["default"]
|
|
1528
1543
|
server["vpc"] ||= {}
|
|
1529
1544
|
server["vpc"]["vpc_id"] = vpcs["default"].self_link
|
|
@@ -1538,7 +1553,7 @@ next if !create
|
|
|
1538
1553
|
if !server['vpc']['subnet_id'] and server['vpc']['subnet_name'].nil?
|
|
1539
1554
|
if !subnets
|
|
1540
1555
|
if server["vpc"]["vpc_id"]
|
|
1541
|
-
vpcs = MU::Cloud
|
|
1556
|
+
vpcs = MU::Cloud.resourceClass("Google", "VPC").find(cloud_id: server["vpc"]["vpc_id"])
|
|
1542
1557
|
subnets = vpcs["default"].subnetworks.sample
|
|
1543
1558
|
end
|
|
1544
1559
|
end
|
|
@@ -89,8 +89,8 @@ module MU
|
|
|
89
89
|
machine_type: size,
|
|
90
90
|
service_accounts: [@service_acct],
|
|
91
91
|
labels: labels,
|
|
92
|
-
disks: MU::Cloud
|
|
93
|
-
network_interfaces: MU::Cloud
|
|
92
|
+
disks: MU::Cloud.resourceClass("Google", "Server").diskConfig(@config, false, false, credentials: @config['credentials']),
|
|
93
|
+
network_interfaces: MU::Cloud.resourceClass("Google", "Server").interfaceConfig(@config, @vpc),
|
|
94
94
|
metadata: metadata,
|
|
95
95
|
tags: MU::Cloud::Google.compute(:Tags).new(items: [MU::Cloud::Google.nameStr(@mu_name)])
|
|
96
96
|
)
|
|
@@ -324,11 +324,11 @@ end
|
|
|
324
324
|
def self.schema(config)
|
|
325
325
|
toplevel_required = []
|
|
326
326
|
schema = {
|
|
327
|
-
"ssh_user" => MU::Cloud
|
|
328
|
-
"metadata" => MU::Cloud
|
|
329
|
-
"service_account" => MU::Cloud
|
|
330
|
-
"scopes" => MU::Cloud
|
|
331
|
-
"network_tags" => MU::Cloud
|
|
327
|
+
"ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
|
|
328
|
+
"metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
|
|
329
|
+
"service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
|
|
330
|
+
"scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
|
|
331
|
+
"network_tags" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["network_tags"],
|
|
332
332
|
"availability_zone" => {
|
|
333
333
|
"type" => "string",
|
|
334
334
|
"description" => "Target a specific availability zone for this pool, which will create zonal instance managers and scalers instead of regional ones."
|
|
@@ -382,7 +382,7 @@ end
|
|
|
382
382
|
if pool['basis']['launch_config']
|
|
383
383
|
launch = pool["basis"]["launch_config"]
|
|
384
384
|
|
|
385
|
-
launch['size'] = MU::Cloud
|
|
385
|
+
launch['size'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(launch["size"], pool["region"])
|
|
386
386
|
ok = false if launch['size'].nil?
|
|
387
387
|
|
|
388
388
|
if launch['image_id'].nil?
|
|
@@ -397,7 +397,7 @@ end
|
|
|
397
397
|
|
|
398
398
|
real_image = nil
|
|
399
399
|
begin
|
|
400
|
-
real_image = MU::Cloud
|
|
400
|
+
real_image = MU::Cloud.resourceClass("Google", "Server").fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
|
|
401
401
|
rescue ::Google::Apis::ClientError => e
|
|
402
402
|
MU.log e.inspect, MU::WARN
|
|
403
403
|
end
|
|
@@ -433,7 +433,7 @@ end
|
|
|
433
433
|
# @return [void]
|
|
434
434
|
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
435
435
|
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
|
436
|
-
return if !MU::Cloud
|
|
436
|
+
return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
|
|
437
437
|
filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
|
|
438
438
|
if !ignoremaster and MU.mu_public_ip
|
|
439
439
|
filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
|
|
@@ -26,10 +26,12 @@ module MU
|
|
|
26
26
|
# If we're being reverse-engineered from a cloud descriptor, use that
|
|
27
27
|
# to determine what sort of account we are.
|
|
28
28
|
if args[:from_cloud_desc]
|
|
29
|
+
@cloud_desc_cache = args[:from_cloud_desc]
|
|
29
30
|
MU::Cloud::Google.admin_directory
|
|
30
31
|
MU::Cloud::Google.iam
|
|
31
32
|
if args[:from_cloud_desc].class == ::Google::Apis::AdminDirectoryV1::User
|
|
32
33
|
@config['type'] = "interactive"
|
|
34
|
+
@cloud_id = args[:from_cloud_desc].primary_email
|
|
33
35
|
elsif args[:from_cloud_desc].class == ::Google::Apis::IamV1::ServiceAccount
|
|
34
36
|
@config['type'] = "service"
|
|
35
37
|
@config['name'] = args[:from_cloud_desc].display_name
|
|
@@ -48,6 +50,10 @@ module MU
|
|
|
48
50
|
@config['name']
|
|
49
51
|
end
|
|
50
52
|
|
|
53
|
+
if @config['type'] == "interactive" and @config['email']
|
|
54
|
+
@cloud_id ||= @config['email']
|
|
55
|
+
end
|
|
56
|
+
|
|
51
57
|
end
|
|
52
58
|
|
|
53
59
|
# Called automatically by {MU::Deploy#createResources}
|
|
@@ -58,7 +64,7 @@ module MU
|
|
|
58
64
|
account_id: acct_id,
|
|
59
65
|
service_account: MU::Cloud::Google.iam(:ServiceAccount).new(
|
|
60
66
|
display_name: @mu_name,
|
|
61
|
-
description: @config['scrub_mu_isms'] ?
|
|
67
|
+
description: @config['scrub_mu_isms'] ? @config['description'] : @deploy.deploy_id
|
|
62
68
|
)
|
|
63
69
|
)
|
|
64
70
|
if @config['use_if_exists']
|
|
@@ -90,7 +96,7 @@ module MU
|
|
|
90
96
|
end
|
|
91
97
|
elsif @config['external']
|
|
92
98
|
@cloud_id = @config['email']
|
|
93
|
-
MU::Cloud
|
|
99
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
|
|
94
100
|
else
|
|
95
101
|
if !@config['email']
|
|
96
102
|
domains = MU::Cloud::Google.admin_directory(credentials: @credentials).list_domains(@customer)
|
|
@@ -122,10 +128,10 @@ module MU
|
|
|
122
128
|
# Called automatically by {MU::Deploy#createResources}
|
|
123
129
|
def groom
|
|
124
130
|
if @config['external']
|
|
125
|
-
MU::Cloud
|
|
131
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
|
|
126
132
|
elsif @config['type'] == "interactive"
|
|
127
133
|
need_update = false
|
|
128
|
-
MU::Cloud
|
|
134
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
|
|
129
135
|
|
|
130
136
|
if @config['force_password_change'] and !cloud_desc.change_password_at_next_login
|
|
131
137
|
MU.log "Forcing #{@mu_name} to change their password at next login", MU::NOTICE
|
|
@@ -170,7 +176,7 @@ module MU
|
|
|
170
176
|
end
|
|
171
177
|
|
|
172
178
|
else
|
|
173
|
-
MU::Cloud
|
|
179
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
|
|
174
180
|
if @config['create_api_key']
|
|
175
181
|
resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(
|
|
176
182
|
cloud_desc.name
|
|
@@ -195,6 +201,7 @@ module MU
|
|
|
195
201
|
if @config['type'] == "interactive" or !@config['type']
|
|
196
202
|
@config['type'] ||= "interactive"
|
|
197
203
|
if !@config['external']
|
|
204
|
+
@cloud_id ||= @config['email']
|
|
198
205
|
@cloud_desc_cache = MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
|
|
199
206
|
else
|
|
200
207
|
return nil
|
|
@@ -226,7 +233,7 @@ module MU
|
|
|
226
233
|
else
|
|
227
234
|
{}
|
|
228
235
|
end
|
|
229
|
-
description.delete(:etag)
|
|
236
|
+
description.delete(:etag) if description
|
|
230
237
|
description
|
|
231
238
|
end
|
|
232
239
|
|
|
@@ -275,7 +282,7 @@ module MU
|
|
|
275
282
|
next if user_email.nil?
|
|
276
283
|
next if !user_email.match(/^[^\/]+@[^\/]+$/)
|
|
277
284
|
|
|
278
|
-
MU::Cloud
|
|
285
|
+
MU::Cloud.resourceClass("Google", "Role").removeBindings("user", user_email, credentials: credentials, noop: noop)
|
|
279
286
|
}
|
|
280
287
|
|
|
281
288
|
end
|
|
@@ -356,8 +363,10 @@ module MU
|
|
|
356
363
|
else
|
|
357
364
|
if cred_cfg['masquerade_as']
|
|
358
365
|
resp = MU::Cloud::Google.admin_directory(credentials: args[:credentials]).list_users(customer: MU::Cloud::Google.customerID(args[:credentials]), show_deleted: false)
|
|
366
|
+
# XXX this ain't exactly performant, do some caching or something
|
|
359
367
|
if resp and resp.users
|
|
360
368
|
resp.users.each { |u|
|
|
369
|
+
next if args[:cloud_id] and !args[:cloud_id] != u.primary_email
|
|
361
370
|
found[u.primary_email] = u
|
|
362
371
|
}
|
|
363
372
|
end
|
|
@@ -374,6 +383,7 @@ module MU
|
|
|
374
383
|
name.match(/\b\d+\-compute@developer\.gserviceaccount\.com$/) or
|
|
375
384
|
name.match(/\bproject-\d+@storage-transfer-service\.iam\.gserviceaccount\.com$/) or
|
|
376
385
|
name.match(/\b\d+@cloudbuild\.gserviceaccount\.com$/) or
|
|
386
|
+
name.match(/\b\d+@cloudservices\.gserviceaccount\.com$/) or
|
|
377
387
|
name.match(/\bservice-\d+@containerregistry\.iam\.gserviceaccount\.com$/) or
|
|
378
388
|
name.match(/\bservice-\d+@container-analysis\.iam\.gserviceaccount\.com$/) or
|
|
379
389
|
name.match(/\bservice-\d+@compute-system\.iam\.gserviceaccount\.com$/) or
|
|
@@ -416,7 +426,7 @@ module MU
|
|
|
416
426
|
return nil
|
|
417
427
|
end
|
|
418
428
|
|
|
419
|
-
user_roles = MU::Cloud
|
|
429
|
+
user_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
|
|
420
430
|
|
|
421
431
|
if cloud_desc.nil?
|
|
422
432
|
MU.log "FAILED TO FIND CLOUD DESCRIPTOR FOR #{self}", MU::ERR, details: @config
|
|
@@ -429,6 +439,10 @@ module MU
|
|
|
429
439
|
|
|
430
440
|
if bok['type'] == "service"
|
|
431
441
|
bok['name'].gsub!(/@.*/, '')
|
|
442
|
+
if cloud_desc.description and !cloud_desc.description.empty? and
|
|
443
|
+
!cloud_desc.description.match(/^[A-Z0-9_-]+-[A-Z0-9_-]+-\d{10}-[A-Z]{2}$/)
|
|
444
|
+
bok['description'] = cloud_desc.description
|
|
445
|
+
end
|
|
432
446
|
bok['project'] = @project_id
|
|
433
447
|
keys = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(@cloud_id)
|
|
434
448
|
|
|
@@ -439,13 +453,13 @@ module MU
|
|
|
439
453
|
if user_roles["serviceAccount"] and
|
|
440
454
|
user_roles["serviceAccount"][bok['cloud_id']] and
|
|
441
455
|
user_roles["serviceAccount"][bok['cloud_id']].size > 0
|
|
442
|
-
bok['roles'] = MU::Cloud
|
|
456
|
+
bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
|
|
443
457
|
end
|
|
444
458
|
else
|
|
445
459
|
if user_roles["user"] and
|
|
446
460
|
user_roles["user"][bok['cloud_id']] and
|
|
447
461
|
user_roles["user"][bok['cloud_id']].size > 0
|
|
448
|
-
bok['roles'] = MU::Cloud
|
|
462
|
+
bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
|
|
449
463
|
end
|
|
450
464
|
bok['given_name'] = cloud_desc.name.given_name if cloud_desc.name.given_name and !cloud_desc.name.given_name.empty?
|
|
451
465
|
bok['family_name'] = cloud_desc.name.family_name if cloud_desc.name.family_name and !cloud_desc.name.family_name.empty?
|
|
@@ -501,6 +515,10 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
|
501
515
|
"type" => "string",
|
|
502
516
|
"description" => "Alias for +family_name+"
|
|
503
517
|
},
|
|
518
|
+
"description" => {
|
|
519
|
+
"type" => "string",
|
|
520
|
+
"description" => "Comment field for service accounts, which we normally use to store the originating deploy's deploy id, since GCP service accounts do not have labels. This field is only honored if +scrub_mu_isms+ is set."
|
|
521
|
+
},
|
|
504
522
|
"email" => {
|
|
505
523
|
"type" => "string",
|
|
506
524
|
"description" => "Canonical email address for a +directory+ user. If not specified, will be set to +name@domain+."
|
|
@@ -528,7 +546,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
|
528
546
|
"roles" => {
|
|
529
547
|
"type" => "array",
|
|
530
548
|
"description" => "One or more Google IAM roles to associate with this user.",
|
|
531
|
-
"items" => MU::Cloud
|
|
549
|
+
"items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
|
|
532
550
|
}
|
|
533
551
|
}
|
|
534
552
|
[toplevel_required, schema]
|
|
@@ -614,15 +632,11 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
|
614
632
|
ok = false
|
|
615
633
|
end
|
|
616
634
|
|
|
617
|
-
user['dependencies'] ||= []
|
|
618
635
|
if user['roles']
|
|
619
636
|
user['roles'].each { |r|
|
|
620
637
|
if r['role'] and r['role']['name'] and
|
|
621
638
|
(!r['role']['deploy_id'] and !r['role']['id'])
|
|
622
|
-
user['
|
|
623
|
-
"type" => "role",
|
|
624
|
-
"name" => r['role']['name']
|
|
625
|
-
}
|
|
639
|
+
MU::Config.addDependency(user, r['role']['name'], "role")
|
|
626
640
|
end
|
|
627
641
|
|
|
628
642
|
if !r["projects"] and !r["organizations"] and !r["folders"]
|
|
@@ -661,7 +675,6 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
|
661
675
|
user['roles'] = parent['roles'].dup
|
|
662
676
|
end
|
|
663
677
|
configurator.insertKitten(user, "users", true)
|
|
664
|
-
parent['dependencies'] ||= []
|
|
665
678
|
parent['service_account'] = MU::Config::Ref.get(
|
|
666
679
|
type: "users",
|
|
667
680
|
cloud: "Google",
|
|
@@ -669,10 +682,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
|
669
682
|
project: user["project"],
|
|
670
683
|
credentials: user["credentials"]
|
|
671
684
|
)
|
|
672
|
-
parent['
|
|
673
|
-
"type" => "user",
|
|
674
|
-
"name" => user["name"]
|
|
675
|
-
}
|
|
685
|
+
MU::Config.addDependency(parent, user['name'], "user")
|
|
676
686
|
|
|
677
687
|
parent
|
|
678
688
|
end
|