cloud-mu 3.1.6 → 3.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +1 -1
- data/bin/mu-adopt +15 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +37 -1
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +4 -4
- data/cookbooks/mu-tools/attributes/default.rb +7 -0
- data/cookbooks/mu-tools/libraries/helper.rb +87 -3
- data/cookbooks/mu-tools/recipes/apply_security.rb +39 -23
- data/cookbooks/mu-tools/recipes/aws_api.rb +13 -0
- data/cookbooks/mu-tools/recipes/google_api.rb +4 -0
- data/cookbooks/mu-tools/recipes/rsyslog.rb +8 -1
- data/cookbooks/mu-tools/resources/disk.rb +33 -12
- data/cookbooks/mu-tools/resources/mommacat_request.rb +1 -2
- data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb +215 -0
- data/extras/clean-stock-amis +10 -2
- data/extras/generate-stock-images +7 -3
- data/extras/image-generators/AWS/centos7.yaml +19 -16
- data/extras/image-generators/AWS/{rhel7.yaml → rhel71.yaml} +0 -0
- data/extras/image-generators/AWS/{win2k12.yaml → win2k12r2.yaml} +0 -0
- data/modules/mommacat.ru +2 -2
- data/modules/mu.rb +84 -97
- data/modules/mu/adoption.rb +359 -59
- data/modules/mu/cleanup.rb +67 -44
- data/modules/mu/cloud.rb +108 -1754
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +929 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +178 -0
- data/modules/mu/config.rb +122 -80
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +2 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +4 -5
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +81 -9
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +30 -34
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +5 -13
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +10 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +13 -17
- data/modules/mu/defaults/AWS.yaml +106 -106
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +33 -19
- data/modules/mu/groomer.rb +15 -0
- data/modules/mu/groomers/chef.rb +3 -0
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +22 -1
- data/modules/mu/mommacat.rb +71 -26
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +82 -3
- data/modules/mu/mommacat/search.rb +59 -16
- data/modules/mu/mommacat/storage.rb +119 -48
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +248 -62
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +65 -63
- data/modules/mu/providers/aws/database.rb +1747 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +291 -133
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +469 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +112 -78
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
- data/modules/mu/{clouds → providers}/aws/server.rb +120 -145
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/vpc.rb +141 -73
- data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
- data/modules/mu/{clouds → providers}/azure.rb +4 -1
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +15 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +2 -2
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +29 -14
- data/modules/mu/{clouds → providers}/google/database.rb +2 -9
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +4 -4
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +2 -2
- data/modules/mu/{clouds → providers}/google/role.rb +46 -35
- data/modules/mu/{clouds → providers}/google/server.rb +26 -11
- data/modules/mu/{clouds → providers}/google/server_pool.rb +11 -11
- data/modules/mu/{clouds → providers}/google/user.rb +32 -22
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +38 -3
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +2 -2
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +1 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +2 -2
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +126 -98
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -492,7 +492,7 @@ next if !create
|
|
492
492
|
return nil if @config.nil? or @deploy.nil?
|
493
493
|
|
494
494
|
nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
|
495
|
-
if !@config["vpc"].nil? and !MU::Cloud
|
495
|
+
if !@config["vpc"].nil? and !MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
|
496
496
|
|
497
497
|
if !@nat.nil?
|
498
498
|
if @nat.cloud_desc.nil?
|
@@ -623,7 +623,7 @@ next if !create
|
|
623
623
|
end
|
624
624
|
|
625
625
|
_nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
|
626
|
-
if !nat_ssh_host and !MU::Cloud
|
626
|
+
if !nat_ssh_host and !MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
|
627
627
|
# XXX check if canonical_ip is in the private ranges
|
628
628
|
# raise MuError, "#{node} has no NAT host configured, and I have no other route to it"
|
629
629
|
end
|
@@ -992,7 +992,7 @@ next if !create
|
|
992
992
|
# Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
|
993
993
|
# which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
|
994
994
|
# The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
|
995
|
-
if MU::Cloud
|
995
|
+
if MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
|
996
996
|
@config['canonical_ip'] = private_ips.first
|
997
997
|
return private_ips.first
|
998
998
|
else
|
@@ -1001,6 +1001,22 @@ next if !create
|
|
1001
1001
|
end
|
1002
1002
|
end
|
1003
1003
|
|
1004
|
+
# Return all of the IP addresses, public and private, from all of our
|
1005
|
+
# network interfaces.
|
1006
|
+
# @return [Array<String>]
|
1007
|
+
def listIPs
|
1008
|
+
ips = []
|
1009
|
+
cloud_desc.network_interfaces.each { |iface|
|
1010
|
+
ips << iface.network_ip
|
1011
|
+
if iface.access_configs
|
1012
|
+
iface.access_configs.each { |acfg|
|
1013
|
+
ips << acfg.nat_ip if acfg.nat_ip
|
1014
|
+
}
|
1015
|
+
end
|
1016
|
+
}
|
1017
|
+
ips
|
1018
|
+
end
|
1019
|
+
|
1004
1020
|
# return [String]: A password string.
|
1005
1021
|
def getWindowsAdminPassword(use_cache: true)
|
1006
1022
|
@config['windows_auth_vault'] ||= {
|
@@ -1274,9 +1290,9 @@ next if !create
|
|
1274
1290
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
1275
1291
|
# @param region [String]: The cloud provider region
|
1276
1292
|
# @return [void]
|
1277
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
1293
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
1278
1294
|
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
1279
|
-
return if !MU::Cloud
|
1295
|
+
return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
|
1280
1296
|
|
1281
1297
|
# XXX make damn sure MU.deploy_id is set
|
1282
1298
|
filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
|
@@ -1340,7 +1356,7 @@ next if !create
|
|
1340
1356
|
def self.schema(config)
|
1341
1357
|
toplevel_required = []
|
1342
1358
|
schema = {
|
1343
|
-
"roles" => MU::Cloud
|
1359
|
+
"roles" => MU::Cloud.resourceClass("Google", "User").schema(config)[1]["roles"],
|
1344
1360
|
"windows_admin_username" => {
|
1345
1361
|
"type" => "string",
|
1346
1362
|
"default" => "muadmin"
|
@@ -1451,8 +1467,7 @@ next if !create
|
|
1451
1467
|
foundmatch = false
|
1452
1468
|
MU::Cloud.availableClouds.each { |cloud|
|
1453
1469
|
next if cloud == "Google"
|
1454
|
-
|
1455
|
-
foreign_types = (cloudbase.listInstanceTypes).values.first
|
1470
|
+
foreign_types = (MU::Cloud.cloudClass(cloud).listInstanceTypes).values.first
|
1456
1471
|
if foreign_types.size == 1
|
1457
1472
|
foreign_types = foreign_types.values.first
|
1458
1473
|
end
|
@@ -1518,12 +1533,12 @@ next if !create
|
|
1518
1533
|
ok = false
|
1519
1534
|
end
|
1520
1535
|
else
|
1521
|
-
server = MU::Cloud
|
1536
|
+
server = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(server, configurator)
|
1522
1537
|
end
|
1523
1538
|
|
1524
1539
|
subnets = nil
|
1525
1540
|
if !server['vpc']
|
1526
|
-
vpcs = MU::Cloud
|
1541
|
+
vpcs = MU::Cloud.resourceClass("Google", "VPC").find(credentials: server['credentials'])
|
1527
1542
|
if vpcs["default"]
|
1528
1543
|
server["vpc"] ||= {}
|
1529
1544
|
server["vpc"]["vpc_id"] = vpcs["default"].self_link
|
@@ -1538,7 +1553,7 @@ next if !create
|
|
1538
1553
|
if !server['vpc']['subnet_id'] and server['vpc']['subnet_name'].nil?
|
1539
1554
|
if !subnets
|
1540
1555
|
if server["vpc"]["vpc_id"]
|
1541
|
-
vpcs = MU::Cloud
|
1556
|
+
vpcs = MU::Cloud.resourceClass("Google", "VPC").find(cloud_id: server["vpc"]["vpc_id"])
|
1542
1557
|
subnets = vpcs["default"].subnetworks.sample
|
1543
1558
|
end
|
1544
1559
|
end
|
@@ -89,8 +89,8 @@ module MU
|
|
89
89
|
machine_type: size,
|
90
90
|
service_accounts: [@service_acct],
|
91
91
|
labels: labels,
|
92
|
-
disks: MU::Cloud
|
93
|
-
network_interfaces: MU::Cloud
|
92
|
+
disks: MU::Cloud.resourceClass("Google", "Server").diskConfig(@config, false, false, credentials: @config['credentials']),
|
93
|
+
network_interfaces: MU::Cloud.resourceClass("Google", "Server").interfaceConfig(@config, @vpc),
|
94
94
|
metadata: metadata,
|
95
95
|
tags: MU::Cloud::Google.compute(:Tags).new(items: [MU::Cloud::Google.nameStr(@mu_name)])
|
96
96
|
)
|
@@ -324,11 +324,11 @@ end
|
|
324
324
|
def self.schema(config)
|
325
325
|
toplevel_required = []
|
326
326
|
schema = {
|
327
|
-
"ssh_user" => MU::Cloud
|
328
|
-
"metadata" => MU::Cloud
|
329
|
-
"service_account" => MU::Cloud
|
330
|
-
"scopes" => MU::Cloud
|
331
|
-
"network_tags" => MU::Cloud
|
327
|
+
"ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
|
328
|
+
"metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
|
329
|
+
"service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
|
330
|
+
"scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
|
331
|
+
"network_tags" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["network_tags"],
|
332
332
|
"availability_zone" => {
|
333
333
|
"type" => "string",
|
334
334
|
"description" => "Target a specific availability zone for this pool, which will create zonal instance managers and scalers instead of regional ones."
|
@@ -382,7 +382,7 @@ end
|
|
382
382
|
if pool['basis']['launch_config']
|
383
383
|
launch = pool["basis"]["launch_config"]
|
384
384
|
|
385
|
-
launch['size'] = MU::Cloud
|
385
|
+
launch['size'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(launch["size"], pool["region"])
|
386
386
|
ok = false if launch['size'].nil?
|
387
387
|
|
388
388
|
if launch['image_id'].nil?
|
@@ -397,7 +397,7 @@ end
|
|
397
397
|
|
398
398
|
real_image = nil
|
399
399
|
begin
|
400
|
-
real_image = MU::Cloud
|
400
|
+
real_image = MU::Cloud.resourceClass("Google", "Server").fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
|
401
401
|
rescue ::Google::Apis::ClientError => e
|
402
402
|
MU.log e.inspect, MU::WARN
|
403
403
|
end
|
@@ -431,9 +431,9 @@ end
|
|
431
431
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
432
432
|
# @param region [String]: The cloud provider region
|
433
433
|
# @return [void]
|
434
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
434
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
435
435
|
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
436
|
-
return if !MU::Cloud
|
436
|
+
return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
|
437
437
|
filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
|
438
438
|
if !ignoremaster and MU.mu_public_ip
|
439
439
|
filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
|
@@ -26,10 +26,12 @@ module MU
|
|
26
26
|
# If we're being reverse-engineered from a cloud descriptor, use that
|
27
27
|
# to determine what sort of account we are.
|
28
28
|
if args[:from_cloud_desc]
|
29
|
+
@cloud_desc_cache = args[:from_cloud_desc]
|
29
30
|
MU::Cloud::Google.admin_directory
|
30
31
|
MU::Cloud::Google.iam
|
31
32
|
if args[:from_cloud_desc].class == ::Google::Apis::AdminDirectoryV1::User
|
32
33
|
@config['type'] = "interactive"
|
34
|
+
@cloud_id = args[:from_cloud_desc].primary_email
|
33
35
|
elsif args[:from_cloud_desc].class == ::Google::Apis::IamV1::ServiceAccount
|
34
36
|
@config['type'] = "service"
|
35
37
|
@config['name'] = args[:from_cloud_desc].display_name
|
@@ -48,6 +50,10 @@ module MU
|
|
48
50
|
@config['name']
|
49
51
|
end
|
50
52
|
|
53
|
+
if @config['type'] == "interactive" and @config['email']
|
54
|
+
@cloud_id ||= @config['email']
|
55
|
+
end
|
56
|
+
|
51
57
|
end
|
52
58
|
|
53
59
|
# Called automatically by {MU::Deploy#createResources}
|
@@ -58,7 +64,7 @@ module MU
|
|
58
64
|
account_id: acct_id,
|
59
65
|
service_account: MU::Cloud::Google.iam(:ServiceAccount).new(
|
60
66
|
display_name: @mu_name,
|
61
|
-
description: @config['scrub_mu_isms'] ?
|
67
|
+
description: @config['scrub_mu_isms'] ? @config['description'] : @deploy.deploy_id
|
62
68
|
)
|
63
69
|
)
|
64
70
|
if @config['use_if_exists']
|
@@ -90,7 +96,7 @@ module MU
|
|
90
96
|
end
|
91
97
|
elsif @config['external']
|
92
98
|
@cloud_id = @config['email']
|
93
|
-
MU::Cloud
|
99
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
|
94
100
|
else
|
95
101
|
if !@config['email']
|
96
102
|
domains = MU::Cloud::Google.admin_directory(credentials: @credentials).list_domains(@customer)
|
@@ -122,10 +128,10 @@ module MU
|
|
122
128
|
# Called automatically by {MU::Deploy#createResources}
|
123
129
|
def groom
|
124
130
|
if @config['external']
|
125
|
-
MU::Cloud
|
131
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
|
126
132
|
elsif @config['type'] == "interactive"
|
127
133
|
need_update = false
|
128
|
-
MU::Cloud
|
134
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
|
129
135
|
|
130
136
|
if @config['force_password_change'] and !cloud_desc.change_password_at_next_login
|
131
137
|
MU.log "Forcing #{@mu_name} to change their password at next login", MU::NOTICE
|
@@ -170,7 +176,7 @@ module MU
|
|
170
176
|
end
|
171
177
|
|
172
178
|
else
|
173
|
-
MU::Cloud
|
179
|
+
MU::Cloud.resourceClass("Google", "Role").bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
|
174
180
|
if @config['create_api_key']
|
175
181
|
resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(
|
176
182
|
cloud_desc.name
|
@@ -195,6 +201,7 @@ module MU
|
|
195
201
|
if @config['type'] == "interactive" or !@config['type']
|
196
202
|
@config['type'] ||= "interactive"
|
197
203
|
if !@config['external']
|
204
|
+
@cloud_id ||= @config['email']
|
198
205
|
@cloud_desc_cache = MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
|
199
206
|
else
|
200
207
|
return nil
|
@@ -226,7 +233,7 @@ module MU
|
|
226
233
|
else
|
227
234
|
{}
|
228
235
|
end
|
229
|
-
description.delete(:etag)
|
236
|
+
description.delete(:etag) if description
|
230
237
|
description
|
231
238
|
end
|
232
239
|
|
@@ -247,7 +254,7 @@ module MU
|
|
247
254
|
# @param noop [Boolean]: If true, will only print what would be done
|
248
255
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
249
256
|
# @return [void]
|
250
|
-
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
257
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
251
258
|
MU::Cloud::Google.getDomains(credentials)
|
252
259
|
my_org = MU::Cloud::Google.getOrg(credentials)
|
253
260
|
|
@@ -275,7 +282,7 @@ module MU
|
|
275
282
|
next if user_email.nil?
|
276
283
|
next if !user_email.match(/^[^\/]+@[^\/]+$/)
|
277
284
|
|
278
|
-
MU::Cloud
|
285
|
+
MU::Cloud.resourceClass("Google", "Role").removeBindings("user", user_email, credentials: credentials, noop: noop)
|
279
286
|
}
|
280
287
|
|
281
288
|
end
|
@@ -356,8 +363,10 @@ module MU
|
|
356
363
|
else
|
357
364
|
if cred_cfg['masquerade_as']
|
358
365
|
resp = MU::Cloud::Google.admin_directory(credentials: args[:credentials]).list_users(customer: MU::Cloud::Google.customerID(args[:credentials]), show_deleted: false)
|
366
|
+
# XXX this ain't exactly performant, do some caching or something
|
359
367
|
if resp and resp.users
|
360
368
|
resp.users.each { |u|
|
369
|
+
next if args[:cloud_id] and !args[:cloud_id] != u.primary_email
|
361
370
|
found[u.primary_email] = u
|
362
371
|
}
|
363
372
|
end
|
@@ -374,6 +383,7 @@ module MU
|
|
374
383
|
name.match(/\b\d+\-compute@developer\.gserviceaccount\.com$/) or
|
375
384
|
name.match(/\bproject-\d+@storage-transfer-service\.iam\.gserviceaccount\.com$/) or
|
376
385
|
name.match(/\b\d+@cloudbuild\.gserviceaccount\.com$/) or
|
386
|
+
name.match(/\b\d+@cloudservices\.gserviceaccount\.com$/) or
|
377
387
|
name.match(/\bservice-\d+@containerregistry\.iam\.gserviceaccount\.com$/) or
|
378
388
|
name.match(/\bservice-\d+@container-analysis\.iam\.gserviceaccount\.com$/) or
|
379
389
|
name.match(/\bservice-\d+@compute-system\.iam\.gserviceaccount\.com$/) or
|
@@ -416,7 +426,7 @@ module MU
|
|
416
426
|
return nil
|
417
427
|
end
|
418
428
|
|
419
|
-
user_roles = MU::Cloud
|
429
|
+
user_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
|
420
430
|
|
421
431
|
if cloud_desc.nil?
|
422
432
|
MU.log "FAILED TO FIND CLOUD DESCRIPTOR FOR #{self}", MU::ERR, details: @config
|
@@ -429,6 +439,10 @@ module MU
|
|
429
439
|
|
430
440
|
if bok['type'] == "service"
|
431
441
|
bok['name'].gsub!(/@.*/, '')
|
442
|
+
if cloud_desc.description and !cloud_desc.description.empty? and
|
443
|
+
!cloud_desc.description.match(/^[A-Z0-9_-]+-[A-Z0-9_-]+-\d{10}-[A-Z]{2}$/)
|
444
|
+
bok['description'] = cloud_desc.description
|
445
|
+
end
|
432
446
|
bok['project'] = @project_id
|
433
447
|
keys = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(@cloud_id)
|
434
448
|
|
@@ -439,13 +453,13 @@ module MU
|
|
439
453
|
if user_roles["serviceAccount"] and
|
440
454
|
user_roles["serviceAccount"][bok['cloud_id']] and
|
441
455
|
user_roles["serviceAccount"][bok['cloud_id']].size > 0
|
442
|
-
bok['roles'] = MU::Cloud
|
456
|
+
bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
|
443
457
|
end
|
444
458
|
else
|
445
459
|
if user_roles["user"] and
|
446
460
|
user_roles["user"][bok['cloud_id']] and
|
447
461
|
user_roles["user"][bok['cloud_id']].size > 0
|
448
|
-
bok['roles'] = MU::Cloud
|
462
|
+
bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
|
449
463
|
end
|
450
464
|
bok['given_name'] = cloud_desc.name.given_name if cloud_desc.name.given_name and !cloud_desc.name.given_name.empty?
|
451
465
|
bok['family_name'] = cloud_desc.name.family_name if cloud_desc.name.family_name and !cloud_desc.name.family_name.empty?
|
@@ -501,6 +515,10 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
501
515
|
"type" => "string",
|
502
516
|
"description" => "Alias for +family_name+"
|
503
517
|
},
|
518
|
+
"description" => {
|
519
|
+
"type" => "string",
|
520
|
+
"description" => "Comment field for service accounts, which we normally use to store the originating deploy's deploy id, since GCP service accounts do not have labels. This field is only honored if +scrub_mu_isms+ is set."
|
521
|
+
},
|
504
522
|
"email" => {
|
505
523
|
"type" => "string",
|
506
524
|
"description" => "Canonical email address for a +directory+ user. If not specified, will be set to +name@domain+."
|
@@ -528,7 +546,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
528
546
|
"roles" => {
|
529
547
|
"type" => "array",
|
530
548
|
"description" => "One or more Google IAM roles to associate with this user.",
|
531
|
-
"items" => MU::Cloud
|
549
|
+
"items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
|
532
550
|
}
|
533
551
|
}
|
534
552
|
[toplevel_required, schema]
|
@@ -614,15 +632,11 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
614
632
|
ok = false
|
615
633
|
end
|
616
634
|
|
617
|
-
user['dependencies'] ||= []
|
618
635
|
if user['roles']
|
619
636
|
user['roles'].each { |r|
|
620
637
|
if r['role'] and r['role']['name'] and
|
621
638
|
(!r['role']['deploy_id'] and !r['role']['id'])
|
622
|
-
user['
|
623
|
-
"type" => "role",
|
624
|
-
"name" => r['role']['name']
|
625
|
-
}
|
639
|
+
MU::Config.addDependency(user, r['role']['name'], "role")
|
626
640
|
end
|
627
641
|
|
628
642
|
if !r["projects"] and !r["organizations"] and !r["folders"]
|
@@ -661,7 +675,6 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
661
675
|
user['roles'] = parent['roles'].dup
|
662
676
|
end
|
663
677
|
configurator.insertKitten(user, "users", true)
|
664
|
-
parent['dependencies'] ||= []
|
665
678
|
parent['service_account'] = MU::Config::Ref.get(
|
666
679
|
type: "users",
|
667
680
|
cloud: "Google",
|
@@ -669,10 +682,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
669
682
|
project: user["project"],
|
670
683
|
credentials: user["credentials"]
|
671
684
|
)
|
672
|
-
parent['
|
673
|
-
"type" => "user",
|
674
|
-
"name" => user["name"]
|
675
|
-
}
|
685
|
+
MU::Config.addDependency(parent, user['name'], "user")
|
676
686
|
|
677
687
|
parent
|
678
688
|
end
|
File without changes
|
File without changes
|
File without changes
|
@@ -537,9 +537,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
537
537
|
# @param noop [Boolean]: If true, will only print what would be done
|
538
538
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
539
539
|
# @return [void]
|
540
|
-
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
540
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
541
541
|
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
542
|
-
return if !MU::Cloud
|
542
|
+
return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
|
543
543
|
filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
|
544
544
|
if !ignoremaster and MU.mu_public_ip
|
545
545
|
filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
|
@@ -565,7 +565,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
565
565
|
MU.log e.message, MU::WARN
|
566
566
|
if e.message.match(/Failed to delete network (.+)/)
|
567
567
|
network_name = Regexp.last_match[1]
|
568
|
-
fwrules = MU::Cloud
|
568
|
+
fwrules = MU::Cloud.resourceClass("Google", "FirewallRule").find(project: flags['habitat'], credentials: credentials)
|
569
569
|
fwrules.reject! { |_name, desc|
|
570
570
|
!desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
|
571
571
|
}
|
@@ -946,6 +946,41 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
946
946
|
createRoute(route, network: @url, tags: [MU::Cloud::Google.nameStr(server.mu_name)])
|
947
947
|
end
|
948
948
|
|
949
|
+
# Looks at existing subnets, and attempts to find the next available
|
950
|
+
# IP block that's roughly similar to the ones we already have. This
|
951
|
+
# checks against secondary IP ranges, as well as each subnet's primary
|
952
|
+
# CIDR block.
|
953
|
+
# @param exclude [Array<String>]: One or more CIDRs to treat as unavailable, in addition to those allocated to existing subnets
|
954
|
+
# @return [String]
|
955
|
+
def getUnusedAddressBlock(exclude: [], max_bits: 28)
|
956
|
+
used_ranges = exclude.map { |cidr| NetAddr::IPv4Net.parse(cidr) }
|
957
|
+
subnets.each { |s|
|
958
|
+
used_ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
|
959
|
+
if s.cloud_desc.secondary_ip_ranges
|
960
|
+
used_ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
|
961
|
+
end
|
962
|
+
}
|
963
|
+
# XXX sort used_ranges
|
964
|
+
candidate = used_ranges.first.next_sib
|
965
|
+
|
966
|
+
begin
|
967
|
+
if candidate.netmask.prefix_len > max_bits
|
968
|
+
candidate = candidate.resize(max_bits)
|
969
|
+
end
|
970
|
+
try_again = false
|
971
|
+
used_ranges.each { |cidr|
|
972
|
+
if !cidr.rel(candidate).nil?
|
973
|
+
candidate = candidate.next_sib
|
974
|
+
try_again = true
|
975
|
+
break
|
976
|
+
end
|
977
|
+
}
|
978
|
+
try_again = false if candidate.nil?
|
979
|
+
end while try_again
|
980
|
+
|
981
|
+
candidate.to_s
|
982
|
+
end
|
983
|
+
|
949
984
|
private
|
950
985
|
|
951
986
|
def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# clouds: AWS
|
2
|
+
---
|
3
|
+
appname: smoketest
|
4
|
+
jobs:
|
5
|
+
- name: event1
|
6
|
+
schedule:
|
7
|
+
minute: '0'
|
8
|
+
hour: '1'
|
9
|
+
day_of_month: '1'
|
10
|
+
month: "*"
|
11
|
+
day_of_week: "?"
|
12
|
+
year: "*"
|
13
|
+
targets:
|
14
|
+
- type: functions
|
15
|
+
name: python-function
|
16
|
+
- name: event2
|
17
|
+
disabled: true
|
18
|
+
schedule:
|
19
|
+
minute: '0'
|
20
|
+
hour: '2'
|
21
|
+
day_of_month: '1'
|
22
|
+
month: "*"
|
23
|
+
day_of_week: "?"
|
24
|
+
year: "*"
|
25
|
+
targets:
|
26
|
+
- type: functions
|
27
|
+
name: node-function
|
28
|
+
|
29
|
+
functions:
|
30
|
+
- name: python-function
|
31
|
+
handler: lambda_function.lambda_handler
|
32
|
+
memory: 128
|
33
|
+
runtime: python3.6
|
34
|
+
timeout: 300
|
35
|
+
code:
|
36
|
+
path: functions/python-function
|
37
|
+
environment_variable:
|
38
|
+
- key: foo
|
39
|
+
value: bar
|
40
|
+
- name: node-function
|
41
|
+
runtime: nodejs12.x
|
42
|
+
handler: lambda_function.lambda_handler
|
43
|
+
memory: 256
|
44
|
+
timeout: 60
|
45
|
+
code:
|
46
|
+
path: functions/node-function
|