cloud-mu 3.1.3 → 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +15 -3
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +21 -13
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +4 -4
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +147 -37
- data/cloud-mu.gemspec +22 -20
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +3 -2
- data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/disk.rb +1 -1
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/extras/image-generators/Google/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +1 -1
- data/modules/mommacat.ru +6 -16
- data/modules/mu.rb +158 -111
- data/modules/mu/adoption.rb +404 -71
- data/modules/mu/cleanup.rb +221 -306
- data/modules/mu/cloud.rb +129 -1633
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +926 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +171 -1767
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +4 -4
- data/modules/mu/config/container_cluster.rb +9 -4
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +10 -9
- data/modules/mu/config/doc_helpers.rb +516 -0
- data/modules/mu/config/endpoint.rb +5 -4
- data/modules/mu/config/firewall_rule.rb +103 -4
- data/modules/mu/config/folder.rb +4 -4
- data/modules/mu/config/function.rb +19 -10
- data/modules/mu/config/group.rb +4 -4
- data/modules/mu/config/habitat.rb +4 -4
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +60 -14
- data/modules/mu/config/log.rb +4 -4
- data/modules/mu/config/msg_queue.rb +4 -4
- data/modules/mu/config/nosqldb.rb +4 -4
- data/modules/mu/config/notifier.rb +10 -21
- data/modules/mu/config/ref.rb +411 -0
- data/modules/mu/config/role.rb +4 -4
- data/modules/mu/config/schema_helpers.rb +509 -0
- data/modules/mu/config/search_domain.rb +4 -4
- data/modules/mu/config/server.rb +98 -71
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +5 -9
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +200 -0
- data/modules/mu/config/user.rb +4 -4
- data/modules/mu/config/vpc.rb +71 -27
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +91 -68
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +3 -2
- data/modules/mu/deploy.rb +43 -26
- data/modules/mu/groomer.rb +17 -2
- data/modules/mu/groomers/ansible.rb +188 -41
- data/modules/mu/groomers/chef.rb +116 -55
- data/modules/mu/logger.rb +127 -148
- data/modules/mu/master.rb +410 -2
- data/modules/mu/master/chef.rb +3 -4
- data/modules/mu/master/ldap.rb +3 -3
- data/modules/mu/master/ssl.rb +12 -3
- data/modules/mu/mommacat.rb +218 -2612
- data/modules/mu/mommacat/daemon.rb +403 -0
- data/modules/mu/mommacat/naming.rb +473 -0
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +722 -0
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +380 -122
- data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
- data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
- data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
- data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
- data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
- data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
- data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
- data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
- data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
- data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
- data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +29 -9
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
- data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
- data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
- data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
- data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +68 -30
- data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
- data/modules/mu/{clouds → providers}/google/database.rb +11 -21
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
- data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
- data/modules/mu/{clouds → providers}/google/function.rb +140 -168
- data/modules/mu/{clouds → providers}/google/group.rb +29 -34
- data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
- data/modules/mu/{clouds → providers}/google/role.rb +94 -58
- data/modules/mu/{clouds → providers}/google/server.rb +243 -156
- data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
- data/modules/mu/{clouds → providers}/google/user.rb +95 -31
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/bucket.yml +4 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/aws-iam.yaml +201 -0
- data/modules/tests/regrooms/bucket.yml +19 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +2 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/modules/tests/win2k12.yaml +17 -5
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +240 -154
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1985
- data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -129,13 +129,13 @@ module MU
|
|
129
129
|
|
130
130
|
if launch_desc["storage"]
|
131
131
|
launch_desc["storage"].each { |vol|
|
132
|
-
mapping, cfm_mapping = MU::Cloud
|
132
|
+
mapping, cfm_mapping = MU::Cloud.resourceClass("AWS", "Server").convertBlockDeviceMapping(vol)
|
133
133
|
if cfm_mapping.size > 0
|
134
134
|
MU::Cloud::CloudFormation.setCloudFormationProp(@cfm_template[@cfm_launch_name], "BlockDeviceMappings", cfm_mapping)
|
135
135
|
end
|
136
136
|
}
|
137
137
|
end
|
138
|
-
MU::Cloud
|
138
|
+
MU::Cloud.resourceClass("AWS", "Server.ephemeral_mappings").each { |mapping|
|
139
139
|
MU::Cloud::CloudFormation.setCloudFormationProp(@cfm_template[@cfm_launch_name], "BlockDeviceMappings", { "DeviceName" => mapping[:device_name], "VirtualName" => mapping[:virtual_name] })
|
140
140
|
}
|
141
141
|
|
@@ -263,7 +263,7 @@ module MU
|
|
263
263
|
# @param config [MU::Config]: The calling MU::Config object
|
264
264
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
265
265
|
def self.schema(config)
|
266
|
-
MU::Cloud
|
266
|
+
MU::Cloud.resourceClass("AWS", "ServerPool").schema(config)
|
267
267
|
end
|
268
268
|
|
269
269
|
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
|
@@ -271,14 +271,14 @@ module MU
|
|
271
271
|
# @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
272
272
|
# @return [Boolean]: True if validation succeeded, False otherwise
|
273
273
|
def self.validateConfig(server, configurator)
|
274
|
-
MU::Cloud
|
274
|
+
MU::Cloud.resourceClass("AWS", "ServerPool").validateConfig(server, configurator)
|
275
275
|
end
|
276
276
|
|
277
277
|
# Does this resource type exist as a global (cloud-wide) artifact, or
|
278
278
|
# is it localized to a region/zone?
|
279
279
|
# @return [Boolean]
|
280
280
|
def self.isGlobal?
|
281
|
-
MU::Cloud
|
281
|
+
MU::Cloud.resourceClass("AWS", "ServerPool").isGlobal?
|
282
282
|
end
|
283
283
|
|
284
284
|
end
|
@@ -240,8 +240,6 @@ module MU
|
|
240
240
|
{}
|
241
241
|
end
|
242
242
|
|
243
|
-
protected
|
244
|
-
|
245
243
|
# Subnets are almost a first-class resource. So let's kinda sorta treat
|
246
244
|
# them like one. This should only be invoked on objects that already
|
247
245
|
# exists in the cloud layer.
|
@@ -288,13 +286,13 @@ module MU
|
|
288
286
|
|
289
287
|
# Placeholder. This is a NOOP for CloudFormation, which doesn't build
|
290
288
|
# resources directly.
|
291
|
-
def self.find(
|
289
|
+
def self.find(**args)
|
292
290
|
MU.log "find() not implemented for CloudFormation layer", MU::DEBUG
|
293
291
|
nil
|
294
292
|
end
|
295
293
|
# Placeholder. This is a NOOP for CloudFormation, which doesn't build
|
296
294
|
# resources directly.
|
297
|
-
def self.cleanup(
|
295
|
+
def self.cleanup(**args)
|
298
296
|
MU.log "cleanup() not implemented for CloudFormation layer", MU::DEBUG
|
299
297
|
nil
|
300
298
|
end
|
@@ -303,7 +301,7 @@ module MU
|
|
303
301
|
# @param config [MU::Config]: The calling MU::Config object
|
304
302
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
305
303
|
def self.schema(config)
|
306
|
-
MU::Cloud
|
304
|
+
MU::Cloud.resourceClass("AWS", "VPC").schema(config)
|
307
305
|
end
|
308
306
|
|
309
307
|
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
|
@@ -311,14 +309,14 @@ module MU
|
|
311
309
|
# @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
312
310
|
# @return [Boolean]: True if validation succeeded, False otherwise
|
313
311
|
def self.validateConfig(server, configurator)
|
314
|
-
MU::Cloud
|
312
|
+
MU::Cloud.resourceClass("AWS", "VPC").validateConfig(server, configurator)
|
315
313
|
end
|
316
314
|
|
317
315
|
# Does this resource type exist as a global (cloud-wide) artifact, or
|
318
316
|
# is it localized to a region/zone?
|
319
317
|
# @return [Boolean]
|
320
318
|
def self.isGlobal?
|
321
|
-
MU::Cloud
|
319
|
+
MU::Cloud.resourceClass("AWS", "VPC").isGlobal?
|
322
320
|
end
|
323
321
|
|
324
322
|
end #class
|
File without changes
|
@@ -52,6 +52,22 @@ module MU
|
|
52
52
|
[:url]
|
53
53
|
end
|
54
54
|
|
55
|
+
# Is this a "real" cloud provider, or a stub like CloudFormation?
|
56
|
+
def self.virtual?
|
57
|
+
false
|
58
|
+
end
|
59
|
+
|
60
|
+
# Most of our resource implementation +find+ methods have to mangle their
|
61
|
+
# args to make sure they've extracted a project or location argument from
|
62
|
+
# other available information. This does it for them.
|
63
|
+
# @return [Hash]
|
64
|
+
def self.findLocationArgs(**args)
|
65
|
+
args[:project] ||= args[:habitat]
|
66
|
+
args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
|
67
|
+
args[:location] ||= args[:region] || args[:availability_zone] || "-"
|
68
|
+
args
|
69
|
+
end
|
70
|
+
|
55
71
|
# A hook that is always called just before any of the instance method of
|
56
72
|
# our resource implementations gets invoked, so that we can ensure that
|
57
73
|
# repetitive setup tasks (like resolving +:resource_group+ for Azure
|
@@ -141,10 +157,9 @@ module MU
|
|
141
157
|
def self.habitat(cloudobj, nolookup: false, deploy: nil)
|
142
158
|
@@habmap ||= {}
|
143
159
|
# XXX whaddabout config['habitat'] HNNNGH
|
144
|
-
|
145
160
|
return nil if !cloudobj.cloudclass.canLiveIn.include?(:Habitat)
|
146
161
|
|
147
|
-
# XXX
|
162
|
+
# XXX these are assholes because they're valid two different ways ugh ugh
|
148
163
|
return nil if [MU::Cloud::Google::Group, MU::Cloud::Google::Folder].include?(cloudobj.cloudclass)
|
149
164
|
if cloudobj.config and cloudobj.config['project']
|
150
165
|
if nolookup
|
@@ -154,7 +169,6 @@ module MU
|
|
154
169
|
return @@habmap[cloudobj.config['project']]
|
155
170
|
end
|
156
171
|
deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy)
|
157
|
-
|
158
172
|
projectobj = projectLookup(cloudobj.config['project'], deploy, raise_on_fail: false)
|
159
173
|
|
160
174
|
if projectobj
|
@@ -222,7 +236,7 @@ module MU
|
|
222
236
|
# @param sibling_only [Boolean]
|
223
237
|
# @return [MU::Config::Habitat,nil]
|
224
238
|
def self.projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false)
|
225
|
-
project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy
|
239
|
+
project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy if !caller.grep(/`findLitterMate'/) # XXX the dumbest
|
226
240
|
|
227
241
|
if !project_obj and !sibling_only
|
228
242
|
resp = MU::MommaCat.findStray(
|
@@ -328,6 +342,7 @@ module MU
|
|
328
342
|
# etc)
|
329
343
|
# @param deploy_id [MU::MommaCat]
|
330
344
|
def self.cleanDeploy(deploy_id, credentials: nil, noop: false)
|
345
|
+
removeDeploySecretsAndRoles(deploy_id, noop: noop, credentials: credentials)
|
331
346
|
end
|
332
347
|
|
333
348
|
# Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
|
@@ -363,7 +378,7 @@ module MU
|
|
363
378
|
cfg = credConfig(credentials)
|
364
379
|
return if !cfg or !cfg['project']
|
365
380
|
flags["project"] ||= cfg['project']
|
366
|
-
|
381
|
+
|
367
382
|
resp = MU::Cloud::Google.storage(credentials: credentials).list_objects(
|
368
383
|
adminBucketName(credentials),
|
369
384
|
prefix: deploy_id
|
@@ -420,7 +435,7 @@ module MU
|
|
420
435
|
MU.log e.message, MU::WARN, details: e.inspect
|
421
436
|
if e.inspect.match(/body: "Not Found"/)
|
422
437
|
raise MuError, "Google admin bucket #{adminBucketName(credentials)} or key #{name} does not appear to exist or is not visible with #{credentials ? credentials : "default"} credentials"
|
423
|
-
elsif e.message.match(/notFound: |Unknown user
|
438
|
+
elsif e.message.match(/notFound: |Unknown user:|conflict: /)
|
424
439
|
if retries < 5
|
425
440
|
sleep 5
|
426
441
|
retries += 1
|
@@ -429,7 +444,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
429
444
|
raise e
|
430
445
|
end
|
431
446
|
elsif e.inspect.match(/The metadata for object "null" was edited during the operation/)
|
432
|
-
MU.log e.message+" - Google admin bucket #{adminBucketName(credentials)}/#{name} with #{credentials ? credentials : "default"} credentials", MU::
|
447
|
+
MU.log e.message+" - Google admin bucket #{adminBucketName(credentials)}/#{name} with #{credentials ? credentials : "default"} credentials", MU::DEBUG, details: aclobj
|
433
448
|
sleep 10
|
434
449
|
retry
|
435
450
|
else
|
@@ -539,8 +554,8 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
539
554
|
begin
|
540
555
|
listRegions(credentials: credentials)
|
541
556
|
listInstanceTypes(credentials: credentials)
|
542
|
-
|
543
|
-
rescue ::Google::Apis::ClientError
|
557
|
+
listHabitats(credentials)
|
558
|
+
rescue ::Google::Apis::ClientError
|
544
559
|
MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes
|
545
560
|
@@authorizers.delete(credentials)
|
546
561
|
return nil
|
@@ -691,13 +706,26 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
691
706
|
nil
|
692
707
|
end
|
693
708
|
|
709
|
+
@allprojects = []
|
710
|
+
|
694
711
|
# List all Google Cloud Platform projects available to our credentials
|
695
|
-
def self.
|
712
|
+
def self.listHabitats(credentials = nil, use_cache: true)
|
696
713
|
cfg = credConfig(credentials)
|
697
|
-
return [] if !cfg
|
714
|
+
return [] if !cfg
|
715
|
+
if cfg['restrict_to_habitats'] and cfg['restrict_to_habitats'].is_a?(Array)
|
716
|
+
cfg['restrict_to_habitats'] << cfg['project'] if cfg['project']
|
717
|
+
return cfg['restrict_to_habitats'].uniq
|
718
|
+
end
|
719
|
+
if @allprojects and !@allprojects.empty? and use_cache
|
720
|
+
return @allprojects
|
721
|
+
end
|
698
722
|
result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
|
699
723
|
result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" }
|
700
|
-
result.projects.map { |p| p.project_id }
|
724
|
+
@allprojects = result.projects.map { |p| p.project_id }
|
725
|
+
if cfg['ignore_habitats'] and cfg['ignore_habitats'].is_a?(Array)
|
726
|
+
@allprojects.reject! { |p| cfg['ignore_habitats'].include?(p) }
|
727
|
+
end
|
728
|
+
@allprojects
|
701
729
|
end
|
702
730
|
|
703
731
|
@@regions = {}
|
@@ -717,7 +745,6 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
717
745
|
raise e
|
718
746
|
end
|
719
747
|
|
720
|
-
regions = []
|
721
748
|
result.items.each { |region|
|
722
749
|
@@regions[region.name] = []
|
723
750
|
region.zones.each { |az|
|
@@ -846,7 +873,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
846
873
|
if subclass.nil?
|
847
874
|
begin
|
848
875
|
@@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true)
|
849
|
-
rescue Signet::AuthorizationError
|
876
|
+
rescue Signet::AuthorizationError
|
850
877
|
MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN
|
851
878
|
@@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
|
852
879
|
@@readonly[credentials] ||= {}
|
@@ -997,14 +1024,17 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
997
1024
|
# @return [Array<OpenStruct>],nil]
|
998
1025
|
def self.getOrg(credentials = nil, with_id: nil)
|
999
1026
|
creds = MU::Cloud::Google.credConfig(credentials)
|
1027
|
+
return nil if !creds
|
1000
1028
|
credname = if creds and creds['name']
|
1001
1029
|
creds['name']
|
1002
1030
|
else
|
1003
1031
|
"default"
|
1004
1032
|
end
|
1005
1033
|
|
1034
|
+
with_id ||= creds['org'] if creds['org']
|
1006
1035
|
return @@orgmap[credname] if @@orgmap.has_key?(credname)
|
1007
1036
|
resp = MU::Cloud::Google.resource_manager(credentials: credname).search_organizations
|
1037
|
+
|
1008
1038
|
if resp and resp.organizations
|
1009
1039
|
# XXX no idea if it's possible to be a member of multiple orgs
|
1010
1040
|
if !with_id
|
@@ -1012,7 +1042,8 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1012
1042
|
return resp.organizations.first
|
1013
1043
|
else
|
1014
1044
|
resp.organizations.each { |org|
|
1015
|
-
if org.name == with_id
|
1045
|
+
if org.name == with_id or org.display_name == with_id or
|
1046
|
+
org.name == "organizations/#{with_id}"
|
1016
1047
|
@@orgmap[credname] = org
|
1017
1048
|
return org
|
1018
1049
|
end
|
@@ -1052,8 +1083,6 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1052
1083
|
@@customer_ids_cache[credentials]
|
1053
1084
|
end
|
1054
1085
|
|
1055
|
-
private
|
1056
|
-
|
1057
1086
|
# Wrapper class for Google APIs, so that we can catch some common
|
1058
1087
|
# transient endpoint errors without having to spray rescues all over the
|
1059
1088
|
# codebase.
|
@@ -1109,12 +1138,13 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1109
1138
|
# @param filter [String]: The Compute API filter string to use to isolate appropriate resources
|
1110
1139
|
def delete(type, project, region = nil, noop = false, filter = "description eq #{MU.deploy_id}", credentials: nil)
|
1111
1140
|
list_sym = "list_#{type.sub(/y$/, "ie")}s".to_sym
|
1141
|
+
credentials ||= @credentials
|
1112
1142
|
resp = nil
|
1113
1143
|
begin
|
1114
1144
|
if region
|
1115
|
-
resp = MU::Cloud::Google.compute(credentials:
|
1145
|
+
resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, region, filter: filter, mu_gcp_enable_apis: false)
|
1116
1146
|
else
|
1117
|
-
resp = MU::Cloud::Google.compute(credentials:
|
1147
|
+
resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, filter: filter, mu_gcp_enable_apis: false)
|
1118
1148
|
end
|
1119
1149
|
|
1120
1150
|
rescue ::Google::Apis::ClientError => e
|
@@ -1137,9 +1167,9 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1137
1167
|
resp = nil
|
1138
1168
|
failed = false
|
1139
1169
|
if region
|
1140
|
-
resp = MU::Cloud::Google.compute(credentials:
|
1170
|
+
resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, region, obj.name)
|
1141
1171
|
else
|
1142
|
-
resp = MU::Cloud::Google.compute(credentials:
|
1172
|
+
resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, obj.name)
|
1143
1173
|
end
|
1144
1174
|
|
1145
1175
|
if resp.error and resp.error.errors and resp.error.errors.size > 0
|
@@ -1195,11 +1225,19 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1195
1225
|
retval = nil
|
1196
1226
|
retries = 0
|
1197
1227
|
wait_backoff = 5
|
1198
|
-
if next_page_token
|
1199
|
-
if
|
1200
|
-
arguments
|
1201
|
-
|
1202
|
-
|
1228
|
+
if next_page_token
|
1229
|
+
if method_sym != :list_entry_log_entries
|
1230
|
+
if arguments.size == 1 and arguments.first.is_a?(Hash)
|
1231
|
+
arguments[0][:page_token] = next_page_token
|
1232
|
+
else
|
1233
|
+
arguments << { :page_token => next_page_token }
|
1234
|
+
end
|
1235
|
+
elsif arguments.first.class == ::Google::Apis::LoggingV2::ListLogEntriesRequest
|
1236
|
+
arguments[0] = ::Google::Apis::LoggingV2::ListLogEntriesRequest.new(
|
1237
|
+
resource_names: arguments.first.resource_names,
|
1238
|
+
filter: arguments.first.filter,
|
1239
|
+
page_token: next_page_token
|
1240
|
+
)
|
1203
1241
|
end
|
1204
1242
|
end
|
1205
1243
|
begin
|
@@ -1318,7 +1356,6 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1318
1356
|
if retval.class.name.match(/.*?::Operation$/)
|
1319
1357
|
|
1320
1358
|
retries = 0
|
1321
|
-
orig_target = retval.name
|
1322
1359
|
|
1323
1360
|
# Check whether the various types of +Operation+ responses say
|
1324
1361
|
# they're done, without knowing which specific API they're from
|
@@ -1390,7 +1427,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1390
1427
|
# take advantage.
|
1391
1428
|
# XXX might want to do something similar for delete ops? just the
|
1392
1429
|
# but where we wait for the operation to definitely be done
|
1393
|
-
had_been_found = false
|
1430
|
+
# had_been_found = false
|
1394
1431
|
if method_sym.to_s.match(/^(insert|create|patch)_/)
|
1395
1432
|
get_method = method_sym.to_s.gsub(/^(insert|patch|create_disk|create)_/, "get_").to_sym
|
1396
1433
|
cloud_id = if retval.respond_to?(:target_link)
|
@@ -1417,7 +1454,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1417
1454
|
#if method_sym == :insert_instance
|
1418
1455
|
#MU.log "actual_resource", MU::WARN, details: actual_resource
|
1419
1456
|
#end
|
1420
|
-
had_been_found = true
|
1457
|
+
# had_been_found = true
|
1421
1458
|
if actual_resource.respond_to?(:status) and
|
1422
1459
|
["PROVISIONING", "STAGING", "PENDING", "CREATING", "RESTORING"].include?(actual_resource.status)
|
1423
1460
|
retries = 0
|
@@ -1440,6 +1477,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1440
1477
|
if overall_retval
|
1441
1478
|
if method_sym.to_s.match(/^list_(.*)/)
|
1442
1479
|
require 'google/apis/iam_v1'
|
1480
|
+
require 'google/apis/logging_v2'
|
1443
1481
|
what = Regexp.last_match[1].to_sym
|
1444
1482
|
whatassign = (Regexp.last_match[1]+"=").to_sym
|
1445
1483
|
if overall_retval.class == ::Google::Apis::IamV1::ListServiceAccountsResponse
|
@@ -1451,7 +1489,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1451
1489
|
newarray = retval.public_send(what) + overall_retval.public_send(what)
|
1452
1490
|
overall_retval.public_send(whatassign, newarray)
|
1453
1491
|
end
|
1454
|
-
|
1492
|
+
elsif !retval.respond_to?(:next_page_token) or retval.next_page_token.nil? or retval.next_page_token.empty?
|
1455
1493
|
MU.log "Not sure how to append #{method_sym.to_s} results to #{overall_retval.class.name} (apparently #{what.to_s} and #{whatassign.to_s} aren't it), returning first page only", MU::WARN, details: retval
|
1456
1494
|
return retval
|
1457
1495
|
end
|
@@ -34,7 +34,7 @@ module MU
|
|
34
34
|
|
35
35
|
# Called automatically by {MU::Deploy#createResources}
|
36
36
|
def groom
|
37
|
-
@project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).
|
37
|
+
@project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloud_id
|
38
38
|
|
39
39
|
current = cloud_desc
|
40
40
|
changed = false
|
@@ -143,15 +143,14 @@ module MU
|
|
143
143
|
# Remove all buckets associated with the currently loaded deployment.
|
144
144
|
# @param noop [Boolean]: If true, will only print what would be done
|
145
145
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
146
|
-
# @param region [String]: The cloud provider region
|
147
146
|
# @return [void]
|
148
|
-
def self.cleanup(noop: false,
|
149
|
-
flags["
|
147
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
148
|
+
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
150
149
|
|
151
|
-
resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['
|
150
|
+
resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['habitat'])
|
152
151
|
if resp and resp.items
|
153
152
|
resp.items.each { |bucket|
|
154
|
-
if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase
|
153
|
+
if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or bucket.labels['mu-master-ip'] == MU.mu_public_ip.gsub(/\./, "_"))
|
155
154
|
MU.log "Deleting Cloud Storage bucket #{bucket.name}"
|
156
155
|
if !noop
|
157
156
|
MU::Cloud::Google.storage(credentials: credentials).delete_bucket(bucket.name)
|
@@ -172,8 +171,7 @@ module MU
|
|
172
171
|
# Locate an existing bucket.
|
173
172
|
# @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
|
174
173
|
def self.find(**args)
|
175
|
-
args
|
176
|
-
args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
|
174
|
+
args = MU::Cloud::Google.findLocationArgs(args)
|
177
175
|
|
178
176
|
found = {}
|
179
177
|
if args[:cloud_id]
|
@@ -198,7 +196,7 @@ module MU
|
|
198
196
|
# Reverse-map our cloud description into a runnable config hash.
|
199
197
|
# We assume that any values we have in +@config+ are placeholders, and
|
200
198
|
# calculate our own accordingly based on what's live in the cloud.
|
201
|
-
def toKitten(
|
199
|
+
def toKitten(**_args)
|
202
200
|
bok = {
|
203
201
|
"cloud" => "Google",
|
204
202
|
"credentials" => @config['credentials'],
|
@@ -245,7 +243,7 @@ module MU
|
|
245
243
|
grantees[binding.role] << { "id" => grantee }
|
246
244
|
elsif grantee.match(/^serviceAccount:(.*)/)
|
247
245
|
sa_name = Regexp.last_match[1]
|
248
|
-
if MU::Cloud
|
246
|
+
if MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(sa_name)
|
249
247
|
grantees[binding.role] << { "id" => grantee }
|
250
248
|
else
|
251
249
|
grantees[binding.role] << MU::Config::Ref.get(
|
@@ -300,14 +298,14 @@ module MU
|
|
300
298
|
end
|
301
299
|
|
302
300
|
# Cloud-specific configuration properties.
|
303
|
-
# @param
|
301
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
304
302
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
305
|
-
def self.schema(
|
303
|
+
def self.schema(_config)
|
306
304
|
toplevel_required = []
|
307
305
|
schema = {
|
308
306
|
"storage_class" => {
|
309
307
|
"type" => "string",
|
310
|
-
"enum" => ["MULTI_REGIONAL", "REGIONAL", "STANDARD", "NEARLINE", "COLDLINE", "DURABLE_REDUCED_AVAILABILITY"],
|
308
|
+
"enum" => ["MULTI_REGIONAL", "REGIONAL", "STANDARD", "NEARLINE", "COLDLINE", "DURABLE_REDUCED_AVAILABILITY", "ARCHIVE"],
|
311
309
|
"default" => "STANDARD"
|
312
310
|
},
|
313
311
|
"bucket_wide_acls" => {
|
@@ -322,9 +320,9 @@ module MU
|
|
322
320
|
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::bucket}, bare and unvalidated.
|
323
321
|
|
324
322
|
# @param bucket [Hash]: The resource to process and validate
|
325
|
-
# @param
|
323
|
+
# @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
326
324
|
# @return [Boolean]: True if validation succeeded, False otherwise
|
327
|
-
def self.validateConfig(bucket,
|
325
|
+
def self.validateConfig(bucket, _configurator)
|
328
326
|
ok = true
|
329
327
|
bucket['project'] ||= MU::Cloud::Google.defaultProject(bucket['credentials'])
|
330
328
|
|
@@ -250,7 +250,7 @@ module MU
|
|
250
250
|
@config['master_az'] = @config['region']
|
251
251
|
parent_arg = "projects/"+@config['project']+"/locations/"+@config['master_az']
|
252
252
|
|
253
|
-
|
253
|
+
MU::Cloud::Google.container(credentials: @config['credentials']).create_project_location_cluster(
|
254
254
|
parent_arg,
|
255
255
|
requestobj
|
256
256
|
)
|
@@ -277,11 +277,9 @@ module MU
|
|
277
277
|
|
278
278
|
me = cloud_desc
|
279
279
|
|
280
|
-
parent_arg = "projects/"+@config['project']+"/locations/"+me.location
|
281
|
-
|
282
280
|
# Enable/disable basic auth
|
283
281
|
authcfg = {}
|
284
|
-
|
282
|
+
|
285
283
|
if @config['master_user'] and (me.master_auth.username != @config['master_user'] or !me.master_auth.password)
|
286
284
|
authcfg[:username] = @config['master_user']
|
287
285
|
authcfg[:password] = Password.pronounceable(16..18)
|
@@ -368,15 +366,24 @@ module MU
|
|
368
366
|
}
|
369
367
|
end
|
370
368
|
|
369
|
+
# map from GKE Kuberentes addon parameter names to our BoK equivalent
|
370
|
+
# fields so we can check all these programmatically
|
371
|
+
addon_map = {
|
372
|
+
:horizontal_pod_autoscaling => 'horizontal_pod_autoscaling',
|
373
|
+
:http_load_balancing => 'http_load_balancing',
|
374
|
+
:kubernetes_dashboard => 'dashboard',
|
375
|
+
:network_policy_config => 'network_policy_addon'
|
376
|
+
}
|
377
|
+
|
371
378
|
if @config['kubernetes']
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
379
|
+
have_changes = false
|
380
|
+
addon_map.each_pair { |param, bok_param|
|
381
|
+
if (me.addons_config.send(param).disabled and @config['kubernetes'][bok_param]) or
|
382
|
+
(!me.addons_config.send(param) and !@config['kubernetes'][bok_param])
|
383
|
+
have_changes = true
|
384
|
+
end
|
385
|
+
}
|
386
|
+
if have_changes
|
380
387
|
updates << { :desired_addons_config => MU::Cloud::Google.container(:AddonsConfig).new(
|
381
388
|
horizontal_pod_autoscaling: MU::Cloud::Google.container(:HorizontalPodAutoscaling).new(
|
382
389
|
disabled: !@config['kubernetes']['horizontal_pod_autoscaling']
|
@@ -467,13 +474,10 @@ module MU
|
|
467
474
|
MU.log %Q{How to interact with your GKE cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
|
468
475
|
end
|
469
476
|
|
470
|
-
|
471
477
|
# Locate an existing ContainerCluster or ContainerClusters and return an array containing matching GCP resource descriptors for those that match.
|
472
478
|
# @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ContainerClusters
|
473
479
|
def self.find(**args)
|
474
|
-
args
|
475
|
-
args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
|
476
|
-
location = args[:region] || args[:availability_zone] || "-"
|
480
|
+
args = MU::Cloud::Google.findLocationArgs(args)
|
477
481
|
|
478
482
|
found = {}
|
479
483
|
|
@@ -486,7 +490,7 @@ module MU
|
|
486
490
|
found[args[:cloud_id]] = resp if resp
|
487
491
|
else
|
488
492
|
resp = begin
|
489
|
-
MU::Cloud::Google.container(credentials: args[:credentials]).list_project_location_clusters("projects/#{args[:project]}/locations/#{location}")
|
493
|
+
MU::Cloud::Google.container(credentials: args[:credentials]).list_project_location_clusters("projects/#{args[:project]}/locations/#{args[:location]}")
|
490
494
|
rescue ::Google::Apis::ClientError => e
|
491
495
|
raise e if !e.message.match(/forbidden:/)
|
492
496
|
end
|
@@ -503,7 +507,7 @@ module MU
|
|
503
507
|
# Reverse-map our cloud description into a runnable config hash.
|
504
508
|
# We assume that any values we have in +@config+ are placeholders, and
|
505
509
|
# calculate our own accordingly based on what's live in the cloud.
|
506
|
-
def toKitten(
|
510
|
+
def toKitten(**_args)
|
507
511
|
|
508
512
|
bok = {
|
509
513
|
"cloud" => "Google",
|
@@ -562,22 +566,24 @@ module MU
|
|
562
566
|
bok['kubernetes']['network_policy_addon'] = true
|
563
567
|
end
|
564
568
|
|
565
|
-
if cloud_desc.ip_allocation_policy
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
569
|
+
if cloud_desc.ip_allocation_policy
|
570
|
+
if cloud_desc.ip_allocation_policy.use_ip_aliases
|
571
|
+
bok['ip_aliases'] = true
|
572
|
+
end
|
573
|
+
if cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
|
574
|
+
bok['pod_ip_block'] = cloud_desc.ip_allocation_policy.cluster_ipv4_cidr_block
|
575
|
+
end
|
576
|
+
if cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
|
577
|
+
bok['services_ip_block'] = cloud_desc.ip_allocation_policy.services_ipv4_cidr_block
|
578
|
+
end
|
574
579
|
|
575
|
-
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
580
|
+
if cloud_desc.ip_allocation_policy.create_subnetwork
|
581
|
+
bok['custom_subnet'] = {
|
582
|
+
"name" => (cloud_desc.ip_allocation_policy.subnetwork_name || cloud_desc.subnetwork)
|
583
|
+
}
|
584
|
+
if cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
|
585
|
+
bok['custom_subnet']['node_ip_block'] = cloud_desc.ip_allocation_policy.node_ipv4_cidr_block
|
586
|
+
end
|
581
587
|
end
|
582
588
|
end
|
583
589
|
|
@@ -651,7 +657,7 @@ module MU
|
|
651
657
|
end
|
652
658
|
|
653
659
|
if bok['service_account']
|
654
|
-
found = MU::Cloud
|
660
|
+
found = MU::Cloud.resourceClass("Google", "User").find(
|
655
661
|
credentials: bok['credentials'],
|
656
662
|
project: bok['project'],
|
657
663
|
cloud_id: bok['service_account']
|
@@ -738,25 +744,26 @@ module MU
|
|
738
744
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
739
745
|
# @param region [String]: The cloud provider region in which to operate
|
740
746
|
# @return [void]
|
741
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
742
|
-
skipsnapshots = flags["skipsnapshots"]
|
747
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
743
748
|
|
744
|
-
flags["
|
745
|
-
return if !MU::Cloud
|
749
|
+
flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
|
750
|
+
return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
|
746
751
|
clusters = []
|
747
752
|
|
748
753
|
# Make sure we catch regional *and* zone clusters
|
749
|
-
found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['
|
754
|
+
found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{region}")
|
750
755
|
clusters.concat(found.clusters) if found and found.clusters
|
751
756
|
MU::Cloud::Google.listAZs(region).each { |az|
|
752
|
-
found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['
|
757
|
+
found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{az}")
|
753
758
|
clusters.concat(found.clusters) if found and found.clusters
|
754
759
|
}
|
755
760
|
|
756
761
|
clusters.uniq.each { |cluster|
|
757
762
|
if !cluster.resource_labels or (
|
758
763
|
!cluster.name.match(/^#{Regexp.quote(MU.deploy_id)}\-/i) and
|
759
|
-
cluster.resource_labels['mu-id'] != MU.deploy_id.downcase
|
764
|
+
(cluster.resource_labels['mu-id'] != MU.deploy_id.downcase or
|
765
|
+
(!ignoremaster and cluster.resource_labels['mu-master-ip'] != MU.mu_public_ip.gsub(/\./, "_"))
|
766
|
+
)
|
760
767
|
)
|
761
768
|
next
|
762
769
|
end
|
@@ -810,10 +817,10 @@ module MU
|
|
810
817
|
"type" => "integer",
|
811
818
|
"description" => "The number of local SSD disks to be attached to workers. See https://cloud.google.com/compute/docs/disks/local-ssd#local_ssd_limits"
|
812
819
|
},
|
813
|
-
"ssh_user" => MU::Cloud
|
814
|
-
"metadata" => MU::Cloud
|
815
|
-
"service_account" => MU::Cloud
|
816
|
-
"scopes" => MU::Cloud
|
820
|
+
"ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
|
821
|
+
"metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
|
822
|
+
"service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
|
823
|
+
"scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
|
817
824
|
"private_cluster" => {
|
818
825
|
"description" => "Set a GKE cluster to be private, that is segregated into its own hidden VPC.",
|
819
826
|
"type" => "object",
|
@@ -1014,6 +1021,25 @@ module MU
|
|
1014
1021
|
cluster['ip_aliases'] = true
|
1015
1022
|
end
|
1016
1023
|
|
1024
|
+
# try to stake out some nice /21s for our networking config
|
1025
|
+
if cluster['ip_aliases'] and cluster["vpc"] and cluster["vpc"]["id"]
|
1026
|
+
habarg = if cluster["vpc"]["habitat"] and cluster["vpc"]["habitat"]["id"]
|
1027
|
+
cluster["vpc"]["habitat"]["id"]
|
1028
|
+
else
|
1029
|
+
cluster["project"]
|
1030
|
+
end
|
1031
|
+
found = MU::MommaCat.findStray("Google", "vpcs", cloud_id: cluster["vpc"]["id"], credentials: cluster["credentials"], habitats: [habarg], dummy_ok: true)
|
1032
|
+
if found and found.size == 1
|
1033
|
+
myvpc = found.first
|
1034
|
+
# XXX this might not make sense with custom_subnet
|
1035
|
+
cluster['pod_ip_block'] ||= myvpc.getUnusedAddressBlock(max_bits: 21)
|
1036
|
+
cluster['services_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block']], max_bits: 21)
|
1037
|
+
if cluster['tpu']
|
1038
|
+
cluster['tpu_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block'], cluster['services_ip_block']], max_bits: 21)
|
1039
|
+
end
|
1040
|
+
end
|
1041
|
+
end
|
1042
|
+
|
1017
1043
|
if cluster['service_account']
|
1018
1044
|
cluster['service_account']['cloud'] = "Google"
|
1019
1045
|
cluster['service_account']['habitat'] ||= MU::Config::Ref.get(
|
@@ -1023,12 +1049,9 @@ module MU
|
|
1023
1049
|
type: "habitats"
|
1024
1050
|
)
|
1025
1051
|
if cluster['service_account']['name'] and
|
1026
|
-
!cluster['service_account']['id']
|
1027
|
-
|
1028
|
-
cluster['
|
1029
|
-
"type" => "user",
|
1030
|
-
"name" => cluster['service_account']['name']
|
1031
|
-
}
|
1052
|
+
!cluster['service_account']['id'] and
|
1053
|
+
!cluster['service_account']['deploy_id']
|
1054
|
+
MU::Config.addDependency(cluster, cluster['service_account']['name'], "user")
|
1032
1055
|
end
|
1033
1056
|
found = MU::Config::Ref.get(cluster['service_account'])
|
1034
1057
|
# XXX verify that found.kitten fails when it's supposed to
|
@@ -1037,29 +1060,7 @@ module MU
|
|
1037
1060
|
ok = false
|
1038
1061
|
end
|
1039
1062
|
else
|
1040
|
-
|
1041
|
-
"name" => cluster['name'],
|
1042
|
-
"cloud" => "Google",
|
1043
|
-
"project" => cluster["project"],
|
1044
|
-
"credentials" => cluster["credentials"],
|
1045
|
-
"type" => "service"
|
1046
|
-
}
|
1047
|
-
if user["name"].length < 6
|
1048
|
-
user["name"] += Password.pronounceable(6)
|
1049
|
-
end
|
1050
|
-
configurator.insertKitten(user, "users", true)
|
1051
|
-
cluster['dependencies'] ||= []
|
1052
|
-
cluster['service_account'] = MU::Config::Ref.get(
|
1053
|
-
type: "users",
|
1054
|
-
cloud: "Google",
|
1055
|
-
name: cluster["name"],
|
1056
|
-
project: cluster["project"],
|
1057
|
-
credentials: cluster["credentials"]
|
1058
|
-
)
|
1059
|
-
cluster['dependencies'] << {
|
1060
|
-
"type" => "user",
|
1061
|
-
"name" => user["name"]
|
1062
|
-
}
|
1063
|
+
cluster = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(cluster, configurator)
|
1063
1064
|
end
|
1064
1065
|
|
1065
1066
|
if cluster['dependencies']
|
@@ -1110,7 +1111,7 @@ module MU
|
|
1110
1111
|
}
|
1111
1112
|
if !match
|
1112
1113
|
MU.log "No version matching #{cluster['kubernetes']['version']} available, will try floating minor revision", MU::WARN
|
1113
|
-
cluster['kubernetes']['version'].sub!(/^(\d+\.\d
|
1114
|
+
cluster['kubernetes']['version'].sub!(/^(\d+\.\d+)\..*/i, '\1')
|
1114
1115
|
master_versions.each { |v|
|
1115
1116
|
if v.match(/^#{Regexp.quote(cluster['kubernetes']['version'])}/)
|
1116
1117
|
match = true
|
@@ -1155,9 +1156,13 @@ module MU
|
|
1155
1156
|
end
|
1156
1157
|
end
|
1157
1158
|
|
1158
|
-
cluster['instance_type'] = MU::Cloud
|
1159
|
+
cluster['instance_type'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
|
1159
1160
|
ok = false if cluster['instance_type'].nil?
|
1160
1161
|
|
1162
|
+
if !MU::Master.kubectl
|
1163
|
+
MU.log "Since I can't find a kubectl executable, you will have to handle all service account, user, and role bindings manually!", MU::WARN
|
1164
|
+
end
|
1165
|
+
|
1161
1166
|
ok
|
1162
1167
|
end
|
1163
1168
|
|
@@ -1204,7 +1209,8 @@ module MU
|
|
1204
1209
|
labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
|
1205
1210
|
|
1206
1211
|
labelset = MU::Cloud::Google.container(:SetLabelsRequest).new(
|
1207
|
-
resource_labels: labels
|
1212
|
+
resource_labels: labels,
|
1213
|
+
label_fingerprint: cloud_desc.label_fingerprint
|
1208
1214
|
)
|
1209
1215
|
MU::Cloud::Google.container(credentials: @config['credentials']).set_project_location_cluster_resource_labels(@cloud_id, labelset)
|
1210
1216
|
end
|
@@ -1223,6 +1229,7 @@ module MU
|
|
1223
1229
|
@@server_config[credentials][az] = MU::Cloud::Google.container(credentials: credentials).get_project_location_server_config(parent_arg)
|
1224
1230
|
@@server_config[credentials][az]
|
1225
1231
|
end
|
1232
|
+
private_class_method :defaults
|
1226
1233
|
|
1227
1234
|
def writeKubeConfig
|
1228
1235
|
kube_conf = @deploy.deploy_dir+"/kubeconfig-#{@config['name']}"
|
@@ -1247,7 +1254,7 @@ module MU
|
|
1247
1254
|
# Take this opportunity to ensure that the 'client' service account
|
1248
1255
|
# used by certificate authentication exists and has appropriate
|
1249
1256
|
# privilege
|
1250
|
-
if @username and @password
|
1257
|
+
if @username and @password and MU::Master.kubectl
|
1251
1258
|
File.open(client_binding, "w"){ |k|
|
1252
1259
|
k.puts <<-EOF
|
1253
1260
|
kind: ClusterRoleBinding
|