cloud-mu 3.1.1 → 3.1.2beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Berksfile.lock +179 -0
- data/bin/mu-adopt +1 -1
- data/bin/mu-azure-tests +46 -0
- data/bin/mu-cleanup +3 -1
- data/bin/mu-configure +7 -0
- data/cloud-mu.gemspec +4 -3
- data/cookbooks/mu-tools/files/default/Mu_CA.pem +33 -0
- data/extras/clean-stock-amis +0 -0
- data/extras/generate-stock-images +0 -0
- data/extras/list-stock-amis +0 -0
- data/extras/vault_tools/export_vaults.sh +0 -0
- data/extras/vault_tools/recreate_vaults.sh +0 -0
- data/extras/vault_tools/test_vaults.sh +0 -0
- data/modules/mu/cleanup.rb +15 -3
- data/modules/mu/clouds/aws/bucket.rb +6 -0
- data/modules/mu/clouds/aws/endpoint.rb +2 -0
- data/modules/mu/clouds/aws/firewall_rule.rb +31 -0
- data/modules/mu/clouds/aws/function.rb +45 -2
- data/modules/mu/clouds/aws/loadbalancer.rb +3 -1
- data/modules/mu/clouds/aws/role.rb +3 -1
- data/modules/mu/clouds/aws/server.rb +10 -4
- data/modules/mu/clouds/aws/server_pool.rb +4 -3
- data/modules/mu/clouds/aws/vpc.rb +28 -3
- data/modules/mu/clouds/google/function.rb +679 -0
- data/modules/mu/clouds/google/vpc.rb +1 -1
- data/modules/mu/clouds/google.rb +54 -9
- data/modules/mu/config/function.rb +12 -40
- data/modules/mu/config.rb +2 -1
- data/modules/mu/groomers/ansible.rb +7 -5
- data/modules/mu/kittens.rb +22134 -0
- data/modules/mu/mu.yaml.rb +282 -0
- data/modules/mu.rb +4 -1
- metadata +41 -20
data/modules/mu/clouds/google.rb
CHANGED
@@ -345,7 +345,7 @@ module MU
|
|
345
345
|
bucket: adminBucketName(credentials),
|
346
346
|
name: name
|
347
347
|
)
|
348
|
-
|
348
|
+
MU::Cloud::Google.storage(credentials: credentials).insert_object(
|
349
349
|
adminBucketName(credentials),
|
350
350
|
objectobj,
|
351
351
|
upload_source: f.path
|
@@ -364,6 +364,21 @@ module MU
|
|
364
364
|
return if !cfg or !cfg['project']
|
365
365
|
flags["project"] ||= cfg['project']
|
366
366
|
name = deploy_id+"-secret"
|
367
|
+
resp = MU::Cloud::Google.storage(credentials: credentials).list_objects(
|
368
|
+
adminBucketName(credentials),
|
369
|
+
prefix: deploy_id
|
370
|
+
)
|
371
|
+
if resp and resp.items
|
372
|
+
resp.items.each { |obj|
|
373
|
+
MU.log "Deleting gs://#{adminBucketName(credentials)}/#{obj.name}"
|
374
|
+
if !noop
|
375
|
+
MU::Cloud::Google.storage(credentials: credentials).delete_object(
|
376
|
+
adminBucketName(credentials),
|
377
|
+
obj.name
|
378
|
+
)
|
379
|
+
end
|
380
|
+
}
|
381
|
+
end
|
367
382
|
end
|
368
383
|
|
369
384
|
# Grant access to appropriate Cloud Storage objects in our log/secret bucket for a deploy member.
|
@@ -392,7 +407,7 @@ module MU
|
|
392
407
|
entity: "user-"+acct
|
393
408
|
)
|
394
409
|
|
395
|
-
[name
|
410
|
+
[name].each { |obj|
|
396
411
|
MU.log "Granting #{acct} access to #{obj} in Cloud Storage bucket #{adminBucketName(credentials)}"
|
397
412
|
|
398
413
|
MU::Cloud::Google.storage(credentials: credentials).insert_object_access_control(
|
@@ -951,6 +966,19 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
951
966
|
end
|
952
967
|
end
|
953
968
|
|
969
|
+
# Google's Cloud Function Service API
|
970
|
+
# @param subclass [<Google::Apis::CloudfunctionsV1>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
|
971
|
+
def self.function(subclass = nil, credentials: nil)
|
972
|
+
require 'google/apis/cloudfunctions_v1'
|
973
|
+
|
974
|
+
if subclass.nil?
|
975
|
+
@@function_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudfunctionsV1::CloudFunctionsService", scopes: ['cloud-platform'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
|
976
|
+
return @@function_api[credentials]
|
977
|
+
elsif subclass.is_a?(Symbol)
|
978
|
+
return Object.const_get("::Google").const_get("Apis").const_get("CloudfunctionsV1").const_get(subclass)
|
979
|
+
end
|
980
|
+
end
|
981
|
+
|
954
982
|
# Retrieve the domains, if any, which these credentials can manage via
|
955
983
|
# GSuite or Cloud Identity.
|
956
984
|
# @param credentials [String]
|
@@ -1237,7 +1265,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1237
1265
|
|
1238
1266
|
@@enable_semaphores[project] ||= Mutex.new
|
1239
1267
|
enable_obj = MU::Cloud::Google.service_manager(:EnableServiceRequest).new(
|
1240
|
-
consumer_id: "project:"+project
|
1268
|
+
consumer_id: "project:"+project.gsub(/^projects\/([^\/]+)\/.*/, '\1')
|
1241
1269
|
)
|
1242
1270
|
# XXX dumbass way to get this string
|
1243
1271
|
if e.message.match(/by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//)
|
@@ -1248,12 +1276,12 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1248
1276
|
retries += 1
|
1249
1277
|
@@enable_semaphores[project].synchronize {
|
1250
1278
|
MU.setLogging(MU::Logger::NORMAL)
|
1251
|
-
MU.log "Attempting to enable #{svc_name} in project #{project}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
|
1279
|
+
MU.log "Attempting to enable #{svc_name} in project #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
|
1252
1280
|
MU.setLogging(save_verbosity)
|
1253
1281
|
begin
|
1254
1282
|
MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj)
|
1255
1283
|
rescue ::Google::Apis::ClientError => e
|
1256
|
-
MU.log "Error enabling #{svc_name} in #{project} for #{method_sym.to_s}: "+ e.message, MU::ERR, details: enable_obj
|
1284
|
+
MU.log "Error enabling #{svc_name} in #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')} for #{method_sym.to_s}: "+ e.message, MU::ERR, details: enable_obj
|
1257
1285
|
raise e
|
1258
1286
|
end
|
1259
1287
|
}
|
@@ -1332,6 +1360,15 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1332
1360
|
retval.self_link.sub(/.*?\/projects\//, 'projects/')
|
1333
1361
|
)
|
1334
1362
|
retval = resp
|
1363
|
+
elsif retval.class.name.match(/::Cloudfunctions[^:]*::/)
|
1364
|
+
resp = MU::Cloud::Google.function(credentials: @credentials).get_operation(
|
1365
|
+
retval.name
|
1366
|
+
)
|
1367
|
+
retval = resp
|
1368
|
+
#MU.log method_sym.to_s, MU::WARN, details: retval
|
1369
|
+
if retval.error
|
1370
|
+
raise MuError, retval.error.message
|
1371
|
+
end
|
1335
1372
|
else
|
1336
1373
|
pp retval
|
1337
1374
|
raise MuError, "I NEED TO IMPLEMENT AN OPERATION HANDLER FOR #{retval.class.name}"
|
@@ -1354,10 +1391,15 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1354
1391
|
# XXX might want to do something similar for delete ops? just the
|
1355
1392
|
# but where we wait for the operation to definitely be done
|
1356
1393
|
had_been_found = false
|
1357
|
-
if method_sym.to_s.match(/^(insert|create)_/)
|
1358
|
-
|
1359
|
-
|
1360
|
-
|
1394
|
+
if method_sym.to_s.match(/^(insert|create|patch)_/)
|
1395
|
+
get_method = method_sym.to_s.gsub(/^(insert|patch|create_disk|create)_/, "get_").to_sym
|
1396
|
+
cloud_id = if retval.respond_to?(:target_link)
|
1397
|
+
retval.target_link.sub(/^.*?\/([^\/]+)$/, '\1')
|
1398
|
+
elsif retval.respond_to?(:metadata) and retval.metadata["target"]
|
1399
|
+
retval.metadata["target"]
|
1400
|
+
else
|
1401
|
+
arguments[0] # if we're lucky
|
1402
|
+
end
|
1361
1403
|
faked_args = arguments.dup
|
1362
1404
|
faked_args.pop
|
1363
1405
|
if get_method == :get_snapshot
|
@@ -1368,6 +1410,8 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1368
1410
|
if get_method == :get_project_location_cluster
|
1369
1411
|
faked_args[0] = faked_args[0]+"/clusters/"+faked_args[1]
|
1370
1412
|
faked_args.pop
|
1413
|
+
elsif get_method == :get_project_location_function
|
1414
|
+
faked_args = [cloud_id]
|
1371
1415
|
end
|
1372
1416
|
actual_resource = @api.method(get_method).call(*faked_args)
|
1373
1417
|
#if method_sym == :insert_instance
|
@@ -1483,6 +1527,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1483
1527
|
@@firestore_api = {}
|
1484
1528
|
@@admin_directory_api = {}
|
1485
1529
|
@@billing_api = {}
|
1530
|
+
@@function_api = {}
|
1486
1531
|
end
|
1487
1532
|
end
|
1488
1533
|
end
|
@@ -29,15 +29,18 @@ module MU
|
|
29
29
|
"properties" => {
|
30
30
|
"cloud" => MU::Config.cloud_primitive,
|
31
31
|
"name" => {"type" => "string"},
|
32
|
-
"runtime" => {
|
33
|
-
"type" => "string",
|
34
|
-
"enum" => %w{nodejs nodejs4.3 nodejs6.10 nodejs8.10 nodejs10.x nodejs12.x java8 java11 python2.7 python3.6 python3.7 python3.8 dotnetcore1.0 dotnetcore2.0 dotnetcore2.1 nodejs4.3-edge go1.x ruby2.5 provided}
|
35
|
-
},
|
36
32
|
"region" => MU::Config.region_primitive,
|
37
33
|
"vpc" => MU::Config::VPC.reference(MU::Config::VPC::ONE_SUBNET+MU::Config::VPC::MANY_SUBNETS, MU::Config::VPC::NO_NAT_OPTS, "all_private"),
|
34
|
+
"triggers" => {
|
35
|
+
"type" => "array",
|
36
|
+
"items" => {
|
37
|
+
"type" => "object",
|
38
|
+
"description" => "Triggers which will cause this function to be invoked."
|
39
|
+
}
|
40
|
+
},
|
38
41
|
"handler" => {
|
39
42
|
"type" => "string",
|
40
|
-
"description" => "The function within your code that
|
43
|
+
"description" => "The function within your code that is should be called to begin execution. For Node.js, it is the module-name.export value in your function. For Java, it can be package.class-name::handler or package.class-name. For more information, see https://docs.aws.amazon.com/lambda/latest/dg/java-programming-model-handler-types.html"
|
41
44
|
},
|
42
45
|
"timeout" => {
|
43
46
|
"type" => "integer",
|
@@ -61,43 +64,10 @@ module MU
|
|
61
64
|
"description" => "Memory to allocation for function, in MB. The value must be a multiple of 64 MB."
|
62
65
|
},
|
63
66
|
"dependencies" => MU::Config.dependencies_primitive,
|
64
|
-
"triggers" => {
|
65
|
-
"type" => "array",
|
66
|
-
"items" => {
|
67
|
-
"type" => "object",
|
68
|
-
"description" => "Trigger for lambda function",
|
69
|
-
"required" => ["service"],
|
70
|
-
"additionalProperties" => false,
|
71
|
-
"properties" => {
|
72
|
-
"service" => {
|
73
|
-
"type" => "string",
|
74
|
-
"enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot},
|
75
|
-
"description" => "The name of the AWS service that will trigger this function"
|
76
|
-
},
|
77
|
-
"name" => {
|
78
|
-
"type" => "string",
|
79
|
-
"description" => "The name of the API Gateway, Cloudwatch Event, or other event trigger object"
|
80
|
-
}
|
81
|
-
}
|
82
|
-
}
|
83
|
-
},
|
84
67
|
"code" => {
|
85
68
|
"type" => "object",
|
86
|
-
"description" => "Zipped deployment package to upload to
|
87
|
-
"additionalProperties" => false,
|
69
|
+
"description" => "Zipped deployment package to upload to our function.",
|
88
70
|
"properties" => {
|
89
|
-
"s3_bucket" => {
|
90
|
-
"type" => "string",
|
91
|
-
"description" => "An S3 bucket where the deployment package can be found. Must be used in conjunction with s3_key."
|
92
|
-
},
|
93
|
-
"s3_key" => {
|
94
|
-
"type" => "string",
|
95
|
-
"description" => "Key in s3_bucket where the deployment package can be found. Must be used in conjunction with s3_bucket."
|
96
|
-
},
|
97
|
-
"s3_object_version" => {
|
98
|
-
"type" => "string",
|
99
|
-
"description" => "Specify an S3 object version for the deployment package, instead of the current default"
|
100
|
-
},
|
101
71
|
"zip_file" => {
|
102
72
|
"type" => "string",
|
103
73
|
"description" => "Path to a zipped deployment package to upload."
|
@@ -138,8 +108,10 @@ module MU
|
|
138
108
|
end
|
139
109
|
if function['code'] and function['code']['zip_file']
|
140
110
|
if !File.readable?(function['code']['zip_file'])
|
141
|
-
MU.log "Can't read deployment package #{function['code']['zip_file']}", MU::ERR
|
111
|
+
MU.log "Can't read Function deployment package #{function['code']['zip_file']}", MU::ERR
|
142
112
|
ok = false
|
113
|
+
else
|
114
|
+
function['code']['zip_file'] = File.realpath(File.expand_path(function['code']['zip_file']))
|
143
115
|
end
|
144
116
|
end
|
145
117
|
|
data/modules/mu/config.rb
CHANGED
@@ -2548,7 +2548,8 @@ $CONFIGURABLES
|
|
2548
2548
|
example += "# #{cfg_plural}:\n"
|
2549
2549
|
firstline = true
|
2550
2550
|
erb.result(binding).split(/\n/).each { |l|
|
2551
|
-
l.
|
2551
|
+
l.chomp!
|
2552
|
+
l.sub!(/#.*/, "") if !l.match(/#(?:INTERNET|NAT|DENY)/)
|
2552
2553
|
next if l.empty? or l.match(/^\s+$/)
|
2553
2554
|
if firstline
|
2554
2555
|
l = "- "+l
|
@@ -441,11 +441,13 @@ module MU
|
|
441
441
|
end
|
442
442
|
|
443
443
|
# Hook up any Ansible roles listed in our platform repos
|
444
|
-
$MU_CFG['repos']
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
444
|
+
if $MU_CFG and $MU_CFG['repos']
|
445
|
+
$MU_CFG['repos'].each { |repo|
|
446
|
+
repo.match(/\/([^\/]+?)(\.git)?$/)
|
447
|
+
shortname = Regexp.last_match(1)
|
448
|
+
repodirs << MU.dataDir + "/" + shortname
|
449
|
+
}
|
450
|
+
end
|
449
451
|
|
450
452
|
repodirs.each { |repodir|
|
451
453
|
["roles", "ansible/roles"].each { |subdir|
|