cloud-mu 3.1.1 → 3.1.2beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -345,7 +345,7 @@ module MU
345
345
  bucket: adminBucketName(credentials),
346
346
  name: name
347
347
  )
348
- ebs_key = MU::Cloud::Google.storage(credentials: credentials).insert_object(
348
+ MU::Cloud::Google.storage(credentials: credentials).insert_object(
349
349
  adminBucketName(credentials),
350
350
  objectobj,
351
351
  upload_source: f.path
@@ -364,6 +364,21 @@ module MU
364
364
  return if !cfg or !cfg['project']
365
365
  flags["project"] ||= cfg['project']
366
366
  name = deploy_id+"-secret"
367
+ resp = MU::Cloud::Google.storage(credentials: credentials).list_objects(
368
+ adminBucketName(credentials),
369
+ prefix: deploy_id
370
+ )
371
+ if resp and resp.items
372
+ resp.items.each { |obj|
373
+ MU.log "Deleting gs://#{adminBucketName(credentials)}/#{obj.name}"
374
+ if !noop
375
+ MU::Cloud::Google.storage(credentials: credentials).delete_object(
376
+ adminBucketName(credentials),
377
+ obj.name
378
+ )
379
+ end
380
+ }
381
+ end
367
382
  end
368
383
 
369
384
  # Grant access to appropriate Cloud Storage objects in our log/secret bucket for a deploy member.
@@ -392,7 +407,7 @@ module MU
392
407
  entity: "user-"+acct
393
408
  )
394
409
 
395
- [name, "log_vol_ebs_key"].each { |obj|
410
+ [name].each { |obj|
396
411
  MU.log "Granting #{acct} access to #{obj} in Cloud Storage bucket #{adminBucketName(credentials)}"
397
412
 
398
413
  MU::Cloud::Google.storage(credentials: credentials).insert_object_access_control(
@@ -951,6 +966,19 @@ MU.log e.message, MU::WARN, details: e.inspect
951
966
  end
952
967
  end
953
968
 
969
+ # Google's Cloud Function Service API
970
+ # @param subclass [<Google::Apis::CloudfunctionsV1>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
971
+ def self.function(subclass = nil, credentials: nil)
972
+ require 'google/apis/cloudfunctions_v1'
973
+
974
+ if subclass.nil?
975
+ @@function_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudfunctionsV1::CloudFunctionsService", scopes: ['cloud-platform'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
976
+ return @@function_api[credentials]
977
+ elsif subclass.is_a?(Symbol)
978
+ return Object.const_get("::Google").const_get("Apis").const_get("CloudfunctionsV1").const_get(subclass)
979
+ end
980
+ end
981
+
954
982
  # Retrieve the domains, if any, which these credentials can manage via
955
983
  # GSuite or Cloud Identity.
956
984
  # @param credentials [String]
@@ -1237,7 +1265,7 @@ MU.log e.message, MU::WARN, details: e.inspect
1237
1265
 
1238
1266
  @@enable_semaphores[project] ||= Mutex.new
1239
1267
  enable_obj = MU::Cloud::Google.service_manager(:EnableServiceRequest).new(
1240
- consumer_id: "project:"+project
1268
+ consumer_id: "project:"+project.gsub(/^projects\/([^\/]+)\/.*/, '\1')
1241
1269
  )
1242
1270
  # XXX dumbass way to get this string
1243
1271
  if e.message.match(/by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//)
@@ -1248,12 +1276,12 @@ MU.log e.message, MU::WARN, details: e.inspect
1248
1276
  retries += 1
1249
1277
  @@enable_semaphores[project].synchronize {
1250
1278
  MU.setLogging(MU::Logger::NORMAL)
1251
- MU.log "Attempting to enable #{svc_name} in project #{project}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
1279
+ MU.log "Attempting to enable #{svc_name} in project #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
1252
1280
  MU.setLogging(save_verbosity)
1253
1281
  begin
1254
1282
  MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj)
1255
1283
  rescue ::Google::Apis::ClientError => e
1256
- MU.log "Error enabling #{svc_name} in #{project} for #{method_sym.to_s}: "+ e.message, MU::ERR, details: enable_obj
1284
+ MU.log "Error enabling #{svc_name} in #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')} for #{method_sym.to_s}: "+ e.message, MU::ERR, details: enable_obj
1257
1285
  raise e
1258
1286
  end
1259
1287
  }
@@ -1332,6 +1360,15 @@ MU.log e.message, MU::WARN, details: e.inspect
1332
1360
  retval.self_link.sub(/.*?\/projects\//, 'projects/')
1333
1361
  )
1334
1362
  retval = resp
1363
+ elsif retval.class.name.match(/::Cloudfunctions[^:]*::/)
1364
+ resp = MU::Cloud::Google.function(credentials: @credentials).get_operation(
1365
+ retval.name
1366
+ )
1367
+ retval = resp
1368
+ #MU.log method_sym.to_s, MU::WARN, details: retval
1369
+ if retval.error
1370
+ raise MuError, retval.error.message
1371
+ end
1335
1372
  else
1336
1373
  pp retval
1337
1374
  raise MuError, "I NEED TO IMPLEMENT AN OPERATION HANDLER FOR #{retval.class.name}"
@@ -1354,10 +1391,15 @@ MU.log e.message, MU::WARN, details: e.inspect
1354
1391
  # XXX might want to do something similar for delete ops? just the
1355
1392
  # but where we wait for the operation to definitely be done
1356
1393
  had_been_found = false
1357
- if method_sym.to_s.match(/^(insert|create)_/) and retval.target_link
1358
- # service["#MU_CLOUDCLASS"].instance_methods(false).include?(:groom)
1359
- get_method = method_sym.to_s.gsub(/^(insert|create_disk|create)_/, "get_").to_sym
1360
- cloud_id = retval.target_link.sub(/^.*?\/([^\/]+)$/, '\1')
1394
+ if method_sym.to_s.match(/^(insert|create|patch)_/)
1395
+ get_method = method_sym.to_s.gsub(/^(insert|patch|create_disk|create)_/, "get_").to_sym
1396
+ cloud_id = if retval.respond_to?(:target_link)
1397
+ retval.target_link.sub(/^.*?\/([^\/]+)$/, '\1')
1398
+ elsif retval.respond_to?(:metadata) and retval.metadata["target"]
1399
+ retval.metadata["target"]
1400
+ else
1401
+ arguments[0] # if we're lucky
1402
+ end
1361
1403
  faked_args = arguments.dup
1362
1404
  faked_args.pop
1363
1405
  if get_method == :get_snapshot
@@ -1368,6 +1410,8 @@ MU.log e.message, MU::WARN, details: e.inspect
1368
1410
  if get_method == :get_project_location_cluster
1369
1411
  faked_args[0] = faked_args[0]+"/clusters/"+faked_args[1]
1370
1412
  faked_args.pop
1413
+ elsif get_method == :get_project_location_function
1414
+ faked_args = [cloud_id]
1371
1415
  end
1372
1416
  actual_resource = @api.method(get_method).call(*faked_args)
1373
1417
  #if method_sym == :insert_instance
@@ -1483,6 +1527,7 @@ MU.log e.message, MU::WARN, details: e.inspect
1483
1527
  @@firestore_api = {}
1484
1528
  @@admin_directory_api = {}
1485
1529
  @@billing_api = {}
1530
+ @@function_api = {}
1486
1531
  end
1487
1532
  end
1488
1533
  end
@@ -29,15 +29,18 @@ module MU
29
29
  "properties" => {
30
30
  "cloud" => MU::Config.cloud_primitive,
31
31
  "name" => {"type" => "string"},
32
- "runtime" => {
33
- "type" => "string",
34
- "enum" => %w{nodejs nodejs4.3 nodejs6.10 nodejs8.10 nodejs10.x nodejs12.x java8 java11 python2.7 python3.6 python3.7 python3.8 dotnetcore1.0 dotnetcore2.0 dotnetcore2.1 nodejs4.3-edge go1.x ruby2.5 provided}
35
- },
36
32
  "region" => MU::Config.region_primitive,
37
33
  "vpc" => MU::Config::VPC.reference(MU::Config::VPC::ONE_SUBNET+MU::Config::VPC::MANY_SUBNETS, MU::Config::VPC::NO_NAT_OPTS, "all_private"),
34
+ "triggers" => {
35
+ "type" => "array",
36
+ "items" => {
37
+ "type" => "object",
38
+ "description" => "Triggers which will cause this function to be invoked."
39
+ }
40
+ },
38
41
  "handler" => {
39
42
  "type" => "string",
40
- "description" => "The function within your code that Lambda calls to begin execution. For Node.js, it is the module-name.export value in your function. For Java, it can be package.class-name::handler or package.class-name. For more information, see https://docs.aws.amazon.com/lambda/latest/dg/java-programming-model-handler-types.html"
43
+ "description" => "The function within your code that is should be called to begin execution. For Node.js, it is the module-name.export value in your function. For Java, it can be package.class-name::handler or package.class-name. For more information, see https://docs.aws.amazon.com/lambda/latest/dg/java-programming-model-handler-types.html"
41
44
  },
42
45
  "timeout" => {
43
46
  "type" => "integer",
@@ -61,43 +64,10 @@ module MU
61
64
  "description" => "Memory to allocation for function, in MB. The value must be a multiple of 64 MB."
62
65
  },
63
66
  "dependencies" => MU::Config.dependencies_primitive,
64
- "triggers" => {
65
- "type" => "array",
66
- "items" => {
67
- "type" => "object",
68
- "description" => "Trigger for lambda function",
69
- "required" => ["service"],
70
- "additionalProperties" => false,
71
- "properties" => {
72
- "service" => {
73
- "type" => "string",
74
- "enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot},
75
- "description" => "The name of the AWS service that will trigger this function"
76
- },
77
- "name" => {
78
- "type" => "string",
79
- "description" => "The name of the API Gateway, Cloudwatch Event, or other event trigger object"
80
- }
81
- }
82
- }
83
- },
84
67
  "code" => {
85
68
  "type" => "object",
86
- "description" => "Zipped deployment package to upload to Lambda. You must specify either s3_bucket+s3_key or zip_file.",
87
- "additionalProperties" => false,
69
+ "description" => "Zipped deployment package to upload to our function.",
88
70
  "properties" => {
89
- "s3_bucket" => {
90
- "type" => "string",
91
- "description" => "An S3 bucket where the deployment package can be found. Must be used in conjunction with s3_key."
92
- },
93
- "s3_key" => {
94
- "type" => "string",
95
- "description" => "Key in s3_bucket where the deployment package can be found. Must be used in conjunction with s3_bucket."
96
- },
97
- "s3_object_version" => {
98
- "type" => "string",
99
- "description" => "Specify an S3 object version for the deployment package, instead of the current default"
100
- },
101
71
  "zip_file" => {
102
72
  "type" => "string",
103
73
  "description" => "Path to a zipped deployment package to upload."
@@ -138,8 +108,10 @@ module MU
138
108
  end
139
109
  if function['code'] and function['code']['zip_file']
140
110
  if !File.readable?(function['code']['zip_file'])
141
- MU.log "Can't read deployment package #{function['code']['zip_file']}", MU::ERR
111
+ MU.log "Can't read Function deployment package #{function['code']['zip_file']}", MU::ERR
142
112
  ok = false
113
+ else
114
+ function['code']['zip_file'] = File.realpath(File.expand_path(function['code']['zip_file']))
143
115
  end
144
116
  end
145
117
 
data/modules/mu/config.rb CHANGED
@@ -2548,7 +2548,8 @@ $CONFIGURABLES
2548
2548
  example += "# #{cfg_plural}:\n"
2549
2549
  firstline = true
2550
2550
  erb.result(binding).split(/\n/).each { |l|
2551
- l.sub!(/#.*/, "")
2551
+ l.chomp!
2552
+ l.sub!(/#.*/, "") if !l.match(/#(?:INTERNET|NAT|DENY)/)
2552
2553
  next if l.empty? or l.match(/^\s+$/)
2553
2554
  if firstline
2554
2555
  l = "- "+l
@@ -441,11 +441,13 @@ module MU
441
441
  end
442
442
 
443
443
  # Hook up any Ansible roles listed in our platform repos
444
- $MU_CFG['repos'].each { |repo|
445
- repo.match(/\/([^\/]+?)(\.git)?$/)
446
- shortname = Regexp.last_match(1)
447
- repodirs << MU.dataDir + "/" + shortname
448
- }
444
+ if $MU_CFG and $MU_CFG['repos']
445
+ $MU_CFG['repos'].each { |repo|
446
+ repo.match(/\/([^\/]+?)(\.git)?$/)
447
+ shortname = Regexp.last_match(1)
448
+ repodirs << MU.dataDir + "/" + shortname
449
+ }
450
+ end
449
451
 
450
452
  repodirs.each { |repodir|
451
453
  ["roles", "ansible/roles"].each { |subdir|