cloud-mu 3.1.1 → 3.1.2beta2

data/modules/mu/clouds/google.rb

@@ -345,7 +345,7 @@ module MU
             bucket: adminBucketName(credentials),
             name: name
           )
-          ebs_key = MU::Cloud::Google.storage(credentials: credentials).insert_object(
+          MU::Cloud::Google.storage(credentials: credentials).insert_object(
             adminBucketName(credentials),
             objectobj,
             upload_source: f.path
@@ -364,6 +364,21 @@ module MU
         return if !cfg or !cfg['project']
         flags["project"] ||= cfg['project']
         name = deploy_id+"-secret"
+        resp = MU::Cloud::Google.storage(credentials: credentials).list_objects(
+          adminBucketName(credentials),
+          prefix: deploy_id
+        )
+        if resp and resp.items
+          resp.items.each { |obj|
+            MU.log "Deleting gs://#{adminBucketName(credentials)}/#{obj.name}"
+            if !noop
+              MU::Cloud::Google.storage(credentials: credentials).delete_object(
+                adminBucketName(credentials),
+                obj.name
+              )
+            end
+          }
+        end
       end
 
       # Grant access to appropriate Cloud Storage objects in our log/secret bucket for a deploy member.
@@ -392,7 +407,7 @@ module MU
             entity: "user-"+acct
           )
 
-          [name, "log_vol_ebs_key"].each { |obj|
+          [name].each { |obj|
             MU.log "Granting #{acct} access to #{obj} in Cloud Storage bucket #{adminBucketName(credentials)}"
 
             MU::Cloud::Google.storage(credentials: credentials).insert_object_access_control(
@@ -951,6 +966,19 @@ MU.log e.message, MU::WARN, details: e.inspect
         end
       end
 
+      # Google's Cloud Function Service API
+      # @param subclass [<Google::Apis::CloudfunctionsV1>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
+      def self.function(subclass = nil, credentials: nil)
+        require 'google/apis/cloudfunctions_v1'
+
+        if subclass.nil?
+          @@function_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudfunctionsV1::CloudFunctionsService", scopes: ['cloud-platform'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
+          return @@function_api[credentials]
+        elsif subclass.is_a?(Symbol)
+          return Object.const_get("::Google").const_get("Apis").const_get("CloudfunctionsV1").const_get(subclass)
+        end
+      end
+
       # Retrieve the domains, if any, which these credentials can manage via
       # GSuite or Cloud Identity.
       # @param credentials [String]
@@ -1237,7 +1265,7 @@ MU.log e.message, MU::WARN, details: e.inspect
 
                 @@enable_semaphores[project] ||= Mutex.new
                 enable_obj = MU::Cloud::Google.service_manager(:EnableServiceRequest).new(
-                  consumer_id: "project:"+project
+                  consumer_id: "project:"+project.gsub(/^projects\/([^\/]+)\/.*/, '\1')
                 )
                 # XXX dumbass way to get this string
                 if e.message.match(/by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//)
@@ -1248,12 +1276,12 @@ MU.log e.message, MU::WARN, details: e.inspect
                     retries += 1
                     @@enable_semaphores[project].synchronize {
                       MU.setLogging(MU::Logger::NORMAL)
-                      MU.log "Attempting to enable #{svc_name} in project #{project}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
+                      MU.log "Attempting to enable #{svc_name} in project #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
                       MU.setLogging(save_verbosity)
                       begin
                         MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj)
                       rescue ::Google::Apis::ClientError => e
-                        MU.log "Error enabling #{svc_name} in #{project} for #{method_sym.to_s}: "+ e.message, MU::ERR, details: enable_obj
+                        MU.log "Error enabling #{svc_name} in #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')} for #{method_sym.to_s}: "+ e.message, MU::ERR, details: enable_obj
                         raise e
                       end
                     }
@@ -1332,6 +1360,15 @@ MU.log e.message, MU::WARN, details: e.inspect
                         retval.self_link.sub(/.*?\/projects\//, 'projects/')
                       )
                       retval = resp
+                    elsif retval.class.name.match(/::Cloudfunctions[^:]*::/)
+                      resp = MU::Cloud::Google.function(credentials: @credentials).get_operation(
+                        retval.name
+                      )
+                      retval = resp
+#MU.log method_sym.to_s, MU::WARN, details: retval
+                      if retval.error
+                        raise MuError, retval.error.message
+                      end
                     else
                       pp retval
                       raise MuError, "I NEED TO IMPLEMENT AN OPERATION HANDLER FOR #{retval.class.name}"
@@ -1354,10 +1391,15 @@ MU.log e.message, MU::WARN, details: e.inspect
               # XXX might want to do something similar for delete ops? just the
               # but where we wait for the operation to definitely be done
               had_been_found = false
-              if method_sym.to_s.match(/^(insert|create)_/) and retval.target_link
-#                service["#MU_CLOUDCLASS"].instance_methods(false).include?(:groom)
-                get_method = method_sym.to_s.gsub(/^(insert|create_disk|create)_/, "get_").to_sym
-                cloud_id = retval.target_link.sub(/^.*?\/([^\/]+)$/, '\1')
+              if method_sym.to_s.match(/^(insert|create|patch)_/)
+                get_method = method_sym.to_s.gsub(/^(insert|patch|create_disk|create)_/, "get_").to_sym
+                cloud_id = if retval.respond_to?(:target_link)
+                  retval.target_link.sub(/^.*?\/([^\/]+)$/, '\1')
+                elsif retval.respond_to?(:metadata) and retval.metadata["target"]
+                  retval.metadata["target"]
+                else
+                  arguments[0] # if we're lucky
+                end
                 faked_args = arguments.dup
                 faked_args.pop
                 if get_method == :get_snapshot
@@ -1368,6 +1410,8 @@ MU.log e.message, MU::WARN, details: e.inspect
                 if get_method == :get_project_location_cluster
                   faked_args[0] = faked_args[0]+"/clusters/"+faked_args[1]
                   faked_args.pop
+                elsif get_method == :get_project_location_function
+                  faked_args = [cloud_id]
                 end
                 actual_resource = @api.method(get_method).call(*faked_args)
 #if method_sym == :insert_instance
@@ -1483,6 +1527,7 @@ MU.log e.message, MU::WARN, details: e.inspect
       @@firestore_api = {}
       @@admin_directory_api = {}
       @@billing_api = {}
+      @@function_api = {}
     end
   end
 end

data/modules/mu/config/function.rb

@@ -29,15 +29,18 @@ module MU
         "properties" => {
           "cloud" => MU::Config.cloud_primitive,
           "name" => {"type" => "string"},
-          "runtime" => {
-            "type" => "string",
-            "enum" => %w{nodejs nodejs4.3 nodejs6.10 nodejs8.10 nodejs10.x nodejs12.x java8 java11 python2.7 python3.6 python3.7 python3.8 dotnetcore1.0 dotnetcore2.0 dotnetcore2.1 nodejs4.3-edge go1.x ruby2.5 provided}
-          },
           "region" => MU::Config.region_primitive,
           "vpc" => MU::Config::VPC.reference(MU::Config::VPC::ONE_SUBNET+MU::Config::VPC::MANY_SUBNETS, MU::Config::VPC::NO_NAT_OPTS, "all_private"),
+          "triggers" => {
+            "type" => "array",
+            "items" => {
+              "type" => "object",
+              "description" => "Triggers which will cause this function to be invoked."
+            }
+          },
           "handler" => {
             "type" => "string",
-            "description" => "The function within your code that Lambda calls to begin execution. For Node.js, it is the module-name.export value in your function. For Java, it can be package.class-name::handler or package.class-name. For more information, see https://docs.aws.amazon.com/lambda/latest/dg/java-programming-model-handler-types.html"
+            "description" => "The function within your code that is should be called to begin execution. For Node.js, it is the module-name.export value in your function. For Java, it can be package.class-name::handler or package.class-name. For more information, see https://docs.aws.amazon.com/lambda/latest/dg/java-programming-model-handler-types.html"
           }, 
           "timeout" => {
             "type" => "integer",
@@ -61,43 +64,10 @@ module MU
             "description" => "Memory to allocation for function, in MB. The value must be a multiple of 64 MB."
           },
           "dependencies" => MU::Config.dependencies_primitive,
-          "triggers" => {
-            "type" => "array",
-            "items" => {
-              "type" => "object",
-              "description" => "Trigger for lambda function",
-              "required" => ["service"],
-              "additionalProperties" => false,
-              "properties" => {
-                "service" => {
-                  "type" => "string",
-                  "enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot},
-                  "description" => "The name of the AWS service that will trigger this function"
-                },
-                "name" => {
-                  "type" => "string",
-                  "description" => "The name of the API Gateway, Cloudwatch Event, or other event trigger object"
-                }
-              }
-            }
-          },
           "code" => {
             "type" => "object",  
-            "description" => "Zipped deployment package to upload to Lambda. You must specify either s3_bucket+s3_key or zip_file.", 
-            "additionalProperties" => false,
+            "description" => "Zipped deployment package to upload to our function.", 
             "properties" => {  
-              "s3_bucket" => {
-                "type" => "string",
-                "description" => "An S3 bucket where the deployment package can be found. Must be used in conjunction with s3_key."
-              }, 
-              "s3_key" => {
-                "type" => "string",
-                "description" => "Key in s3_bucket where the deployment package can be found. Must be used in conjunction with s3_bucket."
-              }, 
-              "s3_object_version" => {
-                "type" => "string",
-                "description" => "Specify an S3 object version for the deployment package, instead of the current default"
-              }, 
               "zip_file" => {
                 "type" => "string",
                 "description" => "Path to a zipped deployment package to upload."
@@ -138,8 +108,10 @@ module MU
         end
         if function['code'] and function['code']['zip_file']
           if !File.readable?(function['code']['zip_file'])
-            MU.log "Can't read deployment package #{function['code']['zip_file']}", MU::ERR
+            MU.log "Can't read Function deployment package #{function['code']['zip_file']}", MU::ERR
             ok = false
+          else
+            function['code']['zip_file'] = File.realpath(File.expand_path(function['code']['zip_file']))
           end
         end
 

data/modules/mu/config.rb

@@ -2548,7 +2548,8 @@ $CONFIGURABLES
                     example += "#      #{cfg_plural}:\n"
                     firstline = true
                     erb.result(binding).split(/\n/).each { |l|
-                      l.sub!(/#.*/, "")
+                      l.chomp!
+                      l.sub!(/#.*/, "") if !l.match(/#(?:INTERNET|NAT|DENY)/)
                       next if l.empty? or l.match(/^\s+$/)
                       if firstline
                         l = "- "+l

data/modules/mu/groomers/ansible.rb

@@ -441,11 +441,13 @@ module MU
         end
 
         # Hook up any Ansible roles listed in our platform repos
-        $MU_CFG['repos'].each { |repo|
-          repo.match(/\/([^\/]+?)(\.git)?$/)
-          shortname = Regexp.last_match(1)
-          repodirs << MU.dataDir + "/" + shortname
-        }
+        if $MU_CFG and $MU_CFG['repos']
+          $MU_CFG['repos'].each { |repo|
+            repo.match(/\/([^\/]+?)(\.git)?$/)
+            shortname = Regexp.last_match(1)
+            repodirs << MU.dataDir + "/" + shortname
+          }
+        end
 
         repodirs.each { |repodir|
           ["roles", "ansible/roles"].each { |subdir|