cloud-mu 3.2.0 → 3.5.0

data/modules/mu/providers/google/bucket.rb

@@ -144,7 +144,7 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
 
           resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['habitat'])

data/modules/mu/providers/google/container_cluster.rb

@@ -744,7 +744,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region in which to operate
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
 
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)

data/modules/mu/providers/google/database.rb

@@ -100,7 +100,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region in which to operate
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
 
 #          instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['habitat'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"})

data/modules/mu/providers/google/firewall_rule.rb

@@ -207,7 +207,7 @@ end
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}

data/modules/mu/providers/google/folder.rb

@@ -162,7 +162,7 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -265,8 +265,12 @@ module MU
 
           if args[:cloud_id]
             raw_id = args[:cloud_id].sub(/^folders\//, "")
-            resp = MU::Cloud::Google.folder(credentials: args[:credentials]).get_folder("folders/"+raw_id)
-            found[resp.name] = resp if resp
+            begin
+              resp = MU::Cloud::Google.folder(credentials: args[:credentials]).get_folder("folders/"+raw_id)
+              found[resp.name] = resp if resp
+            rescue ::Google::Apis::ClientError => e
+              raise e if e.message !~ /forbidden: /
+            end
 
           elsif args[:flags] and args[:flags]['display_name']
 

data/modules/mu/providers/google/function.rb

@@ -119,6 +119,9 @@ module example.com/cloudfunction
         # Called automatically by {MU::Deploy#createResources}
         def groom
           desc = {}
+
+          func_obj = buildDesc
+
           labels = Hash[@tags.keys.map { |k|
             [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
           ]
@@ -140,6 +143,10 @@ module example.com/cloudfunction
           if cloud_desc.available_memory_mb != @config['memory']
             need_update = true
           end
+          if cloud_desc.service_account_email != func_obj.service_account_email
+            need_update = true
+          end
+
           if @config['environment_variable']
             @config['environment_variable'].each { |var|
               if !cloud_desc.environment_variables or
@@ -161,7 +168,17 @@ module example.com/cloudfunction
             File.read("#{dir}/current.zip")
           }
 
-          new = if @config['code']['zip_file']
+          tempfile = nil
+          new = if @config['code']['zip_file'] or @config['code']['path']
+            if @config['code']['path']
+              tempfile = Tempfile.new(["function", ".zip"])
+              MU.log "#{@mu_name} using code at #{@config['code']['path']}"
+              MU::Master.zipDir(@config['code']['path'], tempfile.path)
+              @config['code']['zip_file'] = tempfile.path
+            else
+              MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
+            end
+#            @code_sha256 = Base64.encode64(Digest::SHA256.digest(zip)).chomp
             File.read(@config['code']['zip_file'])
           elsif @config['code']['gs_url']
             @config['code']['gs_url'].match(/^gs:\/\/([^\/]+)\/(.*)/)
@@ -172,25 +189,31 @@ module example.com/cloudfunction
               File.read(dir+"/new.zip")
             }
           end
+
           if @config['code']['gs_url'] and
              (@config['code']['gs_url'] != cloud_desc.source_archive_url or
              current != new)
             need_update = true
-          elsif @config['code']['zip_file'] and current != new
+          elsif (@config['code']['zip_file'] or @config['code']['path']) and current != new
             need_update = true
-            desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
+          end
+
+          if @config['vpc_connector']
+            if cloud_desc.vpc_connector != @config['vpc_connector'] or
+               cloud_desc.vpc_connector_egress_settings != (@config['vpc_connector_allow_all_egress'] ? "ALL_TRAFFIC" : "PRIVATE_RANGES_ONLY")
+              need_update = true
+            end
           end
 
           if need_update
-            func_obj = buildDesc
-            MU.log "Updating Cloud Function #{@mu_name}", MU::NOTICE, details: func_obj
+            MU.log "Updating Cloud Function #{@cloud_id}", MU::NOTICE, details: func_obj
             begin
-#              MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
-#                @cloud_id, 
-#                func_obj
-#              )
-            rescue ::Google::Apis::ClientError
-              MU.log "Error updating Cloud Function #{@mu_name}.", MU::ERR
+              MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
+                @cloud_id, 
+                func_obj
+              )
+            rescue ::Google::Apis::ClientError => e
+              MU.log "Error updating Cloud Function #{@mu_name}.", MU::ERR, e.message
               if desc[:source_archive_url]
                 main_file = nil
                 HELLO_WORLDS.each_pair { |runtime, code|
@@ -207,6 +230,11 @@ module example.com/cloudfunction
 #            service_account_email: sa.kitten.cloud_desc.email,
 #            labels: labels,
 
+          if tempfile
+            tempfile.close
+            tempfile.unlink
+          end
+
         end
 
         # Return the metadata for this project's configuration
@@ -233,7 +261,7 @@ module example.com/cloudfunction
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           # Make sure we catch regional *and* zone functions
@@ -354,6 +382,7 @@ module example.com/cloudfunction
         def self.schema(config)
           toplevel_required = ["runtime"]
           schema = {
+            "roles" => MU::Cloud.resourceClass("Google", "User").schema(config)[1]["roles"],
             "triggers" => {
               "type" => "array",
               "items" => {
@@ -448,6 +477,7 @@ module example.com/cloudfunction
             content_type: "application/zip",
             name: filename
           )
+
           MU::Cloud::Google.storage(credentials: credentials).insert_object(
             bucket,
             obj_obj,
@@ -487,7 +517,7 @@ module example.com/cloudfunction
           end
 # XXX list_project_locations
 
-          if !function['code'] or (!function['code']['zip_file'] and !function['code']['gs_url'])
+          if !function['code'] or (!function['code']['zip_file'] and !function['code']['gs_url'] and !function['code']['path'])
             MU.log "Must specify a code source in Cloud Function #{function['name']}", MU::ERR
             ok = false
           elsif function['code']['zip_file']
@@ -557,22 +587,14 @@ module example.com/cloudfunction
 
           location = "projects/"+@config['project']+"/locations/"+@config['region']
           sa = nil
-          retries = 0
-          begin
-            sa_ref = MU::Config::Ref.get(@config['service_account'])
-            sa = @deploy.findLitterMate(name: sa_ref.name, type: "users")
-            if !sa or !sa.cloud_desc
-              sleep 10
-            end
-          rescue ::Google::Apis::ClientError => e
-            if e.message.match(/notFound:/)
-              sleep 10
-              retries += 1
-              retry
-            end
-          end while !sa or !sa.cloud_desc and retries < 5
+          need_sa = Proc.new {
+            !sa or !sa.kitten or !sa.kitten.cloud_desc
+          }
+          MU.retrier(loop_if: need_sa, wait: 10, max: 6) { |retries, _wait|
+            sa = MU::Config::Ref.get(@config['service_account'])
+          }
 
-          if !sa or !sa.cloud_desc
+          if need_sa.call()
             raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
           end
 
@@ -583,7 +605,7 @@ module example.com/cloudfunction
 #            entry_point: "hello_world",
             entry_point: @config['handler'],
             description: @deploy.deploy_id,
-            service_account_email: sa.cloud_desc.email,
+            service_account_email: sa.kitten.cloud_desc.email,
             labels: labels,
             available_memory_mb: @config['memory']
           }
@@ -596,7 +618,6 @@ module example.com/cloudfunction
           if @config['vpc_connector']
             desc[:vpc_connector] = @config['vpc_connector']
             desc[:vpc_connector_egress_settings] = @config['vpc_connector_allow_all_egress'] ? "ALL_TRAFFIC" : "PRIVATE_RANGES_ONLY"
-            pp desc
           elsif @vpc
             desc[:network] = @vpc.url.sub(/^.*?\/projects\//, 'projects/')
           end
@@ -627,8 +648,22 @@ module example.com/cloudfunction
 #          }
           if @config['code']['gs_url']
             desc[:source_archive_url] = @config['code']['gs_url']
-          elsif @config['code']['zip_file']
+          elsif @config['code']['zip_file'] or @config['code']['path']
+            tempfile = nil
+            if @config['code']['path']
+              tempfile = Tempfile.new(["function", ".zip"])
+              MU.log "#{@mu_name} using code at #{@config['code']['path']}"
+              MU::Master.zipDir(@config['code']['path'], tempfile.path)
+              @config['code']['zip_file'] = tempfile.path
+            else
+              MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
+            end
             desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
+
+            if tempfile
+              tempfile.close
+              tempfile.unlink
+            end
           end
 
 #          Dir.mktmpdir(@mu_name) { |dir|

data/modules/mu/providers/google/group.rb

@@ -140,7 +140,7 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           MU::Cloud::Google.getDomains(credentials)
           my_org = MU::Cloud::Google.getOrg(credentials)
 

data/modules/mu/providers/google/habitat.rb

@@ -222,7 +222,7 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           resp = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
 
           if resp and resp.projects

data/modules/mu/providers/google/loadbalancer.rb

@@ -146,7 +146,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: nil, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: nil, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}

data/modules/mu/providers/google/role.rb

@@ -465,7 +465,7 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           customer = MU::Cloud::Google.customerID(credentials)
           my_org = MU::Cloud::Google.getOrg(credentials)
 
@@ -581,7 +581,7 @@ module MU
               }
             end
             if args[:cloud_id]
-              found.reject! { |k, _v| k != role.name }
+              found.reject! { |k, _v| k != args[:cloud_id] }
             end
 
             # Now go get everything that's bound here
@@ -745,6 +745,7 @@ module MU
                   end
 
                   entity_types.each_pair { |entity_type, entities|
+                    next if entity_type == "deleted"
                     mu_entitytype = (entity_type == "serviceAccount" ? "user" : entity_type)+"s"
                     entities.each { |entity|
                       next if entity.nil?
@@ -925,7 +926,9 @@ module MU
               }
 
               MU::Cloud.resourceClass("Google", "Folder").find(credentials: credentials).keys.each { |folder|
-                MU::Cloud.resourceClass("Google", "Folder").bindings(folder, credentials: credentials).each { |binding|
+                folder_bindings = MU::Cloud.resourceClass("Google", "Folder").bindings(folder, credentials: credentials)
+                next if !folder_bindings
+                folder_bindings.each { |binding|
                   insertBinding("folders", folder, binding)
                 }
               }

data/modules/mu/providers/google/server.rb

@@ -1290,7 +1290,7 @@ next if !create
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
 

data/modules/mu/providers/google/server_pool.rb

@@ -431,7 +431,7 @@ end
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}

data/modules/mu/providers/google/user.rb

@@ -254,7 +254,7 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           MU::Cloud::Google.getDomains(credentials)
           my_org = MU::Cloud::Google.getOrg(credentials)
 

data/modules/mu/providers/google/vpc.rb

@@ -364,6 +364,12 @@ end
             }
           end
 
+
+          # The API is filled with lies
+          @subnets.reject! { |s|
+            !MU::Cloud::Google.listRegions(credentials: @credentials).include?(s.az)
+          }
+
           return @subnets
         end
 
@@ -442,14 +448,19 @@ end
 
         # Check for a subnet in this VPC matching one or more of the specified
         # criteria, and return it if found.
-        def getSubnet(cloud_id: nil, name: nil, tag_key: nil, tag_value: nil, ip_block: nil, region: nil)
+        def getSubnet(cloud_id: nil, name: nil, tag_key: nil, tag_value: nil, ip_block: nil, region: nil, subnet_mu_name: nil)
           if !cloud_id.nil? and cloud_id.match(/^https:\/\//)
             cloud_id.match(/\/regions\/([^\/]+)\/subnetworks\/([^\/]+)$/)
             region = Regexp.last_match[1]
             cloud_id = Regexp.last_match[2]
             cloud_id.gsub!(/.*?\//, "")
           end
-          MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block}, region: #{region})", MU::DEBUG, details: caller[0]
+          
+          if name
+            subnet_mu_name ||= @config['scrub_mu_isms'] ? @cloud_id+name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(name, max_length: 61))
+          end
+
+          MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block}, region: #{region}, subnet_mu_name: #{subnet_mu_name})", MU::DEBUG, details: caller[0]
           subnets.each { |subnet|
             next if region and subnet.az != region
             if !cloud_id.nil? and !subnet.cloud_id.nil? and subnet.cloud_id.to_s == cloud_id.to_s
@@ -457,6 +468,9 @@ end
             elsif !name.nil? and !subnet.name.nil? and
                   subnet.name.downcase.to_s == name.downcase.to_s
               return subnet
+            elsif !subnet_mu_name.nil? and !subnet.name.nil? and
+                  subnet.name.downcase.to_s == subnet_mu_name.downcase.to_s
+              return subnet
             end
           }
           return nil
@@ -537,7 +551,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
@@ -931,6 +945,14 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                 else
                   route['nat_host_name'] = nat['name']
                   route['priority'] = 100
+                  MU::Config.addDependency(vpc, nat['name'], "server", their_phase: "groom", my_phase: "groom")
+                  vpc["bastion"] = MU::Config::Ref.get(
+                    name: nat['name'],
+                    cloud: vpc['cloud'],
+                    credentials: vpc['credentials'],
+                    type: "servers"
+                  )
+
                 end
               end
             }
@@ -1172,6 +1194,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
               if e.message.match(/notFound: /)
                 MU.log "Failed to fetch cloud description for Google subnet #{@cloud_id}", MU::WARN, details: { "project" => @parent.habitat_id, "region" => @az, "name" => @cloud_id }
                 return nil
+              elsif e.message.match(/Unknown region\. /)
+                MU.log "Google subnet #{@cloud_id} seems like it should live in #{@az}, but that's not a valid region", MU::WARN, details: { "project" => @parent.habitat_id, "region" => @az, "name" => @cloud_id }
+                return nil
               else
                 raise e
               end

data/modules/tests/aws-jobs-functions.yaml

@@ -0,0 +1,46 @@
+# clouds: AWS
+---
+appname: smoketest
+jobs:
+- name: event1
+  schedule:
+    minute: '0'
+    hour: '1'
+    day_of_month: '1'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: python-function
+- name: event2
+  disabled: true
+  schedule:
+    minute: '0'
+    hour: '2'
+    day_of_month: '1'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: node-function
+
+functions:
+- name: python-function
+  handler: lambda_function.lambda_handler
+  memory: 128
+  runtime: python3.6
+  timeout: 300
+  code:
+    path: functions/python-function
+  environment_variable:
+  - key: foo
+    value: bar
+- name: node-function
+  runtime: nodejs12.x
+  handler: lambda_function.lambda_handler
+  memory: 256
+  timeout: 60
+  code:
+    path: functions/node-function