cloud-mu 3.1.6 → 3.4.0

data/modules/mu/{clouds → providers}/google/server.rb

@@ -492,7 +492,7 @@ next if !create
           return nil if @config.nil? or @deploy.nil?
 
           nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
-          if !@config["vpc"].nil? and !MU::Cloud::Google::VPC.haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
+          if !@config["vpc"].nil? and !MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
 
             if !@nat.nil?
               if @nat.cloud_desc.nil?
@@ -623,7 +623,7 @@ next if !create
           end
 
           _nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
-          if !nat_ssh_host and !MU::Cloud::Google::VPC.haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
+          if !nat_ssh_host and !MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
 # XXX check if canonical_ip is in the private ranges
 #            raise MuError, "#{node} has no NAT host configured, and I have no other route to it"
           end
@@ -992,7 +992,7 @@ next if !create
           # Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
           # which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
           # The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
-          if MU::Cloud::Google::VPC.haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
+          if MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
             @config['canonical_ip'] = private_ips.first
             return private_ips.first
           else
@@ -1001,6 +1001,22 @@ next if !create
           end
         end
 
+        # Return all of the IP addresses, public and private, from all of our
+        # network interfaces.
+        # @return [Array<String>]
+        def listIPs
+          ips = []
+          cloud_desc.network_interfaces.each { |iface|
+            ips << iface.network_ip
+            if iface.access_configs
+              iface.access_configs.each { |acfg|
+                ips << acfg.nat_ip if acfg.nat_ip
+              }
+            end
+          }
+          ips
+        end
+
         # return [String]: A password string.
         def getWindowsAdminPassword(use_cache: true)
           @config['windows_auth_vault'] ||= {
@@ -1274,9 +1290,9 @@ next if !create
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
 
 # XXX make damn sure MU.deploy_id is set
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
@@ -1340,7 +1356,7 @@ next if !create
         def self.schema(config)
           toplevel_required = []
           schema = {
-            "roles" => MU::Cloud::Google::User.schema(config)[1]["roles"],
+            "roles" => MU::Cloud.resourceClass("Google", "User").schema(config)[1]["roles"],
             "windows_admin_username" => {
               "type" => "string",
               "default" => "muadmin"
@@ -1451,8 +1467,7 @@ next if !create
             foundmatch = false
             MU::Cloud.availableClouds.each { |cloud|
               next if cloud == "Google"
-              cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-              foreign_types = (cloudbase.listInstanceTypes).values.first
+              foreign_types = (MU::Cloud.cloudClass(cloud).listInstanceTypes).values.first
               if foreign_types.size == 1
                 foreign_types = foreign_types.values.first
               end
@@ -1518,12 +1533,12 @@ next if !create
               ok = false
             end
           else
-            server = MU::Cloud::Google::User.genericServiceAccount(server, configurator)
+            server = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(server, configurator)
           end
 
           subnets = nil
           if !server['vpc']
-            vpcs = MU::Cloud::Google::VPC.find(credentials: server['credentials'])
+            vpcs = MU::Cloud.resourceClass("Google", "VPC").find(credentials: server['credentials'])
             if vpcs["default"]
               server["vpc"] ||= {}
               server["vpc"]["vpc_id"] = vpcs["default"].self_link
@@ -1538,7 +1553,7 @@ next if !create
           if !server['vpc']['subnet_id'] and server['vpc']['subnet_name'].nil?
             if !subnets
               if server["vpc"]["vpc_id"]
-                vpcs = MU::Cloud::Google::VPC.find(cloud_id: server["vpc"]["vpc_id"])
+                vpcs = MU::Cloud.resourceClass("Google", "VPC").find(cloud_id: server["vpc"]["vpc_id"])
                 subnets = vpcs["default"].subnetworks.sample
               end
             end

data/modules/mu/{clouds → providers}/google/server_pool.rb

@@ -89,8 +89,8 @@ module MU
             machine_type: size,
             service_accounts: [@service_acct],
             labels: labels,
-            disks: MU::Cloud::Google::Server.diskConfig(@config, false, false, credentials: @config['credentials']),
-            network_interfaces: MU::Cloud::Google::Server.interfaceConfig(@config, @vpc),
+            disks: MU::Cloud.resourceClass("Google", "Server").diskConfig(@config, false, false, credentials: @config['credentials']),
+            network_interfaces: MU::Cloud.resourceClass("Google", "Server").interfaceConfig(@config, @vpc),
             metadata: metadata,
             tags: MU::Cloud::Google.compute(:Tags).new(items: [MU::Cloud::Google.nameStr(@mu_name)])
           )
@@ -324,11 +324,11 @@ end
         def self.schema(config)
           toplevel_required = []
           schema = {
-            "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
-            "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
-            "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
-            "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
-            "network_tags" => MU::Cloud::Google::Server.schema(config)[1]["network_tags"],
+            "ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
+            "metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
+            "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
+            "scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
+            "network_tags" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["network_tags"],
             "availability_zone" => {
               "type" => "string",
               "description" => "Target a specific availability zone for this pool, which will create zonal instance managers and scalers instead of regional ones."
@@ -382,7 +382,7 @@ end
           if pool['basis']['launch_config']
             launch = pool["basis"]["launch_config"]
 
-            launch['size'] = MU::Cloud::Google::Server.validateInstanceType(launch["size"], pool["region"])
+            launch['size'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(launch["size"], pool["region"])
             ok = false if launch['size'].nil?
 
             if launch['image_id'].nil?
@@ -397,7 +397,7 @@ end
 
             real_image = nil
             begin
-              real_image = MU::Cloud::Google::Server.fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
+              real_image = MU::Cloud.resourceClass("Google", "Server").fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
             rescue ::Google::Apis::ClientError => e
               MU.log e.inspect, MU::WARN
             end
@@ -431,9 +431,9 @@ end
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}

data/modules/mu/{clouds → providers}/google/user.rb

@@ -26,10 +26,12 @@ module MU
           # If we're being reverse-engineered from a cloud descriptor, use that
           # to determine what sort of account we are.
           if args[:from_cloud_desc]
+            @cloud_desc_cache = args[:from_cloud_desc]
             MU::Cloud::Google.admin_directory
             MU::Cloud::Google.iam
             if args[:from_cloud_desc].class == ::Google::Apis::AdminDirectoryV1::User
               @config['type'] = "interactive"
+              @cloud_id = args[:from_cloud_desc].primary_email
             elsif args[:from_cloud_desc].class == ::Google::Apis::IamV1::ServiceAccount
               @config['type'] = "service"
               @config['name'] = args[:from_cloud_desc].display_name
@@ -48,6 +50,10 @@ module MU
             @config['name']
           end
 
+          if @config['type'] == "interactive" and @config['email']
+            @cloud_id ||= @config['email']
+          end
+
         end
 
         # Called automatically by {MU::Deploy#createResources}
@@ -58,7 +64,7 @@ module MU
               account_id: acct_id,
               service_account: MU::Cloud::Google.iam(:ServiceAccount).new(
                 display_name: @mu_name,
-                description: @config['scrub_mu_isms'] ? nil : @deploy.deploy_id
+                description: @config['scrub_mu_isms'] ? @config['description'] : @deploy.deploy_id
               )
             )
             if @config['use_if_exists']
@@ -90,7 +96,7 @@ module MU
             end
           elsif @config['external']
             @cloud_id = @config['email']
-            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
           else
             if !@config['email']
               domains = MU::Cloud::Google.admin_directory(credentials: @credentials).list_domains(@customer)
@@ -122,10 +128,10 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def groom
           if @config['external']
-            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
           elsif @config['type'] == "interactive"
             need_update = false
-            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
 
             if @config['force_password_change'] and !cloud_desc.change_password_at_next_login
               MU.log "Forcing #{@mu_name} to change their password at next login", MU::NOTICE
@@ -170,7 +176,7 @@ module MU
             end
 
           else
-            MU::Cloud::Google::Role.bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
             if @config['create_api_key']
               resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(
                 cloud_desc.name
@@ -195,6 +201,7 @@ module MU
           if @config['type'] == "interactive" or !@config['type']
              @config['type'] ||= "interactive"
             if !@config['external']
+              @cloud_id ||= @config['email']
               @cloud_desc_cache = MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
             else
               return nil
@@ -226,7 +233,7 @@ module MU
           else
             {}
           end
-          description.delete(:etag)
+          description.delete(:etag) if description
           description
         end
 
@@ -247,7 +254,7 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           MU::Cloud::Google.getDomains(credentials)
           my_org = MU::Cloud::Google.getOrg(credentials)
 
@@ -275,7 +282,7 @@ module MU
                 next if user_email.nil?
                 next if !user_email.match(/^[^\/]+@[^\/]+$/)
 
-                MU::Cloud::Google::Role.removeBindings("user", user_email, credentials: credentials, noop: noop)
+                MU::Cloud.resourceClass("Google", "Role").removeBindings("user", user_email, credentials: credentials, noop: noop)
               }
 
             end
@@ -356,8 +363,10 @@ module MU
           else
             if cred_cfg['masquerade_as']
               resp = MU::Cloud::Google.admin_directory(credentials: args[:credentials]).list_users(customer: MU::Cloud::Google.customerID(args[:credentials]), show_deleted: false)
+# XXX this ain't exactly performant, do some caching or something
               if resp and resp.users
                 resp.users.each { |u|
+                  next if args[:cloud_id] and !args[:cloud_id] != u.primary_email
                   found[u.primary_email] = u
                 }
               end
@@ -374,6 +383,7 @@ module MU
           name.match(/\b\d+\-compute@developer\.gserviceaccount\.com$/) or
           name.match(/\bproject-\d+@storage-transfer-service\.iam\.gserviceaccount\.com$/) or
           name.match(/\b\d+@cloudbuild\.gserviceaccount\.com$/) or
+          name.match(/\b\d+@cloudservices\.gserviceaccount\.com$/) or
           name.match(/\bservice-\d+@containerregistry\.iam\.gserviceaccount\.com$/) or
           name.match(/\bservice-\d+@container-analysis\.iam\.gserviceaccount\.com$/) or
           name.match(/\bservice-\d+@compute-system\.iam\.gserviceaccount\.com$/) or
@@ -416,7 +426,7 @@ module MU
             return nil
           end
 
-          user_roles = MU::Cloud::Google::Role.getAllBindings(@config['credentials'])["by_entity"]
+          user_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
 
           if cloud_desc.nil?
             MU.log "FAILED TO FIND CLOUD DESCRIPTOR FOR #{self}", MU::ERR, details: @config
@@ -429,6 +439,10 @@ module MU
 
           if bok['type'] == "service"
             bok['name'].gsub!(/@.*/, '')
+            if cloud_desc.description and !cloud_desc.description.empty? and
+               !cloud_desc.description.match(/^[A-Z0-9_-]+-[A-Z0-9_-]+-\d{10}-[A-Z]{2}$/)
+              bok['description'] = cloud_desc.description
+            end
             bok['project'] = @project_id
             keys = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(@cloud_id)
 
@@ -439,13 +453,13 @@ module MU
             if user_roles["serviceAccount"] and
                user_roles["serviceAccount"][bok['cloud_id']] and
                user_roles["serviceAccount"][bok['cloud_id']].size > 0
-              bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
+              bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
             end
           else
             if user_roles["user"] and
                user_roles["user"][bok['cloud_id']] and
                user_roles["user"][bok['cloud_id']].size > 0
-              bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
+              bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
             end
             bok['given_name'] = cloud_desc.name.given_name if cloud_desc.name.given_name and !cloud_desc.name.given_name.empty?
             bok['family_name'] = cloud_desc.name.family_name if cloud_desc.name.family_name and !cloud_desc.name.family_name.empty?
@@ -501,6 +515,10 @@ If we are binding (rather than creating) a user and no roles are specified, we w
               "type" => "string",
               "description" => "Alias for +family_name+"
             },
+            "description" => {
+              "type" => "string",
+              "description" => "Comment field for service accounts, which we normally use to store the originating deploy's deploy id, since GCP service accounts do not have labels. This field is only honored if +scrub_mu_isms+ is set."
+            },
             "email" => {
               "type" => "string",
               "description" => "Canonical email address for a +directory+ user. If not specified, will be set to +name@domain+."
@@ -528,7 +546,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             "roles" => {
               "type" => "array",
               "description" => "One or more Google IAM roles to associate with this user.",
-              "items" => MU::Cloud::Google::Role.ref_schema
+              "items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
             }
           }
           [toplevel_required, schema]
@@ -614,15 +632,11 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             ok = false
           end
 
-          user['dependencies'] ||= []
           if user['roles']
             user['roles'].each { |r|
               if r['role'] and r['role']['name'] and
                  (!r['role']['deploy_id'] and !r['role']['id'])
-                user['dependencies'] << {
-                  "type" => "role",
-                  "name" => r['role']['name']
-                }
+                MU::Config.addDependency(user, r['role']['name'], "role")
               end
 
               if !r["projects"] and !r["organizations"] and !r["folders"]
@@ -661,7 +675,6 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             user['roles'] = parent['roles'].dup
           end
           configurator.insertKitten(user, "users", true)
-          parent['dependencies'] ||= []
           parent['service_account'] = MU::Config::Ref.get(
             type: "users",
             cloud: "Google",
@@ -669,10 +682,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             project: user["project"],
             credentials: user["credentials"]
           )
-          parent['dependencies'] << {
-            "type" => "user",
-            "name" => user["name"]
-          }
+          MU::Config.addDependency(parent, user['name'], "user")
 
           parent
         end

data/modules/mu/{clouds → providers}/google/vpc.rb

@@ -537,9 +537,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -565,7 +565,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                   MU.log e.message, MU::WARN
                   if e.message.match(/Failed to delete network (.+)/)
                     network_name = Regexp.last_match[1]
-                    fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['habitat'], credentials: credentials)
+                    fwrules = MU::Cloud.resourceClass("Google", "FirewallRule").find(project: flags['habitat'], credentials: credentials)
                     fwrules.reject! { |_name, desc|
                       !desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
                     }
@@ -946,6 +946,41 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           createRoute(route, network: @url, tags: [MU::Cloud::Google.nameStr(server.mu_name)])
         end
 
+        # Looks at existing subnets, and attempts to find the next available
+        # IP block that's roughly similar to the ones we already have. This
+        # checks against secondary IP ranges, as well as each subnet's primary
+        # CIDR block.
+        # @param exclude [Array<String>]: One or more CIDRs to treat as unavailable, in addition to those allocated to existing subnets
+        # @return [String]
+        def getUnusedAddressBlock(exclude: [], max_bits: 28)
+          used_ranges = exclude.map { |cidr| NetAddr::IPv4Net.parse(cidr) }
+          subnets.each { |s|
+            used_ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
+            if s.cloud_desc.secondary_ip_ranges
+              used_ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
+            end
+          }
+# XXX sort used_ranges
+          candidate = used_ranges.first.next_sib
+
+          begin
+            if candidate.netmask.prefix_len > max_bits
+              candidate = candidate.resize(max_bits)
+            end
+            try_again = false
+            used_ranges.each { |cidr|
+              if !cidr.rel(candidate).nil?
+                candidate = candidate.next_sib
+                try_again = true
+                break
+              end
+            }
+            try_again = false if candidate.nil?
+          end while try_again
+
+          candidate.to_s
+        end
+
         private
 
         def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)

data/modules/tests/aws-jobs-functions.yaml

@@ -0,0 +1,46 @@
+# clouds: AWS
+---
+appname: smoketest
+jobs:
+- name: event1
+  schedule:
+    minute: '0'
+    hour: '1'
+    day_of_month: '1'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: python-function
+- name: event2
+  disabled: true
+  schedule:
+    minute: '0'
+    hour: '2'
+    day_of_month: '1'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: node-function
+
+functions:
+- name: python-function
+  handler: lambda_function.lambda_handler
+  memory: 128
+  runtime: python3.6
+  timeout: 300
+  code:
+    path: functions/python-function
+  environment_variable:
+  - key: foo
+    value: bar
+- name: node-function
+  runtime: nodejs12.x
+  handler: lambda_function.lambda_handler
+  memory: 256
+  timeout: 60
+  code:
+    path: functions/node-function