cloud-mu 3.1.4 → 3.3.1

data/modules/mu/{clouds → providers}/google/server_pool.rb

@@ -89,8 +89,8 @@ module MU
             machine_type: size,
             service_accounts: [@service_acct],
             labels: labels,
-            disks: MU::Cloud::Google::Server.diskConfig(@config, false, false, credentials: @config['credentials']),
-            network_interfaces: MU::Cloud::Google::Server.interfaceConfig(@config, @vpc),
+            disks: MU::Cloud.resourceClass("Google", "Server").diskConfig(@config, false, false, credentials: @config['credentials']),
+            network_interfaces: MU::Cloud.resourceClass("Google", "Server").interfaceConfig(@config, @vpc),
             metadata: metadata,
             tags: MU::Cloud::Google.compute(:Tags).new(items: [MU::Cloud::Google.nameStr(@mu_name)])
           )
@@ -324,11 +324,11 @@ end
         def self.schema(config)
           toplevel_required = []
           schema = {
-            "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
-            "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
-            "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
-            "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
-            "network_tags" => MU::Cloud::Google::Server.schema(config)[1]["network_tags"],
+            "ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
+            "metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
+            "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
+            "scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
+            "network_tags" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["network_tags"],
             "availability_zone" => {
               "type" => "string",
               "description" => "Target a specific availability zone for this pool, which will create zonal instance managers and scalers instead of regional ones."
@@ -382,7 +382,7 @@ end
           if pool['basis']['launch_config']
             launch = pool["basis"]["launch_config"]
 
-            launch['size'] = MU::Cloud::Google::Server.validateInstanceType(launch["size"], pool["region"])
+            launch['size'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(launch["size"], pool["region"])
             ok = false if launch['size'].nil?
 
             if launch['image_id'].nil?
@@ -397,7 +397,7 @@ end
 
             real_image = nil
             begin
-              real_image = MU::Cloud::Google::Server.fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
+              real_image = MU::Cloud.resourceClass("Google", "Server").fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
             rescue ::Google::Apis::ClientError => e
               MU.log e.inspect, MU::WARN
             end
@@ -431,9 +431,9 @@ end
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -444,7 +444,7 @@ end
             ["region_autoscaler", "region_instance_group_manager"].each { |type|
               MU::Cloud::Google.compute(credentials: credentials).delete(
                 type,
-                flags["project"],
+                flags["habitat"],
                 region,
                 noop
               )
@@ -452,7 +452,7 @@ end
           else
             MU::Cloud::Google.compute(credentials: credentials).delete(
               "instance_template",
-              flags["project"],
+              flags["habitat"],
               noop
             )
           end

data/modules/mu/{clouds → providers}/google/user.rb

@@ -26,10 +26,12 @@ module MU
           # If we're being reverse-engineered from a cloud descriptor, use that
           # to determine what sort of account we are.
           if args[:from_cloud_desc]
+            @cloud_desc_cache = args[:from_cloud_desc]
             MU::Cloud::Google.admin_directory
             MU::Cloud::Google.iam
             if args[:from_cloud_desc].class == ::Google::Apis::AdminDirectoryV1::User
               @config['type'] = "interactive"
+              @cloud_id = args[:from_cloud_desc].primary_email
             elsif args[:from_cloud_desc].class == ::Google::Apis::IamV1::ServiceAccount
               @config['type'] = "service"
               @config['name'] = args[:from_cloud_desc].display_name
@@ -48,6 +50,10 @@ module MU
             @config['name']
           end
 
+          if @config['type'] == "interactive" and @config['email']
+            @cloud_id ||= @config['email']
+          end
+
         end
 
         # Called automatically by {MU::Deploy#createResources}
@@ -58,7 +64,7 @@ module MU
               account_id: acct_id,
               service_account: MU::Cloud::Google.iam(:ServiceAccount).new(
                 display_name: @mu_name,
-                description: @config['scrub_mu_isms'] ? nil : @deploy.deploy_id
+                description: @config['scrub_mu_isms'] ? @config['description'] : @deploy.deploy_id
               )
             )
             if @config['use_if_exists']
@@ -90,7 +96,7 @@ module MU
             end
           elsif @config['external']
             @cloud_id = @config['email']
-            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
           else
             if !@config['email']
               domains = MU::Cloud::Google.admin_directory(credentials: @credentials).list_domains(@customer)
@@ -122,10 +128,10 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def groom
           if @config['external']
-            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
           elsif @config['type'] == "interactive"
             need_update = false
-            MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
 
             if @config['force_password_change'] and !cloud_desc.change_password_at_next_login
               MU.log "Forcing #{@mu_name} to change their password at next login", MU::NOTICE
@@ -170,7 +176,7 @@ module MU
             end
 
           else
-            MU::Cloud::Google::Role.bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
             if @config['create_api_key']
               resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(
                 cloud_desc.name
@@ -195,6 +201,7 @@ module MU
           if @config['type'] == "interactive" or !@config['type']
              @config['type'] ||= "interactive"
             if !@config['external']
+              @cloud_id ||= @config['email']
               @cloud_desc_cache = MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
             else
               return nil
@@ -226,7 +233,7 @@ module MU
           else
             {}
           end
-          description.delete(:etag)
+          description.delete(:etag) if description
           description
         end
 
@@ -247,7 +254,7 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
           MU::Cloud::Google.getDomains(credentials)
           my_org = MU::Cloud::Google.getOrg(credentials)
 
@@ -275,15 +282,15 @@ module MU
                 next if user_email.nil?
                 next if !user_email.match(/^[^\/]+@[^\/]+$/)
 
-                MU::Cloud::Google::Role.removeBindings("user", user_email, credentials: credentials, noop: noop)
+                MU::Cloud.resourceClass("Google", "Role").removeBindings("user", user_email, credentials: credentials, noop: noop)
               }
 
             end
           end
 
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           resp = MU::Cloud::Google.iam(credentials: credentials).list_project_service_accounts(
-            "projects/"+flags["project"]
+            "projects/"+flags["habitat"]
           )
 
           if resp and resp.accounts and MU.deploy_id
@@ -356,8 +363,10 @@ module MU
           else
             if cred_cfg['masquerade_as']
               resp = MU::Cloud::Google.admin_directory(credentials: args[:credentials]).list_users(customer: MU::Cloud::Google.customerID(args[:credentials]), show_deleted: false)
+# XXX this ain't exactly performant, do some caching or something
               if resp and resp.users
                 resp.users.each { |u|
+                  next if args[:cloud_id] and !args[:cloud_id] != u.primary_email
                   found[u.primary_email] = u
                 }
               end
@@ -374,6 +383,7 @@ module MU
           name.match(/\b\d+\-compute@developer\.gserviceaccount\.com$/) or
           name.match(/\bproject-\d+@storage-transfer-service\.iam\.gserviceaccount\.com$/) or
           name.match(/\b\d+@cloudbuild\.gserviceaccount\.com$/) or
+          name.match(/\b\d+@cloudservices\.gserviceaccount\.com$/) or
           name.match(/\bservice-\d+@containerregistry\.iam\.gserviceaccount\.com$/) or
           name.match(/\bservice-\d+@container-analysis\.iam\.gserviceaccount\.com$/) or
           name.match(/\bservice-\d+@compute-system\.iam\.gserviceaccount\.com$/) or
@@ -416,7 +426,7 @@ module MU
             return nil
           end
 
-          user_roles = MU::Cloud::Google::Role.getAllBindings(@config['credentials'])["by_entity"]
+          user_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
 
           if cloud_desc.nil?
             MU.log "FAILED TO FIND CLOUD DESCRIPTOR FOR #{self}", MU::ERR, details: @config
@@ -429,6 +439,10 @@ module MU
 
           if bok['type'] == "service"
             bok['name'].gsub!(/@.*/, '')
+            if cloud_desc.description and !cloud_desc.description.empty? and
+               !cloud_desc.description.match(/^[A-Z0-9_-]+-[A-Z0-9_-]+-\d{10}-[A-Z]{2}$/)
+              bok['description'] = cloud_desc.description
+            end
             bok['project'] = @project_id
             keys = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(@cloud_id)
 
@@ -439,13 +453,13 @@ module MU
             if user_roles["serviceAccount"] and
                user_roles["serviceAccount"][bok['cloud_id']] and
                user_roles["serviceAccount"][bok['cloud_id']].size > 0
-              bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
+              bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
             end
           else
             if user_roles["user"] and
                user_roles["user"][bok['cloud_id']] and
                user_roles["user"][bok['cloud_id']].size > 0
-              bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
+              bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
             end
             bok['given_name'] = cloud_desc.name.given_name if cloud_desc.name.given_name and !cloud_desc.name.given_name.empty?
             bok['family_name'] = cloud_desc.name.family_name if cloud_desc.name.family_name and !cloud_desc.name.family_name.empty?
@@ -501,6 +515,10 @@ If we are binding (rather than creating) a user and no roles are specified, we w
               "type" => "string",
               "description" => "Alias for +family_name+"
             },
+            "description" => {
+              "type" => "string",
+              "description" => "Comment field for service accounts, which we normally use to store the originating deploy's deploy id, since GCP service accounts do not have labels. This field is only honored if +scrub_mu_isms+ is set."
+            },
             "email" => {
               "type" => "string",
               "description" => "Canonical email address for a +directory+ user. If not specified, will be set to +name@domain+."
@@ -528,7 +546,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             "roles" => {
               "type" => "array",
               "description" => "One or more Google IAM roles to associate with this user.",
-              "items" => MU::Cloud::Google::Role.ref_schema
+              "items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
             }
           }
           [toplevel_required, schema]
@@ -614,15 +632,11 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             ok = false
           end
 
-          user['dependencies'] ||= []
           if user['roles']
             user['roles'].each { |r|
               if r['role'] and r['role']['name'] and
                  (!r['role']['deploy_id'] and !r['role']['id'])
-                user['dependencies'] << {
-                  "type" => "role",
-                  "name" => r['role']['name']
-                }
+                MU::Config.addDependency(user, r['role']['name'], "role")
               end
 
               if !r["projects"] and !r["organizations"] and !r["folders"]
@@ -661,7 +675,6 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             user['roles'] = parent['roles'].dup
           end
           configurator.insertKitten(user, "users", true)
-          parent['dependencies'] ||= []
           parent['service_account'] = MU::Config::Ref.get(
             type: "users",
             cloud: "Google",
@@ -669,10 +682,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
             project: user["project"],
             credentials: user["credentials"]
           )
-          parent['dependencies'] << {
-            "type" => "user",
-            "name" => user["name"]
-          }
+          MU::Config.addDependency(parent, user['name'], "user")
 
           parent
         end

data/modules/mu/{clouds → providers}/google/vpc.rb

@@ -113,7 +113,7 @@ module MU
         # Describe this VPC
         # @return [Hash]
         def notify
-          base = MU.structToHash(cloud_desc)
+          base = MU.structToHash(cloud_desc, stringify_keys: true)
           base["cloud_id"] = @cloud_id
           base["project_id"] = habitat_id
           base.merge!(@config.to_h)
@@ -301,14 +301,10 @@ end
              @deploy.deployment["vpcs"][@config['name']]["subnets"] and
              @deploy.deployment["vpcs"][@config['name']]["subnets"].size > 0
             @deploy.deployment["vpcs"][@config['name']]["subnets"].each { |desc|
-              subnet = {}
-              subnet["ip_block"] = desc['ip_block']
-              subnet["name"] = desc["name"]
+              subnet = desc.clone
               subnet['mu_name'] = @config['scrub_mu_isms'] ? @cloud_id+subnet['name'].downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet['name'], max_length: 61))
-              subnet["cloud_id"] = desc['cloud_id']
               subnet["cloud_id"] ||= desc['self_link'].gsub(/.*?\/([^\/]+)$/, '\1')
               subnet["cloud_id"] ||= subnet['mu_name']
-              subnet['az'] = desc["az"]
               subnet['az'] ||= desc["region"].gsub(/.*?\/([^\/]+)$/, '\1')
               @subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet, precache_description: false)
             }
@@ -513,7 +509,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # @param target_subnets_key [String]: The subnet/subnets on the other side of the peered VPC.
         # @param instance_id [String]: The instance ID in the target subnet/subnets.
         # @return [Boolean]
-        def self.have_route_peered_vpc?(source_subnets_key, target_subnets_key, instance_id)
+        def self.can_route_to_master_peer?(source_subnets_key, target_subnets_key, instance_id)
         end
 
         # Retrieves the route tables of used by subnets
@@ -541,16 +537,16 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
           end
           MU.log "Placeholder: Google VPC artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
 
-          purge_subnets(noop, project: flags['project'], credentials: credentials)
+          purge_subnets(noop, project: flags['habitat'], credentials: credentials)
           ["route", "network"].each { |type|
 # XXX tagged routes aren't showing up in list, and the networks that own them
 # fail to delete silently
@@ -559,7 +555,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             begin
               MU::Cloud::Google.compute(credentials: credentials).delete(
                 type,
-                flags["project"],
+                flags["habitat"],
                 nil,
                 noop
               )
@@ -569,13 +565,13 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                   MU.log e.message, MU::WARN
                   if e.message.match(/Failed to delete network (.+)/)
                     network_name = Regexp.last_match[1]
-                    fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['project'], credentials: credentials)
+                    fwrules = MU::Cloud.resourceClass("Google", "FirewallRule").find(project: flags['habitat'], credentials: credentials)
                     fwrules.reject! { |_name, desc|
                       !desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
                     }
                     fwrules.keys.each { |name|
                       MU.log "Attempting to delete firewall rule #{name} so that VPC #{network_name} can be removed", MU::NOTICE
-                      MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['project'], name)
+                      MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['habitat'], name)
                     }
                   end
                 end
@@ -950,6 +946,41 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           createRoute(route, network: @url, tags: [MU::Cloud::Google.nameStr(server.mu_name)])
         end
 
+        # Looks at existing subnets, and attempts to find the next available
+        # IP block that's roughly similar to the ones we already have. This
+        # checks against secondary IP ranges, as well as each subnet's primary
+        # CIDR block.
+        # @param exclude [Array<String>]: One or more CIDRs to treat as unavailable, in addition to those allocated to existing subnets
+        # @return [String]
+        def getUnusedAddressBlock(exclude: [], max_bits: 28)
+          used_ranges = exclude.map { |cidr| NetAddr::IPv4Net.parse(cidr) }
+          subnets.each { |s|
+            used_ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
+            if s.cloud_desc.secondary_ip_ranges
+              used_ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
+            end
+          }
+# XXX sort used_ranges
+          candidate = used_ranges.first.next_sib
+
+          begin
+            if candidate.netmask.prefix_len > max_bits
+              candidate = candidate.resize(max_bits)
+            end
+            try_again = false
+            used_ranges.each { |cidr|
+              if !cidr.rel(candidate).nil?
+                candidate = candidate.next_sib
+                try_again = true
+                break
+              end
+            }
+            try_again = false if candidate.nil?
+          end while try_again
+
+          candidate.to_s
+        end
+
         private
 
         def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)
@@ -1120,7 +1151,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           # Describe this VPC Subnet
           # @return [Hash]
           def notify
-            MU.structToHash(cloud_desc)
+            MU.structToHash(cloud_desc, stringify_keys: true)
           end
 
           # Return the +self_link+ to this subnet

data/modules/tests/aws-jobs-functions.yaml

@@ -0,0 +1,46 @@
+# clouds: AWS
+---
+appname: smoketest
+jobs:
+- name: event1
+  schedule:
+    minute: '0'
+    hour: '1'
+    day_of_month: '1'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: python-function
+- name: event2
+  disabled: true
+  schedule:
+    minute: '0'
+    hour: '2'
+    day_of_month: '1'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: node-function
+
+functions:
+- name: python-function
+  handler: lambda_function.lambda_handler
+  memory: 128
+  runtime: python3.6
+  timeout: 300
+  code:
+    path: functions/python-function
+  environment_variable:
+  - key: foo
+    value: bar
+- name: node-function
+  runtime: nodejs12.x
+  handler: lambda_function.lambda_handler
+  memory: 256
+  timeout: 60
+  code:
+    path: functions/node-function