cloud-mu 3.1.6 → 3.2.0

data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb

@@ -129,13 +129,13 @@ module MU
 
             if launch_desc["storage"]
               launch_desc["storage"].each { |vol|
-                mapping, cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
+                mapping, cfm_mapping = MU::Cloud.resourceClass("AWS", "Server").convertBlockDeviceMapping(vol)
                 if cfm_mapping.size > 0
                   MU::Cloud::CloudFormation.setCloudFormationProp(@cfm_template[@cfm_launch_name], "BlockDeviceMappings", cfm_mapping)
                 end
               }
             end
-            MU::Cloud::AWS::Server.ephemeral_mappings.each { |mapping|
+            MU::Cloud.resourceClass("AWS", "Server.ephemeral_mappings").each { |mapping|
               MU::Cloud::CloudFormation.setCloudFormationProp(@cfm_template[@cfm_launch_name], "BlockDeviceMappings", { "DeviceName" => mapping[:device_name], "VirtualName" => mapping[:virtual_name] })
             }
 
@@ -263,7 +263,7 @@ module MU
         # @param config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
         def self.schema(config)
-          MU::Cloud::AWS::ServerPool.schema(config)
+          MU::Cloud.resourceClass("AWS", "ServerPool").schema(config)
         end
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -271,14 +271,14 @@ module MU
         # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
         def self.validateConfig(server, configurator)
-          MU::Cloud::AWS::ServerPool.validateConfig(server, configurator)
+          MU::Cloud.resourceClass("AWS", "ServerPool").validateConfig(server, configurator)
         end
 
         # Does this resource type exist as a global (cloud-wide) artifact, or
         # is it localized to a region/zone?
         # @return [Boolean]
         def self.isGlobal?
-          MU::Cloud::AWS::ServerPool.isGlobal?
+          MU::Cloud.resourceClass("AWS", "ServerPool").isGlobal?
         end
 
       end

data/modules/mu/{clouds → providers}/cloudformation/vpc.rb

@@ -301,7 +301,7 @@ module MU
         # @param config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
         def self.schema(config)
-          MU::Cloud::AWS::VPC.schema(config)
+          MU::Cloud.resourceClass("AWS", "VPC").schema(config)
         end
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -309,14 +309,14 @@ module MU
         # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
         def self.validateConfig(server, configurator)
-          MU::Cloud::AWS::VPC.validateConfig(server, configurator)
+          MU::Cloud.resourceClass("AWS", "VPC").validateConfig(server, configurator)
         end
 
         # Does this resource type exist as a global (cloud-wide) artifact, or
         # is it localized to a region/zone?
         # @return [Boolean]
         def self.isGlobal?
-          MU::Cloud::AWS::VPC.isGlobal?
+          MU::Cloud.resourceClass("AWS", "VPC").isGlobal?
         end
 
       end #class

data/modules/mu/{clouds → providers}/google.rb

@@ -236,7 +236,7 @@ module MU
       # @param sibling_only [Boolean]
       # @return [MU::Config::Habitat,nil]
       def self.projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false)
-        project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy
+        project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy if !caller.grep(/`findLitterMate'/) # XXX the dumbest
 
         if !project_obj and !sibling_only
           resp = MU::MommaCat.findStray(
@@ -706,21 +706,26 @@ MU.log e.message, MU::WARN, details: e.inspect
         nil
       end
 
+      @allprojects = []
+
       # List all Google Cloud Platform projects available to our credentials
-      def self.listHabitats(credentials = nil)
+      def self.listHabitats(credentials = nil, use_cache: true)
         cfg = credConfig(credentials)
         return [] if !cfg
         if cfg['restrict_to_habitats'] and cfg['restrict_to_habitats'].is_a?(Array)
           cfg['restrict_to_habitats'] << cfg['project'] if cfg['project']
           return cfg['restrict_to_habitats'].uniq
         end
+        if @allprojects and !@allprojects.empty? and use_cache
+          return @allprojects
+        end
         result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
         result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" }
-        allprojects = result.projects.map { |p| p.project_id }
+        @allprojects = result.projects.map { |p| p.project_id }
         if cfg['ignore_habitats'] and cfg['ignore_habitats'].is_a?(Array)
-          allprojects.reject! { |p| cfg['ignore_habitats'].include?(p) }
+          @allprojects.reject! { |p| cfg['ignore_habitats'].include?(p) }
         end
-        allprojects
+        @allprojects
       end
 
       @@regions = {}
@@ -1025,8 +1030,10 @@ MU.log e.message, MU::WARN, details: e.inspect
           "default"
         end 
 
+        with_id ||= creds['org'] if creds['org'] 
         return @@orgmap[credname] if @@orgmap.has_key?(credname)
         resp = MU::Cloud::Google.resource_manager(credentials: credname).search_organizations
+
         if resp and resp.organizations
           # XXX no idea if it's possible to be a member of multiple orgs
           if !with_id
@@ -1034,7 +1041,8 @@ MU.log e.message, MU::WARN, details: e.inspect
             return resp.organizations.first
           else
             resp.organizations.each { |org|
-              if org.name == with_id
+              if org.name == with_id or org.display_name == with_id or
+                 org.name == "organizations/#{with_id}"
                 @@orgmap[credname] = org
                 return org
               end

data/modules/mu/{clouds → providers}/google/bucket.rb

@@ -243,7 +243,7 @@ module MU
                   grantees[binding.role] << { "id" => grantee }
                 elsif grantee.match(/^serviceAccount:(.*)/)
                   sa_name = Regexp.last_match[1]
-                  if MU::Cloud::Google::User.cannedServiceAcctName?(sa_name)
+                  if MU::Cloud.resourceClass("Google", "User").cannedServiceAcctName?(sa_name)
                     grantees[binding.role] << { "id" => grantee }
                   else
                     grantees[binding.role] << MU::Config::Ref.get(

data/modules/mu/{clouds → providers}/google/container_cluster.rb

@@ -657,7 +657,7 @@ module MU
           end
 
           if bok['service_account']
-            found = MU::Cloud::Google::User.find(
+            found = MU::Cloud.resourceClass("Google", "User").find(
               credentials: bok['credentials'],
               project: bok['project'],
               cloud_id: bok['service_account']
@@ -747,7 +747,7 @@ module MU
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
 
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           clusters = []
 
           # Make sure we catch regional *and* zone clusters
@@ -817,10 +817,10 @@ module MU
               "type" => "integer",
               "description" => "The number of local SSD disks to be attached to workers. See https://cloud.google.com/compute/docs/disks/local-ssd#local_ssd_limits"
             },
-            "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
-            "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
-            "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
-            "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
+            "ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
+            "metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
+            "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
+            "scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
             "private_cluster" => {
               "description" => "Set a GKE cluster to be private, that is segregated into its own hidden VPC.",
               "type" => "object",
@@ -1021,6 +1021,25 @@ module MU
             cluster['ip_aliases'] = true
           end
 
+          # try to stake out some nice /21s for our networking config
+          if cluster['ip_aliases'] and cluster["vpc"] and cluster["vpc"]["id"]
+            habarg = if cluster["vpc"]["habitat"] and cluster["vpc"]["habitat"]["id"]
+              cluster["vpc"]["habitat"]["id"]
+            else
+              cluster["project"]
+            end
+            found = MU::MommaCat.findStray("Google", "vpcs", cloud_id: cluster["vpc"]["id"], credentials: cluster["credentials"], habitats: [habarg], dummy_ok: true)
+            if found and found.size == 1
+              myvpc = found.first
+# XXX this might not make sense with custom_subnet
+              cluster['pod_ip_block'] ||= myvpc.getUnusedAddressBlock(max_bits: 21)
+              cluster['services_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block']], max_bits: 21)
+              if cluster['tpu']
+                cluster['tpu_ip_block'] ||= myvpc.getUnusedAddressBlock(exclude: [cluster['pod_ip_block'], cluster['services_ip_block']], max_bits: 21)
+              end
+            end
+          end
+
           if cluster['service_account']
             cluster['service_account']['cloud'] = "Google"
             cluster['service_account']['habitat'] ||= MU::Config::Ref.get(
@@ -1032,11 +1051,7 @@ module MU
             if cluster['service_account']['name'] and
                !cluster['service_account']['id'] and
                !cluster['service_account']['deploy_id']
-              cluster['dependencies'] ||= []
-              cluster['dependencies'] << {
-                "type" => "user",
-                "name" => cluster['service_account']['name']
-              }
+              MU::Config.addDependency(cluster, cluster['service_account']['name'], "user")
             end
             found = MU::Config::Ref.get(cluster['service_account'])
             # XXX verify that found.kitten fails when it's supposed to
@@ -1045,7 +1060,7 @@ module MU
               ok = false
             end
           else
-            cluster = MU::Cloud::Google::User.genericServiceAccount(cluster, configurator)
+            cluster = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(cluster, configurator)
           end
 
           if cluster['dependencies']
@@ -1141,7 +1156,7 @@ module MU
             end
           end
 
-          cluster['instance_type'] = MU::Cloud::Google::Server.validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
+          cluster['instance_type'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
           ok = false if cluster['instance_type'].nil?
 
           if !MU::Master.kubectl

data/modules/mu/{clouds → providers}/google/database.rb

@@ -25,14 +25,7 @@ module MU
           @config["groomer"] = MU::Config.defaultGroomer unless @config["groomer"]
           @groomclass = MU::Groomer.loadGroomer(@config["groomer"])
 
-          @mu_name ||=
-            if @config and @config['engine'] and @config["engine"].match(/^sqlserver/)
-              @deploy.getResourceName(@config["name"], max_length: 15)
-            else
-              @deploy.getResourceName(@config["name"], max_length: 63)
-            end
-
-          @mu_name.gsub(/(--|-$)/i, "").gsub(/(_)/, "-").gsub!(/^[^a-z]/i, "")
+          @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 63)
         end
 
         # Called automatically by {MU::Deploy#createResources}

data/modules/mu/{clouds → providers}/google/firewall_rule.rb

@@ -209,7 +209,7 @@ end
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -440,7 +440,7 @@ end
             elsif acl['vpc']['habitat'] and acl['vpc']['habitat']['name']
               acl['vpc']['project'] = acl['vpc']['habitat']['name']
             end
-            correct_vpc = MU::Cloud::Google::VPC.pickVPC(
+            correct_vpc = MU::Cloud.resourceClass("Google", "VPC").pickVPC(
               acl['vpc'],
               acl,
               "firewall_rule",

data/modules/mu/{clouds → providers}/google/folder.rb

@@ -138,7 +138,7 @@ module MU
         # Return the metadata for this folders's configuration
         # @return [Hash]
         def notify
-          desc = MU.structToHash(MU::Cloud::Google.folder(credentials: @config['credentials']).get_folder("folders/"+@cloud_id))
+          desc = MU.structToHash(cloud_desc)
           desc["mu_name"] = @mu_name
           desc["parent"] = @parent
           desc["cloud_id"] = @cloud_id
@@ -236,10 +236,10 @@ module MU
         # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
         def self.find(**args)
           found = {}
-
           # Recursively search a GCP folder hierarchy for a folder matching our
           # supplied name or identifier.
           def self.find_matching_folder(parent, name: nil, id: nil, credentials: nil)
+
             resp = MU::Cloud::Google.folder(credentials: credentials).list_folders(parent: parent)
             if resp and resp.folders
               resp.folders.each { |f|
@@ -278,6 +278,7 @@ module MU
             end
           else
             resp = MU::Cloud::Google.folder(credentials: args[:credentials]).list_folders(parent: parent)
+
             if resp and resp.folders
               resp.folders.each { |folder|
                 next if folder.lifecycle_state == "DELETE_REQUESTED"
@@ -310,7 +311,6 @@ module MU
           bok['cloud_id'] = cloud_desc.name
           bok['name'] = cloud_desc.display_name#+bok['cloud_id'] # only way to guarantee uniqueness
           if cloud_desc.parent.match(/^folders\/(.*)/)
-MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_desc.parent
             bok['parent'] = MU::Config::Ref.get(
               id: cloud_desc.parent,
               cloud: "Google",
@@ -355,11 +355,7 @@ MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_d
           end
 
           if folder['parent'] and folder['parent']['name'] and !folder['parent']['deploy_id'] and configurator.haveLitterMate?(folder['parent']['name'], "folders")
-            folder["dependencies"] ||= []
-            folder["dependencies"] << {
-              "type" => "folder",
-              "name" => folder['parent']['name']
-            }
+            MU::Config.addDependency(folder, folder['parent']['name'], "folder")
           end
 
           ok

data/modules/mu/{clouds → providers}/google/function.rb

@@ -235,7 +235,7 @@ module example.com/cloudfunction
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           # Make sure we catch regional *and* zone functions
           found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["habitat"])
           found.each_pair { |cloud_id, desc|
@@ -373,7 +373,7 @@ module example.com/cloudfunction
                 }
               }
             },
-            "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
+            "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
             "runtime" => {
               "type" => "string",
               "enum" => %w{nodejs go python nodejs8 nodejs10 python37 go111 go113},
@@ -524,7 +524,7 @@ module example.com/cloudfunction
               ok = false
             end
           else
-            function = MU::Cloud::Google::User.genericServiceAccount(function, configurator)
+            function = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(function, configurator)
           end
 
 #          siblings = configurator.haveLitterMate?(nil, "vpcs", has_multiple: true)

data/modules/mu/{clouds → providers}/google/group.rb

@@ -44,7 +44,7 @@ module MU
             resp = MU::Cloud::Google.admin_directory(credentials: @credentials).insert_group(group_obj)
             @cloud_id = resp.email
 
-            MU::Cloud::Google::Role.bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'])
+            MU::Cloud.resourceClass("Google", "Role").bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'])
           else
             @cloud_id = @config['name'].sub(/@.*/, "")+"@"+@config['domain']
           end
@@ -52,7 +52,7 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
-          MU::Cloud::Google::Role.bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'], debug: true)
+          MU::Cloud.resourceClass("Google", "Role").bindFromConfig("group", @cloud_id, @config['roles'], credentials: @config['credentials'], debug: true)
 
           if @config['members']
             resolved_desired = []
@@ -166,7 +166,7 @@ module MU
 
           if flags['known']
             flags['known'].each { |group|
-              MU::Cloud::Google::Role.removeBindings("group", group, credentials: credentials, noop: noop)
+              MU::Cloud.resourceClass("Google", "Role").removeBindings("group", group, credentials: credentials, noop: noop)
             }
           end
         end
@@ -222,10 +222,10 @@ module MU
 #              type: "users"
 #            )
 #          }
-          group_roles = MU::Cloud::Google::Role.getAllBindings(@config['credentials'])["by_entity"]
+          group_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
           if group_roles["group"] and group_roles["group"][bok['cloud_id']] and
              group_roles["group"][bok['cloud_id']].size > 0
-            bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(group_roles["group"][bok['cloud_id']], credentials: @config['credentials'])
+            bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(group_roles["group"][bok['cloud_id']], credentials: @config['credentials'])
           end
 
           bok
@@ -264,7 +264,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
 
             "roles" => {
               "type" => "array",
-              "items" => MU::Cloud::Google::Role.ref_schema
+              "items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
             }
           }
           [toplevel_required, schema]
@@ -340,11 +340,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
           if group['members']
             group['members'].each { |m|
               if configurator.haveLitterMate?(m, "users")
-                group['dependencies'] ||= []
-                group['dependencies'] << {
-                  "name" => m,
-                  "type" => "user"
-                }
+                MU::Config.addDependency(group, m, "user")
               end
             }
           end
@@ -353,11 +349,7 @@ If we are binding (rather than creating) a group and no roles are specified, we
             group['roles'].each { |r|
               if r['role'] and r['role']['name'] and
                  (!r['role']['deploy_id'] and !r['role']['id'])
-                group['dependencies'] ||= []
-                group['dependencies'] << {
-                  "type" => "role",
-                  "name" => r['role']['name']
-                }
+                MU::Config.addDependency(group, r['role']['name'], "role")
               end
             }
           end

data/modules/mu/{clouds → providers}/google/habitat.rb

@@ -61,7 +61,7 @@ module MU
           if @config['parent']['name'] and !@config['parent']['id']
             @config['parent']['deploy_id'] = @deploy.deploy_id
           end
-          parent = MU::Cloud::Google::Folder.resolveParent(@config['parent'], credentials: @config['credentials'])
+          parent = MU::Cloud.resourceClass("Google", "Folder").resolveParent(@config['parent'], credentials: @config['credentials'])
           if !parent
             MU.log "Unable to resolve parent resource of Google Project #{@config['name']}", MU::ERR, details: @config['parent']
             raise "Unable to resolve parent resource of Google Project #{@config['name']}"
@@ -285,7 +285,7 @@ module MU
               next if p.lifecycle_state == "DELETE_REQUESTED"
               found[p.project_id] = p
             }
-            @@list_projects_cache = found
+            @@list_projects_cache = found.clone
           end
 
           found
@@ -376,11 +376,7 @@ module MU
           end
 
           if habitat['parent'] and habitat['parent']['name'] and !habitat['parent']['deploy_id'] and configurator.haveLitterMate?(habitat['parent']['name'], "folders")
-            habitat["dependencies"] ||= []
-            habitat["dependencies"] << {
-              "type" => "folder",
-              "name" => habitat['parent']['name']
-            }
+            MU::Config.addDependency(habitat, habitat['parent']['name'], "folder")
           end
 
           ok

data/modules/mu/{clouds → providers}/google/loadbalancer.rb

@@ -148,7 +148,7 @@ module MU
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: nil, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}