cloud-mu 2.0.0.pre.beta2 → 2.0.0.pre.beta3

data/modules/mu/clouds/google/bucket.rb

@@ -19,6 +19,7 @@ module MU
       class Bucket < MU::Cloud::Bucket
         @deploy = nil
         @config = nil
+        @project_id = nil
 
         attr_reader :mu_name
         attr_reader :config
@@ -30,17 +31,28 @@ module MU
           @deploy = mommacat
           @config = MU::Config.manxify(kitten_cfg)
           @cloud_id ||= cloud_id
+          if mu_name
+            @mu_name = mu_name
+            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
+            if !@project_id
+              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
+              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
+            end
+          end
           @mu_name ||= @deploy.getResourceName(@config["name"])
         end
 
         # Called automatically by {MU::Deploy#createResources}
         def create
-          MU::Cloud::Google.storage(credentials: credentials).insert_bucket(@config['project'], bucket_descriptor)
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
+          MU::Cloud::Google.storage(credentials: credentials).insert_bucket(@project_id, bucket_descriptor)
           @cloud_id = @mu_name.downcase
         end
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
+
           current = cloud_desc
           changed = false
 
@@ -63,6 +75,52 @@ module MU
           if changed
             MU::Cloud::Google.storage(credentials: credentials).patch_bucket(@cloud_id, bucket_descriptor)
           end
+
+          if @config['policies']
+            @config['policies'].each { |pol|
+              pol['grant_to'].each { |grantee|
+                entity = if grantee["type"]
+                  sibling = deploy_obj.findLitterMate(
+                    name: grantee["identifier"],
+                    type: grantee["type"]
+                  )
+                  if sibling
+                    sibling.cloudobj.cloud_id
+                  else
+                    raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating Cloud Storage access policy"
+                  end
+                else
+                  pol['grant_to'].first['identifier']
+                end
+
+                if entity.match(/@/) and !entity.match(/^(group|user)\-/)
+                  entity = "user-"+entity if entity.match(/@/)
+                end
+
+                bucket_acl_obj = MU::Cloud::Google.storage(:BucketAccessControl).new(
+                  bucket: @cloud_id,
+                  role: pol['permissions'].first,
+                  entity: entity
+                )
+                MU.log "Adding Cloud Storage policy to bucket #{@cloud_id}", MU::NOTICE, details: bucket_acl_obj
+                MU::Cloud::Google.storage(credentials: credentials).insert_bucket_access_control(
+                  @cloud_id,
+                  bucket_acl_obj
+                )
+
+                acl_obj = MU::Cloud::Google.storage(:ObjectAccessControl).new(
+                  bucket: @cloud_id,
+                  role: pol['permissions'].first,
+                  entity: entity
+                )
+                MU::Cloud::Google.storage(credentials: credentials).insert_default_object_access_control(
+                  @cloud_id,
+                  acl_obj
+                )
+              }
+            }
+
+          end
         end
 
         # Does this resource type exist as a global (cloud-wide) artifact, or
@@ -72,6 +130,12 @@ module MU
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all buckets associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
@@ -96,7 +160,9 @@ module MU
         # Return the metadata for this user cofiguration
         # @return [Hash]
         def notify
-          MU.structToHash(cloud_desc)
+          desc = MU.structToHash(cloud_desc)
+          desc["project_id"] = @project_id
+          desc
         end
 
         # Locate an existing bucket.
@@ -104,7 +170,7 @@ module MU
         # @param region [String]: The cloud provider region.
         # @param flags [Hash]: Optional flags
         # @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {}, tag_key: nil, tag_value: nil)
           found = {}
           if cloud_id
             found[cloud_id] = MU::Cloud::Google.storage(credentials: credentials).get_bucket(cloud_id)
@@ -135,6 +201,15 @@ module MU
         def self.validateConfig(bucket, configurator)
           ok = true
 
+          if bucket['policies']
+            bucket['policies'].each { |pol|
+              if !pol['permissions'] or pol['permissions'].empty?
+                pol['permissions'] = ["READER"]
+              end
+            }
+# XXX validate READER OWNER EDITOR w/e
+          end
+
           ok
         end
 

data/modules/mu/clouds/google/container_cluster.rb

@@ -38,6 +38,11 @@ module MU
             @mu_name = mu_name
             deploydata = describe[2]
             @config['availability_zone'] = deploydata['zone']
+            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
+            if !@project_id
+              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
+              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
+            end
           else
             @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 40)
           end
@@ -185,6 +190,7 @@ puts @config['credentials']
           desc = MU.structToHash(MU::Cloud::Google.container(credentials: @config['credentials']).get_zone_cluster(@config["project"], @config['availability_zone'], @mu_name.downcase))
           desc["project"] = @config['project']
           desc["cloud_id"] = @cloud_id
+          desc["project_id"] = @project_id
           desc["mu_name"] = @mu_name.downcase
           desc
         end
@@ -196,6 +202,12 @@ puts @config['credentials']
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::ALPHA
+        end
+
         # Called by {MU::Cleanup}. Locates resources that were created by the
         # currently-loaded deployment, and purges them.
         # @param noop [Boolean]: If true, will only print what would be done

data/modules/mu/clouds/google/database.rb

@@ -18,6 +18,7 @@ module MU
       # A database as configured in {MU::Config::BasketofKittens::databases}
       class Database < MU::Cloud::Database
         @deploy = nil
+        @project_id = nil
         @config = nil
         attr_reader :mu_name
         attr_reader :cloud_id
@@ -36,6 +37,11 @@ module MU
 
           if !mu_name.nil?
             @mu_name = mu_name
+            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
+            if !@project_id
+              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
+              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
+            end
           else
             @mu_name ||=
               if @config and @config['engine'] and @config["engine"].match(/^sqlserver/)
@@ -51,6 +57,7 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         # @return [String]: The cloud provider's identifier for this database instance.
         def create
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
           labels = {}
           MU::MommaCat.listStandardTags.each_pair { |name, value|
             if !value.nil?
@@ -74,7 +81,7 @@ module MU
             instance_type: "CLOUD_SQL_INSTANCE" # TODO: READ_REPLICA_INSTANCE
           )
           pp instance_desc
-          pp MU::Cloud::Google.sql(credentials: @config['credentials']).insert_instance(@config['project'], instance_desc)
+          pp MU::Cloud::Google.sql(credentials: @config['credentials']).insert_instance(@project_id, instance_desc)
         end
 
         # Locate an existing Database or Databases and return an array containing matching GCP resource descriptors for those that match.
@@ -90,6 +97,7 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
         end
 
         # Register a description of this database instance with this deployment's metadata.
@@ -111,6 +119,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::ALPHA
+        end
+
         # Called by {MU::Cleanup}. Locates resources that were created by the
         # currently-loaded deployment, and purges them.
         # @param noop [Boolean]: If true, will only print what would be done

data/modules/mu/clouds/google/firewall_rule.rb

@@ -21,6 +21,7 @@ module MU
 
         @deploy = nil
         @config = nil
+        @project_id = nil
         @admin_sgs = Hash.new
         @admin_sg_semaphore = Mutex.new
 
@@ -38,6 +39,11 @@ module MU
             @mu_name = mu_name
             # This is really a placeholder, since we "own" multiple rule sets
             @cloud_id ||= MU::Cloud::Google.nameStr(@mu_name+"-ingress-allow")
+            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
+            if !@project_id
+              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
+              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
+            end
           else
             if !@vpc.nil?
               @mu_name = @deploy.getResourceName(@config['name'], need_unique_string: true, max_length: 61)
@@ -52,6 +58,8 @@ module MU
 
         # Called by {MU::Deploy#createResources}
         def create
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
+
           vpc_id = @vpc.cloudobj.url if !@vpc.nil? and !@vpc.cloudobj.nil?
           vpc_id ||= @config['vpc']['vpc_id'] if @config['vpc'] and @config['vpc']['vpc_id']
 
@@ -109,8 +117,8 @@ module MU
           allrules.each_value { |fwdesc|
             threads << Thread.new { 
               fwobj = MU::Cloud::Google.compute(:Firewall).new(fwdesc)
-              MU.log "Creating firewall #{fwdesc[:name]} in project #{@config['project']}", details: fwobj
-              resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_firewall(@config['project'], fwobj)
+              MU.log "Creating firewall #{fwdesc[:name]} in project #{@project_id}", details: fwobj
+              resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_firewall(@project_id, fwobj)
 # XXX Check for empty (no hosts) sets
 #  MU.log "Can't create empty firewalls in Google Cloud, skipping #{@mu_name}", MU::WARN
             }
@@ -123,6 +131,7 @@ module MU
 
         # Called by {MU::Deploy#createResources}
         def groom
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
         end
 
         # Log metadata about this ruleset to the currently running deployment
@@ -132,6 +141,7 @@ module MU
           )
           sg_data ||= {}
           sg_data["group_id"] = @cloud_id
+          sg_data["project_id"] = @project_id
           sg_data["cloud_id"] = @cloud_id
 
           return sg_data
@@ -176,6 +186,12 @@ module MU
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all security groups (firewall rulesets) associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/google/folder.rb

@@ -19,6 +19,7 @@ module MU
       class Folder < MU::Cloud::Folder
         @deploy = nil
         @config = nil
+        @parent = nil
 
         attr_reader :mu_name
         attr_reader :config
@@ -30,23 +31,95 @@ module MU
           @deploy = mommacat
           @config = MU::Config.manxify(kitten_cfg)
           @cloud_id ||= cloud_id
-          @mu_name ||= @deploy.getResourceName(@config["name"])
+
+          if !mu_name.nil?
+            @mu_name = mu_name
+          elsif @config['scrub_mu_isms']
+            @mu_name = @config['name']
+          else
+            @mu_name = @deploy.getResourceName(@config['name'])
+          end
         end
 
         # Called automatically by {MU::Deploy#createResources}
         def create
 
-          name_string = @deploy.getResourceName(@config["name"], max_length: 30).downcase
+          name_string = if @config['scrub_mu_isms']
+            @config["name"]
+          else
+            @deploy.getResourceName(@config["name"], max_length: 30).downcase
+          end
 
-          folder_obj = MU::Cloud::Google.folder(:Folder).new(
-            name: name_string,
+          params = {
             display_name: name_string
-          )
-pp folder_obj
-          MU.log "Creating folder #{@mu_name}", details: folder_obj
-          resp = MU::Cloud::Google.folder(credentials: @config['credentials']).create_folder(folder_obj)
+          }
+
+          parent = MU::Cloud::Google::Folder.resolveParent(@config['parent'], credentials: @config['credentials'])
+
+          folder_obj = MU::Cloud::Google.folder(:Folder).new(params)
+
+          MU.log "Creating folder #{name_string} under #{parent}", details: folder_obj
+          resp = MU::Cloud::Google.folder(credentials: @config['credentials']).create_folder(folder_obj, parent: parent)
+
+          # Wait for list_folders output to be consistent (for the folder we
+          # just created to show up)
+          retries = 0
+          begin
+            found = MU::Cloud::Google::Folder.find(credentials: credentials, flags: { 'display_name' => name_string, 'parent_id' => parent })
+            if found.size > 0
+              @cloud_id = found.keys.first
+              @parent = found.values.first.parent
+              MU.log "Folder #{name_string} has identifier #{@cloud_id}"
+            else
+              if retries > 0 and (retries % 3) == 0
+                MU.log "Waiting for Google Cloud folder #{name_string} to appear in list_folder results...", MU::NOTICE
+              end
+              retries += 1
+              sleep 15
+            end
+          end while found.size == 0
 
-          @cloud_id = name_string.downcase
+        end
+
+        # Given a {MU::Config::Folder.reference} configuration block, resolve
+        # to a GCP resource id and type suitable for use in API calls to manage
+        # projects and folders.
+        # @param parentblock [Hash]
+        # @return [String]
+        def self.resolveParent(parentblock, credentials: nil)
+          my_org = MU::Cloud::Google.getOrg(credentials)
+          if !parentblock or parentblock['id'] == my_org.name or
+             parentblock['name'] == my_org.display_name or (parentblock['id'] and
+             "organizations/"+parentblock['id'] == my_org.name)
+            return my_org.name
+          end
+
+          if parentblock['name']
+            sib_folder = MU::MommaCat.findStray(
+              "Google",
+              "folders",
+              deploy_id: parentblock['deploy_id'],
+              credentials: credentials,
+              name: parentblock['name']
+            ).first
+            if sib_folder
+              return "folders/"+sib_folder.cloudobj.cloud_id
+            end
+          end
+
+          begin
+          found = MU::Cloud::Google::Folder.find(cloud_id: parentblock['id'], credentials: credentials, flags: { 'display_name' => parentblock['name'] })
+          rescue ::Google::Apis::ClientError => e
+            if !e.message.match(/Invalid request status_code: 404/)
+              raise e
+            end
+          end
+
+          if found and found.size > 0
+            return found.values.first.name
+          end
+
+          nil
         end
 
         # Return the cloud descriptor for the Folder
@@ -57,8 +130,9 @@ pp folder_obj
         # Return the metadata for this project's configuration
         # @return [Hash]
         def notify
-          desc = MU.structToHash(MU::Cloud::Google.folder(credentials: credentials).get_folder(@cloud_id))
+          desc = MU.structToHash(MU::Cloud::Google.folder(credentials: @config['credentials']).get_folder("folders/"+@cloud_id))
           desc["mu_name"] = @mu_name
+          desc["parent"] = @parent
           desc["cloud_id"] = @cloud_id
           desc
         end
@@ -70,24 +144,105 @@ pp folder_obj
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all Google projects associated with the currently loaded deployment. Try to, anyway.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
-        # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}, region: MU.myRegion)
+          # We can't label GCP folders, and their names are too short to encode
+          # Mu deploy IDs, so all we can do is rely on flags['known'] passed in
+          # from cleanup, which relies on our metadata to know what's ours.
+
+          if flags and flags['known']
+            flags['known'].each { |cloud_id|
+              found = self.find(cloud_id: cloud_id, credentials: credentials)
+              if found.size > 0 and found.values.first.lifecycle_state == "ACTIVE"
+                MU.log "Deleting folder #{found.values.first.display_name} (#{found.keys.first})"
+                if !noop
+                  max_retries = 10
+                  retries = 0
+                  success = false
+                  begin
+                    MU::Cloud::Google.folder(credentials: credentials).delete_folder(
+                      "folders/"+found.keys.first   
+                    )
+                    found = self.find(cloud_id: cloud_id, credentials: credentials)
+                    if found and found.size > 0 and found.values.first.lifecycle_state != "DELETE_REQUESTED"
+                      if retries < max_retries
+                        sleep 30
+                        retries += 1
+                        puts retries
+                      else
+                        MU.log "Folder #{cloud_id} still exists after #{max_retries.to_s} attempts to delete", MU::ERR
+                        break
+                      end
+                    else
+                      success = true
+                    end
+
+                  rescue ::Google::Apis::ClientError => e
+                    if e.message.match(/failedPrecondition/) and retries < max_retries
+                      sleep 30
+                      retries += 1
+                      retry
+                    else
+                      raise e
+                    end
+                  end while !success
+                end
+              end
+            }
+          end
         end
 
         # Locate an existing project
         # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
         # @param flags [Hash]: Optional flags
         # @return [OpenStruct]: The cloud provider's complete descriptions of matching project
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        def self.find(cloud_id: nil, credentials: nil, flags: {}, tag_key: nil, tag_value: nil)
           found = {}
+
+          # Recursively search a GCP folder hierarchy for a folder matching our
+          # supplied name or identifier.
+          def self.find_matching_folder(parent, name: nil, id: nil, credentials: nil)
+            resp = MU::Cloud::Google.folder(credentials: credentials).list_folders(parent: parent)
+            if resp and resp.folders
+              resp.folders.each { |f|
+                if name and name.downcase == f.display_name.downcase
+                  return f
+                elsif id and "folders/"+id== f.name
+                  return f
+                else
+                  found = self.find_matching_folder(f.name, name: name, id: id, credentials: credentials)
+                  return found if found
+                end
+              }
+            end
+            nil
+          end
+
           if cloud_id
-            found[cloud_id] = MU::Cloud::Google.folder(credentials: credentials).get_folder(cloud_id)
-          else
+            found[cloud_id.sub(/^folders\//, "")] = MU::Cloud::Google.folder(credentials: credentials).get_folder("folders/"+cloud_id.sub(/^folders\//, ""))
+          elsif flags['display_name']
+            parent = if flags['parent_id']
+              flags['parent_id']
+            else
+              my_org = MU::Cloud::Google.getOrg(credentials)
+              my_org.name
+            end
+
+            if parent
+              resp = self.find_matching_folder(parent, name: flags['display_name'], credentials: credentials)
+              if resp
+                found[resp.name.sub(/^folders\//, "")] = resp
+              end
+            end
           end
           
           found
@@ -110,6 +265,18 @@ pp folder_obj
         def self.validateConfig(folder, configurator)
           ok = true
 
+          if !MU::Cloud::Google.getOrg(folder['credentials'])
+            MU.log "Cannot manage Google Cloud projects in environments without an organization. See also: https://cloud.google.com/resource-manager/docs/creating-managing-organization", MU::ERR
+            ok = false
+          end
+
+          if folder['parent'] and folder['parent']['name'] and !folder['parent']['deploy_id'] and configurator.haveLitterMate?(folder['parent']['name'], "folders")
+            folder["dependencies"] ||= []
+            folder["dependencies"] << {
+              "type" => "folder",
+              "name" => folder['parent']['name']
+            }
+          end
 
           ok
         end