cloud-mu 2.0.0.pre.beta2 → 2.0.0.pre.beta3

data/modules/mu/clouds/google/group.rb

@@ -56,6 +56,12 @@ module MU
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::ALPHA
+        end
+
         # Remove all groups associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
@@ -69,7 +75,7 @@ module MU
         # @param region [String]: The cloud provider region.
         # @param flags [Hash]: Optional flags
         # @return [OpenStruct]: The cloud provider's complete descriptions of matching group group.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {}, tag_key: nil, tag_value: nil)
           flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
           found = nil
           found

data/modules/mu/clouds/google/habitat.rb

@@ -30,36 +30,112 @@ module MU
           @deploy = mommacat
           @config = MU::Config.manxify(kitten_cfg)
           @cloud_id ||= cloud_id
-          @mu_name ||= @deploy.getResourceName(@config["name"])
+
+          if !mu_name.nil?
+            @mu_name = mu_name
+          elsif @config['scrub_mu_isms']
+            @mu_name = @config['name']
+          else
+            @mu_name = @deploy.getResourceName(@config['name'])
+          end
         end
 
         # Called automatically by {MU::Deploy#createResources}
         def create
           labels = {}
 
-          name_string = @deploy.getResourceName(@config["name"], max_length: 30).downcase
+          name_string = if @config['scrub_mu_isms']
+            @config["name"]
+          else
+            @deploy.getResourceName(@config["name"], max_length: 30).downcase
+          end
+
+          params = {
+            name: name_string,
+            project_id: name_string,
+          }
 
           MU::MommaCat.listStandardTags.each_pair { |name, value|
             if !value.nil?
               labels[name.downcase] = value.downcase.gsub(/[^a-z0-9\-\_]/i, "_")
             end
           }
+          
+          if !@config['scrub_mu_isms']
+            params[:labels] = labels
+          end
 
-          desc = {
-            name: name_string,
-            project_id: name_string,
-            labels: labels
-          }
-          if @config['folder'] and @config['folder']['id']
-            desc["parent"] = @config['folder']['id']
+          parent = MU::Cloud::Google::Folder.resolveParent(@config['parent'], credentials: @config['credentials'])
+          if !parent
+            MU.log "Unable to resolve parent resource of Google Project #{@config['name']}", MU::ERR, details: @config['parent']
+            raise "Unable to resolve parent resource of Google Project #{@config['name']}"
           end
 
-          project_obj = MU::Cloud::Google.resource_manager(:Project).new(desc)
-pp project_obj
-          MU.log "Creating project #{@mu_name}", details: project_obj
-          resp = MU::Cloud::Google.resource_manager(credentials: @config['credentials']).create_project(project_obj)
+          type, parent_id = parent.split(/\//)
+          params[:parent] = MU::Cloud::Google.resource_manager(:ResourceId).new(
+            id: parent_id,
+            type: type.sub(/s$/, "") # I wish these engineering teams would talk to each other
+          )
+
+          project_obj = MU::Cloud::Google.resource_manager(:Project).new(params)
+
+          MU.log "Creating project #{name_string} under #{parent}", details: project_obj
+          MU::Cloud::Google.resource_manager(credentials: @config['credentials']).create_project(project_obj)
+
+
+          found = false
+          retries = 0
+          begin
+            resp = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
+            if resp and resp.projects
+              resp.projects.each { |p|
+                if p.name == name_string.downcase
+                  found = true
+                end
+              }
+            end
+            if !found
+              if retries > 0 and (retries % 3) == 0
+                MU.log "Waiting for Google Cloud project #{name_string} to appear in list_projects results...", MU::NOTICE
+              end
+              retries += 1
+              sleep 15
+            end
+          end while !found
+
 
           @cloud_id = name_string.downcase
+          setProjectBilling
+        end
+
+        # Called automatically by {MU::Deploy#createResources}
+        def groom
+          setProjectBilling
+        end
+
+        # Associate a billing account with this project. If none is specified in
+        # our configuration, use the billing account tied the the default
+        # project of our credential set.
+        def setProjectBilling
+          cur_billing = MU::Cloud::Google.billing(credentials: @config['credentials']).get_project_billing_info("projects/"+@cloud_id)
+
+          if !cur_billing or
+             cur_billing.billing_account_name != "billingAccounts/"+@config['billing_acct'] or
+             !cur_billing.billing_enabled
+
+            billing_obj = MU::Cloud::Google.billing(:ProjectBillingInfo).new(
+              billing_account_name: "billingAccounts/"+@config['billing_acct'],
+              billing_enabled: true,
+              name: "projects/"+@cloud_id+"/billingInfo",
+              project_id: @cloud_id
+            )
+            MU.log "Associating project #{@cloud_id} with billing account #{@config['billing_acct']}"
+            MU::Cloud::Google.billing(credentials: credentials).update_project_billing_info(
+              "projects/"+@cloud_id,
+              billing_obj
+            )
+
+          end
         end
 
         # Return the cloud descriptor for the Habitat
@@ -70,12 +146,7 @@ pp project_obj
         # Return the metadata for this project's configuration
         # @return [Hash]
         def notify
-          desc = MU.structToHash(MU::Cloud::Google.resource_manager(credentials: credentials).list_projects(
-              filter: "name:#{cloud_id}"
-            ).projects.first)
-          desc["mu_name"] = @mu_name
-          desc["cloud_id"] = @cloud_id
-          desc
+          MU.structToHash(MU::Cloud::Google.resource_manager(credentials: @config['credentials']).get_project(@cloud_id))
         end
 
         # Does this resource type exist as a global (cloud-wide) artifact, or
@@ -85,12 +156,38 @@ pp project_obj
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all Google projects associated with the currently loaded deployment. Try to, anyway.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+          resp = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
+          if resp and resp.projects
+            resp.projects.each { |p|
+              if p.labels and p.labels["mu-id"] == MU.deploy_id.downcase and
+                 p.lifecycle_state == "ACTIVE"
+                MU.log "Deleting project #{p.name}", details: p
+                if !noop
+                  begin
+                    MU::Cloud::Google.resource_manager(credentials: credentials).delete_project(p.name)
+                  rescue ::Google::Apis::ClientError => e
+                    if e.message.match(/Cannot delete an inactive project/)
+                      # this is fine
+                    else
+                      raise e
+                    end
+                  end
+                end
+              end
+            }
+          end
         end
 
         # Locate an existing project
@@ -98,13 +195,13 @@ pp project_obj
         # @param region [String]: The cloud provider region.
         # @param flags [Hash]: Optional flags
         # @return [OpenStruct]: The cloud provider's complete descriptions of matching project
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {}, tag_key: nil, tag_value: nil)
           found = {}
           if cloud_id
             resp = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects(
               filter: "name:#{cloud_id}"
-            ).projects.first
-            found[resp.name] = resp
+            )
+            found[resp.name] = resp.projects.first if resp and resp.projects
           else
             resp = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects().projects
             resp.each { |p|
@@ -121,6 +218,10 @@ pp project_obj
         def self.schema(config)
           toplevel_required = []
           schema = {
+            "billing_acct" => {
+              "type" => "string",
+              "description" => "Billing account ID to associate with a newly-created Google Project. If not specified, will attempt to locate a billing account associated with the default project for our credentials."
+            }
           }
           [toplevel_required, schema]
         end
@@ -132,11 +233,25 @@ pp project_obj
         def self.validateConfig(habitat, configurator)
           ok = true
 
-          if habitat['folder'] and habitat['folder']['name'] and !habitat['folder']['deploy_id']
+          if !MU::Cloud::Google.getOrg(habitat['credentials'])
+            MU.log "Cannot manage Google Cloud projects in environments without an organization. See also: https://cloud.google.com/resource-manager/docs/creating-managing-organization", MU::ERR
+            ok = false
+          end
+
+          if !habitat['billing_acct']
+            default_billing = MU::Cloud::Google.billing(credentials: habitat['credentials']).get_project_billing_info("projects/"+MU::Cloud::Google.defaultProject(habitat['credentials']))
+            if !default_billing or !default_billing.billing_account_name
+              MU.log "Google project #{habitat['name']} does not specify 'billing_acct' and I'm unable to locate a default", MU::ERR
+              ok = false
+            end
+            habitat['billing_acct'] = default_billing.billing_account_name.sub(/^billingAccounts\//, "")
+          end
+
+          if habitat['parent'] and habitat['parent']['name'] and !habitat['parent']['deploy_id'] and configurator.haveLitterMate?(habitat['parent']['name'], "folders")
             habitat["dependencies"] ||= []
             habitat["dependencies"] << {
               "type" => "folder",
-              "name" => habitat['folder']['name']
+              "name" => habitat['parent']['name']
             }
           end
 

data/modules/mu/clouds/google/loadbalancer.rb

@@ -18,6 +18,7 @@ module MU
       # A load balancer as configured in {MU::Config::BasketofKittens::loadbalancers}
       class LoadBalancer < MU::Cloud::LoadBalancer
 
+        @project_id = nil
         @deploy = nil
         @lb = nil
         attr_reader :mu_name
@@ -36,6 +37,11 @@ module MU
           @cloud_id ||= cloud_id
           if !mu_name.nil?
             @mu_name = mu_name
+            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
+            if !@project_id
+              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
+              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
+            end
           elsif @config['scrub_mu_isms']
             @mu_name = @config['name']
           else
@@ -45,6 +51,7 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def create
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
 
           parent_thread_id = Thread.current.object_id
 
@@ -96,13 +103,13 @@ module MU
             if @config['global']
               MU.log "Creating Global Forwarding Rule #{@mu_name}", MU::NOTICE, details: ruleobj
               resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_global_forwarding_rule(
-                @config['project'],
+                @project_id,
                 ruleobj
               )
             else
               MU.log "Creating regional Forwarding Rule #{@mu_name} in #{@config['region']}", MU::NOTICE, details: ruleobj
               resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_forwarding_rule(
-                @config['project'],
+                @project_id,
                 @config['region'],
                 ruleobj
               )
@@ -120,12 +127,12 @@ module MU
         def notify
           rules = {}
           resp = MU::Cloud::Google.compute(credentials: @config['credentials']).list_global_forwarding_rules(
-            @config["project"],
+            @project_id,
             filter: "description eq #{@deploy.deploy_id}"
           )
           if resp.nil? or resp.items.nil? or resp.items.size == 0
             resp = MU::Cloud::Google.compute(credentials: @config['credentials']).list_forwarding_rules(
-              @config["project"],
+              @project_id,
               @config['region'],
               filter: "description eq #{@deploy.deploy_id}"
             )
@@ -136,6 +143,7 @@ module MU
               rules[rule.name].delete(:label_fingerprint)
             }
           end
+          rules["project_id"] = @project_id
 
           rules
         end
@@ -154,6 +162,12 @@ module MU
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all load balancers associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
@@ -321,7 +335,7 @@ module MU
           )
           MU.log "Creating url map #{tg['name']}", details: urlmap_obj
           urlmap = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_url_map(
-            @config['project'],
+            @project_id,
             urlmap_obj
           )
 
@@ -335,7 +349,7 @@ module MU
             target_obj = MU::Cloud::Google.compute(:TargetHttpProxy).new(desc)
             MU.log "Creating http target proxy #{tg['name']}", details: target_obj
             MU::Cloud::Google.compute(credentials: @config['credentials']).insert_target_http_proxy(
-              @config['project'],
+              @project_id,
               target_obj
             )
           else
@@ -348,7 +362,7 @@ module MU
             target_obj = MU::Cloud::Google.compute(:TargetHttpsProxy).new(desc)
             MU.log "Creating https target proxy #{tg['name']}", details: target_obj
             MU::Cloud::Google.compute(credentials: @config['credentials']).insert_target_https_proxy(
-              @config['project'],
+              @project_id,
               target_obj
             )
           end
@@ -394,13 +408,13 @@ module MU
           MU.log "Creating backend service #{MU::Cloud::Google.nameStr(@deploy.getResourceName(tg["name"]))}", details: backend_obj
           if @config['private'] and !@config['global']
             return MU::Cloud::Google.compute(credentials: @config['credentials']).insert_region_backend_service(
-              @config['project'],
+              @project_id,
               @config['region'],
               backend_obj
             )
           else
             return MU::Cloud::Google.compute(credentials: @config['credentials']).insert_backend_service(
-              @config['project'],
+              @project_id,
               backend_obj
             )
           end
@@ -430,12 +444,12 @@ module MU
             MU.log "Creating #{proto} health check #{name}", details: hc_obj
             if proto == "HTTP"
               return MU::Cloud::Google.compute(credentials: @config['credentials']).insert_http_health_check(
-                @config['project'],
+                @project_id,
                 hc_obj
               )
             else
               return MU::Cloud::Google.compute(credentials: @config['credentials']).insert_https_health_check(
-                @config['project'],
+                @project_id,
                 hc_obj
               )
             end
@@ -475,7 +489,7 @@ module MU
             hc_obj = MU::Cloud::Google.compute(:HealthCheck).new(desc)
             MU.log "INSERTING HEALTH CHECK", MU::NOTICE, details: hc_obj
             return MU::Cloud::Google.compute(credentials: @config['credentials']).insert_health_check(
-              @config['project'],
+              @project_id,
               hc_obj
             )
           end

data/modules/mu/clouds/google/server.rb

@@ -28,6 +28,7 @@ module MU
       # Google Cloud, this amounts to a single Instance in an Unmanaged
       # Instance Group.
       class Server < MU::Cloud::Server
+        @project_id = nil
 
         attr_reader :mu_name
         attr_reader :config
@@ -69,6 +70,11 @@ module MU
             @config['mu_name'] = @mu_name
             # describe
             @mu_windows_name = @deploydata['mu_windows_name'] if @mu_windows_name.nil? and @deploydata
+            @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
+            if !@project_id
+              project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
+              @project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
+            end
           else
             if kitten_cfg.has_key?("basis")
               @mu_name = @deploy.getResourceName(@config['name'], need_unique_string: true)
@@ -243,11 +249,12 @@ next if !create
 
         # Called automatically by {MU::Deploy#createResources}
         def create
+          @project_id = MU::Cloud::Google.projectLookup(@config['project_id'], @deploy).cloudobj.cloud_id
 
           service_acct = MU::Cloud::Google::Server.createServiceAccount(
             @mu_name.downcase,
             @deploy,
-            project: @config['project'],
+            project: @project_id,
             credentials: @config['credentials']
           )
           MU::Cloud::Google.grantDeploySecretAccess(service_acct.email, credentials: @config['credentials'])
@@ -301,7 +308,7 @@ next if !create
             MU.log "Creating instance #{@mu_name}"
             begin
               instance = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_instance(
-                @config['project'],
+                @project_id,
                 @config['availability_zone'],
                 instanceobj
               )
@@ -379,7 +386,7 @@ next if !create
         def stop
           MU.log "Stopping #{@cloud_id}"
           MU::Cloud::Google.compute(credentials: @config['credentials']).stop_instance(
-            @config['project'],
+            @project_id,
             @config['availability_zone'],
             @cloud_id
           )
@@ -392,7 +399,7 @@ next if !create
         def start
           MU.log "Starting #{@cloud_id}"
           MU::Cloud::Google.compute(credentials: @config['credentials']).start_instance(
-            @config['project'],
+            @project_id,
             @config['availability_zone'],
             @cloud_id
           )
@@ -687,6 +694,7 @@ next if !create
               "image_created" => @config['image_created'],
 #              "iam_role" => @config['iam_role'],
               "cloud_desc_id" => @cloud_id,
+              "project_id" => @project_id,
               "private_ip_address" => private_ips.first,
               "public_ip_address" => public_ips.first,
               "private_ip_list" => private_ips,
@@ -714,6 +722,7 @@ next if !create
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
+          @project_id = MU::Cloud::Google.projectLookup(@config['project_id'], @deploy).cloudobj.cloud_id
 
           MU::MommaCat.lock(@cloud_id+"-groom")
           
@@ -791,7 +800,7 @@ next if !create
                 region: @config['region'],
                 storage: @config['storage'],
                 family: ("mu-"+@config['platform']+"-"+MU.environment).downcase,
-                project: @config['project'],
+                project: @project_id,
                 exclude_storage: img_cfg['image_exclude_storage'],
                 make_public: img_cfg['public'],
                 tags: @config['tags'],
@@ -803,7 +812,7 @@ next if !create
             if img_cfg['image_then_destroy']
               MU.log "Image #{image_id} ready, removing source node #{node}"
               MU::Cloud::Google.compute(credentials: @config['credentials']).delete_instance(
-                @config['project'],
+                @project_id,
                 @config['availability_zone'],
                 @cloud_id
               )
@@ -901,7 +910,7 @@ next if !create
 #          if !@cloud_id.nil?
 #            begin
 #              return MU::Cloud::Google.compute(credentials: @config['credentials']).get_instance(
-#                @config['project'],
+#                @project_id,
 #                @config['availability_zone'],
 #                @cloud_id
 #              )
@@ -975,14 +984,14 @@ next if !create
             description: description,
             zone: @config['availability_zone'],
 #            type: "projects/#{config['project']}/zones/#{config['availability_zone']}/diskTypes/pd-ssd",
-            type: "projects/#{@config['project']}/zones/#{@config['availability_zone']}/diskTypes/pd-standard",
+            type: "projects/#{@project_id}/zones/#{@config['availability_zone']}/diskTypes/pd-standard",
 # Other values include pd-ssd and local-ssd
             name: resname
           )
 
           begin
             newdisk = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_disk(
-              @config['project'],
+              @project_id,
               @config['availability_zone'],
               newdiskobj
             )
@@ -1002,7 +1011,7 @@ next if !create
             type: "PERSISTENT"
           )
           attachment = MU::Cloud::Google.compute(credentials: @config['credentials']).attach_disk(
-            @config['project'],
+            @project_id,
             @config['availability_zone'],
             @cloud_id,
             attachobj
@@ -1024,6 +1033,12 @@ next if !create
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all instances associated with the currently loaded deployment. Also cleans up associated volumes, droppings in the MU master's /etc/hosts and ~/.ssh, and in whatever Groomer was used.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server