cloud-mu 3.1.5 → 3.1.6

data/modules/mu/clouds/google/container_cluster.rb

@@ -474,7 +474,6 @@ module MU
           MU.log %Q{How to interact with your GKE cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
         end
 
-
         # Locate an existing ContainerCluster or ContainerClusters and return an array containing matching GCP resource descriptors for those that match.
         # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ContainerClusters
         def self.find(**args)
@@ -747,15 +746,15 @@ module MU
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
 
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
           clusters = []
 
           # Make sure we catch regional *and* zone clusters
-          found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{region}")
+          found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{region}")
           clusters.concat(found.clusters) if found and found.clusters
           MU::Cloud::Google.listAZs(region).each { |az|
-            found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['project']}/locations/#{az}")
+            found = MU::Cloud::Google.container(credentials: credentials).list_project_location_clusters("projects/#{flags['habitat']}/locations/#{az}")
             clusters.concat(found.clusters) if found and found.clusters
           }
 
@@ -1097,7 +1096,7 @@ module MU
               }
               if !match
                 MU.log "No version matching #{cluster['kubernetes']['version']} available, will try floating minor revision", MU::WARN
-                cluster['kubernetes']['version'].sub!(/^(\d+\.\d+\.).*/i, '\1')
+                cluster['kubernetes']['version'].sub!(/^(\d+\.\d+)\..*/i, '\1')
                 master_versions.each { |v|
                   if v.match(/^#{Regexp.quote(cluster['kubernetes']['version'])}/)
                     match = true
@@ -1145,6 +1144,10 @@ module MU
           cluster['instance_type'] = MU::Cloud::Google::Server.validateInstanceType(cluster["instance_type"], cluster["region"], project: cluster['project'], credentials: cluster['credentials'])
           ok = false if cluster['instance_type'].nil?
 
+          if !MU::Master.kubectl
+            MU.log "Since I can't find a kubectl executable, you will have to handle all service account, user, and role bindings manually!", MU::WARN
+          end
+
           ok
         end
 
@@ -1236,7 +1239,7 @@ module MU
           # Take this opportunity to ensure that the 'client' service account
           # used by certificate authentication exists and has appropriate
           # privilege
-          if @username and @password
+          if @username and @password and MU::Master.kubectl
             File.open(client_binding, "w"){ |k|
               k.puts <<-EOF
 kind: ClusterRoleBinding 

data/modules/mu/clouds/google/database.rb

@@ -108,13 +108,13 @@ module MU
         # @param region [String]: The cloud provider region in which to operate
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
 
-#          instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['project'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"})
+#          instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['habitat'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"})
 #          if instances and instances.items
 #            instances.items.each { |instance|
 #              MU.log "Deleting Cloud SQL instance #{instance.name}"
-#              MU::Cloud::Google.sql(credentials: credentials).delete_instance(flags['project'], instance.name) if !noop
+#              MU::Cloud::Google.sql(credentials: credentials).delete_instance(flags['habitat'], instance.name) if !noop
 #            }
 #          end
         end

data/modules/mu/clouds/google/firewall_rule.rb

@@ -208,8 +208,8 @@ end
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -218,7 +218,7 @@ end
 
           MU::Cloud::Google.compute(credentials: credentials).delete(
             "firewall",
-            flags["project"],
+            flags["habitat"],
             nil,
             noop
           )

data/modules/mu/clouds/google/function.rb

@@ -234,10 +234,10 @@ module example.com/cloudfunction
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
           # Make sure we catch regional *and* zone functions
-          found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["project"])
+          found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["habitat"])
           found.each_pair { |cloud_id, desc|
             if (desc.description and desc.description == MU.deploy_id) or
                (desc.labels and desc.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or desc.labels["mu-master-ip"] == MU.mu_public_ip.gsub(/\./, "_"))) or

data/modules/mu/clouds/google/loadbalancer.rb

@@ -147,8 +147,8 @@ module MU
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: nil, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -159,7 +159,7 @@ module MU
             ["forwarding_rule", "region_backend_service"].each { |type|
               MU::Cloud::Google.compute(credentials: credentials).delete(
                 type,
-                flags["project"],
+                flags["habitat"],
                 region,
                 noop
               )
@@ -170,7 +170,7 @@ module MU
             ["global_forwarding_rule", "target_http_proxy", "target_https_proxy", "url_map", "backend_service", "health_check", "http_health_check", "https_health_check"].each { |type|
               MU::Cloud::Google.compute(credentials: credentials).delete(
                 type,
-                flags["project"],
+                flags["habitat"],
                 nil,
                 noop
               )

data/modules/mu/clouds/google/role.rb

@@ -731,25 +731,34 @@ module MU
                 bindings[scopetype].each_pair { |scope_id, entity_types|
                   # If we've been given a habitat filter, skip over bindings
                   # that don't match it.
-                  if scopetype == "projects" and args[:habitats] and
-                     !args[:habitats].empty? and
-                     !args[:habitats].include?(scope_id)
-                    next
+                  if scopetype == "projects"
+                    if (args[:habitats] and !args[:habitats].empty? and
+                       !args[:habitats].include?(scope_id)) or
+                       !MU::Cloud::Google.listHabitats(@credentials).include?(scope_id)
+                      next
+                    end
                   end
 
                   entity_types.each_pair { |entity_type, entities|
                     mu_entitytype = (entity_type == "serviceAccount" ? "user" : entity_type)+"s"
                     entities.each { |entity|
+                      foreign = if entity_type == "serviceAccount" and entity.match(/@(.*?)\.iam\.gserviceaccount\.com/)
+                        !MU::Cloud::Google.listHabitats(@credentials).include?(Regexp.last_match[1])
+                      end
                       entity_ref = if entity_type == "organizations"
                         { "id" => ((org == my_org.name and @config['credentials']) ? @config['credentials'] : org) }
                       elsif entity_type == "domain"
                         { "id" => entity }
                       else
-                        MU::Config::Ref.get(
-                          id: entity,
-                          cloud: "Google",
-                          type: mu_entitytype
-                        )
+                        if foreign
+                          { "id" => entity }
+                        else
+                          MU::Config::Ref.get(
+                            id: entity,
+                            cloud: "Google",
+                            type: mu_entitytype
+                          )
+                        end
                       end
                       refmap ||= {}
                       refmap[entity_ref] ||= {}

data/modules/mu/clouds/google/server.rb

@@ -1016,7 +1016,6 @@ next if !create
                 item: @config['windows_auth_vault']['item'],
                 field: @config["windows_auth_vault"]["password_field"]
               )
-MU.log "RETURNINATING FROM CACHE", MU::WARN, details: win_admin_password
               return win_admin_password if win_admin_password
             rescue MU::Groomer::MuNoSuchSecret, MU::Groomer::RunError
             end
@@ -1276,8 +1275,8 @@ MU.log "RETURNINATING FROM CACHE", MU::WARN, details: win_admin_password
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
 
 # XXX make damn sure MU.deploy_id is set
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
@@ -1288,13 +1287,12 @@ MU.log "RETURNINATING FROM CACHE", MU::WARN, details: win_admin_password
           MU::Cloud::Google.listAZs(region).each { |az|
             disks = []
             resp = MU::Cloud::Google.compute(credentials: credentials).list_instances(
-              flags["project"],
+              flags["habitat"],
               az,
               filter: filter
             )
             if !resp.items.nil? and resp.items.size > 0
               resp.items.each { |instance|
-                saname = instance.tags.items.first.gsub(/[^a-z]/, "") # XXX this nonsense again
                 MU.log "Terminating instance #{instance.name}"
                 if !instance.disks.nil? and instance.disks.size > 0
                   instance.disks.each { |disk|
@@ -1302,17 +1300,21 @@ MU.log "RETURNINATING FROM CACHE", MU::WARN, details: win_admin_password
                   }
                 end
                 MU::Cloud::Google.compute(credentials: credentials).delete_instance(
-                  flags["project"],
+                  flags["habitat"],
                   az,
                   instance.name
                 ) if !noop
-                MU.log "Removing service account #{saname}"
-                begin
-                  MU::Cloud::Google.iam(credentials: credentials).delete_project_service_account(
-                    "projects/#{flags["project"]}/serviceAccounts/#{saname}@#{flags["project"]}.iam.gserviceaccount.com"
-                  ) if !noop
-                rescue ::Google::Apis::ClientError => e
-                  raise e if !e.message.match(/^notFound: /)
+                if instance.service_accounts
+                  instance.service_accounts.each { |sa|
+                    MU.log "Removing service account #{sa.email}"
+                    begin
+                      MU::Cloud::Google.iam(credentials: credentials).delete_project_service_account(
+                        "projects/#{flags["habitat"]}/serviceAccounts/#{sa.email}"
+                      ) if !noop
+                    rescue ::Google::Apis::ClientError => e
+                      raise e if !e.message.match(/^notFound: /)
+                    end
+                  }
                 end
 # XXX wait-loop on pending?
 #                pp deletia
@@ -1325,7 +1327,7 @@ MU.log "RETURNINATING FROM CACHE", MU::WARN, details: win_admin_password
 # XXX honor snapshotting
             MU::Cloud::Google.compute(credentials: credentials).delete(
               "disk",
-              flags["project"],
+              flags["habitat"],
               az,
               noop
             ) if !noop

data/modules/mu/clouds/google/server_pool.rb

@@ -432,8 +432,8 @@ end
         # @param region [String]: The cloud provider region
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -444,7 +444,7 @@ end
             ["region_autoscaler", "region_instance_group_manager"].each { |type|
               MU::Cloud::Google.compute(credentials: credentials).delete(
                 type,
-                flags["project"],
+                flags["habitat"],
                 region,
                 noop
               )
@@ -452,7 +452,7 @@ end
           else
             MU::Cloud::Google.compute(credentials: credentials).delete(
               "instance_template",
-              flags["project"],
+              flags["habitat"],
               noop
             )
           end

data/modules/mu/clouds/google/user.rb

@@ -281,9 +281,9 @@ module MU
             end
           end
 
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
           resp = MU::Cloud::Google.iam(credentials: credentials).list_project_service_accounts(
-            "projects/"+flags["project"]
+            "projects/"+flags["habitat"]
           )
 
           if resp and resp.accounts and MU.deploy_id

data/modules/mu/clouds/google/vpc.rb

@@ -113,7 +113,7 @@ module MU
         # Describe this VPC
         # @return [Hash]
         def notify
-          base = MU.structToHash(cloud_desc)
+          base = MU.structToHash(cloud_desc, stringify_keys: true)
           base["cloud_id"] = @cloud_id
           base["project_id"] = habitat_id
           base.merge!(@config.to_h)
@@ -301,14 +301,10 @@ end
              @deploy.deployment["vpcs"][@config['name']]["subnets"] and
              @deploy.deployment["vpcs"][@config['name']]["subnets"].size > 0
             @deploy.deployment["vpcs"][@config['name']]["subnets"].each { |desc|
-              subnet = {}
-              subnet["ip_block"] = desc['ip_block']
-              subnet["name"] = desc["name"]
+              subnet = desc.clone
               subnet['mu_name'] = @config['scrub_mu_isms'] ? @cloud_id+subnet['name'].downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet['name'], max_length: 61))
-              subnet["cloud_id"] = desc['cloud_id']
               subnet["cloud_id"] ||= desc['self_link'].gsub(/.*?\/([^\/]+)$/, '\1')
               subnet["cloud_id"] ||= subnet['mu_name']
-              subnet['az'] = desc["az"]
               subnet['az'] ||= desc["region"].gsub(/.*?\/([^\/]+)$/, '\1')
               @subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet, precache_description: false)
             }
@@ -542,15 +538,15 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
           end
           MU.log "Placeholder: Google VPC artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
 
-          purge_subnets(noop, project: flags['project'], credentials: credentials)
+          purge_subnets(noop, project: flags['habitat'], credentials: credentials)
           ["route", "network"].each { |type|
 # XXX tagged routes aren't showing up in list, and the networks that own them
 # fail to delete silently
@@ -559,7 +555,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             begin
               MU::Cloud::Google.compute(credentials: credentials).delete(
                 type,
-                flags["project"],
+                flags["habitat"],
                 nil,
                 noop
               )
@@ -569,13 +565,13 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                   MU.log e.message, MU::WARN
                   if e.message.match(/Failed to delete network (.+)/)
                     network_name = Regexp.last_match[1]
-                    fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['project'], credentials: credentials)
+                    fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['habitat'], credentials: credentials)
                     fwrules.reject! { |_name, desc|
                       !desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
                     }
                     fwrules.keys.each { |name|
                       MU.log "Attempting to delete firewall rule #{name} so that VPC #{network_name} can be removed", MU::NOTICE
-                      MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['project'], name)
+                      MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['habitat'], name)
                     }
                   end
                 end
@@ -1120,7 +1116,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           # Describe this VPC Subnet
           # @return [Hash]
           def notify
-            MU.structToHash(cloud_desc)
+            MU.structToHash(cloud_desc, stringify_keys: true)
           end
 
           # Return the +self_link+ to this subnet

data/modules/mu/config.rb

@@ -77,7 +77,7 @@ module MU
       if config.is_a?(Hash)
         newhash = {}
         config.each_pair { |key, val|
-          next if remove_runtime_keys and key.match(/^#MU_/)
+          next if remove_runtime_keys and (key.nil? or key.match(/^#MU_/))
           next if val.is_a?(Array) and val.empty?
           newhash[key] = self.manxify(val, remove_runtime_keys: remove_runtime_keys)
         }

data/modules/mu/config/container_cluster.rb

@@ -104,6 +104,11 @@ module MU
           cluster["min_size"] ||= [cluster["instance_count"], cluster["min_size"]].reject { |c| c.nil? }.min
         end
 
+        if cluster['kubernetes_resources'] and !MU::Master.kubectl
+          MU.log "Cannot apply kubernetes resources without a working kubectl executable", MU::ERR
+          ok = false
+        end
+
         ok
       end
 

data/modules/mu/config/doc_helpers.rb

@@ -239,7 +239,7 @@ $CONFIGURABLES
 
             if class_hierarchy.size == 1
 
-              _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(name)
+              _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(name, false)
               if cfg_name
                 example_path = MU.myRoot+"/modules/mu/config/"+cfg_name+".yml"
                 if File.exist?(example_path)

data/modules/mu/config/ref.rb

@@ -255,7 +255,7 @@ module MU
         if @obj
           @deploy_id ||= @obj.deploy_id
           @id ||= @obj.cloud_id
-          @name ||= @obj.config['name']
+          @name ||= @obj.config['name'] if @obj.config
           return @obj
         end
 
@@ -266,6 +266,7 @@ module MU
             @mommacat ||= mommacat
             @obj.intoDeploy(@mommacat) # make real sure these are set
             @deploy_id ||= mommacat.deploy_id
+
             if !@name
               if @obj.config and @obj.config['name']
                 @name = @obj.config['name']
@@ -283,6 +284,7 @@ end
         end
 
         if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud::Google::User.cannedServiceAcctName?(@id)) and !shallow
+          try_deploy_id = @deploy_id
 
           begin
             hab_arg = if @habitat.nil?
@@ -300,22 +302,26 @@ end
               @type,
               name: @name,
               cloud_id: @id,
-              deploy_id: @deploy_id,
+              deploy_id: try_deploy_id,
               region: @region,
               habitats: hab_arg,
               credentials: @credentials,
               dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
             )
             @obj ||= found.first if found
+          rescue MU::MommaCat::MultipleMatches => e
+            if try_deploy_id.nil? and MU.deploy_id
+              MU.log "Attempting to narrow down #{@cloud} #{@type} to #{MU.deploy_id}", MU::NOTICE
+              try_deploy_id = MU.deploy_id
+              retry
+            else
+              raise e
+            end
           rescue ThreadError => e
             # Sometimes MommaCat calls us in a potential deadlock situation;
             # don't be the cause of a fatal error if so, we don't need this
             # object that badly.
             raise e if !e.message.match(/recursive locking/)
-rescue SystemExit
-# XXX this is temporary, to cope with some debug stuff that's in findStray
-# for the nonce
-return
           end
         end