cloud-mu 3.1.5 → 3.1.6

data/modules/mu/cloud.rb

@@ -49,7 +49,7 @@ module MU
     generic_instance_methods = [:create, :notify, :mu_name, :cloud_id, :config]
 
     # Class methods which the base of a cloud implementation must implement
-    generic_class_methods_toplevel =  [:required_instance_methods, :myRegion, :listRegions, :listAZs, :hosted?, :hosted_config, :config_example, :writeDeploySecret, :listCredentials, :credConfig, :listInstanceTypes, :adminBucketName, :adminBucketUrl, :habitat]
+    generic_class_methods_toplevel =  [:required_instance_methods, :myRegion, :listRegions, :listAZs, :hosted?, :hosted_config, :config_example, :writeDeploySecret, :listCredentials, :credConfig, :listInstanceTypes, :adminBucketName, :adminBucketUrl, :listHabitats, :habitat, :virtual?]
 
     # Public attributes which will be available on all instantiated cloud resource objects
     #
@@ -529,7 +529,7 @@ module MU
                 images.deep_merge!(YAML.load(response))
                 break
               end
-            rescue StandardError
+            rescue StandardError => e
               if fail_hard
                 raise MuError, "Failed to fetch stock images from #{base_url}/#{cloud}.yaml (#{e.message})"
               else
@@ -644,9 +644,16 @@ module MU
 
     # Shorthand lookup for resource type names. Given any of the shorthand class name, configuration name (singular or plural), or full class name, return all four as a set.
     # @param type [String]: A string that looks like our short or full class name or singular or plural configuration names.
+    # @param assert [Boolean]: Raise an exception if the type isn't valid
     # @return [Array]: Class name (Symbol), singular config name (String), plural config name (String), full class name (Object)
-    def self.getResourceNames(type)
-      return [nil, nil, nil, nil, {}] if !type
+    def self.getResourceNames(type, assert = true)
+      if !type
+        if assert
+          raise MuError, "nil resource type requested in getResourceNames"
+        else
+          return [nil, nil, nil, nil, {}]
+        end
+      end
       @@resource_types.each_pair { |name, cloudclass|
         if name == type.to_sym or
             cloudclass[:cfg_name] == type or
@@ -656,6 +663,10 @@ module MU
           return [type.to_sym, cloudclass[:cfg_name], cloudclass[:cfg_plural], Object.const_get("MU").const_get("Cloud").const_get(name), cloudclass]
         end
       }
+      if assert
+        raise MuError, "Invalid resource type #{type} requested in getResourceNames"
+      end
+
       [nil, nil, nil, nil, {}]
     end
 
@@ -684,6 +695,14 @@ module MU
       @@supportedCloudList
     end
 
+    # Raise an exception if the cloud provider specified isn't valid
+    def self.assertSupportedCloud(cloud)
+      if cloud.nil? or !supportedClouds.include?(cloud.to_s)
+        raise MuError, "Cloud provider #{cloud} is not supported"
+      end
+      Object.const_get("MU").const_get("Cloud").const_get(cloud.to_s)
+    end
+
     # List of known/supported Cloud providers for which we have at least one
     # set of credentials configured.
     # @return [Array<String>]
@@ -701,6 +720,14 @@ module MU
       available
     end
 
+    # Raise an exception if the cloud provider specified isn't valid or we
+    # don't have any credentials configured for it.
+    def self.assertAvailableCloud(cloud)
+      if cloud.nil? or availableClouds.include?(cloud.to_s)
+        raise MuError, "Cloud provider #{cloud} is not available"
+      end
+    end
+
     # Load the container class for each cloud we know about, and inject autoload
     # code for each of its supported resource type classes.
     failed = []
@@ -823,20 +850,20 @@ module MU
       @cloud_class_cache[cloud] = {} if !@cloud_class_cache.has_key?(cloud)
       begin
         cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-        myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
-        @@resource_types[type.to_sym][:class].each { |class_method|
+        myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(shortclass)
+        @@resource_types[shortclass.to_sym][:class].each { |class_method|
           if !myclass.respond_to?(class_method) or myclass.method(class_method).owner.to_s != "#<Class:#{myclass}>"
-            raise MuError, "MU::Cloud::#{cloud}::#{type} has not implemented required class method #{class_method}"
+            raise MuError, "MU::Cloud::#{cloud}::#{shortclass} has not implemented required class method #{class_method}"
           end
         }
-        @@resource_types[type.to_sym][:instance].each { |instance_method|
+        @@resource_types[shortclass.to_sym][:instance].each { |instance_method|
           if !myclass.public_instance_methods.include?(instance_method)
-            raise MuCloudResourceNotImplemented, "MU::Cloud::#{cloud}::#{type} has not implemented required instance method #{instance_method}"
+            raise MuCloudResourceNotImplemented, "MU::Cloud::#{cloud}::#{shortclass} has not implemented required instance method #{instance_method}"
           end
         }
         cloudclass.required_instance_methods.each { |instance_method|
           if !myclass.public_instance_methods.include?(instance_method)
-            MU.log "MU::Cloud::#{cloud}::#{type} has not implemented required instance method #{instance_method}, will declare as attr_accessor", MU::DEBUG
+            MU.log "MU::Cloud::#{cloud}::#{shortclass} has not implemented required instance method #{instance_method}, will declare as attr_accessor", MU::DEBUG
           end
         }
 
@@ -844,7 +871,7 @@ module MU
         return myclass
       rescue NameError => e
         @cloud_class_cache[cloud][type] = nil
-        raise MuCloudResourceNotImplemented, "The '#{type}' resource is not supported in cloud #{cloud} (tried MU::#{cloud}::#{type})", e.backtrace
+        raise MuCloudResourceNotImplemented, "The '#{type}' resource is not supported in cloud #{cloud} (tried MU::Cloud::#{cloud}::#{shortclass})", e.backtrace
       end
     end
 
@@ -867,6 +894,8 @@ module MU
       Object.const_get("MU").const_get("Cloud").const_get(name).class_eval {
         attr_reader :cloudclass
         attr_reader :cloudobj
+        attr_reader :credentials
+        attr_reader :config
         attr_reader :destroyed
         attr_reader :delayed_save
 
@@ -921,14 +950,27 @@ module MU
         # @return [String]: Our new +deploy_id+
         def intoDeploy(mommacat, force: false)
           if force or (!@deploy)
-            MU.log "Inserting #{self} (#{self.object_id}) into #{mommacat.deploy_id}", MU::DEBUG
+            MU.log "Inserting #{self} [#{self.object_id}] into #{mommacat.deploy_id} as a #{@config['name']}", MU::DEBUG
+
             @deploy = mommacat
+            @deploy.addKitten(@cloudclass.cfg_plural, @config['name'], self)
             @deploy_id = @deploy.deploy_id
             @cloudobj.intoDeploy(mommacat, force: force) if @cloudobj
           end
           @deploy_id
         end
 
+        # Return the +virtual_name+ config field, if it is set.
+        # @param name [String]: If set, will only return a value if +virtual_name+ matches this string
+        # @return [String,nil]
+        def virtual_name(name = nil)
+          if @config and @config['virtual_name'] and
+             (!name or name == @config['virtual_name'])
+            return @config['virtual_name']
+          end
+          nil
+        end
+
         # @param mommacat [MU::MommaCat]: The deployment containing this cloud resource
         # @param mu_name [String]: Optional- specify the full Mu resource name of an existing resource to load, instead of creating a new one
         # @param cloud_id [String]: Optional- specify the cloud provider's identifier for an existing resource to load, instead of creating a new one
@@ -955,7 +997,6 @@ module MU
             if my_cloud.nil? or !MU::Cloud.supportedClouds.include?(my_cloud)
               raise MuError, "Can't instantiate a MU::Cloud object without a valid cloud (saw '#{my_cloud}')"
             end
-          
             @cloudclass = MU::Cloud.loadCloudType(my_cloud, self.class.shortname)
             @cloudparentclass = Object.const_get("MU").const_get("Cloud").const_get(my_cloud)
             @cloudobj = @cloudclass.new(
@@ -981,7 +1022,6 @@ module MU
               MU.log "#{self} in #{@deploy.deploy_id} didn't generate a mu_name after being loaded/initialized, dependencies on this resource will probably be confused!", MU::ERR, details: [caller, args.keys]
             end
 
-
           # We are actually a child object invoking this via super() from its
           # own initialize(), so initialize all the attributes and instance
           # variables we know to be universal.
@@ -2021,17 +2061,18 @@ puts "CHOOSING #{@vpc.to_s} 'cause it has #{@config['vpc']['subnet_name']}"
               loglevel = retries > 4 ? MU::NOTICE : MU::DEBUG
               MU.log "Calling WinRM on #{@mu_name}", loglevel, details: opts
               opts = {
-                endpoint: 'https://'+@mu_name+':5986/wsman',
                 retry_limit: winrm_retries,
                 no_ssl_peer_verification: true, # XXX this should not be necessary; we get 'hostname "foo" does not match the server certificate' even when it clearly does match
                 ca_trust_path: "#{MU.mySSLDir}/Mu_CA.pem",
                 transport: :ssl,
                 operation_timeout: timeout,
               }
-              if retries % 2 == 0
+              if retries % 2 == 0 # NTLM password over https
+                opts[:endpoint] = 'https://'+canonical_ip+':5986/wsman'
                 opts[:user] = @config['windows_admin_username']
                 opts[:password] = getWindowsAdminPassword
-              else
+              else # certificate auth over https
+                opts[:endpoint] = 'https://'+@mu_name+':5986/wsman'
                 opts[:client_cert] = "#{MU.mySSLDir}/#{@mu_name}-winrm.crt"
                 opts[:client_key] = "#{MU.mySSLDir}/#{@mu_name}-winrm.key"
               end

data/modules/mu/clouds/aws.rb

@@ -33,6 +33,18 @@ module MU
       module AdditionalResourceMethods
       end
 
+      # Is this a "real" cloud provider, or a stub like CloudFormation?
+      def self.virtual?
+        false
+      end
+
+      # List all AWS projects available to our credentials
+      def self.listHabitats(credentials = nil)
+        cfg = credConfig(credentials)
+        return [] if !cfg or !cfg['account_number']
+        [cfg['account_number']]
+      end
+
       # A hook that is always called just before any of the instance method of
       # our resource implementations gets invoked, so that we can ensure that
       # repetitive setup tasks (like resolving +:resource_group+ for Azure
@@ -344,6 +356,27 @@ end
       # etc)
       # @param deploy_id [MU::MommaCat]
       def self.cleanDeploy(deploy_id, credentials: nil, noop: false)
+
+        if !noop
+          MU.log "Deleting s3://#{adminBucketName(credentials)}/#{deploy_id}-secret"
+          MU::Cloud::AWS.s3(credentials: credentials).delete_object(
+            bucket: adminBucketName(credentials),
+            key: "#{deploy_id}-secret"
+          )
+          listRegions(credentials: credentials).each { |r|
+            resp = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_key_pairs(
+              filters: [{name: "key-name", values: ["deploy-#{MU.deploy_id}"]}]
+            )
+            resp.data.key_pairs.each { |keypair|
+              MU.log "Deleting key pair #{keypair.key_name} from #{r}"
+              MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_key_pair(key_name: keypair.key_name) if !noop
+            }
+          }
+
+        end
+        if hosted?
+          MU::Cloud::AWS.openFirewallForClients
+        end
       end
 
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
@@ -1394,7 +1427,7 @@ end
         # Create an AWS API client
         # @param region [String]: Amazon region so we know what endpoint to use
         # @param api [String]: Which API are we wrapping?
-        def initialize(region: MU.curRegion, api: "EC2", credentials: nil)
+        def initialize(region: nil, api: "EC2", credentials: nil)
           @cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
           @credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
 
@@ -1403,6 +1436,8 @@ end
           end
 
           params = {}
+          region ||= MU::Cloud::AWS.credConfig(credentials)['region']
+          region ||= MU.myRegion
 
           if region
             @region = region

data/modules/mu/clouds/aws/container_cluster.rb

@@ -67,16 +67,15 @@ module MU
               # soul-crushing, yet effective
               if e.message.match(/because (#{Regexp.quote(@config['region'])}[a-z]), the targeted availability zone, does not currently have sufficient capacity/)
                 bad_az = Regexp.last_match(1)
-                deletia = nil
+                deletia = []
                 mySubnets.each { |subnet|
-                  if subnet.az == bad_az
-                    deletia = subnet.cloud_id
-                    break
-                  end
+                  deletia << subnet.cloud_id if subnet.az == bad_az
+                }
+                raise e if deletia.empty?
+                MU.log "#{bad_az} does not have EKS capacity. Dropping unsupported subnets from ContainerCluster '#{@config['name']}' and retrying.", MU::NOTICE, details: deletia
+                deletia.each { |subnet|
+                  params[:resources_vpc_config][:subnet_ids].delete(subnet)
                 }
-                raise e if deletia.nil?
-                MU.log "#{bad_az} does not have EKS capacity. Dropping #{deletia} from ContainerCluster '#{@config['name']}' and retrying.", MU::NOTICE
-                params[:resources_vpc_config][:subnet_ids].delete(deletia)
               end
             }
 
@@ -1372,6 +1371,9 @@ MU.log c.name, MU::NOTICE, details: t
               "name" => cluster["name"]+"pods",
               "phase" => "groom"
             }
+            if !MU::Master.kubectl
+              MU.log "Since I can't find a kubectl executable, you will have to handle all service account, user, and role bindings manually!", MU::WARN
+            end
           end
 
           if MU::Cloud::AWS.isGovCloud?(cluster["region"]) and cluster["flavor"] == "EKS"
@@ -1470,6 +1472,11 @@ MU.log c.name, MU::NOTICE, details: t
             end
 
             if cluster["flavor"] == "EKS"
+
+              if !MU::Master.kubectl
+                MU.log "Without a kubectl executable, I cannot bind IAM roles to EKS worker nodes", MU::ERR
+                ok = false
+              end
               worker_pool["canned_iam_policies"] = [
                 "AmazonEKSWorkerNodePolicy",
                 "AmazonEKS_CNI_Policy",
@@ -1602,19 +1609,21 @@ MU.log c.name, MU::NOTICE, details: t
             raise MuError, "Failed to apply #{authmap_cmd}" if $?.exitstatus != 0
           end
 
-          admin_user_cmd = %Q{#{MU::Master.kubectl} --kubeconfig "#{kube_conf}" apply -f "#{MU.myRoot}/extras/admin-user.yaml"}
-          admin_role_cmd = %Q{#{MU::Master.kubectl} --kubeconfig "#{kube_conf}" apply -f "#{MU.myRoot}/extras/admin-role-binding.yaml"}
-          MU.log "Configuring Kubernetes admin-user and role", MU::NOTICE, details: admin_user_cmd+"\n"+admin_role_cmd
-          %x{#{admin_user_cmd}}
-          %x{#{admin_role_cmd}}
-
-          if @config['kubernetes_resources']
-            MU::Master.applyKubernetesResources(
-              @config['name'], 
-              @config['kubernetes_resources'],
-              kubeconfig: kube_conf,
-              outputdir: @deploy.deploy_dir
-            )
+          if MU::Master.kubectl
+            admin_user_cmd = %Q{#{MU::Master.kubectl} --kubeconfig "#{kube_conf}" apply -f "#{MU.myRoot}/extras/admin-user.yaml"}
+            admin_role_cmd = %Q{#{MU::Master.kubectl} --kubeconfig "#{kube_conf}" apply -f "#{MU.myRoot}/extras/admin-role-binding.yaml"}
+            MU.log "Configuring Kubernetes admin-user and role", MU::NOTICE, details: admin_user_cmd+"\n"+admin_role_cmd
+            %x{#{admin_user_cmd}}
+            %x{#{admin_role_cmd}}
+
+            if @config['kubernetes_resources']
+              MU::Master.applyKubernetesResources(
+                @config['name'], 
+                @config['kubernetes_resources'],
+                kubeconfig: kube_conf,
+                outputdir: @deploy.deploy_dir
+              )
+            end
           end
 
           MU.log %Q{How to interact with your EKS cluster\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY

data/modules/mu/clouds/aws/role.rb

@@ -1186,7 +1186,7 @@ end
                         statement["Resource"] << id+"/*"
                       end
                     else
-                      raise MuError, "Couldn't find a #{target["entity_type"]} named #{target["identifier"]} when generating IAM policy"
+                      raise MuError, "Couldn't find a #{target["type"]} named #{target["identifier"]} when generating IAM policy"
                     end
                   else
                     target["identifier"] += target["path"] if target["path"]

data/modules/mu/clouds/aws/vpc.rb

@@ -1270,7 +1270,11 @@ module MU
         def peerWith(peer)
           peer_ref = MU::Config::Ref.get(peer['vpc'])
           peer_obj = peer_ref.kitten
-          peer_id = peer_ref.cloud_id
+          peer_id = peer_ref.kitten.cloud_id
+          if peer_id == @cloud_id
+            MU.log "#{@mu_name} attempted to peer with itself (#{@cloud_id})", MU::ERR, details: peer
+            raise "#{@mu_name} attempted to peer with itself (#{@cloud_id})"
+          end
 
           if peer_obj and peer_obj.config['peers']
             peer_obj.config['peers'].each { |peerpeer|

data/modules/mu/clouds/azure.rb

@@ -47,6 +47,11 @@ module MU
         guid_chunks.join("-")
       end
 
+      # List all Azure subscriptions available to our credentials
+      def self.listHabitats(credentials = nil)
+        []
+      end
+
       # A hook that is always called just before any of the instance method of
       # our resource implementations gets invoked, so that we can ensure that
       # repetitive setup tasks (like resolving +:resource_group+ for Azure
@@ -77,6 +82,11 @@ module MU
         [:resource_group]
       end
 
+      # Is this a "real" cloud provider, or a stub like CloudFormation?
+      def self.virtual?
+        false
+      end
+
       # Stub class to represent Azure's resource identifiers, which look like:
       # /subscriptions/3d20ddd8-4652-4074-adda-0d127ef1f0e0/resourceGroups/mu/providers/Microsoft.Network/virtualNetworks/mu-vnet
       # Various API calls need chunks of this in different contexts, and this

data/modules/mu/clouds/cloudformation.rb

@@ -28,6 +28,16 @@ module MU
 
       @@cloudformation_mode = false
 
+      # Is this a "real" cloud provider, or a stub like CloudFormation?
+      def self.virtual?
+        true
+      end
+
+      # List all AWS projects available to our credentials
+      def self.listHabitats(credentials = nil)
+        MU::Cloud::AWS.listHabitats(credentials)
+      end
+
       # Return what we think of as a cloud object's habitat. In AWS, this means
       # the +account_number+ in which it's resident. If this is not applicable,
       # such as for a {Habitat} or {Folder}, returns nil.

data/modules/mu/clouds/google.rb

@@ -52,6 +52,11 @@ module MU
         [:url]
       end
 
+      # Is this a "real" cloud provider, or a stub like CloudFormation?
+      def self.virtual?
+        false
+      end
+
       # Most of our resource implementation +find+ methods have to mangle their
       # args to make sure they've extracted a project or location argument from
       # other available information. This does it for them.
@@ -337,6 +342,7 @@ module MU
       # etc)
       # @param deploy_id [MU::MommaCat]
       def self.cleanDeploy(deploy_id, credentials: nil, noop: false)
+        removeDeploySecretsAndRoles(deploy_id, noop: noop, credentials: credentials)
       end
 
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
@@ -548,7 +554,7 @@ MU.log e.message, MU::WARN, details: e.inspect
             begin
               listRegions(credentials: credentials)
               listInstanceTypes(credentials: credentials)
-              listProjects(credentials)
+              listHabitats(credentials)
             rescue ::Google::Apis::ClientError
               MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes
               @@authorizers.delete(credentials)
@@ -701,12 +707,20 @@ MU.log e.message, MU::WARN, details: e.inspect
       end
 
       # List all Google Cloud Platform projects available to our credentials
-      def self.listProjects(credentials = nil)
+      def self.listHabitats(credentials = nil)
         cfg = credConfig(credentials)
-        return [] if !cfg or !cfg['project']
+        return [] if !cfg
+        if cfg['restrict_to_habitats'] and cfg['restrict_to_habitats'].is_a?(Array)
+          cfg['restrict_to_habitats'] << cfg['project'] if cfg['project']
+          return cfg['restrict_to_habitats'].uniq
+        end
         result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
         result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" }
-        result.projects.map { |p| p.project_id }
+        allprojects = result.projects.map { |p| p.project_id }
+        if cfg['ignore_habitats'] and cfg['ignore_habitats'].is_a?(Array)
+          allprojects.reject! { |p| cfg['ignore_habitats'].include?(p) }
+        end
+        allprojects
       end
 
       @@regions = {}

data/modules/mu/clouds/google/bucket.rb

@@ -145,9 +145,9 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
 
-          resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['project'])
+          resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['habitat'])
           if resp and resp.items
             resp.items.each { |bucket|
               if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or bucket.labels['mu-master-ip'] == MU.mu_public_ip.gsub(/\./, "_"))