cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

data/modules/mu/cloud.rb

@@ -39,8 +39,12 @@ module MU
     class MuCloudFlagNotImplemented < StandardError;
     end
 
-    generic_class_methods = [:find, :cleanup, :validateConfig, :schema]
-    generic_instance_methods = [:create, :notify, :mu_name, :cloud_id, :config, :cloud_desc]
+    # Methods which a cloud resource implementation, e.g. Server, must implement
+    generic_class_methods = [:find, :cleanup, :validateConfig, :schema, :isGlobal?]
+    generic_instance_methods = [:create, :notify, :mu_name, :cloud_id, :config]
+
+    # Class methods which the base of a cloud implementation must implement
+    generic_class_methods_toplevel =  [:required_instance_methods, :myRegion, :listRegions, :listAZs, :hosted?, :hosted_config, :config_example, :writeDeploySecret, :listCredentials, :credConfig, :listInstanceTypes, :adminBucketName, :adminBucketUrl]
 
     # Initialize empty classes for each of these. We'll fill them with code
     # later; we're doing this here because otherwise the parser yells about
@@ -80,7 +84,7 @@ module MU
     class Alarm;
     end
     # Stub base class; real implementations generated at runtime
-    class Notification;
+    class Notifier;
     end
     # Stub base class; real implementations generated at runtime
     class Log;
@@ -98,7 +102,7 @@ module MU
     class MsgQueue;
     end
     # Stub base class; real implementations generated at runtime
-    class Project;
+    class Habitat;
     end
     # Stub base class; real implementations generated at runtime
     class Folder;
@@ -227,12 +231,12 @@ module MU
         :class => generic_class_methods,
         :instance => generic_instance_methods + [:groom]
       },
-      :Notification => {
+      :Notifier => {
         :has_multiples => false,
         :can_live_in_vpc => false,
-        :cfg_name => "notification",
-        :cfg_plural => "notifications",
-        :interface => self.const_get("Notification"),
+        :cfg_name => "notifier",
+        :cfg_plural => "notifiers",
+        :interface => self.const_get("Notifier"),
         :deps_wait_on_my_creation => false,
         :waits_on_parent_completion => false,
         :class => generic_class_methods,
@@ -304,12 +308,12 @@ module MU
         :class => generic_class_methods,
         :instance => generic_instance_methods + [:groom]
       },
-      :Project => {
+      :Habitat => {
         :has_multiples => false,
         :can_live_in_vpc => false,
-        :cfg_name => "project",
-        :cfg_plural => "projects",
-        :interface => self.const_get("Project"),
+        :cfg_name => "habitat",
+        :cfg_plural => "habitats",
+        :interface => self.const_get("Habitat"),
         :deps_wait_on_my_creation => true,
         :waits_on_parent_completion => true,
         :class => generic_class_methods,
@@ -399,16 +403,30 @@ module MU
       }
     end
 
+    # List of known/supported Cloud providers. This may be modified at runtime
+    # if an implemention is defective or missing required methods.
+    @@supportedCloudList = ['AWS', 'CloudFormation', 'Google', 'Azure']
+
     # List of known/supported Cloud providers
     def self.supportedClouds
-      ["AWS", "CloudFormation", "Google"]
+      @@supportedCloudList
     end
 
     # Load the container class for each cloud we know about, and inject autoload
     # code for each of its supported resource type classes.
+    failed = []
     MU::Cloud.supportedClouds.each { |cloud|
       require "mu/clouds/#{cloud.downcase}"
+      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+      generic_class_methods_toplevel.each { |method|
+        if !cloudclass.respond_to?(method)
+          MU.log "MU::Cloud::#{cloud} has not implemented required class method #{method}, disabling", MU::ERR
+          failed << cloud
+        end
+      }
     }
+    failed.uniq!
+    @@supportedCloudList = @@supportedCloudList - failed
 
     # @return [Mutex]
     def self.userdata_mutex
@@ -559,6 +577,7 @@ module MU
         attr_reader :deploy_id
         attr_reader :mu_name
         attr_reader :cloud_id
+        attr_reader :credentials
         attr_reader :url
         attr_reader :config
         attr_reader :deploydata
@@ -623,6 +642,7 @@ module MU
         def initialize(mommacat: nil,
                        mu_name: nil,
                        cloud_id: nil,
+                       credentials: nil,
                        kitten_cfg: nil,
                        delayed_save: false)
           raise MuError, "Cannot invoke Cloud objects without a configuration" if kitten_cfg.nil?
@@ -631,7 +651,9 @@ module MU
           @config = kitten_cfg
           @delayed_save = delayed_save
           @cloud_id = cloud_id
-          
+          @credentials = credentials
+          @credentials ||= kitten_cfg['credentials']
+
           if !@deploy.nil?
             @deploy_id = @deploy.deploy_id
             MU.log "Initializing an instance of #{self.class.name} in #{@deploy_id} #{mu_name}", MU::DEBUG, details: kitten_cfg
@@ -720,21 +742,23 @@ module MU
             end
           end
         end
-
-        def cloud_desc
+        
+        def cloud_desc()
           describe
           if !@cloudobj.nil?
-            @cloud_desc = @cloudobj.cloud_desc
+            @cloud_desc_cache ||= @cloudobj.cloud_desc
             @url = @cloudobj.url if @cloudobj.respond_to?(:url)
-          elsif !@config.nil? and !@cloud_id.nil?
+          end
+          if !@config.nil? and !@cloud_id.nil? and @cloud_desc_cache.nil?
             # The find() method should be returning a Hash with the cloud_id
             # as a key and a cloud platform descriptor as the value.
             begin
-              matches = self.class.find(region: @config['region'], cloud_id: @cloud_id, flags: @config)
+
+              matches = self.class.find(region: @config['region'], cloud_id: @cloud_id, flags: @config, credentials: @credentials)
               if !matches.nil? and matches.is_a?(Hash) and matches.has_key?(@cloud_id)
-                @cloud_desc = matches[@cloud_id]
+                @cloud_desc_cache = matches[@cloud_id]
               else
-                MU.log "Failed to find a live #{self.class.shortname} with identifier #{@cloud_id} in #{@config['region']}, which has a record in deploy #{@deploy.deploy_id}", MU::WARN, details: caller
+                MU.log "Failed to find a live #{self.class.shortname} with identifier #{@cloud_id} in #{@credentials}/#{@config['region']}, which has a record in deploy #{@deploy.deploy_id}", MU::WARN, details: caller
               end
             rescue Exception => e
               MU.log "Got #{e.inspect} trying to find cloud handle for #{self.class.shortname} #{@mu_name} (#{@cloud_id})", MU::WARN
@@ -742,7 +766,7 @@ module MU
             end
           end
 
-          return @cloud_desc
+          return @cloud_desc_cache
         end
 
         # Retrieve all of the known metadata for this resource.
@@ -755,6 +779,7 @@ module MU
           end
           res_type = self.class.cfg_plural
           res_name = @config['name'] if !@config.nil?
+          @credentials ||= @config['credentials'] if !@config.nil?
           deploydata = nil
           if !@deploy.nil? and @deploy.is_a?(MU::MommaCat) and
               !@deploy.deployment.nil? and
@@ -836,11 +861,42 @@ module MU
           # Special dependencies: my containing VPC
           if self.class.can_live_in_vpc and !@config['vpc'].nil?
             MU.log "Loading VPC for #{self}", MU::DEBUG, details: @config['vpc']
-            if !@config['vpc']["vpc_name"].nil? and
+            if !@config['vpc']["vpc_name"].nil? and @deploy
+              sib_by_name = @deploy.findLitterMate(name: @config['vpc']['vpc_name'], type: "vpcs", return_all: true)
+              if sib_by_name.is_a?(Array)
+                if sib_by_name.size == 1
+                  @vpc = matches.first
+                else
+# XXX ok but this is the wrong place for this really the config parser needs to sort this out somehow
+                  # we got multiple matches, try to pick one by preferred subnet
+                  # behavior
+                  sib_by_name.each { |sibling|
+                    all_private = sibling.subnets.map { |s| s.private? }.all?(true)
+                    all_public = sibling.subnets.map { |s| s.private? }.all?(false)
+                    if all_private and ["private", "all_private"].include?(@config['vpc']['subnet_pref'])
+                      @vpc = sibling
+                      break
+                    elsif all_public and ["public", "all_public"].include?(@config['vpc']['subnet_pref'])
+                      @vpc = sibling
+                      break
+                    else
+                      MU.log "Got multiple matching VPCs for #{@mu_name}, so I'm arbitrarily choosing #{sibling.mu_name}"
+                      @vpc = sibling
+                      break
+                    end
+                  }
+                end
+              else
+MU.log "in dependencies() and findLitterMate gave me "+sib_by_name.to_s+" on behalf of "+self.to_s, MU::NOTICE, details: @config['vpc']
+                @vpc = sib_by_name
+              end
+            end
+
+            if !@vpc and !@config['vpc']["vpc_name"].nil? and
                 @dependencies.has_key?("vpc") and
                 @dependencies["vpc"].has_key?(@config['vpc']["vpc_name"])
               @vpc = @dependencies["vpc"][@config['vpc']["vpc_name"]]
-            else
+            elsif !@vpc
               tag_key, tag_value = @config['vpc']['tag'].split(/=/, 2) if !@config['vpc']['tag'].nil?
               if !@config['vpc'].has_key?("vpc_id") and
                   !@config['vpc'].has_key?("deploy_id") and !@deploy.nil?
@@ -883,12 +939,14 @@ module MU
                     nat_cloud_id: @config['vpc']['nat_host_id'],
                     nat_filter_key: "vpc-id",
                     region: @config['vpc']["region"],
-                    nat_filter_value: @vpc.cloud_id
+                    nat_filter_value: @vpc.cloud_id,
+                    credentials: @config['credentials']
                   )
                 else
                   @nat = @vpc.findNat(
                     nat_cloud_id: @config['vpc']['nat_host_id'],
-                    region: @config['vpc']["region"]
+                    region: @config['vpc']["region"],
+                    credentials: @config['credentials']
                   )
                 end
               end
@@ -939,9 +997,13 @@ module MU
               # sense there
               cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
               if args[:region] and cloudbase.respond_to?(:listRegions)
-                next if !cloudbase.listRegions.include?(args[:region])
+                next if !cloudbase.listRegions(credentials: args[:credentials]).include?(args[:region])
+              end
+              begin
+                cloudclass = MU::Cloud.loadCloudType(cloud, shortname)
+              rescue MU::MuError => e
+                next
               end
-              cloudclass = MU::Cloud.loadCloudType(cloud, shortname)
 
               found = cloudclass.find(args)
               if !found.nil?

data/modules/mu/clouds/aws.rb

@@ -26,81 +26,112 @@ module MU
     class AWS
       @@myRegion_var = nil
 
-      @@creds_loaded = false
+      @@creds_loaded = {}
 
       # Load some credentials for using the AWS API
-      def self.loadCredentials
-        return if @@creds_loaded
-        if $MU_CFG and $MU_CFG['aws']
-          loaded = false
-          if $MU_CFG['aws']['access_key'] and $MU_CFG['aws']['access_secret'] and
-            # access key and secret just sitting in mu.yaml
-             !$MU_CFG['aws']['access_key'].empty? and
-             !$MU_CFG['aws']['access_secret'].empty?
-            Aws.config = {
-              access_key_id: $MU_CFG['aws']['access_key'],
-              secret_access_key: $MU_CFG['aws']['access_secret'],
-              region: $MU_CFG['aws']['region']
-            }
-            loaded = true
-          elsif $MU_CFG['aws']['credentials_file'] and
-                !$MU_CFG['aws']['credentials_file'].empty?
-            # pull access key and secret from an awscli-style credentials file
-            begin
-              File.read($MU_CFG["aws"]["credentials_file"]) # make sure it's there
-              credfile = IniFile.load($MU_CFG["aws"]["credentials_file"])
+      # @param name [String]: The name of the mu.yaml AWS credential set to use. If not specified, will use the default credentials, and set the global Aws.config credentials to those.
+      # @return [Aws::Credentials]
+      def self.loadCredentials(name = nil)
+        @@creds_loaded ||= {}
 
-              if !credfile.sections or credfile.sections.size == 0
-                raise ::IniFile::Error, "No AWS profiles found in #{$MU_CFG["aws"]["credentials_file"]}"
-              end
-              data = credfile.has_section?("default") ? credfile["default"] : credfile[credfile.sections.first]
-              if data["aws_access_key_id"] and data["aws_secret_access_key"]
-                Aws.config = {
-                  access_key_id: data['aws_access_key_id'],
-                  secret_access_key: data['aws_secret_access_key'],
-                  region: $MU_CFG['aws']['region']
-                }
-                loaded = true
-              else
-                MU.log "AWS credentials in #{$MU_CFG["aws"]["credentials_file"]} specified, but is missing aws_access_key_id or aws_secret_access_key elements", MU::WARN
+        if name.nil?
+          return @@creds_loaded["#default"] if @@creds_loaded["#default"]
+        else
+          return @@creds_loaded[name] if @@creds_loaded[name]
+        end
+
+        cred_cfg = credConfig(name)
+        if cred_cfg.nil?
+          return nil
+        end
+
+        loaded = false
+        cred_obj = nil
+        if cred_cfg['access_key'] and cred_cfg['access_secret'] and
+          # access key and secret just sitting in mu.yaml
+           !cred_cfg['access_key'].empty? and
+           !cred_cfg['access_secret'].empty?
+          cred_obj = Aws::Credentials.new(
+            cred_cfg['access_key'], cred_cfg['access_secret']
+          )
+          if name.nil?
+#            Aws.config = {
+#              access_key_id: cred_cfg['access_key'],
+#              secret_access_key: cred_cfg['access_secret'],
+#              region: cred_cfg['region']
+#            }
+          end
+        elsif cred_cfg['credentials_file'] and
+              !cred_cfg['credentials_file'].empty?
+
+          # pull access key and secret from an awscli-style credentials file
+          begin
+            File.read(cred_cfg["credentials_file"]) # make sure it's there
+            credfile = IniFile.load(cred_cfg["credentials_file"])
+
+            if !credfile.sections or credfile.sections.size == 0
+              raise ::IniFile::Error, "No AWS profiles found in #{cred_cfg["credentials_file"]}"
+            end
+            data = credfile.has_section?("default") ? credfile["default"] : credfile[credfile.sections.first]
+            if data["aws_access_key_id"] and data["aws_secret_access_key"]
+              cred_obj = Aws::Credentials.new(
+                data['aws_access_key_id'], data['aws_secret_access_key']
+              )
+              if name.nil?
+#                Aws.config = {
+#                  access_key_id: data['aws_access_key_id'],
+#                  secret_access_key: data['aws_secret_access_key'],
+#                  region: cred_cfg['region']
+#                }
               end
-            rescue IniFile::Error, Errno::ENOENT, Errno::EACCES => e
-              MU.log "AWS credentials file #{$MU_CFG["aws"]["credentials_file"]} is missing or invalid", MU::WARN, details: e.message
+            else
+              MU.log "AWS credentials in #{cred_cfg["credentials_file"]} specified, but is missing aws_access_key_id or aws_secret_access_key elements", MU::WARN
             end
-          elsif $MU_CFG['aws']['credentials'] and
-                !$MU_CFG['aws']['credentials'].empty?
-            # pull access key and secret from a vault
-            begin
-              vault, item = $MU_CFG["aws"]["credentials"].split(/:/)
-              data = MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h
-              if data["access_key"] and data["access_secret"]
-                Aws.config = {
-                  access_key_id: data['access_key'],
-                  secret_access_key: data['access_secret'],
-                  region: $MU_CFG['aws']['region']
-                }
-                loaded = true
-              else
-                MU.log "AWS credentials vault:item #{$MU_CFG["aws"]["credentials"]} specified, but is missing access_key or access_secret elements", MU::WARN
+          rescue IniFile::Error, Errno::ENOENT, Errno::EACCES => e
+            MU.log "AWS credentials file #{cred_cfg["credentials_file"]} is missing or invalid", MU::WARN, details: e.message
+          end
+        elsif cred_cfg['credentials'] and
+              !cred_cfg['credentials'].empty?
+          # pull access key and secret from a vault
+          begin
+            vault, item = cred_cfg["credentials"].split(/:/)
+            data = MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h
+            if data["access_key"] and data["access_secret"]
+              cred_obj = Aws::Credentials.new(
+                cred_cfg['access_key'], cred_cfg['access_secret']
+              )
+              if name.nil?
+#                Aws.config = {
+#                  access_key_id: data['access_key'],
+#                  secret_access_key: data['access_secret'],
+#                  region: cred_cfg['region']
+#                }
               end
-            rescue MU::Groomer::Chef::MuNoSuchSecret
-              MU.log "AWS credentials vault:item #{$MU_CFG["aws"]["credentials"]} specified, but does not exist", MU::WARN
+            else
+              MU.log "AWS credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", MU::WARN
             end
+          rescue MU::Groomer::Chef::MuNoSuchSecret
+            MU.log "AWS credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", MU::WARN
           end
+        end
 
-          if !loaded and hosted?
-            # assume we've got an IAM profile and hope for the best
-            ENV.delete('AWS_ACCESS_KEY_ID')
-            ENV.delete('AWS_SECRET_ACCESS_KEY')
-            Aws.config = {region: ENV['EC2_REGION']}
-            loaded = true
-          end
+        if !cred_obj and hosted?
+          # assume we've got an IAM profile and hope for the best
+          ENV.delete('AWS_ACCESS_KEY_ID')
+          ENV.delete('AWS_SECRET_ACCESS_KEY')
+          cred_obj = Aws::InstanceProfileCredentials.new
+#          if name.nil?
+#            Aws.config = {region: ENV['EC2_REGION']}
+#          end
+        end
 
-          @@creds_loaded = loaded
-          if !@@creds_loaded
-            raise MuError, "AWS layer is enabled in mu.yaml, but I couldn't find working API credentials anywhere"
-          end
+        if name.nil?
+          @@creds_loaded["#default"] = cred_obj
+        else
+          @@creds_loaded[name] = cred_obj
         end
+
+        cred_obj
       end
 
       # Any cloud-specific instance methods we require our resource
@@ -114,13 +145,17 @@ module MU
       # If we've configured AWS as a provider, or are simply hosted in AWS, 
       # decide what our default region is.
       def self.myRegion
+        return @@myRegion_var if @@myRegion_var
+        return nil if credConfig.nil? and !hosted?
+
         if $MU_CFG and (!$MU_CFG['aws'] or !account_number) and !hosted?
           return nil
         end
+
         if $MU_CFG and $MU_CFG['aws'] and $MU_CFG['aws']['region']
-          @@myRegion_var ||= MU::Cloud::AWS.ec2($MU_CFG['aws']['region']).describe_availability_zones.availability_zones.first.region_name
+          @@myRegion_var ||= MU::Cloud::AWS.ec2(region: $MU_CFG['aws']['region']).describe_availability_zones.availability_zones.first.region_name
         elsif ENV.has_key?("EC2_REGION") and !ENV['EC2_REGION'].empty?
-          @@myRegion_var ||= MU::Cloud::AWS.ec2(ENV['EC2_REGION']).describe_availability_zones.availability_zones.first.region_name
+          @@myRegion_var ||= MU::Cloud::AWS.ec2(region: ENV['EC2_REGION']).describe_availability_zones.availability_zones.first.region_name
         else
           # hacky, but useful in a pinch
           az_str = MU::Cloud::AWS.getAWSMetaData("placement/availability-zone")
@@ -140,7 +175,7 @@ module MU
       # @param value [String]: The value of the tag to remove
       # @param region [String]: The cloud provider region
       def self.removeTag(key, value, resources = [], region: myRegion)
-        MU::Cloud::AWS.ec2(region).delete_tags(
+        MU::Cloud::AWS.ec2(region: region).delete_tags(
           resources: resources,
           tags: [
             {
@@ -158,11 +193,11 @@ module MU
       # @param value [String]: The value of the tag
       # @param region [String]: The cloud provider region
       # @return [void,<Hash>]
-      def self.createTag(key, value, resources = [], region: myRegion)
+      def self.createTag(key, value, resources = [], region: myRegion, credentials: nil)
   
         if !MU::Cloud::CloudFormation.emitCloudFormation
           begin
-            MU::Cloud::AWS.ec2(region).create_tags(
+            MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_tags(
               resources: resources,
               tags: [
                 {
@@ -196,7 +231,7 @@ module MU
       # server resides.
       # @param region [String]: The region to search.
       # @return [Array<String>]: The Availability Zones in this region.
-      def self.listAZs(region = MU.curRegion)
+      def self.listAZs(region: MU.curRegion, account: nil, credentials: nil)
         if $MU_CFG and (!$MU_CFG['aws'] or !account_number)
           return []
         end
@@ -204,7 +239,7 @@ module MU
           return @@azs[region]
         end
         if region
-          azs = MU::Cloud::AWS.ec2(region).describe_availability_zones(
+          azs = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_availability_zones(
             filters: [name: "region-name", values: [region]]
           )
         end
@@ -218,21 +253,57 @@ module MU
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
       # @param deploy_id [String]: The deploy for which we're writing the secret
       # @param value [String]: The contents of the secret
-      def self.writeDeploySecret(deploy_id, value, name = nil)
+      def self.writeDeploySecret(deploy_id, value, name = nil, credentials: nil)
         name ||= deploy_id+"-secret"
         begin
-          MU.log "Writing #{name} to S3 bucket #{MU.adminBucketName}"
-          MU::Cloud::AWS.s3(myRegion).put_object(
+          MU.log "Writing #{name} to S3 bucket #{adminBucketName(credentials)}"
+          MU::Cloud::AWS.s3(region: myRegion, credentials: credentials).put_object(
             acl: "private",
-            bucket: MU.adminBucketName,
+            bucket: adminBucketName(credentials),
             key: name,
             body: value
           )
         rescue Aws::S3::Errors => e
-          raise MU::MommaCat::DeployInitializeError, "Got #{e.inspect} trying to write #{name} to #{MU.adminBucketName}"
+          raise MU::MommaCat::DeployInitializeError, "Got #{e.inspect} trying to write #{name} to #{adminBucketName(credentials)}"
         end
       end
 
+  # Log bucket policy for enabling CloudTrail logging to our log bucket in S3.
+      def self.cloudtrailBucketPolicy(credentials = nil)
+        cfg = credConfig(credentials)
+        policy_json = '{
+      		"Version": "2012-10-17",
+      		"Statement": [
+      			{
+      				"Sid": "AWSCloudTrailAclCheck20131101",
+      				"Effect": "Allow",
+              "Principal": {
+                "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::<%= MU.account_number %>:root",
+                "Service": "cloudtrail.amazonaws.com"
+              },
+      				"Action": "s3:GetBucketAcl",
+      				"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'"
+      			},
+      			{
+      				"Sid": "AWSCloudTrailWrite20131101",
+      				"Effect": "Allow",
+              "Principal": {
+                "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::'+credToAcct(credentials)+':root",
+                "Service": "cloudtrail.amazonaws.com"
+              },
+      				"Action": "s3:PutObject",
+      				"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/AWSLogs/'+credToAcct(credentials)+'/*",
+      				"Condition": {
+      					"StringEquals": {
+      						"s3:x-amz-acl": "bucket-owner-full-control"
+      					}
+      				}
+      			}
+      		]
+      	}'
+        ERB.new(policy_json).result
+      end
+
       @@is_in_aws = nil
 
       # Alias for #{MU::Cloud::AWS.hosted?}
@@ -293,32 +364,137 @@ module MU
         sample
       end
 
-      @my_acct_num = nil
+      @@my_acct_num = nil
+      @@my_hosted_cfg = nil
+      @@acct_to_profile_map = {}
+
+      # Map the name of a credential set back to an AWS account number
+      # @param name [String]
+      def self.credToAcct(name = nil)
+        creds = credConfig(name)
+
+        return creds['account_number'] if creds['account_number']
+
+        user_list = MU::Cloud::AWS.iam(credentials: name).list_users.users
+        acct_num = MU::Cloud::AWS.iam(credentials: name).list_users.users.first.arn.split(/:/)[4]
+        acct_num.to_s
+      end
+
+      # Return the name strings of all known sets of credentials for this cloud
+      # @return [Array<String>]
+      def self.listCredentials
+        if !$MU_CFG['aws']
+          return hosted? ? ["#default"] : nil
+        end
+
+        $MU_CFG['aws'].keys
+      end 
+
+      def self.adminBucketName(credentials = nil)
+         #XXX find a default if this particular account doesn't have a log_bucket_name configured
+        cfg = credConfig(credentials)
+        cfg['log_bucket_name']
+      end
+
+      def self.adminBucketUrl(credentials = nil)
+        "s3://"+adminBucketName+"/"
+      end
+
+      # Return the $MU_CFG data associated with a particular profile/name/set of
+      # credentials. If no account name is specified, will return one flagged as
+      # default. Returns nil if AWS is not configured. Throws an exception if 
+      # an account name is specified which does not exist.
+      # @param name [String]: The name of the key under 'aws' in mu.yaml to return
+      # @return [Hash,nil]
+      def self.credConfig(name = nil, name_only: false)
+        # If there's nothing in mu.yaml (which is wrong), but we're running
+        # on a machine hosted in AWS, *and* that machine has an IAM profile,
+        # fake it with those credentials and hope for the best.
+        if !$MU_CFG['aws'] or !$MU_CFG['aws'].is_a?(Hash) or $MU_CFG['aws'].size == 0
+          return @@my_hosted_cfg if @@my_hosted_cfg
+
+          if hosted?
+            begin
+              iam_data = JSON.parse(getAWSMetaData("iam/info"))
+              if iam_data["InstanceProfileArn"] and !iam_data["InstanceProfileArn"].empty?
+                @@my_hosted_cfg = hosted_config
+                return name_only ? "#default" : @@my_hosted_cfg
+              end
+            rescue JSON::ParserError => e
+            end
+          end
+
+          return nil
+        end
+
+        if name.nil?
+          $MU_CFG['aws'].each_pair { |name, cfg|
+            if cfg['default']
+              return name_only ? name : cfg
+            end
+          }
+        else
+          if $MU_CFG['aws'][name]
+            return name_only ? name : $MU_CFG['aws'][name]
+          elsif @@acct_to_profile_map[name.to_s]
+            return name_only ? name : @@acct_to_profile_map[name.to_s]
+          elsif name.is_a?(Integer) or name.match(/^\d+$/)
+            # Try to map backwards from an account id, if that's what we go
+            $MU_CFG['aws'].each_pair { |acctname, cfg|
+              if cfg['account_number'] and name.to_s == cfg['account_number'].to_s
+                return name_only ? acctname : $MU_CFG['aws'][acctname]
+              end
+            }
+
+            # Check each credential sets' resident account, then
+            $MU_CFG['aws'].each_pair { |acctname, cfg|
+              begin
+                user_list = MU::Cloud::AWS.iam(credentials: acctname).list_users.users
+#             rescue ::Aws::IAM::Errors => e # XXX why does this NameError here?
+              rescue Exception => e
+                MU.log e.inspect, MU::WARN, details: cfg
+                next
+              end
+              acct_num = MU::Cloud::AWS.iam(credentials: acctname).list_users.users.first.arn.split(/:/)[4]
+              if acct_num.to_s ==  name.to_s
+                cfg['account_number'] = acct_num.to_s
+                @@acct_to_profile_map[name.to_s] = cfg
+                return name_only ? name.to_s : cfg
+                return cfg
+              end
+            }
+          end
+
+          raise MuError, "AWS credential set #{name} was requested, but I see no such working credentials in mu.yaml"
+        end
+      end
 
-      # Fetch the AWS account number where this Mu master resides. If it's not in 
-      # AWS at all, or otherwise cannot be determined, return nil.
-      # here.
+      # Fetch the AWS account number where this Mu master resides. If it's not
+      # in AWS at all, or otherwise cannot be determined, return nil.  here.
       # XXX account for Google and non-cloud situations
-      # XXX but what about multi-account uuuugh
+      # XXX this needs to be "myAccountNumber" or somesuch
+      # XXX and maybe do the IAM thing for arbitrary, non-resident accounts
       def self.account_number
-        return nil if !$MU_CFG['aws']
-        return @my_acct_num if @my_acct_num
-
-    		begin
-    	    user_list = MU::Cloud::AWS.iam($MU_CFG['aws']['region']).list_users.users
-#    		rescue ::Aws::IAM::Errors => e # XXX why does this NameError here?
-    		rescue Exception => e
-    			MU.log "Got #{e.inspect} while trying to figure out our account number", MU::WARN, details: caller
-    		end
-        if user_list.nil? or user_list.size == 0
+        return nil if credConfig.nil?
+        return @@my_acct_num if @@my_acct_num
+        loadCredentials
+# XXX take optional credential set argument
+
+#       begin
+#          user_list = MU::Cloud::AWS.iam(region: credConfig['region']).list_users.users
+##        rescue ::Aws::IAM::Errors => e # XXX why does this NameError here?
+#        rescue Exception => e
+#          MU.log "Got #{e.inspect} while trying to figure out our account number", MU::WARN, details: caller
+#        end
+#        if user_list.nil? or user_list.size == 0
           mac = MU::Cloud::AWS.getAWSMetaData("network/interfaces/macs/").split(/\n/)[0]
           acct_num = MU::Cloud::AWS.getAWSMetaData("network/interfaces/macs/#{mac}owner-id")
           acct_num.chomp!
-        else
-          acct_num = MU::Cloud::AWS.iam($MU_CFG['aws']['region']).list_users.users.first.arn.split(/:/)[4]
-        end
+#        else
+#          acct_num = MU::Cloud::AWS.iam(region: credConfig['region']).list_users.users.first.arn.split(/:/)[4]
+#        end
         MU.setVar("acct_num", acct_num)
-        @my_acct_num ||= acct_num
+        @@my_acct_num ||= acct_num
         acct_num
       end
 
@@ -327,17 +503,19 @@ module MU
       # region that is local to this Mu server will be listed first.
       # @param us_only [Boolean]: Restrict results to United States only
       # @return [Array<String>]
-      def self.listRegions(us_only = false)
-        if $MU_CFG and (!$MU_CFG['aws'] or !account_number)
-          return []
-        end
+      def self.listRegions(us_only = false, credentials: nil)
+
         if @@regions.size == 0
-          result = MU::Cloud::AWS.ec2(myRegion).describe_regions.regions
+          return [] if credConfig.nil?
+          result = MU::Cloud::AWS.ec2(region: myRegion, credentials: credentials).describe_regions.regions
           regions = []
           result.each { |r|
-            @@regions[r.region_name] = Proc.new { listAZs(r.region_name) }
+            @@regions[r.region_name] = Proc.new {
+              listAZs(region: r.region_name, credentials: credentials)
+            }
           }
         end
+
         regions = if us_only
           @@regions.keys.delete_if { |r| !r.match(/^us\-/) }.uniq
         else
@@ -368,14 +546,14 @@ module MU
       # @param keyname [String]: The name of the key to create.
       # @param public_key [String]: The public key
       # @return [Array<String>]: keypairname, ssh_private_key, ssh_public_key
-      def self.createEc2SSHKey(keyname, public_key)
+      def self.createEc2SSHKey(keyname, public_key, credentials: nil)
         # We replicate this key in all regions
         if !MU::Cloud::CloudFormation.emitCloudFormation
           MU::Cloud::AWS.listRegions.each { |region|
             MU.log "Replicating #{keyname} to EC2 in #{region}", MU::DEBUG, details: @ssh_public_key
-            MU::Cloud::AWS.ec2(region).import_key_pair(
-                key_name: keyname,
-                public_key_material: public_key
+            MU::Cloud::AWS.ec2(region: region, credentials: credentials).import_key_pair(
+              key_name: keyname,
+              public_key_material: public_key
             )
           }
         end
@@ -389,9 +567,7 @@ module MU
       # @return [Hash]
       def self.listInstanceTypes(region = myRegion)
         return @@instance_types if @@instance_types and @@instance_types[region]
-        if $MU_CFG and (!$MU_CFG['aws'] or !account_number)
-          return {}
-        end
+        return {} if credConfig.nil?
 
         human_region = @@regionLookup[region]
 
@@ -402,7 +578,7 @@ module MU
         begin
           # Pricing API isn't widely available, so ask a region we know supports
           # it
-          resp = MU::Cloud::AWS.pricing("us-east-1").get_products(
+          resp = MU::Cloud::AWS.pricing(region: "us-east-1").get_products(
             service_code: "AmazonEC2",
             filters: [
               {
@@ -456,7 +632,7 @@ module MU
 
         if !name.nil? and !name.empty?
           matches = []
-          acmcerts = MU::Cloud::AWS.acm(region).list_certificates(
+          acmcerts = MU::Cloud::AWS.acm(region: region).list_certificates(
             certificate_statuses: ["ISSUED"]
           )
           acmcerts.certificate_summary_list.each { |cert|
@@ -475,14 +651,14 @@ module MU
           if matches.size == 1
             return matches.first
           elsif matches.size == 0
-            raise MuError, "No IAM or ACM certificate named #{name} was found"
+            raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
           elsif matches.size > 1
-            raise MuError, "Multiple certificates named #{name} were found. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."            
+            raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."            
           end
         end
 
         if id.match(/^arn:aws(?:-us-gov)?:acm/)
-          resp = MU::Cloud::AWS.acm(region).get_certificate(
+          resp = MU::Cloud::AWS.acm(region: region).get_certificate(
             certificate_arn: id
           )
           if resp.nil?
@@ -510,221 +686,255 @@ module MU
       end
 
       # Amazon Certificate Manager API
-      def self.acm(region = MU.curRegion)
+      def self.acm(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@acm_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "ACM", region: region)
-        @@acm_api[region]
+        @@acm_api[credentials] ||= {}
+        @@acm_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "ACM", region: region, credentials: credentials)
+        @@acm_api[credentials][region]
       end
 
       # Amazon's IAM API
-      def self.iam(region = MU.curRegion)
-        region ||= myRegion
-        @@iam_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "IAM", region: region)
-        @@iam_api[region]
+      def self.iam(credentials: nil)
+        @@iam_api[credentials] ||= MU::Cloud::AWS::Endpoint.new(api: "IAM", credentials: credentials)
+        @@iam_api[credentials]
       end
 
       # Amazon's EC2 API
-      def self.ec2(region = MU.curRegion)
+      def self.ec2(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@ec2_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "EC2", region: region)
-        @@ec2_api[region]
+        @@ec2_api[credentials] ||= {}
+        @@ec2_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "EC2", region: region, credentials: credentials)
+        @@ec2_api[credentials][region]
       end
 
       # Amazon's Autoscaling API
-      def self.autoscale(region = MU.curRegion)
+      def self.autoscale(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@autoscale_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "AutoScaling", region: region)
-        @@autoscale_api[region]
+        @@autoscale_api[credentials] ||= {}
+        @@autoscale_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "AutoScaling", region: region, credentials: credentials)
+        @@autoscale_api[credentials][region]
       end
 
       # Amazon's ElasticLoadBalancing API
-      def self.elb(region = MU.curRegion)
+      def self.elb(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@elb_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "ElasticLoadBalancing", region: region)
-        @@elb_api[region]
+        @@elb_api[credentials] ||= {}
+        @@elb_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "ElasticLoadBalancing", region: region, credentials: credentials)
+        @@elb_api[credentials][region]
       end
 
       # Amazon's ElasticLoadBalancingV2 (ALB) API
-      def self.elb2(region = MU.curRegion)
+      def self.elb2(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@elb2_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "ElasticLoadBalancingV2", region: region)
-        @@elb2_api[region]
+        @@elb2_api[credentials] ||= {}
+        @@elb2_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "ElasticLoadBalancingV2", region: region, credentials: credentials)
+        @@elb2_api[credentials][region]
       end
 
       # Amazon's Route53 API
-      def self.route53(region = MU.curRegion)
-        region ||= myRegion
-        @@route53_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "Route53", region: region)
-        @@route53_api[region]
+      def self.route53(credentials: nil)
+        @@route53_api[credentials] ||= MU::Cloud::AWS::Endpoint.new(api: "Route53", credentials: credentials)
+        @@route53_api[credentials]
       end
 
       # Amazon's RDS API
-      def self.rds(region = MU.curRegion)
+      def self.rds(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@rds_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "RDS", region: region)
-        @@rds_api[region]
+        @@rds_api[credentials] ||= {}
+        @@rds_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "RDS", region: region, credentials: credentials)
+        @@rds_api[credentials][region]
       end
 
       # Amazon's CloudFormation API
-      def self.cloudformation(region = MU.curRegion)
+      def self.cloudformation(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@cloudformation_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudFormation", region: region)
-        @@cloudformation_api[region]
+        @@cloudformation_api[credentials] ||= {}
+        @@cloudformation_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudFormation", region: region, credentials: credentials)
+        @@cloudformation_api[credentials][region]
       end
 
       # Amazon's S3 API
-      def self.s3(region = MU.curRegion)
+      def self.s3(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@s3_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "S3", region: region)
-        @@s3_api[region]
+        @@s3_api[credentials] ||= {}
+        @@s3_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "S3", region: region, credentials: credentials)
+        @@s3_api[credentials][region]
       end
 
       # Amazon's CloudTrail API
-      def self.cloudtrail(region = MU.curRegion)
+      def self.cloudtrail(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@cloudtrail_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudTrail", region: region)
-        @@cloudtrail_api[region]
+        @@cloudtrail_api[credentials] ||= {}
+        @@cloudtrail_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudTrail", region: region, credentials: credentials)
+        @@cloudtrail_api[credentials][region]
       end
 
       # Amazon's CloudWatch API
-      def self.cloudwatch(region = MU.curRegion)
+      def self.cloudwatch(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@cloudwatch_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudWatch", region: region)
-        @@cloudwatch_api[region]
+        @@cloudwatch_api[credentials] ||= {}
+        @@cloudwatch_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudWatch", region: region, credentials: credentials)
+        @@cloudwatch_api[credentials][region]
       end
 
       # Amazon's Web Application Firewall API (Global, for CloudFront et al)
-      def self.wafglobal(region = MU.curRegion)
+      def self.wafglobal(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@wafglobal[region] ||= MU::Cloud::AWS::Endpoint.new(api: "WAF", region: region)
-        @@wafglobal[region]
+        @@wafglobal_api[credentials] ||= {}
+        @@wafglobal[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "WAF", region: region, credentials: credentials)
+        @@wafglobal[credentials][region]
       end
 
 
       # Amazon's Web Application Firewall API (Regional, for ALBs et al)
-      def self.waf(region = MU.curRegion)
+      def self.waf(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@waf[region] ||= MU::Cloud::AWS::Endpoint.new(api: "WAFRegional", region: region)
-        @@waf[region]
+        @@waf[credentials] ||= {}
+        @@waf[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "WAFRegional", region: region, credentials: credentials)
+        @@waf[credentials][region]
       end
 
       # Amazon's CloudWatchLogs API
-      def self.cloudwatchlogs(region = MU.curRegion)
+      def self.cloudwatchlogs(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@cloudwatchlogs_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudWatchLogs", region: region)
-        @@cloudwatchlogs_api[region]
+        @@cloudwatchlogs_api[credentials] ||= {}
+        @@cloudwatchlogs_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudWatchLogs", region: region, credentials: credentials)
+        @@cloudwatchlogs_api[credentials][region]
       end
 
       # Amazon's CloudFront API
-      def self.cloudfront(region = MU.curRegion)
+      def self.cloudfront(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@cloudfront_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudFront", region: region)
-        @@cloudfront_api[region]
+        @@cloudfront_api[credentials] ||= {}
+        @@cloudfront_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudFront", region: region, credentials: credentials)
+        @@cloudfront_api[credentials][region]
       end
 
       # Amazon's ElastiCache API
-      def self.elasticache(region = MU.curRegion)
+      def self.elasticache(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@elasticache_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "ElastiCache", region: region)
-        @@elasticache_api[region]
+        @@elasticache_api[credentials] ||= {}
+        @@elasticache_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "ElastiCache", region: region, credentials: credentials)
+        @@elasticache_api[credentials][region]
       end
       
       # Amazon's SNS API
-      def self.sns(region = MU.curRegion)
+      def self.sns(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@sns_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "SNS", region: region)
-        @@sns_api[region]
+        @@sns_api[credentials] ||= {}
+        @@sns_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "SNS", region: region, credentials: credentials)
+        @@sns_api[credentials][region]
       end
       
       # Amazon's SQS API
-      def self.sqs(region = MU.curRegion)
+      def self.sqs(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@sqs_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "SQS", region: region)
-        @@sqs_api[region]
+        @@sqs_api[credentials] ||= {}
+        @@sqs_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "SQS", region: region, credentials: credentials)
+        @@sqs_api[credentials][region]
       end
 
       # Amazon's EFS API
-      def self.efs(region = MU.curRegion)
+      def self.efs(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@efs_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "EFS", region: region)
-        @@efs_api[region]
+        @@efs_api[credentials] ||= {}
+        @@efs_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "EFS", region: region, credentials: credentials)
+        @@efs_api[credentials][region]
       end
 
       # Amazon's Lambda API
-      def self.lambda(region = MU.curRegion)
+      def self.lambda(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@lambda_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "Lambda", region: region)
-        @@lambda_api[region]
+        @@lambda_api[credentials] ||= {}
+        @@lambda_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "Lambda", region: region, credentials: credentials)
+        @@lambda_api[credentials][region]
       end
 
       # Amazon's API Gateway API
-      def self.apig(region = MU.curRegion)
+      def self.apig(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@apig_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "APIGateway", region: region)
-        @@apig_api[region]
+        @@apig_api[credentials] ||= {}
+        @@apig_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "APIGateway", region: region, credentials: credentials)
+        @@apig_api[credentials][region]
       end
       
       # Amazon's Cloudwatch Events API
       def self.cloudwatch_events(region = MU.cureRegion)
         region ||= myRegion
-        @@cloudwatch_events_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudWatchEvents", region: region)
+        @@cloudwatch_events_api[credentials] ||= {}
+        @@cloudwatch_events_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials)
         @@cloudwatch_events_api
       end
 
       # Amazon's ECS API
-      def self.ecs(region = MU.curRegion)
+      def self.ecs(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@ecs_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "ECS", region: region)
-        @@ecs_api[region]
+        @@ecs_api[credentials] ||= {}
+        @@ecs_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "ECS", region: region, credentials: credentials)
+        @@ecs_api[credentials][region]
       end
 
       # Amazon's EKS API
-      def self.eks(region = MU.curRegion)
+      def self.eks(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@eks_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "EKS", region: region)
-        @@eks_api[region]
+        @@eks_api[credentials] ||= {}
+        @@eks_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "EKS", region: region, credentials: credentials)
+        @@eks_api[credentials][region]
       end
 
       # Amazon's Pricing API
-      def self.pricing(region = MU.curRegion)
+      def self.pricing(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@pricing_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "Pricing", region: region)
-        @@pricing_api[region]
+        @@pricing_api[credentials] ||= {}
+        @@pricing_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "Pricing", region: region, credentials: credentials)
+        @@pricing_api[credentials][region]
       end
 
       # Amazon's Simple Systems Manager API
-      def self.ssm(region = MU.curRegion)
+      def self.ssm(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@ssm_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "SSM", region: region)
-        @@ssm_api[region]
+        @@ssm_api[credentials] ||= {}
+        @@ssm_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "SSM", region: region, credentials: credentials)
+        @@ssm_api[credentials][region]
       end
 
       # Amazon's Elasticsearch API
-      def self.elasticsearch(region = MU.curRegion)
+      def self.elasticsearch(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@elasticsearch_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "ElasticsearchService", region: region)
-        @@elasticsearch_api[region]
+        @@elasticsearch_api[credentials] ||= {}
+        @@elasticsearch_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "ElasticsearchService", region: region, credentials: credentials)
+        @@elasticsearch_api[credentials][region]
       end
 
       # Amazon's Cognito Identity API
-      def self.cognito_ident(region = MU.curRegion)
+      def self.cognito_ident(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@cognito_ident_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "CognitoIdentity", region: region)
-        @@cognito_ident_api[region]
+        @@cognito_ident_api[credentials] ||= {}
+        @@cognito_ident_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "CognitoIdentity", region: region, credentials: credentials)
+        @@cognito_ident_api[credentials][region]
       end
 
       # Amazon's Cognito Identity Provider API
-      def self.cognito_user(region = MU.curRegion)
+      def self.cognito_user(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@cognito_user_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "CognitoIdentityProvider", region: region)
-        @@cognito_user_api[region]
+        @@cognito_user_api[credentials] ||= {}
+        @@cognito_user_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "CognitoIdentityProvider", region: region, credentials: credentials)
+        @@cognito_user_api[credentials][region]
       end
 
       # Amazon's KMS API
-      def self.kms(region = MU.curRegion)
+      def self.kms(region: MU.curRegion, credentials: nil)
         region ||= myRegion
-        @@kms_api[region] ||= MU::Cloud::AWS::Endpoint.new(api: "KMS", region: region)
-        @@kms_api[region]
+        @@kms_api[credentials] ||= {}
+        @@kms_api[credentials][region] ||= MU::Cloud::AWS::Endpoint.new(api: "KMS", region: region, credentials: credentials)
+        @@kms_api[credentials][region]
+      end
+
+      # Amazon's Organizations API
+      def self.orgs(credentials: nil)
+        @@organizations_api ||= {}
+        @@organizations_api[credentials] ||= MU::Cloud::AWS::Endpoint.new(api: "Organizations", credentials: credentials)
+        @@organizations_api[credentials]
       end
 
       # Fetch an Amazon instance metadata parameter (example: public-ipv4).
@@ -734,13 +944,13 @@ module MU
         base_url = "http://169.254.169.254/latest/meta-data/"
         begin
           response = nil
-          Timeout.timeout(2) do
+          Timeout.timeout(1) do
             response = open("#{base_url}/#{param}").read
           end
 
           response
         rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH, Net::HTTPServerException, Errno::EHOSTUNREACH => e
-          # This is fairly normal, just handle it gracefully
+          # This is normal on machines checking to see if they're AWS-hosted
           logger = MU::Logger.new
           logger.log "Failed metadata request #{base_url}/#{param}: #{e.inspect}", MU::DEBUG
           return nil
@@ -844,7 +1054,7 @@ module MU
                     rule.from_port == port and rule.to_port == port
                   MU.log "Revoking old rules for port #{port.to_s} from #{sg_id}", MU::NOTICE
                   begin
-                    MU::Cloud::AWS.ec2(myRegion).revoke_security_group_ingress(
+                    MU::Cloud::AWS.ec2(region: myRegion).revoke_security_group_ingress(
                         group_id: sg_id,
                         ip_permissions: [
                             {
@@ -871,7 +1081,7 @@ module MU
             }
 
             begin
-              MU::Cloud::AWS.ec2(myRegion).authorize_security_group_ingress(
+              MU::Cloud::AWS.ec2(region: myRegion).authorize_security_group_ingress(
                   group_id: sg_id,
                   ip_permissions: [
                       {
@@ -919,24 +1129,41 @@ module MU
       class Endpoint
         @api = nil
         @region = nil
+        @cred_obj = nil
+        attr_reader :credentials
+        attr_reader :account
 
         # Create an AWS API client
         # @param region [String]: Amazon region so we know what endpoint to use
         # @param api [String]: Which API are we wrapping?
-        def initialize(region: MU.curRegion, api: "EC2")
-          @region = region
+        def initialize(region: MU.curRegion, api: "EC2", credentials: nil)
+          @cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
+          @credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
+
+          if !@cred_obj
+            raise MuError, "Unable to locate valid AWS credentials for #{api} API. #{credentials ? "Credentials requested were '#{credentials}'": ""}"
+          end
+
+          params = {}
+
           if region
-            @api = Object.const_get("Aws::#{api}::Client").new(region: region)
-          else
-            @api = Object.const_get("Aws::#{api}::Client").new
+            @region = region
+            params[:region] = @region
           end
+
+          params[:credentials] = @cred_obj
+
+          MU.log "Initializing #{api} object with credentials #{credentials}", MU::DEBUG, details: params
+          @api = Object.const_get("Aws::#{api}::Client").new(params)
+
+          @api
         end
 
         @instance_cache = {}
         # Catch-all for AWS client methods. Essentially a pass-through with some
         # rescues for known silly endpoint behavior.
         def method_missing(method_sym, *arguments)
-          MU::Cloud::AWS.loadCredentials
+
           retries = 0
           begin
             MU.log "Calling #{method_sym} in #{@region}", MU::DEBUG, details: arguments
@@ -949,7 +1176,7 @@ module MU
               retval = @api.method(method_sym).call
             end
             return retval
-          rescue Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling => e
+          rescue Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException => e
             if e.class.name == "Seahorse::Client::NetworkingError" and e.message.match(/Name or service not known/)
               MU.log e.inspect, MU::ERR
               raise e
@@ -967,9 +1194,12 @@ module MU
             # elsif retries > 100
               # raise MuError, "Exhausted retries after #{retries} attempts while calling EC2's #{method_sym} in #{@region}.  Args were: #{arguments}"
             end
-            MU.log "Got #{e.inspect} calling EC2's #{method_sym} in #{@region}, waiting #{interval.to_s}s and retrying. Args were: #{arguments}", debuglevel, details: caller
+            MU.log "Got #{e.inspect} calling EC2's #{method_sym} in #{@region} with credentials #{@credentials}, waiting #{interval.to_s}s and retrying. Args were: #{arguments}", debuglevel, details: caller
             sleep interval
             retry
+          rescue Exception => e
+            MU.log "Got #{e.inspect} calling EC2's #{method_sym} in #{@region} with credentials #{@credentials}", MU::DEBUG, details: arguments
+            raise e
           end
         end
       end
@@ -1004,6 +1234,7 @@ module MU
       @@cognito_ident_api ={}
       @@cognito_user_api ={}
       @@kms_api ={}
+      @@organizataion_api ={}
     end
   end
 end