cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

data/modules/mu/clouds/aws/user.rb

@@ -40,7 +40,7 @@ module MU
         def create
 
           begin
-            MU::Cloud::AWS.iam.get_user(user_name: @mu_name, path: @config['path'])
+            MU::Cloud::AWS.iam(credentials: @config['credentials']).get_user(user_name: @mu_name, path: @config['path'])
             if !@config['use_if_exists']
               raise MuError, "IAM user #{@mu_name} already exists and use_if_exists is false"
             end
@@ -48,7 +48,7 @@ module MU
             @config['path'] ||= "/"+@deploy.deploy_id+"/"
             MU.log "Creating IAM user #{@config['path']}/#{@mu_name}"
             tags = get_tag_params
-            MU::Cloud::AWS.iam.create_user(
+            MU::Cloud::AWS.iam(credentials: @config['credentials']).create_user(
               user_name: @mu_name,
               path: @config['path'],
               tags: tags
@@ -59,7 +59,7 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
-          resp = MU::Cloud::AWS.iam.list_user_tags(user_name: @mu_name)
+          resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_user_tags(user_name: @mu_name)
 
           ext_tags = resp.tags.map { |t| t.to_h }
           tag_param = get_tag_params(true)
@@ -67,7 +67,7 @@ module MU
 
           if tag_param.size > 0
             MU.log "Updating tags on IAM user #{@mu_name}", MU::NOTICE, details: tag_param
-            MU::Cloud::AWS.iam.tag_user(user_name: @mu_name, tags: tag_param)
+            MU::Cloud::AWS.iam(credentials: @config['credentials']).tag_user(user_name: @mu_name, tags: tag_param)
           end
           # Note: We don't delete tags, because we often share user accounts
           # managed outside of Mu. We have no way of know what tags might come
@@ -76,12 +76,12 @@ module MU
 
           if @config['create_console_password']
             begin
-              MU::Cloud::AWS.iam.get_login_profile(user_name: @mu_name)
+              MU::Cloud::AWS.iam(credentials: @config['credentials']).get_login_profile(user_name: @mu_name)
             rescue Aws::IAM::Errors::NoSuchEntity
               pw = Password.pronounceable(12..14)
               retries = 0
               begin
-                MU::Cloud::AWS.iam.create_login_profile(
+                MU::Cloud::AWS.iam(credentials: @config['credentials']).create_login_profile(
                   user_name: @mu_name,
                   password: pw
                 )
@@ -101,11 +101,11 @@ module MU
           end
 
           if @config['create_api_keys']
-            resp = MU::Cloud::AWS.iam.list_access_keys(
+            resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_access_keys(
               user_name: @mu_name
             )
             if resp.access_key_metadata.size == 0
-              resp = MU::Cloud::AWS.iam.create_access_key(
+              resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).create_access_key(
                 user_name: @mu_name
               )
               scratchitem = MU::Master.storeScratchPadSecret("AWS Access Key and Secret for user #{@mu_name}:\nKEY: #{resp.access_key.access_key_id}\nSECRET: #{resp.access_key.secret_access_key}")
@@ -124,61 +124,68 @@ module MU
         # Return the metadata for this user cofiguration
         # @return [Hash]
         def notify
-          descriptor = MU.structToHash(MU::Cloud::AWS.iam.get_user(user_name: @mu_name).user)
+          descriptor = MU.structToHash(MU::Cloud::AWS.iam(credentials: @config['credentials']).get_user(user_name: @mu_name).user)
           descriptor["cloud_id"] = @mu_name
           descriptor
         end
 
+        # Does this resource type exist as a global (cloud-wide) artifact, or
+        # is it localized to a region/zone?
+        # @return [Boolean]
+        def self.isGlobal?
+          true
+        end
+
         # Remove all users associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
 
           # XXX this doesn't belong here; maybe under roles, maybe as its own stupid first-class resource
-          resp = MU::Cloud::AWS.iam.list_policies(
+          resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
             path_prefix: "/"+MU.deploy_id+"/"
           )
           if resp and resp.policies
             resp.policies.each { |policy|
               MU.log "Deleting policy /#{MU.deploy_id}/#{policy.policy_name}"
               if !noop
-                attachments = MU::Cloud::AWS.iam.list_entities_for_policy(
+                attachments = MU::Cloud::AWS.iam(credentials: credentials).list_entities_for_policy(
                   policy_arn: policy.arn
                 )
                 attachments.policy_users.each { |u|
-                  MU::Cloud::AWS.iam.detach_user_policy(
+                  MU::Cloud::AWS.iam(credentials: credentials).detach_user_policy(
                     user_name: u.user_name,
                     policy_arn: policy.arn
                   )
                 }
                 attachments.policy_groups.each { |g|
-                  MU::Cloud::AWS.iam.detach_role_policy(
+                  MU::Cloud::AWS.iam(credentials: credentials).detach_role_policy(
                     group_name: g.group_name,
                     policy_arn: policy.arn
                   )
                 }
                 attachments.policy_roles.each { |r|
-                  MU::Cloud::AWS.iam.detach_role_policy(
+                  MU::Cloud::AWS.iam(credentials: credentials).detach_role_policy(
                     role_name: r.role_name,
                     policy_arn: policy.arn
                   )
                 }
-                MU::Cloud::AWS.iam.delete_policy(
+                MU::Cloud::AWS.iam(credentials: credentials).delete_policy(
                   policy_arn: policy.arn
                 )
               end
             }
           end
 
-          resp = MU::Cloud::AWS.iam.list_users
+          resp = MU::Cloud::AWS.iam(credentials: credentials).list_users
 
           # XXX this response includes a tags attribute, but it's always empty,
           # even when the user is tagged. So we go through the extra call for
           # each user. Inefficient. Probably Amazon's bug.
           resp.users.each { |u|
-            tags = MU::Cloud::AWS.iam.list_user_tags(
+            tags = MU::Cloud::AWS.iam(credentials: credentials).list_user_tags(
               user_name: u.user_name
             ).tags
             has_nodelete = false
@@ -194,43 +201,45 @@ module MU
               MU.log "Deleting IAM user #{u.path}#{u.user_name}"
               if !@noop
                 begin
-                  groups = MU::Cloud::AWS.iam.list_groups_for_user(
+                  groups = MU::Cloud::AWS.iam(credentials: credentials).list_groups_for_user(
                     user_name: u.user_name
                   ).groups
 
                   groups.each { |g|
-                    MU::Cloud::AWS.iam.remove_user_from_group(
+                    MU::Cloud::AWS.iam(credentials: credentials).remove_user_from_group(
                       user_name: u.user_name,
                       group_name: g.group_name
                     )
                   }
-                  profile = MU::Cloud::AWS.iam.get_login_profile(
+                  profile = MU::Cloud::AWS.iam(credentials: credentials).get_login_profile(
                     user_name: u.user_name
                   )
                   MU.log "Deleting IAM login profile for #{u.user_name}"
-                  MU::Cloud::AWS.iam.delete_login_profile(
+                  MU::Cloud::AWS.iam(credentials: credentials).delete_login_profile(
                     user_name: u.user_name
                   )
+                rescue Aws::IAM::Errors::EntityTemporarilyUnmodifiable
+                  sleep 10
+                  retry
                 rescue Aws::IAM::Errors::NoSuchEntity
                 end
-                keys = MU::Cloud::AWS.iam.list_access_keys(
+                keys = MU::Cloud::AWS.iam(credentials: credentials).list_access_keys(
                   user_name: u.user_name
                 )
                 if keys.access_key_metadata.size > 0
                   keys.access_key_metadata.each { |key|
                     MU.log "Deleting IAM access key #{key.access_key_id} for #{u.user_name}"
-                    keys = MU::Cloud::AWS.iam.delete_access_key(
+                    keys = MU::Cloud::AWS.iam(credentials: credentials).delete_access_key(
                       user_name: u.user_name,
                       access_key_id: key.access_key_id
                     )
                   }
                 end
-                MU::Cloud::AWS.iam.delete_user(user_name: u.user_name)
+                MU::Cloud::AWS.iam(credentials: credentials).delete_user(user_name: u.user_name)
               end
             end
           }
 
-#          MU.log "CLEANUP CALLED ON AWS::USER", MU::WARN, details: resp
         end
 
         # Canonical Amazon Resource Number for this resource
@@ -244,7 +253,7 @@ module MU
         # @param region [String]: The cloud provider region.
         # @param flags [Hash]: Optional flags
         # @return [OpenStruct]: The cloud provider's complete descriptions of matching user group.
-        def self.find(cloud_id: nil, region: MU.curRegion, flags: {})
+        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
           found = nil
 
           begin

data/modules/mu/clouds/aws/userdata/linux.erb

@@ -20,94 +20,99 @@ region="`curl -s http://169.254.169.254/latest/meta-data/placement/availability-
 
 # cleanse inherited ephemeral devices that don't actually exist
 for d in r s t u ;do
-	if [ ! -f "/dev/xvd$d" ];then
-		sed -Ein "s'^(/dev/xvd$d)'#\\1'" /etc/fstab
-	fi
+  if [ ! -f "/dev/xvd$d" ];then
+    sed -Ein "s'^(/dev/xvd$d)'#\\1'" /etc/fstab
+  fi
 done
 
-if [ -f /etc/debian_version ];then
-	if ! grep '^/bin/sh /var/lib/cloud/instance/user-data.txt$' /etc/rc.local > /dev/null;then
-		echo "/bin/sh /var/lib/cloud/instance/user-data.txt" >> /etc/rc.local
-	fi
-	apt-get update -y
-	if [ ! -f /usr/bin/pip ] ;then /usr/bin/apt-get --fix-missing -y install python-pip;fi
-	if [ ! -f /usr/bin/curl ] ;then /usr/bin/apt-get --fix-missing -y install curl;fi
-	AWSCLI=/usr/local/bin/aws
+if ping -c 5 8.8.8.8 > /dev/null; then
+  if [ -f /etc/debian_version ];then
+    if ! grep '^/bin/sh /var/lib/cloud/instance/user-data.txt$' /etc/rc.local > /dev/null;then
+      echo "/bin/sh /var/lib/cloud/instance/user-data.txt" >> /etc/rc.local
+    fi
+    apt-get update -y
+    if [ ! -f /usr/bin/pip ] ;then /usr/bin/apt-get --fix-missing -y install python-pip;fi
+    if [ ! -f /usr/bin/curl ] ;then /usr/bin/apt-get --fix-missing -y install curl;fi
+    AWSCLI=/usr/local/bin/aws
 <% if !$mu.skipApplyUpdates %>
-	if [ ! -f /.mu-installer-ran-updates ];then
-		service ssh stop
-		apt-get --fix-missing -y upgrade 
-		if [ $? -eq 0 ]
-		then
-		  echo "Successfully updated packages"
-		  updates_run=1
-		else
-		  echo "FAILED PACKAGE UPDATE" >&2
-		fi
-		# Proceed regardless
-		touch /.mu-installer-ran-updates
-
-		# XXX this logic works on Ubuntu, is it Debian-friendly?
-		latest_kernel="`ls -1 /boot/vmlinuz-* | sed -r 's/^\/boot\/vmlinuz-//' | tail -1`"
-		running_kernel="`uname -r`"
-		if [ "$running_kernel" != "$latest_kernel" -a "$latest_kernel" != "" ];then
-			need_reboot=1
-		else
-			service ssh start
-		fi
-	fi
+    if [ ! -f /.mu-installer-ran-updates ];then
+      service ssh stop
+      apt-get --fix-missing -y upgrade 
+      if [ $? -eq 0 ]
+      then
+        echo "Successfully updated packages"
+        updates_run=1
+      else
+        echo "FAILED PACKAGE UPDATE" >&2
+      fi
+      # Proceed regardless
+      touch /.mu-installer-ran-updates
+
+      # XXX this logic works on Ubuntu, is it Debian-friendly?
+      latest_kernel="`ls -1 /boot/vmlinuz-* | sed -r 's/^\/boot\/vmlinuz-//' | tail -1`"
+      running_kernel="`uname -r`"
+      if [ "$running_kernel" != "$latest_kernel" -a "$latest_kernel" != "" ];then
+        need_reboot=1
+      else
+        service ssh start
+      fi
+    fi
 <% end %>
-elif [ -x /usr/bin/yum ];then
-	version=`/bin/rpm -qa \*-release | grep -Ei "redhat|centos" | cut -d"-" -f3`
-	if [ -z "$version" ];then
-		amazon_version=`/bin/rpm -qa \*-release | grep -Ei "system-release"| cut -d"-" -f3 | cut -d"." -f1`
-		if [ "$amazon_version" == "2014" ] || [ "$amazon_version" == "2015" ] || [ "$amazon_version" == "2016" ];then
-			version=6
-		fi
-	fi
-	if [ $version -eq 7 ];then
-		userdata_dir="/var/lib/cloud/instances/$instance_id"
-	else
-		userdata_dir="/var/lib/cloud/instance"
-	fi
-	if ! grep "^/bin/sh $userdata_dir/user-data.txt$" /etc/rc.d/rc.local > /dev/null;then
-    cat /etc/rc.d/rc.local | grep -v '^/bin/sh /var/lib/cloud/instances/' >> /tmp/rc.local.$$
-		echo "/bin/sh $userdata_dir/user-data.txt" >> /tmp/rc.local.$$
-    mv /tmp/rc.local.$$ /etc/rc.d/rc.local
-	fi
-
-  sed -i 's/^Defaults.*requiretty$/Defaults   !requiretty/' /etc/sudoers
-
-	if [ $version == 7 ];then
-		chmod 755 /etc/rc.d/rc.local
-    systemctl reset-failed sshd.service
-	fi
-	if [ ! -f /usr/bin/curl ] ;then /usr/bin/yum -y install curl;fi
-	# Ugh, rando EPEL mirror
-	if [ ! -f /etc/yum.repos.d/epel.repo ];then
-		/bin/rpm -ivh http://mirror.metrocast.net/fedora/epel/epel-release-latest-$version.noarch.rpm
-	fi
+  elif [ -x /usr/bin/yum ];then
+    version=`/bin/rpm -qa \*-release | grep -Ei "redhat|centos" | cut -d"-" -f3`
+    if [ -z "$version" ];then
+      amazon_version=`/bin/rpm -qa \*-release | grep -Ei "system-release"| cut -d"-" -f3 | cut -d"." -f1`
+      if [ "$amazon_version" == "2014" ] || [ "$amazon_version" == "2015" ] || [ "$amazon_version" == "2016" ];then
+        version=6
+      fi
+    fi
+    if [ $version -eq 7 ];then
+      userdata_dir="/var/lib/cloud/instances/$instance_id"
+    else
+      userdata_dir="/var/lib/cloud/instance"
+    fi
+    if ! grep "^/bin/sh $userdata_dir/user-data.txt$" /etc/rc.d/rc.local > /dev/null;then
+      cat /etc/rc.d/rc.local | grep -v '^/bin/sh /var/lib/cloud/instances/' >> /tmp/rc.local.$$
+      echo "/bin/sh $userdata_dir/user-data.txt" >> /tmp/rc.local.$$
+      mv /tmp/rc.local.$$ /etc/rc.d/rc.local
+    fi
+
+    sed -i 's/^Defaults.*requiretty$/Defaults   !requiretty/' /etc/sudoers
+
+    if [ $version == 7 ];then
+      chmod 755 /etc/rc.d/rc.local
+      systemctl reset-failed sshd.service
+    fi
+    if [ ! -f /usr/bin/curl ] ;then /usr/bin/yum -y install curl;fi
+    # Ugh, rando EPEL mirror
+    if [ ! -f /etc/yum.repos.d/epel.repo ];then
+      /bin/rpm -ivh http://mirror.metrocast.net/fedora/epel/epel-release-latest-$version.noarch.rpm
+    fi
 <% if !$mu.skipApplyUpdates %>
-	if [ ! -f /.mu-installer-ran-updates ];then
-		service sshd stop
-		kernel_update=`yum list updates | grep kernel`
-		yum -y update
-		if [ $? -eq 0 ]
-		then
-		  echo "Successfully updated packages"
-		  updates_run=1
-		else
-		  echo "FAILED PACKAGE UPDATE" >&2
-		fi
-		# Proceed regardless
-		touch /.mu-installer-ran-updates
-		if [ -n "$kernel_update" ]; then
-			need_reboot=1
-		else
-			service sshd start
-		fi
-	fi
+    if [ ! -f /.mu-installer-ran-updates ];then
+      service sshd stop
+      kernel_update=`yum list updates | grep kernel`
+      yum -y update
+      if [ $? -eq 0 ]
+      then
+        echo "Successfully updated packages"
+        updates_run=1
+      else
+        echo "FAILED PACKAGE UPDATE" >&2
+      fi
+      # Proceed regardless
+      touch /.mu-installer-ran-updates
+      if [ -n "$kernel_update" ]; then
+        need_reboot=1
+      else
+        service sshd start
+      fi
+    fi
+  fi
 <% end %>
+else
+  /bin/logger "***** Unable to verify internet connectivity, skipping package updates from userdata"
+  touch /.mu-installer-ran-updates
 fi
 
 AWSCLI='command -v aws'
@@ -140,24 +145,27 @@ umask 0077
 
 # Install Chef now, because why not?
 if [ ! -f /opt/chef/embedded/bin/ruby ];then
-	curl https://www.chef.io/chef/install.sh > chef-install.sh
-	set +e
-	# We may run afoul of a synchronous bootstrap process doing the same thing. So
-	# wait until we've managed to run successfully.
-	while ! sh chef-install.sh -v <%= MU.chefVersion %>;do
-		sleep 10
-	done
-	touch /opt/mu_installed_chef
-	set -e
+  curl https://www.chef.io/chef/install.sh > chef-install.sh
+  set +e
+  # We may run afoul of a synchronous bootstrap process doing the same thing. So
+  # wait until we've managed to run successfully.
+  while ! sh chef-install.sh -v <%= MU.chefVersion %>;do
+    sleep 10
+  done
+  touch /opt/mu_installed_chef
+  set -e
 fi
 
 <% if !$mu.skipApplyUpdates %>
 if [ "$need_reboot" == "1" ];then
-	shutdown -r now "Applying new kernel"
+  shutdown -r now "Applying new kernel"
 fi
 <% end %>
+fi
 
-$AWSCLI --region="$region" s3 cp s3://<%= MU.adminBucketName %>/<%= $mu.muID %>-secret .
+if [ "$AWSCLI" != "" ];then
+  $AWSCLI --region="$region" s3 cp s3://<%= MU.adminBucketName %>/<%= $mu.muID %>-secret .
+fi
 
 echo '
 require "openssl"

data/modules/mu/clouds/aws/vpc.rb

@@ -46,21 +46,21 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def create
           MU.log "Creating VPC #{@mu_name}", details: @config
-          resp = MU::Cloud::AWS.ec2(@config['region']).create_vpc(cidr_block: @config['ip_block']).vpc
+          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc(cidr_block: @config['ip_block']).vpc
           vpc_id = @config['vpc_id'] = resp.vpc_id
 
-          MU::MommaCat.createStandardTags(vpc_id, region: @config['region'])
-          MU::MommaCat.createTag(vpc_id, "Name", @mu_name, region: @config['region'])
+          MU::MommaCat.createStandardTags(vpc_id, region: @config['region'], credentials: @config['credentials'])
+          MU::MommaCat.createTag(vpc_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
 
           if @config['tags']
             @config['tags'].each { |tag|
-              MU::MommaCat.createTag(vpc_id, tag['key'], tag['value'], region: @config['region'])
+              MU::MommaCat.createTag(vpc_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
             }
           end
 
           if @config['optional_tags']
             MU::MommaCat.listOptionalTags.each { |key, value|
-              MU::MommaCat.createTag(vpc_id, key, value, region: @config['region'])
+              MU::MommaCat.createTag(vpc_id, key, value, region: @config['region'], credentials: @config['credentials'])
             }
           end
 
@@ -68,10 +68,10 @@ module MU
             begin
               MU.log "Waiting for VPC #{@mu_name} (#{vpc_id}) to be available", MU::NOTICE
               sleep 5
-              resp = MU::Cloud::AWS.ec2(@config['region']).describe_vpcs(vpc_ids: [vpc_id]).vpcs.first
+              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpcs(vpc_ids: [vpc_id]).vpcs.first
             end while resp.state != "available"
             # There's a default route table that comes with. Let's tag it.
-            resp = MU::Cloud::AWS.ec2(@config['region']).describe_route_tables(
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
               filters: [
                 {
                   name: "vpc-id",
@@ -80,18 +80,18 @@ module MU
               ]
             )
             resp.route_tables.each { |rtb|
-              MU::MommaCat.createTag(rtb.route_table_id, "Name", @mu_name+"-#DEFAULTPRIV", region: @config['region'])
+              MU::MommaCat.createTag(rtb.route_table_id, "Name", @mu_name+"-#DEFAULTPRIV", region: @config['region'], credentials: @config['credentials'])
               if @config['tags']
                 @config['tags'].each { |tag|
-                  MU::MommaCat.createTag(rtb.route_table_id, tag['key'], tag['value'], region: @config['region'])
+                  MU::MommaCat.createTag(rtb.route_table_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
                 }
               end
 
-              MU::MommaCat.createStandardTags(rtb.route_table_id, region: @config['region'])
+              MU::MommaCat.createStandardTags(rtb.route_table_id, region: @config['region'], credentials: @config['credentials'])
 
               if @config['optional_tags']
                 MU::MommaCat.listOptionalTags.each { |key, value|
-                  MU::MommaCat.createTag(rtb.route_table_id, key, value, region: @config['region'])
+                  MU::MommaCat.createTag(rtb.route_table_id, key, value, region: @config['region'], credentials: @config['credentials'])
                 }
               end
             }
@@ -101,24 +101,24 @@ module MU
 
           if @config['create_internet_gateway']
             MU.log "Creating Internet Gateway #{@mu_name}"
-            resp = MU::Cloud::AWS.ec2(@config['region']).create_internet_gateway
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_internet_gateway
             internet_gateway_id = resp.internet_gateway.internet_gateway_id
             sleep 5
-            MU::MommaCat.createStandardTags(internet_gateway_id, region: @config['region'])
-            MU::MommaCat.createTag(internet_gateway_id, "Name", @mu_name, region: @config['region'])
+            MU::MommaCat.createStandardTags(internet_gateway_id, region: @config['region'], credentials: @config['credentials'])
+            MU::MommaCat.createTag(internet_gateway_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
             if @config['tags']
               @config['tags'].each { |tag|
-                MU::MommaCat.createTag(internet_gateway_id, tag['key'], tag['value'], region: @config['region'])
+                MU::MommaCat.createTag(internet_gateway_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
               }
             end
 
             if @config['optional_tags']
               MU::MommaCat.listOptionalTags.each { |key, value|
-                MU::MommaCat.createTag(internet_gateway_id, key, value, region: @config['region'])
+                MU::MommaCat.createTag(internet_gateway_id, key, value, region: @config['region'], credentials: @config['credentials'])
               }
             end
 
-            MU::Cloud::AWS.ec2(@config['region']).attach_internet_gateway(vpc_id: vpc_id, internet_gateway_id: internet_gateway_id)
+            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_internet_gateway(vpc_id: vpc_id, internet_gateway_id: internet_gateway_id)
             @config['internet_gateway_id'] = internet_gateway_id
           end
 
@@ -142,7 +142,7 @@ module MU
               config[:policy_document] = statement.to_json
             end
 
-            resp = MU::Cloud::AWS.ec2(@config['region']).create_vpc_endpoint(config).vpc_endpoint
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_endpoint(config).vpc_endpoint
             endpoint_id = resp.vpc_endpoint_id
             MU.log "Creating VPC endpoint #{endpoint_id}"
             attempts = 0
@@ -151,7 +151,7 @@ module MU
               MU.log "Waiting for VPC endpoint #{endpoint_id} to become available" if attempts % 5 == 0
               sleep 10
               begin
-                resp = MU::Cloud::AWS.ec2(@config['region']).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint_id]).vpc_endpoints.first
+                resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint_id]).vpc_endpoints.first
               rescue Aws::EmptyStructure, NoMethodError
                 sleep 5
                 retry
@@ -168,7 +168,7 @@ module MU
             logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
 
             MU.log "Enabling traffic logging on VPC #{@mu_name} to log group #{loggroup.mu_name}"
-            MU::Cloud::AWS.ec2(@config['region']).create_flow_logs(
+            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_flow_logs(
               resource_ids: [@cloud_id],
               resource_type: "VPC",
               traffic_type: "ALL",
@@ -187,7 +187,7 @@ module MU
             @config['subnets'].each { |subnet|
               subnet_name = @config['name']+"-"+subnet['name']
               MU.log "Creating Subnet #{subnet_name} (#{subnet['ip_block']})", details: subnet
-              azs = MU::Cloud::AWS.listAZs if azs.size == 0
+              azs = MU::Cloud::AWS.listAZs(region: @config['region'], credentials: @config['credentials']) if azs.size == 0
               if !subnet['availability_zone'].nil?
                 az = subnet['availability_zone']
               else
@@ -196,23 +196,23 @@ module MU
 
               subnetthreads << Thread.new {
                 MU.dupGlobals(parent_thread_id)
-                resp = MU::Cloud::AWS.ec2(@config['region']).create_subnet(
+                resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_subnet(
                     vpc_id: vpc_id,
                     cidr_block: subnet['ip_block'],
                     availability_zone: az
                 ).subnet
                 subnet_id = subnet['subnet_id'] = resp.subnet_id
-                MU::MommaCat.createStandardTags(subnet_id, region: @config['region'])
-                MU::MommaCat.createTag(subnet_id, "Name", @mu_name+"-"+subnet['name'], region: @config['region'])
+                MU::MommaCat.createStandardTags(subnet_id, region: @config['region'], credentials: @config['credentials'])
+                MU::MommaCat.createTag(subnet_id, "Name", @mu_name+"-"+subnet['name'], region: @config['region'], credentials: @config['credentials'])
                 if @config['tags']
                   @config['tags'].each { |tag|
-                    MU::MommaCat.createTag(subnet_id, tag['key'], tag['value'], region: @config['region'])
+                    MU::MommaCat.createTag(subnet_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
                   }
                 end
 
                 if @config['optional_tags']
                   MU::MommaCat.listOptionalTags.each { |key, value|
-                    MU::MommaCat.createTag(subnet_id, key, value, region: @config['region'])
+                    MU::MommaCat.createTag(subnet_id, key, value, region: @config['region'], credentials: @config['credentials'])
                   }
                 end
 
@@ -222,7 +222,7 @@ module MU
                     begin
                       MU.log "Waiting for Subnet #{subnet_name} (#{subnet_id}) to be available", MU::NOTICE
                       sleep 5
-                      resp = MU::Cloud::AWS.ec2(@config['region']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
+                      resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
                     rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
                       sleep 10
                       retry
@@ -233,7 +233,7 @@ module MU
                     MU.log "Got bogus Aws::EmptyResponse error on #{subnet_id} (retries used: #{retries}/3)", MU::WARN
                     retries = retries + 1
                     sleep 5
-                    resp = MU::Cloud::AWS.ec2(@config['region']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
+                    resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
                     retry
                   else
                     raise e
@@ -252,7 +252,7 @@ module MU
                   MU.log "Associating Route Table '#{subnet['route_table']}' (#{routes[subnet['route_table']]['route_table_id']}) with #{subnet_name}"
                   retries = 0
                   begin
-                    MU::Cloud::AWS.ec2(@config['region']).associate_route_table(
+                    MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_route_table(
                         route_table_id: routes[subnet['route_table']]['route_table_id'],
                         subnet_id: subnet_id
                     )
@@ -268,7 +268,7 @@ module MU
                 end
                 retries = 0
                 begin
-                  resp = MU::Cloud::AWS.ec2(@config['region']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
+                  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
                 rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
                   if retries < 10
                     MU.log "Got #{e.inspect}, waiting and retrying", MU::WARN
@@ -282,7 +282,7 @@ module MU
                 if subnet['is_public'] && subnet['create_nat_gateway']
                   MU::MommaCat.lock("nat-gateway-eipalloc")
                   filters = [{name: "domain", values: ["vpc"]}]
-                  eips = MU::Cloud::AWS.ec2(@config['region']).describe_addresses(filters: filters).addresses
+                  eips = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(filters: filters).addresses
                   allocation_id = nil
                   eips.each { |eip|
                     next if !eip.association_id.nil? and !eip.association_id.empty?
@@ -295,12 +295,12 @@ module MU
                   }
 
                   if allocation_id.nil?
-                    allocation_id = MU::Cloud::AWS.ec2(@config['region']).allocate_address(domain: "vpc").allocation_id
+                    allocation_id = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).allocate_address(domain: "vpc").allocation_id
                     MU::MommaCat.lock(allocation_id, false, true)
                   end
 
                   allocation_ids << allocation_id
-                  resp = MU::Cloud::AWS.ec2(@config['region']).create_nat_gateway(
+                  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_nat_gateway(
                     subnet_id: subnet['subnet_id'],
                     allocation_id: allocation_id
                   ).nat_gateway
@@ -312,7 +312,7 @@ module MU
                     MU.log "Waiting for nat gateway #{nat_gateway_id} () to become available (EIP allocation: #{allocation_id})" if attempts % 5 == 0
                     sleep 30
                     begin
-                      resp = MU::Cloud::AWS.ec2(@config['region']).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
+                      resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
                     rescue Aws::EmptyStructure, NoMethodError
                       sleep 5
                       retry
@@ -332,7 +332,7 @@ module MU
                 if subnet.has_key?("map_public_ips")
                   retries = 0
                   begin
-                    resp = MU::Cloud::AWS.ec2(@config['region']).modify_subnet_attribute(
+                    resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_subnet_attribute(
                       subnet_id: subnet_id,
                       map_public_ip_on_launch: {
                         value: subnet['map_public_ips'],
@@ -353,7 +353,7 @@ module MU
                   loggroup = @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs")
                   logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
                   MU.log "Enabling traffic logging on Subnet #{subnet_name} in VPC #{@mu_name} to log group #{loggroup.mu_name}"
-                  MU::Cloud::AWS.ec2(@config['region']).create_flow_logs(
+                  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_flow_logs(
                     resource_ids: [subnet_id],
                     resource_type: "Subnet",
                     traffic_type: subnet["traffic_type_to_log"],
@@ -387,7 +387,7 @@ module MU
 
                           MU.log "Creating route for #{route['destination_network']} through NAT gatway #{gateway['id']}", details: route_config
                           begin
-                            resp = MU::Cloud::AWS.ec2(@config['region']).create_route(route_config)
+                            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
                           rescue Aws::EC2::Errors::RouteAlreadyExists => e
                             MU.log "Attempt to create duplicate route to #{route['destination_network']} for #{gateway['id']} in #{rtb['route_table_id']}", MU::WARN
                           end
@@ -402,14 +402,14 @@ module MU
 
           if @config['enable_dns_support']
             MU.log "Enabling DNS support in #{@mu_name}"
-            MU::Cloud::AWS.ec2(@config['region']).modify_vpc_attribute(
+            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
                 vpc_id: vpc_id,
                 enable_dns_support: {value: @config['enable_dns_support']}
             )
           end
           if @config['enable_dns_hostnames']
             MU.log "Enabling DNS hostnames in #{@mu_name}"
-            MU::Cloud::AWS.ec2(@config['region']).modify_vpc_attribute(
+            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
                 vpc_id: vpc_id,
                 enable_dns_hostnames: {value: @config['enable_dns_hostnames']}
             )
@@ -435,33 +435,33 @@ module MU
               dhcpopts << {key: "netbios-name-servers", values: @config['dhcp']['netbios_servers']}
             end
 
-            resp = MU::Cloud::AWS.ec2(@config['region']).create_dhcp_options(
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_dhcp_options(
                 dhcp_configurations: dhcpopts
             )
             dhcpopt_id = resp.dhcp_options.dhcp_options_id
-            MU::MommaCat.createStandardTags(dhcpopt_id, region: @config['region'])
-            MU::MommaCat.createTag(dhcpopt_id, "Name", @mu_name, region: @config['region'])
+            MU::MommaCat.createStandardTags(dhcpopt_id, region: @config['region'], credentials: @config['credentials'])
+            MU::MommaCat.createTag(dhcpopt_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
 
             if @config['tags']
               @config['tags'].each { |tag|
-                MU::MommaCat.createTag(dhcpopt_id, tag['key'], tag['value'], region: @config['region'])
+                MU::MommaCat.createTag(dhcpopt_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
               }
             end
 
             if @config['optional_tags']
               MU::MommaCat.listOptionalTags.each { |key, value|
-                MU::MommaCat.createTag(dhcpopt_id, key, value, region: @config['region'])
+                MU::MommaCat.createTag(dhcpopt_id, key, value, region: @config['region'], credentials: @config['credentials'])
               }
             end
 
-            MU::Cloud::AWS.ec2(@config['region']).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id: vpc_id)
+            MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id: vpc_id)
           end
           notify
 
           if !MU::Cloud::AWS.isGovCloud?(@config['region'])
-            mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu").values.first
+            mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: @config['credentials']).values.first
             if !mu_zone.nil?
-              MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: vpc_id, region: @config['region'])
+              MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: vpc_id, region: @config['region'], credentials: @config['credentials'])
             end
           end
 					loadSubnets
@@ -472,7 +472,7 @@ module MU
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU.account_number+":vpc/"+@cloud_id
+          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":vpc/"+@cloud_id
         end
 
         # Describe this VPC
@@ -489,73 +489,99 @@ module MU
           if !@config['peers'].nil? and @config['peers'].size > 0
             @config['peers'].each { |peer|
               peer_obj = nil
+              peer_id = nil
+
               begin
-                if peer['account'].nil? or peer['account'] == MU.account_number
+                # If we know this to be a sibling VPC elsewhere in our stack,
+                # go fetch it, and fix it if we've been misconfigured with a
+                # duplicate peering connection
+                if peer['vpc']['vpc_name'] and !peer['account']
+                  peer_obj = @deploy.findLitterMate(name: peer['vpc']['vpc_name'], type: "vpcs")
+                  if peer_obj
+                    if peer_obj.config['peers']
+                      skipme = false
+                      peer_obj.config['peers'].each { |peerpeer|
+                        if peerpeer['vpc']['vpc_name'] == @config['name'] and
+                           (peer['vpc']['vpc_name'] <=> @config['name']) == -1
+                          skipme = true
+                          MU.log "VPCs #{peer['vpc']['vpc_name']} and #{@config['name']} both declare mutual peering connection, ignoring #{@config['name']}'s redundant declaration", MU::DEBUG
+# XXX and if deploy_id matches or is unset
+                        end
+                      }
+                    end
+                    next if skipme
+                    peer['account'] = MU::Cloud::AWS.credToAcct(peer_obj.credentials)
+                    peer['vpc']['vpc_id'] = peer_obj.cloud_id
+                  end
+                end
+
+                # If we still don't know our peer's vpc identifier, go fishing
+                if !peer_obj
                   tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
                   if peer['vpc']['deploy_id'].nil? and peer['vpc']['vpc_id'].nil? and tag_key.nil?
                     peer['vpc']['deploy_id'] = @deploy.deploy_id
                   end
                   peer_obj = MU::MommaCat.findStray(
-                      "AWS",
-                      "vpcs",
-                      deploy_id: peer['vpc']['deploy_id'],
-                      cloud_id: peer['vpc']['vpc_id'],
-                      name: peer['vpc']['vpc_name'],
-                      tag_key: tag_key,
-                      tag_value: tag_value,
-                      dummy_ok: true,
-                      region: peer['vpc']['region']
+                    "AWS",
+                    "vpcs",
+                    deploy_id: peer['vpc']['deploy_id'],
+                    cloud_id: peer['vpc']['vpc_id'],
+# XXX we need a credentials argument here... maybe
+                    name: peer['vpc']['vpc_name'],
+                    tag_key: tag_key,
+                    tag_value: tag_value,
+                    dummy_ok: true,
+                    region: peer['vpc']['region']
                   )
                   raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil? or peer_obj.first.nil?
                   peer_obj = peer_obj.first
-                  peer_id = peer_obj.cloud_id
-
-                  MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id}"
-                  resp = MU::Cloud::AWS.ec2(@config['region']).create_vpc_peering_connection(
-                    vpc_id: @cloud_id,
-                    peer_vpc_id: peer_id
-                  )
-                else
-                  peer_id = peer['vpc']['vpc_id']
-                  MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id} in account #{peer['account']}", MU::INFO, details: peer
-                  resp = MU::Cloud::AWS.ec2(@config['region']).create_vpc_peering_connection(
-                      vpc_id: @cloud_id,
-                      peer_vpc_id: peer_id,
-                      peer_owner_id: peer['account']
-                  )
+                  peer['account'] ||= MU::Cloud::AWS.credToAcct(peer_obj.credentials)
+                  peer['vpc']['vpc_id'] ||= peer_obj.cloud_id
                 end
+
+                peer_id = peer['vpc']['vpc_id']
+                peer['account'] ||= MU::Cloud::AWS.account_number
+
+                MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id} in account #{MU::Cloud::AWS.credToAcct(@config['credentials'])}) to #{peer_id} in account #{peer['account']}", MU::INFO, details: peer
+                resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_peering_connection(
+                  vpc_id: @cloud_id,
+                  peer_vpc_id: peer_id,
+                  peer_owner_id: peer['account']
+                )
+
               rescue Aws::EC2::Errors::VpcPeeringConnectionAlreadyExists => e
                 MU.log "Attempt to create duplicate peering connection to #{peer_id} from VPC #{@config['name']}", MU::WARN
               end
-              peering_name = @deploy.getResourceName(@config['name']+"-PEER-"+peer_id)
+              peering_name = @deploy.getResourceName(@config['name']+"-PEER-"+peer['vpc']['vpc_id'])
 
               peering_id = resp.vpc_peering_connection.vpc_peering_connection_id
-              MU::MommaCat.createStandardTags(peering_id, region: @config['region'])
-              MU::MommaCat.createTag(peering_id, "Name", peering_name, region: @config['region'])
+              MU::MommaCat.createStandardTags(peering_id, region: @config['region'], credentials: @config['credentials'])
+              MU::MommaCat.createTag(peering_id, "Name", peering_name, region: @config['region'], credentials: @config['credentials'])
 
               if @config['optional_tags']
                 MU::MommaCat.listOptionalTags.each { |key, value|
-                  MU::MommaCat.createTag(peering_id, key, value, region: @config['region'])
+                  MU::MommaCat.createTag(peering_id, key, value, region: @config['region'], credentials: @config['credentials'])
                 }
               end
 
               if @config['tags']
                 @config['tags'].each { |tag|
-                  MU::MommaCat.createTag(peering_id, tag['key'], tag['value'], region: @config['region'])
+                  MU::MommaCat.createTag(peering_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
                 }
               end
 
               # Create routes to our new friend.
-              MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @config['region']).each { |rtb_id|
+              MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @config['region'], credentials: @config['credentials']).each { |rtb_id|
                 my_route_config = {
                   :route_table_id => rtb_id,
                   :destination_cidr_block => peer_obj.cloud_desc.cidr_block,
                   :vpc_peering_connection_id => peering_id
                 }
                 begin
-                  resp = MU::Cloud::AWS.ec2(@config['region']).create_route(my_route_config)
+                  MU.log "Creating peering route to #{peer_obj.cloud_desc.cidr_block} from VPC #{@config['name']}"
+                  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(my_route_config)
                 rescue Aws::EC2::Errors::RouteAlreadyExists => e
-                  rtbdesc = MU::Cloud::AWS.ec2(@config['region']).describe_route_tables(
+                  rtbdesc = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
                     route_table_ids: [rtb_id]
                   ).route_tables.first
                   rtbdesc.routes.each { |r|
@@ -569,35 +595,51 @@ module MU
                     end
                   }
                 end
-              }
+              } # MU::Cloud::AWS::VPC.listAllSubnetRouteTables
 
               begin
-                cnxn = MU::Cloud::AWS.ec2(@config['region']).describe_vpc_peering_connections(
-                    vpc_peering_connection_ids: [peering_id]
+                cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
+                  vpc_peering_connection_ids: [peering_id]
                 ).vpc_peering_connections.first
 
                 if cnxn.status.code == "pending-acceptance"
                   if ((!peer_obj.nil? and !peer_obj.deploydata.nil? and peer_obj.deploydata['auto_accept_peers']) or $MU_CFG['allow_invade_foreign_vpcs'])
                     MU.log "Auto-accepting peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id}", MU::NOTICE
                     begin
-                      MU::Cloud::AWS.ec2(@config['region']).accept_vpc_peering_connection(
-                          vpc_peering_connection_id: peering_id
+                      MU::Cloud::AWS.ec2(region: @config['region'], credentials: peer['account']).accept_vpc_peering_connection(
+                        vpc_peering_connection_id: peering_id
                       )
+                      if peer['account'] != MU::Cloud::AWS.credToAcct(@config['credentials'])
+                        # this seems to take a while across accounts
+                        sleep 5
+                      end
+                      cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
+                        vpc_peering_connection_ids: [peering_id]
+                      ).vpc_peering_connections.first
                     rescue Aws::EC2::Errors::VpcPeeringConnectionAlreadyExists => e
                       MU.log "Attempt to create duplicate peering connection to #{peer_id} from VPC #{@config['name']}", MU::WARN
                     end
 
                     # Create routes back from our new friend to us.
-                    MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_id, region: peer['vpc']['region']).each { |rtb_id|
+                    MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_id, region: peer['vpc']['region'], credentials: peer['account']).uniq.each { |rtb_id|
                       peer_route_config = {
-                          :route_table_id => rtb_id,
-                          :destination_cidr_block => @config['ip_block'],
-                          :vpc_peering_connection_id => peering_id
+                        :route_table_id => rtb_id,
+                        :destination_cidr_block => @config['ip_block'],
+                        :vpc_peering_connection_id => peering_id
                       }
                       begin
-                        resp = MU::Cloud::AWS.ec2(@config['region']).create_route(peer_route_config)
+                        resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: peer['account']).create_route(peer_route_config)
                       rescue Aws::EC2::Errors::RouteAlreadyExists => e
-                        MU.log "Attempt to create duplicate route to #{@config['ip_block']} from VPC #{peer_id}", MU::WARN
+                        rtbdesc = MU::Cloud::AWS.ec2(region: @config['region'], credentials: peer['account']).describe_route_tables(
+                          route_table_ids: [rtb_id]
+                        ).route_tables.first
+                        rtbdesc.routes.each { |r|
+                          if r.destination_cidr_block == @config['ip_block']
+                            if r.vpc_peering_connection_id != peering_id
+                              MU.log "Attempt to create duplicate route to VPC #{@config['name']} (#{@config['ip_block']}) from peer VPC #{peer_id}'s route table #{rtb_id}", MU::ERR
+                            end
+                          end
+                        }
                       end
                     }
                   else
@@ -608,15 +650,16 @@ module MU
                 if cnxn.status.code == "failed" or cnxn.status.code == "rejected" or cnxn.status.code == "expired" or cnxn.status.code == "deleted"
                   MU.log "VPC peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id} #{cnxn.status.code}: #{cnxn.status.message}", MU::ERR
                   begin
-                    MU::Cloud::AWS.ec2(@config['region']).delete_vpc_peering_connection(
-                        vpc_peering_connection_id: peering_id
+                    MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).delete_vpc_peering_connection(
+                      vpc_peering_connection_id: peering_id
                     )
                   rescue Aws::EC2::Errors::InvalidStateTransition => e
                     # XXX apparently this is normal?
                   end
                   raise MuError, "VPC peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id} #{cnxn.status.code}: #{cnxn.status.message}"
                 end
-              end while cnxn.status.code != "active" and !(cnxn.status.code == "pending-acceptance" and (peer_obj.nil? or peer_obj.deploydata.nil? or !peer_obj.deploydata['auto_accept_peers']))
+
+              end while cnxn.status.code != "active" and !((peer_obj.nil? or peer_obj.deploydata.nil? or !peer_obj.deploydata['auto_accept_peers']) and cnxn.status.code == "pending-acceptance")
 
             }
           end
@@ -644,7 +687,7 @@ module MU
                   route_config[:instance_id] = nat_instance.cloud_id
 
                   MU.log "Creating route for #{route['destination_network']} through NAT host #{nat_instance.cloud_id}", details: route_config
-                  resp = MU::Cloud::AWS.ec2(@config['region']).create_route(route_config)
+                  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
                 end
               }
 
@@ -659,7 +702,7 @@ module MU
         # @param tag_key [String]: A tag key to search.
         # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
         # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching VPCs
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
+        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
 
           retries = 0
           map = {}
@@ -668,7 +711,7 @@ module MU
 
             if tag_value
               MU.log "Searching for VPC by tag:#{tag_key}=#{tag_value}", MU::DEBUG
-              resp = MU::Cloud::AWS.ec2(region).describe_vpcs(
+              resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs(
                 filters: [
                   {name: "tag:#{tag_key}", values: [tag_value]}
                 ]
@@ -686,7 +729,7 @@ module MU
             if !cloud_id.nil?
               MU.log "Searching for VPC id '#{cloud_id}' in #{region}", MU::DEBUG
               begin
-                resp = MU::Cloud::AWS.ec2(region).describe_vpcs(vpc_ids: [cloud_id.to_s])
+                resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs(vpc_ids: [cloud_id.to_s])
                 resp.vpcs.each { |vpc|
                   map[vpc.vpc_id] = vpc
                 }
@@ -720,7 +763,7 @@ module MU
         # @return [Array<Hash>]: A list of cloud provider identifiers of subnets associated with this VPC.
         def loadSubnets
           if @cloud_id
-            resp = MU::Cloud::AWS.ec2(@config['region']).describe_subnets(
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(
               filters: [
                 { name: "vpc-id", values: [@cloud_id] }
               ]
@@ -741,6 +784,7 @@ module MU
               @config['subnets'].each { |subnet|
                 subnet['mu_name'] = @mu_name+"-"+subnet['name'] if !subnet.has_key?("mu_name")
                 subnet['region'] = @config['region']
+                subnet['credentials'] = @config['credentials']
                 if !resp.nil? and !resp.data.nil? and !resp.data.subnets.nil?
                   resp.data.subnets.each { |desc|
                     if desc.cidr_block == subnet["ip_block"]
@@ -776,6 +820,7 @@ module MU
                 subnet["tags"] = MU.structToHash(desc.tags)
                 subnet["cloud_id"] = desc.subnet_id
                 subnet['region'] = @config['region']
+                subnet['credentials'] = @config['credentials']
                 if !ext_ids.include?(desc.subnet_id)
                   @subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
                 end
@@ -790,16 +835,16 @@ module MU
         # @param nat_filter_key [String]: A cloud provider filter to help identify the resource, used in conjunction with nat_filter_value.
         # @param nat_filter_value [String]: A cloud provider filter to help identify the resource, used in conjunction with nat_filter_key.
         # @param region [String]: The cloud provider region of the target instance.
-        def findNat(nat_cloud_id: nil, nat_filter_key: nil, nat_filter_value: nil, region: MU.curRegion)
+        def findNat(nat_cloud_id: nil, nat_filter_key: nil, nat_filter_value: nil, region: MU.curRegion, credentials: nil)
           # Discard the nat_cloud_id if it's an AWS instance ID
           nat_cloud_id = nil if nat_cloud_id && nat_cloud_id.start_with?("i-")
 
           if @gateways.nil?
             @gateways = 
               if nat_cloud_id
-                MU::Cloud::AWS.ec2(region).describe_nat_gateways(nat_gateway_ids: [nat_cloud_id])
+                MU::Cloud::AWS.ec2(region: region, credentials: nil).describe_nat_gateways(nat_gateway_ids: [nat_cloud_id])
               elsif nat_filter_key && nat_filter_value
-                MU::Cloud::AWS.ec2(region).describe_nat_gateways(
+                MU::Cloud::AWS.ec2(region: region, credentials: nil).describe_nat_gateways(
                   filter: [
                     {
                       name: nat_filter_key,
@@ -893,13 +938,13 @@ module MU
         # @param instance [String]: A cloud descriptor for the instance, to save us an API call if we already have it
         # @param region [String]: The cloud provider region of the target instance
         # @return [Array<String>]
-        def self.getInstanceSubnets(instance_id: nil, instance: nil, region: MU.curRegion)
+        def self.getInstanceSubnets(instance_id: nil, instance: nil, region: MU.curRegion, credentials: nil)
           return [] if instance_id.nil? and instance.nil?
           my_subnets = []
 
           if instance.nil?
             begin
-              instance = MU::Cloud::AWS.ec2(region).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
+              instance = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_instances(instance_ids: [instance_id]).reservations.first.instances.first
             rescue NoMethodError, Aws::EC2::Errors::InvalidInstanceIDNotFound => e
               MU.log "Failed to identify instance #{instance_id} in MU::Cloud::AWS::VPC.getInstanceSubnets", MU::WARN
               return []
@@ -923,7 +968,7 @@ module MU
         # @param target_instance [OpenStruct]: The cloud descriptor of the instance to check.
         # @param region [String]: The cloud provider region of the target subnet.
         # @return [Boolean]
-        def self.haveRouteToInstance?(target_instance, region: MU.curRegion)
+        def self.haveRouteToInstance?(target_instance, region: MU.curRegion, credentials: nil)
           return false if target_instance.nil?
           return false if MU.myCloud != "AWS"
           instance_id = target_instance.instance_id
@@ -941,21 +986,20 @@ module MU
 
           return @route_cache[instance_id] if @route_cache.has_key?(instance_id) && @route_cache[instance_id]
           my_subnets = MU::Cloud::AWS::VPC.getInstanceSubnets(instance: MU.myCloudDescriptor)
-          target_subnets = MU::Cloud::AWS::VPC.getInstanceSubnets(instance: target_instance, region: region)
-# XXX make sure accounts for being in different regions
+          target_subnets = MU::Cloud::AWS::VPC.getInstanceSubnets(instance: target_instance, region: region, credentials: credentials)
 
           resp = nil
           my_subnets_key = my_subnets.join(",")
           target_subnets_key = target_subnets.join(",")
           MU::Cloud::AWS::VPC.update_route_tables_cache(my_subnets_key, region: MU.myRegion)
-          MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, region: region)
+          MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, region: region, credentials: credentials)
 
           if MU::Cloud::AWS::VPC.have_route_peered_vpc?(my_subnets_key, target_subnets_key, instance_id)
             return true
           else
             # The cache can be out of date at times, check again without it
             MU::Cloud::AWS::VPC.update_route_tables_cache(my_subnets_key, use_cache: false, region: MU.myRegion)
-            MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, use_cache: false, region: region)
+            MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, use_cache: false, region: region, credentials: credentials)
 
             return MU::Cloud::AWS::VPC.have_route_peered_vpc?(my_subnets_key, target_subnets_key, instance_id)
           end
@@ -968,7 +1012,7 @@ module MU
         # @param subnet_key [String]: The subnet/subnets route tables will be extracted from.
         # @param use_cache [Boolean]: If to use the existing cache and add records to cache only if missing, or to also replace exising records in cache.
         # @param region [String]: The cloud provider region of the target subnet.
-        def self.update_route_tables_cache(subnet_key, use_cache: true, region: MU.curRegion)
+        def self.update_route_tables_cache(subnet_key, use_cache: true, region: MU.curRegion, credentials: nil)
           @rtb_cache_semaphore.synchronize {
             update = 
               if !use_cache
@@ -980,12 +1024,12 @@ module MU
               end
 
             if update
-              route_tables = MU::Cloud::AWS::VPC.get_route_tables(subnet_ids: subnet_key.split(","), region: region)
+              route_tables = MU::Cloud::AWS::VPC.get_route_tables(subnet_ids: subnet_key.split(","), region: region, credentials: credentials)
 
               if route_tables.empty? && !subnet_key.empty?
-                vpc_id = MU::Cloud::AWS.ec2(region).describe_subnets(subnet_ids: subnet_key.split(",")).subnets.first.vpc_id
+                vpc_id = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_subnets(subnet_ids: subnet_key.split(",")).subnets.first.vpc_id
                 MU.log "No route table associations found for #{subnet_key}, falling back to the default table for #{vpc_id}", MU::NOTICE
-                route_tables = MU::Cloud::AWS::VPC.get_route_tables(vpc_ids: [vpc_id], region: region)
+                route_tables = MU::Cloud::AWS::VPC.get_route_tables(vpc_ids: [vpc_id], region: region, credentials: credentials)
               end
 
               @rtb_cache[subnet_key] = route_tables
@@ -1004,16 +1048,19 @@ module MU
 
           @rtb_cache[source_subnets_key].each { |route_table|
             route_table.routes.each { |route|
-              if route.destination_cidr_block != "0.0.0.0/0" and route.state == "active" and !route.destination_cidr_block.nil?
+              if route.destination_cidr_block != "0.0.0.0/0" and !route.destination_cidr_block.nil?
                 my_routes << NetAddr::IPv4Net.parse(route.destination_cidr_block)
                 if !route.vpc_peering_connection_id.nil?
+                  if route.state == "blackhole"
+                    MU.log "Ignoring blackhole route to #{route.destination_cidr_block} over #{route.vpc_peering_connection_id}", MU::WARN
+                  end
+                  next if route.state != "active"
                   vpc_peer_mapping[route.vpc_peering_connection_id] = route.destination_cidr_block
                 end
               end
             }
           }
           my_routes.uniq!
-
           target_routes = []
           @rtb_cache[target_subnets_key].each { |route_table|
             route_table.routes.each { |route|
@@ -1021,6 +1068,7 @@ module MU
               cidr = NetAddr::IPv4Net.parse(route.destination_cidr_block)
               shared_ip_space = false
               my_routes.each { |my_cidr|
+                target_routes << NetAddr::IPv4Net.parse(route.destination_cidr_block)
                 if my_cidr.contains(NetAddr::IPv4Net.parse(route.destination_cidr_block).nth(2)) or my_cidr.cmp(cidr)
                   shared_ip_space = true
                   break
@@ -1043,10 +1091,10 @@ module MU
         # @param vpc_ids [Array]: The cloud identifier of the VPCs to retrieve route tables for.
         # @param region [String]: The cloud provider region of the target subnet.
         # @return [Array<OpenStruct>]: The cloud provider's complete descriptions of the route tables
-        def self.get_route_tables(subnet_ids: [], vpc_ids: [], region: MU.curRegion)
+        def self.get_route_tables(subnet_ids: [], vpc_ids: [], region: MU.curRegion, credentials: nil)
           resp = []
           if !subnet_ids.empty?
-            resp = MU::Cloud::AWS.ec2(region).describe_route_tables(
+            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_route_tables(
               filters: [
                 {
                   name: "association.subnet-id", 
@@ -1055,7 +1103,7 @@ module MU
               ]
             ).route_tables
           elsif !vpc_ids.empty?
-            resp = MU::Cloud::AWS.ec2(region).describe_route_tables(
+            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_route_tables(
               filters: [
                 {
                   name: "vpc-id", 
@@ -1068,21 +1116,28 @@ module MU
               ]
             ).route_tables
           else
-            resp = MU::Cloud::AWS.ec2(region).describe_route_tables.route_tables
+            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_route_tables.route_tables
           end
 
           return resp
         end
 
+        # Does this resource type exist as a global (cloud-wide) artifact, or
+        # is it localized to a region/zone?
+        # @return [Boolean]
+        def self.isGlobal?
+          false
+        end
+
         # Remove all VPC resources associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
 
           tagfilters = [
-              {name: "tag:MU-ID", values: [MU.deploy_id]}
+            {name: "tag:MU-ID", values: [MU.deploy_id]}
           ]
           if !ignoremaster
             tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
@@ -1091,7 +1146,7 @@ module MU
           vpcs = []
           retries = 0
           begin
-            resp = MU::Cloud::AWS.ec2(region).describe_vpcs(filters: tagfilters).vpcs
+            resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs(filters: tagfilters).vpcs
             vpcs = resp if !resp.empty?
           rescue Aws::EC2::Errors::InvalidVpcIDNotFound => e
             if retries < 5
@@ -1106,8 +1161,8 @@ module MU
             vpcs.each { |vpc|
               # NAT gateways don't have any tags, and we can't assign them a name. Lets find them based on a VPC ID
               gwthreads << Thread.new {
-                purge_nat_gateways(noop, vpc_id: vpc.vpc_id, region: region)
-                purge_endpoints(noop, vpc_id: vpc.vpc_id, region: region)
+                purge_nat_gateways(noop, vpc_id: vpc.vpc_id, region: region, credentials: credentials)
+                purge_endpoints(noop, vpc_id: vpc.vpc_id, region: region, credentials: credentials)
               }
             }
             gwthreads.each { |t|
@@ -1115,12 +1170,12 @@ module MU
             }
           end
 
-          purge_gateways(noop, tagfilters, region: region)
-          purge_routetables(noop, tagfilters, region: region)
-          purge_interfaces(noop, tagfilters, region: region)
-          purge_subnets(noop, tagfilters, region: region)
-          purge_vpcs(noop, tagfilters, region: region)
-          purge_dhcpopts(noop, tagfilters, region: region)
+          purge_gateways(noop, tagfilters, region: region, credentials: credentials)
+          purge_routetables(noop, tagfilters, region: region, credentials: credentials)
+          purge_interfaces(noop, tagfilters, region: region, credentials: credentials)
+          purge_subnets(noop, tagfilters, region: region, credentials: credentials)
+          purge_vpcs(noop, tagfilters, region: region, credentials: credentials)
+          purge_dhcpopts(noop, tagfilters, region: region, credentials: credentials)
 
           unless noop
             MU::Cloud::AWS.iam.list_roles.roles.each{ |role|
@@ -1339,7 +1394,7 @@ module MU
 
           if (!vpc['subnets'] or vpc['subnets'].empty?) and vpc['create_standard_subnets']
             if vpc['availability_zones'].nil? or vpc['availability_zones'].empty?
-              vpc['availability_zones'] = MU::Cloud::AWS.listAZs(vpc['region'])
+              vpc['availability_zones'] = MU::Cloud::AWS.listAZs(region: vpc['region'])
             else
               # turn into a hash so we can use list parameters easily
               vpc['availability_zones'] = vpc['availability_zones'].map { |val| val['zone'] }
@@ -1390,8 +1445,8 @@ module MU
 
 
         # List the route tables for each subnet in the given VPC
-        def self.listAllSubnetRouteTables(vpc_id, region: MU.curRegion)
-          resp = MU::Cloud::AWS.ec2(region).describe_subnets(
+        def self.listAllSubnetRouteTables(vpc_id, region: MU.curRegion, credentials: nil)
+          resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_subnets(
               filters: [
                   {
                       name: "vpc-id",
@@ -1402,7 +1457,7 @@ module MU
 
           subnets = resp.subnets.map { |subnet| subnet.subnet_id }
 
-          tables = MU::Cloud::AWS.ec2(region).describe_route_tables(
+          tables = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_route_tables(
               filters: [
                   {
                       name: "vpc-id",
@@ -1417,7 +1472,7 @@ module MU
 
           if tables.nil? or tables.route_tables.size == 0
             MU.log "No route table associations found for #{subnets}, falling back to the default table for #{vpc_id}", MU::NOTICE
-            tables = MU::Cloud::AWS.ec2(MU.myRegion).describe_route_tables(
+            tables = MU::Cloud::AWS.ec2(region: MU.myRegion).describe_route_tables(
               filters: [
                 {name: "vpc-id", values: [vpc_id]},
                 {name: "association.main", values: ["true"]},
@@ -1440,24 +1495,24 @@ module MU
           vpc_id = @cloud_id
           vpc_name = @config['name']
           MU.setVar("curRegion", @config['region']) if !@config['region'].nil?
-          resp = MU::Cloud::AWS.ec2.create_route_table(vpc_id: vpc_id).route_table
+          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route_table(vpc_id: vpc_id).route_table
           route_table_id = rtb['route_table_id'] = resp.route_table_id
           sleep 5
-          MU::MommaCat.createTag(route_table_id, "Name", vpc_name+"-"+rtb['name'].upcase)
+          MU::MommaCat.createTag(route_table_id, "Name", vpc_name+"-"+rtb['name'].upcase, credentials: @config['credentials'])
 
           if @config['tags']
             @config['tags'].each { |tag|
-              MU::MommaCat.createTag(route_table_id, tag['key'], tag['value'])
+              MU::MommaCat.createTag(route_table_id, tag['key'], tag['value'], credentials: @config['credentials'])
             }
           end
 
           if @config['optional_tags']
             MU::MommaCat.listOptionalTags.each { |key, value|
-              MU::MommaCat.createTag(route_table_id, key, value, region: @config['region'])
+              MU::MommaCat.createTag(route_table_id, key, value, region: @config['region'], credentials: @config['credentials'])
             }
           end
 
-          MU::MommaCat.createStandardTags(route_table_id)
+          MU::MommaCat.createStandardTags(route_table_id, credentials: @config['credentials'])
           rtb['routes'].each { |route|
             if route['nat_host_id'].nil? and route['nat_host_name'].nil?
               route_config = {
@@ -1473,7 +1528,7 @@ module MU
               unless route['gateway'] == '#NAT'
                 # Need to change the order of how things are created to create the route here
                 MU.log "Creating route for #{route['destination_network']}", details: route_config
-                resp = MU::Cloud::AWS.ec2.create_route(route_config)
+                resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
               end
             end
           }
@@ -1485,8 +1540,8 @@ module MU
         # @param noop [Boolean]: If true, will only print what would be done
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.purge_gateways(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
-          resp = MU::Cloud::AWS.ec2(region).describe_internet_gateways(
+        def self.purge_gateways(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_internet_gateways(
               filters: tagfilters
           )
           gateways = resp.data.internet_gateways
@@ -1495,7 +1550,7 @@ module MU
             gateway.attachments.each { |attachment|
               MU.log "Detaching Internet Gateway #{gateway.internet_gateway_id} from #{attachment.vpc_id}"
               begin
-                MU::Cloud::AWS.ec2(region).detach_internet_gateway(
+                MU::Cloud::AWS.ec2(credentials: credentials, region: region).detach_internet_gateway(
                     internet_gateway_id: gateway.internet_gateway_id,
                     vpc_id: attachment.vpc_id
                 ) if !noop
@@ -1505,7 +1560,7 @@ module MU
             }
             MU.log "Deleting Internet Gateway #{gateway.internet_gateway_id}"
             begin
-              MU::Cloud::AWS.ec2(region).delete_internet_gateway(internet_gateway_id: gateway.internet_gateway_id) if !noop
+              MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_internet_gateway(internet_gateway_id: gateway.internet_gateway_id) if !noop
             rescue Aws::EC2::Errors::InvalidInternetGatewayIDNotFound
               MU.log "Gateway #{gateway.internet_gateway_id} was already destroyed by the time I got to it", MU::WARN
             end
@@ -1518,8 +1573,8 @@ module MU
         # @param vpc_id [String]: The cloud provider's unique VPC identifier
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.purge_nat_gateways(noop = false, vpc_id: nil, region: MU.curRegion)
-          gateways = MU::Cloud::AWS.ec2(region).describe_nat_gateways(
+        def self.purge_nat_gateways(noop = false, vpc_id: nil, region: MU.curRegion, credentials: nil)
+          gateways = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(
             filter: [
               {
                 name: "vpc-id",
@@ -1537,15 +1592,15 @@ module MU
                 MU.log "Deleting NAT Gateway #{gateway.nat_gateway_id}"
                 if !noop
                   begin
-                    MU::Cloud::AWS.ec2(region).delete_nat_gateway(nat_gateway_id: gateway.nat_gateway_id)
-                    resp = MU::Cloud::AWS.ec2(region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
+                    MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_nat_gateway(nat_gateway_id: gateway.nat_gateway_id)
+                    resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
 
                     attempts = 0
                     while resp.state != "deleted" and resp.state != "failed"
                       MU.log "Waiting for nat gateway #{gateway.nat_gateway_id} to delete" if attempts % 2 == 0
                       sleep 30
                       begin
-                        resp = MU::Cloud::AWS.ec2(region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
+                        resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
                       rescue Aws::EmptyStructure, NoMethodError
                         sleep 5
                         retry
@@ -1575,8 +1630,8 @@ module MU
         # @param vpc_id [String]: The cloud provider's unique VPC identifier
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.purge_endpoints(noop = false, vpc_id: nil, region: MU.curRegion)
-          vpc_endpoints = MU::Cloud::AWS.ec2(region).describe_vpc_endpoints(
+        def self.purge_endpoints(noop = false, vpc_id: nil, region: MU.curRegion, credentials: nil)
+          vpc_endpoints = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(
             filters: [
               {
                 name:"vpc-id",
@@ -1594,15 +1649,15 @@ module MU
                 MU.log "Deleting VPC endpoint #{endpoint.vpc_endpoint_id}"
                 if !noop
                   begin
-                    MU::Cloud::AWS.ec2(region).delete_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id])
-                    resp = MU::Cloud::AWS.ec2(region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
+                    MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id])
+                    resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
 
                     attempts = 0
                     while resp.state != "deleted"
                       MU.log "Waiting for VPC endpoint #{endpoint.vpc_endpoint_id} to delete" if attempts % 5 == 0
                       sleep 30
                       begin
-                        resp = MU::Cloud::AWS.ec2(region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
+                        resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
                       rescue Aws::EmptyStructure, NoMethodError
                         sleep 5
                         retry
@@ -1634,8 +1689,8 @@ module MU
         # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.purge_routetables(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
-          resp = MU::Cloud::AWS.ec2(region).describe_route_tables(
+        def self.purge_routetables(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
               filters: tagfilters
           )
           route_tables = resp.data.route_tables
@@ -1647,7 +1702,7 @@ module MU
               if !route.network_interface_id.nil?
                 MU.log "Deleting Network Interface #{route.network_interface_id}"
                 begin
-                  MU::Cloud::AWS.ec2(region).delete_network_interface(network_interface_id: route.network_interface_id) if !noop
+                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_network_interface(network_interface_id: route.network_interface_id) if !noop
                 rescue Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound => e
                   MU.log "Network Interface #{route.network_interface_id} has already been deleted", MU::WARN
                 end
@@ -1655,7 +1710,7 @@ module MU
               if route.gateway_id != "local"
                 MU.log "Deleting #{table.route_table_id}'s route for #{route.destination_cidr_block}"
                 begin
-                  MU::Cloud::AWS.ec2(region).delete_route(
+                  MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
                     route_table_id: table.route_table_id,
                     destination_cidr_block: route.destination_cidr_block
                   ) if !noop
@@ -1667,7 +1722,7 @@ module MU
             can_delete = true
             table.associations.each { |assoc|
               begin
-                MU::Cloud::AWS.ec2(region).disassociate_route_table(association_id: assoc.route_table_association_id) if !noop
+                MU::Cloud::AWS.ec2(credentials: credentials, region: region).disassociate_route_table(association_id: assoc.route_table_association_id) if !noop
               rescue Aws::EC2::Errors::InvalidAssociationIDNotFound => e
                 MU.log "Route table association #{assoc.route_table_association_id} already removed", MU::WARN
               rescue Aws::EC2::Errors::InvalidParameterValue => e
@@ -1679,7 +1734,7 @@ module MU
             next if !can_delete
             MU.log "Deleting Route Table #{table.route_table_id}"
             begin
-              MU::Cloud::AWS.ec2(region).delete_route_table(route_table_id: table.route_table_id) if !noop
+              MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route_table(route_table_id: table.route_table_id) if !noop
             rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
               MU.log "Route table #{table.route_table_id} already removed", MU::WARN
             end
@@ -1693,8 +1748,8 @@ module MU
         # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.purge_interfaces(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
-          resp = MU::Cloud::AWS.ec2(region).describe_network_interfaces(
+        def self.purge_interfaces(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
               filters: tagfilters
           )
           ifaces = resp.data.network_interfaces
@@ -1703,7 +1758,7 @@ module MU
 
           ifaces.each { |iface|
             MU.log "Deleting Network Interface #{iface.network_interface_id}"
-            MU::Cloud::AWS.ec2(region).delete_network_interface(network_interface_id: iface.network_interface_id)
+            MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_network_interface(network_interface_id: iface.network_interface_id)
           }
         end
 
@@ -1712,8 +1767,8 @@ module MU
         # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
-          resp = MU::Cloud::AWS.ec2(region).describe_subnets(
+        def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_subnets(
               filters: tagfilters
           )
           subnets = resp.data.subnets
@@ -1728,7 +1783,7 @@ module MU
                 sleep 30
               else
                 MU.log "Deleting Subnet #{subnet.subnet_id}"
-                MU::Cloud::AWS.ec2(region).delete_subnet(subnet_id: subnet.subnet_id) if !noop
+                MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_subnet(subnet_id: subnet.subnet_id) if !noop
               end
             rescue Aws::EC2::Errors::DependencyViolation => e
               if retries < 7
@@ -1752,8 +1807,8 @@ module MU
         # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.purge_dhcpopts(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
-          resp = MU::Cloud::AWS.ec2(region).describe_dhcp_options(
+        def self.purge_dhcpopts(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_dhcp_options(
               filters: tagfilters
           )
           sets = resp.data.dhcp_options
@@ -1763,7 +1818,7 @@ module MU
           sets.each { |optset|
             begin
               MU.log "Deleting DHCP Option Set #{optset.dhcp_options_id}"
-              MU::Cloud::AWS.ec2(region).delete_dhcp_options(dhcp_options_id: optset.dhcp_options_id)
+              MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_dhcp_options(dhcp_options_id: optset.dhcp_options_id)
             rescue Aws::EC2::Errors::DependencyViolation => e
               MU.log e.inspect, MU::ERR
 #        rescue Aws::EC2::Errors::InvalidSubnetIDNotFound
@@ -1778,8 +1833,8 @@ module MU
         # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.purge_vpcs(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
-          resp = MU::Cloud::AWS.ec2(region).describe_vpcs(
+        def self.purge_vpcs(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpcs(
               filters: tagfilters
           )
 
@@ -1787,7 +1842,7 @@ module MU
           return if vpcs.nil? or vpcs.size == 0
 
           vpcs.each { |vpc|
-            my_peer_conns = MU::Cloud::AWS.ec2(region).describe_vpc_peering_connections(
+            my_peer_conns = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
                 filters: [
                     {
                         name: "requester-vpc-info.vpc-id",
@@ -1795,7 +1850,7 @@ module MU
                     }
                 ]
             ).vpc_peering_connections
-            my_peer_conns.concat(MU::Cloud::AWS.ec2(region).describe_vpc_peering_connections(
+            my_peer_conns.concat(MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
               filters: [
                 {
                   name: "accepter-vpc-info.vpc-id",
@@ -1806,15 +1861,19 @@ module MU
             my_peer_conns.each { |cnxn|
 
               [cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
-                MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region).each { |rtb_id|
-                  resp = MU::Cloud::AWS.ec2(region).describe_route_tables(
+                MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
+                  begin
+                    resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
                       route_table_ids: [rtb_id]
-                  )
+                    )
+                  rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound => e
+                    next
+                  end
                   resp.route_tables.each { |rtb|
                     rtb.routes.each { |route|
                       if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
                         MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
-                        MU::Cloud::AWS.ec2(region).delete_route(
+                        MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
                             route_table_id: rtb_id,
                             destination_cidr_block: route.destination_cidr_block
                         ) if !noop
@@ -1825,18 +1884,20 @@ module MU
               }
               MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
               begin
-                MU::Cloud::AWS.ec2(region).delete_vpc_peering_connection(
+                MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
                     vpc_peering_connection_id: cnxn.vpc_peering_connection_id
                 ) if !noop
               rescue Aws::EC2::Errors::InvalidStateTransition => e
                 MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
+              rescue Aws::EC2::Errors::OperationNotPermitted => e
+                MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
               end
             }
 
             MU.log "Deleting VPC #{vpc.vpc_id}"
             retries = 0
             begin
-              MU::Cloud::AWS.ec2(region).delete_vpc(vpc_id: vpc.vpc_id) if !noop
+              MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc(vpc_id: vpc.vpc_id) if !noop
             rescue Aws::EC2::Errors::InvalidVpcIDNotFound
               MU.log "VPC #{vpc.vpc_id} has already been deleted", MU::WARN
             rescue Aws::EC2::Errors::DependencyViolation => e
@@ -1851,9 +1912,9 @@ module MU
             end
 
             if !MU::Cloud::AWS.isGovCloud?(region)
-              mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", region: region).values.first
+              mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", region: region, credentials: credentials).values.first
               if !mu_zone.nil?
-                MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true)
+                MU::Cloud::AWS::DNSZone.toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true, credentials: credentials)
               end
             end
           }
@@ -1882,20 +1943,20 @@ module MU
             @mu_name = config['mu_name']
             @name = config['name']
             @deploydata = config # This is a dummy for the sake of describe()
-            resp = MU::Cloud::AWS.ec2(@config['region']).describe_subnets(subnet_ids: [@cloud_id]).subnets.first
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [@cloud_id]).subnets.first
             @az = resp.availability_zone
             @ip_block = resp.cidr_block
-            @cloud_desc = resp
+            @cloud_desc = resp # XXX this really isn't the cloud implementation's business
 
           end
 
           # Return the cloud identifier for the default route of this subnet.
           def defaultRoute
-            resp = MU::Cloud::AWS.ec2(@config['region']).describe_route_tables(
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
                 filters: [{name: "association.subnet-id", values: [@cloud_id]}]
             )
             if resp.route_tables.size == 0 # use default route table for the VPC
-              resp = MU::Cloud::AWS.ec2(@config['region']).describe_route_tables(
+              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
                  filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
               )
             end
@@ -1916,11 +1977,11 @@ module MU
           # @return [Boolean]
           def private?
             return false if @cloud_id.nil?
-            resp = MU::Cloud::AWS.ec2(@config['region']).describe_route_tables(
+            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
                 filters: [{name: "association.subnet-id", values: [@cloud_id]}]
             )
             if resp.route_tables.size == 0 # use default route table for the VPC
-              resp = MU::Cloud::AWS.ec2(@config['region']).describe_route_tables(
+              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
                  filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
               )
             end