cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

data/modules/mu/clouds/aws/server_pool.rb

@@ -51,7 +51,7 @@ module MU
 
           zones_to_try = @config["zones"]
           begin
-            asg = MU::Cloud::AWS.autoscale(@config['region']).create_auto_scaling_group(asg_options)
+            asg = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_auto_scaling_group(asg_options)
           rescue Aws::AutoScaling::Errors::ValidationError => e
             if zones_to_try != nil and zones_to_try.size > 0
               MU.log "#{e.message}, retrying with individual AZs", MU::WARN
@@ -66,7 +66,7 @@ module MU
           if zones_to_try != nil and zones_to_try.size < @config["zones"].size
             zones_to_try.each { |zone|
               begin
-                MU::Cloud::AWS.autoscale(@config['region']).update_auto_scaling_group(
+                MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(
                     auto_scaling_group_name: @mu_name,
                     availability_zones: [zone]
                 )
@@ -84,11 +84,11 @@ module MU
           attempts = 0
           begin
             sleep 5
-            desc = MU::Cloud::AWS.autoscale(@config['region']).describe_auto_scaling_groups(auto_scaling_group_names: [@mu_name]).auto_scaling_groups.first
+            desc = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(auto_scaling_group_names: [@mu_name]).auto_scaling_groups.first
             MU.log "Looking for #{desc.min_size} instances in #{@mu_name}, found #{desc.instances.size}", MU::DEBUG
             attempts = attempts + 1
             if attempts > 25 and desc.instances.size == 0
-              MU.log "No instances spun up after #{5*attempts} seconds, something's wrong with Autoscale group #{@mu_name}", MU::ERR, details: MU::Cloud::AWS.autoscale(@config['region']).describe_scaling_activities(auto_scaling_group_name: @mu_name).activities
+              MU.log "No instances spun up after #{5*attempts} seconds, something's wrong with Autoscale group #{@mu_name}", MU::ERR, details: MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_scaling_activities(auto_scaling_group_name: @mu_name).activities
               raise MuError, "No instances spun up after #{5*attempts} seconds, something's wrong with Autoscale group #{@mu_name}"
             end
           end while desc.instances.size < desc.min_size
@@ -136,7 +136,7 @@ module MU
               t.join
             }
             MU.log "Setting min_size to #{@config['min_size']} and max_size to #{@config['max_size']}"
-            MU::Cloud::AWS.autoscale(@config['region']).update_auto_scaling_group(
+            MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(
               auto_scaling_group_name: @mu_name,
               min_size: @config['min_size'],
               max_size: @config['max_size']
@@ -158,7 +158,7 @@ module MU
         def setScaleInProtection(need_instances = @config['min_size'])
           live_instances = []
           begin
-            desc = MU::Cloud::AWS.autoscale(@config['region']).describe_auto_scaling_groups(auto_scaling_group_names: [@mu_name]).auto_scaling_groups.first
+            desc = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(auto_scaling_group_names: [@mu_name]).auto_scaling_groups.first
 
             live_instances = desc.instances.map { |i| i.instance_id }
             already_set = 0
@@ -170,7 +170,7 @@ module MU
             elsif already_set > need_instances
               unset_me = live_instances.sample(already_set - need_instances)
               MU.log "Disabling scale-in protection for #{unset_me.size.to_s} instances in #{@mu_name}", MU::NOTICE, details: unset_me
-              MU::Cloud::AWS.autoscale(@config['region']).set_instance_protection(
+              MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).set_instance_protection(
                 auto_scaling_group_name: @mu_name,
                 instance_ids: unset_me,
                 protected_from_scale_in: false
@@ -179,7 +179,7 @@ module MU
               live_instances = live_instances.sample(need_instances)
               MU.log "Enabling scale-in protection for #{@config['scale_in_protection']} instances in #{@mu_name}", details: live_instances
               begin
-                MU::Cloud::AWS.autoscale(@config['region']).set_instance_protection(
+                MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).set_instance_protection(
                   auto_scaling_group_name: @mu_name,
                   instance_ids: live_instances,
                   protected_from_scale_in: true
@@ -213,7 +213,7 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def groom
           if @config['schedule']
-            ext_actions = MU::Cloud::AWS.autoscale(@config['region']).describe_scheduled_actions(
+            ext_actions = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_scheduled_actions(
               auto_scaling_group_name: @mu_name
             ).scheduled_update_group_actions
 
@@ -234,7 +234,7 @@ module MU
                 if s['action_name'] == ext.scheduled_action_name
                   if !MU.hashCmp(MU.structToHash(ext), sched_config, missing_is_default: true)
                     MU.log "Removing scheduled action #{s['action_name']} from AutoScale group #{@mu_name}"
-                    MU::Cloud::AWS.autoscale(@config['region']).delete_scheduled_action(
+                    MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_scheduled_action(
                       auto_scaling_group_name: @mu_name,
                       scheduled_action_name: s['action_name']
                     )
@@ -246,7 +246,7 @@ module MU
               }
               if !action_already_correct
                 MU.log "Adding scheduled action to AutoScale group #{@mu_name}", MU::NOTICE, details: sched_config
-                MU::Cloud::AWS.autoscale(@config['region']).put_scheduled_update_group_action(
+                MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).put_scheduled_update_group_action(
                   sched_config
                 )
               end
@@ -274,10 +274,10 @@ module MU
           if need_tag_update
             MU.log "Updating ServerPool #{@mu_name} with new tags", MU::NOTICE, details: tag_conf[:tags]
 
-            MU::Cloud::AWS.autoscale(@config['region']).create_or_update_tags(tag_conf)
+            MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_or_update_tags(tag_conf)
             current.instances.each { |instance|
               tag_conf[:tags].each { |t|
-                MU::MommaCat.createTag(instance.instance_id, t[:key], t[:value], region: @config['region'])
+                MU::MommaCat.createTag(instance.instance_id, t[:key], t[:value], region: @config['region'], credentials: @config['credentials'])
               }
             }
           end
@@ -291,7 +291,7 @@ module MU
           asg_options[:new_instances_protected_from_scale_in] = (@config['scale_in_protection'] == "all")
           tg_arns = []
           if asg_options[:target_group_arns]
-            MU::Cloud::AWS.autoscale(@config['region']).attach_load_balancer_target_groups(
+            MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).attach_load_balancer_target_groups(
               auto_scaling_group_name: @mu_name,
               target_group_arns: asg_options[:target_group_arns]
             )
@@ -299,7 +299,7 @@ module MU
             asg_options.delete(:target_group_arns)
           end
 
-          MU::Cloud::AWS.autoscale(@config['region']).update_auto_scaling_group(asg_options)
+          MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(asg_options)
 
           if @config['scale_in_protection']
             if @config['scale_in_protection'] == "all"
@@ -313,7 +313,7 @@ module MU
             setScaleInProtection(0)
           end
 
-          ext_pols = MU::Cloud::AWS.autoscale(@config['region']).describe_policies(
+          ext_pols = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_policies(
             auto_scaling_group_name: @mu_name
           ).scaling_policies
           if @config["scaling_policies"] and @config["scaling_policies"].size > 0
@@ -325,7 +325,7 @@ module MU
             ext_pols.each { |ext|
               if !legit_policies.include?(ext.policy_name)
                 MU.log "Scaling policy #{ext.policy_name} is not named in scaling_policies, removing from #{@mu_name}", MU::NOTICE, details: ext
-                MU::Cloud::AWS.autoscale(@config['region']).delete_policy(
+                MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_policy(
                   auto_scaling_group_name: @mu_name,
                   policy_name: ext.policy_name
                 )
@@ -389,7 +389,7 @@ module MU
               ext_pols.each { |ext|
                 if ext.policy_name == policy_name
                   if !MU.hashCmp(MU.structToHash(ext), policy_params, missing_is_default: true)
-                    MU::Cloud::AWS.autoscale(@config['region']).delete_policy(
+                    MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_policy(
                       auto_scaling_group_name: @mu_name,
                       policy_name: policy_name
                     )
@@ -401,44 +401,9 @@ module MU
               }
               if !policy_already_correct
                 MU.log "Putting scaling policy #{policy_name} for #{@mu_name}", MU::NOTICE, details: policy_params
-                resp = MU::Cloud::AWS.autoscale(@config['region']).put_scaling_policy(policy_params)
+                resp = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).put_scaling_policy(policy_params)
               end
 
-
-              # If we are creating alarms for scaling policies we need to have the autoscaling policy ARN
-              # To make life easier we're creating the alarms here
-              if policy.has_key?("alarms") && !policy["alarms"].empty?
-                policy["alarms"].each { |alarm|
-                  alarm["alarm_actions"] = [] if !alarm.has_key?("alarm_actions")
-                  alarm["ok_actions"] = [] if !alarm.has_key?("ok_actions")
-                  alarm["alarm_actions"] << resp.policy_arn
-                  alarm["dimensions"] = [{name: "AutoScalingGroupName", value: asg_options[:auto_scaling_group_name]}]
-
-                  if alarm["enable_notifications"]
-                    topic_arn = MU::Cloud::AWS::Notification.createTopic(alarm["notification_group"], region: @config["region"])
-                    MU::Cloud::AWS::Notification.subscribe(arn: topic_arn, protocol: alarm["notification_type"], endpoint: alarm["notification_endpoint"], region: @config["region"])
-                    alarm["alarm_actions"] << topic_arn
-                    alarm["ok_actions"] << topic_arn
-                  end
-
-                  MU::Cloud::AWS::Alarm.setAlarm(
-                    name: "#{MU.deploy_id}-#{alarm["name"]}".upcase,
-                    ok_actions: alarm["ok_actions"],
-                    alarm_actions: alarm["alarm_actions"],
-                    insufficient_data_actions: alarm["no_data_actions"],
-                    metric_name: alarm["metric_name"],
-                    namespace: alarm["namespace"],
-                    statistic: alarm["statistic"],
-                    dimensions: alarm["dimensions"],
-                    period: alarm["period"],
-                    unit: alarm["unit"],
-                    evaluation_periods: alarm["evaluation_periods"],
-                    threshold: alarm["threshold"],
-                    comparison_operator: alarm["comparison_operator"],
-                    region: @config["region"]
-                  )
-                }
-              end
             }
           end
 
@@ -447,7 +412,7 @@ module MU
         # Retrieve the AWS descriptor for this Autoscale group
         # @return [OpenStruct]
         def cloud_desc
-          MU::Cloud::AWS.autoscale(@config['region']).describe_auto_scaling_groups(
+          MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(
             auto_scaling_group_names: [@mu_name]
           ).auto_scaling_groups.first
         end
@@ -471,10 +436,10 @@ module MU
         # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
         # @param flags [Hash]: Optional flags
         # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ServerPools
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
+        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
           found = []
           if cloud_id
-            resp = MU::Cloud::AWS.autoscale(region).describe_auto_scaling_groups({
+            resp = MU::Cloud::AWS.autoscale(region: region, credentials: credentials).describe_auto_scaling_groups({
               auto_scaling_group_names: [
                 cloud_id
               ], 
@@ -751,7 +716,7 @@ module MU
           ok = true
 
           if pool["termination_policy"]
-            valid_policies = MU::Cloud::AWS.autoscale(pool['region']).describe_termination_policy_types.termination_policy_types
+            valid_policies = MU::Cloud::AWS.autoscale(region: pool['region']).describe_termination_policy_types.termination_policy_types
             if !valid_policies.include?(pool["termination_policy"])
               ok = false
               MU.log "Termination policy #{pool["termination_policy"]} is not valid in region #{pool['region']}", MU::ERR, details: valid_policies
@@ -853,6 +818,7 @@ module MU
 # XXX maybe break this down into policies and add those?
               end
 
+              role['credentials'] = pool['credentials'] if pool['credentials']
               configurator.insertKitten(role, "roles")
               pool["dependencies"] ||= []
               pool["dependencies"] << {
@@ -949,7 +915,9 @@ module MU
                   alarm['dimensions'] << { "name" => pool["name"], "cloud_class" => "AutoScalingGroupName" }
                   alarm["namespace"] = "AWS/EC2" if alarm["namespace"].nil?
                   alarm['cloud'] = pool['cloud']
-#                  ok = false if !insertKitten(alarm, "alarms")
+                  alarm['credentials'] = pool['credentials']
+                  alarm['region'] = pool['region']
+                  ok = false if !configurator.insertKitten(alarm, "alarms")
                 }
               end
             }
@@ -957,17 +925,24 @@ module MU
           ok
         end
 
+        # Does this resource type exist as a global (cloud-wide) artifact, or
+        # is it localized to a region/zone?
+        # @return [Boolean]
+        def self.isGlobal?
+          false
+        end
+
         # Remove all autoscale groups associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           filters = [{name: "key", values: ["MU-ID"]}]
           if !ignoremaster
             filters << {name: "key", values: ["MU-MASTER-IP"]}
           end
-          resp = MU::Cloud::AWS.autoscale(region).describe_tags(
+          resp = MU::Cloud::AWS.autoscale(credentials: credentials, region: region).describe_tags(
             filters: filters,
             max_records: 100
           )
@@ -995,7 +970,7 @@ module MU
             next if noop
             retries = 0
             begin
-              MU::Cloud::AWS.autoscale(region).delete_auto_scaling_group(
+              MU::Cloud::AWS.autoscale(credentials: credentials, region: region).delete_auto_scaling_group(
                   auto_scaling_group_name: resource_id,
                   # XXX this should obey @force
                   force_delete: true
@@ -1017,7 +992,7 @@ module MU
             retries = 0
             begin
               MU.log "Removing AutoScale Launch Configuration #{resource_id}"
-              MU::Cloud::AWS.autoscale(region).delete_launch_configuration(
+              MU::Cloud::AWS.autoscale(credentials: credentials, region: region).delete_launch_configuration(
                 launch_configuration_name: resource_id
               )
             rescue Aws::AutoScaling::Errors::ValidationError => e
@@ -1054,12 +1029,13 @@ module MU
           elsif !@config['basis']['launch_config']["instance_id"].nil?
             @config['basis']['launch_config']["ami_id"] = MU::Cloud::AWS::Server.createImage(
               name: @mu_name,
-              instance_id: @config['basis']['launch_config']["instance_id"]
+              instance_id: @config['basis']['launch_config']["instance_id"],
+              credentials: @config['credentials']
             )
           end
-          MU::Cloud::AWS::Server.waitForAMI(@config['basis']['launch_config']["ami_id"])
+          MU::Cloud::AWS::Server.waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
 
-          oldlaunch = MU::Cloud::AWS.autoscale(@config['region']).describe_launch_configurations(
+          oldlaunch = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_launch_configurations(
             launch_configuration_names: [@mu_name]
           ).launch_configurations.first
 
@@ -1131,7 +1107,7 @@ module MU
             # Put our Autoscale group onto a temporary launch config
             begin
 
-              MU::Cloud::AWS.autoscale(@config['region']).create_launch_configuration(
+              MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_launch_configuration(
                 launch_configuration_name: @mu_name+"-TMP",
                 user_data: Base64.encode64(olduserdata),
                 image_id: oldlaunch.image_id,
@@ -1154,12 +1130,12 @@ module MU
             end
 
 
-            MU::Cloud::AWS.autoscale(@config['region']).update_auto_scaling_group(
+            MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(
               auto_scaling_group_name: @mu_name,
               launch_configuration_name: @mu_name+"-TMP"
             )
             # ...now back to an identical one with the "real" name
-            MU::Cloud::AWS.autoscale(@config['region']).delete_launch_configuration(
+            MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_launch_configuration(
               launch_configuration_name: @mu_name
             )
           end
@@ -1197,9 +1173,16 @@ module MU
           }
 
           if @config['basis']['launch_config']['generate_iam_role']
-            # Using ARN instead of IAM instance profile name to hopefully get around some random AWS failures
-            rolename, cfm_role_name, cfm_prof_name, arn = MU::Cloud::AWS::Server.createIAMProfile(@mu_name, base_profile: @config['basis']['launch_config']['iam_role'], extra_policies: @config['basis']['launch_config']['iam_policies'], canned_policies: @config['basis']['launch_config']['canned_iam_policies'])
-            launch_options[:iam_instance_profile] = rolename
+            role = @deploy.findLitterMate(name: @config['name'], type: "roles")
+# XXX are these the right patterns for a pool, or did we need wildcards?
+            s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
+              'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU.adminBucketName+'/'+file
+            }
+            role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
+
+            @config['iam_role'] = role.mu_name
+
+            launch_options[:iam_instance_profile] = role.cloudobj.createInstanceProfile
           elsif @config['basis']['launch_config']['iam_role'].nil?
             raise MuError, "#{@mu_name} has generate_iam_role set to false, but no iam_role assigned."
           else
@@ -1208,18 +1191,12 @@ module MU
 
           @config['iam_role'] = rolename ? rolename : launch_options[:iam_instance_profile]
 
-          if rolename
-            MU::Cloud::AWS::Server.addStdPoliciesToIAMProfile(rolename, region: @config['region'])
-          else
-            MU::Cloud::AWS::Server.addStdPoliciesToIAMProfile(@config['iam_role'], region: @config['region'])
-          end
-
           lc_attempts = 0
           begin
-            MU::Cloud::AWS.autoscale(@config['region']).create_launch_configuration(launch_options)
+            MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_launch_configuration(launch_options)
           rescue Aws::AutoScaling::Errors::ValidationError => e
             if lc_attempts > 3
-              MU.log "Got error while creating #{@mu_name} Launch Config: #{e.message}, retrying in 10s", MU::WARN
+              MU.log "Got error while creating #{@mu_name} Launch Config#{@config['credentials'] ? " with credentials #{@config['credentials']}" : ""}: #{e.message}, retrying in 10s", MU::WARN, details: launch_options.reject { |k,v | k == :user_data }
             end
             sleep 5
             lc_attempts += 1
@@ -1228,11 +1205,11 @@ module MU
 
           if !oldlaunch.nil?
             # Tell the ASG to use the new one, and nuke the old one
-            MU::Cloud::AWS.autoscale(@config['region']).update_auto_scaling_group(
+            MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(
               auto_scaling_group_name: @mu_name,
               launch_configuration_name: @mu_name
             )
-            MU::Cloud::AWS.autoscale(@config['region']).delete_launch_configuration(
+            MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_launch_configuration(
               launch_configuration_name: @mu_name+"-TMP"
             )
             MU.log "Launch Configuration #{@mu_name} replaced"
@@ -1375,7 +1352,7 @@ module MU
           # Do the dance of specifying individual zones if we haven't asked to
           # use particular VPC subnets.
           if @config['zones'].nil? and asg_options[:vpc_zone_identifier].nil?
-            @config["zones"] = MU::Cloud::AWS.listAZs(@config['region'])
+            @config["zones"] = MU::Cloud::AWS.listAZs(region: @config['region'])
             MU.log "Using zones from #{@config['region']}", MU::DEBUG, details: @config['zones']
           end
           asg_options[:availability_zones] = @config["zones"] if @config["zones"] != nil

data/modules/mu/clouds/aws/storage_pool.rb

@@ -36,7 +36,7 @@ module MU
         # @return [String]: The cloud provider's identifier for this storage pool.
         def create
           MU.log "Creating storage pool #{@mu_name}"
-          resp = MU::Cloud::AWS.efs(@config['region']).create_file_system(
+          resp = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).create_file_system(
             creation_token: @mu_name,
             performance_mode: @config['storage_type']
           )
@@ -44,7 +44,7 @@ module MU
           attempts = 0
           loop do
             MU.log "Waiting for #{@mu_name}: #{resp.file_system_id} to become available" if attempts % 5 == 0
-            storage_pool = MU::Cloud::AWS.efs(@config['region']).describe_file_systems(
+            storage_pool = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_file_systems(
               creation_token: @mu_name
             ).file_systems.first
             break if storage_pool.life_cycle_state == "available"
@@ -54,7 +54,7 @@ module MU
             raise MuError, "timed out waiting for #{resp.mount_target_id }" if attempts >= 20
           end
 
-          addStandardTags(cloud_id: resp.file_system_id, region: @config['region'])
+          addStandardTags(cloud_id: resp.file_system_id, region: @config['region'], credentials: @config['credentials'])
           @cloud_id = resp.file_system_id
 
           if @config['mount_points'] && !@config['mount_points'].empty?
@@ -89,6 +89,7 @@ module MU
                   ip_address: target['ip_address'],
                   subnet_id: target['vpc']['subnet_id'],
                   security_groups: sgs,
+                  credentials: @config['credentials'],
                   region: @config['region']
                 )
                 target['cloud_id'] = mount_target.mount_target_id
@@ -106,7 +107,7 @@ module MU
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":elasticfilesystem:"+@config['region']+":"+MU.account_number+":file-system/"+@cloud_id
+          "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":elasticfilesystem:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":file-system/"+@cloud_id
         end
 
         # Locate an existing storage pool and return an array containing matching AWS resource descriptors for those that match.
@@ -116,10 +117,10 @@ module MU
         # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
         # @param flags [Hash]: Optional flags
         # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching storage pool
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
+        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
           map = {}
           if cloud_id
-            storge_pool = MU::Cloud::AWS.efs(region).describe_file_systems(
+            storge_pool = MU::Cloud::AWS.efs(region: region, credentials: credentials).describe_file_systems(
               file_system_id: cloud_id
             ).file_systems.first
             
@@ -127,11 +128,11 @@ module MU
           end
 
           if tag_value
-            storage_pools = MU::Cloud::AWS.efs(region).describe_file_systems.file_systems
+            storage_pools = MU::Cloud::AWS.efs(region: region, credentials: credentials).describe_file_systems.file_systems
           
             if !storage_pools.empty?
               storage_pools.each{ |pool|
-                tags = MU::Cloud::AWS.efs(region).describe_tags(
+                tags = MU::Cloud::AWS.efs(region: region, credentials: credentials).describe_tags(
                   file_system_id: pool.file_system_id
                 ).tags
 
@@ -157,7 +158,7 @@ module MU
         # Add our standard tag set to an Amazon EFS File System.
         # @param cloud_id [String]: The cloud provider's identifier for this resource.
         # @param region [String]: The cloud provider region
-        def addStandardTags(cloud_id: nil, region: MU.curRegion)
+        def addStandardTags(cloud_id: nil, region: MU.curRegion, credentials: nil)
           if cloud_id
             tags = []
             MU::MommaCat.listStandardTags.each_pair { |name, value|
@@ -180,7 +181,7 @@ module MU
 
             tags << {key: "Name", value: @mu_name} unless name_tag
 
-            MU::Cloud::AWS.efs(region).create_tags(
+            MU::Cloud::AWS.efs(region: region, credentials: credentials).create_tags(
               file_system_id: cloud_id,
               tags: tags
             )
@@ -200,10 +201,10 @@ module MU
         # @param subnet_id [String]: The subnet_id to associate the mount point with
         # @param security_groups [Array]: A list of security groups to associate with the mount point.
         # @param region [String]: The cloud provider region
-        def self.create_mount_target(cloud_id: nil, ip_address: nil, subnet_id: nil, security_groups: [], region: MU.curRegion)
+        def self.create_mount_target(cloud_id: nil, ip_address: nil, subnet_id: nil, security_groups: [], region: MU.curRegion, credentials: nil)
           MU.log "Creating mount target for filesystem #{cloud_id}"
 
-          resp = MU::Cloud::AWS.efs(region).create_mount_target(
+          resp = MU::Cloud::AWS.efs(region: region, credentials: credentials).create_mount_target(
             file_system_id: cloud_id,
             subnet_id: subnet_id,
             ip_address: ip_address,
@@ -215,12 +216,12 @@ module MU
           loop do
             MU.log "Waiting for #{resp.mount_target_id} to become available", MU::NOTICE if attempts % 10 == 0
             begin
-              mount_target = MU::Cloud::AWS.efs(region).describe_mount_targets(
+              mount_target = MU::Cloud::AWS.efs(region: region, credentials: credentials).describe_mount_targets(
                 mount_target_id: resp.mount_target_id 
               ).mount_targets.first
             rescue Aws::EFS::Errors::MountTargetNotFound
               if retries <= 3
-                sleep 5
+                sleep 10
                 retry
               else
                 return nil
@@ -244,7 +245,7 @@ module MU
         # @param region [String]: The cloud provider region
         def self.modify_security_groups(cloud_id: nil, replace: false , security_groups: [], region: MU.curRegion)
           unless replace
-            extisting_sgs = MU::Cloud::AWS.efs(region).describe_mount_target_security_groups(
+            extisting_sgs = MU::Cloud::AWS.efs(region: region).describe_mount_target_security_groups(
               mount_target_id: cloud_id
             ).security_groups
 
@@ -252,7 +253,7 @@ module MU
           end
 
           security_groups.uniq!
-          resp = MU::Cloud::AWS.efs(region).modify_mount_target_security_groups(
+          resp = MU::Cloud::AWS.efs(region: region).modify_mount_target_security_groups(
             mount_target_id: cloud_id,
             security_groups: security_groups
           )
@@ -261,7 +262,7 @@ module MU
         # Register a description of this storage pool with this deployment's metadata.
         def notify
           deploy_struct = {}
-          storage_pool = MU::Cloud::AWS.efs(@config['region']).describe_file_systems(
+          storage_pool = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_file_systems(
             creation_token: @mu_name
           ).file_systems.first
 
@@ -272,19 +273,33 @@ module MU
               subnet = nil
               dependencies
               mp_vpc = if mp['vpc'] and mp['vpc']['vpc_name']
-                @deploy.findLitterMate(type: "vpc", name: mp['vpc']['vpc_name'])
+                @deploy.findLitterMate(type: "vpc", name: mp['vpc']['vpc_name'], credentials: @config['credentials'])
+              elsif mp['vpc']
+                MU::MommaCat.findStray(
+                  @config['cloud'],
+                  "vpcs",
+                  deploy_id: mp['vpc']["deploy_id"],
+                  credentials: @config['credentials'],
+                  mu_name: mp['vpc']["mu_name"],
+                  cloud_id: mp['vpc']['vpc_id'],
+                  region: @config['region'],
+                  dummy_ok: false
+                ).first
 # XXX non-sibling, findStray version
               end
 
-              mount_targets = MU::Cloud::AWS.efs(@config['region']).describe_mount_targets(
+              mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
                 file_system_id: storage_pool.file_system_id
               ).mount_targets
 
+#              subnet_obj = mp_vpc.subnets.select { |s|
+#                s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
+#              }.first
               mount_target = nil
-              subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
               mp_vpc.subnets.each { |subnet_obj|
                 mount_targets.map { |t|
-                  if subnet_cidr_obj.contains(t.ip_address)
+                  subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
+                  if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
                     mount_target = t
                     subnet = subnet_obj.cloud_desc
                   end
@@ -292,7 +307,7 @@ module MU
                 break if mount_target
               }
 
-              # mount_target = MU::Cloud::AWS.efs(@config['region']).describe_mount_targets(
+              # mount_target = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
                 # mount_target_id: mp["cloud_id"]
               # ).mount_targets.first
 
@@ -303,7 +318,7 @@ module MU
                 "file_system_id" => mount_target.file_system_id,
                 "mount_directory" => mp["directory"],
                 "subnet_id" => mount_target.subnet_id,
-                "vpc_id" => subnet.vpc_id,
+                "vpc_id" => mp_vpc.cloud_id,
                 "availability_zone" => subnet.availability_zone,
                 "state" => mount_target.life_cycle_state,
                 "ip_address" => mount_target.ip_address,
@@ -327,17 +342,24 @@ module MU
           return deploy_struct
         end
 
+        # Does this resource type exist as a global (cloud-wide) artifact, or
+        # is it localized to a region/zone?
+        # @return [Boolean]
+        def self.isGlobal?
+          false
+        end
+
         # Called by {MU::Cleanup}. Locates resources that were created by the
         # currently-loaded deployment, and purges them.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region in which to operate
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           supported_regions = %w{us-west-2 us-east-1 eu-west-1}
           if supported_regions.include?(region)
             begin 
-              resp = MU::Cloud::AWS.efs(region).describe_file_systems
+              resp = MU::Cloud::AWS.efs(credentials: credentials, region: region).describe_file_systems
               return if resp.nil? or resp.file_systems.nil?
               storage_pools = resp.file_systems
             rescue Aws::EFS::Errors::AccessDeniedException
@@ -350,7 +372,7 @@ module MU
 
             if !storage_pools.empty?
               storage_pools.each{ |pool|
-                tags = MU::Cloud::AWS.efs(region).describe_tags(
+                tags = MU::Cloud::AWS.efs(credentials: credentials, region: region).describe_tags(
                   file_system_id: pool.file_system_id
                 ).tags
 
@@ -372,7 +394,7 @@ module MU
 
             # How to identify mount points in a reliable way? Mount points are not tagged, which means we can only reliably identify mount points based on a filesystem ID. We can you our deployment metadata, but it isn’t necessarily reliable
             # begin
-              # resp = MU::Cloud::AWS.efs(region).delete_mount_target(
+              # resp = MU::Cloud::AWS.efs(credentials: credentials, region: region).delete_mount_target(
                 # mount_target_id: "MountTargetId"
               # )
               # MU.log "Deleted mount target"
@@ -382,7 +404,7 @@ module MU
 
             if !our_pools.empty?
               our_pools.each{ |pool|
-                mount_targets = MU::Cloud::AWS.efs(region).describe_mount_targets(
+                mount_targets = MU::Cloud::AWS.efs(credentials: credentials, region: region).describe_mount_targets(
                   file_system_id: pool.file_system_id
                 ).mount_targets
 
@@ -391,7 +413,7 @@ module MU
                     MU.log "Deleting mount target #{mp.mount_target_id} for filesystem #{pool.name}: #{pool.file_system_id}"
                     unless noop
                       begin
-                        resp = MU::Cloud::AWS.efs(region).delete_mount_target(
+                        resp = MU::Cloud::AWS.efs(credentials: credentials, region: region).delete_mount_target(
                           mount_target_id: mp.mount_target_id
                           )
                       rescue Aws::EFS::Errors::BadRequest => e
@@ -405,7 +427,7 @@ module MU
                 unless noop
                   attempts = 0
                   begin
-                    resp = MU::Cloud::AWS.efs(region).delete_file_system(
+                    resp = MU::Cloud::AWS.efs(credentials: credentials, region: region).delete_file_system(
                       file_system_id: pool.file_system_id
                     )
                   rescue Aws::EFS::Errors::BadRequest => e
@@ -474,7 +496,13 @@ module MU
             pool['mount_points'].each{ |mp|
               if mp['ingress_rules']
                 fwname = "storage-#{mp['name']}"
-                acl = {"name" => fwname, "rules" => mp['ingress_rules'], "region" => pool['region'], "optional_tags" => pool['optional_tags']}
+                acl = {
+                  "name" => fwname,
+                  "rules" => mp['ingress_rules'],
+                  "region" => pool['region'],
+                  "credentials" => pool['credentials'],
+                  "optional_tags" => pool['optional_tags']
+                }
                 acl["tags"] = pool['tags'] if pool['tags'] && !pool['tags'].empty?
                 acl["vpc"] = mp['vpc'].dup if mp['vpc']
                 ok = false if !configurator.insertKitten(acl, "firewall_rules")