cloud-mu 3.1.3 → 3.3.0

data/modules/mu/{clouds → providers}/README.md

@@ -89,7 +89,7 @@ Looking elsewhere in `cloud.rb` let's see what all we have to do:
     generic_instance_methods = [:create, :notify, :mu_name, :cloud_id, :config]
 ```
 
-Just the basics, for now. Here's what that will look like in the AWS layer, in the file `modules/mu/clouds/aws/function.rb`:
+Just the basics, for now. Here's what that will look like in the AWS layer, in the file `modules/mu/providers/aws/function.rb`:
 
 ```
 module MU

data/modules/mu/{clouds → providers}/aws.rb

@@ -33,13 +33,25 @@ module MU
       module AdditionalResourceMethods
       end
 
+      # Is this a "real" cloud provider, or a stub like CloudFormation?
+      def self.virtual?
+        false
+      end
+
+      # List all AWS projects available to our credentials
+      def self.listHabitats(credentials = nil, use_cache: true)
+        cfg = credConfig(credentials)
+        return [] if !cfg or !cfg['account_number']
+        [cfg['account_number']]
+      end
+
       # A hook that is always called just before any of the instance method of
       # our resource implementations gets invoked, so that we can ensure that
       # repetitive setup tasks (like resolving +:resource_group+ for Azure
       # resources) have always been done.
       # @param cloudobj [MU::Cloud]
-      # @param deploy [MU::MommaCat]
-      def self.resourceInitHook(cloudobj, deploy)
+      # @param _deploy [MU::MommaCat]
+      def self.resourceInitHook(cloudobj, _deploy)
         class << self
           attr_reader :cloudformation_data
         end
@@ -63,7 +75,6 @@ module MU
           return nil
         end
 
-        loaded = false
         cred_obj = nil
         if cred_cfg['access_key'] and cred_cfg['access_secret'] and
           # access key and secret just sitting in mu.yaml
@@ -137,11 +148,22 @@ module MU
           # assume we've got an IAM profile and hope for the best
           ENV.delete('AWS_ACCESS_KEY_ID')
           ENV.delete('AWS_SECRET_ACCESS_KEY')
-          cred_obj = Aws::InstanceProfileCredentials.new
+          retries = 0
+          begin
+            cred_obj = Aws::InstanceProfileCredentials.new
+            if cred_obj.nil?
+              retries += 1
+              MU.log "Failed to fetch AWS instance profile credentials, attempt #{retries.to_s}/10", MU::WARN
+              sleep 3
+            end
+          end while cred_obj.nil? and retries < 10
 #          if name.nil?
 #            Aws.config = {region: ENV['EC2_REGION']}
 #          end
         end
+if cred_obj.nil?
+MU.log "cred_obj is nil and hosted? says #{hosted?.to_s}", MU::WARN, details: name
+end
 
         if name.nil?
           @@creds_loaded["#default"] = cred_obj
@@ -172,18 +194,34 @@ module MU
         end
       end
 
-      # Tag a resource with all of our standard identifying tags.
+      # Tag an EC2 resource
       #
       # @param resource [String]: The cloud provider identifier of the resource to tag
       # @param region [String]: The cloud provider region
+      # @param credentials [String]: Credentials to authorize API requests
+      # @param optional [Boolean]: Whether to apply our optional generic tags
+      # @param nametag [String]: A +Name+ tag to apply
+      # @param othertags [Array<Hash>]: Miscellaneous custom tags, in Basket of Kittens style
       # @return [void]
-      def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil)
+      def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil, optional: true, nametag: nil, othertags: nil)
         tags = []
         MU::MommaCat.listStandardTags.each_pair { |name, value|
-          if !value.nil?
-            tags << {key: name, value: value}
-          end
+          tags << {key: name, value: value} if !value.nil?
         }
+        if optional
+          MU::MommaCat.listOptionalTags.each { |key, value|
+            tags << {key: name, value: value} if !value.nil?
+          }
+        end
+        if nametag
+          tags << { key: "Name", value: nametag }
+        end
+        if othertags
+          othertags.each { |tag|
+            tags << { key: tag['key'], value: tag['value'] }
+          }
+        end
+
         if MU::Cloud::CloudFormation.emitCloudFormation
           return tags
         end
@@ -205,6 +243,7 @@ module MU
           end
         end
         MU.log "Created standard tags for resource #{resource}", MU::DEBUG, details: caller
+
       end
 
       @@myVPCObj = nil
@@ -283,52 +322,13 @@ module MU
         )
       end
 
-      # Tag EC2 resources. 
-      #
-      # @param resources [Array<String>]: The cloud provider identifier of the resource to tag
-      # @param key [String]: The name of the tag to create
-      # @param value [String]: The value of the tag
-      # @param region [String]: The cloud provider region
-      # @return [void,<Hash>]
-      def self.createTag(key, value, resources = [], region: myRegion, credentials: nil)
-  
-        if !MU::Cloud::CloudFormation.emitCloudFormation
-          begin
-            MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_tags(
-              resources: resources,
-              tags: [
-                {
-                  key: key,
-                  value: value
-                }
-              ]
-            )
-          rescue Aws::EC2::Errors::ServiceError => e
-            MU.log "Got #{e.inspect} tagging #{resources.size.to_s} resources with #{key}=#{value}", MU::WARN, details: resources if attempts > 1
-            if attempts < 5
-              attempts = attempts + 1
-              sleep 15
-              retry
-            else
-              raise e
-            end
-          end
-          MU.log "Created tag #{key} with value #{value}", MU::DEBUG, details: resources
-        else
-          return {
-            "Key" =>  key,
-            "Value" => value
-          }
-        end
-      end
-
       @@azs = {}
       # List the Availability Zones associated with a given Amazon Web Services
       # region. If no region is given, search the one in which this MU master
       # server resides.
       # @param region [String]: The region to search.
       # @return [Array<String>]: The Availability Zones in this region.
-      def self.listAZs(region: MU.curRegion, account: nil, credentials: nil)
+      def self.listAZs(region: MU.curRegion, credentials: nil)
         cfg = credConfig(credentials)
         return [] if !cfg
         if !region.nil? and @@azs[region]
@@ -356,6 +356,27 @@ module MU
       # etc)
       # @param deploy_id [MU::MommaCat]
       def self.cleanDeploy(deploy_id, credentials: nil, noop: false)
+
+        if !noop
+          MU.log "Deleting s3://#{adminBucketName(credentials)}/#{deploy_id}-secret"
+          MU::Cloud::AWS.s3(credentials: credentials).delete_object(
+            bucket: adminBucketName(credentials),
+            key: "#{deploy_id}-secret"
+          )
+          listRegions(credentials: credentials).each { |r|
+            resp = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_key_pairs(
+              filters: [{name: "key-name", values: ["deploy-#{MU.deploy_id}"]}]
+            )
+            resp.data.key_pairs.each { |keypair|
+              MU.log "Deleting key pair #{keypair.key_name} from #{r}"
+              MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_key_pair(key_name: keypair.key_name) if !noop
+            }
+          }
+
+        end
+        if hosted?
+          MU::Cloud::AWS.openFirewallForClients
+        end
       end
 
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
@@ -435,7 +456,7 @@ module MU
         end
 
         begin
-          Timeout.timeout(2) do
+          Timeout.timeout(4) do
             instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read
             if !instance_id.nil? and instance_id.size > 0
               @@is_in_aws = true
@@ -491,7 +512,32 @@ module MU
       # @param cloudobj [MU::Cloud::AWS]: The resource from which to extract the habitat id
       # @return [String,nil]
       def self.habitat(cloudobj, nolookup: false, deploy: nil)
-        cloudobj.respond_to?(:account_number) ? cloudobj.account_number : nil
+        @@habmap ||= {}
+# XXX whaddabout config['habitat'] HNNNGH
+
+        if cloudobj.respond_to?(:account_number) and cloudobj.account_number and
+           !cloudobj.account_number.empty?
+          return cloudobj.account_number
+        elsif cloudobj.config and cloudobj.config['account']
+          if nolookup
+            return cloudobj.config['account']
+          end
+          if @@habmap[cloudobj.config['account']]
+            return @@habmap[cloudobj.config['account']]
+          end
+          deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy)
+
+          MU.log "Incomplete implementation: MU::Cloud::AWS.habitat", MU::DEBUG, details: deploy
+
+#          accountobj = accountLookup(cloudobj.config['account'], deploy, raise_on_fail: false)
+
+#          if accountobj
+#            @@habmap[cloudobj.config['account']] = accountobj.cloud_id
+#            return accountobj.cloud_id
+#          end
+        end
+
+        nil
       end
 
 
@@ -506,7 +552,6 @@ module MU
 
         return creds['account_number'] if creds['account_number']
 
-        user_list = MU::Cloud::AWS.iam(credentials: name).list_users.users
         acct_num = MU::Cloud::AWS.iam(credentials: name).list_users.users.first.arn.split(/:/)[4]
         acct_num.to_s
       end
@@ -543,8 +588,8 @@ module MU
         if !found
           MU.log "Attempting to create log bucket #{cfg['log_bucket_name']} for credentials #{credentials}", MU::WARN
           begin
-            resp = MU::Cloud::AWS.s3(credentials: credentials).create_bucket(bucket: cfg['log_bucket_name'], acl: "private")
-          rescue Aws::S3::Errors::BucketAlreadyExists => e
+            MU::Cloud::AWS.s3(credentials: credentials).create_bucket(bucket: cfg['log_bucket_name'], acl: "private")
+          rescue Aws::S3::Errors::BucketAlreadyExists
             raise MuError, "AWS credentials #{credentials} need a log bucket, and the name #{cfg['log_bucket_name']} is unavailable. Use mu-configure to edit credentials '#{credentials}' or 'hostname'"
           end
         end
@@ -620,9 +665,9 @@ module MU
             # Check each credential sets' resident account, then
             $MU_CFG['aws'].each_pair { |acctname, cfg|
               begin
-                user_list = MU::Cloud::AWS.iam(credentials: acctname).list_users.users
+                MU::Cloud::AWS.iam(credentials: acctname).list_users.users
 #             rescue ::Aws::IAM::Errors => e # XXX why does this NameError here?
-              rescue Exception => e
+              rescue StandardError => e
                 MU.log e.inspect, MU::WARN, details: cfg
                 next
               end
@@ -653,7 +698,7 @@ module MU
 #       begin
 #          user_list = MU::Cloud::AWS.iam(region: credConfig['region']).list_users.users
 ##        rescue ::Aws::IAM::Errors => e # XXX why does this NameError here?
-#        rescue Exception => e
+#        rescue StandardError => e
 #          MU.log "Got #{e.inspect} while trying to figure out our account number", MU::WARN, details: caller
 #        end
 #        if user_list.nil? or user_list.size == 0
@@ -682,7 +727,6 @@ module MU
         if @@regions.size == 0
           return [] if credConfig.nil?
           result = MU::Cloud::AWS.ec2(region: myRegion, credentials: credentials).describe_regions.regions
-          regions = []
           @@regions_semaphore.synchronize {
             begin
               result.each { |r|
@@ -761,50 +805,47 @@ module MU
 
         @@instance_types ||= {}
         @@instance_types[region] ||= {}
-        next_token = nil
 
-        begin
-          # Pricing API isn't widely available, so ask a region we know supports
-          # it
-          resp = MU::Cloud::AWS.pricing(region: "us-east-1").get_products(
-            service_code: "AmazonEC2",
-            filters: [
-              {
-                field: "productFamily",
-                value: "Compute Instance",
-                type: "TERM_MATCH"
-              },
-              {
-                field: "tenancy",
-                value: "Shared",
-                type: "TERM_MATCH"
-              },
-              {
-                field: "location",
-                value: human_region,
-                type: "TERM_MATCH"
-              }
-            ],
-            next_token: next_token
-          )
-          resp.price_list.each { |pricing|
-            data = JSON.parse(pricing)
-            type = data["product"]["attributes"]["instanceType"]
-            next if @@instance_types[region].has_key?(type)
-            @@instance_types[region][type] = {}
-            ["ecu", "vcpu", "memory", "storage"].each { |a|
-              @@instance_types[region][type][a] = data["product"]["attributes"][a]
+        # Pricing API isn't widely available, so ask a region we know supports
+        # it
+        resp = MU::Cloud::AWS.pricing(region: "us-east-1").get_products(
+          service_code: "AmazonEC2",
+          filters: [
+            {
+              field: "productFamily",
+              value: "Compute Instance",
+              type: "TERM_MATCH"
+            },
+            {
+              field: "tenancy",
+              value: "Shared",
+              type: "TERM_MATCH"
+            },
+            {
+              field: "location",
+              value: human_region,
+              type: "TERM_MATCH"
             }
-            @@instance_types[region][type]["memory"].sub!(/ GiB/, "")
-            @@instance_types[region][type]["memory"] = @@instance_types[region][type]["memory"].to_f
-            @@instance_types[region][type]["vcpu"] = @@instance_types[region][type]["vcpu"].to_f
+          ]
+        )
+        resp.price_list.each { |pricing|
+          data = JSON.parse(pricing)
+          type = data["product"]["attributes"]["instanceType"]
+          next if @@instance_types[region].has_key?(type)
+          @@instance_types[region][type] = {}
+          ["ecu", "vcpu", "memory", "storage"].each { |a|
+            @@instance_types[region][type][a] = data["product"]["attributes"][a]
           }
-          next_token = resp.next_token
-        end while resp and next_token
+          @@instance_types[region][type]["memory"].sub!(/ GiB/, "")
+          @@instance_types[region][type]["memory"] = @@instance_types[region][type]["memory"].to_f
+          @@instance_types[region][type]["vcpu"] = @@instance_types[region][type]["vcpu"].to_f
+        }
 
         @@instance_types
       end
 
+      @@certificates = {}
+
       # AWS can stash API-available certificates in Amazon Certificate Manager
       # or in IAM. Rather than make people crazy trying to get the syntax
       # correct in our Baskets of Kittens, let's have a helper that tries to do
@@ -813,21 +854,24 @@ module MU
       # @param name [String]: The name of the cert. For IAM certs this can be any IAM name; for ACM, it's usually the domain name. If multiple matches are found, or no matches, an exception is raised.
       # @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied.
       # @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired.
-      def self.findSSLCertificate(name: nil, id: nil, region: myRegion)
-        if name.nil? and name.empty? and id.nil? and id.empty?
+      def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true)
+        if (name.nil? or name.empty?) and (id.nil? or id.empty?)
           raise MuError, "Can't call findSSLCertificate without specifying either a name or an id"
         end
+        if id and @@certificates[id]
+          return [id, @@certificates[id]]
+        end
 
         if !name.nil? and !name.empty?
           matches = []
-          acmcerts = MU::Cloud::AWS.acm(region: region).list_certificates(
+          acmcerts = MU::Cloud::AWS.acm(region: region, credentials: credentials).list_certificates(
             certificate_statuses: ["ISSUED"]
           )
           acmcerts.certificate_summary_list.each { |cert|
             matches << cert.certificate_arn if cert.domain_name == name
           }
           begin
-            iamcert = MU::Cloud::AWS.iam.get_server_certificate(
+            iamcert = MU::Cloud::AWS.iam(credentials: credentials).get_server_certificate(
               server_certificate_name: name
             )
           rescue Aws::IAM::Errors::ValidationError, Aws::IAM::Errors::NoSuchEntity
@@ -837,32 +881,45 @@ module MU
             matches << iamcert.server_certificate.server_certificate_metadata.arn
           end
           if matches.size == 1
-            return matches.first
+            id = matches.first
           elsif matches.size == 0
-            raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
+            if raise_on_missing
+              raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
+            else
+              return nil
+            end
           elsif matches.size > 1
             raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."            
           end
         end
 
+        domains = []
+
         if id.match(/^arn:aws(?:-us-gov)?:acm/)
-          resp = MU::Cloud::AWS.acm(region: region).get_certificate(
+          resp = MU::Cloud::AWS.acm(region: region).describe_certificate(
             certificate_arn: id
           )
-          if resp.nil?
+
+          if resp.nil? or resp.certificate.nil?
             raise MuError, "No such ACM certificate '#{id}'"
           end
+          domains << resp.certificate.domain_name
+          if resp.certificate.subject_alternative_names
+            domains.concat(resp.certificate.subject_alternative_names)
+          end
         elsif id.match(/^arn:aws(?:-us-gov)?:iam/)
           resp = MU::Cloud::AWS.iam.list_server_certificates
           if resp.nil?
             raise MuError, "No such IAM certificate '#{id}'"
           end
           resp.server_certificate_metadata_list.each { |cert|
+
             if cert.arn == id
               if cert.expiration < Time.now
                 MU.log "IAM SSL certificate #{cert.server_certificate_name} (#{id}) is EXPIRED", MU::WARN
               end
-              return id
+              @@certificates[id] = [cert.server_certificate_name]
+              return [id, [cert.server_certificate_name]]
             end
           }
           raise MuError, "No such IAM certificate '#{id}'"
@@ -870,7 +927,56 @@ module MU
           raise MuError, "The format of '#{id}' doesn't look like an ARN for either Amazon Certificate Manager or IAM"
         end
 
-        id
+        @@certificates[id] = domains.uniq
+        [id, domains.uniq]
+      end
+
+      # Given a domain name and an ACM or IAM certificate identifier, sort out
+      # whether the domain name is "covered" by the certificate
+      # @param name [String]
+      # @param cert_id [String]
+      # @return [Boolean]
+      def self.nameMatchesCertificate(name, cert_id)
+        _id, domains = findSSLCertificate(id: cert_id)
+        return false if !domains
+        domains.each { |dom|
+          if dom == name or
+             (dom =~ /^\*/ and name =~ /.*#{Regexp.quote(dom[1..-1])}/)
+            return true
+          end
+        }
+        false
+      end
+
+      # Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate,
+      # look up and validate the specified certificate. This is intended to be
+      # invoked from resource implementations' +validateConfig+ methods.
+      # @param certblock [Hash,MU::Config::Ref]: 
+      # @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any
+      # @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any
+      # @return [Boolean]
+      def self.resolveSSLCertificate(certblock, region: nil, credentials: nil)
+        return false if !certblock
+        ok = true
+
+        certblock['region'] ||= region if !certblock['id']
+        certblock['credentials'] ||= credentials
+        cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate(
+          name: certblock["name"],
+          id: certblock["id"],
+          region: certblock['region'],
+          credentials: certblock['credentials']
+        )
+
+        if cert_arn
+          certblock['id'] ||= cert_arn
+        end
+
+        ['region', 'credentials'].each { |field|
+          certblock.delete(field) if certblock[field].nil?
+        }
+
+        [cert_arn, cert_domains]
       end
 
       # Amazon Certificate Manager API
@@ -990,6 +1096,14 @@ module MU
         @@cloudwatchlogs_api[credentials][region]
       end
 
+      # Amazon's CloudWatchEvents API
+      def self.cloudwatchevents(region: MU.curRegion, credentials: nil)
+        region ||= myRegion
+        @@cloudwatchevents_api[credentials] ||= {}
+        @@cloudwatchevents_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials)
+        @@cloudwatchevents_api[credentials][region]
+      end
+
       # Amazon's CloudFront API
       def self.cloudfront(region: MU.curRegion, credentials: nil)
         region ||= myRegion
@@ -1078,6 +1192,14 @@ module MU
         @@dynamo_api[credentials][region]
       end
 
+      # Amazon's DynamoStream API
+      def self.dynamostream(region: MU.curRegion, credentials: nil)
+        region ||= myRegion
+        @@dynamostream_api[credentials] ||= {}
+        @@dynamostream_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "DynamoDBStreams", region: region, credentials: credentials)
+        @@dynamostream_api[credentials][region]
+      end
+
       # Amazon's Pricing API
       def self.pricing(region: MU.curRegion, credentials: nil)
         region ||= myRegion
@@ -1126,6 +1248,14 @@ module MU
         @@kms_api[credentials][region]
       end
 
+      # Amazon's CloudFront API
+      def self.cloudfront(region: MU.curRegion, credentials: nil)
+        region ||= myRegion
+        @@cloudfront_api[credentials] ||= {}
+        @@cloudfront_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudFront", region: region, credentials: credentials)
+        @@cloudfront_api[credentials][region]
+      end
+
       # Amazon's Organizations API
       def self.orgs(credentials: nil)
         @@organizations_api ||= {}
@@ -1154,12 +1284,60 @@ module MU
         end
       end
 
+      # Tag a resource. Defaults to applying our MU deployment identifier, if no
+      # arguments other than the resource identifier are given.
+      #
+      # @param resource [String]: The cloud provider identifier of the resource to tag
+      # @param tag_name [String]: The name of the tag to create
+      # @param tag_value [String]: The value of the tag
+      # @param region [String]: The cloud provider region
+      # @return [void]
+      def self.createTag(resource = nil,
+          tag_name="MU-ID",
+          tag_value=MU.deploy_id,
+          region: MU.curRegion,
+          credentials: nil)
+        attempts = 0
+
+        return nil if resource.nil?
+        resource = [resource] if resource.is_a?(String)
+
+        if !MU::Cloud::CloudFormation.emitCloudFormation
+          begin
+            MU::Cloud::AWS.ec2(credentials: credentials, region: region).create_tags(
+              resources: resource,
+              tags: [
+                {
+                  key: tag_name,
+                  value: tag_value
+                }
+              ]
+            )
+          rescue Aws::EC2::Errors::ServiceError => e
+            MU.log "Got #{e.inspect} tagging #{resource} with #{tag_name}=#{tag_value}", MU::WARN if attempts > 1
+            if attempts < 5
+              attempts = attempts + 1
+              sleep 15
+              retry
+            else
+              raise e
+            end
+          end
+          MU.log "Created tag #{tag_name} with value #{tag_value} for resource #{resource}", MU::DEBUG
+        else
+          return {
+            "Key" =>  tag_name,
+            "Value" => tag_value
+          }
+        end
+      end
+
       @syslog_port_semaphore = Mutex.new
       # Punch AWS security group holes for client nodes to talk back to us, the
       # Mu Master, if we're in AWS.
       # @return [void]
       def self.openFirewallForClients
-        MU::Cloud.loadCloudType("AWS", :FirewallRule)
+        MU::Cloud.resourceClass("AWS", :FirewallRule)
         begin
           if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
             ::Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
@@ -1205,8 +1383,8 @@ module MU
             )
             sg_id = group.group_id
             my_sgs << sg_id
-            MU::MommaCat.createTag sg_id, "Name", my_client_sg_name
-            MU::MommaCat.createTag sg_id, "MU-MASTER-IP", MU.mu_public_ip
+            MU::Cloud::AWS.createTag sg_id, "Name", my_client_sg_name
+            MU::Cloud::AWS.createTag sg_id, "MU-MASTER-IP", MU.mu_public_ip
             MU::Cloud::AWS.ec2.modify_instance_attribute(
                 instance_id: my_instance_id,
                 groups: my_sgs
@@ -1237,7 +1415,7 @@ module MU
         end
 
         allow_ips = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
-        MU::MommaCat.listAllNodes.each_pair { |node, data|
+        MU::MommaCat.listAllNodes.values.each { |data|
           next if data.nil? or !data.is_a?(Hash)
           ["public_ip_address"].each { |key|
             if data.has_key?(key) and !data[key].nil? and !data[key].empty?
@@ -1266,7 +1444,7 @@ module MU
                             }
                         ]
                     )
-                  rescue Aws::EC2::Errors::InvalidPermissionNotFound => e
+                  rescue Aws::EC2::Errors::InvalidPermissionNotFound
                     MU.log "Permission disappeared from #{sg_id} (port #{port.to_s}) before I could remove it", MU::WARN, details: MU.structToHash(rule.ip_ranges)
                   end
                 end
@@ -1300,8 +1478,6 @@ module MU
         }
       end
 
-      private
-
       # XXX we shouldn't have to do this, but AWS does not provide a way to look
       # it up, and the pricing API only returns the human-readable strings.
       @@regionLookup = {
@@ -1337,7 +1513,7 @@ module MU
         # Create an AWS API client
         # @param region [String]: Amazon region so we know what endpoint to use
         # @param api [String]: Which API are we wrapping?
-        def initialize(region: MU.curRegion, api: "EC2", credentials: nil)
+        def initialize(region: nil, api: "EC2", credentials: nil)
           @cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
           @credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
 
@@ -1346,6 +1522,8 @@ module MU
           end
 
           params = {}
+          region ||= MU::Cloud::AWS.credConfig(credentials)['region']
+          region ||= MU.myRegion
 
           if region
             @region = region
@@ -1362,21 +1540,98 @@ module MU
         # Catch-all for AWS client methods. Essentially a pass-through with some
         # rescues for known silly endpoint behavior.
         def method_missing(method_sym, *arguments)
+          # make sure error symbols are loaded for our exception handling later
           require "aws-sdk-core"
+          require "aws-sdk-core/rds"
+          require "aws-sdk-core/ec2"
+          require "aws-sdk-core/route53"
+          require "aws-sdk-core/iam"
+          require "aws-sdk-core/efs"
+          require "aws-sdk-core/pricing"
+          require "aws-sdk-core/apigateway"
+          require "aws-sdk-core/ecs"
+          require "aws-sdk-core/eks"
+          require "aws-sdk-core/cloudwatchlogs"
+          require "aws-sdk-core/cloudwatchevents"
+          require "aws-sdk-core/elasticloadbalancing"
+          require "aws-sdk-core/elasticloadbalancingv2"
+          require "aws-sdk-core/autoscaling"
+          require "aws-sdk-core/client_waiters"
+          require "aws-sdk-core/waiters/errors"
 
           retries = 0
           begin
             MU.log "Calling #{method_sym} in #{@region}", MU::DEBUG, details: arguments
-            retval = nil
-            if !arguments.nil? and arguments.size == 1
-              retval = @api.method(method_sym).call(arguments[0])
-            elsif !arguments.nil? and arguments.size > 0
-              retval = @api.method(method_sym).call(*arguments)
-            else
-              retval = @api.method(method_sym).call
+
+              retval = if !arguments.nil? and arguments.size == 1
+                @api.method(method_sym).call(arguments[0])
+              elsif !arguments.nil? and arguments.size > 0
+                @api.method(method_sym).call(*arguments)
+              else
+                @api.method(method_sym).call
+              end
+
+            if !retval.nil?
+              begin
+              page_markers = {
+                :marker => :marker,
+                :next_token => :next_token,
+                :next_marker => :marker
+              }
+              paginator = nil
+              new_page = nil
+              page_markers.each_key { |m|
+                if !retval.nil? and retval.respond_to?(m)
+                  paginator = m
+                  new_page = retval.send(m)
+                  break
+                end
+              }
+
+              if paginator and new_page and !new_page.empty?
+                resp = retval.respond_to?(:__getobj__) ? retval.__getobj__ : retval
+                concat_to = resp.class.instance_methods(false).reject { |m|
+                  m.to_s.match(/=$/) or m == paginator or resp.send(m).nil? or !resp.send(m).is_a?(Array)
+                }
+                if concat_to.size != 1
+                  MU.log "Tried to figure out where I might append paginated results for a #{resp.class.name}, but failed", MU::DEBUG, details: concat_to
+                else
+                  concat_to = concat_to.first
+                  new_args = arguments ? arguments.dup : [{}]
+                  begin
+                    if new_args.is_a?(Array)
+                      new_args << {} if new_args.empty?
+                      if new_args.size == 1 and new_args.first.is_a?(Hash)
+                        new_args[0][page_markers[paginator]] = new_page
+                      else
+                        MU.log "I don't know how to insert a #{paginator} into these arguments for #{method_sym}", MU::WARN, details: new_args
+                      end
+                    elsif new_args.is_a?(Hash)
+                      new_args[page_markers[paginator]] = new_page
+                    end
+
+                    MU.log "Attempting magic pagination for #{method_sym}", MU::DEBUG, details: new_args
+
+#                    resp = if !arguments.nil? and arguments.size == 1
+#                      @api.method(method_sym).call(new_args[0])
+#                    elsif !arguments.nil? and arguments.size > 0
+                    resp = @api.method(method_sym).call(*new_args)
+#                    end
+                    break if resp.nil?
+                    resp = resp.__getobj__ if resp.respond_to?(:__getobj__)
+                    retval.send(concat_to).concat(resp.send(concat_to))
+                    new_page = resp.send(paginator) if !resp.nil?
+                  end while !resp.nil? and !new_page.nil? and !new_page.empty?
+                end
+              end
+              rescue StandardError => e
+                MU.log "Made a good-faith effort to auto-paginate API call to #{method_sym} and failed with #{e.message}", MU::DEBUG, details: arguments
+                raise e
+              end
             end
+
             return retval
-          rescue Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
+          rescue Aws::Lambda::Errors::TooManyRequestsException, Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
             if e.class.name == "Seahorse::Client::NetworkingError" and e.message.match(/Name or service not known/)
               MU.log e.inspect, MU::ERR
               raise e
@@ -1397,7 +1652,7 @@ module MU
             MU.log "Got #{e.inspect} calling EC2's #{method_sym} in #{@region} with credentials #{@credentials}, waiting #{interval.to_s}s and retrying. Args were: #{arguments}", debuglevel, details: caller
             sleep interval
             retry
-          rescue Exception => e
+          rescue StandardError => e
             MU.log "Got #{e.inspect} calling EC2's #{method_sym} in #{@region} with credentials #{@credentials}", MU::DEBUG, details: arguments
             raise e
           end
@@ -1418,6 +1673,7 @@ module MU
       @@wafglobal = {}
       @@waf = {}
       @@cloudwatchlogs_api = {}
+      @@cloudwatchevents_api = {}
       @@cloudfront_api = {}
       @@elasticache_api = {}
       @@sns_api = {}
@@ -1436,6 +1692,8 @@ module MU
       @@kms_api ={}
       @@organization_api ={}
       @@dynamo_api ={}
+      @@dynamostream_api ={}
+      @@cloudfront_api ={}
     end
   end
 end