cloud-mu 3.2.0 → 3.5.0

data/modules/mu/providers/aws.rb

@@ -16,7 +16,6 @@ require "net/http"
 require 'open-uri'
 require 'timeout'
 require 'inifile'
-gem 'aws-sdk-core'
 autoload :Aws, "aws-sdk-core"
 
 
@@ -54,14 +53,19 @@ module MU
       def self.resourceInitHook(cloudobj, _deploy)
         class << self
           attr_reader :cloudformation_data
+          attr_reader :region
         end
+        return if !cloudobj
         cloudobj.instance_variable_set(:@cloudformation_data, {})
+
+        cloudobj.instance_variable_set(:@region, cloudobj.config['region'])
       end
 
       # Load some credentials for using the AWS API
       # @param name [String]: The name of the mu.yaml AWS credential set to use. If not specified, will use the default credentials, and set the global Aws.config credentials to those.
       # @return [Aws::Credentials]
       def self.loadCredentials(name = nil)
+        gem 'aws-sdk-core'
         @@creds_loaded ||= {}
 
         if name.nil?
@@ -124,10 +128,14 @@ module MU
           # pull access key and secret from a vault
           begin
             vault, item = cred_cfg["credentials"].split(/:/)
-            data = MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h
-            if data["access_key"] and data["access_secret"]
+            data = if !vault or !item
+              raise MuError.new "AWS #{name} credentials field value '#{cred_cfg["credentials"]}' malformed, should be vaultname:itemname", details: cred_cfg
+            else
+              MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h
+            end
+            if data and data["access_key"] and data["access_secret"]
               cred_obj = Aws::Credentials.new(
-                cred_cfg['access_key'], cred_cfg['access_secret']
+                data['access_key'], data['access_secret']
               )
               if name.nil?
 #                Aws.config = {
@@ -137,10 +145,10 @@ module MU
 #                }
               end
             else
-              MU.log "AWS credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", MU::WARN
+              raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", details: cred_cfg
             end
           rescue MU::Groomer::MuNoSuchSecret
-            MU.log "AWS credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", MU::WARN
+            raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", details: cred_cfg
           end
         end
 
@@ -186,6 +194,7 @@ end
       # @param r [String]
       # @return [String]
       def self.validate_region(r, credentials: nil)
+        require "aws-sdk-ec2"
         begin
           MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_availability_zones.availability_zones.first.region_name
         rescue ::Aws::EC2::Errors::UnauthorizedOperation => e
@@ -204,6 +213,7 @@ end
       # @param othertags [Array<Hash>]: Miscellaneous custom tags, in Basket of Kittens style
       # @return [void]
       def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil, optional: true, nametag: nil, othertags: nil)
+        require "aws-sdk-ec2"
         tags = []
         MU::MommaCat.listStandardTags.each_pair { |name, value|
           tags << {key: name, value: value} if !value.nil?
@@ -261,20 +271,33 @@ end
         @@myVPCObj
       end
 
-      # If we've configured AWS as a provider, or are simply hosted in AWS, 
+      # If we've configured AWS as a provider, or are simply hosted in AWS,
       # decide what our default region is.
-      def self.myRegion(credentials = nil)
-        return @@myRegion_var if @@myRegion_var
+      def self.myRegion(credentials = nil, debug: false)
+        loglevel = debug ? MU::NOTICE : MU::DEBUG
+        if @@myRegion_var
+          MU.log "AWS.myRegion: returning #{@@myRegion_var} from cache", loglevel
+          return @@myRegion_var
+        end
 
+        MU.log "AWS.myRegion: credConfig", loglevel, details: credConfig
+        MU.log "AWS.myRegion: hosted?", loglevel, details: hosted?.to_s
+        MU.log "AWS.myRegion: ENV['EC2_REGION']", loglevel, details: ENV['EC2_REGION']
+        MU.log "AWS.myRegion: $MU_CFG['aws']", loglevel, details: $MU_CFG['aws']
         if credConfig.nil? and !hosted? and !ENV['EC2_REGION']
+          MU.log "AWS.myRegion: nothing of use set, returning", loglevel
           return nil
         end
 
         if $MU_CFG and $MU_CFG['aws']
           $MU_CFG['aws'].each_pair { |credset, cfg|
+            MU.log "AWS.myRegion: #{credset} != #{credentials} ?", loglevel, details: cfg
             next if credentials and credset != credentials
+            MU.log "AWS.myRegion: validating credset #{credset}", loglevel, details: cfg
             next if !cfg['region']
-            if (cfg['default'] or !@@myRegion_var) and validate_region(cfg['region'], credentials: credset)
+            MU.log "AWS.myRegion: validation response", loglevel, details: validate_region(cfg['region'], credentials: credset)
+            if (cfg['default'] or !@@myRegion_var or $MU_CFG['aws'].size == 1) and validate_region(cfg['region'], credentials: credset)
+              MU.log "AWS.myRegion: liking this set", loglevel, details: cfg
               @@myRegion_var = cfg['region']
               break if cfg['default'] or credentials
             end
@@ -286,15 +309,21 @@ end
                (Aws.config['access_key'] and Aws.config['access_secret'])
               )
           # Make sure this string is valid by way of the API
+          MU.log "AWS.myRegion: using ENV", loglevel, details: ENV
           @@myRegion_var = ENV['EC2_REGION']
         end
 
         if hosted? and !@@myRegion_var
           # hacky, but useful in a pinch (and if we're hosted in AWS)
           az_str = MU::Cloud::AWS.getAWSMetaData("placement/availability-zone")
+          MU.log "AWS.myRegion: using hosted", loglevel, details: az_str
           @@myRegion_var = az_str.sub(/[a-z]$/i, "") if az_str
         end
 
+        if credConfig and credConfig["region"]
+          @@myRegion_var ||= credConfig["region"]
+        end
+
         @@myRegion_var
       end
 
@@ -382,8 +411,9 @@ end
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
       # @param deploy_id [String]: The deploy for which we're writing the secret
       # @param value [String]: The contents of the secret
-      def self.writeDeploySecret(deploy_id, value, name = nil, credentials: nil)
-        name ||= deploy_id+"-secret"
+      def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
+        require "aws-sdk-s3"
+        name ||= deploy.deploy_id+"-secret"
         begin
           MU.log "Writing #{name} to S3 bucket #{adminBucketName(credentials)}"
           MU::Cloud::AWS.s3(region: myRegion, credentials: credentials).put_object(
@@ -401,35 +431,35 @@ end
       def self.cloudtrailBucketPolicy(credentials = nil)
         cfg = credConfig(credentials)
         policy_json = '{
-      		"Version": "2012-10-17",
-      		"Statement": [
-      			{
-      				"Sid": "AWSCloudTrailAclCheck20131101",
-      				"Effect": "Allow",
+                "Version": "2012-10-17",
+                "Statement": [
+                        {
+                                "Sid": "AWSCloudTrailAclCheck20131101",
+                                "Effect": "Allow",
               "Principal": {
                 "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::<%= MU.account_number %>:root",
                 "Service": "cloudtrail.amazonaws.com"
               },
-      				"Action": "s3:GetBucketAcl",
-      				"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'"
-      			},
-      			{
-      				"Sid": "AWSCloudTrailWrite20131101",
-      				"Effect": "Allow",
+                                "Action": "s3:GetBucketAcl",
+                                "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'"
+                        },
+                        {
+                                "Sid": "AWSCloudTrailWrite20131101",
+                                "Effect": "Allow",
               "Principal": {
                 "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::'+credToAcct(credentials)+':root",
                 "Service": "cloudtrail.amazonaws.com"
               },
-      				"Action": "s3:PutObject",
-      				"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/AWSLogs/'+credToAcct(credentials)+'/*",
-      				"Condition": {
-      					"StringEquals": {
-      						"s3:x-amz-acl": "bucket-owner-full-control"
-      					}
-      				}
-      			}
-      		]
-      	}'
+                                "Action": "s3:PutObject",
+                                "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/AWSLogs/'+credToAcct(credentials)+'/*",
+                                "Condition": {
+                                        "StringEquals": {
+                                                "s3:x-amz-acl": "bucket-owner-full-control"
+                                        }
+                                }
+                        }
+                ]
+        }'
         ERB.new(policy_json).result
       end
 
@@ -440,6 +470,53 @@ end
         MU::Cloud::AWS.hosted?
       end
 
+      # If we're in AWS and NVME-aware, return a mapping of AWS-side device
+      # names to actual NVME devices.
+      # @return [Hash]
+      def self.attachedNVMeDisks
+        if !hosted? or !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme")
+          return {}
+        end
+        map = {}
+        devices = MU::Master.listBlockDevices
+        return {} if !devices
+        devices.each { |d|
+          if d =~ /^\/dev\/nvme/
+            %x{/sbin/nvme id-ctrl -v #{d}}.each_line { |desc|
+              if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./)
+                virt_dev = Regexp.last_match[1]
+                map[virt_dev] = d
+                break
+              end
+            }
+          end
+        }
+        map
+      end
+
+      # Map our own idea of what a block device is called back to whatever AWS
+      # and the operating system decided on amongst themselves. This currently
+      # exists to map generic "xvd[a-z]" style names back to real NVMe devices.
+      # @param dev [String]
+      def self.realDevicePath(dev)
+        return dev if !hosted?
+        value = nil
+        should_retry = Proc.new {
+          !value and MU::Master.nvme?
+        }
+        MU.retrier(loop_if: should_retry, wait: 5, max: 6) {
+          map = attachedNVMeDisks
+          value = if map[dev]
+            map[dev]
+          elsif map[dev.gsub(/.*?\//, '')]
+            map[dev.gsub(/.*?\//, '')]
+          else
+            dev # be nice to actually handle this too
+          end
+        }
+        value
+      end
+
       # Determine whether we (the Mu master, presumably) are hosted in this
       # cloud.
       # @return [Boolean]
@@ -457,7 +534,7 @@ end
 
         begin
           Timeout.timeout(4) do
-            instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read
+            instance_id = URI.open("http://169.254.169.254/latest/meta-data/instance-id").read
             if !instance_id.nil? and instance_id.size > 0
               @@is_in_aws = true
               region = getAWSMetaData("placement/availability-zone").sub(/[a-z]$/i, "")
@@ -550,7 +627,9 @@ end
       def self.credToAcct(name = nil)
         creds = credConfig(name)
 
-        return creds['account_number'] if creds['account_number']
+        if creds['account_number'] and !creds['account_number'].empty?
+          return creds['account_number']
+        end
 
         acct_num = MU::Cloud::AWS.iam(credentials: name).list_users.users.first.arn.split(/:/)[4]
         acct_num.to_s
@@ -564,17 +643,18 @@ end
         end
 
         $MU_CFG['aws'].keys
-      end 
+      end
 
       # Resolve the administrative S3 bucket for a given credential set, or
       # return a default.
       # @param credentials [String]
       # @return [String]
       def self.adminBucketName(credentials = nil)
+        require "aws-sdk-s3"
         cfg = credConfig(credentials)
         return nil if !cfg
         if !cfg['log_bucket_name']
-          cfg['log_bucket_name'] = $MU_CFG['hostname'] 
+          cfg['log_bucket_name'] = $MU_CFG['hostname']
           MU.log "No AWS log bucket defined for credentials #{credentials}, attempting to use default of #{cfg['log_bucket_name']}", MU::WARN
         end
         resp = MU::Cloud::AWS.s3(credentials: credentials).list_buckets
@@ -608,7 +688,7 @@ end
 
       # Return the $MU_CFG data associated with a particular profile/name/set of
       # credentials. If no account name is specified, will return one flagged as
-      # default. Returns nil if AWS is not configured. Throws an exception if 
+      # default. Returns nil if AWS is not configured. Throws an exception if
       # an account name is specified which does not exist.
       # @param name [String]: The name of the key under 'aws' in mu.yaml to return
       # @return [Hash,nil]
@@ -623,10 +703,13 @@ end
 
           if hosted?
             begin
-              iam_data = JSON.parse(getAWSMetaData("iam/info"))
-              if iam_data["InstanceProfileArn"] and !iam_data["InstanceProfileArn"].empty?
-                @@my_hosted_cfg = hosted_config
-                return name_only ? "#default" : @@my_hosted_cfg
+              iam_blob = getAWSMetaData("iam/info")
+              if iam_blob
+                iam_data = JSON.parse(iam_blob)
+                if iam_data["InstanceProfileArn"] and !iam_data["InstanceProfileArn"].empty?
+                  @@my_hosted_cfg = hosted_config
+                  return name_only ? "#default" : @@my_hosted_cfg
+                end
               end
             rescue JSON::ParserError => e
             end
@@ -672,8 +755,8 @@ end
                 next
               end
               acct_num = MU::Cloud::AWS.iam(credentials: acctname).list_users.users.first.arn.split(/:/)[4]
-              if acct_num.to_s ==  name.to_s
-                cfg['account_number'] = acct_num.to_s
+              cfg['account_number'] ||= acct_num.to_s
+              if acct_num.to_s == name.to_s
                 @@acct_to_profile_map[name.to_s] = cfg
                 return name_only ? name.to_s : cfg
               end
@@ -690,6 +773,7 @@ end
       # XXX this needs to be "myAccountNumber" or somesuch
       # XXX and maybe do the IAM thing for arbitrary, non-resident accounts
       def self.account_number
+        require "aws-sdk-ec2"
         return nil if credConfig.nil?
         return @@my_acct_num if @@my_acct_num
         loadCredentials
@@ -747,7 +831,7 @@ end
           @@regions.keys.uniq
         end
 
-# XXX GovCloud doesn't show up if you query a commercial endpoint... that's 
+# XXX GovCloud doesn't show up if you query a commercial endpoint... that's
 # *probably* ok for most purposes? We can't call listAZs on it from out here
 # apparently, so getting around it is nontrivial
 #        if !@@regions.has_key?("us-gov-west-1")
@@ -772,6 +856,7 @@ end
       # @param public_key [String]: The public key
       # @return [Array<String>]: keypairname, ssh_private_key, ssh_public_key
       def self.createEc2SSHKey(keyname, public_key, credentials: nil)
+        require "aws-sdk-ec2"
         # We replicate this key in all regions
         if !MU::Cloud::CloudFormation.emitCloudFormation
           MU::Cloud::AWS.listRegions.each { |region|
@@ -800,8 +885,16 @@ end
       def self.listInstanceTypes(region = myRegion)
         return @@instance_types if @@instance_types and @@instance_types[region]
         return {} if credConfig.nil?
+        if region.nil?
+          region = myRegion(debug: true)
+        end
+        return {} if region.nil?
 
         human_region = @@regionLookup[region]
+        if human_region.nil?
+          MU.log "Failed to map a Pricing API region name from #{region}", MU::ERR
+          return {}
+        end
 
         @@instance_types ||= {}
         @@instance_types[region] ||= {}
@@ -844,6 +937,8 @@ end
         @@instance_types
       end
 
+      @@certificates = {}
+
       # AWS can stash API-available certificates in Amazon Certificate Manager
       # or in IAM. Rather than make people crazy trying to get the syntax
       # correct in our Baskets of Kittens, let's have a helper that tries to do
@@ -852,21 +947,25 @@ end
       # @param name [String]: The name of the cert. For IAM certs this can be any IAM name; for ACM, it's usually the domain name. If multiple matches are found, or no matches, an exception is raised.
       # @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied.
       # @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired.
-      def self.findSSLCertificate(name: nil, id: nil, region: myRegion)
-        if name.nil? and name.empty? and id.nil? and id.empty?
+      def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true)
+        require "aws-sdk-iam"
+        if (name.nil? or name.empty?) and (id.nil? or id.empty?)
           raise MuError, "Can't call findSSLCertificate without specifying either a name or an id"
         end
+        if id and @@certificates[id]
+          return [id, @@certificates[id]]
+        end
 
         if !name.nil? and !name.empty?
           matches = []
-          acmcerts = MU::Cloud::AWS.acm(region: region).list_certificates(
+          acmcerts = MU::Cloud::AWS.acm(region: region, credentials: credentials).list_certificates(
             certificate_statuses: ["ISSUED"]
           )
           acmcerts.certificate_summary_list.each { |cert|
             matches << cert.certificate_arn if cert.domain_name == name
           }
           begin
-            iamcert = MU::Cloud::AWS.iam.get_server_certificate(
+            iamcert = MU::Cloud::AWS.iam(credentials: credentials).get_server_certificate(
               server_certificate_name: name
             )
           rescue Aws::IAM::Errors::ValidationError, Aws::IAM::Errors::NoSuchEntity
@@ -876,32 +975,45 @@ end
             matches << iamcert.server_certificate.server_certificate_metadata.arn
           end
           if matches.size == 1
-            return matches.first
+            id = matches.first
           elsif matches.size == 0
-            raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
+            if raise_on_missing
+              raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
+            else
+              return nil
+            end
           elsif matches.size > 1
-            raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."            
+            raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."
           end
         end
 
+        domains = []
+
         if id.match(/^arn:aws(?:-us-gov)?:acm/)
-          resp = MU::Cloud::AWS.acm(region: region).get_certificate(
+          resp = MU::Cloud::AWS.acm(region: region).describe_certificate(
             certificate_arn: id
           )
-          if resp.nil?
+
+          if resp.nil? or resp.certificate.nil?
             raise MuError, "No such ACM certificate '#{id}'"
           end
+          domains << resp.certificate.domain_name
+          if resp.certificate.subject_alternative_names
+            domains.concat(resp.certificate.subject_alternative_names)
+          end
         elsif id.match(/^arn:aws(?:-us-gov)?:iam/)
           resp = MU::Cloud::AWS.iam.list_server_certificates
           if resp.nil?
             raise MuError, "No such IAM certificate '#{id}'"
           end
           resp.server_certificate_metadata_list.each { |cert|
+
             if cert.arn == id
               if cert.expiration < Time.now
                 MU.log "IAM SSL certificate #{cert.server_certificate_name} (#{id}) is EXPIRED", MU::WARN
               end
-              return id
+              @@certificates[id] = [cert.server_certificate_name]
+              return [id, [cert.server_certificate_name]]
             end
           }
           raise MuError, "No such IAM certificate '#{id}'"
@@ -909,7 +1021,56 @@ end
           raise MuError, "The format of '#{id}' doesn't look like an ARN for either Amazon Certificate Manager or IAM"
         end
 
-        id
+        @@certificates[id] = domains.uniq
+        [id, domains.uniq]
+      end
+
+      # Given a domain name and an ACM or IAM certificate identifier, sort out
+      # whether the domain name is "covered" by the certificate
+      # @param name [String]
+      # @param cert_id [String]
+      # @return [Boolean]
+      def self.nameMatchesCertificate(name, cert_id)
+        _id, domains = findSSLCertificate(id: cert_id)
+        return false if !domains
+        domains.each { |dom|
+          if dom == name or
+             (dom =~ /^\*/ and name =~ /.*#{Regexp.quote(dom[1..-1])}/)
+            return true
+          end
+        }
+        false
+      end
+
+      # Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate,
+      # look up and validate the specified certificate. This is intended to be
+      # invoked from resource implementations' +validateConfig+ methods.
+      # @param certblock [Hash,MU::Config::Ref]:
+      # @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any
+      # @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any
+      # @return [Boolean]
+      def self.resolveSSLCertificate(certblock, region: nil, credentials: nil)
+        return false if !certblock
+        ok = true
+
+        certblock['region'] ||= region if !certblock['id']
+        certblock['credentials'] ||= credentials
+        cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate(
+          name: certblock["name"],
+          id: certblock["id"],
+          region: certblock['region'],
+          credentials: certblock['credentials']
+        )
+
+        if cert_arn
+          certblock['id'] ||= cert_arn
+        end
+
+        ['region', 'credentials'].each { |field|
+          certblock.delete(field) if certblock[field].nil?
+        }
+
+        [cert_arn, cert_domains]
       end
 
       # Amazon Certificate Manager API
@@ -1029,6 +1190,14 @@ end
         @@cloudwatchlogs_api[credentials][region]
       end
 
+      # Amazon's CloudWatchEvents API
+      def self.cloudwatchevents(region: MU.curRegion, credentials: nil)
+        region ||= myRegion
+        @@cloudwatchevents_api[credentials] ||= {}
+        @@cloudwatchevents_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials)
+        @@cloudwatchevents_api[credentials][region]
+      end
+
       # Amazon's CloudFront API
       def self.cloudfront(region: MU.curRegion, credentials: nil)
         region ||= myRegion
@@ -1044,7 +1213,7 @@ end
         @@elasticache_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElastiCache", region: region, credentials: credentials)
         @@elasticache_api[credentials][region]
       end
-      
+
       # Amazon's SNS API
       def self.sns(region: MU.curRegion, credentials: nil)
         region ||= myRegion
@@ -1052,7 +1221,7 @@ end
         @@sns_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SNS", region: region, credentials: credentials)
         @@sns_api[credentials][region]
       end
-      
+
       # Amazon's SQS API
       def self.sqs(region: MU.curRegion, credentials: nil)
         region ||= myRegion
@@ -1084,7 +1253,7 @@ end
         @@apig_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "APIGateway", region: region, credentials: credentials)
         @@apig_api[credentials][region]
       end
-      
+
       # Amazon's Cloudwatch Events API
       def self.cloudwatch_events(region = MU.cureRegion)
         region ||= myRegion
@@ -1117,6 +1286,14 @@ end
         @@dynamo_api[credentials][region]
       end
 
+      # Amazon's DynamoStream API
+      def self.dynamostream(region: MU.curRegion, credentials: nil)
+        region ||= myRegion
+        @@dynamostream_api[credentials] ||= {}
+        @@dynamostream_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "DynamoDBStreams", region: region, credentials: credentials)
+        @@dynamostream_api[credentials][region]
+      end
+
       # Amazon's Pricing API
       def self.pricing(region: MU.curRegion, credentials: nil)
         region ||= myRegion
@@ -1165,6 +1342,14 @@ end
         @@kms_api[credentials][region]
       end
 
+      # Amazon's CloudFront API
+      def self.cloudfront(region: MU.curRegion, credentials: nil)
+        region ||= myRegion
+        @@cloudfront_api[credentials] ||= {}
+        @@cloudfront_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudFront", region: region, credentials: credentials)
+        @@cloudfront_api[credentials][region]
+      end
+
       # Amazon's Organizations API
       def self.orgs(credentials: nil)
         @@organizations_api ||= {}
@@ -1181,7 +1366,7 @@ end
         begin
           response = nil
           Timeout.timeout(1) do
-            response = open("#{base_url}/#{param}").read
+            response = URI.open("#{base_url}/#{param}").read
           end
 
           response
@@ -1206,6 +1391,7 @@ end
           tag_value=MU.deploy_id,
           region: MU.curRegion,
           credentials: nil)
+        require "aws-sdk-ec2"
         attempts = 0
 
         return nil if resource.nil?
@@ -1246,6 +1432,7 @@ end
       # Mu Master, if we're in AWS.
       # @return [void]
       def self.openFirewallForClients
+        require "aws-sdk-ec2"
         MU::Cloud.resourceClass("AWS", :FirewallRule)
         begin
           if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
@@ -1425,6 +1612,7 @@ end
         def initialize(region: nil, api: "EC2", credentials: nil)
           @cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
           @credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
+          @api_name = api
 
           if !@cred_obj
             raise MuError, "Unable to locate valid AWS credentials for #{api} API. #{credentials ? "Credentials requested were '#{credentials}'": ""}"
@@ -1442,6 +1630,8 @@ end
           params[:credentials] = @cred_obj
 
           MU.log "Initializing #{api} object with credentials #{credentials}", MU::DEBUG, details: params
+          require "aws-sdk-#{api.downcase}"
+
           @api = Object.const_get("Aws::#{api}::Client").new(params)
         end
 
@@ -1450,26 +1640,31 @@ end
         # rescues for known silly endpoint behavior.
         def method_missing(method_sym, *arguments)
           # make sure error symbols are loaded for our exception handling later
-          require "aws-sdk-core"
-          require "aws-sdk-core/rds"
-          require "aws-sdk-core/ec2"
-          require "aws-sdk-core/route53"
-          require "aws-sdk-core/iam"
-          require "aws-sdk-core/efs"
-          require "aws-sdk-core/pricing"
-          require "aws-sdk-core/apigateway"
-          require "aws-sdk-core/ecs"
-          require "aws-sdk-core/eks"
-          require "aws-sdk-core/cloudwatchlogs"
-          require "aws-sdk-core/elasticloadbalancing"
-          require "aws-sdk-core/elasticloadbalancingv2"
-          require "aws-sdk-core/autoscaling"
-          require "aws-sdk-core/client_waiters"
-          require "aws-sdk-core/waiters/errors"
+          require "aws-sdk-lambda"
+          require "aws-sdk-rds"
+          require "aws-sdk-ec2"
+          require "aws-sdk-route53"
+          require "aws-sdk-iam"
+          require "aws-sdk-efs"
+          require "aws-sdk-pricing"
+          require "aws-sdk-apigateway"
+          require "aws-sdk-ecs"
+          require "aws-sdk-eks"
+          require "aws-sdk-cloudwatchlogs"
+          require "aws-sdk-cloudwatchevents"
+          require "aws-sdk-elasticloadbalancing"
+          require "aws-sdk-elasticloadbalancingv2"
+          require "aws-sdk-autoscaling"
+
+          known_concats = {
+            "Pricing" => {
+              :get_products => :price_list
+            }
+          }
 
           retries = 0
           begin
-            MU.log "Calling #{method_sym} in #{@region}", MU::DEBUG, details: arguments
+            MU.log "Calling #{@api_name}.#{method_sym} in #{@region}", MU::DEBUG, details: arguments
 
               retval = if !arguments.nil? and arguments.size == 1
                 @api.method(method_sym).call(arguments[0])
@@ -1481,24 +1676,39 @@ end
 
             if !retval.nil?
               begin
-              page_markers = [:marker, :next_token]
+              page_markers = {
+                :marker => :marker,
+                :next_token => :next_token,
+                :next_marker => :marker
+              }
               paginator = nil
               new_page = nil
-              [:next_token, :marker].each { |m|
+              page_markers.each_key { |m|
                 if !retval.nil? and retval.respond_to?(m)
                   paginator = m
-                  new_page = retval.send(paginator)
+                  new_page = retval.send(m)
                   break
                 end
               }
 
               if paginator and new_page and !new_page.empty?
                 resp = retval.respond_to?(:__getobj__) ? retval.__getobj__ : retval
-                concat_to = resp.class.instance_methods(false).reject { |m|
+                concat_to = MU.structToHash(resp).keys.reject { |m|
                   m.to_s.match(/=$/) or m == paginator or resp.send(m).nil? or !resp.send(m).is_a?(Array)
                 }
+
+                if concat_to.empty? and known_concats[@api_name] and
+                   known_concats[@api_name][method_sym]
+                  concat_to << known_concats[@api_name][method_sym]
+                end
+
+                if concat_to.empty? and method_sym.to_s.match(/^(?:describe|list)_(.*)/)
+                  my_attr = Regexp.last_match[1].to_sym
+                  concat_to << my_attr if resp.respond_to?(my_attr)
+                end
+
                 if concat_to.size != 1
-                  MU.log "Tried to figure out where I might append paginated results for a #{resp.class.name}, but failed", MU::DEBUG, details: concat_to
+                  raise MuError.new "Tried to figure out where I might append paginated results for a #{@api_name}.#{method_sym}, but failed", details: MU.structToHash(resp).keys
                 else
                   concat_to = concat_to.first
                   new_args = arguments ? arguments.dup : [{}]
@@ -1506,12 +1716,12 @@ end
                     if new_args.is_a?(Array)
                       new_args << {} if new_args.empty?
                       if new_args.size == 1 and new_args.first.is_a?(Hash)
-                        new_args[0][paginator] = new_page
+                        new_args[0][page_markers[paginator]] = new_page
                       else
                         MU.log "I don't know how to insert a #{paginator} into these arguments for #{method_sym}", MU::WARN, details: new_args
                       end
                     elsif new_args.is_a?(Hash)
-                      new_args[paginator] = new_page
+                      new_args[page_markers[paginator]] = new_page
                     end
 
                     MU.log "Attempting magic pagination for #{method_sym}", MU::DEBUG, details: new_args
@@ -1535,7 +1745,7 @@ end
             end
 
             return retval
-          rescue Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
+          rescue Aws::Lambda::Errors::TooManyRequestsException, Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
             if e.class.name == "Seahorse::Client::NetworkingError" and e.message.match(/Name or service not known/)
               MU.log e.inspect, MU::ERR
               raise e
@@ -1577,6 +1787,7 @@ end
       @@wafglobal = {}
       @@waf = {}
       @@cloudwatchlogs_api = {}
+      @@cloudwatchevents_api = {}
       @@cloudfront_api = {}
       @@elasticache_api = {}
       @@sns_api = {}
@@ -1595,6 +1806,8 @@ end
       @@kms_api ={}
       @@organization_api ={}
       @@dynamo_api ={}
+      @@dynamostream_api ={}
+      @@cloudfront_api ={}
     end
   end
 end