cloud-mu 3.0.0beta → 3.0.0

data/bin/mu-gen-docs

@@ -47,6 +47,7 @@ if !Dir.exist?(docdir)
   FileUtils.mkdir_p(docdir, mode: 0755)
 end
 
+MU::Config.emitConfigAsRuby
 MU::Config.emitSchemaAsRuby
 if Process.uid == 0
   MU.log "Generating YARD documentation in #{docdir} (see http://#{$MU_CFG['public_address']}/docs/frames.html)"
@@ -68,6 +69,7 @@ Dir.chdir(MU.myRoot) do
 EOF
 
   impl_counts = {}
+  cloud_is_useful = {}
   cloudlist = MU::Cloud.supportedClouds.sort { |a, b|
     counts = {
       a => 0,
@@ -80,9 +82,11 @@ EOF
           myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
           case myclass.quality
           when MU::Cloud::RELEASE
+            cloud_is_useful[cloud] = true
             counts[cloud] += 4
             impl_counts[type] += 4
           when MU::Cloud::BETA
+            cloud_is_useful[cloud] = true
             counts[cloud] += 2
             impl_counts[type] += 2
           when MU::Cloud::ALPHA
@@ -96,6 +100,8 @@ EOF
     counts[b] <=> counts[a]
   }
 
+  cloudlist.reject! { |c| !cloud_is_useful[c] }
+
   readme += "\n\n<table><tr><th></th>"
   cloudlist.each { |cloud|
     readme += "<th>"+cloud+"</th>"

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.0.0beta'
-  s.date        = '2019-11-01'
+  s.version     = '3.0.0'
+  s.date        = '2019-11-11'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"

data/cookbooks/mu-tools/recipes/gcloud.rb

@@ -41,7 +41,14 @@ if platform_family?("rhel") or platform_family?("amazon")
       cwd "/opt"
       code <<-EOH
         tar -xzf #{Chef::Config[:file_cache_path]}/gcloud-cli.tar.gz
-        CLOUDSDK_PYTHON="`/bin/rpm -ql muthon | grep '/bin/python$'`" ./google-cloud-sdk/install.sh -q
+        if [ -f /opt/rh/python27/root/usr/bin/python ];then
+          if [ ! -f /etc/ld.so.conf.d/python27.conf ];then
+            echo "/opt/rh/python27/root/usr/lib64" > /etc/ld.so.conf.d/python27.conf
+            echo "/opt/rh/python27/root/usr/lib" >> /etc/ld.so.conf.d/python27.conf
+            /sbin/ldconfig
+          fi
+        fi
+        CLOUDSDK_PYTHON="`/bin/rpm -ql muthon python27-python | grep '/bin/python$'`" ./google-cloud-sdk/install.sh -q
       EOH
       notifies :create, "remote_file[#{Chef::Config[:file_cache_path]}/gcloud-cli.sh]", :before
       notifies :create, "remote_file[#{Chef::Config[:file_cache_path]}/gcloud-cli.tar.gz]", :before

data/modules/mommacat.ru

@@ -46,7 +46,7 @@ end
 Signal.trap("URG") do
   puts "------------------------------"
   puts "Open flock() locks:"
-  pp MU::MommaCat.locks
+  pp MU::MommaCat.trapSafeLocks
   puts "------------------------------"
 end
 

data/modules/mu.rb

@@ -404,41 +404,49 @@ module MU
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.mommacat;
+    @@globals[Thread.current.object_id] ||= {}
     @@globals[Thread.current.object_id]['mommacat']
   end
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.deploy_id;
+    @@globals[Thread.current.object_id] ||= {}
     @@globals[Thread.current.object_id]['deploy_id']
   end
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.appname;
+    @@globals[Thread.current.object_id] ||= {}
     @@globals[Thread.current.object_id]['appname']
   end
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.environment;
+    @@globals[Thread.current.object_id] ||= {}
     @@globals[Thread.current.object_id]['environment']
   end
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.timestamp;
+    @@globals[Thread.current.object_id] ||= {}
     @@globals[Thread.current.object_id]['timestamp']
   end
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.seed;
+    @@globals[Thread.current.object_id] ||= {}
     @@globals[Thread.current.object_id]['seed']
   end
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.handle;
+    @@globals[Thread.current.object_id] ||= {}
     @@globals[Thread.current.object_id]['handle']
   end
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.chef_user;
+    @@globals[Thread.current.object_id] ||= {}
     if @@globals.has_key?(Thread.current.object_id) and @@globals[Thread.current.object_id].has_key?('chef_user')
       @@globals[Thread.current.object_id]['chef_user']
     elsif Etc.getpwuid(Process.uid).name == "root"
@@ -450,6 +458,7 @@ module MU
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.mu_user
+    @@globals[Thread.current.object_id] ||= {}
     if @@globals.has_key?(Thread.current.object_id) and @@globals[Thread.current.object_id].has_key?('mu_user')
       return @@globals[Thread.current.object_id]['mu_user']
     elsif Etc.getpwuid(Process.uid).name == "root"
@@ -461,11 +470,13 @@ module MU
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.curRegion
+    @@globals[Thread.current.object_id] ||= {}
     @@globals[Thread.current.object_id]['curRegion'] ||= myRegion || ENV['EC2_REGION']
   end
 
   # Accessor for per-thread global variable. There is probably a Ruby-clever way to define this.
   def self.syncLitterThread;
+    @@globals[Thread.current.object_id] ||= {}
     @@globals[Thread.current.object_id]['syncLitterThread']
   end
 
@@ -487,26 +498,13 @@ module MU
     end
   end
 
-  # The verbose logging flag merits a default value.
+  # Return the verbosity setting of the default @@logger object
   def self.verbosity
-    if @@globals[Thread.current.object_id].nil? or @@globals[Thread.current.object_id]['verbosity'].nil?
-      MU.setVar("verbosity", MU::Logger::NORMAL)
-    end
-    @@globals[Thread.current.object_id]['verbosity']
-  end
-  
-  # The color logging flag merits a default value.
-  def self.color
-    if @@globals[Thread.current.object_id].nil? or @@globals[Thread.current.object_id]['color'].nil?
-      MU.setVar("color", true)
-    end
-    @@globals[Thread.current.object_id]['color']
+    @@logger ? @@logger.verbosity : MU::Logger::NORMAL
   end
 
   # Set parameters parameters for calls to {MU#log}
   def self.setLogging(verbosity, webify_logs = false, handle = STDOUT, color = true)
-    MU.setVar("verbosity", verbosity)
-    MU.setVar("color", color)
     @@logger ||= MU::Logger.new(verbosity, webify_logs, handle, color)
     @@logger.html = webify_logs
     @@logger.verbosity = verbosity
@@ -523,33 +521,25 @@ module MU
   end
 
   # Shortcut to invoke {MU::Logger#log}
-  def self.log(msg, level = MU::INFO, details: nil, html: false, verbosity: MU.verbosity, color: true)
-    return if (level == MU::DEBUG and verbosity <= MU::Logger::LOUD)
-    return if verbosity == MU::Logger::SILENT
+  def self.log(msg, level = MU::INFO, details: nil, html: false, verbosity: nil, color: true)
+    return if (level == MU::DEBUG and verbosity and verbosity <= MU::Logger::LOUD)
+    return if verbosity and verbosity == MU::Logger::SILENT
 
     if (level == MU::ERR or
         level == MU::WARN or
         level == MU::DEBUG or
-        verbosity >= MU::Logger::LOUD or
-        (level == MU::NOTICE and !details.nil?)
-    )
-      # TODO add more stuff to details here (e.g. call stack)
-      extra = nil
-      if Thread.current.thread_variable_get("name") and (level > MU::NOTICE or verbosity >= MU::Logger::LOUD)
-        extra = Hash.new
-        extra = {
-          :thread => Thread.current.object_id,
-          :name => Thread.current.thread_variable_get("name")
-        }
-      end
-      if !details.nil?
-        extra = Hash.new if extra.nil?
-        extra[:details] = details
-      end
-      @@logger.log(msg, level, details: extra, verbosity: MU::Logger::LOUD, html: html, color: color)
-    else
-      @@logger.log(msg, level, html: html, verbosity: verbosity, color: color)
+        (verbosity and verbosity >= MU::Logger::LOUD) or
+        (level == MU::NOTICE and !details.nil?)) and
+        Thread.current.thread_variable_get("name")
+      newdetails = {
+        :thread => Thread.current.object_id,
+        :name => Thread.current.thread_variable_get("name")
+      }
+      newdetails[:details] = details.dup if details
+      details = newdetails
     end
+
+    @@logger.log(msg, level, details: details, html: html, verbosity: verbosity, color: color)
   end
 
   # For log entries that should only be logged when we're in verbose mode
@@ -894,9 +884,11 @@ module MU
 
   @@myCloudDescriptor = nil
   if MU.myCloud
-    found = MU::MommaCat.findStray(MU.myCloud, "server", cloud_id: @@myInstanceId, dummy_ok: true, region: MU.myRegion)
+    svrclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud).const_get("Server")
+    found = svrclass.find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
+#    found = MU::MommaCat.findStray(MU.myCloud, "server", cloud_id: @@myInstanceId, dummy_ok: true, region: MU.myRegion)
     if !found.nil? and found.size == 1
-      @@myCloudDescriptor = found.first.cloud_desc
+      @@myCloudDescriptor = found.values.first
     end
   end
 

data/modules/mu/cloud.rb

@@ -1351,6 +1351,8 @@ module MU
 
           # Special dependencies: my containing VPC
           if self.class.can_live_in_vpc and !@config['vpc'].nil?
+            @config['vpc']["id"] ||= @config['vpc']["vpc_id"] # old deploys
+            @config['vpc']["name"] ||= @config['vpc']["vpc_name"] # old deploys
             # If something hash-ified a MU::Config::Ref here, fix it
             if !@config['vpc']["id"].nil? and @config['vpc']["id"].is_a?(Hash)
               @config['vpc']["id"] = MU::Config::Ref.new(@config['vpc']["id"])
@@ -1930,6 +1932,8 @@ puts "CHOOSING #{@vpc.to_s} 'cause it has #{@config['vpc']['subnet_name']}"
             session = nil
             retries = 0
 
+            vpc_class = Object.const_get("MU").const_get("Cloud").const_get(@cloud).const_get("VPC")
+
             # XXX WHY is this a thing
             Thread.handle_interrupt(Errno::ECONNREFUSED => :never) {
             }
@@ -1953,6 +1957,7 @@ puts "CHOOSING #{@vpc.to_s} 'cause it has #{@config['vpc']['subnet_name']}"
                     :proxy => proxy
                 )
               else
+
                 MU.log "Attempting SSH to #{canonical_ip} (#{@mu_name}) as #{ssh_user} with key #{ssh_keydir}/#{@deploy.ssh_key_name}" if retries == 0
                 session = Net::SSH.start(
                     canonical_ip,
@@ -1986,9 +1991,14 @@ puts "CHOOSING #{@vpc.to_s} 'cause it has #{@config['vpc']['subnet_name']}"
 
               if retries < max_retries
                 retries = retries + 1
-                msg = "ssh #{ssh_user}@#{@mu_name}: #{e.message}, waiting #{retry_interval}s (attempt #{retries}/#{max_retries})", MU::WARN
+                msg = "ssh #{ssh_user}@#{@mu_name}: #{e.message}, waiting #{retry_interval}s (attempt #{retries}/#{max_retries})"
                 if retries == 1 or (retries/max_retries <= 0.5 and (retries % 3) == 0)
                   MU.log msg, MU::NOTICE
+                  if !vpc_class.haveRouteToInstance?(cloud_desc, credentials: @credentials) and
+                     canonical_ip.match(/(^127\.)|(^192\.168\.)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^::1$)|(^[fF][cCdD])/) and
+                     !nat_ssh_host
+                    MU.log "Node #{@mu_name} at #{canonical_ip} looks like it's in a private address space, and I don't appear to have a direct route to it. It may not be possible to connect with this routing!", MU::WARN
+                  end
                 elsif retries/max_retries > 0.5
                   MU.log msg, MU::WARN, details: e.inspect
                 end

data/modules/mu/clouds/aws.rb

@@ -164,6 +164,7 @@ module MU
       # @param r [String]
       # @return [String]
       def self.validate_region(r, credentials: nil)
+        require "aws-sdk-core"
         begin
           MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_availability_zones.availability_zones.first.region_name
         rescue ::Aws::EC2::Errors::UnauthorizedOperation => e
@@ -207,15 +208,19 @@ module MU
         MU.log "Created standard tags for resource #{resource}", MU::DEBUG, details: caller
       end
 
+      @@myVPCObj = nil
+
       # If we reside in this cloud, return the VPC in which we, the Mu Master, reside.
       # @return [MU::Cloud::VPC]
       def self.myVPCObj
+        return @@myVPCObj if @@myVPCObj
         return nil if !hosted?
         instance = MU.myCloudDescriptor
         return nil if !instance or !instance.vpc_id
-        vpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: instance.vpc_id, dummy_ok: true)
+        vpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: instance.vpc_id, dummy_ok: true, no_deploy_search: true)
         return nil if vpc.nil? or vpc.size == 0
-        vpc.first
+        @@myVPCObj = vpc.first
+        @@myVPCObj
       end
 
       # If we've configured AWS as a provider, or are simply hosted in AWS, 
@@ -1334,8 +1339,8 @@ module MU
         # @param region [String]: Amazon region so we know what endpoint to use
         # @param api [String]: Which API are we wrapping?
         def initialize(region: MU.curRegion, api: "EC2", credentials: nil)
-          @cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
           @credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
+          @cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
 
           if !@cred_obj
             raise MuError, "Unable to locate valid AWS credentials for #{api} API. #{credentials ? "Credentials requested were '#{credentials}'": ""}"

data/modules/mu/clouds/aws/alarm.rb

@@ -147,12 +147,9 @@ module MU
         end
 
         # Locate an existing alarm.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
         # @return [OpenStruct]: The cloud provider's complete descriptions of matching alarm.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
-          MU::Cloud::AWS::Alarm.getAlarmByName(cloud_id, region: region, credentials: credentials)
+        def self.find(**args)
+          MU::Cloud::AWS::Alarm.getAlarmByName(args[:cloud_id], region: args[:region], credentials: args[:credentials])
         end
 
         # Create an alarm.
@@ -260,13 +257,13 @@ module MU
           alarm["dimensions"] ||= []
 
           if alarm["#TARGETCLASS"] == "cache_cluster"
-            alarm['dimensions'] << { "name" => alarm["#TARGETCLASS"], "cloud_class" => "CacheClusterId" }
+            alarm['dimensions'] << { "name" => alarm["#TARGETNAME"], "cloud_class" => "CacheClusterId" }
             alarm["namespace"] = "AWS/ElastiCache" if alarm["namespace"].nil?
           elsif alarm["#TARGETCLASS"] == "server"
-            alarm['dimensions'] << { "name" => alarm["#TARGETCLASS"], "cloud_class" => "InstanceId" }
+            alarm['dimensions'] << { "name" => alarm["#TARGETNAME"], "cloud_class" => "InstanceId" }
             alarm["namespace"] = "AWS/EC2" if alarm["namespace"].nil?
           elsif alarm["#TARGETCLASS"] == "database"
-            alarm['dimensions'] << { "name" => alarm["#TARGETCLASS"], "cloud_class" => "DBInstanceIdentifier" }
+            alarm['dimensions'] << { "name" => alarm["#TARGETNAME"], "cloud_class" => "DBInstanceIdentifier" }
             alarm["namespace"] = "AWS/RDS" if alarm["namespace"].nil?
           end
 

data/modules/mu/clouds/aws/bucket.rb

@@ -33,11 +33,17 @@ module MU
           bucket_name = @deploy.getResourceName(@config["name"], max_length: 63).downcase
 
           MU.log "Creating S3 bucket #{bucket_name}"
-          MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).create_bucket(
+          resp = MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).create_bucket(
             acl: @config['acl'],
             bucket: bucket_name
           )
+
           @cloud_id = bucket_name
+          is_live = MU::Cloud::AWS::Bucket.find(cloud_id: @cloud_id, region: @config['region'], credentials: @credentials).values.first
+          begin
+            is_live = MU::Cloud::AWS::Bucket.find(cloud_id: @cloud_id, region: @config['region'], credentials: @credentials).values.first
+            sleep 3
+          end while !is_live
 
           @@region_cache_semaphore.synchronize {
             @@region_cache[@cloud_id] ||= @config['region']
@@ -216,7 +222,7 @@ puts path
                     else
                       @@region_cache[bucket.name] = location
                     end
-                  rescue Aws::S3::Errors::AccessDenied => e
+                  rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::AccessDenied
                     # this is routine- we saw a bucket that's not our business
                     next
                   end
@@ -261,14 +267,14 @@ puts path
         end
 
         # Locate an existing bucket.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching bucket.
+        def self.find(**args)
           found = {}
-          if cloud_id
-            found[cloud_id] = describe_bucket(cloud_id, minimal: true, credentials: credentials, region: region)
+          if args[:cloud_id]
+            begin
+              found[args[:cloud_id]] = describe_bucket(args[:cloud_id], minimal: true, credentials: args[:credentials], region: args[:region])
+            rescue ::Aws::S3::Errors::NoSuchBucket
+            end
           end
           found
         end

data/modules/mu/clouds/aws/cache_cluster.rb

@@ -39,27 +39,22 @@ module MU
         end
 
         # Locate an existing Cache Cluster or Cache Clusters and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching Cache Clusters.
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching Cache Clusters.
+        def self.find(**args)
           map = {}
-          if cloud_id
-            cache_cluster = MU::Cloud::AWS::CacheCluster.getCacheClusterById(cloud_id, region: region)
-            map[cloud_id] = cache_cluster if cache_cluster
+          if args[:cloud_id]
+            cache_cluster = MU::Cloud::AWS::CacheCluster.getCacheClusterById(args[:cloud_id], region: args[:region], credentials: args[:credentials])
+            map[args[:cloud_id]] = cache_cluster if cache_cluster
           end
 
-          if tag_value
-            MU::Cloud::AWS.elasticache(region: region, credentials: credentials).describe_cache_clusters.cache_clusters.each { |cc|
-              resp = MU::Cloud::AWS.elasticache(region: region, credentials: credentials).list_tags_for_resource(
-                  resource_name: MU::Cloud::AWS::CacheCluster.getARN(cc.cache_cluster_id, "cluster", "elasticache", region: region, credentials: credentials)
+          if args[:tag_value]
+            MU::Cloud::AWS.elasticache(region: args[:region], credentials: args[:credentials]).describe_cache_clusters.cache_clusters.each { |cc|
+              resp = MU::Cloud::AWS.elasticache(region: args[:region], credentials: args[:credentials]).list_tags_for_resource(
+                  resource_name: MU::Cloud::AWS::CacheCluster.getARN(cc.cache_cluster_id, "cluster", "elasticache", region: args[:region], credentials: args[:credentials])
               )
               if resp && resp.tag_list && !resp.tag_list.empty?
                 resp.tag_list.each { |tag|
-                    map[cc.cache_cluster_id] = cc if tag.key == tag_key and tag.value == tag_value
+                    map[cc.cache_cluster_id] = cc if tag.key == args[:tag_key] and tag.value == args[:tag_value]
                 }
               end
             }
@@ -222,14 +217,25 @@ module MU
             @cloud_id = resp.replication_group_id
           else
             config_struct[:cache_cluster_id] = @config['identifier']
-            config_struct[:az_mode] = @config["az_mode"]
+            config_struct[:az_mode] = @config["multi_az"] ? "cross-az" : "single-az"
             config_struct[:num_cache_nodes] = @config["node_count"]
             # config_struct[:replication_group_id] = @config["replication_group_id"] if @config["replication_group_id"]
             # config_struct[:preferred_availability_zone] = @config["preferred_availability_zone"] if @config["preferred_availability_zone"] && @config["az_mode"] == "single-az"
             # config_struct[:preferred_availability_zones] = @config["preferred_availability_zones"] if @config["preferred_availability_zones"] && @config["az_mode"] == "cross-az"
 
             MU.log "Creating cache cluster #{@config['identifier']}"
-            resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
+            begin
+              resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
+            rescue ::Aws::ElastiCache::Errors::InvalidParameterValue => e
+              if e.message.match(/security group (sg-[^\s]+)/)
+                bad_sg = Regexp.last_match[1]
+                MU.log "Removing invalid security group #{bad_sg} from Cache Cluster #{@mu_name}", MU::WARN, details: e.message
+                config_struct[:security_group_ids].delete(bad_sg)
+                retry
+              else
+                raise e
+              end
+            end
 
             wait_start_time = Time.now
             retries = 0
@@ -260,7 +266,6 @@ module MU
         # Create a subnet group for a Cache Cluster with the given config.
         def createSubnetGroup
           subnet_ids = []
-
           if @config["vpc"] && !@config["vpc"].empty?
             raise MuError, "Didn't find the VPC specified in #{@config["vpc"]}" unless @vpc
 
@@ -306,8 +311,8 @@ module MU
               }
 
               @config['vpc'] = {
-                  "vpc_id" => vpc_id,
-                  "subnets" => mu_subnets
+                "vpc_id" => vpc_id,
+                "subnets" => mu_subnets
               }
               using_default_vpc = true
               MU.log "Using default VPC for cache cluster #{@config['identifier']}"
@@ -346,8 +351,8 @@ module MU
 
             if @dependencies.has_key?('firewall_rule')
               @config["security_group_ids"] = []
-                @dependencies['firewall_rule'].values.each { |sg|
-                  @config["security_group_ids"] << sg.cloud_id
+              @dependencies['firewall_rule'].values.each { |sg|
+                @config["security_group_ids"] << sg.cloud_id
               }
             end
           end
@@ -691,6 +696,10 @@ module MU
         def self.schema(config)
           toplevel_required = []
           schema = {
+            "create_replication_group" => {
+              "type" => "boolean",
+              "description" => "Create a replication group; will be set automatically if +engine+ is +redis+ and +node_count+ is greated than one."
+            },
             "ingress_rules" => {
               "items" => {
                 "properties" => {
@@ -722,6 +731,32 @@ module MU
         def self.validateConfig(cache, configurator)
           ok = true
 
+          if !cache['vpc']
+            siblings = configurator.haveLitterMate?(nil, "vpcs", has_multiple: true)
+            if siblings.size == 1
+              MU.log "CacheCluster #{cache['name']} did not declare a VPC. Inserting into sibling VPC #{siblings[0]['name']}.", MU::WARN
+              cache["vpc"] = {
+                "name" => siblings[0]['name'],
+                "subnet_pref" => "all_private"
+              }
+            elsif MU::Cloud::AWS.hosted? and MU::Cloud::AWS.myVPCObj
+              cache["vpc"] = {
+                "id" => MU.myVPC,
+                "subnet_pref" => "all_private"
+              }
+            else
+              MU.log "CacheCluster #{cache['name']} must declare a VPC", MU::ERR
+              ok = false
+            end
+
+            # Re-insert ourselves with this modification so that our child
+            # resources get this VPC we just shoved in
+            if ok and cache['vpc']
+              cache.delete("#MU_VALIDATED")
+              return configurator.insertKitten(cache, "cache_clusters", overwrite: true)
+            end
+          end
+
           if cache.has_key?("parameter_group_parameters") && cache["parameter_group_family"].nil?
             MU.log "parameter_group_family must be set when setting parameter_group_parameters", MU::ERR
             ok = false
@@ -743,7 +778,6 @@ module MU
             end
           elsif cache["engine"] == "memcached"
             cache["create_replication_group"] = false
-            cache["az_mode"] = cache["multi_az"] ? "cross-az" : "single-az"
 
             if cache["node_count"] > 20
               MU.log "#{cache['engine']} supports up to 20 nodes per cache cluster", MU::ERR
@@ -868,7 +902,7 @@ module MU
         # @param region [String]: The cloud provider's region in which to operate.
         # @param cloud_id [String]: The cloud provider's identifier for this resource.
         # @return [void]
-        def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, cloud_id: nil)
+        def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, cloud_id: nil, credentials: nil)
           raise MuError, "terminate_replication_group requires a non-nil cache replication group descriptor" if repl_group.nil? || repl_group.empty?
 
           repl_group_id = repl_group.replication_group_id
@@ -908,9 +942,9 @@ module MU
                 )
               end
 
-              def self.createSnap(repl_group_id, region)
+              def self.createSnap(repl_group_id, region, credentials)
                 MU.log "Terminating #{repl_group_id}. Final snapshot name: #{repl_group_id}-mufinal"
-                MU::Cloud::AWS.elasticache(region: region).delete_replication_group(
+                MU::Cloud::AWS.elasticache(region: region, credentials: credentials).delete_replication_group(
                   replication_group_id: repl_group_id,
                   retain_primary_cluster: false,
                   final_snapshot_identifier: "#{repl_group_id}-mufinal"