cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/config/server_pool.rb

@@ -180,7 +180,7 @@ module MU
         pool['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
         ok = false if !MU::Config.check_vault_refs(pool)
 
-        if !pool['scrub_mu_isms']
+        if !pool['scrub_mu_isms'] and pool["cloud"] != "Azure"
           pool['dependencies'] << configurator.adminFirewallRuleset(vpc: pool['vpc'], region: pool['region'], cloud: pool['cloud'], credentials: pool['credentials'])
         end
 

data/modules/mu/config/storage_pool.rb

@@ -93,7 +93,7 @@ module MU
                 siblingvpc = configurator.haveLitterMate?(mp["vpc"]["vpc_name"], "vpcs")
                 if !MU::Config::VPC.processReference(mp['vpc'],
                                         "storage_pools",
-                                        "storagepool '#{pool['name']}'",
+                                        pool,
                                         configurator,
                                         dflt_region: pool['region'],
                                         credentials: pool['credentials'],
@@ -104,7 +104,7 @@ module MU
               else
                 if !MU::Config::VPC.processReference(mp["vpc"],
                                         "storage_pools",
-                                        "storagepool #{pool['name']}",
+                                        pool,
                                         configurator,
                                         dflt_region: pool['region'],
                                         credentials: pool['credentials'])

data/modules/mu/config/user.rb

@@ -41,6 +41,10 @@ module MU
               "description" => "If we attempt to create or associate a user that already exists, simply modify that user in-place and use it, rather than throwing an error. If this flag is set, the user will *not* be deleted on cleanup, nor will we overwrite any existing tags on cloud platforms that support user tagging.",
               "default" => true
             },
+            "force_password_change" => {
+              "type" => "boolean",
+              "description" => "For supported platforms and user types, require the user to reset their password on their next login. Our default behavior is to set this flag when initially creating an account. Setting it explicitly +true+ will set this flag on every subsequent +groom+ of the user, which may not be desired behavior."
+            },
             "create_api_key" => {
               "type" => "boolean",
               "default" => false,

data/modules/mu/config/vpc.rb

@@ -27,15 +27,21 @@ module MU
           "description" => "Create Virtual Private Clouds with custom public or private subnets.",
           "properties" => {
             "name" => {"type" => "string"},
+            "habitat" => MU::Config::Habitat.reference,
             "cloud" => MU::Config.cloud_primitive,
             "ip_block" => {
               "type" => "string",
               "pattern" => MU::Config::CIDR_PATTERN,
-              "description" => MU::Config::CIDR_DESCRIPTION,
-              "default" => "10.0.0.0/16"
+              "description" => MU::Config::CIDR_DESCRIPTION
             },
             "tags" => MU::Config.tags_primitive,
             "optional_tags" => MU::Config.optional_tags_primitive,
+            "create_bastion" => {
+              "type" => "boolean",
+              "description" => "If we have private subnets and our Mu Master will not be able to route directly to them, create a small instance to serve as an ssh relay.",
+              "default" => true
+            },
+            "bastion" => MU::Config::Ref.schema(type: "servers", desc: "A reference to a bastion host that can be used to tunnel into private address space in this VPC."),
             "create_standard_subnets" => {
               "type" => "boolean",
               "description" => "If the 'subnets' parameter to this VPC is not specified, we will instead create one set of public subnets and one set of private, with a public/private pair in each Availability Zone in the target region.",
@@ -117,19 +123,45 @@ module MU
                 }
             },
             "route_tables" => {
-                "type" => "array",
-                "items" => {
-                    "type" => "object",
-                    "required" => ["name", "routes"],
-                    "description" => "A table of route entries, typically for use inside a VPC.",
-                    "properties" => {
-                        "name" => {"type" => "string"},
-                        "routes" => {
-                            "type" => "array",
-                            "items" => routeschema
-                        }
+              "default_if" => [
+                {
+                  "key_is" => "create_standard_subnets",
+                  "value_is" => true,
+                  "set" => [
+                    {
+                      "name" => "internet",
+                      "routes" => [ { "destination_network" => "0.0.0.0/0", "gateway" => "#INTERNET" } ]
+                    },
+                    {
+                      "name" => "private",
+                      "routes" => [ { "destination_network" => "0.0.0.0/0", "gateway" => "#NAT" } ]
                     }
+                  ]
+                },
+                {
+                  "key_is" => "create_standard_subnets",
+                  "value_is" => false,
+                  "set" => [
+                    {
+                      "name" => "private",
+                      "routes" => [ { "destination_network" => "0.0.0.0/0" } ]
+                    }
+                  ]
                 }
+              ],
+              "type" => "array",
+              "items" => {
+                "type" => "object",
+                "required" => ["name", "routes"],
+                "description" => "A table of route entries, typically for use inside a VPC.",
+                "properties" => {
+                  "name" => {"type" => "string"},
+                  "routes" => {
+                    "type" => "array",
+                    "items" => routeschema
+                  }
+                }
+              }
             },
             "subnets" => {
                 "type" => "array",
@@ -223,34 +255,40 @@ module MU
       # @param subnet_pref [String]:
       # @return [Hash]
       def self.reference(subnets = MANY_SUBNETS, nat_opts = NAT_OPTS, subnet_pref = nil)
-        vpc_ref_schema = {
-          "type" => "object",
-          "description" => "Deploy, attach, allow access from, or peer this resource with a VPC of VPCs.",
-          "minProperties" => 1,
-          "additionalProperties" => false,
-          "properties" => {
-            "vpc_id" => {
-              "type" => "string",
-              "description" => "Discover this VPC by looking for this cloud provider identifier."
-            },
-            "credentials" => MU::Config.credentials_primitive,
-            "vpc_name" => {
-              "type" => "string",
-              "description" => "Discover this VPC by Mu-internal name; typically the shorthand 'name' field of a VPC declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
-            },
-            "region" => MU::Config.region_primitive,
-            "cloud" => MU::Config.cloud_primitive,
-            "tag" => {
-              "type" => "string",
-              "description" => "Discover this VPC by a cloud provider tag (key=value); note that this tag must not match more than one resource.",
-              "pattern" => "^[^=]+=.+"
-            },
-            "deploy_id" => {
-              "type" => "string",
-              "description" => "Search for this VPC in an existing Mu deploy; specify a Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
-            }
-          }
-        }
+        schema_aliases = [
+          { "vpc_id" => "id" },
+          { "vpc_name" => "name" }
+        ]
+        vpc_ref_schema = MU::Config::Ref.schema(schema_aliases, type: "vpcs")
+
+#        vpc_ref_schema = {
+#          "type" => "object",
+#          "description" => "Deploy, attach, allow access from, or peer this resource with a VPC of VPCs.",
+#          "minProperties" => 1,
+#          "additionalProperties" => false,
+#          "properties" => {
+#            "vpc_id" => {
+#              "type" => "string",
+#              "description" => "Discover this VPC by looking for this cloud provider identifier."
+#            },
+#            "credentials" => MU::Config.credentials_primitive,
+#            "vpc_name" => {
+#              "type" => "string",
+#              "description" => "Discover this VPC by Mu-internal name; typically the shorthand 'name' field of a VPC declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
+#            },
+#            "region" => MU::Config.region_primitive,
+#            "cloud" => MU::Config.cloud_primitive,
+#            "tag" => {
+#              "type" => "string",
+#              "description" => "Discover this VPC by a cloud provider tag (key=value); note that this tag must not match more than one resource.",
+#              "pattern" => "^[^=]+=.+"
+#            },
+#            "deploy_id" => {
+#              "type" => "string",
+#              "description" => "Search for this VPC in an existing Mu deploy; specify a Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
+#            }
+#          }
+#        }
 
         if nat_opts
           vpc_ref_schema["properties"].merge!(
@@ -374,19 +412,142 @@ module MU
       def self.validate(vpc, configurator)
         ok = true
 
+        have_public = false
+        have_private = false
+
+        using_default_cidr = false
+        if !vpc['ip_block']
+          using_default_cidr = true
+          vpc['ip_block'] = "10.0.0.0/16"
+        end
+
         # Look for a common YAML screwup in route table land
-        if vpc['route_tables']
-          vpc['route_tables'].each { |rtb|
-            next if !rtb['routes']
-            rtb['routes'].each { |r|
-              if r.has_key?("gateway") and (!r["gateway"] or r["gateway"].to_s.empty?)
-                MU.log "Route gateway in VPC #{vpc['name']} cannot be nil- did you forget to puts quotes around a #INTERNET, #NAT, or #DENY?", MU::ERR, details: rtb
-                ok = false
+        vpc['route_tables'].each { |rtb|
+          next if !rtb['routes']
+          rtb['routes'].each { |r|
+            have_public = true if r['gateway'] == "#INTERNET"
+            have_private = true if r['gateway'] == "#NAT" or r['gateway'] == "#DENY"
+            # XXX the above logic doesn't cover VPN ids, peering connections, or
+            # instances used as routers. If you're doing anything that complex
+            # you should probably be declaring your own bastion hosts and 
+            # routing behaviors, rather than relying on our inferred defaults.
+            if r.has_key?("gateway") and (!r["gateway"] or r["gateway"].to_s.empty?)
+              MU.log "Route gateway in VPC #{vpc['name']} cannot be nil- did you forget to puts quotes around a #INTERNET, #NAT, or #DENY?", MU::ERR, details: rtb
+              ok = false
+            end
+          }
+          rtb['routes'].uniq!
+        }
+
+        # if we're peering with other on-the-fly VPCs who might be using
+        # the default range, make sure our ip_blocks don't overlap
+        peer_blocks = []
+        my_cidr = NetAddr::IPv4Net.parse(vpc['ip_block'].to_s)
+        if vpc["peers"]
+          siblings = configurator.haveLitterMate?(nil, "vpcs", has_multiple: true)
+          siblings.each { |v|
+            next if v['name'] == vpc['name']
+            peer_blocks << v['ip_block'] if v['ip_block']
+          }
+          if peer_blocks.size > 0 and using_default_cidr
+            begin
+              have_overlaps = false
+              peer_blocks.each { |cidr|
+                sibling_cidr = NetAddr::IPv4Net.parse(cidr)
+                have_overlaps = true if my_cidr.rel(sibling_cidr) != nil
+              }
+              if have_overlaps
+                my_cidr = my_cidr.next_sib
+                my_cidr = nil if my_cidr.to_s.match(/^10\.255\./)
               end
+            end while have_overlaps
+            if !my_cidr.nil? and vpc['ip_block'] != my_cidr.to_s
+              vpc['ip_block'] = my_cidr.to_s
+            else
+              my_cidr = NetAddr::IPv4Net.parse(vpc['ip_block'])
+            end
+          end
+        end
+
+        # Work out what we'll do 
+        if have_private
+          vpc["cloud"] ||= MU.defaultCloud
+
+          # See if we'll be able to create peering connections
+          can_peer = false
+          if MU.myCloud == vpc["cloud"] and MU.myVPCObj
+            peer_blocks.concat(MU.myVPCObj.routes)
+            begin
+              can_peer = true
+              peer_blocks.each { |cidr|
+                cidr_obj = NetAddr::IPv4Net.parse(cidr)
+                if my_cidr.rel(cidr_obj) != nil
+                  can_peer = false
+                end
+              }
+              if !can_peer and using_default_cidr
+                my_cidr = my_cidr.next_sib
+                my_cidr = nil if my_cidr.to_s.match(/^10\.255\./)
+              end
+            end while !can_peer and using_default_cidr and !my_cidr.nil?
+            if !my_cidr.nil? and vpc['ip_block'] != my_cidr.to_s
+              vpc['ip_block'] = my_cidr.to_s
+            end
+            if using_default_cidr
+              MU.log "Defaulting address range for VPC #{vpc['name']} to #{vpc['ip_block']}", MU::NOTICE
+            end
+            if can_peer
+              vpc['peers'] ||= []
+              vpc['peers'] << {
+                "vpc" => { "id" => MU.myVPC, "type" => "vpcs" }
+              }
+            else
+              MU.log "#{vpc['ip_block']} overlaps with existing routes, will not be able to peer with Master's VPC", MU::WARN
+            end
+          end
+
+
+          # Feeling that, generate a generic bastion/NAT host to do the job.
+          # Clouds that don't have some kind of native NAT gateway can also
+          # leverage this host to honor "gateway" => "#NAT" situations.
+          if !can_peer and have_public and vpc["create_bastion"]
+            serverclass = Object.const_get("MU").const_get("Cloud").const_get(vpc["cloud"]).const_get("Server")
+            bastion = serverclass.genericNAT.dup
+            bastion['name'] = vpc['name']+"-natstion" # XXX account for multiples somehow
+            bastion['credentials'] = vpc['credentials']
+            bastion['ingress_rules'] ||= []
+            ["tcp", "udp", "icmp"].each { |proto|
+              bastion['ingress_rules'] << {
+                "hosts" => [vpc["ip_block"].to_s],
+                "proto" => proto
+              }
             }
-          }
+            bastion["application_attributes"] = {
+              "nat" => {
+                "private_net" => vpc["ip_block"].to_s
+              }
+            }
+            bastion["vpc"] = {
+              "name" => vpc["name"],
+              "subnet_pref" => "public"
+            }
+            vpc["dependencies"] << {
+              "type" => "server",
+              "name" => bastion['name'],
+            }
+            vpc["bastion"] = MU::Config::Ref.get(
+              name: bastion['name'],
+              cloud: vpc['cloud'],
+              credentials: vpc['credentials'],
+              type: "servers"
+            )
+
+            ok = false if !configurator.insertKitten(bastion, "servers", true)
+          end
+
         end
 
+
         ok = false if !resolvePeers(vpc, configurator)
 
         ok
@@ -404,22 +565,26 @@ module MU
           append = []
           delete = []
           vpc["peers"].each { |peer|
+            if peer.nil? or !peer.is_a?(Hash) or !peer["vpc"]
+              MU.log "Skipping malformed VPC peer in #{vpc['name']}", MU::ERR, details: peer
+              next
+            end
             peer["#MU_CLOUDCLASS"] = Object.const_get("MU").const_get("Cloud").const_get("VPC")
             # We check for multiple siblings because some implementations
             # (Google) can split declared VPCs into parts to get the mimic the
             # routing behaviors we expect.
-            siblings = configurator.haveLitterMate?(peer['vpc']["vpc_name"], "vpcs", has_multiple: true)
+            siblings = configurator.haveLitterMate?(peer['vpc']["name"], "vpcs", has_multiple: true)
 
             # If we're peering with a VPC in this deploy, set it as a dependency
-            if !peer['vpc']["vpc_name"].nil? and siblings.size > 0 and
+            if !peer['vpc']["name"].nil? and siblings.size > 0 and
                peer["vpc"]['deploy_id'].nil? and peer["vpc"]['vpc_id'].nil?
 
               peer['vpc']['cloud'] = vpc['cloud'] if peer['vpc']['cloud'].nil?
               siblings.each { |sib|
-                if sib['name'] != peer['vpc']["vpc_name"]
+                if sib['name'] != peer['vpc']["name"]
                   if sib['name'] != vpc['name']
                     append_me = { "vpc" => peer["vpc"].dup }
-                    append_me['vpc']['vpc_name'] = sib['name']
+                    append_me['vpc']['name'] = sib['name']
                     append << append_me
                     vpc["dependencies"] << {
                       "type" => "vpc",
@@ -430,7 +595,7 @@ module MU
                 else
                   vpc["dependencies"] << {
                     "type" => "vpc",
-                    "name" => peer['vpc']["vpc_name"]
+                    "name" => peer['vpc']["name"]
                   }
                 end
                 delete << peer if sib['name'] == vpc['name']
@@ -461,17 +626,19 @@ module MU
         ok
       end
 
+
+      @@reference_cache = {}
+
       # Pick apart an external VPC reference, validate it, and resolve it and its
       # various subnets and NAT hosts to live resources.
       # @param vpc_block [Hash]:
       # @param parent_type [String]:
-      # @param parent_name [String]:
+      # @param parent [MU::Cloud::VPC]:
       # @param configurator [MU::Config]:
-      # @param is_sibling [Boolean]:
       # @param sibling_vpcs [Array]:
       # @param dflt_region [String]:
-      def self.processReference(vpc_block, parent_type, parent_name, configurator, is_sibling: false, sibling_vpcs: [], dflt_region: MU.curRegion, credentials: nil)
-        puts vpc_block.ancestors if !vpc_block.is_a?(Hash)
+      def self.processReference(vpc_block, parent_type, parent, configurator, sibling_vpcs: [], dflt_region: MU.curRegion, dflt_project: nil, credentials: nil)
+
         if !vpc_block.is_a?(Hash) and vpc_block.kind_of?(MU::Cloud::VPC)
           return true
         end
@@ -480,8 +647,56 @@ module MU
         if vpc_block['region'].nil? and dflt_region and !dflt_region.empty?
           vpc_block['region'] = dflt_region.to_s
         end
+        dflt_region ||= vpc_block['region']
+        vpc_block['name'] ||= vpc_block['vpc_name'] if vpc_block['vpc_name']
+        vpc_block['id'] ||= vpc_block['vpc_id'] if vpc_block['vpc_id']
 
         vpc_block['credentials'] ||= credentials if credentials
+        vpc_block['project'] ||= dflt_project if dflt_project
+        vpc_block["cloud"] ||= parent["cloud"]
+
+# XXX the right thing to do here is have a per-cloud callback hook for resolving
+# projects/accounts/whatever, but for now let's get it working with Google's case
+        if vpc_block["cloud"] and vpc_block["cloud"] == "Google" and
+           vpc_block['project']
+          vpc_block["habitat"] ||= MU::Cloud::Google.projectToRef(vpc_block['project'], config: configurator, credentials: vpc_block['credentials']).to_h
+          vpc_block.delete("project")
+        end
+
+        # If this appears to be a sibling VPC that's destined to live in a
+        # sibling habitat, then by definition it doesn't exist yet. So don't
+        # try to do anything else clever here.
+# XXX except maybe there's some stuff we should still do
+        if vpc_block["habitat"] and vpc_block["habitat"]["name"] and
+           !vpc_block["habitat"]["id"]
+          return ok
+        end
+
+        # Resolve "forked" Google VPCs to the correct literal resources, based
+        # on the original reference to the (now virtual) parent VPC and, if
+        # set, subnet_pref or subnet_name
+        sibling_vpcs.each { |sibling|
+          if sibling['virtual_name'] and
+             sibling['virtual_name'] == vpc_block['name']
+            if vpc_block['region'] and
+               sibling['regions'].include?(vpc_block['region'])
+              gateways = sibling['route_tables'].map { |rtb|
+                rtb['routes'].map { |r| r["gateway"] }
+              }.flatten.uniq
+              if ["public", "all_public"].include?(vpc_block['subnet_pref']) and
+                 gateways.include?("#INTERNET")
+                vpc_block['name'] = sibling['name']
+                break
+              elsif ["private", "all_private"].include?(vpc_block['subnet_pref']) and
+                 !gateways.include?("#INTERNET")
+                vpc_block['name'] = sibling['name']
+                break
+              end
+            end
+          end
+        }
+
+        is_sibling = (vpc_block['name'] and configurator.haveLitterMate?(vpc_block["name"], "vpcs"))
 
         # Sometimes people set subnet_pref to "private" or "public" when they
         # mean "all_private" or "all_public." Help them out.
@@ -496,27 +711,47 @@ module MU
 
         flags = {}
         flags["subnet_pref"] = vpc_block["subnet_pref"] if !vpc_block["subnet_pref"].nil?
+        hab_arg = if vpc_block['habitat']
+          if vpc_block['habitat'].is_a?(MU::Config::Ref)
+            [vpc_block['habitat'].id] # XXX actually, findStray it
+          elsif vpc_block['habitat'].is_a?(Hash)
+            [vpc_block['habitat']['id']] # XXX actually, findStray it
+          else
+            [vpc_block['habitat'].to_s]
+          end
+        elsif vpc_block['project']
+          [vpc_block['project']]
+        else
+          []
+        end
 
         # First, dig up the enclosing VPC 
         tag_key, tag_value = vpc_block['tag'].split(/=/, 2) if !vpc_block['tag'].nil?
         if !is_sibling
           begin
             if vpc_block['cloud'] != "CloudFormation"
-              found = MU::MommaCat.findStray(
-                vpc_block['cloud'],
-                "vpc",
-                deploy_id: vpc_block["deploy_id"],
-                cloud_id: vpc_block["vpc_id"],
-                name: vpc_block["vpc_name"],
-                credentials: vpc_block["credentials"],
-                tag_key: tag_key,
-                tag_value: tag_value,
-                region: vpc_block["region"],
-                flags: flags,
-                dummy_ok: true
-              )
-
-              ext_vpc = found.first if found.size == 1
+              ext_vpc = if @@reference_cache[vpc_block]
+MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
+                @@reference_cache[vpc_block]
+              else
+                found = MU::MommaCat.findStray(
+                  vpc_block['cloud'],
+                  "vpc",
+                  deploy_id: vpc_block["deploy_id"],
+                  cloud_id: vpc_block["id"],
+                  name: vpc_block["name"],
+                  credentials: vpc_block["credentials"],
+                  tag_key: tag_key,
+                  tag_value: tag_value,
+                  region: vpc_block["region"],
+                  flags: flags,
+                  habitats: hab_arg,
+                  dummy_ok: true
+                )
+
+                found.first if found and found.size == 1
+              end
+              @@reference_cache[vpc_block] ||= ext_vpc
 
               # Make sure we don't have a weird mismatch between requested
               # credential sets and the VPC we actually found
@@ -525,26 +760,27 @@ module MU
                 if vpc_block['credentials'] and # probably can't happen
                    vpc_block['credentials'] != ext_vpc.cloudobj.config["credentials"]
                   ok = false
-                  MU.log "#{parent_type} #{parent_name} requested a VPC on credentials '#{vpc_block['credentials']}' but matched VPC is under credentials '#{ext_vpc.cloudobj.config["credentials"]}'", MU::ERR, details: vpc_block
+                  MU.log "#{parent_type} #{parent['name']} requested a VPC on credentials '#{vpc_block['credentials']}' but matched VPC is under credentials '#{ext_vpc.cloudobj.config["credentials"]}'", MU::ERR, details: vpc_block
                 end
                 if credentials and
                    credentials != ext_vpc.cloudobj.config["credentials"]
                   ok = false
-                  MU.log "#{parent_type} #{parent_name} is using credentials '#{credentials}' but matched VPC is under credentials '#{ext_vpc.cloudobj.config["credentials"]}'", MU::ERR, details: vpc_block
+                  MU.log "#{parent_type} #{parent['name']} is using credentials '#{credentials}' but matched VPC is under credentials '#{ext_vpc.cloudobj.config["credentials"]}'", MU::ERR, details: vpc_block
                 end
+                @@reference_cache[vpc_block] ||= ext_vpc if ok
                 vpc_block['credentials'] ||= ext_vpc.cloudobj.config["credentials"]
               end
-
+              @@reference_cache[vpc_block] ||= ext_vpc if ok
             end
           rescue Exception => e
             raise MuError, e.inspect, e.backtrace
           ensure
             if !ext_vpc and vpc_block['cloud'] != "CloudFormation"
-              MU.log "Couldn't resolve VPC reference to a unique live VPC in #{parent_name} (called by #{caller[0]})", MU::ERR, details: vpc_block
+              MU.log "Couldn't resolve VPC reference to a unique live VPC in #{parent_type} #{parent['name']} (called by #{caller[0]})", MU::ERR, details: vpc_block
               return false
-            elsif !vpc_block["vpc_id"]
-              MU.log "Resolved VPC to #{ext_vpc.cloud_id} in #{parent_name}", MU::DEBUG, details: vpc_block
-              vpc_block["vpc_id"] = configurator.getTail("#{parent_name} Target VPC", value: ext_vpc.cloud_id, prettyname: "#{parent_name} Target VPC", cloudtype: "AWS::EC2::VPC::Id")
+            elsif !vpc_block["id"]
+              MU.log "Resolved VPC to #{ext_vpc.cloud_id} in #{parent['name']}", MU::DEBUG, details: vpc_block
+              vpc_block["id"] = configurator.getTail("#{parent['name']} Target VPC", value: ext_vpc.cloud_id, prettyname: "#{parent['name']} Target VPC", cloudtype: "AWS::EC2::VPC::Id")
             end
           end
 
@@ -565,20 +801,20 @@ module MU
               nat_ip: vpc_block['nat_host_ip']
             )
             ssh_keydir = Etc.getpwnam(MU.mu_user).dir+"/.ssh"
-            if !vpc_block['nat_ssh_key'].nil? and !File.exists?(ssh_keydir+"/"+vpc_block['nat_ssh_key'])
-              MU.log "Couldn't find alternate NAT key #{ssh_keydir}/#{vpc_block['nat_ssh_key']} in #{parent_name}", MU::ERR, details: vpc_block
+            if !vpc_block['nat_ssh_key'].nil? and !File.exist?(ssh_keydir+"/"+vpc_block['nat_ssh_key'])
+              MU.log "Couldn't find alternate NAT key #{ssh_keydir}/#{vpc_block['nat_ssh_key']} in #{parent['name']}", MU::ERR, details: vpc_block
               return false
             end
 
             if !ext_nat
               if vpc_block["nat_host_id"].nil? and nat_tag_key.nil? and vpc_block['nat_host_ip'].nil? and vpc_block["deploy_id"].nil?
-                MU.log "Couldn't resolve NAT host to a live instance in #{parent_name}.", MU::DEBUG, details: vpc_block
+                MU.log "Couldn't resolve NAT host to a live instance in #{parent['name']}.", MU::DEBUG, details: vpc_block
               else
-                MU.log "Couldn't resolve NAT host to a live instance in #{parent_name}", MU::ERR, details: vpc_block
+                MU.log "Couldn't resolve NAT host to a live instance in #{parent['name']}", MU::ERR, details: vpc_block
                 return false
               end
             elsif !vpc_block["nat_host_id"]
-              MU.log "Resolved NAT host to #{ext_nat.cloud_id} in #{parent_name}", MU::DEBUG, details: vpc_block
+              MU.log "Resolved NAT host to #{ext_nat.cloud_id} in #{parent['name']}", MU::DEBUG, details: vpc_block
               vpc_block["nat_host_id"] = ext_nat.cloud_id
               vpc_block.delete('nat_host_name')
               vpc_block.delete('nat_host_ip')
@@ -600,13 +836,13 @@ module MU
 
               if ext_subnet.nil? and vpc_block["cloud"] != "CloudFormation"
                 ok = false
-                MU.log "Couldn't resolve subnet reference (list) in #{parent_name} to a live subnet", MU::ERR, details: subnet
+                MU.log "Couldn't resolve subnet reference (list) in #{parent['name']} to a live subnet", MU::ERR, details: subnet
               elsif !subnet['subnet_id']
                 subnet['subnet_id'] = ext_subnet.cloud_id
                 subnet['az'] = ext_subnet.az
                 subnet.delete('subnet_name')
                 subnet.delete('tag')
-                MU.log "Resolved subnet reference in #{parent_name} to #{ext_subnet.cloud_id}", MU::DEBUG, details: subnet
+                MU.log "Resolved subnet reference in #{parent['name']} to #{ext_subnet.cloud_id}", MU::DEBUG, details: subnet
               end
             }
             # ...others single subnets
@@ -619,13 +855,13 @@ module MU
 
             if ext_subnet.nil?
               ok = false
-              MU.log "Couldn't resolve subnet reference (name/id) in #{parent_name} to a live subnet", MU::ERR, details: vpc_block
+              MU.log "Couldn't resolve subnet reference (name/id) in #{parent['name']} to a live subnet", MU::ERR, details: vpc_block
             elsif !vpc_block['subnet_id']
               vpc_block['subnet_id'] = ext_subnet.cloud_id
               vpc_block['az'] = ext_subnet.az
               vpc_block.delete('subnet_name')
               vpc_block.delete('subnet_pref')
-              MU.log "Resolved subnet reference in #{parent_name} to #{ext_subnet.cloud_id}", MU::DEBUG, details: vpc_block
+              MU.log "Resolved subnet reference in #{parent['name']} to #{ext_subnet.cloud_id}", MU::DEBUG, details: vpc_block
             end
           end
         end
@@ -641,7 +877,7 @@ module MU
               honor_subnet_prefs=false
             end
             if !subnet['subnet_id'].nil? and subnet['subnet_id'].is_a?(String)
-              subnet['subnet_id'] = configurator.getTail("Subnet #{count} for #{parent_name}", value: subnet['subnet_id'], prettyname: "Subnet #{count} for #{parent_name}", cloudtype: "AWS::EC2::Subnet::Id")
+              subnet['subnet_id'] = configurator.getTail("Subnet #{count} for #{parent['name']}", value: subnet['subnet_id'], prettyname: "Subnet #{count} for #{parent['name']}", cloudtype: "AWS::EC2::Subnet::Id")
               count = count + 1
             end
           }
@@ -662,11 +898,11 @@ module MU
             ext_vpc.subnets.each { |subnet|
               next if dflt_region and vpc_block["cloud"] == "Google" and subnet.az != dflt_region
               if subnet.private? and (vpc_block['subnet_pref'] != "all_public" and vpc_block['subnet_pref'] != "public")
-                private_subnets << { "subnet_id" => configurator.getTail("#{parent_name} Private Subnet #{priv}", value: subnet.cloud_id, prettyname: "#{parent_name} Private Subnet #{priv}",  cloudtype:  "AWS::EC2::Subnet::Id"), "az" => subnet.az }
+                private_subnets << { "subnet_id" => configurator.getTail("#{parent['name']} Private Subnet #{priv}", value: subnet.cloud_id, prettyname: "#{parent['name']} Private Subnet #{priv}",  cloudtype:  "AWS::EC2::Subnet::Id"), "az" => subnet.az }
                 private_subnets_map[subnet.cloud_id] = subnet
                 priv = priv + 1
               elsif !subnet.private? and vpc_block['subnet_pref'] != "all_private" and vpc_block['subnet_pref'] != "private"
-                public_subnets << { "subnet_id" => configurator.getTail("#{parent_name} Public Subnet #{pub}", value: subnet.cloud_id, prettyname: "#{parent_name} Public Subnet #{pub}",  cloudtype: "AWS::EC2::Subnet::Id"), "az" => subnet.az }
+                public_subnets << { "subnet_id" => configurator.getTail("#{parent['name']} Public Subnet #{pub}", value: subnet.cloud_id, prettyname: "#{parent['name']} Public Subnet #{pub}",  cloudtype: "AWS::EC2::Subnet::Id"), "az" => subnet.az }
                 public_subnets_map[subnet.cloud_id] = subnet
                 pub = pub + 1
               else
@@ -674,12 +910,15 @@ module MU
               end
             }
           else
-            sibling_vpcs.each { |ext_vpc|
-              if ext_vpc['name'].to_s == vpc_block['vpc_name'].to_s and ext_vpc['subnets']
+            sibling_vpcs.each { |sibling_vpc|
+              if (sibling_vpc['name'].to_s == vpc_block['name'].to_s or
+                 sibling_vpc['virtual_name'].to_s == vpc_block['name'].to_s) and
+                 sibling_vpc['subnets']
                 subnet_ptr = "subnet_name"
-                ext_vpc['subnets'].each { |subnet|
-                  next if dflt_region and vpc_block["cloud"] == "Google" and subnet['availability_zone'] != dflt_region
-                  if subnet['is_public'] # NAT nonsense calculated elsewhere, ew
+
+                sibling_vpc['subnets'].each { |subnet|
+                  next if dflt_region and vpc_block["cloud"].to_s == "Google" and subnet['availability_zone'] != dflt_region
+                  if subnet['is_public']
                     public_subnets << {"subnet_name" => subnet['name'].to_s}
                   else
                     private_subnets << {"subnet_name" => subnet['name'].to_s}
@@ -689,13 +928,12 @@ module MU
                     end
                   end
                 }
-                break
               end
             }
           end
 
           if public_subnets.size == 0 and private_subnets == 0
-            MU.log "Couldn't find any subnets for #{parent_name}", MU::ERR
+            MU.log "Couldn't find any subnets for #{parent['name']}", MU::ERR
             return false
           end
           all_subnets = public_subnets + private_subnets
@@ -705,14 +943,16 @@ module MU
               if !public_subnets.nil? and public_subnets.size > 0
                 vpc_block.merge!(public_subnets[rand(public_subnets.length)]) if public_subnets
               else
-                MU.log "Public subnet requested for #{parent_name}, but none found in #{vpc_block}", MU::ERR
+                MU.log "Public subnet requested for #{parent_type} #{parent['name']}, but none found among #{all_subnets.join(", ")}", MU::ERR, details: vpc_block.to_h
+                pp is_sibling
                 return false
               end
             when "private"
               if !private_subnets.nil? and private_subnets.size > 0
                 vpc_block.merge!(private_subnets[rand(private_subnets.length)])
               else
-                MU.log "Private subnet requested for #{parent_name}, but none found in #{vpc_block}", MU::ERR
+                MU.log "Private subnet requested for #{parent_type} #{parent['name']}, but none found among #{all_subnets.join(", ")}", MU::ERR, details: vpc_block.to_h
+                pp is_sibling
                 return false
               end
               if !is_sibling and !private_subnets_map[vpc_block[subnet_ptr]].nil?
@@ -748,9 +988,9 @@ module MU
             else
               vpc_block['subnets'] ||= []
 
-              sibling_vpcs.each { |ext_vpc|
-                next if ext_vpc["name"] != vpc_block["vpc_name"]
-                ext_vpc["subnets"].each { |subnet|
+              sibling_vpcs.each { |sibling_vpc|
+                next if sibling_vpc["name"] != vpc_block["name"]
+                sibling_vpc["subnets"].each { |subnet|
                   if subnet["route_table"] == vpc_block["subnet_pref"]
                     vpc_block["subnets"] << subnet
                   end
@@ -784,16 +1024,16 @@ module MU
             }
           end
 
-          vpc_block.delete('deploy_id')
-          vpc_block.delete('vpc_name') if vpc_block.has_key?('vpc_id')
+          vpc_block.delete('id') if vpc_block['id'].nil?
+          vpc_block.delete('name') if vpc_block.has_key?('id')
           vpc_block.delete('tag')
-          MU.log "Resolved VPC resources for #{parent_name}", MU::DEBUG, details: vpc_block
+          MU.log "Resolved VPC resources for #{parent['name']}", MU::DEBUG, details: vpc_block
         end
 
-        if !vpc_block["vpc_id"].nil? and vpc_block["vpc_id"].is_a?(String)
-          vpc_block["vpc_id"] = configurator.getTail("#{parent_name}vpc_id", value: vpc_block["vpc_id"], prettyname: "#{parent_name} Target VPC",  cloudtype: "AWS::EC2::VPC::Id")
+        if !vpc_block["id"].nil? and vpc_block["id"].is_a?(String)
+          vpc_block["id"] = configurator.getTail("#{parent['name']}_id", value: vpc_block["id"], prettyname: "#{parent['name']} Target VPC",  cloudtype: "AWS::EC2::VPC::Id")
         elsif !vpc_block["nat_host_name"].nil? and vpc_block["nat_host_name"].is_a?(String)
-          vpc_block["nat_host_name"] = MU::Config::Tail.new("#{parent_name}nat_host_name", vpc_block["nat_host_name"])
+          vpc_block["nat_host_name"] = MU::Config::Tail.new("#{parent['name']}nat_host_name", vpc_block["nat_host_name"])
 
         end