cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

data/cookbooks/mu-master/recipes/init.rb

@@ -39,7 +39,7 @@ CHEF_SERVER_VERSION="12.17.15-1"
 CHEF_CLIENT_VERSION="14.4.56"
 KNIFE_WINDOWS="1.9.0"
 MU_BASE="/opt/mu"
-MU_BRANCH="i_yam_what_i_yam" # GIT HOOK EDITABLE DO NOT TOUCH
+MU_BRANCH="Azure_you_want_azure" # GIT HOOK EDITABLE DO NOT TOUCH
 realbranch=`cd #{MU_BASE}/lib && git rev-parse --abbrev-ref HEAD`
 
 if ENV.key?('MU_BRANCH')
@@ -234,7 +234,7 @@ file  "#{MU_BASE}/lib/.git/hooks/pre-commit" do
   action :delete
 end
 
-[MU_BASE+"/var", MU_BASE+"/install", MU_BASE+"/deprecated-bash-library.sh"].each do |dir|
+[MU_BASE+"/var", MU_BASE+"/var/ssl"].each do |dir|
   directory dir do
     recursive true
     mode 0755
@@ -308,7 +308,9 @@ rpms.each_pair { |pkg, src|
     end
   end
 }
-package "jq"
+package "jq" do
+  ignore_failure true # sometimes we can't see EPEL immediately
+end
 package removepackages do
   action :remove
 end
@@ -343,6 +345,7 @@ file "#{MU_BASE}/var/users/mu/email" do
     content "#{$MU_CFG['mu_admin_email']}\n"
   else
     content "root@example.com\n"
+    action :create_if_missing
   end
 end
 file "#{MU_BASE}/var/users/mu/realname" do
@@ -350,6 +353,7 @@ file "#{MU_BASE}/var/users/mu/realname" do
     content "#{$MU_CFG['mu_admin_name']}\n"
   else
     content "Mu Administrator\n"
+    action :create_if_missing
   end
 end
 

data/cookbooks/mu-master/recipes/ssl-certs.rb

@@ -25,6 +25,7 @@
 include_recipe 'mu-master::firewall-holes'
 service_certs = ["rsyslog", "mommacat", "ldap", "consul", "vault"]
 
+directory "#{$MU_CFG['datadir']}"
 directory "#{$MU_CFG['datadir']}/ssl"
 template "#{$MU_CFG['datadir']}/ssl/openssl.cnf" do
   source "openssl.cnf.erb"

data/cookbooks/mu-mongo/Berksfile

@@ -0,0 +1,10 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+
+# Supermarket Cookbooks
+cookbook 'mongodb', '~> 0.16.2'
+cookbook 'chef-vault', '~> 3.1.1'

data/cookbooks/mu-openvpn/Berksfile

@@ -0,0 +1,11 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+cookbook 'mu-utility'
+cookbook 'mu-firewall'
+
+# Supermarket Cookbooks
+cookbook 'chef-vault', '~> 3.1.1'

data/cookbooks/mu-php54/Berksfile

@@ -0,0 +1,13 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+cookbook 'mu-utility'
+
+# Supermarket Cookbooks
+cookbook 'simple_iptables', '~> 0.8.0'
+cookbook 'apache2', '< 4.0'
+cookbook 'mysql', '~> 8.5.1'
+cookbook 'yum-epel', '~> 3.2.0'

data/cookbooks/mu-splunk/Berksfile

@@ -0,0 +1,10 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+
+# Supermarket Cookbooks
+cookbook 'chef-vault', '~> 3.1.1'
+cookbook 'windows', '~> 5.1.1'

data/cookbooks/mu-tools/Berksfile

@@ -0,0 +1,21 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+cookbook "nagios"
+cookbook "mu-utility"
+cookbook "mu-splunk"
+cookbook "mu-firewall"
+cookbook "mu-activedirectory"
+
+# Supermarket Cookbooks
+cookbook "oracle-instantclient", '~> 1.1.0'
+cookbook "database", '~> 6.1.1'
+cookbook "postgresql", '~> 7.1.0'
+cookbook "java", '~> 2.2.0'
+cookbook "windows", '~> 5.1.1'
+cookbook "chef-vault", '~> 3.1.1'
+cookbook "poise-python", '~> 1.7.0'
+cookbook "yum-epel", '~> 3.2.0'

data/cookbooks/mu-tools/files/default/Mu_CA.pem

@@ -1,8 +1,8 @@
 -----BEGIN CERTIFICATE-----
-MIIF2zCCA8OgAwIBAgIJALVaC3iJgQA6MA0GCSqGSIb3DQEBDQUAMF0xFjAUBgNV
+MIIF2zCCA8OgAwIBAgIJAJSp1wu4cHj9MA0GCSqGSIb3DQEBDQUAMF0xFjAUBgNV
 BAMMDTU0LjE3NS44Ni4xOTQxIDAeBgNVBAsMF011IFNlcnZlciA1NC4xNzUuODYu
-MTk0MRQwEgYDVQQKDAtlR2xvYmFsVGVjaDELMAkGA1UEBhMCVVMwHhcNMTgxMjAz
-MTM1OTUxWhcNMjEwOTIyMTM1OTUxWjBdMRYwFAYDVQQDDA01NC4xNzUuODYuMTk0
+MTk0MRQwEgYDVQQKDAtlR2xvYmFsVGVjaDELMAkGA1UEBhMCVVMwHhcNMTkwMTIy
+MTUzMjM4WhcNMjExMTExMTUzMjM4WjBdMRYwFAYDVQQDDA01NC4xNzUuODYuMTk0
 MSAwHgYDVQQLDBdNdSBTZXJ2ZXIgNTQuMTc1Ljg2LjE5NDEUMBIGA1UECgwLZUds
 b2JhbFRlY2gxCzAJBgNVBAYTAlVTMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
 CgKCAgEAo7rntOFj/WPNvh00SN55aJBusppsY9arq7QF5gt/9+cBPsjcXn7jJMu0
@@ -19,16 +19,16 @@ e4Q3VnxhRfmkS1NqEzIvPabVLg9qvN419cubpE6HAtBJw/f3ocUCAwEAAaOBnTCB
 mjBKBgNVHREEQzBBhwQ2r1bCgglsb2NhbGhvc3SHBH8AAAGCGXN0YW5nZS1tdS1k
 ZXYucGxhdGZvcm0tbXWCDXN0YW5nZS1tdS1kZXYwHQYDVR0OBBYEFK/EmtGebCwd
 5QpM8y/3EKdYNVbcMB8GA1UdIwQYMBaAFK/EmtGebCwd5QpM8y/3EKdYNVbcMAwG
-A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQENBQADggIBAIFFoIWaS2vsiMjEeFo/FXKd
-Gy5lrvbBMaKgeFBI7Yqrz7wxmPol5de1KuuYANrcHtgpTuqnNRPMKt4VF6NW6CRY
-FTaaG0kwJU0Y3FMAdrmUUFK0bqNhchrwSv3Zqs68ifQ7vxwN0xLZrKJ5kMWjfQCn
-XDOImEe14ZQYwAf0kxZXm9qGoVQZK+ObIfNDPTPAOADQ3ZeawRrogZ2M6694mN+4
-eS/PDibpcZ4Dl8Cw4nuLIG4ct9Dm/8kZ0XRAoxXTEUfMIrJQpKmjnhux8Kzjy/DU
-eovi+530klyeyV0909lN9l8JJmBd3Zw1EArSB2PgSSTfGXrdGN/A6TlE50QahZS6
-wlE4/P2ISDCyyc+Zobu9e+6WII7DFcNwzyFuC3WO6h2I2IXnZvUfVCowjipfdSKx
-+qQvevSmbprs+AJVfvkyaejKYK5PPe+fGMHJo80Pqc2LiODoChs6NZh10xAG0Sd6
-zQ05A4ZUmZjlC0lpFkgVPBaAlUAW28y6CdlRNW6H52KgvctJecGrBYZ52cTdju1b
-AXdlwMbPHHoA0HDCT7vGhGb/zUWkWYJVpXQ7EwwQWdCEejegtCRNWqTZC0s4mjUa
-Yw4ISVSAaTkzWWxBkizBGJcIxUfukuhnEEGs1G09hqpaXMWyXLg3wf9GkCsn+tD+
-PY7N1R6ysc8wA8nByPeR
+A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQENBQADggIBAEQ1l3JqlK7hQHhhA7Y4Odhk
+mGsEJ8m1Kdb6wq9/hMAAOhzFPtzIzxPaDUuN29nGT/d4urTdJ+fSa7f8uhFyzKEL
+retbsal0n4MFD7Iu+QopYBSTsS3w7y9a4HGj2AW6IMrYjWFBtl1QjRMKPymwTV6Y
+ngjfJdJy4ySUP1FTl7hG1LdN+My6Q2ykQznyY/g51RzSNzmreWkK9Qfy/bk7G8ss
+vDPQOqocwQE/5GoO4WqDn+6IFw1bgs+rPt897MOv0YjLZR3B5Q8SYivp+7CVvLZT
+hKhIHL41k3H8R7khmV+Ak6ok9k+hCnT6pWC40g0jMyuy/HnlfeXcz9betiNm0Vhg
+98pxFjISd7GaSmzqH4L+NbNshgRBhgTA+qm5Nu7hETf5b4tV1gBi0MA/CzpD3cq5
+MadyWDkDnbGnSPqrLHvZwKjvDwUhPYw+6jYU9ejkG8I3G8ntmo4j7el+bYNDa1Yu
+o60AIxVn2LpDIPICibf2cpspOqxQMB0fClAttajklXCwZ73TomQP/9TegjMeLJMl
+p2KreRF3lvflajjrV7KHQ6tm8t9NIqljd3q5KW1RqatRwyMQUBWo8i8DlD6HQ+w0
+gipxlcwMUqjVEIQyUtCNax6RDpysdOsB6uPPkUxb825bRwOWnAHPVCTc/HaTVzmA
+FTNwHY0pyDGREcu+cb/A
 -----END CERTIFICATE-----

data/cookbooks/mu-utility/Berksfile

@@ -0,0 +1,9 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+
+# Supermarket Cookbooks
+cookbook 'windows', '~> 5.1.1'

data/cookbooks/mu-utility/metadata.rb

@@ -13,4 +13,5 @@ version '0.6.0'
 	supports os
 end
 
-depends 'windows', '~> 5.1.1'
+depends 'windows', '~> 5.1.1'
+depends 'mu-firewall'

data/cookbooks/nagios/Berksfile

@@ -1,8 +1,11 @@
 source 'https://supermarket.chef.io'
+source chef_repo: ".."
 
 metadata
 
-group :integration do
-  cookbook 'apt'
-  cookbook 'nagios_test', path: './test/fixtures/cookbooks/nagios_test'
-end
+# Mu Cookbooks
+
+# Supermarket Cookbooks
+cookbook 'apache2', '< 4.0'
+cookbook 'php', '< 6.0'
+cookbook 'zap', '>= 0.6.0'

data/cookbooks/s3fs/Berksfile

@@ -0,0 +1,9 @@
+source 'https://supermarket.chef.io'
+source chef_repo: ".."
+
+metadata
+
+# Mu Cookbooks
+cookbook "mu-utility"
+
+# Supermarket Cookbooks

data/environments/dev.json

@@ -1,8 +1,8 @@
 {
-  "name": "dev",
-  "default_attributes": {
-  },
-  "json_class": "Chef::Environment",
-  "description": "Infrastructure development environment",
-  "chef_type": "environment"
+	"name": "DEV",
+	"default_attributes": {
+	  },
+	"json_class": "Chef::Environment",
+	"description": "Infrastructure development environment",
+	"chef_type": "environment"
 }

data/environments/prod.json

@@ -1,8 +1,8 @@
 {
-  "name": "prod",
-  "default_attributes": {
-  },
-  "json_class": "Chef::Environment",
-  "description": "Infrastructure production environment",
-  "chef_type": "environment"
+	"name": "PROD",
+	"default_attributes": {
+	  },
+	"json_class": "Chef::Environment",
+	"description": "Infrastructure production environment",
+	"chef_type": "environment"
 }

data/modules/mu.rb

@@ -498,7 +498,7 @@ module MU
       begin
         @@myAZ_var ||= MU.myCloudDescriptor.placement.availability_zone
       rescue Aws::EC2::Errors::InternalError => e
-        MU.log "Got #{e.inspect} on MU::Cloud::AWS.ec2(#{MU.myRegion}).describe_instances(instance_ids: [#{@@myInstanceId}])", MU::WARN
+        MU.log "Got #{e.inspect} on MU::Cloud::AWS.ec2(region: #{MU.myRegion}).describe_instances(instance_ids: [#{@@myInstanceId}])", MU::WARN
         sleep 10
       end
     end
@@ -514,7 +514,7 @@ module MU
     )
   elsif MU::Cloud::AWS.hosted?
     begin
-      @@myCloudDescriptor = MU::Cloud::AWS.ec2(MU.myRegion).describe_instances(instance_ids: [MU.myInstanceId]).reservations.first.instances.first
+      @@myCloudDescriptor = MU::Cloud::AWS.ec2(region: MU.myRegion).describe_instances(instance_ids: [MU.myInstanceId]).reservations.first.instances.first
     rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
     rescue Aws::Errors::MissingCredentialsError => e
       MU.log "I'm hosted in AWS, but I can't make API calls. Does this instance have an appropriate IAM profile?", MU::WARN
@@ -536,7 +536,7 @@ module MU
         nil
       end
     rescue Aws::EC2::Errors::InternalError => e
-      MU.log "Got #{e.inspect} on MU::Cloud::AWS.ec2(#{MU.myRegion}).describe_instances(instance_ids: [#{@@myInstanceId}])", MU::WARN
+      MU.log "Got #{e.inspect} on MU::Cloud::AWS.ec2(region: #{MU.myRegion}).describe_instances(instance_ids: [#{@@myInstanceId}])", MU::WARN
       sleep 10
     end
     @@myVPC_var
@@ -546,7 +546,7 @@ module MU
   # The AWS Subnets associated with the VPC this MU Master is in
   # XXX account for Google and non-cloud situations
   def self.mySubnets
-    @@mySubnets_var ||= MU::Cloud::AWS.ec2(MU.myRegion).describe_subnets(
+    @@mySubnets_var ||= MU::Cloud::AWS.ec2(region: MU.myRegion).describe_subnets(
       filters: [
         {
           name: "vpc-id", 
@@ -678,47 +678,25 @@ module MU
   end
 
 
-  # Return the name of the S3 Mu log and key bucket for this Mu server.
+  # Return the name of the Mu log and key bucket for this Mu server. Not
+  # necessarily in any specific cloud provider.
   # @return [String]
-  # XXX account for Google and non-cloud situations
-  def self.adminBucketName
-    bucketname = $MU_CFG['aws']['log_bucket_name']
-    if bucketname.nil? or bucketname.empty?
-      bucketname = "Mu_Logs_"+Socket.gethostname+"_"+MU::Cloud::AWS.getAWSMetaData("instance-id")
-    end
+  def self.adminBucketName(platform = nil, credentials: nil)
+    return nil if platform and !MU::Cloud.supportedClouds.include?(platform)
+
+    clouds = platform.nil? ? MU::Cloud.supportedClouds : [platform]
+    clouds.each { |cloud|
+      cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+      bucketname = cloudclass.adminBucketName(credentials)
+      begin
+        if platform or (cloudclass.hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
+          return bucketname
+        end
+      end
+    }
+
     return bucketname
   end
 
-  # Log bucket policy for enabling CloudTrail logging to our log bucket in S3.
-  CLOUDTRAIL_BUCKET_POLICY = '{
-		"Version": "2012-10-17",
-		"Statement": [
-			{
-				"Sid": "AWSCloudTrailAclCheck20131101",
-				"Effect": "Allow",
-        "Principal": {
-          "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+':iam::<%= MU.account_number %>:root",
-          "Service": "cloudtrail.amazonaws.com"
-        },
-				"Action": "s3:GetBucketAcl",
-				"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+':s3:::<%= $bucketname %>"
-			},
-			{
-				"Sid": "AWSCloudTrailWrite20131101",
-				"Effect": "Allow",
-        "Principal": {
-          "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+':iam::<%= MU.account_number %>:root",
-          "Service": "cloudtrail.amazonaws.com"
-        },
-				"Action": "s3:PutObject",
-				"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+':s3:::<%= $bucketname %>/AWSLogs/<%= MU.account_number %>/*",
-				"Condition": {
-					"StringEquals": {
-						"s3:x-amz-acl": "bucket-owner-full-control"
-					}
-				}
-			}
-		]
-	}'
 
 end

data/modules/mu/cleanup.rb

@@ -60,6 +60,10 @@ module MU
         MU.setVar("dataDir", MU.mainDataDir)
       end
 
+
+# XXX AWS needs to check MU::Cloud::AWS.isGovCloud? on some things, or gracefully handle the API not existing
+      types_in_order = ["Collection", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "DNSZone", "Collection"]
+
       # Load up our deployment metadata
       if !mommacat.nil?
         @mommacat = mommacat
@@ -82,124 +86,122 @@ module MU
         end
       end
 
-      projects = {
-        "Google" => MU::Cloud::Google.listProjects,
-        "AWS" => ["dummy"]
-      }
-
       if !@skipcloud
+        creds = {}
+        MU::Cloud.supportedClouds.each { |cloud|
+          if $MU_CFG[cloud.downcase] and $MU_CFG[cloud.downcase].size > 0
+            cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+            creds[cloud] ||= {}
+            $MU_CFG[cloud.downcase].keys.each { |credset|
+              creds[cloud][credset] = cloudclass.listRegions(credentials: credset)
+            }
+          end
+        }
         parent_thread_id = Thread.current.object_id
-        regions = {}
-        regions['AWS'] = MU::Cloud::AWS.listRegions
-        regions['Google'] = MU::Cloud::Google.listRegions
         deleted_nodes = 0
         @regionthreads = []
         keyname = "deploy-#{MU.deploy_id}"
 # XXX blindly checking for all of these resources in all clouds is now prohibitively slow. We should only do this when we don't see deployment metadata to work from.
-        regions.each_pair { |provider, list|
-          list.each { |r|
-            @regionthreads << Thread.new {
-              MU.dupGlobals(parent_thread_id)
-              MU.setVar("curRegion", r)
-              if projects[provider].size == 1
-                MU.log "Checking for #{provider} resources from #{MU.deploy_id} in #{r}", MU::NOTICE
-              end
-
-              # We do these in an order that unrolls dependent resources
-              # sensibly, and we hit :Collection twice because AWS
-              # CloudFormation sometimes fails internally.
-              projectthreads = []
-              projects[provider].each { |project|
-                projectthreads << Thread.new {
-                  MU.dupGlobals(parent_thread_id)
-                  MU.setVar("curRegion", r)
-                  if projects[provider].size > 1
-                    MU.log "Checking for #{provider} resources from #{MU.deploy_id} in #{r}, project #{project}", MU::NOTICE
-                  end
-                  MU.dupGlobals(parent_thread_id)
-                  flags = { "project" => project }
-                  MU::Cloud::Collection.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["Collection"]) > 0
-                  MU::Cloud::Function.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["Function"]) > 0
-                  MU::Cloud::ServerPool.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["ServerPool"]) > 0
-                  begin
-                    MU::Cloud::ContainerCluster.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["ContainerCluster"]) > 0
-                  rescue Seahorse::Client::NetworkingError => e
-                    MU.log "Service not available in AWS region #{r}, skipping", MU::DEBUG, details: e.message
-                  end
-                  MU::Cloud::SearchDomain.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["SearchDomain"]) > 0
-                  MU::Cloud::Server.cleanup(skipsnapshots: @skipsnapshots, onlycloud: @onlycloud, noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["Server"]) > 0
-                  if provider == "AWS"
-                    MU::Cloud::MsgQueue.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["MsgQueue"]) > 0
-                    MU::Cloud::Database.cleanup(skipsnapshots: @skipsnapshots, noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["Database"]) > 0
-                  end
-                  MU::Cloud::CacheCluster.cleanup(skipsnapshots: @skipsnapshots, noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["CacheCluster"]) > 0
-                  MU::Cloud::StoragePool.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["StoragePool"]) > 0
-                  if provider == "AWS"
-                    MU::Cloud::FirewallRule.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["FirewallRule", "Server", "ServerPool", "Database", "StoragePool"]) > 0
-                  end
-                  if @mommacat.nil? or @mommacat.numKittens(types: ["LoadBalancer"]) > 0
-                    MU::Cloud::LoadBalancer.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags)
-                    if provider == "AWS"
-                      MU::Cloud::FirewallRule.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags)
+        creds.each_pair { |provider, credsets|
+          credsets.each_pair { |credset, regions|
+            global_vs_region_semaphore = Mutex.new
+            global_done = []
+            regions.each { |r|
+              @regionthreads << Thread.new {
+                MU.dupGlobals(parent_thread_id)
+                MU.setVar("curRegion", r)
+                projects = []
+                if $MU_CFG[provider.downcase][credset]["project"]
+# XXX GCP credential schema needs an array for projects
+                  projects << $MU_CFG[provider.downcase][credset]["project"]
+                end
+
+                if projects == [""]
+                  MU.log "Checking for #{provider}/#{credset} resources from #{MU.deploy_id} in #{r}", MU::NOTICE
+                end
+
+                # We do these in an order that unrolls dependent resources
+                # sensibly, and we hit :Collection twice because AWS
+                # CloudFormation sometimes fails internally.
+                projectthreads = []
+                projects.each { |project|
+                  projectthreads << Thread.new {
+                    MU.dupGlobals(parent_thread_id)
+                    MU.setVar("curRegion", r)
+                    if project != ""
+                      MU.log "Checking for #{provider}/#{credset} resources from #{MU.deploy_id} in #{r}, project #{project}", MU::NOTICE
                     end
-                  end
-                  MU::Cloud::Alarm.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["Alarm"]) > 0 # XXX other resources can make these appear, I think- which ones?
-                  MU::Cloud::Notification.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["Notification"]) > 0 # XXX other resources can make these appear, I think- which ones?
-                  MU::Cloud::Log.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["Log"]) > 0 # XXX other resources can make these appear, I think- which ones?
-                  if provider == "AWS" and (@mommacat.nil? or @mommacat.numKittens(types: ["VPC"]) > 0)
-                    MU::Cloud::FirewallRule.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags)
-                    MU::Cloud::VPC.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, cloud: provider, flags: flags)
-                  end
-                  MU::Cloud::Collection.cleanup(noop: @noop, ignoremaster: @ignoremaster, region: r, wait: true, cloud: provider, flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["Collection"]) > 0
-                }
-              }
-              projectthreads.each do |t|
-                t.join
-              end
 
-              if provider == "AWS"
-                resp = MU::Cloud::AWS.ec2(r).describe_key_pairs(
-                    filters: [{name: "key-name", values: [keyname]}]
-                )
-                resp.data.key_pairs.each { |keypair|
-                  MU.log "Deleting key pair #{keypair.key_name} from #{r}"
-                  MU::Cloud::AWS.ec2(r).delete_key_pair(key_name: keypair.key_name) if !@noop
+                    MU.dupGlobals(parent_thread_id)
+                    flags = {
+                      "project" => project,
+                      "onlycloud" => @onlycloud,
+                      "skipsnapshots" => @skipsnapshots,
+                    }
+                    types_in_order.each { |t|
+                      begin
+                        skipme = false
+                        global_vs_region_semaphore.synchronize {
+                          if Object.const_get("MU").const_get("Cloud").const_get(provider).const_get(t).isGlobal?
+                            if !global_done.include?(t)
+                              global_done << t
+                              flags['global'] = true
+                            else
+                              skipme = true
+                            end
+                          end
+                        }
+                        next if skipme
+                      rescue MU::Cloud::MuCloudResourceNotImplemented => e
+                        next
+                      rescue MU::MuError, NoMethodError => e
+                        MU.log e.message, MU::WARN
+                        next
+                      end
+
+                      if @mommacat.nil? or @mommacat.numKittens(types: [t]) > 0
+                        begin
+                          resclass = Object.const_get("MU").const_get("Cloud").const_get(t)
+                          resclass.cleanup(
+                            noop: @noop,
+                            ignoremaster: @ignoremaster,
+                            region: r,
+                            cloud: provider,
+                            flags: flags,
+                            credentials: credset
+                          )
+                        rescue Seahorse::Client::NetworkingError => e
+                          MU.log "Service not available in AWS region #{r}, skipping", MU::DEBUG, details: e.message
+                        end
+                      end
+                    }
+                  }
                 }
-              end
+                projectthreads.each do |t|
+                  t.join
+                end
+
+                # XXX move to MU::AWS
+                if provider == "AWS"
+                  resp = MU::Cloud::AWS.ec2(region: r, credentials: credset).describe_key_pairs(
+                      filters: [{name: "key-name", values: [keyname]}]
+                  )
+                  resp.data.key_pairs.each { |keypair|
+                    MU.log "Deleting key pair #{keypair.key_name} from #{r}"
+                    MU::Cloud::AWS.ec2(region: r, credentials: credset).delete_key_pair(key_name: keypair.key_name) if !@noop
+                  }
+                end
+              }
             }
           }
-          MU::Cloud::Role.cleanup(noop: @noop, ignoremaster: @ignoremaster, cloud: provider) if @mommacat.nil? or @mommacat.numKittens(types: ["Role"]) > 0
-          MU::Cloud::Group.cleanup(noop: @noop, ignoremaster: @ignoremaster, cloud: provider) if @mommacat.nil? or @mommacat.numKittens(types: ["Group"]) > 0
-          MU::Cloud::User.cleanup(noop: @noop, ignoremaster: @ignoremaster, cloud: provider) if @mommacat.nil? or @mommacat.numKittens(types: ["User"]) > 0
         }
 
-        # knock over region-agnostic resources
-
         @regionthreads.each do |t|
           t.join
         end
         @projectthreads = []
 
 
-        projects["Google"].each { |project|
-          @projectthreads << Thread.new {
-            MU.dupGlobals(parent_thread_id)
-            flags = { "global" => true, "project" => project }
-            MU::Cloud::ServerPool.cleanup(noop: @noop, ignoremaster: @ignoremaster, cloud: "Google", flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["ServerPool"]) > 0
-            MU::Cloud::FirewallRule.cleanup(noop: @noop, ignoremaster: @ignoremaster, cloud: "Google", flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["FirewallRule"]) > 0
-            MU::Cloud::LoadBalancer.cleanup(noop: @noop, ignoremaster: @ignoremaster, cloud: "Google", flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["LoadBalancer"]) > 0
-            MU::Cloud::Database.cleanup(skipsnapshots: @skipsnapshots, noop: @noop, ignoremaster: @ignoremaster, cloud: "Google", flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["Database"]) > 0
-            MU::Cloud::VPC.cleanup(noop: @noop, ignoremaster: @ignoremaster, cloud: "Google", flags: flags) if @mommacat.nil? or @mommacat.numKittens(types: ["VPC"]) > 0
-    
-          }
-        }
-
-        if !MU::Cloud::AWS.isGovCloud?
-          if $MU_CFG['aws'] and $MU_CFG['aws']['account_number']
-            MU::Cloud::DNSZone.cleanup(noop: @noop, cloud: "AWS", ignoremaster: @ignoremaster) if @mommacat.nil? or @mommacat.numKittens(types: ["DNSZone"]) > 0
-          end
-        end
-
         @projectthreads.each do |t|
           t.join
         end
@@ -310,7 +312,7 @@ module MU
 
       if !@noop and !@skipcloud
         if $MU_CFG['aws'] and $MU_CFG['aws']['account_number']
-          MU::Cloud::AWS.s3(MU.myRegion).delete_object(
+          MU::Cloud::AWS.s3(region: MU.myRegion).delete_object(
             bucket: MU.adminBucketName,
             key: "#{MU.deploy_id}-secret"
           )