cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/cleanup.rb

@@ -41,7 +41,7 @@ module MU
     # @param web [Boolean]: Generate web-friendly output.
     # @param ignoremaster [Boolean]: Ignore the tags indicating the originating MU master server when deleting.
     # @return [void]
-    def self.run(deploy_id, noop: false, skipsnapshots: false, onlycloud: false, verbosity: MU::Logger::NORMAL, web: false, ignoremaster: false, skipcloud: false, mommacat: nil)
+    def self.run(deploy_id, noop: false, skipsnapshots: false, onlycloud: false, verbosity: MU::Logger::NORMAL, web: false, ignoremaster: false, skipcloud: false, mommacat: nil, credsets: nil, regions: nil)
       MU.setLogging(verbosity, web)
       @noop = noop
       @skipsnapshots = skipsnapshots
@@ -61,7 +61,7 @@ module MU
       end
 
 
-      types_in_order = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection", "Habitat", "Folder"]
+      types_in_order = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"]
 
       # Load up our deployment metadata
       if !mommacat.nil?
@@ -75,190 +75,229 @@ module MU
             FileUtils.touch("#{deploy_dir}/.cleanup") if !@noop
           else
             MU.log "I don't see a deploy named #{deploy_id}.", MU::WARN
-            MU.log "Known deployments:\n#{Dir.entries(deploy_dir).reject { |item| item.match(/^\./) or !File.exists?(deploy_dir+"/"+item+"/public_key") }.join("\n")}", MU::WARN
+            MU.log "Known deployments:\n#{Dir.entries(deploy_dir).reject { |item| item.match(/^\./) or !File.exist?(deploy_dir+"/"+item+"/public_key") }.join("\n")}", MU::WARN
             MU.log "Searching for remnants of #{deploy_id}, though this may be an invalid MU-ID.", MU::WARN
           end
-          @mommacat = MU::MommaCat.new(deploy_id, mu_user: MU.mu_user)
+          @mommacat = MU::MommaCat.new(deploy_id, mu_user: MU.mu_user, delay_descriptor_load: true)
         rescue Exception => e
           MU.log "Can't load a deploy record for #{deploy_id} (#{e.inspect}), cleaning up resources by guesswork", MU::WARN, details: e.backtrace
           MU.setVar("deploy_id", deploy_id)
+
         end
       end
 
+      regionsused = @mommacat.regionsUsed if @mommacat
+      credsused = @mommacat.credsUsed if @mommacat
+
       if !@skipcloud
         creds = {}
-        MU::Cloud.supportedClouds.each { |cloud|
+        MU::Cloud.availableClouds.each { |cloud|
           if $MU_CFG[cloud.downcase] and $MU_CFG[cloud.downcase].size > 0
             cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
             creds[cloud] ||= {}
             cloudclass.listCredentials.each { |credset|
+              next if credsets and credsets.size > 0 and !credsets.include?(credset)
+              next if credsused and credsused.size > 0 and !credsused.include?(credset)
+              MU.log "Will scan #{cloud} with credentials #{credset}"
               creds[cloud][credset] = cloudclass.listRegions(credentials: credset)
             }
           end
         }
+
         parent_thread_id = Thread.current.object_id
         deleted_nodes = 0
-        @regionthreads = []
+        cloudthreads = []
         keyname = "deploy-#{MU.deploy_id}"
-# XXX blindly checking for all of these resources in all clouds is now prohibitively slow. We should only do this when we don't see deployment metadata to work from.
-        creds.each_pair { |provider, credsets|
-          credsets.each_pair { |credset, regions|
-            global_vs_region_semaphore = Mutex.new
-            global_done = []
-            regions.each { |r|
-              @regionthreads << Thread.new {
-                MU.dupGlobals(parent_thread_id)
-                MU.setVar("curRegion", r)
-                projects = []
-                if $MU_CFG[provider.downcase][credset]["project"]
-# XXX GCP credential schema needs an array for projects
-                  projects << $MU_CFG[provider.downcase][credset]["project"]
-                end
 
-                if projects == []
-                  projects << "" # dummy
-                  MU.log "Checking for #{provider}/#{credset} resources from #{MU.deploy_id} in #{r}", MU::NOTICE
+        creds.each_pair { |provider, credsets_outer|
+          cloudthreads << Thread.new(provider, credsets_outer) { |cloud, credsets_inner|
+            MU.dupGlobals(parent_thread_id)
+            Thread.abort_on_exception = false
+            cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+            habitatclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get("Habitat")
+            credsets_inner.each_pair { |credset, acct_regions|
+              next if credsused and !credsused.include?(credset)
+              global_vs_region_semaphore = Mutex.new
+              global_done = {}
+              habitats_done = {}
+              regionthreads = []
+              acct_regions.each { |r|
+                if regionsused
+                  if regionsused.size > 0
+                    next if !regionsused.include?(r)
+                  else
+                    next if r != cloudclass.myRegion(credset)
+                  end
                 end
-
-                # We do these in an order that unrolls dependent resources
-                # sensibly, and we hit :Collection twice because AWS
-                # CloudFormation sometimes fails internally.
-                projectthreads = []
-                projects.each { |project|
-                  projectthreads << Thread.new {
-                    MU.dupGlobals(parent_thread_id)
-                    MU.setVar("curRegion", r)
-                    if project != ""
-                      MU.log "Checking for #{provider}/#{credset} resources from #{MU.deploy_id} in #{r}, project #{project}", MU::NOTICE
-                    end
-
-                    MU.dupGlobals(parent_thread_id)
-                    flags = {
-                      "project" => project,
-                      "onlycloud" => @onlycloud,
-                      "skipsnapshots" => @skipsnapshots,
-                    }
-                    types_in_order.each { |t|
-                      begin
-                        skipme = false
-                        global_vs_region_semaphore.synchronize {
-                          if Object.const_get("MU").const_get("Cloud").const_get(provider).const_get(t).isGlobal?
-                            if !global_done.include?(t)
-                              global_done << t
-                              flags['global'] = true
-                            else
-                              skipme = true
-                            end
-                          end
-                        }
-                        next if skipme
-                      rescue MU::Cloud::MuCloudResourceNotImplemented => e
-                        next
-                      rescue MU::MuError, NoMethodError => e
-                        MU.log e.message, MU::WARN
-                        next
-                      rescue ::Aws::EC2::Errors::AuthFailure => e
-                        # AWS has been having transient auth problems with ap-east-1 lately
-                        MU.log e.message+" in "+r, MU::ERR
-                        next
+                if regions and !regions.empty?
+                  next if !regions.include?(r)
+                  MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}...", MU::NOTICE
+                end
+                regionthreads << Thread.new {
+                  MU.dupGlobals(parent_thread_id)
+                  Thread.abort_on_exception = false
+                  MU.setVar("curRegion", r)
+                  projects = []
+                  if $MU_CFG[cloud.downcase][credset]["project"]
+# XXX GCP credential schema needs an array for projects
+                    projects << $MU_CFG[cloud.downcase][credset]["project"]
+                  end
+                  begin
+                    projects.concat(cloudclass.listProjects(credset))
+                  rescue NoMethodError
+                  end
+
+                  if projects == []
+                    projects << "" # dummy
+                    MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}", MU::NOTICE
+                  end
+                  projects.uniq!
+
+                  # We do these in an order that unrolls dependent resources
+                  # sensibly, and we hit :Collection twice because AWS
+                  # CloudFormation sometimes fails internally.
+                  projectthreads = []
+                  projects.each { |project|
+                    next if !habitatclass.isLive?(project, credset)
+
+                    projectthreads << Thread.new {
+                      MU.dupGlobals(parent_thread_id)
+                      MU.setVar("curRegion", r)
+                      Thread.abort_on_exception = false
+                      if project != ""
+                        MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}, project #{project}", MU::NOTICE
                       end
 
-                      if @mommacat.nil? or @mommacat.numKittens(types: [t]) > 0
-                        if @mommacat
-                          found = @mommacat.findLitterMate(type: t, return_all: true, credentials: credset)
-                          flags['known'] ||= []
-                          if found.is_a?(Array)
-                            found.each { |k|
-                              flags['known'] << k.cloud_id
-                            }
-                          elsif found and found.is_a?(Hash)
-                            flags['known'] << found['cloud_id']
-                          elsif found
-                            flags['known'] << found.cloud_id                            
-                          end
+                      MU.dupGlobals(parent_thread_id)
+                      flags = {
+                        "project" => project,
+                        "onlycloud" => @onlycloud,
+                        "skipsnapshots" => @skipsnapshots,
+                      }
+                      types_in_order.each { |t|
+                        shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(t)
+                        begin
+                          skipme = false
+                          global_vs_region_semaphore.synchronize {
+                            MU::Cloud.loadCloudType(cloud, t)
+                            if Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(t).isGlobal?
+                              global_done[project] ||= []
+                              if !global_done[project].include?(t)
+                                global_done[project] << t
+                                flags['global'] = true
+                              else
+                                skipme = true
+                              end
+                            end
+                          }
+                          next if skipme
+                        rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented => e
+                          next
+                        rescue MU::MuError, NoMethodError => e
+                          MU.log "While checking mu/clouds/#{cloud.downcase}/#{cloudclass.cfg_name} for global-ness in cleanup: "+e.message, MU::WARN
+                          next
+                        rescue ::Aws::EC2::Errors::AuthFailure, ::Google::Apis::ClientError => e
+                          MU.log e.message+" in "+r, MU::ERR
+                          next
                         end
+
                         begin
-                          resclass = Object.const_get("MU").const_get("Cloud").const_get(t)
-                          resclass.cleanup(
-                            noop: @noop,
-                            ignoremaster: @ignoremaster,
-                            region: r,
-                            cloud: provider,
-                            flags: flags,
-                            credentials: credset
-                          )
-                        rescue Seahorse::Client::NetworkingError => e
-                          MU.log "Service not available in AWS region #{r}, skipping", MU::DEBUG, details: e.message
+                          self.call_cleanup(t, credset, cloud, flags, r)
+                        rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented => e
+                          next
                         end
-                      end
+                      }
+                    } # types_in_order.each { |t|
+                  } # projects.each { |project|
+                  projectthreads.each do |t|
+                    t.join
+                  end
+
+                  # XXX move to MU::AWS
+                  if cloud == "AWS"
+                    resp = MU::Cloud::AWS.ec2(region: r, credentials: credset).describe_key_pairs(
+                      filters: [{name: "key-name", values: [keyname]}]
+                    )
+                    resp.data.key_pairs.each { |keypair|
+                      MU.log "Deleting key pair #{keypair.key_name} from #{r}"
+                      MU::Cloud::AWS.ec2(region: r, credentials: credset).delete_key_pair(key_name: keypair.key_name) if !@noop
                     }
-                  }
-                }
-                projectthreads.each do |t|
-                  t.join
-                end
+                  end
+                } # regionthreads << Thread.new {
+              } # acct_regions.each { |r|
+              regionthreads.each do |t|
+                t.join
+              end
 
-                # XXX move to MU::AWS
-                if provider == "AWS"
-                  resp = MU::Cloud::AWS.ec2(region: r, credentials: credset).describe_key_pairs(
-                    filters: [{name: "key-name", values: [keyname]}]
-                  )
-                  resp.data.key_pairs.each { |keypair|
-                    MU.log "Deleting key pair #{keypair.key_name} from #{r}"
-                    MU::Cloud::AWS.ec2(region: r, credentials: credset).delete_key_pair(key_name: keypair.key_name) if !@noop
-                  }
-                end
+            } # credsets.each_pair { |credset, acct_regions|
+          } # cloudthreads << Thread.new(provider, credsets) { |cloud, credsets_outer|
+          cloudthreads.each do |t|
+            t.join
+          end
+        } # creds.each_pair { |provider, credsets|
+
+
+        # Knock habitats and folders, which would contain the above resources,
+        # once they're all done.
+        creds.each_pair { |provider, credsets_inner|
+          credsets_inner.keys.each { |credset|
+            next if credsused and !credsused.include?(credset)
+            ["Habitat", "Folder"].each { |t|
+              flags = {
+                "onlycloud" => @onlycloud,
+                "skipsnapshots" => @skipsnapshots
               }
+              self.call_cleanup(t, credset, provider, flags, nil)
             }
           }
         }
 
-        @regionthreads.each do |t|
-          t.join
-        end
-        @projectthreads = []
-
-
-        @projectthreads.each do |t|
-          t.join
-        end
-
         MU::Cloud::Google.removeDeploySecretsAndRoles(MU.deploy_id) 
 # XXX port AWS equivalent behavior and add a MU::Cloud wrapper
+
+        creds.each_pair { |provider, credsets_inner|
+          cloudclass = Object.const_get("MU").const_get("Cloud").const_get(provider)
+          credsets_inner.keys.each { |c|
+            cloudclass.cleanDeploy(MU.deploy_id, credentials: c, noop: @noop)
+          }
+        }
       end
 
       # Scrub any residual Chef records with matching tags
-      if !@onlycloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0)
-        MU::Groomer::Chef.loadChefLib
-        if File.exists?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
-          Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
-        end
-        deadnodes = []
-        Chef::Config[:environment] = MU.environment
-        q = Chef::Search::Query.new
+      if !@onlycloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0) and !(Gem.paths and Gem.paths.home and !Dir.exist?("/opt/mu/lib"))
         begin
-          q.search("node", "tags_MU-ID:#{MU.deploy_id}").each { |item|
-            next if item.is_a?(Integer)
-            item.each { |node|
-              deadnodes << node.name
+          MU::Groomer::Chef.loadChefLib
+          if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
+            Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
+          end
+          deadnodes = []
+          Chef::Config[:environment] = MU.environment
+          q = Chef::Search::Query.new
+          begin
+            q.search("node", "tags_MU-ID:#{MU.deploy_id}").each { |item|
+              next if item.is_a?(Integer)
+              item.each { |node|
+                deadnodes << node.name
+              }
             }
-          }
-        rescue Net::HTTPServerException
-        end
+          rescue Net::HTTPServerException
+          end
 
-        begin
-          q.search("node", "name:#{MU.deploy_id}-*").each { |item|
-            next if item.is_a?(Integer)
-            item.each { |node|
-              deadnodes << node.name
+          begin
+            q.search("node", "name:#{MU.deploy_id}-*").each { |item|
+              next if item.is_a?(Integer)
+              item.each { |node|
+                deadnodes << node.name
+              }
             }
+          rescue Net::HTTPServerException
+          end
+          MU.log "Missed some Chef resources in node cleanup, purging now", MU::NOTICE if deadnodes.size > 0
+          deadnodes.uniq.each { |node|
+            MU::Groomer::Chef.cleanup(node, [], noop)
           }
-        rescue Net::HTTPServerException
+        rescue LoadError
         end
-        MU.log "Missed some Chef resources in node cleanup, purging now", MU::NOTICE if deadnodes.size > 0
-        deadnodes.uniq.each { |node|
-          MU::Groomer::Chef.cleanup(node, [], noop)
-        }
       end
 
       if !@onlycloud and !@noop and @mommacat
@@ -270,18 +309,18 @@ module MU
       sshconf = "#{sshdir}/config"
       ssharchive = "#{sshdir}/archive"
 
-      Dir.mkdir(sshdir, 0700) if !Dir.exists?(sshdir) and !@noop
-      Dir.mkdir(ssharchive, 0700) if !Dir.exists?(ssharchive) and !@noop
+      Dir.mkdir(sshdir, 0700) if !Dir.exist?(sshdir) and !@noop
+      Dir.mkdir(ssharchive, 0700) if !Dir.exist?(ssharchive) and !@noop
 
       keyname = "deploy-#{MU.deploy_id}"
-      if File.exists?("#{sshdir}/#{keyname}")
+      if File.exist?("#{sshdir}/#{keyname}")
         MU.log "Moving #{sshdir}/#{keyname} to #{ssharchive}/#{keyname}"
         if !@noop
           File.rename("#{sshdir}/#{keyname}", "#{ssharchive}/#{keyname}")
         end
       end
 
-      if File.exists?(sshconf) and File.open(sshconf).read.match(/\/deploy\-#{MU.deploy_id}$/)
+      if File.exist?(sshconf) and File.open(sshconf).read.match(/\/deploy\-#{MU.deploy_id}$/)
         MU.log "Expunging #{MU.deploy_id} from #{sshconf}"
         if !@noop
           FileUtils.copy(sshconf, "#{ssharchive}/config-#{MU.deploy_id}")
@@ -309,21 +348,25 @@ module MU
       # XXX refactor with above? They're similar, ish.
       hostsfile = "/etc/hosts"
       if File.open(hostsfile).read.match(/ #{MU.deploy_id}\-/)
-        MU.log "Expunging traces of #{MU.deploy_id} from #{hostsfile}"
-        if !@noop
-          FileUtils.copy(hostsfile, "#{hostsfile}.cleanup-#{deploy_id}")
-          File.open(hostsfile, File::CREAT|File::RDWR, 0644) { |f|
-            f.flock(File::LOCK_EX)
-            newlines = Array.new
-            f.readlines.each { |line|
-              newlines << line if !line.match(/ #{MU.deploy_id}\-/)
+        if Process.uid == 0
+          MU.log "Expunging traces of #{MU.deploy_id} from #{hostsfile}"
+          if !@noop
+            FileUtils.copy(hostsfile, "#{hostsfile}.cleanup-#{deploy_id}")
+            File.open(hostsfile, File::CREAT|File::RDWR, 0644) { |f|
+              f.flock(File::LOCK_EX)
+              newlines = Array.new
+              f.readlines.each { |line|
+                newlines << line if !line.match(/ #{MU.deploy_id}\-/)
+              }
+              f.rewind
+              f.truncate(0)
+              f.puts(newlines)
+              f.flush
+              f.flock(File::LOCK_UN)
             }
-            f.rewind
-            f.truncate(0)
-            f.puts(newlines)
-            f.flush
-            f.flock(File::LOCK_UN)
-          }
+          end
+        else
+          MU.log "Residual /etc/hosts entries for #{MU.deploy_id} must be removed by root user", MU::WARN
         end
       end
 
@@ -354,5 +397,39 @@ module MU
       end
 
     end
+
+    private
+
+    def self.call_cleanup(type, credset, provider, flags, region)
+      if @mommacat.nil? or @mommacat.numKittens(types: [type]) > 0
+        if @mommacat
+          found = @mommacat.findLitterMate(type: type, return_all: true, credentials: credset)
+          flags['known'] ||= []
+          if found.is_a?(Array)
+            found.each { |k|
+              flags['known'] << k.cloud_id
+            }
+          elsif found and found.is_a?(Hash)
+            flags['known'] << found['cloud_id']
+          elsif found
+            flags['known'] << found.cloud_id                            
+          end
+        end
+#        begin
+          resclass = Object.const_get("MU").const_get("Cloud").const_get(type)
+
+          resclass.cleanup(
+            noop: @noop,
+            ignoremaster: @ignoremaster,
+            region: region,
+            cloud: provider,
+            flags: flags,
+            credentials: credset
+          )
+#                        rescue ::Seahorse::Client::NetworkingError => e
+#                          MU.log "Service not available in AWS region #{r}, skipping", MU::DEBUG, details: e.message
+#                        end
+      end
+    end
   end #class
 end #module