cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/defaults/{amazon_images.yaml → AWS.yaml}

@@ -1,5 +1,5 @@
 ---
-rhel71: &rhel71
+rhel71: &4
   us-east-1: ami-001ac4e5f414fc5b7
   ap-northeast-1: ami-0094aa64967e9267d
   ap-northeast-2: ami-072783bf50ca01ef7
@@ -16,24 +16,24 @@ rhel71: &rhel71
   us-east-2: ami-00cc9f964320129d7
   us-west-1: ami-0110a4f8a96b24369
   us-west-2: ami-0006d7ea260e40777
-centos6: &centos6
-  us-east-1: ami-02f468ed191eda16e
-  ap-northeast-1: ami-0211c03a05f62bd88
-  ap-northeast-2: ami-0c4e210038a6c79a0
-  ap-south-1: ami-0679b5acba08097b4
-  ap-southeast-1: ami-0d06903ec09d67b84
-  ap-southeast-2: ami-07fcb5609d11b08c2
-  ca-central-1: ami-0090ac3d146715c76
-  eu-central-1: ami-0402323ff6cf59dc3
-  eu-north-1: ami-0c8c61870da56e00c
-  eu-west-1: ami-02b5e95243d740d41
-  eu-west-2: ami-0d3086afa04c0bac9
-  eu-west-3: ami-07810e092578b4d32
-  sa-east-1: ami-05f5c9f4fd81c0ae8
-  us-east-2: ami-039668d561db19b15
-  us-west-1: ami-09403e40a997f60bf
-  us-west-2: ami-0728d0eef40a5430b
-centos7: &centos7
+centos6: &3
+  us-east-1: ami-06a4e11df81ad5f36
+  ap-northeast-1: ami-0c777a90d1239a89f
+  ap-northeast-2: ami-00a536043e60d99e0
+  ap-south-1: ami-0dbe9e035d242ab19
+  ap-southeast-1: ami-07a0370b2aaca3d83
+  ap-southeast-2: ami-0df17df004fb2d85a
+  ca-central-1: ami-0852062afaf78369b
+  eu-central-1: ami-058aab5f423dea3fa
+  eu-north-1: ami-040fd7c04441f6c6b
+  eu-west-1: ami-0a0595aeafc8430b2
+  eu-west-2: ami-0f45a65b48147cbd2
+  eu-west-3: ami-0fa2dea3f867cd713
+  sa-east-1: ami-0e49de35e61a9bf6a
+  us-east-2: ami-06c0d5b6dbb7494c7
+  us-west-1: ami-08c8ff991d27c3a25
+  us-west-2: ami-0b62b81cae1d662c9
+centos7:
   us-east-1: ami-0d98f625837fb042e
   ap-northeast-1: ami-0f86cf4470a454938
   ap-northeast-2: ami-012efba9ff5419589
@@ -50,29 +50,29 @@ centos7: &centos7
   us-east-2: ami-00c5f50b43ecdb248
   us-west-1: ami-0cc4055a6417e4df6
   us-west-2: ami-0c250c14d31f0847f
-ubuntu16: &ubuntu16
+ubuntu16: &2
   us-east-1: ami-bcdc16c6
   us-west-1: ami-1b17257b
   us-west-2: ami-19e92861
-  eu-west-1:  ami-eed00d97
+  eu-west-1: ami-eed00d97
   eu-central-1: ami-e613ac89
   sa-east-1: ami-1ca7d970
   ap-northeast-1: ami-6959870f
   ap-northeast-2: ami-08d77266
-  ap-southeast-1: ami-d9dca7ba  
+  ap-southeast-1: ami-d9dca7ba
   ap-southeast-2: ami-02ad4060
-ubuntu14: &ubuntu14
+ubuntu14:
   us-east-1: ami-663a6e0c
   us-west-1: ami-13988772
   us-west-2: ami-b885eed8
-  eu-west-1:  ami-b265c7c1
+  eu-west-1: ami-b265c7c1
   eu-central-1: ami-ad8894c1
   sa-east-1: ami-4196112d
   ap-northeast-1: ami-575b6e39
   ap-northeast-2: ami-f0ac629e
-  ap-southeast-1: ami-2855964b  
+  ap-southeast-1: ami-2855964b
   ap-southeast-2: ami-d19fc4b2
-win2k12r2: &win2k12r2
+win2k12r2: &1
   us-east-1: ami-d4409aae
   us-east-2: ami-fbbe929e
   us-west-1: ami-ec91ac8c
@@ -85,14 +85,13 @@ win2k12r2: &win2k12r2
   ap-southeast-1: ami-b61657d5
   ap-southeast-2: ami-9a7b97f8
   ap-south-1: ami-99a8eaf6
-  ca-central-1: ami-608b3304 
-win2k16: &win2k16
+  ca-central-1: ami-608b3304
+win2k16:
   us-east-1: ami-d2cb25a8
   us-east-2: ami-2db59748
   us-west-1: ami-2db59748
   us-west-2: ami-3b47ba43
   eu-central-1: ami-37d46558
-  eu-west-1: ami-53408c2a
   eu-west-1: ami-06c5d662
   sa-east-1: ami-53fd803f
   ap-northeast-1: ami-ce8b42a8
@@ -101,11 +100,12 @@ win2k16: &win2k16
   ap-southeast-2: ami-792bcd1b
   ap-south-1: ami-448dcb2b
   ca-central-1: ami-a39920c7
-amazon: &amazon2016
+amazon:
   us-east-1: ami-b73b63a0
   us-east-2: ami-58277d3d
   us-west-1: ami-23e8a343
   us-west-2: ami-5ec1673e
+  us-gov-east-1: ami-fdaf4e8c
   eu-central-1: ami-f9619996
   eu-west-1: ami-9398d3e0
   sa-east-1: ami-97831ffb
@@ -113,12 +113,10 @@ amazon: &amazon2016
   ap-northeast-2: ami-983ce8f6
   ap-southeast-1: ami-b953f2da
   ap-southeast-2: ami-db704cb8
-win2k12: *win2k12r2
-win2k16: *win2k16
-windows: *win2k12r2
-ubuntu: *ubuntu16
-centos: *centos6
-rhel7: *rhel71
-rhel: *rhel71
-linux: *centos6
-amazon: *amazon2016
+win2k12: *1
+windows: *1
+ubuntu: *2
+centos: *3
+rhel7: *4
+rhel: *4
+linux: *3

data/modules/mu/defaults/Azure.yaml

@@ -0,0 +1,17 @@
+---
+centos6: &centos6 OpenLogic/CentOS/6
+#centos7: &centos7 westernoceansoftwaresprivatelimited/centos-7-6/centos-7-6-server
+centos7: &centos7 OpenLogic/CentOS/7
+rhel8: &rhel8 RedHat/RHEL/8
+rhel7: &rhel7 RedHat/RHEL/7
+rhel6: &rhel6 RedHat/RHEL/6
+debian10: &debian10 Debian/debian-10/10
+win2k12r2: &win2k12r2 MicrosoftWindowsServer/WindowsServer/2012-R2-Datacenter
+win2k16: &win2k16 MicrosoftWindowsServer/WindowsServer/2016-Datacenter
+win2k19: &win2k19 MicrosoftWindowsServer/WindowsServer/2019-Datacenter
+win2k12: *win2k12r2
+windows: *win2k16
+centos: *centos7
+rhel: *rhel7
+linux: *centos7
+debian: *debian10

data/modules/mu/defaults/Google.yaml

@@ -0,0 +1,24 @@
+---
+centos6: &centos6 centos-cloud/centos-6
+centos7: &centos7 centos-cloud/centos-7
+rhel71: &rhel71 rhel-cloud/rhel-7
+rhel6: &rhel6 rhel-cloud/rhel-6
+debian10: &debian10 debian-cloud/debian-10
+debian9: &debian9 debian-cloud/debian-9
+ubuntu14: &ubuntu14 ubuntu-os-cloud/ubuntu-1404-lts
+ubuntu16: &ubuntu16 ubuntu-os-cloud/ubuntu-1604-lts
+ubuntu18: &ubuntu18 ubuntu-os-cloud/ubuntu-1804-lts
+ubuntu19: &ubuntu19 ubuntu-os-cloud/ubuntu-1904
+win2k12r2: &win2k12r2 windows-cloud/windows-2012-r2
+win2k16: &win2k16 windows-cloud/windows-2016
+win2k19: &win2k19 windows-cloud/windows-2019
+google: &google coreos-cloud/coreos-stable
+coreos: *google
+win2k12: *win2k12r2
+windows: *win2k16
+ubuntu: *ubuntu18
+centos: *centos7
+rhel7: *rhel71
+rhel: *rhel71
+linux: *centos7
+debian: *debian10

data/modules/mu/defaults/README.md

@@ -1,2 +1,2 @@
 Do not edit these files in place! Instead, copy them to $install_prefix/etc
-(e.g. /opt/mu/etc/amazon_images.yaml) and make customizations there.
+(e.g. /opt/mu/etc/AWS.yaml) and make customizations there.

data/modules/mu/deploy.rb

@@ -50,6 +50,9 @@ module MU
     # We just pass this flag to MommaCat, telling it not to save any metadata.
     attr_reader :no_artifacts
 
+    # The deployment object we create for our stack
+    attr_reader :mommacat
+
     # Indicates whether we are updating an existing deployment, as opposed to
     # creating a new one.
     attr_reader :updating
@@ -66,6 +69,7 @@ module MU
     # @param deploy_id [String]: Reload and re-process an existing deploy
     def initialize(environment,
                    verbosity: MU::Logger::NORMAL,
+                   color: true,
                    webify_logs: false,
                    nocleanup: false,
                    cloudformation_path: nil,
@@ -76,12 +80,14 @@ module MU
                    deploy_id: nil,
                    deploy_obj: nil)
       MU.setVar("verbosity", verbosity)
+      MU.setVar("color", color)
       @webify_logs = webify_logs
       @verbosity = verbosity
+      @color = color
       @nocleanup = nocleanup
       @no_artifacts = no_artifacts
       @reraise_thread = reraise_thread
-      MU.setLogging(verbosity, webify_logs)
+      MU.setLogging(verbosity, webify_logs, STDOUT, color)
 
       MU::Cloud::CloudFormation.emitCloudFormation(set: force_cloudformation)
       @cloudformation_output = cloudformation_path
@@ -96,7 +102,7 @@ module MU
       @dependency_semaphore = Mutex.new
 
       @main_config = stack_conf
-      @original_config = Marshal.load(Marshal.dump(stack_conf))
+      @original_config = Marshal.load(Marshal.dump(MU.structToHash(stack_conf.dup)))
       @original_config.freeze
       @admins = stack_conf["admins"]
       @mommacat = deploy_obj
@@ -109,7 +115,7 @@ module MU
         @updating = false
         time=Time.new
         @appname = stack_conf["appname"]
-        @timestamp = time.strftime("%Y%m%d%H").to_s;
+        @timestamp = time.strftime("%Y%m%d%H").to_s
         @timestamp.freeze
         @timestart = time.to_s;
         @timestart.freeze
@@ -141,7 +147,7 @@ module MU
         if !@main_config[data[:cfg_plural]].nil? and @main_config[data[:cfg_plural]].size > 0
           @main_config[data[:cfg_plural]].each { |resource|
             if force_cloudformation
-              if resource['cloud'] = "AWS"
+              if resource['cloud'] == "AWS"
                 resource['cloud'] = "CloudFormation"
                 if resource.has_key?("vpc") and resource["vpc"].is_a?(Hash)
                   resource["vpc"]['cloud'] = "CloudFormation"
@@ -205,12 +211,26 @@ module MU
         if !die
           puts "Received SIGINT, hit ctrl-C again within five seconds to kill this deployment."
         else
-          raise "Terminated by user"
+          Thread.list.each do |t|
+            next if !t.status
+            if t.object_id != Thread.current.object_id and
+               t.thread_variable_get("name") != "main_thread" and 
+               t.thread_variable_get("owned_by_mu")
+              t.kill
+            end
+          end
+
+          if @main_thread
+            @main_thread.raise "Terminated by user"
+          else
+            raise "Terminated by user"
+          end
         end
         @last_sigterm = Time.now.to_i
       end
 
       begin
+        @main_thread = Thread.current
         if !@mommacat
           metadata = {
             "appname" => @appname,
@@ -238,16 +258,25 @@ module MU
         @admins.each { |admin|
           @mommacat.notify("admins", admin['name'], admin)
         }
+        if @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0
+          MU::MommaCat.start
+        end
 
         @deploy_semaphore = Mutex.new
         parent_thread_id = Thread.current.object_id
-        @main_thread = Thread.current
+
+        # Run cloud provider-specific deploy meta-artifact creation (ssh keys,
+        # resource groups, etc)
+        @mommacat.cloudsUsed.each { |cloud|
+          cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
+          cloudclass.initDeploy(@mommacat)
+        }
 
         # Kick off threads to create each of our new servers.
         @my_threads << Thread.new {
           MU.dupGlobals(parent_thread_id)
           Thread.current.thread_variable_set("name", "mu_create_container")
-          Thread.abort_on_exception = true
+#          Thread.abort_on_exception = false
           MU::Cloud.resource_types.each { |cloudclass, data|
             if !@main_config[data[:cfg_plural]].nil? and
                 @main_config[data[:cfg_plural]].size > 0 and
@@ -261,7 +290,7 @@ module MU
         @my_threads << Thread.new {
           MU.dupGlobals(parent_thread_id)
           Thread.current.thread_variable_set("name", "mu_groom_container")
-          Thread.abort_on_exception = true
+#          Thread.abort_on_exception = false
           MU::Cloud.resource_types.each { |cloudclass, data|
             if !@main_config[data[:cfg_plural]].nil? and
                 @main_config[data[:cfg_plural]].size > 0 and
@@ -283,9 +312,13 @@ module MU
           t.join
         end
 
+        @mommacat.save!
+
       rescue Exception => e
         @my_threads.each do |t|
-          if t.object_id != Thread.current.object_id and t.thread_variable_get("name") != "main_thread" and t.object_id != parent_thread_id
+          if t.object_id != Thread.current.object_id and
+             t.thread_variable_get("name") != "main_thread" and
+             t.object_id != parent_thread_id
             MU::MommaCat.unlockAll
             t.kill
           end
@@ -294,14 +327,17 @@ module MU
         # If it was a regular old exit, we assume something deeper in already
         # handled logging and cleanup for us, and just quietly go away.
         if e.class.to_s != "SystemExit"
-          MU.log e.inspect, MU::ERR, details: e.backtrace if @verbosity != MU::Logger::SILENT
+          MU.log e.class.name+": "+e.message, MU::ERR, details: e.backtrace if @verbosity != MU::Logger::SILENT
           if !@nocleanup
-            Thread.list.each do |t|
-              if t.object_id != Thread.current.object_id and t.thread_variable_get("name") != "main_thread" and t.object_id != parent_thread_id
-                t.kill
-              end
-            end
-            MU::Cleanup.run(MU.deploy_id, skipsnapshots: true, verbosity: @verbosity, mommacat: @mommacat)
+
+            # Wrap this in a thread to protect the Azure SDK from imploding
+            # because it mistakenly thinks there's a deadlock.
+            cleanup_thread = Thread.new {
+              MU.dupGlobals(parent_thread_id)
+              Thread.abort_on_exception = false
+              MU::Cleanup.run(MU.deploy_id, skipsnapshots: true, verbosity: @verbosity, mommacat: @mommacat)
+            }
+            cleanup_thread.join
             @nocleanup = true # so we don't run this again later
           end
         end
@@ -333,7 +369,7 @@ module MU
         if MU.myCloud == "AWS" 
           MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
         end
-        MU::MommaCat.getLitter(MU.deploy_id, use_cache: false)
+#        MU::MommaCat.getLitter(MU.deploy_id, use_cache: false)
         if @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0
 #          MU::MommaCat.syncMonitoringConfig # TODO only invoke if Server or ServerPool actually changed something when @updating
         end
@@ -401,19 +437,7 @@ module MU
         }
       end
 
-      if $MU_CFG['slack'] and $MU_CFG['slack']['webhook'] and
-         (!$MU_CFG['slack']['skip_environments'] or !$MU_CFG['slack']['skip_environments'].any?{ |s| s.casecmp(MU.environment)==0 })
-        require 'slack-notifier'
-        slack =  Slack::Notifier.new $MU_CFG['slack']['webhook']
-
-        slack.ping "Mu deployment #{MU.appname} *\"#{MU.handle}\"* (`#{MU.deploy_id}`) successfully completed on *#{$MU_CFG['hostname']}* (#{$MU_CFG['public_address']})", channel: $MU_CFG['slack']['channel']
-        if MU.summary.size > 0
-          MU.summary.each { |msg|
-            slack.ping msg, channel: $MU_CFG['slack']['channel']
-          }
-        end
-      end
-
+      @mommacat.sendAdminSlack("Deploy completed succesfully", msg: MU.summary.join("\n"))
     end
 
     private
@@ -431,6 +455,7 @@ module MU
       $str += JSON.pretty_generate(@mommacat.deployment)
 
       admin_addrs = @admins.map { |admin|
+        admin['name'] ||= ""
         admin['name']+" <"+admin['email']+">"
       }
 
@@ -516,7 +541,7 @@ MESSAGE_END
 
       services.each { |resource|
         if !resource["#MU_CLOUDCLASS"]
-          pp resource
+#          pp resource
         end
         res_type = resource["#MU_CLOUDCLASS"].cfg_name
         name = res_type+"_"+resource["name"]
@@ -531,9 +556,10 @@ MESSAGE_END
         if resource["dependencies"] != nil then
           resource["dependencies"].each { |dependency|
             parent_class = nil
-            MU::Cloud.resource_types.each_pair { |name, attrs|
-              if attrs[:cfg_name] == dependency['type']
-                parent_class = Object.const_get("MU").const_get("Cloud").const_get(name)
+            MU::Cloud.resource_types.each_pair { |res_class, attrs|
+              if attrs[:cfg_name] == dependency['type'] or
+                 attrs[:cfg_plural] == dependency['type']
+                parent_class = Object.const_get("MU").const_get("Cloud").const_get(res_class)
                 break
               end
             }
@@ -576,109 +602,126 @@ MESSAGE_END
       parent_thread = Thread.current
       services.uniq!
       services.each do |service|
-        @my_threads << Thread.new(service) { |myservice|
-          MU.dupGlobals(parent_thread_id)
-          threadname = service["#MU_CLOUDCLASS"].cfg_name+"_"+myservice["name"]+"_#{mode}"
-          Thread.current.thread_variable_set("name", threadname)
-          Thread.abort_on_exception = true
-          waitOnThreadDependencies(threadname)
-
-          if service["#MU_CLOUDCLASS"].instance_methods(false).include?(:groom) and !service['dependencies'].nil? and !service['dependencies'].size == 0
-            if mode == "create"
-              MU::MommaCat.lock(service["#MU_CLOUDCLASS"].cfg_name+"_"+myservice["name"]+"-dependencies")
-            elsif mode == "groom"
-              MU::MommaCat.unlock(service["#MU_CLOUDCLASS"].cfg_name+"_"+myservice["name"]+"-dependencies")
+        begin
+          @my_threads << Thread.new(service) { |myservice|
+            MU.dupGlobals(parent_thread_id)
+            threadname = myservice["#MU_CLOUDCLASS"].cfg_name+"_"+myservice["name"]+"_#{mode}"
+            Thread.current.thread_variable_set("name", threadname)
+            Thread.current.thread_variable_set("owned_by_mu", true)
+#            Thread.abort_on_exception = false
+            waitOnThreadDependencies(threadname)
+
+            if myservice["#MU_CLOUDCLASS"].instance_methods(false).include?(:groom) and !myservice['dependencies'].nil? and !myservice['dependencies'].size == 0
+              if mode == "create"
+                MU::MommaCat.lock(myservice["#MU_CLOUDCLASS"].cfg_name+"_"+myservice["name"]+"-dependencies")
+              elsif mode == "groom"
+                MU::MommaCat.unlock(myservice["#MU_CLOUDCLASS"].cfg_name+"_"+myservice["name"]+"-dependencies")
+              end
             end
-          end
 
-          MU.log "Launching thread #{threadname}", MU::DEBUG
-          begin
-            if service['#MUOBJECT'].nil?
-              service['#MUOBJECT'] = service["#MU_CLOUDCLASS"].new(mommacat: @mommacat, kitten_cfg: myservice, delayed_save: @updating)
-            end
-          rescue Exception => e
-            MU::MommaCat.unlockAll
-            @main_thread.raise MuError, "Error instantiating object from #{service["#MU_CLOUDCLASS"]} (#{e.inspect})", e.backtrace
-            raise e
-          end
-          begin
-            run_this_method = service['#MUOBJECT'].method(mode)
-          rescue Exception => e
-            MU::MommaCat.unlockAll
-            @main_thread.raise MuError, "Error invoking #{service["#MU_CLOUDCLASS"]}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
-            raise e
-          end
-          begin
-            MU.log "Checking whether to run #{service['#MUOBJECT']}.#{mode} (updating: #{@updating})", MU::DEBUG
-            if !@updating or mode != "create"
-              myservice = run_this_method.call
-            else
-
-              # XXX experimental create behavior for --liveupdate flag, only works on a couple of resource types. Inserting new resources into an old deploy is tricky.
-              opts = {}
-              if service["#MU_CLOUDCLASS"].cfg_name == "loadbalancer"
-                opts['classic'] = service['classic'] ? true : false
+            MU.log "Launching thread #{threadname}", MU::DEBUG
+            begin
+              if myservice['#MUOBJECT'].nil?
+                if @mommacat
+                  ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false)
+                  if @updating
+                    raise MuError, "Failed to findLitterMate(type: #{myservice["#MU_CLOUDCLASS"].cfg_plural}, name: #{myservice['name']}, credentials: #{myservice['credentials']}, created_only: true, return_all: false) in deploy #{@mommacat.deploy_id}" if !ext_obj
+                    ext_obj.config!(myservice)
+                  end
+                  myservice['#MUOBJECT'] = ext_obj
+                end
+                myservice['#MUOBJECT'] ||= myservice["#MU_CLOUDCLASS"].new(mommacat: @mommacat, kitten_cfg: myservice, delayed_save: @updating)
               end
+            rescue Exception => e
+              MU::MommaCat.unlockAll
+              @main_thread.raise MuError, "Error instantiating object from #{myservice["#MU_CLOUDCLASS"]} (#{e.inspect})", e.backtrace
+              raise e
+            end
+            begin
+              run_this_method = myservice['#MUOBJECT'].method(mode)
+            rescue Exception => e
+              MU::MommaCat.unlockAll
+              @main_thread.raise MuError, "Error invoking #{myservice["#MU_CLOUDCLASS"]}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
+              raise e
+            end
+            begin
+              MU.log "Checking whether to run #{myservice['#MUOBJECT']}.#{mode} (updating: #{@updating})", MU::DEBUG
+              if !@updating or mode != "create"
+                myservice = run_this_method.call
+              else
 
-              found = MU::MommaCat.findStray(service['cloud'],
-                                 service["#MU_CLOUDCLASS"].cfg_name,
-                                 name: service['name'],
-                                 region: service['region'],
-                                 deploy_id: @mommacat.deploy_id,
-#                                 allow_multi: service["#MU_CLOUDCLASS"].has_multiple,
-                                 tag_key: "MU-ID",
-                                 tag_value: @mommacat.deploy_id,
-                                 flags: opts,
-                                 dummy_ok: false
-                                )
-
-              found = found.delete_if { |x|
-                x.cloud_id.nil? and x.cloudobj.cloud_id.nil?
-              }
+                # XXX experimental create behavior for --liveupdate flag, only works on a couple of resource types. Inserting new resources into an old deploy is tricky.
+                opts = {}
+                if myservice["#MU_CLOUDCLASS"].cfg_name == "loadbalancer"
+                  opts['classic'] = myservice['classic'] ? true : false
+                end
 
-              if found.size == 0
-                if service["#MU_CLOUDCLASS"].cfg_name == "loadbalancer" or
-                   service["#MU_CLOUDCLASS"].cfg_name == "firewall_rule" or
-                   service["#MU_CLOUDCLASS"].cfg_name == "msg_queue" or
-                   service["#MU_CLOUDCLASS"].cfg_name == "server_pool" or
-                   service["#MU_CLOUDCLASS"].cfg_name == "container_cluster"
-# XXX only know LBs to be safe, atm
-                  MU.log "#{service["#MU_CLOUDCLASS"].name} #{service['name']} not found, creating", MU::NOTICE
+                found = MU::MommaCat.findStray(myservice['cloud'],
+                                   myservice["#MU_CLOUDCLASS"].cfg_name,
+                                   name: myservice['name'],
+                                   credentials: myservice['credentials'],
+                                   region: myservice['region'],
+                                   deploy_id: @mommacat.deploy_id,
+#                                 allow_multi: myservice["#MU_CLOUDCLASS"].has_multiple,
+                                   tag_key: "MU-ID",
+                                   tag_value: @mommacat.deploy_id,
+                                   flags: opts,
+                                   dummy_ok: false
+                                  )
+
+                found = found.delete_if { |x|
+                  x.cloud_id.nil? and x.cloudobj.cloud_id.nil?
+                }
+
+                if found.size == 0
+                  MU.log "#{myservice["#MU_CLOUDCLASS"].name} #{myservice['name']} not found, creating", MU::NOTICE
                   myservice = run_this_method.call
+                else
+                  real_descriptor = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_name, name: myservice['name'], created_only: true)
+
+                  if !real_descriptor
+                    MU.log "Invoking #{run_this_method.to_s} #{myservice['name']} #{myservice['name']}", MU::NOTICE
+                    myservice = run_this_method.call
+                  end
+#MU.log "#{myservice["#MU_CLOUDCLASS"].cfg_name} #{myservice['name']}", MU::NOTICE
                 end
-              else
-                real_descriptor = @mommacat.findLitterMate(type: service["#MU_CLOUDCLASS"].cfg_name, name: service['name'], created_only: true)
-
-                if !real_descriptor and (
-                    service["#MU_CLOUDCLASS"].cfg_name == "loadbalancer" or
-                    service["#MU_CLOUDCLASS"].cfg_name == "firewall_rule" or
-                    service["#MU_CLOUDCLASS"].cfg_name == "msg_queue" or
-                    service["#MU_CLOUDCLASS"].cfg_name == "server_pool" or
-                    service["#MU_CLOUDCLASS"].cfg_name == "container_cluster"
-                   )
-                  MU.log "Invoking #{run_this_method.to_s} #{service['name']} #{service['name']}", MU::NOTICE
-                  myservice = run_this_method.call
+
+              end
+            rescue ThreadError => e
+              MU.log "Waiting for threads to complete (#{e.message})", MU::NOTICE
+              @my_threads.each do |thr|
+                next if thr.object_id == Thread.current.object_id
+                thr.join(0.1)
+              end
+              @my_threads.reject! { |thr| !thr.alive? }
+              sleep 10+Random.rand(20)
+              retry
+            rescue Exception => e
+              MU.log e.inspect, MU::ERR, details: e.backtrace if @verbosity != MU::Logger::SILENT
+              MU::MommaCat.unlockAll
+              Thread.list.each do |t|
+                if t.object_id != Thread.current.object_id and t.thread_variable_get("name") != "main_thread" and t.object_id != parent_thread_id and t.thread_variable_get("owned_by_mu")
+                  t.kill
                 end
-#MU.log "#{service["#MU_CLOUDCLASS"].cfg_name} #{service['name']}", MU::NOTICE
               end
-
-            end
-          rescue Exception => e
-            MU.log e.inspect, MU::ERR, details: e.backtrace if @verbosity != MU::Logger::SILENT
-            MU::MommaCat.unlockAll
-            Thread.list.each do |t|
-              if t.object_id != Thread.current.object_id and t.thread_variable_get("name") != "main_thread" and t.object_id != parent_thread_id
-                t.kill
+              if !@nocleanup
+                MU::Cleanup.run(MU.deploy_id, verbosity: @verbosity, skipsnapshots: true)
+                @nocleanup = true # so we don't run this again later
               end
+              @main_thread.raise MuError, e.message, e.backtrace
             end
-            if !@nocleanup
-              MU::Cleanup.run(MU.deploy_id, verbosity: @verbosity, skipsnapshots: true)
-              @nocleanup = true # so we don't run this again later
-            end
-            @main_thread.raise MuError, e.message, e.backtrace
+            MU.purgeGlobals
+          }
+        rescue ThreadError => e
+          MU.log "Waiting for threads to complete (#{e.message})", MU::NOTICE
+          @my_threads.each do |thr|
+            next if thr.object_id == Thread.current.object_id
+            thr.join(0.1)
           end
-          MU.purgeGlobals
-        }
+          @my_threads.reject! { |thr| !thr.alive? }
+          sleep 10+Random.rand(20)
+          retry
+        end
       end
     end