cloud-mu 3.2.0 → 3.5.0

data/modules/mu/groomers/chef.rb

@@ -70,8 +70,8 @@ module MU
             # XXX kludge to get at knife-windows when it's installed from 
             # a git repo and bundler sticks it somewhere in a corner
             $LOAD_PATH.each { |path|
-              if path.match(/\/gems\/aws\-sdk\-core\-\d+\.\d+\.\d+\/lib$/)
-                addpath = path.sub(/\/gems\/aws\-sdk\-core\-\d+\.\d+\.\d+\/lib$/, "")+"/bundler/gems"
+              if path.match(/\/gems\/chef\-\d+\.\d+\.\d+\/lib$/)
+                addpath = path.sub(/\/gems\/chef\-\d+\.\d+\.\d+\/lib$/, "")+"/bundler/gems"
                 Dir.glob(addpath+"/knife-windows-*").each { |version|
                   $LOAD_PATH << version+"/lib"
                 }

data/modules/mu/master.rb

@@ -195,9 +195,12 @@ module MU
       temp_dev = "/dev/#{ramdisk}"
 
       if !File.open("/etc/mtab").read.match(/ #{path} /)
-        realdevice = device.dup
-        if MU::Cloud::Google.hosted?
-          realdevice = "/dev/disk/by-id/google-"+device.gsub(/.*?\/([^\/]+)$/, '\1')
+        realdevice = if MU::Cloud::Google.hosted?
+          "/dev/disk/by-id/google-"+device.gsub(/.*?\/([^\/]+)$/, '\1')
+        elsif MU::Cloud::AWS.hosted?
+          MU::Cloud::AWS.realDevicePath(device.dup)
+        else
+          device.dup
         end
         alias_device = cryptfile ? "/dev/mapper/"+path.gsub(/[^0-9a-z_\-]/i, "_") : realdevice
 
@@ -216,6 +219,9 @@ module MU
               tag_name: "Name",
               tag_value: "#{$MU_CFG['hostname']} #{path}"
             )
+            # the device might be on some arbitrary NVMe slot
+            realdevice = MU::Cloud::AWS.realDevicePath(realdevice)
+            alias_device = cryptfile ? "/dev/mapper/"+path.gsub(/[^0-9a-z_\-]/i, "_") : realdevice
           elsif MU::Cloud::Google.hosted?
             dummy_svr = MU::Cloud::Google::Server.new(
               mu_name: "MU-MASTER",
@@ -880,5 +886,66 @@ module MU
       end
     end
 
+    # Recursively zip a directory
+    # @param srcdir [String]
+    # @param outfile [String]
+    def self.zipDir(srcdir, outfile)
+      require 'zip'
+      ::Zip::File.open(outfile, ::Zip::File::CREATE) { |zipfile|
+        addpath = Proc.new { |zip_path, parent_path|
+          Dir.entries(parent_path).reject{ |d| [".", ".."].include?(d) }.each { |entry|
+            src = File.join(parent_path, entry)
+            dst = File.join(zip_path, entry).sub(/^\//, '')
+            if File.directory?(src)
+              addpath.call(dst, src)
+            else
+              zipfile.add(dst, src)
+            end
+          }
+        }
+        addpath.call("", srcdir)
+      }
+    end
+
+    # Just list our block devices
+    # @return [Array<String>]
+    def self.listBlockDevices
+      if File.executable?("/bin/lsblk")
+        %x{/bin/lsblk -i -p -r -n | egrep ' disk( |$)'}.each_line.map { |l|
+          l.chomp.sub(/ .*/, '')
+        }
+      else
+        # XXX something dumber
+        nil
+      end
+    end
+
+
+    # Retrieve the UUID of a block device, if available
+    # @param dev [String]
+    def self.diskUUID(dev)
+      realdev = if MU::Cloud::Google.hosted?
+        "/dev/disk/by-id/google-"+dev.gsub(/.*?\/([^\/]+)$/, '\1')
+      elsif MU::Cloud::AWS.hosted?
+        MU::Cloud::AWS.realDevicePath(dev)
+      else
+        dev
+      end
+      %x{/sbin/blkid #{realdev} -o export | grep ^UUID=}.chomp
+    end
+
+    # Determine whether we're running in an NVMe-enabled environment
+    def self.nvme?
+      if File.executable?("/bin/lsblk")
+        %x{/bin/lsblk -i -p -r -n}.each_line { |l|
+          return true if l =~ /^\/dev\/nvme\d/
+        }
+      else
+        return true if File.exists?("/dev/nvme0n1")
+      end
+      false
+    end
+
+
   end
 end

data/modules/mu/mommacat.rb

@@ -173,6 +173,7 @@ module MU
       @public_key = nil
       @secrets = Hash.new
       @secrets['instance_secret'] = Hash.new
+      @secrets['windows_admin_password'] = Hash.new
       @ssh_key_name = ssh_key_name
       @ssh_private_key = ssh_private_key
       @ssh_public_key = ssh_public_key
@@ -512,6 +513,8 @@ module MU
       @ssh_private_key = File.read("#{ssh_dir}/#{@ssh_key_name}")
       @ssh_private_key.chomp!
 
+# XXX the following mess belongs in cloud layers, probably in their initDeploy
+# methods
       if numKittens(clouds: ["AWS"], types: ["Server", "ServerPool", "ContainerCluster"]) > 0
         creds_used = []
         ["servers", "server_pools", "container_clusters"].each { |type|
@@ -547,15 +550,26 @@ module MU
     # @param remove [Boolean]: Remove this resource from the deploy structure, instead of adding it.
     # @return [void]
     def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, delayed_save: false)
+      no_write = (@no_artifacts or !caller.grep(/\/mommacat\.rb:\d+:in `notify'/).empty?)
 
       begin
-        MU::MommaCat.lock("deployment-notification", deploy_id: @deploy_id) if !@no_artifacts
+        if !no_write
+          if !MU::MommaCat.lock("deployment-notification", deploy_id: @deploy_id, retries: 300)
+            raise MuError, "Failed to get deployment-notifcation lock for #{@deploy_id}"
+          end
+        end
 
         if !@need_deploy_flush or @deployment.nil? or @deployment.empty?
           loadDeploy(true) # make sure we're saving the latest and greatest
         end
 
-        _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type, false)
+        @timestamp ||= @deployment['timestamp']
+        @seed ||= @deployment['seed']
+        @appname ||= @deployment['appname']
+        @handle ||= @deployment['handle']
+        
+        _shortclass, _cfg_name, mu_type, _classname, attrs = MU::Cloud.getResourceNames(type, false)
+        type = mu_type if mu_type
         has_multiples = attrs[:has_multiples] ? true : false
 
         mu_name ||= if !data.nil? and !data["mu_name"].nil?
@@ -569,6 +583,7 @@ module MU
         end
 
         @need_deploy_flush = true
+        @last_modified = Time.now
 
         if !remove
           if data.nil?
@@ -588,7 +603,9 @@ module MU
             @deployment[type][key] = data
             MU.log "Adding to @deployment[#{type}][#{key}]", MU::DEBUG, details: data
           end
-          save!(key) if !delayed_save and !@no_artifacts
+          if !delayed_save and !no_write
+            save!(key)
+          end
         else
           have_deploy = true
           if @deployment[type].nil? or @deployment[type][key].nil?
@@ -613,10 +630,10 @@ module MU
               end
             }
           end
-          save! if !delayed_save and !@no_artifacts
+          save! if !delayed_save and !no_write
         end
       ensure
-        MU::MommaCat.unlock("deployment-notification", deploy_id: @deploy_id) if !@no_artifacts
+        MU::MommaCat.unlock("deployment-notification", deploy_id: @deploy_id) if !no_write
       end
     end
 
@@ -811,6 +828,7 @@ MAIL_HEAD_END
 
       threads = []
       update_servers.each { |sibling|
+        next if sibling.config.has_key?("groom") and !sibling.config["groom"]
         threads << Thread.new {
           Thread.abort_on_exception = true
           Thread.current.thread_variable_set("name", "sync-"+sibling.mu_name.downcase)
@@ -869,13 +887,13 @@ MAIL_HEAD_END
         if resource and resource.config and resource.config['cloud']
           cloudclass = MU::Cloud.cloudClass(resource.config['cloud'])
 
-          cloudclass.writeDeploySecret(@deploy_id, cert.to_pem, cert_cn+".crt", credentials: resource.config['credentials'])
-          cloudclass.writeDeploySecret(@deploy_id, key.to_pem, cert_cn+".key", credentials: resource.config['credentials'])
+          cloudclass.writeDeploySecret(self, cert.to_pem, cert_cn+".crt", credentials: resource.config['credentials'])
+          cloudclass.writeDeploySecret(self, key.to_pem, cert_cn+".key", credentials: resource.config['credentials'])
           if pfx_cert
-            cloudclass.writeDeploySecret(@deploy_id, pfx_cert.to_der, cert_cn+".pfx", credentials: resource.config['credentials'])
+            cloudclass.writeDeploySecret(self, pfx_cert.to_der, cert_cn+".pfx", credentials: resource.config['credentials'])
           end
           if winrm_cert
-            cloudclass.writeDeploySecret(@deploy_id, winrm_cert.to_pem, cert_cn+"-winrm.crt", credentials: resource.config['credentials'])
+            cloudclass.writeDeploySecret(self, winrm_cert.to_pem, cert_cn+"-winrm.crt", credentials: resource.config['credentials'])
           end
         end
 
@@ -895,6 +913,7 @@ MAIL_HEAD_END
     ###########################################################################
     ###########################################################################
     def setThreadContextToMe
+
       ["appname", "environment", "timestamp", "seed", "handle"].each { |var|
         @deployment[var] ||= instance_variable_get("@#{var}".to_sym)
         if @deployment[var]

data/modules/mu/mommacat/daemon.rb

@@ -288,8 +288,8 @@ module MU
 
     # Path to the PID file used by the Momma Cat daemon
     # @return [String]
-    def self.daemonPidFile
-      base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
+    def self.daemonPidFile(root = false)
+      base = ((Process.uid == 0 or root) and !MU.localOnly) ? "/var" : MU.dataDir
       "#{base}/run/mommacat.pid"
     end
 
@@ -306,8 +306,14 @@ module MU
           Dir.mkdir(dir)
         end
       }
-      return 0 if status
+      if (Process.uid != 0 and
+           (!$MU_CFG['overridden_keys'] or !$MU_CFG['overridden_keys'].include?("mommacat_port")) and
+            status(true)
+         ) or status
+        return 0
+      end
     
+      File.unlink(daemonPidFile) if File.exists?(daemonPidFile)
       MU.log "Starting Momma Cat on port #{MU.mommaCatPort}, logging to #{daemonLogFile}, PID file #{daemonPidFile}"
       origdir = Dir.getwd
       Dir.chdir(MU.myRoot+"/modules")
@@ -346,12 +352,12 @@ module MU
 
     # Return true if the Momma Cat daemon appears to be running
     # @return [Boolean]
-    def self.status
+    def self.status(root = false)
       if MU.inGem? and MU.muCfg['disable_mommacat']
         return true
       end
-      if File.exist?(daemonPidFile)
-        pid = File.read(daemonPidFile).chomp.to_i
+      if File.exist?(daemonPidFile(root))
+        pid = File.read(daemonPidFile(root)).chomp.to_i
         begin
           Process.getpgid(pid)
           MU.log "Momma Cat running with pid #{pid.to_s}", (@@notified_on_pid[pid] ? MU::DEBUG : MU::INFO) # shush
@@ -360,7 +366,7 @@ module MU
         rescue Errno::ESRCH
         end
       end
-      MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile
+      MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile(root)
       false
     end
     

data/modules/mu/mommacat/naming.rb

@@ -164,7 +164,7 @@ module MU
     # @param scrub_mu_isms [Boolean]: Don't bother with generating names specific to this deployment. Used to generate generic CloudFormation templates, amongst other purposes.
     # @param disallowed_chars [Regexp]: A pattern of characters that are illegal for this resource name, such as +/[^a-zA-Z0-9-]/+
     # @return [String]: A full name string for this resource
-    def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], disallowed_chars: nil)
+    def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], disallowed_chars: nil, never_gen_unique: false)
       if name.nil?
         raise MuError, "Got no argument to MU::MommaCat.getResourceName"
       end
@@ -219,7 +219,7 @@ module MU
           else
             # If we have to strip anything, assume we've lost uniqueness and
             # will have to compensate with #genUniquenessString.
-            need_unique_string = true
+            need_unique_string = true if !never_gen_unique
             reserved = 4
             basename.sub!(/-[^-]+-#{@seed.upcase}-#{Regexp.escape(name.upcase)}$/, "")
             basename = basename + "-" + @seed.upcase + "-" + name.upcase

data/modules/mu/mommacat/search.rb

@@ -107,8 +107,17 @@ module MU
       matches = []
 
       credlist.each { |creds|
-#            next if region and region.is_a?(Array) and !region.empty? and !region.include?(r)
-        cloud_descs = search_cloud_provider(type, cloud, habitats, region, cloud_id: cloud_id, tag_key: tag_key, tag_value: tag_value, credentials: creds, flags: flags)
+        cur_habitats = []
+
+        if habitats and !habitats.empty? and habitats != [nil]
+          valid_habitats = cloudclass.listHabitats(creds)
+          cur_habitats = (habitats & valid_habitats)
+          next if cur_habitats.empty?
+        else
+          cur_habitats = cloudclass.listHabitats(creds)
+        end
+
+        cloud_descs = search_cloud_provider(type, cloud, cur_habitats, region, cloud_id: cloud_id, tag_key: tag_key, tag_value: tag_value, credentials: creds, flags: flags)
 
         cloud_descs.each_pair.each { |p, regions|
           regions.each_pair.each { |r, results|
@@ -138,7 +147,7 @@ module MU
     # @param created_only [Boolean]: Only return the littermate if its cloud_id method returns a value
     # @param return_all [Boolean]: Return a Hash of matching objects indexed by their mu_name, instead of a single match. Only valid for resource types where has_multiples is true.
     # @return [MU::Cloud]
-    def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, debug: false, **flags)
+    def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, ignore_missing: false, debug: false, **flags)
       _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type)
 
       # If we specified a habitat, which we may also have done by its shorthand
@@ -173,8 +182,10 @@ module MU
             end
           end
           if @object_load_fails or !@kittens[type]
-            MU.log "#{@deploy_id}'s original config has #{@original_config[type].size == 1 ? "a" : @original_config[type].size.to_s} #{type}, but loadObjects could not populate anything from deployment metadata", MU::ERR if !@object_load_fails
-            @object_load_fails = true
+            if !ignore_missing
+              MU.log "#{@deploy_id}'s original config has #{@original_config[type].size == 1 ? "a" : @original_config[type].size.to_s} #{type}, but loadObjects could not populate anything from deployment metadata", MU::ERR if !@object_load_fails
+              @object_load_fails = true
+            end
             return nil
           end
         end

data/modules/mu/mommacat/storage.rb

@@ -123,11 +123,12 @@ module MU
       MU::Cloud.resource_types.each_pair { |res_type, attrs|
         next if !@deployment.has_key?(attrs[:cfg_plural])
         deletia = []
+# existing_deploys
         @deployment[attrs[:cfg_plural]].each_pair { |res_name, data|
           orig_cfg = findResourceConfig(attrs[:cfg_plural], res_name, (scrub_with || @original_config))
 
-          if orig_cfg.nil?
-            MU.log "#{res_type} #{res_name} no longer configured, will remove deployment metadata", MU::NOTICE
+          if orig_cfg.nil? and (!data['mu_name'] or data['mu_name'] =~ /^#{Regexp.quote(@deploy_id)}/)
+            MU.log "#{res_type} #{res_name} no longer configured, will remove deployment metadata", MU::NOTICE, details: data
             deletia << res_name
           end
         }
@@ -176,7 +177,7 @@ module MU
     # @param id [String]: The lock identifier to release.
     # @param nonblock [Boolean]: Whether to block while waiting for the lock. In non-blocking mode, we simply return false if the lock is not available.
     # return [false, nil]
-    def self.lock(id, nonblock = false, global = false, deploy_id: MU.deploy_id)
+    def self.lock(id, nonblock = false, global = false, retries: 0, deploy_id: MU.deploy_id)
       raise MuError, "Can't pass a nil id to MU::MommaCat.lock" if id.nil?
 
       if !global
@@ -189,6 +190,7 @@ module MU
         MU.log "Creating #{lockdir}", MU::DEBUG
         Dir.mkdir(lockdir, 0700)
       end
+      nonblock = true if retries > 0
 
       @lock_semaphore.synchronize {
         if @locks[Thread.current.object_id].nil?
@@ -197,11 +199,39 @@ module MU
 
         @locks[Thread.current.object_id][id] = File.open("#{lockdir}/#{id}.lock", File::CREAT|File::RDWR, 0600)
       }
-      MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG
+
+      MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG, details: caller
+      show_relevant = Proc.new {
+        @lock_semaphore.synchronize {
+          @locks.each_pair { |thread_id, lock|
+            lock.each_pair { |lockid, lockpath|
+              if lockid == id
+                thread = Thread.list.select { |t| t.object_id == thread_id }.first
+                if thread.object_id != Thread.current.object_id
+                  MU.log "#{thread_id} sitting on #{id} (#{thread.thread_variables.map { |v| "#{v.to_s}: #{thread.thread_variable_get(v).to_s}" }.join(", ")})", MU::WARN, thread.backtrace
+                end
+              end
+            }
+          }
+        }
+      }
+
       begin
         if nonblock
           if !@locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
-            return false
+            if retries > 0
+              success = false
+              MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries, wait: 1, logmsg_interval: 0) { |cur_retries, _wait|
+                success = @locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
+                if !success and cur_retries > 0 and (cur_retries % 45) == 0
+                  show_relevant.call
+                end
+              }
+              show_relevant.call if !success
+              return success
+            else
+              return false
+            end
           end
         else
           @locks[Thread.current.object_id][id].flock(File::LOCK_EX)
@@ -390,6 +420,7 @@ module MU
           deploy.flock(File::LOCK_UN)
           deploy.close
           @need_deploy_flush = false
+          @last_modified = nil
           MU::MommaCat.updateLitter(@deploy_id, self)
         end
 
@@ -512,6 +543,22 @@ module MU
       return Dir.exist?(deploy_path)
     end
 
+    # Write our shared deploy secret out to wherever the cloud provider layers
+    # like to stash it.
+    def writeDeploySecret
+      return if !@deploy_secret
+      credsets = credsUsed
+      return if !credsets
+      if !@original_config['scrub_mu_isms'] and !@no_artifacts
+        cloudsUsed.each { |cloud|
+          credsets.each { |credentials|
+            next if MU::Cloud.cloudClass(cloud).credConfig(credentials).nil? # XXX this is a dumb way to check this, should be able to get credsUsed by cloud
+            MU::Cloud.cloudClass(cloud).writeDeploySecret(self, @deploy_secret, credentials: credentials)
+          }
+        }
+      end
+    end
+
     private
         
     def writeFile(filename, contents)
@@ -522,30 +569,8 @@ module MU
 
     # Helper for +initialize+
     def setDeploySecret
-      credsets = {}
-      MU::Cloud.resource_types.values.each { |attrs|
-        if !@original_config[attrs[:cfg_plural]].nil? and @original_config[attrs[:cfg_plural]].size > 0
-          @original_config[attrs[:cfg_plural]].each { |resource|
-
-            credsets[resource['cloud']] ||= []
-            credsets[resource['cloud']] << resource['credentials']
-            @clouds[resource['cloud']] = 0 if !@clouds.has_key?(resource['cloud'])
-            @clouds[resource['cloud']] = @clouds[resource['cloud']] + 1
-
-          }
-        end
-      }
-
       MU.log "Creating deploy secret for #{MU.deploy_id}"
       @deploy_secret = Password.random(256)
-      if !@original_config['scrub_mu_isms'] and !@no_artifacts
-        credsets.each_pair { |cloud, creds|
-          creds.uniq!
-          creds.each { |credentials|
-            MU::Cloud.cloudClass(cloud).writeDeploySecret(@deploy_id, @deploy_secret, credentials: credentials)
-          }
-        }
-      end
     end
 
     def loadObjects(delay_descriptor_load)
@@ -618,6 +643,14 @@ module MU
     def loadDeployFromCache(set_context_to_me = true)
       return false if !File.size?(deploy_dir+"/deployment.json")
 
+      lastmod = File.mtime("#{deploy_dir}/deployment.json")
+      if @last_modified and lastmod < @last_modified
+        MU.log "#{deploy_dir}/deployment.json last written at #{lastmod}, live meta at #{@last_modified}, not loading", MU::WARN if @last_modified
+        # this is a weird place for this
+        setThreadContextToMe if set_context_to_me
+        return true
+      end
+
       deploy = File.open("#{deploy_dir}/deployment.json", File::RDONLY)
       MU.log "Getting lock to read #{deploy_dir}/deployment.json", MU::DEBUG
       # deploy.flock(File::LOCK_EX)
@@ -629,6 +662,7 @@ module MU
 
       begin
         @deployment = JSON.parse(File.read("#{deploy_dir}/deployment.json"))
+# XXX is it worthwhile to merge fuckery?
       rescue JSON::ParserError => e
         MU.log "JSON parse failed on #{deploy_dir}/deployment.json", MU::ERR, details: e.message
       end
@@ -638,20 +672,21 @@ module MU
 
       setThreadContextToMe if set_context_to_me
 
-      @timestamp = @deployment['timestamp']
-      @seed = @deployment['seed']
-      @appname = @deployment['appname']
-      @handle = @deployment['handle']
-
       true
     end
 
+
     ###########################################################################
     ###########################################################################
     def loadDeploy(deployment_json_only = false, set_context_to_me: true)
       MU::MommaCat.deploy_struct_semaphore.synchronize {
         success = loadDeployFromCache(set_context_to_me)
 
+        @timestamp ||= @deployment['timestamp']
+        @seed ||= @deployment['seed']
+        @appname ||= @deployment['appname']
+        @handle ||= @deployment['handle']
+
         return if deployment_json_only and success
 
         if File.exist?(deploy_dir+"/private_key")