cloud-mu 3.1.2 → 3.2.0

data/modules/mu/master.rb

@@ -24,6 +24,12 @@ module MU
     autoload :LDAP, 'mu/master/ldap'
     autoload :SSL, 'mu/master/ssl'
 
+    # Home directory of the invoking user
+    MY_HOME = Etc.getpwuid(Process.uid).dir
+
+    # Home directory of the Nagios user, if we're in a non-gem context
+    NAGIOS_HOME = "/opt/mu/var/nagios_user_home" # XXX gross
+
     # @param users [Hash]: User metadata of the type returned by listUsers
     def self.printUsersToTerminal(users = MU::Master.listUsers)
       labeled = false
@@ -228,7 +234,7 @@ module MU
             begin
               resp = MU::Cloud::AWS.s3.get_object(bucket: MU.adminBucketName, key: cryptfile)
               body = resp.body
-            rescue Exception => e
+            rescue StandardError => e
               MU.log "Failed to fetch #{cryptfile} from S3 bucket #{MU.adminBucketName}", MU::ERR, details: e.inspect
               %x{/bin/dd if=/dev/urandom of=#{temp_dev} bs=1M count=1 > /dev/null 2>&1}
               raise e
@@ -236,7 +242,7 @@ module MU
           elsif MU::Cloud::Google.hosted?
             begin
               body = MU::Cloud::Google.storage.get_object(MU.adminBucketName, cryptfile)
-            rescue Exception => e
+            rescue StandardError => e
               MU.log "Failed to fetch #{cryptfile} from Cloud Storage bucket #{MU.adminBucketName}", MU::ERR, details: e.inspect
               %x{/bin/dd if=/dev/urandom of=#{temp_dev} bs=1M count=1 > /dev/null 2>&1}
               raise e
@@ -380,6 +386,7 @@ module MU
       best = nil
       best_version = nil
       paths.uniq.each { |path|
+        path.sub!(/^~/, MY_HOME)
         if File.exist?(path+"/kubectl")
           version = %x{#{path}/kubectl version --short --client}.chomp.sub(/.*Client version:\s+v/i, '')
           next if !$?.success?
@@ -493,5 +500,385 @@ module MU
       end
     end
 
+    # Clean a node's entries out of /etc/hosts
+    # @param node [String]: The node's name
+    # @return [void]
+    def self.removeInstanceFromEtcHosts(node)
+      return if MU.mu_user != "mu"
+      hostsfile = "/etc/hosts"
+      FileUtils.copy(hostsfile, "#{hostsfile}.bak-#{MU.deploy_id}")
+      File.open(hostsfile, File::CREAT|File::RDWR, 0644) { |f|
+        f.flock(File::LOCK_EX)
+        newlines = Array.new
+        f.readlines.each { |line|
+          newlines << line if !line.match(/ #{node}(\s|$)/)
+        }
+        f.rewind
+        f.truncate(0)
+        f.puts(newlines)
+        f.flush
+
+        f.flock(File::LOCK_UN)
+      }
+    end
+
+
+    # Insert node names associated with a new instance into /etc/hosts so we
+    # can treat them as if they were real DNS entries. Especially helpful when
+    # Chef/Ohai mistake the proper hostname, e.g. when bootstrapping Windows.
+    # @param public_ip [String]: The node's IP address
+    # @param chef_name [String]: The node's Chef node name
+    # @param system_name [String]: The node's local system name
+    # @return [void]
+    def self.addInstanceToEtcHosts(public_ip, chef_name = nil, system_name = nil)
+
+      # XXX cover ipv6 case
+      if public_ip.nil? or !public_ip.match(/^\d+\.\d+\.\d+\.\d+$/) or (chef_name.nil? and system_name.nil?)
+        raise MuError, "addInstanceToEtcHosts requires public_ip and one or both of chef_name and system_name!"
+      end
+      if chef_name == "localhost" or system_name == "localhost"
+        raise MuError, "Can't set localhost as a name in addInstanceToEtcHosts"
+      end
+
+      if !["mu", "root"].include?(MU.mu_user)
+        response = nil
+        begin
+          response = open("https://127.0.0.1:#{MU.mommaCatPort.to_s}/rest/hosts_add/#{chef_name}/#{public_ip}").read
+        rescue Errno::ECONNRESET, Errno::ECONNREFUSED
+        end
+        if response != "ok"
+          MU.log "Unable to add #{public_ip} to /etc/hosts via MommaCat request", MU::WARN
+        end
+        return
+      end
+
+      File.readlines("/etc/hosts").each { |line|
+        if line.match(/^#{public_ip} /) or (chef_name != nil and line.match(/ #{chef_name}(\s|$)/)) or (system_name != nil and line.match(/ #{system_name}(\s|$)/))
+          MU.log "Ignoring attempt to add duplicate /etc/hosts entry: #{public_ip} #{chef_name} #{system_name}", MU::DEBUG
+          return
+        end
+      }
+      File.open("/etc/hosts", 'a') { |etc_hosts|
+        etc_hosts.flock(File::LOCK_EX)
+        etc_hosts.puts("#{public_ip} #{chef_name} #{system_name}")
+        etc_hosts.flock(File::LOCK_UN)
+      }
+      MU.log("Added to /etc/hosts: #{public_ip} #{chef_name} #{system_name}")
+    end
+
+    @ssh_semaphore = Mutex.new
+    # Insert a definition for a node into our SSH config.
+    # @param server [MU::Cloud::Server]: The name of the node.
+    # @param names [Array<String>]: Other names that we'd like this host to be known by for SSH purposes
+    # @param ssh_dir [String]: The configuration directory of the SSH config to emit.
+    # @param ssh_conf [String]: A specific SSH configuration file to write entries into.
+    # @param ssh_owner [String]: The preferred owner of the SSH configuration files.
+    # @param timeout [Integer]: An alternate timeout value for connections to this server.
+    # @return [void]
+    def self.addHostToSSHConfig(server,
+        ssh_dir: "#{Etc.getpwuid(Process.uid).dir}/.ssh",
+        ssh_conf: "#{Etc.getpwuid(Process.uid).dir}/.ssh/config",
+        ssh_owner: Etc.getpwuid(Process.uid).name,
+        names: [],
+        timeout: 0
+    )
+      if server.nil?
+        MU.log "Called addHostToSSHConfig without a MU::Cloud::Server object", MU::ERR, details: caller
+        return nil
+      end
+
+      _nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name = begin
+        server.getSSHConfig
+      rescue MU::MuError
+        return
+      end
+
+      if ssh_user.nil? or ssh_user.empty?
+        MU.log "Failed to extract ssh_user for #{server.mu_name} addHostToSSHConfig", MU::ERR
+        return
+      end
+      if canonical_ip.nil? or canonical_ip.empty?
+        MU.log "Failed to extract canonical_ip for #{server.mu_name} addHostToSSHConfig", MU::ERR
+        return
+      end
+      if ssh_key_name.nil? or ssh_key_name.empty?
+        MU.log "Failed to extract ssh_key_name for #{server.mu_name} in addHostToSSHConfig", MU::ERR
+        return
+      end
+
+      @ssh_semaphore.synchronize {
+
+        if File.exist?(ssh_conf)
+          File.readlines(ssh_conf).each { |line|
+            if line.match(/^Host #{server.mu_name} /)
+              MU.log("Attempt to add duplicate #{ssh_conf} entry for #{server.mu_name}", MU::WARN)
+              return
+            end
+          }
+        end
+
+        File.open(ssh_conf, 'a', 0600) { |ssh_config|
+          ssh_config.flock(File::LOCK_EX)
+          host_str = "Host #{server.mu_name} #{server.canonicalIP}"
+          if !names.nil? and names.size > 0
+            host_str = host_str+" "+names.join(" ")
+          end
+          ssh_config.puts host_str
+          ssh_config.puts "  Hostname #{server.canonicalIP}"
+          if !nat_ssh_host.nil? and server.canonicalIP != nat_ssh_host
+            ssh_config.puts "  ProxyCommand ssh -W %h:%p #{nat_ssh_user}@#{nat_ssh_host}"
+          end
+          if timeout > 0
+            ssh_config.puts "  ConnectTimeout #{timeout}"
+          end
+
+          ssh_config.puts "  User #{ssh_user}"
+# XXX I'd rather add the host key to known_hosts, but Net::SSH is a little dumb
+          ssh_config.puts "  StrictHostKeyChecking no"
+          ssh_config.puts "  ServerAliveInterval 60"
+
+          ssh_config.puts "  IdentityFile #{ssh_dir}/#{ssh_key_name}"
+          if !File.exist?("#{ssh_dir}/#{ssh_key_name}")
+            MU.log "#{server.mu_name} - ssh private key #{ssh_dir}/#{ssh_key_name} does not exist", MU::WARN
+          end
+
+          ssh_config.flock(File::LOCK_UN)
+          ssh_config.chown(Etc.getpwnam(ssh_owner).uid, Etc.getpwnam(ssh_owner).gid)
+        }
+        MU.log "Wrote #{server.mu_name} ssh key to #{ssh_dir}/config", MU::DEBUG
+        return "#{ssh_dir}/#{ssh_key_name}"
+      }
+    end
+
+    # Clean an IP address out of ~/.ssh/known hosts
+    # @param ip [String]: The IP to remove
+    # @return [void]
+    def self.removeIPFromSSHKnownHosts(ip, noop: false)
+      return if ip.nil?
+      sshdir = "#{MY_HOME}/.ssh"
+      knownhosts = "#{sshdir}/known_hosts"
+
+      if File.exist?(knownhosts) and File.open(knownhosts).read.match(/^#{Regexp.quote(ip)} /)
+        MU.log "Expunging old #{ip} entry from #{knownhosts}", MU::NOTICE
+        if !noop
+          File.open(knownhosts, File::CREAT|File::RDWR, 0600) { |f|
+            f.flock(File::LOCK_EX)
+            newlines = Array.new
+            f.readlines.each { |line|
+              next if line.match(/^#{Regexp.quote(ip)} /)
+              newlines << line
+            }
+            f.rewind
+            f.truncate(0)
+            f.puts(newlines)
+            f.flush
+            f.flock(File::LOCK_UN)
+          }
+        end
+      end
+    end
+
+    # Clean a node's entries out of ~/.ssh/config
+    # @param nodename [String]: The node's name
+    # @return [void]
+    def self.removeHostFromSSHConfig(nodename, noop: false)
+      sshdir = "#{MY_HOME}/.ssh"
+      sshconf = "#{sshdir}/config"
+
+      if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{nodename} /)
+        MU.log "Expunging old #{nodename} entry from #{sshconf}", MU::DEBUG
+        if !noop
+          File.open(sshconf, File::CREAT|File::RDWR, 0600) { |f|
+            f.flock(File::LOCK_EX)
+            newlines = Array.new
+            delete_block = false
+            f.readlines.each { |line|
+              if line.match(/^Host #{nodename}(\s|$)/)
+                delete_block = true
+              elsif line.match(/^Host /)
+                delete_block = false
+              end
+              newlines << line if !delete_block
+            }
+            f.rewind
+            f.truncate(0)
+            f.puts(newlines)
+            f.flush
+            f.flock(File::LOCK_UN)
+          }
+        end
+      end
+    end
+
+    # Evict ssh keys associated with a particular deploy from our ssh config
+    # and key directory.
+    # @param deploy_id [String]
+    # @param noop [Boolean]
+    def self.purgeDeployFromSSH(deploy_id, noop: false)
+      myhome = Etc.getpwuid(Process.uid).dir
+      sshdir = "#{myhome}/.ssh"
+      sshconf = "#{sshdir}/config"
+      ssharchive = "#{sshdir}/archive"
+
+      Dir.mkdir(sshdir, 0700) if !Dir.exist?(sshdir) and !noop
+      Dir.mkdir(ssharchive, 0700) if !Dir.exist?(ssharchive) and !noop
+
+      keyname = "deploy-#{deploy_id}"
+      if File.exist?("#{sshdir}/#{keyname}")
+        MU.log "Moving #{sshdir}/#{keyname} to #{ssharchive}/#{keyname}"
+        if !noop
+          File.rename("#{sshdir}/#{keyname}", "#{ssharchive}/#{keyname}")
+        end
+      end
+      if File.exist?(sshconf) and File.open(sshconf).read.match(/\/deploy\-#{deploy_id}$/)
+        MU.log "Expunging #{deploy_id} from #{sshconf}"
+        if !noop
+          FileUtils.copy(sshconf, "#{ssharchive}/config-#{deploy_id}")
+          File.open(sshconf, File::CREAT|File::RDWR, 0600) { |f|
+            f.flock(File::LOCK_EX)
+            newlines = Array.new
+            delete_block = false
+            f.readlines.each { |line|
+              if line.match(/^Host #{deploy_id}\-/)
+                delete_block = true
+              elsif line.match(/^Host /)
+                delete_block = false
+              end
+              newlines << line if !delete_block
+            }
+            f.rewind
+            f.truncate(0)
+            f.puts(newlines)
+            f.flush
+            f.flock(File::LOCK_UN)
+          }
+        end
+      end
+      # XXX refactor with above? They're similar, ish.
+      hostsfile = "/etc/hosts"
+      if File.open(hostsfile).read.match(/ #{deploy_id}\-/)
+        if Process.uid == 0
+          MU.log "Expunging traces of #{deploy_id} from #{hostsfile}"
+          if !noop
+            FileUtils.copy(hostsfile, "#{hostsfile}.cleanup-#{deploy_id}")
+            File.open(hostsfile, File::CREAT|File::RDWR, 0644) { |f|
+              f.flock(File::LOCK_EX)
+              newlines = Array.new
+              f.readlines.each { |line|
+                newlines << line if !line.match(/ #{deploy_id}\-/)
+              }
+              f.rewind
+              f.truncate(0)
+              f.puts(newlines)
+              f.flush
+              f.flock(File::LOCK_UN)
+            }
+          end
+        else
+          MU.log "Residual /etc/hosts entries for #{deploy_id} must be removed by root user", MU::WARN
+        end
+      end
+
+    end
+
+    # Ensure that the Nagios configuration local to the MU master has been
+    # updated, and make sure Nagios has all of the ssh keys it needs to tunnel
+    # to client nodes.
+    # @return [void]
+    def self.syncMonitoringConfig(blocking = true)
+      return if Etc.getpwuid(Process.uid).name != "root" or (MU.mu_user != "mu" and MU.mu_user != "root")
+      parent_thread_id = Thread.current.object_id
+      nagios_threads = []
+      nagios_threads << Thread.new {
+        MU.dupGlobals(parent_thread_id)
+        realhome = Etc.getpwnam("nagios").dir
+        [NAGIOS_HOME, "#{NAGIOS_HOME}/.ssh"].each { |dir|
+          Dir.mkdir(dir, 0711) if !Dir.exist?(dir)
+          File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, dir)
+        }
+        if realhome != NAGIOS_HOME and Dir.exist?(realhome) and !File.symlink?("#{realhome}/.ssh")
+          File.rename("#{realhome}/.ssh", "#{realhome}/.ssh.#{$$}") if Dir.exist?("#{realhome}/.ssh")
+          File.symlink("#{NAGIOS_HOME}/.ssh", Etc.getpwnam("nagios").dir+"/.ssh")
+        end
+        MU.log "Updating #{NAGIOS_HOME}/.ssh/config..."
+        ssh_lock = File.new("#{NAGIOS_HOME}/.ssh/config.mu.lock", File::CREAT|File::TRUNC|File::RDWR, 0600)
+        ssh_lock.flock(File::LOCK_EX)
+        ssh_conf = File.new("#{NAGIOS_HOME}/.ssh/config.tmp", File::CREAT|File::TRUNC|File::RDWR, 0600)
+        ssh_conf.puts "Host MU-MASTER localhost"
+        ssh_conf.puts "  Hostname localhost"
+        ssh_conf.puts "  User root"
+        ssh_conf.puts "  IdentityFile #{NAGIOS_HOME}/.ssh/id_rsa"
+        ssh_conf.puts "  StrictHostKeyChecking no"
+        ssh_conf.close
+        FileUtils.cp("#{Etc.getpwuid(Process.uid).dir}/.ssh/id_rsa", "#{NAGIOS_HOME}/.ssh/id_rsa")
+        File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, "#{NAGIOS_HOME}/.ssh/id_rsa")
+        threads = []
+
+        parent_thread_id = Thread.current.object_id
+        MU::MommaCat.listDeploys.sort.each { |deploy_id|
+          begin
+            # We don't want to use cached litter information here because this is also called by cleanTerminatedInstances.
+            deploy = MU::MommaCat.getLitter(deploy_id)
+            if deploy.ssh_key_name.nil? or deploy.ssh_key_name.empty?
+              MU.log "Failed to extract ssh key name from #{deploy_id} in syncMonitoringConfig", MU::ERR if deploy.kittens.has_key?("servers")
+              next
+            end
+            FileUtils.cp("#{Etc.getpwuid(Process.uid).dir}/.ssh/#{deploy.ssh_key_name}", "#{NAGIOS_HOME}/.ssh/#{deploy.ssh_key_name}")
+            File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, "#{NAGIOS_HOME}/.ssh/#{deploy.ssh_key_name}")
+            if deploy.kittens.has_key?("servers")
+              deploy.kittens["servers"].values.each { |nodeclasses|
+                nodeclasses.values.each { |nodes|
+                  nodes.values.each { |server|
+                    next if !server.cloud_desc
+                    MU.dupGlobals(parent_thread_id)
+                    threads << Thread.new {
+                      MU::MommaCat.setThreadContext(deploy)
+                      MU.log "Adding #{server.mu_name} to #{NAGIOS_HOME}/.ssh/config", MU::DEBUG
+                      MU::Master.addHostToSSHConfig(
+                          server,
+                          ssh_dir: "#{NAGIOS_HOME}/.ssh",
+                          ssh_conf: "#{NAGIOS_HOME}/.ssh/config.tmp",
+                          ssh_owner: "nagios"
+                      )
+                      MU.purgeGlobals
+                    }
+                  }
+                }
+              }
+            end
+          rescue StandardError => e
+            MU.log "#{e.inspect} while generating Nagios SSH config in #{deploy_id}", MU::ERR, details: e.backtrace
+          end
+        }
+        threads.each { |t|
+          t.join
+        }
+        ssh_lock.flock(File::LOCK_UN)
+        ssh_lock.close
+        File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, "#{NAGIOS_HOME}/.ssh/config.tmp")
+        File.rename("#{NAGIOS_HOME}/.ssh/config.tmp", "#{NAGIOS_HOME}/.ssh/config")
+
+        MU.log "Updating Nagios monitoring config, this may take a while..."
+        output = nil
+        if $MU_CFG and !$MU_CFG['master_runlist_extras'].nil?
+          output = %x{#{MU::Groomer::Chef.chefclient} -o 'role[mu-master-nagios-only],#{$MU_CFG['master_runlist_extras'].join(",")}' 2>&1}
+        else
+          output = %x{#{MU::Groomer::Chef.chefclient} -o 'role[mu-master-nagios-only]' 2>&1}
+        end
+
+        if $?.exitstatus != 0
+          MU.log "Nagios monitoring config update returned a non-zero exit code!", MU::ERR, details: output
+        else
+          MU.log "Nagios monitoring config update complete."
+        end
+      }
+
+      if blocking
+        nagios_threads.each { |t|
+          t.join
+        }
+      end
+    end
+
   end
 end

data/modules/mu/master/chef.rb

@@ -48,7 +48,7 @@ module MU
       def self.deleteOrg(org)
         begin
           Timeout::timeout(45) {
-            response = chefAPI.delete("organizations/#{org}")
+            chefAPI.delete("organizations/#{org}")
           }
           MU.log "Removed Chef organization #{org}", MU::NOTICE
           return true
@@ -82,7 +82,7 @@ module MU
 
         begin
           Timeout::timeout(45) {
-            response = chefAPI.delete("users/#{chef_user}")
+            chefAPI.delete("users/#{chef_user}")
           }
           MU.log "Removed Chef user #{chef_user}", MU::NOTICE
           return true
@@ -200,7 +200,6 @@ module MU
 
         # This organization does not yet exist, create it
         if !existing_org
-          name = org.dup if fullname.nil?
           begin
             org_data = {
               :name => org.dup,
@@ -255,7 +254,7 @@ module MU
               }
               remove_users.each { |user|
                 begin
-                  response = chefAPI.delete("organizations/#{org}/users/#{user}")
+                  chefAPI.delete("organizations/#{org}/users/#{user}")
                   MU.log "Removed Chef user #{user} from organization #{org}", MU::NOTICE
                 rescue Net::HTTPServerException => e
                 end