cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/logger.rb

@@ -36,21 +36,24 @@ module MU
     @verbosity = MU::Logger::NORMAL
     @quiet = false
     @html = false
+    @color = true
     @handle = STDOUT
 
     @@log_semaphere = Mutex.new
 
     # @param verbosity [Integer]: See {MU::Logger.QUIET}, {MU::Logger.NORMAL}, {MU::Logger.LOUD}
     # @param html [Boolean]: Enable web-friendly log output.
-    def initialize(verbosity=MU::Logger::NORMAL, html=false, handle=STDOUT)
+    def initialize(verbosity=MU::Logger::NORMAL, html=false, handle=STDOUT, color=true)
       @verbosity = verbosity
       @html = html
       @handle = handle
+      @color = color
       @summary = []
     end
 
     attr_reader :summary
     attr_accessor :verbosity
+    attr_accessor :color
     attr_accessor :quiet
     attr_accessor :html
     attr_accessor :handle
@@ -65,7 +68,8 @@ module MU
             details: nil,
             html: @html,
             verbosity: @verbosity,
-            handle: @handle
+            handle: @handle,
+            color: @color
     )
       verbosity = MU::Logger::NORMAL if verbosity.nil?
       return if verbosity == MU::Logger::SILENT
@@ -98,12 +102,14 @@ module MU
       # We get passed literal quoted newlines sometimes, fix 'em. Get Windows'
       # ugly line feeds too.
       if !details.nil?
+        details = details.dup # in case it's frozen or something
         details.gsub!(/\\n/, "\n")
         details.gsub!(/(\\r|\r)/, "")
       end
 
       msg = msg.first if msg.is_a?(Array)
       msg = "" if msg == nil
+      msg = msg.to_s if !msg.is_a?(String) and msg.respond_to?(:to_s)
 
       @@log_semaphere.synchronize {
         case level
@@ -114,9 +120,12 @@ module MU
               if @html
                 html_out "#{time} - #{caller_name} - #{msg}", "orange"
                 html_out " #{details}" if details
-              else
+              elsif color
                 handle.puts "#{time} - #{caller_name} - #{msg}".yellow.on_black
                 handle.puts "#{details}".white.on_black if details
+              else
+                handle.puts "#{time} - #{caller_name} - #{msg}"
+                handle.puts "#{details}" if details
               end
               Syslog.log(Syslog::LOG_DEBUG, msg.gsub(/%/, ''))
               Syslog.log(Syslog::LOG_DEBUG, details.gsub(/%/, '')) if details
@@ -125,14 +134,18 @@ module MU
             if verbosity >= MU::Logger::NORMAL
               if @html
                 html_out "#{time} - #{caller_name} - #{msg}", "green"
-              else
+              elsif color
                 handle.puts "#{time} - #{caller_name} - #{msg}".green.on_black
+              else
+                handle.puts "#{time} - #{caller_name} - #{msg}"
               end
               if verbosity >= MU::Logger::LOUD
                 if @html
                   html_out " #{details}"
-                else
+                elsif color
                   handle.puts "#{details}".white.on_black if details
+                else
+                  handle.puts "#{details}" if details
                 end
               end
               Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
@@ -141,14 +154,18 @@ module MU
           when NOTICE
             if @html
               html_out "#{time} - #{caller_name} - #{msg}", "yellow"
-            else
+            elsif color
               handle.puts "#{time} - #{caller_name} - #{msg}".yellow.on_black
+            else
+              handle.puts "#{time} - #{caller_name} - #{msg}"
             end
             if verbosity >= MU::Logger::LOUD
               if @html
                 html_out "#{caller_name} - #{msg}"
-              else
+              elsif color
                 handle.puts "#{details}".white.on_black if details
+              else
+                handle.puts "#{details}" if details
               end
             end
             Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
@@ -156,14 +173,18 @@ module MU
           when WARN
             if @html
               html_out "#{time} - #{caller_name} - #{msg}", "orange"
-            else
+            elsif color
               handle.puts "#{time} - #{caller_name} - #{msg}".light_red.on_black
+            else
+              handle.puts "#{time} - #{caller_name} - #{msg}"
             end
             if verbosity >= MU::Logger::LOUD
               if @html
                 html_out "#{caller_name} - #{msg}"
-              else
+              elsif color
                 handle.puts "#{details}".white.on_black if details
+              else
+                handle.puts "#{details}" if details
               end
             end
             Syslog.log(Syslog::LOG_WARNING, msg.gsub(/%/, ''))
@@ -172,9 +193,12 @@ module MU
             if @html
               html_out "#{time} - #{caller_name} - #{msg}", "red"
               html_out " #{details}" if details
-            else
+            elsif color
               handle.puts "#{time} - #{caller_name} - #{msg}".red.on_black
               handle.puts "#{details}".white.on_black if details
+            else
+              handle.puts "#{time} - #{caller_name} - #{msg}"
+              handle.puts "#{details}" if details
             end
             Syslog.log(Syslog::LOG_ERR, msg.gsub(/%/, ''))
             Syslog.log(Syslog::LOG_ERR, details.gsub(/%/, '')) if details
@@ -182,9 +206,12 @@ module MU
             if @html
               html_out "#{time} - #{caller_name} - #{msg}"
               html_out " #{details}" if details
-            else
+            elsif color
               handle.puts "#{time} - #{caller_name} - #{msg}".white.on_black
               handle.puts "#{details}".white.on_black if details
+            else
+              handle.puts "#{time} - #{caller_name} - #{msg}"
+              handle.puts "#{details}" if details
             end
             Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
             Syslog.log(Syslog::LOG_NOTICE, details.gsub(/%/, '')) if details

data/modules/mu/master.rb

@@ -1,4 +1,3 @@
-#!/usr/local/ruby-current/bin/ruby
 # Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
 #
 # Licensed under the BSD-3 license (the "License");
@@ -23,6 +22,7 @@ module MU
     require 'fileutils'
     autoload :Chef, 'mu/master/chef'
     autoload :LDAP, 'mu/master/ldap'
+    autoload :SSL, 'mu/master/ssl'
 
     # @param users [Hash]: User metadata of the type returned by listUsers
     def self.printUsersToTerminal(users = MU::Master.listUsers)
@@ -169,7 +169,7 @@ module MU
           itemname = Password.pronounceable(32)
           # Make sure this itemname isn't already in use
           MU::Groomer::Chef.getSecret(vault: "scratchpad", item: itemname)
-        rescue MU::Groomer::Chef::MuNoSuchSecret
+        rescue MU::Groomer::MuNoSuchSecret
           MU::Groomer::Chef.saveSecret(vault: "scratchpad", item: itemname, data: data)
           return itemname
         end while true
@@ -195,7 +195,7 @@ module MU
         end
         alias_device = cryptfile ? "/dev/mapper/"+path.gsub(/[^0-9a-z_\-]/i, "_") : realdevice
 
-        if !File.exists?(realdevice)
+        if !File.exist?(realdevice)
           MU.log "Creating #{path} volume"
           if MU::Cloud::AWS.hosted?
             dummy_svr = MU::Cloud::AWS::Server.new(
@@ -250,7 +250,7 @@ module MU
           keyfile.close
 
           # we can assume that mu-master::init installed cryptsetup-luks
-          if !File.exists?(alias_device)
+          if !File.exist?(alias_device)
             MU.log "Initializing crypto on #{alias_device}", MU::NOTICE
             %x{/sbin/cryptsetup luksFormat #{realdevice} #{keyfile.path} --batch-mode}
             %x{/sbin/cryptsetup luksOpen #{realdevice} #{alias_device.gsub(/.*?\/([^\/]+)$/, '\1')} --key-file #{keyfile.path}}
@@ -264,7 +264,7 @@ module MU
           %x{/sbin/mkfs.xfs "#{alias_device}"}
           %x{/usr/sbin/xfs_admin -L "#{path.gsub(/[^0-9a-z_\-]/i, "_")}" "#{alias_device}"}
         end
-        Dir.mkdir(path, 0700) if !Dir.exists?(path) # XXX recursive
+        Dir.mkdir(path, 0700) if !Dir.exist?(path) # XXX recursive
         %x{/usr/sbin/xfs_info "#{alias_device}" > /dev/null 2>&1}
         if $?.exitstatus != 0
           MU.log "Mounting #{alias_device} to #{path}"
@@ -292,7 +292,7 @@ module MU
 
     # Remove Scratchpad entries which have exceeded their maximum age.
     def self.cleanExpiredScratchpads
-      return if !$MU_CFG['scratchpad'].has_key?('max_age') or $MU_CFG['scratchpad']['max_age'] < 1
+      return if !$MU_CFG['scratchpad'] or !$MU_CFG['scratchpad'].has_key?('max_age') or $MU_CFG['scratchpad']['max_age'] < 1
       @scratchpad_semaphore.synchronize {
         entries = MU::Groomer::Chef.getSecret(vault: "scratchpad")
         entries.each { |pad|
@@ -347,8 +347,8 @@ module MU
       ldap_users['mu'] = {}
       ldap_users['mu']['admin'] = true
       ldap_users['mu']['non_ldap'] = true
-      ldap_users.each_pair { |username, data|
-        key = username.to_s
+      ldap_users.each_pair { |uname, data|
+        key = uname.to_s
         all_user_data[key] = {}
         userdir = $MU_CFG['installdir']+"/var/users/#{key}"
         if !Dir.exist?(userdir)
@@ -369,6 +369,88 @@ module MU
       all_user_data
     end
 
+
+    @@kubectl_path = nil
+    # Locate a working +kubectl+ executable and return its fully-qualified
+    # path.
+    def self.kubectl
+      return @@kubectl_path if @@kubectl_path
+
+      paths = ["/opt/mu/bin"]+ENV['PATH'].split(/:/)
+      best = nil
+      best_version = nil
+      paths.uniq.each { |path|
+        if File.exist?(path+"/kubectl")
+          version = %x{#{path}/kubectl version --short --client}.chomp.sub(/.*Client version:\s+v/i, '')
+          next if !$?.success?
+          if !best_version or MU.version_sort(best_version, version) > 0
+            best_version = version
+            best = path+"/kubectl"
+          end
+        end
+      }
+      if !best
+        MU.log "Failed to find a working kubectl executable in any path", MU::WARN, details: paths.uniq.sort
+        return nil
+      else
+        MU.log "Kubernetes commands will use #{best} (#{best_version})"
+      end
+
+      @@kubectl_path = best
+      @@kubectl_path
+    end
+
+    # Given an array of hashes representing Kubernetes resources, 
+    def self.applyKubernetesResources(name, blobs = [], kubeconfig: nil, outputdir: nil)
+      use_tmp = false
+      if !outputdir
+        require 'tempfile'
+        use_tmp = true
+      end
+
+      count = 0
+      blobs.each { |blob|
+        f = nil
+        blobfile = if use_tmp
+          f = Tempfile.new("k8s-resource-#{count.to_s}-#{name}")
+          f.puts blob.to_yaml
+          f.close
+          f.path
+        else
+          path = outputdir+"/k8s-resource-#{count.to_s}-#{name}"
+          File.open(path, "w") { |fh|
+            fh.puts blob.to_yaml
+          }
+          path
+        end
+        next if !kubectl
+        done = false
+        retries = 0
+        begin
+          %x{#{kubectl} --kubeconfig "#{kubeconfig}" get -f #{blobfile} > /dev/null 2>&1}
+          arg = $?.exitstatus == 0 ? "apply" : "create"
+          cmd = %Q{#{kubectl} --kubeconfig "#{kubeconfig}" #{arg} -f #{blobfile}}
+          MU.log "Applying Kubernetes resource #{count.to_s} with kubectl #{arg}", MU::NOTICE, details: cmd
+          output = %x{#{cmd} 2>&1}
+          if $?.exitstatus == 0
+            MU.log "Kubernetes resource #{count.to_s} #{arg} was successful: #{output}", details: blob.to_yaml
+            done = true
+          else
+            MU.log "Kubernetes resource #{count.to_s} #{arg} failed: #{output}", MU::WARN, details: blob.to_yaml
+            if retries < 5
+              sleep 5
+            else
+              MU.log "Giving up on Kubernetes resource #{count.to_s} #{arg}"
+              done = true
+            end
+            retries += 1
+          end
+          f.unlink if use_tmp
+        end while !done
+        count += 1
+      }
+    end
+
     # Update Mu's local cache/metadata for the given user, fixing permissions
     # and updating stored values. Create a single-user group for the user, as
     # well.

data/modules/mu/master/chef.rb

@@ -1,4 +1,3 @@
-#!/usr/local/ruby-current/bin/ruby
 # Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
 #
 # Licensed under the BSD-3 license (the "License");
@@ -112,7 +111,7 @@ module MU
           f.puts "chef_server_url  'https://#{$MU_CFG["public_address"]}/organizations/#{chef_user}'"
           f.puts "validation_client_name '#{chef_user}-validator'"
         }
-        if !File.exists?("#{chefdir}/client.rb") or
+        if !File.exist?("#{chefdir}/client.rb") or
             File.read("#{chefdir}/client.rb") != File.read("#{chefdir}/client.rb.tmp.#{Process.pid}")
           File.rename(chefdir+"/client.rb.tmp.#{Process.pid}", chefdir+"/client.rb")
           FileUtils.chown_R(user, user+".mu-user", Etc.getpwnam(user).dir+"/.chef")
@@ -143,7 +142,7 @@ module MU
           # f.puts "verify_api_cert    false"
           # f.puts "ssl_verify_mode    :verify_none"
         }
-        if !File.exists?("#{chefdir}/knife.rb") or
+        if !File.exist?("#{chefdir}/knife.rb") or
             File.read("#{chefdir}/knife.rb") != File.read("#{chefdir}/knife.rb.tmp.#{Process.pid}")
           File.rename(chefdir+"/knife.rb.tmp.#{Process.pid}", chefdir+"/knife.rb")
           FileUtils.chown_R(user, user+".mu-user", Etc.getpwnam(user).dir+"/.chef")

data/modules/mu/master/ldap.rb

@@ -1,4 +1,3 @@
-#!/usr/local/ruby-current/bin/ruby
 # Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
 #
 # Licensed under the BSD-3 license (the "License");

data/modules/mu/master/ssl.rb

@@ -0,0 +1,250 @@
+# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Master
+    # Create and manage our own internal SSL signing authority
+    class SSL
+
+      # List of Mu services for which we'll generate SSL certs signed by our
+      # authority.
+      SERVICES = ["rsyslog", "mommacat", "ldap", "consul", "vault"]
+
+      # Exception class for when we can't find the +openssl+ command
+      class MuSSLNotFound < MU::MuError;end
+
+# TODO set file/dir ownerships to honor for_user if we were invoked as root
+
+      # @param for_user [String]
+      def self.bootstrap(for_user: MU.mu_user)
+        ssldir = MU.dataDir(for_user)+"/ssl"
+        Dir.mkdir(ssldir, 0755) if !Dir.exist?(ssldir)
+
+        alt_names = [MU.mu_public_ip, MU.my_private_ip, MU.mu_public_addr, Socket.gethostbyname(Socket.gethostname).first, "localhost", "127.0.0.1"].uniq
+        alt_names.reject! { |s| s.nil? }
+
+        getCert("Mu_CA", "/CN=#{MU.mu_public_addr}/OU=Mu Server at #{MU.mu_public_addr}/O=eGlobalTech/C=US", sans: alt_names, ca: true)
+
+        SERVICES.each { |service|
+          getCert(service, "/CN=#{MU.mu_public_addr}/OU=Mu #{service}/O=eGlobalTech/C=US", sans: alt_names)
+        }
+
+      end
+
+      # @param name [String]
+      # @param for_user [String]
+      # @return [OpenSSL::PKey::RSA]
+      def self.getKey(name, for_user: MU.mu_user, keysize: 4096)
+        ssldir = MU.dataDir(for_user)+"/ssl"
+        if !File.exist?(ssldir+"/"+name+".key")
+          key = OpenSSL::PKey::RSA.new keysize
+          File.write(ssldir+"/"+name+".key", key)
+        end
+        File.chmod(0400, ssldir+"/"+name+".key")
+        OpenSSL::PKey::RSA.new(File.read(ssldir+"/"+name+".key"))
+      end
+
+      # @param for_user [String]
+      # @return [Integer]
+      def self.incrementCASerial(for_user: MU.mu_user)
+        ssldir = MU.dataDir(for_user)+"/ssl"
+        cur = 0
+        if File.exist?(ssldir+"/serial")
+          cur = File.read(ssldir+"/serial").chomp.to_i
+        end
+        File.open("#{ssldir}/serial", File::CREAT|File::RDWR, 0600) { |f|
+          f.flock(File::LOCK_EX)
+          cur += 1
+          f.rewind
+          f.truncate(0)
+          f.puts cur
+          f.flush
+          f.flock(File::LOCK_UN)
+        }
+        cur
+      end
+
+
+      # Given a Certificate Signing Request, sign it with our internal CA and
+      # write the resulting signed certificate. Only works on local files.
+      # @param csr_path [String]: The CSR to sign, as a file.
+      def self.sign(csr_path, sans = [], for_user: MU.mu_user)
+        certdir = File.dirname(csr_path)
+        certname = File.basename(csr_path, ".csr")
+        if File.exist?("#{certdir}/#{certname}.crt")
+          MU.log "Not re-signing SSL certificate request #{csr_path}, #{certdir}/#{certname}.crt already exists", MU::DEBUG
+          return
+        end
+        MU.log "Signing SSL certificate request #{csr_path} with #{MU.mySSLDir}/Mu_CA.pem"
+
+        begin
+          csr = OpenSSL::X509::Request.new File.read csr_path
+        rescue Exception => e
+          MU.log e.message, MU::ERR, details: File.read(csr_path)
+          raise e
+        end
+
+        cakey = getKey("Mu_CA")
+        cacert = getCert("Mu_CA", ca: true).first
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.serial = incrementCASerial(for_user: for_user)
+        cert.version = 0x2
+        cert.not_before = Time.now
+        cert.not_after = Time.now + 180000000
+        cert.subject = csr.subject
+        cert.public_key = csr.public_key
+        cert.issuer = cacert.subject
+        ef = OpenSSL::X509::ExtensionFactory.new
+        ef.issuer_certificate = cacert
+        ef.subject_certificate = cert
+        ef.subject_request = csr
+        if !sans.nil? and !sans.empty? and
+           !formatSANS(sans).nil? and !formatSANS(sans).empty?
+          cert.add_extension(ef.create_extension("subjectAltName",formatSANS(sans),false))
+        end
+        cert.add_extension(ef.create_extension("keyUsage","nonRepudiation,digitalSignature,keyEncipherment", false))
+        cert.add_extension(ef.create_extension("extendedKeyUsage","clientAuth,serverAuth,codeSigning,emailProtection",false))
+        cert.sign cakey, OpenSSL::Digest::SHA256.new
+
+        File.open("#{certdir}/#{certname}.crt", 'w', 0644) { |f|
+          f.write cert.to_pem
+        }
+
+        cert
+      end
+
+      # @param name [String]
+      # @param cn_str [String]
+      # @param sans [Array<String>]
+      # @param ca [Array<String>]
+      # @param for_user [String]
+      # @return [OpenSSL::X509::Certificate]
+      def self.getReq(name, cn_str = nil, sans: [], ca: false, for_user: MU.mu_user)
+      end
+
+      # @param name [String]
+      # @param cn_str [String]
+      # @param sans [Array<String>]
+      # @param ca [Array<String>]
+      # @param for_user [String]
+      # @param pfx [Boolean]
+      # @return [OpenSSL::X509::Certificate]
+      def self.getCert(name, cn_str = nil, sans: [], ca: false, for_user: MU.mu_user, pfx: false)
+        ssldir = MU.dataDir(for_user)+"/ssl"
+        filename = ca ? "#{ssldir}/#{name}.pem" : "#{ssldir}/#{name}.crt"
+        keyfile = "#{ssldir}/#{name}.key"
+        pfxfile = "#{ssldir}/#{name}.pfx"
+        pfx_cert = nil
+
+        if File.exist?(filename)
+          pfx_cert = toPfx(filename, keyfile, pfxfile) if pfx
+          cert = OpenSSL::X509::Certificate.new(File.read(filename))
+          return [cert, pfx_cert]
+        end
+
+        if cn_str.nil?
+          raise MuError, "Can't generate an SSL cert for #{name} without a CN"
+        end
+
+        key = getKey(name, for_user: for_user)
+
+puts cn_str
+        cn = OpenSSL::X509::Name.parse(cn_str)
+
+        # If we're generating our local CA, we're not really doing a CSR, but
+        # the operation is close to identical.
+        csr = if ca
+          MU.log "Generating Mu CA certificate", MU::NOTICE, details: filename
+          csr = OpenSSL::X509::Certificate.new
+          csr.not_before = Time.now
+          csr.not_after = Time.now + 180000000
+          csr 
+        else
+          MU.log "Generating Mu-signed certificate for #{name}", MU::NOTICE, details: filename
+          OpenSSL::X509::Request.new
+        end
+
+        csr.version = 0x2 # by which we mean '3'
+        csr.subject = cn
+        csr.public_key = key.public_key
+
+
+        # If we're the CA certificate, declare ourselves our own issuer and
+        # write, instead of going through the rest of the motions.
+        if ca
+          csr.issuer = csr.subject
+          ef = OpenSSL::X509::ExtensionFactory.new
+          csr.serial = 1
+          ef.subject_certificate = csr
+          ef.issuer_certificate = csr
+          csr.add_extension(ef.create_extension("subjectAltName",formatSANS(sans),false))
+          csr.add_extension(ef.create_extension("basicConstraints", "CA:TRUE", true))
+          csr.add_extension(ef.create_extension("keyUsage","keyCertSign, cRLSign", true))
+          csr.add_extension(ef.create_extension("subjectKeyIdentifier", "hash", false))
+          csr.add_extension(ef.create_extension("authorityKeyIdentifier", "keyid:always", false))
+        end
+
+        csr.sign key, OpenSSL::Digest::SHA256.new
+
+        cert = if !ca
+          File.open("#{ssldir}/#{name}.csr", 'w', 0644) { |f|
+            f.write csr.to_pem
+          }
+					sign("#{ssldir}/#{name}.csr", sans, for_user: for_user)
+        else
+          csr
+        end
+
+        File.open(filename, 'w', 0644) { |f|
+          f.write cert.to_pem
+        }
+        pfx_cert = toPfx(filename, keyfile, pfxfile) if pfx
+
+        if MU.mu_user != "mu" and Process.uid == 0
+          owner_uid = Etc.getpwnam(for_user).uid
+          File.chown(owner_uid, nil, filename)
+          File.chown(owner_uid, nil, pfxfile) if pfx
+        end
+
+
+        [cert, pfx_cert]
+      end
+
+      private
+
+      def self.toPfx(certfile, keyfile, pfxfile)
+        cacert = getCert("Mu_CA", ca: true).first
+        cert = OpenSSL::X509::Certificate.new(File.read(certfile))
+        key = OpenSSL::PKey::RSA.new(File.read(keyfile))
+        pfx = OpenSSL::PKCS12.create(nil, nil, key, cert, [cacert], nil, nil, nil, nil)
+        File.open(pfxfile, 'w', 0644) { |f|
+          f.write pfx.to_der
+        }
+        pfx
+      end
+
+      def self.formatSANS(sans)
+        sans.map { |s|
+          if s.match(/^\d+\.\d+\.\d+\.\d+$/)
+            "IP:"+s
+          else
+            "DNS:"+s
+          end
+        }.join(",")
+      end
+
+    end
+  end
+end