cloud-mu 3.3.2 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 831bdab20f2750e0080d2282d6bbb14710339b442bd484ae539b496f9e9cbe07
-  data.tar.gz: 1e8369e3e7937fa4bda0002a13cada2d7d99eb5c9bede0bd11b5009d5ac62add
+  metadata.gz: c0a85c9f70be756955896aaeb1ea32d462178402d4eec97279454337f839fc96
+  data.tar.gz: 3bee42f370ebb5ac6caa2fb52a36ec61d4aae204410a4aed13472cad130e222a
 SHA512:
-  metadata.gz: 7677217cea3460bccc8bfcfc6cd39400e6efc659ceaf35c80742885b541988ad42776385dc232d72848acd34d9dcbed5d44abe6ea6e5af688b3d6ec86ac5a6d6
-  data.tar.gz: 2e26064eb275a1f68f73766558bd39513c50a4218aeb8968779f6592d3b1210c4b84ee2808455d81bc3eac3bf8dad3027e97f451a99c9dd5bd87d05c77fb8880
+  metadata.gz: 2910888a4c3061b4536bd84d60ec2c6b2b4170043983c603d5ff5b0af22ed43adee32e283e13da844fd949a9761a88c171a1eeb6b2b99bd7a08a93ed1efae772
+  data.tar.gz: 5b6e371475a5768895d5618865d42d5005265fa0810d5bf71e2af93032bb2767e843612ef7ec9e401b7be48dc10827cb3445bd005cdc9e4a8dd088488f30fed7

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.3.2'
-  s.date        = '2020-10-04'
+  s.version     = '3.4.0'
+  s.date        = '2020-10-22'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -36,7 +36,7 @@ EOF
     'https://github.com/cloudamatic/mu'
   s.license       = 'BSD-3-Clause-Attribution'
   s.add_runtime_dependency 'addressable', '~> 2.5'
-  s.add_runtime_dependency "aws-sdk-core", "< 3"
+  s.add_runtime_dependency "aws-sdk", "~> 3.0"
   s.add_runtime_dependency 'azure_sdk', "~> 0.52"
   s.add_runtime_dependency 'bundler', "~> 1.17"
   s.add_runtime_dependency 'chronic_duration', "~> 0.10"

data/cookbooks/mu-tools/attributes/default.rb

@@ -21,6 +21,13 @@ if disk_name_str == "CAP-MASTER" or disk_name_str == "MU-MASTER" and !node['host
   disk_name_str = node['hostname']
 end rescue NoMethodError
 
+diskdevs = :xvd
+if !platform_family?("windows")
+  if default['kernel']['modules'].keys.include?("nvme")
+    diskdevs = :nvme
+  end
+end
+
 default['os_updates_using_chef'] = false
 
 default['application_attributes']['application_volume']['mount_directory'] = '/apps'

data/cookbooks/mu-tools/libraries/helper.rb

@@ -45,6 +45,70 @@ module Mutools
       nil
     end
 
+    # Just list our block devices
+    # @return [Array<String>]
+    def list_disk_devices
+      if File.executable?("/bin/lsblk")
+        shell_out(%Q{/bin/lsblk -i -p -r -n | egrep ' disk( |$)'}).stdout.each_line.map { |l|
+          l.chomp.sub(/ .*/, '')
+        }
+      else
+        # XXX something dumber
+        nil
+      end
+    end
+
+    # If we're in AWS and NVME-aware, return a mapping of AWS-side device names
+    # to actual NVME devices.
+    # @return [Hash]
+    def attached_nvme_disks
+      if get_aws_metadata("meta-data/instance-id").nil? or
+         !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme")
+        return {}
+      end
+      map = {}
+      devices = list_disk_devices
+      return {} if !devices
+      devices.each { |d|
+        if d =~ /^\/dev\/nvme/
+          shell_out(%Q{/sbin/nvme id-ctrl -v #{d}}).stdout.each_line { |desc|
+            if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./)
+              virt_dev = Regexp.last_match[1]
+              map[virt_dev] = d
+              if !File.exists?(virt_dev)
+                begin
+                  File.symlink(d, virt_dev)
+                rescue Errno::EEXIST # XXX whyyyyy is this needed
+                end
+              end
+              break
+            end
+          }
+        end
+      }
+      map
+    end
+
+    def real_devicepath(dev)
+      map = attached_nvme_disks
+      if map[dev]
+        map[dev]
+      else
+        dev # be nice to actually handle this too
+      end
+    end
+
+    def nvme?
+      if File.executable?("/bin/lsblk")
+        shell_out(%Q{/bin/lsblk -i -p -r -n}).stdout.each_line { |l|
+          return true if l =~ /^\/dev\/nvme\d/
+        }
+      else
+        return true if File.exists?("/dev/nvme0n1")
+      end
+      false
+    end
+
     @project = nil
     @authorizer = nil
     def set_gcp_cfg_params
@@ -186,12 +250,12 @@ module Mutools
       if cloud == "AWS"
         resp = nil
         begin
+          Chef::Log.info("Fetch deploy secret from s3://#{bucket}/#{filename}")
           resp = s3.get_object(bucket: bucket, key: filename)
         rescue ::Aws::S3::Errors::PermanentRedirect => e
           tmps3 = Aws::S3::Client.new(region: "us-east-1")
           resp = tmps3.get_object(bucket: bucket, key: filename)
         end
-        Chef::Log.info("Fetch deploy secret from s3://#{bucket}/#{filename}")
         secret = resp.body.read
       elsif cloud == "Google"
         include_recipe "mu-tools::gcloud"
@@ -230,6 +294,7 @@ module Mutools
     end
 
     def mommacat_request(action, arg)
+      params = Base64.urlsafe_encode64(JSON.generate(arg)) if arg
       uri = URI("https://#{get_mu_master_ips.first}:2260/")
       req = Net::HTTP::Post.new(uri)
       res_type = (node['deployment'].has_key?(:server_pools) and node['deployment']['server_pools'].has_key?(node['service_name'])) ? "server_pool" : "server"
@@ -241,6 +306,8 @@ module Mutools
         end
 
         Chef::Log.info("Sending Momma Cat #{action} request to #{uri} from #{get_aws_metadata("meta-data/instance-id")}")
+        disks_before = list_disk_devices if action == "add_volume"
+
         req.set_form_data(
           "mu_id" => mu_get_tag_value("MU-ID"),
           "mu_resource_name" => node['service_name'],
@@ -248,7 +315,7 @@ module Mutools
           "mu_resource_type" => res_type,
           "mu_user" => node['deployment']['mu_user'] || node['deployment']['chef_user'],
           "mu_deploy_secret" => secret,
-          action => arg
+          action => params
         )
         http = Net::HTTP.new(uri.hostname, uri.port)
         http.use_ssl = true
@@ -256,6 +323,23 @@ module Mutools
         response = http.request(req)
         if response.code != "200"
           Chef::Log.error("Got #{response.code} back from #{uri} on #{action} => #{arg}")
+        else
+          if action == "add_volume" and arg and arg.is_a?(Hash) and arg[:dev]
+            seen_requested = false
+            retries = 0
+            begin
+              list_disk_devices.each { |d|
+                if d == arg[:dev] or
+                   (nvme? and d == attached_nvme_disks[arg[:dev]])
+                  seen_requested = true
+                end
+              }
+              if !seen_requested
+                sleep 6
+                retries += 1
+              end
+            end while retries < 5 and !seen_requested
+          end
         end
       rescue EOFError => e
         # Sometimes deployment metadata is incomplete and missing a

data/cookbooks/mu-tools/recipes/apply_security.rb

@@ -23,13 +23,37 @@ if !node['application_attributes']['skip_recipes'].include?('apply_security')
       include_recipe "mu-tools::aws_api"
       include_recipe "mu-tools::google_api"
   
+      if node['platform_version'].to_i < 6
+        package "policycoreutils"
+      elsif node['platform_version'].to_i < 8
+        package "policycoreutils-python"
+      else
+        package "xfsprogs"
+        package "xfsprogs-devel"
+        package "policycoreutils-python-utils"
+      end
   
-      %w{ policycoreutils-python authconfig ntp aide }.each do |pkg|
+      %w{ authconfig aide }.each do |pkg|
         package "apply_security package #{pkg}" do
           package_name pkg
         end
       end
 
+      if node['platform_version'].to_i < 8
+        package "ntp"
+        bash "NTP" do
+          user "root"
+          code <<-EOH
+    				chkconfig ntpd on
+  	  			ntpdate pool.ntp.org
+  		  		service ntpd start
+          EOH
+        end
+      else
+        package "chrony"
+        service "chronyd"
+      end
+
       execute "enable manual auditd restarts" do
         command "sed -i s/RefuseManualStop=yes/#RefuseManualStop=yes/ /usr/lib/systemd/system/auditd.service ; pkill auditd"
         ignore_failure true
@@ -60,14 +84,6 @@ if !node['application_attributes']['skip_recipes'].include?('apply_security')
         content "set -r autologout 15\n"
       end
   
-      bash "NTP" do
-        user "root"
-        code <<-EOH
-  				chkconfig ntpd on
-  				ntpdate pool.ntp.org
-  				service ntpd start
-        EOH
-      end
   
       #File integrity checking. Default configuration
       bash "AIDE" do

data/cookbooks/mu-tools/recipes/aws_api.rb

@@ -30,3 +30,7 @@ if platform_family?("rhel") or platform_family?("amazon")
     end
   end
 end
+
+package "nvme-cli" do
+  ignore_failure true
+end

data/cookbooks/mu-tools/recipes/google_api.rb

@@ -23,3 +23,7 @@
 		only_if { !get_google_metadata("instance/name").nil? }
   end
 }
+
+package "nvme-cli" do
+  ignore_failure true
+end

data/cookbooks/mu-tools/recipes/rsyslog.rb

@@ -33,7 +33,14 @@ if !node['application_attributes']['skip_recipes'].include?('rsyslog')
     if platform_family?("rhel") or platform_family?("amazon")
       $rsyslog_ssl_ca_path = "/etc/pki/Mu_CA.pem"
       if !platform?("amazon")
-        package node['platform_version'].to_i < 6 ? "policycoreutils" : "policycoreutils-python"
+        semanage_pkg = if node['platform_version'].to_i < 6
+          "policycoreutils"
+        elsif node['platform_version'].to_i < 8
+          "policycoreutils-python"
+        else
+          "policycoreutils-python-utils"
+        end
+        package semanage_pkg
         execute "allow rsyslog to meddle with port 10514" do
           command "/usr/sbin/semanage port -a -t syslogd_port_t -p tcp 10514"
           not_if "/usr/sbin/semanage port -l | grep '^syslog.*10514'"

data/cookbooks/mu-tools/resources/disk.rb

@@ -10,25 +10,43 @@ actions :create # ~FC092
 default_action :create
 
 action :create do
-  device = new_resource.device
+  devicepath = new_resource.device
   path = new_resource.mountpoint
-  devicename = device
+  devicename = devicepath.dup
 
   if set_gcp_cfg_params
     devicename= devicename.gsub(/.*?\//, "")
-    device = "/dev/disk/by-id/google-"+devicename
+    devicepath = "/dev/disk/by-id/google-"+devicename
   end
 
-  mu_tools_mommacat_request "create #{path}" do
+#  if devicename =~ /^\/dev\/(?:sd|xvd)([a-z])/
+#    if nvme?
+#      map = attached_nvme_disks
+#      if map[devicename]
+#        devicepath = map[devicename]
+#      end
+#    end
+#  end
+
+  mu_tools_mommacat_request "create #{devicepath} for #{path}" do
     request "add_volume"
     passparams(
       :dev => devicename,
       :size => new_resource.size,
       :delete_on_termination => new_resource.delete_on_termination
     )
-    not_if { ::File.exist?(device) }
+    not_if { ::File.exist?(real_devicepath(devicepath)) }
   end
 
+#  if nvme? and device.nil?
+#    map = attached_nvme_disks
+#    if map[devicename]
+#      devicepath = map[devicename]
+#    else
+#      Chef::Application.fatal!("In NVME mode and attempted to allocate disk #{devicename}, but didn't find it in metadata of any of our NVME block devices (#{map.values.join(", ")})")
+#    end
+#  end
+
   reboot "Rebooting after adding #{path}" do
     action :nothing
   end
@@ -38,7 +56,7 @@ action :create do
     action :nothing
   end
   mount "/mnt#{backupname}" do
-    device device
+    device real_devicepath(devicepath)
     options "nodev"
     action :nothing
     notifies :create, "directory[/mnt#{backupname}]", :before
@@ -51,10 +69,11 @@ action :create do
     action :nothing
   end
 
-  mkfs_cmd = node['platform_version'].to_i == 6 ? "mkfs.ext4 -F #{device}" : "mkfs.xfs -i size=512 #{device}"
-  guard_cmd = node['platform_version'].to_i == 6 ? "tune2fs -l #{device} > /dev/null" : "xfs_admin -l #{device} > /dev/null"
+#  mkfs_cmd = node['platform_version'].to_i == 6 ? "mkfs.ext4 -F #{real_devicepath(devicepath)}" : "mkfs.xfs -i size=512 #{real_devicepath(devicepath)}"
+#  guard_cmd = node['platform_version'].to_i == 6 ? "tune2fs -l #{real_devicepath(devicepath)} > /dev/null" : "xfs_admin -l #{real_devicepath(devicepath)} > /dev/null"
 
-  execute mkfs_cmd do
+  execute "format #{devicename}" do
+    command (node['platform_version'].to_i == 6 ? "mkfs.ext4 -F #{real_devicepath(devicepath)}" : "mkfs.xfs -i size=512 #{real_devicepath(devicepath)}")
     if new_resource.preserve_data
       notifies :mount, "mount[/mnt#{backupname}]", :immediately
       notifies :run, "execute[back up #{backupname}]", :immediately
@@ -63,11 +82,13 @@ action :create do
     if new_resource.reboot_after_create
       notifies :request_reboot, "reboot[Rebooting after adding #{path}]", :delayed
     end
-    not_if guard_cmd
+    retries 5 # sometimes there's a bit of lag
+    retry_delay 6
+    not_if (node['platform_version'].to_i == 6 ? "tune2fs -l #{real_devicepath(devicepath)} > /dev/null" : "xfs_admin -l #{real_devicepath(devicepath)} > /dev/null")
   end
 
   if !new_resource.reboot_after_create
-    directory "Ensure existence of #{path} for #{device}" do
+    directory "Ensure existence of #{path} for #{real_devicepath(devicepath)}" do
       recursive true
       path path
     end
@@ -78,7 +99,7 @@ action :create do
     end
 
     mount path do
-      device device
+      device real_devicepath(devicepath)
       options "nodev"
       action [:mount, :enable]
       notifies :run, "execute[/sbin/restorecon -R #{path}]", :immediately

data/cookbooks/mu-tools/resources/mommacat_request.rb

@@ -6,6 +6,5 @@ actions :run # ~FC092
 default_action :run
 
 action :run do
-  params = Base64.urlsafe_encode64(JSON.generate(new_resource.passparams))
-  mommacat_request(new_resource.request, params)
+  mommacat_request(new_resource.request, new_resource.passparams)
 end

data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb

@@ -0,0 +1,215 @@
+#       $OpenBSD: sshd_config,v 1.93 2014/01/10 05:59:19 djm Exp $
+
+# This is the sshd server system-wide configuration file.  See
+# sshd_config(5) for more information.
+
+# This sshd was compiled with PATH=/usr/local/bin:/usr/bin
+
+# The strategy used for options in the default sshd_config shipped with
+# OpenSSH is to specify options with their default value where
+# possible, but leave them commented.  Uncommented options override the
+# default value.
+
+# If you want to change the port on a SELinux system, you have to tell
+# SELinux about this change.
+# semanage port -a -t ssh_port_t -p tcp #PORTNUMBER
+#
+#Port 22
+#AddressFamily any
+#ListenAddress 0.0.0.0
+#ListenAddress ::
+
+# The default requires explicit activation of protocol 1
+#Protocol 2
+
+# HostKey for protocol version 1
+#HostKey /etc/ssh/ssh_host_key
+# HostKeys for protocol version 2
+HostKey /etc/ssh/ssh_host_rsa_key
+#HostKey /etc/ssh/ssh_host_dsa_key
+HostKey /etc/ssh/ssh_host_ecdsa_key
+HostKey /etc/ssh/ssh_host_ed25519_key
+
+# Lifetime and size of ephemeral version 1 server key
+#KeyRegenerationInterval 1h
+#ServerKeyBits 1024
+
+# Logging
+# obsoletes QuietMode and FascistLogging
+#SyslogFacility AUTH
+SyslogFacility AUTHPRIV
+LogLevel INFO
+
+# Authentication:
+
+#LoginGraceTime 2m
+#PermitRootLogin yes
+#StrictModes yes
+MaxAuthTries 4
+#MaxSessions 10
+
+#RSAAuthentication yes
+#PubkeyAuthentication yes
+
+# The default is to check both .ssh/authorized_keys and .ssh/authorized_keys2
+# but this is overridden so installations will only check .ssh/authorized_keys
+AuthorizedKeysFile .ssh/authorized_keys
+
+#AuthorizedPrincipalsFile none
+
+#AuthorizedKeysCommand none
+#AuthorizedKeysCommandUser nobody
+
+# For this to work you will also need host keys in /etc/ssh/ssh_known_hosts
+#RhostsRSAAuthentication no
+# similar for protocol version 2
+HostbasedAuthentication no
+# Change to yes if you don't trust ~/.ssh/known_hosts for
+# RhostsRSAAuthentication and HostbasedAuthentication
+#IgnoreUserKnownHosts no
+# Don't read the user's ~/.rhosts and ~/.shosts files
+IgnoreRhosts yes
+
+# To disable tunneled clear text passwords, change to no here!
+#PasswordAuthentication yes
+PermitEmptyPasswords no
+<% begin
+	if node['application_attributes']['sshd_allow_password_auth'] %>
+PasswordAuthentication yes
+<%
+  else %>
+    PasswordAuthentication no
+<%  end
+rescue NoMethodError %>
+PasswordAuthentication no
+<% end %>
+
+
+# Change to no to disable s/key passwords
+#ChallengeResponseAuthentication yes
+ChallengeResponseAuthentication no
+
+# Kerberos options
+#KerberosAuthentication no
+#KerberosOrLocalPasswd yes
+#KerberosTicketCleanup yes
+#KerberosGetAFSToken no
+#KerberosUseKuserok yes
+
+# GSSAPI options
+#GSSAPIAuthentication no
+GSSAPIAuthentication yes
+GSSAPICleanupCredentials yes
+#GSSAPIStrictAcceptorCheck yes
+#GSSAPIKeyExchange no
+#GSSAPIEnablek5users no
+
+# Set this to 'yes' to enable PAM authentication, account processing,
+# and session processing. If this is enabled, PAM authentication will
+# be allowed through the ChallengeResponseAuthentication and
+# PasswordAuthentication.  Depending on your PAM configuration,
+# PAM authentication via ChallengeResponseAuthentication may bypass
+# the setting of "PermitRootLogin without-password".
+# If you just want the PAM account and session checks to run without
+# PAM authentication, then enable this but set PasswordAuthentication
+# and ChallengeResponseAuthentication to 'no'.
+# WARNING: 'UsePAM no' is not supported in Red Hat Enterprise Linux and may cause several
+# problems.
+UsePAM yes
+
+# Accept locale-related environment variables
+AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
+AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
+AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
+AcceptEnv XMODIFIERS
+
+#AllowAgentForwarding yes
+#AllowTcpForwarding yes
+#GatewayPorts no
+X11Forwarding no
+#X11DisplayOffset 10
+#X11UseLocalhost yes
+#PermitTTY yes
+#PrintMotd yes
+#PrintLastLog yes
+#TCPKeepAlive yes
+#UseLogin no
+UsePrivilegeSeparation sandbox          # Default for new installations.
+#PermitUserEnvironment no
+#Compression delayed
+ClientAliveInterval 300
+ClientAliveCountMax 0
+#ShowPatchLevel no
+#UseDNS yes
+#PidFile /var/run/sshd.pid
+#MaxStartups 10:30:100
+#PermitTunnel no
+#ChrootDirectory none
+#VersionAddendum none
+
+# no default banner path
+#Banner none
+Banner /etc/issue.net
+
+# override default of no subsystems
+Subsystem sftp  /usr/libexec/openssh/sftp-server
+
+# Example of overriding settings on a per-user basis
+#Match User anoncvs
+#       X11Forwarding no
+#       AllowTcpForwarding no
+#       PermitTTY no
+#       ForceCommand cvs server
+PermitRootLogin without-password
+UseDNS no
+
+Ciphers aes128-ctr,aes192-ctr,aes256-ctr
+
+# If we've set AllowGroups, use that instead of restricting to centos
+<%
+begin
+	if !node['application_attributes']['sshd_allow_groups'].empty? %>
+AllowGroups <%= node['application_attributes']['sshd_allow_groups'] %> root
+<%
+	else
+	%>
+AllowUsers centos root
+<%
+	end
+rescue NoMethodError %>
+AllowUsers centos root
+<%
+end
+%>
+
+# Support SVN-only servers, while we're at it
+<%
+begin
+	if node['application_attributes']['svn_only_group'] %>
+Match Group <%= node['application_attributes']['svn_only_group'] %>
+	ForceCommand /usr/bin/svnserve -t
+<%
+	end
+rescue NoMethodError
+end
+%>
+
+# Support SFTP-only servers, while we're at it
+<%
+begin
+	if node['application_attributes']['sftp_only_group'] %>
+Match Group <%= node['application_attributes']['sftp_only_group'] %>
+	ForceCommand internal-sftp
+<%  begin
+			if node['application_attributes']['sftp_chroot'] %>
+	ChrootDirectory <%= node['application_attributes']['sftp_chroot'] %>
+<%
+			end
+		rescue NoMethodError %>
+	ChrootDirectory /home/
+<%
+		end
+	end
+rescue NoMethodError
+end
+%>