cloud-mu 3.5.0 → 3.6.3

data/cookbooks/mu-tools/recipes/nrpe.rb

@@ -45,7 +45,7 @@ if !node['application_attributes']['skip_recipes'].include?('nrpe')
   
     case elversion
     when 7
-      %w{nrpe_file.pp nrpe_file.te nrpe_check_disk.te nrpe_check_disk.pp}.each { |f|
+      %w{nrpe_file.pp nrpe_file.te nrpe_check_disk.te nrpe_check_disk.pp nrpe_conf_d.pp}.each { |f|
         cookbook_file "#{Chef::Config[:file_cache_path]}/#{f}" do
           source f
         end
@@ -64,6 +64,23 @@ if !node['application_attributes']['skip_recipes'].include?('nrpe')
         not_if "/usr/sbin/semodule -l | grep nrpe_check_disk"
         notifies :restart, "service[nrpe]", :delayed
       end
+  
+      execute "Allow NRPE to read /etc/nagios/nrpe.d through SELinux" do
+        command "/usr/sbin/semodule -i nrpe_conf_d.pp"
+        cwd Chef::Config[:file_cache_path]
+        not_if "/usr/sbin/semodule -l | grep nrpe_conf_d"
+        notifies :restart, "service[nrpe]", :delayed
+      end
+
+      if node['platform'] == "centos"
+        package "nagios-plugins-check-updates"
+        nrpe_check "check_updates" do
+          command "#{node['nrpe']['plugin_dir']}/check_updates --security-only"
+          action :add
+          notifies :run, 'execute[selinux permissions]', :immediately if node['platform'] != 'amazon'
+          notifies :restart, "service[nrpe]", :delayed
+        end
+      end
     when 6
       if node['platform'] != 'amazon'
         cookbook_file "nrpe_disk.pp" do
@@ -96,7 +113,7 @@ if !node['application_attributes']['skip_recipes'].include?('nrpe')
       notifies :run, 'execute[selinux permissions]', :immediately if node['platform'] != 'amazon'
       notifies :restart, "service[nrpe]", :delayed
     end
-  
+
     # execute "chmod o+r /etc/nagios/nrpe.d/check_disk.cfg"
     # file "/etc/nagios/nrpe.d/check_disk.cfg" do
       # mode 0640
@@ -113,6 +130,7 @@ if !node['application_attributes']['skip_recipes'].include?('nrpe')
       end
     end
 
+    execute "restorecon -Rv /etc/nagios/nrpe.d"
     service "nrpe" do
       action [:enable, :start]
     end

data/cookbooks/mu-tools/recipes/rsyslog.rb

@@ -20,7 +20,11 @@ if !node['application_attributes']['skip_recipes'].include?('rsyslog')
   case node['platform_family']
   when "rhel", "debian", "amazon"
     package "rsyslog"
-    package "rsyslog-gnutls"
+    if platform_family?("amazon") and node['platform_version'].to_i == 2023
+      package "rsyslog-crypto"
+    else
+      package "rsyslog-gnutls"
+    end
     execute "chcon -R -h -t var_log_t /Mu_Logs" do
       action :nothing
       only_if { ::Dir.exist?("/Mu_Logs") }
@@ -71,3 +75,10 @@ if !node['application_attributes']['skip_recipes'].include?('rsyslog')
     end
   end
 end
+
+cookbook_file "/etc/logrotate.conf" do
+  source "logrotate.conf"
+  mode 0644
+  owner "root"
+  group "root"
+end

data/cookbooks/mu-tools/recipes/set_local_fw.rb

@@ -19,7 +19,7 @@
 if !node['application_attributes']['skip_recipes'].include?('set_local_fw')
   master_ips = get_mu_master_ips
   case node['platform_family']
-  when 'rhel', 'amazon'
+  when 'rhel'#, 'amazon'
     include_recipe 'mu-firewall'
 
     if elversion >= 7 and node['platform_family'] != "amazon" # Can use firewalld, but not if iptables is already rigged

data/data_bags/nagios_services/apm_backend_connect.json

@@ -0,0 +1,5 @@
+{
+  "id": "apm_backend_connect",
+  "hostgroup_name": "mu-logstash",
+  "command_line": "$USER1$/check_nrpe -H $HOSTADDRESS$ -c check_apm"
+}

data/data_bags/nagios_services/apm_listen.json

@@ -0,0 +1,5 @@
+{
+  "id": "apm_listen",
+  "hostgroup_name": "mu-logstash",
+  "command_line": "$USER1$/check_http $HOSTADDRESS$ -p 8200"
+}

data/data_bags/nagios_services/elastic_shards.json

@@ -0,0 +1,5 @@
+{
+  "id": "elastic_shards",
+  "hostgroup_name": "mu-elastic",
+  "command_line": "$USER1$/check_nrpe -H $HOSTADDRESS$ -c check_elastic_shards"
+}

data/data_bags/nagios_services/logstash.json

@@ -0,0 +1,5 @@
+{
+  "id": "logstash",
+  "hostgroup_name": "mu-logstash",
+  "command_line": "$USER1$/check_nrpe -H $HOSTADDRESS$ -c check_logstash"
+}

data/data_bags/nagios_services/rhel7_updates.json

@@ -0,0 +1,8 @@
+{
+	"id": "rhel7_updates",
+  "hostgroup_name": "mu-node",
+	"command_line": "$USER1$/check_nrpe -H $HOSTADDRESS$ -t 60 -c check_updates",
+  "check_interval": 120,
+  "retry_interval": 3600,
+  "notification_interval": 14400
+}

data/extras/image-generators/AWS/centos7.yaml

@@ -1,5 +1,6 @@
 ---
 appname: mu
+us_only: true
 servers: 
 - name: centos7
   platform: centos7

data/extras/image-generators/AWS/rhel7.yaml

@@ -0,0 +1,21 @@
+---
+appname: mu
+us_only: true
+servers: 
+- name: rhel7
+  platform: rhel7
+  size: m4.large
+  vpc:
+    name: r7vpc
+  scrub_groomer: true
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]
+  create_image:
+    image_then_destroy: true
+    public: true
+    copy_to_regions:
+    - "#ALL"
+vpcs:
+- name: r7vpc

data/extras/image-generators/AWS/win2k12r2.yaml

@@ -1,5 +1,6 @@
 ---
   appname: mu
+  us_only: true
   vpcs:
   - name: windowsbuild
   servers: 

data/extras/image-generators/AWS/win2k16.yaml

@@ -1,5 +1,6 @@
 ---
   appname: mu
+  us_only: true
   vpcs:
   - name: windowsbuild
   servers: 

data/extras/image-generators/AWS/win2k19.yaml

@@ -1,5 +1,6 @@
 ---
   appname: mu
+  us_only: true
   vpcs:
   - name: windowsbuild
   servers: 

data/extras/ruby_rpm/muby.spec

@@ -1,16 +1,18 @@
 Summary: Ruby for Mu(by)
 BuildArch: x86_64
 Name: muby
-Version: 2.7.2
+Version: 3.3.5
 Release: 1%{dist}
 Group: Development/Languages
 License: Ruby License/GPL - see COPYING
 URL: http://www.ruby-lang.org/
 Prefix: /opt/rubies
-Source: https://cache.ruby-lang.org/pub/ruby/2.7/ruby-%{version}.tar.gz
+Source: https://cache.ruby-lang.org/pub/ruby/3.3/ruby-%{version}.tar.gz
 
 BuildRequires: zlib
 BuildRequires: zlib-devel
+BuildRequires: libyaml
+BuildRequires: libyaml-devel
 BuildRequires: openssl
 
 %description
@@ -19,21 +21,22 @@ I was drunk when I wrote this spec file
 %prep
 rm -rf $RPM_BUILD_DIR/ruby-%{version}
 rm -rf %{prefix}
-test -f $RPM_SOURCE_DIR/ruby-%{version}.tar.gz || ( cd $RPM_SOURCE_DIR && curl -O https://cache.ruby-lang.org/pub/ruby/2.7/ruby-%{version}.tar.gz )
+test -f $RPM_SOURCE_DIR/ruby-%{version}.tar.gz || ( cd $RPM_SOURCE_DIR && curl -O https://cache.ruby-lang.org/pub/ruby/3.3/ruby-%{version}.tar.gz )
 tar -xzvf $RPM_SOURCE_DIR/ruby-%{version}.tar.gz
 mkdir -p $RPM_BUILD_ROOT%{prefix}
 ln -s %{prefix}/ruby-%{version} $RPM_BUILD_ROOT%{prefix}/ruby-%{version}
     
 %build
 cd $RPM_BUILD_DIR/ruby-%{version}
-./configure --prefix=%{prefix}/ruby-%{version}  --enable-load-relative --enable-shared
+./configure --prefix=%{prefix}/ruby-%{version}  --enable-load-relative --enable-shared --disable-install-doc
 make
 
 %install
 cd $RPM_BUILD_DIR/ruby-%{version}
 make install
 mkdir -p %{prefix}
-yes | %{prefix}/ruby-%{version}/bin/gem install bundler --version '~> 2.1.4' --force
+yes | %{prefix}/ruby-%{version}/bin/gem install bundler --version '~> 2.5.18' --force
+yes | %{prefix}/ruby-%{version}/bin/gem install psych --force
 mkdir -p $RPM_BUILD_ROOT%{prefix}
 mv %{prefix}/ruby-%{version} $RPM_BUILD_ROOT%{prefix}/
 mkdir -p $RPM_BUILD_ROOT/usr/local/bin

data/extras/vault_tools/export_vaults.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 # Exports existing vaults to a vaults directory for use by test_vaults and recreate_vaults
-mkdir -p ~/vaults ; for i in `knife data bag list | grep -v -- -`;do echo $i; mkdir -p vaults/$i ; for j in `knife data bag show $i | grep -v '_keys$'`;do echo "   $j"; knife vault show $i $j -F json > vaults/$i/$j.json;done;done ; find vaults -empty -delete
+mkdir -p ~/vaults ; for i in `knife data bag list | grep -vE -- '-[0-9]{10}-'`;do echo $i; mkdir -p vaults/$i ; for j in `knife data bag show $i | grep -v '_keys$'`;do echo "   $j"; knife vault show $i $j -F json > vaults/$i/$j.json;done;done ; find vaults -empty -delete

data/install/deprecated-bash-library.sh

@@ -1299,7 +1299,7 @@ install_chef()
     status_message "Installing current Chef client"
     yum -y erase chef || rpm -e chef # one of these will get it
     rm -rf /opt/chef # and stay out
-    curl https://www.chef.io/chef/install.sh > /root/chef-install.sh
+    curl https://omnitruck.chef.io/install.sh > /root/chef-install.sh
     sh /root/chef-install.sh -v $CHEF_CLIENT_VERSION
   fi
   if [ -f /opt/chef/embedded/bin/gem ];then

data/install/installer

@@ -1,8 +1,10 @@
 #!/bin/sh
 
+
+yum -y install git dmidecode
 BOLD=`tput bold`
 NORM=`tput sgr0`
-CHEF_CLIENT_VERSION="16.9.29"
+CHEF_CLIENT_VERSION="18.5.0"
 if [ "$MU_BRANCH" == "" ];then
   MU_BRANCH="master"
   mydir="`dirname $0`"
@@ -45,7 +47,7 @@ if ! /bin/rpm -q $CHEF_CLIENT_PKG > /dev/null ;then
   # Drop any old Chef packages laying around, first
   /usr/bin/yum -y erase chef || /bin/rpm -e chef
   /bin/rm -rf /opt/chef ~/.berkshelf ~/.chef /etc/chef
-  /usr/bin/curl https://www.chef.io/chef/install.sh > /root/chef-install.sh
+  /usr/bin/curl https://omnitruck.chef.io/install.sh > /root/chef-install.sh
   /bin/sh /root/chef-install.sh -v $CHEF_CLIENT_VERSION
 fi
 

data/modules/mommacat.ru

@@ -417,12 +417,14 @@ app = proc do |env|
 # XXX make sure we handle mangled input safely
           params = JSON.parse(Base64.decode64(req["add_volume"]))
           MU.log "add_volume request", MU::NOTICE, details: params
-          instance.addVolume(params["dev"], params["size"], delete_on_termination: params["delete_on_termination"])
+          Thread.current.thread_variable_set("addVolume", req["mu_instance_id"])
+          instance.addVolume(dev: params["dev"], size: params["size"], delete_on_termination: params["delete_on_termination"])
         else
           returnval = throw500 "I don't know how to add a volume for #{instance}"
         end
       elsif !instance.nil?
         if !req["mu_bootstrap"].nil?
+          Thread.current.thread_variable_set("groomRequest", req["mu_instance_id"])
           kittenpile.groomNode(req["mu_instance_id"], req["mu_resource_name"], req["mu_resource_type"], mu_name: mu_name, sync_wait: true)
           returnval[2] = ["Grooming asynchronously, check Momma Cat logs on the master for details."]
         else

data/modules/mu/adoption.rb

@@ -285,7 +285,7 @@ module MU
               rescue StandardError
               ensure
                 if !obj_desc
-                  MU.log cloud+" "+type.to_s+" "+obj_thr.cloud_id+" did not return a cloud descriptor, skipping", MU::WARN
+                  MU.log cloud+" "+type.to_s+" "+obj_thr.cloud_id+" #{cloud == "Google" ? "in org #{MU::Cloud::Google.getOrg(obj_thr.credentials).display_name} ": ""}did not return a cloud descriptor, skipping", MU::WARN
                   next
                 end
               end

data/modules/mu/cloud/dnszone.rb

@@ -22,9 +22,9 @@ module MU
 
       # Set a generic .platform-mu DNS entry for a resource, and return the name
       # that was set.
-      def self.genericMuDNSEntry(*flags)
+      def self.genericMuDNSEntry(**flags)
 # XXX have this switch on a global config for where Mu puts its DNS
-        MU::Cloud.resourceClass(MU::Config.defaultCloud, "DNSZone").genericMuDNSEntry(flags.first)
+        MU::Cloud.resourceClass(MU::Config.defaultCloud, "DNSZone").genericMuDNSEntry(**flags)
       end
 
       # Wrapper for {MU::Cloud::AWS::DNSZone.manageRecord}. Spawns threads to create all

data/modules/mu/cloud/machine_images.rb

@@ -27,14 +27,14 @@ module MU
     # Aliases for platform names, in case we don't have actual images built for
     # them.
     PLATFORM_ALIASES = {
-      "linux" => "centos7",
+      "linux" => "amazon2023",
       "windows" => "win2k12r2",
       "win2k12" => "win2k12r2",
       "ubuntu" => "ubuntu16",
       "centos" => "centos7",
       "rhel7" => "rhel71",
       "rhel" => "rhel71",
-      "amazon" => "amazon2016"
+      "amazon" => "amazon2023"
     }
 
     @@image_fetch_cache = {}
@@ -87,28 +87,29 @@ module MU
       end
       
       images = nil
-      urls.each { |base_url|
-        @@image_fetch_semaphore.synchronize {
-          if @@image_fetch_cache[cloud] and (Time.now - @@image_fetch_cache[cloud]['time']) < 30
-            images = @@image_fetch_cache[cloud]['contents'].dup
-          else
-            begin
-              Timeout.timeout(2) do
-                response = URI.open("#{base_url}/#{cloud}.yaml").read
-                images ||= {}
-                images.deep_merge!(YAML.load(response))
-                break
-              end
-            rescue StandardError => e
-              if fail_hard
-                raise MuError, "Failed to fetch stock images from #{base_url}/#{cloud}.yaml (#{e.message})"
-              else
-                MU.log "Failed to fetch stock images from #{base_url}/#{cloud}.yaml (#{e.message})", MU::WARN if !quiet
-              end
-            end
-          end
-        }
-      }
+# XXX no ability to update this cache anymore, and it's pointless now anyway
+#      urls.each { |base_url|
+#        @@image_fetch_semaphore.synchronize {
+#          if @@image_fetch_cache[cloud] and (Time.now - @@image_fetch_cache[cloud]['time']) < 30
+#            images = @@image_fetch_cache[cloud]['contents'].dup
+#          else
+#            begin
+#              Timeout.timeout(2) do
+#                response = URI.open("#{base_url}/#{cloud}.yaml").read
+#                images ||= {}
+#                images.deep_merge!(YAML.load(response))
+#                break
+#              end
+#            rescue StandardError => e
+#              if fail_hard
+#                raise MuError, "Failed to fetch stock images from #{base_url}/#{cloud}.yaml (#{e.message})"
+#              else
+#                MU.log "Failed to fetch stock images from #{base_url}/#{cloud}.yaml (#{e.message})", MU::WARN if !quiet
+#              end
+#            end
+#          end
+#        }
+#      }
 
       @@image_fetch_semaphore.synchronize {
         @@image_fetch_cache[cloud] = {
@@ -127,7 +128,7 @@ module MU
         [backwards_compat[cloud], cloud].each { |file|
           next if file.nil?
           if File.exist?("#{MU.myRoot}/modules/mu/defaults/#{file}.yaml")
-            images = YAML.load(File.read("#{MU.myRoot}/modules/mu/defaults/#{file}.yaml"))
+            images = YAML.load(File.read("#{MU.myRoot}/modules/mu/defaults/#{file}.yaml"), aliases: true)
             break
           end
         }