cloud-mu 3.5.0 → 3.6.3

data/cookbooks/mu-master/files/default/check_kibana.rb

@@ -0,0 +1,45 @@
+#!/usr/local/ruby-current/bin/ruby
+
+require 'optimist'
+require 'json'
+require 'net/http'
+
+OPTS = Optimist::options do
+  opt :host, "Kibana host to check", :required => true, :type => :string
+  opt :username, "Kibana username", :required => false, :type => :string
+  opt :password, "Kibana password", :required => false, :type => :string
+  opt :port, "Port to check for Kibana", :required => false, :default => 5601, :type => :integer
+  opt :basepath, "Path prefix for API requests", :required => false, :default => "", :type => :string
+end
+
+uri = "https://"+OPTS[:host]+":"+OPTS[:port].to_s+OPTS[:basepath]+"/api/status"
+req = Net::HTTP::Get.new(uri)
+if OPTS[:username] and OPTS[:password]
+  req.basic_auth OPTS[:username], OPTS[:password]
+end
+begin
+  Net::HTTP.start(OPTS[:host], OPTS[:port], :use_ssl => true) do |http|
+    resp = JSON.parse(http.request(req).body)
+    status = resp["status"]["overall"]
+    output = status["nickname"]+" since "+status["since"]
+    if resp["metrics"] and resp["metrics"] and resp["metrics"]["requests"]
+      output += "\n"+resp["metrics"]["requests"]["total"].to_s+" requests since "+resp["metrics"]["last_updated"]
+    end
+
+    puts output
+    if status["state"] == "green"
+      exit 0
+    elsif status["state"] == "yellow"
+      exit 1
+    else
+      exit 2
+    end
+  end
+rescue Net::HTTPServerException, SocketError, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNREFUSED => e
+  puts e.inspect
+  exit 2
+rescue StandardError => e
+  puts e.inspect
+  exit 3
+end
+

data/cookbooks/mu-master/libraries/mu.rb

@@ -26,4 +26,28 @@ elsif File.readable?("/opt/mu/lib/modules/mu-load-config.rb")
   require "/opt/mu/lib/modules/mu-load-config.rb"
 end
 
+# for some reason aaws-sigv4 isn't getting picked up by Mu's requires
+require "aws-sigv4"
 require "mu"
+
+def baskets
+  baskets = {}
+  if Dir.exists?("/opt/mu/var/deployments")
+    Dir.glob("/opt/mu/var/deployments/*/basket_of_kittens.json").each { |basket_json|
+      basket_json =~ /^\/opt\/mu\/var\/deployments\/([^\/]+)\/basket_of_kittens.json$/
+      baskets[Regexp.last_match[1]] = JSON.parse(File.read(basket_json))
+    }
+  end
+  baskets
+end
+
+def deployments
+  deploys = {}
+  if Dir.exists?("/opt/mu/var/deployments")
+    Dir.glob("/opt/mu/var/deployments/*/deployment.json").each { |dep_json|
+      dep_json =~ /^\/opt\/mu\/var\/deployments\/([^\/]+)\/deployment.json$/
+      deploys[Regexp.last_match[1]] = JSON.parse(File.read(dep_json))
+    }
+  end
+  deploys
+end

data/cookbooks/mu-master/metadata.rb

@@ -7,13 +7,13 @@ long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
 source_url 'https://github.com/cloudamatic/mu'
 issues_url 'https://github.com/cloudamatic/mu/issues'
 chef_version '>= 12.1' if respond_to?(:chef_version)
-version '0.9.7'
+version '0.9.9'
 
 %w( centos amazon redhat ).each do |os|
 	supports os
 end
 
-depends 'mu-nagios'
+depends 'nagios'
 depends 'nrpe', '~> 2.0.3'
 depends 'mu-utility'
 depends 'mu-tools'
@@ -23,9 +23,9 @@ depends 'postfix', '~> 5.3.1'
 depends 'bind', '~> 2.2.0'
 depends 'bind9-ng', '~> 0.1.0'
 depends 'mu-firewall'
-depends 'vault-cluster', '~> 2.1.0'
-depends 'consul-cluster', '~> 2.0.0'
+#depends 'vault-cluster', '~> 2.1.0'
+#depends 'consul-cluster', '~> 2.0.0'
 depends 'chef-sugar' # undeclared dependency of consul 2.1, which can't be upgraded without creating a conflict with consul-cluster and vault-cluster -zr2d2
 depends 'hostsfile', '~> 3.0.1'
 depends 'chef-vault', '~> 3.1.1'
-depends 'apache2', '< 8.0.0'
+depends 'apache2', '~> 9.0.3'

data/cookbooks/mu-master/recipes/default.rb

@@ -160,32 +160,33 @@ if !node['update_nagios_only']
   end
 end
 
-include_recipe "mu-master::update_nagios_only"
+include_recipe "mu-master::update_nagios_only" if !$MU_CFG['disable_nagios']
 
 if !node['update_nagios_only']
-
-  package %w(nagios-plugins-breeze nagios-plugins-by_ssh nagios-plugins-cluster nagios-plugins-dhcp nagios-plugins-dig nagios-plugins-disk nagios-plugins-disk_smb nagios-plugins-dns nagios-plugins-dummy nagios-plugins-file_age nagios-plugins-flexlm nagios-plugins-fping nagios-plugins-game nagios-plugins-hpjd nagios-plugins-http nagios-plugins-icmp nagios-plugins-ide_smart nagios-plugins-ircd nagios-plugins-ldap nagios-plugins-load nagios-plugins-log nagios-plugins-mailq nagios-plugins-mrtg nagios-plugins-mrtgtraf nagios-plugins-nagios nagios-plugins-nt nagios-plugins-ntp nagios-plugins-ntp-perl nagios-plugins-nwstat nagios-plugins-oracle nagios-plugins-overcr nagios-plugins-pgsql nagios-plugins-ping nagios-plugins-procs nagios-plugins-real nagios-plugins-rpc nagios-plugins-sensors nagios-plugins-smtp nagios-plugins-snmp nagios-plugins-ssh nagios-plugins-swap nagios-plugins-tcp nagios-plugins-time nagios-plugins-ups nagios-plugins-users nagios-plugins-wave) do
-    action :install
-  end
-
-  package %w(nagios-plugins-mysql) do
+  if !$MU_CFG['disable_nagios']
+    package %w(nagios-plugins-breeze nagios-plugins-by_ssh nagios-plugins-cluster nagios-plugins-dhcp nagios-plugins-dig nagios-plugins-disk nagios-plugins-disk_smb nagios-plugins-dns nagios-plugins-dummy nagios-plugins-file_age nagios-plugins-flexlm nagios-plugins-fping nagios-plugins-game nagios-plugins-hpjd nagios-plugins-http nagios-plugins-icmp nagios-plugins-ide_smart nagios-plugins-ircd nagios-plugins-ldap nagios-plugins-load nagios-plugins-log nagios-plugins-mailq nagios-plugins-mrtg nagios-plugins-mrtgtraf nagios-plugins-nagios nagios-plugins-nt nagios-plugins-ntp nagios-plugins-ntp-perl nagios-plugins-nwstat nagios-plugins-oracle nagios-plugins-overcr nagios-plugins-pgsql nagios-plugins-ping nagios-plugins-procs nagios-plugins-real nagios-plugins-rpc nagios-plugins-sensors nagios-plugins-smtp nagios-plugins-snmp nagios-plugins-ssh nagios-plugins-swap nagios-plugins-tcp nagios-plugins-time nagios-plugins-ups nagios-plugins-users nagios-plugins-wave) do
       action :install
-      not_if { node['platform'] == 'amazon' }
-  end
+    end
 
-  directory "/home/nagios" do
-    owner "nagios"
-    mode 0711
-  end
+    package %w(nagios-plugins-mysql) do
+        action :install
+        not_if { node['platform'] == 'amazon' }
+    end
 
-  directory "/home/nagios/.ssh" do
-    owner "nagios"
-    mode 0711
-  end
+    directory "/home/nagios" do
+      owner "nagios"
+      mode 0711
+    end
 
-  file "/home/nagios/.ssh/config" do
-    owner "nagios"
-    mode 0600
+    directory "/home/nagios/.ssh" do
+      owner "nagios"
+      mode 0711
+    end
+
+    file "/home/nagios/.ssh/config" do
+      owner "nagios"
+      mode 0600
+    end
   end
 
   execute "dhclient-script" do
@@ -220,6 +221,7 @@ if !node['update_nagios_only']
   apache2_install "" do
     docroot_dir "/var/www/html"
     modules %w{status alias auth_basic authn_core authn_file authz_core authz_groupfile authz_host authz_user autoindex deflate dir env mime negotiation setenvif log_config logio unixd systemd headers proxy proxy_http rewrite ssl ldap authnz_ldap slotmem_shm}
+    listen [80, 443, 8443]
   end
   package "mod_ldap"
 
@@ -229,7 +231,12 @@ if !node['update_nagios_only']
   apache2_mod_cgid ""
   apache2_mod_ssl ""
 
+  link "/usr/lib64/httpd/modules/mod_php5.so" do
+    to "/usr/lib64/httpd/modules/libphp5.so"
+  end
   apache2_mod "php"
+  apache2_module "php5" 
+  apache2_module "cgi" 
   apache2_default_site "" do
     action :enable
     notifies :start, "service[apache2]", :delayed
@@ -414,6 +421,10 @@ if !node['update_nagios_only']
   "
   end
 
+  execute "chcon -R -h -t var_log_t /Mu_Logs" do
+    not_if "ls -aZ /Mu_Logs | grep ':var_log_t'"
+  end
+
 # XXX this will catch the occasional 4am groom. Need a way to graceful-restart momma.
   file "/etc/logrotate.d/Mu_momma_cat" do
     content "/var/log/mu-momma-cat.log

data/cookbooks/mu-master/recipes/firewall-holes.rb

@@ -23,6 +23,11 @@ firewall_rule "MU Master default ports" do
   port [MU.mommaCatPort, 7443, 8443, 9443, 10514, 443, 80, 25]
 end
 
+firewall_rule "Logstash port" do
+  port [5044]
+  source "10.0.0.0/8"
+end
+
 local_chef_ports = [4321, 9463, 9583, 16379, 8983, 8000, 9680, 9683, 9090, 5432]
 firewall_rule "Chef Server ports on 127.0.0.1" do
   port local_chef_ports

data/cookbooks/mu-master/recipes/init.rb

@@ -22,6 +22,8 @@
 # references to other cookbooks, no include_recipes, no cookbook_files, no
 # templates.
 
+chef_gem "aws-sigv4"
+
 require 'etc'
 require 'open-uri'
 require 'socket'
@@ -35,8 +37,8 @@ ENV['PATH'] = ENV['PATH']+":/bin:/opt/opscode/embedded/bin"
 
 # XXX We want to be able to override these things when invoked from chef-apply,
 # but, like, how?
-CHEF_SERVER_VERSION="14.0.65-1"
-CHEF_CLIENT_VERSION="16.9.29"
+CHEF_SERVER_VERSION="14.11.31-1"
+CHEF_CLIENT_VERSION="18.5.0"
 KNIFE_WINDOWS="1.9.0"
 MU_BASE="/opt/mu"
 
@@ -227,7 +229,13 @@ when 'amazon'
   when '2'
     basepackages.concat(['libX11', 'mariadb-devel', 'cryptsetup', 'ncurses-devel', 'ncurses-compat-libs', 'iptables-services'])
     removepackages = ['nagios', 'firewalld']
-    elversion = '7' #HACK TO FORCE AMAZON LINUX 2 TO BE TREATED LIKE RHEL 7
+    elversion = '7'
+
+  when '2023'
+    basepackages.concat(['libX11', 'mariadb105-devel', 'cryptsetup', 'ncurses-devel', 'ncurses-compat-libs', 'iptables-services', 'libxcrypt-compat', 'ruby'])
+    basepackages.delete('curl')
+    removepackages = ['nagios', 'firewalld']
+    elversion = '7'
 
   else
     raise "Mu Masters on Amazon-family hosts must be equivalent to Amazon Linux 1 or 2 (got #{node['platform_version'].split('.')[0]})"
@@ -237,18 +245,27 @@ else
 end
 
 rpms = {
-  "epel-release" => "http://dl.fedoraproject.org/pub/epel/epel-release-latest-#{elversion}.noarch.rpm",
   "chef-server-core" => "https://packages.chef.io/files/stable/chef-server/#{CHEF_SERVER_VERSION.sub(/\-\d+$/, "")}/el/#{elversion}/chef-server-core-#{CHEF_SERVER_VERSION}.el#{elversion}.x86_64.rpm"
 }
 
-rpms["ruby27"] = "https://s3.amazonaws.com/cloudamatic/muby-2.7.2-1.el#{elversion}.x86_64.rpm"
-if elversion.to_i == 6
-  rpms["openssl"] = "https://s3.amazonaws.com/cloudamatic/mussl-1.1.1h-1.el6.x86_64.rpm"
-  rpms["sqlite"] = "https://s3.amazonaws.com/cloudamatic/muqlite-3.33-1.el6.x86_64.rpm"
+unless node['platform_family'] == "amazon"
+  rpms["epel-release"] = "http://dl.fedoraproject.org/pub/epel/epel-release-latest-#{elversion}.noarch.rpm"
 end
-if elversion.to_i == 7
-  rpms["mugit"] = "https://s3.amazonaws.com/cloudamatic/mugit-2.30.0-1.el7.x86_64.rpm"
+
+shorthand = "el"
+shorthand = "amzn" if node['platform_family'] == "amazon"
+
+rpms["ruby33"] = "https://s3.amazonaws.com/icras-ruby/muby-3.3.5-1.#{shorthand}#{node['platform_version'].split('.')[0]}.x86_64.rpm"
+unless node['platform_family'] == "amazon"
+  if elversion.to_i == 6
+    rpms["openssl"] = "https://s3.amazonaws.com/cloudamatic/mussl-1.1.1h-1.el6.x86_64.rpm"
+    rpms["sqlite"] = "https://s3.amazonaws.com/cloudamatic/muqlite-3.33-1.el6.x86_64.rpm"
+  end
+  if elversion.to_i == 7
+    rpms["mugit"] = "https://s3.amazonaws.com/cloudamatic/mugit-2.30.0-1.el7.x86_64.rpm"
+  end
 end
+
 # this takes up a huge amount of space, save it until we're fully operational
 if !RUNNING_STANDALONE
   rpms["python38"] = "https://s3.amazonaws.com/cloudamatic/muthon-3.8.3-1.el#{elversion}.x86_64.rpm"
@@ -354,7 +371,7 @@ end
 rpms.each_pair { |pkg, src|
   rpm_package pkg do
     source src
-    if pkg == "ruby27" 
+    if pkg == "ruby33" 
       options '--prefix=/opt/rubies/'
     end
     if pkg == "epel-release" 
@@ -376,6 +393,21 @@ rpms.each_pair { |pkg, src|
   end
 }
 
+execute "/sbin/ldconfig" do
+  action :nothing
+end
+
+["/opt/rubies/ruby-3.3.5/lib/pkgconfig/ruby-3.3.pc", "/opt/rubies/ruby-3.3.5/lib/ruby/3.3.0/x86_64-linux/rbconfig.rb"].each { |f|
+  execute "scrub rpmbuild cruft from #{f}" do
+    command "sed -i 's/-Wl,-dT,\\/root\\/rpmbuild\\/BUILD\\/.package_note-muby-3.3.5-1.amzn2023.x86_64.ld//' #{f}"
+  end
+}
+
+file "/etc/ld.so.conf.d/muby.conf" do
+  content "/usr/local/ruby-current/lib\n"
+  notifies :run, "execute[/sbin/ldconfig]", :immediately
+end
+
 package ["jq"] do
   ignore_failure true # sometimes we can't see EPEL immediately
 end
@@ -473,7 +505,7 @@ gemfile_dir = if RUNNING_STANDALONE and !File.readlines("/etc/mtab").grep(/\s\/o
         end
       }
       dmiout = shell_out!(%Q{PATH=/sbin:/usr/sbin:/bin:/usr/bin dmidecode})
-      if dmiout.match(/Google/)
+      if dmiout.stdout.match(/Google/)
         exclude_gems.delete("google-api-client")
       end
 
@@ -569,14 +601,14 @@ end
 
 # Get a 'mu' Chef org in place and populate it with artifacts
 directory "/root/.chef"
-execute "knife ssl fetch" do
+execute "env -i HOME=/root:PATH=/opt/chef/embedded/bin:/bin:/usr/bin /opt/chef/embedded/bin/knife ssl fetch" do
   action :nothing
 end
 execute "initial Chef artifact upload" do
   command "MU_INSTALLDIR=#{MU_BASE} MU_LIBDIR=#{MU_BASE}/lib MU_DATADIR=#{MU_BASE}/var #{MU_BASE}/lib/bin/mu-upload-chef-artifacts"
   action :nothing
   notifies :stop, "service[iptables]", :before
-  notifies :run, "execute[knife ssl fetch]", :before
+  notifies :run, "execute[env -i knife ssl fetch]", :before
   if !RUNNING_STANDALONE
     notifies :start, "service[iptables]", :immediately
   end
@@ -587,11 +619,13 @@ chef_gem "simple-password-gen" do
 end
 require "simple-password-gen"
 
-# XXX this would make an awesome library
 execute "create mu Chef user" do
   command "/opt/opscode/bin/chef-server-ctl user-create mu Mu Master root@example.com #{Password.pronounceable} -f #{MU_BASE}/var/users/mu/mu.user.key"
   umask "0277"
   not_if "/opt/opscode/bin/chef-server-ctl user-list | grep '^mu$'"
+  if !File.exist?("/etc/opscode/pivotal.rb")
+    notifies :run, "execute[reconfigure Chef server]", :immediately
+  end
   notifies :start, "service[chef-server]", :before
 end
 execute "create mu Chef org" do
@@ -679,12 +713,12 @@ knife_cfg = "-c /root/.chef/knife.rb"
 execute "create MU-MASTER Chef client" do
 # XXX I dislike --ssh-verify-host-key=never intensely, but the CLI-documented 'accept_new' doesn't actually work
   if SSH_USER == "root"
-    command "/opt/chef/bin/knife bootstrap #{knife_cfg} -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -U #{SSH_USER} --ssh-identity-file=/root/.ssh/id_rsa --ssh-verify-host-key=never 127.0.0.1"
+    command "PATH='/opt/chef/bin:/opt/chef/embedded/bin' knife bootstrap #{knife_cfg} -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -U #{SSH_USER} --ssh-identity-file=/root/.ssh/id_rsa --ssh-verify-host-key=never 127.0.0.1"
   else
-    command "/opt/chef/bin/knife bootstrap #{knife_cfg} -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -U #{SSH_USER} --ssh-identity-file=/root/.ssh/id_rsa --ssh-verify-host-key=never --sudo 127.0.0.1"
+    command "PATH='/opt/chef/bin:/opt/chef/embedded/bin' knife bootstrap #{knife_cfg} -N MU-MASTER --no-node-verify-api-cert --node-ssl-verify-mode=none -U #{SSH_USER} --ssh-identity-file=/root/.ssh/id_rsa --ssh-verify-host-key=never --sudo 127.0.0.1"
   end
-  only_if "/opt/chef/bin/knife node #{knife_cfg} list" # don't do crazy stuff just because knife isn't working
-  not_if "/opt/chef/bin/knife node #{knife_cfg} list | grep '^MU-MASTER$'"
+  only_if "PATH='/opt/chef/bin:/opt/chef/embedded/bin' knife node #{knife_cfg} list" # don't do crazy stuff just because knife isn't working
+  not_if "PATH='/opt/chef/bin:/opt/chef/embedded/bin' knife node #{knife_cfg} list | grep '^MU-MASTER$'"
   notifies :run, "execute[add localhost key to authorized_keys]", :before
   notifies :delete, "file[/etc/chef/client.rb]", :before
   notifies :delete, "file[/etc/chef/client.pem]", :before
@@ -719,6 +753,11 @@ bash "fix misc permissions" do
   EOH
 end
 
+# https://github.com/chef/chef-server/issues/3109#issuecomment-1022084825
+execute "ensure Chef indexes aren't read-only" do
+  command %Q{curl -XPUT -H "Content-Type: application/json" http://127.0.0.1:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}'}
+end
+
 directory TMPDIR do
   action :delete
   recursive true