cloud-mu 3.3.0 → 3.5.1

data/cookbooks/mu-master/recipes/update_nagios_only.rb

@@ -139,7 +139,7 @@ Dir.glob("/usr/lib/cgi-bin/*.cgi").each { |script|
 
 ["/usr/lib/cgi-bin"].each { |cgidir|
   if Dir.exist?(cgidir)
-    execute "chcon -R -h system_u:object_r:httpd_sys_script_exec_t #{cgidir}" do
+    execute "chcon -R -t httpd_sys_script_exec_t #{cgidir}" do
       not_if "ls -aZ #{cgidir} | grep ':httpd_sys_script_exec_t:'"
       notifies :reload, "service[apache2]", :delayed
     end

data/cookbooks/mu-master/recipes/vault.rb

@@ -25,87 +25,88 @@
 include_recipe 'mu-master::firewall-holes'
 
 # Mangle a bunch of values used by the Consul and Vault community cookbooks
-node.normal['consul']['config']['bootstrap_expect'] = 1 # XXX we only want this on our first run, maybe figure out how to toss it later
-node.normal['consul']['config']['start_join'] = ["127.0.0.1"]
-node.normal['consul']['config']['ca_file'] = "#{$MU_CFG['datadir']}/ssl/Mu_CA.pem"
-node.normal['consul']['config']['key_file'] = "#{$MU_CFG['datadir']}/ssl/consul.key"
-node.normal['consul']['config']['cert_file'] = "#{$MU_CFG['datadir']}/ssl/consul.crt"
-consul_public = $MU_CFG['public_address']
-if !consul_public.match(/^\d+\.\d+\.\d+\.\d+$/)
-  resolver = Resolv::DNS.new
-  begin
-    consul_public = resolver.getaddress(consul_public).to_s
-  end
-end
-# strictly speaking we could split internal vs. external IPs here, but atm
-# we're treating everything not local to this machine as public anyway
-node.normal['consul']['config']['advertise_addr'] = consul_public
-node.normal['consul']['config']['advertise_addr_wan'] = consul_public
-node.normal['consul']['config']['bind_addr'] = "0.0.0.0"
-node.normal['hashicorp-vault']['config']['tls_key_file'] = "#{$MU_CFG['datadir']}/ssl/vault.key"
-node.normal['hashicorp-vault']['config']['tls_cert_file'] = "#{$MU_CFG['datadir']}/ssl/vault.crt"
-node.normal['hashicorp-vault']['config']['address'] = '0.0.0.0:8200'
-node.save
+#node.normal['consul']['config']['bootstrap_expect'] = 1 # XXX we only want this on our first run, maybe figure out how to toss it later
+#node.normal['consul']['config']['start_join'] = ["127.0.0.1"]
+#node.normal['consul']['config']['ca_file'] = "#{$MU_CFG['datadir']}/ssl/Mu_CA.pem"
+#node.normal['consul']['config']['key_file'] = "#{$MU_CFG['datadir']}/ssl/consul.key"
+#node.normal['consul']['config']['cert_file'] = "#{$MU_CFG['datadir']}/ssl/consul.crt"
+#consul_public = $MU_CFG['public_address']
+#if !consul_public.match(/^\d+\.\d+\.\d+\.\d+$/)
+#  resolver = Resolv::DNS.new
+#  begin
+#    consul_public = resolver.getaddress(consul_public).to_s
+#  end
+#end
+## strictly speaking we could split internal vs. external IPs here, but atm
+## we're treating everything not local to this machine as public anyway
+#node.normal['consul']['config']['advertise_addr'] = consul_public
+#node.normal['consul']['config']['advertise_addr_wan'] = consul_public
+#node.normal['consul']['config']['bind_addr'] = "0.0.0.0"
+#node.normal['consul-cluster']['tls']
+#node.normal['hashicorp-vault']['config']['tls_key_file'] = "#{$MU_CFG['datadir']}/ssl/vault.key"
+#node.normal['hashicorp-vault']['config']['tls_cert_file'] = "#{$MU_CFG['datadir']}/ssl/vault.crt"
+#node.normal['hashicorp-vault']['config']['address'] = '0.0.0.0:8200'
+#node.save
 
-["consul", "vault"].each { |cert|
-  # These community cookbooks aren't bright enough to deal with a stringent
-  # umask, and create these unreadable by the application if we don't do it for
-  # them.
-  directory "fix /opt/#{cert} permissions" do
-    path "/opt/#{cert}"
-    mode 0755
-    notifies :restart, "service[#{cert}]", :delayed
-  end
-}
+#["consul", "vault"].each { |cert|
+#  # These community cookbooks aren't bright enough to deal with a stringent
+#  # umask, and create these unreadable by the application if we don't do it for
+#  # them.
+#  directory "fix /opt/#{cert} permissions" do
+#    path "/opt/#{cert}"
+#    mode 0755
+#    notifies :restart, "service[#{cert}]", :delayed
+#  end
+#}
 
-include_recipe "consul-cluster"
-include_recipe "vault-cluster"
+#include_recipe "consul-cluster"
+#include_recipe "vault-cluster"
 
-["consul", "vault"].each { |cert|
-  file "fix #{cert} cert permissions" do
-    path "#{$MU_CFG['datadir']}/ssl/#{cert}.crt"
-    owner cert
-    notifies :restart, "service[#{cert}]", :delayed
-  end
-  file "fix #{cert} key permissions" do
-    path "#{$MU_CFG['datadir']}/ssl/#{cert}.key"
-    notifies :restart, "service[#{cert}]", :delayed
-    owner cert
-  end
-  }
+#["consul", "vault"].each { |cert|
+#  file "fix #{cert} cert permissions" do
+#    path "#{$MU_CFG['datadir']}/ssl/#{cert}.crt"
+#    owner cert
+#    notifies :restart, "service[#{cert}]", :delayed
+#  end
+#  file "fix #{cert} key permissions" do
+#    path "#{$MU_CFG['datadir']}/ssl/#{cert}.key"
+#    notifies :restart, "service[#{cert}]", :delayed
+#    owner cert
+#  end
+#  }
 
-directory "/opt/vault/#{node['hashicorp-vault']['version']}" do
-  mode 0755
-  notifies :restart, "service[vault]", :delayed
-end
+#directory "/opt/vault/#{node['hashicorp-vault']['version']}" do
+#  mode 0755
+#  notifies :restart, "service[vault]", :delayed
+#end
 
-directory "/etc/consul/ssl" do
-  owner "consul"
-  group "consul"
-  mode 0755
-end
-directory "/etc/vault" do
-  owner "root"
-  mode 0755
-end
-directory "/etc/vault/ssl" do
-  owner "root"
-  mode 0755
-end
-directory "/etc/consul/ssl/CA" do
-  owner "root"
-  mode 0755
-end
-include_recipe 'chef-vault'
+#directory "/etc/consul/ssl" do
+#  owner "consul"
+#  group "consul"
+#  mode 0755
+#end
+#directory "/etc/vault" do
+#  owner "root"
+#  mode 0755
+#end
+#directory "/etc/vault/ssl" do
+#  owner "root"
+#  mode 0755
+#end
+#directory "/etc/consul/ssl/CA" do
+#  owner "root"
+#  mode 0755
+#end
+#include_recipe 'chef-vault'
 
-file "/etc/consul/ssl/CA/ca.crt" do
-  mode 0644
-  content chef_vault_item("secrets", "consul")["ca_certificate"]
-end
+#file "/etc/consul/ssl/CA/ca.crt" do
+#  mode 0644
+#  content chef_vault_item("secrets", "consul")["ca_certificate"]
+#end
 
-service "consul" do
-  action [:enable, :start]
-end
-service "vault" do
-  action [:enable, :start]
-end
+#service "consul" do
+#  action [:enable, :start]
+#end
+#service "vault" do
+#  action [:enable, :start]
+#end

data/cookbooks/mu-master/templates/default/mods/rewrite.conf.erb

@@ -0,0 +1 @@
+LoadModule rewrite_module /usr/lib64/httpd/modules/mod_rewrite.so

data/cookbooks/mu-master/templates/default/nagios.conf.erb

@@ -0,0 +1,103 @@
+# Autogenerated by Chef.
+
+<% unless node['nagios']['ldap_verify_cert'].nil? %>LDAPVerifyServerCert <%= node['nagios']['ldap_verify_cert'] %><% end %>
+<% unless node['nagios']['ldap_trusted_mode'].nil? -%>LDAPTrustedMode <%= node['nagios']['ldap_trusted_mode'] %> <% end -%>
+<% unless node['nagios']['ldap_trusted_global_cert'].nil? -%>LDAPTrustedGlobalCert <%= node['nagios']['ldap_trusted_global_cert'] %> <% end -%>
+
+<VirtualHost *:<%= node['nagios']['http_port'] %>>
+  ServerAdmin     <%= node['nagios']['sysadmin_email'] %>
+<% if @nagios_url %>
+  ServerName <%= @nagios_url %>
+<% else %>
+  ServerName <%= @server_name %>
+<% end %>
+  ServerAlias <% @server_aliases.each do |a| %><%= a %> <% end %>
+  DocumentRoot    <%= node['nagios']['docroot'] %>
+#  CustomLog       <%= node['apache']['log_dir'] %>/nagios_access.log combined
+#  ErrorLog        <%= node['apache']['log_dir'] %>/nagios_error.log
+
+<% if node['platform_family'] == 'debian' && node['nagios']['server']['install_method'] == 'package'-%>
+  Alias /stylesheets /etc/<%= node['nagios']['server']['vname'] %>/stylesheets
+  Alias /nagios3/stylesheets /etc/<%= node['nagios']['server']['vname'] %>/stylesheets
+<% end -%>
+  ScriptAlias <%= node['nagios']['cgi-path'] %> <%= node['nagios']['cgi-bin'] %>
+  ScriptAlias /cgi-bin/statusjson.cgi <%= node['nagios']['cgi-bin'] %>/statusjson.cgi
+  Alias /<%= node['nagios']['server']['vname'] %> <%= node['nagios']['docroot'] %>
+
+  <Directory "<%= node['nagios']['cgi-bin'] %>">
+     Options ExecCGI
+     <% if node['nagios']['default_user_name'] -%>
+     require all granted
+     <% end -%>
+  </Directory>
+
+  <FilesMatch ".+\.ph(p[345]?|t|tml)$">
+     SetHandler application/x-httpd-php
+  </FilesMatch>
+
+<% if @https -%>
+  SSLEngine On
+  SSLProtocol <%= node['nagios']['ssl_protocols'] %>
+<% if node['nagios']['ssl_ciphers'] != nil -%>
+  SSLCipherSuite <%= node['nagios']['ssl_ciphers'] %>
+<% end -%>
+  SSLCertificateFile <%= @ssl_cert_file %>
+<% if node['nagios']['ssl_cert_chain_file'] %>
+  SSLCertificateChainFile <%= node['nagios']['ssl_cert_chain_file'] %>
+<% end -%>
+  SSLCertificateKeyFile <%= @ssl_cert_key %>
+
+<% end -%>
+<% case node['nagios']['server_auth_method'] -%>
+<% when "openid" -%>
+  <Location />
+    AuthName "Nagios Server"
+    AuthType OpenID
+    require user <%= node['apache']['allowed_openids'].join(' ') %>
+    AuthOpenIDDBLocation <%= node['apache']['mod_auth_openid']['dblocation'] %>
+  </Location>
+<% when "cas" -%>
+  CASLoginURL <%= node['nagios']['cas_login_url'] %>
+  CASValidateURL <%= node['nagios']['cas_validate_url'] %>
+  CASValidateServer <%= node['nagios']['cas_validate_server'] %>
+  <% if node['nagios']['cas_root_proxy_url'] -%>
+  CASRootProxiedAs <%= node['nagios']['cas_root_proxy_url'] %>
+  <% end -%>
+
+  <Location />
+    AuthType CAS
+    require <%= node['nagios']['server_auth_require'] %>
+  </Location>
+<% when "ldap" -%>
+  <Location />
+    AuthName "Nagios Server"
+    AuthType Basic
+    AuthBasicProvider ldap
+    <% unless node['nagios']['ldap_group_attribute_is_dn'].nil? %>AuthLDAPGroupAttributeIsDN <%= node['nagios']['ldap_group_attribute_is_dn'] %><% end %>
+    <% unless node['nagios']['ldap_group_attribute'].nil? -%>AuthLDAPGroupAttribute "<%= node['nagios']['ldap_group_attribute'] %>" <% end -%>
+    <% unless node['nagios']['ldap_bind_dn'].nil? -%>AuthLDAPBindDN "<%= node['nagios']['ldap_bind_dn'] %>" <% end -%>
+    <% unless node['nagios']['ldap_bind_password'].nil? -%>AuthLDAPBindPassword "<%= node['nagios']['ldap_bind_password'] %>"<% end -%>
+    AuthLDAPURL "<%= node['nagios']['ldap_url'] %>"
+    <% if !node['apache']['version'].nil? and node['apache']['version'] < "2.4" %>
+      <% unless node['nagios']['ldap_authoritative'].nil? %>AuthzLDAPAuthoritative <%= node['nagios']['ldap_authoritative'] %><% end %>
+    <% end %>
+    require <%= node['nagios']['server_auth_require'] %>
+  </Location>
+<% else -%>
+  <Location />
+    AuthName "Nagios Server"
+    AuthType Basic
+    AuthUserFile "<%= node['nagios']['conf_dir'] %>/htpasswd.users"
+    require <%= node['nagios']['server_auth_require'] %>
+    <% unless node['nagios']['allowed_ips'].empty? -%>
+    Order Deny,Allow
+    Deny from All
+    Allow from <%=node['nagios']['allowed_ips'].join(' ') %>
+    Satisfy Any
+    <% end -%>
+  </Location>
+<% end -%>
+
+  SetEnv TZ "<%= node['nagios']['conf']['use_timezone'] %>"
+
+</VirtualHost>

data/cookbooks/mu-master/templates/default/web_app.conf.erb

@@ -1,12 +1,12 @@
-<VirtualHost *:<%= @params[:server_port] || node['apache']['listen'].first %>>
-  ServerName <%= @params[:server_name] %>
-  ServerAlias <% @params[:server_aliases].each do |a| %><%= a %> <% end %>
+<VirtualHost *:<%= @server_port || (node['apache'] and node['apache']['listen'].first) %>>
+  ServerName <%= @server_name %>
+  ServerAlias <% @server_aliases.each do |a| %><%= a %> <% end %>
+  DocumentRoot <%= @docroot %>
   FileETag -INode
-  DocumentRoot <%= @params[:docroot] %>
   RewriteEngine On
   RewriteRule ^/(nagios|jenkins|scratchpad)$ https://%{HTTP_HOST}/$1/ [R=301,NC,L]
 
-<% if @params[:server_port].to_s.match(/443$/) %>
+<% if @server_port.to_s.match(/443$/) %>
   SSLEngine On
   SSLCertificateFile <%= $MU_CFG['ssl']['cert'] %>
   SSLCertificateKeyFile <%= $MU_CFG['ssl']['key'] %>
@@ -15,12 +15,7 @@
 <% end %>
   SSLProxyEngine on
   <Proxy *>
-<% if node['apache']['version'] == "2.2" %>
-    Order allow,deny
-    Allow from all
-<% elsif node['apache']['version'] == "2.4" %>
     Require all granted
-<% end %>
   </Proxy>
 
   ProxyPreserveHost on
@@ -48,19 +43,14 @@
   RewriteRule ^/(nagios|jenkins|scratchpad)/(.*) https://%{HTTP_HOST}/$1/$2 [R=301,NC,L]
 <% end %>
 
-	RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK)
-	RewriteRule .* - [F]
+  RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK)
+  RewriteRule .* - [F]
 
-  <Directory <%= @params[:docroot] %>>
-    Options <%= [@params[:directory_options] || "FollowSymLinks" ].flatten.join " " %>
-    AllowOverride <%= [@params[:allow_override] || "None" ].flatten.join " " %>
-<% if node['apache']['version'] == "2.2" %>
-    Order allow,deny
-    Allow from all
-<% elsif node['apache']['version'] == "2.4" %>
-	Require all granted
-<% end %>
-  </Directory>
+  <DirectoryMatch "<%= @docroot %>\/.*">
+    Options <%= [@directory_options || "FollowSymLinks" ].flatten.join " " %>
+    AllowOverride <%= [@allow_override || "None" ].flatten.join " " %>
+    Require all granted
+  </DirectoryMatch>
 
   <Directory />
     Options FollowSymLinks
@@ -69,18 +59,12 @@
 
   <Location /server-status>
     SetHandler server-status
-<% if node['apache']['version'] == "2.2" %>
-    Order Deny,Allow
-    Deny from all
-    Allow from 127.0.0.1
-<% elsif node['apache']['version'] == "2.4" %>
     Require host 127.0.0.1
-<% end %>
   </Location>
 
 
-  <% if @params[:directory_index] -%>
-  DirectoryIndex <%= [@params[:directory_index]].flatten.join " " %>
+  <% if @directory_index -%>
+  DirectoryIndex <%= [@directory_index].flatten.join " " %>
   <% end -%>
 
 </VirtualHost>

data/cookbooks/mu-tools/attributes/default.rb

@@ -21,6 +21,13 @@ if disk_name_str == "CAP-MASTER" or disk_name_str == "MU-MASTER" and !node['host
   disk_name_str = node['hostname']
 end rescue NoMethodError
 
+diskdevs = :xvd
+if !platform_family?("windows")
+  if default['kernel']['modules'].keys.include?("nvme")
+    diskdevs = :nvme
+  end
+end
+
 default['os_updates_using_chef'] = false
 
 default['application_attributes']['application_volume']['mount_directory'] = '/apps'
@@ -107,6 +114,11 @@ default['sec']['pwd'] = {
   end
 }
 
+default['application_attributes']['swap']["volume_size_gb"] = 4
+default['application_attributes']['swap']['mount_device'] = "/dev/xvdm"
+default['application_attributes']['swap']['label'] = "#{disk_name_str} swap"
+default['application_attributes']['swap']['mount_directory'] = "swap"
+
 default['application_attributes']['home']["volume_size_gb"] = 2
 default['application_attributes']['home']['mount_device'] = "/dev/xvdn"
 default['application_attributes']['home']['label'] = "#{disk_name_str} /home"

data/cookbooks/mu-tools/files/centos-6/CentOS-Base.repo

@@ -0,0 +1,47 @@
+# CentOS-Base.repo
+#
+# The mirror system uses the connecting IP address of the client and the
+# update status of each mirror to pick mirrors that are updated to and
+# geographically close to the client.  You should use this for CentOS updates
+# unless you are manually picking other mirrors.
+#
+# If the mirrorlist= does not work for you, as a fall back you can try the
+# remarked out baseurl= line instead.
+#
+#
+
+[base]
+name=CentOS-$releasever - Base
+baseurl=http://vault.centos.org/6.10/os/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+
+#released updates
+[updates]
+name=CentOS-$releasever - Updates
+baseurl=http://vault.centos.org/6.10/updates/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+
+#additional packages that may be useful
+[extras]
+name=CentOS-$releasever - Extras
+baseurl=http://vault.centos.org/6.10/extras/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+
+#additional packages that extend functionality of existing packages
+[centosplus]
+name=CentOS-$releasever - Plus
+baseurl=http://vault.centos.org/6.10/centosplus/$basearch/
+gpgcheck=1
+enabled=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+
+#contrib - packages by Centos Users
+[contrib]
+name=CentOS-$releasever - Contrib
+baseurl=http://vault.centos.org/6.10/contrib/$basearch/
+gpgcheck=1
+enabled=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6

data/cookbooks/mu-tools/libraries/helper.rb

@@ -45,6 +45,77 @@ module Mutools
       nil
     end
 
+    # Just list our block devices
+    # @return [Array<String>]
+    def list_disk_devices
+      if File.executable?("/bin/lsblk")
+        shell_out(%Q{/bin/lsblk -i -p -r -n | egrep ' disk( |$)'}).stdout.each_line.map { |l|
+          l.chomp.sub(/ .*/, '')
+        }
+      else
+        # XXX something dumber
+        nil
+      end
+    end
+
+    # If we're in AWS and NVME-aware, return a mapping of AWS-side device names
+    # to actual NVME devices.
+    # @return [Hash]
+    def attached_nvme_disks
+      if get_aws_metadata("meta-data/instance-id").nil? or
+         !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme")
+        return {}
+      end
+      map = {}
+      devices = list_disk_devices
+      return {} if !devices
+      devices.each { |d|
+        if d =~ /^\/dev\/nvme/
+          shell_out(%Q{/sbin/nvme id-ctrl -v #{d}}).stdout.each_line { |desc|
+            if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./)
+              virt_dev = Regexp.last_match[1]
+              map[virt_dev] = d
+              if !File.exists?(virt_dev)
+                begin
+                  File.symlink(d, virt_dev)
+                rescue Errno::EEXIST # XXX whyyyyy is this needed
+                end
+              end
+              break
+            end
+          }
+        end
+      }
+      map
+    end
+
+    def real_devicepath(dev)
+      map = attached_nvme_disks
+      if map[dev]
+        map[dev]
+      elsif map[dev.gsub(/.*?\//, '')]
+        map[dev.gsub(/.*?\//, '')]
+      else
+        dev # be nice to actually handle this too
+      end
+    end
+
+    def uuid_line(dev)
+      realdev = real_devicepath(dev)
+      shell_out(%Q{/sbin/blkid #{realdev} -o export | grep ^UUID=}).stdout.chomp
+    end
+
+    def nvme?
+      if File.executable?("/bin/lsblk")
+        shell_out(%Q{/bin/lsblk -i -p -r -n}).stdout.each_line { |l|
+          return true if l =~ /^\/dev\/nvme\d/
+        }
+      else
+        return true if File.exists?("/dev/nvme0n1")
+      end
+      false
+    end
+
     @project = nil
     @authorizer = nil
     def set_gcp_cfg_params
@@ -65,7 +136,7 @@ module Mutools
     @region = nil
     def set_aws_cfg_params
       begin
-        require 'aws-sdk-core'
+        require 'aws-sdk'
         instance_identity = get_aws_metadata("dynamic/instance-identity/document")
         return false if instance_identity.nil? # Not in AWS, most likely
         @region = JSON.parse(instance_identity)["region"]
@@ -90,6 +161,7 @@ module Mutools
 
     @ec2 = nil
     def ec2
+      require 'aws-sdk-ec2'
       if set_aws_cfg_params
         @ec2 ||= Aws::EC2::Client.new(region: @region)
       end
@@ -97,6 +169,7 @@ module Mutools
     end
     @s3 = nil
     def s3
+      require 'aws-sdk-s3'
       if set_aws_cfg_params
         @s3 ||= Aws::S3::Client.new(region: @region)
       end
@@ -186,12 +259,12 @@ module Mutools
       if cloud == "AWS"
         resp = nil
         begin
+          Chef::Log.info("Fetch deploy secret from s3://#{bucket}/#{filename}")
           resp = s3.get_object(bucket: bucket, key: filename)
         rescue ::Aws::S3::Errors::PermanentRedirect => e
           tmps3 = Aws::S3::Client.new(region: "us-east-1")
           resp = tmps3.get_object(bucket: bucket, key: filename)
         end
-        Chef::Log.info("Fetch deploy secret from s3://#{bucket}/#{filename}")
         secret = resp.body.read
       elsif cloud == "Google"
         include_recipe "mu-tools::gcloud"
@@ -230,9 +303,10 @@ module Mutools
     end
 
     def mommacat_request(action, arg)
+      params = Base64.urlsafe_encode64(JSON.generate(arg)) if arg
       uri = URI("https://#{get_mu_master_ips.first}:2260/")
       req = Net::HTTP::Post.new(uri)
-      res_type = (node['deployment'].has_key?(:server_pools) and node['deployment']['server_pools'].has_key?(node['service_name'])) ? "server_pool" : "server"
+      res_type = (node['deployment'].has_key?('server_pools') and node['deployment']['server_pools'].has_key?(node['service_name'])) ? "server_pool" : "server"
       response = nil
       begin
         secret = get_deploy_secret
@@ -241,6 +315,8 @@ module Mutools
         end
 
         Chef::Log.info("Sending Momma Cat #{action} request to #{uri} from #{get_aws_metadata("meta-data/instance-id")}")
+        disks_before = list_disk_devices if action == "add_volume"
+
         req.set_form_data(
           "mu_id" => mu_get_tag_value("MU-ID"),
           "mu_resource_name" => node['service_name'],
@@ -248,7 +324,7 @@ module Mutools
           "mu_resource_type" => res_type,
           "mu_user" => node['deployment']['mu_user'] || node['deployment']['chef_user'],
           "mu_deploy_secret" => secret,
-          action => arg
+          action => params
         )
         http = Net::HTTP.new(uri.hostname, uri.port)
         http.use_ssl = true
@@ -256,10 +332,28 @@ module Mutools
         response = http.request(req)
         if response.code != "200"
           Chef::Log.error("Got #{response.code} back from #{uri} on #{action} => #{arg}")
+        else
+          if action == "add_volume" and arg and arg.is_a?(Hash) and arg[:dev]
+            seen_requested = false
+            retries = 0
+            begin
+              list_disk_devices.each { |d|
+                if d == arg[:dev] or
+                   (nvme? and d == attached_nvme_disks[arg[:dev]])
+                  seen_requested = true
+                end
+              }
+              if !seen_requested
+                sleep 6
+                retries += 1
+              end
+            end while retries < 5 and !seen_requested
+          end
         end
       rescue EOFError => e
         # Sometimes deployment metadata is incomplete and missing a
         # server_pool entry. Try to help it out.
+        # XXX find some awsmetadata way to determine that we're in an Autoscale Group before trying this
         if res_type == "server"
           res_type = "server_pool"
           retry