cloud-mu 3.1.6 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 447346eba4a1cd7ee0df18c2c85aea32d1a45af0f4d22474fc902007d1f30a2c
-  data.tar.gz: 8f44c2cf180c0748c712b9c244d0c21335147c6a3c9f6a1472772546e13a9b85
+  metadata.gz: c0a85c9f70be756955896aaeb1ea32d462178402d4eec97279454337f839fc96
+  data.tar.gz: 3bee42f370ebb5ac6caa2fb52a36ec61d4aae204410a4aed13472cad130e222a
 SHA512:
-  metadata.gz: 8008e86471d5596337e3b642f5740a2bfe3b178646dd36a37f23dcfb8e0eacfcfbab4bac148ec5855f74f803543fd333480636249ec938387523e6f0c1fddde8
-  data.tar.gz: '08466a848ca7b54fc6460e240f472cc812e93730650545f240a92ea67bde875b96a5d8398b770182355721c6628887665f938dcb7a3103209f7e170cffa246b9'
+  metadata.gz: 2910888a4c3061b4536bd84d60ec2c6b2b4170043983c603d5ff5b0af22ed43adee32e283e13da844fd949a9761a88c171a1eeb6b2b99bd7a08a93ed1efae772
+  data.tar.gz: 5b6e371475a5768895d5618865d42d5005265fa0810d5bf71e2af93032bb2767e843612ef7ec9e401b7be48dc10827cb3445bd005cdc9e4a8dd088488f30fed7

data/Dockerfile

@@ -8,7 +8,7 @@ RUN df -h
 
 RUN apt-get update
 
-RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl
+RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential python-pip curl openssh-client
 
 RUN apt-get upgrade -y
 

data/bin/mu-adopt

@@ -21,12 +21,6 @@ require 'bundler/setup'
 require 'optimist'
 require 'mu'
 
-available_clouds = MU::Cloud.supportedClouds
-available_clouds.reject! { |cloud|
-  cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-  cloudclass.listCredentials.nil? or cloudclass.listCredentials.size == 0
-}
-
 available_types = MU::Cloud.resource_types.keys.map { |t| t.to_s }
 grouping_options = {
   "logical" => "Group resources in logical layers (folders and habitats together, users/roles/groups together, network resources together, etc)",
@@ -39,16 +33,19 @@ $opt = Optimist::options do
   EOS
   opt :appname, "The overarching name of the application stack we will generate", :required => false, :default => "mu", :type => :string
   opt :types, "The resource types to scan and import. Valid types: #{available_types.join(", ")}", :required => false, :type => :strings, :default => available_types
-  opt :clouds, "The cloud providers to scan and import.", :required => false, :type => :strings, :default => available_clouds
+  opt :clouds, "The cloud providers to scan and import.", :required => false, :type => :strings, :default => MU::Cloud.availableClouds
   opt :parent, "Where applicable, resources which reside in the root folder or organization are configured with the specified parent in our target BoK", :required => false, :type => :string
   opt :billing, "Force-set this billing entity on created resources, instead of copying from the live resources", :required => false, :type => :string
   opt :sources, "One or more sets of credentials to use when importing resources. By default we will search and import from all sets of available credentials for each cloud provider specified with --clouds", :required => false, :type => :strings
   opt :credentials, "Override the 'credentials' value in our generated Baskets of Kittens to target a single, specific account. Our default behavior is to set each resource to deploy into the account from which it was sourced.", :required => false, :type => :string
   opt :savedeploys, "Generate actual deployment metadata in #{MU.dataDir}/deployments, as though the resources we found were created with mu-deploy. If we are generating more than one configuration, and a resource needs to reference another resource (e.g. to declare a VPC in which to reside), this will allow us to reference them as virtual resource, rather than by raw cloud identifier.", :required => false, :type => :boolean, :default => false
   opt :diff, "List the differences between what we find and an existing, saved deploy from a previous run, if one exists.", :required => false, :type => :boolean
+  opt :merge_changes, "When using --diff, merge detected changes into the baseline deploy after reporting on them.", :required => false, :type => :boolean, :default => false
   opt :grouping, "Methods for grouping found resources into separate Baskets.\n\n"+MU::Adoption::GROUPMODES.keys.map { |g| "* "+g.to_s+": "+MU::Adoption::GROUPMODES[g] }.join("\n")+"\n\n", :required => false, :type => :string, :default => "logical"
   opt :habitats, "Limit scope of searches to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
+  opt :regions, "Restrict to operating on a subset of available regions, instead of all that we know about.", :require => false, :type => :strings
   opt :scrub, "Whether to set scrub_mu_isms in the BoKs we generate", :default => $MU_CFG.has_key?('adopt_scrub_mu_isms') ? $MU_CFG['adopt_scrub_mu_isms'] : false
+  opt :pattern, "Only adopt resources whose resource name would match this pattern. Must be a valid regular expression. Alphabetical characters will be treated case-insensitively.", :required => false, :type => :string
 end
 
 ok = true
@@ -64,6 +61,16 @@ if $opt[:diff]
   $opt[:savedeploys] = false
 end
 
+pattern = nil
+if $opt[:pattern]
+  begin
+    pattern = Regexp.new($opt[:pattern], true)
+  rescue RegexpError => e
+    MU.log "Invalid --pattern option: #{e.message}", MU::ERR
+    exit 1
+  end
+end
+
 types = []
 $opt[:types].each { |t|
   t_name = t.gsub(/-/, "_")
@@ -102,8 +109,7 @@ if !ok
   exit 1
 end
 
-
-adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub])
+adoption = MU::Adoption.new(clouds: clouds, types: types, parent: $opt[:parent], billing: $opt[:billing], sources: $opt[:sources], credentials: $opt[:credentials], group_by: $opt[:grouping].to_sym, savedeploys: $opt[:savedeploys], diff: $opt[:diff], habitats: $opt[:habitats], scrub_mu_isms: $opt[:scrub], regions: $opt[:regions], merge: $opt[:merge_changes], pattern: pattern)
 found = adoption.scrapeClouds
 if found.nil? or found.empty?
   MU.log "No resources found to adopt", MU::WARN, details: {"clouds" => clouds, "types" => types }
@@ -117,10 +123,7 @@ boks.each_pair { |appname, bok|
   File.open("#{appname}.yaml", "w") { |f|
     f.write JSON.parse(JSON.generate(bok)).to_yaml
   }
-  conf_engine = MU::Config.new("#{appname}.yaml")
-  stack_conf = conf_engine.config
 #  puts stack_conf.to_yaml
-  MU.log "#{appname}.yaml validated successfully", MU::NOTICE
   MU::Cloud.resource_types.each_pair { |type, cfg|
     if bok[cfg[:cfg_plural]]
       MU.log "#{bok[cfg[:cfg_plural]].size.to_s} #{cfg[:cfg_plural]}", MU::NOTICE

data/bin/mu-azure-tests

@@ -0,0 +1,57 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+(0..100000).to_a.each { |n|
+retries = 0
+seed = nil
+#        begin
+#          raise MuError, "Failed to allocate an unused MU-ID after #{retries} tries!" if retries > 70
+#          seedsize = 1 + (retries/10).abs
+#          seed = (0...seedsize+1).map { ('a'..'z').to_a[rand(26)] }.join
+#        end while seed == "mu" or seed[0] == seed[1]
+seed = "nn"
+handle = MU::MommaCat.generateHandle(seed)
+puts handle
+}
+exit
+
+#pp MU::Cloud::Azure.listRegions
+#pp MU::Cloud::Azure::Habitat.testcalls
+#pp MU::Cloud::Azure::VPC.find(cloud_id: MU::Cloud::Azure::Id.new(resource_group: "mu", name: "mu-vnet"))
+#pp MU::Cloud::Azure.authorization.role_assignments.list_for_resource_group("AKS-DEV-2019062015-KA-EASTUS")
+#pp MU::Cloud::Azure::Role.find(role_name: "Azure Kubernetes Service Cluster Admin Role")
+#puts MU::Cloud::Azure.default_subscription
+#pp MU::Cloud::Azure.fetchPublicIP("MYVPC-DEV-2019061911-XI-EASTUS", "ip-addr-thingy")
+#pp MU::Cloud::Azure.ensureProvider("egtazure", "Microsoft.ContainerService", force: true)
+pp MU::Cloud::Azure::Server.find(cloud_id: "mu")
+exit
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/6")
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/8")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/6")
+pp MU::Cloud::Azure::Server.fetchImage("Debian/debian-10/10")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2012-R2-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2016-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2019-Datacenter")

data/bin/mu-cleanup

@@ -24,10 +24,8 @@ require 'mu'
 Dir.chdir(MU.installDir)
 
 credentials = []
-MU::Cloud.supportedClouds.each { |cloud|
-  cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-  next if cloudclass.listCredentials.nil? or cloudclass.listCredentials.size == 0
-  credentials.concat(cloudclass.listCredentials)
+MU::Cloud.availableClouds.each { |cloud|
+  credentials.concat(MU::Cloud.cloudClass(cloud).listCredentials)
 }
 credentials.uniq!
 

data/bin/mu-configure

@@ -113,12 +113,44 @@ $CONFIGURABLES = {
     "desc" => "Disable the Momma Cat grooming daemon. Nodes which require asynchronous Ansible/Chef bootstraps will not function. This option is only honored in gem-based installations.",
     "boolean" => true
   },
+  "adopt_change_notify" => {
+    "title" => "Adoption Change Notifications",
+    "subtree" => {
+      "slack" => {
+        "title" => "Send to Slack",
+        "desc" => "Report modifications to adopted resources, detected by mu-adopt --diff, to the Slack webhook and channel configured under Slack Configuration.",
+        "boolean" => true
+      },
+      "slack_snippet_threshold" => {
+        "title" => "Attachment Threshold",
+        "desc" => "If a list of details about a modified resources is longer than this number of lines (in JSON), it will be sent as an \"attachment,\" which in Slack means a blockquote that displays a few lines with a \"Show more\" button. The internal default is 5 lines."
+      },
+#      "email" => {
+#        "title" => "Send Email",
+#        "desc" => "",
+#        "boolean" => true
+#      }
+    }
+  },
   "adopt_scrub_mu_isms" => {
-    "title" => "Disable Momma Cat",
+    "title" => "Scrub Mu-isms from Baskets of Kittens",
     "default" => false,
     "desc" => "Ordinarily, Mu will automatically name, tag and generate auxiliary resources in a standard Mu-ish fashion that allows for deployment of multiple clones of a given stack. Toggling this flag will change the default behavior of mu-adopt, when it creates stack descriptors from found resources, to enable or disable this behavior (see also mu-adopt's --scrub option).",
     "boolean" => true
   },
+  "slack" => {
+    "title" => "Slack Configuration",
+    "subtree" => {
+      "webhook" => {
+        "title" => "Webhook",
+        "desc" => "The hooks.slack.com URL for the webook to which we'll send deploy notifications"
+      },
+      "channel" => {
+        "title" => "Channel",
+        "desc" => "The channel name (without leading #) to which alerts should be sent."
+      }
+    }
+  },
   "mommacat_port" => {
     "title" => "Momma Cat Listen Port",
     "pattern" => /^[0-9]+$/i,
@@ -247,6 +279,10 @@ $CONFIGURABLES = {
         "required" => false,
         "desc" => "For Google Cloud projects which are attached to a GSuite domain. GCP service accounts cannot view or manage GSuite resources (groups, users, etc) directly, but must instead masquerade as a GSuite user which has delegated authority to the service account. See also: https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority"
       },
+      "org" => {
+        "title" => "Default Org/Domain",
+        "desc" => "For credential sets which have access to multiple GSuite or Cloud Identity orgs, you must specify a default organization (e.g. my.domain.com)."
+      },
       "customer_id" => {
         "title" => "GSuite Customer ID",
         "required" => false,

data/bin/mu-deploy

@@ -105,7 +105,7 @@ if $opts[:dryrun]
     Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :never) {
       begin
         Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :immediate) {
-          MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::WARN, verbosity: MU::Logger::NORMAL
+          MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::NOTICE, verbosity: MU::Logger::NORMAL
           Thread.current.exit
         }
       ensure
@@ -124,7 +124,7 @@ if $opts[:dryrun]
       )
       cost_dummy_deploy.run
     rescue MU::Cloud::MuCloudResourceNotImplemented, MU::Cloud::MuCloudFlagNotImplemented
-      MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::WARN, verbosity: MU::Logger::NORMAL
+      MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::NOTICE, verbosity: MU::Logger::NORMAL
     end
   end
   exit
@@ -135,7 +135,7 @@ if $opts[:update]
 # TODO consider whether this is useful/valid
 #  old_conf = JSON.parse(File.read(deploy.deploy_dir+"/basket_of_kittens.json"))
 #  stack_conf = old_conf.merge(stack_conf)
-  deploy.updateBasketofKittens(stack_conf)
+  deploy.updateBasketofKittens(stack_conf, skip_validation: true)
   deployer = MU::Deploy.new(
     deploy.environment,
     verbosity: verbosity,

data/bin/mu-findstray-tests

@@ -0,0 +1,25 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+MU::MommaCat.findStray("AWS", "firewall_rule", region: MU.myRegion, dummy_ok: true, debug: true)

data/bin/mu-gen-docs

@@ -79,8 +79,7 @@ EOF
       impl_counts[type] ||= 0
       [a, b].each { |cloud|
         begin
-          myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
-          case myclass.quality
+          case MU::Cloud.resourceClass(cloud, type).quality
           when MU::Cloud::RELEASE
             cloud_is_useful[cloud] = true
             counts[cloud] += 4
@@ -114,8 +113,7 @@ EOF
     cloudlist.each { |cloud|
       readme += "<td><center>"
       begin
-        myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
-        case myclass.quality
+        case MU::Cloud.resourceClass(cloud, type).quality
         when MU::Cloud::RELEASE
           readme += "<img src='release.png' style='#{icon_style}' title='Release Quality' alt='[Release Quality]'>"
         when MU::Cloud::BETA

data/bin/mu-load-config.rb

@@ -134,7 +134,7 @@ def loadMuConfig(default_cfg_overrides = nil)
     }
   end
 
-  global_cfg = { "config_files" => [] }
+  global_cfg = { "config_files" => [], "overridden_keys" => [] }
   if File.exist?(cfgPath)
     global_cfg = YAML.load(File.read(cfgPath))
     global_cfg["config_files"] = [cfgPath]
@@ -147,6 +147,7 @@ def loadMuConfig(default_cfg_overrides = nil)
     if localfile
       global_cfg.merge!(localfile)
       global_cfg["config_files"] << "#{home}/.mu.yaml"
+      global_cfg["overridden_keys"] = localfile.keys
     end
   end
   if !global_cfg.has_key?("installdir")

data/bin/mu-run-tests

@@ -34,6 +34,7 @@ Usage:
 #{$0} [-m <#>] [-f] [-v] [specific test BoK to run [...]]
   EOS
   opt :max_threads, "Environment to set on creation.", :require => false, :default => 3, :type => :integer
+  opt :max_retries, "Number of times to retry failed tests in --dryrun mode.", :require => false, :default => 2, :type => :integer
   opt :full, "Actually run deploys, instead of --dryrun", :require => false, :default => false
   opt :verbose, "Show more information while running", :require => false, :default => false
 end
@@ -42,7 +43,7 @@ only = ARGV
 
 files = Dir.glob("*.yaml", base: dir)
 files.concat(Dir.glob("*.yml", base: dir))
-baseclouds = MU::Cloud.supportedClouds.reject { |c| c == "CloudFormation" }
+baseclouds = MU::Cloud.availableClouds.reject { |c| c == "CloudFormation" }
 
 commands = {}
 failures = []
@@ -56,20 +57,33 @@ end
 
 files.each { |f|
   clouds = baseclouds.dup
+  groomer_match = true
   File.open(dir+"/"+f).readlines.each { |l|
     l.chomp!
-    next if !l.match(/^\s*#\s*clouds: (.*)/)
-    clouds = []
-    cloudstr = Regexp.last_match[1]
-    cloudstr.split(/\s*,\s*/).each { |c|
-      baseclouds.each { |cloud|
-        if cloud.match(/^#{Regexp.quote(c)}$/i)
-          clouds << cloud
+    if l.match(/^\s*#\s*clouds: (.*)/)
+      clouds = []
+      cloudstr = Regexp.last_match[1]
+      cloudstr.split(/\s*,\s*/).each { |c|
+        baseclouds.each { |cloud|
+          if cloud.match(/^#{Regexp.quote(c)}$/i)
+            clouds << cloud
+          end
+        }
+      }
+    elsif l.match(/^\s*#\s*groomers: (.*)/)
+      groomerstr = Regexp.last_match[1]
+      groomerstr.split(/\s*,\s*/).each { |g|
+        if !MU::Groomer.availableGroomers.include?(g)
+          MU.log "#{f} requires groomer #{g}, which is not available. This test will be skipped.", MU::NOTICE
+          groomer_match = false
         end
       }
-    }
-    break
+    end
   }
+  if !groomer_match
+    next
+  end
+
   clouds.each { |cloud|
     cmd = "mu-deploy #{f} --cloud #{cloud} #{$opts[:full] ? "" : "--dryrun"}"
     commands[cmd] = {
@@ -108,8 +122,19 @@ def execCommand(cmd, results_stash)
   }
 
   ok = true
-  output = %x{#{cmd} 2>&1}
-  ok = false if $?.exitstatus != 0
+  retries = 0
+  begin
+    output = %x{#{cmd} 2>&1}
+    if $?.exitstatus != 0
+      ok = false
+      retries += 1
+      if $opts[:verbose] and !$opts[:full] and retries <= $opts[:max_retries]
+        puts "#{cmd} RETRY #{retries.to_s}".light_red
+      end
+    else
+      ok = true
+    end
+  end while !ok and !$opts[:full] and retries <= $opts[:max_retries]
 
   results_stash["output"] += output
 

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.1.6'
-  s.date        = '2020-03-20'
+  s.version     = '3.4.0'
+  s.date        = '2020-10-22'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -36,7 +36,7 @@ EOF
     'https://github.com/cloudamatic/mu'
   s.license       = 'BSD-3-Clause-Attribution'
   s.add_runtime_dependency 'addressable', '~> 2.5'
-  s.add_runtime_dependency "aws-sdk-core", "< 3"
+  s.add_runtime_dependency "aws-sdk", "~> 3.0"
   s.add_runtime_dependency 'azure_sdk', "~> 0.52"
   s.add_runtime_dependency 'bundler', "~> 1.17"
   s.add_runtime_dependency 'chronic_duration', "~> 0.10"
@@ -57,7 +57,7 @@ EOF
   s.add_runtime_dependency 'rack', "~> 2.0"
   s.add_runtime_dependency 'ruby-graphviz', "~> 1.2"
   s.add_runtime_dependency 'rubocop', '~> 0.58'
-  s.add_runtime_dependency 'rubyzip', "~> 2.0"
+  s.add_runtime_dependency 'rubyzip', "~> 2.3"
   s.add_runtime_dependency 'simple-password-gen', "~> 0.1"
   s.add_runtime_dependency 'slack-notifier', "~> 2.3"
   s.add_runtime_dependency 'solve', '~> 4.0'

data/cookbooks/mu-tools/attributes/default.rb

@@ -21,6 +21,13 @@ if disk_name_str == "CAP-MASTER" or disk_name_str == "MU-MASTER" and !node['host
   disk_name_str = node['hostname']
 end rescue NoMethodError
 
+diskdevs = :xvd
+if !platform_family?("windows")
+  if default['kernel']['modules'].keys.include?("nvme")
+    diskdevs = :nvme
+  end
+end
+
 default['os_updates_using_chef'] = false
 
 default['application_attributes']['application_volume']['mount_directory'] = '/apps'

data/cookbooks/mu-tools/libraries/helper.rb

@@ -45,6 +45,70 @@ module Mutools
       nil
     end
 
+    # Just list our block devices
+    # @return [Array<String>]
+    def list_disk_devices
+      if File.executable?("/bin/lsblk")
+        shell_out(%Q{/bin/lsblk -i -p -r -n | egrep ' disk( |$)'}).stdout.each_line.map { |l|
+          l.chomp.sub(/ .*/, '')
+        }
+      else
+        # XXX something dumber
+        nil
+      end
+    end
+
+    # If we're in AWS and NVME-aware, return a mapping of AWS-side device names
+    # to actual NVME devices.
+    # @return [Hash]
+    def attached_nvme_disks
+      if get_aws_metadata("meta-data/instance-id").nil? or
+         !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme")
+        return {}
+      end
+      map = {}
+      devices = list_disk_devices
+      return {} if !devices
+      devices.each { |d|
+        if d =~ /^\/dev\/nvme/
+          shell_out(%Q{/sbin/nvme id-ctrl -v #{d}}).stdout.each_line { |desc|
+            if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./)
+              virt_dev = Regexp.last_match[1]
+              map[virt_dev] = d
+              if !File.exists?(virt_dev)
+                begin
+                  File.symlink(d, virt_dev)
+                rescue Errno::EEXIST # XXX whyyyyy is this needed
+                end
+              end
+              break
+            end
+          }
+        end
+      }
+      map
+    end
+
+    def real_devicepath(dev)
+      map = attached_nvme_disks
+      if map[dev]
+        map[dev]
+      else
+        dev # be nice to actually handle this too
+      end
+    end
+
+    def nvme?
+      if File.executable?("/bin/lsblk")
+        shell_out(%Q{/bin/lsblk -i -p -r -n}).stdout.each_line { |l|
+          return true if l =~ /^\/dev\/nvme\d/
+        }
+      else
+        return true if File.exists?("/dev/nvme0n1")
+      end
+      false
+    end
+
     @project = nil
     @authorizer = nil
     def set_gcp_cfg_params
@@ -186,12 +250,12 @@ module Mutools
       if cloud == "AWS"
         resp = nil
         begin
+          Chef::Log.info("Fetch deploy secret from s3://#{bucket}/#{filename}")
           resp = s3.get_object(bucket: bucket, key: filename)
         rescue ::Aws::S3::Errors::PermanentRedirect => e
           tmps3 = Aws::S3::Client.new(region: "us-east-1")
           resp = tmps3.get_object(bucket: bucket, key: filename)
         end
-        Chef::Log.info("Fetch deploy secret from s3://#{bucket}/#{filename}")
         secret = resp.body.read
       elsif cloud == "Google"
         include_recipe "mu-tools::gcloud"
@@ -230,17 +294,20 @@ module Mutools
     end
 
     def mommacat_request(action, arg)
+      params = Base64.urlsafe_encode64(JSON.generate(arg)) if arg
       uri = URI("https://#{get_mu_master_ips.first}:2260/")
       req = Net::HTTP::Post.new(uri)
       res_type = (node['deployment'].has_key?(:server_pools) and node['deployment']['server_pools'].has_key?(node['service_name'])) ? "server_pool" : "server"
       response = nil
       begin
         secret = get_deploy_secret
-        if secret.nil?
+        if secret.nil? or secret.empty?
           raise "Failed to fetch deploy secret, and I can't communicate with Momma Cat without it"
         end
 
         Chef::Log.info("Sending Momma Cat #{action} request to #{uri} from #{get_aws_metadata("meta-data/instance-id")}")
+        disks_before = list_disk_devices if action == "add_volume"
+
         req.set_form_data(
           "mu_id" => mu_get_tag_value("MU-ID"),
           "mu_resource_name" => node['service_name'],
@@ -248,7 +315,7 @@ module Mutools
           "mu_resource_type" => res_type,
           "mu_user" => node['deployment']['mu_user'] || node['deployment']['chef_user'],
           "mu_deploy_secret" => secret,
-          action => arg
+          action => params
         )
         http = Net::HTTP.new(uri.hostname, uri.port)
         http.use_ssl = true
@@ -256,6 +323,23 @@ module Mutools
         response = http.request(req)
         if response.code != "200"
           Chef::Log.error("Got #{response.code} back from #{uri} on #{action} => #{arg}")
+        else
+          if action == "add_volume" and arg and arg.is_a?(Hash) and arg[:dev]
+            seen_requested = false
+            retries = 0
+            begin
+              list_disk_devices.each { |d|
+                if d == arg[:dev] or
+                   (nvme? and d == attached_nvme_disks[arg[:dev]])
+                  seen_requested = true
+                end
+              }
+              if !seen_requested
+                sleep 6
+                retries += 1
+              end
+            end while retries < 5 and !seen_requested
+          end
         end
       rescue EOFError => e
         # Sometimes deployment metadata is incomplete and missing a