cloud-mu 3.1.1 → 3.1.2beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e410b5b2d1b9bb3bb28993cda8852ba50bc0d4a32218a64226af3e66ae86465a
-  data.tar.gz: ec28f00264ebea17f917428b12b44f2eaf2361711c1ee8ab0b8db6ac94d4bad8
+  metadata.gz: 5df44eebc605db5e8965c18414c18f1802a565df95014216609799ff912311d7
+  data.tar.gz: 7978b64d27f1fd467c0215ad8b7c6246f4bb4cbba4eeb1f834a35dd4adf08aae
 SHA512:
-  metadata.gz: 5232f7dc31462aa58f307ded9bb7f46d7dd63a7b373ba88d4546620b8531ae5e71167d8d86d29a49638f159ea21dd5fd2e88a7bb7529c8df350e8627ca29fd71
-  data.tar.gz: '09b513e485274453b5a3242e9747c9bc0cd41e06ea9a6744ffe4ab7cab38b43845c8ca124159d0ab418f39441d894f626b48505992de913615a1fb2a8c4b77cf'
+  metadata.gz: ad79d77971ecf24de81294e302f5f0284529da90468e6793be7334d08a59248a026a5b0b36552fe7fb54f80c604f539da0fe2a0f3f2b130a05df9d5993f2808a
+  data.tar.gz: df69d04d5c0f7799b52e60955ea3d1d1a21f63d509a44e369824f011f88a159a425dfdf9d8ebb02975d085b3ba6c3f604937d292aa1c9330ecc43ffa5b8b6c40

data/Berksfile.lock

@@ -0,0 +1,179 @@
+DEPENDENCIES
+  awscli
+  chocolatey
+  firewall
+    path: cookbooks/firewall
+  mu-activedirectory
+  mu-firewall
+  mu-glusterfs
+  mu-master
+  mu-mongo
+  mu-nagios
+    git: https://github.com/cloudamatic/mu-nagios.git
+    revision: c1e3f6155d5ab9952c8403693b118664f8d50973
+  mu-openvpn
+  mu-splunk
+  mu-tools
+  mu-utility
+
+GRAPH
+  apache2 (5.2.1)
+  apt (7.2.0)
+  awscli (1.1.2)
+    python (~> 1.4)
+  bind (2.2.1)
+  bind9-ng (0.1.0)
+  build-essential (8.2.1)
+    mingw (>= 1.1)
+    seven_zip (>= 0.0.0)
+  chef-sugar (5.1.8)
+  chef-vault (3.1.2)
+  chocolatey (2.0.1)
+  consul (2.3.0)
+    build-essential (>= 0.0.0)
+    firewall (~> 2.0)
+    golang (>= 0.0.0)
+    nssm (>= 0.0.0)
+    poise (~> 2.2)
+    poise-archive (~> 1.3)
+    poise-service (~> 1.4)
+  consul-cluster (2.0.0)
+    consul (~> 2.1)
+    ssl_certificate (~> 1.11)
+  cpan (0.1.0)
+  database (6.1.1)
+    postgresql (>= 1.0.0)
+  firewall (2.7.1)
+  golang (1.7.0)
+  hashicorp-vault (2.5.0)
+    build-essential (>= 0.0.0)
+    golang (~> 1.7)
+    poise (~> 2.6)
+    poise-service (~> 1.1)
+    rubyzip (~> 1.0)
+  homebrew (5.0.8)
+  hostsfile (3.0.1)
+  java (2.2.1)
+    homebrew (>= 0.0.0)
+    windows (>= 0.0.0)
+  mingw (2.1.0)
+    seven_zip (>= 0.0.0)
+  mongodb (0.16.2)
+    apt (>= 1.8.2)
+    python (>= 0.0.0)
+    runit (>= 1.5.0)
+    yum (>= 3.0)
+  mu-activedirectory (0.2.0)
+    chef-vault (~> 3.1.1)
+    windows (~> 5.1.1)
+    yum-epel (~> 3.2.0)
+  mu-firewall (0.1.2)
+    firewall (~> 2.7.1)
+  mu-glusterfs (0.1.0)
+    mu-firewall (>= 0.0.0)
+    yum (~> 5.1.0)
+  mu-master (0.9.6)
+    apache2 (< 6.0.0)
+    bind (~> 2.2.0)
+    bind9-ng (~> 0.1.0)
+    chef-sugar (>= 0.0.0)
+    chef-vault (~> 3.1.1)
+    consul-cluster (~> 2.0.0)
+    hostsfile (~> 3.0.1)
+    mu-activedirectory (>= 0.0.0)
+    mu-firewall (>= 0.0.0)
+    mu-nagios (>= 0.0.0)
+    mu-tools (>= 0.0.0)
+    mu-utility (>= 0.0.0)
+    nrpe (~> 2.0.3)
+    postfix (~> 5.3.1)
+    s3fs (>= 0.0.0)
+    vault-cluster (~> 2.1.0)
+  mu-mongo (0.5.0)
+    chef-vault (~> 3.1.1)
+    mongodb (~> 0.16.2)
+  mu-nagios (8.2.2)
+    apache2 (< 6.0.0)
+    build-essential (>= 5.0)
+    nginx (>= 7.0)
+    nrpe (>= 0.0.0)
+    php (>= 0.0.0)
+    php-fpm (>= 0.7.9)
+    yum-epel (>= 0.0.0)
+    zap (>= 0.6.0)
+  mu-openvpn (0.1.0)
+    chef-vault (~> 3.1.1)
+    mu-firewall (>= 0.0.0)
+    mu-utility (>= 0.0.0)
+  mu-splunk (1.3.0)
+    chef-vault (>= 1.0.4)
+  mu-tools (1.1.0)
+    chef-vault (~> 3.1.1)
+    chocolatey (>= 0.0.0)
+    database (~> 6.1.1)
+    firewall (>= 0.0.0)
+    java (~> 2.2.0)
+    mu-activedirectory (>= 0.0.0)
+    mu-firewall (>= 0.0.0)
+    mu-nagios (>= 0.0.0)
+    mu-splunk (>= 0.0.0)
+    mu-utility (>= 0.0.0)
+    oracle-instantclient (~> 1.1.0)
+    poise-python (~> 1.7.0)
+    postgresql (~> 7.1.0)
+    selinux (~> 3.0.0)
+    windows (~> 5.1.1)
+    yum-epel (~> 3.2.0)
+  mu-utility (0.6.0)
+    mu-firewall (>= 0.0.0)
+    windows (~> 5.1.1)
+  nginx (10.0.2)
+    ohai (~> 5.2)
+  nrpe (2.0.5)
+    build-essential (>= 0.0.0)
+    yum-epel (>= 0.0.0)
+  nssm (4.0.1)
+    windows (>= 0.0.0)
+  ohai (5.3.0)
+  oracle-instantclient (1.1.0)
+    build-essential (>= 0.0.0)
+    cpan (>= 0.0.0)
+    php (>= 0.0.0)
+  packagecloud (1.0.1)
+  php (7.0.0)
+    yum-epel (>= 0.0.0)
+  php-fpm (0.8.0)
+  poise (2.8.2)
+  poise-archive (1.5.0)
+    poise (~> 2.6)
+  poise-languages (2.1.2)
+    poise (~> 2.5)
+    poise-archive (~> 1.0)
+  poise-python (1.7.0)
+    poise (~> 2.7)
+    poise-languages (~> 2.0)
+  poise-service (1.5.2)
+    poise (~> 2.0)
+  postfix (5.3.1)
+  postgresql (7.1.5)
+  python (1.4.6)
+    build-essential (>= 0.0.0)
+    yum-epel (>= 0.0.0)
+  rubyzip (1.3.1)
+    poise (~> 2.2)
+  runit (5.1.2)
+    packagecloud (>= 0.0.0)
+    yum-epel (>= 0.0.0)
+  s3fs (3.0.1)
+  selinux (3.0.1)
+  seven_zip (3.1.2)
+    windows (>= 0.0.0)
+  ssl_certificate (1.12.0)
+  vault-cluster (2.1.0)
+    consul-cluster (~> 2.0)
+    hashicorp-vault (~> 2.1)
+    ssl_certificate (~> 1.11)
+  windows (5.1.6)
+  yum (5.1.0)
+  yum-epel (3.2.0)
+  zap (1.2.0)

data/bin/mu-adopt

@@ -47,7 +47,7 @@ $opt = Optimist::options do
   opt :savedeploys, "Generate actual deployment metadata in #{MU.dataDir}/deployments, as though the resources we found were created with mu-deploy. If we are generating more than one configuration, and a resource needs to reference another resource (e.g. to declare a VPC in which to reside), this will allow us to reference them as virtual resource, rather than by raw cloud identifier.", :required => false, :type => :boolean, :default => false
   opt :diff, "List the differences between what we find and an existing, saved deploy from a previous run, if one exists.", :required => false, :type => :boolean
   opt :grouping, "Methods for grouping found resources into separate Baskets.\n\n"+MU::Adoption::GROUPMODES.keys.map { |g| "* "+g.to_s+": "+MU::Adoption::GROUPMODES[g] }.join("\n")+"\n\n", :required => false, :type => :string, :default => "logical"
-  opt :habitats, "Limit scope of research searching to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
+  opt :habitats, "Limit scope of searches to the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
 end
 
 ok = true

data/bin/mu-azure-tests

@@ -0,0 +1,46 @@
+#!/usr/local/ruby-current/bin/ruby
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler/setup'
+require 'json'
+require 'erb'
+require 'optimist'
+require 'json-schema'
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+require 'mu'
+
+puts $stdout.class.name
+exit
+
+#pp MU::Cloud::Azure.listRegions
+#pp MU::Cloud::Azure::Habitat.testcalls
+#pp MU::Cloud::Azure::VPC.find(cloud_id: MU::Cloud::Azure::Id.new(resource_group: "mu", name: "mu-vnet"))
+#pp MU::Cloud::Azure.authorization.role_assignments.list_for_resource_group("AKS-DEV-2019062015-KA-EASTUS")
+#pp MU::Cloud::Azure::Role.find(role_name: "Azure Kubernetes Service Cluster Admin Role")
+#puts MU::Cloud::Azure.default_subscription
+#pp MU::Cloud::Azure.fetchPublicIP("MYVPC-DEV-2019061911-XI-EASTUS", "ip-addr-thingy")
+#pp MU::Cloud::Azure.ensureProvider("egtazure", "Microsoft.ContainerService", force: true)
+pp MU::Cloud::Azure::Server.find(cloud_id: "mu")
+exit
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/6")
+pp MU::Cloud::Azure::Server.fetchImage("OpenLogic/CentOS/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/8")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/7")
+pp MU::Cloud::Azure::Server.fetchImage("RedHat/RHEL/6")
+pp MU::Cloud::Azure::Server.fetchImage("Debian/debian-10/10")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2012-R2-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2016-Datacenter")
+pp MU::Cloud::Azure::Server.fetchImage("MicrosoftWindowsServer/WindowsServer/2019-Datacenter")

data/bin/mu-cleanup

@@ -45,6 +45,7 @@ Usage:
   opt :verbose, "Display debugging output.", :require => false, :default => false, :type => :boolean
   opt :credentials, "Restrict to operating on a subset of available credential sets, instead of all that we know about.", :require => false, :default => credentials, :type => :strings
   opt :regions, "Restrict to operating on a subset of available regions, instead of all that we know about.", :require => false, :type => :strings
+  opt :habitats, "Restrict to operating on the named accounts/projects/subscriptions, instead of search all habitats visible to our credentials.", :required => false, :type => :strings
   opt :quiet, "Display minimal output.", :require => false, :default => false, :type => :boolean
 end
 verbosity = MU::Logger::NORMAL
@@ -77,5 +78,6 @@ MU::Cleanup.run(
   skipcloud: $opts[:skipcloud],
   ignoremaster: $opts[:ignoremaster],
   credsets: $opts[:credentials],
-  regions: $opts[:regions]
+  regions: $opts[:regions],
+  habitats: $opts[:habitats]
 )

data/bin/mu-configure

@@ -1469,3 +1469,10 @@ if !$NOOP
     puts "source #{HOMEDIR}/.bashrc".bold
   end
 end
+
+if $IN_GEM
+  ansible_exec_path = MU::Groomer::Ansible.ansibleExecDir
+  if !ansible_exec_path or ansible_exec_path.empty?
+    puts "No Ansible executables found. Will not be able to groom servers!".red
+  end
+end

data/cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.1.1'
-  s.date        = '2020-01-11'
+  s.version     = '3.1.2beta2'
+  s.date        = '2020-01-27'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -54,11 +54,12 @@ EOF
   s.add_runtime_dependency 'net-ssh', "~> 4.2"
   s.add_runtime_dependency 'net-ssh-multi', '~> 1.2', '>= 1.2.1'
   s.add_runtime_dependency 'googleauth', "~> 0.6"
-  s.add_runtime_dependency 'google-api-client', "~> 0.30.8"
+  s.add_runtime_dependency 'google-api-client', "~> 0.36.4"
   s.add_runtime_dependency 'rubocop', '~> 0.58'
   s.add_runtime_dependency 'addressable', '~> 2.5'
   s.add_runtime_dependency 'slack-notifier', "~> 2.3"
   s.add_runtime_dependency 'azure_sdk', "~> 0.37"
   s.add_runtime_dependency 'rack', "~> 2.0"
   s.add_runtime_dependency 'thin', "~> 1.7"
+  s.add_runtime_dependency 'rubyzip', "~> 2.0"
 end

data/cookbooks/mu-tools/files/default/Mu_CA.pem

@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIIFvzCCA6egAwIBAgIJANg7fTwivzSDMA0GCSqGSIb3DQEBDQUAMF0xFjAUBgNV
+BAMMDTU0LjE3NS44Ni4xOTQxIDAeBgNVBAsMF011IFNlcnZlciA1NC4xNzUuODYu
+MTk0MRQwEgYDVQQKDAtlR2xvYmFsVGVjaDELMAkGA1UEBhMCVVMwHhcNMTkwODEx
+MjExMzMwWhcNMjIwNTMxMjExMzMwWjBdMRYwFAYDVQQDDA01NC4xNzUuODYuMTk0
+MSAwHgYDVQQLDBdNdSBTZXJ2ZXIgNTQuMTc1Ljg2LjE5NDEUMBIGA1UECgwLZUds
+b2JhbFRlY2gxCzAJBgNVBAYTAlVTMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
+CgKCAgEAo7rntOFj/WPNvh00SN55aJBusppsY9arq7QF5gt/9+cBPsjcXn7jJMu0
+vD9RFqkR8fpkvs01MiTToKHDli30FYSO+pybW/3R8VMby3jU7Df+i20tnB8gZqkc
+XQGU4c8cGwdu1J/DpRoX5oCOlO2by+2+5nebJd7ABpzl9eE2/1HBJVaHROCVzmbu
+UCXVIlKAOccgwzPj+r4EHwH4Nyv8cSnh67Fg8jehW21ZltZNXek7upc9421MQLka
+9TtbBod7DWVQNfc8hAxATlupOnKsKa1n8vZD9bj9xvK2wz1E6lVYbkuxzpOzqBqy
+PO/6Svt8zTH3pEJMbxwtiwJ8cCLiqSoxj8hOKvvsSmvboN9DwN73JQjOY/pXHaU1
+/w9syNORnwEKMzs5Eu14dAV1+w7Nk8xff4LHjIYoTWD+zuK6ETVnX8j7f1zwebok
+HLF0qlnfZhU4uiE8+wU1h6oeGZG9fLV63wlGdUXA+HermzovuJ0d2ocy0O93QQDt
+Y92dr6UcPfAmzFyX3Rj9FFMYb2/n1G8l5pEd/Qkx3sH04aoxEmyQU0zugo3zQsL9
+KNyIbp2BTlSh2R/4hWJpWiXFliRvotiJu1s2wdNQ1D3SZgxDbfxf/3j04xgdi5eW
+e4Q3VnxhRfmkS1NqEzIvPabVLg9qvN419cubpE6HAtBJw/f3ocUCAwEAAaOBgTB/
+MC8GA1UdEQQoMCaHBDavVsKCCWxvY2FsaG9zdIcEfwAAAYINc3RhbmdlLW11LWRl
+djAdBgNVHQ4EFgQUr8Sa0Z5sLB3lCkzzL/cQp1g1VtwwHwYDVR0jBBgwFoAUr8Sa
+0Z5sLB3lCkzzL/cQp1g1VtwwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQ0FAAOC
+AgEAISgwMuoA0es7f8a8aZHuxeUP/160yyMzzoSolKW+JXHDvJjRi/uM5IICkspR
+19ucWB5NJjp6oLaRTA+Recfpk8rc14GICcjhj/455xlhbg/Dnpwi4S58XEeFnoMY
+9o/z9xWHafM579oZPrUzT2un/1xZuYaOshXa3hZQa5R/aK24P4rW/oCCmifBm8ij
+Mdx24gbI2/1aijWXkUrSMpQ1GVTBKs1ArUokrNWHrXeWInGPp3pEj+9C4t6fnzGu
+QA8zL61yt2ZL5bAedYolWklIkZpbo/5U33tdQP8Jm/HUnbrMLucW1Ar2WV556+1S
+2D3DyJ6gkJ17wR/6XwwQAwZvvNtBIKtWvjS+pCgKzlb2l+jyFeUDaFdCKoxCsYvw
+8UMjBNcWYzA6jqmseR+iCxTiGz/kXScOZ9RiFAARGP8yaLNjNZQDPv2Mdm6w7BGB
+E2K/gxNjq5v6aq2YH8uWkN+/A19UzKwr0GItXWFZHFMUQId5gQre57hvYYlcKbbk
+wBQoEmE5IfyLizIOHVUZ8HwTLRXi3eZjuGcDM4cviGdCsCfPJSLrLwQXcKKdmXB7
+6PbucNbPWgHH7V3ny/yi1OeKn2EPM8izxuOZmE6ck4akf+HuAY/NJI2D7dYhZs2P
+GbrvG4NaRQwTbrrykAcKvFfRb+Wle4YNCf11akm5bHLxAwQ=
+-----END CERTIFICATE-----

data/modules/mu/cleanup.rb

@@ -40,8 +40,10 @@ module MU
     # @param verbosity [Integer]: Debug level for MU.log output
     # @param web [Boolean]: Generate web-friendly output.
     # @param ignoremaster [Boolean]: Ignore the tags indicating the originating MU master server when deleting.
+    # @param regions [Array<String>]: Operate only on these regions
+    # @param habitats [Array<String>]: Operate only on these accounts/projects/subscriptions
     # @return [void]
-    def self.run(deploy_id, noop: false, skipsnapshots: false, onlycloud: false, verbosity: MU::Logger::NORMAL, web: false, ignoremaster: false, skipcloud: false, mommacat: nil, credsets: nil, regions: nil)
+    def self.run(deploy_id, noop: false, skipsnapshots: false, onlycloud: false, verbosity: MU::Logger::NORMAL, web: false, ignoremaster: false, skipcloud: false, mommacat: nil, credsets: nil, regions: nil, habitats: nil)
       MU.setLogging(verbosity, web)
       @noop = noop
       @skipsnapshots = skipsnapshots
@@ -92,8 +94,8 @@ module MU
       if !@skipcloud
         creds = {}
         MU::Cloud.availableClouds.each { |cloud|
+          cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
           if $MU_CFG[cloud.downcase] and $MU_CFG[cloud.downcase].size > 0
-            cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
             creds[cloud] ||= {}
             cloudclass.listCredentials.each { |credset|
               next if credsets and credsets.size > 0 and !credsets.include?(credset)
@@ -101,6 +103,11 @@ module MU
               MU.log "Will scan #{cloud} with credentials #{credset}"
               creds[cloud][credset] = cloudclass.listRegions(credentials: credset)
             }
+          else
+            if cloudclass.hosted?
+              creds[cloud] ||= {}
+              creds[cloud]["#default"] = cloudclass.listRegions
+            end
           end
         }
 
@@ -139,7 +146,9 @@ module MU
                   Thread.abort_on_exception = false
                   MU.setVar("curRegion", r)
                   projects = []
-                  if $MU_CFG[cloud.downcase][credset]["project"]
+                  if $MU_CFG and $MU_CFG[cloud.downcase] and
+                     $MU_CFG[cloud.downcase][credset] and
+                     $MU_CFG[cloud.downcase][credset]["project"]
 # XXX GCP credential schema needs an array for projects
                     projects << $MU_CFG[cloud.downcase][credset]["project"]
                   end
@@ -160,6 +169,9 @@ module MU
                   projectthreads = []
                   projects.each { |project|
                     next if !habitatclass.isLive?(project, credset)
+                    if habitats and !habitats.empty? and project != ""
+                      next if !habitats.include?(project)
+                    end
 
                     projectthreads << Thread.new {
                       MU.dupGlobals(parent_thread_id)

data/modules/mu/clouds/aws/bucket.rb

@@ -92,6 +92,12 @@ module MU
           current = cloud_desc
 
           if @config['policies']
+            @config['policies'].each { |pol|
+              pol['grant_to'] ||= [
+                { "id" => "*" }
+              ]
+            }
+
             policy_docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['policies'], deploy_obj: @deploy)
             policy_docs.each { |doc|
               MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: doc.values.first

data/modules/mu/clouds/aws/endpoint.rb

@@ -35,6 +35,8 @@ module MU
 
           # TODO guard this crap so we don't touch it if there are no changes
           @config['methods'].each { |m|
+            m["auth"] ||= m["iam_role"] ? "AWS_IAM" : "NONE"
+
             method_arn = "arn:#{MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws"}:execute-api:#{@config["region"]}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{@cloud_id}/*/#{m['type']}/#{m['path']}"
 
             resp = MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials']).get_resources(

data/modules/mu/clouds/aws/firewall_rule.rb

@@ -495,6 +495,37 @@ module MU
             rescue Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse
               if retries < 10
                 MU.log "EC2 Security Group #{sg.group_name} is still in use, waiting...", MU::NOTICE
+                # try to get out from under loose network interfaces with which
+                # we're associated
+                if sg.vpc_id
+                  # get the default SG for this VPC
+                  default_resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
+                    filters: [
+                      { name: "group-name", values: ["default"] },
+                      { name: "vpc-id", values: [sg.vpc_id] }
+                    ]
+                  ).security_groups
+                  if default_resp and default_resp.size == 1
+                    default_sg = default_resp.first.group_id
+                    eni_resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
+                      filters: [ {name: "group-id", values: [sg.group_id]} ]
+                    )
+                    if eni_resp and eni_resp.data and
+                       eni_resp.data.network_interfaces
+                      eni_resp.data.network_interfaces.each { |iface|
+                        iface_groups = iface.groups.map { |sg| sg.group_id }
+                        iface_groups.delete(sg.group_id)
+                        iface_groups << default_sg if iface_groups.empty?
+                        MU.log "Attempting to remove #{sg.group_id} from ENI #{iface.network_interface_id}"
+                        MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_network_interface_attribute(
+                          network_interface_id: iface.network_interface_id,
+                          groups: iface_groups
+                        )
+                      }
+                    end
+                  end
+                end
+
                 sleep 10
                 retries = retries + 1
                 retry

data/modules/mu/clouds/aws/function.rb

@@ -97,8 +97,11 @@ module MU
                 sgs << sg.cloud_id if sg and sg.cloud_id
               }
             end
+            if !@vpc
+              raise MuError, "Function #{@config['name']} had a VPC configured, but none was loaded"
+            end
             lambda_properties[:vpc_config] = {
-              :subnet_ids => @config['vpc']['subnets'].map { |s| s["subnet_id"] },
+              :subnet_ids => @vpc.subnets.map { |s| s.cloud_id },
               :security_group_ids => sgs
             }
           end
@@ -407,8 +410,48 @@ MU.log shortname, MU::NOTICE, details: function.configuration.role
         # @param config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
         def self.schema(config)
-          toplevel_required = []
+          toplevel_required = ["runtime"]
           schema = {
+            "triggers" => {
+              "type" => "array",
+              "items" => {
+                "type" => "object",
+                "description" => "Trigger for lambda function",
+                "required" => ["service"],
+                "properties" => {
+                  "service" => {
+                    "type" => "string",
+                    "enum" => %w{apigateway events s3 sns sqs dynamodb kinesis ses cognito alexa iot},
+                    "description" => "The name of the AWS service that will trigger this function"
+                  },
+                  "name" => {
+                    "type" => "string",
+                    "description" => "The name of the API Gateway, Cloudwatch Event, or other event trigger object"
+                  }
+                }
+              }
+            },
+            "runtime" => {
+              "type" => "string",
+              "enum" => %w{nodejs nodejs4.3 nodejs6.10 nodejs8.10 nodejs10.x nodejs12.x java8 java11 python2.7 python3.6 python3.7 python3.8 dotnetcore1.0 dotnetcore2.0 dotnetcore2.1 nodejs4.3-edge go1.x ruby2.5 provided},
+            },
+            "code" => {
+              "type" => "object",  
+              "properties" => {  
+                "s3_bucket" => {
+                  "type" => "string",
+                  "description" => "An S3 bucket where the deployment package can be found. Must be used in conjunction with s3_key."
+                }, 
+                "s3_key" => {
+                  "type" => "string",
+                  "description" => "Key in s3_bucket where the deployment package can be found. Must be used in conjunction with s3_bucket."
+                }, 
+                "s3_object_version" => {
+                  "type" => "string",
+                  "description" => "Specify an S3 object version for the deployment package, instead of the current default"
+                }, 
+              }
+            },
             "iam_role" => {
               "type" => "string",
               "description" => "Deprecated, +role+ is now preferred. The name of an IAM role for our Lambda function to assume. Can refer to an existing IAM role, or a sibling 'role' resource in Mu. If not specified, will create a default role with permissions listed in `permissions` (and if none are listed, we will set `AWSLambdaBasicExecutionRole`)."

data/modules/mu/clouds/aws/loadbalancer.rb

@@ -598,6 +598,7 @@ module MU
         # @param targetgroups [Array<String>] The target group(s) of which this node should be made a member. Not applicable to classic LoadBalancers. If not supplied, the node will be registered to all available target groups on this LoadBalancer.
         def registerNode(instance_id, targetgroups: nil)
           if @config['classic'] or !@config.has_key?("classic")
+            MU.log "Registering #{instance_id} to ELB #{@cloud_id}"
             MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).register_instances_with_load_balancer(
               load_balancer_name: @cloud_id,
               instances: [
@@ -608,11 +609,12 @@ module MU
             if targetgroups.nil? or !targetgroups.is_a?(Array) or targetgroups.size == 0
               if @targetgroups.nil?
                 cloud_desc
-                return
+                return if @targetgroups.nil?
               end
               targetgroups = @targetgroups.keys
             end
             targetgroups.each { |tg|
+              MU.log "Registering #{instance_id} to Target Group #{tg}"
               MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).register_targets(
                 target_group_arn: @targetgroups[tg].target_group_arn,
                 targets: [

data/modules/mu/clouds/aws/role.rb

@@ -1126,6 +1126,7 @@ end
               if policy["grant_to"] # XXX factor this with target, they're too similar
                 statement["Principal"] ||= []
                 policy["grant_to"].each { |grantee|
+                  grantee["identifier"] ||= grantee["id"]
                   if grantee["type"] and deploy_obj
                     sibling = deploy_obj.findLitterMate(
                       name: grantee["identifier"],
@@ -1147,6 +1148,7 @@ end
               end
               if policy["targets"]
                 policy["targets"].each { |target|
+                  target["identifier"] ||= target["id"]
                   if target["type"] and deploy_obj
                     sibling = deploy_obj.findLitterMate(
                       name: target["identifier"],
@@ -1154,7 +1156,7 @@ end
                     )
                     if sibling
                       id = sibling.cloudobj.arn
-                      id.sub!(/:([^:]+)$/, ":"+target["path"]) if target["path"]
+                      id.sub!(/:([^:]+)$/, ":"+'\1'+target["path"]) if target["path"]
                       statement["Resource"] << id
                       if id.match(/:log-group:/)
                         stream_id = id.sub(/:([^:]+)$/, ":log-stream:*")

data/modules/mu/clouds/aws/server.rb

@@ -756,7 +756,7 @@ module MU
 
           nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name = getSSHConfig
           if subnet.private? and !nat_ssh_host and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
-            raise MuError, "#{node} is in a private subnet (#{subnet}), but has no NAT host configured, and I have no other route to it"
+            raise MuError, "#{node} is in a private subnet (#{subnet}), but has no bastion host configured, and I have no other route to it"
           end
 
           # If we've asked for additional subnets (and this @config is not a
@@ -764,7 +764,7 @@ module MU
           # extra interfaces to accomodate.
           if !@config['vpc']['subnets'].nil? and @config['basis'].nil?
             device_index = 1
-            @vpc.subnets { |s|
+            @vpc.subnets.each { |s|
               subnet_id = s.cloud_id
               MU.log "Adding network interface on subnet #{subnet_id} for #{node}"
               iface = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_network_interface(subnet_id: subnet_id).network_interface
@@ -942,6 +942,8 @@ module MU
         # we're done.
         if MU.inGem?
           MU.log "Deploying from a gem, not grooming"
+          MU::MommaCat.unlock(instance.instance_id+"-orchestrate")
+          MU::MommaCat.unlock(instance.instance_id+"-groom")
 
           return true
         elsif @groomer.haveBootstrapped?
@@ -2233,8 +2235,12 @@ module MU
 
           if ips.size > 0 and !onlycloud
             known_hosts_files = [Etc.getpwuid(Process.uid).dir+"/.ssh/known_hosts"]
-            if Etc.getpwuid(Process.uid).name == "root"
-              known_hosts_files << Etc.getpwnam("nagios").dir+"/.ssh/known_hosts"
+            if Etc.getpwuid(Process.uid).name == "root" and !MU.inGem?
+              begin
+                known_hosts_files << Etc.getpwnam("nagios").dir+"/.ssh/known_hosts"
+              rescue ArgumentError
+                # we're in a non-nagios environment and that's ok
+              end
             end
             known_hosts_files.each { |known_hosts|
               next if !File.exist?(known_hosts)

data/modules/mu/clouds/aws/server_pool.rb

@@ -1409,7 +1409,7 @@ module MU
                 found = false
                 @deploy.deployment["loadbalancers"].each_pair { |lb_name, deployed_lb|
                   if lb_name == lb['concurrent_load_balancer']
-                    lbs << deployed_lb["awsname"]
+                    lbs << deployed_lb["awsname"] # XXX check for classic
                     if deployed_lb.has_key?("targetgroups")
                       deployed_lb["targetgroups"].each_pair { |tg_name, tg_arn|
                         tg_arns << tg_arn
@@ -1423,8 +1423,9 @@ module MU
             }
             if tg_arns.size > 0
               asg_options[:target_group_arns] = tg_arns
-            else
-              asg_options[:load_balancer_names] = lbs
+            end
+            if lbs.size > 0
+#              asg_options[:load_balancer_names] = lbs
             end
           end
           asg_options[:termination_policies] = @config["termination_policies"] if @config["termination_policies"]