cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

data/modules/mu/config/database.yml

@@ -1,7 +1,7 @@
 <% if $complexity == "complex" %>
 
 name: database-complex
-size: <%= db_size %>
+size: db.r4.large
 engine: postgres
 engine_version: 9.6.6
 storage: 5
@@ -20,7 +20,7 @@ region: us-west-2
 
 name: database-simple
 size: <%= db_size %>
-engine: postgres
+engine: mariadb
 storage: 5
 
-<% end %>
+<% end %>

data/modules/mu/config/function.rb

@@ -24,7 +24,7 @@ module MU
         "type" => "object",
         "title" => "Function",
         "description" => "Create a cloud function.",
-        "required" => ["name", "cloud","runtime","iam_role","handler","code","region"],
+        "required" => ["name", "cloud","runtime","handler","code","region"],
         "additionalProperties" => false,
         "properties" => {
           "cloud" => MU::Config.cloud_primitive,
@@ -33,7 +33,6 @@ module MU
             "type" => "string",
             "enum" => %w{nodejs nodejs4.3 nodejs6.10 nodejs8.10 java8 python2.7 python3.6 dotnetcore1.0 dotnetcore2.0 dotnetcore2.1 nodejs4.3-edge go1.x}
           },
-          "iam_role" => {"type" => "string"},
           "region" => MU::Config.region_primitive,
           "vpc" => MU::Config::VPC.reference(MU::Config::VPC::ONE_SUBNET+MU::Config::VPC::MANY_SUBNETS, MU::Config::VPC::NO_NAT_OPTS, "all_private"),
           "handler" => {

data/modules/mu/config/{project.rb → habitat.rb}

@@ -15,15 +15,15 @@
 module MU
   class Config
     # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/project.rb
-    class Project
+    class Habitat
 
-      # Base configuration schema for a Project
+      # Base configuration schema for a Habitat
       # @return [Hash]
       def self.schema
         {
           "type" => "object",
           "additionalProperties" => false,
-          "description" => "Set up a cloud account (AWS account, Google Cloud project, etc)",
+          "description" => "Generate a cloud habitat (AWS account, Google Cloud project, Azure Directory, etc)",
           "properties" => {
             "name" => { "type" => "string" },
             "folder" => MU::Config::Folder.reference
@@ -36,32 +36,32 @@ module MU
       def self.reference
         {
           "type" => "object",
-          "description" => "Deploy into or connect with resources in a specific account/project",
+          "description" => "Deploy into or connect with resources in a specific habitat (AWS account, GCP project, etc)",
           "minProperties" => 1,
           "additionalProperties" => false,
           "properties" => {
             "id" => {
               "type" => "string",
-              "description" => "Discover this account/project by looking for this cloud provider identifier, such as 836541910896 (an AWS account number) or my-project-196124 (a Google Cloud project id)"
+              "description" => "Discover this habitat by looking for this cloud provider identifier, such as 836541910896 (an AWS account number) or my-project-196124 (a Google Cloud project id)"
             },
             "name" => {
               "type" => "string",
-              "description" => "Discover this account/project by Mu-internal name; typically the shorthand 'name' field of an Account object declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
+              "description" => "Discover this habitat by Mu-internal name; typically the shorthand 'name' field of a Habitat object declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
             },
             "cloud" => MU::Config.cloud_primitive,
             "deploy_id" => {
               "type" => "string",
-              "description" => "Search for this account/project in an existing Mu deploy; specify a Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
+              "description" => "Search for this Habitat in an existing Mu deploy by Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
             }
           }
         }
       end
 
-      # Generic pre-processing of {MU::Config::BasketofKittens::project}, bare and unvalidated.
-      # @param project [Hash]: The resource to process and validate
+      # Generic pre-processing of {MU::Config::BasketofKittens::habitat}, bare and unvalidated.
+      # @param habitat [Hash]: The resource to process and validate
       # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(project, configurator)
+      def self.validate(habitat, configurator)
         ok = true
         ok
       end

data/modules/mu/config/notifier.rb

@@ -0,0 +1,85 @@
+# Copyright:: Copyright (c) 2018 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Config
+    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/notifier.rb
+    class Notifier
+
+      # Base configuration schema for a Notifier
+      # @return [Hash]
+      def self.schema
+        {
+          "type" => "object",
+          "additionalProperties" => false,
+          "description" => "A stub for inline resource that generate SNS notifications in AWS. This should really be expanded.",
+          "properties" => {
+            "name" => {
+              "type" => "string"
+            },
+            "region" => MU::Config.region_primitive,
+            "credentials" => MU::Config.credentials_primitive,
+            "subscriptions" => {
+              "type" => "array",
+              "description" => "A list of people or resources which should receive notifications",
+              "items" => {
+                "type" => "object",
+                "description" => "A list of people or resources which should receive notifications",
+                "required" => ["endpoint"],
+                "properties" => {
+                  "endpoint" => {
+                    "type" => "string",
+                    "description" => "The endpoint which should be subscribed to this notifier, typically an email address or SMS-enabled phone number."
+                  }
+                }
+              }
+            }
+          }
+        }
+      end
+
+      # Generic pre-processing of {MU::Config::BasketofKittens::notifiers}, bare and unvalidated.
+      # @param notifier [Hash]: The resource to process and validate
+      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @return [Boolean]: True if validation succeeded, False otherwise
+      def self.validate(notifier, configurator)
+        ok = true
+
+        if notifier['subscriptions']
+          notifier['subscriptions'].each { |sub|
+            if !sub["type"]
+              if sub["endpoint"].match(/^http:/i)
+                sub["type"] = "http"
+              elsif sub["endpoint"].match(/^https:/i)
+                sub["type"] = "https"
+              elsif sub["endpoint"].match(/^sqs:/i)
+                sub["type"] = "sqs"
+              elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
+                sub["type"] = "sms"
+              elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
+                sub["type"] = "email"
+              else
+                MU.log "Notifier #{notifier['name']} subscription #{sub['endpoint']} did not specify a type, and I'm unable to guess one", MU::ERR
+                ok = false
+              end
+            end
+          }
+        end
+
+        ok
+      end
+
+    end
+  end
+end

data/modules/mu/config/notifier.yml

@@ -0,0 +1,9 @@
+<% if $complexity == 'complex' %>
+name: notifier
+<% else %>
+name: notifier
+subscriptions:
+- endpoint: admin@example.com
+  type: email-json
+- endpoint: 703-555-5555
+<% end %>

data/modules/mu/config/role.rb

@@ -28,7 +28,7 @@ module MU
             "name" => {
               "type" => "string",
               "description" => "The name of a cloud provider role to create",
-              "pattern" => '^[a-zA-Z0-9]+$'
+              "pattern" => '^[a-zA-Z0-9_\-]+$'
             },
             "import" => {
               "type" => "array",

data/modules/mu/config/search_domain.yml

@@ -1,6 +1,6 @@
 <% if $complexity == "complex" %>
 
-name: search_domain-complex
+name: searchdomain-complex
 instance_type: t2.small.elasticsearch
 instance_count: 4
 dedicated_masters: 1
@@ -20,6 +20,6 @@ vpc:
 
 <% else %> # IF NOT COMPLEX THEN ASSUME SIMPLE
 
-name: search_domain-simple
+name: searchdomain-simple
 instance_type: t2.small.elasticsearch
 <% end %>

data/modules/mu/config/server.rb

@@ -136,6 +136,16 @@ module MU
               "default" => MU::Config.defaultGroomer,
               "enum" => MU.supportedGroomers
           },
+          "groom" => {
+              "type" => "boolean",
+              "default" => true,
+              "description" => "Whether to run a host configuration agent, e.g. Chef, when bootstrapping"
+          },
+          "groomer_timeout" => {
+              "type" => "integer",
+              "default" => 1800,
+              "description" => "Maximum execution time for a groomer run"
+          },
           "scrub_groomer" => {
               "type" => "boolean",
               "default" => false,
@@ -555,7 +565,9 @@ module MU
         server['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
         ok = false if !MU::Config.check_vault_refs(server)
 
-        server['dependencies'] << configurator.adminFirewallRuleset(vpc: server['vpc'], region: server['region'], cloud: server['cloud']) if !server['scrub_mu_isms']
+        if !server['scrub_mu_isms']
+          server['dependencies'] << configurator.adminFirewallRuleset(vpc: server['vpc'], region: server['region'], cloud: server['cloud'], credentials: server['credentials'])
+        end
 
         if !server["vpc"].nil?
           # Common mistake- using all_public or all_private subnet_pref for

data/modules/mu/config/server.yml

@@ -1,8 +1,8 @@
 <% if $complexity == 'complex' %>
-name: server-complex
+name: servercomplex
 size: <%= instance_type %>
 # TODO: BUILD OUT COMPLEX EXAMPLE
 <% else %>
-name: server-simple
+name: serversimple
 size: <%= instance_type %>
-<% end %>
+<% end %>

data/modules/mu/config/server_pool.rb

@@ -180,7 +180,9 @@ module MU
         pool['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
         ok = false if !MU::Config.check_vault_refs(pool)
 
-        pool['dependencies'] << configurator.adminFirewallRuleset(vpc: pool['vpc'], region: pool['region'], cloud: pool['cloud']) if !pool['scrub_mu_isms']
+        if !pool['scrub_mu_isms']
+          pool['dependencies'] << configurator.adminFirewallRuleset(vpc: pool['vpc'], region: pool['region'], cloud: pool['cloud'], credentials: pool['credentials'])
+        end
 
         if !pool["vpc"].nil?
           if !pool["vpc"]["subnet_name"].nil? and configurator.nat_routes.has_key?(pool["vpc"]["subnet_name"])

data/modules/mu/config/storage_pool.rb

@@ -96,6 +96,7 @@ module MU
                                         "storagepool '#{pool['name']}'",
                                         configurator,
                                         dflt_region: pool['region'],
+                                        credentials: pool['credentials'],
                                         is_sibling: true,
                                         sibling_vpcs: [siblingvpc])
                   ok = false
@@ -105,7 +106,8 @@ module MU
                                         "storage_pools",
                                         "storagepool #{pool['name']}",
                                         configurator,
-                                        dflt_region: pool['region'])
+                                        dflt_region: pool['region'],
+                                        credentials: pool['credentials'])
                   ok = false
                 end
               end

data/modules/mu/config/storage_pool.yml

@@ -0,0 +1,19 @@
+<% if $complexity == "complex" %>
+# XXX glue to other resources in test BoKs, if you want to test complexity
+name: efs
+mount_points:
+- name: mountpoint
+  directory: /efs
+  ingress_rules:
+  - port: 2049
+    hosts:
+    - 0.0.0.0/0
+<% else %> # IF NOT COMPLEX THEN ASSUME SIMPLE
+name: efs
+mount_points:
+- name: mountpoint
+  directory: /efs
+  ingress_rules:
+  - port: 2049
+    hosts:
+<% end %>

data/modules/mu/config/vpc.rb

@@ -102,8 +102,8 @@ module MU
                     "description" => "One or more other VPCs with which to attempt to create a peering connection.",
                     "properties" => {
                         "account" => {
-                            "type" => "string",
-                            "description" => "The AWS account which owns the target VPC."
+                          "type" => "string",
+                          "description" => "The AWS account which owns the target VPC."
                         },
                         "vpc" => reference(MANY_SUBNETS, NO_NAT_OPTS, "all")
                         #             "route_tables" => {
@@ -233,6 +233,7 @@ module MU
               "type" => "string",
               "description" => "Discover this VPC by looking for this cloud provider identifier."
             },
+            "credentials" => MU::Config.credentials_primitive,
             "vpc_name" => {
               "type" => "string",
               "description" => "Discover this VPC by Mu-internal name; typically the shorthand 'name' field of a VPC declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
@@ -400,16 +401,39 @@ module MU
       def self.resolvePeers(vpc, configurator)
         ok = true
         if !vpc["peers"].nil?
+          append = []
+          delete = []
           vpc["peers"].each { |peer|
             peer["#MU_CLOUDCLASS"] = Object.const_get("MU").const_get("Cloud").const_get("VPC")
+            # We check for multiple siblings because some implementations
+            # (Google) can split declared VPCs into parts to get the mimic the
+            # routing behaviors we expect.
+            siblings = configurator.haveLitterMate?(peer['vpc']["vpc_name"], "vpcs", has_multiple: true)
+
             # If we're peering with a VPC in this deploy, set it as a dependency
-            if !peer['vpc']["vpc_name"].nil? and
-               configurator.haveLitterMate?(peer['vpc']["vpc_name"], "vpcs") and
+            if !peer['vpc']["vpc_name"].nil? and siblings.size > 0 and
                peer["vpc"]['deploy_id'].nil? and peer["vpc"]['vpc_id'].nil?
+
               peer['vpc']['cloud'] = vpc['cloud'] if peer['vpc']['cloud'].nil?
-              vpc["dependencies"] << {
-                "type" => "vpc",
-                "name" => peer['vpc']["vpc_name"]
+              siblings.each { |sib|
+                if sib['name'] != peer['vpc']["vpc_name"]
+                  if sib['name'] != vpc['name']
+                    append_me = { "vpc" => peer["vpc"].dup }
+                    append_me['vpc']['vpc_name'] = sib['name']
+                    append << append_me
+                    vpc["dependencies"] << {
+                      "type" => "vpc",
+                      "name" => sib['name']
+                    }
+                  end
+                  delete << peer
+                else
+                  vpc["dependencies"] << {
+                    "type" => "vpc",
+                    "name" => peer['vpc']["vpc_name"]
+                  }
+                end
+                delete << peer if sib['name'] == vpc['name']
               }
               # If we're using a VPC from somewhere else, make sure the flippin'
               # thing exists, and also fetch its id now so later search routines
@@ -426,6 +450,12 @@ module MU
               end
             end
           }
+          append.each { |append_me|
+            vpc["peers"] << append_me
+          }
+          delete.each { |delete_me|
+            vpc["peers"].delete(delete_me)
+          }
         end
         ok
       end
@@ -439,7 +469,7 @@ module MU
       # @param is_sibling [Boolean]:
       # @param sibling_vpcs [Array]:
       # @param dflt_region [String]:
-      def self.processReference(vpc_block, parent_type, parent_name, configurator, is_sibling: false, sibling_vpcs: [], dflt_region: MU.curRegion)
+      def self.processReference(vpc_block, parent_type, parent_name, configurator, is_sibling: false, sibling_vpcs: [], dflt_region: MU.curRegion, credentials: nil)
         puts vpc_block.ancestors if !vpc_block.is_a?(Hash)
         if !vpc_block.is_a?(Hash) and vpc_block.kind_of?(MU::Cloud::VPC)
           return true
@@ -450,6 +480,19 @@ module MU
           vpc_block['region'] = dflt_region.to_s
         end
 
+        vpc_block['credentials'] ||= credentials if credentials
+
+        # Sometimes people set subnet_pref to "private" or "public" when they
+        # mean "all_private" or "all_public." Help them out.
+        if parent_type and 
+           MU::Config.schema["properties"][parent_type] and
+           MU::Config.schema["properties"][parent_type]["items"]["properties"]["vpc"] and
+           MU::Config.schema["properties"][parent_type]["items"]["properties"]["vpc"]["properties"].has_key?("subnets") and
+           !MU::Config.schema["properties"][parent_type]["items"]["properties"]["vpc"]["properties"].has_key?("subnet_id")
+           vpc_block["subnet_pref"] = "all_public" if vpc_block["subnet_pref"] == "public"
+           vpc_block["subnet_pref"] = "all_private" if vpc_block["subnet_pref"] == "private"
+        end
+
         flags = {}
         flags["subnet_pref"] = vpc_block["subnet_pref"] if !vpc_block["subnet_pref"].nil?
 
@@ -464,6 +507,7 @@ module MU
                 deploy_id: vpc_block["deploy_id"],
                 cloud_id: vpc_block["vpc_id"],
                 name: vpc_block["vpc_name"],
+                credentials: vpc_block["credentials"],
                 tag_key: tag_key,
                 tag_value: tag_value,
                 region: vpc_block["region"],
@@ -472,6 +516,24 @@ module MU
               )
 
               ext_vpc = found.first if found.size == 1
+
+              # Make sure we don't have a weird mismatch between requested
+              # credential sets and the VPC we actually found
+              if ext_vpc and ext_vpc.cloudobj and ext_vpc.cloudobj.config and
+                 ext_vpc.cloudobj.config["credentials"]
+                if vpc_block['credentials'] and # probably can't happen
+                   vpc_block['credentials'] != ext_vpc.cloudobj.config["credentials"]
+                  ok = false
+                  MU.log "#{parent_type} #{parent_name} requested a VPC on credentials '#{vpc_block['credentials']}' but matched VPC is under credentials '#{ext_vpc.cloudobj.config["credentials"]}'", MU::ERR, details: vpc_block
+                end
+                if credentials and
+                   credentials != ext_vpc.cloudobj.config["credentials"]
+                  ok = false
+                  MU.log "#{parent_type} #{parent_name} is using credentials '#{credentials}' but matched VPC is under credentials '#{ext_vpc.cloudobj.config["credentials"]}'", MU::ERR, details: vpc_block
+                end
+                vpc_block['credentials'] ||= ext_vpc.cloudobj.config["credentials"]
+              end
+
             end
           rescue Exception => e
             raise MuError, e.inspect, e.backtrace

data/modules/mu/groomers/chef.rb

@@ -253,18 +253,17 @@ module MU
       # @param max_retries [Integer]: The maximum number of attempts at a successful run to make before giving up.
       # @param output [Boolean]: Display Chef's regular (non-error) output to the console
       # @param override_runlist [String]: Use the specified run list instead of the node's configured list
-      def run(purpose: "Chef run", update_runlist: true, max_retries: 5, output: true, override_runlist: nil, reboot_first_fail: false)
+      def run(purpose: "Chef run", update_runlist: true, max_retries: 5, output: true, override_runlist: nil, reboot_first_fail: false, timeout: 1800)
         self.class.loadChefLib
         if update_runlist and !@config['run_list'].nil?
           knifeAddToRunList(multiple: @config['run_list'])
         end
 
-        timeout = @server.windows? ? 1800 : 600
         pending_reboot_count = 0
         chef_node = ::Chef::Node.load(@server.mu_name)
         if !@config['application_attributes'].nil?
           MU.log "Setting node:#{@server.mu_name} application_attributes", MU::DEBUG, details: @config['application_attributes']
-          chef_node.normal.application_attributes = @config['application_attributes']
+          chef_node.normal['application_attributes'] = @config['application_attributes']
           chef_node.save
         end
         if @server.deploy.original_config.has_key?('parameters')