cloud-mu 3.1.3 → 3.3.0

data/modules/mu/config/search_domain.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/search_domain.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/search_domain.rb
     class SearchDomain
 
       # Base configuration schema for a SearchDomain
@@ -47,10 +47,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::search_domains}, bare and unvalidated.
-      # @param dom [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _dom [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(dom, configurator)
+      def self.validate(_dom, _configurator)
         ok = true
         # This resource basically only exists in AWS, so the validation lives
         # there. If some other provider comes up with it we can factor

data/modules/mu/config/server.rb

@@ -14,9 +14,68 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/server.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/server.rb
     class Server
 
+      # Verify that a server or server_pool has a valid LDAP config referencing
+      # valid Vaults for credentials.
+      # @param server [Hash]
+      def self.checkVaultRefs(server)
+        ok = true
+        server['vault_access'] = [] if server['vault_access'].nil?
+        server['groomer'] ||= self.defaultGroomer
+        groomclass = MU::Groomer.loadGroomer(server['groomer'])
+
+        begin
+          if !server['active_directory'].nil?
+            ["domain_admin_vault", "domain_join_vault"].each { |vault_class|
+              server['vault_access'] << {
+                  "vault" => server['active_directory'][vault_class]['vault'],
+                  "item" => server['active_directory'][vault_class]['item']
+              }
+              item = groomclass.getSecret(
+                  vault: server['active_directory'][vault_class]['vault'],
+                  item: server['active_directory'][vault_class]['item'],
+              )
+              ["username_field", "password_field"].each { |field|
+                if !item.has_key?(server['active_directory'][vault_class][field])
+                  ok = false
+                  MU.log "I don't see a value named #{field} in Chef Vault #{server['active_directory'][vault_class]['vault']}:#{server['active_directory'][vault_class]['item']}", MU::ERR
+                end
+              }
+            }
+          end
+
+          if !server['windows_auth_vault'].nil?
+            server['use_cloud_provider_windows_password'] = false
+
+            server['vault_access'] << {
+                "vault" => server['windows_auth_vault']['vault'],
+                "item" => server['windows_auth_vault']['item']
+            }
+            item = groomclass.getSecret(
+              vault: server['windows_auth_vault']['vault'],
+              item: server['windows_auth_vault']['item']
+            )
+            ["password_field", "ec2config_password_field", "sshd_password_field"].each { |field|
+              if !item.has_key?(server['windows_auth_vault'][field])
+                MU.log "No value named #{field} in Chef Vault #{server['windows_auth_vault']['vault']}:#{server['windows_auth_vault']['item']}, will use a generated password.", MU::NOTICE
+                server['windows_auth_vault'].delete(field)
+              end
+            }
+          end
+          # Check all of the non-special ones while we're at it
+          server['vault_access'].each { |v|
+            next if v['vault'] == "splunk" and v['item'] == "admin_user"
+            item = groomclass.getSecret(vault: v['vault'], item: v['item'])
+          }
+        rescue MuError
+          MU.log "Can't load a Chef Vault I was configured to use. Does it exist?", MU::ERR
+          ok = false
+        end
+        return ok
+      end
+
       # Generate schema for a storage volume
       # @return [Hash]
       def self.storage_primitive
@@ -332,9 +391,8 @@ module MU
           },
           "userdata_script" => userdata_primitive,
           "windows_admin_username" => {
-              "type" => "string",
-              "default" => "Administrator",
-              "description" => "Use an alternate Windows account for Administrator functions. Will change the name of the Administrator account, if it has not already been done."
+            "type" => "string",
+            "description" => "Use an alternate Windows account for Administrator functions. Will change the name of the Administrator account, if it has not already been done."
           },
           "windows_auth_vault" => {
               "type" => "object",
@@ -370,60 +428,30 @@ module MU
               }
           },
           "ssh_user" => {
-              "type" => "string",
-              "default" => "root",
-              "default_if" => [
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "windows",
-                      "set" => "Administrator"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "win2k12",
-                      "set" => "Administrator"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "win2k12r2",
-                      "set" => "Administrator"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "win2k16",
-                      "set" => "Administrator"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "ubuntu",
-                      "set" => "ubuntu"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "ubuntu14",
-                      "set" => "ubuntu"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "centos7",
-                      "set" => "centos"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "rhel7",
-                      "set" => "ec2-user"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "rhel71",
-                      "set" => "ec2-user"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "amazon",
-                      "set" => "ec2-user"
-                  }
-              ]
+            "type" => "string",
+            "default" => "root",
+            "default_if" => [
+              {
+                "key_is" => "platform",
+                "value_is" => "centos",
+                "set" => "centos"
+              },
+              {
+                "key_is" => "platform",
+                "value_is" => "centos6",
+                "set" => "centos"
+              },
+              {
+                "key_is" => "platform",
+                "value_is" => "centos7",
+                "set" => "centos"
+              },
+              {
+                "key_is" => "platform",
+                "value_is" => "centos8",
+                "set" => "centos"
+              }
+            ]
           },
           "use_cloud_provider_windows_password" => {
               "type" => "boolean",
@@ -518,7 +546,7 @@ module MU
           "additionalProperties" => false,
           "description" => "Create individual server instances.",
           "properties" => {
-              "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true),
+              "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true, embedded_type: "server"),
               "bastion" => {
                 "type" => "boolean",
                 "default" => false,
@@ -595,7 +623,14 @@ module MU
         server['ingress_rules'] ||= []
         server['vault_access'] ||= []
         server['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
-        ok = false if !MU::Config.check_vault_refs(server)
+        ok = false if !MU::Config::Server.checkVaultRefs(server)
+
+        server['groomer'] ||= self.defaultGroomer
+        groomclass = MU::Groomer.loadGroomer(server['groomer'])
+        if !groomclass.available?(server['platform'].match(/^win/))
+          MU.log "Groomer #{server['groomer']} for #{server['name']} is missing or has incomplete dependencies", MU::ERR
+          ok = false
+        end
 
         if server["cloud"] != "Azure"
           server['dependencies'] << configurator.adminFirewallRuleset(vpc: server['vpc'], region: server['region'], cloud: server['cloud'], credentials: server['credentials'])
@@ -615,20 +650,12 @@ module MU
           end
 
           if !server["vpc"]["subnet_name"].nil? and configurator.nat_routes.has_key?(server["vpc"]["subnet_name"]) and !configurator.nat_routes[server["vpc"]["subnet_name"]].empty?
-            server["dependencies"] << {
-              "type" => "server",
-              "name" => configurator.nat_routes[server["vpc"]["subnet_name"]],
-              "phase" => "groom"
-            }
+            MU::Config.addDependency(server, configurator.nat_routes[server["vpc"]["subnet_name"]], "server", phase: "groom", no_create_wait: true)
           elsif !server["vpc"]["name"].nil?
             siblingvpc = configurator.haveLitterMate?(server["vpc"]["name"], "vpcs")
             if siblingvpc and siblingvpc['bastion'] and
-               server['name'] != siblingvpc['bastion'].to_h['name']
-              server["dependencies"] << {
-                "type" => "server",
-                "name" => siblingvpc['bastion'].to_h['name'],
-                "phase" => "groom"
-              }
+               server['name'] != siblingvpc['bastion']['name']
+              MU::Config.addDependency(server, siblingvpc['bastion']['name'], "server", phase: "groom", no_create_wait: true)
             end
           end
         end

data/modules/mu/config/server.yml

@@ -7,6 +7,7 @@ vpc:
 platform: ubuntu
 ssh_user: ubuntu
 associate_public_ip: true
+add_private_ips: 3
 canned_iam_policies:
 - AmazonDynamoDBReadOnlyAccess
 - AmazonElastiCacheFullAccess

data/modules/mu/config/server_pool.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/server_pool.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/server_pool.rb
     class ServerPool
 
       # Base configuration schema for a ServerPool
@@ -178,7 +178,7 @@ module MU
         pool['ingress_rules'] ||= []
         pool['vault_access'] ||= []
         pool['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
-        ok = false if !MU::Config.check_vault_refs(pool)
+        ok = false if !MU::Config::Server.checkVaultRefs(pool)
 
         if !pool['scrub_mu_isms'] and pool["cloud"] != "Azure"
           pool['dependencies'] << configurator.adminFirewallRuleset(vpc: pool['vpc'], region: pool['region'], cloud: pool['cloud'], credentials: pool['credentials'])
@@ -186,11 +186,7 @@ module MU
 
         if !pool["vpc"].nil?
           if !pool["vpc"]["subnet_name"].nil? and configurator.nat_routes.has_key?(pool["vpc"]["subnet_name"])
-            pool["dependencies"] << {
-              "type" => "pool",
-              "name" => configurator.nat_routes[pool["vpc"]["subnet_name"]],
-              "phase" => "groom"
-            }
+            MU::Config.addDependency(pool, configurator.nat_routes[pool["vpc"]["subnet_name"]], "server", phase: "groom", no_create_wait: true)
           end
         end
 # TODO make sure this is handled... somewhere
@@ -202,8 +198,8 @@ module MU
 #            ok = false if !insertKitten(alarm, "alarms")
 #          }
 #        end
-        if pool["basis"]["server"] != nil
-          pool["dependencies"] << {"type" => "server", "name" => pool["basis"]["server"]}
+        if pool["basis"] and pool["basis"]["server"]
+          MU::Config.addDependency(pool, pool["basis"]["server"], "server", phase: "groom")
         end
         if !pool['static_ip'].nil? and !pool['ip'].nil?
           ok = false

data/modules/mu/config/storage_pool.rb

@@ -14,7 +14,7 @@
 
 module MU
   class Config
-    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/storage_pool.rb
+    # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/storage_pool.rb
     class StoragePool
 
       # Base configuration schema for a StoragePool

data/modules/mu/config/tail.rb

@@ -0,0 +1,200 @@
+# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+
+  # Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
+  class Config
+
+    # A wrapper for config leaves that came from ERB parameters instead of raw
+    # YAML or JSON. Will behave like a string for things that expect that
+    # sort of thing. Code that needs to know that this leaf was the result of
+    # a parameter will be able to tell by the object class being something
+    # other than a plain string, array, or hash.
+    class Tail
+      @value = nil
+      @name = nil
+      @prettyname = nil
+      @description = nil
+      @prefix = ""
+      @suffix = ""
+      @is_list_element = false
+      @pseudo = false
+      @runtimecode = nil
+      @valid_values = []
+      @index = 0
+      attr_reader :description
+      attr_reader :pseudo
+      attr_reader :index
+      attr_reader :value
+      attr_reader :runtimecode
+      attr_reader :valid_values
+      attr_reader :is_list_element
+
+      def initialize(name, value, prettyname = nil, cloudtype = "String", valid_values = [], description = "", is_list_element = false, prefix: "", suffix: "", pseudo: false, runtimecode: nil, index: 0)
+        @name = name
+        @bindings = {}
+        @value = value
+        @valid_values = valid_values
+        @pseudo = pseudo
+        @index = index
+        @runtimecode = runtimecode
+        @cloudtype = cloudtype
+        @is_list_element = is_list_element
+        @description ||= 
+          if !description.nil?
+            description
+          else
+            ""
+          end
+        @prettyname ||= 
+          if !prettyname.nil?
+            prettyname
+          else
+            @name.capitalize.gsub(/[^a-z0-9]/i, "")
+          end
+        @prefix = prefix if !prefix.nil?
+        @suffix = suffix if !suffix.nil?
+      end
+ 
+      # Return the parameter name of this Tail
+      def getName
+        @name
+      end
+      # Return the platform-specific cloud type of this Tail
+      def getCloudType
+        @cloudtype
+      end
+      # Return the human-friendly name of this Tail
+      def getPrettyName
+        @prettyname
+      end
+      # Walk like a String
+      def to_s
+        @prefix.to_s+@value.to_s+@suffix.to_s
+      end
+      # Quack like a String
+      def to_str
+        to_s
+      end
+      # Upcase like a String
+      def upcase
+        to_s.upcase
+      end
+      # Downcase like a String
+      def downcase
+        to_s.downcase
+      end
+      # Check for emptiness like a String
+      def empty?
+        to_s.empty?
+      end
+      # Match like a String
+      def match(*args)
+        to_s.match(*args)
+      end
+      # Check for equality like a String
+      def ==(o)
+        (o.class == self.class or o.class == "String") && o.to_s == to_s
+      end
+      # Concatenate like a string
+      def +(o)
+        return to_s if o.nil?
+        to_s + o.to_s
+      end
+      # Perform global substitutions like a String
+      def gsub(*args)
+        to_s.gsub(*args)
+      end
+
+      # Lets callers access us like a {Hash}
+      # @param attribute [String,Symbol]
+      def [](attribute)
+        if respond_to?(attribute.to_sym)
+          send(attribute.to_sym)
+        else
+          nil
+        end
+      end
+    end
+
+    # Wrapper method for creating a {MU::Config::Tail} object as a reference to
+    # a parameter that's valid in the loaded configuration.
+    # @param param [<String>]: The name of the parameter to which this should be tied.
+    # @param value [<String>]: The value of the parameter to return when asked
+    # @param prettyname [<String>]: A human-friendly parameter name to be used when generating CloudFormation templates and the like
+    # @param cloudtype [<String>]: A platform-specific identifier used by cloud layers to identify a parameter's type, e.g. AWS::EC2::VPC::Id
+    # @param valid_values [Array<String>]: A list of acceptable String values for the given parameter.
+    # @param description [<String>]: A long-form description of what the parameter does.
+    # @param list_of [<String>]: Indicates that the value should be treated as a member of a list (array) by the cloud layer.
+    # @param prefix [<String>]: A static String that should be prefixed to the stored value when queried
+    # @param suffix [<String>]: A static String that should be appended to the stored value when queried
+    # @param pseudo [<Boolean>]: This is a pseudo-parameter, automatically provided, and not available as user input.
+    # @param runtimecode [<String>]: Actual code to allow the cloud layer to interpret literally in its own idiom, e.g. '"Ref" : "AWS::StackName"' for CloudFormation
+    def getTail(param, value: nil, prettyname: nil, cloudtype: "String", valid_values: [], description: nil, list_of: nil, prefix: "", suffix: "", pseudo: false, runtimecode: nil)
+      param = param.gsub(/[^a-z0-9_]/i, "_")
+      if value.nil?
+        if @@parameters.nil? or !@@parameters.has_key?(param)
+          MU.log "Parameter '#{param}' (#{param.class.name}) referenced in config but not provided (#{caller[0]})", MU::DEBUG, details: @@parameters
+          return nil
+#          raise DeployParamError
+        else
+          value = @@parameters[param]
+        end
+      end
+      if !prettyname.nil?
+        prettyname.gsub!(/[^a-z0-9]/i, "") # comply with CloudFormation restrictions
+      end
+      if value.is_a?(MU::Config::Tail)
+        MU.log "Parameter #{param} is using a nested parameter as a value. This rarely works, depending on the target cloud. YMMV.", MU::WARN
+        tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
+      elsif !list_of.nil? or (@@tails.has_key?(param) and @@tails[param].is_a?(Array))
+        tail = []
+        count = 0
+        value.split(/\s*,\s*/).each { |subval|
+          if @@tails.has_key?(param) and !@@tails[param][count].nil?
+            subval = @@tails[param][count].values.first.to_s if subval.nil?
+            list_of = @@tails[param][count].values.first.getName if list_of.nil?
+            prettyname = @@tails[param][count].values.first.getPrettyName if prettyname.nil?
+            description = @@tails[param][count].values.first.description if description.nil?
+            valid_values = @@tails[param][count].values.first.valid_values if valid_values.nil? or valid_values.empty?
+            cloudtype = @@tails[param][count].values.first.getCloudType if @@tails[param][count].values.first.getCloudType != "String"
+          end
+          prettyname = param.capitalize if prettyname.nil?
+          tail << { list_of => MU::Config::Tail.new(list_of, subval, prettyname, cloudtype, valid_values, description, true, pseudo: pseudo, index: count) }
+          count = count + 1
+        }
+      else
+        if @@tails.has_key?(param)
+          pseudo = @@tails[param].pseudo
+          value = @@tails[param].to_s if value.nil?
+          prettyname = @@tails[param].getPrettyName if prettyname.nil?
+          description = @@tails[param].description if description.nil?
+          valid_values = @@tails[param].valid_values if valid_values.nil? or valid_values.empty?
+          cloudtype = @@tails[param].getCloudType if @@tails[param].getCloudType != "String"
+        end
+        tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
+      end
+
+      if valid_values and valid_values.size > 0 and value
+        if !valid_values.include?(value)
+          raise DeployParamError, "Invalid parameter value '#{value}' supplied for '#{param}'"
+        end
+      end
+      @@tails[param] = tail
+
+      tail
+    end
+  end
+end