cloud-mu 3.1.3 → 3.1.4

data/modules/mu/config/search_domain.rb

@@ -47,10 +47,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::search_domains}, bare and unvalidated.
-      # @param dom [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _dom [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(dom, configurator)
+      def self.validate(_dom, _configurator)
         ok = true
         # This resource basically only exists in AWS, so the validation lives
         # there. If some other provider comes up with it we can factor

data/modules/mu/config/server.rb

@@ -17,6 +17,65 @@ module MU
     # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/server.rb
     class Server
 
+      # Verify that a server or server_pool has a valid LDAP config referencing
+      # valid Vaults for credentials.
+      # @param server [Hash]
+      def self.checkVaultRefs(server)
+        ok = true
+        server['vault_access'] = [] if server['vault_access'].nil?
+        server['groomer'] ||= self.defaultGroomer
+        groomclass = MU::Groomer.loadGroomer(server['groomer'])
+
+        begin
+          if !server['active_directory'].nil?
+            ["domain_admin_vault", "domain_join_vault"].each { |vault_class|
+              server['vault_access'] << {
+                  "vault" => server['active_directory'][vault_class]['vault'],
+                  "item" => server['active_directory'][vault_class]['item']
+              }
+              item = groomclass.getSecret(
+                  vault: server['active_directory'][vault_class]['vault'],
+                  item: server['active_directory'][vault_class]['item'],
+              )
+              ["username_field", "password_field"].each { |field|
+                if !item.has_key?(server['active_directory'][vault_class][field])
+                  ok = false
+                  MU.log "I don't see a value named #{field} in Chef Vault #{server['active_directory'][vault_class]['vault']}:#{server['active_directory'][vault_class]['item']}", MU::ERR
+                end
+              }
+            }
+          end
+
+          if !server['windows_auth_vault'].nil?
+            server['use_cloud_provider_windows_password'] = false
+
+            server['vault_access'] << {
+                "vault" => server['windows_auth_vault']['vault'],
+                "item" => server['windows_auth_vault']['item']
+            }
+            item = groomclass.getSecret(
+              vault: server['windows_auth_vault']['vault'],
+              item: server['windows_auth_vault']['item']
+            )
+            ["password_field", "ec2config_password_field", "sshd_password_field"].each { |field|
+              if !item.has_key?(server['windows_auth_vault'][field])
+                MU.log "No value named #{field} in Chef Vault #{server['windows_auth_vault']['vault']}:#{server['windows_auth_vault']['item']}, will use a generated password.", MU::NOTICE
+                server['windows_auth_vault'].delete(field)
+              end
+            }
+          end
+          # Check all of the non-special ones while we're at it
+          server['vault_access'].each { |v|
+            next if v['vault'] == "splunk" and v['item'] == "admin_user"
+            item = groomclass.getSecret(vault: v['vault'], item: v['item'])
+          }
+        rescue MuError
+          MU.log "Can't load a Chef Vault I was configured to use. Does it exist?", MU::ERR
+          ok = false
+        end
+        return ok
+      end
+
       # Generate schema for a storage volume
       # @return [Hash]
       def self.storage_primitive
@@ -332,9 +391,8 @@ module MU
           },
           "userdata_script" => userdata_primitive,
           "windows_admin_username" => {
-              "type" => "string",
-              "default" => "Administrator",
-              "description" => "Use an alternate Windows account for Administrator functions. Will change the name of the Administrator account, if it has not already been done."
+            "type" => "string",
+            "description" => "Use an alternate Windows account for Administrator functions. Will change the name of the Administrator account, if it has not already been done."
           },
           "windows_auth_vault" => {
               "type" => "object",
@@ -370,60 +428,30 @@ module MU
               }
           },
           "ssh_user" => {
-              "type" => "string",
-              "default" => "root",
-              "default_if" => [
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "windows",
-                      "set" => "Administrator"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "win2k12",
-                      "set" => "Administrator"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "win2k12r2",
-                      "set" => "Administrator"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "win2k16",
-                      "set" => "Administrator"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "ubuntu",
-                      "set" => "ubuntu"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "ubuntu14",
-                      "set" => "ubuntu"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "centos7",
-                      "set" => "centos"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "rhel7",
-                      "set" => "ec2-user"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "rhel71",
-                      "set" => "ec2-user"
-                  },
-                  {
-                      "key_is" => "platform",
-                      "value_is" => "amazon",
-                      "set" => "ec2-user"
-                  }
-              ]
+            "type" => "string",
+            "default" => "root",
+            "default_if" => [
+              {
+                "key_is" => "platform",
+                "value_is" => "centos",
+                "set" => "centos"
+              },
+              {
+                "key_is" => "platform",
+                "value_is" => "centos6",
+                "set" => "centos"
+              },
+              {
+                "key_is" => "platform",
+                "value_is" => "centos7",
+                "set" => "centos"
+              },
+              {
+                "key_is" => "platform",
+                "value_is" => "centos8",
+                "set" => "centos"
+              }
+            ]
           },
           "use_cloud_provider_windows_password" => {
               "type" => "boolean",
@@ -595,7 +623,7 @@ module MU
         server['ingress_rules'] ||= []
         server['vault_access'] ||= []
         server['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
-        ok = false if !MU::Config.check_vault_refs(server)
+        ok = false if !MU::Config::Server.checkVaultRefs(server)
 
         if server["cloud"] != "Azure"
           server['dependencies'] << configurator.adminFirewallRuleset(vpc: server['vpc'], region: server['region'], cloud: server['cloud'], credentials: server['credentials'])

data/modules/mu/config/server_pool.rb

@@ -178,7 +178,7 @@ module MU
         pool['ingress_rules'] ||= []
         pool['vault_access'] ||= []
         pool['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
-        ok = false if !MU::Config.check_vault_refs(pool)
+        ok = false if !MU::Config::Server.checkVaultRefs(pool)
 
         if !pool['scrub_mu_isms'] and pool["cloud"] != "Azure"
           pool['dependencies'] << configurator.adminFirewallRuleset(vpc: pool['vpc'], region: pool['region'], cloud: pool['cloud'], credentials: pool['credentials'])
@@ -202,7 +202,7 @@ module MU
 #            ok = false if !insertKitten(alarm, "alarms")
 #          }
 #        end
-        if pool["basis"]["server"] != nil
+        if pool["basis"] and pool["basis"]["server"]
           pool["dependencies"] << {"type" => "server", "name" => pool["basis"]["server"]}
         end
         if !pool['static_ip'].nil? and !pool['ip'].nil?

data/modules/mu/config/tail.rb

@@ -0,0 +1,189 @@
+# Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+
+  # Methods and structures for parsing Mu's configuration files. See also {MU::Config::BasketofKittens}.
+  class Config
+
+    # A wrapper for config leaves that came from ERB parameters instead of raw
+    # YAML or JSON. Will behave like a string for things that expect that
+    # sort of thing. Code that needs to know that this leaf was the result of
+    # a parameter will be able to tell by the object class being something
+    # other than a plain string, array, or hash.
+    class Tail
+      @value = nil
+      @name = nil
+      @prettyname = nil
+      @description = nil
+      @prefix = ""
+      @suffix = ""
+      @is_list_element = false
+      @pseudo = false
+      @runtimecode = nil
+      @valid_values = []
+      @index = 0
+      attr_reader :description
+      attr_reader :pseudo
+      attr_reader :index
+      attr_reader :value
+      attr_reader :runtimecode
+      attr_reader :valid_values
+      attr_reader :is_list_element
+
+      def initialize(name, value, prettyname = nil, cloudtype = "String", valid_values = [], description = "", is_list_element = false, prefix: "", suffix: "", pseudo: false, runtimecode: nil, index: 0)
+        @name = name
+        @bindings = {}
+        @value = value
+        @valid_values = valid_values
+        @pseudo = pseudo
+        @index = index
+        @runtimecode = runtimecode
+        @cloudtype = cloudtype
+        @is_list_element = is_list_element
+        @description ||= 
+          if !description.nil?
+            description
+          else
+            ""
+          end
+        @prettyname ||= 
+          if !prettyname.nil?
+            prettyname
+          else
+            @name.capitalize.gsub(/[^a-z0-9]/i, "")
+          end
+        @prefix = prefix if !prefix.nil?
+        @suffix = suffix if !suffix.nil?
+      end
+ 
+      # Return the parameter name of this Tail
+      def getName
+        @name
+      end
+      # Return the platform-specific cloud type of this Tail
+      def getCloudType
+        @cloudtype
+      end
+      # Return the human-friendly name of this Tail
+      def getPrettyName
+        @prettyname
+      end
+      # Walk like a String
+      def to_s
+        @prefix.to_s+@value.to_s+@suffix.to_s
+      end
+      # Quack like a String
+      def to_str
+        to_s
+      end
+      # Upcase like a String
+      def upcase
+        to_s.upcase
+      end
+      # Downcase like a String
+      def downcase
+        to_s.downcase
+      end
+      # Check for emptiness like a String
+      def empty?
+        to_s.empty?
+      end
+      # Match like a String
+      def match(*args)
+        to_s.match(*args)
+      end
+      # Check for equality like a String
+      def ==(o)
+        (o.class == self.class or o.class == "String") && o.to_s == to_s
+      end
+      # Concatenate like a string
+      def +(o)
+        return to_s if o.nil?
+        to_s + o.to_s
+      end
+      # Perform global substitutions like a String
+      def gsub(*args)
+        to_s.gsub(*args)
+      end
+    end
+
+    # Wrapper method for creating a {MU::Config::Tail} object as a reference to
+    # a parameter that's valid in the loaded configuration.
+    # @param param [<String>]: The name of the parameter to which this should be tied.
+    # @param value [<String>]: The value of the parameter to return when asked
+    # @param prettyname [<String>]: A human-friendly parameter name to be used when generating CloudFormation templates and the like
+    # @param cloudtype [<String>]: A platform-specific identifier used by cloud layers to identify a parameter's type, e.g. AWS::EC2::VPC::Id
+    # @param valid_values [Array<String>]: A list of acceptable String values for the given parameter.
+    # @param description [<String>]: A long-form description of what the parameter does.
+    # @param list_of [<String>]: Indicates that the value should be treated as a member of a list (array) by the cloud layer.
+    # @param prefix [<String>]: A static String that should be prefixed to the stored value when queried
+    # @param suffix [<String>]: A static String that should be appended to the stored value when queried
+    # @param pseudo [<Boolean>]: This is a pseudo-parameter, automatically provided, and not available as user input.
+    # @param runtimecode [<String>]: Actual code to allow the cloud layer to interpret literally in its own idiom, e.g. '"Ref" : "AWS::StackName"' for CloudFormation
+    def getTail(param, value: nil, prettyname: nil, cloudtype: "String", valid_values: [], description: nil, list_of: nil, prefix: "", suffix: "", pseudo: false, runtimecode: nil)
+      if value.nil?
+        if @@parameters.nil? or !@@parameters.has_key?(param)
+          MU.log "Parameter '#{param}' (#{param.class.name}) referenced in config but not provided (#{caller[0]})", MU::DEBUG, details: @@parameters
+          return nil
+#          raise DeployParamError
+        else
+          value = @@parameters[param]
+        end
+      end
+      if !prettyname.nil?
+        prettyname.gsub!(/[^a-z0-9]/i, "") # comply with CloudFormation restrictions
+      end
+      if value.is_a?(MU::Config::Tail)
+        MU.log "Parameter #{param} is using a nested parameter as a value. This rarely works, depending on the target cloud. YMMV.", MU::WARN
+        tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
+      elsif !list_of.nil? or (@@tails.has_key?(param) and @@tails[param].is_a?(Array))
+        tail = []
+        count = 0
+        value.split(/\s*,\s*/).each { |subval|
+          if @@tails.has_key?(param) and !@@tails[param][count].nil?
+            subval = @@tails[param][count].values.first.to_s if subval.nil?
+            list_of = @@tails[param][count].values.first.getName if list_of.nil?
+            prettyname = @@tails[param][count].values.first.getPrettyName if prettyname.nil?
+            description = @@tails[param][count].values.first.description if description.nil?
+            valid_values = @@tails[param][count].values.first.valid_values if valid_values.nil? or valid_values.empty?
+            cloudtype = @@tails[param][count].values.first.getCloudType if @@tails[param][count].values.first.getCloudType != "String"
+          end
+          prettyname = param.capitalize if prettyname.nil?
+          tail << { list_of => MU::Config::Tail.new(list_of, subval, prettyname, cloudtype, valid_values, description, true, pseudo: pseudo, index: count) }
+          count = count + 1
+        }
+      else
+        if @@tails.has_key?(param)
+          pseudo = @@tails[param].pseudo
+          value = @@tails[param].to_s if value.nil?
+          prettyname = @@tails[param].getPrettyName if prettyname.nil?
+          description = @@tails[param].description if description.nil?
+          valid_values = @@tails[param].valid_values if valid_values.nil? or valid_values.empty?
+          cloudtype = @@tails[param].getCloudType if @@tails[param].getCloudType != "String"
+        end
+        tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
+      end
+
+      if valid_values and valid_values.size > 0 and value
+        if !valid_values.include?(value)
+          raise DeployParamError, "Invalid parameter value '#{value}' supplied for '#{param}'"
+        end
+      end
+      @@tails[param] = tail
+
+      tail
+    end
+  end
+end

data/modules/mu/config/user.rb

@@ -68,10 +68,10 @@ module MU
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::user}, bare and unvalidated.
-      # @param user [Hash]: The resource to process and validate
-      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @param _user [Hash]: The resource to process and validate
+      # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
       # @return [Boolean]: True if validation succeeded, False otherwise
-      def self.validate(user, configurator)
+      def self.validate(_user, _configurator)
         ok = true
 
         ok

data/modules/mu/config/vpc.rb

@@ -418,7 +418,6 @@ module MU
         if !vpc['ip_block']
           if configurator.updating and configurator.existing_deploy and
              configurator.existing_deploy.original_config['vpcs']
-            pieces = []
             configurator.existing_deploy.original_config['vpcs'].each { |v|
               if v['name'] == vpc['name']
                 vpc['ip_block'] = v['ip_block']
@@ -799,7 +798,7 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
               end
               @@reference_cache[vpc_block] ||= ext_vpc if ok
             end
-          rescue Exception => e
+          rescue StandardError => e
             raise MuError, e.inspect, e.backtrace
           ensure
             if !ext_vpc and vpc_block['cloud'] != "CloudFormation"
@@ -877,7 +876,7 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
             tag_key, tag_value = vpc_block['tag'].split(/=/, 2) if !vpc_block['tag'].nil?
             begin
               ext_subnet = ext_vpc.getSubnet(cloud_id: vpc_block['subnet_id'], name: vpc_block['subnet_name'], tag_key: tag_key, tag_value: tag_value)
-            rescue MuError => e
+            rescue MuError
             end
 
             if ext_subnet.nil?
@@ -918,7 +917,6 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
           public_subnets = []
           public_subnets_map = {}
           subnet_ptr = "subnet_id"
-          all_subnets = []
           if !is_sibling
             pub = priv = 0
             raise MuError, "No subnets found in #{ext_vpc}" if ext_vpc.subnets.nil?
@@ -1068,5 +1066,47 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
       end
 
     end
+
+    # Take an IP block and split it into a more-or-less arbitrary number of
+    # subnets.
+    # @param ip_block [String]: CIDR of the network to subdivide
+    # @param subnets_desired [Integer]: Number of subnets we want back
+    # @param max_mask [Integer]: The highest netmask we're allowed to use for a subnet (various by cloud provider)
+    # @return [MU::Config::Tail]: Resulting subnet tails, or nil if an error occurred.
+    def divideNetwork(ip_block, subnets_desired, max_mask = 28)
+      cidr = NetAddr::IPv4Net.parse(ip_block.to_s)
+
+      # Ugly but reliable method of landing on the right subnet size
+      subnet_bits = cidr.netmask.prefix_len
+      begin
+        subnet_bits += 1
+        if subnet_bits > max_mask
+          MU.log "Can't subdivide #{cidr.to_s} into #{subnets_desired.to_s}", MU::ERR
+          raise MuError, "Subnets smaller than /#{max_mask} not permitted"
+        end
+      end while cidr.subnet_count(subnet_bits) < subnets_desired
+
+      if cidr.subnet_count(subnet_bits) > subnets_desired
+        MU.log "Requested #{subnets_desired.to_s} subnets from #{cidr.to_s}, leaving #{(cidr.subnet_count(subnet_bits)-subnets_desired).to_s} unused /#{subnet_bits.to_s}s available", MU::NOTICE
+      end
+
+      begin
+        subnets = []
+        (0..subnets_desired).each { |x|
+          subnets << cidr.nth_subnet(subnet_bits, x).to_s
+        }
+      rescue RuntimeError => e
+        if e.message.match(/exceeds subnets available for allocation/)
+          MU.log e.message, MU::ERR
+          MU.log "I'm attempting to create #{subnets_desired} subnets (one public and one private for each Availability Zone), of #{subnet_size} addresses each, but that's too many for a /#{cidr.netmask.prefix_len} network. Either declare a larger network, or explicitly declare a list of subnets with few enough entries to fit.", MU::ERR
+          return nil
+        else
+          raise e
+        end
+      end
+
+      subnets = getTail("subnetblocks", value: subnets.join(","), cloudtype: "CommaDelimitedList", description: "IP Address ranges to be used for VPC subnets", prettyname: "SubnetIpBlocks", list_of: "ip_block").map { |tail| tail["ip_block"] }
+      subnets
+    end
   end
 end