cloud-mu 3.1.3 → 3.3.0

data/modules/mu/{clouds → providers}/azure/habitat.rb

@@ -59,8 +59,10 @@ module MU
         def groom
         end
 
+        @cached_cloud_desc = nil
         # Return the cloud descriptor for the Habitat
-        def cloud_desc
+        def cloud_desc(use_cache: true)
+          return @cached_cloud_desc if @cached_cloud_desc and use_cache
           @cached_cloud_desc ||= MU::Cloud::Azure::Habitat.find(cloud_id: @cloud_id).values.first
 #          @habitat_id ||= @cached_cloud_desc.parent.id if @cached_cloud_desc
           @cached_cloud_desc
@@ -131,7 +133,7 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil)
+        def toKitten(**args)
           bok = {
             "cloud" => "Azure",
             "credentials" => @config['credentials']
@@ -141,9 +143,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
           }
@@ -152,9 +154,9 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::habitats}, bare and unvalidated.
         # @param habitat [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(habitat, configurator)
+        def self.validateConfig(habitat, _configurator)
           ok = true
           habitat['region'] ||= MU::Cloud::Azure.myRegion(habitat['credentials'])
 

data/modules/mu/{clouds → providers}/azure/loadbalancer.rb

@@ -74,9 +74,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
 #            "named_ports" => {
@@ -102,9 +102,9 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::loadbalancers}, bare and unvalidated.
         # @param lb [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(lb, configurator)
+        def self.validateConfig(lb, _configurator)
           ok = true
           lb['region'] ||= MU::Cloud::Azure.myRegion(lb['credentials'])
 
@@ -144,7 +144,7 @@ module MU
                 found[Id.new(lb.id)] = lb
               }
             else
-              MU::Cloud::Azure.network(credentials: args[:credentials]).load_balancers.list_all.each { |net|
+              MU::Cloud::Azure.network(credentials: args[:credentials]).load_balancers.list_all.each { |lb|
                 found[Id.new(lb.id)] = lb
               }
             end

data/modules/mu/{clouds → providers}/azure/role.rb

@@ -65,8 +65,8 @@ module MU
         end
 
         # Assign this role object to a given principal (create a RoleAssignment)
-        # @param principal [MU::Cloud::Azure::Id]
-        def assignTo(principal)
+        # @param principal_id [MU::Cloud::Azure::Id]
+        def assignTo(principal_id)
           MU::Cloud::Azure::Role.assignTo(principal_id, role_id: @cloud_id)
         end
 
@@ -145,7 +145,7 @@ module MU
             begin
               resp = MU::Cloud::Azure.authorization(credentials: args[:credentials]).role_definitions.get(scope, id_str)
               found[Id.new(resp.id)] = resp
-            rescue MsRestAzure::AzureOperationError => e
+            rescue MsRestAzure::AzureOperationError
               # this is fine, we're doing a blind search after all
             end
           else
@@ -155,7 +155,7 @@ module MU
               end
             }
             if args[:role_name]
-              @@role_list_cache[scope].each_pair { |key, role|
+              @@role_list_cache[scope].values.each { |role|
                 begin
                   if role.role_name == args[:role_name]
                     found[Id.new(role.id)] = role
@@ -183,9 +183,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
           }
@@ -194,17 +194,15 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::roles}, bare and unvalidated.
         # @param role [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(role, configurator)
+        def self.validateConfig(role, _configurator)
           ok = true
           role['region'] ||= MU::Cloud::Azure.myRegion(role['credentials'])
 
           ok
         end
 
-        private
-
       end
     end
   end

data/modules/mu/{clouds → providers}/azure/server.rb

@@ -139,21 +139,21 @@ module MU
         # Figure out what's needed to SSH into this server.
         # @return [Array<String>]: nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name, alternate_names
         def getSSHConfig
-          node, config, deploydata = describe(cloud_id: @cloud_id)
+          describe(cloud_id: @cloud_id)
 # XXX add some awesome alternate names from metadata and make sure they end
 # up in MU::MommaCat's ssh config wangling
           ssh_keydir = Etc.getpwuid(Process.uid).dir+"/.ssh"
           return nil if @config.nil? or @deploy.nil?
 
           nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
-          if !@config["vpc"].nil? and !MU::Cloud::Azure::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
+          if !@config["vpc"].nil? and !MU::Cloud.resourceClass("Azure", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
 
             if !@nat.nil? and @nat.mu_name != @mu_name
               if @nat.cloud_desc.nil?
                 MU.log "NAT was missing cloud descriptor when called in #{@mu_name}'s getSSHConfig", MU::ERR
                 return nil
               end
-              foo, bar, baz, nat_ssh_host, nat_ssh_user, nat_ssh_key  = @nat.getSSHConfig
+              _foo, _bar, _baz, nat_ssh_host, nat_ssh_user, nat_ssh_key  = @nat.getSSHConfig
               if nat_ssh_user.nil? and !nat_ssh_host.nil?
                 MU.log "#{@config["name"]} (#{MU.deploy_id}) is configured to use #{@config['vpc']} NAT #{nat_ssh_host}, but username isn't specified. Guessing root.", MU::ERR, details: caller
                 nat_ssh_user = "root"
@@ -163,7 +163,7 @@ module MU
 
           if @config['ssh_user'].nil?
             if windows?
-              @config['ssh_user'] = "Administrator"
+              @config['ssh_user'] = "muadmin"
             else
               @config['ssh_user'] = "root"
             end
@@ -188,8 +188,8 @@ module MU
             @named = true
           end
 
-          nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name = getSSHConfig
-          if !nat_ssh_host and !MU::Cloud::Azure::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
+          _nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
+          if !nat_ssh_host and !MU::Cloud.resourceClass("Azure", "VPC").haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
 # XXX check if canonical_ip is in the private ranges
 #            raise MuError, "#{node} has no NAT host configured, and I have no other route to it"
           end
@@ -236,7 +236,7 @@ module MU
                 resp = MU::Cloud::Azure.compute(credentials: args[:credentials]).virtual_machines.get(rg, id_str)
                 next if resp.nil?
                 found[Id.new(resp.id)] = resp
-              rescue MU::Cloud::Azure::APIError => e
+              rescue MU::Cloud::Azure::APIError
                 # this is fine, we're doing a blind search after all
               end
             }
@@ -267,7 +267,7 @@ module MU
 
           MU::MommaCat.lock(@cloud_id.to_s+"-groom")
           
-          node, config, deploydata = describe(cloud_id: @cloud_id)
+          node, _config, deploydata = describe(cloud_id: @cloud_id)
 
           if node.nil? or node.empty?
             raise MuError, "MU::Cloud::Azure::Server.groom was called without a mu_name"
@@ -357,7 +357,7 @@ module MU
         # bastion hosts that may be in the path, see getSSHConfig if that's what
         # you need.
         def canonicalIP
-          mu_name, config, deploydata = describe(cloud_id: @cloud_id)
+          describe(cloud_id: @cloud_id)
 
           if !cloud_desc
             raise MuError, "Couldn't retrieve cloud descriptor for server #{self}"
@@ -384,7 +384,7 @@ module MU
           # Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
           # which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
           # The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
-          if MU::Cloud::Azure::VPC.haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
+          if MU::Cloud.resourceClass("Azure", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
             @config['canonical_ip'] = private_ips.first
             return private_ips.first
           else
@@ -393,6 +393,28 @@ module MU
           end
         end
 
+        # Return all of the IP addresses, public and private, from all of our
+        # network interfaces.
+        # @return [Array<String>]
+        def listIPs
+          ips = []
+          cloud_desc.network_profile.network_interfaces.each { |iface|
+            iface_id = Id.new(iface.is_a?(Hash) ? iface['id'] : iface.id)
+            iface_desc = MU::Cloud::Azure.network(credentials: @credentials).network_interfaces.get(@resource_group, iface_id.to_s)
+            iface_desc.ip_configurations.each { |ipcfg|
+              ips << ipcfg.private_ipaddress
+              if ipcfg.respond_to?(:public_ipaddress) and ipcfg.public_ipaddress
+                ip_id = Id.new(ipcfg.public_ipaddress.id)
+                ip_desc = MU::Cloud::Azure.network(credentials: @credentials).public_ipaddresses.get(@resource_group, ip_id.to_s)
+                if ip_desc
+                  ips << ip_desc.ip_address
+                end
+              end
+            }
+          }
+          ips
+        end
+
         # return [String]: A password string.
         def getWindowsAdminPassword
         end
@@ -430,7 +452,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
         end
 
         # Cloud-specific configuration properties.
@@ -441,7 +463,7 @@ module MU
           hosts_schema = MU::Config::CIDR_PRIMITIVE
           hosts_schema["pattern"] = "^(\\d+\\.\\d+\\.\\d+\\.\\d+\/[0-9]{1,2}|\\*)$"
           schema = {
-            "roles" => MU::Cloud::Azure::User.schema(config)[1]["roles"],
+            "roles" => MU::Cloud.resourceClass("Azure", "User").schema(config)[1]["roles"],
             "ingress_rules" => {
               "items" => {
                 "properties" => {
@@ -451,6 +473,34 @@ module MU
                   }
                 }
               }
+            },
+            "windows_admin_username" => {
+              "type" => "string",
+              "default" => "muadmin",
+            },
+            "ssh_user" => {
+              "default_if" => [
+                {
+                  "key_is" => "platform",
+                  "value_is" => "windows",
+                  "set" => "muadmin"
+                },
+                {
+                  "key_is" => "platform",
+                  "value_is" => "win2k12",
+                  "set" => "muadmin"
+                },
+                {
+                  "key_is" => "platform",
+                  "value_is" => "win2k12r2",
+                  "set" => "muadmin"
+                },
+                {
+                  "key_is" => "platform",
+                  "value_is" => "win2k16",
+                  "set" => "muadmin"
+                }
+              ]
             }
           }
           [toplevel_required, schema]
@@ -469,8 +519,7 @@ module MU
             foundmatch = false
             MU::Cloud.availableClouds.each { |cloud|
               next if cloud == "Azure"
-              cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-              foreign_types = (cloudbase.listInstanceTypes).values.first
+              foreign_types = (MU::Cloud.cloudClass(cloud).listInstanceTypes).values.first
               if foreign_types.size == 1
                 foreign_types = foreign_types.values.first
               end
@@ -479,6 +528,7 @@ module MU
                 mem = foreign_types[size]["memory"]
                 ecu = foreign_types[size]["ecu"]
                 types.keys.sort.reverse.each { |type|
+                  next if type.match(/_Promo$/i)
                   features = types[type]
                   next if ecu == "Variable" and ecu != features["ecu"]
                   next if features["vcpu"] != vcpu
@@ -495,7 +545,6 @@ module MU
 
             if !foundmatch
               MU.log "Invalid size '#{size}' for Azure Compute instance in #{region}. Supported types:", MU::ERR, details: types.keys.sort.join(", ")
-exit
               return nil
             end
           end
@@ -512,6 +561,10 @@ exit
 
           server['region'] ||= MU::Cloud::Azure.myRegion(server['credentials'])
           server['ssh_user'] ||= "muadmin"
+          if server['windows_admin_username'] == "Administrator"
+            MU.log "Azure does not permit admin user to be 'Administrator'", MU::ERR
+            ok = false
+          end
 
           server['size'] = validateInstanceType(server["size"], server["region"])
           ok = false if server['size'].nil?
@@ -527,17 +580,17 @@ exit
           end
 
           image_desc = MU::Cloud::Azure::Server.fetchImage(server['image_id'].to_s, credentials: server['credentials'], region: server['region'])
-          if image_desc.plan
-            terms = MU::Cloud::Azure.marketplace(credentials: @credentials).marketplace_agreements.get(image_desc.plan.publisher, image_desc.plan.product, image_desc.plan.name)
-            if !terms.accepted
-              MU.log "Deploying #{server['name']} will automatically agree to the licensing terms for #{terms.product}", MU::NOTICE, details: terms.license_text_link
-            end
-          end
 
           if !image_desc
             MU.log "Failed to locate an Azure VM image for #{server['name']} from #{server['image_id']} in #{server['region']}", MU::ERR
             ok = false
           else
+            if image_desc.plan
+              terms = MU::Cloud::Azure.marketplace(credentials: @credentials).marketplace_agreements.get(image_desc.plan.publisher, image_desc.plan.product, image_desc.plan.name)
+              if !terms.accepted
+                MU.log "Deploying #{server['name']} will automatically agree to the licensing terms for #{terms.product}", MU::NOTICE, details: terms.license_text_link
+              end
+            end
             server['image_id'] = image_desc.id
           end
 
@@ -558,18 +611,8 @@ exit
             if !configurator.insertKitten(vpc, "vpcs", true)
               ok = false
             end
-            server['dependencies'] ||= []
-
-            server['dependencies'] << {
-              "type" => "vpc",
-              "name" => server['name']+"vpc"
-            }
-# XXX what happens if there's no natstion here?
-            server['dependencies'] << {
-              "type" => "server",
-              "name" => server['name']+"vpc-natstion",
-              "phase" => "groom"
-            }
+            MU::Config.addDependency(server, server['name']+"vpc", "vpc")
+            MU::Config.addDependency(server, server['name']+"vpc-natstion", "server", phase: "groom")
             server['vpc'] = {
               "name" => server['name']+"vpc",
               "subnet_pref" => "private"
@@ -586,17 +629,14 @@ exit
             "credentials" => server["credentials"],
             "roles" => server["roles"]
           }
-          server['dependencies'] ||= []
-          server['dependencies'] << {
-            "type" => "user",
-            "name" => server["name"]+"user"
-          }
+          MU::Config.addDependency(server, server['name']+"user", "user")
 
           ok = false if !configurator.insertKitten(svcacct_desc, "users")
 
           ok
         end
 
+        # stub
         def self.diskConfig(config, create = true, disk_as_url = true, credentials: nil)
         end
 
@@ -624,17 +664,22 @@ exit
           skus = MU::Cloud::Azure.compute(credentials: credentials).virtual_machine_images.list_skus(region, publisher, offer).map { |s| s.name }
 
           if !skus.include?(sku)
-            skus.sort { |a, b| MU.version_sort(a, b) }.reverse.each { |s|
-              if s.match(/^#{Regexp.quote(sku)}/)
-                sku = s
-                break
-              end
-            }
+            skus.reject! { |s| !s.match(/^#{Regexp.quote(sku)}/) }
+            skus.sort! { |a, b| MU.version_sort(a, b) }.reverse!
+            sku = skus.first
           end
 
-          versions = MU::Cloud::Azure.compute(credentials: credentials).virtual_machine_images.list(region, publisher, offer, sku).map { |v| v.name }
+          version = nil
+          begin
+            versions = MU::Cloud::Azure.compute(credentials: credentials).virtual_machine_images.list(region, publisher, offer, sku).map { |v| v.name }
+            if versions.nil? or versions.empty?
+              skus.delete(sku)
+              sku = skus.first
+            end
+          end while skus.size > 0 and (versions.nil? or versions.empty?)
+
           if versions.nil? or versions.empty?
-            MU.log "Azure API returned empty machine image version list for publisher #{publisher} offer #{offer} sku #{sku}", MU::ERR
+            MU.log "Azure API returned empty machine image version list for publisher #{publisher} offer #{offer} sku #{sku}", MU::ERR, details: skus
             return nil
           end
 
@@ -724,12 +769,15 @@ exit
           hw_obj.vm_size = @config['size']
 
           os_obj = MU::Cloud::Azure.compute(:OSProfile).new
-          os_obj.admin_username = @config['ssh_user']
-          os_obj.computer_name = @mu_name
           if windows?
             win_obj = MU::Cloud::Azure.compute(:WindowsConfiguration).new
             os_obj.windows_configuration = win_obj
+            os_obj.admin_username = @config['windows_admin_username']
+            os_obj.admin_password = MU.generateWindowsPassword
+            os_obj.computer_name = @deploy.getResourceName(@config["name"], max_length: 15, disallowed_chars: /[~!@#$%^&*()=+_\[\]{}\\\|;:\.'",<>\/\?]/)
           else
+            os_obj.admin_username = @config['ssh_user']
+            os_obj.computer_name = @mu_name
             key_obj = MU::Cloud::Azure.compute(:SshPublicKey).new
             key_obj.key_data = @deploy.ssh_public_key
             key_obj.path = "/home/#{@config['ssh_user']}/.ssh/authorized_keys"
@@ -772,7 +820,7 @@ exit
               begin
 # XXX this doesn't actually work as documented
                 MU::Cloud::Azure.marketplace(credentials: @credentials).marketplace_agreements.sign(image_desc.plan.publisher, image_desc.plan.product, image_desc.plan.name)
-              rescue Exception => e
+              rescue StandardError => e
                 MU.log e.message, MU::ERR
                 vm_obj.plan = nil
               end

data/modules/mu/{clouds → providers}/azure/user.rb

@@ -86,7 +86,6 @@ module MU
 
             client = ::MsRest::ServiceClient.new(cred_obj)
             cloud_desc.client_secret_url.match(/^(http.*?\.azure\.net)(\/.*)/)
-            base = Regexp.last_match[1]
             path = Regexp.last_match[2]
 #MU.log "Calling into #{base} #{path}"
             promise = client.make_request_async(
@@ -97,7 +96,7 @@ module MU
 
             # XXX this is async, need to stop and wait somehow
             promise.then do | result|
-              resp = result.response
+              result.response
 #              MU.log "RESPONSE", MU::WARN, details: resp
             end
           end
@@ -106,10 +105,9 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
-          rgroup_name = @deploy.deploy_id+"-"+@config['region'].upcase
           if @config['roles']
             @config['roles'].each { |role|
-              MU::Cloud::Azure::Role.assignTo(cloud_desc.principal_id, role_name: role, credentials: @config['credentials'])
+              MU::Cloud.resourceClass("Azure", "Role").assignTo(cloud_desc.principal_id, role_name: role, credentials: @config['credentials'])
             }
           end
         end
@@ -191,9 +189,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "region" => MU::Config.region_primitive,
@@ -221,9 +219,9 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::users}, bare and unvalidated.
         # @param user [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(user, configurator)
+        def self.validateConfig(user, _configurator)
           ok = true
           user['region'] ||= MU::Cloud::Azure.myRegion(user['credentials'])