cloud-mu 3.4.0 → 3.5.0

data/modules/mu/providers/azure.rb

@@ -288,9 +288,14 @@ module MU
           raise MuError, "Nil response from Azure API attempting list_locations(#{subscription})"
         end
 
-        sdk_response.value.each do | region |
-          @@regions.push(region.name)
-        end
+        sdk_response.value.each { |region|
+          begin
+            listInstanceTypes(region.name) # use this to filter for broken regions
+            @@regions.push(region.name)
+          rescue APIError => e
+            MU.log "Azure region "+region.name+" does not appear operational, skipping", MU::WARN
+          end
+        }
 
         return us_only ? @@regions.reject { |r| !r.match(/us\d?$/) } : @@regions
       end
@@ -350,13 +355,42 @@ module MU
           listRegions.each { |region|
             next if !deploy.regionsUsed.include?(region)
             begin
-              createResourceGroup(deploy.deploy_id+"-"+region.upcase, region, credentials: creds)
+              rg_obj = createResourceGroup(deploy.deploy_id+"-"+region.upcase, region, credentials: creds)
+              createVault(rg_obj.name, region, deploy, credentials: creds)
             rescue ::MsRestAzure::AzureOperationError
             end
           }
         }
       end
 
+      # Arguably this should be a first class resource, but for now we'll do
+      # it here since we're going to have a generic deployment vault in every
+      # resource group.
+      # @param rg [String]: The name of the resource group in which we'll reside
+      # @param region [String]: The region in which we'll reside
+      # @param deploy [MU::MommaCat]: The deployment which we serve
+      # @param credentials [String]:
+      def self.createVault(rg, region, deploy, credentials: nil)
+        cred_hash = MU::Cloud::Azure.getSDKOptions(credentials)
+        vaultname = deploy.getResourceName(region, max_length: 23, disallowed_chars: /[^a-z0-9-]/i, never_gen_unique: true)
+        MU::Cloud::Azure.ensureProvider("Microsoft.KeyVault", credentials: credentials)
+        sku = MU::Cloud::Azure.keyvault(:Sku).new
+        sku.name = "standard" # ...I'm angry about this
+
+        props = MU::Cloud::Azure.keyvault(:VaultProperties).new
+        props.tenant_id = cred_hash[:tenant_id]
+        props.enabled_for_deployment = true
+        props.sku = sku
+        props.access_policies = []
+
+        params = MU::Cloud::Azure.keyvault(:VaultCreateOrUpdateParameters).new
+        params.location = region
+        params.properties = props
+
+        MU.log "Creating KeyVault #{vaultname} in #{region}"
+        MU::Cloud::Azure.keyvault(credentials: credentials).vaults.create_or_update(rg, vaultname, params)
+      end
+
       @@rg_semaphore = Mutex.new
 
       # Purge cloud-specific deploy meta-artifacts (SSH keys, resource groups,
@@ -400,17 +434,30 @@ module MU
           end
         }
         MU.log "Configuring resource group #{name} in #{region}", details: rg_obj
-        MU::Cloud::Azure.resources(credentials: credentials).resource_groups.create_or_update(
+        rg = MU::Cloud::Azure.resources(credentials: credentials).resource_groups.create_or_update(
           name,
           rg_obj
         )
+
+        rg
       end
 
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
-      # @param deploy_id [String]: The deploy for which we're writing the secret
+      # @param deploy [MU::MommaCat]: The deploy for which we're writing the secret
       # @param value [String]: The contents of the secret
-      def self.writeDeploySecret(deploy_id, value, name = nil, credentials: nil)
-# XXX this ain't it hoss
+      def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
+        deploy_id = deploy.deploy_id
+
+        listRegions.each { |region|
+          next if !deploy.regionsUsed.include?(region)
+          rg = deploy_id+"-"+region.upcase
+          vaultname = deploy.getResourceName(region, max_length: 23, disallowed_chars: /[^a-z0-9-]/i, never_gen_unique: true)
+
+          resp = MU::Cloud::Azure.keyvault(credentials: credentials).vaults.get(rg, vaultname)
+          next if !resp
+MU.log "vault existence check #{vaultname}", MU::WARN, details: resp
+
+        }
       end
 
       # Return the name strings of all known sets of credentials for this cloud
@@ -522,7 +569,7 @@ module MU
 
         begin
           Timeout.timeout(2) do
-            resp = JSON.parse(open("#{base_url}/?#{arg_str}","Metadata"=>"true").read)
+            resp = JSON.parse(URI.open("#{base_url}/?#{arg_str}","Metadata"=>"true").read)
             MU.log "curl -H Metadata:true "+"#{base_url}/?#{arg_str}", loglevel, details: resp
             if svc != "instance"
               return resp
@@ -777,6 +824,24 @@ module MU
         return @@resources_api[credentials]
       end
 
+      # The Azure KeyVault API
+      # @param model [<Azure::Apis::KeyVault::Mgmt::V2018_02_14::Models>]: If specified, will return the class ::Azure::Apis::KeyVault::Mgmt::V2018_02_14::Models::model instead of an API client instance
+      # @param model_version [String]: Use an alternative model version supported by the SDK when requesting a +model+
+      # @param alt_object [String]: Return an instance of something other than the usual API client object
+      # @param credentials [String]: The credential set (subscription, effectively) in which to operate
+      # @return [MU::Cloud::Azure::SDKClient]
+      def self.keyvault(model = nil, alt_object: nil, credentials: nil, model_version: "V2018_02_14")
+        require 'azure_mgmt_key_vault'
+
+        if model and model.is_a?(Symbol)
+          return Object.const_get("Azure").const_get("KeyVault").const_get("Mgmt").const_get(model_version).const_get("Models").const_get(model)
+        else
+          @@keyvault_api[credentials] ||= MU::Cloud::Azure::SDKClient.new(api: "KeyVault", credentials: credentials, subclass: alt_object)
+        end
+
+        return @@keyvault_api[credentials]
+      end
+
       # The Azure Features API
       # @param model [<Azure::Apis::Features::Mgmt::V2015_12_01::Models>]: If specified, will return the class ::Azure::Apis::Features::Mgmt::V2015_12_01::Models::model instead of an API client instance
       # @param model_version [String]: Use an alternative model version supported by the SDK when requesting a +model+
@@ -931,6 +996,7 @@ module MU
       @@resources_api = {}
       @@containers_api = {}
       @@features_api = {}
+      @@keyvault_api = {}
       @@apis_api = {}
       @@marketplace_api = {}
       @@service_identity_api = {}
@@ -1069,9 +1135,9 @@ module MU
                       retry
                     end
 
-                    MU.log "#{@parent.api.class.name}.#{@myname}.#{method_sym.to_s} returned '"+err["code"]+"' - "+err["message"], MU::WARN, details: caller
-                    MU.log e.backtrace[0], MU::WARN, details: parsed
-                    raise MU::Cloud::Azure::APIError, err["code"]+": "+err["message"]+" (call was #{@parent.api.class.name}.#{@myname}.#{method_sym.to_s})"
+#                    MU.log "#{@parent.api.class.name}.#{@myname}.#{method_sym.to_s} returned '"+err["code"]+"' - "+err["message"], MU::WARN, details: caller
+#                    MU.log e.backtrace[0], MU::WARN, details: parsed
+                    raise MU::Cloud::Azure::APIError.new err["code"]+": "+err["message"]+" (call was #{@parent.api.class.name}.#{@myname}.#{method_sym.to_s})", details: parsed, silent: true
                   end
                 end
               rescue JSON::ParserError

data/modules/mu/providers/azure/server.rb

@@ -150,7 +150,7 @@ module MU
 
             if !@nat.nil? and @nat.mu_name != @mu_name
               if @nat.cloud_desc.nil?
-                MU.log "NAT was missing cloud descriptor when called in #{@mu_name}'s getSSHConfig", MU::ERR
+                MU.log "NAT #{@nat} was missing cloud descriptor when called in #{@mu_name}'s getSSHConfig", MU::ERR
                 return nil
               end
               _foo, _bar, _baz, nat_ssh_host, nat_ssh_user, nat_ssh_key  = @nat.getSSHConfig
@@ -220,6 +220,7 @@ module MU
         # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching instances
         def self.find(**args)
           found = {}
+MU.log "Azure::Server.find called", MU::NOTICE, details: args
           # told one, we may have to search all the ones we can see.
           resource_groups = if args[:resource_group]
             [args[:resource_group]]
@@ -417,6 +418,7 @@ module MU
 
         # return [String]: A password string.
         def getWindowsAdminPassword
+          @deploy.fetchSecret(@mu_name, "windows_admin_password")
         end
 
         # Add a volume to this instance
@@ -612,7 +614,7 @@ module MU
               ok = false
             end
             MU::Config.addDependency(server, server['name']+"vpc", "vpc")
-            MU::Config.addDependency(server, server['name']+"vpc-natstion", "server", phase: "groom")
+            MU::Config.addDependency(server, server['name']+"vpc-natstion", "server", their_phase: "groom")
             server['vpc'] = {
               "name" => server['name']+"vpc",
               "subnet_pref" => "private"
@@ -770,10 +772,23 @@ module MU
 
           os_obj = MU::Cloud::Azure.compute(:OSProfile).new
           if windows?
+            winrm_listen = MU::Cloud::Azure.compute(:WinRMListener).new
+            winrm_listen.certificate_url = "goddamn stupid ass thing"
+            winrm_listen.protocol = "https"
+            winrm = MU::Cloud::Azure.compute(:WinRMConfiguration).new
+            winrm.listeners = [winrm_listen]
+
             win_obj = MU::Cloud::Azure.compute(:WindowsConfiguration).new
+            win_obj.win_rmconfiguration = winrm
             os_obj.windows_configuration = win_obj
             os_obj.admin_username = @config['windows_admin_username']
-            os_obj.admin_password = MU.generateWindowsPassword
+            os_obj.admin_password = begin
+              @deploy.fetchSecret(@mu_name, "windows_admin_password")
+            rescue MU::MommaCat::SecretError
+              pw = MU.generateWindowsPassword
+              @deploy.saveNodeSecret(@mu_name, pw, "windows_admin_password")
+              pw
+            end
             os_obj.computer_name = @deploy.getResourceName(@config["name"], max_length: 15, disallowed_chars: /[~!@#$%^&*()=+_\[\]{}\\\|;:\.'",<>\/\?]/)
           else
             os_obj.admin_username = @config['ssh_user']

data/modules/mu/providers/cloudformation/server.rb

@@ -304,7 +304,7 @@ module MU
                     role_name: baserole.role_name,
                     policy_name: name
                 )
-                policies[name] = URI.decode(resp.policy_document)
+                policies[name] = CGI.unescape(resp.policy_document)
               }
             }
           end

data/modules/mu/providers/google.rb

@@ -348,7 +348,8 @@ module MU
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
       # @param deploy_id [String]: The deploy for which we're writing the secret
       # @param value [String]: The contents of the secret
-      def self.writeDeploySecret(deploy_id, value, name = nil, credentials: nil)
+      def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
+        deploy_id = deploy.deploy_id
         name ||= deploy_id+"-secret"
         begin
           MU.log "Writing #{name} to Cloud Storage bucket #{adminBucketName(credentials)}"
@@ -487,7 +488,7 @@ MU.log e.message, MU::WARN, details: e.inspect
         base_url = "http://metadata.google.internal/computeMetadata/v1"
         begin
           Timeout.timeout(2) do
-            response = open(
+            response = URI.open(
               "#{base_url}/#{param}",
               "Metadata-Flavor" => "Google"
             ).read
@@ -981,7 +982,20 @@ MU.log e.message, MU::WARN, details: e.inspect
       end
 
       # Google's Cloud Billing Service API
-      # @param subclass [<Google::Apis::LoggingV2>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
+      # @param subclass [<Google::Apis::CloudbillingV1>]: If specified, will return the class ::Google::Apis::CloudbillingV1::subclass instead of an API client instance
+      def self.budgets(subclass = nil, credentials: nil)
+        require 'google/apis/billingbudgets_v1'
+
+        if subclass.nil?
+          @@budgets_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "BillingbudgetsV1::CloudBillingBudgetService", scopes: ['cloud-platform', 'cloud-billing'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
+          return @@budgets_api[credentials]
+        elsif subclass.is_a?(Symbol)
+          return Object.const_get("::Google").const_get("Apis").const_get("BillingbudgetsV1").const_get(subclass)
+        end
+      end
+
+      # Google's Cloud Billing Budget Service API
+      # @param subclass [<Google::Apis::CloudbillingV1>]: If specified, will return the class ::Google::Apis::CloudbillingV1::subclass instead of an API client instance
       def self.billing(subclass = nil, credentials: nil)
         require 'google/apis/cloudbilling_v1'
 
@@ -1310,7 +1324,7 @@ MU.log e.message, MU::WARN, details: e.inspect
 
                   svc_name = Regexp.last_match[1]
                   save_verbosity = MU.verbosity
-                  if svc_name != "servicemanagement.googleapis.com" and method_sym != :delete
+                  if !["servicemanagement.googleapis.com", "billingbudgets.googleapis.com"].include?(svc_name) and method_sym != :delete
                     retries += 1
                     @@enable_semaphores[project].synchronize {
                       MU.setLogging(MU::Logger::NORMAL)
@@ -1565,6 +1579,7 @@ MU.log e.message, MU::WARN, details: e.inspect
       @@firestore_api = {}
       @@admin_directory_api = {}
       @@billing_api = {}
+      @@budgets_api = {}
       @@function_api = {}
     end
   end

data/modules/mu/providers/google/folder.rb

@@ -265,8 +265,12 @@ module MU
 
           if args[:cloud_id]
             raw_id = args[:cloud_id].sub(/^folders\//, "")
-            resp = MU::Cloud::Google.folder(credentials: args[:credentials]).get_folder("folders/"+raw_id)
-            found[resp.name] = resp if resp
+            begin
+              resp = MU::Cloud::Google.folder(credentials: args[:credentials]).get_folder("folders/"+raw_id)
+              found[resp.name] = resp if resp
+            rescue ::Google::Apis::ClientError => e
+              raise e if e.message !~ /forbidden: /
+            end
 
           elsif args[:flags] and args[:flags]['display_name']
 

data/modules/mu/providers/google/function.rb

@@ -119,6 +119,9 @@ module example.com/cloudfunction
         # Called automatically by {MU::Deploy#createResources}
         def groom
           desc = {}
+
+          func_obj = buildDesc
+
           labels = Hash[@tags.keys.map { |k|
             [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
           ]
@@ -140,6 +143,10 @@ module example.com/cloudfunction
           if cloud_desc.available_memory_mb != @config['memory']
             need_update = true
           end
+          if cloud_desc.service_account_email != func_obj.service_account_email
+            need_update = true
+          end
+
           if @config['environment_variable']
             @config['environment_variable'].each { |var|
               if !cloud_desc.environment_variables or
@@ -161,7 +168,17 @@ module example.com/cloudfunction
             File.read("#{dir}/current.zip")
           }
 
-          new = if @config['code']['zip_file']
+          tempfile = nil
+          new = if @config['code']['zip_file'] or @config['code']['path']
+            if @config['code']['path']
+              tempfile = Tempfile.new(["function", ".zip"])
+              MU.log "#{@mu_name} using code at #{@config['code']['path']}"
+              MU::Master.zipDir(@config['code']['path'], tempfile.path)
+              @config['code']['zip_file'] = tempfile.path
+            else
+              MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
+            end
+#            @code_sha256 = Base64.encode64(Digest::SHA256.digest(zip)).chomp
             File.read(@config['code']['zip_file'])
           elsif @config['code']['gs_url']
             @config['code']['gs_url'].match(/^gs:\/\/([^\/]+)\/(.*)/)
@@ -172,25 +189,31 @@ module example.com/cloudfunction
               File.read(dir+"/new.zip")
             }
           end
+
           if @config['code']['gs_url'] and
              (@config['code']['gs_url'] != cloud_desc.source_archive_url or
              current != new)
             need_update = true
-          elsif @config['code']['zip_file'] and current != new
+          elsif (@config['code']['zip_file'] or @config['code']['path']) and current != new
             need_update = true
-            desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
+          end
+
+          if @config['vpc_connector']
+            if cloud_desc.vpc_connector != @config['vpc_connector'] or
+               cloud_desc.vpc_connector_egress_settings != (@config['vpc_connector_allow_all_egress'] ? "ALL_TRAFFIC" : "PRIVATE_RANGES_ONLY")
+              need_update = true
+            end
           end
 
           if need_update
-            func_obj = buildDesc
-            MU.log "Updating Cloud Function #{@mu_name}", MU::NOTICE, details: func_obj
+            MU.log "Updating Cloud Function #{@cloud_id}", MU::NOTICE, details: func_obj
             begin
-#              MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
-#                @cloud_id, 
-#                func_obj
-#              )
-            rescue ::Google::Apis::ClientError
-              MU.log "Error updating Cloud Function #{@mu_name}.", MU::ERR
+              MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
+                @cloud_id, 
+                func_obj
+              )
+            rescue ::Google::Apis::ClientError => e
+              MU.log "Error updating Cloud Function #{@mu_name}.", MU::ERR, e.message
               if desc[:source_archive_url]
                 main_file = nil
                 HELLO_WORLDS.each_pair { |runtime, code|
@@ -207,6 +230,11 @@ module example.com/cloudfunction
 #            service_account_email: sa.kitten.cloud_desc.email,
 #            labels: labels,
 
+          if tempfile
+            tempfile.close
+            tempfile.unlink
+          end
+
         end
 
         # Return the metadata for this project's configuration
@@ -354,6 +382,7 @@ module example.com/cloudfunction
         def self.schema(config)
           toplevel_required = ["runtime"]
           schema = {
+            "roles" => MU::Cloud.resourceClass("Google", "User").schema(config)[1]["roles"],
             "triggers" => {
               "type" => "array",
               "items" => {
@@ -448,6 +477,7 @@ module example.com/cloudfunction
             content_type: "application/zip",
             name: filename
           )
+
           MU::Cloud::Google.storage(credentials: credentials).insert_object(
             bucket,
             obj_obj,
@@ -487,7 +517,7 @@ module example.com/cloudfunction
           end
 # XXX list_project_locations
 
-          if !function['code'] or (!function['code']['zip_file'] and !function['code']['gs_url'])
+          if !function['code'] or (!function['code']['zip_file'] and !function['code']['gs_url'] and !function['code']['path'])
             MU.log "Must specify a code source in Cloud Function #{function['name']}", MU::ERR
             ok = false
           elsif function['code']['zip_file']
@@ -557,22 +587,14 @@ module example.com/cloudfunction
 
           location = "projects/"+@config['project']+"/locations/"+@config['region']
           sa = nil
-          retries = 0
-          begin
-            sa_ref = MU::Config::Ref.get(@config['service_account'])
-            sa = @deploy.findLitterMate(name: sa_ref.name, type: "users")
-            if !sa or !sa.cloud_desc
-              sleep 10
-            end
-          rescue ::Google::Apis::ClientError => e
-            if e.message.match(/notFound:/)
-              sleep 10
-              retries += 1
-              retry
-            end
-          end while !sa or !sa.cloud_desc and retries < 5
+          need_sa = Proc.new {
+            !sa or !sa.kitten or !sa.kitten.cloud_desc
+          }
+          MU.retrier(loop_if: need_sa, wait: 10, max: 6) { |retries, _wait|
+            sa = MU::Config::Ref.get(@config['service_account'])
+          }
 
-          if !sa or !sa.cloud_desc
+          if need_sa.call()
             raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
           end
 
@@ -583,7 +605,7 @@ module example.com/cloudfunction
 #            entry_point: "hello_world",
             entry_point: @config['handler'],
             description: @deploy.deploy_id,
-            service_account_email: sa.cloud_desc.email,
+            service_account_email: sa.kitten.cloud_desc.email,
             labels: labels,
             available_memory_mb: @config['memory']
           }
@@ -596,7 +618,6 @@ module example.com/cloudfunction
           if @config['vpc_connector']
             desc[:vpc_connector] = @config['vpc_connector']
             desc[:vpc_connector_egress_settings] = @config['vpc_connector_allow_all_egress'] ? "ALL_TRAFFIC" : "PRIVATE_RANGES_ONLY"
-            pp desc
           elsif @vpc
             desc[:network] = @vpc.url.sub(/^.*?\/projects\//, 'projects/')
           end
@@ -627,8 +648,22 @@ module example.com/cloudfunction
 #          }
           if @config['code']['gs_url']
             desc[:source_archive_url] = @config['code']['gs_url']
-          elsif @config['code']['zip_file']
+          elsif @config['code']['zip_file'] or @config['code']['path']
+            tempfile = nil
+            if @config['code']['path']
+              tempfile = Tempfile.new(["function", ".zip"])
+              MU.log "#{@mu_name} using code at #{@config['code']['path']}"
+              MU::Master.zipDir(@config['code']['path'], tempfile.path)
+              @config['code']['zip_file'] = tempfile.path
+            else
+              MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
+            end
             desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
+
+            if tempfile
+              tempfile.close
+              tempfile.unlink
+            end
           end
 
 #          Dir.mktmpdir(@mu_name) { |dir|