cloud-mu 3.2.0 → 3.5.0

data/modules/mu/providers/aws/vpc_subnet.rb

@@ -29,18 +29,21 @@ module MU
           attr_reader :mu_name
           attr_reader :name
           attr_reader :az
+          attr_reader :config
           attr_reader :cloud_desc
 
           # @param parent [MU::Cloud::AWS::VPC]: The parent VPC of this subnet.
           # @param config [Hash<String>]:
           def initialize(parent, config)
-            @parent = parent
             @config = MU::Config.manxify(config)
+            MU::Cloud::AWS.resourceInitHook(self, @deploy)
+            @parent = parent
             @cloud_id = config['cloud_id']
+            @credentials ||= config['credentials']
             @mu_name = config['mu_name']
             @name = config['name']
             @deploydata = config # This is a dummy for the sake of describe()
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [@cloud_id]).subnets.first
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_subnets(subnet_ids: [@cloud_id]).subnets.first
             @az = resp.availability_zone
             @ip_block = resp.cidr_block
             @cloud_desc = resp # XXX this really isn't the cloud implementation's business
@@ -50,11 +53,11 @@ module MU
           # Return the cloud identifier for the default route of this subnet.
           # @return [String,nil]
           def defaultRoute
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_route_tables(
                 filters: [{name: "association.subnet-id", values: [@cloud_id]}]
             )
             if resp.route_tables.size == 0 # use default route table for the VPC
-              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+              resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_route_tables(
                  filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
               )
             end
@@ -75,11 +78,11 @@ module MU
           # @return [Boolean]
           def private?
             return false if @cloud_id.nil?
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_route_tables(
                 filters: [{name: "association.subnet-id", values: [@cloud_id]}]
             )
             if resp.route_tables.size == 0 # use default route table for the VPC
-              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
+              resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_route_tables(
                  filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
               )
             end
@@ -106,14 +109,14 @@ module MU
 
           subnetthreads = Array.new
 
-          azs = MU::Cloud::AWS.listAZs(region: @config['region'], credentials: @config['credentials'])
+          azs = MU::Cloud::AWS.listAZs(region: @region, credentials: @credentials)
           @config['subnets'].each { |subnet|
             subnet_name = @config['name']+"-"+subnet['name']
             az = subnet['availability_zone'] ? subnet['availability_zone'] : azs.op
             MU.log "Creating Subnet #{subnet_name} (#{subnet['ip_block']}) in #{az}", details: subnet
 
             subnetthreads << Thread.new {
-              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_subnet(
+              resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_subnet(
                 vpc_id: @cloud_id,
                 cidr_block: subnet['ip_block'],
                 availability_zone: az
@@ -123,7 +126,7 @@ module MU
               tag_me(subnet_id, @mu_name+"-"+subnet['name'])
 
               loop_if = Proc.new {
-                resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
+                resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_subnets(subnet_ids: [subnet_id]).subnets.first
                 (!resp or resp.state != "available")
               }
 
@@ -141,7 +144,7 @@ module MU
                 end
                 MU.log "Associating Route Table '#{subnet['route_table']}' (#{routes[subnet['route_table']]['route_table_id']}) with #{subnet_name}"
                 MU.retrier([Aws::EC2::Errors::InvalidRouteTableIDNotFound], wait: 10, max: 10) {
-                  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_route_table(
+                  MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).associate_route_table(
                     route_table_id: routes[subnet['route_table']]['route_table_id'],
                     subnet_id: subnet_id
                   )
@@ -150,7 +153,7 @@ module MU
 
               if subnet.has_key?("map_public_ips")
                 MU.retrier([Aws::EC2::Errors::InvalidSubnetIDNotFound], wait: 10, max: 10) {
-                  resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_subnet_attribute(
+                  resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).modify_subnet_attribute(
                     subnet_id: subnet_id,
                     map_public_ip_on_launch: {
                       value: subnet['map_public_ips'],
@@ -167,7 +170,7 @@ module MU
                 loggroup = @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs")
                 logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
                 MU.log "Enabling traffic logging on Subnet #{subnet_name} in VPC #{@mu_name} to log group #{loggroup.mu_name}"
-                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_flow_logs(
+                MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_flow_logs(
                   resource_ids: [subnet_id],
                   resource_type: "Subnet",
                   traffic_type: subnet["traffic_type_to_log"],
@@ -188,30 +191,31 @@ module MU
         def allocate_eip_for_nat
           MU::MommaCat.lock("nat-gateway-eipalloc")
 
-          eips = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(
-            filters: [
-              {
-                name: "domain",
-                values: ["vpc"]
-              }
-            ]
-          ).addresses
-
-          allocation_id = nil
-          eips.each { |eip|
-            next if !eip.association_id.nil? and !eip.association_id.empty?
-            if (eip.private_ip_address.nil? || eip.private_ip_address.empty?) and MU::MommaCat.lock(eip.allocation_id, true, true)
-              if !@eip_allocation_ids.include?(eip.allocation_id)
-                allocation_id = eip.allocation_id
-                break
-              end
-            end
-          }
-
-          if allocation_id.nil?
-            allocation_id = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).allocate_address(domain: "vpc").allocation_id
-            MU::MommaCat.lock(allocation_id, false, true)
-          end
+#          eips = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_addresses(
+#            filters: [
+#              {
+#                name: "domain",
+#                values: ["vpc"]
+#              }
+#            ]
+#          ).addresses
+
+#          allocation_id = nil
+#          eips.each { |eip|
+#            next if !eip.association_id.nil? and !eip.association_id.empty?
+#            if (eip.private_ip_address.nil? || eip.private_ip_address.empty?) and MU::MommaCat.lock(eip.allocation_id, true, true)
+#              if !@eip_allocation_ids.include?(eip.allocation_id)
+#                allocation_id = eip.allocation_id
+#                break
+#              end
+#            end
+#          }
+
+#          if allocation_id.nil?
+            allocation_id = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).allocate_address(domain: "vpc").allocation_id
+            tag_me(allocation_id)
+#            MU::MommaCat.lock(allocation_id, false, true)
+#          end
 
           @eip_allocation_ids << allocation_id
 
@@ -223,15 +227,15 @@ module MU
         def create_nat_gateway(subnet)
           allocation_id = allocate_eip_for_nat
 
-          nat_gateway_id = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_nat_gateway(
+          nat_gateway_id = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_nat_gateway(
             subnet_id: subnet['subnet_id'],
             allocation_id: allocation_id,
           ).nat_gateway.nat_gateway_id
 
           ensure_unlock = Proc.new { MU::MommaCat.unlock(allocation_id, true) }
-          resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
+          resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
           loop_if = Proc.new {
-            resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
+            resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
             resp.class != Aws::EC2::Types::NatGateway or resp.state == "pending"
           }
 

data/modules/mu/providers/azure.rb

@@ -288,9 +288,14 @@ module MU
           raise MuError, "Nil response from Azure API attempting list_locations(#{subscription})"
         end
 
-        sdk_response.value.each do | region |
-          @@regions.push(region.name)
-        end
+        sdk_response.value.each { |region|
+          begin
+            listInstanceTypes(region.name) # use this to filter for broken regions
+            @@regions.push(region.name)
+          rescue APIError => e
+            MU.log "Azure region "+region.name+" does not appear operational, skipping", MU::WARN
+          end
+        }
 
         return us_only ? @@regions.reject { |r| !r.match(/us\d?$/) } : @@regions
       end
@@ -350,13 +355,42 @@ module MU
           listRegions.each { |region|
             next if !deploy.regionsUsed.include?(region)
             begin
-              createResourceGroup(deploy.deploy_id+"-"+region.upcase, region, credentials: creds)
+              rg_obj = createResourceGroup(deploy.deploy_id+"-"+region.upcase, region, credentials: creds)
+              createVault(rg_obj.name, region, deploy, credentials: creds)
             rescue ::MsRestAzure::AzureOperationError
             end
           }
         }
       end
 
+      # Arguably this should be a first class resource, but for now we'll do
+      # it here since we're going to have a generic deployment vault in every
+      # resource group.
+      # @param rg [String]: The name of the resource group in which we'll reside
+      # @param region [String]: The region in which we'll reside
+      # @param deploy [MU::MommaCat]: The deployment which we serve
+      # @param credentials [String]:
+      def self.createVault(rg, region, deploy, credentials: nil)
+        cred_hash = MU::Cloud::Azure.getSDKOptions(credentials)
+        vaultname = deploy.getResourceName(region, max_length: 23, disallowed_chars: /[^a-z0-9-]/i, never_gen_unique: true)
+        MU::Cloud::Azure.ensureProvider("Microsoft.KeyVault", credentials: credentials)
+        sku = MU::Cloud::Azure.keyvault(:Sku).new
+        sku.name = "standard" # ...I'm angry about this
+
+        props = MU::Cloud::Azure.keyvault(:VaultProperties).new
+        props.tenant_id = cred_hash[:tenant_id]
+        props.enabled_for_deployment = true
+        props.sku = sku
+        props.access_policies = []
+
+        params = MU::Cloud::Azure.keyvault(:VaultCreateOrUpdateParameters).new
+        params.location = region
+        params.properties = props
+
+        MU.log "Creating KeyVault #{vaultname} in #{region}"
+        MU::Cloud::Azure.keyvault(credentials: credentials).vaults.create_or_update(rg, vaultname, params)
+      end
+
       @@rg_semaphore = Mutex.new
 
       # Purge cloud-specific deploy meta-artifacts (SSH keys, resource groups,
@@ -400,17 +434,30 @@ module MU
           end
         }
         MU.log "Configuring resource group #{name} in #{region}", details: rg_obj
-        MU::Cloud::Azure.resources(credentials: credentials).resource_groups.create_or_update(
+        rg = MU::Cloud::Azure.resources(credentials: credentials).resource_groups.create_or_update(
           name,
           rg_obj
         )
+
+        rg
       end
 
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
-      # @param deploy_id [String]: The deploy for which we're writing the secret
+      # @param deploy [MU::MommaCat]: The deploy for which we're writing the secret
       # @param value [String]: The contents of the secret
-      def self.writeDeploySecret(deploy_id, value, name = nil, credentials: nil)
-# XXX this ain't it hoss
+      def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
+        deploy_id = deploy.deploy_id
+
+        listRegions.each { |region|
+          next if !deploy.regionsUsed.include?(region)
+          rg = deploy_id+"-"+region.upcase
+          vaultname = deploy.getResourceName(region, max_length: 23, disallowed_chars: /[^a-z0-9-]/i, never_gen_unique: true)
+
+          resp = MU::Cloud::Azure.keyvault(credentials: credentials).vaults.get(rg, vaultname)
+          next if !resp
+MU.log "vault existence check #{vaultname}", MU::WARN, details: resp
+
+        }
       end
 
       # Return the name strings of all known sets of credentials for this cloud
@@ -522,7 +569,7 @@ module MU
 
         begin
           Timeout.timeout(2) do
-            resp = JSON.parse(open("#{base_url}/?#{arg_str}","Metadata"=>"true").read)
+            resp = JSON.parse(URI.open("#{base_url}/?#{arg_str}","Metadata"=>"true").read)
             MU.log "curl -H Metadata:true "+"#{base_url}/?#{arg_str}", loglevel, details: resp
             if svc != "instance"
               return resp
@@ -777,6 +824,24 @@ module MU
         return @@resources_api[credentials]
       end
 
+      # The Azure KeyVault API
+      # @param model [<Azure::Apis::KeyVault::Mgmt::V2018_02_14::Models>]: If specified, will return the class ::Azure::Apis::KeyVault::Mgmt::V2018_02_14::Models::model instead of an API client instance
+      # @param model_version [String]: Use an alternative model version supported by the SDK when requesting a +model+
+      # @param alt_object [String]: Return an instance of something other than the usual API client object
+      # @param credentials [String]: The credential set (subscription, effectively) in which to operate
+      # @return [MU::Cloud::Azure::SDKClient]
+      def self.keyvault(model = nil, alt_object: nil, credentials: nil, model_version: "V2018_02_14")
+        require 'azure_mgmt_key_vault'
+
+        if model and model.is_a?(Symbol)
+          return Object.const_get("Azure").const_get("KeyVault").const_get("Mgmt").const_get(model_version).const_get("Models").const_get(model)
+        else
+          @@keyvault_api[credentials] ||= MU::Cloud::Azure::SDKClient.new(api: "KeyVault", credentials: credentials, subclass: alt_object)
+        end
+
+        return @@keyvault_api[credentials]
+      end
+
       # The Azure Features API
       # @param model [<Azure::Apis::Features::Mgmt::V2015_12_01::Models>]: If specified, will return the class ::Azure::Apis::Features::Mgmt::V2015_12_01::Models::model instead of an API client instance
       # @param model_version [String]: Use an alternative model version supported by the SDK when requesting a +model+
@@ -931,6 +996,7 @@ module MU
       @@resources_api = {}
       @@containers_api = {}
       @@features_api = {}
+      @@keyvault_api = {}
       @@apis_api = {}
       @@marketplace_api = {}
       @@service_identity_api = {}
@@ -1069,9 +1135,9 @@ module MU
                       retry
                     end
 
-                    MU.log "#{@parent.api.class.name}.#{@myname}.#{method_sym.to_s} returned '"+err["code"]+"' - "+err["message"], MU::WARN, details: caller
-                    MU.log e.backtrace[0], MU::WARN, details: parsed
-                    raise MU::Cloud::Azure::APIError, err["code"]+": "+err["message"]+" (call was #{@parent.api.class.name}.#{@myname}.#{method_sym.to_s})"
+#                    MU.log "#{@parent.api.class.name}.#{@myname}.#{method_sym.to_s} returned '"+err["code"]+"' - "+err["message"], MU::WARN, details: caller
+#                    MU.log e.backtrace[0], MU::WARN, details: parsed
+                    raise MU::Cloud::Azure::APIError.new err["code"]+": "+err["message"]+" (call was #{@parent.api.class.name}.#{@myname}.#{method_sym.to_s})", details: parsed, silent: true
                   end
                 end
               rescue JSON::ParserError

data/modules/mu/providers/azure/server.rb

@@ -150,7 +150,7 @@ module MU
 
             if !@nat.nil? and @nat.mu_name != @mu_name
               if @nat.cloud_desc.nil?
-                MU.log "NAT was missing cloud descriptor when called in #{@mu_name}'s getSSHConfig", MU::ERR
+                MU.log "NAT #{@nat} was missing cloud descriptor when called in #{@mu_name}'s getSSHConfig", MU::ERR
                 return nil
               end
               _foo, _bar, _baz, nat_ssh_host, nat_ssh_user, nat_ssh_key  = @nat.getSSHConfig
@@ -220,6 +220,7 @@ module MU
         # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching instances
         def self.find(**args)
           found = {}
+MU.log "Azure::Server.find called", MU::NOTICE, details: args
           # told one, we may have to search all the ones we can see.
           resource_groups = if args[:resource_group]
             [args[:resource_group]]
@@ -417,6 +418,7 @@ module MU
 
         # return [String]: A password string.
         def getWindowsAdminPassword
+          @deploy.fetchSecret(@mu_name, "windows_admin_password")
         end
 
         # Add a volume to this instance
@@ -452,7 +454,7 @@ module MU
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
         end
 
         # Cloud-specific configuration properties.
@@ -612,7 +614,7 @@ module MU
               ok = false
             end
             MU::Config.addDependency(server, server['name']+"vpc", "vpc")
-            MU::Config.addDependency(server, server['name']+"vpc-natstion", "server", phase: "groom")
+            MU::Config.addDependency(server, server['name']+"vpc-natstion", "server", their_phase: "groom")
             server['vpc'] = {
               "name" => server['name']+"vpc",
               "subnet_pref" => "private"
@@ -636,6 +638,7 @@ module MU
           ok
         end
 
+        # stub
         def self.diskConfig(config, create = true, disk_as_url = true, credentials: nil)
         end
 
@@ -769,10 +772,23 @@ module MU
 
           os_obj = MU::Cloud::Azure.compute(:OSProfile).new
           if windows?
+            winrm_listen = MU::Cloud::Azure.compute(:WinRMListener).new
+            winrm_listen.certificate_url = "goddamn stupid ass thing"
+            winrm_listen.protocol = "https"
+            winrm = MU::Cloud::Azure.compute(:WinRMConfiguration).new
+            winrm.listeners = [winrm_listen]
+
             win_obj = MU::Cloud::Azure.compute(:WindowsConfiguration).new
+            win_obj.win_rmconfiguration = winrm
             os_obj.windows_configuration = win_obj
             os_obj.admin_username = @config['windows_admin_username']
-            os_obj.admin_password = MU.generateWindowsPassword
+            os_obj.admin_password = begin
+              @deploy.fetchSecret(@mu_name, "windows_admin_password")
+            rescue MU::MommaCat::SecretError
+              pw = MU.generateWindowsPassword
+              @deploy.saveNodeSecret(@mu_name, pw, "windows_admin_password")
+              pw
+            end
             os_obj.computer_name = @deploy.getResourceName(@config["name"], max_length: 15, disallowed_chars: /[~!@#$%^&*()=+_\[\]{}\\\|;:\.'",<>\/\?]/)
           else
             os_obj.admin_username = @config['ssh_user']

data/modules/mu/providers/cloudformation/server.rb

@@ -304,7 +304,7 @@ module MU
                     role_name: baserole.role_name,
                     policy_name: name
                 )
-                policies[name] = URI.decode(resp.policy_document)
+                policies[name] = CGI.unescape(resp.policy_document)
               }
             }
           end

data/modules/mu/providers/google.rb

@@ -236,7 +236,7 @@ module MU
       # @param sibling_only [Boolean]
       # @return [MU::Config::Habitat,nil]
       def self.projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false)
-        project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy if !caller.grep(/`findLitterMate'/) # XXX the dumbest
+        project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy and caller.grep(/`findLitterMate'/).empty? # XXX the dumbest
 
         if !project_obj and !sibling_only
           resp = MU::MommaCat.findStray(
@@ -348,7 +348,8 @@ module MU
       # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
       # @param deploy_id [String]: The deploy for which we're writing the secret
       # @param value [String]: The contents of the secret
-      def self.writeDeploySecret(deploy_id, value, name = nil, credentials: nil)
+      def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
+        deploy_id = deploy.deploy_id
         name ||= deploy_id+"-secret"
         begin
           MU.log "Writing #{name} to Cloud Storage bucket #{adminBucketName(credentials)}"
@@ -487,7 +488,7 @@ MU.log e.message, MU::WARN, details: e.inspect
         base_url = "http://metadata.google.internal/computeMetadata/v1"
         begin
           Timeout.timeout(2) do
-            response = open(
+            response = URI.open(
               "#{base_url}/#{param}",
               "Metadata-Flavor" => "Google"
             ).read
@@ -981,7 +982,20 @@ MU.log e.message, MU::WARN, details: e.inspect
       end
 
       # Google's Cloud Billing Service API
-      # @param subclass [<Google::Apis::LoggingV2>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
+      # @param subclass [<Google::Apis::CloudbillingV1>]: If specified, will return the class ::Google::Apis::CloudbillingV1::subclass instead of an API client instance
+      def self.budgets(subclass = nil, credentials: nil)
+        require 'google/apis/billingbudgets_v1'
+
+        if subclass.nil?
+          @@budgets_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "BillingbudgetsV1::CloudBillingBudgetService", scopes: ['cloud-platform', 'cloud-billing'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
+          return @@budgets_api[credentials]
+        elsif subclass.is_a?(Symbol)
+          return Object.const_get("::Google").const_get("Apis").const_get("BillingbudgetsV1").const_get(subclass)
+        end
+      end
+
+      # Google's Cloud Billing Budget Service API
+      # @param subclass [<Google::Apis::CloudbillingV1>]: If specified, will return the class ::Google::Apis::CloudbillingV1::subclass instead of an API client instance
       def self.billing(subclass = nil, credentials: nil)
         require 'google/apis/cloudbilling_v1'
 
@@ -1024,6 +1038,7 @@ MU.log e.message, MU::WARN, details: e.inspect
       # @return [Array<OpenStruct>],nil]
       def self.getOrg(credentials = nil, with_id: nil)
         creds = MU::Cloud::Google.credConfig(credentials)
+        return nil if !creds
         credname = if creds and creds['name']
           creds['name']
         else
@@ -1309,7 +1324,7 @@ MU.log e.message, MU::WARN, details: e.inspect
 
                   svc_name = Regexp.last_match[1]
                   save_verbosity = MU.verbosity
-                  if svc_name != "servicemanagement.googleapis.com" and method_sym != :delete
+                  if !["servicemanagement.googleapis.com", "billingbudgets.googleapis.com"].include?(svc_name) and method_sym != :delete
                     retries += 1
                     @@enable_semaphores[project].synchronize {
                       MU.setLogging(MU::Logger::NORMAL)
@@ -1564,6 +1579,7 @@ MU.log e.message, MU::WARN, details: e.inspect
       @@firestore_api = {}
       @@admin_directory_api = {}
       @@billing_api = {}
+      @@budgets_api = {}
       @@function_api = {}
     end
   end