cloud-mu 3.1.3 → 3.1.4

data/modules/mu/clouds/azure/user.rb

@@ -86,7 +86,6 @@ module MU
 
             client = ::MsRest::ServiceClient.new(cred_obj)
             cloud_desc.client_secret_url.match(/^(http.*?\.azure\.net)(\/.*)/)
-            base = Regexp.last_match[1]
             path = Regexp.last_match[2]
 #MU.log "Calling into #{base} #{path}"
             promise = client.make_request_async(
@@ -97,7 +96,7 @@ module MU
 
             # XXX this is async, need to stop and wait somehow
             promise.then do | result|
-              resp = result.response
+              result.response
 #              MU.log "RESPONSE", MU::WARN, details: resp
             end
           end
@@ -106,7 +105,6 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
-          rgroup_name = @deploy.deploy_id+"-"+@config['region'].upcase
           if @config['roles']
             @config['roles'].each { |role|
               MU::Cloud::Azure::Role.assignTo(cloud_desc.principal_id, role_name: role, credentials: @config['credentials'])
@@ -191,9 +189,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "region" => MU::Config.region_primitive,
@@ -221,9 +219,9 @@ module MU
 
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::users}, bare and unvalidated.
         # @param user [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(user, configurator)
+        def self.validateConfig(user, _configurator)
           ok = true
           user['region'] ||= MU::Cloud::Azure.myRegion(user['credentials'])
 

data/modules/mu/clouds/azure/vpc.rb

@@ -53,7 +53,6 @@ module MU
         def groom
 
           if @config['peers']
-            count = 0
             @config['peers'].each { |peer|
               if peer['vpc']['name']
                 peer_obj = @deploy.findLitterMate(name: peer['vpc']['name'], type: "vpcs", habitat: peer['vpc']['project'])
@@ -113,17 +112,16 @@ module MU
         # Describe this VPC
         # @return [Hash]
         def notify
-          base = {}
           base = MU.structToHash(cloud_desc)
           base["cloud_id"] = @cloud_id.name
           base.merge!(@config.to_h)
           base
         end
-#
+
         # Describe this VPC from the cloud platform's perspective
         # @return [Hash]
-        def cloud_desc
-          if @cloud_desc_cache
+        def cloud_desc(use_cache: true)
+          if @cloud_desc_cache and use_cache
             return @cloud_desc_cache
           end
           @cloud_desc_cache = MU::Cloud::Azure::VPC.find(cloud_id: @cloud_id, resource_group: @resource_group).values.first
@@ -192,10 +190,9 @@ module MU
         # @param use_cache [Boolean]: If available, use saved deployment metadata to describe subnets, instead of querying the cloud API
         # @return [Array<Hash>]: A list of cloud provider identifiers of subnets associated with this VPC.
         def loadSubnets(use_cache: false)
-          desc = cloud_desc
           @subnets = []
 
-          MU::Cloud::Azure.network(credentials: @credentials).subnets.list(@resource_group, cloud_desc.name).each { |subnet|
+          MU::Cloud::Azure.network(credentials: @credentials).subnets.list(@resource_group, cloud_desc(use_cache: use_cache).name).each { |subnet|
             subnet_cfg = {
               "cloud_id" => subnet.name,
               "mu_name" => subnet.name,
@@ -334,7 +331,7 @@ module MU
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
         # XXX add flag to return the diff between @config and live cloud
-        def toKitten(rootparent: nil, billing: nil)
+        def toKitten(**_args)
           return nil if cloud_desc.name == "default" # parent project builds these
           bok = {
             "cloud" => "Azure",
@@ -346,9 +343,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config = nil)
+        def self.schema(_config = nil)
           toplevel_required = []
           schema = {
             "peers" => {
@@ -522,7 +519,6 @@ MU.structToHash(ext_vpc).diff(MU.structToHash(vpc_obj))
           # this is slow, so maybe thread it
           rtb_map = {}
           routethreads = []
-          create_nat_gateway = false
           @config['route_tables'].each { |rtb_cfg|
             routethreads << Thread.new(rtb_cfg) { |rtb|
               rtb_name = @mu_name+"-"+rtb['name'].upcase
@@ -573,6 +569,9 @@ MU.structToHash(ext_vpc).diff(MU.structToHash(vpc_obj))
                 routename = rtb_name+"-"+route['destination_network'].gsub(/[^a-z0-9]/i, "_")
                 route_obj.next_hop_type = if route['gateway'] == "#NAT" and @config['bastion']
                   routename = rtb_name+"-NAT"
+                  if @config['bastion'].is_a?(Hash) and !@config['bastion']['id'] and !@config['bastion']['deploy_id']
+                    @config['bastion']['deploy_id'] = @deploy.deploy_id
+                  end
                   bastion_ref = MU::Config::Ref.get(@config['bastion'])
                   if bastion_ref.kitten and bastion_ref.kitten.cloud_desc
                     iface_id = Id.new(bastion_ref.kitten.cloud_desc.network_profile.network_interfaces.first.id)
@@ -718,8 +717,6 @@ MU.structToHash(ext_subnet).diff(MU.structToHash(subnet_obj))
           loadSubnets
         end
 
-        protected
-
         # Subnets are almost a first-class resource. So let's kinda sorta treat
         # them like one. This should only be invoked on objects that already
         # exists in the cloud layer.
@@ -772,8 +769,8 @@ MU.structToHash(ext_subnet).diff(MU.structToHash(subnet_obj))
           end
 
           # Describe this VPC Subnet from the cloud platform's perspective
-          def cloud_desc
-            return @cloud_desc_cache if !@cloud_desc_cache.nil?
+          def cloud_desc(use_cache: true)
+            return @cloud_desc_cache if @cloud_desc_cache and use_cache
             @cloud_desc_cache = MU::Cloud::Azure.network(credentials: @parent.credentials).subnets.get(@parent.resource_group, @parent.cloud_desc.name, @cloud_id.to_s)
             @cloud_desc_cache
           end

data/modules/mu/clouds/cloudformation.rb

@@ -73,8 +73,8 @@ module MU
       # Stub method- there's no such thing as being "hosted" in a CloudFormation
       # environment. Calls {MU::Cloud::AWS.listAZs} to return sensible
       # values, if we happen to have AWS credentials configured.
-      def self.listAZs(region: MU.curRegion, account: nil, credentials: nil)
-        MU::Cloud::AWS.listAZs(region: region, account: account, credentials: credentials)
+      def self.listAZs(region: MU.curRegion, credentials: nil)
+        MU::Cloud::AWS.listAZs(region: region, credentials: credentials)
       end
 
       # Stub method- there's no such thing as being "hosted" in a CloudFormation
@@ -466,7 +466,7 @@ module MU
         desc["DependsOn"] = []
         if !cloudobj.nil? and cloudobj.respond_to?(:dependencies) and type != "subnet"
           cloudobj.dependencies(use_cache: true).first.each_pair { |resource_classname, resources|
-            resources.each_pair { |sibling_name, sibling_obj|
+            resources.each_pair { |_sibling_name, sibling_obj|
               next if sibling_obj == cloudobj
 #              desc["DependsOn"] << (resource_classname+sibling_obj.cloudobj.mu_name).gsub!(/[^a-z0-9]/i, "")
               desc["DependsOn"] << sibling_obj.cloudobj.cfm_name
@@ -498,7 +498,6 @@ module MU
       # @param name [String]: The name of key we're creating/appending
       # @param value [MU::Config::Tail|String]: The value to set. If it's a {MU::Config::Tail} object, we'll treat it as a reference to a parameter.
       def self.setCloudFormationProp(resource, name, value)
-        realvalue = value
         is_list_element = false
 
         # Recursively resolve MU::Config::Tail references
@@ -508,9 +507,11 @@ module MU
               tree[key] = self.resolveTails(val)
             }
           elsif tree.is_a?(Array)
+            newtree = []
             tree.each { |elt|
-              elt = self.resolveTails(elt)
+              newtree << self.resolveTails(elt)
             }
+            tree = newtree
           elsif tree.class.to_s == "MU::Config::Tail"
             if tree.is_list_element
               return { "Fn::Select" => [tree.index, { "Ref" => "#{tree.getPrettyName}" }] }
@@ -584,7 +585,7 @@ module MU
             cfm_template["Conditions"][cond['name']] = JSON.parse(cond['cloudcode'])
           }
         end
-        tails.each_pair { |param, data|
+        tails.each_pair { |_param, data|
           tail = data
           next if tail.is_a?(MU::Config::Tail) and (tail.pseudo or !tail.runtimecode.nil?)
           default = ""
@@ -638,7 +639,7 @@ module MU
             }
           end
         }
-        MU::Cloud.resource_types.each { |cloudclass, data|
+        MU::Cloud.resource_types.values.each { |data|
           if !config[data[:cfg_plural]].nil? and
               config[data[:cfg_plural]].size > 0
             config[data[:cfg_plural]].each { |resource|

data/modules/mu/clouds/cloudformation/vpc.rb

@@ -240,8 +240,6 @@ module MU
           {}
         end
 
-        protected
-
         # Subnets are almost a first-class resource. So let's kinda sorta treat
         # them like one. This should only be invoked on objects that already
         # exists in the cloud layer.
@@ -288,13 +286,13 @@ module MU
 
         # Placeholder. This is a NOOP for CloudFormation, which doesn't build
         # resources directly.
-        def self.find(*args)
+        def self.find(**args)
           MU.log "find() not implemented for CloudFormation layer", MU::DEBUG
           nil
         end
         # Placeholder. This is a NOOP for CloudFormation, which doesn't build
         # resources directly.
-        def self.cleanup(*args)
+        def self.cleanup(**args)
           MU.log "cleanup() not implemented for CloudFormation layer", MU::DEBUG
           nil
         end

data/modules/mu/clouds/google.rb

@@ -52,6 +52,17 @@ module MU
         [:url]
       end
 
+      # Most of our resource implementation +find+ methods have to mangle their
+      # args to make sure they've extracted a project or location argument from
+      # other available information. This does it for them.
+      # @return [Hash]
+      def self.findLocationArgs(**args)
+        args[:project] ||= args[:habitat]
+        args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
+        args[:location] ||= args[:region] || args[:availability_zone] || "-"
+        args
+      end
+
       # A hook that is always called just before any of the instance method of
       # our resource implementations gets invoked, so that we can ensure that
       # repetitive setup tasks (like resolving +:resource_group+ for Azure
@@ -141,10 +152,9 @@ module MU
       def self.habitat(cloudobj, nolookup: false, deploy: nil)
         @@habmap ||= {}
 # XXX whaddabout config['habitat'] HNNNGH
-
         return nil if !cloudobj.cloudclass.canLiveIn.include?(:Habitat)
 
-# XXX users are assholes because they're valid two different ways ugh ugh
+# XXX these are assholes because they're valid two different ways ugh ugh
         return nil if [MU::Cloud::Google::Group, MU::Cloud::Google::Folder].include?(cloudobj.cloudclass)
         if cloudobj.config and cloudobj.config['project']
           if nolookup
@@ -154,7 +164,6 @@ module MU
             return @@habmap[cloudobj.config['project']]
           end
           deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy)
-
           projectobj = projectLookup(cloudobj.config['project'], deploy, raise_on_fail: false)
 
           if projectobj
@@ -363,7 +372,7 @@ module MU
         cfg = credConfig(credentials)
         return if !cfg or !cfg['project']
         flags["project"] ||= cfg['project']
-        name = deploy_id+"-secret"
+
         resp = MU::Cloud::Google.storage(credentials: credentials).list_objects(
           adminBucketName(credentials),
           prefix: deploy_id
@@ -420,7 +429,7 @@ module MU
 MU.log e.message, MU::WARN, details: e.inspect
           if e.inspect.match(/body: "Not Found"/)
             raise MuError, "Google admin bucket #{adminBucketName(credentials)} or key #{name} does not appear to exist or is not visible with #{credentials ? credentials : "default"} credentials"
-          elsif e.message.match(/notFound: |Unknown user:/)
+          elsif e.message.match(/notFound: |Unknown user:|conflict: /)
             if retries < 5
               sleep 5
               retries += 1
@@ -429,7 +438,7 @@ MU.log e.message, MU::WARN, details: e.inspect
               raise e
             end
           elsif e.inspect.match(/The metadata for object "null" was edited during the operation/)
-            MU.log e.message+" - Google admin bucket #{adminBucketName(credentials)}/#{name} with #{credentials ? credentials : "default"} credentials", MU::WARN, details: aclobj
+            MU.log e.message+" - Google admin bucket #{adminBucketName(credentials)}/#{name} with #{credentials ? credentials : "default"} credentials", MU::DEBUG, details: aclobj
             sleep 10
             retry
           else
@@ -540,7 +549,7 @@ MU.log e.message, MU::WARN, details: e.inspect
               listRegions(credentials: credentials)
               listInstanceTypes(credentials: credentials)
               listProjects(credentials)
-            rescue ::Google::Apis::ClientError => e
+            rescue ::Google::Apis::ClientError
               MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes
               @@authorizers.delete(credentials)
               return nil
@@ -717,7 +726,6 @@ MU.log e.message, MU::WARN, details: e.inspect
             raise e
           end
 
-          regions = []
           result.items.each { |region|
             @@regions[region.name] = []
             region.zones.each { |az|
@@ -846,7 +854,7 @@ MU.log e.message, MU::WARN, details: e.inspect
           if subclass.nil?
             begin
               @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true)
-            rescue Signet::AuthorizationError => e
+            rescue Signet::AuthorizationError
               MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN
               @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
               @@readonly[credentials] ||= {}
@@ -1052,8 +1060,6 @@ MU.log e.message, MU::WARN, details: e.inspect
         @@customer_ids_cache[credentials]
       end
 
-      private
-
       # Wrapper class for Google APIs, so that we can catch some common
       # transient endpoint errors without having to spray rescues all over the
       # codebase.
@@ -1109,12 +1115,13 @@ MU.log e.message, MU::WARN, details: e.inspect
         # @param filter [String]: The Compute API filter string to use to isolate appropriate resources
         def delete(type, project, region = nil, noop = false, filter = "description eq #{MU.deploy_id}", credentials: nil)
           list_sym = "list_#{type.sub(/y$/, "ie")}s".to_sym
+          credentials ||= @credentials
           resp = nil
           begin
             if region
-              resp = MU::Cloud::Google.compute(credentials: @credentials).send(list_sym, project, region, filter: filter, mu_gcp_enable_apis: false)
+              resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, region, filter: filter, mu_gcp_enable_apis: false)
             else
-              resp = MU::Cloud::Google.compute(credentials: @credentials).send(list_sym, project, filter: filter, mu_gcp_enable_apis: false)
+              resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, filter: filter, mu_gcp_enable_apis: false)
             end
 
           rescue ::Google::Apis::ClientError => e
@@ -1137,9 +1144,9 @@ MU.log e.message, MU::WARN, details: e.inspect
                     resp = nil
                     failed = false
                     if region
-                      resp = MU::Cloud::Google.compute(credentials: @credentials).send(delete_sym, project, region, obj.name)
+                      resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, region, obj.name)
                     else
-                      resp = MU::Cloud::Google.compute(credentials: @credentials).send(delete_sym, project, obj.name)
+                      resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, obj.name)
                     end
 
                     if resp.error and resp.error.errors and resp.error.errors.size > 0
@@ -1195,11 +1202,19 @@ MU.log e.message, MU::WARN, details: e.inspect
             retval = nil
             retries = 0
             wait_backoff = 5
-            if next_page_token
-              if arguments.size == 1 and arguments.first.is_a?(Hash)
-                arguments[0][:page_token] = next_page_token
-              else
-                arguments << { :page_token => next_page_token }
+            if next_page_token 
+              if method_sym != :list_entry_log_entries
+                if arguments.size == 1 and arguments.first.is_a?(Hash)
+                  arguments[0][:page_token] = next_page_token
+                else
+                  arguments << { :page_token => next_page_token }
+                end
+              elsif arguments.first.class == ::Google::Apis::LoggingV2::ListLogEntriesRequest
+                arguments[0] = ::Google::Apis::LoggingV2::ListLogEntriesRequest.new(
+                  resource_names: arguments.first.resource_names,
+                  filter: arguments.first.filter,
+                  page_token: next_page_token
+                )
               end
             end
             begin
@@ -1318,7 +1333,6 @@ MU.log e.message, MU::WARN, details: e.inspect
             if retval.class.name.match(/.*?::Operation$/)
 
               retries = 0
-              orig_target = retval.name
 
               # Check whether the various types of +Operation+ responses say
               # they're done, without knowing which specific API they're from
@@ -1390,7 +1404,7 @@ MU.log e.message, MU::WARN, details: e.inspect
               # take advantage.
               # XXX might want to do something similar for delete ops? just the
               # but where we wait for the operation to definitely be done
-              had_been_found = false
+#              had_been_found = false
               if method_sym.to_s.match(/^(insert|create|patch)_/)
                 get_method = method_sym.to_s.gsub(/^(insert|patch|create_disk|create)_/, "get_").to_sym
                 cloud_id = if retval.respond_to?(:target_link)
@@ -1417,7 +1431,7 @@ MU.log e.message, MU::WARN, details: e.inspect
 #if method_sym == :insert_instance
 #MU.log "actual_resource", MU::WARN, details: actual_resource
 #end
-                had_been_found = true
+#                had_been_found = true
                 if actual_resource.respond_to?(:status) and
                   ["PROVISIONING", "STAGING", "PENDING", "CREATING", "RESTORING"].include?(actual_resource.status)
                   retries = 0
@@ -1440,6 +1454,7 @@ MU.log e.message, MU::WARN, details: e.inspect
             if overall_retval
               if method_sym.to_s.match(/^list_(.*)/)
                 require 'google/apis/iam_v1'
+                require 'google/apis/logging_v2'
                 what = Regexp.last_match[1].to_sym
                 whatassign = (Regexp.last_match[1]+"=").to_sym
                 if overall_retval.class == ::Google::Apis::IamV1::ListServiceAccountsResponse
@@ -1451,7 +1466,7 @@ MU.log e.message, MU::WARN, details: e.inspect
                     newarray = retval.public_send(what) + overall_retval.public_send(what)
                     overall_retval.public_send(whatassign, newarray)
                   end
-                else
+                elsif !retval.respond_to?(:next_page_token) or retval.next_page_token.nil? or retval.next_page_token.empty?
                   MU.log "Not sure how to append #{method_sym.to_s} results to #{overall_retval.class.name} (apparently #{what.to_s} and #{whatassign.to_s} aren't it), returning first page only", MU::WARN, details: retval
                   return retval
                 end

data/modules/mu/clouds/google/bucket.rb

@@ -34,7 +34,7 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
-          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
+          @project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloud_id
 
           current = cloud_desc
           changed = false
@@ -143,15 +143,14 @@ module MU
         # Remove all buckets associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
-        # @param region [String]: The cloud provider region
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
+        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
           flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
 
           resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['project'])
           if resp and resp.items
             resp.items.each { |bucket|
-              if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase
+              if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or bucket.labels['mu-master-ip'] == MU.mu_public_ip.gsub(/\./, "_"))
                 MU.log "Deleting Cloud Storage bucket #{bucket.name}"
                 if !noop
                   MU::Cloud::Google.storage(credentials: credentials).delete_bucket(bucket.name)
@@ -172,8 +171,7 @@ module MU
         # Locate an existing bucket.
         # @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
         def self.find(**args)
-          args[:project] ||= args[:habitat]
-          args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
+          args = MU::Cloud::Google.findLocationArgs(args)
 
           found = {}
           if args[:cloud_id]
@@ -198,7 +196,7 @@ module MU
         # Reverse-map our cloud description into a runnable config hash.
         # We assume that any values we have in +@config+ are placeholders, and
         # calculate our own accordingly based on what's live in the cloud.
-        def toKitten(rootparent: nil, billing: nil, habitats: nil)
+        def toKitten(**_args)
           bok = {
             "cloud" => "Google",
             "credentials" => @config['credentials'],
@@ -300,9 +298,9 @@ module MU
         end
 
         # Cloud-specific configuration properties.
-        # @param config [MU::Config]: The calling MU::Config object
+        # @param _config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
-        def self.schema(config)
+        def self.schema(_config)
           toplevel_required = []
           schema = {
             "storage_class" => {
@@ -322,9 +320,9 @@ module MU
         # Cloud-specific pre-processing of {MU::Config::BasketofKittens::bucket}, bare and unvalidated.
 
         # @param bucket [Hash]: The resource to process and validate
-        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
         # @return [Boolean]: True if validation succeeded, False otherwise
-        def self.validateConfig(bucket, configurator)
+        def self.validateConfig(bucket, _configurator)
           ok = true
           bucket['project'] ||= MU::Cloud::Google.defaultProject(bucket['credentials'])