cloud-mu 2.0.0.pre.beta2 → 2.0.0.pre.beta3

data/modules/mu/clouds/aws/log.rb

@@ -216,6 +216,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all logs associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/msg_queue.rb

@@ -133,6 +133,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all msg_queues associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/nosqldb.rb

@@ -160,6 +160,12 @@ pp params
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all buckets associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/notifier.rb

@@ -62,6 +62,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all notifiers associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/role.rb

@@ -292,6 +292,12 @@ module MU
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all roles associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
@@ -485,6 +491,42 @@ module MU
           resp.instance_profile.arn
         end
 
+        # Schema fragment for IAM policy conditions, which some other resource
+        # types may need to import.
+        def self.condition_schema
+          {
+            "items" => {
+              "properties" => {
+                "conditions" => {
+                  "type" => "array",
+                  "items" => {
+                    "type" => "object",
+                    "description" => "One or more conditions under which to apply this policy. See also: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition.html",
+                    "required" => ["comparison", "variable", "values"],
+                    "properties" => {
+                      "comparison" => {
+                        "type" => "string",
+                        "description" => "A comparison to make, like +DateGreaterThan+ or +IpAddress+."
+                      },
+                      "variable" => {
+                        "type" => "string",
+                        "description" => "The variable which we will compare, like +aws:CurrentTime+ or +aws:SourceIp+."
+                      },
+                      "values" => {
+                        "type" => "array",
+                        "items" => {
+                          "type" => "string",
+                          "description" => "Value(s) to which we will compare our variable, like +2013-08-16T15:00:00Z+ or +192.0.2.0/24+."
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        end
+
         # Cloud-specific configuration properties.
         # @param config [MU::Config]: The calling MU::Config object
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
@@ -499,9 +541,11 @@ module MU
             end
           }.map { |t| MU::Cloud.resource_types[t][:cfg_name] }.sort
 
+
           schema = {
             "tags" => MU::Config.tags_primitive,
             "optional_tags" => MU::Config.optional_tags_primitive,
+            "policies" => self.condition_schema,
             "import" => {
               "items" => {
                 "description" => "Can be a shorthand reference to a canned IAM policy like +AdministratorAccess+, or a full ARN like +arn:aws:iam::aws:policy/AmazonESCognitoAccess+"
@@ -611,14 +655,15 @@ module MU
           ok
         end
 
-        private
-
-        # Convert entries from the cloud-neutral @config['policies'] list into
-        # AWS syntax.
-        def convert_policies_to_iam
+        # Convert our generic internal representation of access policies into
+        # structures suitable for AWS IAM policy documents.
+        # @param policies [Array<Hash>]: One or more policy chunks
+        # @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
+        # @return [Array<Hash>]
+        def self.genPolicyDocument(policies, deploy_obj: nil)
           iam_policies = []
-          if @config['policies']
-            @config['policies'].each { |policy|
+          if policies
+            policies.each { |policy|
               doc = {
                 "Version" => "2012-10-17",
                 "Statement" => [
@@ -633,19 +678,52 @@ module MU
               policy["permissions"].each { |perm|
                 doc["Statement"].first["Action"] << perm
               }
+              if policy["conditions"]
+                doc["Statement"].first["Condition"] ||= {}
+                policy["conditions"].each { |cond|
+                  doc["Statement"].first["Condition"][cond['comparison']] = {
+                    cond["variable"] => cond["values"]
+                  }
+                }
+              end
+              if policy["grant_to"] # XXX factor this with target, they're too similar
+                doc["Statement"].first["Principal"] ||= []
+                policy["grant_to"].each { |grantee|
+                  if grantee["type"] and deploy_obj
+                    sibling = deploy_obj.findLitterMate(
+                      name: grantee["identifier"],
+                      type: grantee["type"]
+                    )
+                    if sibling
+                      id = sibling.cloudobj.arn
+                      doc["Statement"].first["Principal"] << id
+                    else
+                      raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy"
+                    end
+                  else
+                    doc["Statement"].first["Principal"] << grantee["identifier"]
+                  end
+                }
+                if policy["grant_to"].size == 1
+                  doc["Statement"].first["Principal"] = doc["Statement"].first["Principal"].first
+                end
+              end
               if policy["targets"]
                 policy["targets"].each { |target|
-                  if target["type"]
-                    sibling = @deploy.findLitterMate(
+                  if target["type"] and deploy_obj
+                    sibling = deploy_obj.findLitterMate(
                       name: target["identifier"],
                       type: target["type"]
                     )
                     if sibling
-                      doc["Statement"].first["Resource"] << sibling.cloudobj.arn
+                      id = sibling.cloudobj.arn
+                      id += target["path"] if target["path"]
+                      doc["Statement"].first["Resource"] << id
                     else
-                      raise MuError, "Couldn't find a #{target["entity_type"]} named #{target["identifier"]} when generating IAM policy in role #{@mu_name}"
+                      raise MuError, "Couldn't find a #{target["entity_type"]} named #{target["identifier"]} when generating IAM policy"
                     end
                   else
+                    target["identifier"] += target["path"] if target["path"]
                     doc["Statement"].first["Resource"] << target["identifier"]
                   end
                 }
@@ -657,6 +735,14 @@ module MU
           iam_policies
         end
 
+        private
+
+        # Convert entries from the cloud-neutral @config['policies'] list into
+        # AWS syntax.
+        def convert_policies_to_iam
+          MU::Cloud::AWS::Role.genPolicyDocument(@config['policies'], deploy_obj: @deploy)
+        end
+
         def get_tag_params(strip_std = false)
           @config['tags'] ||= []
 

data/modules/mu/clouds/aws/search_domain.rb

@@ -108,6 +108,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all search_domains associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/server.rb

@@ -1794,6 +1794,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all instances associated with the currently loaded deployment. Also cleans up associated volumes, droppings in the MU master's /etc/hosts and ~/.ssh, and in whatever Groomer was used.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/server_pool.rb

@@ -989,6 +989,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all autoscale groups associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/storage_pool.rb

@@ -349,6 +349,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Called by {MU::Cleanup}. Locates resources that were created by the
         # currently-loaded deployment, and purges them.
         # @param noop [Boolean]: If true, will only print what would be done

data/modules/mu/clouds/aws/user.rb

@@ -136,6 +136,12 @@ module MU
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all users associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/vpc.rb

@@ -1129,6 +1129,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all VPC resources associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
@@ -1440,10 +1446,28 @@ module MU
           ok
         end
 
+        # Remove all network interfaces associated with the currently loaded deployment.
+        # @param noop [Boolean]: If true, will only print what would be done
+        # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
+        # @param region [String]: The cloud provider region
+        # @return [void]
+        def self.purge_interfaces(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
+          resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
+              filters: tagfilters
+          )
+          ifaces = resp.data.network_interfaces
+
+          return if ifaces.nil? or ifaces.size == 0
+
+          ifaces.each { |iface|
+            MU.log "Deleting Network Interface #{iface.network_interface_id}"
+            MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_network_interface(network_interface_id: iface.network_interface_id)
+          }
+        end
+
 
         private
 
-
         # List the route tables for each subnet in the given VPC
         def self.listAllSubnetRouteTables(vpc_id, region: MU.curRegion, credentials: nil)
           resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_subnets(

data/modules/mu/clouds/google.rb

@@ -28,6 +28,7 @@ module MU
       @@my_hosted_cfg = nil
       @@authorizers = {}
       @@acct_to_profile_map = {}
+      @@enable_semaphores = {}
 
       # Any cloud-specific instance methods we require our resource
       # implementations to have, above and beyond the ones specified by
@@ -71,6 +72,36 @@ module MU
         $MU_CFG['google'].keys
       end
 
+      # A shortcut for {MU::MommaCat.findStray} to resolve a shorthand project
+      # name into a cloud object, whether it refers to a sibling by internal
+      # name or by cloud identifier.
+      # @param name [String]
+      # @param deploy [String]
+      # @param raise_on_fail [Boolean]
+      # @param sibling_only [Boolean]
+      # @return [MU::Cloud::Habitat,nil]
+      def self.projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false)
+        project_obj = deploy.findLitterMate(type: "habitats", name: name)
+
+        if !project_obj and !sibling_only
+          resp = MU::MommaCat.findStray(
+            "Google",
+            "habitats",
+            deploy_id: deploy.deploy_id,
+            cloud_id: name,
+            name: name,
+            dummy_ok: true
+          )
+          project_obj = resp.first if resp
+        end
+
+        if (!project_obj or !project_obj.cloud_id) and raise_on_fail
+          raise MuError, "Failed to find project '#{name}' in deploy #{deploy.deploy_id}"
+        end
+
+        project_obj
+      end
+
       # Resolve the administrative Cloud Storage bucket for a given credential
       # set, or return a default.
       # @param credentials [String]
@@ -211,7 +242,7 @@ module MU
 
           [name, "log_vol_ebs_key"].each { |obj|
             MU.log "Granting #{acct} access to #{obj} in Cloud Storage bucket #{adminBucketName(credentials)}"
-            pp aclobj
+
             MU::Cloud::Google.storage(credentials: credentials).insert_object_access_control(
               adminBucketName(credentials),
               obj,
@@ -323,6 +354,10 @@ module MU
 
         cfg = credConfig(credentials)
 
+        if cfg['project']
+          @@enable_semaphores[cfg['project']] ||= Mutex.new
+        end
+
         if cfg
           data = nil
           @@authorizers[credentials] ||= {}
@@ -597,7 +632,8 @@ module MU
         require 'google/apis/cloudresourcemanager_v1'
 
         if subclass.nil?
-          @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
+#          @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/cloudplatformprojects'], masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
+          @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/cloudplatformprojects'], credentials: credentials)
           return @@resource_api[credentials]
         elsif subclass.is_a?(Symbol)
           return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV1").const_get(subclass)
@@ -605,15 +641,15 @@ module MU
       end
 
       # Google's Cloud Resource Manager API V2, which apparently has all the folder bits
-      # @param subclass [<Google::Apis::CloudresourcemanagerV2beta1>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV2beta1::subclass instead of an API client instance
+      # @param subclass [<Google::Apis::CloudresourcemanagerV2>]: If specified, will return the class ::Google::Apis::CloudresourcemanagerV2::subclass instead of an API client instance
       def self.folder(subclass = nil, credentials: nil)
-        require 'google/apis/cloudresourcemanager_v2beta1'
+        require 'google/apis/cloudresourcemanager_v2'
 
         if subclass.nil?
-          @@resource2_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV2beta1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
+          @@resource2_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV2::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/cloudplatformfolders'], credentials: credentials)
           return @@resource2_api[credentials]
         elsif subclass.is_a?(Symbol)
-          return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV2beta1").const_get(subclass)
+          return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV2").const_get(subclass)
         end
       end
 
@@ -682,6 +718,31 @@ module MU
         end
       end
 
+      # Google's Cloud Billing Service API
+      # @param subclass [<Google::Apis::LoggingV2>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
+      def self.billing(subclass = nil, credentials: nil)
+        require 'google/apis/cloudbilling_v1'
+
+        if subclass.nil?
+          @@billing_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudbillingV1::CloudbillingService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
+          return @@billing_api[credentials]
+        elsif subclass.is_a?(Symbol)
+          return Object.const_get("::Google").const_get("Apis").const_get("CloudbillingV1").const_get(subclass)
+        end
+      end
+
+
+      # Retrieve the organization, if any, to which these credentials belong.
+      # @param credentials [String]
+      # @return [Array<OpenStruct>],nil]
+      def self.getOrg(credentials = nil)
+        resp = MU::Cloud::Google.resource_manager(credentials: credentials).search_organizations
+        if resp and resp.organizations
+          # XXX no idea if it's possible to be a member of multiple orgs
+          return resp.organizations.first
+        end
+        nil
+      end
 
       private
 
@@ -762,7 +823,7 @@ module MU
                     end
 # TODO validate that the resource actually went away, because it seems not to do so very reliably
                   rescue ::Google::Apis::ClientError => e
-                    raise e if !e.message.match(/^notFound: /)
+                    raise e if !e.message.match(/(^notFound: |operation in progress)/)
                   end while failed and retries < 6
                 end
               }
@@ -798,30 +859,37 @@ module MU
               else
                 raise MU::MuError, "Service account #{MU::Cloud::Google.svc_account_name} has insufficient privileges to call #{method_sym}"
               end
-            rescue ::Google::Apis::ClientError => e
+            rescue ::Google::Apis::ClientError, OpenSSL::SSL::SSLError => e
               if e.message.match(/^invalidParameter:/)
                 MU.log "#{method_sym.to_s}: "+e.message, MU::ERR, details: arguments
 # uncomment for debugging stuff; this can occur in benign situations so we don't normally want it logging
               elsif e.message.match(/^forbidden:/)
                 MU.log "Using credentials #{@credentials}: #{method_sym.to_s}: "+e.message, MU::ERR, details: caller
               end
-              if retries <= 1 and e.message.match(/^accessNotConfigured/)
+              @@enable_semaphores ||= {}
+              max_retries = 3
+              wait_time = 90
+              if retries <= max_retries and e.message.match(/^accessNotConfigured/)
                 enable_obj = nil
                 project = arguments.size > 0 ? arguments.first.to_s : MU::Cloud::Google.defaultProject(@credentials)
+                @@enable_semaphores[project] ||= Mutex.new
                 enable_obj = MU::Cloud::Google.service_manager(:EnableServiceRequest).new(
                   consumer_id: "project:"+project
                 )
                 # XXX dumbass way to get this string
-                e.message.match(/Enable it by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//)
+                e.message.match(/by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//)
+
                 svc_name = Regexp.last_match[1]
                 save_verbosity = MU.verbosity
                 if svc_name != "servicemanagement.googleapis.com"
-                  MU.setLogging(MU::Logger::NORMAL)
-                  MU.log "Attempting to enable #{svc_name} in project #{project}, then waiting for 30s", MU::WARN
-                  MU.setLogging(save_verbosity)
-                  MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj)
-                  sleep 30
                   retries += 1
+                  @@enable_semaphores[project].synchronize {
+                    MU.setLogging(MU::Logger::NORMAL)
+                    MU.log "Attempting to enable #{svc_name} in project #{project}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
+                    MU.setLogging(save_verbosity)
+                    MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj)
+                  }
+                  sleep wait_time/retries
                   retry
                 else
                   MU.setLogging(MU::Logger::NORMAL)
@@ -831,7 +899,8 @@ module MU
                 end
               elsif retries <= 10 and
                  e.message.match(/^resourceNotReady:/) or
-                 (e.message.match(/^resourceInUseByAnotherResource:/) and method_sym.to_s.match(/^delete_/))
+                 (e.message.match(/^resourceInUseByAnotherResource:/) and method_sym.to_s.match(/^delete_/)) or
+                 e.message.match(/SSL_connect/)
                 if retries > 0 and retries % 3 == 0
                   MU.log "Will retry #{method_sym} after #{e.message} (retry #{retries})", MU::NOTICE, details: arguments
                 else
@@ -968,6 +1037,7 @@ module MU
       @@service_api = {}
       @@firestore_api = {}
       @@admin_directory_api = {}
+      @@billing_api = {}
     end
   end
 end