cloud-mu 3.0.0beta → 3.0.0

data/modules/mu/clouds/aws/collection.rb

@@ -297,13 +297,13 @@ module MU
         end
 
         # placeholder
-        def self.find(cloud_id: nil, region: MU.myRegion, credentials: nil)
+        def self.find(**args)
           found = nil
-          resp = MU::Cloud::AWS.cloudformation(region: region, credentials: credentials).describe_stacks(
-            stack_name: cloud_id
+          resp = MU::Cloud::AWS.cloudformation(region: args[:region], credentials: args[:credentials]).describe_stacks(
+            stack_name: args[:cloud_id]
           )
           if resp and resp.stacks
-            found[cloud_id] = resp.stacks.first
+            found[args[:cloud_id]] = resp.stacks.first
           end
 
           found

data/modules/mu/clouds/aws/container_cluster.rb

@@ -213,7 +213,7 @@ module MU
               )
             end
 
-            MU.log %Q{How to interact with your Kubernetes cluster\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
+            MU.log %Q{How to interact with your EKS cluster\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
           elsif @config['flavor'] != "Fargate"
             resp = MU::Cloud::AWS.ecs(region: @config['region'], credentials: @config['credentials']).list_container_instances({
               cluster: @mu_name
@@ -495,12 +495,28 @@ module MU
               if @config['vpc']
                 subnet_ids = []
                 all_public = true
-                subnet_names = @config['vpc']['subnets'].map { |s| s.values.first }
-                @vpc.subnets.each { |subnet_obj|
-                  next if !subnet_names.include?(subnet_obj.config['name'])
+
+                subnets =
+                  if @config["vpc"]["subnets"].empty?
+                    @vpc.subnets
+                  else
+                    subnet_objects= []
+                    @config["vpc"]["subnets"].each { |subnet|
+                      sobj = @vpc.getSubnet(cloud_id: subnet["subnet_id"], name: subnet["subnet_name"])
+                      if sobj.nil?
+                        MU.log "Got nil result from @vpc.getSubnet(cloud_id: #{subnet["subnet_id"]}, name: #{subnet["subnet_name"]})", MU::WARN
+                      else
+                        subnet_objects << sobj
+                      end
+                    }
+                    subnet_objects
+                  end
+
+                subnets.each { |subnet_obj|
                   subnet_ids << subnet_obj.cloud_id
                   all_public = false if subnet_obj.private?
                 }
+
                 service_params[:network_configuration] = {
                   :awsvpc_configuration => {
                     :subnets => subnet_ids,
@@ -905,15 +921,10 @@ MU.log c.name, MU::NOTICE, details: t
         end
 
         # Locate an existing container_clusters.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching container_clusters.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
-          MU.log cloud_id, MU::WARN, details: flags
-          MU.log region, MU::WARN
-          resp = MU::Cloud::AWS.ecs(region: region, credentials: credentials).list_clusters
-          resp = MU::Cloud::AWS.eks(region: region, credentials: credentials).list_clusters
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching container_clusters.
+        def self.find(**args)
+          resp = MU::Cloud::AWS.ecs(region: args[:region], credentials: args[:credentials]).list_clusters
+          resp = MU::Cloud::AWS.eks(region: args[:region], credentials: args[:credentials]).list_clusters
 # XXX uh, this ain't complete
         end
 
@@ -1528,6 +1539,32 @@ MU.log c.name, MU::NOTICE, details: t
             ok = false
           end
 
+          if ["Fargate", "EKS"].include?(cluster["flavor"]) and !cluster["vpc"]
+            siblings = configurator.haveLitterMate?(nil, "vpcs", has_multiple: true)
+            if siblings.size == 1
+              MU.log "ContainerCluster #{cluster['name']} did not declare a VPC. Inserting into sibling VPC #{siblings[0]['name']}.", MU::WARN
+              cluster["vpc"] = {
+                "name" => siblings[0]['name'],
+                "subnet_pref" => "all_private"
+              }
+            elsif MU::Cloud::AWS.hosted? and MU::Cloud::AWS.myVPCObj
+              cluster["vpc"] = {
+                "id" => MU.myVPC,
+                "subnet_pref" => "all_private"
+              }
+            else
+              MU.log "ContainerCluster #{cluster['name']} must declare a VPC", MU::ERR
+              ok = false
+            end
+
+            # Re-insert ourselves with this modification so that our child
+            # resources get this VPC we just shoved in
+            if ok and cluster['vpc']
+              cluster.delete("#MU_VALIDATED")
+              return configurator.insertKitten(cluster, "container_clusters", overwrite: true)
+            end
+          end
+
           if cluster["volumes"]
             cluster["volumes"].each { |v|
               if v["type"] == "docker"
@@ -1627,26 +1664,6 @@ MU.log c.name, MU::NOTICE, details: t
             ok = false
           end
 
-          if cluster["flavor"] == "EKS" and !cluster["vpc"]
-            if !MU::Cloud::AWS.hosted? or !MU::Cloud::AWS.myVPCObj
-              siblings = configurator.haveLitterMate?(nil, "vpcs", has_multiple: true)
-              if siblings.size == 1
-                MU.log "EKS cluster #{cluster['name']} did not declare a VPC. Inserting into an available sibling VPC.", MU::WARN
-                cluster["vpc"] = {
-                  "name" => siblings[0]['name'],
-                  "subnet_pref" => "all_private"
-                }
-              else
-                MU.log "EKS cluster #{cluster['name']} must declare a VPC", MU::ERR
-                ok = false
-              end
-            else
-              cluster["vpc"] = {
-                "id" => MU.myVPC,
-                "subnet_pref" => "all_private"
-              }
-            end
-          end
 
           if ["ECS", "EKS"].include?(cluster["flavor"])
             std_ami = getStandardImage(cluster["flavor"], cluster['region'], version: cluster['kubernetes']['version'], gpu: cluster['gpu'])

data/modules/mu/clouds/aws/database.rb

@@ -164,27 +164,22 @@ module MU
 
 
         # Locate an existing Database or Databases and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching Databases
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching Databases
+        def self.find(**args)
           map = {}
-          if cloud_id
-            resp = MU::Cloud::AWS::Database.getDatabaseById(cloud_id, region: region, credentials: credentials)
-            map[cloud_id] = resp if resp
+          if args[:cloud_id]
+            resp = MU::Cloud::AWS::Database.getDatabaseById(args[:cloud_id], region: args[:region], credentials: args[:credentials])
+            map[args[:cloud_id]] = resp if resp
           end
 
-          if tag_value
-            MU::Cloud::AWS.rds(credentials: credentials, region: region).describe_db_instances.db_instances.each { |db|
-              resp = MU::Cloud::AWS.rds(credentials: credentials, region: region).list_tags_for_resource(
-                  resource_name: MU::Cloud::AWS::Database.getARN(db.db_instance_identifier, "db", "rds", region: region, credentials: credentials)
+          if args[:tag_value]
+            MU::Cloud::AWS.rds(credentials: args[:credentials], region: args[:region]).describe_db_instances.db_instances.each { |db|
+              resp = MU::Cloud::AWS.rds(credentials: args[:credentials], region: args[:region]).list_tags_for_resource(
+                  resource_name: MU::Cloud::AWS::Database.getARN(db.db_instance_identifier, "db", "rds", region: args[:region], credentials: args[:credentials])
               )
               if resp && resp.tag_list && !resp.tag_list.empty?
                 resp.tag_list.each { |tag|
-                  map[db.db_instance_identifier] = db if tag.key == tag_key and tag.value == tag_value
+                  map[db.db_instance_identifier] = db if tag.key == args[:tag_key] and tag.value == args[:tag_value]
                 }
               end
             }
@@ -370,7 +365,7 @@ module MU
               # Does create_db_instance implement wait_until_available ?
               waiter.max_attempts = nil
               waiter.before_attempt do |w_attempts|
-                MU.log "Waiting for RDS database #{@config['identifier']} to be ready..", MU::NOTICE if w_attempts % 10 == 0
+                MU.log "Waiting for RDS database #{@config['identifier']} to be ready...", MU::NOTICE if w_attempts % 10 == 0
               end
               waiter.before_wait do |w_attempts, r|
                 throw :success if r.db_instances.first.db_instance_status == "available"
@@ -457,6 +452,14 @@ module MU
           # Maybe wait for DB instance to be in available state. DB should still be writeable at this state
           if @config['allow_major_version_upgrade'] && @config["creation_style"] == "new"
             MU.log "Setting major database version upgrade on #{@config['identifier']}'"
+            database = MU::Cloud::AWS::Database.getDatabaseById(@config['identifier'], region: @config['region'], credentials: @config['credentials'])
+            begin
+              if database.db_instance_status != "available"
+                sleep 5
+                database = MU::Cloud::AWS::Database.getDatabaseById(@config['identifier'], region: @config['region'], credentials: @config['credentials'])
+              end
+            end while database.db_instance_status != "available"
+
             MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).modify_db_instance(
               db_instance_identifier: @config['identifier'],
               apply_immediately: true,
@@ -611,11 +614,8 @@ module MU
 
             subnets.each{ |subnet|
               next if subnet.nil?
-              if @config["publicly_accessible"]
-                subnet_ids << subnet.cloud_id if !subnet.private?
-              elsif !@config["publicly_accessible"]
-                subnet_ids << subnet.cloud_id if subnet.private?
-              end
+              next if @config["publicly_accessible"] and subnet.private?
+              subnet_ids << subnet.cloud_id
             }
           else
             # If we didn't specify a VPC try to figure out if the account has a default VPC
@@ -1566,6 +1566,10 @@ module MU
             end
           end
 
+          if db['engine'] == "aurora-postgresql"
+            db.delete('cloudwatch_logs')
+          end
+
           if db['engine'].match(/^aurora/) and !db['create_cluster'] and !db['add_cluster_node']
             MU.log "Database #{db['name']}: #{db['engine']} looks like a cluster engine, but create_cluster is not set. Add 'create_cluster: true' if you're building an RDS cluster.", MU::ERR
             ok = false

data/modules/mu/clouds/aws/dnszone.rb

@@ -542,7 +542,8 @@ module MU
 
           if !mu_zone.nil? and !MU.myVPC.nil?
             subdomain = cloudclass.cfg_name
-            dns_name = name.downcase+"."+subdomain+"."+MU.myInstanceId
+            dns_name = name.downcase+"."+subdomain
+            dns_name += "."+MU.myInstanceId if MU.myInstanceId
             record_type = "CNAME"
             record_type = "A" if target.match(/^\d+\.\d+\.\d+\.\d+/)
             ip = nil
@@ -869,36 +870,33 @@ module MU
         end
 
         # Locate an existing DNSZone or DNSZones and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource. Can also use the domain name, we'll check for both.
-        # @param region [String]: The cloud provider region
-        # @param flags [Hash]: Optional flags
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching DNSZones
-        def self.find(cloud_id: nil, deploy_id: MU.deploy_id, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching DNSZones
+        def self.find(**args)
           matches = {}
 
-          resp = MU::Cloud::AWS.route53(credentials: credentials).list_hosted_zones(
+          resp = MU::Cloud::AWS.route53(credentials: args[:credentials]).list_hosted_zones(
               max_items: 100
           )
 
           resp.hosted_zones.each { |zone|
-            if !cloud_id.nil? and !cloud_id.empty?
-              if zone.id == cloud_id
+            if !args[:cloud_id].nil? and !args[:cloud_id].empty?
+              if zone.id == args[:cloud_id]
                 begin 
-                  matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
+                  matches[zone.id] = MU::Cloud::AWS.route53(credentials: args[:credentials]).get_hosted_zone(id: zone.id).hosted_zone
                 rescue Aws::Route53::Errors::NoSuchHostedZone
                   MU.log "Hosted zone #{zone.id} doesn't exist"
                 end
-              elsif zone.name == cloud_id or zone.name == cloud_id+"."
+              elsif zone.name == args[:cloud_id] or zone.name == args[:cloud_id]+"."
                 begin 
-                  matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
+                  matches[zone.id] = MU::Cloud::AWS.route53(credentials: args[:credentials]).get_hosted_zone(id: zone.id).hosted_zone
                 rescue Aws::Route53::Errors::NoSuchHostedZone
                   MU.log "Hosted zone #{zone.id} doesn't exist"
                 end
               end
             end
-            if !deploy_id.nil? and !deploy_id.empty? and zone.config.comment == deploy_id
+            if !args[:deploy_id].nil? and !args[:deploy_id].empty? and zone.config.comment == args[:deploy_id]
               begin 
-                matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
+                matches[zone.id] = MU::Cloud::AWS.route53(credentials: args[:credentials]).get_hosted_zone(id: zone.id).hosted_zone
               rescue Aws::Route53::Errors::NoSuchHostedZone
                 MU.log "Hosted zone #{zone.id} doesn't exist"
               end

data/modules/mu/clouds/aws/endpoint.rb

@@ -256,14 +256,11 @@ MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials
         end
 
         # Locate an existing API.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching API.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
-          if cloud_id
-            return MU::Cloud::AWS.apig(region: region, credentials: credentials).get_rest_api(
-              rest_api_id: cloud_id
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching API.
+        def self.find(**args)
+          if args[:cloud_id]
+            return MU::Cloud::AWS.apig(region: args[:region], credentials: args[:credentials]).get_rest_api(
+              rest_api_id: args[:cloud_id]
             )
           end
 #          resp = MU::Cloud::AWS.apig(region: region, credentials: credentials).get_rest_apis

data/modules/mu/clouds/aws/firewall_rule.rb

@@ -562,10 +562,15 @@ module MU
                     rescue Aws::EC2::Errors::InvalidPermissionNotFound => e
                     end
                   end
-                  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
-                    group_id: @cloud_id,
-                    ip_permissions: [rule]
-                  )
+                  begin
+                    MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
+                      group_id: @cloud_id,
+                      ip_permissions: [rule]
+                    )
+                  rescue Aws::EC2::Errors::InvalidParameterCombination => e
+                    MU.log "FirewallRule #{@mu_name} had a bogus rule: #{e.message}", MU::ERR, details: rule
+                    raise e
+                  end
                 end
 
                 if egress

data/modules/mu/clouds/aws/folder.rb

@@ -63,16 +63,13 @@ module MU
         end
 
         # Locate an existing AWS organization. If no identifying parameters are specified, this will return a description of the Organization which owns the account for our credentials.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching log group.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching log group.
+        def self.find(**args)
           found = nil
 
-          if cloud_id
+          if args[:cloud_id]
           else
-            resp = MU::Cloud::AWS.orgs(credentials: credentials).describe_organization
+            resp = MU::Cloud::AWS.orgs(credentials: args[:credentials]).describe_organization
             found ||= {}
             found[resp.organization.id] = resp.organization
           end

data/modules/mu/clouds/aws/function.rb

@@ -292,17 +292,14 @@ module MU
         end
 
         # Locate an existing function.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching function.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching function.
+        def self.find(**args)
           matches = {}
 
-          if !cloud_id.nil?
-            all_functions = MU::Cloud::AWS.lambda(region: region, credentials: credentials).list_functions
+          if !args[:cloud_id].nil?
+            all_functions = MU::Cloud::AWS.lambda(region: args[:region], credentials: args[:credentials]).list_functions
             all_functions.functions.each do |x|
-              if x.function_name == cloud_id
+              if x.function_name == args[:cloud_id]
                 matches[x.function_name] = x
                 break
               end

data/modules/mu/clouds/aws/group.rb

@@ -159,18 +159,15 @@ module MU
         end
 
         # Locate an existing group group.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching group group.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching group group.
+        def self.find(**args)
           found = nil
           begin
-            resp = MU::Cloud::AWS.iam(credentials: credentials).get_group(
-              group_name: cloud_id
+            resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_group(
+              group_name: args[:cloud_id]
             )
             found ||= {}
-            found[cloud_id] = resp
+            found[args[:cloud_id]] = resp
           rescue Aws::IAM::Errors::NoSuchEntity
           end
           found

data/modules/mu/clouds/aws/habitat.rb

@@ -107,11 +107,8 @@ module MU
         end
 
         # Locate an existing account
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching account
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching account
+        def self.find(**args)
           {}
         end
 

data/modules/mu/clouds/aws/loadbalancer.rb

@@ -852,43 +852,39 @@ module MU
         end
 
         # Locate an existing LoadBalancer or LoadBalancers and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching LoadBalancers
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
-          classic = flags['classic'] ? true : false
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching LoadBalancers
+        def self.find(**args)
+          args[:flags] ||= {}
+          classic = args[:flags]['classic'] ? true : false
 
           matches = {}
           list = {}
           arn2name = {}
           resp = nil
           if classic
-            resp = MU::Cloud::AWS.elb(region: region, credentials: credentials).describe_load_balancers().load_balancer_descriptions
+            resp = MU::Cloud::AWS.elb(region: args[:region], credentials: args[:credentials]).describe_load_balancers().load_balancer_descriptions
           else
-            resp = MU::Cloud::AWS.elb2(region: region, credentials: credentials).describe_load_balancers().load_balancers
+            resp = MU::Cloud::AWS.elb2(region: args[:region], credentials: args[:credentials]).describe_load_balancers().load_balancers
           end
 
           resp.each { |lb|
             list[lb.load_balancer_name] = lb
             arn2name[lb.load_balancer_arn] = lb.load_balancer_name if !classic
-            if !cloud_id.nil? and lb.load_balancer_name == cloud_id
-              matches[cloud_id] = lb
+            if !args[:cloud_id].nil? and lb.load_balancer_name == args[:cloud_id]
+              matches[args[:cloud_id]] = lb
             end
           }
 
           return matches if matches.size > 0
 
-          if !tag_key.nil? and !tag_value.nil? and !tag_key.empty? and list.size > 0
+          if !args[:tag_key].nil? and !args[:tag_value].nil? and !args[:tag_key].empty? and list.size > 0
             tag_descriptions = nil
             if classic
-              tag_descriptions = MU::Cloud::AWS.elb(region: region, credentials: credentials).describe_tags(
+              tag_descriptions = MU::Cloud::AWS.elb(region: args[:region], credentials: args[:credentials]).describe_tags(
                 load_balancer_names: list.keys
               ).tag_descriptions
             else
-              tag_descriptions = MU::Cloud::AWS.elb2(region: region, credentials: credentials).describe_tags(
+              tag_descriptions = MU::Cloud::AWS.elb2(region: args[:region], credentials: args[:credentials]).describe_tags(
                 resource_arns: list.values.map { |l| l.load_balancer_arn }
               ).tag_descriptions
             end
@@ -896,7 +892,7 @@ module MU
               tag_descriptions.each { |lb|
                 lb_name = classic ? lb.load_balancer_name : arn2name[lb.resource_arn]
                 lb.tags.each { |tag|
-                  if tag.key == tag_key and tag.value == tag_value
+                  if tag.key == args[:tag_key] and tag.value == args[:tag_value]
                     matches[lb_name] = list[lb_name]
                   end
                 }