cloud-mu 3.0.0beta → 3.0.0

data/modules/mu/clouds/aws/log.rb

@@ -263,18 +263,15 @@ module MU
         end
 
         # Locate an existing log group.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching log group.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching log group.
+        def self.find(**args)
           found = nil
-          if !cloud_id.nil? and !cloud_id.match(/^arn:/i)
+          if !args[:cloud_id].nil? and !args[:cloud_id].match(/^arn:/i)
             found ||= {}
-            found[cloud_id] = MU::Cloud::AWS::Log.getLogGroupByName(cloud_id, region: region, credentials: nil)
+            found[args[:cloud_id]] = MU::Cloud::AWS::Log.getLogGroupByName(args[:cloud_id], region: args[:region], credentials: args[:credentials])
           else
-            resp = MU::Cloud::AWS.cloudwatchlogs(region: region, credentials: credentials).describe_log_groups.log_groups.each { |group|
-              if group.arn == cloud_id or group.arn.sub(/:\*$/, "") == cloud_id
+            resp = MU::Cloud::AWS.cloudwatchlogs(region: args[:region], credentials: args[:credentials]).describe_log_groups.log_groups.each { |group|
+              if group.arn == args[:cloud_id] or group.arn.sub(/:\*$/, "") == args[:cloud_id]
                 found ||= {}
                 found[group.log_group_name] = group
                 break

data/modules/mu/clouds/aws/msg_queue.rb

@@ -153,47 +153,44 @@ module MU
         end
 
         # Locate an existing msg_queue.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
         # @return [Hash]: AWS doesn't return anything but the SQS URL, so supplement with attributes
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
-          flags['account'] ||= MU.account_number
-          return nil if !cloud_id
+        def self.find(**args)
+          args[:flags] ||= {}
+          args[:flags]['account'] ||= MU.account_number
+          return nil if !args[:cloud_id]
 
           # If it's a URL, make sure it's good
           begin
-            if cloud_id.match(/^https?:/i)
-              resp = MU::Cloud::AWS.sqs(region: region, credentials: credentials).get_queue_attributes(
-                queue_url: cloud_id,
+            if args[:cloud_id].match(/^https?:/i)
+              resp = MU::Cloud::AWS.sqs(region: args[:region], credentials: args[:credentials]).get_queue_attributes(
+                queue_url: args[:cloud_id],
                 attribute_names: ["All"]
               )
               if resp and resp.attributes
                 desc = resp.attributes.dup
-                desc["Url"] = cloud_id
+                desc["Url"] = args[:cloud_id]
                 return desc
               end
             else
               # If it's a plain queue name, resolve it to a URL
-              resp = MU::Cloud::AWS.sqs(region: region, credentials: credentials).get_queue_url(
-                queue_name: cloud_id,
-                queue_owner_aws_account_id: flags['account']
+              resp = MU::Cloud::AWS.sqs(region: args[:region], credentials: args[:credentials]).get_queue_url(
+                queue_name: args[:cloud_id],
+                queue_owner_aws_account_id: args[:flags]['account']
               )
-              cloud_id = resp.queue_url if resp and resp.queue_url
+              args[:cloud_id] = resp.queue_url if resp and resp.queue_url
             end
           rescue ::Aws::SQS::Errors::NonExistentQueue => e
           end
 
           # Go fetch its attributes
-          if cloud_id
-            resp = MU::Cloud::AWS.sqs(region: region, credentials: credentials).get_queue_attributes(
-              queue_url: cloud_id,
+          if args[:cloud_id]
+            resp = MU::Cloud::AWS.sqs(region: args[:region], credentials: args[:credentials]).get_queue_attributes(
+              queue_url: args[:cloud_id],
               attribute_names: ["All"]
             )
             if resp and resp.attributes
               desc = resp.attributes.dup
-              desc["Url"] = cloud_id
-MU.log "RETURNING FROM FIND ON #{cloud_id}", MU::WARN, details: caller
+              desc["Url"] = args[:cloud_id]
               return desc
             end
           end

data/modules/mu/clouds/aws/nosqldb.rb

@@ -98,7 +98,7 @@ module MU
               end
             }
           end
-pp params
+
           MU.log "Creating DynamoDB table #{@mu_name}", details: params
 
           resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).create_table(params)
@@ -169,16 +169,25 @@ pp params
             resp.table_names.each { |table|
               desc = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).describe_table(table_name: table).table
               next if desc.table_status == "DELETING"
-              tags = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tags_of_resource(resource_arn: desc.table_arn)
-              if tags and tags.tags
-                tags.tags.each { |tag|
-                  if tag.key == "MU-ID" and tag.value == MU.deploy_id
-                    MU.log "Deleting DynamoDB table #{desc.table_name}"
-                    if !noop
-                      MU::Cloud::AWS.dynamo(credentials: credentials, region: region).delete_table(table_name: desc.table_name)
+              if desc.table_status == "CREATING"
+                begin
+                  desc = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).describe_table(table_name: table).table
+                  sleep 1
+                end while desc.table_status == "CREATING"
+              end
+              begin
+                tags = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tags_of_resource(resource_arn: desc.table_arn)
+                if tags and tags.tags
+                  tags.tags.each { |tag|
+                    if tag.key == "MU-ID" and tag.value == MU.deploy_id
+                      MU.log "Deleting DynamoDB table #{desc.table_name}"
+                      if !noop
+                        MU::Cloud::AWS.dynamo(credentials: credentials, region: region).delete_table(table_name: desc.table_name)
+                      end
                     end
-                  end
-                }
+                  }
+                end
+              rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
               end
 
             }
@@ -200,15 +209,15 @@ pp params
         end
 
         # Locate an existing DynamoDB table
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching bucket.
+        def self.find(**args)
           found = {}
-          if cloud_id
-            resp = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).describe_table(table_name: cloud_id)
-            found[cloud_id] = resp.table if resp and resp.table
+          if args[:cloud_id]
+            begin
+              resp = MU::Cloud::AWS.dynamo(credentials: args[:credentials], region: args[:region]).describe_table(table_name: args[:cloud_id])
+            rescue ::Aws::DynamoDB::Errors::ResourceNotFoundException
+            end
+            found[args[:cloud_id]] = resp.table if resp and resp.table
           end
           found
         end

data/modules/mu/clouds/aws/notifier.rb

@@ -91,16 +91,13 @@ module MU
         end
 
         # Locate an existing notifier.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching notifier.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching notifier.
+        def self.find(**args)
           found = {}
-          if cloud_id
-            arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(region) ? "aws-us-gov" : "aws")+":sns:"+region+":"+MU::Cloud::AWS.credToAcct(credentials)+":"+cloud_id
-            desc = MU::Cloud::AWS.sns(region: region, credentials: credentials).get_topic_attributes(topic_arn: arn).attributes
-            found[cloud_id] = desc if desc
+          if args[:cloud_id]
+            arn = "arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id]
+            desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes
+            found[args[:cloud_id]] = desc if desc
           end
           found
         end

data/modules/mu/clouds/aws/role.rb

@@ -123,12 +123,12 @@ module MU
                     version_id: desc.policy.default_version_id
                   )
 
-                  if version.policy_version.document != URI.encode_www_form(JSON.generate(policy.values.first), /[^a-z0-9\-]/i)
+                  if version.policy_version.document != URI.encode(JSON.generate(policy.values.first))#, /[^a-z0-9\-]/i)
                     # Special exception- we don't want to overwrite extra rules
                     # in MuSecrets policies, because our siblings might have 
                     # (will have) injected those and they should stay.
                     if policy.size == 1 and policy["MuSecrets"]
-                      ext = JSON.parse(URI.decode_www_form(version.policy_version.document))
+                      ext = JSON.parse(URI.decode(version.policy_version.document))
                       if (ext["Statement"][0]["Resource"] & policy["MuSecrets"]["Statement"][0]["Resource"]).sort == policy["MuSecrets"]["Statement"][0]["Resource"].sort
                         next
                       end
@@ -409,11 +409,8 @@ module MU
         end
 
         # Locate an existing user group.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching user group.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching user group.
+        def self.find(**args)
           found = nil
 
           found

data/modules/mu/clouds/aws/search_domain.rb

@@ -32,7 +32,6 @@ module MU
           params = genParams
 
           MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params
-          pp params
           resp = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).create_elasticsearch_domain(params).domain_status
 
           tagDomain
@@ -86,6 +85,11 @@ module MU
           deploy_struct['tags'] = tags.map { |t| { t.key => t.value } }
           if deploy_struct['endpoint']
             deploy_struct['kibana'] = deploy_struct['endpoint']+"/_plugin/kibana/"
+          elsif deploy_struct['endpoints']
+            deploy_struct['kibana'] = {}
+            deploy_struct['endpoints'].each_pair { |k, v|
+              deploy_struct['kibana'][k] = v+"/_plugin/kibana/"
+            }
           end
           deploy_struct['domain_name'] ||= @config['domain_name'] if @config['domain_name']
           deploy_struct
@@ -112,20 +116,25 @@ module MU
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
           list = MU::Cloud::AWS.elasticsearch(region: region).list_domain_names
           if list and list.domain_names and list.domain_names.size > 0
-            descs = MU::Cloud::AWS.elasticsearch(region: region).describe_elasticsearch_domains(domain_names: list.domain_names.map { |d| d.domain_name } )
-
-            descs.domain_status_list.each { |domain|
-              tags = MU::Cloud::AWS.elasticsearch(region: region).list_tags(arn: domain.arn)
-              tags.tag_list.each { |tag|
-                if tag.key == "MU-ID" and tag.value == MU.deploy_id
-                  MU.log "Deleting ElasticSearch Domain #{domain.domain_name}"
-                  if !noop
-                    MU::Cloud::AWS.elasticsearch(region: region).delete_elasticsearch_domain(domain_name: domain.domain_name)
+            names = list.domain_names.map { |d| d.domain_name }
+            begin
+              # why is this API so obnoxious?
+              sample = names.slice!(0, (names.length >= 5 ? 5 : names.length))
+              descs = MU::Cloud::AWS.elasticsearch(region: region).describe_elasticsearch_domains(domain_names: sample)
+
+              descs.domain_status_list.each { |domain|
+                tags = MU::Cloud::AWS.elasticsearch(region: region).list_tags(arn: domain.arn)
+                tags.tag_list.each { |tag|
+                  if tag.key == "MU-ID" and tag.value == MU.deploy_id
+                    MU.log "Deleting ElasticSearch Domain #{domain.domain_name}"
+                    if !noop
+                      MU::Cloud::AWS.elasticsearch(region: region).delete_elasticsearch_domain(domain_name: domain.domain_name)
+                    end
+                    break
                   end
-                  break
-                end
+                }
               }
-            }
+            end while names.size > 0
           end
 
           unless noop
@@ -142,18 +151,15 @@ module MU
         end
 
         # Locate an existing search_domain.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching search_domain.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
-          if cloud_id
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching search_domain.
+        def self.find(**args)
+          if args[:cloud_id]
             # Annoyingly, we might expect one of several possible artifacts,
             # since AWS couldn't decide what the real identifier of these
             # things should be
-            list = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).list_domain_names
+            list = MU::Cloud::AWS.elasticsearch(region: args[:region], credentials: args[:credentials]).list_domain_names
             if list and list.domain_names and list.domain_names.size > 0
-              descs = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).describe_elasticsearch_domains(domain_names: list.domain_names.map { |d| d.domain_name } )
+              descs = MU::Cloud::AWS.elasticsearch(region: args[:region], credentials: args[:credentials]).describe_elasticsearch_domains(domain_names: list.domain_names.map { |d| d.domain_name } )
               descs.domain_status_list.each { |domain|
                 return domain if domain.arn == cloud_id
                 return domain if domain.domain_name == cloud_id
@@ -305,6 +311,10 @@ module MU
 
           if dom["dedicated_masters"] > 0 and dom["master_instance_type"].nil?
             dom["master_instance_type"] = dom["instance_type"]
+            if dom["dedicated_masters"] != 3 and dom["dedicated_masters"] != 5
+              MU.log "SearchDomain #{dom['name']}: You must choose either three or five dedicated master nodes", MU::ERR
+              ok = false
+            end
           end
 
           if dom["instance_count"] < 1
@@ -312,6 +322,11 @@ module MU
             ok = false
           end
 
+          if dom["ebs_iops"]
+            MU.log "SearchDomain #{dom['name']} declared ebs_iops, setting volume type to io1", MU::NOTICE
+            dom["ebs_type"] = "io1"
+          end
+
           if dom["zone_aware"] and (dom["instance_count"] % 2) != 0
             MU.log "Must set an even number for instance_count when enabling Zone Awareness in SearchDomain '#{dom['name']}'", MU::ERR
             ok = false
@@ -554,6 +569,10 @@ module MU
               }
             end
 
+            # XXX this will break on regroom, revisit and make deterministic
+            # or remembered
+            subnet_ids = subnet_ids.sample(3) if subnet_ids.size > 3
+
             if ext.nil? or
                ext.vpc_options.subnet_ids != subnet_ids or
                ext.vpc_options.security_group_ids != sgs
@@ -561,6 +580,11 @@ module MU
               params[:vpc_options][:subnet_ids] = subnet_ids
               params[:vpc_options][:security_group_ids] = sgs
             end
+            if @config['zone_aware'] and params[:elasticsearch_cluster_config]
+              params[:elasticsearch_cluster_config][:zone_awareness_config] = {
+                :availability_zone_count => subnet_ids.size
+              }
+            end
           end
 
           if @config['ebs_type']
@@ -650,13 +674,16 @@ module MU
 
           begin
             resp = cloud_desc
-            if (resp.endpoint.nil? or resp.endpoint.empty?) and !resp.deleted
+
+            if (resp.endpoint.nil? or resp.endpoint.empty?) and
+               (resp.endpoints.nil? or resp.endpoints.empty?) and
+               !resp.deleted
               loglevel = (retries > 0 and retries % 3 == 0) ? MU::NOTICE : MU::DEBUG
               MU.log "Waiting for Elasticsearch domain #{@mu_name} (#{@config['domain_name']}) to finish creating", loglevel
               sleep interval
             end
             retries += 1
-          end while (resp.endpoint.nil? or resp.endpoint.empty?) and !resp.deleted
+          end while (resp.endpoint.nil? or resp.endpoint.empty?) and (resp.endpoints.nil? or resp.endpoints.empty?) and !resp.deleted
         end
 
       end

data/modules/mu/clouds/aws/server.rb

@@ -391,8 +391,13 @@ module MU
 #				end
 
           retries = 0
-          begin
+          instance = begin
             response = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).run_instances(instance_descriptor)
+            if response and response.instances and response.instances.size > 0
+              instance = response.instances.first
+            else
+              MU.log "halp", MU::ERR, details: response
+            end
           rescue Aws::EC2::Errors::InvalidRequest => e
             MU.log e.message, MU::ERR, details: instance_descriptor
             raise e
@@ -409,11 +414,9 @@ module MU
             end
           end
 
-          instance = response.instances.first
           MU.log "#{node} (#{instance.instance_id}) coming online"
 
-          return instance
-
+          instance
         end
 
         # Ask the Amazon API to restart this node
@@ -967,13 +970,7 @@ module MU
         # postBoot
 
         # Locate an existing instance or instances and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching instances
-#        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching instances
         def self.find(**args)
           ip ||= args[:flags]['ip'] if args[:flags] and args[:flags]['ip']
 
@@ -1278,7 +1275,12 @@ module MU
           retries = 0
           if !@cloud_id.nil?
             begin
-              return MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(instance_ids: [@cloud_id]).reservations.first.instances.first
+              resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_instances(instance_ids: [@cloud_id])
+              if resp and resp.reservations and resp.reservations.first and
+                 resp.reservations.first.instances and
+                 resp.reservations.first.instances.first
+                return resp.reservations.first.instances.first
+              end
             rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
               return nil
             rescue NoMethodError => e
@@ -2126,7 +2128,8 @@ module MU
             "cloud" => "AWS",
             "bastion" => true,
             "size" => "t2.small",
-            "run_list" => [ "mu-utility::nat" ],
+            "run_list" => [ "mu-nat" ],
+            "groomer" => "Ansible",
             "platform" => "centos7",
             "ssh_user" => "centos",
             "associate_public_ip" => true,
@@ -2214,7 +2217,10 @@ module MU
             MU::Cloud.availableClouds.each { |cloud|
               next if cloud == "AWS"
               cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-              foreign_types = (cloudbase.listInstanceTypes)[cloudbase.myRegion]
+              foreign_types = (cloudbase.listInstanceTypes).values.first
+              if foreign_types.size == 1
+                foreign_types = foreign_types.values.first
+              end
               if foreign_types and foreign_types.size > 0 and foreign_types.has_key?(size)
                 vcpu = foreign_types[size]["vcpu"]
                 mem = foreign_types[size]["memory"]

data/modules/mu/clouds/aws/server_pool.rb

@@ -89,11 +89,20 @@ module MU
             desc.instances.each { |member|
               begin
                 groomthreads << Thread.new {
-                  Thread.abort_on_exception = false
                   MU.dupGlobals(parent_thread_id)
                   MU.log "Initializing #{member.instance_id} in ServerPool #{@mu_name}"
                   MU::MommaCat.lock(member.instance_id+"-mommagroom")
-                  kitten = MU::Cloud::Server.new(mommacat: @deploy, kitten_cfg: @config, cloud_id: member.instance_id)
+                  begin
+                    kitten = MU::Cloud::Server.new(mommacat: @deploy, kitten_cfg: @config, cloud_id: member.instance_id)
+                  rescue RuntimeError => e
+                    if e.message.match(/can't add a new key into hash during iteration/)
+                      MU.log e.message+", retrying", MU::WARN
+                      sleep 3
+                      retry
+                    else
+                      raise e
+                    end
+                  end
                   MU::MommaCat.lock("#{kitten.cloudclass.name}_#{kitten.config["name"]}-dependencies")
                   MU::MommaCat.unlock("#{kitten.cloudclass.name}_#{kitten.config["name"]}-dependencies")
                   if !kitten.postBoot(member.instance_id)
@@ -435,18 +444,13 @@ module MU
         end
 
         # Locate an existing ServerPool or ServerPools and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ServerPools
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching ServerPools
+        def self.find(**args)
           found = []
-          if cloud_id
-            resp = MU::Cloud::AWS.autoscale(region: region, credentials: credentials).describe_auto_scaling_groups({
+          if args[:cloud_id]
+            resp = MU::Cloud::AWS.autoscale(region: args[:region], credentials: args[:credentials]).describe_auto_scaling_groups({
               auto_scaling_group_names: [
-                cloud_id
+                args[:cloud_id]
               ], 
             })
             return resp.auto_scaling_groups
@@ -1361,9 +1365,15 @@ module MU
           if @config["vpc_zone_identifier"]
             asg_options[:vpc_zone_identifier] = @config["vpc_zone_identifier"]
           elsif @config["vpc"]
+            if !@vpc and @config['vpc'].is_a?(MU::Config::Ref)
+              @vpc = @config['vpc'].kitten
+            end
 
             subnet_ids = []
 
+            if !@vpc
+              raise MuError, "Failed to load vpc for Autoscale Group #{@mu_name}"
+            end
             if !@config["vpc"]["subnets"].nil? and @config["vpc"]["subnets"].size > 0
               @config["vpc"]["subnets"].each { |subnet|
                 subnet_obj = @vpc.getSubnet(cloud_id: subnet["subnet_id"], name: subnet["subnet_name"])