cloud-mu 2.0.4 → 2.1.0beta

data/modules/mu/clouds/aws/firewall_rule.rb

@@ -106,10 +106,10 @@ module MU
           # XXX the egress logic here is a crude hack, this really needs to be
           # done at config level
           setRules(
-              [],
-              add_to_self: @config['self_referencing'],
-              ingress: true,
-              egress: egress
+            [],
+            add_to_self: @config['self_referencing'],
+            ingress: true,
+            egress: egress
           )
 
           MU.log "EC2 Security Group #{groupname} is #{secgroup.group_id}", MU::DEBUG
@@ -124,10 +124,10 @@ module MU
             # XXX the egress logic here is a crude hack, this really needs to be
             # done at config level
             setRules(
-                @config['rules'],
-                add_to_self: @config['self_referencing'],
-                ingress: true,
-                egress: egress
+              @config['rules'],
+              add_to_self: @config['self_referencing'],
+              ingress: true,
+              egress: egress
             )
           end
         end
@@ -150,36 +150,58 @@ module MU
         # @param egress [Boolean]: Whether this is an egress ruleset, instead of ingress.
         # @param port_range [String]: A port range descriptor (e.g. 0-65535). Only valid with udp or tcp.
         # @return [void]
-        def addRule(hosts, proto: "tcp", port: nil, egress: false, port_range: "0-65535")
+        def addRule(hosts, proto: "tcp", port: nil, egress: false, port_range: "0-65535", comment: nil)
           rule = Hash.new
           rule["proto"] = proto
-          if hosts.is_a?(String)
-            rule["hosts"] = [hosts]
-          else
-            rule["hosts"] = hosts
-          end
+          sgs = []
+          hosts = [hosts] if hosts.is_a?(String)
+          hosts.each { |h|
+            if h.match(/^sg-/)
+              sgs << h
+            end
+          }
+          rule["sgs"] = sgs if sgs.size > 0
+          hosts = hosts - sgs
+          rule["hosts"] = hosts if hosts.size > 0
+
           if port != nil
             port = port.to_s if !port.is_a?(String)
             rule["port"] = port
           else
             rule["port_range"] = port_range
           end
+          rule["description"] = comment if comment
           ec2_rule = convertToEc2([rule])
 
           begin
             if egress
               MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
-                  group_id: @cloud_id,
-                  ip_permissions: ec2_rule
+                group_id: @cloud_id,
+                ip_permissions: ec2_rule
               )
             else
               MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
-                  group_id: @cloud_id,
-                  ip_permissions: ec2_rule
+                group_id: @cloud_id,
+                ip_permissions: ec2_rule
               )
             end
           rescue Aws::EC2::Errors::InvalidPermissionDuplicate => e
             MU.log "Attempt to add duplicate rule to #{@cloud_id}", MU::DEBUG, details: ec2_rule
+            # Ensure that, at least, the description field gets updated on
+            # existing rules
+            if comment
+              if egress
+                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).update_security_group_rule_descriptions_egress(
+                  group_id: @cloud_id,
+                  ip_permissions: ec2_rule
+                )
+              else
+                MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).update_security_group_rule_descriptions_ingress(
+                  group_id: @cloud_id,
+                  ip_permissions: ec2_rule
+                )
+              end
+            end
           end
         end
 
@@ -554,7 +576,11 @@ module MU
                 rule['hosts'].each { |cidr|
                   next if cidr.nil? # XXX where is that coming from?
                   cidr = cidr + "/32" if cidr.match(/^\d+\.\d+\.\d+\.\d+$/)
-                  ec2_rule[:ip_ranges] << {cidr_ip: cidr}
+                  if rule['description']
+                    ec2_rule[:ip_ranges] << {cidr_ip: cidr, description: rule['description']}
+                  else
+                    ec2_rule[:ip_ranges] << {cidr_ip: cidr}
+                  end
                 }
               end
 

data/modules/mu/clouds/aws/log.rb

@@ -191,13 +191,14 @@ module MU
 
         # Return the cloud descriptor for the Log Group
         def cloud_desc
-          MU::Cloud::AWS::Log.find(cloud_id: @cloud_id).values.first
+          found = MU::Cloud::AWS::Log.find(cloud_id: @cloud_id)
+          found ? found.values.first : nil
         end
 
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn
-          cloud_desc.arn
+          cloud_desc ? cloud_desc.arn : nil
         end
 
         # Return the metadata for this log configuration

data/modules/mu/clouds/aws/role.rb

@@ -30,6 +30,7 @@ module MU
           @config = MU::Config.manxify(kitten_cfg)
           @cloud_id ||= cloud_id
           @mu_name = mu_name
+          @cloud_id ||= @mu_name # should be the same
           @mu_name ||= @deploy.getResourceName(@config["name"])
         end
 
@@ -54,8 +55,10 @@ module MU
 
           if !@config['bare_policies']
             MU.log "Creating IAM role #{@mu_name}"
+            @cloud_id = @mu_name
+            path = @config['strip_path'] ? nil : "/"+@deploy.deploy_id+"/"
             resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).create_role(
-              path: "/"+@deploy.deploy_id+"/",
+              path: nil,
               role_name: @mu_name,
               description: "Generated by Mu",
               assume_role_policy_document: gen_role_policy_doc,
@@ -128,6 +131,7 @@ module MU
                     policy_arn: arn,
                     version_id: desc.policy.default_version_id
                   )
+
                   if version.policy_version.document != URI.encode(JSON.generate(policy.values.first), /[^a-z0-9\-]/i)
                     MU.log "Updating IAM policy #{policy_name}", MU::NOTICE, details: policy.values.first
                     update_policy(arn, policy.values.first)
@@ -196,6 +200,7 @@ module MU
             end
 
           end
+          desc['cloud_id'] ||= @cloud_id
 
           desc
         end
@@ -216,6 +221,7 @@ module MU
           end
 
           my_policies = cloud_desc["policies"]
+          my_policies ||= []
           
           my_policies.each { |p|
             if p.policy_name == policy
@@ -563,6 +569,11 @@ module MU
             "tags" => MU::Config.tags_primitive,
             "optional_tags" => MU::Config.optional_tags_primitive,
             "policies" => self.condition_schema,
+            "strip_path" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "Normally we namespace IAM roles with a +path+ set to match our +deploy_id+; this disables that behavior. Temporary workaround for a bug in EKS/IAM integration."
+            },
             "import" => {
               "items" => {
                 "description" => "Can be a shorthand reference to a canned IAM policy like +AdministratorAccess+, or a full ARN like +arn:aws:iam::aws:policy/AmazonESCognitoAccess+"
@@ -734,8 +745,13 @@ module MU
                     )
                     if sibling
                       id = sibling.cloudobj.arn
-                      id += target["path"] if target["path"]
+                      id.sub!(/:([^:]+)$/, ":"+target["path"]) if target["path"]
                       doc["Statement"].first["Resource"] << id
+                      if id.match(/:log-group:/)
+                        stream_id = id.sub(/:([^:]+)$/, ":log-stream:*")
+#                        "arn:aws:logs:us-east-2:accountID:log-group:log_group_name:log-stream:CloudTrail_log_stream_name_prefix*"
+                        doc["Statement"].first["Resource"] << stream_id
+                      end
                     else
                       raise MuError, "Couldn't find a #{target["entity_type"]} named #{target["identifier"]} when generating IAM policy"
                     end

data/modules/mu/clouds/aws/server.rb

@@ -424,7 +424,10 @@ module MU
               groupname = resp.auto_scaling_instances.first.auto_scaling_group_name
               MU.log "Pausing Autoscale processes in #{groupname}", MU::NOTICE
               MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).suspend_processes(
-                auto_scaling_group_name: groupname
+                auto_scaling_group_name: groupname,
+                scaling_processes: [
+                  "Terminate",
+                ], 
               )
             end
             begin
@@ -445,7 +448,10 @@ module MU
               if !groupname.nil?
                 MU.log "Resuming Autoscale processes in #{groupname}", MU::NOTICE
                 MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).resume_processes(
-                  auto_scaling_group_name: groupname
+                  auto_scaling_group_name: groupname,
+                  scaling_processes: [
+                    "Terminate",
+                  ],
                 )
               end
             end
@@ -1218,7 +1224,7 @@ module MU
                 purgecmd = "rm -rf /cygdrive/c/chef/ /home/#{@config['windows_admin_username']}/.ssh/authorized_keys /home/Administrator/.ssh/authorized_keys /cygdrive/c/mu-installer-ran-updates /cygdrive/c/mu_installed_chef"
                 # session.exec!("powershell -Command \"& {(Get-WmiObject -Class Win32_Product -Filter \"Name='UniversalForwarder'\").Uninstall()}\"")
               else
-                purgecmd = "#{sudo} rm -rf /root/.ssh/authorized_keys /etc/ssh/ssh_host_*key* /etc/chef /etc/opscode/* /.mu-installer-ran-updates /var/chef /opt/mu_installed_chef /opt/chef ; #{sudo} sed -i 's/^HOSTNAME=.*//' /etc/sysconfig/network"
+                purgecmd = "#{sudo} rm -rf /var/lib/cloud/instances/i-* /root/.ssh/authorized_keys /etc/ssh/ssh_host_*key* /etc/chef /etc/opscode/* /.mu-installer-ran-updates /var/chef /opt/mu_installed_chef /opt/chef ; #{sudo} sed -i 's/^HOSTNAME=.*//' /etc/sysconfig/network"
               end
             end
             session.exec!(purgecmd)
@@ -2205,8 +2211,8 @@ module MU
             if server['iam_policies']
               role['iam_policies'] = server['iam_policies'].dup
             end
-            if server['canned_policies']
-              role['import'] = server['canned_policies'].dup
+            if server['canned_iam_policies']
+              role['import'] = server['canned_iam_policies'].dup
             end
             if server['iam_role']
 # XXX maybe break this down into policies and add those?

data/modules/mu/clouds/aws/server_pool.rb

@@ -368,18 +368,7 @@ module MU
                 policy_params[:scaling_adjustment] = policy['adjustment']
                 policy_params[:adjustment_type] = policy['type']
               elsif policy["policy_type"] == "TargetTrackingScaling"
-                def strToSym(hash)
-                  newhash = {}
-                  hash.each_pair { |k, v|
-                    if v.is_a?(Hash)
-                      newhash[k.to_sym] = strToSym(v)
-                    else
-                      newhash[k.to_sym] = v
-                    end
-                  }
-                  newhash
-                end
-                policy_params[:target_tracking_configuration] = strToSym(policy['target_tracking_configuration'])
+                policy_params[:target_tracking_configuration] = MU.strToSym(policy['target_tracking_configuration'])
                 policy_params[:target_tracking_configuration].delete(:preferred_target_group)
                 if policy_params[:target_tracking_configuration][:predefined_metric_specification] and
                    policy_params[:target_tracking_configuration][:predefined_metric_specification][:predefined_metric_type] == "ALBRequestCountPerTarget"
@@ -489,6 +478,11 @@ module MU
           toplevel_required = []
           
           schema = {
+            "role_strip_path" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "Normally we namespace IAM roles with a +path+ set to match our +deploy_id+; this disables that behavior. Temporary workaround for a bug in EKS/IAM integration."
+            },
             "notifications" => {
               "type" => "object",
               "description" => "Send notifications to an SNS topic for basic AutoScaling events",
@@ -845,8 +839,11 @@ module MU
                 ok = false
               end
             else
+              s3_objs = ['arn:'+(MU::Cloud::AWS.isGovCloud?(pool['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU.adminBucketName+'/Mu_CA.pem']
+
               role = {
                 "name" => pool["name"],
+                "strip_path" => pool["role_strip_path"],
                 "can_assume" => [
                   {
                     "entity_id" => "ec2.amazonaws.com",
@@ -857,19 +854,15 @@ module MU
                   {
                     "name" => "MuSecrets",
                     "permissions" => ["s3:GetObject"],
-                    "targets" => [
-                      {
-                        "identifier" => 'arn:'+(MU::Cloud::AWS.isGovCloud?(pool['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU.adminBucketName+'/Mu_CA.pem'
-                      }
-                    ]
+                    "targets" => s3_objs.map { |f| { "identifier" => f } }
                   }
                 ]
               }
               if launch['iam_policies']
                 role['iam_policies'] = launch['iam_policies'].dup
               end
-              if pool['canned_policies']
-                role['import'] = pool['canned_policies'].dup
+              if pool['canned_iam_policies']
+                role['import'] = pool['canned_iam_policies'].dup
               end
               if pool['iam_role']
 # XXX maybe break this down into policies and add those?
@@ -1154,6 +1147,14 @@ module MU
 
           storage.concat(MU::Cloud::AWS::Server.ephemeral_mappings)
 
+          if @config['basis']['launch_config']['generate_iam_role']
+            role = @deploy.findLitterMate(name: @config['name'], type: "roles")
+            s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
+              'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU.adminBucketName+'/'+file
+            }
+            role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
+          end
+
           if !oldlaunch.nil?
             olduserdata = Base64.decode64(oldlaunch.user_data)
             if userdata != olduserdata or
@@ -1237,11 +1238,6 @@ module MU
 
           if @config['basis']['launch_config']['generate_iam_role']
             role = @deploy.findLitterMate(name: @config['name'], type: "roles")
-# XXX are these the right patterns for a pool, or did we need wildcards?
-            s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
-              'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU.adminBucketName+'/'+file
-            }
-            role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
 
             @config['iam_role'] = role.mu_name
 

data/modules/mu/clouds/aws/userdata/linux.erb

@@ -108,8 +108,8 @@ if ping -c 5 8.8.8.8 > /dev/null; then
         service sshd start
       fi
     fi
-  fi
 <% end %>
+  fi
 else
   /bin/logger "***** Unable to verify internet connectivity, skipping package updates from userdata"
   touch /.mu-installer-ran-updates

data/modules/mu/clouds/aws/vpc.rb

@@ -386,8 +386,17 @@ module MU
                           }
 
                           MU.log "Creating route for #{route['destination_network']} through NAT gatway #{gateway['id']}", details: route_config
+                          nat_retries = 0
                           begin
                             resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
+                          rescue Aws::EC2::Errors::InvalidNatGatewayIDNotFound => e
+                            if nat_retries < 5
+                              nat_retries += 1 
+                              sleep 10
+                              retry
+                            else
+                              raise e
+                            end
                           rescue Aws::EC2::Errors::RouteAlreadyExists => e
                             MU.log "Attempt to create duplicate route to #{route['destination_network']} for #{gateway['id']} in #{rtb['route_table_id']}", MU::WARN
                           end

data/modules/mu/clouds/google/server.rb

@@ -623,6 +623,8 @@ next if !create
                       az,
                       cloud_id
                     )
+                  rescue ::OpenSSL::SSL::SSLError => e
+                    MU.log "Got #{e.message} looking for instance #{cloud_id} in project #{flags["project"]} (#{az}). Usually this means we've tried to query a non-functional region.", MU::DEBUG
                   rescue ::Google::Apis::ClientError => e
                     raise e if !e.message.match(/^notFound: /)
                   end

data/modules/mu/config.rb

@@ -1214,7 +1214,7 @@ module MU
         "type" => "array",
         "minItems" => 1,
         "items" => {
-          "description" => "Tags to apply to this resource. Will apply at the cloud provider level and in Chef, where applicable.",
+          "description" => "Tags to apply to this resource. Will apply at the cloud provider level and in node groomers, where applicable.",
           "type" => "object",
           "title" => "tags",
           "required" => ["key", "value"],
@@ -1500,7 +1500,7 @@ module MU
           }
         end
         if conf_chunk.nil? and schema_chunk["default"] != nil
-          return schema_chunk["default"]
+          return schema_chunk["default"].dup
         end
       end
       return conf_chunk
@@ -1567,7 +1567,7 @@ module MU
     def self.check_vault_refs(server)
       ok = true
       server['vault_access'] = [] if server['vault_access'].nil?
-      server['groomer'] ||= "Chef"
+      server['groomer'] ||= self.defaultGroomer
       groomclass = MU::Groomer.loadGroomer(server['groomer'])
 
       begin

data/modules/mu/config/container_cluster.rb

@@ -41,7 +41,7 @@ module MU
               "properties" => {
                 "version" => {
                   "type" => "string",
-                  "default" => "1.10",
+                  "default" => "1.11",
                   "description" => "Version of Kubernetes control plane to deploy",
                 },
                 "max_pods" => {

data/modules/mu/config/firewall_rule.rb

@@ -78,6 +78,10 @@ module MU
               "type" => "boolean",
               "default" => false
             },
+            "comment" => {
+              "type" => "string",
+              "description" => "String description of this firewall rule, where supported"
+            },
             "hosts" => {
               "type" => "array",
               "items" => MU::Config::CIDR_PRIMITIVE

data/modules/mu/config/role.rb

@@ -45,6 +45,32 @@ module MU
         }
       end
 
+      # Chunk of schema to reference an account/project, here to be embedded
+      # into the schemas of other resources.
+      def self.reference
+        {
+          "type" => "object",
+          "description" => "An IAM role to associate with this resource",
+          "minProperties" => 1,
+          "additionalProperties" => false,
+          "properties" => {
+            "id" => {
+              "type" => "string",
+              "description" => "Discover this role by looking for this cloud provider identifier, such as an AWS ARN"
+            },
+            "name" => {
+              "type" => "string",
+              "description" => "Discover this role by Mu-internal name; typically the shorthand 'name' field of a Role object declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
+            },
+            "cloud" => MU::Config.cloud_primitive,
+            "deploy_id" => {
+              "type" => "string",
+              "description" => "Search for this Role in an existing Mu deploy by Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
+            }
+          }
+        }
+      end
+
       # A generic, cloud-neutral descriptor for a policy that grants or denies
       # permissions to some entity over some other entity.
       # @param subobjects [Boolean]: Whether the returned schema should include a +path+ parameter
@@ -89,6 +115,9 @@ module MU
                   "identifier" => {
                     "type" => "string",
                     "description" => "Either the name of a sibling Mu resource in this stack (used in conjunction with +entity_type+), or the full cloud identifier for a resource, such as an ARN in Amazon Web Services."
+                  },
+                  "path" => {
+                    "type" => "string",
                   }
                 }
               }