cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/clouds/aws/server_pool.rb

@@ -18,25 +18,11 @@ module MU
       # A server pool as configured in {MU::Config::BasketofKittens::server_pools}
       class ServerPool < MU::Cloud::ServerPool
 
-        @deploy = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :cloud_id
-        attr_reader :config
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::server_pools}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
-          if !mu_name.nil?
-            @mu_name = mu_name
-          elsif @config['scrub_mu_isms']
-            @mu_name = @config['name']
-          else
-            @mu_name = @deploy.getResourceName(@config['name'])
-          end
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
+          @mu_name ||= @deploy.getResourceName(@config['name'])
         end
 
         # Called automatically by {MU::Deploy#createResources}
@@ -148,8 +134,6 @@ module MU
             setScaleInProtection(need_instances)
           end
 
-          MU.log "See /var/log/mu-momma-cat.log for asynchronous bootstrap progress.", MU::NOTICE
-
           return asg
         end
 
@@ -476,6 +460,8 @@ module MU
         # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
         def self.schema(config)
           toplevel_required = []
+
+          term_policies = MU::Cloud::AWS.credConfig ? MU::Cloud::AWS.autoscale.describe_termination_policy_types.termination_policy_types : ["AllocationStrategy", "ClosestToNextInstanceHour", "Default", "NewestInstance", "OldestInstance", "OldestLaunchConfiguration", "OldestLaunchTemplate"]
           
           schema = {
             "role_strip_path" => {
@@ -586,7 +572,7 @@ module MU
               "items" => {
                 "type" => "String",
                 "default" => "Default",
-                "enum" => MU::Cloud::AWS.autoscale.describe_termination_policy_types.termination_policy_types
+                "enum" => term_policies
               }
             },
             "scaling_policies" => {
@@ -839,10 +825,11 @@ module MU
                 ok = false
               end
             else
-              s3_objs = ['arn:'+(MU::Cloud::AWS.isGovCloud?(pool['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU.adminBucketName+'/Mu_CA.pem']
+              s3_objs = ['arn:'+(MU::Cloud::AWS.isGovCloud?(pool['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(pool['credentials'])+'/Mu_CA.pem']
 
               role = {
                 "name" => pool["name"],
+                "cloud" => "AWS",
                 "strip_path" => pool["role_strip_path"],
                 "can_assume" => [
                   {
@@ -878,9 +865,9 @@ module MU
             end
             launch["ami_id"] ||= launch["image_id"]
             if launch["server"].nil? and launch["instance_id"].nil? and launch["ami_id"].nil?
-              if MU::Config.amazon_images.has_key?(pool['platform']) and
-                  MU::Config.amazon_images[pool['platform']].has_key?(pool['region'])
-                launch['ami_id'] = configurator.getTail("pool"+pool['name']+"AMI", value: MU::Config.amazon_images[pool['platform']][pool['region']], prettyname: "pool"+pool['name']+"AMI", cloudtype: "AWS::EC2::Image::Id")
+              img_id = MU::Cloud.getStockImage("AWS", platform: pool['platform'], region: pool['region'])
+              if img_id
+                launch['ami_id'] = configurator.getTail("pool"+pool['name']+"AMI", value: img_id, prettyname: "pool"+pool['name']+"AMI", cloudtype: "AWS::EC2::Image::Id")
   
               else
                 ok = false
@@ -961,6 +948,7 @@ module MU
               if policy["alarms"] && !policy["alarms"].empty?
                 policy["alarms"].each { |alarm|
                   alarm["name"] = "scaling-policy-#{pool["name"]}-#{alarm["name"]}"
+                  alarm["cloud"] = "AWS",
                   alarm['dimensions'] = [] if !alarm['dimensions']
                   alarm['dimensions'] << { "name" => pool["name"], "cloud_class" => "AutoScalingGroupName" }
                   alarm["namespace"] = "AWS/EC2" if alarm["namespace"].nil?
@@ -1086,8 +1074,9 @@ module MU
             @config['basis']['launch_config']["ami_id"] = MU::Cloud::AWS::Server.createImage(
               name: @mu_name,
               instance_id: @config['basis']['launch_config']["instance_id"],
-              credentials: @config['credentials']
-            )
+              credentials: @config['credentials'],
+              region: @config['region']
+            )[@config['region']]
           end
           MU::Cloud::AWS::Server.waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
 
@@ -1097,13 +1086,17 @@ module MU
 
           userdata = MU::Cloud.fetchUserdata(
             platform: @config["platform"],
-            cloud: "aws",
+            cloud: "AWS",
+            credentials: @config['credentials'],
             template_variables: {
               "deployKey" => Base64.urlsafe_encode64(@deploy.public_key),
               "deploySSHKey" => @deploy.ssh_public_key,
               "muID" => @deploy.deploy_id,
               "muUser" => MU.chef_user,
               "publicIP" => MU.mu_public_ip,
+              "mommaCatPort" => MU.mommaCatPort,
+              "chefVersion" => MU.chefVersion,
+              "adminBucketName" => MU::Cloud::AWS.adminBucketName(@credentials),
               "windowsAdminName" => @config['windows_admin_username'],
               "skipApplyUpdates" => @config['skipinitialupdates'],
               "resourceName" => @config["name"],
@@ -1149,19 +1142,21 @@ module MU
 
           if @config['basis']['launch_config']['generate_iam_role']
             role = @deploy.findLitterMate(name: @config['name'], type: "roles")
-            s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
-              'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU.adminBucketName+'/'+file
-            }
-            role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
+            if role
+              s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file| 
+                'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(@credentials)+'/'+file
+              }
+              role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
+            end
           end
 
           if !oldlaunch.nil?
             olduserdata = Base64.decode64(oldlaunch.user_data)
-            if userdata != olduserdata or
-                oldlaunch.image_id != @config["basis"]["launch_config"]["ami_id"] or
-                oldlaunch.ebs_optimized != @config["basis"]["launch_config"]["ebs_optimized"] or
-                oldlaunch.instance_type != @config["basis"]["launch_config"]["size"] or
-                oldlaunch.instance_monitoring.enabled != @config["basis"]["launch_config"]["monitoring"]
+            if userdata == olduserdata and
+                oldlaunch.image_id == @config["basis"]["launch_config"]["ami_id"] and
+                oldlaunch.ebs_optimized == @config["basis"]["launch_config"]["ebs_optimized"] and
+                oldlaunch.instance_type == @config["basis"]["launch_config"]["size"] and
+                oldlaunch.instance_monitoring.enabled == @config["basis"]["launch_config"]["monitoring"]
                 # XXX check more things
 #                launch.block_device_mappings != storage
 #                XXX block device comparison isn't this simple

data/modules/mu/clouds/aws/storage_pool.rb

@@ -17,18 +17,11 @@ module MU
     class AWS
       # A storage pool as configured in {MU::Config::BasketofKittens::storage_pools}
       class StoragePool < MU::Cloud::StoragePool
-        @deploy = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :cloud_id
-        attr_reader :config
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::storage_pools}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
+
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
           @mu_name ||= @deploy.getResourceName(@config['name'])
         end
 

data/modules/mu/clouds/aws/user.rb

@@ -17,18 +17,11 @@ module MU
     class AWS
       # A user as configured in {MU::Config::BasketofKittens::users}
       class User < MU::Cloud::User
-        @deploy = nil
-        @config = nil
-        attr_reader :mu_name
-        attr_reader :config
-        attr_reader :cloud_id
-
-        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
-        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::users}
-        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
-          @deploy = mommacat
-          @config = MU::Config.manxify(kitten_cfg)
-          @cloud_id ||= cloud_id
+
+        # Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
+        # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
+        def initialize(**args)
+          super
           @mu_name ||= if @config['unique_name']
             @deploy.getResourceName(@config["name"])
           else
@@ -157,30 +150,63 @@ module MU
             resp.policies.each { |policy|
               MU.log "Deleting policy /#{MU.deploy_id}/#{policy.policy_name}"
               if !noop
-                attachments = MU::Cloud::AWS.iam(credentials: credentials).list_entities_for_policy(
-                  policy_arn: policy.arn
-                )
-                attachments.policy_users.each { |u|
-                  MU::Cloud::AWS.iam(credentials: credentials).detach_user_policy(
-                    user_name: u.user_name,
-                    policy_arn: policy.arn
-                  )
-                }
-                attachments.policy_groups.each { |g|
-                  MU::Cloud::AWS.iam(credentials: credentials).detach_role_policy(
-                    group_name: g.group_name,
+                attachments = begin
+                  MU::Cloud::AWS.iam(credentials: credentials).list_entities_for_policy(
                     policy_arn: policy.arn
                   )
-                }
-                attachments.policy_roles.each { |r|
-                  MU::Cloud::AWS.iam(credentials: credentials).detach_role_policy(
-                    role_name: r.role_name,
+                rescue ::Aws::IAM::Errors::NoSuchEntity
+                end
+                if attachments
+                  begin
+                    attachments.policy_users.each { |u|
+                      MU::Cloud::AWS.iam(credentials: credentials).detach_user_policy(
+                        user_name: u.user_name,
+                        policy_arn: policy.arn
+                      )
+                    }
+                  rescue ::Aws::IAM::Errors::NoSuchEntity
+                  end
+                  begin
+                    attachments.policy_groups.each { |g|
+                      MU::Cloud::AWS.iam(credentials: credentials).detach_role_policy(
+                        group_name: g.group_name,
+                        policy_arn: policy.arn
+                      )
+                    }
+                  rescue ::Aws::IAM::Errors::NoSuchEntity
+                  end
+                  begin
+                    attachments.policy_roles.each { |r|
+                      MU::Cloud::AWS.iam(credentials: credentials).detach_role_policy(
+                        role_name: r.role_name,
+                        policy_arn: policy.arn
+                      )
+                    }
+                  rescue ::Aws::IAM::Errors::NoSuchEntity
+                  end
+                end
+
+                begin
+                  MU::Cloud::AWS.iam(credentials: credentials).delete_policy(
                     policy_arn: policy.arn
                   )
-                }
-                MU::Cloud::AWS.iam(credentials: credentials).delete_policy(
-                  policy_arn: policy.arn
-                )
+                rescue ::Aws::IAM::Errors::DeleteConflict
+                  versions = MU::Cloud::AWS.iam(credentials: credentials).list_policy_versions(
+                    policy_arn: policy.arn,
+                  ).versions
+                  versions.each { |v|
+                    next if v.is_default_version
+                    begin
+                      MU::Cloud::AWS.iam(credentials: credentials).delete_policy_version(
+                        policy_arn: policy.arn,
+                        version_id: v.version_id
+                      )
+                    rescue ::Aws::IAM::Errors::NoSuchEntity
+                    end
+                  }
+                  retry
+                rescue ::Aws::IAM::Errors::NoSuchEntity
+                end
               end
             }
           end
@@ -282,7 +308,7 @@ module MU
           schema = {
             "name" => {
               "type" => "string",
-              "description" => "A plain IAM user. If the user already exists, we will operate on that existing user. Otherwise, we will attempt to create a new user."
+              "description" => "A plain IAM user. If the user already exists, we will operate on that existing user. Otherwise, we will attempt to create a new user. AWS IAM does not distinguish between human user accounts and machine accounts."
             },
             "path" => {
               "type" => "string",

data/modules/mu/clouds/aws/userdata/linux.erb

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 updates_run=0
-need_reboot=0
 instance_id="`curl http://169.254.169.254/latest/meta-data/instance-id`"
 region="`curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed 's/[a-z]$//'`"
 
@@ -35,27 +34,25 @@ if ping -c 5 8.8.8.8 > /dev/null; then
     if [ ! -f /usr/bin/curl ] ;then /usr/bin/apt-get --fix-missing -y install curl;fi
     AWSCLI=/usr/local/bin/aws
 <% if !$mu.skipApplyUpdates %>
+    set +e
     if [ ! -f /.mu-installer-ran-updates ];then
       service ssh stop
       apt-get --fix-missing -y upgrade 
+      touch /.mu-installer-ran-updates
       if [ $? -eq 0 ]
       then
         echo "Successfully updated packages"
         updates_run=1
+        # XXX this logic works on Ubuntu, is it Debian-friendly?
+        latest_kernel="`ls -1 /boot/vmlinuz-* | sed -r 's/^\/boot\/vmlinuz-//' | tail -1`"
+        running_kernel="`uname -r`"
+        if [ "$running_kernel" != "$latest_kernel" -a "$latest_kernel" != "" ];then
+          shutdown -r now "Applying new kernel"
+        fi
       else
         echo "FAILED PACKAGE UPDATE" >&2
       fi
-      # Proceed regardless
-      touch /.mu-installer-ran-updates
-
-      # XXX this logic works on Ubuntu, is it Debian-friendly?
-      latest_kernel="`ls -1 /boot/vmlinuz-* | sed -r 's/^\/boot\/vmlinuz-//' | tail -1`"
-      running_kernel="`uname -r`"
-      if [ "$running_kernel" != "$latest_kernel" -a "$latest_kernel" != "" ];then
-        need_reboot=1
-      else
-        service ssh start
-      fi
+      service ssh start
     fi
 <% end %>
   elif [ -x /usr/bin/yum ];then
@@ -89,24 +86,23 @@ if ping -c 5 8.8.8.8 > /dev/null; then
       /bin/rpm -ivh http://mirror.metrocast.net/fedora/epel/epel-release-latest-$version.noarch.rpm
     fi
 <% if !$mu.skipApplyUpdates %>
+    set +e
     if [ ! -f /.mu-installer-ran-updates ];then
       service sshd stop
       kernel_update=`yum list updates | grep kernel`
       yum -y update
+      touch /.mu-installer-ran-updates
       if [ $? -eq 0 ]
       then
         echo "Successfully updated packages"
         updates_run=1
+        if [ -n "$kernel_update" ]; then
+          shutdown -r now "Applying new kernel"
+        fi
       else
         echo "FAILED PACKAGE UPDATE" >&2
       fi
-      # Proceed regardless
-      touch /.mu-installer-ran-updates
-      if [ -n "$kernel_update" ]; then
-        need_reboot=1
-      else
-        service sshd start
-      fi
+      service sshd start
     fi
 <% end %>
   fi
@@ -143,27 +139,19 @@ fi
 
 umask 0077
 
-# Install Chef now, because why not?
 if [ ! -f /opt/chef/embedded/bin/ruby ];then
   curl https://www.chef.io/chef/install.sh > chef-install.sh
   set +e
   # We may run afoul of a synchronous bootstrap process doing the same thing. So
   # wait until we've managed to run successfully.
-  while ! sh chef-install.sh -v <%= MU.chefVersion %>;do
+  while ! sh chef-install.sh -v <%= $mu.chefVersion %>;do
     sleep 10
   done
   touch /opt/mu_installed_chef
-  set -e
-fi
-
-<% if !$mu.skipApplyUpdates %>
-if [ "$need_reboot" == "1" ];then
-  shutdown -r now "Applying new kernel"
 fi
-<% end %>
 
 if [ "$AWSCLI" != "" ];then
-  $AWSCLI --region="$region" s3 cp s3://<%= MU.adminBucketName %>/<%= $mu.muID %>-secret .
+  $AWSCLI --region="$region" s3 cp s3://<%= $mu.adminBucketName %>/<%= $mu.muID %>-secret .
 fi
 
 echo '
@@ -181,6 +169,6 @@ if ! ( netstat -na | grep LISTEN | grep ':22 ' );then
   service sshd start
 fi
 
-/usr/bin/curl -k --data mu_id="<%= $mu.muID %>" --data mu_resource_name="<%= $mu.resourceName %>" --data mu_resource_type="<%= $mu.resourceType %>" --data mu_instance_id="$instance_id" --data mu_bootstrap="1" --data mu_user="<%= $mu.muUser %>" --data mu_deploy_secret="`/opt/chef/embedded/bin/ruby encrypt_deploy_secret.rb`" https://<%= $mu.publicIP %>:2260/
+/usr/bin/curl -k --data mu_id="<%= $mu.muID %>" --data mu_resource_name="<%= $mu.resourceName %>" --data mu_resource_type="<%= $mu.resourceType %>" --data mu_instance_id="$instance_id" --data mu_bootstrap="1" --data mu_user="<%= $mu.muUser %>" --data mu_deploy_secret="`/opt/chef/embedded/bin/ruby encrypt_deploy_secret.rb`" https://<%= $mu.publicIP %>:<%= $mu.mommaCatPort %>/
 /bin/rm -f <%= $mu.muID %>-secret mu_deploy_key.pub chef-install.sh encrypt_deploy_secret.rb
 touch /.mu_userdata_complete

data/modules/mu/clouds/aws/userdata/windows.erb

@@ -23,8 +23,8 @@ function log
 }
 
 function fetchSecret([string]$file){
-  log "Fetching s3://<%= MU.adminBucketName %>/$file to $tmp/$file"
-  aws.cmd --region $region s3 cp s3://<%= MU.adminBucketName %>/$file $tmp/$file
+  log "Fetching s3://<%= $mu.adminBucketName %>/$file to $tmp/$file"
+  aws.cmd --region $region s3 cp s3://<%= $mu.adminBucketName %>/$file $tmp/$file
 }
 
 function importCert([string]$cert, [string]$store){
@@ -113,7 +113,7 @@ function removeChef($location){
   $install_chef = $false
   $my_chef = (Get-ItemProperty $location | Where-Object {$_.DisplayName -like "chef client*"}).DisplayName
   if ($my_chef) {
-    if ($my_chef -match '<%= MU.chefVersion %>'.split('-')[0]) {
+    if ($my_chef -match '<%= $mu.chefVersion %>'.split('-')[0]) {
       $install_chef = $false
     } else{
       log "Uninstalling Chef"
@@ -143,13 +143,13 @@ If (!(Test-Path "c:\opscode\chef\embedded\bin\ruby.exe")){
 }
 
 If ($install_chef){
-  log "Installing Chef <%= MU.chefVersion %>"
-  If (!(Test-Path $env:Temp/chef-installer-<%= MU.chefVersion %>.msi)){
+  log "Installing Chef <%= $mu.chefVersion %>"
+  If (!(Test-Path $env:Temp/chef-installer-<%= $mu.chefVersion %>.msi)){
     log "Downloading Chef installer"
-    $WebClient.DownloadFile("https://www.chef.io/chef/download?p=windows&pv=2012&m=x86_64&v=<%= MU.chefVersion %>","$env:Temp/chef-installer-<%= MU.chefVersion %>.msi")
+    $WebClient.DownloadFile("https://www.chef.io/chef/download?p=windows&pv=2012&m=x86_64&v=<%= $mu.chefVersion %>","$env:Temp/chef-installer-<%= $mu.chefVersion %>.msi")
   }
   log "Running Chef installer"
-  (Start-Process -FilePath msiexec -ArgumentList "/i $env:Temp\chef-installer-<%= MU.chefVersion %>.msi ALLUSERS=1 /le $env:Temp\chef-client-install.log /qn" -Wait -Passthru).ExitCode
+  (Start-Process -FilePath msiexec -ArgumentList "/i $env:Temp\chef-installer-<%= $mu.chefVersion %>.msi ALLUSERS=1 /le $env:Temp\chef-client-install.log /qn" -Wait -Passthru).ExitCode
   Set-Content "c:/mu_installed_chef" "yup"
 }
 
@@ -163,9 +163,9 @@ $deploy_secret = & "c:\opscode\chef\embedded\bin\ruby" -ropenssl -rbase64 -e "ke
 function callMomma([string]$act)
 {
   $params = @{mu_id='<%= $mu.muID %>';mu_resource_name='<%= $mu.resourceName %>';mu_resource_type='<%= $mu.resourceType %>';mu_instance_id="$awsid";mu_user='<%= $mu.muUser %>';mu_deploy_secret="$deploy_secret";$act="1"}
-  log "Calling Momma Cat at https://<%= $mu.publicIP %>:2260 with $act"
+  log "Calling Momma Cat at https://<%= $mu.publicIP %>:<%= $mu.mommaCatPort %> with $act"
   [System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true}
-  $resp = Invoke-WebRequest -Uri https://<%= $mu.publicIP %>:2260 -Method POST -Body $params
+  $resp = Invoke-WebRequest -Uri https://<%= $mu.publicIP %>:<%= $mu.mommaCatPort %> -Method POST -Body $params
   return $resp.Content
 }