cloud-mu 3.0.0beta → 3.0.0

data/modules/mu/clouds/aws/storage_pool.rb

@@ -104,40 +104,35 @@ module MU
         end
 
         # Locate an existing storage pool and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @param flags [Hash]: Optional flags
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching storage pool
-        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching storage pool
+        def self.find(**args)
           map = {}
-          if cloud_id
-            storge_pool = MU::Cloud::AWS.efs(region: region, credentials: credentials).describe_file_systems(
-              file_system_id: cloud_id
+          if args[:cloud_id]
+            storge_pool = MU::Cloud::AWS.efs(region: args[:region], credentials: args[:credentials]).describe_file_systems(
+              file_system_id: args[:cloud_id]
             ).file_systems.first
             
             map[cloud_id] = storge_pool if storge_pool
           end
 
           if tag_value
-            storage_pools = MU::Cloud::AWS.efs(region: region, credentials: credentials).describe_file_systems.file_systems
+            storage_pools = MU::Cloud::AWS.efs(region: args[:region], credentials: args[:credentials]).describe_file_systems.file_systems
           
             if !storage_pools.empty?
               storage_pools.each{ |pool|
-                tags = MU::Cloud::AWS.efs(region: region, credentials: credentials).describe_tags(
+                tags = MU::Cloud::AWS.efs(region: args[:region], credentials: args[:credentials]).describe_tags(
                   file_system_id: pool.file_system_id
                 ).tags
 
                 value = nil
                 tags.each{ |tag|
-                  if tag.key == tag_key
+                  if tag.key == args[:tag_key]
                     value = tag.value
                     break
                   end
                 }
                 
-                if value == tag_value
+                if value == args[:tag_value]
                   map[pool.file_system_id] = pool
                   break
                 end

data/modules/mu/clouds/aws/user.rb

@@ -280,19 +280,16 @@ module MU
           cloud_desc.arn
         end
 
-        # Locate an existing user group.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region.
-        # @param flags [Hash]: Optional flags
-        # @return [OpenStruct]: The cloud provider's complete descriptions of matching user group.
-        def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
+        # Locate an existing IAM user
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching user group.
+        def self.find(**args)
           found = nil
 
           begin
-            resp = MU::Cloud::AWS.iam.get_user(user_name: cloud_id)
+            resp = MU::Cloud::AWS.iam.get_user(user_name: args[:cloud_id])
             if resp and resp.user
               found ||= {}
-              found[cloud_id] = resp.user
+              found[args[:cloud_id]] = resp.user
             end
           rescue ::Aws::IAM::Errors::NoSuchEntity
           end

data/modules/mu/clouds/aws/userdata/linux.erb

@@ -24,6 +24,12 @@ for d in r s t u ;do
   fi
 done
 
+for f in /etc/rc.local /etc/rc.d/rc.local;do
+  if [ -f $f ];then
+    chmod 755 $f
+  fi
+done
+
 if ping -c 5 8.8.8.8 > /dev/null; then
   if [ -f /etc/debian_version ];then
     if ! grep '^/bin/sh /var/lib/cloud/instance/user-data.txt$' /etc/rc.local > /dev/null;then
@@ -72,12 +78,12 @@ if ping -c 5 8.8.8.8 > /dev/null; then
       cat /etc/rc.d/rc.local | grep -v '^/bin/sh /var/lib/cloud/instances/' >> /tmp/rc.local.$$
       echo "/bin/sh $userdata_dir/user-data.txt" >> /tmp/rc.local.$$
       mv /tmp/rc.local.$$ /etc/rc.d/rc.local
+      chmod 755 /etc/rc.d/rc.local
     fi
 
     sed -i 's/^Defaults.*requiretty$/Defaults   !requiretty/' /etc/sudoers
 
     if [ "$version" == "7" ];then
-      chmod 755 /etc/rc.d/rc.local
       systemctl reset-failed sshd.service
     fi
     if [ ! -f /usr/bin/curl ] ;then /usr/bin/yum -y install curl;fi

data/modules/mu/clouds/aws/vpc.rb

@@ -208,7 +208,7 @@ module MU
                 begin
                   if resp.state != "available"
                     begin
-                      MU.log "Waiting for Subnet #{subnet_name} (#{subnet_id}) to be available", MU::NOTICE
+                      MU.log "Waiting for Subnet #{subnet_name} (#{subnet_id}) to be available", MU::NOTICE if retries > 0 and (retries % 3) == 0
                       sleep 5
                       resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
                     rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
@@ -296,7 +296,7 @@ module MU
                   nat_gateway_id = resp.nat_gateway_id
                   attempts = 0
                   MU::MommaCat.unlock("nat-gateway-eipalloc")
-                  while resp.state == "pending"
+                  while resp.class.name != "Aws::EC2::Types::NatGateway" or resp.state == "pending"
                     MU.log "Waiting for nat gateway #{nat_gateway_id} () to become available (EIP allocation: #{allocation_id})" if attempts % 5 == 0
                     sleep 30
                     begin
@@ -554,7 +554,7 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
                   },
                   {
                     name: "accepter-vpc-info.vpc-id",
-                    values: [peer_id]
+                    values: [peer_id.to_s]
                   }
                 ]
               )
@@ -717,12 +717,7 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
         end
 
         # Locate an existing VPC or VPCs and return an array containing matching AWS resource descriptors for those that match.
-        # @param cloud_id [String]: The cloud provider's identifier for this resource.
-        # @param region [String]: The cloud provider region
-        # @param tag_key [String]: A tag key to search.
-        # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
-        # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching VPCs
-#        def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
+        # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching VPCs
         def self.find(**args)
           cloud_id = args[:cloud_id]
           region = args[:region] || MU.curRegion
@@ -1870,25 +1865,32 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
 
           retries = 0
           subnets.each { |subnet|
+            MU.log "Deleting Subnet #{subnet.subnet_id}"
             begin
               if subnet.state != "available"
                 MU.log "Waiting for #{subnet.subnet_id} to be in a removable state...", MU::NOTICE
                 sleep 30
               else
-                MU.log "Deleting Subnet #{subnet.subnet_id}"
                 MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_subnet(subnet_id: subnet.subnet_id) if !noop
               end
             rescue Aws::EC2::Errors::DependencyViolation => e
-              if retries < 7
-                MU.log "#{e.inspect}, retrying in 10s", MU::WARN
-                sleep 10
+              # We're often stuck waiting for an RDS database or something else
+              # that takes 5-ever to delete.
+              if retries < 19
+                loglevel = (retries > 0 and (retries % 3) == 0) ? MU::NOTICE : MU::DEBUG
+                MU.log "#{e.message} (retry #{retries.to_s}/20)", loglevel
+                sleep 30
+                retries = retries + 1
+                retry
+              elsif retries < 20
+                MU.log "#{e.message} (final attempt)", MU::WARN
+                sleep 60
                 retries = retries + 1
                 retry
               else
                 raise e
               end
             rescue Aws::EC2::Errors::InvalidSubnetIDNotFound
-              MU.log "Subnet #{subnet.subnet_id} disappeared before I could remove it", MU::WARN
               next
             end while subnet.state != "available"
           }

data/modules/mu/clouds/azure.rb

@@ -376,6 +376,7 @@ module MU
         rg_obj = MU::Cloud::Azure.resources(:ResourceGroup).new
         rg_obj.location = region
         rg_obj.tags = MU::MommaCat.listStandardTags
+        rg_obj.tags.reject! { |k, v| v.nil? }
 
         MU::Cloud::Azure.resources(credentials: credentials).resource_groups.list.each { |rg|
           if rg.name == name and rg.location == region and rg.tags == rg_obj.tags
@@ -384,7 +385,6 @@ module MU
           end
         }
         MU.log "Configuring resource group #{name} in #{region}", details: rg_obj
-
         MU::Cloud::Azure.resources(credentials: credentials).resource_groups.create_or_update(
           name,
           rg_obj

data/modules/mu/clouds/azure/container_cluster.rb

@@ -71,7 +71,7 @@ module MU
             )
           end
 
-          MU.log %Q{How to interact with your Kubernetes cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
+          MU.log %Q{How to interact with your AKS cluster\nkubectl --kubeconfig "#{kube_conf}" get events --all-namespaces\nkubectl --kubeconfig "#{kube_conf}" get all\nkubectl --kubeconfig "#{kube_conf}" create -f some_k8s_deploy.yml\nkubectl --kubeconfig "#{kube_conf}" get nodes}, MU::SUMMARY
 
         end
 

data/modules/mu/clouds/azure/server.rb

@@ -111,7 +111,8 @@ module MU
             "src_dst_check" => false,
             "bastion" => true,
             "size" => "Standard_B2s",
-            "run_list" => [ "mu-utility::nat" ],
+            "run_list" => [ "mu-nat" ],
+            "groomer" => "Ansible",
             "platform" => "centos7",
             "associate_public_ip" => true,
             "static_ip" => { "assign_ip" => true },
@@ -469,7 +470,10 @@ module MU
             MU::Cloud.availableClouds.each { |cloud|
               next if cloud == "Azure"
               cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
-              foreign_types = (cloudbase.listInstanceTypes)[cloudbase.myRegion]
+              foreign_types = (cloudbase.listInstanceTypes).values.first
+              if foreign_types.size == 1
+                foreign_types = foreign_types.values.first
+              end
               if foreign_types and foreign_types.size > 0 and foreign_types.has_key?(size)
                 vcpu = foreign_types[size]["vcpu"]
                 mem = foreign_types[size]["memory"]
@@ -491,6 +495,7 @@ module MU
 
             if !foundmatch
               MU.log "Invalid size '#{size}' for Azure Compute instance in #{region}. Supported types:", MU::ERR, details: types.keys.sort.join(", ")
+exit
               return nil
             end
           end
@@ -509,6 +514,8 @@ module MU
           server['ssh_user'] ||= "muadmin"
 
           server['size'] = validateInstanceType(server["size"], server["region"])
+          ok = false if server['size'].nil?
+
           if server['image_id'].nil?
             img_id = MU::Cloud.getStockImage("Azure", platform: server['platform'])
             if img_id
@@ -797,8 +804,15 @@ module MU
 if !@cloud_id
 # XXX actually guard this correctly
           MU.log "Creating VM #{@mu_name}", details: vm_obj
+          begin
           vm = MU::Cloud::Azure.compute(credentials: @credentials).virtual_machines.create_or_update(@resource_group, @mu_name, vm_obj)
           @cloud_id = Id.new(vm.id)
+          rescue ::MU::Cloud::Azure::APIError => e
+            if e.message.match(/InvalidParameter: /)
+              MU.log e.message, MU::ERR, details: vm_obj
+            end
+            raise e
+          end
 end
 
         end

data/modules/mu/clouds/azure/user.rb

@@ -25,7 +25,7 @@ module MU
 
           if !mu_name.nil?
             @mu_name = mu_name
-            @cloud_id = Id.new(cloud_desc.id) if @cloud_id
+            @cloud_id = Id.new(cloud_desc.id) if @cloud_id and cloud_desc
           else
             @mu_name ||= @deploy.getResourceName(@config["name"], max_length: 31)
           end

data/modules/mu/clouds/azure/userdata/linux.erb

@@ -15,84 +15,89 @@
 
 updates_run=0
 need_reboot=0
-instance_id="`curl http://metadata.google.internal/computeMetadata/v1/instance/name`"
+instance_id="`curl -H Metadata:tttp://169.254.169.254/metadata/instance/compute/name?api-version=2017-08-01&format=text'`"
+
+for f in /etc/rc.local /etc/rc.d/rc.local;do
+  if [ -f $f ];then
+    chmod 755 $f
+  fi
+done
+
 if [ -f /etc/debian_version ];then
-	if ! grep '^/bin/sh /var/lib/cloud/instance/user-data.txt$' /etc/rc.local > /dev/null;then
-		echo "/bin/sh /var/lib/cloud/instance/user-data.txt" >> /etc/rc.local
-	fi
-	apt-get update -y
-	if [ ! -f /usr/bin/curl ] ;then /usr/bin/apt-get --fix-missing -y install curl;fi
+  if ! grep '^/bin/sh /var/lib/cloud/instance/user-data.txt$' /etc/rc.local > /dev/null;then
+    echo "/bin/sh /var/lib/cloud/instance/user-data.txt" >> /etc/rc.local
+  fi
+  apt-get update -y
+  if [ ! -f /usr/bin/curl ] ;then /usr/bin/apt-get --fix-missing -y install curl;fi
 <% if !$mu.skipApplyUpdates %>
-	if [ ! -f /.mu-installer-ran-updates ];then
-		service ssh stop
-		apt-get --fix-missing -y upgrade 
-		if [ $? -eq 0 ]
-		then
-		  echo "Successfully updated packages"
-		  updates_run=1
-		else
-		  echo "FAILED PACKAGE UPDATE" >&2
-		fi
-		# Proceed regardless
-		touch /.mu-installer-ran-updates
+  if [ ! -f /.mu-installer-ran-updates ];then
+    service ssh stop
+    apt-get --fix-missing -y upgrade 
+    if [ $? -eq 0 ]
+    then
+      echo "Successfully updated packages"
+      updates_run=1
+    else
+      echo "FAILED PACKAGE UPDATE" >&2
+    fi
+    # Proceed regardless
+    touch /.mu-installer-ran-updates
 
-		# XXX this logic works on Ubuntu, is it Debian-friendly?
-		latest_kernel="`ls -1 /boot/vmlinuz-* | sed -r 's/^\/boot\/vmlinuz-//' | tail -1`"
-		running_kernel="`uname -r`"
-		if [ "$running_kernel" != "$latest_kernel" -a "$latest_kernel" != "" ];then
-			need_reboot=1
-		else
-			service ssh start
-		fi
-	fi
+    # XXX this logic works on Ubuntu, is it Debian-friendly?
+    latest_kernel="`ls -1 /boot/vmlinuz-* | sed -r 's/^\/boot\/vmlinuz-//' | tail -1`"
+    running_kernel="`uname -r`"
+    if [ "$running_kernel" != "$latest_kernel" -a "$latest_kernel" != "" ];then
+      need_reboot=1
+    else
+      service ssh start
+    fi
+  fi
 <% end %>
 elif [ -x /usr/bin/yum ];then
-	version=`/bin/rpm -qa \*-release | grep -Ei "redhat|centos" | cut -d"-" -f3`
-	if [ -z "$version" ];then
-		amazon_version=`/bin/rpm -qa \*-release | grep -Ei "system-release"| cut -d"-" -f3 | cut -d"." -f1`
-		if [ "$amazon_version" == "2014" ] || [ "$amazon_version" == "2015" ] || [ "$amazon_version" == "2016" ];then
-			version=6
-		fi
-	fi
-	if [ $version -eq 7 ];then
-		userdata_dir="/var/lib/cloud/instances/$instance_id"
-	else
-		userdata_dir="/var/lib/cloud/instance"
-	fi
-	if ! grep "^/bin/sh $userdata_dir/user-data.txt$" /etc/rc.d/rc.local > /dev/null;then
-		echo "/bin/sh $userdata_dir/user-data.txt" >> /etc/rc.d/rc.local
-	fi
+  version=`/bin/rpm -qa \*-release | grep -Ei "redhat|centos" | cut -d"-" -f3`
+  if [ -z "$version" ];then
+    amazon_version=`/bin/rpm -qa \*-release | grep -Ei "system-release"| cut -d"-" -f3 | cut -d"." -f1`
+    if [ "$amazon_version" == "2014" ] || [ "$amazon_version" == "2015" ] || [ "$amazon_version" == "2016" ];then
+      version=6
+    fi
+  fi
+  if [ $version -eq 7 ];then
+    userdata_dir="/var/lib/cloud/instances/$instance_id"
+  else
+    userdata_dir="/var/lib/cloud/instance"
+  fi
+  if ! grep "^/bin/sh $userdata_dir/user-data.txt$" /etc/rc.d/rc.local > /dev/null;then
+    echo "/bin/sh $userdata_dir/user-data.txt" >> /etc/rc.d/rc.local
+  fi
 
   sed -i 's/^Defaults.*requiretty$/Defaults   !requiretty/' /etc/sudoers
 
-	if [ $version == 7 ];then
-		chmod 755 /etc/rc.d/rc.local
-	fi
-	if [ ! -f /usr/bin/curl ] ;then /usr/bin/yum -y install curl;fi
-	# Ugh, rando EPEL mirror
-	if [ ! -f /etc/yum.repos.d/epel.repo ];then
-		/bin/rpm -ivh http://mirror.metrocast.net/fedora/epel/epel-release-latest-$version.noarch.rpm
-	fi
+  chmod 755 /etc/rc.d/rc.local
+  if [ ! -f /usr/bin/curl ] ;then /usr/bin/yum -y install curl;fi
+  # Ugh, rando EPEL mirror
+  if [ ! -f /etc/yum.repos.d/epel.repo ];then
+    /bin/rpm -ivh http://mirror.metrocast.net/fedora/epel/epel-release-latest-$version.noarch.rpm
+  fi
 <% if !$mu.skipApplyUpdates %>
-	if [ ! -f /.mu-installer-ran-updates ];then
-		service sshd stop
-		kernel_update=`yum list updates | grep kernel`
-		yum -y update
-		if [ $? -eq 0 ]
-		then
-		  echo "Successfully updated packages"
-		  updates_run=1
-		else
-		  echo "FAILED PACKAGE UPDATE" >&2
-		fi
-		# Proceed regardless
-		touch /.mu-installer-ran-updates
-		if [ -n "$kernel_update" ]; then
-			need_reboot=1
-		else
-			service sshd start
-		fi
-	fi
+  if [ ! -f /.mu-installer-ran-updates ];then
+    service sshd stop
+    kernel_update=`yum list updates | grep kernel`
+    yum -y update
+    if [ $? -eq 0 ]
+    then
+      echo "Successfully updated packages"
+      updates_run=1
+    else
+      echo "FAILED PACKAGE UPDATE" >&2
+    fi
+    # Proceed regardless
+    touch /.mu-installer-ran-updates
+    if [ -n "$kernel_update" ]; then
+      need_reboot=1
+    else
+      service sshd start
+    fi
+  fi
 <% end %>
 fi
 
@@ -100,20 +105,20 @@ umask 0077
 
 # Install Chef now, because why not?
 if [ ! -f /opt/chef/embedded/bin/ruby ];then
-	curl https://www.chef.io/chef/install.sh > chef-install.sh
-	set +e
-	# We may run afoul of a synchronous bootstrap process doing the same thing. So
-	# wait until we've managed to run successfully.
-	while ! sh chef-install.sh -v <%= $mu.chefVersion %>;do
-		sleep 10
-	done
-	touch /opt/mu_installed_chef
-	set -e
+  curl https://www.chef.io/chef/install.sh > chef-install.sh
+  set +e
+  # We may run afoul of a synchronous bootstrap process doing the same thing. So
+  # wait until we've managed to run successfully.
+  while ! sh chef-install.sh -v <%= $mu.chefVersion %>;do
+    sleep 10
+  done
+  touch /opt/mu_installed_chef
+  set -e
 fi
 
 <% if !$mu.skipApplyUpdates %>
 if [ "$need_reboot" == "1" ];then
-	shutdown -r now "Applying new kernel"
+  shutdown -r now "Applying new kernel"
 fi
 <% end %>
 
@@ -127,7 +132,6 @@ print Base64.urlsafe_encode64(key.public_encrypt(File.read("<%= $mu.muID %>-secr
 ' > encrypt_deploy_secret.rb
 
 deploykey="<%= $mu.deployKey %>"
-instance_id="`curl http://metadata.google.internal/computeMetadata/v1/instance/name`"
 
 # Make double-sure sshd is actually up
 service sshd restart