cloud-mu 1.9.0.pre.beta

data/modules/mu/clouds/aws/user.rb

@@ -0,0 +1,382 @@
+# Copyright:: Copyright (c) 2018 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Cloud
+    class AWS
+      # A user as configured in {MU::Config::BasketofKittens::users}
+      class User < MU::Cloud::User
+        @deploy = nil
+        @config = nil
+        attr_reader :mu_name
+        attr_reader :config
+        attr_reader :cloud_id
+
+        # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
+        # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::users}
+        def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
+          @deploy = mommacat
+          @config = MU::Config.manxify(kitten_cfg)
+          @cloud_id ||= cloud_id
+          @mu_name ||= if @config['unique_name']
+            @deploy.getResourceName(@config["name"])
+          else
+            @config['name']
+          end
+        end
+
+        # Called automatically by {MU::Deploy#createResources}
+        def create
+
+          begin
+            MU::Cloud::AWS.iam.get_user(user_name: @mu_name, path: @config['path'])
+            if !@config['use_if_exists']
+              raise MuError, "IAM user #{@mu_name} already exists and use_if_exists is false"
+            end
+          rescue Aws::IAM::Errors::NoSuchEntity => e
+            @config['path'] ||= "/"+@deploy.deploy_id+"/"
+            MU.log "Creating IAM user #{@config['path']}/#{@mu_name}"
+            tags = get_tag_params
+            MU::Cloud::AWS.iam.create_user(
+              user_name: @mu_name,
+              path: @config['path'],
+              tags: tags
+            )
+          end
+
+        end
+
+        # Called automatically by {MU::Deploy#createResources}
+        def groom
+          resp = MU::Cloud::AWS.iam.list_user_tags(user_name: @mu_name)
+
+          ext_tags = resp.tags.map { |t| t.to_h }
+          tag_param = get_tag_params(true)
+          tag_param.reject! { |t| ext_tags.include?(t) }
+
+          if tag_param.size > 0
+            MU.log "Updating tags on IAM user #{@mu_name}", MU::NOTICE, details: tag_param
+            MU::Cloud::AWS.iam.tag_user(user_name: @mu_name, tags: tag_param)
+          end
+          # Note: We don't delete tags, because we often share user accounts
+          # managed outside of Mu. We have no way of know what tags might come
+          # from other things, so we err on the side of caution instead of 
+          # deleting stuff.
+
+          if @config['create_console_password']
+            begin
+              MU::Cloud::AWS.iam.get_login_profile(user_name: @mu_name)
+            rescue Aws::IAM::Errors::NoSuchEntity
+              pw = Password.pronounceable(12..14)
+              retries = 0
+              begin
+                MU::Cloud::AWS.iam.create_login_profile(
+                  user_name: @mu_name,
+                  password: pw
+                )
+                scratchitem = MU::Master.storeScratchPadSecret("AWS Console password for user #{@mu_name}:\n<pre>#{pw}</pre>")
+                MU.log "User #{@mu_name}'s AWS Console password can be retrieved from: https://#{$MU_CFG['public_address']}/scratchpad/#{scratchitem}", MU::SUMMARY
+              rescue Aws::IAM::Errors::PasswordPolicyViolation => e
+                if retries < 1
+                  pw = MU.generateWindowsPassword
+                  retries += 1
+                  sleep 1
+                  retry
+                else
+                  MU.log "Error setting password for #{e.message}", MU::WARN
+                end
+              end
+            end
+          end
+
+          if @config['create_api_keys']
+            resp = MU::Cloud::AWS.iam.list_access_keys(
+              user_name: @mu_name
+            )
+            if resp.access_key_metadata.size == 0
+              resp = MU::Cloud::AWS.iam.create_access_key(
+                user_name: @mu_name
+              )
+              scratchitem = MU::Master.storeScratchPadSecret("AWS Access Key and Secret for user #{@mu_name}:\nKEY: #{resp.access_key.access_key_id}\nSECRET: #{resp.access_key.secret_access_key}")
+              MU.log "User #{@mu_name}'s AWS Key and Secret can be retrieved from: https://#{$MU_CFG['public_address']}/scratchpad/#{scratchitem}", MU::SUMMARY
+            end
+          end
+
+          if @config['iam_policies']
+             @dependencies["role"].each_pair { |rolename, roleobj|
+               roleobj.cloudobj.bindTo("user", @cloud_id)
+             }
+          end
+        end
+
+
+        # Return the metadata for this user cofiguration
+        # @return [Hash]
+        def notify
+          descriptor = MU.structToHash(MU::Cloud::AWS.iam.get_user(user_name: @mu_name).user)
+          descriptor["cloud_id"] = @mu_name
+          descriptor
+        end
+
+        # Remove all users associated with the currently loaded deployment.
+        # @param noop [Boolean]: If true, will only print what would be done
+        # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
+        # @param region [String]: The cloud provider region
+        # @return [void]
+        def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
+
+          # XXX this doesn't belong here; maybe under roles, maybe as its own stupid first-class resource
+          resp = MU::Cloud::AWS.iam.list_policies(
+            path_prefix: "/"+MU.deploy_id+"/"
+          )
+          if resp and resp.policies
+            resp.policies.each { |policy|
+              MU.log "Deleting policy /#{MU.deploy_id}/#{policy.policy_name}"
+              if !noop
+                attachments = MU::Cloud::AWS.iam.list_entities_for_policy(
+                  policy_arn: policy.arn
+                )
+                attachments.policy_users.each { |u|
+                  MU::Cloud::AWS.iam.detach_user_policy(
+                    user_name: u.user_name,
+                    policy_arn: policy.arn
+                  )
+                }
+                attachments.policy_groups.each { |g|
+                  MU::Cloud::AWS.iam.detach_role_policy(
+                    group_name: g.group_name,
+                    policy_arn: policy.arn
+                  )
+                }
+                attachments.policy_roles.each { |r|
+                  MU::Cloud::AWS.iam.detach_role_policy(
+                    role_name: r.role_name,
+                    policy_arn: policy.arn
+                  )
+                }
+                MU::Cloud::AWS.iam.delete_policy(
+                  policy_arn: policy.arn
+                )
+              end
+            }
+          end
+
+          resp = MU::Cloud::AWS.iam.list_users
+
+          # XXX this response includes a tags attribute, but it's always empty,
+          # even when the user is tagged. So we go through the extra call for
+          # each user. Inefficient. Probably Amazon's bug.
+          resp.users.each { |u|
+            tags = MU::Cloud::AWS.iam.list_user_tags(
+              user_name: u.user_name
+            ).tags
+            has_nodelete = false
+            has_ourdeploy = false
+            tags.each { |tag|
+              if tag.key == "MU-ID" and tag.value == MU.deploy_id
+                has_ourdeploy = true
+              elsif tag.key == "MU-NO-DELETE" and tag.value == "true"
+                has_nodelete = true
+              end
+            }
+            if has_ourdeploy and !has_nodelete
+              MU.log "Deleting IAM user #{u.path}#{u.user_name}"
+              if !@noop
+                begin
+                  groups = MU::Cloud::AWS.iam.list_groups_for_user(
+                    user_name: u.user_name
+                  ).groups
+
+                  groups.each { |g|
+                    MU::Cloud::AWS.iam.remove_user_from_group(
+                      user_name: u.user_name,
+                      group_name: g.group_name
+                    )
+                  }
+                  profile = MU::Cloud::AWS.iam.get_login_profile(
+                    user_name: u.user_name
+                  )
+                  MU.log "Deleting IAM login profile for #{u.user_name}"
+                  MU::Cloud::AWS.iam.delete_login_profile(
+                    user_name: u.user_name
+                  )
+                rescue Aws::IAM::Errors::NoSuchEntity
+                end
+                keys = MU::Cloud::AWS.iam.list_access_keys(
+                  user_name: u.user_name
+                )
+                if keys.access_key_metadata.size > 0
+                  keys.access_key_metadata.each { |key|
+                    MU.log "Deleting IAM access key #{key.access_key_id} for #{u.user_name}"
+                    keys = MU::Cloud::AWS.iam.delete_access_key(
+                      user_name: u.user_name,
+                      access_key_id: key.access_key_id
+                    )
+                  }
+                end
+                MU::Cloud::AWS.iam.delete_user(user_name: u.user_name)
+              end
+            end
+          }
+
+#          MU.log "CLEANUP CALLED ON AWS::USER", MU::WARN, details: resp
+        end
+
+        # Canonical Amazon Resource Number for this resource
+        # @return [String]
+        def arn
+          cloud_desc.arn
+        end
+
+        # Locate an existing user group.
+        # @param cloud_id [String]: The cloud provider's identifier for this resource.
+        # @param region [String]: The cloud provider region.
+        # @param flags [Hash]: Optional flags
+        # @return [OpenStruct]: The cloud provider's complete descriptions of matching user group.
+        def self.find(cloud_id: nil, region: MU.curRegion, flags: {})
+          found = nil
+
+          begin
+            resp = MU::Cloud::AWS.iam.get_user(user_name: cloud_id)
+            if resp and resp.user
+              found ||= {}
+              found[cloud_id] = resp.user
+            end
+          rescue ::Aws::IAM::Errors::NoSuchEntity
+          end
+
+          found
+        end
+
+        # Cloud-specific configuration properties.
+        # @param config [MU::Config]: The calling MU::Config object
+        # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
+        def self.schema(config)
+          toplevel_required = []
+          schema = {
+            "name" => {
+              "type" => "string",
+              "description" => "A plain IAM user. If the user already exists, we will operate on that existing user. Otherwise, we will attempt to create a new user."
+            },
+            "path" => {
+              "type" => "string",
+              "description" => "AWS IAM users can be namespaced with a path (ex: +/organization/unit/user+). If not specified, and if we do not see a matching existing user under +/+ with +use_if_exists+ set, we will prepend the deploy identifier to the path of users we create. Ex: +/IAMTESTS-DEV-2018112910-GR/myuser+.",
+              "pattern" => '^\/(?:[^\/]+(?:\/[^\/]+)*\/$)?'
+            },
+            "tags" => MU::Config.tags_primitive,
+            "optional_tags" => MU::Config.optional_tags_primitive,
+            "unique_name" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "Instead of creating/updating a user account with
+ the exact name specified in the 'name' field, generate a unique-per-deploy Mu-
+style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
+            },
+            "create_console_password" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "Generate a password for this user, for use logging into the AWS Console. It will be shared via Scratchpad for one-time retrieval."
+            },
+            "iam_policies" => {
+              "type" => "array",
+              "items" => {
+                "description" => "A key (name) with a value that is an Amazon-compatible policy document. See https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_examples.html for example policies.",
+                "type" => "object"
+              }
+            }
+          }
+          [toplevel_required, schema]
+        end
+
+        # Cloud-specific pre-processing of {MU::Config::BasketofKittens::users}, bare and unvalidated.
+        # @param user [Hash]: The resource to process and validate
+        # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+        # @return [Boolean]: True if validation succeeded, False otherwise
+        def self.validateConfig(user, configurator)
+          ok = true
+
+          if user['iam_policies'] and user['iam_policies'].size > 0
+            roledesc = {
+              "name" => user["name"]+"role",
+              "bare_policies" => true,
+              "iam_policies" => user['iam_policies'].dup
+            }
+            configurator.insertKitten(roledesc, "roles")
+            user["dependencies"] ||= []
+            user["dependencies"] << {
+              "type" => "role",
+              "name" => user["name"]+"role"
+            }
+          end
+
+          if user['groups']
+            user['groups'].each { |group|
+              need_dependency = false
+              if configurator.haveLitterMate?(group, "groups")
+                need_dependency = true
+              else
+                found = MU::Cloud::AWS::Group.find(cloud_id: group)
+                if found.nil? or found.empty? or (configurator.updating and
+                   found.values.first.group.path == "/"+configurator.updating+"/")
+                  groupdesc = {
+                    "name" => group
+                  }
+                  configurator.insertKitten(groupdesc, "groups")
+                  need_dependency = true
+                end
+              end
+
+              if need_dependency
+                user["dependencies"] ||= []
+                user["dependencies"] << {
+                  "type" => "group",
+                  "name" => group
+                }
+              end
+            }
+          end
+
+          ok
+        end
+
+        private
+
+        def get_tag_params(strip_std = false)
+          @config['tags'] ||= []
+
+          if !strip_std
+            MU::MommaCat.listStandardTags.each_pair { |key, value|
+              @config['tags'] << { "key" => key, "value" => value }
+            }
+
+            if @config['optional_tags']
+              MU::MommaCat.listOptionalTags.each { |key, value|
+                @config['tags'] << { "key" => key, "value" => value }
+              }
+            end
+          end
+
+          if @config['preserve_on_cleanup']
+            @config['tags'] << { "key" => "MU-NO-DELETE", "value" => "true" }
+          end
+
+          @config['tags'].map { |t|
+            { :key => t["key"], :value => t["value"] }
+          }
+        end
+
+      end
+    end
+  end
+end

data/modules/mu/clouds/aws/userdata/README.md

@@ -0,0 +1,4 @@
+
+Baseline CloudInit userdata scripts for MU nodes to self-configure.
+
+See also: https://help.ubuntu.com/community/CloudInit

data/modules/mu/clouds/aws/userdata/linux.erb

@@ -0,0 +1,179 @@
+#!/bin/sh
+# Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+updates_run=0
+need_reboot=0
+instance_id="`curl http://169.254.169.254/latest/meta-data/instance-id`"
+region="`curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed 's/[a-z]$//'`"
+
+# cleanse inherited ephemeral devices that don't actually exist
+for d in r s t u ;do
+	if [ ! -f "/dev/xvd$d" ];then
+		sed -Ein "s'^(/dev/xvd$d)'#\\1'" /etc/fstab
+	fi
+done
+
+if [ -f /etc/debian_version ];then
+	if ! grep '^/bin/sh /var/lib/cloud/instance/user-data.txt$' /etc/rc.local > /dev/null;then
+		echo "/bin/sh /var/lib/cloud/instance/user-data.txt" >> /etc/rc.local
+	fi
+	apt-get update -y
+	if [ ! -f /usr/bin/pip ] ;then /usr/bin/apt-get --fix-missing -y install python-pip;fi
+	if [ ! -f /usr/bin/curl ] ;then /usr/bin/apt-get --fix-missing -y install curl;fi
+	AWSCLI=/usr/local/bin/aws
+<% if !$mu.skipApplyUpdates %>
+	if [ ! -f /.mu-installer-ran-updates ];then
+		service ssh stop
+		apt-get --fix-missing -y upgrade 
+		if [ $? -eq 0 ]
+		then
+		  echo "Successfully updated packages"
+		  updates_run=1
+		else
+		  echo "FAILED PACKAGE UPDATE" >&2
+		fi
+		# Proceed regardless
+		touch /.mu-installer-ran-updates
+
+		# XXX this logic works on Ubuntu, is it Debian-friendly?
+		latest_kernel="`ls -1 /boot/vmlinuz-* | sed -r 's/^\/boot\/vmlinuz-//' | tail -1`"
+		running_kernel="`uname -r`"
+		if [ "$running_kernel" != "$latest_kernel" -a "$latest_kernel" != "" ];then
+			need_reboot=1
+		else
+			service ssh start
+		fi
+	fi
+<% end %>
+elif [ -x /usr/bin/yum ];then
+	version=`/bin/rpm -qa \*-release | grep -Ei "redhat|centos" | cut -d"-" -f3`
+	if [ -z "$version" ];then
+		amazon_version=`/bin/rpm -qa \*-release | grep -Ei "system-release"| cut -d"-" -f3 | cut -d"." -f1`
+		if [ "$amazon_version" == "2014" ] || [ "$amazon_version" == "2015" ] || [ "$amazon_version" == "2016" ];then
+			version=6
+		fi
+	fi
+	if [ $version -eq 7 ];then
+		userdata_dir="/var/lib/cloud/instances/$instance_id"
+	else
+		userdata_dir="/var/lib/cloud/instance"
+	fi
+	if ! grep "^/bin/sh $userdata_dir/user-data.txt$" /etc/rc.d/rc.local > /dev/null;then
+    cat /etc/rc.d/rc.local | grep -v '^/bin/sh /var/lib/cloud/instances/' >> /tmp/rc.local.$$
+		echo "/bin/sh $userdata_dir/user-data.txt" >> /tmp/rc.local.$$
+    mv /tmp/rc.local.$$ /etc/rc.d/rc.local
+	fi
+
+  sed -i 's/^Defaults.*requiretty$/Defaults   !requiretty/' /etc/sudoers
+
+	if [ $version == 7 ];then
+		chmod 755 /etc/rc.d/rc.local
+    systemctl reset-failed sshd.service
+	fi
+	if [ ! -f /usr/bin/curl ] ;then /usr/bin/yum -y install curl;fi
+	# Ugh, rando EPEL mirror
+	if [ ! -f /etc/yum.repos.d/epel.repo ];then
+		/bin/rpm -ivh http://mirror.metrocast.net/fedora/epel/epel-release-latest-$version.noarch.rpm
+	fi
+<% if !$mu.skipApplyUpdates %>
+	if [ ! -f /.mu-installer-ran-updates ];then
+		service sshd stop
+		kernel_update=`yum list updates | grep kernel`
+		yum -y update
+		if [ $? -eq 0 ]
+		then
+		  echo "Successfully updated packages"
+		  updates_run=1
+		else
+		  echo "FAILED PACKAGE UPDATE" >&2
+		fi
+		# Proceed regardless
+		touch /.mu-installer-ran-updates
+		if [ -n "$kernel_update" ]; then
+			need_reboot=1
+		else
+			service sshd start
+		fi
+	fi
+<% end %>
+fi
+
+AWSCLI='command -v aws'
+PIP='command -v pip'
+
+if ! $AWSCLI > /dev/null;then
+  if ! $PIP > /dev/null;then
+    amazon=`rpm -qa \*yum | grep amzn`
+    if [ $amazon ];then
+      yum -y install python27-pip
+    else
+      /usr/bin/curl -o get-pip.py https://bootstrap.pypa.io/get-pip.py
+      /usr/bin/python get-pip.py
+    fi
+    PIP='command -v pip'
+  fi
+  `$PIP` install awscli
+fi
+
+# TO DO - This should use "command" instead
+if [ -f /usr/bin/aws ];then
+  AWSCLI=/usr/bin/aws
+elif [ -f /usr/local/bin/aws ];then
+  AWSCLI=/usr/local/bin/aws
+else
+  AWSCLI=/bin/aws
+fi
+
+umask 0077
+
+# Install Chef now, because why not?
+if [ ! -f /opt/chef/embedded/bin/ruby ];then
+	curl https://www.chef.io/chef/install.sh > chef-install.sh
+	set +e
+	# We may run afoul of a synchronous bootstrap process doing the same thing. So
+	# wait until we've managed to run successfully.
+	while ! sh chef-install.sh -v <%= MU.chefVersion %>;do
+		sleep 10
+	done
+	touch /opt/mu_installed_chef
+	set -e
+fi
+
+<% if !$mu.skipApplyUpdates %>
+if [ "$need_reboot" == "1" ];then
+	shutdown -r now "Applying new kernel"
+fi
+<% end %>
+
+$AWSCLI --region="$region" s3 cp s3://<%= MU.adminBucketName %>/<%= $mu.muID %>-secret .
+
+echo '
+require "openssl"
+require "base64"
+key = OpenSSL::PKey::RSA.new(Base64.urlsafe_decode64("<%= $mu.deployKey %>"))
+print Base64.urlsafe_encode64(key.public_encrypt(File.read("<%= $mu.muID %>-secret")))
+' > encrypt_deploy_secret.rb
+
+deploykey="<%= $mu.deployKey %>"
+instance_id="`curl http://169.254.169.254/latest/meta-data/instance-id`"
+
+# Make double-sure sshd is actually up
+if ! ( netstat -na | grep LISTEN | grep ':22 ' );then
+  service sshd start
+fi
+
+/usr/bin/curl -k --data mu_id="<%= $mu.muID %>" --data mu_resource_name="<%= $mu.resourceName %>" --data mu_resource_type="<%= $mu.resourceType %>" --data mu_instance_id="$instance_id" --data mu_bootstrap="1" --data mu_user="<%= $mu.muUser %>" --data mu_deploy_secret="`/opt/chef/embedded/bin/ruby encrypt_deploy_secret.rb`" https://<%= $mu.publicIP %>:2260/
+/bin/rm -f <%= $mu.muID %>-secret mu_deploy_key.pub chef-install.sh encrypt_deploy_secret.rb
+touch /.mu_userdata_complete