aws-ec2 1.0.0 → 1.1.0

data/lib/aws_ec2/scripts/auto_terminate.sh

@@ -1,95 +1,14 @@
 #!/bin/bash -exu
-# The shebang line is here in case there's is currently an empty user-data script.
-# It wont hurt if already there.
-##################
-# auto_terminate.sh script
-# When creating an AMI, a aws ec2 create-image command is added to the end of
-# the user-data script. Creating AMIs prevent the script going any further.
-#
-# To get around this the this is script is added before that happens.
-#
-# https://stackoverflow.com/questions/27920806/how-to-avoid-heredoc-expanding-variables
-cat >/root/terminate-myself.sh << 'EOL'
-#!/bin/bash -exu
-
-# install jq dependencies
-function install_jq() {
-  if ! type jq > /dev/null ; then
-    wget "https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64"
-    mv jq-linux64 /usr/local/bin/jq
-    chmod a+x /usr/local/bin/jq
-  fi
-}
-
-function configure_aws_cli() {
-  local home_dir=$1
-  # Configure aws cli in case it is not yet configured
-  mkdir -p $home_dir/.aws
-  if [ ! -f $home_dir/.aws/config ]; then
-    EC2_AVAIL_ZONE=`curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone`
-    EC2_REGION="`echo \"$EC2_AVAIL_ZONE\" | sed -e 's:\([0-9][0-9]*\)[a-z]*\$:\\1:'`"
-    cat >$home_dir/.aws/config <<CONFIGURE_AWS_CLI
-[default]
-region = $EC2_REGION
-output = json
-CONFIGURE_AWS_CLI
-  fi
-}
-
-function terminate_instance() {
-  aws ec2 terminate-instances --instance-ids $INSTANCE_ID
-}
-
-# on-demand instance example:
-# $ aws ec2 describe-instances --instance-ids i-09482b1a6e330fbf7 | jq '.Reservations[].Instances[].SpotInstanceRequestId'
-# null
-# spot instance example:
-# $ aws ec2 describe-instances --instance-ids i-08318bb7f33c216bd | jq '.Reservations[].Instances[].SpotInstanceRequestId'
-# "sir-dzci5wsh"
-function cancel_spot_request() {
-  aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $SPOT_INSTANCE_REQUEST_ID
-}
-
-###
-# program starts here
-###
-export PATH=/usr/local/bin:$PATH
-install_jq
-configure_aws_cli /root
-
-AMI_NAME=$1
-if [ $AMI_NAME != "NO-WAIT" ]; then
-  # wait for the ami to be successfully created before terminating the instance
-  # https://docs.aws.amazon.com/cli/latest/reference/ec2/wait/image-available.html
-  # It will poll every 15 seconds until a successful state has been reached. This will exit with a return code of 255 after 40 failed checks.
-  # so it'll wait for 10 mins max
-  aws ec2 wait image-available --filters "Name=name,Values=$AMI_NAME" --owners self
-fi
-
-
-INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
-SPOT_INSTANCE_REQUEST_ID=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].SpotInstanceRequestId')
-
-# Remove this script so it is only allowed to be ran once ever
-# Or else whenenver we launch the AMI, it will kill itself
-rm -f /root/terminate-myself.sh
-grep -v terminate-myself /etc/rc.d/rc.local > /etc/rc.d/rc.local.tmp
-mv /etc/rc.d/rc.local.tmp /etc/rc.d/rc.local
 
-if [ -n "$SPOT_INSTANCE_REQUEST_ID" ]; then
-  cancel_spot_request
+if [ $# -eq 0 ]; then
+  command=$(basename "$0")
+  echo "Usage: $command WHEN"
+  echo "Examples:"
+  echo "  $command now"
+  echo "  $command later"
+  exit 1
 fi
-terminate_instance
-EOL
-chmod a+x /root/terminate-myself.sh
+WHEN=$1 # now or later
 
-<% if @options[:auto_terminate] %>
-<% if @options[:ami_name] %>
-# schedule termination upon reboot
-chmod +x /etc/rc.d/rc.local
-echo "/root/terminate-myself.sh <%= @options[:ami_name] %> >> /var/log/terminate-myself.log 2>&1" >> /etc/rc.d/rc.local
-<% else %>
-# terminate immediately
-/root/terminate-myself.sh NO-WAIT >> /var/log/terminate-myself.log 2>&1
-<% end %>
-<% end %>
+source /opt/aws-ec2/auto_terminate/functions.sh
+terminate "$WHEN"

data/lib/aws_ec2/scripts/auto_terminate/after_timeout.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -exu
+
+# This is another copy the /opt/aws-ec2/auto_terminate.sh script because the
+# /opt/aws-ec2/auto_terminate.sh also gets removed when it gets called.  Specifically,
+# it gets gets removed when "terminate after_ami" is called.
+
+# There's this extra script that terminates after a timeout because sometimes:
+#   1. the script stalls: IE: aws ec2 wait image-available and a custom wait_ami
+#      does this.
+#   2. the user_data script breaks and stops before finishing, never reaching
+#      the terminate_later or terminate_now functions.
+#
+
+source /opt/aws-ec2/auto_terminate/functions.sh
+# remove itself since at jobs survive reboots and if the ami gets created
+# successfully we do not want this to be captured as part of the ami
+rm -f /opt/aws-ec2/auto_terminate/after_timeout.sh
+terminate now # hard code to now since only gets called via an at job

data/lib/aws_ec2/scripts/auto_terminate/functions.sh

@@ -0,0 +1,128 @@
+#!/bin/bash -eux
+
+# Key is that instance will not be terminated if source image is the same as the
+# original image id.
+function terminate_instance() {
+  local SOURCE_AMI_ID
+  local AMI_ID
+
+  SOURCE_AMI_ID=$(curl -s http://169.254.169.254/latest/meta-data/ami-id)
+  AMI_ID=$(cat /opt/aws-ec2/data/ami-id.txt | jq -r '.ImageId')
+  if [ "$SOURCE_AMI_ID" = "$AMI_ID" ]; then
+    echo "The source ami and ami_id are the same: $AMI_ID"
+    echo "WILL NOT TERMINATE!"
+    return
+  fi
+
+  INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
+  SPOT_INSTANCE_REQUEST_ID=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" | jq -r '.Reservations[].Instances[].SpotInstanceRequestId')
+
+  if [ -n "$SPOT_INSTANCE_REQUEST_ID" ]; then
+    cancel_spot_request
+  fi
+  aws ec2 terminate-instances --instance-ids "$INSTANCE_ID"
+}
+
+# on-demand instance example:
+# $ aws ec2 describe-instances --instance-ids i-09482b1a6e330fbf7 | jq '.Reservations[].Instances[].SpotInstanceRequestId'
+# null
+# spot instance example:
+# $ aws ec2 describe-instances --instance-ids i-08318bb7f33c216bd | jq '.Reservations[].Instances[].SpotInstanceRequestId'
+# "sir-dzci5wsh"
+function cancel_spot_request() {
+  aws ec2 cancel-spot-instance-requests --spot-instance-request-ids "$SPOT_INSTANCE_REQUEST_ID"
+}
+
+# When image doesnt exist at all, an empty string is returned.
+function ami_state() {
+  local ami_id
+  ami_id=$1
+  aws ec2 describe-images --image-ids "$ami_id" --owners self | jq -r '.Images[].State'
+}
+
+function wait_for_ami() {
+  local name
+  name=$1
+
+  local x
+  local state
+  x=0
+
+  state=$(ami_state "$name")
+  while [ "$x" -lt 10 ] && [ "$state" != "available" ]; do
+    x=$((x+1))
+
+    state=$(ami_state "$name")
+    echo "state $state"
+    echo "sleeping for 60 seconds... times out at 10 minutes total"
+
+    type sleep
+    sleep 60
+  done
+
+  echo "final state $state"
+}
+
+function terminate() {
+  local when
+  when=$1
+
+  export PATH=/usr/local/bin:$PATH # for jq
+
+  if [ "$when" == "later" ]; then
+    terminate_later
+  elif [ "$when" == "after_ami" ]; then
+    terminate_after_ami
+  elif [ "$when" == "after_timeout" ]; then
+    terminate_after_timeout
+  else
+    terminate_now
+  fi
+}
+
+function terminate_later() {
+  schedule_termination
+}
+
+# This gets set up at the very beginning of the user_data script.  This ensures
+# that after a 45 minute timeout the instance will get cleaned up and terminated.
+function terminate_after_timeout() {
+  rm -f /opt/aws-ec2/data/ami-id.txt # rm file possible stale file from previous ami
+
+  # Remove all old at jobs.
+  # Assume all at job are previous after_timeout.sh job from previous AMI.
+  # Note: Each new at job increments the id by 1.  So each AMI will have a different
+  # at job number.
+  for i in $(atq | awk '{print $1}') ; do
+    at -r $i
+  done
+  echo "/opt/aws-ec2/auto_terminate/after_timeout.sh now" | at now + 45 minutes
+}
+
+function terminate_after_ami() {
+  local AMI_ID
+
+  unschedule_termination
+  AMI_ID=$(cat /opt/aws-ec2/data/ami-id.txt | jq -r '.ImageId')
+  if [ -n "$AMI_ID" ]; then
+    # wait for the ami to be successfully created before terminating the instance
+    # https://docs.aws.amazon.com/cli/latest/reference/ec2/wait/image-available.html
+    # It will poll every 15 seconds until a successful state has been reached. This will exit with a return code of 255 after 40 failed checks.
+    # so it'll wait for 10 mins max
+    # aws ec2 wait image-available --image-ids "$AMI_ID" --owners self
+
+    # For some reason aws ec2 wait image-available didnt work for amazonlinux2
+    # so using a custom version
+    wait_for_ami "$AMI_ID"
+  fi
+
+  terminate_instance
+}
+
+function terminate_now() {
+  terminate_instance
+}
+
+source "/opt/aws-ec2/shared/functions.sh"
+os=$(os_name)
+source "/opt/aws-ec2/auto_terminate/functions/${os}.sh"

data/lib/aws_ec2/scripts/auto_terminate/functions/amazonlinux2.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -eux
+function schedule_termination() {
+  chmod +x /etc/rc.d/rc.local
+  echo "/opt/aws-ec2/auto_terminate.sh after_ami >> /var/log/auto-terminate.log 2>&1" >> /etc/rc.d/rc.local
+}
+
+function unschedule_termination() {
+  grep -v auto_terminate.sh /etc/rc.d/rc.local > /etc/rc.d/rc.local.tmp
+  mv /etc/rc.d/rc.local.tmp /etc/rc.d/rc.local
+}

data/lib/aws_ec2/scripts/auto_terminate/functions/ubuntu.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -eux
+function schedule_termination() {
+  chmod +x /etc/rc.local
+  sed -i 's/exit 0//' /etc/rc.local
+  echo "/opt/aws-ec2/auto_terminate.sh after_ami >> /var/log/auto-terminate.log 2>&1" >> /etc/rc.local
+}
+
+function unschedule_termination() {
+  grep -v terminate-myself /etc/rc.local > /etc/rc.local.tmp
+  mv /etc/rc.local{.tmp,}
+}

data/lib/aws_ec2/scripts/auto_terminate/setup.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -eux
+function install_jq() {
+  if ! type jq > /dev/null ; then
+    wget "https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64"
+    mv jq-linux64 /usr/local/bin/jq
+    chmod a+x /usr/local/bin/jq
+  fi
+}
+
+function configure_aws_cli() {
+  local home_dir
+  home_dir=${1:-/root} # default to /root
+  # Configure aws cli in case it is not yet configured
+  mkdir -p "$home_dir/.aws"
+  if [ ! -f "$home_dir/.aws/config" ]; then
+    EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
+    EC2_REGION=${EC2_AVAIL_ZONE::-1}
+    cat >"$home_dir/.aws/config" <<CONFIGURE_AWS_CLI
+[default]
+region = $EC2_REGION
+output = json
+CONFIGURE_AWS_CLI
+  fi
+}
+
+function setup() {
+  install_jq
+  configure_aws_cli /root
+}
+
+setup

data/lib/aws_ec2/scripts/cloudwatch.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -eux
+
+if [ $# -eq 0 ]; then
+  command=$(basename "$0")
+  echo "Usage: $command LOG_GROUP_NAME"
+  echo "Examples:"
+  echo "  $command aws-ec2"
+  echo "  $command ec2"
+  exit 1
+fi
+LOG_GROUP_NAME=$1
+
+source "/opt/aws-ec2/shared/functions.sh"
+os=$(os_name)
+if [ "$os" != "amazonlinux2" ]; then
+  echo "Sorry, enable cloudwatch logging with the aws-ec2 tool is only supported for amazonlinux2 currently"
+  exit
+fi
+
+/opt/aws-ec2/cloudwatch/install.sh
+/opt/aws-ec2/cloudwatch/configure.sh "$LOG_GROUP_NAME"
+/opt/aws-ec2/cloudwatch/service.sh

data/lib/aws_ec2/scripts/cloudwatch/configure.sh

@@ -0,0 +1,74 @@
+#!/bin/bash -eux
+
+if [ $# -eq 0 ]; then
+  command=$(basename "$0")
+  echo "Usage: $command LOG_GROUP_NAME"
+  echo "Examples:"
+  echo "  $command aws-ec2"
+  echo "  $command ec2"
+  exit 1
+fi
+LOG_GROUP_NAME=$1
+
+# Inject the CloudWatch Logs configuration file contents
+cat > /etc/awslogs/awslogs.conf <<- EOF
+[general]
+state_file = /var/lib/awslogs/agent-state
+
+[/var/log/dmesg]
+file = /var/log/dmesg
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/dmesg
+
+[/var/log/messages]
+file = /var/log/messages
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/messages
+datetime_format = %b %d %H:%M:%S
+
+[/var/log/cloud-init.log]
+file = /var/log/cloud-init.log
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/cloud-init.log
+datetime_format =
+
+[/var/log/cloud-init-output.log]
+file = /var/log/cloud-init-output.log
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/cloud-init-output.log
+datetime_format =
+
+[/var/log/secure]
+file = /var/log/secure
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/secure
+datetime_format =
+
+[/var/log/audit/audit.log]
+file = /var/log/audit/audit.log
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/audit/audit.log
+datetime_format =
+
+[/var/lib/cloud/instance/user-data.txt]
+file = /var/lib/cloud/instance/user-data.txt
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/lib/cloud/instance/user-data.txt
+datetime_format =
+
+[/var/log/messages]
+file = /var/log/messages
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/messages
+datetime_format =
+
+[/var/log/auto-terminate.log]
+file = /var/log/auto-terminate.log
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/auto-terminate.log
+datetime_format =
+
+EOF
+
+region=$(curl 169.254.169.254/latest/meta-data/placement/availability-zone | sed s'/.$//')
+sed -i -e "s/region = us-east-1/region = $region/g" /etc/awslogs/awscli.conf

data/lib/aws_ec2/scripts/cloudwatch/install.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -eux
+
+# Install awslogs and the jq JSON parser
+yum install -y awslogs jq

data/lib/aws_ec2/scripts/cloudwatch/service.sh

@@ -0,0 +1,19 @@
+cat > /etc/init/awslogs.conf <<- EOL
+#upstart-job
+description "Configure and start CloudWatch Logs agent EC2 instance"
+author "Tung Nguyen"
+start on startup
+
+script
+  exec 2>>/var/log/cloudwatch-logs-start.log
+  set -x
+
+  service awslogs start
+  chkconfig awslogs on
+end script
+EOL
+
+initctl list
+initctl reload-configuration
+initctl start awslogs
+initctl list

data/lib/aws_ec2/scripts/shared/functions.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -eux
+
+function configure_aws_cli() {
+  local home_dir
+  home_dir=${1:-/root} # default to /root
+  # Configure aws cli in case it is not yet configured
+  mkdir -p "$home_dir/.aws"
+  if [ ! -f "$home_dir/.aws/config" ]; then
+    EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
+    EC2_REGION=${EC2_AVAIL_ZONE::-1}
+    cat >"$home_dir/.aws/config" <<CONFIGURE_AWS_CLI
+[default]
+region = $EC2_REGION
+output = json
+CONFIGURE_AWS_CLI
+  fi
+}
+
+# Example OS values at this point:
+#   Ubuntu
+#   Amazon Linux AMI
+function os_name() {
+  # https://askubuntu.com/questions/459402/how-to-know-if-the-running-platform-is-ubuntu-or-centos-with-help-of-a-bash-scri
+  # Method 1 works for amazonlinux2 and ubuntu
+  # Method 3 the complex script, did not work for amazonlinux2
+  local OS
+  OS=$(gawk -F= '/^NAME/{print $2}' /etc/os-release) # text surrounded by double quotes
+  # strip surrounding quotes: https://stackoverflow.com/questions/9733338/shell-script-remove-first-and-last-quote-from-a-variable
+  OS="${OS%\"}"
+  OS="${OS#\"}"
+  # Example OS values at this point:
+  #   Ubuntu
+  #   Amazon Linux AMI
+
+  # normalize values
+  case "$OS" in
+    Ubuntu)
+      echo "ubuntu"
+      ;;
+    *)
+      echo "amazonlinux2" # default
+      ;;
+  esac
+}