aws-ec2 0.3.0 → 0.4.0

data/lib/aws_ec2/create/params.rb

@@ -0,0 +1,157 @@
+require "byebug"
+
+
+class AwsEc2::Create
+  class Params
+    include AwsEc2::TemplateHelper
+
+    def initialize(options)
+      @options = options
+    end
+
+    # deep_symbolize_keys is ran at the very end only.
+    # up until that point we're dealing with String keys.
+    def generate
+      cleanup
+      params = load_profiles(profile_name)
+      decorate_params(params)
+      normalize_launch_template(params).deep_symbolize_keys
+    end
+
+    def decorate_params(params)
+      upsert_name_tag!(params)
+      replace_runtime_options!(params)
+      params
+    end
+
+    # Expose a list of runtime params that are convenient. Try to limit the
+    # number of options from the cli to keep tool simple. Most options can
+    # be easily control through profile files. The runtime options that are
+    # very convenient to have at the CLI are modified here.
+    def replace_runtime_options!(params)
+      params["image_id"] = @options[:source_ami_id] if @options[:source_ami_id]
+      params
+    end
+
+    def cleanup
+      FileUtils.rm_f("/tmp/aws-ec2/user-data.txt")
+    end
+
+    # Adds instance ec2 tag if not already provided
+    def upsert_name_tag!(params)
+      specs = params["tag_specifications"] || []
+
+      # insert an empty spec placeholder if one not found
+      spec = specs.find do |s|
+        s["resource_type"] == "instance"
+      end
+      unless spec
+        spec = {
+            "resource_type" => "instance",
+            "tags" => []
+          }
+        specs << spec
+      end
+      # guaranteed there's a tag_specifications with resource_type instance at this point
+
+      tags = spec["tags"] || []
+
+      unless tags.map { |t| t["key"] }.include?("Name")
+        tags << { "key" => "Name", "value" => @options[:name] }
+      end
+
+      specs = specs.map do |s|
+        # replace the name tag value
+        if s["resource_type"] == "instance"
+          {
+            "resource_type" => "instance",
+            "tags" => tags
+          }
+        else
+          s
+        end
+      end
+
+      params["tag_specifications"] = specs
+      params
+    end
+
+    # Allow adding launch template as a simple string.
+    #
+    # Standard structure:
+    # {
+    #   launch_template: { launch_template_name: "TestLaunchTemplate" },
+    # }
+    #
+    # Simple string:
+    # {
+    #   launch_template: "TestLaunchTemplate",
+    # }
+    #
+    # When launch_template is a simple String it will get transformed to the
+    # standard structure.
+    def normalize_launch_template(params)
+      if params["launch_template"].is_a?(String)
+        launch_template_identifier = params["launch_template"]
+        launch_template = if launch_template_identifier =~ /^lt-/
+            { "launch_template_id" => launch_template_identifier }
+          else
+            { "launch_template_name" => launch_template_identifier }
+          end
+        params["launch_template"] = launch_template
+      end
+      params
+    end
+
+    # Hard coded sensible defaults.
+    # Can be overridden easily with profiles
+    def defaults
+      {
+        max_count: 1,
+        min_count: 1,
+      }
+    end
+
+    def load_profiles(profile_name)
+      return @profile_params if @profile_params
+
+      profile_file = "#{AwsEc2.root}/profiles/#{profile_name}.yml"
+      base_path = File.dirname(profile_file)
+      default_file = "#{base_path}/default.yml"
+
+      params_exit_check!(profile_file, default_file)
+
+      params = File.exist?(profile_file) ?
+                  load_profile(profile_file) :
+                  load_profile(default_file)
+      @profile_params = params
+    end
+
+    def params_exit_check!(profile_file, default_file)
+      return if File.exist?(profile_file) or File.exist?(default_file)
+
+      puts "Unable to find a #{profile_file} or #{default_file} profile file."
+      puts "Please double check."
+      exit # EXIT HERE
+    end
+
+    def load_profile(file)
+      return {} unless File.exist?(file)
+
+      puts "Using profile: #{file}"
+      data = YAML.load(erb_result(file))
+      data ? data : {} # in case the file is empty
+      data.has_key?("run_instances") ? data["run_instances"] : data
+    end
+
+    def profile_name
+      # allow user to specify the path also
+      if @options[:profile] && File.exist?(@options[:profile])
+        profile = File.basename(@options[:profile], '.yml')
+      end
+
+      # conventional profile is the name of the ec2 instance
+      profile || @options[:profile] || @options[:name]
+    end
+  end
+end

data/lib/aws_ec2/dotenv.rb

@@ -0,0 +1,30 @@
+require 'dotenv'
+require 'pathname'
+
+class AwsEc2::Dotenv
+  class << self
+    def load!
+      ::Dotenv.load(*dotenv_files)
+    end
+
+    # dotenv files will load the following files, starting from the bottom. The first value set (or those already defined in the environment) take precedence:
+
+    # - `.env` - The Original®
+    # - `.env.development`, `.env.test`, `.env.production` - Environment-specific settings.
+    # - `.env.local` - Local overrides. This file is loaded for all environments _except_ `test`.
+    # - `.env.development.local`, `.env.test.local`, `.env.production.local` - Local overrides of environment-specific settings.
+    #
+    def dotenv_files
+      [
+        root.join(".env.#{AwsEc2.env}.local"),
+        (root.join(".env.local") unless AwsEc2.env == "test"),
+        root.join(".env.#{AwsEc2.env}"),
+        root.join(".env")
+      ].compact
+    end
+
+    def root
+      AwsEc2.root || Pathname.new(ENV["AWS_EC2_ROOT"] || Dir.pwd)
+    end
+  end
+end

data/lib/aws_ec2/help/ami.md

@@ -0,0 +1,11 @@
+Examples:
+
+  $ aws-ec2 ami myrubyami --profile ruby --noop
+
+The launches an EC2 instance with using the profile running it's user-data script.  An ami creation script is appended to the end of the user-data script. The ami creation script uses the AWS CLI `aws ec2 create-image` command to create an AMI.  It is useful to include to timestamp as a part of the ami name with the date command.
+
+  $ aws-ec2 ami $(date "+ruby-2.5.0_%Y-%m-%d-%H-%M") --profile ruby --noop
+
+Note, it is recommended to use the `set -e` option in your user-data script so that if the script fails, the ami creation script is never reached and instance is left behind so you can debug.
+
+The instance also automatically gets terminated and cleaned up.

data/lib/aws_ec2/help/create.md

@@ -1,11 +1,11 @@
 Examples:
 
-$ aws-ec2 create my-instance
+  $ aws-ec2 create my-instance
 
 If you want to create an ami at the end of of a successful user-data script run you can use the `--ami` option. Example:
 
-$ aws-ec2 create my-instance --ami myname
+  $ aws-ec2 create my-instance --ami myname
 
 To see the snippet of code that gets added to the user-data script you can use the aws-ec2 userdata command.
 
-$ aws-ec2 userdata myscript --ami myname
+  $ aws-ec2 userdata myscript --ami myname

data/lib/aws_ec2/help/user_data.md

@@ -1,13 +1,13 @@
 Displays the generated user data script. Useful for debugging since ERB can be ran on the user-data scripts.
 
-Given a user data script in profiles/user-data/myscript.sh, run:
+Given a user data script in app/user-data/myscript.sh, run:
 
-$ aws-ec2 userdata myscript
+  $ aws-ec2 userdata myscript
 
 You can have an ami creation snippet of code added to the end of the user data script with the `--ami` option.
 
-$ aws-ec2 userdata myscript --ami myname
+  $ aws-ec2 userdata myscript --ami myname
 
 If you want to include a timestamp in the name you can use this:
 
-$ aws-ec2 userdata myscript --ami '`date "+myname_%Y-%m-%d-%H-%M"`'
+  $ aws-ec2 userdata myscript --ami '`date "+myname_%Y-%m-%d-%H-%M"`'

data/lib/aws_ec2/hook.rb

@@ -0,0 +1,32 @@
+require 'yaml'
+
+module AwsEc2
+  class Hook
+    def initialize(options={})
+      @options = options
+    end
+
+    def run(name)
+      return if @options[:noop]
+      return unless hooks[name]
+      command = hooks[name]
+      puts "Running hook #{name}: #{command}"
+      sh(command)
+    end
+
+    def hooks
+      hooks_path = "#{AwsEc2.root}/config/hooks.yml"
+      data = File.exist?(hooks_path) ? YAML.load_file(hooks_path) : {}
+      data ? data : {} # in case the file is empty
+    end
+
+    def sh(command)
+      puts "=> #{command}".colorize(:cyan)
+      system(command)
+    end
+
+    def self.run(name, options={})
+      Hook.new(options).run(name.to_s)
+    end
+  end
+end

data/lib/aws_ec2/script.rb

@@ -0,0 +1,26 @@
+module AwsEc2
+  class Script
+    def initialize(options={})
+      @options = options
+    end
+
+    def auto_terminate
+      @ami_name = @options[:ami_name]
+      load_template("auto_terminate.sh")
+    end
+
+    def create_ami
+      # set some variables for the template
+      @ami_name = @options[:ami_name]
+      @region = `aws configure get region`.strip rescue 'us-east-1'
+      load_template("ami_creation.sh")
+    end
+
+  private
+    def load_template(name)
+      template = IO.read(File.expand_path("../scripts/#{name}", __FILE__))
+      text = ERB.new(template, nil, "-").result(binding)
+      "#" * 60 + "\n#{text}"
+    end
+  end
+end

data/lib/aws_ec2/scripts/ami_creation.sh

@@ -1,27 +1,35 @@
 #!/bin/bash -exu
-
-# Configure aws cli in case it is not yet configured
-mkdir -p /home/ec2-user/.aws
-if [ ! -f /home/ec2-user/.aws/config ]; then
-  cat >/home/ec2-user/.aws/config <<EOL
+# The shebang line is here in case there's is currently an empty user-data script.
+# It wont hurt if already there.
+######################################
+# ami_creation.sh: added to the end of user-data automatically.
+function configure_aws_cli() {
+  local home_dir=$1
+  # Configure aws cli in case it is not yet configured
+  mkdir -p $home_dir/.aws
+  if [ ! -f $home_dir/.aws/config ]; then
+    cat >$home_dir/.aws/config <<EOL
 [default]
-region = <%= region %>
+region = <%= @region %>
 output = json
 EOL
-fi
+  fi
+}
 
-# The aws ec2 create-image command below reboots the instance.
-# So before rebooting the instance, schedule a job to terminate the instance
-# in 20 mins after the machine has rebooted
-cat >~/terminate-myself.sh <<EOL
-#!/bin/bash -exu
-INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
-aws ec2 terminate-instances --instance-ids \$INSTANCE_ID
-EOL
-at now + 1 minutes -f ~/terminate-myself.sh
+configure_aws_cli /home/ec2-user
+configure_aws_cli /root
+
+echo "############################################" >> /var/log/user-data.log
+echo "# Logs above is from the original AMI baking at: $(date)" >> /var/log/user-data.log
+echo "# New logs below" >> /var/log/user-data.log
+echo "############################################" >> /var/log/user-data.log
 
 # Create AMI Bundle
 AMI_NAME="<%= @ami_name %>"
 INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
 REGION=$(aws configure get region)
+# Note this will cause the instance to reboot.  Not using the --no-reboot flag
+# to ensure consistent AMI creation.
+SOURCE_AMI_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/ami-id)
+echo $SOURCE_AMI_ID > /var/log/source-ami-id.txt
 aws ec2 create-image --name $AMI_NAME --instance-id $INSTANCE_ID --region $REGION

data/lib/aws_ec2/scripts/auto_terminate.sh

@@ -0,0 +1,95 @@
+#!/bin/bash -exu
+# The shebang line is here in case there's is currently an empty user-data script.
+# It wont hurt if already there.
+##################
+# auto_terminate.sh script
+# When creating an AMI, a aws ec2 create-image command is added to the end of
+# the user-data script. Creating AMIs prevent the script going any further.
+#
+# To get around this the this is script is added before that happens.
+#
+# https://stackoverflow.com/questions/27920806/how-to-avoid-heredoc-expanding-variables
+cat >/root/terminate-myself.sh << 'EOL'
+#!/bin/bash -exu
+
+# install jq dependencies
+function install_jq() {
+  if ! type jq > /dev/null ; then
+    wget "https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64"
+    mv jq-linux64 /usr/local/bin/jq
+    chmod a+x /usr/local/bin/jq
+  fi
+}
+
+function configure_aws_cli() {
+  local home_dir=$1
+  # Configure aws cli in case it is not yet configured
+  mkdir -p $home_dir/.aws
+  if [ ! -f $home_dir/.aws/config ]; then
+    EC2_AVAIL_ZONE=`curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone`
+    EC2_REGION="`echo \"$EC2_AVAIL_ZONE\" | sed -e 's:\([0-9][0-9]*\)[a-z]*\$:\\1:'`"
+    cat >$home_dir/.aws/config <<CONFIGURE_AWS_CLI
+[default]
+region = $EC2_REGION
+output = json
+CONFIGURE_AWS_CLI
+  fi
+}
+
+function terminate_instance() {
+  aws ec2 terminate-instances --instance-ids $INSTANCE_ID
+}
+
+# on-demand instance example:
+# $ aws ec2 describe-instances --instance-ids i-09482b1a6e330fbf7 | jq '.Reservations[].Instances[].SpotInstanceRequestId'
+# null
+# spot instance example:
+# $ aws ec2 describe-instances --instance-ids i-08318bb7f33c216bd | jq '.Reservations[].Instances[].SpotInstanceRequestId'
+# "sir-dzci5wsh"
+function cancel_spot_request() {
+  aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $SPOT_INSTANCE_REQUEST_ID
+}
+
+###
+# program starts here
+###
+export PATH=/usr/local/bin:$PATH
+install_jq
+configure_aws_cli /root
+
+AMI_NAME=$1
+if [ $AMI_NAME != "NO-WAIT" ]; then
+  # wait for the ami to be successfully created before terminating the instance
+  # https://docs.aws.amazon.com/cli/latest/reference/ec2/wait/image-available.html
+  # It will poll every 15 seconds until a successful state has been reached. This will exit with a return code of 255 after 40 failed checks.
+  # so it'll wait for 10 mins max
+  aws ec2 wait image-available --filters "Name=name,Values=$AMI_NAME" --owners self
+fi
+
+
+INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
+SPOT_INSTANCE_REQUEST_ID=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[].Instances[].SpotInstanceRequestId')
+
+# Remove this script so it is only allowed to be ran once ever
+# Or else whenenver we launch the AMI, it will kill itself
+rm -f /root/terminate-myself.sh
+grep -v terminate-myself /etc/rc.d/rc.local > /etc/rc.d/rc.local.tmp
+mv /etc/rc.d/rc.local.tmp /etc/rc.d/rc.local
+
+if [ -n "$SPOT_INSTANCE_REQUEST_ID" ]; then
+  cancel_spot_request
+fi
+terminate_instance
+EOL
+chmod a+x /root/terminate-myself.sh
+
+<% if @options[:auto_terminate] %>
+<% if @options[:ami_name] %>
+# schedule termination upon reboot
+chmod +x /etc/rc.d/rc.local
+echo "/root/terminate-myself.sh <%= @options[:ami_name] %> >> /var/log/terminate-myself.log 2>&1" >> /etc/rc.d/rc.local
+<% else %>
+# terminate immediately
+/root/terminate-myself.sh NO-WAIT >> /var/log/terminate-myself.log 2>&1
+<% end %>
+<% end %>