aws-ec2 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41c5379df446fa7ebaeb8a4e7243b70c6b8ef3c91afc6a61af1ab42181695c8a
-  data.tar.gz: 8d4558040a99f9496bfdf74b1d43aeb470229f370ae9b14cda174a4096e83f34
+  metadata.gz: 479b890e45d465cd48dad82e9063f5b9ef3074ca05a47461f68ed93ce8b0fd55
+  data.tar.gz: 062a19ef9048e2ff0e284f97b1e1a21a7c44034642ac5639dfef53581f61c23a
 SHA512:
-  metadata.gz: 0ce576737f3e0dc3c225c581d1507e9a9d338e048db588f7776cd2faa45e73d580b1789e43103d5e9d61298016457c4ec011e3eddf387eb14573fea292550d78
-  data.tar.gz: 94969a46aeb9d0e203b141dc7114df127b1bf353405c2d0574996493baaeea4d79c2863773e895d2d47b4228968c0876e94cba8e0606d628a69d893fac045524
+  metadata.gz: 7e0452f43bb90b093a39e889a65b581620532ac7dc8803e70d74c5b2a0dc56da645273bcf7814aee9de09d666cc61f05305ab94132e7d745d0b69994b27d3cf3
+  data.tar.gz: 3731a44ca9cd7597fbee6f36cb62817ce801fb9404c26f05a7c98f8ade9303f487eca36df301e1c27f0e92d405f8e4f44ac10527fcb1ed32137c3454f865e73d

data/.rspec

@@ -1,2 +1,3 @@
 --color
 --format documentation
+--require spec_helper

data/CHANGELOG.md

@@ -3,6 +3,16 @@
 All notable changes to this project will be documented in this file.
 This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [1.1.0]
+- Merge pull request #5 from tongueroo/auto-terminate
+- allow AWS_RDS_CODE_VERSION env variable
+- restructure way scripts work for sanity
+- bash scripts: ami_creation.sh and auto_terminate.sh
+- support only amazonlinux2 and ubuntu
+
+## [1.0.1]
+- update ordering of the info displayed
+
 ## [1.0.0]
 - Merge pull request #1 from tongueroo/cli-template-upgrade
 - Merge pull request #2 from tongueroo/render_me_pretty

data/README.md

@@ -54,10 +54,10 @@ Directory  | Description
 ------------- | -------------
 app/helpers  | Custom helpers methods.  Define them as modules and their methods are made available whenever ERB is available: `profiles`, `app/scripts`, `app/user-data` files, etc. For example, you would define a `module FooHelper` in `app/helpers/foo_helper.rb`.
 app/partials  | Your partials that can to be included in other scripts.  This is used in conjunction with the `partial` helper method. With great power comes great responsibility.  It is recommended to use partials sparely to keep scripts more straightforward.
-app/scripts  | Where you define common scripts that can be used to configure the server. These scripts can be automatically uploaded to an s3 bucket for later downloading in your user-data script by setting the `scripts_s3_bucket` config option.
+app/scripts  | Where you define common scripts that can be used to configure the server. These scripts can be automatically uploaded to an s3 bucket for later downloading in your user-data script by setting the `s3_folder` settings option.
 app/user-data  | Your user-data scripts that are used to bootstrap EC2 instance.
-config/[AWS_EC2_ENV].yml  | The config file where you set configs that you want available in your templating logic.  Examples are: `config/development.yml` and `config/production.yml`. You access the config variables with the `<%= config["var"] %>` helper.
 app/user-data/layouts  | user-data scripts support layouts. You user-data layouts go in here.
+config/[AWS_EC2_ENV].yml  | The config file where you set configs that you want available in your templating logic.  Examples are: `config/development.yml` and `config/production.yml`. You access the config variables with the `<%= config["var"] %>` helper.
 profiles  | Your profile files.  These files mainly contain parameters that are passed to the aws-sdk run_instances API method.
 tmp  | Where the generated scripts get compiled to. You can manually invoke the compilation via `aws-ec2 compile` to inspect what is generated.  This is automatically done as part of the `aws-ec2` create command.
 
@@ -71,8 +71,9 @@ user_data | Allows you to embed a generated user_data script.  More details on t
 config | Access to the variables set in config/[AWS_EC2_ENV].yml.  Examples are `config/development.yml` and `config/production.yml`.
 latest_ami | Returns an AMI id by searching the AMI name pattern and sorting in reverse order.  Example: `latest_ami("ruby-2.5.0_*")` would return the latest ruby AMIs are named with timestamps at the end like so: `ruby-2.5.0_2018-01-30-05-36-02` and `ruby-2.5.0_2018-01-29-05-36-02`.
 search_ami | Returns a collection of AMI image objects based on a search pattern. The query searches on the AMI name.
+extract_scripts | Use this in your bash script to extract the `app/scripts` files that get uploaded to s3.
 
-For a full list of all the template helpers check out: [aws_ec2/template_helper](lib/aws_ec2/template_helper).
+For a full list of all the template helpers check out: [lib/aws_ec2/template/helper](lib/aws_ec2/template/helper).
 
 You can also define custom helpers in the `app/helpers` folder as ruby modules with the naming convention `*_helper.rb`.  For example, you would define a `module FooHelper` in `app/helpers/foo_helper.rb`.  Custom helpers are first-class citizens and have access to the same variables, methods, and scope as built-in helpers.
 
@@ -152,9 +153,6 @@ subnets:
   - subnet-789
 security_group_ids:
   - sg-123
-scripts_s3_bucket: mybucket # enables s3 uploading of generated app/scripts
-# compile_clean: true # uncomment to automatically remove the
-                      # compiled scripts in tmp after aws-ec2 create
 ```
 
 The variables are accessed via the `config` helper method. Here's a filtered example where it shows the relevant part of a profile: `profiles/default.yml`:
@@ -167,17 +165,23 @@ subnet_id: <%= config["subnets"].shuffle %>
 ...
 ```
 
-#### Config Options
-
-There are some config options that change the behavior of the ec2-aws:
-
-Option | Description
---- | ---
-scripts_s3_bucket | Set this to the bucket name where you want the generated scripts in app/scripts and app/user-data to be uploaded to.  The upload sync happens right before the internal to run_instances call that launches the instance.  If you need more custom logic, you can use the `before_run_instances` hook, covered in the Hooks section.
-
 ### Settings
 
-A `config/settings.yml` file controls the internal behavior of aws-ec2. It is different from config files which are meant for user defined varibles.  Settings variables are for internal use.
+A `config/settings.yml` file controls the internal behavior of aws-ec2. It is different from config files which are meant for user defined varibles.  Settings variables are for internal use.  Example:
+
+```yaml
+development:
+  # By setting s3_folder, aws-ec2 will automatically tarball and upload your scripts
+  # to set. You then can then use the extract_scripts helper method to download
+  # the scripts onto the server.
+  s3_folder: boltops-infra-stag/ec2
+  # compile_clean: true # uncomment to clean at the end of a compile
+  # extract_scripts:
+  #   to: "/opt"
+  #   as: "ec2-user"
+
+production:
+```
 
 ### Hooks
 

data/docs/example/app/user-data/bootstrap.sh

@@ -1,18 +1,6 @@
 #!/bin/bash -exu
 
-exec > >(tee "/var/log/user-data.log" | logger -t user-data -s 2>/dev/console) 2>&1
 export HOME=/root # user-data env runs in weird shell where user is root but HOME is not set
-<% pubkey_path = "#{ENV['HOME']}/.ssh/id_rsa.pub" -%>
-<% if File.exist?(pubkey_path) -%>
-<% pubkey = IO.read(pubkey_path).strip -%>
-# Automatically add user's public key
-echo <%= pubkey %> >> ~/.ssh/authorized_keys
-echo <%= pubkey %> >> /home/ec2-user/.ssh/authorized_keys
-chown ec2-user:ec2-user /home/ec2-user/.ssh/authorized_keys
-<% else %>
-# WARN: unable to find a ~/.ssh/id_rsa.pub locally on your machine.  user: <%= ENV['USER'] %>
-# Unable to automatically add the public key
-<% end -%>
 
 sudo yum install -y postgresql
 

data/lib/aws_ec2/cli.rb

@@ -3,19 +3,24 @@ module AwsEc2
     class_option :noop, type: :boolean
     class_option :profile, desc: "profile name to use"
 
+    common_options = Proc.new do
+      option :auto_terminate, type: :boolean, default: false, desc: "automatically terminate the instance at the end of user-data"
+      option :cloudwatch, type: :boolean, default: false, desc: "enable cloudwatch logging, only supported for amazonlinux"
+    end
+
     desc "create NAME", "create ec2 instance"
     long_desc Help.text(:create)
     option :ami_name, desc: "when specified, an ami creation script is appended to the user-data script"
-    option :auto_terminate, type: :boolean, default: false, desc: "automatically terminate the instance at the end of user-data"
-    option :source_ami, desc: "override the source image_id in profile"
     option :randomize, type: :boolean, desc: "append random characters to end of name"
+    option :source_ami, desc: "override the source image_id in profile"
+    common_options.call
     def create(name)
       Create.new(options.merge(name: name)).run
     end
 
     desc "ami NAME", "launches instance and uses it create AMI"
     long_desc Help.text(:ami)
-    option :auto_terminate, type: :boolean, default: true, desc: "automatically terminate the instance at the end of user-data"
+    common_options.call
     def ami(name)
       Ami.new(options.merge(name: name)).run
     end

data/lib/aws_ec2/core.rb

@@ -1,4 +1,5 @@
 require 'pathname'
+require 'yaml'
 
 module AwsEc2
   module Core
@@ -11,10 +12,6 @@ module AwsEc2
       Setting.new.data
     end
 
-    def env
-      ENV['AWS_EC2_ENV'] || 'development'
-    end
-
     def root
       path = ENV['AWS_EC2_ROOT'] || '.'
       Pathname.new(path)
@@ -26,5 +23,29 @@ module AwsEc2
         exit
       end
     end
+
+    @@env = nil
+    def env
+      return @@env if @@env
+      env = env_from_profile(ENV['AWS_PROFILE']) || 'development'
+      env = ENV['AWS_EC2_ENV'] if ENV['AWS_EC2_ENV'] # highest precedence
+      @@env = env
+    end
+
+    private
+    # Do not use the Setting class to load the profile because it can cause an
+    # infinite loop then if we decide to use AwsEc2.env from within settings class.
+    def env_from_profile(aws_profile)
+      settings_path = "#{AwsEc2.root}/config/settings.yml"
+      return unless File.exist?(settings_path)
+
+      data = YAML.load_file(settings_path)
+      env = data.find do |_env, setting|
+        setting ||= {}
+        profiles = setting['aws_profiles']
+        profiles && profiles.include?(aws_profile)
+      end
+      env.first if env
+    end
   end
 end

data/lib/aws_ec2/create.rb

@@ -12,18 +12,19 @@ module AwsEc2
     def run
       Profile.new(@options).check!
 
-      puts "Creating EC2 instance #{@name}..."
-      display_info
-
       Hook.run(:before_run_instances, @options)
       sync_scripts_to_s3
 
+      puts "Creating EC2 instance #{@name.colorize(:green)}"
+      display_ec2_info
       if @options[:noop]
         puts "NOOP mode enabled. EC2 instance not created."
         return
       end
-      run_instances(params)
-      puts "EC2 instance #{@name} created! 🎉"
+      resp = run_instances(params)
+      instance_id = resp.instances.first.instance_id
+      display_spot_info(instance_id)
+      puts "EC2 instance #{@name} created: #{instance_id} 🎉"
       puts "Visit https://console.aws.amazon.com/ec2/home to check on the status"
     end
 
@@ -33,10 +34,11 @@ module AwsEc2
       handle_ec2_service_error!(e)
     end
 
-    # Configured by config/[AWS_EC2_ENV].yml.
-    # Example: config/development.yml:
+    # Configured by config/settings.yml.
+    # Example: config/settings.yml:
     #
-    #   scripts_s3_bucket: my-bucket
+    #   development:
+    #     s3_folder: my-bucket/folder
     def sync_scripts_to_s3
       if AwsEc2.settings["s3_folder"]
         Script::Upload.new(@options).run
@@ -48,7 +50,15 @@ module AwsEc2
       @params ||= Params.new(@options).generate
     end
 
-    def display_info
+    def display_spot_info(instance_id)
+      resp = ec2.describe_instances(instance_ids: [instance_id])
+      spot_id = resp.reservations.first.instances.first.spot_instance_request_id
+      return unless spot_id
+
+      puts "Spot instance request id: #{spot_id}"
+    end
+
+    def display_ec2_info
       puts "Using the following parameters:"
       pretty_display(params)
 

data/lib/aws_ec2/help/upload.md

@@ -2,8 +2,9 @@ Examples:
 
   $ aws-ec2 upload
 
-Compiles the app/scripts and app/user-data files to the tmp folder. Then uploads the files to an s3 bucket that is configured in config/[AWS_EC2_ENV].yml.  Example scripts_s3_bucket config:
+Compiles the app/scripts and app/user-data files to the tmp folder. Then uploads the files to an s3 bucket that is configured in config/settings.yml.  Example s3_folder setting:
 
 ```yaml
-scripts_s3_bucket: my-bucket # enables auto sync to s3
+development:
+  s3_folder: my-bucket/folder # enables auto sync to s3
 ```

data/lib/aws_ec2/profile.rb

@@ -25,7 +25,16 @@ module AwsEc2
 
       puts "Using profile: #{file}".colorize(:green)
       text = RenderMePretty.result(file, context: context)
-      data = YAML.load(text)
+      begin
+        data = YAML.load(text)
+      rescue Psych::SyntaxError => e
+        tmp_file = file.sub("profiles", "tmp")
+        IO.write(tmp_file, text)
+        puts "There was an error evaluating in your yaml file #{file}".colorize(:red)
+        puts "The evaludated yaml file has been saved at #{tmp_file} for debugging."
+        puts "ERROR: #{e.message}"
+        exit 1
+      end
       data ? data : {} # in case the file is empty
       data.has_key?("run_instances") ? data["run_instances"] : data
     end

data/lib/aws_ec2/script.rb

@@ -8,23 +8,39 @@ module AwsEc2
       @options = options
     end
 
+    def add_to_user_data!(user_data)
+      user_data
+    end
+
+    def auto_terminate_after_timeout
+      load_template("auto_terminate_after_timeout.sh")
+    end
+
     def auto_terminate
+      # set variables for the template
       @ami_name = @options[:ami_name]
       load_template("auto_terminate.sh")
     end
 
+    def cloudwatch
+      load_template("cloudwatch.sh")
+    end
+
     def create_ami
-      # set some variables for the template
+      # set variables for the template
       @ami_name = @options[:ami_name]
       @region = `aws configure get region`.strip rescue 'us-east-1'
       load_template("ami_creation.sh")
     end
 
+    def extract_aws_ec2_scripts
+      load_template("extract_aws_ec2_scripts.sh")
+    end
+
   private
     def load_template(name)
-      template = IO.read(File.expand_path("../scripts/#{name}", __FILE__))
+      template = IO.read(File.expand_path("script/templates/#{name}", File.dirname(__FILE__)))
       text = ERB.new(template, nil, "-").result(binding)
-      "#" * 60 + "\n#{text}"
     end
   end
 end

data/lib/aws_ec2/script/compile.rb

@@ -20,14 +20,14 @@ class AwsEc2::Script
     end
 
     def compile_folder(folder, layout_path=false)
-      puts "Compiling app/#{folder}:".colorize(:green)
+      puts "Compiling app/#{folder} to tmp/app/#{folder}.".colorize(:green)
       Dir.glob("#{AwsEc2.root}/app/#{folder}/**/*").each do |path|
         next if File.directory?(path)
         next if path.include?("layouts")
 
         result = RenderMePretty.result(path, layout: layout_path, context: context)
         tmp_path = path.sub(%r{.*/app/}, "#{BUILD_ROOT}/app/")
-        puts "  #{tmp_path}"
+        puts "  #{tmp_path}" if @options[:verbose]
         FileUtils.mkdir_p(File.dirname(tmp_path))
         IO.write(tmp_path, result)
       end

data/lib/aws_ec2/script/compress.rb

@@ -4,7 +4,7 @@ class AwsEc2::Script
   class Compress < AwsEc2::Base
     def compress
       reset
-      puts "Tarballing #{BUILD_ROOT}/app/scripts folder to scripts.tgz"
+      puts "Tarballing #{BUILD_ROOT}/app/scripts folder to scripts.tgz".colorize(:green)
       tarball_path = create_tarball
       save_scripts_info(tarball_path)
       puts "Tarball created at #{tarball_path}"

data/lib/aws_ec2/script/templates/ami_creation.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -eux
+
+# Create AMI Bundle
+AMI_NAME="<%= @ami_name %>"
+INSTANCE_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/instance-id)
+REGION=$(aws configure get region)
+# Note this will cause the instance to reboot.  Not using the --no-reboot flag
+# to ensure consistent AMI creation.
+SOURCE_AMI_ID=$(wget -q -O - http://169.254.169.254/latest/meta-data/ami-id)
+echo "$SOURCE_AMI_ID" > /var/log/source-ami-id.txt
+mkdir -p /opt/aws-ec2/data
+aws ec2 create-image --name "$AMI_NAME" --instance-id "$INSTANCE_ID" --region "$REGION" > /opt/aws-ec2/data/ami-id.txt

data/lib/aws_ec2/script/templates/auto_terminate.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -eux
+
+/opt/aws-ec2/auto_terminate/setup.sh
+
+<% if @options[:auto_terminate] -%>
+  <% if @options[:ami_name] %>
+/opt/aws-ec2/auto_terminate.sh later
+  <% else %>
+/opt/aws-ec2/auto_terminate.sh now # terminate immediately
+  <% end -%>
+<% end -%>

data/lib/aws_ec2/script/templates/auto_terminate_after_timeout.sh

@@ -0,0 +1,5 @@
+#!/bin/bash -eux
+
+/opt/aws-ec2/auto_terminate/setup.sh
+
+/opt/aws-ec2/auto_terminate.sh after_timeout

data/lib/aws_ec2/script/templates/cloudwatch.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -eux
+
+/opt/aws-ec2/cloudwatch.sh ec2

data/lib/aws_ec2/script/templates/extract_aws_ec2_scripts.sh

@@ -0,0 +1,48 @@
+#!/bin/bash -eux
+
+# Downloads and extract the scripts.
+# The extracted folder from github looks like this:
+#   branch-name.tar.gz => aws-ec2-branch-name
+#   master.tar.gz => aws-ec2-master
+#   v1.0.0.tar.gz => aws-ec2-1.0.0
+function extract_aws_ec2_scripts() {
+  local temp_folder
+  local url
+  local filename
+
+  rm -rf /opt/aws-ec2   # clean start
+
+  temp_folder="/opt/aws-ec2-temp"
+  rm -rf "$temp_folder"
+  mkdir -p "$temp_folder"
+
+  (
+    cd "$temp_folder"
+
+  <%
+    # Examples:
+    #   AWS_EC2_CODE=v1.0.0
+    #   AWS_EC2_CODE=master
+    #   AWS_EC2_CODE=branch-name
+    #
+    #   https://github.com/tongueroo/aws-ec2/archive/v1.0.0.tar.gz
+    #   https://github.com/tongueroo/aws-ec2/archive/master.tar.gz
+    code_version = ENV['AWS_EC2_CODE']
+    code_version ||= "v#{AwsEc2::VERSION}"
+  %>
+    url="https://github.com/tongueroo/aws-ec2/archive/<%= code_version %>.tar.gz"
+    filename=$(basename "$url")
+    folder="${filename%.tar.gz}" # remove extension
+    folder="${folder#v}" # remove leading v character
+    folder="aws-ec2-$folder" # IE: aws-ec2-1.0.0
+
+    wget "$url"
+    tar zxf "$filename"
+
+    mv "$temp_folder/$folder/lib/aws_ec2/scripts" /opt/aws-ec2
+    rm -rf "$temp_folder"
+    chmod a+x -R /opt/aws-ec2
+  )
+}
+
+extract_aws_ec2_scripts

data/lib/aws_ec2/script/upload.rb

@@ -19,7 +19,7 @@ class AwsEc2::Script
     end
 
     def upload(tarball_path)
-      puts "Uploading scripts.tgz (#{filesize}) to #{s3_dest}"
+      puts "Uploading scripts.tgz (#{filesize}) to #{s3_dest}".colorize(:green)
       obj = s3_resource.bucket(bucket_name).object(key)
       start_time = Time.now
       if @options[:noop]