aws-ec2 0.8.3 → 0.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4780c822aef006f9c4720bb069a2d81accea2f56b3a97e7e3598500cab748436
-  data.tar.gz: bd5aa36b830c6e89a88d60bccecaf411bc5c24dbba6ccb46a8a5e2ad172dd616
+  metadata.gz: 7ecc2b042e1fabbb3a4a343edb5766601a51e54259f5029c19d37f93ebed7443
+  data.tar.gz: d0a388114c37362c2f8b1590f71d1e2d16d54dee0631bd36a42c9f5ed89fd877
 SHA512:
-  metadata.gz: cc28865e6b2e57e535b5634d00b2314443b07716920b774901111b65018c3df41d6f54fc135c29b8f82a43857745923b7933bdeed8f62c2b64772d2d6b32f06f
-  data.tar.gz: f913d913014d2c8382901958fc98a139aa092c60d0cce84a8d51e867f68cd740171e9c76e5fd79c7e7055477fd2a67f284f0fb6d5c654e09390a3cc83e445100
+  metadata.gz: c77a9ba413bec00c46b387370df141d10e64286ac94689c194437ff0aeecd3e8ccfbcb50a548549090344df3628affe8210014a4c902e40466da4360cc66bb6a
+  data.tar.gz: 63e9c1864910f2a8c7d57a7005e64297272e6c1764642c29ba55fde4722073d46fd854a5f6afe7f08763e47f9d454e66fafdd790ff4599660dc02a1723738a67

data/.gitignore

@@ -6,7 +6,6 @@
 InstalledFiles
 _yardoc
 coverage
-doc/
 lib/bundler/man
 pkg
 rdoc

data/CHANGELOG.md

@@ -3,6 +3,12 @@
 All notable changes to this project will be documented in this file.
 This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [0.8.4]
+- add doc/example
+- rename to commands to compile and upload
+- rename to core_helper, only require specfic activesupport dependencies
+- generate user-data in tmp/user-data.txt instead
+
 ## [0.8.3]
 - change compile_keep to compile_clean option instead
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    aws-ec2 (0.5.1)
+    aws-ec2 (0.8.3)
       activesupport
       aws-sdk-ec2
       colorize

data/README.md

@@ -1,10 +1,10 @@
 # AWS EC2 Tool
 
-Simple tool to create AWS ec2 instances consistently with pre-configured settings.  The pre-configured settings are stored in the profiles folder of the current directory.
+Simple tool to create AWS ec2 instances consistently with pre-configured settings.  The pre-configured settings are stored in the profiles folder of the current project directory.
 Example:
 
-* profiles/default.yml: default settings. used when no profile is specified.
-* profiles/myserver.yml: myserver settings.
+* profiles/default.yml: Default settings. Used when no profile is specified.
+* profiles/myserver.yml: myserver profile.  Used when `--profile myserver` is specified.
 
 ## Usage
 
@@ -12,55 +12,62 @@ Example:
 aws-ec2 create myserver --profile myserver
 ```
 
-In a nutshell, the profile parameters are passed to the ruby aws-sdk [AWS::EC2::Client#run_instances](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Client.html#run_instances-instance_method) method.  So you can specify any parameter you wish that is available there. To check out what a profile looks like look at the [example default](example/profiles/default.yml)
+In a nutshell, the profile parameters are passed to the ruby aws-sdk [AWS::EC2::Client#run_instances](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Client.html#run_instances-instance_method) method.  This allows you to specify any parameter you wish that is available in the aws-sdk. To check out what a profile looks like check out [example default](example/profiles/default.yml)
 
-You can use ERB in the profile files. Some useful helper methods in the profile files are documented here:
+You can use ERB in the profile files. Some useful helper methods are documented here:
 
 Helper  | Description
 ------------- | -------------
-user_data | Allows you to embed a generated user_data script.  More details on the are provided in the user data section below.
-config | Access to the variables set in config/[AWS_EC2_ENV].yml.
-latest_ami | Returns an AMI id by searching the ami name pattern and sorting in reverse older.  Example: `latest_ami("ruby-2.5.0_*")`
+user_data | Allows you to embed a generated user_data script.  More details on the user-data are provided in the user data section below.
+config | Access to the variables set in config/[AWS_EC2_ENV].yml.  Examples are `config/development.yml`, `config/staging.yml`, and `config/production.yml`.
+latest_ami | Returns an AMI id by searching the ami name pattern and sorting in reverse older.  Example: `latest_ami("ruby-2.5.0_*")` would return the latest ruby AMIs are named with timestamps at the end like so: `ruby-2.5.0_2018-01-30-05-36-02` and `ruby-2.5.0_2018-01-29-05-36-02`.
 search_ami | Returns a collection of AMI image objects based on a search pattern. The query searches on the AMI name.
 
 For a full list of all the template helpers checkout: [aws_ec2/template_helper](lib/aws_ec2/template_helper).
 
-You can also define your own custom helpers in the `app/helpers` folder as ruby modules with the naming convention `'*_helper.rb`.  Example, the module FooHelper  should be defined in `app/helpers/foo_helper.rb`.  The custom helpers are first class citizens and have access to the same variables and methods as built in helpers.
+You can also define custom helpers in the `app/helpers` folder as ruby modules with the naming convention `*_helper.rb`.  Example, you would define a `module FooHelper` in `app/helpers/foo_helper.rb`.  Custom helpers are first-class citizens and have access to the same variables, methods and scope as built-in helpers.
 
 ## Noop mode
 
-You can do a test run with the `--noop` flag.  This will print out what settings will be used to launch the instance.  This is a good way to inspect the generated user-data script.
+You can do a test run with the `--noop` flag.  This will print out what settings will be used to launch the instance.  This is one good way to inspect the generated user-data script.
 
 ```sh
 aws-ec2 create myserver --profile myserver --noop
+cat tmp/user-data.txt # to view generated user-data script
 ```
 
 ## Project Structure
 
 Directory  | Description
 ------------- | -------------
-app/helpers  | Your own custom helpers methods.  Define them in modules and the methods made available to your `config`, `profiles`, `app/scripts`, `app/user-data` files.
-app/partials  | Your partials that can be use to be included in other scripts.  This is used in conjunction with the partial helper method.
-app/scripts  | Where you define common scripts that can be used to configure the server.
+app/helpers  | Custom helpers methods.  Define them as modules and their methods available whenever ERB is available: `config`, `profiles`, `app/scripts`, `app/user-data` files, etc. Example, you would define a `module FooHelper` in `app/helpers/foo_helper.rb`.
+app/partials  | Your partials that can be use to be included in other scripts.  This is used in conjunction with the `partial` helper method.
+app/scripts  | Where you define common scripts that can be used to configure the server. These scripts can be automatically uploaded to an s3 bucket for later downloading in your user-data script by setting the `scripts_s3_bucket` config option.
 app/user-data  | Your user-data scripts that are used to bootstrap EC2 instance.
-config/[AWS_EC2_ENV].yml  | AWS_EC2_ENV can be development, staging or production. Use this config file to set configs that you want available in your templating logic.
+config/[AWS_EC2_ENV].yml  | The config file where you set configs that you want available in your templating logic.  Examples are: `config/development.yml`, `config/staging.yml`, or `config/production.yml`. You access the config with the `<%= config["var"] %>` helper.
 profiles  | Your profile files.  These files mainly contain parameters that are passed to the aws-sd2 run_instances API.
-tmp  | Where the generated scripts get compiled to. You can manually invoke the complilation via `aws-ec2 compile`.
+tmp  | Where the generated scripts get compiled to. You can manually invoke the complilation via `aws-ec2 compile` if you wish to inspect what is generated, though this is automatically done as part of the `aws-ec2` create command.
 
 ## User-Data
 
-You can provide user-data script to customize the server upon launch.  The user-data scripts are under the `app/user-data` folder.
+You can provide a user-data script to customize the server upon launch.  The user-data scripts are located under the `app/user-data` folder.  Example:
 
 * app/user-data/myserver.yml
 
-The user-data script is generated on the machine that is running the aws-ec2 command. If this is your local macosx machine, then the context is your local macosx machine is available. To see the generated user-data script, you can use the run the create command in noop mode and then inspect the generated script.  Example:
+The user-data script is generated on the machine that is running the aws-ec2 command. If this is your local macosx machine, then the context of your local macosx machine is available. To see the generated user-data script, you can use the run the create command in noop mode and then inspect the generated script.  Example:
 
 ```sh
-aws create myserver --noop
+aws-ec2 create myserver --noop
 cat /tmp/aws-ec2/user-data.txt
 ```
 
-To use the user-data script when creating an EC2 instance, you can use the helper method in the profile.  Here's a grep of a profile that users the helper:
+Another way of viewing the generated user-data scripts is the `aws-ec2 compile` command.  It generates the files in the tmp folder.  Example:
+
+```
+aws-ec2 compile # generates files in tmp folder
+```
+
+To use the user-data script when creating an EC2 instance, you use the `user_data` helper method in the profile.  Here's a grep of an example profile that uses the helper to show you want it looks like:
 
 ```
 $ grep user_data profiles/default.yml
@@ -69,7 +76,7 @@ user_data: "<%= user_data("bootstrap") %>"
 
 ### Config
 
-You can set a config file and define variables in there that are available to in your profiles and user_data scripts.  Example `config/development.yml`:
+You can set variables in a config file and they are available when ERB is available, which are your profiles, user_data, scripts, etc.  Example `config/development.yml`:
 
 ```yaml
 ---
@@ -81,13 +88,11 @@ db_subnet_group_name: default
 security_group_ids:
   - sg-123
 scripts_s3_bucket: mybucket # enables s3 uploading of generated app/scripts
-# compile_clean: true # uncomment if you want to automatically clean the
-                      # compiled scripts in tmp that aws-ec2 creates
+# compile_clean: true # uncomment to automatically remove the
+                      # compiled scripts in tmp after aws-ec2 create
 ```
 
-The variables are accessible via the `config` helper method. Example (only showing the part of the profile), `profiles/default.yml`:
-
-The `scripts_s3_bucket` is an option that changes the behavior of aws-ec2 to automatically sync the generated scripts from app/scripts to the s3 bucket right before the internal call to run_instances to launch the instance.  If you need more custom logic, look at the hooks section.
+The variables are accessed via the `config` helper method. Here's a filtered example where it shows shows the relevant part of a profile: `profiles/default.yml`:
 
 ```yaml
 image_id: ami-4fffc834 # Amazon Lambda AMI
@@ -97,6 +102,14 @@ subnet_id: <%= config["subnets"].shuffle %>
 ...
 ```
 
+#### Config Options
+
+There are some config options that change the behavior of the ec2-aws:
+
+Option | Description
+--- | ---
+scripts_s3_bucket | Set this to the bucket name where you want the generated scripts in app/scripts and app/user-data to be automatically sync.  The sync happens right before the the internal call to run_instances that launches the instance.  If you need more custom logic, you can use the `before_run_instances` hook, covered in the Hooks section.
+
 ## Dotenv File Support
 
 You can set and configure environment variables in `.env*` files.  Examples of this is in the [doc/example](doc/example) project.
@@ -110,19 +123,23 @@ There is only one hook: before_run_instances.  You can configure this with `conf
 before_run_instances: /path/to/my/script.sh
 ```
 
+This will run `/path/to/my/script.sh` as a shelled out command before the `run_instances` call.
+
 ## AMI Creation
 
-To create AMIs you can use the `aws-ec2 ami` command.  This command will run launch an EC2 instance with one of the profiles and create an AMI after the user-data script completes successfully. It does this by adding an AMI creation script at the end of the user-data script.  It is recommended to use the `set -e` option so that any error halts the script and the AMI does not get created.
+To create AMIs you can use the `aws-ec2 ami` command.  This command launches an EC2 instance with the specified profile and create an AMI after the user-data script completes successfully. It does this by adding an AMI creation script at the end of the user-data script.  It is recommended to use the `set -e` option so that any error halts the script and the AMI does not get created.
 
 After the AMI is created successfully, the instance will also terminate itself automatically so you do not have to worry about cleanup.  For more help run `aws-ec2 ami help`.
 
 ## Spot Instance Support
 
-Spot instance support natively supported by the AWS run_instances command.  Simply add `instance_market_options` to the parameters to request for a spot instance.  The available spot market options are available here:
+Spot instance is natively supported by the AWS run_instances command.  So, simply add `instance_market_options` to the parameters to request for a spot instance.  The available spot market options are available here:
 
 * [https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_LaunchTemplateSpotMarketOptionsRequest.html](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_LaunchTemplateSpotMarketOptionsRequest.html)
 * [https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Types/SpotMarketOptions.html](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Types/SpotMarketOptions.html)
 
+An example of a spot instance profile is provided in [doc/example/profiles/spot.yml](doc/example/profiles/spot.yml).
+
 ## More Help
 
 ```sh

data/doc/example/.env

@@ -0,0 +1,2 @@
+# Example .env, meant to be updated.
+# Variables in here are available and shared across all environments: development, production, etc.

data/doc/example/.env.development

@@ -0,0 +1,2 @@
+# Example .env.development, meant to be updated.
+ENV_DEVELOPMENT_KEY=example1

data/doc/example/.env.production

@@ -0,0 +1,3 @@
+# Example .env.production, meant to be updated.
+# This file is normally .gitignore.
+MYKEY=example1

data/doc/example/app/scripts/hello.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -exu
+
+echo "hello world"

data/doc/example/app/user-data/bootstrap.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -exu
+
+exec > >(tee "/var/log/user-data.log" | logger -t user-data -s 2>/dev/console) 2>&1
+export HOME=/root # user-data env runs in weird shell where user is root but HOME is not set
+<% pubkey_path = "#{ENV['HOME']}/.ssh/id_rsa.pub" -%>
+<% if File.exist?(pubkey_path) -%>
+<% pubkey = IO.read(pubkey_path).strip -%>
+# Automatically add user's public key
+echo <%= pubkey %> >> ~/.ssh/authorized_keys
+echo <%= pubkey %> >> /home/ec2-user/.ssh/authorized_keys
+chown ec2-user:ec2-user /home/ec2-user/.ssh/authorized_keys
+<% else %>
+# WARN: unable to find a ~/.ssh/id_rsa.pub locally on your machine.  user: <%= ENV['USER'] %>
+# Unable to automatically add the public key
+<% end -%>
+
+sudo yum install -y postgresql
+
+# https://gist.github.com/juno/1330165
+# Install developer tools
+yum install -y git gcc make readline-devel openssl-devel
+
+# Install ruby-build system-widely
+git clone git://github.com/sstephenson/ruby-build.git /tmp/ruby-build
+cd /tmp/ruby-build
+./install.sh
+echo 'export PATH="/usr/local/bin:$PATH"' >> ~/.bashrc
+
+# Install rbenv for root
+git clone git://github.com/sstephenson/rbenv.git ~/.rbenv
+echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
+echo 'eval "$(rbenv init -)"' >> ~/.bashrc
+set +u
+source ~/.bashrc
+set -u
+
+# Install and enable ruby
+rbenv install 2.5.0
+
+# Install ruby for ec2-user also
+cp -R ~/.rbenv /home/ec2-user/
+chown -R ec2-user:ec2-user /home/ec2-user/.rbenv
+echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> /home/ec2-user/.bashrc
+echo 'eval "$(rbenv init -)"' >> /home/ec2-user/.bashrc
+echo '2.5.0' > /home/ec2-user/.ruby-version
+
+uptime | tee /var/log/boot-time.log

data/doc/example/config/development.yml

@@ -0,0 +1,8 @@
+---
+vpc_id: vpc-111
+subnets:
+  - subnet-111
+  - subnet-222
+  - subnet-333
+security_group_ids:
+  - sg-111

data/doc/example/profiles/default.yml

@@ -0,0 +1,11 @@
+---
+image_id: ami-97785bed
+instance_type: t2.medium
+key_name: default
+max_count: 1
+min_count: 1
+security_group_ids: <%= config["security_group_ids"] %>
+subnet_id: <%= config["subnets"].shuffle.first %>
+user_data: "<%= user_data("bootstrap") %>"
+iam_instance_profile:
+  name: IAMProfileName

data/doc/example/profiles/spot.yml

@@ -0,0 +1,20 @@
+---
+image_id: ami-97785bed
+instance_type: t2.medium
+key_name: default
+max_count: 1
+min_count: 1
+security_group_ids:
+- sg-111
+subnet_id: <%= %w[subnet-111 subnet-222].shuffle.first %>
+user_data: "<%= user_data("dev") %>"
+iam_instance_profile:
+  name: IAMProfileName
+instance_market_options:
+  market_type: spot
+  # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_LaunchTemplateSpotMarketOptionsRequest.html
+  # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Types/SpotMarketOptions.html
+  spot_options:
+    max_price: "0.02"
+    spot_instance_type: one-time
+    # instance_interruption_behavior: hibernate

data/lib/aws_ec2/cli.rb

@@ -19,16 +19,16 @@ module AwsEc2
       Ami.new(options.merge(name: name)).run
     end
 
-    desc "compile_scripts", "compiles app/scripts and app/user-data to tmp folder"
-    long_desc Help.text(:compile_scripts)
-    def compile_scripts
+    desc "compile", "compiles app/scripts and app/user-data to tmp folder"
+    long_desc Help.text(:compile)
+    def compile
       Script::Compile.new(options).compile
     end
 
-    desc "upload_scripts", "compiles and uploads scripts to s3"
-    long_desc Help.text(:upload_scripts)
+    desc "upload", "compiles and uploads scripts to s3"
+    long_desc Help.text(:upload)
     option :compile, type: :boolean, default: true, desc: "compile scripts before uploading"
-    def upload_scripts
+    def upload
       Script::Upload.new(options).upload
     end
   end

data/lib/aws_ec2/create.rb

@@ -69,7 +69,7 @@ module AwsEc2
       data = data.deep_stringify_keys
 
       if data["user_data"]
-        message = "base64-encoded: cat /tmp/aws-ec2/user-data.txt to view"
+        message = "base64-encoded: cat tmp/user-data.txt to view"
         data["user_data"] = message
       end
 

data/lib/aws_ec2/create/params.rb

@@ -29,7 +29,7 @@ class AwsEc2::Create
     end
 
     def cleanup
-      FileUtils.rm_f("/tmp/aws-ec2/user-data.txt")
+      FileUtils.rm_f("#{AwsEc2.root}/tmp/user-data.txt")
     end
 
     # Adds instance ec2 tag if not already provided

data/lib/aws_ec2/help/{compile_scripts.md → compile.md}

@@ -1,5 +1,5 @@
 Examples:
 
-  $ aws-ec2 compile_scripts
+  $ aws-ec2 compile
 
 Compiles app/scripts and app/user-data files to the tmp folder. Useful for inspection.

data/lib/aws_ec2/help/create.md

@@ -2,6 +2,6 @@ Examples:
 
   $ aws-ec2 create my-instance
 
-To see the snippet of code that gets added to the user-data script you can use the `--noop` option and then view the generated /tmp/aws-ec2/user-data.txt.
+To see the snippet of code that gets added to the user-data script you can use the `--noop` option and then view the generated tmp/user-data.txt.
 
   $ aws-ec2 create myscript --noop

data/lib/aws_ec2/help/{upload_scripts.md → upload.md}

@@ -1,6 +1,6 @@
 Examples:
 
-  $ aws-ec2 upload_scripts
+  $ aws-ec2 upload
 
 Compiles the app/scripts and app/user-data files to the tmp folder. Then uploads the files to an s3 bucket that is configured in config/[AWS_EC2_ENV].yml.  Example scripts_s3_bucket config:
 

data/lib/aws_ec2/script/compile.rb

@@ -1,6 +1,6 @@
 require 'fileutils'
 
-# Class for aws-ec2 compile_scripts command
+# Class for aws-ec2 compile command
 class AwsEc2::Script
   class Compile < AwsEc2::Base
     include AwsEc2::TemplateHelper

data/lib/aws_ec2/script/upload.rb

@@ -1,6 +1,6 @@
 require 'fileutils'
 
-# Class for aws-ec2 upload_scripts command
+# Class for aws-ec2 upload command
 class AwsEc2::Script
   class Upload < AwsEc2::Base
     def initialize(options={})

data/lib/aws_ec2/template_helper.rb

@@ -1,4 +1,5 @@
-require "active_support/all"
+require "active_support" # for autoload
+require "active_support/core_ext/string"
 
 module AwsEc2
   module TemplateHelper

data/lib/aws_ec2/template_helper/{core.rb → core_helper.rb}

@@ -1,7 +1,7 @@
 require "base64"
 require "erb"
 
-module AwsEc2::TemplateHelper::Core
+module AwsEc2::TemplateHelper::CoreHelper
   def user_data(name, base64=true)
     # allow user to specify the path also
     if File.exist?(name)
@@ -13,7 +13,7 @@ module AwsEc2::TemplateHelper::Core
     result = append_scripts(result)
 
     # save the unencoded user-data script for easy debugging
-    temp_path = "/tmp/aws-ec2/user-data.txt"
+    temp_path = "#{AwsEc2.root}/tmp/user-data.txt"
     FileUtils.mkdir_p(File.dirname(temp_path))
     IO.write(temp_path, result)
 

data/lib/aws_ec2/version.rb

@@ -1,3 +1,3 @@
 module AwsEc2
-  VERSION = "0.8.3"
+  VERSION = "0.8.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws-ec2
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.8.4
 platform: ruby
 authors:
 - Tung Nguyen
@@ -197,6 +197,14 @@ files:
 - README.md
 - Rakefile
 - aws-ec2.gemspec
+- doc/example/.env
+- doc/example/.env.development
+- doc/example/.env.production
+- doc/example/app/scripts/hello.sh
+- doc/example/app/user-data/bootstrap.sh
+- doc/example/config/development.yml
+- doc/example/profiles/default.yml
+- doc/example/profiles/spot.yml
 - exe/aws-ec2
 - lib/aws-ec2.rb
 - lib/aws_ec2/ami.rb
@@ -211,9 +219,9 @@ files:
 - lib/aws_ec2/dotenv.rb
 - lib/aws_ec2/help.rb
 - lib/aws_ec2/help/ami.md
-- lib/aws_ec2/help/compile_scripts.md
+- lib/aws_ec2/help/compile.md
 - lib/aws_ec2/help/create.md
-- lib/aws_ec2/help/upload_scripts.md
+- lib/aws_ec2/help/upload.md
 - lib/aws_ec2/hook.rb
 - lib/aws_ec2/profile.rb
 - lib/aws_ec2/script.rb
@@ -223,7 +231,7 @@ files:
 - lib/aws_ec2/scripts/auto_terminate.sh
 - lib/aws_ec2/template_helper.rb
 - lib/aws_ec2/template_helper/ami_helper.rb
-- lib/aws_ec2/template_helper/core.rb
+- lib/aws_ec2/template_helper/core_helper.rb
 - lib/aws_ec2/template_helper/partial_helper.rb
 - lib/aws_ec2/version.rb
 - spec/fixtures/demo_project/config/test.yml