forger 2.0.5 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24b35480802991ec9af7a567180f74723adabcf858c5b4ddcc6fc14a3944d7f7
-  data.tar.gz: 046b74aae0f9b3d7cdeefbcaabd6409f0485cb2974c111667e6e03b26f2e473c
+  metadata.gz: c6a2c42b3903e698ada5e5ea866c96132f5de9013ab48e6a3ebab3164ef376a2
+  data.tar.gz: 8aee63e946e05d816eb6978d9ce3b504a50f8ae413287d181d2cf05717020078
 SHA512:
-  metadata.gz: fb5996ebc2e3059ff16f6ffb4487be82c2348495167479c834de6f33550b282995a717ecfa67ebd739b1773b61e6c231e0744c3b3546fe6c54ddcd9959773f19
-  data.tar.gz: da660ea8453c5905d74507a19a34e9243610d943e60fcacc2f3abcc341141926abf5c3dabdbe9ebc7b0eaae0cc5a8d952bc86e4d0b44c01a32b4026110c244f4
+  metadata.gz: 3c8acac50ce6b424cb2dfe29069cb92b774903b8ff6d7eb07ab043a98c82ad916d8f610841f9ad1ecd18b39829903d259221f588b726bc8d80d867a9e81f80f0
+  data.tar.gz: 90bfbe3f61a0f05a0ac893b0250678204b25c0be844ef300f2c2d0be3df849512beac1708729fe857c4b7382310922d8e4d3c7730c73f52e201d370949a55efa

data/.circleci/config.yml

@@ -32,6 +32,7 @@ jobs:
       - run:
           name: install dependencies
           command: |
+            sed -i '/BUNDLED WITH/Q' Gemfile.lock # hack to fix bundler issue: allow different versions of bundler to work
             bundle install --jobs=4 --retry=3 --path vendor/bundle
 
       # - save_cache:

data/CHANGELOG.md

@@ -3,6 +3,18 @@
 All notable changes to this project will be documented in this file.
 This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [3.0.0]
+* there are some breaking changes with this release
+* automatically created a forger managed s3 bucket when extract_scripts usage detected
+* `config/variables` support instead of `config/FORGER_ENV.yml` files
+* variables definitions are Ruby instead of YAML
+* removed config helper. use instance variables directly.
+* introduced base.yml profile concept
+* fix os_name and cloudwatch logging for amzn2
+* cleanup: remove vars helper
+* use zeitwerk for autoloading
+* change aws\_profiles option to aws\_profile
+
 ## [2.0.5]
 - add circle config: circleci 2.0
 - use rainbow gem for terminal colors

data/Gemfile.lock

@@ -3,8 +3,10 @@ PATH
   specs:
     forger (2.0.5)
       activesupport
+      aws-sdk-cloudformation
       aws-sdk-ec2
       aws-sdk-s3
+      cfn-status
       dotenv
       filesize
       hashie
@@ -12,39 +14,47 @@ PATH
       rainbow
       render_me_pretty
       thor
+      zeitwerk
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.2)
+    activesupport (6.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    aws-eventstream (1.0.1)
-    aws-partitions (1.135.0)
-    aws-sdk-core (3.46.0)
-      aws-eventstream (~> 1.0)
+      zeitwerk (~> 2.1, >= 2.1.8)
+    aws-eventstream (1.0.3)
+    aws-partitions (1.207.0)
+    aws-sdk-cloudformation (1.25.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-core (3.65.1)
+      aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1.0)
-      aws-sigv4 (~> 1.0)
+      aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-ec2 (1.67.0)
-      aws-sdk-core (~> 3, >= 3.39.0)
-      aws-sigv4 (~> 1.0)
-    aws-sdk-kms (1.13.0)
-      aws-sdk-core (~> 3, >= 3.39.0)
-      aws-sigv4 (~> 1.0)
-    aws-sdk-s3 (1.30.1)
-      aws-sdk-core (~> 3, >= 3.39.0)
+    aws-sdk-ec2 (1.106.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-kms (1.24.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-s3 (1.48.0)
+      aws-sdk-core (~> 3, >= 3.61.1)
       aws-sdk-kms (~> 1)
-      aws-sigv4 (~> 1.0)
-    aws-sigv4 (1.0.3)
-    byebug (10.0.2)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.1.0)
+      aws-eventstream (~> 1.0, >= 1.0.2)
+    byebug (11.0.1)
+    cfn-status (0.2.0)
+      aws-sdk-cloudformation
     coderay (1.1.2)
-    concurrent-ruby (1.1.4)
+    concurrent-ruby (1.1.5)
     diff-lcs (1.3)
-    dotenv (2.6.0)
-    ffi (1.10.0)
+    dotenv (2.7.5)
+    ffi (1.11.1)
     filesize (0.2.0)
     formatador (0.2.5)
     guard (2.15.0)
@@ -66,7 +76,7 @@ GEM
       guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
     hashie (3.6.0)
-    i18n (1.5.3)
+    i18n (1.6.0)
       concurrent-ruby (~> 1.0)
     jmespath (1.4.0)
     listen (3.1.5)
@@ -78,18 +88,18 @@ GEM
     method_source (0.9.2)
     minitest (5.11.3)
     nenv (0.3.0)
-    notiffany (0.1.1)
+    notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     rainbow (3.0.0)
-    rake (12.3.2)
+    rake (12.3.3)
     rb-fsevent (0.10.3)
     rb-inotify (0.10.0)
       ffi (~> 1.0)
-    render_me_pretty (0.8.2)
+    render_me_pretty (0.8.3)
       activesupport
       rainbow
       tilt
@@ -97,15 +107,15 @@ GEM
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
+    rspec-core (3.8.2)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
+    rspec-support (3.8.2)
     ruby_dep (1.5.0)
     shellany (0.0.1)
     thor (0.20.3)
@@ -113,6 +123,7 @@ GEM
     tilt (2.0.9)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
+    zeitwerk (2.1.9)
 
 PLATFORMS
   ruby
@@ -127,4 +138,4 @@ DEPENDENCIES
   rake
 
 BUNDLED WITH
-   1.17.2
+   2.0.2

data/README.md

@@ -65,7 +65,7 @@ app/partials  | Your partials that can to be included in other scripts.  This is
 app/scripts  | Where you define common scripts that can be used to configure the server. These scripts can be automatically uploaded to an s3 bucket for later downloading in your user-data script by setting the `s3_folder` settings option.
 app/user_data  | Your user-data scripts that are used to bootstrap EC2 instance.
 app/user_data/layouts  | user-data scripts support layouts. You user-data layouts go in here.
-config/[FORGER_ENV].yml  | The config file where you set configs that you want available in your templating logic.  Examples are: `config/development.yml` and `config/production.yml`. You access the config variables with the `<%= config["var"] %>` helper.
+config/[FORGER_ENV].yml  | The config file where you set configs that you want available in your templating logic.  Examples are: `config/variables/development.rb` and `config/variables/production.rb`. You access the config variables with ERB `<%= @var %>`.
 profiles  | Your profile files.  These files mainly contain parameters that are passed to the aws-sdk run_instances API method.
 tmp  | Where the generated scripts get compiled to. You can manually invoke the compilation via `forger compile` to inspect what is generated.  This is automatically done as part of the `forger` create command.
 
@@ -76,7 +76,7 @@ You can use ERB in the profile files. Some useful helper methods are documented
 Helper  | Description
 ------------- | -------------
 user_data | Allows you to embed a generated user_data script.  More details on the user-data are provided in the user data section below.
-config | Access to the variables set in config/[AWS\_EC2\_ENV].yml.  Examples are `config/development.yml` and `config/production.yml`.
+config | Access to the variables set in config/[AWS\_EC2\_ENV].yml.  Examples are `config/variables/development.rb` and `config/variables/production.rb`.
 latest_ami | Returns an AMI id by searching the AMI name pattern and sorting in reverse order.  Example: `latest_ami("ruby-2.5.0_*")` would return the latest ruby AMIs are named with timestamps at the end like so: `ruby-2.5.0_2018-01-30-05-36-02` and `ruby-2.5.0_2018-01-29-05-36-02`.
 search_ami | Returns a collection of AMI image objects based on a search pattern. The query searches on the AMI name.
 extract_scripts | Use this in your bash script to extract the `app/scripts` files that get uploaded to s3.
@@ -107,42 +107,11 @@ To use the user-data script when creating an EC2 instance, use the `user_data` h
 
 ### User-Data Layouts
 
-User-data scripts support layouts.  This is useful if you have common setup and finish code with your user-data scripts. Here's an example: `app/user_data/layouts/default.sh`:
-
-```bash
-#!/bin/bash
-# do some setup
-<%= yield %>
-# finish work
-```
-
-And `app/user_data/box.sh`:
-
-    yum install -y vim
-
-The resulting generated user-data script will be:
-
-```bash
-#!/bin/bash
-# do some setup
-yum install -y vim
-# finish work
-```
-
-You can specify the layout to use when you call the `user_data` helper method in your profile. Example: `profiles/box.yml`:
-
-```yaml
----
-...
-user_data: <%= user_data("box.sh", layout: "mylayout" ) %>
-...
-```
-
-If there's a `layouts/default.sh`, then it will automatically be used without having to specify the layout option.  You can disable this behavior by passing in `layout: false` or by deleting the `layouts/default.sh` file.
+Refer to [docs/layouts.md](docs/layouts.md)
 
 ### Config
 
-You can set variables in a config file and they are available when ERB is available: profiles, user-data, scripts, etc.  Example `config/development.yml`:
+You can set variables in a config file and they are available when ERB is available: profiles, user-data, scripts, etc.  Example `config/variables/development.rb`:
 
 ```yaml
 ---
@@ -159,8 +128,8 @@ The variables are accessed via the `config` helper method. Here's a filtered exa
 ```yaml
 image_id: ami-4fffc834 # Amazon Lambda AMI
 instance_type: t2.medium
-security_group_ids: <%= config["security_group_ids"] %>
-subnet_id: <%= config["subnets"].shuffle %>
+security_group_ids: <%= @security_group_ids %>
+subnet_id: <%= @subnets.shuffle %>
 ...
 ```
 

data/docs/example/config/variables/development.rb

@@ -0,0 +1,2 @@
+@subnets = %w[subnet-111 subnet-222 subnet-333]
+@security_group_ids = %w[sg-111]

data/docs/example/profiles/default.yml

@@ -4,8 +4,8 @@ instance_type: t2.medium
 key_name: default
 max_count: 1
 min_count: 1
-security_group_ids: <%= config["security_group_ids"] %>
-subnet_id: <%= config["subnets"].shuffle.first %>
+security_group_ids: <%= @security_group_ids %>
+subnet_id: <%= @subnets.shuffle.first %>
 user_data: "<%= user_data("bootstrap") %>"
 iam_instance_profile:
   name: IAMProfileName

data/docs/extract-scripts.md

@@ -0,0 +1,40 @@
+# Extract Scripts
+
+## Overview
+
+Forger supports uploading files to s3 and making them available to your the launched EC2 instances.
+
+## How it works
+
+1. Add your scripts to the `app/scripts` folder.
+2. In your user data script you called the helper method: `extract_scripts`
+
+The `extract_scripts` in the user_data script can look something like this:
+
+app/user_data/bootstrap.sh:
+
+    #!/bin/bash
+
+    <%= extract_scripts(to: "/opt") %>
+
+The generates `extract_scripts` helper, generates a snippet of bash that looks something like this:
+
+    mkdir -p /opt
+    aws s3 cp s3://forger-bucket-EXAMPLE/development/scripts/scripts-md5.tgz /opt/
+    (
+      cd /opt
+      rm -rf /opt/scripts
+      tar zxf /opt/scripts-md5.tgz
+      chmod -R a+x /opt/scripts
+      chown -R ec2-user:ec2-user /opt/scripts
+    )
+
+It essentially extracts the scripts from the `app/scripts` to `/opt/scripts`.
+
+## ERB Support
+
+You can use ERB in the `app/scripts` files. So you can add dynamic logic based on ENV variables.
+
+## S3 Bucket
+
+Forger will automatically create the s3 bucket as needed. The s3 bucket is defined in the `forger` CloudFormation stack.

data/docs/layouts.md

@@ -0,0 +1,35 @@
+# User Data Layouts
+
+User-data scripts support layouts.  This is useful if you have common setup and finish code with your user-data scripts. Here's an example: `app/user_data/layouts/default.sh`:
+
+```bash
+#!/bin/bash
+# do some setup
+<%= yield %>
+# finish work
+```
+
+And `app/user_data/box.sh`:
+
+    yum install -y vim
+
+The resulting generated user-data script will be:
+
+```bash
+#!/bin/bash
+# do some setup
+yum install -y vim
+# finish work
+```
+
+You can specify the layout to use when you call the `user_data` helper method in your profile. Example: `profiles/box.yml`:
+
+```yaml
+---
+...
+user_data: <%= user_data("box.sh", layout: "mylayout" ) %>
+...
+```
+
+If there's a `layouts/default.sh`, then it will automatically be used without having to specify the layout option.  You can disable this behavior by passing in `layout: false` or by deleting the `layouts/default.sh` file.
+

data/docs/profiles.md

@@ -0,0 +1,79 @@
+# Profiles
+
+## Overview
+
+The `profiles` files are YAML files that set the parameters passed to the `ec2.run_instances` API call.  Example:
+
+`profiles/box.yml`:
+
+```yaml
+---
+image_id: ami-97785bed
+instance_type: t2.medium
+key_name: default
+max_count: 1
+min_count: 1
+security_group_ids: <%= @security_group_ids %>
+subnet_id: <%= @subnets.shuffle.first %>
+user_data: "<%= user_data("bootstrap") %>"
+iam_instance_profile:
+  name: IAMProfileName
+```
+
+The forger call to create the ec2 instance would be:
+
+    forger create box
+
+## Base Profile
+
+If you have multiple profiles with a common base, you can create a `profiles/base.yml` file which will have the shared settings.  Example structure:
+
+    profiles
+    ├── base.yml
+    ├── box.yml
+    └── test.yml
+
+The `base.yml` gets merged with `box.yml`.
+The `base.yml` also gets merged with `test.yml`.
+
+### Examples
+
+profiles/base.yml:
+
+```yaml
+---
+instance_type: t2.medium
+key_name: default
+max_count: 1
+min_count: 1
+security_group_ids: <%= @security_group_ids %>
+subnet_id: <%= @subnets.shuffle.first %>
+iam_instance_profile:
+  name: IAMProfileName
+```
+
+profiles/box.yml:
+
+```yaml
+---
+image_id: ami-1111111
+user_data: "<%= user_data("box") %>"
+```
+
+profiles/test.yml:
+
+```yaml
+---
+image_id: ami-2222222
+user_data: "<%= user_data("test") %>"
+```
+
+## Profile Conventions
+
+By convention the profile file will be same as the name to you pass to the the `forger create NAME`.  So
+
+    forger create box
+
+Automatically matches to the `profiles/box.yml`.  If you need to override the convention you can use the `--profile` option.  Example:
+
+    forger create box-2 --profile box

data/docs/variables.md

@@ -0,0 +1,53 @@
+# Variables
+
+## Overview
+
+You can create variables that are accessible in the `app/scripts` and `profiles` files.  You define the variables in the `config/variables/FORGER_ENV.rb` files.
+
+## Structure
+
+    config
+    └── variables
+        ├── base.rb
+        ├── development.rb
+        └── production.rb
+
+## Variable Definitions
+
+Examples of variables definitions:
+
+config/variables/base.rb:
+
+```ruby
+@keypair = "my-keypair-1"
+```
+
+config/variables/development.rb:
+
+```ruby
+---
+instance_type: t2.medium
+key_name: default
+max_count: 1
+min_count: 1
+security_group_ids: <%= @security_group_ids %>
+subnet_id: <%= @subnets.shuffle.first %>
+iam_instance_profile:
+  name: IAMProfileName
+```
+
+## Variables Layering
+
+The variable files are layered together.  The base.rb variable file always get evaulated. Then environment specific variables get evaluated according to the `FORGER_ENV` value. For example, `FORGER_ENV=development` results in `config/variables/development.rb` getting used.
+
+## Accessing Variables
+
+You access the variables files with ERB. Example:
+
+profiles/server.yml:
+
+```yaml
+---
+security_group_ids: <%= @security_group_ids %>
+subnet_id: <%= @subnets.shuffle.first %>
+```