forger 1.6.0 → 2.0.0

data/lib/forger/scripts/cloudwatch/install/amzn2.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -eux
+
+# Install awslogs and the jq JSON parser
+yum install -y awslogs jq

data/lib/forger/scripts/cloudwatch/service/amzn.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -exu
+
+cat > /etc/init/awslogs.conf <<- EOL
+#upstart-job
+description "Configure and start CloudWatch Logs agent on Amazon instance"
+author "BoltOps"
+start on runlevel [2345]
+
+script
+  exec 2>>/var/log/cloudwatch-logs-start.log
+  set -x
+
+  service awslogs start
+  chkconfig awslogs on
+end script
+EOL

data/lib/forger/scripts/shared/functions.sh

@@ -56,7 +56,7 @@ function os_name() {
     # https://askubuntu.com/questions/459402/how-to-know-if-the-running-platform-is-ubuntu-or-centos-with-help-of-a-bash-scri
     # Method 1 works for amazonlinux and ubuntu
     # Method 3 the complex script, did not work for amazonlinux
-    OS=$(gawk -F= '/^NAME/{print $2}' /etc/os-release)
+    OS=$(gawk -F= '/^ID=/{print $2}' /etc/os-release)
   fi
 
   OS="${OS// /}" # remove spaces

data/lib/forger/sequence.rb

@@ -0,0 +1,25 @@
+require 'fileutils'
+require 'colorize'
+require 'active_support/core_ext/string'
+require 'thor'
+require 'bundler'
+
+class Forger::Sequence < Thor::Group
+  include Thor::Actions
+  # include Forger::Helper
+
+  def self.source_root
+    template = ENV['TEMPLATE'] || 'default'
+    File.expand_path("../../templates/#{template}", __FILE__)
+  end
+
+private
+  def copy_project
+    puts "Creating new forger project called #{project_name}."
+    directory ".", project_name
+  end
+
+  def git_installed?
+    system("type git > /dev/null")
+  end
+end

data/lib/forger/template/helper/core_helper.rb

@@ -4,16 +4,14 @@ require "erb"
 module Forger::Template::Helper::CoreHelper
   # assuming user-data script is a bash script for simplicity for now
   def user_data(name, base64:true, layout:"default")
-    # allow user to specify the path also
-    if File.exist?(name)
-      name = File.basename(name) # normalize name, change path to name
-    end
-    name = File.basename(name, '.sh')
-
+    name = normalize_user_data_name_input(name)
     layout_path = layout_path(layout)
-    ext = File.extname(name)
-    name += ".sh" if ext.empty?
-    path = "#{Forger.root}/app/user-data/#{name}"
+
+    path = "#{Forger.root}/app/user_data/#{name}"
+    unless File.exist?(path)
+      puts "ERROR: user-data script #{path.colorize(:red)} does not exist"
+      exit
+    end
     result = RenderMePretty.result(path, context: self, layout: layout_path)
     # Must prepend and append scripts in user_data here because we need to
     # encode the user_data script for valid yaml to load in the profile.
@@ -43,7 +41,7 @@ module Forger::Template::Helper::CoreHelper
 
     ext = File.extname(name)
     name += ".sh" if ext.empty?
-    layout_path = "#{Forger.root}/app/user-data/layouts/#{name}"
+    layout_path = "#{Forger.root}/app/user_data/layouts/#{name}"
 
     # special rule for default in case there's no default layout
     if name.include?("default") and !File.exist?(layout_path)
@@ -78,9 +76,13 @@ module Forger::Template::Helper::CoreHelper
   end
 
 private
+  def cloudwatch_enabled?
+    Forger.cloudwatch_enabled?(@options)
+  end
+
   # TODO: move script combining logic into class
   def prepend_scripts(scripts)
-    scripts.unshift(script.cloudwatch) if @options[:cloudwatch]
+    scripts.unshift(script.cloudwatch) if cloudwatch_enabled?
     scripts.unshift(script.auto_terminate_after_timeout) if @options[:auto_terminate]
     add_setup_script(scripts, :prepend)
     scripts
@@ -97,7 +99,7 @@ private
     return if @already_setup
     @already_setup = true
 
-    requires_setup = @options[:cloudwatch] ||
+    requires_setup = cloudwatch_enabled? ||
                      @options[:auto_terminate] ||
                      @options[:ami_name]
 
@@ -126,4 +128,11 @@ private
       self.class.send :include, module_name.constantize
     end
   end
+
+private
+  def normalize_user_data_name_input(name)
+    ext = File.extname(name)
+    name += ".sh" if ext.empty?
+    name
+  end
 end

data/lib/forger/template/helper/script_helper.rb

@@ -34,7 +34,7 @@ private
   def check_s3_folder_settings!
     return if settings["s3_folder"]
 
-    puts "Helper method called that requires the s3_folder to be set at:"
+    puts "The extract_scripts helper method aws called.  It requires the s3_folder to be set at:"
     lines = caller.reject { |l| l =~ %r{lib/forger} } # hide internal forger trace
     puts "  #{lines[0]}"
 

data/lib/forger/version.rb

@@ -1,3 +1,3 @@
 module Forger
-  VERSION = "1.6.0"
+  VERSION = "2.0.0"
 end

data/lib/templates/default/.env.example

@@ -0,0 +1 @@
+KEY1=examplevalue1

data/lib/templates/default/.gitignore

@@ -0,0 +1,2 @@
+/tmp
+.env

data/lib/templates/default/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gem 'forger'

data/lib/templates/default/README.md

@@ -0,0 +1,61 @@
+# EC2 Profiles
+
+This folder contains EC2 profile files that can be used with the [forger](https://github.com/tongueroo/forger) tool to quickly launch EC2 instances consistently with some pre-configured settings.
+
+## Generate a Forger Project
+
+    forger new ec2 # project name is ec2
+    cd ec2
+
+## EC2 Box
+
+To create an AWS EC2 instance you can run:
+
+    forger create box
+
+This launches an instance.
+
+Note, you can run forger with `--noop` mode to preview the user-data script that the instance will launch with:
+
+    forger create box --noop
+
+## S3 App Scripts
+
+The generated starter project creates some example `app/scripts` files.  The `app/scripts` are disabled until you configure the `s3_folder` setting `config/settings.yml` the `s3_folder` setting. 
+
+The `app/scripts` files get uploaded to s3 as part of the `forger create` command.  You can use it in conjunction with the `extract_scripts` helper method in your user-data file. The `extract_scripts` helper generates a snippet of bash code that downloads and untars the files so user-data has access to the scripts. The scripts are extracted to `/opt/scripts` by default.  Be sure to add extract_scripts to your user-data script.
+
+You can also specify the `--s3-folder` option as part of the `forger new` command to spare you from manually editing all the necessary files like `config/settings.yml` and the user-data scripts.
+
+### CloudWatch Support
+
+You can also have forger insert a script to the generated user-data script that sends logs to CloudWatch with the `--cloudwatch` option.
+
+    forger create box
+
+It is useful to verify that the instance has launched and completed its bootstrapping scripting successfully with cloudwatch.  The command above should show you a cloudwatch log url to visit.  Here's an example with the output filtered to put the focus on the cloudwatch log message:
+
+    $ forger create box --cloudwatch
+    ...
+    Spot instance request id: sir-sb5r4e1j
+    EC2 instance box created: i-03f3c96eaec8ea359 🎉
+    Visit https://console.aws.amazon.com/ec2/home to check on the status
+    To view instance's cloudwatch logs visit:
+      https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logEventViewer:group=ec2;stream=i-03f3c96eaec8ea359/var/log/cloud-init-output.log
+      cw tail -f ec2 i-03f3c96eaec8ea359/var/log/cloud-init-output.log
+    Note: It takes a little time for the instance to launch and report logs.
+    Pro tip: The CloudWatch Console Link has been added to your copy-and-paste clipboard.
+    $
+
+Note, it is detected that the [cw](https://github.com/lucagrulla/cw) tool installed on your machine it will also add that message.  The cw is a command line tool that allows you to tail the cloudwatch log from the terminal instead of the AWS console website.
+
+## Setup .env File
+
+There are some settings that are environment specific.  To configure these, copy the [.env.example](.env.example) file to `.env` and update them with your specific values.
+
+You can have multiple .env files.  The load in this order of precedence.  You are able to reference these values in the `config/[FORGER_ENV].yml` files with ERB.
+
+1. .env.[FORGER_ENV].local
+2. .env.local
+3. .env.[FORGER_ENV]
+4. .env

data/lib/templates/default/app/helpers/application_helper.rb

@@ -0,0 +1,12 @@
+module ApplicationHelper
+  def personalize_script
+    path = File.expand_path("../../scripts/personalize/#{ENV['USER']}.sh", __FILE__)
+    if File.exist?(path)
+      script =<<EOL
+#######################################
+# personalize script added by ApplicationHelper#personalize_script for #{ENV['USER']}
+/opt/scripts/personalize/tung.sh
+EOL
+    end
+  end
+end

data/lib/templates/default/app/scripts/install/common.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -exu
+
+# https://forums.aws.amazon.com/thread.jspa?threadID=270511
+(
+  cd /tmp
+  wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+  yum install -y ./epel-release-latest-7.noarch.rpm
+)
+
+source /opt/scripts/shared/functions.sh
+install_jq
+
+# Some useful tools
+yum install -y tree vim less

data/lib/templates/default/app/scripts/personalize/tung.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -exu
+
+yum install htop -y

data/lib/templates/default/app/scripts/shared/functions.sh

@@ -0,0 +1,22 @@
+function install_jq() {
+  if ! type jq > /dev/null ; then
+    wget "https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64"
+    mv jq-linux64 /usr/local/bin/jq
+    chmod a+x /usr/local/bin/jq
+  fi
+}
+
+function configure_aws_cli() {
+  local home_dir=$1
+  # Configure aws cli in case it is not yet configured
+  mkdir -p "$home_dir/.aws"
+  if [ ! -f "$home_dir/.aws/config" ]; then
+    EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
+    EC2_REGION=${EC2_AVAIL_ZONE::-1}
+    cat >"$home_dir/.aws/config" <<CONFIGURE_AWS_CLI
+[default]
+region = $EC2_REGION
+output = json
+CONFIGURE_AWS_CLI
+  fi
+}

data/lib/templates/default/app/user_data/bootstrap.sh.tt

@@ -0,0 +1,12 @@
+<% if @options[:s3_folder] -%>
+# setup: shared functions, path, awscli, etc
+source /opt/scripts/shared/functions.sh
+export PATH=/usr/local/bin/:$PATH
+configure_aws_cli /root
+configure_aws_cli /home/ec2-user
+
+# install software
+/opt/scripts/install/common.sh
+<% end -%>
+
+<%%= personalize_script %>

data/lib/templates/default/app/user_data/layouts/default.sh.tt

@@ -0,0 +1,17 @@
+#!/bin/bash -exu
+
+export HOME=/root # In user-data env USER=root but HOME is not set
+cd ~ # user-data starts off in / so go to $HOME
+
+<%%= add_ssh_key %>
+
+<% if options[:s3_folder] -%>
+<%%= extract_scripts(to: "/opt") %>
+<% else -%>
+# Uncomment to use extract_scripts and use app/scripts
+<%%# extract_scripts(to: "/opt") %>
+<% end -%>
+
+<%%= yield %>
+
+uptime | tee /var/log/boot-time.log

data/lib/templates/default/config/development.yml.tt

@@ -0,0 +1,6 @@
+---
+subnets:
+  - <%= @subnet %>
+security_group_ids:
+  - <%= @security_group %>
+key1: <%%= ENV['KEY1'] %>

data/lib/templates/default/config/settings.yml.tt

@@ -0,0 +1,25 @@
+---
+base:
+  # cloudwatch: true # forger create --cloudwatch # TODO: add to docs
+
+# Settings control internal forger behavior.
+# Settings are different from the config files.  The config files are meant to
+# expose config variables that you can use in your ERB code.
+# There are separate files to separate user defined variables and internal
+# setting configs.
+development:
+  # By setting s3_folder, forger will automatically tarball and upload your scripts
+  # to set. You then can then use the extract_scripts helper method to download
+  # the scripts onto the server.
+  # s3_folder: mybucket/path/to/folder # simple string
+  # compile_clean: true # uncomment to clean
+  # extract_scripts:
+  #   to: "/opt"
+  #   as: "ec2-user"
+  aws_profiles:
+    - my-aws-profile
+<% if @options[:s3_folder] -%>
+  s3_folder: <%= @options[:s3_folder] %>
+<% else -%>
+  # s3_folder: my-s3-bucket/ec2
+<% end -%>

data/lib/templates/default/profiles/default.yml.tt

@@ -0,0 +1,41 @@
+---
+image_id: ami-6cd6f714 # Amazon Linux 2 AMI (HVM), SSD Volume Type
+instance_type: t3.small
+<% if @options[:key_name] -%>
+key_name: <%= @options[:key_name] %>
+<% else -%>
+# key_name: default # make sure that the "default" keypair exist in the region
+<% end -%>
+max_count: 1
+min_count: 1
+user_data: "<%%= user_data("bootstrap") %>"
+<% if @options[:iam] -%>
+iam_instance_profile:
+  name: <%= @options[:iam] %>
+<% else -%>
+# iam_instance_profile:
+#   name: ExampleIamProfile
+<% end -%>
+# public network settings
+security_group_ids: <%%= config["security_group_ids"] %>
+subnet_id: <%%= config["subnets"].shuffle.first %>
+# block_device_mappings:
+#   - device_name: /dev/xvda
+#     ebs:
+#       volume_size: 100 # in GB
+<%% if ENV['SPOT'] %>
+instance_market_options:
+  market_type: spot
+  # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_LaunchTemplateSpotMarketOptionsRequest.html
+  # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/EC2/Types/SpotMarketOptions.html
+  spot_options:
+    max_price: "0.018" # $0.020/hr = $14.40/mo
+                       # $0.018/hr = $12.96/mo
+    # valid combinations:
+    #   spot_instance_type: persistent
+    #   instance_interruption_behavior: hibernate
+    # or
+    #   spot_instance_type: one-time
+    # More info: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html
+    spot_instance_type: one-time
+<%% end %>

data/spec/lib/core_helper_spec.rb

@@ -0,0 +1,19 @@
+describe Forger::Template::Helper::CoreHelper do
+  let(:thing) do
+    thing = Class.new
+    Forger::Template::Helper::CoreHelper.send :public, :normalize_user_data_name_input
+    thing.extend(Forger::Template::Helper::CoreHelper)
+    thing
+  end
+
+  it 'normalize user_data name input' do
+    result = thing.normalize_user_data_name_input("ecs.sh")
+    expect(result).to eq "ecs.sh"
+
+    result = thing.normalize_user_data_name_input("ecs")
+    expect(result).to eq "ecs.sh"
+
+    result = thing.normalize_user_data_name_input("test/ecs")
+    expect(result).to eq "test/ecs.sh"
+  end
+end

data/spec/spec_helper.rb

@@ -4,10 +4,6 @@ ENV["FORGER_ROOT"] = "spec/fixtures/demo_project"
 # Ensures aws api never called. Fixture home folder does not contain ~/.aws/credentails
 ENV['HOME'] = "spec/fixtures/home"
 
-# CodeClimate test coverage: https://docs.codeclimate.com/docs/configuring-test-coverage
-# require 'simplecov'
-# SimpleCov.start
-
 require "pp"
 require "byebug"
 root = File.expand_path("../", File.dirname(__FILE__))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: forger
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-28 00:00:00.000000000 Z
+date: 2018-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -109,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: render_me_pretty
+  name: memoist
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: render_me_pretty
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -137,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: memoist
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -256,7 +256,7 @@ files:
 - docs/example/.env.development
 - docs/example/.env.production
 - docs/example/app/scripts/hello.sh
-- docs/example/app/user-data/bootstrap.sh
+- docs/example/app/user_data/bootstrap.sh
 - docs/example/config/development.yml
 - docs/example/profiles/default.yml
 - docs/example/profiles/spot.yml
@@ -291,9 +291,12 @@ files:
 - lib/forger/help/completion_script.md
 - lib/forger/help/create.md
 - lib/forger/help/destroy.md
+- lib/forger/help/new.md
 - lib/forger/help/upload.md
 - lib/forger/help/wait/ami.md
 - lib/forger/hook.rb
+- lib/forger/network.rb
+- lib/forger/new.rb
 - lib/forger/profile.rb
 - lib/forger/script.rb
 - lib/forger/script/compile.rb
@@ -307,18 +310,22 @@ files:
 - lib/forger/scripts/auto_terminate.sh
 - lib/forger/scripts/auto_terminate/after_timeout.sh
 - lib/forger/scripts/auto_terminate/functions.sh
-- lib/forger/scripts/auto_terminate/functions/amazonlinux2.sh
+- lib/forger/scripts/auto_terminate/functions/amzn.sh
+- lib/forger/scripts/auto_terminate/functions/amzn2.sh
 - lib/forger/scripts/auto_terminate/functions/ubuntu.sh
 - lib/forger/scripts/auto_terminate/setup.sh
 - lib/forger/scripts/cloudwatch.sh
 - lib/forger/scripts/cloudwatch/configure.sh
 - lib/forger/scripts/cloudwatch/install.sh
-- lib/forger/scripts/cloudwatch/install/amazonlinux2.sh
+- lib/forger/scripts/cloudwatch/install/amzn.sh
+- lib/forger/scripts/cloudwatch/install/amzn2.sh
 - lib/forger/scripts/cloudwatch/install/ubuntu.sh
 - lib/forger/scripts/cloudwatch/service.sh
-- lib/forger/scripts/cloudwatch/service/amazonlinux2.sh
+- lib/forger/scripts/cloudwatch/service/amzn.sh
+- lib/forger/scripts/cloudwatch/service/amzn2.sh
 - lib/forger/scripts/cloudwatch/service/ubuntu.sh
 - lib/forger/scripts/shared/functions.sh
+- lib/forger/sequence.rb
 - lib/forger/setting.rb
 - lib/forger/template.rb
 - lib/forger/template/context.rb
@@ -332,12 +339,26 @@ files:
 - lib/forger/wait.rb
 - lib/forger/waiter.rb
 - lib/forger/waiter/ami.rb
+- lib/templates/default/.env.example
+- lib/templates/default/.gitignore
+- lib/templates/default/Gemfile
+- lib/templates/default/README.md
+- lib/templates/default/app/helpers/application_helper.rb
+- lib/templates/default/app/scripts/install/common.sh
+- lib/templates/default/app/scripts/personalize/tung.sh
+- lib/templates/default/app/scripts/shared/functions.sh
+- lib/templates/default/app/user_data/bootstrap.sh.tt
+- lib/templates/default/app/user_data/layouts/default.sh.tt
+- lib/templates/default/config/development.yml.tt
+- lib/templates/default/config/settings.yml.tt
+- lib/templates/default/profiles/default.yml.tt
 - spec/fixtures/demo_project/.env.test
-- spec/fixtures/demo_project/app/user-data/bootstrap.sh
+- spec/fixtures/demo_project/app/user_data/bootstrap.sh
 - spec/fixtures/demo_project/config/settings.yml
 - spec/fixtures/demo_project/config/test.yml
 - spec/fixtures/demo_project/profiles/default.yml
 - spec/lib/cli_spec.rb
+- spec/lib/core_helper_spec.rb
 - spec/lib/params_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/tongueroo/forger
@@ -360,16 +381,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Tool to create AWS ec2 instances
 test_files:
 - spec/fixtures/demo_project/.env.test
-- spec/fixtures/demo_project/app/user-data/bootstrap.sh
+- spec/fixtures/demo_project/app/user_data/bootstrap.sh
 - spec/fixtures/demo_project/config/settings.yml
 - spec/fixtures/demo_project/config/test.yml
 - spec/fixtures/demo_project/profiles/default.yml
 - spec/lib/cli_spec.rb
+- spec/lib/core_helper_spec.rb
 - spec/lib/params_spec.rb
 - spec/spec_helper.rb