forger 1.5.0

data/lib/forger/scripts/cloudwatch.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -eux
+
+if [ $# -eq 0 ]; then
+  command=$(basename "$0")
+  echo "Usage: $command LOG_GROUP_NAME"
+  echo "Examples:"
+  echo "  $command forger"
+  echo "  $command ec2"
+  exit 1
+fi
+LOG_GROUP_NAME=$1
+
+# shellcheck disable=SC1091
+source "/opt/forger/shared/functions.sh"
+OS=$(os_name)
+if [ "$OS" != "amazonlinux2" ] && [ "$OS" != "ubuntu" ] ; then
+  echo "Sorry, cloudwatch logging with the forger tool is supported for amazonlinux2 and ubuntu only"
+  exit
+fi
+
+export OS # used by the scripts to delegate to the right OS script
+/opt/forger/cloudwatch/install.sh
+/opt/forger/cloudwatch/configure.sh "$LOG_GROUP_NAME"
+/opt/forger/cloudwatch/service.sh

data/lib/forger/scripts/cloudwatch/configure.sh

@@ -0,0 +1,84 @@
+#!/bin/bash -eux
+
+if [ $# -eq 0 ]; then
+  command=$(basename "$0")
+  echo "Usage: $command LOG_GROUP_NAME"
+  echo "Examples:"
+  echo "  $command forger"
+  echo "  $command ec2"
+  exit 1
+fi
+LOG_GROUP_NAME=$1
+
+if [ "$OS" == "ubuntu" ]; then
+  CONFIG_FILE=/var/awslogs/etc/awslogs.conf
+  STATE_FILE=/var/awslogs/state/agent-state
+else # amazonlinux2
+  CONFIG_FILE=/etc/awslogs/awslogs.conf
+  STATE_FILE=/var/lib/awslogs/agent-state
+fi
+
+# Inject the CloudWatch Logs configuration file contents
+cat > $CONFIG_FILE <<- EOF
+[general]
+state_file = $STATE_FILE
+
+[/var/log/dmesg]
+file = /var/log/dmesg
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/dmesg
+
+[/var/log/messages]
+file = /var/log/messages
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/messages
+datetime_format = %b %d %H:%M:%S
+
+[/var/log/cloud-init.log]
+file = /var/log/cloud-init.log
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/cloud-init.log
+datetime_format =
+
+[/var/log/cloud-init-output.log]
+file = /var/log/cloud-init-output.log
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/cloud-init-output.log
+datetime_format =
+
+[/var/log/secure]
+file = /var/log/secure
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/secure
+datetime_format =
+
+[/var/log/audit/audit.log]
+file = /var/log/audit/audit.log
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/audit/audit.log
+datetime_format =
+
+[/var/lib/cloud/instance/user-data.txt]
+file = /var/lib/cloud/instance/user-data.txt
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/lib/cloud/instance/user-data.txt
+datetime_format =
+
+[/var/log/messages]
+file = /var/log/messages
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/messages
+datetime_format =
+
+[/var/log/auto-terminate.log]
+file = /var/log/auto-terminate.log
+log_group_name = ${LOG_GROUP_NAME}
+log_stream_name = {instance_id}/var/log/auto-terminate.log
+datetime_format =
+
+EOF
+
+if [ -f /etc/awslogs/awscli.conf ]; then
+  region=$(curl -s 169.254.169.254/latest/meta-data/placement/availability-zone | sed s'/.$//')
+  sed -i -e "s/region = us-east-1/region = $region/g" /etc/awslogs/awscli.conf
+fi

data/lib/forger/scripts/cloudwatch/install.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -eux
+
+/opt/forger/cloudwatch/install/"$OS".sh

data/lib/forger/scripts/cloudwatch/install/amazonlinux2.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -eux
+
+# Install awslogs and the jq JSON parser
+yum install -y awslogs jq

data/lib/forger/scripts/cloudwatch/install/ubuntu.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -eux
+
+# https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/QuickStartEC2Instance.html
+
+REGION=$(curl -s 169.254.169.254/latest/meta-data/placement/availability-zone | sed s'/.$//')
+
+if ! type python ; then
+  apt-get update
+  apt-get install -y python-pip
+fi
+
+# Install awslogs and the jq JSON parser
+curl -s https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py -O
+
+# in order to install awslogs non-interactively we need a filler configfile
+mkdir -p /etc/awslogs
+cat > /etc/awslogs/awslogs.conf <<- EOL
+[general]
+state_file = /var/awslogs/state/agent-state
+## filler config file, will get replaced by configure.sh script
+EOL
+
+python ./awslogs-agent-setup.py --region "$REGION" --non-interactive --configfile=/etc/awslogs/awslogs.conf

data/lib/forger/scripts/cloudwatch/service.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -eux
+
+/opt/forger/cloudwatch/service/"$OS".sh

data/lib/forger/scripts/cloudwatch/service/amazonlinux2.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -exu
+
+# yum install already has configured
+#   /usr/lib/systemd/system/awslogsd.service
+# Restart because we adjust the config with configure.sh
+# The yum awslogs package creates a systemd unit called awslogsd.
+systemctl daemon-reload
+systemctl restart awslogsd
+systemctl enable awslogsd
+# systemctl status awslogsd
+# systemctl is-enabled awslogsd

data/lib/forger/scripts/cloudwatch/service/ubuntu.sh

@@ -0,0 +1,8 @@
+#/bin/bash -exu
+
+# The awslogs-agent-setup.py setup creates a systemd unit is called awslogs.
+systemctl daemon-reload
+systemctl restart awslogs
+systemctl enable awslogs
+# systemctl status awslogs
+# systemctl is-enabled awslogs

data/lib/forger/scripts/shared/functions.sh

@@ -0,0 +1,78 @@
+#!/bin/bash -eux
+
+function configure_aws_cli() {
+  local home_dir
+  home_dir=${1:-/root} # default to /root
+  # Configure aws cli in case it is not yet configured
+  mkdir -p "$home_dir/.aws"
+  if [ ! -f "$home_dir/.aws/config" ]; then
+    EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
+    EC2_REGION=${EC2_AVAIL_ZONE::-1}
+    cat >"$home_dir/.aws/config" <<CONFIGURE_AWS_CLI
+[default]
+region = $EC2_REGION
+output = json
+CONFIGURE_AWS_CLI
+  fi
+}
+
+# Normalize os name so we can delegate out to os specific scripts.
+#
+# Amazon Linux 2
+#   $ cat /etc/os-release
+#   NAME="Amazon Linux"
+#   VERSION="2 (2017.12)"
+#   ID="amzn"
+#   ID_LIKE="centos rhel fedora"
+#   VERSION_ID="2"
+#   PRETTY_NAME="Amazon Linux 2 (2017.12) LTS Release Candidate"
+#   ANSI_COLOR="0;33"
+#   CPE_NAME="cpe:2.3:o:amazon:amazon_linux:2"
+#   HOME_URL="https://amazonlinux.com/"
+#
+# Ubuntu
+#   ubuntu@ip-172-31-6-8:~$ cat /etc/os-release
+#   NAME="Ubuntu"
+#   VERSION="16.04.3 LTS (Xenial Xerus)"
+#   ID=ubuntu
+#   ID_LIKE=debian
+#   PRETTY_NAME="Ubuntu 16.04.3 LTS"
+#   VERSION_ID="16.04"
+#   HOME_URL="http://www.ubuntu.com/"
+#   SUPPORT_URL="http://help.ubuntu.com/"
+#   BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"
+#   VERSION_CODENAME=xenial
+#   UBUNTU_CODENAME=xenial
+#
+# Note: Amazon Linux and Amazon Linux 2 have the same name
+#
+function os_name() {
+  local OS
+  local VERSION
+
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    OS="macosx" # gawk not available on macosx usually
+  else
+    # https://askubuntu.com/questions/459402/how-to-know-if-the-running-platform-is-ubuntu-or-centos-with-help-of-a-bash-scri
+    # Method 1 works for amazonlinux and ubuntu
+    # Method 3 the complex script, did not work for amazonlinux
+    OS=$(gawk -F= '/^NAME/{print $2}' /etc/os-release)
+  fi
+
+  OS="${OS// /}" # remove spaces
+  OS=$(echo "$OS" | tr '[:upper:]' '[:lower:]')
+  # https://stackoverflow.com/questions/9733338/shell-script-remove-first-and-last-quote-from-a-variable
+  OS="${OS#\"}" # remove leading "
+  OS="${OS%\"}" # remove trailing "
+
+  if [ "$OS" == "amazonlinux" ]; then
+    VERSION=$(gawk -F= '/^VERSION/{print $2}' /etc/os-release)
+    VERSION="${VERSION#\"}" # remove leading "
+    if [[ "$VERSION" =~ ^2 ]] ; then
+      OS="${OS}2"
+    fi
+  fi
+
+  echo "$OS"
+}
+

data/lib/forger/setting.rb

@@ -0,0 +1,52 @@
+require 'yaml'
+
+module Forger
+  class Setting
+    def initialize(check_project=true)
+      @check_project = check_project
+    end
+
+    # data contains the settings.yml config.  The order or precedence for settings
+    # is the project lono/settings.yml and then the ~/.lono/settings.yml.
+    @@data = nil
+    def data
+      return @@data if @@data
+
+      if @check_project && !File.exist?(project_settings_path)
+        puts "ERROR: No settings file at #{project_settings_path}.  Are you sure you are in a forger project?".colorize(:red)
+        exit 1
+      end
+
+      all_envs = load_file(project_settings_path)
+      all_envs = merge_base(all_envs)
+      @@data = all_envs[Forger.env] || all_envs["base"] || {}
+    end
+
+  private
+    def load_file(path)
+      return Hash.new({}) unless File.exist?(path)
+
+      content = RenderMePretty.result(path)
+      data = YAML.load(content)
+      # If key is is accidentally set to nil it screws up the merge_base later.
+      # So ensure that all keys with nil value are set to {}
+      data.each do |env, _setting|
+        data[env] ||= {}
+      end
+      data
+    end
+
+    # automatically add base settings to the rest of the environments
+    def merge_base(all_envs)
+      base = all_envs["base"] || {}
+      all_envs.each do |env, settings|
+        all_envs[env] = base.merge(settings) unless env == "base"
+      end
+      all_envs
+    end
+
+    def project_settings_path
+      "#{Forger.root}/config/settings.yml"
+    end
+  end
+end

data/lib/forger/template.rb

@@ -0,0 +1,13 @@
+require "active_support" # for autoload
+require "active_support/core_ext/string"
+
+module Forger
+  module Template
+    autoload :Context, "forger/template/context"
+    autoload :Helper, "forger/template/helper"
+
+    def context
+      @context ||= Forger::Template::Context.new(@options)
+    end
+  end
+end

data/lib/forger/template/context.rb

@@ -0,0 +1,32 @@
+require "active_support/core_ext/string"
+
+# Encapsulates helper methods and instance variables to be rendered in the ERB
+# templates.
+module Forger::Template
+  class Context
+    include Forger::Template::Helper
+
+    def initialize(options={})
+      @options = options
+      load_custom_helpers
+    end
+
+  private
+    # Load custom helper methods from project
+    def load_custom_helpers
+      Dir.glob("#{Forger.root}/app/helpers/**/*_helper.rb").each do |path|
+        filename = path.sub(%r{.*/},'').sub('.rb','')
+        module_name = filename.classify
+
+        # Prepend a period so require works AWS_EC2_ROOT is set to a relative path
+        # without a period.
+        #
+        # Example: AWS_EC2_ROOT=tmp/project
+        first_char = path[0..0]
+        path = "./#{path}" unless %w[. /].include?(first_char)
+        require path
+        self.class.send :include, module_name.constantize
+      end
+    end
+  end
+end

data/lib/forger/template/helper.rb

@@ -0,0 +1,17 @@
+require "active_support/core_ext/string"
+
+module Forger::Template
+  module Helper
+    def autoinclude(klass)
+      autoload klass, "forger/template/helper/#{klass.to_s.underscore}"
+      include const_get(klass)
+    end
+    extend self
+
+    autoinclude :AmiHelper
+    autoinclude :CoreHelper
+    autoinclude :PartialHelper
+    autoinclude :ScriptHelper
+    autoinclude :SshKeyHelper
+  end
+end

data/lib/forger/template/helper/ami_helper.rb

@@ -0,0 +1,33 @@
+module Forger::Template::Helper::AmiHelper
+  include Forger::AwsService
+
+  # Example:
+  #
+  #   latest_ami("ruby-2.5.0_*") => ami-122
+  #
+  # Equivalent aws cli test command:
+  #
+  #   $ aws ec2 describe-images --owners self --filters="Name=name,Values=ruby-2.5.0_*" | jq '.Images | length'
+  #
+  # Returns latest ami ami
+  def latest_ami(query, owners=["self"])
+    images = search_ami(query, owners)
+    image = images.sort_by(&:name).reverse.first
+    if image
+      image.image_id
+    else
+      puts "latest_ami helper method could not find an AMI with the query of: #{query.inspect}".colorize(:red)
+      exit 1
+    end
+  end
+
+  def search_ami(query, owners=["self"])
+    ec2.describe_images(
+      owners: owners,
+      filters: [
+        {name: "name", values: [query]},
+        {name: "state", values: ["available"]}
+      ]
+    ).images
+  end
+end

data/lib/forger/template/helper/core_helper.rb

@@ -0,0 +1,127 @@
+require "base64"
+require "erb"
+
+module Forger::Template::Helper::CoreHelper
+  # assuming user-data script is a bash script for simplicity for now
+  def user_data(name, base64:true, layout:"default")
+    # allow user to specify the path also
+    if File.exist?(name)
+      name = File.basename(name) # normalize name, change path to name
+    end
+    name = File.basename(name, '.sh')
+
+    layout_path = layout_path(layout)
+    path = "#{Forger.root}/app/user-data/#{name}.sh"
+    result = RenderMePretty.result(path, context: self, layout: layout_path)
+    # Must prepend and append scripts in user_data here because we need to
+    # encode the user_data script for valid yaml to load in the profile.
+    # Tried moving this logic to the params but that is too late and produces
+    # invalid yaml.  Unless we want to encode and dedode twice.
+    scripts = [result]
+    scripts = prepend_scripts(scripts)
+    scripts = append_scripts(scripts)
+    divider = "\n############################## DIVIDER ##############################\n"
+    result = scripts.join(divider)
+
+    # save the unencoded user-data script for easy debugging
+    temp_path = "#{Forger.root}/tmp/user-data.txt"
+    FileUtils.mkdir_p(File.dirname(temp_path))
+    IO.write(temp_path, result)
+
+    base64 ? Base64.encode64(result).strip : result
+  end
+
+  # Get full path of layout from layout name
+  #
+  #   layout_name=false - dont use layout at all
+  #   layout_name=nil - default to default.sh layout if available
+  def layout_path(name="default")
+    return false if name == false # disable layout
+    name = "default" if name.nil? # in case user passes in nil
+
+    ext = File.extname(name)
+    name += ".sh" if ext.empty?
+    layout_path = "#{Forger.root}/app/user-data/layouts/#{name}"
+
+    # special rule for default in case there's no default layout
+    if name.include?("default") and !File.exist?(layout_path)
+      return false
+    end
+
+    # other named layouts should error if it doesnt exit
+    unless File.exist?(layout_path)
+      puts "ERROR: Layout #{layout_path} does not exist. Are you sure it exists?  Exiting".colorize(:red)
+      exit 1
+    end
+
+    layout_path
+  end
+
+  # provides access to config/* settings as variables
+  #   AWS_EC2_ENV=development => config/development.yml
+  #   AWS_EC2_ENV=production => config/production.yml
+  def config
+    Forger.config
+  end
+
+  # provides access to config/settings.yml as variables
+  def settings
+    Forger.settings
+  end
+
+  # pretty timestamp that is useful for ami ids.
+  # the timestamp is generated once and cached.
+  def timestamp
+    @timestamp ||= Time.now.strftime("%Y-%m-%d-%H-%M-%S")
+  end
+
+private
+  # TODO: move script combining logic into class
+  def prepend_scripts(scripts)
+    scripts.unshift(script.cloudwatch) if @options[:cloudwatch]
+    scripts.unshift(script.auto_terminate_after_timeout) if @options[:auto_terminate]
+    add_setup_script(scripts, :prepend)
+    scripts
+  end
+
+  def append_scripts(scripts)
+    add_setup_script(scripts, :append)
+    scripts << script.auto_terminate if @options[:auto_terminate]
+    scripts << script.create_ami if @options[:ami_name]
+    scripts
+  end
+
+  def add_setup_script(scripts, how)
+    return if @already_setup
+    @already_setup = true
+
+    requires_setup = @options[:cloudwatch] ||
+                     @options[:auto_terminate] ||
+                     @options[:ami_name]
+
+    return unless requires_setup
+
+    if how == :prepend
+      scripts.unshift(script.extract_forger_scripts)
+    else
+      scripts << script.extract_forger_scripts
+    end
+
+    scripts
+  end
+
+  def script
+    @script ||= Forger::Script.new(@options)
+  end
+
+  # Load custom helper methods from the project repo
+  def load_custom_helpers
+    Dir.glob("#{Forger.root}/app/helpers/**/*_helper.rb").each do |path|
+      filename = path.sub(%r{.*/},'').sub('.rb','')
+      module_name = filename.classify
+
+      require path
+      self.class.send :include, module_name.constantize
+    end
+  end
+end