forger 2.0.5 → 3.0.0

data/lib/forger/s3/bucket.rb

@@ -0,0 +1,131 @@
+require "cfn/status"
+
+class Forger::S3
+  class Bucket
+    STACK_NAME = ENV['FORGER_STACK_NAME'] || "forger"
+    include Forger::AwsServices
+    extend Forger::AwsServices
+
+    class << self
+      @@name = nil
+      def name
+        return @@name if @@name # only memoize once bucket has been created
+
+        stack = new.find_stack
+        return unless stack
+
+        resp = cfn.describe_stack_resources(stack_name: STACK_NAME)
+        bucket = resp.stack_resources.find { |r| r.logical_resource_id == "Bucket" }
+        @@name = bucket.physical_resource_id # actual bucket name
+      end
+
+      def ensure_exists!
+        bucket = new
+        return if bucket.exist?
+        bucket.create
+      end
+    end
+
+    def initialize(options={})
+      @options = options
+    end
+
+    def deploy
+      stack = find_stack
+      if stack
+        update
+      else
+        create
+      end
+    end
+
+    def exist?
+      !!bucket_name
+    end
+
+    def bucket_name
+      self.class.name
+    end
+
+    def show
+      if bucket_name
+        puts "Forger bucket name: #{bucket_name}"
+      else
+        puts "Forger bucket does not exist yet."
+      end
+    end
+
+    # Launches a cloudformation to create an s3 bucket
+    def create
+      puts "Creating #{STACK_NAME} stack with the s3 bucket"
+      cfn.create_stack(stack_name: STACK_NAME, template_body: template_body)
+      status = ::Cfn::Status.new(STACK_NAME)
+      status.wait
+    end
+
+    def update
+      puts "Updating #{STACK_NAME} stack with the s3 bucket"
+      cfn.update_stack(stack_name: STACK_NAME, template_body: template_body)
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      raise unless e.message.include?("No updates are to be performed")
+    end
+
+    def delete
+      are_you_sure?
+
+      puts "Deleting #{STACK_NAME} stack with the s3 bucket"
+      empty_bucket!
+      cfn.delete_stack(stack_name: STACK_NAME)
+    end
+
+    def find_stack
+      resp = cfn.describe_stacks(stack_name: STACK_NAME)
+      resp.stacks.first
+    rescue Aws::CloudFormation::Errors::ValidationError
+      nil
+    end
+
+  private
+
+    def empty_bucket!
+      resp = s3.list_objects(bucket: bucket_name)
+      if resp.contents.size > 0
+        # IE: objects = [{key: "objectkey1"}, {key: "objectkey2"}]
+        objects = resp.contents.map { |item| {key: item.key} }
+        s3.delete_objects(
+          bucket: bucket_name,
+          delete: {
+            objects: objects,
+            quiet: false,
+          }
+        )
+        empty_bucket! # keep deleting objects until bucket is empty
+      end
+    end
+
+
+    def are_you_sure?
+      return true if @options[:sure]
+
+      puts "Are you sure you want the forger bucket #{bucket_name.color(:green)} to be emptied and deleted? (y/N)"
+      sure = $stdin.gets.strip
+      yes = sure =~ /^Y/i
+      unless yes
+        puts "Phew that was close."
+        exit
+      end
+    end
+
+    def template_body
+      <<~YAML
+        Resources:
+          Bucket:
+            Type: AWS::S3::Bucket
+            Properties:
+              Tags:
+                - Key: Name
+                  Value: forger
+      YAML
+    end
+  end
+end

data/lib/forger/script.rb

@@ -1,9 +1,5 @@
 module Forger
   class Script
-    autoload :Compile, "forger/script/compile"
-    autoload :Compress, "forger/script/compress"
-    autoload :Upload, "forger/script/upload"
-
     def initialize(options={})
       @options = options
     end

data/lib/forger/script/upload.rb

@@ -15,12 +15,17 @@ class Forger::Script
 
     def run
       compiler.compile_scripts if @compile
+      ensure_bucket_exists
       compressor.compress
       upload(tarball_path)
       compressor.clean
       compiler.clean if @compile and Forger.settings["compile_clean"]
     end
 
+    def ensure_bucket_exists
+      Forger::S3::Bucket.ensure_exists! if Forger::Template::Helper.extract_scripts_registered?
+    end
+
     def upload(tarball_path)
       if @options[:noop]
         puts "NOOP: Not uploading file to s3"
@@ -28,32 +33,13 @@ class Forger::Script
       end
 
       puts "Uploading scripts.tgz (#{filesize}) to #{s3_dest}".color(:green)
-      obj = s3_resource.bucket(bucket_name).object(key)
+      obj = s3_resource.bucket(bucket_name).object(s3_key)
       start_time = Time.now
-      upload_to_s3(obj, tarball_path)
+      obj.upload_file(tarball_path)
       time_took = pretty_time(Time.now-start_time).color(:green)
       puts "Time to upload code to s3: #{time_took}"
     end
 
-    def upload_to_s3(obj, tarball_path)
-      obj.upload_file(tarball_path)
-    rescue Aws::S3::Errors::PermanentRedirect => e
-      puts "ERROR: #{e.class} #{e.message}".color(:red)
-      puts "The bucket you are trying to upload scripts to is in a different region than the region the instance is being launched in."
-      puts "You must configured FORGER_S3_ENDPOINT env variable to prevent this error. Example:"
-      puts "  FORGER_S3_ENDPOINT=https://s3.us-west-2.amazonaws.com"
-      puts "Check your ~/.aws/config for the region being used for the ec2 instance."
-      exit 1
-    rescue Aws::S3::Errors::AccessDenied, Aws::S3::Errors::AllAccessDisabled
-      e = $!
-      puts "ERROR: #{e.class} #{e.message}".color(:red)
-      puts "You do not have permission to upload scripts to this bucket: #{bucket_name}.  Are you sure the right bucket is configured?"
-      if ENV['AWS_PROFILE']
-        puts "Also maybe check your AWS_PROFILE env. Current AWS_PROFILE=#{ENV['AWS_PROFILE']}"
-      end
-      exit 1
-    end
-
     def empty?
       Dir.glob("#{Forger.root}/app/scripts/**/*").select do |path|
         File.file?(path)
@@ -69,33 +55,17 @@ class Forger::Script
     end
 
     def s3_dest
-      "s3://#{bucket_name}/#{key}"
+      "s3://#{bucket_name}/#{s3_key}"
     end
 
-    def key
-      # Example key: ec2/development/scripts/scripts-md5
+    def s3_key
+      # Example s3_key: ec2/development/scripts/scripts-md5
+      dest_folder = "#{Forger.env}/scripts"
       "#{dest_folder}/#{File.basename(tarball_path)}"
     end
 
-    # Example:
-    #   s3_folder: s3://infra-bucket/ec2
-    #   bucket_name: infra-bucket
     def bucket_name
-      s3_folder.sub('s3://','').split('/').first
-    end
-
-    # Removes s3://bucket-name and adds Forger.env. Example:
-    #   s3_folder: s3://infra-bucket/ec2
-    #   bucket_name: ec2/development/scripts
-    def dest_folder
-      folder = s3_folder.sub('s3://','').split('/')[1..-1].join('/')
-      folder = nil if folder == ''
-      [folder, "#{Forger.env}/scripts"].compact.join('/')
-    end
-
-    # s3_folder example:
-    def s3_folder
-      Forger.settings["s3_folder"]
+      Forger::S3::Bucket.name
     end
 
     def s3_resource

data/lib/forger/scripts/cloudwatch.sh

@@ -13,8 +13,8 @@ LOG_GROUP_NAME=$1
 # shellcheck disable=SC1091
 source "/opt/forger/shared/functions.sh"
 OS=$(os_name)
-if [ "$OS" != "amzn" ] && [ "$OS" != "amazonlinux2" ] && [ "$OS" != "ubuntu" ] ; then
-  echo "Sorry, cloudwatch logging with the forger tool is supported for amazonlinux2 and ubuntu only"
+if [ "$OS" != "amzn" ] && [ "$OS" != "amzn2" ] && [ "$OS" != "ubuntu" ] ; then
+  echo "Sorry, cloudwatch logging with the forger tool is supported for amzn2 and ubuntu only"
   exit
 fi
 

data/lib/forger/scripts/shared/functions.sh

@@ -63,7 +63,7 @@ function os_name() {
   OS="${OS#\"}" # remove leading "
   OS="${OS%\"}" # remove trailing "
 
-  if [ "$OS" == "amazonlinux" ]; then
+  if [ "$OS" == "amzn" ]; then
     VERSION=$(gawk -F= '/^VERSION/{print $2}' /etc/os-release)
     VERSION="${VERSION#\"}" # remove leading "
     if [[ "$VERSION" =~ ^2 ]] ; then

data/lib/forger/setting.rb

@@ -20,41 +20,9 @@ module Forger
       all_envs = load_file(project_settings_path)
       all_envs = merge_base(all_envs)
       @@data = all_envs[Forger.env] || all_envs["base"] || {}
-      @@data = flatten_s3_folder(@@data)
       @@data
     end
 
-    # So we can access settings['s3_folder'] transparently
-    def flatten_s3_folder(data)
-      # data = data.clone
-      if data['s3_folder'].is_a?(Hash)
-        data['s3_folder'] = s3_folder
-      end
-      data
-    end
-
-    # Special helper method to support multiple formats for s3_folder setting.
-    # Format 1: Simple String
-    #
-    #   development:
-    #     s3_folder: mybucket/path/to/folder
-    #
-    # Format 2: Hash
-    #
-    #   development:
-    #     s3_folder:
-    #       default: mybucket/path/to/folder
-    #       dev_profile1: mybucket/path/to/folder
-    #       dev_profile1: another-bucket/storage/path
-    #
-    def s3_folder
-      s3_folder = data['s3_folder']
-      return s3_folder if s3_folder.nil? or s3_folder.is_a?(String)
-
-      # If reach here then the s3_folder is a Hash
-      s3_folder[ENV['AWS_PROFILE']] || s3_folder["default"]
-    end
-
   private
     def load_file(path)
       return Hash.new({}) unless File.exist?(path)

data/lib/forger/template.rb

@@ -3,9 +3,6 @@ require "active_support/core_ext/string"
 
 module Forger
   module Template
-    autoload :Context, "forger/template/context"
-    autoload :Helper, "forger/template/helper"
-
     def context
       @context ||= Forger::Template::Context.new(@options)
     end

data/lib/forger/template/context.rb

@@ -8,15 +8,30 @@ module Forger::Template
 
     def initialize(options={})
       @options = options
+      load_variables
       load_custom_helpers
     end
 
   private
+    # Load variables from:
+    #   config/variables/development.rb
+    #   config/variables/production.rb
+    #   etc
+    def load_variables
+      load_variables_file(:base)
+      load_variables_file(Forger.env)
+    end
+
+    def load_variables_file(type)
+      path = "#{Forger.root}/config/variables/#{type}.rb"
+      instance_eval(IO.read(path), path) if File.exist?(path)
+    end
+
     # Load custom helper methods from project
     def load_custom_helpers
       Dir.glob("#{Forger.root}/app/helpers/**/*_helper.rb").each do |path|
         filename = path.sub(%r{.*/},'').sub('.rb','')
-        module_name = filename.classify
+        module_name = filename.camelize
 
         # Prepend a period so require works FORGER_ROOT is set to a relative path
         # without a period.

data/lib/forger/template/helper.rb

@@ -2,16 +2,16 @@ require "active_support/core_ext/string"
 
 module Forger::Template
   module Helper
-    def autoinclude(klass)
-      autoload klass, "forger/template/helper/#{klass.to_s.underscore}"
-      include const_get(klass)
-    end
+    include AmiHelper
+    include CoreHelper
+    include PartialHelper
+    include ScriptHelper
+    include SshKeyHelper
     extend self
 
-    autoinclude :AmiHelper
-    autoinclude :CoreHelper
-    autoinclude :PartialHelper
-    autoinclude :ScriptHelper
-    autoinclude :SshKeyHelper
+    @@extract_scripts_registered = false
+    def extract_scripts_registered?
+      @@extract_scripts_registered
+    end
   end
 end

data/lib/forger/template/helper/ami_helper.rb

@@ -1,5 +1,5 @@
 module Forger::Template::Helper::AmiHelper
-  include Forger::AwsService
+  include Forger::AwsServices
 
   # Example:
   #

data/lib/forger/template/helper/core_helper.rb

@@ -10,7 +10,7 @@ module Forger::Template::Helper::CoreHelper
     path = "#{Forger.root}/app/user_data/#{name}"
     unless File.exist?(path)
       puts "ERROR: user-data script #{path.color(:red)} does not exist"
-      exit
+      exit 1
     end
     result = RenderMePretty.result(path, context: self, layout: layout_path)
     # Must prepend and append scripts in user_data here because we need to
@@ -58,11 +58,12 @@ module Forger::Template::Helper::CoreHelper
   end
 
   # provides access to config/* settings as variables
-  #   FORGER_ENV=development => config/development.yml
-  #   FORGER_ENV=production => config/production.yml
-  def config
-    Forger.config
+  #   FORGER_ENV=development => config/variables/development.rb
+  #   FORGER_ENV=production  => config/variables/production.rb
+  def vars
+    Forger.vars
   end
+  alias_method :variables, :vars
 
   # provides access to config/settings.yml as variables
   def settings
@@ -122,7 +123,7 @@ private
   def load_custom_helpers
     Dir.glob("#{Forger.root}/app/helpers/**/*_helper.rb").each do |path|
       filename = path.sub(%r{.*/},'').sub('.rb','')
-      module_name = filename.classify
+      module_name = filename.camelize
 
       require path
       self.class.send :include, module_name.constantize

data/lib/forger/template/helper/script_helper.rb

@@ -1,7 +1,7 @@
 module Forger::Template::Helper::ScriptHelper
   # Bash code that is meant to included in user-data
   def extract_scripts(options={})
-    check_s3_folder_settings!
+    @@extract_scripts_registered = true
 
     settings_options = settings["extract_scripts"] || {}
     options = settings_options.merge(options)
@@ -23,6 +23,7 @@ mkdir -p #{to}
 aws s3 cp #{scripts_s3_path} #{to}/
 (
   cd #{to}
+  rm -rf #{to}/scripts # cleanup
   tar zxf #{to}/#{scripts_name}
   chmod -R a+x #{to}/scripts
   chown -R #{user}:#{user} #{to}/scripts
@@ -31,15 +32,8 @@ BASH_CODE
   end
 
 private
-  def check_s3_folder_settings!
-    return if settings["s3_folder"]
+  def register_extract_scripts
 
-    puts "The extract_scripts helper method aws called.  It requires the s3_folder to be set at:"
-    lines = caller.reject { |l| l =~ %r{lib/forger} } # hide internal forger trace
-    puts "  #{lines[0]}"
-
-    puts "Please configure your config/settings.yml with an s3_folder.".color(:red)
-    exit 1
   end
 
   def scripts_name

data/lib/forger/version.rb

@@ -1,3 +1,3 @@
 module Forger
-  VERSION = "2.0.5"
+  VERSION = "3.0.0"
 end