aws-ec2 0.9.0 → 1.0.0

data/lib/aws_ec2/script/compile.rb

@@ -3,20 +3,29 @@ require 'fileutils'
 # Class for aws-ec2 compile command
 class AwsEc2::Script
   class Compile < AwsEc2::Base
-    include AwsEc2::TemplateHelper
-    BUILD_ROOT = "tmp"
+    include AwsEc2::Template
 
-    def compile
+    # used in upload
+    def compile_scripts
       clean
       compile_folder("scripts")
-      compile_folder("user-data")
     end
 
-    def compile_folder(folder)
+    # use in compile cli command
+    def compile_all
+      clean
+      compile_folder("scripts")
+      layout_path = context.layout_path(@options[:layout])
+      compile_folder("user-data", layout_path)
+    end
+
+    def compile_folder(folder, layout_path=false)
       puts "Compiling app/#{folder}:".colorize(:green)
       Dir.glob("#{AwsEc2.root}/app/#{folder}/**/*").each do |path|
         next if File.directory?(path)
-        result = erb_result(path)
+        next if path.include?("layouts")
+
+        result = RenderMePretty.result(path, layout: layout_path, context: context)
         tmp_path = path.sub(%r{.*/app/}, "#{BUILD_ROOT}/app/")
         puts "  #{tmp_path}"
         FileUtils.mkdir_p(File.dirname(tmp_path))

data/lib/aws_ec2/script/compress.rb

@@ -0,0 +1,62 @@
+require 'fileutils'
+
+class AwsEc2::Script
+  class Compress < AwsEc2::Base
+    def compress
+      reset
+      puts "Tarballing #{BUILD_ROOT}/app/scripts folder to scripts.tgz"
+      tarball_path = create_tarball
+      save_scripts_info(tarball_path)
+      puts "Tarball created at #{tarball_path}"
+    end
+
+    def create_tarball
+      # https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them
+      sh "cd #{BUILD_ROOT}/app && dot_clean ." if system("type dot_clean > /dev/null")
+
+      # https://serverfault.com/questions/110208/different-md5sums-for-same-tar-contents
+      # Using tar czf directly results in a new m5sum each time because the gzip
+      # timestamp is included.  So using:  tar -c ... | gzip -n
+      sh "cd #{BUILD_ROOT}/app && tar -c scripts | gzip -n > scripts.tgz" # temporary app/scripts.tgz file
+
+      rename_with_md5!
+    end
+
+    def clean
+      FileUtils.rm_f("#{BUILD_ROOT}/scripts/scripts-#{md5sum}.tgz")
+    end
+
+    # Apppend a md5 to file after it's been created and moves it to
+    # output/scripts/scripts-[MD5].tgz
+    def rename_with_md5!
+      md5_path = "#{BUILD_ROOT}/scripts/scripts-#{md5sum}.tgz"
+      FileUtils.mkdir_p(File.dirname(md5_path))
+      FileUtils.mv("#{BUILD_ROOT}/app/scripts.tgz", md5_path)
+      md5_path
+    end
+
+    def save_scripts_info(scripts_name)
+      FileUtils.mkdir_p(File.dirname(SCRIPTS_INFO_PATH))
+      IO.write(SCRIPTS_INFO_PATH, scripts_name)
+    end
+
+    # cache this because the file will get removed
+    def md5sum
+      @md5sum ||= Digest::MD5.file("#{BUILD_ROOT}/app/scripts.tgz").to_s[0..7]
+    end
+
+    def sh(command)
+      puts "=> #{command}"
+      system command
+    end
+
+    # Only avaialble after script has been built.
+    def scripts_name
+      IO.read(SCRIPTS_INFO_PATH).strip
+    end
+
+    def reset
+      FileUtils.rm_f(SCRIPTS_INFO_PATH)
+    end
+  end
+end

data/lib/aws_ec2/script/upload.rb

@@ -1,3 +1,5 @@
+require 'filesize'
+require 'aws-sdk-s3'
 require 'fileutils'
 
 # Class for aws-ec2 upload command
@@ -8,17 +10,77 @@ class AwsEc2::Script
       @compile = @options[:compile] ? @options[:compile] : true
     end
 
-    def upload
-      compiler.compile if @compile
-      sync_scripts_to_s3
-      compiler.clean if @compile and AwsEc2.config["compile_clean"]
+    def run
+      compiler.compile_scripts if @compile
+      compressor.compress
+      upload(tarball_path)
+      compressor.clean
+      compiler.clean if @compile and AwsEc2.settings["compile_clean"]
     end
 
-    def sync_scripts_to_s3
-      puts "Uploading tmp/app to s3...".colorize(:green)
-      s3_bucket = AwsEc2.config["scripts_s3_bucket"]
-      s3_path = AwsEc2.config["scripts_s3_path"] || "ec2/app"
-      sh "aws s3 sync tmp/app s3://#{s3_bucket}/#{s3_path}"
+    def upload(tarball_path)
+      puts "Uploading scripts.tgz (#{filesize}) to #{s3_dest}"
+      obj = s3_resource.bucket(bucket_name).object(key)
+      start_time = Time.now
+      if @options[:noop]
+        puts "NOOP: Not uploading file to s3"
+      else
+        obj.upload_file(tarball_path)
+      end
+      time_took = pretty_time(Time.now-start_time).colorize(:green)
+      puts "Time to upload code to s3: #{time_took}"
+    end
+
+    def tarball_path
+      IO.read(SCRIPTS_INFO_PATH).strip
+    end
+
+    def filesize
+      Filesize.from(File.size(tarball_path).to_s + " B").pretty
+    end
+
+    def s3_dest
+      "s3://#{bucket_name}/#{key}"
+    end
+
+    def key
+      # Example key: ec2/development/scripts/scripts-md5
+      "#{dest_folder}/#{File.basename(tarball_path)}"
+    end
+
+    # Example:
+    #   s3_folder: s3://infra-bucket/ec2
+    #   bucket_name: infra-bucket
+    def bucket_name
+      s3_folder.sub('s3://','').split('/').first
+    end
+
+    # Removes s3://bucket-name and adds AwsEc2.env. Example:
+    #   s3_folder: s3://infra-bucket/ec2
+    #   bucket_name: ec2/development/scripts
+    def dest_folder
+      folder = s3_folder.sub('s3://','').split('/')[1..-1].join('/')
+      "#{folder}/#{AwsEc2.env}/scripts"
+    end
+
+    # s3_folder example:
+    def s3_folder
+      AwsEc2.settings["s3_folder"]
+    end
+
+    def s3_resource
+      @s3_resource ||= Aws::S3::Resource.new
+    end
+
+    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
+    def pretty_time(total_seconds)
+      minutes = (total_seconds / 60) % 60
+      seconds = total_seconds % 60
+      if total_seconds < 60
+        "#{seconds.to_i}s"
+      else
+        "#{minutes.to_i}m #{seconds.to_i}s"
+      end
     end
 
     def sh(command)
@@ -29,5 +91,9 @@ class AwsEc2::Script
     def compiler
       @compiler ||= Compile.new(@options)
     end
+
+    def compressor
+      @compressor ||= Compress.new(@options)
+    end
   end
 end

data/lib/aws_ec2/setting.rb

@@ -0,0 +1,41 @@
+require 'yaml'
+
+module AwsEc2
+  class Setting
+    def initialize(check_project=true)
+      @check_project = check_project
+    end
+
+    # data contains the settings.yml config.  The order or precedence for settings
+    # is the project lono/settings.yml and then the ~/.lono/settings.yml.
+    @@data = nil
+    def data
+      return @@data if @@data
+
+      if @check_project && !File.exist?(project_settings_path)
+        puts "ERROR: No settings file at #{project_settings_path}.  Are you sure you are in a aws-ec2 project?".colorize(:red)
+        exit 1
+      end
+
+      all_envs = load_file(project_settings_path)
+      @@data = all_envs[AwsEc2.env]
+    end
+
+  private
+    def load_file(path)
+      return Hash.new({}) unless File.exist?(path)
+
+      content = RenderMePretty.result(path)
+      data = YAML.load(content)
+      # ensure no nil values
+      data.each do |key, value|
+        data[key] = {} if value.nil?
+      end
+      data
+    end
+
+    def project_settings_path
+      "#{AwsEc2.root}/config/settings.yml"
+    end
+  end
+end

data/lib/aws_ec2/template.rb

@@ -0,0 +1,13 @@
+require "active_support" # for autoload
+require "active_support/core_ext/string"
+
+module AwsEc2
+  module Template
+    autoload :Context, "aws_ec2/template/context"
+    autoload :Helper, "aws_ec2/template/helper"
+
+    def context
+      @context ||= AwsEc2::Template::Context.new(@options)
+    end
+  end
+end

data/lib/aws_ec2/template/context.rb

@@ -0,0 +1,32 @@
+require "active_support/core_ext/string"
+
+# Encapsulates helper methods and instance variables to be rendered in the ERB
+# templates.
+module AwsEc2::Template
+  class Context
+    include AwsEc2::Template::Helper
+
+    def initialize(options={})
+      @options = options
+      load_custom_helpers
+    end
+
+  private
+    # Load custom helper methods from project
+    def load_custom_helpers
+      Dir.glob("#{AwsEc2.root}/app/helpers/**/*_helper.rb").each do |path|
+        filename = path.sub(%r{.*/},'').sub('.rb','')
+        module_name = filename.classify
+
+        # Prepend a period so require works AWS_EC2_ROOT is set to a relative path
+        # without a period.
+        #
+        # Example: AWS_EC2_ROOT=tmp/project
+        first_char = path[0..0]
+        path = "./#{path}" unless %w[. /].include?(first_char)
+        require path
+        self.class.send :include, module_name.constantize
+      end
+    end
+  end
+end

data/lib/aws_ec2/template/helper.rb

@@ -0,0 +1,17 @@
+require "active_support/core_ext/string"
+
+module AwsEc2::Template
+  module Helper
+    def autoinclude(klass)
+      autoload klass, "aws_ec2/template/helper/#{klass.to_s.underscore}"
+      include const_get(klass)
+    end
+    extend self
+
+    autoinclude :AmiHelper
+    autoinclude :CoreHelper
+    autoinclude :PartialHelper
+    autoinclude :ScriptHelper
+    autoinclude :SshKeyHelper
+  end
+end

data/lib/aws_ec2/{template_helper → template/helper}/ami_helper.rb

@@ -1,4 +1,4 @@
-module AwsEc2::TemplateHelper::AmiHelper
+module AwsEc2::Template::Helper::AmiHelper
   include AwsEc2::AwsServices
 
   # Example:
@@ -13,11 +13,16 @@ module AwsEc2::TemplateHelper::AmiHelper
   def latest_ami(query, owners=["self"])
     images = search_ami(query, owners)
     image = images.sort_by(&:name).reverse.first
-    image.image_id
+    if image
+      image.image_id
+    else
+      puts "latest_ami helper method could not find an AMI with the query of: #{query.inspect}".colorize(:red)
+      exit 1
+    end
   end
 
   def search_ami(query, owners=["self"])
-    images = ec2.describe_images(
+    ec2.describe_images(
       owners: owners,
       filters: [
         {name: "name", values: [query]}

data/lib/aws_ec2/template/helper/core_helper.rb

@@ -0,0 +1,88 @@
+require "base64"
+require "erb"
+
+module AwsEc2::Template::Helper::CoreHelper
+  def user_data(name, base64:true, layout:"default")
+    # allow user to specify the path also
+    if File.exist?(name)
+      name = File.basename(name) # normalize name, change path to name
+    end
+    name = File.basename(name, '.sh')
+
+    layout_path = layout_path(layout)
+    path = "#{AwsEc2.root}/app/user-data/#{name}.sh"
+    result = RenderMePretty.result(path, context: self, layout: layout_path)
+    result = append_scripts(result)
+
+    # save the unencoded user-data script for easy debugging
+    temp_path = "#{AwsEc2.root}/tmp/user-data.txt"
+    FileUtils.mkdir_p(File.dirname(temp_path))
+    IO.write(temp_path, result)
+
+    base64 ? Base64.encode64(result).strip : result
+  end
+
+  # Get full path of layout from layout name
+  #
+  #   layout_name=false - dont use layout at all
+  #   layout_name=nil - default to default.sh layout if available
+  def layout_path(name="default")
+    return false if name == false # disable layout
+    name = "default" if name.nil? # in case user passes in nil
+
+    ext = File.extname(name)
+    name += ".sh" if ext.empty?
+    layout_path = "#{AwsEc2.root}/app/user-data/layouts/#{name}"
+
+    # special rule for default in case there's no default layout
+    if name.include?("default") and !File.exist?(layout_path)
+      return false
+    end
+
+    # other named layouts should error if it doesnt exit
+    unless File.exist?(layout_path)
+      puts "ERROR: Layout #{layout_path} does not exist. Are you sure it exists?  Exiting".colorize(:red)
+      exit 1
+    end
+
+    layout_path
+  end
+
+  # provides access to config/* settings as variables
+  #   AWS_EC2_ENV=development => config/development.yml
+  #   AWS_EC2_ENV=production => config/production.yml
+  def config
+    AwsEc2.config
+  end
+
+  # provides access to config/settings.yml as variables
+  def settings
+    AwsEc2.settings
+  end
+
+  # pretty timestamp that is useful for ami ids.
+  # the timestamp is generated once and cached.
+  def timestamp
+    @timestamp ||= Time.now.strftime("%Y-%m-%d-%H-%M-%S")
+  end
+
+private
+  def append_scripts(user_data)
+    # assuming user-data script is a bash script for simplicity
+    script = AwsEc2::Script.new(@options)
+    user_data += script.auto_terminate if @options[:auto_terminate]
+    user_data += script.create_ami if @options[:ami_name]
+    user_data
+  end
+
+  # Load custom helper methods from the project repo
+  def load_custom_helpers
+    Dir.glob("#{AwsEc2.root}/app/helpers/**/*_helper.rb").each do |path|
+      filename = path.sub(%r{.*/},'').sub('.rb','')
+      module_name = filename.classify
+
+      require path
+      self.class.send :include, module_name.constantize
+    end
+  end
+end

data/lib/aws_ec2/{template_helper → template/helper}/partial_helper.rb

@@ -1,4 +1,4 @@
-module AwsEc2::TemplateHelper::PartialHelper
+module AwsEc2::Template::Helper::PartialHelper
   def partial_exist?(path)
     path = partial_path_for(path)
     path = auto_add_format(path)
@@ -17,7 +17,7 @@ module AwsEc2::TemplateHelper::PartialHelper
     path = partial_path_for(path)
     path = auto_add_format(path)
 
-    result = erb_result(path)
+    result = RenderMePretty.result(path, context: self)
     result = indent(result, options[:indent]) if options[:indent]
     if options[:indent]
       # Add empty line at beginning because empty lines gets stripped during

data/lib/aws_ec2/template/helper/script_helper.rb

@@ -0,0 +1,53 @@
+module AwsEc2::Template::Helper::ScriptHelper
+  # Bash code that is meant to included in user-data
+  def extract_scripts(options={})
+    check_s3_folder_settings!
+
+    settings_options = settings["extract_scripts"] || {}
+    options = settings_options.merge(options)
+    # defaults also here in case they are removed from settings
+    to = options[:to] || "/opt"
+    user = options[:as] || "ec2-user"
+
+    if Dir.glob("#{AwsEc2.root}/app/scripts*").empty?
+      puts "WARN: you are using the extract_scripts helper method but you do not have any app/scripts.".colorize(:yellow)
+      calling_line = caller[0].split(':')[0..1].join(':')
+      puts "Called from: #{calling_line}"
+      return ""
+    end
+
+    <<-BASH_CODE
+# Generated from the aws-ec2 extract_scripts helper.
+# Downloads scripts from s3, extract them, and setup.
+mkdir -p #{to}
+aws s3 cp #{scripts_s3_path} #{to}/
+(
+  cd #{to}
+  tar zxf #{to}/#{scripts_name}
+  chmod -R a+x #{to}/scripts
+  chown -R #{user}:#{user} #{to}/scripts
+)
+BASH_CODE
+  end
+
+private
+  def check_s3_folder_settings!
+    return if settings["s3_folder"]
+
+    puts "Helper method called that requires the s3_folder to be set at:"
+    lines = caller.reject { |l| l =~ %r{lib/aws-ec2} } # hide internal aws-ec2 trace
+    puts "  #{lines[0]}"
+
+    puts "Please configure your config/settings.yml with an s3_folder.".colorize(:red)
+    exit 1
+  end
+
+  def scripts_name
+    File.basename(scripts_s3_path)
+  end
+
+  def scripts_s3_path
+    upload = AwsEc2::Script::Upload.new
+    upload.s3_dest
+  end
+end