lono 2.1.0 → 3.0.0

data/lib/lono/cfn/create.rb

@@ -26,9 +26,34 @@ class Lono::Cfn::Create < Lono::Cfn::Base
     cfn.create_stack(
       stack_name: @stack_name,
       template_body: template_body,
-      parameters: params#,
-      # capabilities: ["CAPABILITY_IAM"]
+      parameters: params,
+      capabilities: capabilities, # ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
+      disable_rollback: !@options[:rollback],
     )
     puts message unless @options[:mute]
   end
+
+  # Appends a short random string at the end of a stack name.
+  # Later we will strip this same random string from the template name.
+  # Very makes it convenient.  We can just type:
+  #
+  #   lono cfn create main --randomize-stack-name
+  #
+  # instead of:
+  #
+  #   lono cfn create main-[RANDOM] --template main
+  #
+  # The randomize_stack_name can be specified at the CLI but can also be saved as a
+  # preference.
+  #
+  # It is not a default setting because it might confuse new lono users.
+  def randomize(stack_name)
+    if randomize_stack_name?
+      random = (0...3).map { (65 + rand(26)).chr }.join.downcase # Ex: jhx
+      [stack_name, random].join('-')
+    else
+      stack_name
+    end
+  end
+
 end

data/lib/lono/cfn/help.rb

@@ -12,17 +12,17 @@ The above command will generate and use the template in output/my-stack.json and
 
 Here are examples of overriding the template and params name conventions.
 
-$ lono cfn create my-stack --template different-name1
+$ lono cfn create my-stack --template different1
 
-The template that will be use is output/different-name1.json and the parameters will use params/different-name1.json.
+The template used is output/different1.json and the parameters used is output/params/prod/different1.json.
 
-$ lono cfn create my-stack --params different-name2
+$ lono cfn create my-stack --params different2
 
-The template that will be use is output/different-name2.json and the parameters will use params/different-name2.json.
+The template used is output/my-stack.json and the parameters used is output/params/prod/different2.json.
 
-$ lono cfn create my-stack --template different-name3 --params different-name4
+$ lono cfn create my-stack --template different3 --params different4
 
-The template that will be use is output/different-name3.json and the parameters will use params/different-name4.json.
+The template used is output/different3.json and the parameters used is output/params/prod/different4.json.
 
 EOL
     end
@@ -39,17 +39,17 @@ The above command will generate and use the template in output/my-stack.json and
 
 Here are examples of overriding the template and params name conventions.
 
-$ lono cfn update my-stack --template different-name1
+$ lono cfn update my-stack --template different1
 
-The template that will be use is output/different-name1.json and the parameters will use params/different-name1.json.
+The template used is output/different1.json and the parameters used is output/params/prod/different1.json.
 
-$ lono cfn update my-stack --params different-name2
+$ lono cfn update my-stack --params different2
 
-The template that will be use is output/different-name2.json and the parameters will use params/different-name2.json.
+The template used is output/my-stack.json and the parameters used is output/params/prod/different2.json.
 
-$ lono cfn update my-stack --template different-name3 --params different-name4
+$ lono cfn update my-stack --template different3 --params different4
 
-The template that will be use is output/different-name3.json and the parameters will use params/different-name4.json.
+The template used is output/different3.json and the parameters used is output/params/prod/different4.json.
 
 EOL
     end

data/lib/lono/cfn/preview.rb

@@ -23,22 +23,23 @@ class Lono::Cfn::Preview < Lono::Cfn::Base
     exist_unless_updatable(stack_status(@stack_name))
 
     template_body = IO.read(@template_path)
-    # begin
+    begin
       cfn.create_change_set(
         change_set_name: change_set_name,
         stack_name: @stack_name,
         template_body: template_body,
-        parameters: params
+        parameters: params,
+        capabilities: capabilities, # ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
       )
-    # rescue Aws::CloudFormation::Errors::ValidationError => e
-    #   if e.message =~ /^Parameters: /
-    #     puts "Error creating CloudFormation preview because invalid CloudFormation parameters. Full error message:".colorize(:red)
-    #     puts e.message
-    #     quit(1)
-    #   else
-    #     raise
-    #   end
-    # end
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      if e.message =~ /^Parameters: /
+        puts "Error creating CloudFormation preview because invalid CloudFormation parameters. Full error message:".colorize(:red)
+        puts e.message
+        quit(1)
+      else
+        raise
+      end
+    end
     true
   end
 

data/lib/lono/cfn/update.rb

@@ -38,8 +38,9 @@ class Lono::Cfn::Update < Lono::Cfn::Base
       cfn.update_stack(
         stack_name: @stack_name,
         template_body: template_body,
-        parameters: params#,
-        # capabilities: ["CAPABILITY_IAM"]
+        parameters: params,
+        capabilities: capabilities, # ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
+        disable_rollback: !@options[:rollback],
       )
     rescue Aws::CloudFormation::Errors::ValidationError => e
       puts "ERROR: #{e.message}".red

data/lib/lono/default/settings.yml

@@ -0,0 +1,13 @@
+aws_profile_lono_env_map:
+  default: prod
+  # More examples:
+  # aws_profile1: prod
+  # aws_profile2: stag
+  # aws_profile3: dev
+s3:
+  path:
+    default:  # default is nil
+    # You can specify different buckets for different LONO_ENV.  Examples:
+    # prod: s3://mybucket/templates/storage/path
+    # stag: s3://mybucket/templates/storage/path
+randomize_stack_name: false

data/lib/lono/env.rb

@@ -0,0 +1,11 @@
+class Lono::Env
+  def self.setup!(project_root='.')
+    settings = Lono::Settings.new(project_root).data
+    map = settings['aws_profile_lono_env_map']
+
+    lono_env = map[ENV['AWS_PROFILE']] || map['default'] || 'prod' # defaults to prod
+    lono_env = ENV['LONO_ENV'] if ENV['LONO_ENV'] # highest precedence
+
+    Kernel.const_set(:LONO_ENV, lono_env)
+  end
+end

data/lib/lono/param.rb

@@ -9,7 +9,7 @@ class Lono::Param < Lono::Command
   class_option :mute, type: :boolean
   class_option :project_root, desc: "project root to use", default: '.'
 
-  desc "generate NAME", "generate parameter json file for NAME"
+  desc "generate", "generate all parameter files to json format"
   long_desc Help.generate
   option :path, desc: "Name of the source that maps to the params txt file.  name -> params/NAME.txt.  Use this to override the params/NAME.txt convention"
   def generate

data/lib/lono/param/generator.rb

@@ -2,44 +2,134 @@ class Lono::Param::Generator
   def self.generate_all(options)
     puts "Generating params files"
     project_root = options[:project_root] || '.'
-    Dir.glob("#{project_root}/params/**/*.txt").each do |path|
-      next if File.directory?(path)
-      name = path.sub(/.*params\//, '').sub('.txt', '')
+
+    params = param_names(project_root, "base") + param_names(project_root, LONO_ENV)
+    params.uniq.each do |name|
       param = Lono::Param::Generator.new(name, options)
       param.generate
     end
   end
 
+  # Returns param names
+  # Example:
+  # Given params:
+  #   params/base/a.txt params/base/b.txt params/base/c.txt
+  # Returns:
+  #   param_names("base") => ["a", "b", "c"]
+  def self.param_names(project_root, folder)
+    base_folder = "#{project_root}/params/#{folder}" # Example: "./params/base"
+    Dir.glob("#{base_folder}/**/*.txt").map do |path|
+      path.sub("#{base_folder}/", '').sub('.txt','')
+    end
+  end
+
   def initialize(name, options)
-    @name = name
-    @options = options
-    @project_root = options[:project_root] || '.'
-    @source_path = options[:path] || "#{@project_root}/params/#{@name}.txt"
+    @_name = "#{LONO_ENV}/#{name}"
+    @_options = options
+    @_project_root = options[:project_root] || '.'
+    @_env_path = options[:path] || "#{@_project_root}/params/#{@_name}.txt"
+    @_base_path = @_env_path.sub("/#{LONO_ENV}/", "/base/")
   end
 
   def generate
     # useful option for lono cfn
-    return if @options[:allow_no_file] && !File.exist?(@source_path)
+    return if @_options[:allow_no_file] && !source_exist?
 
-    if File.exist?(@source_path)
-      contents = IO.read(@source_path)
+    if source_exist?
+      contents = overlay_sources
       data = convert_to_cfn_format(contents)
       json = JSON.pretty_generate(data)
       write_output(json)
-      puts "Params file generated for #{@name} at #{output_path}" unless @options[:mute]
+      # Example: @_name = stag/ecs/private
+      #          pretty_name = ecs/private
+      pretty_name = @_name.sub("#{LONO_ENV}/", '')
+      puts "Params file generated for #{pretty_name} at #{output_path}" unless @_options[:mute]
     else
-      puts "#{@source_path} could not be found?  Are you sure it exist?"
+      puts "#{@_base_path} or #{@_env_path} could not be found?  Are you sure it exist?"
       exit 1
     end
+    json
+  end
+
+  # Reads both the base source and env source and overlay the two
+  # Example 1:
+  #   params/base/mystack.txt - base path
+  #   params/prod/mystack.txt - env path
+  #
+  #   the base/mystack.txt gets combined with the prod/mystack.txt
+  #   it produces a final prod/mystack.txt
+  #
+  # Example 2:
+  #   params/base/mystack.txt - base path
+  #
+  #   the base/mystack.txt is used to produced a prod/mystack.txt
+  #
+  # Example 3:
+  #   params/prod/mystack.txt - env path
+  #
+  #   the prod/mystack.txt is used to produced a prod/mystack.txt
+  def overlay_sources
+    contents = []
+    contents << process_erb(@_base_path)
+    contents << process_erb(@_env_path)
+    contents.compact.join("\n")
+  end
+
+  def process_erb(path)
+    if File.exist?(path)
+      template = IO.read(path)
+      erb_result(path, template)
+    end
+  end
+
+  def erb_result(path, template)
+    load_variables
+    begin
+      ERB.new(template, nil, "-").result(binding)
+    rescue Exception => e
+      puts e
+      puts e.backtrace if ENV['DEBUG']
+
+      # how to know where ERB stopped? - https://www.ruby-forum.com/topic/182051
+      # syntax errors have the (erb):xxx info in e.message
+      # undefined variables have (erb):xxx info in e.backtrac
+      error_info = e.message.split("\n").grep(/\(erb\)/)[0]
+      error_info ||= e.backtrace.grep(/\(erb\)/)[0]
+      raise unless error_info # unable to find the (erb):xxx: error line
+      line = error_info.split(':')[1].to_i
+      puts "Error evaluating ERB template on line #{line.to_s.colorize(:red)} of: #{path.sub(/^\.\//, '').colorize(:green)}"
+
+      template_lines = template.split("\n")
+      context = 5 # lines of context
+      top, bottom = [line-context-1, 0].max, line+context-1
+      spacing = template_lines.size.to_s.size
+      template_lines[top..bottom].each_with_index do |line_content, index|
+        line_number = top+index+1
+        if line_number == line
+          printf("%#{spacing}d %s\n".colorize(:red), line_number, line_content)
+        else
+          printf("%#{spacing}d %s\n", line_number, line_content)
+        end
+      end
+      exit 1 unless ENV['TEST']
+    end
+  end
+
+  # Checks both base and source path for existing of the param file.
+  # Example:
+  #   params/base/mystack.txt - base path
+  #   params/prod/mystack.txt - source path
+  def source_exist?
+    File.exist?(@_base_path) || File.exist?(@_env_path)
   end
 
   # useful for when calling CloudFormation via the aws-sdk gem
-  def params
+  def params(casing = :underscore)
     # useful option for lono cfn
-    return {} if @options[:allow_no_file] && !File.exist?(@source_path)
+    return {} if @_options[:allow_no_file] && !source_exist?
 
-    contents = IO.read(@source_path)
-    convert_to_cfn_format(contents, :underscore)
+    contents = overlay_sources
+    convert_to_cfn_format(contents, casing)
   end
 
   def parse_contents(contents)
@@ -55,18 +145,27 @@ class Lono::Param::Generator
 
   def convert_to_cfn_format(contents, casing=:camel)
     lines = parse_contents(contents)
-    params = []
+
+    # First use a Hash structure so that overlay env files will override
+    # the base param file.
+    data = {}
     lines.each do |line|
       key,value = line.strip.split("=").map {|x| x.strip}
+      data[key] = value
+    end
+
+    # Now build up the aws json format for parameters
+    params = []
+    data.each do |key,value|
       param = if value == "use_previous_value"
                 {
-                  ParameterKey: key,
-                  UsePreviousValue: true
+                  "ParameterKey": key,
+                  "UsePreviousValue": true
                 }
               elsif value
                 {
-                  ParameterKey: key,
-                  ParameterValue: value
+                  "ParameterKey": key,
+                  "ParameterValue": value
                 }
               end
       if param
@@ -78,7 +177,7 @@ class Lono::Param::Generator
   end
 
   def output_path
-    "#{@project_root}/output/params/#{@name}.json".sub(/\.\//,'')
+    "#{@_project_root}/output/params/#{@_name}.json".sub(/\.\//,'')
   end
 
   def write_output(json)
@@ -87,4 +186,27 @@ class Lono::Param::Generator
     IO.write(output_path, json)
   end
 
+
+  def load_variables
+    load_variables_folder("base")
+    load_variables_folder(LONO_ENV)
+  end
+
+  # Load the variables defined in config/variables/* to make available the params/*.txt files
+  #
+  # Example:
+  #
+  #   `config/variables/base/variables.rb`:
+  #      @ami = 123
+  #
+  #   `params/ecs/private.txt`:
+  #     AmiId=<%= @ami %>
+  #
+  def load_variables_folder(folder)
+    paths = Dir.glob("#{@_project_root}/config/variables/#{folder}/**/*")
+    paths.select{ |e| File.file? e }.each do |path|
+      instance_eval(IO.read(path))
+    end
+  end
+
 end

data/lib/lono/settings.rb

@@ -0,0 +1,28 @@
+class Lono::Settings
+  def initialize(project_root=nil)
+    @project_root = project_root || '.'
+  end
+
+  def data
+    return @settings_yaml if @settings_yaml
+
+    project_file = "#{@project_root}/.lono/settings.yml"
+    project = File.exist?(project_file) ? YAML.load_file(project_file) : {}
+
+    user_file = "#{ENV['HOME']}/.lono/settings.yml"
+    user = File.exist?(user_file) ? YAML.load_file(user_file) : {}
+
+    default_file = File.expand_path("../default/settings.yml", __FILE__)
+    default = YAML.load_file(default_file)
+
+    @settings_yaml = default.merge(user.merge(project))
+  end
+
+
+  def s3_path
+    s3 = data['s3']
+    # s3['default']['path'] - key will always exist because of default lono/settings.yml
+    #   defauult value is nil though
+    s3['path'][LONO_ENV] || s3['path']['default']
+  end
+end

data/lib/lono/template.rb

@@ -3,18 +3,29 @@ require_relative "command"
 
 class Lono::Template < Lono::Command
   autoload :Help, 'lono/template/help'
+  autoload :Helpers, 'lono/template/helpers'
   autoload :Bashify, 'lono/template/bashify'
   autoload :DSL, 'lono/template/dsl'
   autoload :Template, 'lono/template/template'
+  autoload :Upload, 'lono/template/upload'
+  autoload :AwsServices, 'lono/template/aws_services'
+
+  class_option :quiet, type: :boolean, aliases: "-q", desc: "silence the output"
+  class_option :noop, type: :boolean, desc: "noop mode, do nothing destructive"
 
   desc "generate", "Generate the CloudFormation templates"
   Help.generate
   option :clean, type: :boolean, aliases: "-c", desc: "remove all output files before generating"
   option :project_root, default: ".", aliases: "-r", desc: "project root"
-  option :quiet, type: :boolean, aliases: "-q", desc: "silence the output"
   option :pretty, type: :boolean, default: true, desc: "json pretty the output.  only applies with json format"
   def generate
-   DSL.new(options.clone).run
+    DSL.new(options.clone).run
+  end
+
+  desc "upload", "Uploads templates to configured s3 folder"
+  option :project_root, default: ".", aliases: "-r", desc: "project root"
+  def upload
+    Upload.new(options.clone).run
   end
 
   desc "bashify [URL-OR-PATH]", "Convert the UserData section of an existing CloudFormation Template to a starter bash script that is compatiable with lono"