lono 2.1.0 → 3.0.0

data/lib/lono/template/aws_services.rb

@@ -0,0 +1,7 @@
+require "aws-sdk"
+
+module Lono::Template::AwsServices
+  def s3
+    @s3 ||= Aws::S3::Client.new
+  end
+end

data/lib/lono/template/dsl.rb

@@ -2,7 +2,7 @@ class Lono::Template::DSL
   def initialize(options={})
     @options = options
     @project_root = @options[:project_root] || '.'
-    @path = "#{@project_root}/config/lono.rb"
+    @config_path = "#{@project_root}/config"
     Lono::ProjectChecker.check(@project_root)
     @templates = []
     @results = {}
@@ -15,12 +15,23 @@ class Lono::Template::DSL
     write_output
   end
 
+  # Instance eval's all the files within each folder under
+  #   config/lono/base and config/lono/[LONO_ENV]
+  # Base gets base first and then the LONO_ENV configs get evaluate second.
+  # This means the env specific configs override the base configs.
   def evaluate_templates
-    evaluate_template(@path)
-    load_subfolder
+    evaluate_folder("base")
+    evaluate_folder(LONO_ENV)
     @detected_format = detect_format
   end
 
+  def evaluate_folder(folder)
+    paths = Dir.glob("#{@config_path}/templates/#{folder}/**/*")
+    paths.select{ |e| File.file?(e) }.each do |path|
+      evaluate_template(path)
+    end
+  end
+
   def evaluate_template(path)
     begin
       instance_eval(File.read(path), path)
@@ -55,35 +66,14 @@ class Lono::Template::DSL
     end
   end
 
-  # Detects the format of the templates.  Simply checks the extension of all the
+  # Detects the format of the templates.  Checks the extension of all the
   # templates files.
   # All the templates must be of the same format, either all json or all yaml.
   def detect_format
-    # @templates contains Array of Hashes. Example:
-    # [{name: ""blog-web-prod.json", block: ...},
-    #  {name: ""api-web-prod.json", block: ...}]
-    formats = @templates.map{ |t| File.extname(t[:name]) }.uniq
-    if formats.size > 1
-      puts "ERROR: Detected multiple formats: #{formats.join(", ")}".colorize(:red)
-      puts "All the source values in the template blocks in the config folder must have the same format extension."
-      exit 1
-    else
-      found_format = formats.first
-      if found_format
-        detected_format = found_format.sub(/^\./,'')
-        detected_format = "yaml" if detected_format == "yml"
-      else # empty templates, no templates defined yet
-        detected_format = "yaml" # defaults to yaml
-      end
-    end
-    detected_format
-  end
-
-  # load any templates defined in project/config/lono/*
-  def load_subfolder
-    Dir.glob("#{File.dirname(@path)}/lono/**/*").select{ |e| File.file? e }.each do |path|
-      evaluate_template(path)
-    end
+    extensions = Dir.glob("#{@project_root}/templates/**/*").map do |path|
+      File.extname(path).sub(/^\./,'')
+    end.reject(&:empty?).uniq
+    extensions.include?('yml') ? 'yml' : 'json' # defaults to yml - falls back to json
   end
 
   def template(name, &block)
@@ -91,8 +81,9 @@ class Lono::Template::DSL
   end
 
   def build_templates
+    options = @options.merge(detected_format: @detected_format)
     @templates.each do |t|
-      @results[t[:name]] = Lono::Template::Template.new(t[:name], t[:block], @options).build
+      @results[t[:name]] = Lono::Template::Template.new(t[:name], t[:block], options).build
     end
   end
 
@@ -102,7 +93,8 @@ class Lono::Template::DSL
     FileUtils.mkdir(output_path) unless File.exist?(output_path)
     puts "Generating CloudFormation templates:" unless @options[:quiet]
     @results.each do |name,text|
-      path = "#{output_path}/#{name}".sub(/^\.\//,'')
+      path = "#{output_path}/#{name}".sub(/^\.\//,'') # strip leading '.'
+      path += ".#{@detected_format}"
       puts "  #{path}" unless @options[:quiet]
       ensure_parent_dir(path)
       validate(text, path)
@@ -112,7 +104,6 @@ class Lono::Template::DSL
     end
   end
 
-  # TODO: set @detected_format upon DSL.new
   def validate(text, path)
     if @detected_format == "json"
       validate_json(text, path)
@@ -156,7 +147,7 @@ class Lono::Template::DSL
   def yaml_format(text)
     comment =<<~EOS
       # This file was generated with lono. Do not edit directly, the changes will be lost.
-      # More info: https://github.com/tongueroo/lono
+      # More info: http://lono.cloud
     EOS
     "#{comment}#{remove_blank_lines(text)}"
   end

data/lib/lono/template/helpers.rb

@@ -0,0 +1,144 @@
+module Lono::Template::Helpers
+  def template_s3_path(template_name)
+    format = @_detected_format.sub('yaml','yml')
+    template_path = "#{template_name}.#{format}"
+
+    # must have settings.s3_path for this to owrk
+    settings = Lono::Settings.new(@project_root)
+    if settings.s3_path
+      # high jacking Upload for useful s3_https_url method
+      upload = Lono::Template::Upload.new(@_options)
+      upload.s3_https_url(template_path)
+    else
+      message = "template_s3_path helper called but s3.path not configured in lono/settings.yml"
+      puts "WARN: #{message}".colorize(:yellow)
+      message
+    end
+  end
+
+  def template_params(param_name)
+    param_path = "params/#{LONO_ENV}/#{param_name}.txt"
+    generator_options = {
+      project_root: @_project_root,
+      path: param_path,
+      allow_no_file: true
+    }.merge(@_options)
+    generator = Lono::Param::Generator.new(param_name, generator_options)
+    # do not generate because lono cfn calling logic already generated it we only need the values
+    generator.params    # Returns Array in underscore keys format
+  end
+
+  def user_data(path, vars={})
+    path = "#{@_project_root}/templates/user_data/#{path}"
+    template = IO.read(path)
+    variables(vars)
+    result = erb_result(path, template)
+    output = []
+    result.split("\n").each do |line|
+      output += transform(line)
+    end
+    json = output.to_json
+    json[0] = '' # remove first char: [
+    json.chop!   # remove last char:  ]
+  end
+
+  def ref(name)
+    %Q|{"Ref"=>"#{name}"}|
+  end
+
+  def find_in_map(*args)
+    %Q|{"Fn::FindInMap" => [ #{transform_array(args)} ]}|
+  end
+
+  def base64(value)
+    %Q|{"Fn::Base64"=>"#{value}"}|
+  end
+
+  def get_att(*args)
+    %Q|{"Fn::GetAtt" => [ #{transform_array(args)} ]}|
+  end
+
+  def get_azs(region="AWS::Region")
+    %Q|{"Fn::GetAZs"=>"#{region}"}|
+  end
+
+  def join(delimiter, values)
+    %Q|{"Fn::Join" => ["#{delimiter}", [ #{transform_array(values)} ]]}|
+  end
+
+  def select(index, list)
+    %Q|{"Fn::Select" => ["#{index}", [ #{transform_array(list)} ]]}|
+  end
+
+  def partial_exist?(path)
+    path = partial_path_for(path)
+    path = auto_add_format(path)
+    path && File.exist?(path)
+  end
+
+  # The partial's path is a relative path given without the extension and
+  #
+  # Example:
+  # Given: file in templates/partial/iam/docker.yml
+  # The path should be: iam/docker
+  #
+  # If the user specifies the extension then use that instead of auto-adding
+  # the detected format.
+  def partial(path,vars={}, options={})
+    path = partial_path_for(path)
+    path = auto_add_format(path)
+
+    template = IO.read(path)
+    variables(vars)
+    result = erb_result(path, template)
+    result = indent(result, options[:indent]) if options[:indent]
+    if options[:indent]
+      # Add empty line at beginning because empty lines gets stripped during
+      # processing anyway. This allows the user to call partial without having
+      # to put the partial call at very beginning of the line.
+      # This only should happen if user is using indent option.
+      ["\n", result].join("\n")
+    else
+      result
+    end
+  end
+
+  # add indentation
+  def indent(text, indentation_amount)
+    text.split("\n").map do |line|
+      " " * indentation_amount + line
+    end.join("\n")
+  end
+
+private
+  def partial_path_for(path)
+    "#{@_project_root}/templates/partial/#{path}"
+  end
+
+  def auto_add_format(path)
+    # Return immediately if user provided explicit extension
+    extension = File.extname(path) # current extension
+    return path if !extension.empty?
+
+    # Else let's auto detect
+    paths = Dir.glob("#{path}.*")
+
+    if paths.size == 1 # non-ambiguous match
+      return paths.first
+    end
+
+    if paths.size > 1 # ambiguous match
+      puts "ERROR: Multiple possible partials found:".colorize(:red)
+      paths.each do |path|
+        puts "  #{path}"
+      end
+      puts "Please specify an extension in the name to remove the ambiguity.".colorize(:green)
+      exit 1
+    end
+
+    # Account for case when user wants to include a file with no extension at all
+    return path if File.exist?(path) && !File.directory?(path)
+
+    path # original path if this point is reached
+  end
+end

data/lib/lono/template/template.rb

@@ -3,45 +3,83 @@ require 'json'
 require 'base64'
 
 class Lono::Template::Template
+  include Lono::Template::Helpers
   include ERB::Util
 
-  attr_reader :name
-  def initialize(name, block, options={})
-    @name = name
-    @block = block
-    @options = options
+  def initialize(name, block=nil, options={})
+    # Taking care to name instance variables with _ in front because we load the
+    # variables from config/variables and those instance variables can clobber these
+    # instance variables
+    @_name = name
+    @_options = options
+    @_detected_format = options[:detected_format]
+    @_block = block
+    @_project_root = options[:project_root] || '.'
+    @_config_path = "#{@_project_root}/config"
+    @_source = default_source(name)
+  end
+
+  def default_source(name)
+    "#{@_project_root}/templates/#{name}.#{@_detected_format}" # defaults to name, source method overrides
   end
 
   def build
-    instance_eval(&@block)
-    template = IO.read(@source)
-    erb_result(@source, template)
+    load_variables
+    load_custom_helpers
+    instance_eval(&@_block) if @_block
+    template = IO.read(@_source)
+    erb_result(@_source, template)
   end
 
-  def source(path)
-    @source = path[0..0] == '/' ? path : "#{@options[:project_root]}/templates/#{path}"
+  def load_variables
+    load_variables_folder("base")
+    load_variables_folder(LONO_ENV)
   end
 
-  def variables(vars={})
-    vars.each do |var,value|
-      instance_variable_set("@#{var}", value)
+  # Load the variables defined in config/variables/* to make available in the
+  # template blocks in config/templates/*.
+  #
+  # Example:
+  #
+  #   `config/variables/base/variables.rb`:
+  #     @foo = 123
+  #
+  #   `config/templates/base/resources.rb`:
+  #      template "mytemplate.yml" do
+  #        source "mytemplate.yml.erb"
+  #        variables(foo: @foo)
+  #      end
+  #
+  # NOTE: Only able to make instance variables avaialble with instance_eval
+  #   Wasnt able to make local variables available.
+  def load_variables_folder(folder)
+    paths = Dir.glob("#{@_config_path}/variables/#{folder}/**/*")
+    paths.select{ |e| File.file? e }.each do |path|
+      instance_eval(IO.read(path))
     end
   end
 
-  def partial(path,vars={}, options={})
-    path = "#{@options[:project_root]}/templates/partial/#{path}"
-    template = IO.read(path)
-    variables(vars)
-    result = erb_result(path, template)
-    result = indent(result, options[:indent]) if options[:indent]
-    result
+  # Load custom helper methods from the user's infra repo
+  def load_custom_helpers
+    Dir.glob("#{@_project_root}/helpers/**/*_helper.rb").each do |path|
+      filename = path.sub(%r{.*/},'').sub('.rb','')
+      module_name = filename.classify
+
+      require path
+      self.class.send :include, module_name.constantize
+    end
+
   end
 
-  # add indentation
-  def indent(result, indentation_amount)
-    result.split("\n").map do |line|
-      " " * indentation_amount + line
-    end.join("\n")
+  def source(path)
+    @_source = path[0..0] == '/' ? path : "#{@_project_root}/templates/#{path}"
+    @_source += ".#{@_detected_format}"
+  end
+
+  def variables(vars={})
+    vars.each do |var,value|
+      instance_variable_set("@#{var}", value)
+    end
   end
 
   def erb_result(path, template)
@@ -49,6 +87,7 @@ class Lono::Template::Template
       ERB.new(template, nil, "-").result(binding)
     rescue Exception => e
       puts e
+      puts e.backtrace if ENV['DEBUG']
 
       # how to know where ERB stopped? - https://www.ruby-forum.com/topic/182051
       # syntax errors have the (erb):xxx info in e.message
@@ -57,7 +96,7 @@ class Lono::Template::Template
       error_info ||= e.backtrace.grep(/\(erb\)/)[0]
       raise unless error_info # unable to find the (erb):xxx: error line
       line = error_info.split(':')[1].to_i
-      puts "Error evaluating ERB template on line #{line.to_s.colorize(:red)} of: #{path.sub(/^\.\//, '')}"
+      puts "Error evaluating ERB template on line #{line.to_s.colorize(:red)} of: #{path.sub(/^\.\//, '').colorize(:green)}"
 
       template_lines = template.split("\n")
       context = 5 # lines of context
@@ -75,48 +114,6 @@ class Lono::Template::Template
     end
   end
 
-  def user_data(path, vars={})
-    path = "#{@options[:project_root]}/templates/user_data/#{path}"
-    template = IO.read(path)
-    variables(vars)
-    result = erb_result(path, template)
-    output = []
-    result.split("\n").each do |line|
-      output += transform(line)
-    end
-    json = output.to_json
-    json[0] = '' # remove first char: [
-    json.chop!   # remove last char:  ]
-  end
-
-  def ref(name)
-    %Q|{"Ref"=>"#{name}"}|
-  end
-
-  def find_in_map(*args)
-    %Q|{"Fn::FindInMap" => [ #{transform_array(args)} ]}|
-  end
-
-  def base64(value)
-    %Q|{"Fn::Base64"=>"#{value}"}|
-  end
-
-  def get_att(*args)
-    %Q|{"Fn::GetAtt" => [ #{transform_array(args)} ]}|
-  end
-
-  def get_azs(region="AWS::Region")
-    %Q|{"Fn::GetAZs"=>"#{region}"}|
-  end
-
-  def join(delimiter, values)
-    %Q|{"Fn::Join" => ["#{delimiter}", [ #{transform_array(values)} ]]}|
-  end
-
-  def select(index, list)
-    %Q|{"Fn::Select" => ["#{index}", [ #{transform_array(list)} ]]}|
-  end
-
   def transform_array(arr)
     arr.map! {|x| x =~ /=>/ ? x : x.inspect }
     arr.join(',')
@@ -248,4 +245,8 @@ class Lono::Template::Template
   def encode_base64(text)
     Base64.strict_encode64(text).strip
   end
+
+  def name
+    @_name
+  end
 end

data/lib/lono/template/upload.rb

@@ -0,0 +1,75 @@
+require 'erb'
+require 'json'
+require 'base64'
+
+class Lono::Template::Upload
+  include Lono::Template::AwsServices
+
+  def initialize(options={})
+    @options = options
+    @project_root = options[:project_root] || '.'
+  end
+
+  def run
+    ensure_s3_setup!
+    paths = Dir.glob("#{@project_root}/output/**/*")
+    paths.reject { |p| p =~ %r{output/params} }.
+          select { |p| File.file?(p) }.each do |path|
+      upload(path)
+    end
+    say "Templates uploaded to s3."
+  end
+
+  def upload(path)
+    pretty_path = path.sub(/^\.\//, '')
+    key = "#{s3_path}/#{LONO_ENV}/#{pretty_path.sub(/^output\//,'')}"
+    s3_full_path = "s3://#{s3_bucket}/#{key}"
+
+    resp = s3.put_object(
+      body: IO.read(path),
+      bucket: s3_bucket,
+      key: key,
+      storage_class: "REDUCED_REDUNDANCY"
+    ) unless @options[:noop]
+
+    message = "Uploaded: #{pretty_path} to #{s3_full_path}"
+    message = "NOOP: #{message}" if @options[:noop]
+    say message
+  end
+
+  # https://s3.amazonaws.com/mybucket/cloudformation-templates/prod/parent.yml
+  def s3_https_url(template_path)
+    ensure_s3_setup!
+    "https://s3.amazonaws.com/#{s3_bucket}/#{s3_path}/#{LONO_ENV}/#{template_path}"
+  end
+
+  # Example:
+  #    s3_bucket('s3://mybucket/templates/storage/path') => mybucket
+  def s3_bucket
+    @s3_full_path.sub('s3://','').split('/').first
+  end
+
+  # Example:
+  #    s3_bucket('s3://mybucket/templates/storage/path') => templates/storage/path
+  def s3_path
+    @s3_full_path.sub('s3://','').split('/')[1..-1].join('/')
+  end
+
+  # nice warning if the s3 path not found
+  def ensure_s3_setup!
+    return if @options[:noop]
+
+    settings = Lono::Settings.new(@project_root)
+    if settings.s3_path
+      @s3_full_path = settings.s3_path
+    else
+      say "Unable to upload templates to s3 because you have not configured the s3.path option in .lono/settings.yml.".colorize(:red)
+      say "Please configure .lono/settings.yml with s3.path.  Refer to http://lono.cloud/docs/settings/ for more help.".colorize(:red)
+      exit 1
+    end
+  end
+
+  def say(message)
+    puts message unless @options[:quiet]
+  end
+end