lono 3.5.0 → 4.0.0

data/lib/lono/template/dsl.rb

@@ -1,11 +1,10 @@
 class Lono::Template::DSL
   def initialize(options={})
     @options = options
-    @config_path = "#{Lono.root}/config"
     Lono::ProjectChecker.check
+    Lono::ProjectChecker.empty_templates
     @templates = []
     @results = {}
-    @detected_format = nil
   end
 
   def run(options={})
@@ -14,24 +13,29 @@ class Lono::Template::DSL
     write_output
   end
 
-  # Instance eval's all the files within each folder under
-  #   config/lono/base and config/lono/[Lono.env]
-  # Base gets base first and then the Lono.env configs get evaluate second.
-  # This means the env specific configs override the base configs.
+  # Instance eval's the template declarations in app/definitions in this order:
+  #
+  #   app/definitions/base.rb
+  #   app/definitions/base - all files in folder
+  #   app/definitions/[Lono.env].rb
+  #   app/definitions/[Lono.env] - all files in folder
+  #
+  # So Lono.env specific template declarations override base template declarations.
   def evaluate_templates
+    evaluate_template("base")
     evaluate_folder("base")
+    evaluate_template(Lono.env)
     evaluate_folder(Lono.env)
-    @detected_format = detect_format
   end
 
-  def evaluate_folder(folder)
-    paths = Dir.glob("#{@config_path}/templates/#{folder}/**/*")
-    paths.select{ |e| File.file?(e) }.each do |path|
-      evaluate_template(path)
-    end
+  def evaluate_template(name)
+    path = "#{Lono.config.definitions_path}/#{name}.rb"
+    evaluate_template_path(path)
   end
 
-  def evaluate_template(path)
+  def evaluate_template_path(path)
+    return unless File.exist?(path)
+
     begin
       instance_eval(File.read(path), path)
     rescue Exception => e
@@ -41,6 +45,13 @@ class Lono::Template::DSL
     end
   end
 
+  def evaluate_folder(folder)
+    paths = Dir.glob("#{Lono.config.definitions_path}/#{folder}/**/*")
+    paths.select{ |e| File.file?(e) }.each do |path|
+      evaluate_template_path(path)
+    end
+  end
+
   # Prints out a user friendly task_definition error message
   def template_evaluation_error(e)
     error_info = e.backtrace.first
@@ -65,85 +76,73 @@ class Lono::Template::DSL
     end
   end
 
-  # Detects the format of the templates.  Checks the extension of all the
-  # templates files.
-  # All the templates must be of the same format, either all json or all yaml.
-  def detect_format
-    extensions = Dir.glob("#{Lono.root}/templates/**/*").map do |path|
-      File.extname(path).sub(/^\./,'')
-    end.reject(&:empty?).uniq
-    extensions.include?('yml') ? 'yml' : 'json' # defaults to yml - falls back to json
-  end
-
   def template(name, &block)
     @templates << {name: name, block: block}
   end
 
   def build_templates
-    options = @options.merge(detected_format: @detected_format)
     @templates.each do |t|
-      @results[t[:name]] = Lono::Template::Template.new(t[:name], t[:block], options).build
+      @results[t[:name]] = Lono::Template::Template.new(t[:name], t[:block], @options).build
     end
   end
 
   def write_output
-    output_path = "#{Lono.root}/output"
+    output_path = "#{Lono.config.output_path}/templates"
     FileUtils.rm_rf(output_path) if @options[:clean]
-    FileUtils.mkdir(output_path) unless File.exist?(output_path)
+    FileUtils.mkdir_p(output_path)
     puts "Generating CloudFormation templates:" unless @options[:quiet]
     @results.each do |name,text|
       path = "#{output_path}/#{name}".sub(/^\.\//,'') # strip leading '.'
-      path += ".#{@detected_format}"
+      path += ".yml"
       puts "  #{path}" unless @options[:quiet]
       ensure_parent_dir(path)
-      validate(text, path)
-      File.open(path, 'w') do |f|
-        f.write(output_format(text))
-      end
-    end
-  end
-
-  def validate(text, path)
-    if @detected_format == "json"
-      validate_json(text, path)
-    else
-      validate_yaml(text, path)
+      text = commented(text)
+      IO.write(path, text) # write file first so validate method is simpler
+      validate(path)
     end
   end
 
-  def validate_yaml(yaml, path)
+  def validate(path)
+    text = IO.read(path)
     begin
-      YAML.load(yaml)
+      YAML.load(text)
     rescue Psych::SyntaxError => e
-      puts "Invalid yaml.  Output written to #{path} for debugging".colorize(:red)
-      puts "ERROR: #{e.message}".colorize(:red)
-      File.open(path, 'w') {|f| f.write(yaml) }
-      exit 1
+      handle_yaml_syntax_error(e, path)
     end
   end
 
-  def validate_json(json, path)
-    begin
-      JSON.parse(json)
-    rescue JSON::ParserError => e
-      puts "Invalid json.  Output written to #{path} for debugging".colorize(:red)
-      puts "ERROR: #{e.message}".colorize(:red)
-      File.open(path, 'w') {|f| f.write(json) }
-      exit 1
-    end
-  end
+  def handle_yaml_syntax_error(e, path)
+    io = StringIO.new
+    io.puts "Invalid yaml.  Output written to debugging: #{path}".colorize(:red)
+    io.puts "ERROR: #{e.message}".colorize(:red)
 
-  def output_format(text)
-    @options[:pretty] ? prettify(text) : text
-  end
+    # Grab line info.  Example error:
+    #   ERROR: (<unknown>): could not find expected ':' while scanning a simple key at line 2 column 1
+    md = e.message.match(/at line (\d+) column (\d+)/)
+    line = md[1].to_i
 
-  # Input text is either yaml or json.
-  # Do not prettify yaml format because it removes the !Ref like CloudFormation notation
-  def prettify(text)
-    @detected_format == "json" ? JSON.pretty_generate(JSON.parse(text)) : yaml_format(text)
+    lines = IO.read(path).split("\n")
+    context = 5 # lines of context
+    top, bottom = [line-context-1, 0].max, line+context-1
+    spacing = lines.size.to_s.size
+    lines[top..bottom].each_with_index do |line_content, index|
+      line_number = top+index+1
+      if line_number == line
+        io.printf("%#{spacing}d %s\n".colorize(:red), line_number, line_content)
+      else
+        io.printf("%#{spacing}d %s\n", line_number, line_content)
+      end
+    end
+
+    if ENV['TEST']
+      io.string
+    else
+      puts io.string
+      exit 1
+    end
   end
 
-  def yaml_format(text)
+  def commented(text)
     comment =<<~EOS
       # This file was generated with lono. Do not edit directly, the changes will be lost.
       # More info: http://lono.cloud

data/lib/lono/template/helper.rb

@@ -0,0 +1,201 @@
+# This is included into Lono::Template::Context.
+# It has access to the original thor CLI options via @options.
+#
+# @options gets passed into:
+#
+#   Lono::Template::Context.new(@options)
+module Lono::Template::Helper
+  # Bash code that is meant to included in user-data
+  def extract_scripts(options={})
+    check_s3_folder_settings!
+
+    settings = setting.data["extract_scripts"] || {}
+    options = settings.merge(options)
+    # defaults also here in case they are removed from settings
+    to = options[:to] || "/opt"
+    user = options[:as] || "ec2-user"
+
+    if Dir.glob("#{Lono.config.scripts_path}/*").empty?
+      puts "WARN: you are using the extract_scripts helper method but you do not have any app/scripts.".colorize(:yellow)
+      calling_line = caller[0].split(':')[0..1].join(':')
+      puts "Called from: #{calling_line}"
+      return ""
+    end
+
+    <<-BASH_CODE
+# Generated from the lono extract_scripts helper.
+# Downloads scripts from s3, extract them, and setup.
+mkdir -p #{to}
+aws s3 cp #{scripts_s3_path} #{to}/
+cd #{to}
+tar zxf #{to}/#{scripts_name}
+chmod -R a+x #{to}/scripts
+chown -R #{user}:#{user} #{to}/scripts
+BASH_CODE
+  end
+
+  def check_s3_folder_settings!
+    return if setting.s3_folder
+
+    puts "Helper method called that requires the s3_folder to be set at:"
+    lines = caller.reject { |l| l =~ %r{lib/lono} } # hide internal lono trace
+    puts "  #{lines[0]}"
+
+    puts "Please configure your settings.yml with an s3_folder.".colorize(:red)
+    puts "Detected AWS_PROFILE #{ENV['AWS_PROFILE'].inspect}"
+    exit 1
+  end
+
+  def scripts_name
+    File.basename(scripts_s3_path)
+  end
+
+  def scripts_s3_path
+    upload = Lono::Script::Upload.new
+    upload.s3_dest
+  end
+
+  def template_s3_path(template_name)
+    check_s3_folder_settings!
+    # high jacking Upload for useful s3_https_url method
+    template_path = "#{template_name}.yml"
+    upload = Lono::Template::Upload.new(@options)
+    upload.s3_https_url(template_path)
+  end
+
+  def template_params(param_name)
+    generator_options = {
+      allow_no_file: true
+    }.merge(@options)
+    generator = Lono::Param::Generator.new(param_name, generator_options)
+    # do not generate because lono cfn calling logic already generated it we only need the values
+    generator.params    # Returns Array in underscore keys format
+  end
+
+  # Adjust the partial path so that it will use app/user_data
+  def user_data(path,vars={}, options={})
+    options.merge!(user_data: true)
+    partial(path,vars, options)
+  end
+
+  # The partial's path is a relative path.
+  #
+  # Example:
+  # Given file in app/partials/iam/docker.yml
+  #
+  #   <%= partial("iam/docker", {}, indent: 10) %>
+  #   <%= partial("iam/docker.yml", {}, indent: 10) %>
+  #
+  # If the user specifies the extension then use that instead of auto-adding
+  # the detected format.
+  def partial(path,vars={}, options={})
+    path = options[:user_data] ?
+              user_data_path_for(path) :
+              partial_path_for(path)
+    path = auto_add_format(path)
+
+    instance_variables!(vars)
+    result = render_path(path)
+
+    result = indent(result, options[:indent]) if options[:indent]
+    result + "\n"
+  end
+
+  # add indentation
+  def indent(text, indentation_amount)
+    text.split("\n").map do |line|
+      " " * indentation_amount + line
+    end.join("\n")
+  end
+
+  def partial_exist?(path)
+    path = partial_path_for(path)
+    path = auto_add_format(path)
+    path && File.exist?(path)
+  end
+
+  def current_region
+    region = Aws.config[:region]
+    region ||= ENV['AWS_REGION']
+    return region if region
+
+    default_region = 'us-east-1' # fallback if default not found in ~/.aws/config
+    if ENV['AWS_PROFILE']
+      path = "#{ENV['HOME']}/.aws/config"
+      if File.exist?(path)
+        lines = IO.readlines(path)
+        capture_default, capture_current = false, false
+        lines.each do | line|
+          if line.include?('[default]')
+            capture_default = true # next line
+            next
+          end
+          if capture_default && line.match(/region = /)
+            # over default from above
+            default_region = line.split(' = ').last.strip
+            capture_default = false
+          end
+
+          md = line.match(/\[profile (.*)\]/)
+          if md && md[1] == ENV['AWS_PROFILE']
+            capture_current = true
+            next
+          end
+          if capture_current && line.match(/region = /)
+            region = line.split(' = ').last.strip
+            capture_current = false
+          end
+        end
+      end
+
+      region ||= default_region
+      return region if region
+    end
+
+    'us-east-1' # default
+  end
+
+private
+  def render_path(path)
+    RenderMePretty.result(path, context: self)
+  end
+
+  def user_data_path_for(path)
+    "#{Lono.config.user_data_path}/#{path}"
+  end
+
+  def partial_path_for(path)
+    "#{Lono.config.partials_path}/#{path}"
+  end
+
+  def auto_add_format(path)
+    # Return immediately if user provided explicit extension
+    extension = File.extname(path) # current extension
+    return path if !extension.empty?
+
+    # Else let's auto detect
+    paths = Dir.glob("#{path}.*")
+
+    if paths.size == 1 # non-ambiguous match
+      return paths.first
+    end
+
+    if paths.size > 1 # ambiguous match
+      puts "ERROR: Multiple possible partials found:".colorize(:red)
+      paths.each do |path|
+        puts "  #{path}"
+      end
+      puts "Please specify an extension in the name to remove the ambiguity.".colorize(:green)
+      exit 1
+    end
+
+    # Account for case when user wants to include a file with no extension at all
+    return path if File.exist?(path) && !File.directory?(path)
+
+    path # original path if this point is reached
+  end
+
+  def setting
+    @setting ||= Lono::Setting.new
+  end
+end

data/lib/lono/template/template.rb

@@ -3,249 +3,57 @@ require 'json'
 require 'base64'
 
 class Lono::Template::Template
-  include Lono::Template::Helpers
-  include Lono::CurrentRegion
   include ERB::Util
 
+  # Main template DSL methods are: source and variables
+  #
+  #   template "example-2" do
+  #     source "example"
+  #     variables(test: 1)
+  #   end
+  #
+  attr_reader :name
   def initialize(name, block=nil, options={})
     # Taking care to name instance variables with _ in front because we load the
     # variables from config/variables and those instance variables can clobber these
     # instance variables
-    @_name = name
-    @_options = options
-    @_detected_format = options[:detected_format]
-    @_block = block
-    @_config_path = "#{Lono.root}/config"
-    @_source = default_source(name)
-  end
-
-  def default_source(name)
-    "#{Lono.root}/templates/#{name}.#{@_detected_format}" # defaults to name, source method overrides
-  end
-
-  def build
-    load_variables
-    load_custom_helpers
-    instance_eval(&@_block) if @_block
-    template = IO.read(@_source)
-    erb_result(@_source, template)
-  end
-
-  def load_variables
-    load_variables_folder("base")
-    load_variables_folder(Lono.env)
-  end
-
-  # Load the variables defined in config/variables/* to make available in the
-  # template blocks in config/templates/*.
-  #
-  # Example:
-  #
-  #   `config/variables/base/variables.rb`:
-  #     @foo = 123
-  #
-  #   `config/templates/base/resources.rb`:
-  #      template "mytemplate.yml" do
-  #        source "mytemplate.yml.erb"
-  #        variables(foo: @foo)
-  #      end
-  #
-  # NOTE: Only able to make instance variables avaialble with instance_eval
-  #   Wasnt able to make local variables available.
-  def load_variables_folder(folder)
-    paths = Dir.glob("#{@_config_path}/variables/#{folder}/**/*")
-    paths.select{ |e| File.file? e }.each do |path|
-      instance_eval(IO.read(path))
-    end
-  end
-
-  # Load custom helper methods from the user's infra repo
-  def load_custom_helpers
-    Dir.glob("#{Lono.root}/helpers/**/*_helper.rb").each do |path|
-      filename = path.sub(%r{.*/},'').sub('.rb','')
-      module_name = filename.classify
-
-      require path
-      self.class.send :include, module_name.constantize
-    end
+    @name = name
+    @block = block
+    @options = options
+    @source_path = default_source_path(name)
   end
 
+  # Returns path, example: ./app/templates/example.yml
   def source(path)
-    @_source = path[0..0] == '/' ? path : "#{Lono.root}/templates/#{path}"
-    @_source += ".#{@_detected_format}"
+    @source_path = path[0..0] == '/' ? path : "#{Lono.config.templates_path}/#{path}"
+    @source_path += ".yml"
   end
 
   def variables(vars={})
     vars.each do |var,value|
-      instance_variable_set("@#{var}", value)
+      context.instance_variable_set("@#{var}", value)
     end
   end
 
-  def erb_result(path, template)
-    begin
-      ERB.new(template, nil, "-").result(binding)
-    rescue Exception => e
-      puts e
-      puts e.backtrace if ENV['DEBUG']
-
-      # how to know where ERB stopped? - https://www.ruby-forum.com/topic/182051
-      # syntax errors have the (erb):xxx info in e.message
-      # undefined variables have (erb):xxx info in e.backtrac
-      error_info = e.message.split("\n").grep(/\(erb\)/)[0]
-      error_info ||= e.backtrace.grep(/\(erb\)/)[0]
-      raise unless error_info # unable to find the (erb):xxx: error line
-      line = error_info.split(':')[1].to_i
-      puts "Error evaluating ERB template on line #{line.to_s.colorize(:red)} of: #{path.sub(/^\.\//, '').colorize(:green)}"
-
-      template_lines = template.split("\n")
-      context = 5 # lines of context
-      top, bottom = [line-context-1, 0].max, line+context-1
-      spacing = template_lines.size.to_s.size
-      template_lines[top..bottom].each_with_index do |line_content, index|
-        line_number = top+index+1
-        if line_number == line
-          printf("%#{spacing}d %s\n".colorize(:red), line_number, line_content)
-        else
-          printf("%#{spacing}d %s\n", line_number, line_content)
-        end
-      end
-      exit 1 unless ENV['TEST']
-    end
+  # internal methods
+  def default_source_path(name)
+    "#{Lono.config.templates_path}/#{name}.yml" # defaults to name, source method overrides
   end
 
-  def transform_array(arr)
-    arr.map! {|x| x =~ /=>/ ? x : x.inspect }
-    arr.join(',')
-  end
+  def build
+    instance_eval(&@block) if @block
 
-  # transform each line of bash script to array with cloudformation template objects
-  def transform(data)
-    data = evaluate(data)
-    if data[-1].is_a?(String)
-      data[0..-2] + ["#{data[-1]}\n"]
+    if File.exist?(@source_path)
+      RenderMePretty.result(@source_path, context: context)
     else
-      data + ["\n"]
+      puts "ERROR: #{@source_path} does not exist, but it was used as a template source.".colorize(:red)
+      exit 1
     end
   end
 
-  # Input:
-  #   String
-  # Output:
-  #   Array of parse positions
-  #
-  # The positions of tokens taking into account when brackets start and close,
-  # handles nested brackets.
-  def bracket_positions(line)
-    positions,pair,count = [],[],0
-
-    line.split('').each_with_index do |char,i|
-      pair << i if pair.empty?
-
-      first_pair_char = line[pair[0]]
-      if first_pair_char == '{' # object logic
-        if char == '{'
-          count += 1
-        end
-
-        if char == '}'
-          count -= 1
-          if count == 0
-            pair << i
-            positions << pair
-            pair = []
-          end
-        end
-      else # string logic
-        lookahead = line[i+1]
-        if lookahead == '{'
-          pair << i
-          positions << pair
-          pair = []
-        end
-      end
-    end # end of loop
-
-    # for string logic when lookahead does not contain a object token
-    # need to clear out what's left to match the final pair
-    if !pair.empty?
-      pair << line.size - 1
-      positions << pair
-    end
-
-    positions
-  end
-
-  # Input:
-  #   Array - bracket_positions
-  # Ouput:
-  #   Array - positions that can be use to determine what to parse
-  def parse_positions(line)
-    positions = bracket_positions(line)
-    positions.flatten
-  end
-
-  # Input
-  #   String line of code to decompose into chunks, some can be transformed into objects
-  # Output
-  #   Array of strings, some can be transformed into objects
-  #
-  # Example:
-  # line = 'a{b}c{d{d}d}e' # nested brackets
-  # template.decompose(line).should == ['a','{b}','c','{d{d}d}','e']
-  def decompose(line)
-    positions = parse_positions(line)
-    return [line] if positions.empty?
-
-    result = []
-    str = ''
-    until positions.empty?
-      left = positions.shift
-      right = positions.shift
-      token = line[left..right]
-      # if cfn object, add to the result set but after clearing out
-      # the temp str that is being built up when the token is just a string
-      if cfn_object?(token)
-        unless str.empty? # first token might be a object
-          result << str
-          str = ''
-        end
-        result << token
-      else
-        str << token # keeps building up the string
-      end
-    end
-
-    # at the of the loop there's a leftover string, unless the last token
-    # is an object
-    result << str unless str.empty?
-
-    result
-  end
-
-  def cfn_object?(s)
-    exact = %w[Ref]
-    pattern = %w[Fn::]
-    exact_match = !!exact.detect {|word| s.include?(word)}
-    pattern_match = !!pattern.detect {|p| s =~ Regexp.new(p)}
-    (exact_match || pattern_match) && s =~ /^{/ && s =~ /=>/
-  end
-
-  def recompose(decomposition)
-    decomposition.map { |s| cfn_object?(s) ? eval(s) : s }
-  end
-
-  def evaluate(line)
-    recompose(decompose(line))
-  end
-
-  # For simple just parameters files that can also be generated with lono, the CFN
-  # Fn::Base64 function is not available and as lono is not being used in the context
-  # of CloudFormation.  So this can be used in it's place.
-  def encode_base64(text)
-    Base64.strict_encode64(text).strip
-  end
-
-  def name
-    @_name
+  # Context for ERB rendering.
+  # This is where we control what references get passed to the ERB rendering.
+  def context
+    @context ||= Lono::Template::Context.new(@options)
   end
 end