lono 4.2.7 → 5.0.1

data/lib/lono/template/erb.rb

@@ -0,0 +1,82 @@
+class Lono::Template
+  class Erb < Base
+    include Evaluate
+    include Util
+
+    def initialize(blueprint, options={})
+      super
+      @templates = []
+      @results = {}
+    end
+
+    def run(options={})
+      evaluate_templates
+      build_templates
+      write_output
+    end
+
+    # Instance eval's the template declarations in app/definitions in this order:
+    #
+    #   app/definitions/base.rb
+    #   app/definitions/base - all files in folder
+    #   app/definitions/[Lono.env].rb
+    #   app/definitions/[Lono.env] - all files in folder
+    #
+    # So Lono.env specific template declarations override base template declarations.
+    def evaluate_templates
+      evaluate_template("base")
+      evaluate_folder("base")
+      evaluate_template(Lono.env)
+      evaluate_folder(Lono.env)
+    end
+
+    def evaluate_template(name)
+      path = "#{Lono.config.definitions_path}/#{name}.rb"
+      evaluate_template_path(path)
+    end
+
+    def evaluate_folder(folder)
+      paths = Dir.glob("#{Lono.config.definitions_path}/#{folder}/**/*")
+      paths.select{ |e| File.file?(e) }.each do |path|
+        evaluate_template_path(path)
+      end
+    end
+
+    def template(name, &block)
+      @templates << {name: name, block: block}
+    end
+
+    def build_templates
+      @templates.each do |t|
+        @results[t[:name]] = Lono::Template::Template.new(@blueprint, t[:name], t[:block], @options).build
+      end
+    end
+
+    def write_output
+      output_path = "#{Lono.config.output_path}/#{@blueprint}/templates"
+      FileUtils.rm_rf(Lono.config.output_path) if @options[:clean] # removes entire output folder. params and templates
+      FileUtils.mkdir_p(output_path)
+      puts "Generating CloudFormation templates for blueprint #{@blueprint.color(:green)}:" unless @options[:quiet]
+      @results.each do |name,text|
+        path = "#{output_path}/#{name}".sub(/^\.\//,'') # strip leading '.'
+        path += ".yml"
+        unless @options[:quiet]
+          pretty_path = path.sub("#{Lono.root}/",'')
+          puts "  #{pretty_path}"
+        end
+        ensure_parent_dir(path)
+        text = commented(text)
+        IO.write(path, text) # write file first so validate method is simpler
+        validate_yaml(path)
+      end
+    end
+
+    def commented(text)
+      comment =<<~EOS
+        # This file was generated with lono. Do not edit directly, the changes will be lost.
+        # More info: http://lono.cloud
+      EOS
+      "#{comment}#{text}"
+    end
+  end
+end

data/lib/lono/template/evaluate.rb

@@ -0,0 +1,39 @@
+class Lono::Template
+  module Evaluate
+    def evaluate_template_path(path)
+      return unless File.exist?(path)
+
+      begin
+        instance_eval(File.read(path), path)
+      rescue Exception => e
+        template_evaluation_error(e)
+        puts "\nFull error:"
+        raise
+      end
+    end
+
+    # Prints out a user friendly task_definition error message
+    def template_evaluation_error(e)
+      error_info = e.backtrace.first
+      path, line_no, _ = error_info.split(':')
+      line_no = line_no.to_i
+      puts "Error evaluating #{path}:".color(:red)
+      puts e.message
+      puts "Here's the line in #{path} with the error:\n\n"
+
+      contents = IO.read(path)
+      content_lines = contents.split("\n")
+      context = 5 # lines of context
+      top, bottom = [line_no-context-1, 0].max, line_no+context-1
+      spacing = content_lines.size.to_s.size
+      content_lines[top..bottom].each_with_index do |line_content, index|
+        line_number = top+index+1
+        if line_number == line_no
+          printf("%#{spacing}d %s\n".color(:red), line_number, line_content)
+        else
+          printf("%#{spacing}d %s\n", line_number, line_content)
+        end
+      end
+    end
+  end
+end

data/lib/lono/template/generator.rb

@@ -0,0 +1,29 @@
+require "yaml"
+
+class Lono::Template
+  class Generator
+    include Lono::Blueprint::Root
+
+    def initialize(blueprint, options={})
+      @blueprint, @options = blueprint, options
+      @template = @options[:template] || @blueprint
+      Lono::ProjectChecker.check
+      set_blueprint_root(@blueprint)
+    end
+
+    def run
+      # Examples:
+      #   Erb.new(b, options.clone).run
+      #   Dsl.new(b, options.clone).run
+      generator_class = "Lono::Template::#{template_type.classify}"
+      generator_class = Object.const_get(generator_class)
+      generator_class.new(@blueprint, @options.clone).run
+    end
+
+    def template_type
+      meta_config = "#{Lono.blueprint_root}/.meta/config.yml"
+      data = YAML.load_file(meta_config)
+      data["template_type"] || "dsl"
+    end
+  end
+end

data/lib/lono/template/helper.rb

@@ -5,12 +5,10 @@ require "aws-sdk-core"
 #
 # @options gets passed into:
 #
-#   Lono::Template::Context.new(@options)
+#   Lono::Template::Context.new(blueprint, @options)
 module Lono::Template::Helper
   # Bash code that is meant to included in user-data
   def extract_scripts(options={})
-    check_s3_folder_settings!
-
     settings = setting.data["extract_scripts"] || {}
     options = settings.merge(options)
     # defaults also here in case they are removed from settings
@@ -38,40 +36,28 @@ aws s3 cp #{scripts_s3_path} #{to}/
 BASH_CODE
   end
 
-  def check_s3_folder_settings!
-    return if setting.s3_folder
-
-    puts "Helper method called that requires the s3_folder to be set at:"
-    lines = caller.reject { |l| l =~ %r{lib/lono} } # hide internal lono trace
-    puts "  #{lines[0]}"
-
-    puts "Please configure your settings.yml with an s3_folder.".color(:red)
-    puts "Detected AWS_PROFILE #{ENV['AWS_PROFILE'].inspect}"
-    exit 1
-  end
-
   def scripts_name
     File.basename(scripts_s3_path)
   end
 
   def scripts_s3_path
-    upload = Lono::Script::Upload.new
+    upload = Lono::Script::Upload.new(@blueprint)
     upload.s3_dest
   end
 
   def template_s3_path(template_name)
-    check_s3_folder_settings!
     # high jacking Upload for useful s3_https_url method
-    template_path = "#{template_name}.yml"
-    upload = Lono::Template::Upload.new(@options)
+    template_path = "output/#{@blueprint}/templates/#{template_name}.yml"
+    upload = Lono::Template::Upload.new(@blueprint, @options)
     upload.s3_https_url(template_path)
   end
 
   def template_params(param_name)
     generator_options = {
-      allow_no_file: true
+      allow_not_exists: true
     }.merge(@options)
-    generator = Lono::Param::Generator.new(param_name, generator_options)
+    generator_options["param"] = param_name
+    generator = Lono::Param::Generator.new(@blueprint, generator_options)
     # do not generate because lono cfn calling logic already generated it we only need the values
     params = generator.params    # Returns Array in underscore keys format
     # convert Array to simplified hash structure
@@ -122,14 +108,6 @@ BASH_CODE
     path && File.exist?(path)
   end
 
-  def file_s3_key(name)
-    s3_folder = setting.s3_folder
-    return nil unless s3_folder
-
-    uploader = Lono::FileUploader.new
-    uploader.md5_key("#{Lono.root}/app/files/#{name}")
-  end
-
   def current_region
     region = Aws.config[:region]
     region ||= ENV['AWS_REGION']

data/lib/lono/template/post_processor.rb

@@ -0,0 +1,69 @@
+class Lono::Template
+  class PostProcessor
+    include Lono::Blueprint::Root
+    extend Memoist
+
+    def initialize(blueprint, options={})
+      @blueprint, @options = blueprint, options
+      @template = @options[:template] || @blueprint
+      Lono::ProjectChecker.check
+      set_blueprint_root(@blueprint)
+    end
+
+    def run
+      replacements.each do |placeholder, replacement|
+        update_template!(template)
+      end
+      write_template!
+    end
+
+    def write_template!
+      IO.write(template_path, YAML.dump(template)) # unless ENV['TEST'] # additional safeguard for testing
+    end
+
+    def replacements
+      map = {}
+      registry_items.each do |item|
+        if item.directory? || item.file?
+          replacement = item.s3_path
+        else
+          puts "WARN: PostProcessor replacements Cannot find file: #{item.path}"
+          next
+        end
+
+        placeholder = "file://app/files/#{item.name}"
+        map[placeholder] = replacement
+      end
+      map
+    end
+
+    def update_template!(hash)
+      hash.each do |k, v|
+        if v.is_a?(String)
+          if v =~ %r{^file://}
+            v.replace(replacements[v]) # replace the placeholder
+          end
+        elsif v.is_a?(Hash)
+          update_template!(v) # recurse
+        elsif v.is_a?(Array)
+          v.each { |x| update_template!(x) if x.is_a?(Hash) }
+        end
+      end
+      hash
+    end
+
+    # Useful for specs
+    def registry_items
+      Lono::AppFile::Registry.items
+    end
+
+    def template
+      YAML.load_file(template_path)
+    end
+    memoize :template
+
+    def template_path
+      "#{Lono.config.output_path}/#{@blueprint}/templates/#{@template}.yml"
+    end
+  end
+end

data/lib/lono/template/template.rb

@@ -5,7 +5,7 @@ require 'base64'
 class Lono::Template::Template
   include ERB::Util
 
-  # Main template DSL methods are: source and variables
+  # Main template Erb methods are: source and variables
   #
   #   template "example-2" do
   #     source "example"
@@ -13,13 +13,8 @@ class Lono::Template::Template
   #   end
   #
   attr_reader :name
-  def initialize(name, block=nil, options={})
-    # Taking care to name instance variables with _ in front because we load the
-    # variables from config/variables and those instance variables can clobber these
-    # instance variables
-    @name = name
-    @block = block
-    @options = options
+  def initialize(blueprint, name, block=nil, options={})
+    @blueprint, @name, @block, @options = blueprint, name, block, options
     @source_path = default_source_path(name)
   end
 
@@ -54,6 +49,6 @@ class Lono::Template::Template
   # Context for ERB rendering.
   # This is where we control what references get passed to the ERB rendering.
   def context
-    @context ||= Lono::Template::Context.new(@options)
+    @context ||= Lono::Template::Context.new(@blueprint, @options)
   end
 end

data/lib/lono/template/upload.rb

@@ -3,150 +3,120 @@ require 'json'
 require 'base64'
 require 'digest'
 
-class Lono::Template::Upload
-  include Lono::Template::AwsService
-
-  def initialize(options={})
-    @options = options
-    @checksums = {}
-    @prefix = "#{folder_key}/#{Lono.env}/templates" # s3://s3-bucket/folder/development/templates
-  end
+class Lono::Template
+  class Upload
+    include Lono::Blueprint::Root
+    include Lono::AwsServices
+
+    def initialize(blueprint, options={})
+      @blueprint, @options = blueprint, options
+      @template = @options[:template] || @blueprint
+      Lono::ProjectChecker.check
+      set_blueprint_root(@blueprint)
+
+      @checksums = {}
+      @prefix = Lono.env # s3://s3-bucket/development
+    end
 
-  def run
-    ensure_s3_setup!
-    load_checksums!
+    def run
+      load_checksums!
 
-    say "Uploading CloudFormation templates..."
-    paths = Dir.glob("#{Lono.config.output_path}/templates/**/*")
-    paths.select { |p| File.file?(p) }.each do |path|
-      upload(path)
+      say "Uploading CloudFormation templates..."
+      paths = Dir.glob("#{Lono.config.output_path}/#{@blueprint}/templates/**/*")
+      paths.select { |p| File.file?(p) }.each do |path|
+        upload(path)
+      end
+      say "Templates uploaded to s3."
     end
-    say "Templates uploaded to s3."
-  end
 
-  # Read existing files on s3 to grab their md5 checksum.
-  # We do this so we can see if we should avoid re-uploading the s3 child template
-  # entirely. If we upload a new child template that does not change AWS CloudFormation
-  # is not smart enough to know that it not has changed. I think all AWS CloudFormation
-  # does is check if the file's timestamp.
-  #
-  # Thought this would result in better AWS Change Set info but AWS still reports child
-  # stacks being changed even though they should not be reported.  Leaving this s3 checksum
-  # in for now.
-  def load_checksums!
-    return if @options[:noop]
-
-    resp = s3.list_objects(bucket: s3_bucket, prefix: @prefix)
-    resp.contents.each do |object|
-      # key does not include the bucket name
-      #    full path = s3://my-bucket/s3_folder/templates/production/my-template.yml
-      #    key = s3_folder/templates/production/my-template.yml
-      # etag is the checksum as long as the file is not a multi-part file upload
-      # it has extra double quotes wrapped around it.
-      #    etag = "\"9cb437490cee2cc96101baf326e5ca81\""
-      @checksums[object.key] = strip_surrounding_quotes(object.etag)
+    # Read existing files on s3 to grab their md5 checksum.
+    # We do this so we can see if we should avoid re-uploading the s3 child template
+    # entirely. If we upload a new child template that does not change AWS CloudFormation
+    # is not smart enough to know that it not has changed. I think all AWS CloudFormation
+    # does is check if the file's timestamp.
+    #
+    # Thought this would result in better AWS Change Set info but AWS still reports child
+    # stacks being changed even though they should not be reported.  Leaving this s3 checksum
+    # in for now.
+    def load_checksums!
+      return if @options[:noop]
+
+      resp = s3.list_objects(bucket: s3_bucket, prefix: @prefix)
+      resp.contents.each do |object|
+        @checksums[object.key] = strip_surrounding_quotes(object.etag)
+      end
+      @checksums
     end
-    @checksums
-  end
-
-  def strip_surrounding_quotes(string)
-    string.sub(/^"/,'').sub(/"$/,'')
-  end
 
-  def upload(path)
-    pretty_path = path.sub(/^\.\//, '')
-    key = "#{@prefix}/#{pretty_path.sub(%r{output/templates/},'')}"
-    s3_full_path = "s3://#{s3_bucket}/#{key}"
-
-    local_checksum = Digest::MD5.hexdigest(IO.read(path))
-    remote_checksum = remote_checksum(path)
-    if local_checksum == remote_checksum
-      say("Not modified: #{pretty_path} to #{s3_full_path}".color(:yellow)) unless @options[:noop]
-      return # do not upload unless the checksum has changed
+    def strip_surrounding_quotes(string)
+      string.sub(/^"/,'').sub(/"$/,'')
     end
 
-    resp = s3.put_object(
-      body: IO.read(path),
-      bucket: s3_bucket,
-      key: key,
-      storage_class: "REDUCED_REDUNDANCY"
-    ) unless @options[:noop]
-
-    # Example output:
-    # Uploaded: output/templates/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
-    # Uploaded: output/templates/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
-    message = "Uploaded: #{pretty_path} to #{s3_full_path}".color(:green)
-    message = "NOOP: #{message}" if @options[:noop]
-    say message
-  end
-
-  # @checksums map has a key format: s3_folder/templates/development/docker.yml
-  #
-  # path = ./output/templates/docker.yml
-  # s3_folder = s3://boltops-dev/s3_folder/templates/development/docker.yml
-  def remote_checksum(path)
-    # first convert the local path to the path format that is stored in @checksums keys
-    # ./output/templates/docker.yml => s3_folder/templates/development/docker.yml
-    pretty_path = path.sub(/^\.\//, '')
-    key = "#{@prefix}/#{pretty_path.sub(%r{output/templates/},'')}"
-    @checksums[key]
-  end
-
-  # https://s3.amazonaws.com/mybucket/s3_folder/templates/production/parent.yml
-  def s3_https_url(template_path)
-    ensure_s3_setup!
-    "https://s3.amazonaws.com/#{s3_bucket}/#{@prefix}/#{template_path}"
-  end
-
-  # used for cfn/base.rb def set_template_body!(params)
-  def s3_presigned_url(template_output_path)
-    template_path = template_output_path.sub('output/templates/','')
-    key = "#{@prefix}/#{template_path}"
-    s3_presigner.presigned_url(:get_object, bucket: s3_bucket, key: key)
-  end
-
-  def s3_presigner
-    @signer ||= Aws::S3::Presigner.new
-  end
-
-  # Parse the s3_folder setting and remove the folder portion to leave the
-  # "s3_bucket" portion
-  # Example:
-  #    s3_bucket('s3://mybucket/templates/storage/path')
-  #    => mybucket
-  def s3_bucket
-    return nil if @options[:noop] # to get spec passing
-    return nil unless s3_folder
-    s3_folder.sub('s3://','').split('/').first
-  end
+    def upload(path)
+      return if @options[:noop]
+
+      path = path.sub("#{Lono.root}/",'')
+      pretty_path = path.sub(/^\.\//, '')
+      key = "#{@prefix}/#{pretty_path.sub(%r{output/templates/},'')}"
+      s3_full_path = "s3://#{s3_bucket}/#{key}"
+
+      local_checksum = Digest::MD5.hexdigest(IO.read(path))
+      remote_checksum = remote_checksum(path)
+      if local_checksum == remote_checksum
+        say("Not modified: #{pretty_path} to #{s3_full_path}".color(:yellow)) unless @options[:noop]
+        return # do not upload unless the checksum has changed
+      end
+
+      resp = s3.put_object(
+        body: IO.read(path),
+        bucket: s3_bucket,
+        key: key,
+        storage_class: "REDUCED_REDUNDANCY"
+      ) unless @options[:noop]
+
+      # Example output:
+      # Uploaded: output/templates/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
+      # Uploaded: output/templates/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
+      message = "Uploaded: #{pretty_path} to #{s3_full_path}".color(:green)
+      message = "NOOP: #{message}" if @options[:noop]
+      say message
+    end
 
-  # The folder_key is the s3_folder setting with the s3 bucket.
-  #
-  # Example:
-  #    s3_bucket('s3://mybucket/templates/storage/path')
-  #    => templates/storage/path
-  def folder_key
-    return nil if @options[:noop] # to get spec passing
-    return nil unless s3_folder
-    s3_folder.sub('s3://','').split('/')[1..-1].join('/')
-  end
+    # @checksums map has a key format: s3_folder/templates/development/docker.yml
+    #
+    # path = ./output/templates/docker.yml
+    # s3_folder = s3://boltops-dev/s3_folder/templates/development/docker.yml
+    def remote_checksum(path)
+      # first convert the local path to the path format that is stored in @checksums keys
+      # ./output/templates/docker.yml => s3_folder/templates/development/docker.yml
+      pretty_path = path.sub(/^\.\//, '')
+      key = "#{@prefix}/#{pretty_path.sub(%r{output/templates/},'')}"
+      @checksums[key]
+    end
 
-  def s3_folder
-    settings = Lono::Setting.new
-    settings.s3_folder
-  end
+    # https://s3.amazonaws.com/mybucket/s3_folder/templates/production/parent.yml
+    def s3_https_url(template_path)
+      "https://s3.amazonaws.com/#{s3_bucket}/#{@prefix}/#{template_path}"
+    end
 
-  # nice warning if the s3 path not found
-  def ensure_s3_setup!
-    return if @options[:noop]
-    return if s3_folder
+    # used for cfn/base.rb def set_template_body!(params)
+    def s3_presigned_url(template_output_path)
+      template_path = template_output_path.sub('output/templates/','')
+      key = "#{@prefix}/#{template_path}"
+      s3_presigner.presigned_url(:get_object, bucket: s3_bucket, key: key)
+    end
 
-    say "Unable to upload templates to s3 because you have not configured the s3_folder option in lono settings.yml.".color(:red)
-    say "Please configure settings.yml with s3_folder.  For more help: http://lono.cloud/docs/settings/".color(:red)
-    exit 1
-  end
+    # Parse the s3_folder setting and remove the folder portion to leave the
+    # "s3_bucket" portion
+    # Example:
+    #    s3_bucket('s3://mybucket/templates/storage/path')
+    #    => mybucket
+    def s3_bucket
+      Lono::S3::Bucket.name
+    end
 
-  def say(message)
-    puts message unless @options[:quiet]
+    def say(message)
+      puts message unless @options[:quiet]
+    end
   end
-end
+end