lono 8.0.0.pre.rc1 → 8.0.0.pre.rc4

data/lib/lono/builder/dsl/finalizer/files/build.rb

@@ -0,0 +1,50 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Build < Base
+    def initialize(options={})
+      super
+      @output_path = "#{Lono.root}/.output/files"
+    end
+
+    def run
+      return if Lono::Files.empty?
+      logger.debug "Building files"
+      clean
+      validate!
+      build_files
+    end
+
+    def build_files
+      Lono::Files.files.each do |file| # using singular file, but is like a "file_list"
+        file.build
+        file.build_lambda_layer(@cfn)
+        file.compress
+        # UploadNote: Instead of part of the build process here, uploading happen as part
+        # of create_stack or execute_change_set after user confirms action. IE:
+        #   Cfn::Deploy#create and Cfn::Deploy#update
+      end
+    end
+
+    def clean
+      FileUtils.rm_rf(@output_path)
+    end
+
+    def validate!
+      missing = Lono::Files.files.select do |file|
+        !File.exist?(file.full_path)
+      end
+      return if missing.empty?
+
+      logger.info "ERROR: These files are missing".color(:red)
+      logger.info <<~EOL
+        The file helper is calling them.
+        They need to exist to be uploaded to s3.
+      EOL
+      missing.each do |file|
+        logger.info "    #{pretty_path(file.full_path)}"
+        logger.info "    Called at #{file.call_line}"
+      end
+      logger.info "Please double check that they exist."
+      exit 1
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/files/replace.rb

@@ -0,0 +1,36 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Replace < Base
+    def run
+      replacements.each do |placeholder, replacement|
+        update_template!(@cfn)
+      end
+      @cfn
+    end
+
+    def replacements
+      data = Lono::Files.files.inject({}) do |result, file|
+        result.merge(file.marker => file.s3_key)
+      end
+      # Edge case when bucket is created for the first time and files_bucket is
+      # not available yet. So we call it at this point.
+      Lono::S3::Bucket.ensure_exist
+      data["LONO://S3_BUCKET"] = Lono::S3::Bucket.name
+      data
+    end
+
+    def update_template!(hash)
+      hash.each do |k, v|
+        if v.is_a?(String)
+          if v =~ %r{^LONO://}
+            v.replace(replacements[v]) # replace the placeholder
+          end
+        elsif v.is_a?(Hash)
+          update_template!(v) # recurse
+        elsif v.is_a?(Array)
+          v.each { |x| update_template!(x) if x.is_a?(Hash) }
+        end
+      end
+      hash
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/files.rb

@@ -0,0 +1,9 @@
+class Lono::Builder::Dsl::Finalizer
+  class Files < Base
+    # Replaces metadata under each logical id resource.
+    def run
+      Build.new(@options).run
+      @cfn = Replace.new(@options).run
+    end
+  end
+end

data/lib/lono/builder/{template/dsl → dsl}/finalizer/parameter_groups.rb

@@ -1,10 +1,11 @@
-class Lono::Builder::Template::Dsl::Finalizer
-  class ParameterGroups
+class Lono::Builder::Dsl::Finalizer
+  class ParameterGroups < Base
     extend Memoist
-    include Lono::Builder::Template::Dsl::Evaluator::Stringify
+    include Lono::Builder::Util::Stringify
 
-    def initialize(cfn, parameters)
-      @cfn, @parameters = cfn, parameters
+    def initialize(options={})
+      super
+      @parameters = options[:parameters]
     end
 
     def run

data/lib/lono/builder/dsl/finalizer.rb

@@ -0,0 +1,15 @@
+class Lono::Builder::Dsl
+  class Finalizer
+    def initialize(cfn, options={})
+      @cfn, @options = cfn, options
+    end
+
+    def run
+      o = @options.merge(cfn: @cfn)
+      @cfn = ParameterGroups.new(o).run
+      @cfn = Configsets.new(o).run
+      @cfn = Files.new(o).run
+      @cfn
+    end
+  end
+end

data/lib/lono/builder/{template/dsl/evaluator/helpers/ec2_helper.rb → dsl/helpers/ec2.rb}

@@ -1,5 +1,5 @@
-module Lono::Builder::Template::Dsl::Evaluator::Helpers
-  module Ec2Helper
+module Lono::Builder::Dsl::Helpers
+  module Ec2
     extend Memoist
 
     def default_vpc
@@ -20,7 +20,7 @@ module Lono::Builder::Template::Dsl::Evaluator::Helpers
     def key_pairs(regexp=nil)
       resp = ec2.describe_key_pairs
       key_names = resp.key_pairs.map(&:key_name)
-      key_names.select! { |k| k =~ regexp }
+      key_names.select! { |k| k =~ regexp } if regexp
       key_names
     end
   end

data/lib/lono/builder/dsl/helpers/files.rb

@@ -0,0 +1,7 @@
+module Lono::Builder::Dsl::Helpers
+  module Files
+    def files(path, o={})
+      Lono::Files.register(@options.merge(path: path).merge(o))
+    end
+  end
+end

data/lib/lono/builder/{template/helpers.rb → dsl/helpers/partials.rb}

@@ -1,7 +1,7 @@
 require "aws-sdk-core"
 
-class Lono::Builder::Template
-  module Helpers
+module Lono::Builder::Dsl::Helpers
+  module Partials
     def template_s3_path(template_name)
       # hi-jacking Uploader for https_url
       template_path = "output/#{@blueprint.name}/templates/#{template_name}.yml"
@@ -9,9 +9,7 @@ class Lono::Builder::Template
     end
 
     def template_params(param_name)
-      o = {
-        allow_not_exists: true
-      }.merge(@options)
+      o = @options.dup
       o["param"] = param_name
       generator = Lono::CLI::Param::Generator.new(o)
       # do not generate because lono cfn calling logic already generated it we only need the values
@@ -22,62 +20,54 @@ class Lono::Builder::Template
       end
     end
 
-    # Adjust the partial path so that it will use app/user_data
-    def user_data(path,vars={}, options={})
-      options.merge!(user_data: true)
-      partial(path,vars, options)
+    # The partial's path is a relative path.
+    #
+    # Example:
+    # Given file in app/partials/iam/docker.yml
+    #
+    #   <%= partial("iam/docker", {}, indent: 10) %>
+    #   <%= partial("iam/docker.yml", {}, indent: 10) %>
+    #
+    # If the user specifies the extension then use that instead of auto-adding
+    # the detected format.
+    def partial(path,vars={}, options={})
+      path = partial_path_for(path)
+      path = auto_add_format(path)
+
+      instance_variables!(vars)
+      result = render_path(path)
+
+      result = indent(result, options[:indent]) if options[:indent]
+      result + "\n"
     end
 
-    def current_region
-      region = Aws.config[:region]
-      region ||= ENV['AWS_REGION']
-      return region if region
-
-      default_region = 'us-east-1' # fallback if default not found in ~/.aws/config
-      if ENV['AWS_PROFILE']
-        path = "#{ENV['HOME']}/.aws/config"
-        if File.exist?(path)
-          lines = IO.readlines(path)
-          capture_default, capture_current = false, false
-          lines.each do | line|
-            if line.include?('[default]')
-              capture_default = true # next line
-              next
-            end
-            if capture_default && line.match(/region = /)
-              # over default from above
-              default_region = line.split(' = ').last.strip
-              capture_default = false
-            end
-
-            md = line.match(/\[profile (.*)\]/)
-            if md && md[1] == ENV['AWS_PROFILE']
-              capture_current = true
-              next
-            end
-            if capture_current && line.match(/region = /)
-              region = line.split(' = ').last.strip
-              capture_current = false
-            end
-          end
-        end
-
-        region ||= default_region
-        return region if region
+    # Take a hash and makes them instance variables in the current scope.
+    # Use this in custom helper methods to make variables accessible to ERB templates.
+    def instance_variables!(variables)
+      variables.each do |key, value|
+        instance_variable_set('@' + key.to_s, value)
       end
+    end
 
-      'us-east-1' # default
+    # add indentation
+    def indent(text, indentation_amount)
+      text.split("\n").map do |line|
+        " " * indentation_amount + line
+      end.join("\n")
     end
 
-  private
-    def render_path(path)
-      RenderMePretty.result(path, context: self)
+    def partial_exist?(path)
+      path = partial_path_for(path)
+      path = auto_add_format(path)
+      path && File.exist?(path)
     end
 
-    def user_data_path_for(path)
-      "#{Lono.config.paths.user_data}/#{path}"
+    def region
+      AwsData.region
     end
+    alias_method :current_region, :region
 
+  private
     def auto_add_format(path)
       # Return immediately if user provided explicit extension
       extension = File.extname(path) # current extension
@@ -107,16 +97,15 @@ class Lono::Builder::Template
 
     # Bash code that is meant to included in user-data
     def extract_scripts(options={})
-      settings = Lono.config.extract_scripts
-      options = settings.merge(options)
       # defaults also here in case they are removed from settings
+      path = options[:path]
       to = options[:to] || "/opt"
       user = options[:as] || "ec2-user"
 
-      if Dir.glob("#{Lono.config.paths.scripts}/*").empty?
+      if Dir.glob("#{@bluepint.root}/#{path}*").empty?
         logger.info "WARN: you are using the extract_scripts helper method but you do not have any scripts.".color(:yellow)
         calling_line = caller[0].split(':')[0..1].join(':')
-        logger.info "Called from: #{calling_line}"
+        logger.info "Called from1: #{calling_line}"
         return ""
       end
 
@@ -142,8 +131,10 @@ class Lono::Builder::Template
       upload.s3_dest
     end
 
-    def setting
-      @setting ||= Lono::Setting.new
+    def indent(text, indentation_amount)
+      text.split("\n").map do |line|
+        " " * indentation_amount + line
+      end.join("\n")
     end
   end
 end

data/lib/lono/builder/dsl/helpers/s3.rb

@@ -0,0 +1,14 @@
+module Lono::Builder::Dsl::Helpers
+  module S3
+    def s3_bucket
+      Lono::S3::Bucket.name
+    end
+    alias_method :lono_bucket, :s3_bucket
+
+    # Edge case when bucket is created for the first time and files_bucket is
+    # not available yet to be use by Finalizer::Files::Replace#replacements
+    def files_bucket
+      "LONO://S3_BUCKET"
+    end
+  end
+end

data/lib/lono/builder/{context/ssm_fetcher.rb → dsl/helpers/ssm/fetcher.rb}

@@ -1,8 +1,9 @@
 require 'aws-sdk-ssm'
 
-class Lono::Builder::Context
-  class SsmFetcher
+module Lono::Builder::Dsl::Helpers::Ssm
+  class Fetcher
     extend Memoist
+    include Lono::Utils::Logging
 
     def get(name)
       fetch_ssm_value(name)
@@ -12,7 +13,8 @@ class Lono::Builder::Context
       resp = ssm.get_parameter(name: name, with_decryption: true)
       resp.parameter.value
     rescue Aws::SSM::Errors::ParameterNotFound
-      'SSM-PARAM-NOT-FOUND'
+      logger.warn 'WARN: SSM-PARAM-NOT-FOUND'
+      nil
     end
 
     def ssm

data/lib/lono/builder/{context/helpers.rb → dsl/helpers/ssm.rb}

@@ -1,5 +1,5 @@
-class Lono::Builder::Context
-  module Helpers
+module Lono::Builder::Dsl::Helpers
+  module Ssm
     extend Memoist
 
     def ssm(name)
@@ -7,7 +7,7 @@ class Lono::Builder::Context
     end
 
     def ssm_fetcher
-      SsmFetcher.new
+      Fetcher.new
     end
     memoize :ssm_fetcher
   end

data/lib/lono/builder/{template/dsl/evaluator/helpers/stack_helper.rb → dsl/helpers/stack.rb}

@@ -1,5 +1,5 @@
-module Lono::Builder::Template::Dsl::Evaluator::Helpers
-  module StackHelper
+module Lono::Builder::Dsl::Helpers
+  module Stack
     extend Memoist
     include Lono::AwsServices
 
@@ -15,7 +15,8 @@ module Lono::Builder::Template::Dsl::Evaluator::Helpers
       if o
         o.output_value
       else
-        logger.info "WARN: NOT FOUND: output #{key} for stack #{stack_name}"
+        logger.info "WARN: STACK OUTPUT NOT FOUND: output #{key} for stack #{stack_name}"
+        nil
       end
     end
 
@@ -29,14 +30,13 @@ module Lono::Builder::Template::Dsl::Evaluator::Helpers
         resource.physical_resource_id
       else
         logger.info "WARN: NOT FOUND: logical_id #{logical_id} for stack #{stack_name}"
+        nil
       end
     end
 
     def stack_name(blueprint)
-      return blueprint unless Lono.config.names.output.expand
       names = Lono::Names.new(blueprint: blueprint)
-      # explicit expansion pattern provided by user
-      pattern = blueprint.include?(':') ? blueprint : Lono.config.names.output.stack
+      pattern = blueprint # blueprint can be a pattern provided by user. IE: :BLUEPRINT-:ENV
       names.expansion(pattern)
     end
 

data/lib/lono/builder/{template/dsl/evaluator/helpers/tags_helper.rb → dsl/helpers/tags.rb}

@@ -1,5 +1,5 @@
-module Lono::Builder::Template::Dsl::Evaluator::Helpers
-  module TagsHelper
+module Lono::Builder::Dsl::Helpers
+  module Tags
     def tags(hash={})
       if hash.empty?
         tag_list(@tags) if @tags # when hash is empty, use @tags variable. If not set then return nil

data/lib/lono/builder/dsl/helpers/template_file.rb

@@ -0,0 +1,66 @@
+module Lono::Builder::Dsl::Helpers
+  module TemplateFile
+    extend Memoist
+    include Lono::Utils::Pretty
+
+    def template_file(path)
+      path = "#{@blueprint.root}/#{path}"
+      if File.exist?(path)
+        render_file(path)
+      else
+        template_file_missing(path)
+      end
+    end
+    # do not memoize :template_file - it'll hide the template_file_missing error
+
+    # Caller lines are different for OSes:
+    #
+    #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/lono-1.1.1/lib/lono/builder.rb:34:in `build'"
+    #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/lono-1.1.1/lib/lono/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
+    #
+    class TempleFileNotFoundError < StandardError; end
+    def template_file_missing(path)
+      message = "ERROR: path #{pretty_path(path)} not found"
+      caller_line = caller.find { |l| l =~ %r{/blueprints/} }
+      logger.error message.color(:red)
+      logger.error "Called from:"
+      logger.error "    #{pretty_path(caller_line)}"
+      # Raise an error so Dsl::Evaluator#template_evaluation_error provides user friendly info
+      raise TempleFileNotFoundError.new
+    end
+
+    def render_file(path)
+      if File.exist?(path)
+        RenderMePretty.result(path, context: self)
+      else
+        lines = caller.select { |l| l.include?(Lono.root.to_s) }
+        caller_line = pretty_path(lines.first)
+        message =<<~EOL
+          WARN: #{pretty_path(path)} does not exist
+          Called from: #{caller_line}
+        EOL
+        logger.info message.color(:yellow)
+        message
+      end
+    end
+    alias_method :render_path, :render_file
+
+    def user_data_script
+      unless @user_data_script
+        script_example = pretty_path("#{@blueprint.root}/template/user_data.sh")
+        return <<~EOL
+          # @user_data_script variable not set. IE: @user_data_script = "#{script_example}"
+          # Also, make sure that "#{script_example}" exists.
+        EOL
+      end
+
+      if File.exist?(@user_data_script)
+        render_file(@user_data_script)
+      else
+        message = "WARN: #{@user_data_script} not found"
+        logger.info message.color(:yellow)
+        "# #{message}"
+      end
+    end
+  end
+end

data/lib/lono/builder/dsl/helpers.rb

@@ -0,0 +1,5 @@
+class Lono::Builder::Dsl
+  module Helpers
+    include_modules("helpers")
+  end
+end

data/lib/lono/builder/{template/dsl/evaluator/section → dsl/syntax/core}/base.rb

@@ -1,8 +1,7 @@
-module Lono::Builder::Template::Dsl::Evaluator::Section
+module Lono::Builder::Dsl::Syntax::Core
   class Base
-    include Lono::Builder::Template::Dsl::Evaluator::Fn
-    include Lono::Builder::Template::Dsl::Evaluator::Helpers
-    include Lono::Builder::Template::Dsl::Evaluator::Stringify
+    include Lono::Builder::Util::Stringify
+    include Lono::Builder::Dsl::Syntax::Fn
 
     def initialize(blueprint, *definition)
       @blueprint = blueprint
@@ -17,7 +16,7 @@ module Lono::Builder::Template::Dsl::Evaluator::Section
 
     # Remove items with nil value automatically
     def clean(data)
-      Lono::Builder::Template::Dsl::Evaluator::Squeezer.new(data).squeeze
+      Squeezer.new(data).squeeze
     end
   end
 end

data/lib/lono/builder/{template/dsl/evaluator/section → dsl/syntax/core}/condition.rb

@@ -2,7 +2,7 @@
 #
 #   template - uses @definition to build a CloudFormation template section
 #
-module Lono::Builder::Template::Dsl::Evaluator::Section
+module Lono::Builder::Dsl::Syntax::Core
   class Condition < Base
     def template
       camelize(standarize(@definition))

data/lib/lono/builder/{template/dsl/evaluator/section → dsl/syntax/core}/mapping.rb

@@ -2,7 +2,7 @@
 #
 #   template - uses @definition to build a CloudFormation template section
 #
-module Lono::Builder::Template::Dsl::Evaluator::Section
+module Lono::Builder::Dsl::Syntax::Core
   class Mapping < Base
     def template
       camelize(standarize(@definition))

data/lib/lono/builder/{template/dsl/evaluator/section → dsl/syntax/core}/output.rb

@@ -2,7 +2,7 @@
 #
 #   template - uses @definition to build a CloudFormation template section
 #
-module Lono::Builder::Template::Dsl::Evaluator::Section
+module Lono::Builder::Dsl::Syntax::Core
   class Output < Base
     def template
       camelize(standarize(@definition))

data/lib/lono/builder/{template/dsl/evaluator/section → dsl/syntax/core}/parameter.rb

@@ -2,7 +2,7 @@
 #
 #   template - uses @definition to build a CloudFormation template section
 #
-module Lono::Builder::Template::Dsl::Evaluator::Section
+module Lono::Builder::Dsl::Syntax::Core
   class Parameter < Base
     attr_accessor :group_label
     attr_reader :conditional, :label

data/lib/lono/builder/{template/dsl/evaluator/section → dsl/syntax/core}/resource/property_mover.rb

@@ -1,12 +1,22 @@
-class Lono::Builder::Template::Dsl::Evaluator::Section::Resource
+class Lono::Builder::Dsl::Syntax::Core::Resource
   # Moves the property to the top-level attributes *destructively*
   class PropertyMover
     def initialize(resource, logical_id, properties)
       @resource, @logical_id, @properties = resource, logical_id, properties
     end
 
+    # AWS Docs: Resource attribute reference
+    # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-product-attribute-reference.html
     def move!
-      %w[Condition DeletionPolicy DependsOn].each do |attribute_name|
+      %w[
+        Condition
+        CreationPolicy
+        DeletionPolicy
+        DependsOn
+        Metadata
+        UpdatePolicy
+        UpdateReplacePolicy
+      ].each do |attribute_name|
         # Account for camelize, underscore, String, and Symbol
         move(attribute_name.to_sym)
         move(attribute_name.camelize.to_sym)

data/lib/lono/builder/{template/dsl/evaluator/section → dsl/syntax/core}/resource.rb

@@ -2,7 +2,7 @@
 #
 #   template - uses @definition to build a CloudFormation template section
 #
-module Lono::Builder::Template::Dsl::Evaluator::Section
+module Lono::Builder::Dsl::Syntax::Core
   class Resource < Base
     def template
       camelize(standarize(@definition))

data/lib/lono/builder/{template/dsl/evaluator/section → dsl/syntax/core}/section.rb

@@ -2,7 +2,7 @@
 #
 #   template - uses @definition to build a CloudFormation template section
 #
-module Lono::Builder::Template::Dsl::Evaluator::Section
+module Lono::Builder::Dsl::Syntax::Core
   class Section < Base
     def template
       hash, _ = @definition

data/lib/lono/builder/dsl/syntax/core/squeezer.rb

@@ -0,0 +1,40 @@
+module Lono::Builder::Dsl::Syntax::Core
+  class Squeezer
+    def initialize(data)
+      @data = data
+    end
+
+    def squeeze(new_data=nil, previous_key=nil)
+      data = new_data.nil? ? @data : new_data
+
+      case data
+      when Array
+        # .compact prevents infinite loop when data = [nil] on accident
+        # IE: data[:key] = [nil]
+        data.compact.map! { |v| squeeze(v) }
+      when Hash
+        data.each_with_object({}) do |(k,v), squeezed|
+          # only remove nil and empty Array values within Hash structures
+          squeezed[k] = squeeze(v, k) unless v.nil? # || empty_array?(v, previous_key)
+          squeezed
+        end
+      else
+        data # do not transform
+      end
+    end
+
+    # Special case do not squeeze empty Array when previous_key is "yum". Handles:
+    #
+    #   Metadata:
+    #     AWS::CloudFormation::Init:
+    #       config:
+    #         packages:
+    #           yum:
+    #             httpd: []
+    #
+    def empty_array?(v, previous_key)
+      return false if previous_key.to_s == "yum"
+      v.is_a?(Array) && v.empty?
+    end
+  end
+end

data/lib/lono/builder/{template/dsl/evaluator/section/methods.rb → dsl/syntax/core.rb}

@@ -1,6 +1,6 @@
 # Organize core section method syntax here
-module Lono::Builder::Template::Dsl::Evaluator::Section
-  module Methods
+module Lono::Builder::Dsl::Syntax
+  module Core
     def aws_template_format_version(version_date)
       @cfn["AWSTemplateFormatVersion"] = version_date
     end

data/lib/lono/builder/{template/dsl/evaluator → dsl/syntax}/fn.rb

@@ -1,4 +1,4 @@
-class Lono::Builder::Template::Dsl::Evaluator
+module Lono::Builder::Dsl::Syntax
   module Fn
     # Also act as documentation on the method signatures
     # Also used in Coder crafting so should always list all the functions here
@@ -102,8 +102,6 @@ class Lono::Builder::Template::Dsl::Evaluator
               else
                 item
               end
-      # list.map!(&:camelize) unless options[:autoformat] == false # TODO: maybe add as an option.
-      # feel this may be to destructive since am going with auto_camelize false for resources now.
       args = [list[0], list[1..-1].join('.')]
       { "Fn::GetAtt" => args }
     end

data/lib/lono/builder/{template/dsl/evaluator → dsl}/syntax/parameter_group.rb

@@ -1,4 +1,4 @@
-module Lono::Builder::Template::Dsl::Evaluator::Syntax
+module Lono::Builder::Dsl::Syntax
   module ParameterGroup
     def parameter_group(label)
       @group_label = label