lono 8.0.0.pre.rc2 → 8.0.0.pre.rc5

data/lib/lono/builder/dsl/finalizer/configsets.rb

@@ -1,41 +1,20 @@
 class Lono::Builder::Dsl::Finalizer
-  class Configsets
-    extend Memoist
-
-    def initialize(cfn)
-      @cfn = cfn
-    end
-
+  class Configsets < Base
+    # Replaces metadata under each logical id resource.
     def run
-      @cfn = add # overwrite
-      @cfn
-    end
-
-    def add
-      metadata_map.each do |logical_id, metadata_configset|
+      dsl = Lono::Builder::Configset::Evaluator.new(@options.merge(cfn: @cfn))
+      metadata_map = dsl.evaluate
+      metadata_map.each do |logical_id, cs|
         resource = @cfn["Resources"][logical_id]
-
         unless resource
-          logger.info "WARN: Resources.#{logical_id} not found in the template. Are you sure you are specifying the correct resource id in your configsets configs?".color(:yellow)
+          puts "WARN: Resources.#{logical_id} not found in the template. Are you sure you specified the correct resource logical id in your configsets.rb?".color(:yellow)
           next
         end
 
-        resource["Metadata"] ||= metadata_configset["Metadata"]
-
-        # metdata = resource["Metadata"] ||= {}
-        # metdata["AWS::CloudFormation::Init"] ||= {}
-        # # The metadata_configset has been combined with the original AWS::CloudFormation::Init if it exists
-        # metdata["AWS::CloudFormation::Init"] = metadata_configset["AWS::CloudFormation::Init"]
+        resource["Metadata"] = cs["Metadata"]
       end
 
       @cfn
     end
-
-    def metadata_map
-      []
-      # combiner = Lono::Configset::Combiner.new(@cfn, @options)
-      # combiner.metadata_map
-    end
-    memoize :metadata_map
   end
 end

data/lib/lono/builder/dsl/finalizer/files/base.rb

@@ -0,0 +1,4 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Base < Lono::Builder::Dsl::Finalizer::Base
+  end
+end

data/lib/lono/builder/dsl/finalizer/files/build.rb

@@ -0,0 +1,50 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Build < Base
+    def initialize(options={})
+      super
+      @output_path = "#{Lono.root}/.output/files"
+    end
+
+    def run
+      return if Lono::Files.empty?
+      logger.debug "Building files"
+      clean
+      validate!
+      build_files
+    end
+
+    def build_files
+      Lono::Files.files.each do |file| # using singular file, but is like a "file_list"
+        file.build
+        file.build_lambda_layer(@cfn)
+        file.compress
+        # UploadNote: Instead of part of the build process here, uploading happen as part
+        # of create_stack or execute_change_set after user confirms action. IE:
+        #   Cfn::Deploy#create and Cfn::Deploy#update
+      end
+    end
+
+    def clean
+      FileUtils.rm_rf(@output_path)
+    end
+
+    def validate!
+      missing = Lono::Files.files.select do |file|
+        !File.exist?(file.full_path)
+      end
+      return if missing.empty?
+
+      logger.info "ERROR: These files are missing".color(:red)
+      logger.info <<~EOL
+        The file helper is calling them.
+        They need to exist to be uploaded to s3.
+      EOL
+      missing.each do |file|
+        logger.info "    #{pretty_path(file.full_path)}"
+        logger.info "    Called at #{file.call_line}"
+      end
+      logger.info "Please double check that they exist."
+      exit 1
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/files/replace.rb

@@ -0,0 +1,36 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Replace < Base
+    def run
+      replacements.each do |placeholder, replacement|
+        update_template!(@cfn)
+      end
+      @cfn
+    end
+
+    def replacements
+      data = Lono::Files.files.inject({}) do |result, file|
+        result.merge(file.marker => file.s3_key)
+      end
+      # Edge case when bucket is created for the first time and files_bucket is
+      # not available yet. So we call it at this point.
+      Lono::S3::Bucket.ensure_exist
+      data["LONO://S3_BUCKET"] = Lono::S3::Bucket.name
+      data
+    end
+
+    def update_template!(hash)
+      hash.each do |k, v|
+        if v.is_a?(String)
+          if v =~ %r{^LONO://}
+            v.replace(replacements[v]) # replace the placeholder
+          end
+        elsif v.is_a?(Hash)
+          update_template!(v) # recurse
+        elsif v.is_a?(Array)
+          v.each { |x| update_template!(x) if x.is_a?(Hash) }
+        end
+      end
+      hash
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/files.rb

@@ -0,0 +1,9 @@
+class Lono::Builder::Dsl::Finalizer
+  class Files < Base
+    # Replaces metadata under each logical id resource.
+    def run
+      Build.new(@options).run
+      @cfn = Replace.new(@options).run
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/parameter_groups.rb

@@ -1,10 +1,11 @@
 class Lono::Builder::Dsl::Finalizer
-  class ParameterGroups
+  class ParameterGroups < Base
     extend Memoist
     include Lono::Builder::Util::Stringify
 
-    def initialize(cfn, parameters)
-      @cfn, @parameters = cfn, parameters
+    def initialize(options={})
+      super
+      @parameters = options[:parameters]
     end
 
     def run

data/lib/lono/builder/dsl/finalizer.rb

@@ -5,8 +5,10 @@ class Lono::Builder::Dsl
     end
 
     def run
-      @cfn = ParameterGroups.new(@cfn, @options[:parameters]).run
-      @cfn = Configsets.new(@cfn).run
+      o = @options.merge(cfn: @cfn)
+      @cfn = ParameterGroups.new(o).run
+      @cfn = Configsets.new(o).run
+      @cfn = Files.new(o).run
       @cfn
     end
   end

data/lib/lono/builder/dsl/helpers/ec2.rb

@@ -2,12 +2,25 @@ module Lono::Builder::Dsl::Helpers
   module Ec2
     extend Memoist
 
+    # Returns vpc object
+    def vpc(name)
+      filters = name == "default" ?
+                [name: "isDefault", values: ["true"]] :
+                [name: "tag:Name", values: [name]]
+      resp = ec2.describe_vpcs(filters: filters)
+      resp.vpcs.first
+    end
+    memoize :vpc
+
     def default_vpc
-      resp = ec2.describe_vpcs(filters: [name: "isDefault", values: ["true"]])
-      vpc = resp.vpcs.first
+      vpc = vpc("default")
       vpc ? vpc.vpc_id : "no default vpc found"
     end
-    memoize :default_vpc
+
+    def default_vpc_cidr
+      vpc = vpc("default")
+      vpc.cidr_block
+    end
 
     def default_subnets
       return "no default subnets because no default vpc found" if default_vpc == "no default vpc found"

data/lib/lono/builder/dsl/helpers/files.rb

@@ -0,0 +1,7 @@
+module Lono::Builder::Dsl::Helpers
+  module Files
+    def files(path, o={})
+      Lono::Files.register(@options.merge(path: path).merge(o))
+    end
+  end
+end

data/lib/lono/builder/dsl/helpers/partials.rb

@@ -9,9 +9,7 @@ module Lono::Builder::Dsl::Helpers
     end
 
     def template_params(param_name)
-      o = {
-        allow_not_exists: true
-      }.merge(@options)
+      o = @options.dup
       o["param"] = param_name
       generator = Lono::CLI::Param::Generator.new(o)
       # do not generate because lono cfn calling logic already generated it we only need the values
@@ -22,62 +20,54 @@ module Lono::Builder::Dsl::Helpers
       end
     end
 
-    # Adjust the partial path so that it will use app/user_data
-    def user_data(path,vars={}, options={})
-      options.merge!(user_data: true)
-      partial(path,vars, options)
+    # The partial's path is a relative path.
+    #
+    # Example:
+    # Given file in app/partials/iam/docker.yml
+    #
+    #   <%= partial("iam/docker", {}, indent: 10) %>
+    #   <%= partial("iam/docker.yml", {}, indent: 10) %>
+    #
+    # If the user specifies the extension then use that instead of auto-adding
+    # the detected format.
+    def partial(path,vars={}, options={})
+      path = partial_path_for(path)
+      path = auto_add_format(path)
+
+      instance_variables!(vars)
+      result = render_path(path)
+
+      result = indent(result, options[:indent]) if options[:indent]
+      result + "\n"
     end
 
-    def current_region
-      region = Aws.config[:region]
-      region ||= ENV['AWS_REGION']
-      return region if region
-
-      default_region = 'us-east-1' # fallback if default not found in ~/.aws/config
-      if ENV['AWS_PROFILE']
-        path = "#{ENV['HOME']}/.aws/config"
-        if File.exist?(path)
-          lines = IO.readlines(path)
-          capture_default, capture_current = false, false
-          lines.each do | line|
-            if line.include?('[default]')
-              capture_default = true # next line
-              next
-            end
-            if capture_default && line.match(/region = /)
-              # over default from above
-              default_region = line.split(' = ').last.strip
-              capture_default = false
-            end
-
-            md = line.match(/\[profile (.*)\]/)
-            if md && md[1] == ENV['AWS_PROFILE']
-              capture_current = true
-              next
-            end
-            if capture_current && line.match(/region = /)
-              region = line.split(' = ').last.strip
-              capture_current = false
-            end
-          end
-        end
-
-        region ||= default_region
-        return region if region
+    # Take a hash and makes them instance variables in the current scope.
+    # Use this in custom helper methods to make variables accessible to ERB templates.
+    def instance_variables!(variables)
+      variables.each do |key, value|
+        instance_variable_set('@' + key.to_s, value)
       end
+    end
 
-      'us-east-1' # default
+    # add indentation
+    def indent(text, indentation_amount)
+      text.split("\n").map do |line|
+        " " * indentation_amount + line
+      end.join("\n")
     end
 
-  private
-    def render_path(path)
-      RenderMePretty.result(path, context: self)
+    def partial_exist?(path)
+      path = partial_path_for(path)
+      path = auto_add_format(path)
+      path && File.exist?(path)
     end
 
-    def user_data_path_for(path)
-      "#{Lono.config.paths.user_data}/#{path}"
+    def region
+      AwsData.region
     end
+    alias_method :current_region, :region
 
+  private
     def auto_add_format(path)
       # Return immediately if user provided explicit extension
       extension = File.extname(path) # current extension
@@ -107,16 +97,15 @@ module Lono::Builder::Dsl::Helpers
 
     # Bash code that is meant to included in user-data
     def extract_scripts(options={})
-      settings = Lono.config.extract_scripts
-      options = settings.merge(options)
       # defaults also here in case they are removed from settings
+      path = options[:path]
       to = options[:to] || "/opt"
       user = options[:as] || "ec2-user"
 
-      if Dir.glob("#{Lono.config.paths.scripts}/*").empty?
+      if Dir.glob("#{@bluepint.root}/#{path}*").empty?
         logger.info "WARN: you are using the extract_scripts helper method but you do not have any scripts.".color(:yellow)
         calling_line = caller[0].split(':')[0..1].join(':')
-        logger.info "Called from: #{calling_line}"
+        logger.info "Called from1: #{calling_line}"
         return ""
       end
 
@@ -141,5 +130,11 @@ module Lono::Builder::Dsl::Helpers
       upload = Lono::Script::Upload.new(@options)
       upload.s3_dest
     end
+
+    def indent(text, indentation_amount)
+      text.split("\n").map do |line|
+        " " * indentation_amount + line
+      end.join("\n")
+    end
   end
 end

data/lib/lono/builder/dsl/helpers/s3.rb

@@ -3,14 +3,12 @@ module Lono::Builder::Dsl::Helpers
     def s3_bucket
       Lono::S3::Bucket.name
     end
-    alias_method :lono_bucket_name, :s3_bucket
+    alias_method :lono_bucket, :s3_bucket
 
-    def s3_key(name, options={})
-      default = {type: "file"}
-      options.reverse_merge!(default)
-      Lono::AppFile::Registry.register(name, @blueprint.name, options)
-      "file://app/files/#{options[:type]}/#{name}" # placeholder for post processing
+    # Edge case when bucket is created for the first time and files_bucket is
+    # not available yet to be use by Finalizer::Files::Replace#replacements
+    def files_bucket
+      "LONO://S3_BUCKET"
     end
-    alias_method :file_s3_key, :s3_key
   end
 end

data/lib/lono/builder/dsl/helpers/ssm/fetcher.rb

@@ -3,6 +3,7 @@ require 'aws-sdk-ssm'
 module Lono::Builder::Dsl::Helpers::Ssm
   class Fetcher
     extend Memoist
+    include Lono::Utils::Logging
 
     def get(name)
       fetch_ssm_value(name)
@@ -12,7 +13,8 @@ module Lono::Builder::Dsl::Helpers::Ssm
       resp = ssm.get_parameter(name: name, with_decryption: true)
       resp.parameter.value
     rescue Aws::SSM::Errors::ParameterNotFound
-      'SSM-PARAM-NOT-FOUND'
+      logger.warn 'WARN: SSM-PARAM-NOT-FOUND'
+      nil
     end
 
     def ssm

data/lib/lono/builder/dsl/helpers/stack.rb

@@ -15,7 +15,7 @@ module Lono::Builder::Dsl::Helpers
       if o
         o.output_value
       else
-        logger.info "WARN2: NOT FOUND: output #{key} for stack #{stack_name}"
+        logger.info "WARN: STACK OUTPUT NOT FOUND: output #{key} for stack #{stack_name}"
         nil
       end
     end
@@ -35,10 +35,8 @@ module Lono::Builder::Dsl::Helpers
     end
 
     def stack_name(blueprint)
-      return blueprint unless Lono.config.names.output.expand
       names = Lono::Names.new(blueprint: blueprint)
-      # explicit expansion pattern provided by user
-      pattern = blueprint.include?(':') ? blueprint : Lono.config.names.output.stack
+      pattern = blueprint # blueprint can be a pattern provided by user. IE: :BLUEPRINT-:ENV
       names.expansion(pattern)
     end
 

data/lib/lono/builder/dsl/helpers/template_file.rb

@@ -1,54 +1,48 @@
 module Lono::Builder::Dsl::Helpers
   module TemplateFile
     extend Memoist
+    include Lono::Utils::CallLine
     include Lono::Utils::Pretty
 
+    # Do not memoize :template_file - it'll hide the template_file_missing error
     def template_file(path)
-      path = "#{@blueprint.root}/#{path}"
+      path = "#{@blueprint.root}/#{path}" unless path.starts_with?('/')
       if File.exist?(path)
-        render_file(path)
+        RenderMePretty.result(path, context: self)
       else
         template_file_missing(path)
       end
     end
-    # do not memoize :template_file - it'll hide the template_file_missing error
+    alias_method :render_file, :template_file
+    alias_method :render_path, :template_file
+    alias_method :user_data, :template_file
+    alias_method :content, :template_file
+
 
     # Caller lines are different for OSes:
     #
-    #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/terraspace-1.1.1/lib/terraspace/builder.rb:34:in `build'"
-    #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/terraspace-1.1.1/lib/terraspace/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
+    #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/lono-1.1.1/lib/lono/builder.rb:34:in `build'"
+    #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/lono-1.1.1/lib/lono/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
     #
-    class TempleFileNotFoundError < StandardError; end
     def template_file_missing(path)
-      message = "ERROR: path #{pretty_path(path)} not found"
-      caller_line = caller.find { |l| l =~ %r{/blueprints/} } # TODO: show code itself
-      logger.error message.color(:red)
-      logger.error "Called from:"
-      logger.error "    #{pretty_path(caller_line)}"
-      # Raise an error so Dsl::Evaluator#template_evaluation_error provides user friendly info
-      raise TempleFileNotFoundError.new
-    end
-
-    def render_file(path)
-      RenderMePretty.result(path, context: template_context)
+      logger.warn "WARN: File path not found: #{pretty_path(path)}".color(:yellow)
+      call_line = lono_call_line
+      DslEvaluator.print_code(call_line) # returns true right now
+      ""
     end
-    memoize :render_file
 
     def user_data_script
-      unless @user_data_script
+      path = @user_data_script || @user_data_script_path
+      unless path
+        # script_example = pretty_path("#{@blueprint.root}/template/bootstrap.sh")
+        script_example = "bootstrap.sh"
         return <<~EOL
-          # @user_data_script variable not set. IE: @user_data_script = "#{pretty_path(@blueprint.root)}/template/user_data.sh"
-          # Also, make sure that "config/#{@blueprint.name}/user_data/boostrap.sh" path you're using exists.
+          # @user_data_script_path variable not set. IE: @user_data_script_path = "#{script_example}"
+          # Also, make sure that "#{script_example}" exists.
         EOL
       end
-
-      if File.exist?(@user_data_script)
-        render_file(@user_data_script)
-      else
-        message = "WARN: #{@user_data_script} not found"
-        logger.info message.color(:yellow)
-        "# #{message}"
-      end
+      user_data(path)
     end
+    alias_method :user_data_script_path, :user_data_script
   end
 end

data/lib/lono/builder/dsl/helpers.rb

@@ -1,15 +1,5 @@
-# Built-in helpers for the DSL form
 class Lono::Builder::Dsl
   module Helpers
-    extend Memoist
-
-    # Auto include all modules in helpers folder
-    # only load one level deep. IE: ssm/fetcher is a class
-    helpers_dir = File.expand_path("helpers", __dir__)
-    Dir.glob("#{helpers_dir}/*.rb").each do |path|
-      next unless File.file?(path)
-      klass = path.gsub(%r{.*/lib/},'').sub(".rb",'').camelize
-      include klass.constantize
-    end
+    include_modules("helpers")
   end
 end

data/lib/lono/builder/dsl/syntax/core/resource/property_mover.rb

@@ -5,8 +5,18 @@ class Lono::Builder::Dsl::Syntax::Core::Resource
       @resource, @logical_id, @properties = resource, logical_id, properties
     end
 
+    # AWS Docs: Resource attribute reference
+    # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-product-attribute-reference.html
     def move!
-      %w[Condition DeletionPolicy DependsOn].each do |attribute_name|
+      %w[
+        Condition
+        CreationPolicy
+        DeletionPolicy
+        DependsOn
+        Metadata
+        UpdatePolicy
+        UpdateReplacePolicy
+      ].each do |attribute_name|
         # Account for camelize, underscore, String, and Symbol
         move(attribute_name.to_sym)
         move(attribute_name.camelize.to_sym)

data/lib/lono/builder/dsl/syntax/core/squeezer.rb

@@ -4,21 +4,37 @@ module Lono::Builder::Dsl::Syntax::Core
       @data = data
     end
 
-    def squeeze(new_data=nil)
+    def squeeze(new_data=nil, previous_key=nil)
       data = new_data.nil? ? @data : new_data
 
       case data
       when Array
-        data.map! { |v| squeeze(v) }
+        # .compact prevents infinite loop when data = [nil] on accident
+        # IE: data[:key] = [nil]
+        data.compact.map! { |v| squeeze(v) }
       when Hash
         data.each_with_object({}) do |(k,v), squeezed|
           # only remove nil and empty Array values within Hash structures
-          squeezed[k] = squeeze(v) unless v.nil? || v.is_a?(Array) && v.empty?
+          squeezed[k] = squeeze(v, k) unless v.nil? # || empty_array?(v, previous_key)
           squeezed
         end
       else
         data # do not transform
       end
     end
+
+    # Special case do not squeeze empty Array when previous_key is "yum". Handles:
+    #
+    #   Metadata:
+    #     AWS::CloudFormation::Init:
+    #       config:
+    #         packages:
+    #           yum:
+    #             httpd: []
+    #
+    def empty_array?(v, previous_key)
+      return false if previous_key.to_s == "yum"
+      v.is_a?(Array) && v.empty?
+    end
   end
 end

data/lib/lono/builder/dsl/syntax/fn.rb

@@ -102,8 +102,6 @@ module Lono::Builder::Dsl::Syntax
               else
                 item
               end
-      # list.map!(&:camelize) unless options[:autoformat] == false # TODO: maybe add as an option.
-      # feel this may be to destructive since am going with auto_camelize false for resources now.
       args = [list[0], list[1..-1].join('.')]
       { "Fn::GetAtt" => args }
     end

data/lib/lono/builder/dsl.rb

@@ -1,4 +1,4 @@
-module Lono::Builder
+class Lono::Builder
   class Dsl < Lono::CLI::Base
     attr_reader :results
     def run
@@ -8,7 +8,7 @@ module Lono::Builder
 
     def build_template
       evaluator = Evaluator.new(@options)
-      evaluator.build
+      evaluator.evaluate
     end
   end
 end

data/lib/lono/builder/param.rb

@@ -1,8 +1,13 @@
-module Lono::Builder
+class Lono::Builder
   class Param < Lono::CLI::Base
     attr_reader :env_path, :base_path # set when build is called
+    include Lono::Builder::Dsl::Syntax
+    include Lono::Builder::Context
+    # Overriding output resource DSL method
+    alias_method :output, :stack_output
 
     def build
+      load_context
       logger.info "Building parameters"
 
       contents = []
@@ -36,7 +41,7 @@ module Lono::Builder
     def render_erb(path)
       return unless path
       if File.exist?(path)
-        RenderMePretty.result(path, context: params_context)
+        RenderMePretty.result(path, context: self)
       end
     end
 

data/lib/lono/builder/template/upload.rb

@@ -1,21 +1,8 @@
-require 'erb'
-require 'json'
-require 'base64'
-require 'digest'
-
 class Lono::Builder::Template
   class Upload < Lono::CLI::Base
-    def initialize(options={})
-      super
-      @checksums = {}
-      @prefix = Lono.env # s3://s3-bucket/development
-    end
-
     def run
-      paths = Dir.glob("#{Lono.root}/output/#{@blueprint.name}/template.yml")
-      paths.select { |p| File.file?(p) }.each do |path|
-        Lono::S3::Uploader.new(path).upload
-      end
+      path = "#{Lono.root}/output/#{@blueprint.name}/template.yml"
+      Lono::S3::Uploader.new(path).upload
     end
   end
 end

data/lib/lono/builder/template.rb

@@ -1,6 +1,4 @@
-require "yaml"
-
-module Lono::Builder
+class Lono::Builder
   class Template < Lono::CLI::Base
     extend Memoist