lono 8.0.0.pre.rc2 → 8.0.0.pre.rc3

data/lib/lono/{configset/strategy → builder/configset/definition}/dsl.rb

@@ -1,29 +1,25 @@
-module Lono::Configset::Strategy
+class Lono::Builder::Configset::Definition
   class Dsl < Base
-    # Configset helpers take higher precedence than Lono DSL Helpers. IE: s3_key is overwritten but s3_bucket is the same
-    include Helpers::Dsl
+    include Syntax
 
     def initialize(options={})
       super
-      @structure = {} # holds in memory the configset hash structure to build AWS::CloudFormation::Init
       @current = "main" # current configset
-      @command_counts = Hash.new(0)
       @tracked = []
+      @structure = {} # holds in memory the configset hash structure to build AWS::CloudFormation::Init
+      @command_counts = Hash.new(0)
       # Also support ability to add AWS::CloudFormation::Authentication
       @authentication = nil # holds IAM policy info to build AWS::CloudFormation::Authentication
     end
 
-    def find_evaluation_path
-      "#{@root}/lib/configset.rb"
-    end
-
-    def load
-      evaluate_file(@evaluation_path)
-      configsets = @configsets || @tracked.uniq
+    def evaluate
+      load_context
+      evaluate_file(@configset.path)
+      configsets = configsets || @tracked.uniq
       configsets = ["main"] if configsets.empty?
       configsets_structure = {"configSets" => {"default" => configsets}}.merge(@structure)
-      full = { "AWS::CloudFormation::Init" => configsets_structure }
-      full.deep_stringify_keys!
+      cloudformation_init = { "AWS::CloudFormation::Init" => configsets_structure }
+      wrap_with_metadata(cloudformation_init)
     end
   end
 end

data/lib/lono/{configset/strategy → builder/configset/definition}/erb.rb

@@ -1,29 +1,22 @@
-require "json"
-
-module Lono::Configset::Strategy
+class Lono::Builder::Configset::Definition
   class Erb < Base
-    include Helpers::Erb
-
-    def find_evaluation_path
-      paths = %w[configset.yml configset.json].map { |p| "#{@root}/lib/#{p}" }
-      paths.find { |p| File.exist?(p) }
+    def evaluate
+      load_context
+      cloudformation_init = build
+      wrap_with_metadata(cloudformation_init)
     end
 
-    def load
-      content = RenderMePretty.result(@evaluation_path, context: self)
-      # init structure
-      if File.extname(@evaluation_path) == ".yml"
+    def build
+      content = RenderMePretty.result(@configset.path, context: self)
+      if File.extname(@configset.path) == ".yml"
         load_yaml(content)
       else
         JSON.load(content)
       end
     end
 
-    def authentication
-      # noop
-    end
-
     def load_yaml(content)
+      # init structure
       # Write to file so can use Yamler::Validator
       path = "/tmp/lono/configset.yml"
       FileUtils.mkdir_p(File.dirname(path))
@@ -31,5 +24,9 @@ module Lono::Configset::Strategy
       Lono::Yamler::Validator.new(path).validate!
       Lono::Yamler::Loader.new(content).load
     end
+
+    def authentication
+      # noop
+    end
   end
 end

data/lib/lono/builder/configset/definition.rb

@@ -0,0 +1,18 @@
+module Lono::Builder::Configset
+  class Definition < Lono::CLI::Base
+    attr_reader :configset
+    def initialize(options={})
+      super
+      @meta = options[:meta]
+      @configset = Lono::Configset.new(@meta)
+    end
+
+    def evaluate
+      strategy_class = configset.path.include?('.rb') ? Dsl : Erb
+      strategy = strategy_class.new(@options.merge(path: configset.path))
+      metadata = strategy.evaluate
+      @configset.metadata = metadata
+      @configset
+    end
+  end
+end

data/lib/lono/builder/configset/evaluator.rb

@@ -0,0 +1,10 @@
+# Dsl or Erb
+module Lono::Builder::Configset
+  class Evaluator < Lono::CLI::Base
+    def evaluate
+      Registration.new(@blueprint).evaluate
+      combiner = Combiner.new(@options.merge(metas: Registration.metas))
+      combiner.combine
+    end
+  end
+end

data/lib/lono/builder/configset/registration.rb

@@ -0,0 +1,35 @@
+module Lono::Builder::Configset
+  class Registration
+    include DslEvaluator
+
+    def initialize(blueprint)
+      @blueprint = blueprint
+    end
+
+    cattr_accessor :metas, default: []
+    def metas
+      self.class.metas
+    end
+
+    def evaluate
+      path = "#{@blueprint.root}/configsets.rb" # plural
+      evaluate_file(path)
+    end
+
+    # Only one syntax method so not in separate module.
+    #
+    # Register configset for later processing.
+    # By registering them all up front, can aggregate errors and show them together
+    # for a user-friendly experience.
+    #
+    # The configset method is different with configset registration vs configset definition
+    #
+    #   definition:   app/configsets/httpd/configset.rb
+    #   registration: app/blueprints/demo/configsets.rb
+    #
+    def configset(name, options={})
+      found = metas.detect { |i| i[:name] == name && i[:resource] == options[:resource] }
+      metas << options.merge(name: name) unless found
+    end
+  end
+end

data/lib/lono/builder/context.rb

@@ -0,0 +1,40 @@
+module Lono::Builder
+  module Context
+    include DslEvaluator
+
+    def load_context
+      load_variables
+      load_helpers
+    end
+
+    # Variables in base.rb are overridden by their environment specific variables
+    # file.  Example, file LONO_ENV=dev:
+    #
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #
+    def load_variables
+      return if seed?
+      layers = Lono::Layering::Layer.new(@blueprint, "vars").paths
+      layers.each do |layer|
+        evaluate_file(layer)
+      end
+    end
+
+    # Load blueprint helpers
+    # blueprint helpers override extension helpers
+    def load_helpers
+      load_helper_files("#{Lono.root}/vendor/helpers",   type: :project)
+      load_helper_files("#{Lono.root}/app/helpers",   type: :project)
+      load_helper_files("#{@blueprint.root}/helpers", type: :blueprint) # takes higher precedence
+    end
+
+    # Dont want any existing files to prevent building the blueprint.
+    # This means that parameters cannot be based on vars. It's a trade-off.
+    def seed?
+      ARGV[0] == "seed"
+    end
+  end
+end

data/lib/lono/builder/dsl/evaluator.rb

@@ -1,6 +1,7 @@
 class Lono::Builder::Dsl
   class Evaluator < Lono::CLI::Base
-    include Lono::Builder::Context::Loaders
+    include DslEvaluator
+    include Lono::Builder::Context
     include Lono::Builder::Dsl::Syntax
     include Lono::Utils::Pretty
 
@@ -11,10 +12,8 @@ class Lono::Builder::Dsl
       @cfn = {}
     end
 
-    def build
-      # TODO: load_all_extension_helpers # after Extensions::Preparer#run
-      load_variables unless seed? # both blueprint and project variables
-      load_blueprint_helpers
+    def evaluate
+      load_context
       evaluate_template_paths(@template_path) # modifies @cfn
       finalize
       to_yaml
@@ -32,62 +31,14 @@ class Lono::Builder::Dsl
       @cfn = YAML.load(@cfn.to_json)
     end
 
-    # Dont want any existing files to prevent building the blueprint.
-    # This means that parameters cannot be based on vars. It's a trade-off.
-    def seed?
-      ARGV[0] == "seed"
-    end
-
     # Example path: /full/path/to/project/app/blueprints/demo/template.rb
     def evaluate_template_paths(path)
       ext = File.extname(path)
       folder = path.sub(ext, '')
       expr = "#{folder}/**/*.rb"
-      evaluate_template_path(path) # process top-level template.rb first
+      evaluate_file(path) # process top-level template.rb first
       Dir.glob(expr).each do |path|
-        evaluate_template_path(path)
-      end
-    end
-
-    def evaluate_template_path(path)
-      return unless File.exist?(path)
-      instance_eval(File.read(path), path)
-    rescue Exception => e
-      template_evaluation_error(e)
-      exit 1
-    end
-
-    # Prints out a user friendly task_definition error message
-    #
-    # Backtrace lines are different for OSes:
-    #
-    #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/terraspace-1.1.1/lib/terraspace/builder.rb:34:in `build'"
-    #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/terraspace-1.1.1/lib/terraspace/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
-    #
-    def template_evaluation_error(e)
-      lines = e.backtrace.reject { |l| l.include?("/lib/lono/") }
-      error_info = lines.first
-      parts = error_info.split(':')
-      windows = error_info.match(/^[a-zA-Z]:/)
-      path = windows ? parts[1] : parts[0]
-      line_no = windows ? parts[2] : parts[1]
-      line_no = line_no.to_i
-      logger.debug e.message
-      logger.error "Error evaluating #{pretty_path(path)}".color(:red)
-      logger.error "Here's the line with the error:\n\n"
-
-      contents = IO.read(path)
-      content_lines = contents.split("\n")
-      context = 5 # lines of context
-      top, bottom = [line_no-context-1, 0].max, line_no+context-1
-      spacing = content_lines.size.to_s.size
-      content_lines[top..bottom].each_with_index do |line_content, index|
-        line_number = top+index+1
-        if line_number == line_no
-          logger.printf("%#{spacing}d %s\n".color(:red), line_number, line_content)
-        else
-          logger.printf("%#{spacing}d %s\n", line_number, line_content)
-        end
+        evaluate_file(path)
       end
     end
   end

data/lib/lono/builder/dsl/finalizer/base.rb

@@ -0,0 +1,8 @@
+class Lono::Builder::Dsl::Finalizer
+  class Base < Lono::CLI::Base
+    def initialize(options={})
+      super
+      @cfn = options[:cfn]
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/configsets.rb

@@ -1,41 +1,20 @@
 class Lono::Builder::Dsl::Finalizer
-  class Configsets
-    extend Memoist
-
-    def initialize(cfn)
-      @cfn = cfn
-    end
-
+  class Configsets < Base
+    # Replaces metadata under each logical id resource.
     def run
-      @cfn = add # overwrite
-      @cfn
-    end
-
-    def add
-      metadata_map.each do |logical_id, metadata_configset|
+      dsl = Lono::Builder::Configset::Evaluator.new(@options.merge(cfn: @cfn))
+      metadata_map = dsl.evaluate
+      metadata_map.each do |logical_id, cs|
         resource = @cfn["Resources"][logical_id]
-
         unless resource
-          logger.info "WARN: Resources.#{logical_id} not found in the template. Are you sure you are specifying the correct resource id in your configsets configs?".color(:yellow)
+          puts "WARN: Resources.#{logical_id} not found in the template. Are you sure you specified the correct resource logical id in your configsets.rb?".color(:yellow)
           next
         end
 
-        resource["Metadata"] ||= metadata_configset["Metadata"]
-
-        # metdata = resource["Metadata"] ||= {}
-        # metdata["AWS::CloudFormation::Init"] ||= {}
-        # # The metadata_configset has been combined with the original AWS::CloudFormation::Init if it exists
-        # metdata["AWS::CloudFormation::Init"] = metadata_configset["AWS::CloudFormation::Init"]
+        resource["Metadata"] = cs["Metadata"]
       end
 
       @cfn
     end
-
-    def metadata_map
-      []
-      # combiner = Lono::Configset::Combiner.new(@cfn, @options)
-      # combiner.metadata_map
-    end
-    memoize :metadata_map
   end
 end

data/lib/lono/builder/dsl/finalizer/files/base.rb

@@ -0,0 +1,4 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Base < Lono::Builder::Dsl::Finalizer::Base
+  end
+end

data/lib/lono/builder/dsl/finalizer/files/build.rb

@@ -0,0 +1,58 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Build < Base
+    def initialize(options={})
+      super
+      @output_path = "#{Lono.root}/.output/files"
+    end
+
+    def run
+      return if Lono::Files.empty?
+      logger.debug "Building files"
+      clean
+      validate!
+      build_files
+      # build_layers
+    end
+
+    def build_files
+      Lono::Files.files.each do |file| # using singular file, but is like a "file_list"
+        file.build
+        file.compress
+        # Note: Uploading files happen right before create_stack or execute_change_set
+        # after user confirms action, instead of part of the build process
+      end
+    end
+
+    # TODO: LambdaLayer support
+    # def build_layers
+    #   layer_items = Registry.layers
+    #   layer_items.each do |item|
+    #     LambdaLayer.new(@blueprint, item).build
+    #   end
+    # end
+    #
+
+    def clean
+      FileUtils.rm_rf(@output_path)
+    end
+
+    def validate!
+      missing = Lono::Files.files.select do |file|
+        !File.exist?(file.full_path)
+      end
+      return if missing.empty?
+
+      logger.info "ERROR: These files are missing".color(:red)
+      logger.info <<~EOL
+        The file helper is calling them.
+        They need to exist to be uploaded to s3.
+      EOL
+      missing.each do |file|
+        logger.info "    #{pretty_path(file.full_path)}"
+        logger.info "    Called at #{file.call_line}"
+      end
+      logger.info "Please double check that they exist."
+      exit 1
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/files/replace.rb

@@ -0,0 +1,31 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Replace < Base
+    def run
+      replacements.each do |placeholder, replacement|
+        update_template!(@cfn)
+      end
+      @cfn
+    end
+
+    def replacements
+      Lono::Files.files.inject({}) do |result, file|
+        result.merge(file.marker => file.s3_key)
+      end
+    end
+
+    def update_template!(hash)
+      hash.each do |k, v|
+        if v.is_a?(String)
+          if v =~ %r{^LONO://}
+            v.replace(replacements[v]) # replace the placeholder
+          end
+        elsif v.is_a?(Hash)
+          update_template!(v) # recurse
+        elsif v.is_a?(Array)
+          v.each { |x| update_template!(x) if x.is_a?(Hash) }
+        end
+      end
+      hash
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/files.rb

@@ -0,0 +1,9 @@
+class Lono::Builder::Dsl::Finalizer
+  class Files < Base
+    # Replaces metadata under each logical id resource.
+    def run
+      Build.new(@options).run
+      @cfn = Replace.new(@options).run
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/parameter_groups.rb

@@ -1,10 +1,11 @@
 class Lono::Builder::Dsl::Finalizer
-  class ParameterGroups
+  class ParameterGroups < Base
     extend Memoist
     include Lono::Builder::Util::Stringify
 
-    def initialize(cfn, parameters)
-      @cfn, @parameters = cfn, parameters
+    def initialize(options={})
+      super
+      @parameters = options[:parameters]
     end
 
     def run

data/lib/lono/builder/dsl/finalizer.rb

@@ -5,8 +5,10 @@ class Lono::Builder::Dsl
     end
 
     def run
-      @cfn = ParameterGroups.new(@cfn, @options[:parameters]).run
-      @cfn = Configsets.new(@cfn).run
+      o = @options.merge(cfn: @cfn)
+      @cfn = ParameterGroups.new(o).run
+      @cfn = Configsets.new(o).run
+      @cfn = Files.new(o).run
       @cfn
     end
   end

data/lib/lono/builder/dsl/helpers/files.rb

@@ -0,0 +1,7 @@
+module Lono::Builder::Dsl::Helpers
+  module Files
+    def files(path)
+      Lono::Files.register(@options.merge(path: path))
+    end
+  end
+end

data/lib/lono/builder/dsl/helpers/partials.rb

@@ -9,9 +9,7 @@ module Lono::Builder::Dsl::Helpers
     end
 
     def template_params(param_name)
-      o = {
-        allow_not_exists: true
-      }.merge(@options)
+      o = @options.dup
       o["param"] = param_name
       generator = Lono::CLI::Param::Generator.new(o)
       # do not generate because lono cfn calling logic already generated it we only need the values
@@ -22,62 +20,54 @@ module Lono::Builder::Dsl::Helpers
       end
     end
 
-    # Adjust the partial path so that it will use app/user_data
-    def user_data(path,vars={}, options={})
-      options.merge!(user_data: true)
-      partial(path,vars, options)
+    # The partial's path is a relative path.
+    #
+    # Example:
+    # Given file in app/partials/iam/docker.yml
+    #
+    #   <%= partial("iam/docker", {}, indent: 10) %>
+    #   <%= partial("iam/docker.yml", {}, indent: 10) %>
+    #
+    # If the user specifies the extension then use that instead of auto-adding
+    # the detected format.
+    def partial(path,vars={}, options={})
+      path = partial_path_for(path)
+      path = auto_add_format(path)
+
+      instance_variables!(vars)
+      result = render_path(path)
+
+      result = indent(result, options[:indent]) if options[:indent]
+      result + "\n"
     end
 
-    def current_region
-      region = Aws.config[:region]
-      region ||= ENV['AWS_REGION']
-      return region if region
-
-      default_region = 'us-east-1' # fallback if default not found in ~/.aws/config
-      if ENV['AWS_PROFILE']
-        path = "#{ENV['HOME']}/.aws/config"
-        if File.exist?(path)
-          lines = IO.readlines(path)
-          capture_default, capture_current = false, false
-          lines.each do | line|
-            if line.include?('[default]')
-              capture_default = true # next line
-              next
-            end
-            if capture_default && line.match(/region = /)
-              # over default from above
-              default_region = line.split(' = ').last.strip
-              capture_default = false
-            end
-
-            md = line.match(/\[profile (.*)\]/)
-            if md && md[1] == ENV['AWS_PROFILE']
-              capture_current = true
-              next
-            end
-            if capture_current && line.match(/region = /)
-              region = line.split(' = ').last.strip
-              capture_current = false
-            end
-          end
-        end
-
-        region ||= default_region
-        return region if region
+    # Take a hash and makes them instance variables in the current scope.
+    # Use this in custom helper methods to make variables accessible to ERB templates.
+    def instance_variables!(variables)
+      variables.each do |key, value|
+        instance_variable_set('@' + key.to_s, value)
       end
+    end
 
-      'us-east-1' # default
+    # add indentation
+    def indent(text, indentation_amount)
+      text.split("\n").map do |line|
+        " " * indentation_amount + line
+      end.join("\n")
     end
 
-  private
-    def render_path(path)
-      RenderMePretty.result(path, context: self)
+    def partial_exist?(path)
+      path = partial_path_for(path)
+      path = auto_add_format(path)
+      path && File.exist?(path)
     end
 
-    def user_data_path_for(path)
-      "#{Lono.config.paths.user_data}/#{path}"
+    def region
+      AwsData.region
     end
+    alias_method :current_region, :region
 
+  private
     def auto_add_format(path)
       # Return immediately if user provided explicit extension
       extension = File.extname(path) # current extension
@@ -107,16 +97,15 @@ module Lono::Builder::Dsl::Helpers
 
     # Bash code that is meant to included in user-data
     def extract_scripts(options={})
-      settings = Lono.config.extract_scripts
-      options = settings.merge(options)
       # defaults also here in case they are removed from settings
+      path = options[:path]
       to = options[:to] || "/opt"
       user = options[:as] || "ec2-user"
 
-      if Dir.glob("#{Lono.config.paths.scripts}/*").empty?
+      if Dir.glob("#{@bluepint.root}/#{path}*").empty?
         logger.info "WARN: you are using the extract_scripts helper method but you do not have any scripts.".color(:yellow)
         calling_line = caller[0].split(':')[0..1].join(':')
-        logger.info "Called from: #{calling_line}"
+        logger.info "Called from1: #{calling_line}"
         return ""
       end
 
@@ -141,5 +130,11 @@ module Lono::Builder::Dsl::Helpers
       upload = Lono::Script::Upload.new(@options)
       upload.s3_dest
     end
+
+    def indent(text, indentation_amount)
+      text.split("\n").map do |line|
+        " " * indentation_amount + line
+      end.join("\n")
+    end
   end
 end

data/lib/lono/builder/dsl/helpers/s3.rb

@@ -4,11 +4,11 @@ module Lono::Builder::Dsl::Helpers
       Lono::S3::Bucket.name
     end
     alias_method :lono_bucket_name, :s3_bucket
+    alias_method :files_bucket, :s3_bucket
 
     def s3_key(name, options={})
       default = {type: "file"}
       options.reverse_merge!(default)
-      Lono::AppFile::Registry.register(name, @blueprint.name, options)
       "file://app/files/#{options[:type]}/#{name}" # placeholder for post processing
     end
     alias_method :file_s3_key, :s3_key

data/lib/lono/builder/dsl/helpers/ssm/fetcher.rb

@@ -3,6 +3,7 @@ require 'aws-sdk-ssm'
 module Lono::Builder::Dsl::Helpers::Ssm
   class Fetcher
     extend Memoist
+    include Lono::Utils::Logging
 
     def get(name)
       fetch_ssm_value(name)
@@ -12,7 +13,8 @@ module Lono::Builder::Dsl::Helpers::Ssm
       resp = ssm.get_parameter(name: name, with_decryption: true)
       resp.parameter.value
     rescue Aws::SSM::Errors::ParameterNotFound
-      'SSM-PARAM-NOT-FOUND'
+      logger.warn 'WARN: SSM-PARAM-NOT-FOUND'
+      nil
     end
 
     def ssm