lono 8.0.0.pre.rc2 → 8.0.0.pre.rc5

data/lib/lono/builder/configset/combiner.rb

@@ -0,0 +1,145 @@
+module Lono::Builder::Configset
+  class Combiner < Lono::CLI::Base
+    def initialize(options={})
+      super
+      @cfn = options[:cfn]     # merge from Dsl::Finalizer::Configsets#run
+      @metas = options[:metas] # merge from Configset::Evaluator#evaluate
+      @configsets = []
+      @map = {} # stores resource logical id => metadata cfn-init
+    end
+
+    # Returns metadata map structure
+    #
+    # {"Instance"=>
+    #   {"Metadata"=>
+    #     {"AWS::CloudFormation::Init"=>
+    #
+    def combine
+      return @map if metas_empty?
+      add_existing
+      add_built
+      build_map # metadata map
+    end
+
+    def metas_empty?
+      @metas.empty?
+    end
+
+    def add_existing
+      existing_configsets.each do |configset|
+        add(configset)
+      end
+    end
+
+    def add_built
+      @metas.each do |meta|
+        definition = Definition.new(@options.merge(meta: meta))
+        configset = definition.evaluate
+        add(configset)
+      end
+    end
+
+    # Useful for specs
+    def add(configset)
+      found = @configsets.detect { |c| c.name == configset.name && c.resource == configset.resource }
+      @configsets << configset unless found
+    end
+
+    # Each metadata has this structure:
+    #
+    #     {"Metadata"=>
+    #       {"AWS::CloudFormation::Init"=>
+    #         {"configSets"=>{"default"=>["aaa1", "aaa2"]},
+    #         "aaa1"=>{"commands"=>{"test"=>{"command"=>"echo from-aaa1 > test1.txt"}}},
+    #         "aaa2"=>
+    #           {"commands"=>{"test"=>{"command"=>"echo from-aaa2 > test1.txt"}}}}}}
+    #
+    def build_map
+      metadata_map = {}
+      configsets_map = {}
+
+      @configsets.each_with_index do |configset, i|
+        padded_i = "%03d" % i
+
+        name, resource, metadata = configset.name, configset.resource, configset.metadata
+        configsets = configsets_map[resource] ||= {}
+
+        validate_structure!(name, metadata)
+
+        new_metadata = metadata["Metadata"].dup
+        init = new_metadata["AWS::CloudFormation::Init"] # important: adjust data by reference
+
+        if init.key?("configSets")
+          # validate_simple!(name, new_metadata["AWS::CloudFormation::Init"]["configSets"]) # validate original configset for only simple elements
+
+          # 1. expand each config as its own config, flattening to top-level
+          cs = init.delete("configSets") # Only support configSets with simple Array of Strings
+          new_config_set = {}
+          new_config_set[name] = cs["default"].map {|c| "#{padded_i}_#{c}" }
+          init.transform_keys! { |c| "#{padded_i}_#{c}" }
+
+          # Rebuild default configSet, append the new complex ConfigSet structure with each iteration
+          configsets["default"] ||= []
+          configsets["default"] << {"ConfigSet" => name}
+          configsets.merge!(new_config_set) # add each config from #1 to the top-level
+
+          init["configSets"] = configsets # replace new configset
+        else # simple config
+          init["configSets"] = configsets # adjust data by reference
+          configsets["default"] ||= []
+          configsets["default"] << {"ConfigSet" => name}
+
+          # build new config
+          config_key = "#{padded_i}_single_generated"
+          configsets[name] = [config_key]
+          new_config = {config_key => init["config"]}
+          # replace old config with new one
+          init.delete("config") # delete original simple config
+          init.merge!(new_config)
+        end
+
+        metadata_map[resource] ||= {"Metadata" => {}}
+        metadata_map[resource]["Metadata"].deep_merge!(new_metadata)
+        @map[resource] = metadata_map[resource]
+      end
+      @map
+    end
+
+    # Normalized/convert cfn template to mimic the registry format
+    def existing_configsets
+      configsets = []
+      @cfn["Resources"].each do |logical_id, attributes|
+        cloudformation_init = attributes.dig("Metadata", "AWS::CloudFormation::Init")
+        next unless cloudformation_init
+
+        configset = Lono::Configset.new(name: "#{logical_id}OriginalConfigset", resource: logical_id)
+        configset.metadata = {"Metadata" => attributes["Metadata"]} # wrap metadata to create right structure
+        configsets << configset
+      end
+      configsets
+    end
+
+    def validate_structure!(name, metadata)
+      return if metadata.is_a?(Hash) && metadata.dig("Metadata", "AWS::CloudFormation::Init")
+
+      puts "ERROR: The #{name} configset does not appear to have a AWS::CloudFormation::Init key".color(:red)
+      puts "Please double check the #{name} configset.yml structure"
+      exit 1
+    end
+
+    def validate_simple!(name, cs)
+      has_complex_type = cs["default"].detect { |s| !s.is_a?(String) }
+      return unless has_complex_type
+      message =<<~EOL
+        ERROR: The configset #{name} has a configSets property with a complex type.
+        configSets:
+
+            #{cs}
+
+        lono configsets only supports combining configSets with an Array of Strings.
+      EOL
+      puts message.color(:red)
+      exit 1
+    end
+  end
+end

data/lib/lono/builder/configset/definition/base.rb

@@ -0,0 +1,47 @@
+class Lono::Builder::Configset::Definition
+  class Base < Lono::CLI::Base
+    include DslEvaluator
+    include Lono::Builder::Dsl::Syntax
+    include Lono::Utils::Pretty
+    include Context
+
+    # Really only use @path in Configset DSL and ERB evaluation.
+    # However, configsets are built within the CloudFormation template and can use
+    # things instrinic functions like `ref` like would normally have access to.
+    # So configsets need the same context
+    #
+    #     Configset::Definition::Base < Lono::CLI::Base
+    #
+    # for
+    #
+    #     include Lono::Builder::Dsl::Syntax
+    #
+    def initialize(options={})
+      super
+      @meta = @options[:meta]
+      @configset = Lono::Configset.new(@meta)
+      expose_instance_variables
+    end
+
+    # This context is used by the DSL evaluation. Expose variables so user can use them in configset definitions.
+    # Example:
+    #
+    # "/etc/cfn/hooks.d/cfn-auto-reloader.conf":
+    #   content:
+    #     Fn::Sub:
+    # ...
+    #       path=Resources.<%= @resource %>.Metadata.AWS::CloudFormation::Init
+    #
+    def expose_instance_variables
+      @name = @meta[:name]
+      @resource = @meta[:resource]
+    end
+
+    def wrap_with_metadata(cloudformation_init)
+      full = {"Metadata" => {}}
+      full["Metadata"]["AWS::CloudFormation::Init"] = cloudformation_init["AWS::CloudFormation::Init"]
+      full["Metadata"]["AWS::CloudFormation::Authentication"] = authentication if authentication # only on dsl
+      full.deep_stringify_keys.dup # metadata
+    end
+  end
+end

data/lib/lono/builder/configset/definition/context.rb

@@ -0,0 +1,70 @@
+class Lono::Builder::Configset::Definition
+  module Context
+    include DslEvaluator
+
+    def load_context
+      load_vars
+      load_helpers
+    end
+
+    # Docs: https://lono.cloud/docs/configsets/layering/
+    # Think layering is simple enough to keep in this method for now.
+    # Consider trying to combine with Lono::Layering::Layer for uniformity though.
+    def load_vars
+      layers.each do |layer|
+        evaluate_file(layer)
+      end
+    end
+
+    def layers
+      paths = [
+        "#{@configset.root}/vars.rb", # source defaults
+        "#{Lono.root}/config/configsets/#{@configset.name}/vars.rb", # source overrides
+        "#{@blueprint.root}/config/configsets/vars/#{@configset.name}.rb", # blueprint overrides
+        "#{Lono.root}/config/blueprints/#{@blueprint.name}/configsets/vars/#{@configset.name}.rb", # user overrides
+      ]
+      show_layers(paths)
+      paths
+    end
+
+    def show_layers(paths)
+      if ENV['LONO_LAYERS_ALL']
+        show_all_layers(paths)
+      else
+        show_existing_layers(paths)
+      end
+    end
+
+    def show_all_layers(paths)
+      show_header
+      paths.each do |path|
+        show_layer(path)
+      end
+    end
+
+    def show_existing_layers(paths)
+      existing = paths.select { |path| File.exist?(path) }
+      return if existing.empty?
+      show_header
+      existing.each do |path|
+        show_layer(path)
+      end
+    end
+
+    def show_header
+      logger.info "    Configset Layers #{@configset.name}"
+    end
+
+    def show_layer(path)
+      logger.info "        #{pretty_path(path)}"
+    end
+
+    def show_layers?
+      Lono.config.layering.show || ENV['LONO_LAYERS']
+    end
+
+    def load_helpers
+      load_helper_files("#{@configset.root}/helpers", type: :configset)
+    end
+  end
+end

data/lib/lono/{configset/strategy/helpers/dsl → builder/configset/definition/dsl/syntax}/auth.rb

@@ -1,4 +1,4 @@
-module Lono::Configset::Strategy::Helpers::Dsl
+module Lono::Builder::Configset::Definition::Dsl::Syntax
   module Auth
     def authentication(data=nil, force: false)
       if data.nil?

data/lib/lono/builder/configset/definition/dsl/syntax/content.rb

@@ -0,0 +1,22 @@
+module Lono::Builder::Configset::Definition::Dsl::Syntax
+  module Content
+    def content_file(path)
+      content_path = "#{content_file_root}/content"
+      file = "#{content_path}/#{path}"
+      if File.exist?(file)
+        IO.read(file)
+      else
+        "File not found: #{file}"
+      end
+    end
+
+  private
+    def content_file_root
+      if @configset
+        @configset.root
+      else
+        @blueprint.root
+      end
+    end
+  end
+end

data/lib/lono/{configset/strategy/helpers/dsl/syntax.rb → builder/configset/definition/dsl/syntax/core.rb}

@@ -1,7 +1,20 @@
-module Lono::Configset::Strategy::Helpers::Dsl
-  module Syntax
+module Lono::Builder::Configset::Definition::Dsl::Syntax
+  module Core
     extend Memoist
 
+    #
+    # The configset method is different with configset registration vs configset definition
+    #
+    #   definition:   app/configsets/httpd/configset.rb
+    #   registration: app/blueprints/demo/configsets.rb
+    #
+    def configset(current)
+      @tracked << current
+      previous, @current = @current, current
+      yield
+      @current = previous
+    end
+
     %w[package group user file service].each do |meth|
       section = meth.pluralize
       define_method(meth) do |k, props={}|
@@ -53,13 +66,6 @@ module Lono::Configset::Strategy::Helpers::Dsl
       current_structure(@current)["sources"].deep_merge!(item)
     end
 
-    def configset(current)
-      @tracked << current
-      previous, @current = @current, current
-      yield
-      @current = previous
-    end
-
   private
     def current_structure(configset)
       @structure[configset] ||= {}

data/lib/lono/{configset/strategy/helpers/dsl → builder/configset/definition/dsl/syntax}/package.rb

@@ -1,4 +1,4 @@
-module Lono::Configset::Strategy::Helpers::Dsl
+module Lono::Builder::Configset::Definition::Dsl::Syntax
   module Package
     # In recent versions of the AmazonLinux2 cfnbootstrap, , the package command doesnt work with updated versions of rubygems.
     # Get this error

data/lib/lono/builder/configset/definition/dsl/syntax.rb

@@ -0,0 +1,5 @@
+class Lono::Builder::Configset::Definition::Dsl
+  module Syntax
+    include_modules "syntax"
+  end
+end

data/lib/lono/{configset/strategy → builder/configset/definition}/dsl.rb

@@ -1,29 +1,25 @@
-module Lono::Configset::Strategy
+class Lono::Builder::Configset::Definition
   class Dsl < Base
-    # Configset helpers take higher precedence than Lono DSL Helpers. IE: s3_key is overwritten but s3_bucket is the same
-    include Helpers::Dsl
+    include Syntax
 
     def initialize(options={})
       super
-      @structure = {} # holds in memory the configset hash structure to build AWS::CloudFormation::Init
       @current = "main" # current configset
-      @command_counts = Hash.new(0)
       @tracked = []
+      @structure = {} # holds in memory the configset hash structure to build AWS::CloudFormation::Init
+      @command_counts = Hash.new(0)
       # Also support ability to add AWS::CloudFormation::Authentication
       @authentication = nil # holds IAM policy info to build AWS::CloudFormation::Authentication
     end
 
-    def find_evaluation_path
-      "#{@root}/lib/configset.rb"
-    end
-
-    def load
-      evaluate_file(@evaluation_path)
-      configsets = @configsets || @tracked.uniq
+    def evaluate
+      load_context
+      evaluate_file(@configset.path)
+      configsets = configsets || @tracked.uniq
       configsets = ["main"] if configsets.empty?
       configsets_structure = {"configSets" => {"default" => configsets}}.merge(@structure)
-      full = { "AWS::CloudFormation::Init" => configsets_structure }
-      full.deep_stringify_keys!
+      cloudformation_init = { "AWS::CloudFormation::Init" => configsets_structure }
+      wrap_with_metadata(cloudformation_init)
     end
   end
 end

data/lib/lono/{configset/strategy → builder/configset/definition}/erb.rb

@@ -1,29 +1,22 @@
-require "json"
-
-module Lono::Configset::Strategy
+class Lono::Builder::Configset::Definition
   class Erb < Base
-    include Helpers::Erb
-
-    def find_evaluation_path
-      paths = %w[configset.yml configset.json].map { |p| "#{@root}/lib/#{p}" }
-      paths.find { |p| File.exist?(p) }
+    def evaluate
+      load_context
+      cloudformation_init = build
+      wrap_with_metadata(cloudformation_init)
     end
 
-    def load
-      content = RenderMePretty.result(@evaluation_path, context: self)
-      # init structure
-      if File.extname(@evaluation_path) == ".yml"
+    def build
+      content = RenderMePretty.result(@configset.path, context: self)
+      if File.extname(@configset.path) == ".yml"
         load_yaml(content)
       else
         JSON.load(content)
       end
     end
 
-    def authentication
-      # noop
-    end
-
     def load_yaml(content)
+      # init structure
       # Write to file so can use Yamler::Validator
       path = "/tmp/lono/configset.yml"
       FileUtils.mkdir_p(File.dirname(path))
@@ -31,5 +24,9 @@ module Lono::Configset::Strategy
       Lono::Yamler::Validator.new(path).validate!
       Lono::Yamler::Loader.new(content).load
     end
+
+    def authentication
+      # noop
+    end
   end
 end

data/lib/lono/builder/configset/definition.rb

@@ -0,0 +1,18 @@
+module Lono::Builder::Configset
+  class Definition < Lono::CLI::Base
+    attr_reader :configset
+    def initialize(options={})
+      super
+      @meta = options[:meta]
+      @configset = Lono::Configset.new(@meta)
+    end
+
+    def evaluate
+      strategy_class = configset.path.include?('.rb') ? Dsl : Erb
+      strategy = strategy_class.new(@options.merge(path: configset.path))
+      metadata = strategy.evaluate
+      @configset.metadata = metadata
+      @configset
+    end
+  end
+end

data/lib/lono/builder/configset/evaluator.rb

@@ -0,0 +1,10 @@
+# Dsl or Erb
+module Lono::Builder::Configset
+  class Evaluator < Lono::CLI::Base
+    def evaluate
+      Registration.new(@blueprint).evaluate
+      combiner = Combiner.new(@options.merge(metas: Registration.metas))
+      combiner.combine
+    end
+  end
+end

data/lib/lono/builder/configset/registration.rb

@@ -0,0 +1,35 @@
+module Lono::Builder::Configset
+  class Registration
+    include DslEvaluator
+
+    def initialize(blueprint)
+      @blueprint = blueprint
+    end
+
+    cattr_accessor :metas, default: []
+    def metas
+      self.class.metas
+    end
+
+    def evaluate
+      path = "#{@blueprint.root}/configsets.rb" # plural
+      evaluate_file(path)
+    end
+
+    # Only one syntax method so not in separate module.
+    #
+    # Register configset for later processing.
+    # By registering them all up front, can aggregate errors and show them together
+    # for a user-friendly experience.
+    #
+    # The configset method is different with configset registration vs configset definition
+    #
+    #   definition:   app/configsets/httpd/configset.rb
+    #   registration: app/blueprints/demo/configsets.rb
+    #
+    def configset(name, options={})
+      found = metas.detect { |i| i[:name] == name && i[:resource] == options[:resource] }
+      metas << options.merge(name: name) unless found
+    end
+  end
+end

data/lib/lono/builder/context.rb

@@ -0,0 +1,40 @@
+class Lono::Builder
+  module Context
+    include DslEvaluator
+
+    def load_context
+      load_helpers # load helpers before variable so user custom helpers are available in vars files
+      load_variables
+    end
+
+    # Variables in base.rb are overridden by their environment specific variables
+    # file.  Example, file LONO_ENV=dev:
+    #
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #
+    def load_variables
+      return if seed?
+      layers = Lono::Layering::Layer.new(@blueprint, "vars").paths
+      layers.each do |layer|
+        evaluate_file(layer)
+      end
+    end
+
+    # Load blueprint helpers
+    # blueprint helpers override extension helpers
+    def load_helpers
+      load_helper_files("#{Lono.root}/vendor/helpers",   type: :project)
+      load_helper_files("#{Lono.root}/app/helpers",   type: :project)
+      load_helper_files("#{@blueprint.root}/helpers", type: :blueprint) # takes higher precedence
+    end
+
+    # Dont want any existing files to prevent building the blueprint.
+    # This means that parameters cannot be based on vars. It's a trade-off.
+    def seed?
+      ARGV[0] == "seed"
+    end
+  end
+end

data/lib/lono/builder/dsl/evaluator.rb

@@ -1,6 +1,7 @@
 class Lono::Builder::Dsl
   class Evaluator < Lono::CLI::Base
-    include Lono::Builder::Context::Loaders
+    include DslEvaluator
+    include Lono::Builder::Context
     include Lono::Builder::Dsl::Syntax
     include Lono::Utils::Pretty
 
@@ -11,10 +12,8 @@ class Lono::Builder::Dsl
       @cfn = {}
     end
 
-    def build
-      # TODO: load_all_extension_helpers # after Extensions::Preparer#run
-      load_variables unless seed? # both blueprint and project variables
-      load_blueprint_helpers
+    def evaluate
+      load_context
       evaluate_template_paths(@template_path) # modifies @cfn
       finalize
       to_yaml
@@ -32,62 +31,14 @@ class Lono::Builder::Dsl
       @cfn = YAML.load(@cfn.to_json)
     end
 
-    # Dont want any existing files to prevent building the blueprint.
-    # This means that parameters cannot be based on vars. It's a trade-off.
-    def seed?
-      ARGV[0] == "seed"
-    end
-
     # Example path: /full/path/to/project/app/blueprints/demo/template.rb
     def evaluate_template_paths(path)
       ext = File.extname(path)
       folder = path.sub(ext, '')
       expr = "#{folder}/**/*.rb"
-      evaluate_template_path(path) # process top-level template.rb first
+      evaluate_file(path) # process top-level template.rb first
       Dir.glob(expr).each do |path|
-        evaluate_template_path(path)
-      end
-    end
-
-    def evaluate_template_path(path)
-      return unless File.exist?(path)
-      instance_eval(File.read(path), path)
-    rescue Exception => e
-      template_evaluation_error(e)
-      exit 1
-    end
-
-    # Prints out a user friendly task_definition error message
-    #
-    # Backtrace lines are different for OSes:
-    #
-    #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/terraspace-1.1.1/lib/terraspace/builder.rb:34:in `build'"
-    #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/terraspace-1.1.1/lib/terraspace/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
-    #
-    def template_evaluation_error(e)
-      lines = e.backtrace.reject { |l| l.include?("/lib/lono/") }
-      error_info = lines.first
-      parts = error_info.split(':')
-      windows = error_info.match(/^[a-zA-Z]:/)
-      path = windows ? parts[1] : parts[0]
-      line_no = windows ? parts[2] : parts[1]
-      line_no = line_no.to_i
-      logger.debug e.message
-      logger.error "Error evaluating #{pretty_path(path)}".color(:red)
-      logger.error "Here's the line with the error:\n\n"
-
-      contents = IO.read(path)
-      content_lines = contents.split("\n")
-      context = 5 # lines of context
-      top, bottom = [line_no-context-1, 0].max, line_no+context-1
-      spacing = content_lines.size.to_s.size
-      content_lines[top..bottom].each_with_index do |line_content, index|
-        line_number = top+index+1
-        if line_number == line_no
-          logger.printf("%#{spacing}d %s\n".color(:red), line_number, line_content)
-        else
-          logger.printf("%#{spacing}d %s\n", line_number, line_content)
-        end
+        evaluate_file(path)
       end
     end
   end

data/lib/lono/builder/dsl/finalizer/base.rb

@@ -0,0 +1,8 @@
+class Lono::Builder::Dsl::Finalizer
+  class Base < Lono::CLI::Base
+    def initialize(options={})
+      super
+      @cfn = options[:cfn]
+    end
+  end
+end