lono 8.0.0.pre.rc2 → 8.0.0.pre.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df8edbab23041858e8bea192749b440727066aeff707fa03edf5caf4dd529df9
-  data.tar.gz: 22f1143dd996e18bd6801080128605db30957c324136ce277c493497adb1f3bb
+  metadata.gz: 99ed66c36726e513ebe394291384f039d6dbbc7beef5a376b28b3b84121f1a42
+  data.tar.gz: ec1902278f840c29539d54fffa9167565a27f1837a4ee2795d8db86badf0030d
 SHA512:
-  metadata.gz: 866583abf80bbf971b345e23e1b54a430542ad4f5b0578e7973761c88065ba8875c3ef6f724a4291e8c16ff35163e461db0ee563eda438b11539b5f7d7fa2409
-  data.tar.gz: 68e43613edfaa724c954541b4788a2365a0faa8e32d726ae84610255ad214b0d40c2ff84c830e8f44c73bbb9e00c9ab06708f614796f13738fcb7d2ca5f24056
+  metadata.gz: 522056fb098c360c2ad59d99dfabaa723aeec157ca5012938c36efcb130e1bf6c2974d158e3c2f864e258c5d4b065d8e949667291f6af7d779bad91d2b451a0f
+  data.tar.gz: 18d17bbf1fcdd0f9d7c59f87b83e463f277958fb34cfb07d5918904b76ae44ecd2995f7e36f09d0a99c15d01070b469abf44bcfc95ec8f413aa54fdb0058fc6c

data/CHANGELOG.md

@@ -1,7 +1,20 @@
 # Change Log
 
-All notable changes to this project will be documented in this file.
-This project *tries* to adhere to [Semantic Versioning](http://semver.org/).
+## [8.0.0.rc3] - 2022-03-06
+
+* lono files support for lambda functions
+  * new structure: remove Appfile
+  * allow regions and envs support
+  * print friend message with exact line of yaml code in case of aws
+  * cloudformation yaml validation error
+  * improve cli help
+* configsets new structure
+  * lono new helper blueprint and project commands
+  * project helpers support
+  * include_modules improvements
+  * support vendor level helpers
+  * improve plan params diff
+* lono plan build all first
 
 ## [8.0.0.rc2] - 2022-02-23
 

data/Gemfile

@@ -3,8 +3,6 @@ source "https://rubygems.org"
 # Specify your gem dependencies in lono.gemspec
 gemspec
 
-if ENV['C9_USER']
-  gem "rspec-lono", path: "~/boltops-tools/rspec-lono"
-else
+group :test do
   gem "rspec-lono", git: "https://github.com/boltops-tools/rspec-lono", branch: "master"
 end

data/lib/lono/app/callable_option/concern.rb

@@ -0,0 +1,12 @@
+class Lono::App::CallableOption
+  module Concern
+    def callable_option(options={})
+      callable_option = Lono::App::CallableOption.new(
+        config_name: options[:config_name],
+        config_value: options[:config_value],
+        passed_args: options[:passed_args],
+      )
+      callable_option.object
+    end
+  end
+end

data/lib/lono/app/callable_option.rb

@@ -0,0 +1,56 @@
+# Class represents a lono option that is possibly callable. Examples:
+#
+#    config.allow.envs
+#    config.allow.regions
+#    config.deny.envs
+#    config.deny.regions
+#
+# Abstraction is definitely obtuse. Using it to get rid of duplication.
+#
+class Lono::App
+  class CallableOption
+    include Lono::Utils::Logging
+
+    def initialize(options={})
+      @options = options
+      # Example:
+      # config_name:  config.allow.envs
+      # config_value: ["dev"]
+      # args:         [@stack_name] # passed to object.call
+      @config_name = options[:config_name]
+      @config_value = options[:config_value]
+      @passed_args = options[:passed_args]
+    end
+
+    # Returns either an Array or nil
+    def object
+      case @config_value
+      when nil
+        return nil
+      when Array
+        return @config_value
+      when -> (c) { c.respond_to?(:public_instance_methods) && c.public_instance_methods.include?(:call) }
+        object= @config_value.new
+      when -> (c) { c.respond_to?(:call) }
+        object = @config_value
+      else
+        raise "Invalid option for #{@config_name}"
+      end
+
+      if object
+        result = @passed_args.empty? ? object.call : object.call(*@passed_args)
+        unless result.is_a?(Array) || result.is_a?(NilClass)
+          message = "ERROR: The #{@config_name} needs to return an Array or nil"
+          logger.info message.color(:yellow)
+          logger.info <<~EOL
+            The #{@config_name} when assigned a class, object, or proc must implement
+            the call method and return an Array or nil.
+            The current return value is a #{result.class}
+          EOL
+          raise message
+        end
+      end
+      result
+    end
+  end
+end

data/lib/lono/app.rb

@@ -13,6 +13,16 @@ module Lono
     def defaults
       config = ActiveSupport::OrderedOptions.new
 
+      config.allow = ActiveSupport::OrderedOptions.new
+      config.allow.envs = nil
+      config.allow.regions = nil
+      config.deny = ActiveSupport::OrderedOptions.new
+      config.deny.envs = nil
+      config.deny.regions = nil
+
+      config.layering = ActiveSupport::OrderedOptions.new
+      config.layering.names = {}
+
       config.log = ActiveSupport::OrderedOptions.new
       config.log.root = Lono.log_root
       config.logger = lono_logger
@@ -20,16 +30,21 @@ module Lono
       config.logger.level = ENV['LONO_LOG_LEVEL'] || :info
 
       config.names = ActiveSupport::OrderedOptions.new
-      config.names.stack = ":APP-:BLUEPRINT-:ENV"
       config.names.output = ActiveSupport::OrderedOptions.new
-      config.names.output.stack = ":BLUEPRINT-:ENV" # does not include APP by default. Think this is more common
       config.names.output.expand = true
+      config.names.output.stack = ":BLUEPRINT-:ENV" # does not include APP by default. Think this is more common
+      config.names.stack = ":APP-:BLUEPRINT-:ENV"
+
+      config.plan = ActiveSupport::OrderedOptions.new
+      config.plan.changeset = true
+      config.plan.params = "full"
+      config.plan.template = "summary" # summary is same as true
 
-      config.paths = ActiveSupport::OrderedOptions.new
-      config.paths.scripts = "scripts"
-      config.paths.content = "content"
-      config.paths.helpers = "helpers"
-      config.paths.user_data = "user_data"
+      config.seed = ActiveSupport::OrderedOptions.new
+      config.seed.where = "config"
+
+      config.test = ActiveSupport::OrderedOptions.new
+      config.test.framework = "rspec"
 
       config.up = ActiveSupport::OrderedOptions.new
       config.up.capabilities = nil
@@ -37,22 +52,6 @@ module Lono
       config.up.rollback = true
       config.up.tags = nil
 
-      config.diff = ActiveSupport::OrderedOptions.new
-      config.diff.changeset = true
-      config.diff.params = "full"
-      config.diff.template = "summary" # summary is same as true
-
-      config.extract_scripts = {}
-
-      config.test = ActiveSupport::OrderedOptions.new
-      config.test.framework = "rspec"
-
-      config.layering = ActiveSupport::OrderedOptions.new
-      config.layering.names = {}
-
-      config.seed = ActiveSupport::OrderedOptions.new
-      config.seed.where = "config"
-
       config
     end
 

data/lib/lono/autoloader.rb

@@ -17,6 +17,7 @@ module Lono
         loader.inflector = Inflector.new
         loader.push_dir(File.dirname(__dir__)) # lib
         loader.log! if ENV["LONO_AUTOLOAD_LOG"]
+        loader.ignore("#{__dir__}/ext.rb")
         loader.setup
       end
     end

data/lib/lono/aws_services/helper.rb

@@ -1,8 +1,6 @@
 module Lono::AwsServices
   module Helper
     def stack_exists?(stack_name)
-      return false if ENV['LONO_NOOP']
-
       exist = nil
       begin
         # When the stack does not exist an exception is raised. Example:

data/lib/lono/blueprint.rb

@@ -1,22 +1,5 @@
 module Lono
-  class Blueprint
-    attr_reader :name
-    def initialize(options={})
-      @options = options
-      @name = options[:blueprint]
-    end
-
-    def exist?
-      !root.nil?
-    end
-
-    def root
-      paths = Dir.glob("#{Lono.root}/{app,vendor}/blueprints/*")
-      paths.find do |path|
-        path.include?("blueprints/#{@name}")
-      end
-    end
-
+  class Blueprint < Component
     def output_path
       "#{Lono.root}/output/#{@name}/template.yml"
     end

data/lib/lono/builder/allow/base.rb

@@ -0,0 +1,54 @@
+class Lono::Builder::Allow
+  class Base < Lono::CLI::Base
+    include Lono::App::CallableOption::Concern
+
+    def check!
+      messages = []
+      unless allowed?
+        messages << message # message is interface method
+      end
+      unless messages.empty?
+        puts "ERROR: The configs do not allow this.".color(:red)
+        puts messages
+        exit 1
+      end
+    end
+
+    def allowed?
+      if allows.nil? && denys.nil?
+        true
+      elsif denys.nil?
+        allows.include?(check_value)
+      elsif allows.nil?
+        !denys.include?(check_value)
+      else
+        allows.include?(check_value) && !denys.include?(check_value)
+      end
+    end
+
+    def allows
+      callable_option(
+        config_name: "config.allow.#{config_name}",
+        config_value: config.dig(:allow, config_name),
+        passed_args: [@blueprint],
+      )
+    end
+
+    def denys
+      callable_option(
+        config_name: "config.deny.#{config_name}",
+        config_value: config.dig(:deny, config_name),
+        passed_args: [@blueprint],
+      )
+    end
+
+  private
+    def config
+      Lono.config
+    end
+
+    def config_name
+      self.class.to_s.split('::').last.underscore.pluralize.to_sym # ActiveSuport::HashWithIndifferentAccess#dig requires symbol
+    end
+  end
+end

data/lib/lono/builder/allow/env.rb

@@ -0,0 +1,17 @@
+class Lono::Builder::Allow
+  class Env < Base
+    # interface method
+    def message
+      messages = []
+      messages << "This env is not allowed to be used: LONO_ENV=#{Lono.env}"
+      messages << "Allow envs: #{allows.join(', ')}" if allows
+      messages << "Deny envs: #{denys.join(', ')}" if denys
+      messages.join("\n")
+    end
+
+    # interface method
+    def check_value
+      Lono.env
+    end
+  end
+end

data/lib/lono/builder/allow/region.rb

@@ -0,0 +1,20 @@
+class Lono::Builder::Allow
+  class Region < Base
+    include Lono::Concerns::AwsInfo
+
+    # interface method
+    def message
+      messages = []
+      word = config_name.to_s # IE: regions or locations
+      messages << "This #{word.singularize} is not allowed to be used: Detected current #{word.singularize}=#{current_region}"
+      messages << "Allow #{word}: #{allows.join(', ')}" if allows
+      messages << "Deny #{word}: #{denys.join(', ')}" if denys
+      messages.join("\n")
+    end
+
+    # interface method
+    def check_value
+      region
+    end
+  end
+end

data/lib/lono/builder/allow.rb

@@ -0,0 +1,8 @@
+module Lono::Builder
+  class Allow < Lono::CLI::Base
+    def check!
+      Env.new(@options).check!
+      Region.new(@options).check!
+    end
+  end
+end

data/lib/lono/builder/configset/combiner.rb

@@ -0,0 +1,145 @@
+module Lono::Builder::Configset
+  class Combiner < Lono::CLI::Base
+    def initialize(options={})
+      super
+      @cfn = options[:cfn]     # merge from Dsl::Finalizer::Configsets#run
+      @metas = options[:metas] # merge from Configset::Evaluator#evaluate
+      @configsets = []
+      @map = {} # stores resource logical id => metadata cfn-init
+    end
+
+    # Returns metadata map structure
+    #
+    # {"Instance"=>
+    #   {"Metadata"=>
+    #     {"AWS::CloudFormation::Init"=>
+    #
+    def combine
+      return @map if metas_empty?
+      add_existing
+      add_built
+      build_map # metadata map
+    end
+
+    def metas_empty?
+      @metas.empty?
+    end
+
+    def add_existing
+      existing_configsets.each do |configset|
+        add(configset)
+      end
+    end
+
+    def add_built
+      @metas.each do |meta|
+        definition = Definition.new(@options.merge(meta: meta))
+        configset = definition.evaluate
+        add(configset)
+      end
+    end
+
+    # Useful for specs
+    def add(configset)
+      found = @configsets.detect { |c| c.name == configset.name && c.resource == configset.resource }
+      @configsets << configset unless found
+    end
+
+    # Each metadata has this structure:
+    #
+    #     {"Metadata"=>
+    #       {"AWS::CloudFormation::Init"=>
+    #         {"configSets"=>{"default"=>["aaa1", "aaa2"]},
+    #         "aaa1"=>{"commands"=>{"test"=>{"command"=>"echo from-aaa1 > test1.txt"}}},
+    #         "aaa2"=>
+    #           {"commands"=>{"test"=>{"command"=>"echo from-aaa2 > test1.txt"}}}}}}
+    #
+    def build_map
+      metadata_map = {}
+      configsets_map = {}
+
+      @configsets.each_with_index do |configset, i|
+        padded_i = "%03d" % i
+
+        name, resource, metadata = configset.name, configset.resource, configset.metadata
+        configsets = configsets_map[resource] ||= {}
+
+        validate_structure!(name, metadata)
+
+        new_metadata = metadata["Metadata"].dup
+        init = new_metadata["AWS::CloudFormation::Init"] # important: adjust data by reference
+
+        if init.key?("configSets")
+          # validate_simple!(name, new_metadata["AWS::CloudFormation::Init"]["configSets"]) # validate original configset for only simple elements
+
+          # 1. expand each config as its own config, flattening to top-level
+          cs = init.delete("configSets") # Only support configSets with simple Array of Strings
+          new_config_set = {}
+          new_config_set[name] = cs["default"].map {|c| "#{padded_i}_#{c}" }
+          init.transform_keys! { |c| "#{padded_i}_#{c}" }
+
+          # Rebuild default configSet, append the new complex ConfigSet structure with each iteration
+          configsets["default"] ||= []
+          configsets["default"] << {"ConfigSet" => name}
+          configsets.merge!(new_config_set) # add each config from #1 to the top-level
+
+          init["configSets"] = configsets # replace new configset
+        else # simple config
+          init["configSets"] = configsets # adjust data by reference
+          configsets["default"] ||= []
+          configsets["default"] << {"ConfigSet" => name}
+
+          # build new config
+          config_key = "#{padded_i}_single_generated"
+          configsets[name] = [config_key]
+          new_config = {config_key => init["config"]}
+          # replace old config with new one
+          init.delete("config") # delete original simple config
+          init.merge!(new_config)
+        end
+
+        metadata_map[resource] ||= {"Metadata" => {}}
+        metadata_map[resource]["Metadata"].deep_merge!(new_metadata)
+        @map[resource] = metadata_map[resource]
+      end
+      @map
+    end
+
+    # Normalized/convert cfn template to mimic the registry format
+    def existing_configsets
+      configsets = []
+      @cfn["Resources"].each do |logical_id, attributes|
+        cloudformation_init = attributes.dig("Metadata", "AWS::CloudFormation::Init")
+        next unless cloudformation_init
+
+        configset = Lono::Configset.new(name: "#{logical_id}OriginalConfigset", resource: logical_id)
+        configset.metadata = {"Metadata" => attributes["Metadata"]} # wrap metadata to create right structure
+        configsets << configset
+      end
+      configsets
+    end
+
+    def validate_structure!(name, metadata)
+      return if metadata.is_a?(Hash) && metadata.dig("Metadata", "AWS::CloudFormation::Init")
+
+      puts "ERROR: The #{name} configset does not appear to have a AWS::CloudFormation::Init key".color(:red)
+      puts "Please double check the #{name} configset.yml structure"
+      exit 1
+    end
+
+    def validate_simple!(name, cs)
+      has_complex_type = cs["default"].detect { |s| !s.is_a?(String) }
+      return unless has_complex_type
+      message =<<~EOL
+        ERROR: The configset #{name} has a configSets property with a complex type.
+        configSets:
+
+            #{cs}
+
+        lono configsets only supports combining configSets with an Array of Strings.
+      EOL
+      puts message.color(:red)
+      exit 1
+    end
+  end
+end

data/lib/lono/builder/configset/definition/base.rb

@@ -0,0 +1,47 @@
+class Lono::Builder::Configset::Definition
+  class Base < Lono::CLI::Base
+    include DslEvaluator
+    include Lono::Builder::Dsl::Syntax
+    include Lono::Utils::Pretty
+    include Context
+
+    # Really only use @path in Configset DSL and ERB evaluation.
+    # However, configsets are built within the CloudFormation template and can use
+    # things instrinic functions like `ref` like would normally have access to.
+    # So configsets need the same context
+    #
+    #     Configset::Definition::Base < Lono::CLI::Base
+    #
+    # for
+    #
+    #     include Lono::Builder::Dsl::Syntax
+    #
+    def initialize(options={})
+      super
+      @meta = @options[:meta]
+      @configset = Lono::Configset.new(@meta)
+      expose_instance_variables
+    end
+
+    # This context is used by the DSL evaluation. Expose variables so user can use them in configset definitions.
+    # Example:
+    #
+    # "/etc/cfn/hooks.d/cfn-auto-reloader.conf":
+    #   content:
+    #     Fn::Sub:
+    # ...
+    #       path=Resources.<%= @resource %>.Metadata.AWS::CloudFormation::Init
+    #
+    def expose_instance_variables
+      @name = @meta[:name]
+      @resource = @meta[:resource]
+    end
+
+    def wrap_with_metadata(cloudformation_init)
+      full = {"Metadata" => {}}
+      full["Metadata"]["AWS::CloudFormation::Init"] = cloudformation_init["AWS::CloudFormation::Init"]
+      full["Metadata"]["AWS::CloudFormation::Authentication"] = authentication if authentication # only on dsl
+      full.deep_stringify_keys.dup # metadata
+    end
+  end
+end

data/lib/lono/builder/configset/definition/context.rb

@@ -0,0 +1,29 @@
+class Lono::Builder::Configset::Definition
+  module Context
+    include DslEvaluator
+
+    def load_context
+      load_vars
+      load_helpers
+    end
+
+    # Docs: https://lono.cloud/docs/configsets/layering/
+    # Layering is simple enough to keep in this method.
+    def load_vars
+      logger.debug "Layers for configset #{@configset.name}:"
+      evaluate_layer("#{@configset.root}/vars.rb") # source defaults
+      evaluate_layer("#{Lono.root}/config/configsets/#{@configset.name}/vars.rb") # source overrides
+      evaluate_layer("#{@blueprint.root}/config/configsets/vars/#{@configset.name}.rb") # blueprint overrides
+      evaluate_layer("#{Lono.root}/config/blueprints/#{@blueprint.name}/configsets/vars/#{@configset.name}.rb") # user overrides
+    end
+
+    def evaluate_layer(path)
+      logger.debug "    #{pretty_path(path)}" if File.exist?(path) || ENV['LONO_SHOW_ALL_LAYERS']
+      evaluate_file(path)
+    end
+
+    def load_helpers
+      load_helper_files("#{@configset.root}/helpers", type: :configset)
+    end
+  end
+end

data/lib/lono/{configset/strategy/helpers/dsl → builder/configset/definition/dsl/syntax}/auth.rb

@@ -1,4 +1,4 @@
-module Lono::Configset::Strategy::Helpers::Dsl
+module Lono::Builder::Configset::Definition::Dsl::Syntax
   module Auth
     def authentication(data=nil, force: false)
       if data.nil?

data/lib/lono/{configset/strategy/helpers/dsl/core.rb → builder/configset/definition/dsl/syntax/content.rb}

@@ -1,7 +1,7 @@
-module Lono::Configset::Strategy::Helpers::Dsl
-  module Core
+module Lono::Builder::Configset::Definition::Dsl::Syntax
+  module Content
     def content_file(path)
-      content_path = "#{@root}/lib/content"
+      content_path = "#{file_root}/content"
       file = "#{content_path}/#{path}"
       if File.exist?(file)
         IO.read(file)
@@ -10,8 +10,16 @@ module Lono::Configset::Strategy::Helpers::Dsl
       end
     end
 
+    def file_root
+      if @configset
+        @configset.root
+      else
+        @blueprint.root
+      end
+    end
+
     def s3_key(name)
-      Lono::Configset::S3File::Registry.register(name, blueprint: @blueprint.name, configset: @configset, root: @root)
+      Lono::Configset::S3File::Registry.register(name, blueprint: @blueprint, configset: @configset, root: @root)
       "file://configset/#{@configset}/#{name}"
     end
   end

data/lib/lono/{configset/strategy/helpers/dsl/syntax.rb → builder/configset/definition/dsl/syntax/core.rb}

@@ -1,7 +1,20 @@
-module Lono::Configset::Strategy::Helpers::Dsl
-  module Syntax
+module Lono::Builder::Configset::Definition::Dsl::Syntax
+  module Core
     extend Memoist
 
+    #
+    # The configset method is different with configset registration vs configset definition
+    #
+    #   definition:   app/configsets/httpd/configset.rb
+    #   registration: app/blueprints/demo/configsets.rb
+    #
+    def configset(current)
+      @tracked << current
+      previous, @current = @current, current
+      yield
+      @current = previous
+    end
+
     %w[package group user file service].each do |meth|
       section = meth.pluralize
       define_method(meth) do |k, props={}|
@@ -53,13 +66,6 @@ module Lono::Configset::Strategy::Helpers::Dsl
       current_structure(@current)["sources"].deep_merge!(item)
     end
 
-    def configset(current)
-      @tracked << current
-      previous, @current = @current, current
-      yield
-      @current = previous
-    end
-
   private
     def current_structure(configset)
       @structure[configset] ||= {}

data/lib/lono/{configset/strategy/helpers/dsl → builder/configset/definition/dsl/syntax}/package.rb

@@ -1,4 +1,4 @@
-module Lono::Configset::Strategy::Helpers::Dsl
+module Lono::Builder::Configset::Definition::Dsl::Syntax
   module Package
     # In recent versions of the AmazonLinux2 cfnbootstrap, , the package command doesnt work with updated versions of rubygems.
     # Get this error

data/lib/lono/builder/configset/definition/dsl/syntax.rb

@@ -0,0 +1,5 @@
+class Lono::Builder::Configset::Definition::Dsl
+  module Syntax
+    include_modules "syntax"
+  end
+end