lono 7.5.1 → 8.0.0.pre.rc2

data/lib/lono/app_file/build/lambda_layer/ruby_packager.rb

@@ -4,10 +4,10 @@ class Lono::AppFile::Build::LambdaLayer
     include Lono::Utils::Rsync
 
     def initialize(blueprint, registry_item)
-      @blueprint, @registry_item = blueprint, registry_item
+      @blueprint.name, @registry_item = blueprint, registry_item
 
       @registry_name = @registry_item.name.sub(/\/$/,'')
-      @app_root = "#{Lono.blueprint_root}/app/files/#{@registry_name}"
+      @app_root = "#{@blueprint.root}/app/files/#{@registry_name}"
     end
 
     def build
@@ -42,7 +42,7 @@ class Lono::AppFile::Build::LambdaLayer
 
       # create symlink in output path not the cache path
       symlink_dest = "#{output_area}/vendor/gems/ruby/#{ruby_folder}"
-      puts "symlink_dest #{symlink_dest}"
+      logger.info "symlink_dest #{symlink_dest}"
       FileUtils.rm_rf(symlink_dest) # blow away original 2.5.0 folder
 
       # Create symlink that will point to the gems in the Lambda Layer:
@@ -68,7 +68,7 @@ class Lono::AppFile::Build::LambdaLayer
     # project gets built again not all the gems from get installed from the
     # beginning.
     def bundle_install
-      puts "Bundling: running bundle install in cache area: #{cache_area}."
+      logger.info "Bundling: running bundle install in cache area: #{cache_area}."
 
       rsync(output_area, cache_area)
 
@@ -85,15 +85,15 @@ class Lono::AppFile::Build::LambdaLayer
       FileUtils.rm_rf("#{cache_area}/.bundle")
       require "bundler" # dynamically require bundler so user can use any bundler
       setup_bundle_config(cache_area)
-      Bundler.with_unbundled_env do
+      ::Bundler.with_unbundled_env do
         sh "cd #{cache_area} && env bundle install"
       end
 
       remove_bundled_with("#{cache_area}/Gemfile.lock")
       # Fixes really tricky bug where Gemfile and Gemfile.lock is out-of-sync. Details: https://gist.github.com/tongueroo/b5b0d0c924a4a1633eee514795e4b04b
-      FileUtils.cp("#{cache_area}/Gemfile.lock", "#{Lono.config.output_path}/#{@blueprint}/files/#{@registry_name}/Gemfile.lock")
+      FileUtils.cp("#{cache_area}/Gemfile.lock", "#{Lono.root}/output/#{@blueprint.name}/files/#{@registry_name}/Gemfile.lock")
 
-      puts 'Bundle install success.'
+      logger.info 'Bundle install success.'
     end
 
     # Remove the BUNDLED WITH line since we don't control the bundler gem version on AWS Lambda
@@ -130,11 +130,11 @@ EOL
     end
 
     def output_area
-      "#{Lono.config.output_path}/#{@blueprint}/files/#{@registry_name}"
+      "#{Lono.root}/output/#{@blueprint.name}/files/#{@registry_name}"
     end
 
     def build_area
-      "#{Lono.config.output_path}/#{@blueprint}/lambda_layers/#{@registry_name}"
+      "#{Lono.root}/output/#{@blueprint.name}/lambda_layers/#{@registry_name}"
     end
 
     def cache_area

data/lib/lono/app_file/build/lambda_layer.rb

@@ -1,19 +1,19 @@
 class Lono::AppFile::Build
   class LambdaLayer
     def initialize(blueprint, registry_item)
-      @blueprint, @registry_item = blueprint, registry_item
+      @blueprint.name, @registry_item = blueprint, registry_item
     end
 
     def build
       lang = @registry_item.options[:lang]
       unless lang =~ /ruby/
-        puts "WARN: Currently only support ruby lambda layers".color(:yellow)
+        logger.info "WARN: Currently only support ruby lambda layers".color(:yellow)
         return
       end
 
       klass_name = "Lono::AppFile::Build::LambdaLayer::#{lang.camelize}Packager"
       klass = klass_name.constantize
-      packager = klass.new(@blueprint, @registry_item)
+      packager = klass.new(@blueprint.name, @registry_item)
       packager.build
     end
   end

data/lib/lono/app_file/build.rb

@@ -5,12 +5,12 @@ module Lono::AppFile
     include Lono::Utils::Item::Zip
 
     def initialize_variables
-      @output_files_path = "#{Lono.config.output_path}/#{@blueprint}/files"
+      @output_files_path = "#{Lono.root}/output/#{@blueprint.name}/files"
     end
 
     def run
       return unless detect_files?
-      puts "Building app/files for blueprint #{@blueprint}"
+      logger.info "Building app/files"
       build_all
     end
 
@@ -29,11 +29,11 @@ module Lono::AppFile
       end
       missing_paths = missing.map { |item| item.src_path }.uniq
       unless missing_paths.empty?
-        puts "ERROR: These app/files are missing were used by the s3_key method but are missing".color(:red)
+        logger.info "ERROR: These app/files are missing were used by the s3_key method but are missing".color(:red)
         missing_paths.each do |path|
-          puts "    #{path}"
+          logger.info "    #{path}"
         end
-        puts "Please double check that they exist."
+        logger.info "Please double check that they exist."
         exit 1
       end
     end
@@ -51,32 +51,27 @@ module Lono::AppFile
         if item.type == "lambda_layer" || item.exist?
           zip(item)
         else
-          puts "WARN: #{item.src_path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
+          logger.info "WARN: #{item.src_path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
         end
       end
     end
 
     def copy_to_output
-      override_source_paths("#{Lono.blueprint_root}/app/files")
+      override_source_paths("#{@blueprint.root}/app/files")
       self.destination_root = @output_files_path
-      directory(".", verbose: false, context: context.get_binding) # Thor::Action
+      directory(".", verbose: false, context: template_context.get_binding) # Thor::Action
     end
 
-    def context
-      Lono::Template::Context.new(@options)
-    end
-    memoize :context
-
     def clean_output
       FileUtils.rm_rf(@output_files_path)
     end
 
     def detect_files?
-      app_files = Dir["#{Lono.blueprint_root}/app/files/*"]
+      app_files = Dir["#{@blueprint.root}/app/files/*"]
       if app_files.empty?
         false
       else
-        puts "Detected app/files for blueprint #{@blueprint}"
+        logger.info "Detected app/files"
         true
       end
     end

data/lib/lono/app_file/registry/item.rb

@@ -5,19 +5,19 @@ class Lono::AppFile::Registry
 
     attr_reader :name, :options, :type
     def initialize(name, blueprint, options={})
-      @name, @blueprint, @options = name, blueprint, options
+      @name, @blueprint.name, @options = name, blueprint, options
       @type = options[:type] || "file"
     end
 
     def src_path
-      "#{Lono.blueprint_root}/app/files/#{@name}"
+      "#{@blueprint.root}/app/files/#{@name}"
     end
 
     def output_path
       if @type == "file"
-        "#{Lono.root}/output/#{@blueprint}/files/#{@name}"
+        "#{Lono.root}/output/#{@blueprint.name}/files/#{@name}"
       else
-        "#{Lono.root}/output/#{@blueprint}/lambda_layers/#{@name}/opt"
+        "#{Lono.root}/output/#{@blueprint.name}/lambda_layers/#{@name}/opt"
       end
     end
   end

data/lib/lono/app_file/upload.rb

@@ -2,7 +2,7 @@ module Lono::AppFile
   class Upload < Base
     def upload
       return unless Registry.items.size > 0
-      puts "Uploading app/files..."
+      logger.info "Uploading app/files..."
 
       Registry.items.each do |item|
         Lono::S3::Uploader.new(item.zip_file_path).upload

data/lib/lono/autoloader.rb

@@ -1,5 +1,5 @@
-require "lono/bundle"
-Lono::Bundle.setup
+require "lono/bundler"
+Lono::Bundler.setup
 require "zeitwerk"
 
 module Lono

data/lib/lono/aws_services/helper.rb

@@ -1,14 +1,51 @@
 module Lono::AwsServices
   module Helper
-    include Stack
-    include StackSet
+    def stack_exists?(stack_name)
+      return false if ENV['LONO_NOOP']
 
-    def rollback_complete?(stack)
-      stack&.stack_status == 'ROLLBACK_COMPLETE'
+      exist = nil
+      begin
+        # When the stack does not exist an exception is raised. Example:
+        # Aws::CloudFormation::Errors::ValidationError: Stack with id blah does not exist
+        cfn.describe_stacks(stack_name: stack_name)
+        exist = true
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        if e.message =~ /does not exist/
+          exist = false
+        elsif e.message.include?("'stackName' failed to satisfy constraint")
+          # Example of e.message when describe_stack with invalid stack name
+          # "1 validation error detected: Value 'instance_and_route53' at 'stackName' failed to satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z][-a-zA-Z0-9]*|arn:[-a-zA-Z0-9:/._+]*"
+          logger.info "Invalid stack name: #{stack_name}"
+          logger.info "Full error message: #{e.message}"
+          exit 1
+        else
+          raise # re-raise exception  because unsure what other errors can happen
+        end
+      end
+      exist
     end
 
-    def testing_update?
-      ENV['LONO_TEST'] && self.class.name == "LonoCfn::Update"
+    def find_stack(stack_name)
+      resp = cfn.describe_stacks(stack_name: stack_name)
+      resp.stacks.first
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      # example: Stack with id demo-web does not exist
+      if e.message =~ /Stack with/ && e.message =~ /does not exist/
+        nil
+      else
+        raise
+      end
+    end
+
+    def find_stack_resources(stack_name)
+      resp = cfn.describe_stack_resources(stack_name: stack_name)
+      resp.stack_resources
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      if e.message.include?("does not exist")
+        nil
+      else
+        raise
+      end
     end
   end
 end

data/lib/lono/aws_services.rb

@@ -11,27 +11,27 @@ module Lono
     include Helper
 
     def cfn
-      Aws::CloudFormation::Client.new
+      Aws::CloudFormation::Client.new(aws_options)
     end
     memoize :cfn
 
     def ec2
-      Aws::EC2::Client.new
+      Aws::EC2::Client.new(aws_options)
     end
     memoize :ec2
 
     def iam
-      Aws::IAM::Client.new
+      Aws::IAM::Client.new(aws_options)
     end
     memoize :iam
 
     def s3
-      Aws::S3::Client.new
+      Aws::S3::Client.new(aws_options)
     end
     memoize :s3
 
     def s3_resource
-      Aws::S3::Resource.new
+      Aws::S3::Resource.new(aws_options)
     end
     memoize :s3_resource
 
@@ -41,8 +41,39 @@ module Lono
     memoize :s3_presigner
 
     def sts
-      Aws::STS::Client.new # part of aws-sdk-core
+      Aws::STS::Client.new(aws_options) # part of aws-sdk-core
     end
     memoize :sts
+
+    # Override the AWS retry settings with Lono AWS clients.
+    #
+    # The aws-sdk-core has exponential backup with this formula:
+    #
+    #   2 ** c.retries * c.config.retry_base_delay
+    #
+    # Source:
+    #   https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/retry_errors.rb
+    #
+    # So the max delay will be 2 ** 7 * 0.6 = 76.8s
+    #
+    # Only scoping this to deploy because dont want to affect people's application that use the aws sdk.
+    #
+    # There is also additional rate backoff logic elsewhere, since this is only scoped to deploys.
+    #
+    # Useful links:
+    #   https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/plugins/retry_errors.rb
+    #   https://docs.aws.amazon.com/apigateway/latest/developerguide/limits.html
+    #
+    def aws_options
+      options = {
+        retry_limit: 7, # default: 3
+        retry_base_delay: 0.6, # default: 0.3
+      }
+      options.merge!(
+        log_level: :debug,
+        logger: Logger.new($stdout),
+      ) if ENV['LONO_DEBUG_AWS_SDK']
+      options
+    end
   end
 end

data/lib/lono/blueprint.rb

@@ -1,15 +1,24 @@
 module Lono
-  class Blueprint < Command
-    long_desc Help.text("blueprint/new")
-    New.cli_options.each do |args|
-      option(*args)
+  class Blueprint
+    attr_reader :name
+    def initialize(options={})
+      @options = options
+      @name = options[:blueprint]
     end
-    register(New, "new", "new NAME", "Generates new lono blueprint.")
 
-    desc "list", "Lists project blueprints"
-    long_desc Help.text("blueprint/list")
-    def list
-      Finder::Blueprint.list
+    def exist?
+      !root.nil?
+    end
+
+    def root
+      paths = Dir.glob("#{Lono.root}/{app,vendor}/blueprints/*")
+      paths.find do |path|
+        path.include?("blueprints/#{@name}")
+      end
+    end
+
+    def output_path
+      "#{Lono.root}/output/#{@name}/template.yml"
     end
   end
 end

data/lib/lono/booter.rb

@@ -0,0 +1,28 @@
+module Lono
+  module Booter
+    def boot
+      run_hooks
+      Lono::Bundler.require # load plugins
+    end
+
+    # Special boot hooks run super early.
+    # Useful for setting env vars and other early things.
+    #
+    #    config/boot.rb
+    #    config/boot/dev.rb
+    #
+    def run_hooks
+      run_hook
+      run_hook(Lono.env)
+      Lono::App::Inits.run_all
+    end
+
+    def run_hook(env=nil)
+      name = env ? "boot/#{env}" : "boot"
+      path = "#{Lono.root}/config/#{name}.rb"
+      require path if File.exist?(path)
+    end
+
+    extend self
+  end
+end

data/lib/lono/builder/context/generic.rb

@@ -0,0 +1,11 @@
+module Lono::Builder::Context
+  class Generic < Lono::CLI::Base
+    include Lono::Builder::Dsl::Syntax
+
+    # For Lono::AppFile::Build usage of Thor::Action directory
+    # For some reason a send(:binding) doesnt work but get_binding like this works.
+    def get_binding
+      binding
+    end
+  end
+end

data/lib/lono/{template/context/loader → builder/context/loaders}/load_files.rb

@@ -1,13 +1,12 @@
-module Lono::Template::Context::Loader
+module Lono::Builder::Context::Loaders
   module LoadFiles
     # Load custom helper methods from project
-    def load_files(root)
-      paths = Dir.glob("#{root}/**/*.rb")
+    def load_files(dir)
+      paths = Dir.glob("#{dir}/**/*.rb")
       paths.sort_by! { |p| p.size } # so namespaces are loaded first
       paths.each do |path|
-        # lib = gem extensions
-        # app = project extensions
-        filename = path.sub(%r{.*/lib/},'').sub(%r{.*/app/helpers/},'').sub('.rb','')
+        # helpers = blueprint or project extensions
+        filename = path.sub(%r{.*/helpers/},'').sub('.rb','')
         module_name = filename.camelize
 
         # Prepend a period so require works LONO_ROOT is set to a relative path without a period.

data/lib/lono/builder/context/loaders.rb

@@ -0,0 +1,35 @@
+module Lono::Builder::Context
+  module Loaders
+    include LoadFiles
+
+    # Variables in base.rb are overridden by their environment specific variables
+    # file.  Example, file LONO_ENV=dev:
+    #
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #
+    def load_variables
+      layers = Lono::Layering::Layer.new(@blueprint, "vars").paths
+      layers.each do |layer|
+        evaluate_variables_file(layer)
+      end
+    end
+
+    # Load the variables defined in config/vars/* to make available in lono scope.
+    #
+    # NOTE: Was only able to make instance variables avaialble with instance_eval, wasnt able to make local variables
+    # available.
+    def evaluate_variables_file(path)
+      return unless File.exist?(path)
+      instance_eval(IO.read(path), path)
+    end
+
+    # Load blueprint helpers
+    # blueprint helpers override extension helpers
+    def load_blueprint_helpers
+      load_files("#{@blueprint.root}/#{Lono.config.paths.helpers}")
+    end
+  end
+end

data/lib/lono/builder/context/params.rb

@@ -0,0 +1,6 @@
+module Lono::Builder::Context
+  class Params < Generic
+    # Overriding output resource DSL method
+    alias_method :output, :stack_output
+  end
+end

data/lib/lono/builder/context/template.rb

@@ -0,0 +1,4 @@
+module Lono::Builder::Context
+  class Template < Generic
+  end
+end

data/lib/lono/builder/dsl/evaluator.rb

@@ -0,0 +1,94 @@
+class Lono::Builder::Dsl
+  class Evaluator < Lono::CLI::Base
+    include Lono::Builder::Context::Loaders
+    include Lono::Builder::Dsl::Syntax
+    include Lono::Utils::Pretty
+
+    def initialize(options={})
+      super
+      @template_path = "#{@blueprint.root}/template.rb"
+      @parameters = [] # built by parameter_groups.rb
+      @cfn = {}
+    end
+
+    def build
+      # TODO: load_all_extension_helpers # after Extensions::Preparer#run
+      load_variables unless seed? # both blueprint and project variables
+      load_blueprint_helpers
+      evaluate_template_paths(@template_path) # modifies @cfn
+      finalize
+      to_yaml
+      @cfn
+    end
+
+    def finalize
+      o = @options.merge(parameters: @parameters)
+      @cfn = Finalizer.new(@cfn, o).run
+    end
+
+    def to_yaml
+      # https://stackoverflow.com/questions/24508364/how-to-emit-yaml-in-ruby-expanding-aliases
+      # Trick to prevent YAML from emitting aliases
+      @cfn = YAML.load(@cfn.to_json)
+    end
+
+    # Dont want any existing files to prevent building the blueprint.
+    # This means that parameters cannot be based on vars. It's a trade-off.
+    def seed?
+      ARGV[0] == "seed"
+    end
+
+    # Example path: /full/path/to/project/app/blueprints/demo/template.rb
+    def evaluate_template_paths(path)
+      ext = File.extname(path)
+      folder = path.sub(ext, '')
+      expr = "#{folder}/**/*.rb"
+      evaluate_template_path(path) # process top-level template.rb first
+      Dir.glob(expr).each do |path|
+        evaluate_template_path(path)
+      end
+    end
+
+    def evaluate_template_path(path)
+      return unless File.exist?(path)
+      instance_eval(File.read(path), path)
+    rescue Exception => e
+      template_evaluation_error(e)
+      exit 1
+    end
+
+    # Prints out a user friendly task_definition error message
+    #
+    # Backtrace lines are different for OSes:
+    #
+    #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/terraspace-1.1.1/lib/terraspace/builder.rb:34:in `build'"
+    #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/terraspace-1.1.1/lib/terraspace/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
+    #
+    def template_evaluation_error(e)
+      lines = e.backtrace.reject { |l| l.include?("/lib/lono/") }
+      error_info = lines.first
+      parts = error_info.split(':')
+      windows = error_info.match(/^[a-zA-Z]:/)
+      path = windows ? parts[1] : parts[0]
+      line_no = windows ? parts[2] : parts[1]
+      line_no = line_no.to_i
+      logger.debug e.message
+      logger.error "Error evaluating #{pretty_path(path)}".color(:red)
+      logger.error "Here's the line with the error:\n\n"
+
+      contents = IO.read(path)
+      content_lines = contents.split("\n")
+      context = 5 # lines of context
+      top, bottom = [line_no-context-1, 0].max, line_no+context-1
+      spacing = content_lines.size.to_s.size
+      content_lines[top..bottom].each_with_index do |line_content, index|
+        line_number = top+index+1
+        if line_number == line_no
+          logger.printf("%#{spacing}d %s\n".color(:red), line_number, line_content)
+        else
+          logger.printf("%#{spacing}d %s\n", line_number, line_content)
+        end
+      end
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/configsets.rb

@@ -0,0 +1,41 @@
+class Lono::Builder::Dsl::Finalizer
+  class Configsets
+    extend Memoist
+
+    def initialize(cfn)
+      @cfn = cfn
+    end
+
+    def run
+      @cfn = add # overwrite
+      @cfn
+    end
+
+    def add
+      metadata_map.each do |logical_id, metadata_configset|
+        resource = @cfn["Resources"][logical_id]
+
+        unless resource
+          logger.info "WARN: Resources.#{logical_id} not found in the template. Are you sure you are specifying the correct resource id in your configsets configs?".color(:yellow)
+          next
+        end
+
+        resource["Metadata"] ||= metadata_configset["Metadata"]
+
+        # metdata = resource["Metadata"] ||= {}
+        # metdata["AWS::CloudFormation::Init"] ||= {}
+        # # The metadata_configset has been combined with the original AWS::CloudFormation::Init if it exists
+        # metdata["AWS::CloudFormation::Init"] = metadata_configset["AWS::CloudFormation::Init"]
+      end
+
+      @cfn
+    end
+
+    def metadata_map
+      []
+      # combiner = Lono::Configset::Combiner.new(@cfn, @options)
+      # combiner.metadata_map
+    end
+    memoize :metadata_map
+  end
+end

data/lib/lono/{template/strategy → builder}/dsl/finalizer/parameter_groups.rb

@@ -1,7 +1,7 @@
-class Lono::Template::Strategy::Dsl::Finalizer
+class Lono::Builder::Dsl::Finalizer
   class ParameterGroups
     extend Memoist
-    include Lono::Template::Strategy::Dsl::Builder::Stringify
+    include Lono::Builder::Util::Stringify
 
     def initialize(cfn, parameters)
       @cfn, @parameters = cfn, parameters

data/lib/lono/{template/strategy → builder}/dsl/finalizer.rb

@@ -1,4 +1,4 @@
-class Lono::Template::Strategy::Dsl
+class Lono::Builder::Dsl
   class Finalizer
     def initialize(cfn, options={})
       @cfn, @options = cfn, options
@@ -6,6 +6,7 @@ class Lono::Template::Strategy::Dsl
 
     def run
       @cfn = ParameterGroups.new(@cfn, @options[:parameters]).run
+      @cfn = Configsets.new(@cfn).run
       @cfn
     end
   end

data/lib/lono/{template/strategy/dsl/builder/helpers/ec2_helper.rb → builder/dsl/helpers/ec2.rb}

@@ -1,5 +1,5 @@
-module Lono::Template::Strategy::Dsl::Builder::Helpers
-  module Ec2Helper
+module Lono::Builder::Dsl::Helpers
+  module Ec2
     extend Memoist
 
     def default_vpc
@@ -20,7 +20,7 @@ module Lono::Template::Strategy::Dsl::Builder::Helpers
     def key_pairs(regexp=nil)
       resp = ec2.describe_key_pairs
       key_names = resp.key_pairs.map(&:key_name)
-      key_names.select! { |k| k =~ regexp }
+      key_names.select! { |k| k =~ regexp } if regexp
       key_names
     end
   end