lono 7.5.2 → 8.0.0.pre.rc1

data/lib/lono/app_file/build/lambda_layer.rb

@@ -1,19 +1,19 @@
 class Lono::AppFile::Build
   class LambdaLayer
     def initialize(blueprint, registry_item)
-      @blueprint, @registry_item = blueprint, registry_item
+      @blueprint.name, @registry_item = blueprint, registry_item
     end
 
     def build
       lang = @registry_item.options[:lang]
       unless lang =~ /ruby/
-        puts "WARN: Currently only support ruby lambda layers".color(:yellow)
+        logger.info "WARN: Currently only support ruby lambda layers".color(:yellow)
         return
       end
 
       klass_name = "Lono::AppFile::Build::LambdaLayer::#{lang.camelize}Packager"
       klass = klass_name.constantize
-      packager = klass.new(@blueprint, @registry_item)
+      packager = klass.new(@blueprint.name, @registry_item)
       packager.build
     end
   end

data/lib/lono/app_file/build.rb

@@ -5,12 +5,12 @@ module Lono::AppFile
     include Lono::Utils::Item::Zip
 
     def initialize_variables
-      @output_files_path = "#{Lono.config.output_path}/#{@blueprint}/files"
+      @output_files_path = "#{Lono.root}/output/#{@blueprint.name}/files"
     end
 
     def run
       return unless detect_files?
-      puts "Building app/files for blueprint #{@blueprint}"
+      logger.info "Building app/files"
       build_all
     end
 
@@ -29,11 +29,11 @@ module Lono::AppFile
       end
       missing_paths = missing.map { |item| item.src_path }.uniq
       unless missing_paths.empty?
-        puts "ERROR: These app/files are missing were used by the s3_key method but are missing".color(:red)
+        logger.info "ERROR: These app/files are missing were used by the s3_key method but are missing".color(:red)
         missing_paths.each do |path|
-          puts "    #{path}"
+          logger.info "    #{path}"
         end
-        puts "Please double check that they exist."
+        logger.info "Please double check that they exist."
         exit 1
       end
     end
@@ -51,32 +51,27 @@ module Lono::AppFile
         if item.type == "lambda_layer" || item.exist?
           zip(item)
         else
-          puts "WARN: #{item.src_path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
+          logger.info "WARN: #{item.src_path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
         end
       end
     end
 
     def copy_to_output
-      override_source_paths("#{Lono.blueprint_root}/app/files")
+      override_source_paths("#{@blueprint.root}/app/files")
       self.destination_root = @output_files_path
       directory(".", verbose: false, context: context.get_binding) # Thor::Action
     end
 
-    def context
-      Lono::Template::Context.new(@options)
-    end
-    memoize :context
-
     def clean_output
       FileUtils.rm_rf(@output_files_path)
     end
 
     def detect_files?
-      app_files = Dir["#{Lono.blueprint_root}/app/files/*"]
+      app_files = Dir["#{@blueprint.root}/app/files/*"]
       if app_files.empty?
         false
       else
-        puts "Detected app/files for blueprint #{@blueprint}"
+        logger.info "Detected app/files"
         true
       end
     end

data/lib/lono/app_file/registry/item.rb

@@ -5,19 +5,19 @@ class Lono::AppFile::Registry
 
     attr_reader :name, :options, :type
     def initialize(name, blueprint, options={})
-      @name, @blueprint, @options = name, blueprint, options
+      @name, @blueprint.name, @options = name, blueprint, options
       @type = options[:type] || "file"
     end
 
     def src_path
-      "#{Lono.blueprint_root}/app/files/#{@name}"
+      "#{@blueprint.root}/app/files/#{@name}"
     end
 
     def output_path
       if @type == "file"
-        "#{Lono.root}/output/#{@blueprint}/files/#{@name}"
+        "#{Lono.root}/output/#{@blueprint.name}/files/#{@name}"
       else
-        "#{Lono.root}/output/#{@blueprint}/lambda_layers/#{@name}/opt"
+        "#{Lono.root}/output/#{@blueprint.name}/lambda_layers/#{@name}/opt"
       end
     end
   end

data/lib/lono/app_file/upload.rb

@@ -2,7 +2,7 @@ module Lono::AppFile
   class Upload < Base
     def upload
       return unless Registry.items.size > 0
-      puts "Uploading app/files..."
+      logger.info "Uploading app/files..."
 
       Registry.items.each do |item|
         Lono::S3::Uploader.new(item.zip_file_path).upload

data/lib/lono/autoloader.rb

@@ -1,5 +1,5 @@
-require "lono/bundle"
-Lono::Bundle.setup
+require "lono/bundler"
+Lono::Bundler.setup
 require "zeitwerk"
 
 module Lono

data/lib/lono/aws_services/helper.rb

@@ -1,14 +1,51 @@
 module Lono::AwsServices
   module Helper
-    include Stack
-    include StackSet
+    def stack_exists?(stack_name)
+      return false if ENV['LONO_NOOP']
 
-    def rollback_complete?(stack)
-      stack&.stack_status == 'ROLLBACK_COMPLETE'
+      exist = nil
+      begin
+        # When the stack does not exist an exception is raised. Example:
+        # Aws::CloudFormation::Errors::ValidationError: Stack with id blah does not exist
+        cfn.describe_stacks(stack_name: stack_name)
+        exist = true
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        if e.message =~ /does not exist/
+          exist = false
+        elsif e.message.include?("'stackName' failed to satisfy constraint")
+          # Example of e.message when describe_stack with invalid stack name
+          # "1 validation error detected: Value 'instance_and_route53' at 'stackName' failed to satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z][-a-zA-Z0-9]*|arn:[-a-zA-Z0-9:/._+]*"
+          logger.info "Invalid stack name: #{stack_name}"
+          logger.info "Full error message: #{e.message}"
+          exit 1
+        else
+          raise # re-raise exception  because unsure what other errors can happen
+        end
+      end
+      exist
     end
 
-    def testing_update?
-      ENV['LONO_TEST'] && self.class.name == "LonoCfn::Update"
+    def find_stack(stack_name)
+      resp = cfn.describe_stacks(stack_name: stack_name)
+      resp.stacks.first
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      # example: Stack with id demo-web does not exist
+      if e.message =~ /Stack with/ && e.message =~ /does not exist/
+        nil
+      else
+        raise
+      end
+    end
+
+    def find_stack_resources(stack_name)
+      resp = cfn.describe_stack_resources(stack_name: stack_name)
+      resp.stack_resources
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      if e.message.include?("does not exist")
+        nil
+      else
+        raise
+      end
     end
   end
 end

data/lib/lono/aws_services.rb

@@ -11,27 +11,27 @@ module Lono
     include Helper
 
     def cfn
-      Aws::CloudFormation::Client.new
+      Aws::CloudFormation::Client.new(aws_options)
     end
     memoize :cfn
 
     def ec2
-      Aws::EC2::Client.new
+      Aws::EC2::Client.new(aws_options)
     end
     memoize :ec2
 
     def iam
-      Aws::IAM::Client.new
+      Aws::IAM::Client.new(aws_options)
     end
     memoize :iam
 
     def s3
-      Aws::S3::Client.new
+      Aws::S3::Client.new(aws_options)
     end
     memoize :s3
 
     def s3_resource
-      Aws::S3::Resource.new
+      Aws::S3::Resource.new(aws_options)
     end
     memoize :s3_resource
 
@@ -41,8 +41,39 @@ module Lono
     memoize :s3_presigner
 
     def sts
-      Aws::STS::Client.new # part of aws-sdk-core
+      Aws::STS::Client.new(aws_options) # part of aws-sdk-core
     end
     memoize :sts
+
+    # Override the AWS retry settings with Lono AWS clients.
+    #
+    # The aws-sdk-core has exponential backup with this formula:
+    #
+    #   2 ** c.retries * c.config.retry_base_delay
+    #
+    # Source:
+    #   https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/retry_errors.rb
+    #
+    # So the max delay will be 2 ** 7 * 0.6 = 76.8s
+    #
+    # Only scoping this to deploy because dont want to affect people's application that use the aws sdk.
+    #
+    # There is also additional rate backoff logic elsewhere, since this is only scoped to deploys.
+    #
+    # Useful links:
+    #   https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/plugins/retry_errors.rb
+    #   https://docs.aws.amazon.com/apigateway/latest/developerguide/limits.html
+    #
+    def aws_options
+      options = {
+        retry_limit: 7, # default: 3
+        retry_base_delay: 0.6, # default: 0.3
+      }
+      options.merge!(
+        log_level: :debug,
+        logger: Logger.new($stdout),
+      ) if ENV['LONO_DEBUG_AWS_SDK']
+      options
+    end
   end
 end

data/lib/lono/blueprint.rb

@@ -1,15 +1,24 @@
 module Lono
-  class Blueprint < Command
-    long_desc Help.text("blueprint/new")
-    New.cli_options.each do |args|
-      option(*args)
+  class Blueprint
+    attr_reader :name
+    def initialize(options={})
+      @options = options
+      @name = options[:blueprint]
     end
-    register(New, "new", "new NAME", "Generates new lono blueprint.")
 
-    desc "list", "Lists project blueprints"
-    long_desc Help.text("blueprint/list")
-    def list
-      Finder::Blueprint.list
+    def exist?
+      !root.nil?
+    end
+
+    def root
+      paths = Dir.glob("#{Lono.root}/{app,vendor}/blueprints/*")
+      paths.find do |path|
+        path.include?("blueprints/#{@name}")
+      end
+    end
+
+    def output_path
+      "#{Lono.root}/output/#{@name}/template.yml"
     end
   end
 end

data/lib/lono/booter.rb

@@ -0,0 +1,28 @@
+module Lono
+  module Booter
+    def boot
+      run_hooks
+      Lono::Bundler.require # load plugins
+    end
+
+    # Special boot hooks run super early.
+    # Useful for setting env vars and other early things.
+    #
+    #    config/boot.rb
+    #    config/boot/dev.rb
+    #
+    def run_hooks
+      run_hook
+      run_hook(Lono.env)
+      Lono::App::Inits.run_all
+    end
+
+    def run_hook(env=nil)
+      name = env ? "boot/#{env}" : "boot"
+      path = "#{Lono.root}/config/#{name}.rb"
+      require path if File.exist?(path)
+    end
+
+    extend self
+  end
+end

data/lib/lono/{template → builder}/context/helpers.rb

@@ -1,4 +1,4 @@
-class Lono::Template::Context
+class Lono::Builder::Context
   module Helpers
     extend Memoist
 

data/lib/lono/{template → builder}/context/loader/load_files.rb

@@ -1,13 +1,12 @@
-module Lono::Template::Context::Loader
+module Lono::Builder::Context::Loader
   module LoadFiles
     # Load custom helper methods from project
-    def load_files(root)
-      paths = Dir.glob("#{root}/**/*.rb")
+    def load_files(dir)
+      paths = Dir.glob("#{dir}/**/*.rb")
       paths.sort_by! { |p| p.size } # so namespaces are loaded first
       paths.each do |path|
-        # lib = gem extensions
-        # app = project extensions
-        filename = path.sub(%r{.*/lib/},'').sub(%r{.*/app/helpers/},'').sub('.rb','')
+        # helpers = blueprint or project extensions
+        filename = path.sub(%r{.*/helpers/},'').sub('.rb','')
         module_name = filename.camelize
 
         # Prepend a period so require works LONO_ROOT is set to a relative path without a period.

data/lib/lono/builder/context/loader.rb

@@ -0,0 +1,35 @@
+class Lono::Builder::Context
+  module Loader
+    include LoadFiles
+
+    # Variables in base.rb are overridden by their environment specific variables
+    # file.  Example, file LONO_ENV=dev:
+    #
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #
+    def load_variables
+      layers = Lono::Layering::Layer.new(@blueprint, "vars").paths
+      layers.each do |layer|
+        evaluate_variables_file(layer)
+      end
+    end
+
+    # Load the variables defined in config/vars/* to make available in lono scope.
+    #
+    # NOTE: Was only able to make instance variables avaialble with instance_eval, wasnt able to make local variables
+    # available.
+    def evaluate_variables_file(path)
+      return unless File.exist?(path)
+      instance_eval(IO.read(path), path)
+    end
+
+    # Load blueprint helpers
+    # blueprint helpers override extension helpers
+    def load_blueprint_helpers
+      load_files("#{@blueprint.root}/#{Lono.config.paths.helpers}")
+    end
+  end
+end

data/lib/lono/builder/context/params.rb

@@ -0,0 +1,7 @@
+class Lono::Builder::Context
+  class Params < Lono::Builder::Context
+    # These helpers are only defined in the params context.
+    # Overriding output resource DSL method
+    alias_method :output, :stack_output
+  end
+end

data/lib/lono/{template → builder}/context/ssm_fetcher.rb

@@ -1,6 +1,6 @@
 require 'aws-sdk-ssm'
 
-class Lono::Template::Context
+class Lono::Builder::Context
   class SsmFetcher
     extend Memoist
 

data/lib/lono/{template → builder}/context.rb

@@ -1,16 +1,12 @@
 # Encapsulates helper methods and instance variables to be rendered in the ERB templates.
-class Lono::Template
-  class Context < Lono::AbstractBase
+module Lono::Builder
+  class Context < Lono::CLI::Base
     extend Memoist
-    include Lono::Template::Helper
+    include Lono::Builder::Template::Helpers
     include Loader
     include Helpers # ERB
-    include Strategy::Dsl::Builder::Syntax # DSL
-
-    def initialize(options={})
-      super
-      load_context
-    end
+    include Template::Dsl::Evaluator::Syntax # DSL
+    include Lono::Utils::Context
 
     # Take a hash and makes them instance variables in the current scope.
     # Use this in custom helper methods to make variables accessible to ERB templates.

data/lib/lono/{param/generator.rb → builder/param.rb}

@@ -1,15 +1,16 @@
-class Lono::Param
-  class Generator < Lono::AbstractBase
-    attr_reader :env_path, :base_path # set when generate is called
+module Lono::Builder
+  class Param < Lono::CLI::Base
+    attr_reader :env_path, :base_path # set when build is called
 
-    def generate
-      puts "Generating parameter files for blueprint #{@blueprint.color(:green)}:"
+    def build
+      logger.info "Building parameters"
 
       contents = []
-      layering = Lono::Layering.new("params", @options, Lono.env)
-      layering.locations.each do |path|
-        contents << render_erb(path)
+
+      layers.each do |layer|
+        contents << render_erb(layer)
       end
+
       contents = contents.compact.join("\n") # result
 
       data = convert_to_cfn_format(contents)
@@ -17,30 +18,28 @@ class Lono::Param
       json = JSON.pretty_generate(camel_data)
       write_output(json)
       unless @options[:mute]
-        short_output_path = output_path.sub("#{Lono.root}/","")
-        puts "  #{short_output_path}"
+        short_param_path = param_path.sub("#{Lono.root}/","")
+        logger.info "    #{short_param_path}"
       end
 
       data
     end
 
+    def layers
+      Lono::Layering::Layer.new(@blueprint, "params").paths
+    end
+
     def parameters
-      generate
+      build
     end
 
     def render_erb(path)
       return unless path
       if File.exist?(path)
-        RenderMePretty.result(path, context: context)
+        RenderMePretty.result(path, context: params_context)
       end
     end
 
-    # Context for ERB rendering.
-    # This is where we control what references get passed to the ERB rendering.
-    def context
-      @context ||= Lono::Template::Context.new(@options)
-    end
-
     def convert_to_cfn_format(contents, casing=:underscore)
       lines = parse_contents(contents)
 
@@ -96,14 +95,14 @@ class Lono::Param
       end
     end
 
-    def output_path
-      "#{Lono.root}/output/#{@blueprint}/params/#{@stack}.json"
+    def param_path
+      "#{Lono.root}/output/#{@blueprint.name}/params.json"
     end
 
     def write_output(json)
-      dir = File.dirname(output_path)
+      dir = File.dirname(param_path)
       FileUtils.mkdir_p(dir)
-      IO.write(output_path, json)
+      IO.write(param_path, json)
     end
   end
 end

data/lib/lono/builder/template/aws_service.rb

@@ -0,0 +1,15 @@
+require "aws-sdk-s3"
+
+class Lono::Builder::Template::AwsService
+  def s3
+    return @s3 if @s3
+
+    options = {}
+    endpoint = ENV['S3_ENDPOINT'] if ENV['S3_ENDPOINT']
+    options[:endpoint] = endpoint if endpoint
+    if options[:endpoint]
+      options[:region] = options[:endpoint].split('.')[1]
+    end
+    @s3 = Aws::S3::Client.new(options)
+  end
+end

data/lib/lono/builder/template/bashify.rb

@@ -0,0 +1,43 @@
+require 'open-uri'
+
+class Lono::Builder::Template
+  class Bashify
+    include Lono::Utils::Logging
+
+    def initialize(options={})
+      @options = options
+      @path = options[:path]
+    end
+
+    def user_data_paths(data,path="")
+      paths = []
+      paths << path
+      data.each do |key,value|
+        if value.is_a?(Hash)
+          paths += user_data_paths(value,"#{path}/#{key}")
+        else
+          paths += ["#{path}/#{key}"]
+        end
+      end
+      paths.select {|p| p =~ /UserData/ && p =~ /Fn::Join/ }
+    end
+
+    def run
+      raw = open(@path).read
+      json = JSON.load(raw)
+      paths = user_data_paths(json)
+      if paths.empty?
+        logger.info "No UserData script found"
+        return
+      end
+      paths.each do |path|
+        logger.info "UserData script for #{path}:"
+        key = path.sub('/','').split("/").map {|x| "['#{x}']"}.join('')
+        user_data = eval("json#{key}")
+        delimiter = user_data[0]
+        script = user_data[1]
+        logger.info script.join(delimiter)
+      end
+    end
+  end
+end

data/lib/lono/{template → builder/template}/configset_injector.rb

@@ -1,5 +1,5 @@
-class Lono::Template
-  class ConfigsetInjector < Lono::AbstractBase
+class Lono::Builder::Template
+  class ConfigsetInjector < Lono::CLI::Base
     def initialize(options={})
       super
     end
@@ -16,7 +16,7 @@ class Lono::Template
         resource = @cfn["Resources"][logical_id]
 
         unless resource
-          puts "WARN: Resources.#{logical_id} not found in the template. Are you sure you are specifying the correct resource id in your configsets configs?".color(:yellow)
+          logger.info "WARN: Resources.#{logical_id} not found in the template. Are you sure you are specifying the correct resource id in your configsets configs?".color(:yellow)
           next
         end
 
@@ -46,7 +46,7 @@ class Lono::Template
     end
 
     def template_path
-      "#{Lono.config.output_path}/#{@blueprint}/templates/#{@template}.yml"
+      "#{Lono.root}/output/#{@blueprint.name}/templates/#{@template}.yml"
     end
   end
 end

data/lib/lono/{template/strategy/dsl/builder → builder/template/dsl/evaluator}/fn.rb

@@ -1,4 +1,4 @@
-class Lono::Template::Strategy::Dsl::Builder
+class Lono::Builder::Template::Dsl::Evaluator
   module Fn
     # Also act as documentation on the method signatures
     # Also used in Coder crafting so should always list all the functions here
@@ -83,7 +83,7 @@ class Lono::Template::Strategy::Dsl::Builder
       end
     end
 
-    def conditional_ref(name, options)
+    def conditional_ref(name, options={})
       fallback = options[:Fallback] || options[:fallback] || ref("AWS::NoValue")
       if!("Has#{name}", ref(name, options), fallback)
     end

data/lib/lono/{template/strategy/dsl/builder → builder/template/dsl/evaluator}/helpers/core_helper.rb

@@ -1,4 +1,4 @@
-module Lono::Template::Strategy::Dsl::Builder::Helpers
+module Lono::Builder::Template::Dsl::Evaluator::Helpers
   module CoreHelper
     extend Memoist
 

data/lib/lono/{template/strategy/dsl/builder → builder/template/dsl/evaluator}/helpers/ec2_helper.rb

@@ -1,4 +1,4 @@
-module Lono::Template::Strategy::Dsl::Builder::Helpers
+module Lono::Builder::Template::Dsl::Evaluator::Helpers
   module Ec2Helper
     extend Memoist