lono 7.5.2 → 8.0.0.pre.rc3

data/lib/lono/app.rb

@@ -0,0 +1,73 @@
+module Lono
+  class App
+    extend Memoist
+    include DslEvaluator
+    include Singleton
+    include Lono::Utils::Logging
+
+    attr_reader :config
+    def initialize
+      @config = defaults
+    end
+
+    def defaults
+      config = ActiveSupport::OrderedOptions.new
+
+      config.allow = ActiveSupport::OrderedOptions.new
+      config.allow.envs = nil
+      config.allow.regions = nil
+      config.deny = ActiveSupport::OrderedOptions.new
+      config.deny.envs = nil
+      config.deny.regions = nil
+
+      config.layering = ActiveSupport::OrderedOptions.new
+      config.layering.names = {}
+
+      config.log = ActiveSupport::OrderedOptions.new
+      config.log.root = Lono.log_root
+      config.logger = lono_logger
+      config.logger.formatter = Logger::Formatter.new
+      config.logger.level = ENV['LONO_LOG_LEVEL'] || :info
+
+      config.names = ActiveSupport::OrderedOptions.new
+      config.names.output = ActiveSupport::OrderedOptions.new
+      config.names.output.expand = true
+      config.names.output.stack = ":BLUEPRINT-:ENV" # does not include APP by default. Think this is more common
+      config.names.stack = ":APP-:BLUEPRINT-:ENV"
+
+      config.plan = ActiveSupport::OrderedOptions.new
+      config.plan.changeset = true
+      config.plan.params = "full"
+      config.plan.template = "summary" # summary is same as true
+
+      config.seed = ActiveSupport::OrderedOptions.new
+      config.seed.where = "config"
+
+      config.test = ActiveSupport::OrderedOptions.new
+      config.test.framework = "rspec"
+
+      config.up = ActiveSupport::OrderedOptions.new
+      config.up.capabilities = nil
+      config.up.notification_arns = nil
+      config.up.rollback = true
+      config.up.tags = nil
+
+      config
+    end
+
+    def lono_logger
+      Logger.new(ENV['LONO_LOG_PATH'] || $stderr)
+    end
+    memoize :lono_logger
+
+    def configure
+      yield(@config)
+    end
+
+    def load_project_config
+      evaluate_file("#{Lono.root}/config/app.rb")
+      path = "#{Lono.root}/config/envs/#{Lono.env}.rb"
+      evaluate_file(path)
+    end
+  end
+end

data/lib/lono/autoloader.rb

@@ -1,5 +1,5 @@
-require "lono/bundle"
-Lono::Bundle.setup
+require "lono/bundler"
+Lono::Bundler.setup
 require "zeitwerk"
 
 module Lono
@@ -17,6 +17,7 @@ module Lono
         loader.inflector = Inflector.new
         loader.push_dir(File.dirname(__dir__)) # lib
         loader.log! if ENV["LONO_AUTOLOAD_LOG"]
+        loader.ignore("#{__dir__}/ext.rb")
         loader.setup
       end
     end

data/lib/lono/aws_services/helper.rb

@@ -1,14 +1,49 @@
 module Lono::AwsServices
   module Helper
-    include Stack
-    include StackSet
+    def stack_exists?(stack_name)
+      exist = nil
+      begin
+        # When the stack does not exist an exception is raised. Example:
+        # Aws::CloudFormation::Errors::ValidationError: Stack with id blah does not exist
+        cfn.describe_stacks(stack_name: stack_name)
+        exist = true
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        if e.message =~ /does not exist/
+          exist = false
+        elsif e.message.include?("'stackName' failed to satisfy constraint")
+          # Example of e.message when describe_stack with invalid stack name
+          # "1 validation error detected: Value 'instance_and_route53' at 'stackName' failed to satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z][-a-zA-Z0-9]*|arn:[-a-zA-Z0-9:/._+]*"
+          logger.info "Invalid stack name: #{stack_name}"
+          logger.info "Full error message: #{e.message}"
+          exit 1
+        else
+          raise # re-raise exception  because unsure what other errors can happen
+        end
+      end
+      exist
+    end
 
-    def rollback_complete?(stack)
-      stack&.stack_status == 'ROLLBACK_COMPLETE'
+    def find_stack(stack_name)
+      resp = cfn.describe_stacks(stack_name: stack_name)
+      resp.stacks.first
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      # example: Stack with id demo-web does not exist
+      if e.message =~ /Stack with/ && e.message =~ /does not exist/
+        nil
+      else
+        raise
+      end
     end
 
-    def testing_update?
-      ENV['LONO_TEST'] && self.class.name == "LonoCfn::Update"
+    def find_stack_resources(stack_name)
+      resp = cfn.describe_stack_resources(stack_name: stack_name)
+      resp.stack_resources
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      if e.message.include?("does not exist")
+        nil
+      else
+        raise
+      end
     end
   end
 end

data/lib/lono/aws_services.rb

@@ -11,27 +11,27 @@ module Lono
     include Helper
 
     def cfn
-      Aws::CloudFormation::Client.new
+      Aws::CloudFormation::Client.new(aws_options)
     end
     memoize :cfn
 
     def ec2
-      Aws::EC2::Client.new
+      Aws::EC2::Client.new(aws_options)
     end
     memoize :ec2
 
     def iam
-      Aws::IAM::Client.new
+      Aws::IAM::Client.new(aws_options)
     end
     memoize :iam
 
     def s3
-      Aws::S3::Client.new
+      Aws::S3::Client.new(aws_options)
     end
     memoize :s3
 
     def s3_resource
-      Aws::S3::Resource.new
+      Aws::S3::Resource.new(aws_options)
     end
     memoize :s3_resource
 
@@ -41,8 +41,39 @@ module Lono
     memoize :s3_presigner
 
     def sts
-      Aws::STS::Client.new # part of aws-sdk-core
+      Aws::STS::Client.new(aws_options) # part of aws-sdk-core
     end
     memoize :sts
+
+    # Override the AWS retry settings with Lono AWS clients.
+    #
+    # The aws-sdk-core has exponential backup with this formula:
+    #
+    #   2 ** c.retries * c.config.retry_base_delay
+    #
+    # Source:
+    #   https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/retry_errors.rb
+    #
+    # So the max delay will be 2 ** 7 * 0.6 = 76.8s
+    #
+    # Only scoping this to deploy because dont want to affect people's application that use the aws sdk.
+    #
+    # There is also additional rate backoff logic elsewhere, since this is only scoped to deploys.
+    #
+    # Useful links:
+    #   https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/plugins/retry_errors.rb
+    #   https://docs.aws.amazon.com/apigateway/latest/developerguide/limits.html
+    #
+    def aws_options
+      options = {
+        retry_limit: 7, # default: 3
+        retry_base_delay: 0.6, # default: 0.3
+      }
+      options.merge!(
+        log_level: :debug,
+        logger: Logger.new($stdout),
+      ) if ENV['LONO_DEBUG_AWS_SDK']
+      options
+    end
   end
 end

data/lib/lono/blueprint.rb

@@ -1,15 +1,7 @@
 module Lono
-  class Blueprint < Command
-    long_desc Help.text("blueprint/new")
-    New.cli_options.each do |args|
-      option(*args)
-    end
-    register(New, "new", "new NAME", "Generates new lono blueprint.")
-
-    desc "list", "Lists project blueprints"
-    long_desc Help.text("blueprint/list")
-    def list
-      Finder::Blueprint.list
+  class Blueprint < Component
+    def output_path
+      "#{Lono.root}/output/#{@name}/template.yml"
     end
   end
 end

data/lib/lono/booter.rb

@@ -0,0 +1,28 @@
+module Lono
+  module Booter
+    def boot
+      run_hooks
+      Lono::Bundler.require # load plugins
+    end
+
+    # Special boot hooks run super early.
+    # Useful for setting env vars and other early things.
+    #
+    #    config/boot.rb
+    #    config/boot/dev.rb
+    #
+    def run_hooks
+      run_hook
+      run_hook(Lono.env)
+      Lono::App::Inits.run_all
+    end
+
+    def run_hook(env=nil)
+      name = env ? "boot/#{env}" : "boot"
+      path = "#{Lono.root}/config/#{name}.rb"
+      require path if File.exist?(path)
+    end
+
+    extend self
+  end
+end

data/lib/lono/builder/allow/base.rb

@@ -0,0 +1,54 @@
+class Lono::Builder::Allow
+  class Base < Lono::CLI::Base
+    include Lono::App::CallableOption::Concern
+
+    def check!
+      messages = []
+      unless allowed?
+        messages << message # message is interface method
+      end
+      unless messages.empty?
+        puts "ERROR: The configs do not allow this.".color(:red)
+        puts messages
+        exit 1
+      end
+    end
+
+    def allowed?
+      if allows.nil? && denys.nil?
+        true
+      elsif denys.nil?
+        allows.include?(check_value)
+      elsif allows.nil?
+        !denys.include?(check_value)
+      else
+        allows.include?(check_value) && !denys.include?(check_value)
+      end
+    end
+
+    def allows
+      callable_option(
+        config_name: "config.allow.#{config_name}",
+        config_value: config.dig(:allow, config_name),
+        passed_args: [@blueprint],
+      )
+    end
+
+    def denys
+      callable_option(
+        config_name: "config.deny.#{config_name}",
+        config_value: config.dig(:deny, config_name),
+        passed_args: [@blueprint],
+      )
+    end
+
+  private
+    def config
+      Lono.config
+    end
+
+    def config_name
+      self.class.to_s.split('::').last.underscore.pluralize.to_sym # ActiveSuport::HashWithIndifferentAccess#dig requires symbol
+    end
+  end
+end

data/lib/lono/builder/allow/env.rb

@@ -0,0 +1,17 @@
+class Lono::Builder::Allow
+  class Env < Base
+    # interface method
+    def message
+      messages = []
+      messages << "This env is not allowed to be used: LONO_ENV=#{Lono.env}"
+      messages << "Allow envs: #{allows.join(', ')}" if allows
+      messages << "Deny envs: #{denys.join(', ')}" if denys
+      messages.join("\n")
+    end
+
+    # interface method
+    def check_value
+      Lono.env
+    end
+  end
+end

data/lib/lono/builder/allow/region.rb

@@ -0,0 +1,20 @@
+class Lono::Builder::Allow
+  class Region < Base
+    include Lono::Concerns::AwsInfo
+
+    # interface method
+    def message
+      messages = []
+      word = config_name.to_s # IE: regions or locations
+      messages << "This #{word.singularize} is not allowed to be used: Detected current #{word.singularize}=#{current_region}"
+      messages << "Allow #{word}: #{allows.join(', ')}" if allows
+      messages << "Deny #{word}: #{denys.join(', ')}" if denys
+      messages.join("\n")
+    end
+
+    # interface method
+    def check_value
+      region
+    end
+  end
+end

data/lib/lono/builder/allow.rb

@@ -0,0 +1,8 @@
+module Lono::Builder
+  class Allow < Lono::CLI::Base
+    def check!
+      Env.new(@options).check!
+      Region.new(@options).check!
+    end
+  end
+end

data/lib/lono/builder/configset/combiner.rb

@@ -0,0 +1,145 @@
+module Lono::Builder::Configset
+  class Combiner < Lono::CLI::Base
+    def initialize(options={})
+      super
+      @cfn = options[:cfn]     # merge from Dsl::Finalizer::Configsets#run
+      @metas = options[:metas] # merge from Configset::Evaluator#evaluate
+      @configsets = []
+      @map = {} # stores resource logical id => metadata cfn-init
+    end
+
+    # Returns metadata map structure
+    #
+    # {"Instance"=>
+    #   {"Metadata"=>
+    #     {"AWS::CloudFormation::Init"=>
+    #
+    def combine
+      return @map if metas_empty?
+      add_existing
+      add_built
+      build_map # metadata map
+    end
+
+    def metas_empty?
+      @metas.empty?
+    end
+
+    def add_existing
+      existing_configsets.each do |configset|
+        add(configset)
+      end
+    end
+
+    def add_built
+      @metas.each do |meta|
+        definition = Definition.new(@options.merge(meta: meta))
+        configset = definition.evaluate
+        add(configset)
+      end
+    end
+
+    # Useful for specs
+    def add(configset)
+      found = @configsets.detect { |c| c.name == configset.name && c.resource == configset.resource }
+      @configsets << configset unless found
+    end
+
+    # Each metadata has this structure:
+    #
+    #     {"Metadata"=>
+    #       {"AWS::CloudFormation::Init"=>
+    #         {"configSets"=>{"default"=>["aaa1", "aaa2"]},
+    #         "aaa1"=>{"commands"=>{"test"=>{"command"=>"echo from-aaa1 > test1.txt"}}},
+    #         "aaa2"=>
+    #           {"commands"=>{"test"=>{"command"=>"echo from-aaa2 > test1.txt"}}}}}}
+    #
+    def build_map
+      metadata_map = {}
+      configsets_map = {}
+
+      @configsets.each_with_index do |configset, i|
+        padded_i = "%03d" % i
+
+        name, resource, metadata = configset.name, configset.resource, configset.metadata
+        configsets = configsets_map[resource] ||= {}
+
+        validate_structure!(name, metadata)
+
+        new_metadata = metadata["Metadata"].dup
+        init = new_metadata["AWS::CloudFormation::Init"] # important: adjust data by reference
+
+        if init.key?("configSets")
+          # validate_simple!(name, new_metadata["AWS::CloudFormation::Init"]["configSets"]) # validate original configset for only simple elements
+
+          # 1. expand each config as its own config, flattening to top-level
+          cs = init.delete("configSets") # Only support configSets with simple Array of Strings
+          new_config_set = {}
+          new_config_set[name] = cs["default"].map {|c| "#{padded_i}_#{c}" }
+          init.transform_keys! { |c| "#{padded_i}_#{c}" }
+
+          # Rebuild default configSet, append the new complex ConfigSet structure with each iteration
+          configsets["default"] ||= []
+          configsets["default"] << {"ConfigSet" => name}
+          configsets.merge!(new_config_set) # add each config from #1 to the top-level
+
+          init["configSets"] = configsets # replace new configset
+        else # simple config
+          init["configSets"] = configsets # adjust data by reference
+          configsets["default"] ||= []
+          configsets["default"] << {"ConfigSet" => name}
+
+          # build new config
+          config_key = "#{padded_i}_single_generated"
+          configsets[name] = [config_key]
+          new_config = {config_key => init["config"]}
+          # replace old config with new one
+          init.delete("config") # delete original simple config
+          init.merge!(new_config)
+        end
+
+        metadata_map[resource] ||= {"Metadata" => {}}
+        metadata_map[resource]["Metadata"].deep_merge!(new_metadata)
+        @map[resource] = metadata_map[resource]
+      end
+      @map
+    end
+
+    # Normalized/convert cfn template to mimic the registry format
+    def existing_configsets
+      configsets = []
+      @cfn["Resources"].each do |logical_id, attributes|
+        cloudformation_init = attributes.dig("Metadata", "AWS::CloudFormation::Init")
+        next unless cloudformation_init
+
+        configset = Lono::Configset.new(name: "#{logical_id}OriginalConfigset", resource: logical_id)
+        configset.metadata = {"Metadata" => attributes["Metadata"]} # wrap metadata to create right structure
+        configsets << configset
+      end
+      configsets
+    end
+
+    def validate_structure!(name, metadata)
+      return if metadata.is_a?(Hash) && metadata.dig("Metadata", "AWS::CloudFormation::Init")
+
+      puts "ERROR: The #{name} configset does not appear to have a AWS::CloudFormation::Init key".color(:red)
+      puts "Please double check the #{name} configset.yml structure"
+      exit 1
+    end
+
+    def validate_simple!(name, cs)
+      has_complex_type = cs["default"].detect { |s| !s.is_a?(String) }
+      return unless has_complex_type
+      message =<<~EOL
+        ERROR: The configset #{name} has a configSets property with a complex type.
+        configSets:
+
+            #{cs}
+
+        lono configsets only supports combining configSets with an Array of Strings.
+      EOL
+      puts message.color(:red)
+      exit 1
+    end
+  end
+end

data/lib/lono/builder/configset/definition/base.rb

@@ -0,0 +1,47 @@
+class Lono::Builder::Configset::Definition
+  class Base < Lono::CLI::Base
+    include DslEvaluator
+    include Lono::Builder::Dsl::Syntax
+    include Lono::Utils::Pretty
+    include Context
+
+    # Really only use @path in Configset DSL and ERB evaluation.
+    # However, configsets are built within the CloudFormation template and can use
+    # things instrinic functions like `ref` like would normally have access to.
+    # So configsets need the same context
+    #
+    #     Configset::Definition::Base < Lono::CLI::Base
+    #
+    # for
+    #
+    #     include Lono::Builder::Dsl::Syntax
+    #
+    def initialize(options={})
+      super
+      @meta = @options[:meta]
+      @configset = Lono::Configset.new(@meta)
+      expose_instance_variables
+    end
+
+    # This context is used by the DSL evaluation. Expose variables so user can use them in configset definitions.
+    # Example:
+    #
+    # "/etc/cfn/hooks.d/cfn-auto-reloader.conf":
+    #   content:
+    #     Fn::Sub:
+    # ...
+    #       path=Resources.<%= @resource %>.Metadata.AWS::CloudFormation::Init
+    #
+    def expose_instance_variables
+      @name = @meta[:name]
+      @resource = @meta[:resource]
+    end
+
+    def wrap_with_metadata(cloudformation_init)
+      full = {"Metadata" => {}}
+      full["Metadata"]["AWS::CloudFormation::Init"] = cloudformation_init["AWS::CloudFormation::Init"]
+      full["Metadata"]["AWS::CloudFormation::Authentication"] = authentication if authentication # only on dsl
+      full.deep_stringify_keys.dup # metadata
+    end
+  end
+end

data/lib/lono/builder/configset/definition/context.rb

@@ -0,0 +1,29 @@
+class Lono::Builder::Configset::Definition
+  module Context
+    include DslEvaluator
+
+    def load_context
+      load_vars
+      load_helpers
+    end
+
+    # Docs: https://lono.cloud/docs/configsets/layering/
+    # Layering is simple enough to keep in this method.
+    def load_vars
+      logger.debug "Layers for configset #{@configset.name}:"
+      evaluate_layer("#{@configset.root}/vars.rb") # source defaults
+      evaluate_layer("#{Lono.root}/config/configsets/#{@configset.name}/vars.rb") # source overrides
+      evaluate_layer("#{@blueprint.root}/config/configsets/vars/#{@configset.name}.rb") # blueprint overrides
+      evaluate_layer("#{Lono.root}/config/blueprints/#{@blueprint.name}/configsets/vars/#{@configset.name}.rb") # user overrides
+    end
+
+    def evaluate_layer(path)
+      logger.debug "    #{pretty_path(path)}" if File.exist?(path) || ENV['LONO_SHOW_ALL_LAYERS']
+      evaluate_file(path)
+    end
+
+    def load_helpers
+      load_helper_files("#{@configset.root}/helpers", type: :configset)
+    end
+  end
+end

data/lib/lono/{configset/strategy/helpers/dsl → builder/configset/definition/dsl/syntax}/auth.rb

@@ -1,4 +1,4 @@
-module Lono::Configset::Strategy::Helpers::Dsl
+module Lono::Builder::Configset::Definition::Dsl::Syntax
   module Auth
     def authentication(data=nil, force: false)
       if data.nil?

data/lib/lono/{configset/strategy/helpers/dsl/core.rb → builder/configset/definition/dsl/syntax/content.rb}

@@ -1,7 +1,7 @@
-module Lono::Configset::Strategy::Helpers::Dsl
-  module Core
+module Lono::Builder::Configset::Definition::Dsl::Syntax
+  module Content
     def content_file(path)
-      content_path = "#{@root}/lib/content"
+      content_path = "#{file_root}/content"
       file = "#{content_path}/#{path}"
       if File.exist?(file)
         IO.read(file)
@@ -10,6 +10,14 @@ module Lono::Configset::Strategy::Helpers::Dsl
       end
     end
 
+    def file_root
+      if @configset
+        @configset.root
+      else
+        @blueprint.root
+      end
+    end
+
     def s3_key(name)
       Lono::Configset::S3File::Registry.register(name, blueprint: @blueprint, configset: @configset, root: @root)
       "file://configset/#{@configset}/#{name}"

data/lib/lono/{configset/strategy/helpers/dsl/syntax.rb → builder/configset/definition/dsl/syntax/core.rb}

@@ -1,7 +1,20 @@
-module Lono::Configset::Strategy::Helpers::Dsl
-  module Syntax
+module Lono::Builder::Configset::Definition::Dsl::Syntax
+  module Core
     extend Memoist
 
+    #
+    # The configset method is different with configset registration vs configset definition
+    #
+    #   definition:   app/configsets/httpd/configset.rb
+    #   registration: app/blueprints/demo/configsets.rb
+    #
+    def configset(current)
+      @tracked << current
+      previous, @current = @current, current
+      yield
+      @current = previous
+    end
+
     %w[package group user file service].each do |meth|
       section = meth.pluralize
       define_method(meth) do |k, props={}|
@@ -53,13 +66,6 @@ module Lono::Configset::Strategy::Helpers::Dsl
       current_structure(@current)["sources"].deep_merge!(item)
     end
 
-    def configset(current)
-      @tracked << current
-      previous, @current = @current, current
-      yield
-      @current = previous
-    end
-
   private
     def current_structure(configset)
       @structure[configset] ||= {}

data/lib/lono/{configset/strategy/helpers/dsl → builder/configset/definition/dsl/syntax}/package.rb

@@ -1,4 +1,4 @@
-module Lono::Configset::Strategy::Helpers::Dsl
+module Lono::Builder::Configset::Definition::Dsl::Syntax
   module Package
     # In recent versions of the AmazonLinux2 cfnbootstrap, , the package command doesnt work with updated versions of rubygems.
     # Get this error

data/lib/lono/builder/configset/definition/dsl/syntax.rb

@@ -0,0 +1,5 @@
+class Lono::Builder::Configset::Definition::Dsl
+  module Syntax
+    include_modules "syntax"
+  end
+end