lono 7.5.2 → 8.0.0.pre.rc3

data/lib/lono/{configset/strategy → builder/configset/definition}/dsl.rb

@@ -1,29 +1,25 @@
-module Lono::Configset::Strategy
+class Lono::Builder::Configset::Definition
   class Dsl < Base
-    # Configset helpers take higher precedence than Lono DSL Helpers. IE: s3_key is overwritten but s3_bucket is the same
-    include Helpers::Dsl
+    include Syntax
 
     def initialize(options={})
       super
-      @structure = {} # holds in memory the configset hash structure to build AWS::CloudFormation::Init
       @current = "main" # current configset
-      @command_counts = Hash.new(0)
       @tracked = []
+      @structure = {} # holds in memory the configset hash structure to build AWS::CloudFormation::Init
+      @command_counts = Hash.new(0)
       # Also support ability to add AWS::CloudFormation::Authentication
       @authentication = nil # holds IAM policy info to build AWS::CloudFormation::Authentication
     end
 
-    def find_evaluation_path
-      "#{@root}/lib/configset.rb"
-    end
-
-    def load
-      evaluate_file(@evaluation_path)
-      configsets = @configsets || @tracked.uniq
+    def evaluate
+      load_context
+      evaluate_file(@configset.path)
+      configsets = configsets || @tracked.uniq
       configsets = ["main"] if configsets.empty?
       configsets_structure = {"configSets" => {"default" => configsets}}.merge(@structure)
-      full = { "AWS::CloudFormation::Init" => configsets_structure }
-      full.deep_stringify_keys!
+      cloudformation_init = { "AWS::CloudFormation::Init" => configsets_structure }
+      wrap_with_metadata(cloudformation_init)
     end
   end
 end

data/lib/lono/{configset/strategy → builder/configset/definition}/erb.rb

@@ -1,29 +1,22 @@
-require "json"
-
-module Lono::Configset::Strategy
+class Lono::Builder::Configset::Definition
   class Erb < Base
-    include Helpers::Erb
-
-    def find_evaluation_path
-      paths = %w[configset.yml configset.json].map { |p| "#{@root}/lib/#{p}" }
-      paths.find { |p| File.exist?(p) }
+    def evaluate
+      load_context
+      cloudformation_init = build
+      wrap_with_metadata(cloudformation_init)
     end
 
-    def load
-      content = RenderMePretty.result(@evaluation_path, context: self)
-      # init structure
-      if File.extname(@evaluation_path) == ".yml"
+    def build
+      content = RenderMePretty.result(@configset.path, context: self)
+      if File.extname(@configset.path) == ".yml"
         load_yaml(content)
       else
         JSON.load(content)
       end
     end
 
-    def authentication
-      # noop
-    end
-
     def load_yaml(content)
+      # init structure
       # Write to file so can use Yamler::Validator
       path = "/tmp/lono/configset.yml"
       FileUtils.mkdir_p(File.dirname(path))
@@ -31,5 +24,9 @@ module Lono::Configset::Strategy
       Lono::Yamler::Validator.new(path).validate!
       Lono::Yamler::Loader.new(content).load
     end
+
+    def authentication
+      # noop
+    end
   end
 end

data/lib/lono/builder/configset/definition.rb

@@ -0,0 +1,18 @@
+module Lono::Builder::Configset
+  class Definition < Lono::CLI::Base
+    attr_reader :configset
+    def initialize(options={})
+      super
+      @meta = options[:meta]
+      @configset = Lono::Configset.new(@meta)
+    end
+
+    def evaluate
+      strategy_class = configset.path.include?('.rb') ? Dsl : Erb
+      strategy = strategy_class.new(@options.merge(path: configset.path))
+      metadata = strategy.evaluate
+      @configset.metadata = metadata
+      @configset
+    end
+  end
+end

data/lib/lono/builder/configset/evaluator.rb

@@ -0,0 +1,10 @@
+# Dsl or Erb
+module Lono::Builder::Configset
+  class Evaluator < Lono::CLI::Base
+    def evaluate
+      Registration.new(@blueprint).evaluate
+      combiner = Combiner.new(@options.merge(metas: Registration.metas))
+      combiner.combine
+    end
+  end
+end

data/lib/lono/builder/configset/registration.rb

@@ -0,0 +1,35 @@
+module Lono::Builder::Configset
+  class Registration
+    include DslEvaluator
+
+    def initialize(blueprint)
+      @blueprint = blueprint
+    end
+
+    cattr_accessor :metas, default: []
+    def metas
+      self.class.metas
+    end
+
+    def evaluate
+      path = "#{@blueprint.root}/configsets.rb" # plural
+      evaluate_file(path)
+    end
+
+    # Only one syntax method so not in separate module.
+    #
+    # Register configset for later processing.
+    # By registering them all up front, can aggregate errors and show them together
+    # for a user-friendly experience.
+    #
+    # The configset method is different with configset registration vs configset definition
+    #
+    #   definition:   app/configsets/httpd/configset.rb
+    #   registration: app/blueprints/demo/configsets.rb
+    #
+    def configset(name, options={})
+      found = metas.detect { |i| i[:name] == name && i[:resource] == options[:resource] }
+      metas << options.merge(name: name) unless found
+    end
+  end
+end

data/lib/lono/builder/context.rb

@@ -0,0 +1,40 @@
+module Lono::Builder
+  module Context
+    include DslEvaluator
+
+    def load_context
+      load_variables
+      load_helpers
+    end
+
+    # Variables in base.rb are overridden by their environment specific variables
+    # file.  Example, file LONO_ENV=dev:
+    #
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #   config/vars/base.rb
+    #   config/vars/dev.rb - will override any variables in base.rb
+    #
+    def load_variables
+      return if seed?
+      layers = Lono::Layering::Layer.new(@blueprint, "vars").paths
+      layers.each do |layer|
+        evaluate_file(layer)
+      end
+    end
+
+    # Load blueprint helpers
+    # blueprint helpers override extension helpers
+    def load_helpers
+      load_helper_files("#{Lono.root}/vendor/helpers",   type: :project)
+      load_helper_files("#{Lono.root}/app/helpers",   type: :project)
+      load_helper_files("#{@blueprint.root}/helpers", type: :blueprint) # takes higher precedence
+    end
+
+    # Dont want any existing files to prevent building the blueprint.
+    # This means that parameters cannot be based on vars. It's a trade-off.
+    def seed?
+      ARGV[0] == "seed"
+    end
+  end
+end

data/lib/lono/builder/dsl/evaluator.rb

@@ -0,0 +1,45 @@
+class Lono::Builder::Dsl
+  class Evaluator < Lono::CLI::Base
+    include DslEvaluator
+    include Lono::Builder::Context
+    include Lono::Builder::Dsl::Syntax
+    include Lono::Utils::Pretty
+
+    def initialize(options={})
+      super
+      @template_path = "#{@blueprint.root}/template.rb"
+      @parameters = [] # built by parameter_groups.rb
+      @cfn = {}
+    end
+
+    def evaluate
+      load_context
+      evaluate_template_paths(@template_path) # modifies @cfn
+      finalize
+      to_yaml
+      @cfn
+    end
+
+    def finalize
+      o = @options.merge(parameters: @parameters)
+      @cfn = Finalizer.new(@cfn, o).run
+    end
+
+    def to_yaml
+      # https://stackoverflow.com/questions/24508364/how-to-emit-yaml-in-ruby-expanding-aliases
+      # Trick to prevent YAML from emitting aliases
+      @cfn = YAML.load(@cfn.to_json)
+    end
+
+    # Example path: /full/path/to/project/app/blueprints/demo/template.rb
+    def evaluate_template_paths(path)
+      ext = File.extname(path)
+      folder = path.sub(ext, '')
+      expr = "#{folder}/**/*.rb"
+      evaluate_file(path) # process top-level template.rb first
+      Dir.glob(expr).each do |path|
+        evaluate_file(path)
+      end
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/base.rb

@@ -0,0 +1,8 @@
+class Lono::Builder::Dsl::Finalizer
+  class Base < Lono::CLI::Base
+    def initialize(options={})
+      super
+      @cfn = options[:cfn]
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/configsets.rb

@@ -0,0 +1,20 @@
+class Lono::Builder::Dsl::Finalizer
+  class Configsets < Base
+    # Replaces metadata under each logical id resource.
+    def run
+      dsl = Lono::Builder::Configset::Evaluator.new(@options.merge(cfn: @cfn))
+      metadata_map = dsl.evaluate
+      metadata_map.each do |logical_id, cs|
+        resource = @cfn["Resources"][logical_id]
+        unless resource
+          puts "WARN: Resources.#{logical_id} not found in the template. Are you sure you specified the correct resource logical id in your configsets.rb?".color(:yellow)
+          next
+        end
+
+        resource["Metadata"] = cs["Metadata"]
+      end
+
+      @cfn
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/files/base.rb

@@ -0,0 +1,4 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Base < Lono::Builder::Dsl::Finalizer::Base
+  end
+end

data/lib/lono/builder/dsl/finalizer/files/build.rb

@@ -0,0 +1,58 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Build < Base
+    def initialize(options={})
+      super
+      @output_path = "#{Lono.root}/.output/files"
+    end
+
+    def run
+      return if Lono::Files.empty?
+      logger.debug "Building files"
+      clean
+      validate!
+      build_files
+      # build_layers
+    end
+
+    def build_files
+      Lono::Files.files.each do |file| # using singular file, but is like a "file_list"
+        file.build
+        file.compress
+        # Note: Uploading files happen right before create_stack or execute_change_set
+        # after user confirms action, instead of part of the build process
+      end
+    end
+
+    # TODO: LambdaLayer support
+    # def build_layers
+    #   layer_items = Registry.layers
+    #   layer_items.each do |item|
+    #     LambdaLayer.new(@blueprint, item).build
+    #   end
+    # end
+    #
+
+    def clean
+      FileUtils.rm_rf(@output_path)
+    end
+
+    def validate!
+      missing = Lono::Files.files.select do |file|
+        !File.exist?(file.full_path)
+      end
+      return if missing.empty?
+
+      logger.info "ERROR: These files are missing".color(:red)
+      logger.info <<~EOL
+        The file helper is calling them.
+        They need to exist to be uploaded to s3.
+      EOL
+      missing.each do |file|
+        logger.info "    #{pretty_path(file.full_path)}"
+        logger.info "    Called at #{file.call_line}"
+      end
+      logger.info "Please double check that they exist."
+      exit 1
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/files/replace.rb

@@ -0,0 +1,31 @@
+class Lono::Builder::Dsl::Finalizer::Files
+  class Replace < Base
+    def run
+      replacements.each do |placeholder, replacement|
+        update_template!(@cfn)
+      end
+      @cfn
+    end
+
+    def replacements
+      Lono::Files.files.inject({}) do |result, file|
+        result.merge(file.marker => file.s3_key)
+      end
+    end
+
+    def update_template!(hash)
+      hash.each do |k, v|
+        if v.is_a?(String)
+          if v =~ %r{^LONO://}
+            v.replace(replacements[v]) # replace the placeholder
+          end
+        elsif v.is_a?(Hash)
+          update_template!(v) # recurse
+        elsif v.is_a?(Array)
+          v.each { |x| update_template!(x) if x.is_a?(Hash) }
+        end
+      end
+      hash
+    end
+  end
+end

data/lib/lono/builder/dsl/finalizer/files.rb

@@ -0,0 +1,9 @@
+class Lono::Builder::Dsl::Finalizer
+  class Files < Base
+    # Replaces metadata under each logical id resource.
+    def run
+      Build.new(@options).run
+      @cfn = Replace.new(@options).run
+    end
+  end
+end

data/lib/lono/{template/strategy → builder}/dsl/finalizer/parameter_groups.rb

@@ -1,10 +1,11 @@
-class Lono::Template::Strategy::Dsl::Finalizer
-  class ParameterGroups
+class Lono::Builder::Dsl::Finalizer
+  class ParameterGroups < Base
     extend Memoist
-    include Lono::Template::Strategy::Dsl::Builder::Stringify
+    include Lono::Builder::Util::Stringify
 
-    def initialize(cfn, parameters)
-      @cfn, @parameters = cfn, parameters
+    def initialize(options={})
+      super
+      @parameters = options[:parameters]
     end
 
     def run

data/lib/lono/builder/dsl/finalizer.rb

@@ -0,0 +1,15 @@
+class Lono::Builder::Dsl
+  class Finalizer
+    def initialize(cfn, options={})
+      @cfn, @options = cfn, options
+    end
+
+    def run
+      o = @options.merge(cfn: @cfn)
+      @cfn = ParameterGroups.new(o).run
+      @cfn = Configsets.new(o).run
+      @cfn = Files.new(o).run
+      @cfn
+    end
+  end
+end

data/lib/lono/{template/strategy/dsl/builder/helpers/ec2_helper.rb → builder/dsl/helpers/ec2.rb}

@@ -1,5 +1,5 @@
-module Lono::Template::Strategy::Dsl::Builder::Helpers
-  module Ec2Helper
+module Lono::Builder::Dsl::Helpers
+  module Ec2
     extend Memoist
 
     def default_vpc
@@ -20,7 +20,7 @@ module Lono::Template::Strategy::Dsl::Builder::Helpers
     def key_pairs(regexp=nil)
       resp = ec2.describe_key_pairs
       key_names = resp.key_pairs.map(&:key_name)
-      key_names.select! { |k| k =~ regexp }
+      key_names.select! { |k| k =~ regexp } if regexp
       key_names
     end
   end

data/lib/lono/builder/dsl/helpers/files.rb

@@ -0,0 +1,7 @@
+module Lono::Builder::Dsl::Helpers
+  module Files
+    def files(path)
+      Lono::Files.register(@options.merge(path: path))
+    end
+  end
+end

data/lib/lono/builder/dsl/helpers/partials.rb

@@ -0,0 +1,140 @@
+require "aws-sdk-core"
+
+module Lono::Builder::Dsl::Helpers
+  module Partials
+    def template_s3_path(template_name)
+      # hi-jacking Uploader for https_url
+      template_path = "output/#{@blueprint.name}/templates/#{template_name}.yml"
+      Lono::S3::Uploader.new(template_path).s3_https_url
+    end
+
+    def template_params(param_name)
+      o = @options.dup
+      o["param"] = param_name
+      generator = Lono::CLI::Param::Generator.new(o)
+      # do not generate because lono cfn calling logic already generated it we only need the values
+      parameters = generator.parameters    # Returns Array in underscore keys format
+      # convert Array to simplified hash structure
+      parameters.inject({}) do |h, param|
+        h.merge(param[:parameter_key] => param[:parameter_value])
+      end
+    end
+
+    # The partial's path is a relative path.
+    #
+    # Example:
+    # Given file in app/partials/iam/docker.yml
+    #
+    #   <%= partial("iam/docker", {}, indent: 10) %>
+    #   <%= partial("iam/docker.yml", {}, indent: 10) %>
+    #
+    # If the user specifies the extension then use that instead of auto-adding
+    # the detected format.
+    def partial(path,vars={}, options={})
+      path = partial_path_for(path)
+      path = auto_add_format(path)
+
+      instance_variables!(vars)
+      result = render_path(path)
+
+      result = indent(result, options[:indent]) if options[:indent]
+      result + "\n"
+    end
+
+    # Take a hash and makes them instance variables in the current scope.
+    # Use this in custom helper methods to make variables accessible to ERB templates.
+    def instance_variables!(variables)
+      variables.each do |key, value|
+        instance_variable_set('@' + key.to_s, value)
+      end
+    end
+
+    # add indentation
+    def indent(text, indentation_amount)
+      text.split("\n").map do |line|
+        " " * indentation_amount + line
+      end.join("\n")
+    end
+
+    def partial_exist?(path)
+      path = partial_path_for(path)
+      path = auto_add_format(path)
+      path && File.exist?(path)
+    end
+
+    def region
+      AwsData.region
+    end
+    alias_method :current_region, :region
+
+  private
+    def auto_add_format(path)
+      # Return immediately if user provided explicit extension
+      extension = File.extname(path) # current extension
+      return path if !extension.empty?
+
+      # Else let's auto detect
+      paths = Dir.glob("#{path}.*")
+
+      if paths.size == 1 # non-ambiguous match
+        return paths.first
+      end
+
+      if paths.size > 1 # ambiguous match
+        logger.info "ERROR: Multiple possible partials found:".color(:red)
+        paths.each do |path|
+          logger.info "  #{path}"
+        end
+        logger.info "Please specify an extension in the name to remove the ambiguity.".color(:green)
+        exit 1
+      end
+
+      # Account for case when user wants to include a file with no extension at all
+      return path if File.exist?(path) && !File.directory?(path)
+
+      path # original path if this point is reached
+    end
+
+    # Bash code that is meant to included in user-data
+    def extract_scripts(options={})
+      # defaults also here in case they are removed from settings
+      path = options[:path]
+      to = options[:to] || "/opt"
+      user = options[:as] || "ec2-user"
+
+      if Dir.glob("#{@bluepint.root}/#{path}*").empty?
+        logger.info "WARN: you are using the extract_scripts helper method but you do not have any scripts.".color(:yellow)
+        calling_line = caller[0].split(':')[0..1].join(':')
+        logger.info "Called from1: #{calling_line}"
+        return ""
+      end
+
+      <<~BASH_CODE
+        # Generated from the lono extract_scripts helper.
+        # Downloads scripts from s3, extract them, and setup.
+        mkdir -p #{to}
+        aws s3 cp #{scripts_s3_path} #{to}/
+        (
+          cd #{to}
+          tar zxf #{to}/#{scripts_name}
+          chown -R #{user}:#{user} #{to}/scripts
+        )
+      BASH_CODE
+    end
+
+    def scripts_name
+      File.basename(scripts_s3_path)
+    end
+
+    def scripts_s3_path
+      upload = Lono::Script::Upload.new(@options)
+      upload.s3_dest
+    end
+
+    def indent(text, indentation_amount)
+      text.split("\n").map do |line|
+        " " * indentation_amount + line
+      end.join("\n")
+    end
+  end
+end

data/lib/lono/{template/strategy/dsl/builder/helpers/s3_helper.rb → builder/dsl/helpers/s3.rb}

@@ -1,14 +1,14 @@
-module Lono::Template::Strategy::Dsl::Builder::Helpers
-  module S3Helper
+module Lono::Builder::Dsl::Helpers
+  module S3
     def s3_bucket
       Lono::S3::Bucket.name
     end
     alias_method :lono_bucket_name, :s3_bucket
+    alias_method :files_bucket, :s3_bucket
 
     def s3_key(name, options={})
       default = {type: "file"}
       options.reverse_merge!(default)
-      Lono::AppFile::Registry.register(name, @blueprint, options)
       "file://app/files/#{options[:type]}/#{name}" # placeholder for post processing
     end
     alias_method :file_s3_key, :s3_key

data/lib/lono/{template/context/ssm_fetcher.rb → builder/dsl/helpers/ssm/fetcher.rb}

@@ -1,8 +1,9 @@
 require 'aws-sdk-ssm'
 
-class Lono::Template::Context
-  class SsmFetcher
+module Lono::Builder::Dsl::Helpers::Ssm
+  class Fetcher
     extend Memoist
+    include Lono::Utils::Logging
 
     def get(name)
       fetch_ssm_value(name)
@@ -12,7 +13,8 @@ class Lono::Template::Context
       resp = ssm.get_parameter(name: name, with_decryption: true)
       resp.parameter.value
     rescue Aws::SSM::Errors::ParameterNotFound
-      'SSM-PARAM-NOT-FOUND'
+      logger.warn 'WARN: SSM-PARAM-NOT-FOUND'
+      nil
     end
 
     def ssm

data/lib/lono/{template/context/helpers.rb → builder/dsl/helpers/ssm.rb}

@@ -1,5 +1,5 @@
-class Lono::Template::Context
-  module Helpers
+module Lono::Builder::Dsl::Helpers
+  module Ssm
     extend Memoist
 
     def ssm(name)
@@ -7,7 +7,7 @@ class Lono::Template::Context
     end
 
     def ssm_fetcher
-      SsmFetcher.new
+      Fetcher.new
     end
     memoize :ssm_fetcher
   end