lono 8.0.0.pre.rc1 → 8.0.0.pre.rc4

data/lib/lono/configset.rb

@@ -0,0 +1,17 @@
+module Lono
+  class Configset < Component
+    attr_reader :resource
+    attr_accessor :metadata
+    def initialize(options={})
+      super
+      @resource = options[:resource]
+    end
+
+    def path
+      return unless root
+      exts = %w[rb yml json] # rb highest precedence
+      paths = exts.map { |ext| "#{root}/configset.#{ext}" }
+      paths.find { |p| File.exist?(p) }
+    end
+  end
+end

data/lib/lono/core.rb

@@ -19,11 +19,21 @@ module Lono
     end
     memoize :app
 
+    def role
+      ENV['LONO_ROLE']
+    end
+    memoize :role
+
     def env
       ENV['LONO_ENV'] || 'dev'
     end
     memoize :env
 
+    def extra
+      ENV['LONO_EXTRA']
+    end
+    memoize :extra
+
     def tmp_root
       ENV['LONO_TMP_ROOT'] || "/tmp/lono"
     end
@@ -37,7 +47,7 @@ module Lono
       App.instance.configure(&block)
     end
 
-    # Generally, use the Terraspace.config instead of App.instance.config since it guarantees the load_project_config call
+    # Generally, use the Lono.config instead of App.instance.config since it guarantees the load_project_config call
     def config
       App.instance.load_project_config
       App.instance.config

data/lib/lono/ext/bundler.rb

@@ -0,0 +1,7 @@
+# Bundler 2.0 does yet not have with_unbundled_env
+# Bundler 2.1 deprecates with_clean_env for with_unbundled_env
+
+require "bundler"
+def Bundler.with_unbundled_env(&block)
+  with_clean_env(&block)
+end unless Bundler.respond_to?(:with_unbundled_env)

data/lib/lono/ext/core/module.rb

@@ -0,0 +1,31 @@
+class Module
+  # Include all modules within the relative folder. IE: for dsl/syntax/mod/*
+  #
+  #    include Common
+  #    include Provider
+  #    # etc
+  #
+  # Caller lines are different for OSes:
+  #
+  #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/lono-1.1.1/lib/lono/builder.rb:34:in `build'"
+  #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/lono-1.1.1/lib/lono/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
+  #
+  def include_modules(dir)
+    caller_line = caller[0]
+    parts = caller_line.split(':')
+    calling_file = caller_line.match(/^[a-zA-Z]:/) ? parts[1] : parts[0]
+    parent_dir = File.dirname(calling_file)
+
+    full_dir = "#{parent_dir}/#{dir}"
+    paths = Dir.glob("#{full_dir}/**/*.rb")
+    if paths.empty?
+      raise "Empty include_modules full_dir: #{full_dir}"
+    end
+    paths.each do |path|
+      regexp = Regexp.new(".*/lib/")
+      mod = path.sub(regexp, '').sub('.rb','').camelize
+      c = mod.constantize
+      include c if c.class == Module
+    end
+  end
+end

data/lib/lono/ext/core/object.rb

@@ -0,0 +1,32 @@
+class Object
+  # Load custom helper methods from project
+  def load_helper_files(dir, type: :blueprint)
+    paths = Dir.glob("#{dir}/**/*.rb")
+    paths.sort_by! { |p| p.size } # so namespaces are loaded first
+    paths.each do |path|
+      next unless File.file?(path)
+
+      filename = if type == :project
+                   path.sub(%r{.*/helpers/[a-zA-Z\-_]+/},'').sub('.rb','')
+                 else # blueprint, configset
+                   path.sub(%r{.*/helpers/},'').sub('.rb','')
+                 end
+      module_name = filename.camelize
+
+      # Prepend a period so require works when LONO_ROOT is set to a relative path without a period.
+      #   Example: LONO_ROOT=tmp/lono_project
+      first_char = path[0..0]
+      path = "./#{path}" unless %w[. /].include?(first_char)
+
+      # Examples:
+      #   project:
+      #     path:        app/helpers/custom/custom_helper.rb
+      #     module_name: CustomHelper
+      #   blueprint:
+      #     path:        app/blueprints/demo/helpers/outputs.rb
+      #     module_name: Outputs
+      require path
+      self.class.send :include, module_name.constantize
+    end
+  end
+end

data/lib/lono/ext/core/string.rb

@@ -0,0 +1,9 @@
+module Lono
+  refine String do
+    def camelcase
+      self.underscore.camelize
+    end
+  end
+end
+
+using Lono

data/lib/lono/ext.rb

@@ -0,0 +1,4 @@
+require_relative "ext/bundler"
+require_relative "ext/core/module"
+require_relative "ext/core/object"
+require_relative "ext/core/string"

data/lib/lono/files/base.rb

@@ -0,0 +1,12 @@
+class Lono::Files
+  class Base < Lono::CLI::Base
+    delegate :full_path, :output_path, :zip_name, :zip_path, to: :files
+    attr_reader :files
+    def initialize(options={})
+      super
+      # @files is reference to Lono::Files self instance
+      # IE: Lono::Files instance is the caller
+      @files = options[:files]
+    end
+  end
+end

data/lib/lono/{utils → files/builder/lambda_layer}/rsync.rb

@@ -1,13 +1,11 @@
 require 'shellwords'
 
-module Lono::Utils
+class Lono::Files::Builder::LambdaLayer
   module Rsync
-    include Lono::Utils::Logging
-
     def sh(command)
-      logger.info "=> #{command}"
+      logger.debug "=> #{command}"
       out = `#{command}`
-      logger.info out if ENV['LONO_DEBUG_SH']
+      logger.debug out
       success = $?.success?
       raise("ERROR: running command #{command}").color(:red) unless success
       success

data/lib/lono/{app_file/build → files/builder}/lambda_layer/ruby_packager.rb

@@ -1,61 +1,27 @@
-class Lono::AppFile::Build::LambdaLayer
-  # Based on jets
-  class RubyPackager
-    include Lono::Utils::Rsync
-
-    def initialize(blueprint, registry_item)
-      @blueprint.name, @registry_item = blueprint, registry_item
-
-      @registry_name = @registry_item.name.sub(/\/$/,'')
-      @app_root = "#{@blueprint.root}/app/files/#{@registry_name}"
-    end
+# Based on jets
+class Lono::Files::Builder::LambdaLayer
+  class RubyPackager < Lono::Files::Base
+    include Rsync
 
     def build
       return unless gemfile_exist?
-
+      pretty_path = pretty_path("#{@blueprint.root}/#{@files.path}")
+      logger.debug "Packaging Lambda Layer #{pretty_path}"
+      check_ruby_version!
       bundle_install
       package
     end
 
+    def check_ruby_version!
+      RubyVersion.new(@options).check!
+    end
+
     # We consolidate all gems to opt
     def package
-      setup_bundle_config(output_area)
-      # copy_cache_gems # TODO: might not need this cache
+      setup_bundle_config(compiled_area)
       consolidate_gems_to_opt
     end
 
-    # Also restructure the folder from:
-    #   vendor/gems/ruby/2.5.0
-    # To:
-    #   ruby/gems/2.5.0
-    #
-    # For Lambda Layer structure
-    def consolidate_gems_to_opt
-      src = "#{cache_area}/vendor/gems/ruby/#{ruby_folder}"
-      dest = "#{opt_area}/ruby/gems/#{ruby_folder}"
-      rsync_and_link(src, dest)
-    end
-
-    def rsync_and_link(src, dest)
-      FileUtils.mkdir_p(dest)
-      rsync(src, dest)
-
-      # create symlink in output path not the cache path
-      symlink_dest = "#{output_area}/vendor/gems/ruby/#{ruby_folder}"
-      logger.info "symlink_dest #{symlink_dest}"
-      FileUtils.rm_rf(symlink_dest) # blow away original 2.5.0 folder
-
-      # Create symlink that will point to the gems in the Lambda Layer:
-      #   stage/opt/ruby/gems/2.5.0 -> /opt/ruby/gems/2.5.0
-      FileUtils.mkdir_p(File.dirname(symlink_dest))
-      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", symlink_dest)
-    end
-
-    def ruby_folder
-      major, minor, _ = RUBY_VERSION.split('.')
-      [major, minor, '0'].join('.') # 2.5.1 => 2.5.0
-    end
-
     # Installs gems on the current target system: both compiled and non-compiled.
     # If user is on a macosx machine, macosx gems will be installed.
     # If user is on a linux machine, linux gems will be installed.
@@ -68,9 +34,9 @@ class Lono::AppFile::Build::LambdaLayer
     # project gets built again not all the gems from get installed from the
     # beginning.
     def bundle_install
-      logger.info "Bundling: running bundle install in cache area: #{cache_area}."
+      logger.debug "Bundling #{cache_area}"
 
-      rsync(output_area, cache_area)
+      rsync(compiled_area, cache_area)
 
       # Uncomment out to always remove the cache/vendor/gems to debug
       # FileUtils.rm_rf("#{cache_area}/vendor/gems")
@@ -85,15 +51,101 @@ class Lono::AppFile::Build::LambdaLayer
       FileUtils.rm_rf("#{cache_area}/.bundle")
       require "bundler" # dynamically require bundler so user can use any bundler
       setup_bundle_config(cache_area)
-      ::Bundler.with_unbundled_env do
+      Bundler.with_unbundled_env do
         sh "cd #{cache_area} && env bundle install"
       end
 
       remove_bundled_with("#{cache_area}/Gemfile.lock")
       # Fixes really tricky bug where Gemfile and Gemfile.lock is out-of-sync. Details: https://gist.github.com/tongueroo/b5b0d0c924a4a1633eee514795e4b04b
-      FileUtils.cp("#{cache_area}/Gemfile.lock", "#{Lono.root}/output/#{@blueprint.name}/files/#{@registry_name}/Gemfile.lock")
+      FileUtils.cp("#{cache_area}/Gemfile.lock", compiled_area)
+
+      logger.debug 'Bundle install success.'
+    end
+
+    def setup_bundle_config(dir)
+      text =<<~EOL
+        ---
+        BUNDLE_PATH: "vendor/gems"
+        BUNDLE_WITHOUT: "development:test"
+      EOL
+      bundle_config = "#{dir}/.bundle/config"
+      FileUtils.mkdir_p(File.dirname(bundle_config))
+      IO.write(bundle_config, text)
+    end
+
+    # Examples:
+    #
+    #   output/BLUEPRINT/lambda_layers/PATH/opt
+    #   output/demo/lambda_layers/files/lambda-function/opt
+    #   output/demo/lambda_layers/files/lambda-function # drop /opt to get dirname
+    #
+    # The build_parent is the parent folder because it can include cache and opt. Example tree:
+    #
+    #     output
+    #     └── demo
+    #         └── lambda_layers
+    #             └── files
+    #                 └── lambda-function  # <= build_area
+    #                     ├── cache
+    #                     └── opt
+    #
+    def build_area
+      File.dirname(@files.output_path)
+    end
+
+    def cache_area
+      "#{build_area}/cache"
+    end
+
+    # Final artifact that will be zipped up
+    def opt_area
+      "#{build_area}/opt"
+    end
 
-      logger.info 'Bundle install success.'
+    # Original source is compiled to output folder
+    #
+    #     app/blueprints/demo/files/lambda-function/ - original
+    #     output/demo/files/lambda-function/         - compiled
+    #
+    # Compiling is done beforehand to support .tt ERB in the files.
+    #
+    # Then it's moved again to a cache area to run `bundle`
+    # Then it's moved yet agaoin to an opt area to produce the final Lambda Layer artifact.
+    #
+    def compiled_area
+      @files.compiled_path
+    end
+
+    # Also restructure the folder from:
+    #   vendor/gems/ruby/3.0.0
+    # To:
+    #   ruby/gems/3.0.0
+    #
+    # For Lambda Layer structure
+    def consolidate_gems_to_opt
+      src = "#{cache_area}/vendor/gems/ruby/#{ruby_folder}"
+      dest = "#{opt_area}/ruby/gems/#{ruby_folder}"
+      rsync_and_link(src, dest)
+    end
+
+    def rsync_and_link(src, dest)
+      FileUtils.mkdir_p(dest)
+      rsync(src, dest)
+
+      # create symlink in output path not the cache path
+      symlink_dest = "#{compiled_area}/vendor/gems/ruby/#{ruby_folder}"
+      logger.debug "symlink_dest #{symlink_dest}"
+      FileUtils.rm_rf(symlink_dest) # blow away original 3.0.0 folder
+
+      # Create symlink that will point to the gems in the Lambda Layer:
+      #   stage/opt/ruby/gems/3.0.0 -> /opt/ruby/gems/3.0.0
+      FileUtils.mkdir_p(File.dirname(symlink_dest))
+      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", symlink_dest)
+    end
+
+    def ruby_folder
+      major, minor, _ = RUBY_VERSION.split('.')
+      [major, minor, '0'].join('.') # 3.0.1 => 3.0.0
     end
 
     # Remove the BUNDLED WITH line since we don't control the bundler gem version on AWS Lambda
@@ -118,35 +170,8 @@ class Lono::AppFile::Build::LambdaLayer
       IO.write(gemfile_lock, content)
     end
 
-    def setup_bundle_config(dir)
-      text =<<-EOL
----
-BUNDLE_PATH: "vendor/gems"
-BUNDLE_WITHOUT: "development:test"
-EOL
-      bundle_config = "#{dir}/.bundle/config"
-      FileUtils.mkdir_p(File.dirname(bundle_config))
-      IO.write(bundle_config, text)
-    end
-
-    def output_area
-      "#{Lono.root}/output/#{@blueprint.name}/files/#{@registry_name}"
-    end
-
-    def build_area
-      "#{Lono.root}/output/#{@blueprint.name}/lambda_layers/#{@registry_name}"
-    end
-
-    def cache_area
-      "#{build_area}/cache"
-    end
-
-    def opt_area
-      "#{build_area}/opt"
-    end
-
     def gemfile_exist?
-      gemfile_path = "#{@app_root}/Gemfile"
+      gemfile_path = "#{@files.full_path}/Gemfile"
       File.exist?(gemfile_path)
     end
   end

data/lib/lono/files/builder/lambda_layer/ruby_version.rb

@@ -0,0 +1,38 @@
+class Lono::Files::Builder::LambdaLayer
+  class RubyVersion < Lono::Files::Base
+    def initialize(options={})
+      # cfn passed from Finalizer::Files::Build#build_files to
+      # Lono::Files#build_lambda_layer(@cfn)
+      # This because Lono::Files at initialization only has registration info
+      # The build info is passed to the build_lambda_layer method
+      @cfn = options[:cfn]
+      super
+    end
+
+    def check!
+      # IE: logical_id = LayerVersion
+      # attrs = attrs with attrs['Properties']
+      logical_id, attrs = @cfn['Resources'].find do |logical_id, attrs|
+        attrs.dig('Properties', 'Content', 'S3Key') == @files.marker
+      end
+      props = attrs['Properties']
+      major, minor, _ = RUBY_VERSION.split('.')
+      current_ruby = "ruby#{major}.#{minor}"
+      ok = props['CompatibleRuntimes'].include?(current_ruby)
+      return if ok
+
+      runtimes = props['CompatibleRuntimes'].map { |s| s.sub('ruby','') }.join(', ')
+      logger.error "ERROR: Current Ruby Version does not match the Lambda Layer Ruby Version".color(:red)
+      logger.error <<~EOL
+        Current Ruby: #{RUBY_VERSION}
+        Lambda Layer Ruby Version: #{runtimes}
+
+        Lono is unable to package up a Lambda Layer
+        unless you use the same Ruby version on this machine.
+      EOL
+      logger.error "Resource: #{logical_id}"
+      logger.error YAML.dump(attrs)
+      exit 1
+    end
+  end
+end

data/lib/lono/files/builder/lambda_layer.rb

@@ -0,0 +1,17 @@
+class Lono::Files::Builder
+  class LambdaLayer < Lono::Files::Base
+    def run
+      layer = @options[:layer]
+      return unless layer # triggers Lambda Layer building
+      unless layer =~ /ruby/
+        logger.warn "WARN: Currently only support Ruby lambda layers automatically".color(:yellow)
+        return
+      end
+
+      klass_name = "Lono::Files::Builder::LambdaLayer::#{layer.camelize}Packager"
+      klass = klass_name.constantize
+      packager = klass.new(@options)
+      packager.build
+    end
+  end
+end

data/lib/lono/files/builder.rb

@@ -0,0 +1,37 @@
+class Lono::Files
+  class Builder < Base
+    extend Memoist
+    include Lono::Builder::Context
+    include Lono::Builder::Dsl::Syntax
+    # Tricky must define files again to avoid conflict with files from loading the Dsl::Syntax
+    attr_reader :files
+
+    def run
+      load_context
+      if File.directory?(full_path)
+        directory(@files)
+      else
+        template(@files)
+      end
+    end
+
+    def template(files)
+      sequence.send(:set_template_paths, @blueprint.root)
+      src = files.path
+      dest = files.output_path
+      sequence.template(src, dest)
+    end
+
+    def directory(files)
+      src = files.full_path
+      sequence.send(:set_template_paths, src)
+      sequence.destination_root = @files.output_path
+      sequence.directory(".", verbose: false, force: true, context: binding) # Thor::Action
+    end
+
+    def sequence
+      Lono::CLI::New::Sequence.new(@options)
+    end
+    memoize :sequence
+  end
+end

data/lib/lono/files/compressor.rb

@@ -0,0 +1,53 @@
+require 'fileutils'
+
+class Lono::Files
+  class Compressor < Base
+    def run
+      if File.directory?(output_path)
+        zip_directory
+      else
+        zip_file
+      end
+    end
+
+    def zip_file
+      logger.debug "Zipping file to #{pretty_path(zip_path)}"
+      # create zipfile at same level of file
+      command = "cd #{File.dirname(output_path)} && zip -q #{zip_name} #{File.basename(output_path)}"
+      execute_zip(command)
+    end
+
+    def zip_directory
+      logger.debug "Zipping folder to #{zip_path}"
+      command = "cd #{output_path} && zip --symlinks -rq #{zip_name} ." # create zipfile witih directory
+      execute_zip(command)
+      FileUtils.mv("#{output_path}/#{zip_name}", "#{File.dirname(output_path)}/#{zip_name}") # move zip back to the parent directory
+    end
+
+  private
+    def execute_zip(command)
+      check_zip_installed!
+      logger.debug "=> #{command}".color(:green)
+      `#{command}`
+      return if $?.success?
+      logger.info "ERROR: Fail trying to zip files".color(:red)
+      logger.info "Command: #{command}".color(:red)
+      exit 1
+    end
+
+    @@zip_installed = false
+    def check_zip_installed!
+      @@zip_installed = system("type zip > /dev/null 2>&1")
+      if @@zip_installed
+        @@zip_installed = true
+        return
+      end
+      logger.error "ERROR: The command 'zip' is not installed.".color(:red)
+      logger.error <<~EOL
+        The file helper requires the zip command.
+        Please install the zip command on this system.
+      EOL
+      exit 1
+    end
+  end
+end

data/lib/lono/files/concerns/post_processing.rb

@@ -0,0 +1,52 @@
+module Lono::Files::Concerns
+  module PostProcessing
+    extend Memoist
+
+    def build
+      Lono::Files::Builder.new(@options.merge(files: self)).run
+    end
+
+    def build_lambda_layer(cfn)
+      Lono::Files::Builder::LambdaLayer.new(@options.merge(files: self, cfn: cfn)).run
+    end
+
+    def compress
+      Lono::Files::Compressor.new(@options.merge(files: self)).run
+    end
+
+    def upload
+      uploader.upload # Lono::S3::Uploader#upload
+    end
+
+    delegate :s3_path, :s3_key, to: :uploader
+    def uploader
+      Lono::S3::Uploader.new(zip_path)
+    end
+    memoize :uploader
+
+    def zip_name
+      name = File.basename(full_path)
+      name << "-layer" if layer
+      "#{name}-#{Lono::Md5.sum(full_path)}.zip"
+    end
+
+    def zip_path
+      "#{File.dirname(output_path)}/#{zip_name}"
+    end
+
+    def output_path
+      if layer
+        # For Lambda Layer the final output path is the opt folder, where final artifact is zipped
+        "#{Lono.root}/output/#{@blueprint.name}/layer/#{path}/opt" # Note: layer and opt
+      else
+        # For normal files the final output path in the compiled path
+        compiled_path
+      end
+    end
+
+    # Also used by LambdaLayer::RubyPackager#compiled_area
+    def compiled_path
+      "#{Lono.root}/output/#{@blueprint.name}/normal/#{path}"
+    end
+  end
+end

data/lib/lono/files/concerns/registration.rb

@@ -0,0 +1,16 @@
+module Lono::Files::Concerns
+  module Registration
+    def call_line
+      @caller.find do |line|
+        line.include?(Lono.root.to_s)
+      end
+    end
+
+    # IE:
+    #   LONO:://files/function-normal
+    #   LONO:://files/function-layer
+    def marker
+      "LONO://#{@path}-#{@type}"
+    end
+  end
+end

data/lib/lono/files/registry.rb

@@ -0,0 +1,6 @@
+class Lono::Files
+  class Registry
+    class_attribute :files
+    self.files = []
+  end
+end

data/lib/lono/files.rb

@@ -0,0 +1,41 @@
+# Class name is Lono::Files because can represent a single file or a directory.
+# Also, it avoids conflicting with the stdlib File class name.
+# Files is like a "FileList". Using shorter Files name.
+module Lono
+  class Files < Lono::CLI::Base
+    include Concerns::Registration
+    include Concerns::PostProcessing
+
+    attr_reader :path, :layer, :type
+    def initialize(options={})
+      super
+      @path = options[:path]
+      @layer = options[:layer]
+      @type = @layer ? "layer" : "normal"
+      @caller = options[:caller] # original caller stack at registration time
+    end
+
+    def full_path
+      "#{@blueprint.root}/#{@path}"
+    end
+
+    class << self
+      def register(options={})
+        path = options[:path]
+        layer = options[:layer]
+        # Registration uniquess based on both path and layer option
+        file = files.find { |f| f.path == path && f.layer == layer }
+        unless file
+          file = Files.new(options.merge(caller: caller))
+          files << file
+        end
+        file.marker
+      end
+
+      delegate :empty?, :files, to: :files
+      def files
+        Registry.files
+      end
+    end
+  end
+end