lono 8.0.0.pre.rc3 → 8.0.0.pre.rc4

data/lib/lono/files/builder/lambda_layer/ruby_packager.rb

@@ -0,0 +1,178 @@
+# Based on jets
+class Lono::Files::Builder::LambdaLayer
+  class RubyPackager < Lono::Files::Base
+    include Rsync
+
+    def build
+      return unless gemfile_exist?
+      pretty_path = pretty_path("#{@blueprint.root}/#{@files.path}")
+      logger.debug "Packaging Lambda Layer #{pretty_path}"
+      check_ruby_version!
+      bundle_install
+      package
+    end
+
+    def check_ruby_version!
+      RubyVersion.new(@options).check!
+    end
+
+    # We consolidate all gems to opt
+    def package
+      setup_bundle_config(compiled_area)
+      consolidate_gems_to_opt
+    end
+
+    # Installs gems on the current target system: both compiled and non-compiled.
+    # If user is on a macosx machine, macosx gems will be installed.
+    # If user is on a linux machine, linux gems will be installed.
+    #
+    # Copies Gemfile* to /tmp/jets/demo/cache folder and installs
+    # gems with bundle install from there.
+    #
+    # We take the time to copy Gemfile and bundle into a separate directory
+    # because it gets left around to act as a 'cache'.  So, when the builds the
+    # project gets built again not all the gems from get installed from the
+    # beginning.
+    def bundle_install
+      logger.debug "Bundling #{cache_area}"
+
+      rsync(compiled_area, cache_area)
+
+      # Uncomment out to always remove the cache/vendor/gems to debug
+      # FileUtils.rm_rf("#{cache_area}/vendor/gems")
+
+      # Remove .bundle folder so .bundle/config doesnt affect how gems are packages
+      # Not using BUNDLE_IGNORE_CONFIG=1 to allow home ~/.bundle/config to affect bundling though.
+      # This is useful if you have private gems sources that require authentication. Example:
+      #
+      #    bundle config gems.myprivatesource.com user:pass
+      #
+
+      FileUtils.rm_rf("#{cache_area}/.bundle")
+      require "bundler" # dynamically require bundler so user can use any bundler
+      setup_bundle_config(cache_area)
+      Bundler.with_unbundled_env do
+        sh "cd #{cache_area} && env bundle install"
+      end
+
+      remove_bundled_with("#{cache_area}/Gemfile.lock")
+      # Fixes really tricky bug where Gemfile and Gemfile.lock is out-of-sync. Details: https://gist.github.com/tongueroo/b5b0d0c924a4a1633eee514795e4b04b
+      FileUtils.cp("#{cache_area}/Gemfile.lock", compiled_area)
+
+      logger.debug 'Bundle install success.'
+    end
+
+    def setup_bundle_config(dir)
+      text =<<~EOL
+        ---
+        BUNDLE_PATH: "vendor/gems"
+        BUNDLE_WITHOUT: "development:test"
+      EOL
+      bundle_config = "#{dir}/.bundle/config"
+      FileUtils.mkdir_p(File.dirname(bundle_config))
+      IO.write(bundle_config, text)
+    end
+
+    # Examples:
+    #
+    #   output/BLUEPRINT/lambda_layers/PATH/opt
+    #   output/demo/lambda_layers/files/lambda-function/opt
+    #   output/demo/lambda_layers/files/lambda-function # drop /opt to get dirname
+    #
+    # The build_parent is the parent folder because it can include cache and opt. Example tree:
+    #
+    #     output
+    #     └── demo
+    #         └── lambda_layers
+    #             └── files
+    #                 └── lambda-function  # <= build_area
+    #                     ├── cache
+    #                     └── opt
+    #
+    def build_area
+      File.dirname(@files.output_path)
+    end
+
+    def cache_area
+      "#{build_area}/cache"
+    end
+
+    # Final artifact that will be zipped up
+    def opt_area
+      "#{build_area}/opt"
+    end
+
+    # Original source is compiled to output folder
+    #
+    #     app/blueprints/demo/files/lambda-function/ - original
+    #     output/demo/files/lambda-function/         - compiled
+    #
+    # Compiling is done beforehand to support .tt ERB in the files.
+    #
+    # Then it's moved again to a cache area to run `bundle`
+    # Then it's moved yet agaoin to an opt area to produce the final Lambda Layer artifact.
+    #
+    def compiled_area
+      @files.compiled_path
+    end
+
+    # Also restructure the folder from:
+    #   vendor/gems/ruby/3.0.0
+    # To:
+    #   ruby/gems/3.0.0
+    #
+    # For Lambda Layer structure
+    def consolidate_gems_to_opt
+      src = "#{cache_area}/vendor/gems/ruby/#{ruby_folder}"
+      dest = "#{opt_area}/ruby/gems/#{ruby_folder}"
+      rsync_and_link(src, dest)
+    end
+
+    def rsync_and_link(src, dest)
+      FileUtils.mkdir_p(dest)
+      rsync(src, dest)
+
+      # create symlink in output path not the cache path
+      symlink_dest = "#{compiled_area}/vendor/gems/ruby/#{ruby_folder}"
+      logger.debug "symlink_dest #{symlink_dest}"
+      FileUtils.rm_rf(symlink_dest) # blow away original 3.0.0 folder
+
+      # Create symlink that will point to the gems in the Lambda Layer:
+      #   stage/opt/ruby/gems/3.0.0 -> /opt/ruby/gems/3.0.0
+      FileUtils.mkdir_p(File.dirname(symlink_dest))
+      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", symlink_dest)
+    end
+
+    def ruby_folder
+      major, minor, _ = RUBY_VERSION.split('.')
+      [major, minor, '0'].join('.') # 3.0.1 => 3.0.0
+    end
+
+    # Remove the BUNDLED WITH line since we don't control the bundler gem version on AWS Lambda
+    # And this can cause issues with require 'bundler/setup'
+    def remove_bundled_with(gemfile_lock)
+      lines = IO.readlines(gemfile_lock)
+
+      # amount is the number of lines to remove
+      new_lines, capture, count, amount = [], true, 0, 2
+      lines.each do |l|
+        capture = false if l.include?('BUNDLED WITH')
+        if capture
+          new_lines << l
+        end
+        if capture == false
+          count += 1
+          capture = count > amount # renable capture
+        end
+      end
+
+      content = new_lines.join('')
+      IO.write(gemfile_lock, content)
+    end
+
+    def gemfile_exist?
+      gemfile_path = "#{@files.full_path}/Gemfile"
+      File.exist?(gemfile_path)
+    end
+  end
+end

data/lib/lono/files/builder/lambda_layer/ruby_version.rb

@@ -0,0 +1,38 @@
+class Lono::Files::Builder::LambdaLayer
+  class RubyVersion < Lono::Files::Base
+    def initialize(options={})
+      # cfn passed from Finalizer::Files::Build#build_files to
+      # Lono::Files#build_lambda_layer(@cfn)
+      # This because Lono::Files at initialization only has registration info
+      # The build info is passed to the build_lambda_layer method
+      @cfn = options[:cfn]
+      super
+    end
+
+    def check!
+      # IE: logical_id = LayerVersion
+      # attrs = attrs with attrs['Properties']
+      logical_id, attrs = @cfn['Resources'].find do |logical_id, attrs|
+        attrs.dig('Properties', 'Content', 'S3Key') == @files.marker
+      end
+      props = attrs['Properties']
+      major, minor, _ = RUBY_VERSION.split('.')
+      current_ruby = "ruby#{major}.#{minor}"
+      ok = props['CompatibleRuntimes'].include?(current_ruby)
+      return if ok
+
+      runtimes = props['CompatibleRuntimes'].map { |s| s.sub('ruby','') }.join(', ')
+      logger.error "ERROR: Current Ruby Version does not match the Lambda Layer Ruby Version".color(:red)
+      logger.error <<~EOL
+        Current Ruby: #{RUBY_VERSION}
+        Lambda Layer Ruby Version: #{runtimes}
+
+        Lono is unable to package up a Lambda Layer
+        unless you use the same Ruby version on this machine.
+      EOL
+      logger.error "Resource: #{logical_id}"
+      logger.error YAML.dump(attrs)
+      exit 1
+    end
+  end
+end

data/lib/lono/files/builder/lambda_layer.rb

@@ -0,0 +1,17 @@
+class Lono::Files::Builder
+  class LambdaLayer < Lono::Files::Base
+    def run
+      layer = @options[:layer]
+      return unless layer # triggers Lambda Layer building
+      unless layer =~ /ruby/
+        logger.warn "WARN: Currently only support Ruby lambda layers automatically".color(:yellow)
+        return
+      end
+
+      klass_name = "Lono::Files::Builder::LambdaLayer::#{layer.camelize}Packager"
+      klass = klass_name.constantize
+      packager = klass.new(@options)
+      packager.build
+    end
+  end
+end

data/lib/lono/files/concerns/post_processing.rb

@@ -6,6 +6,10 @@ module Lono::Files::Concerns
       Lono::Files::Builder.new(@options.merge(files: self)).run
     end
 
+    def build_lambda_layer(cfn)
+      Lono::Files::Builder::LambdaLayer.new(@options.merge(files: self, cfn: cfn)).run
+    end
+
     def compress
       Lono::Files::Compressor.new(@options.merge(files: self)).run
     end
@@ -21,7 +25,9 @@ module Lono::Files::Concerns
     memoize :uploader
 
     def zip_name
-      "#{File.basename(full_path)}-#{Lono::Md5.sum(full_path)}.zip"
+      name = File.basename(full_path)
+      name << "-layer" if layer
+      "#{name}-#{Lono::Md5.sum(full_path)}.zip"
     end
 
     def zip_path
@@ -29,7 +35,18 @@ module Lono::Files::Concerns
     end
 
     def output_path
-      "#{Lono.root}/output/#{@blueprint.name}/#{path}"
+      if layer
+        # For Lambda Layer the final output path is the opt folder, where final artifact is zipped
+        "#{Lono.root}/output/#{@blueprint.name}/layer/#{path}/opt" # Note: layer and opt
+      else
+        # For normal files the final output path in the compiled path
+        compiled_path
+      end
+    end
+
+    # Also used by LambdaLayer::RubyPackager#compiled_area
+    def compiled_path
+      "#{Lono.root}/output/#{@blueprint.name}/normal/#{path}"
     end
   end
 end

data/lib/lono/files/concerns/registration.rb

@@ -6,8 +6,11 @@ module Lono::Files::Concerns
       end
     end
 
+    # IE:
+    #   LONO:://files/function-normal
+    #   LONO:://files/function-layer
     def marker
-      "LONO://#{@path}"
+      "LONO://#{@path}-#{@type}"
     end
   end
 end

data/lib/lono/files.rb

@@ -6,10 +6,12 @@ module Lono
     include Concerns::Registration
     include Concerns::PostProcessing
 
-    attr_reader :path
+    attr_reader :path, :layer, :type
     def initialize(options={})
       super
       @path = options[:path]
+      @layer = options[:layer]
+      @type = @layer ? "layer" : "normal"
       @caller = options[:caller] # original caller stack at registration time
     end
 
@@ -20,7 +22,9 @@ module Lono
     class << self
       def register(options={})
         path = options[:path]
-        file = files.find { |f| f.path == path }
+        layer = options[:layer]
+        # Registration uniquess based on both path and layer option
+        file = files.find { |f| f.path == path && f.layer == layer }
         unless file
           file = Files.new(options.merge(caller: caller))
           files << file

data/lib/lono/hooks/builder.rb

@@ -0,0 +1,50 @@
+module Lono::Hooks
+  class Builder
+    extend Memoist
+    include Dsl
+    include DslEvaluator
+    include Lono::Utils
+
+    # IE: dsl_file: config/hooks.rb
+    attr_accessor :name
+    def initialize(blueprint, name)
+      @blueprint, @name = blueprint, name
+      @hooks = {before: {}, after: {}}
+    end
+
+    def build
+      evaluate_file("#{Lono.root}/config/hooks.rb")
+      evaluate_file("#{@blueprint.root}/config/hooks.rb")
+      @hooks.deep_stringify_keys!
+    end
+    memoize :build
+
+    def run_hooks
+      build
+      run_each_hook("before")
+      out = yield if block_given?
+      run_each_hook("after")
+      out
+    end
+
+    def run_each_hook(type)
+      hooks = @hooks.dig(type, @name) || []
+      hooks.each do |hook|
+        run_hook(type, hook)
+      end
+    end
+
+    def run_hook(type, hook)
+      return unless run?(hook)
+
+      id = "#{type} #{@name}"
+      label = " label: #{hook["label"]}" if hook["label"]
+      logger.info  "Hook: Running #{id} hook#{label}".color(:cyan) if Lono.config.hooks.show
+      Runner.new(@blueprint, hook).run
+    end
+
+    def run?(hook)
+      !!hook["execute"]
+    end
+  end
+end

data/lib/lono/hooks/concern.rb

@@ -0,0 +1,9 @@
+module Lono::Hooks
+  module Concern
+    def run_hooks(name, &block)
+      hooks = Builder.new(@blueprint, name)
+      hooks.build # build hooks
+      hooks.run_hooks(&block)
+    end
+  end
+end

data/lib/lono/hooks/dsl.rb

@@ -0,0 +1,20 @@
+module Lono::Hooks
+  module Dsl
+    def before(*commands, **props)
+      commands.each do |name|
+        each_hook(:before, name, props)
+      end
+    end
+
+    def after(*commands, **props)
+      commands.each do |name|
+        each_hook(:after, name, props)
+      end
+    end
+
+    def each_hook(type, name, props={})
+      @hooks[type][name] ||= []
+      @hooks[type][name] << props
+    end
+  end
+end

data/lib/lono/hooks/runner.rb

@@ -0,0 +1,46 @@
+module Lono::Hooks
+  class Runner
+    include Lono::Utils
+
+    # exposing blueprint and hook so lono hooks have access to them via runner context. IE:
+    #
+    #     class EnvExporter
+    #       def call(runner)
+    #         puts "runner.hook #{runner.hook}"
+    #       end
+    #     end
+    #
+    # Docs: http://lono.cloud/docs/config/hooks/ruby/#method-argument
+    #
+    attr_reader :blueprint, :hook
+    def initialize(blueprint, hook)
+      @blueprint, @hook = blueprint, hook
+      @execute = @hook["execute"]
+    end
+
+    def run
+      case @execute
+      when String
+        sh(@execute, exit_on_fail: @hook["exit_on_fail"])
+      when -> (e) { e.respond_to?(:public_instance_methods) && e.public_instance_methods.include?(:call) }
+        executor = @execute.new
+      when -> (e) { e.respond_to?(:call) }
+        executor = @execute
+      else
+        logger.warn "WARN: execute option not set for hook: #{@hook.inspect}"
+      end
+
+      return unless executor
+
+      meth = executor.method(:call)
+      case meth.arity
+      when 0
+        executor.call # backwards compatibility
+      when 1
+        executor.call(self)
+      else
+        raise "The #{executor} call method definition has been more than 1 arguments and is not supported"
+      end
+    end
+  end
+end

data/lib/lono/layering/layer.rb

@@ -31,30 +31,18 @@ module Lono::Layering
       paths
     end
 
-    def full_layering
-      # layers defined in Lono::Layering module
-      all = layers.map { |layer| layer.sub(/\/$/,'') } # strip trailing slash
-      all.inject([]) do |sum, layer|
-        sum += layer_levels(layer) unless layer.nil?
-        sum
+    def full_layers(dir)
+      layers = layer_levels("#{dir}/#{@type}") # root
+      if Lono.role
+        layers += layer_levels("#{dir}/#{@type}/#{Lono.role}")
       end
-    end
-
-    # interface method
-    def main_layers
-      [
-        '',
-        region,
-        account,
-        "#{account}/#{region}",
-      ]
-    end
-
-    def account
-      name = aws_data.account
-      # friendly name mapping
-      names = Lono.config.layering.names.stringify_keys
-      names[name] || name
+      if Lono.app
+        layers += layer_levels("#{dir}/#{@type}/#{Lono.app}")
+      end
+      if Lono.app && Lono.role
+        layers += layer_levels("#{dir}/#{@type}/#{Lono.app}/#{Lono.role}")
+      end
+      layers
     end
 
     # adds prefix and to each layer pair that has base and Lono.env. IE:
@@ -62,19 +50,53 @@ module Lono::Layering
     #    "#{prefix}/base"
     #    "#{prefix}/#{Lono.env}"
     #
+    # Params Layers:
+    #     config/blueprints/demo/params/txt
+    #     config/blueprints/demo/params/base.txt
+    #     config/blueprints/demo/params/dev.txt
     def layer_levels(prefix=nil)
-      levels = ["base", Lono.env]
+      levels = ["", "base", Lono.env]
+      levels << "#{Lono.env}-#{Lono.extra}" if Lono.extra
       levels.map! do |i|
-        # base layer has prefix of '', reject with blank so it doesnt produce '//'
         [prefix, i].reject(&:blank?).join('/')
       end
-      levels.unshift(prefix) # unless prefix.blank? # IE: params/us-west-2.txt
+      levels.map! { |level| level.sub(/\/$/,'') } # strip trailing slash
+
+      # layers = pre_layers + main_layers + post_layers
+      levels = layers.inject([]) do |sum, layer|
+        layer_levels = levels.map do |level|
+          [level, layer].reject(&:blank?).join('/')
+        end
+        sum += layer_levels
+        sum
+      end
+
       levels
     end
 
+    # Interface method: layers = pre_layers + main_layers + post_layers
+    # Simple layering is default. Can set with:
+    #
+    #      config.layering.mode = "simple" # simple of full
+    #
+    def main_layers
+      if Lono.config.layering.mode == "simple"
+        ['']
+      else # full
+        # includes region, account, and account/region layers
+        # More powerful but less used and more complex to understand
+        [
+          '',
+          region,
+          account,
+          "#{account}/#{region}",
+        ]
+      end
+    end
+
     def add_ext!(paths)
       exts = {
-        params: "txt",
+        params: "env",
         vars: "rb",
       }
       ext = exts[@type.to_sym]
@@ -85,28 +107,28 @@ module Lono::Layering
       end
     end
 
-    def full_layers(dir)
-      layers = full_layering.map do |layer|
-        "#{dir}/#{@type}/#{layer}"
-      end
-      if Lono.app
-        app_layers = full_layering.map do |layer|
-          "#{dir}/#{@type}/#{Lono.app}/#{layer}"
-        end
-        layers += app_layers
-      end
-      layers
-    end
-
     @@shown_layers = {}
     def show_layers(paths)
       return if @@shown_layers[@type]
       logger.debug "#{@type.capitalize} Layers:"
+      show = Lono.config.layering.show || ENV['LONO_LAYERS']
       paths.each do |path|
-        logger.debug "    #{pretty_path(path)}" if File.exist?(path) || ENV['LONO_SHOW_ALL_LAYERS']
+        # next if path.include?("app/blueprints/") # more useful to show only config layers
+        if ENV['LONO_LAYERS_ALL']
+          logger.info "    #{pretty_path(path)}"
+        elsif show
+          logger.info "    #{pretty_path(path)}" if File.exist?(path)
+        end
       end
       logger.debug ""
       @@shown_layers[@type] = true
     end
+
+    def account
+      name = aws_data.account
+      # friendly name mapping
+      names = Lono.config.layering.names.stringify_keys
+      names[name] || name
+    end
   end
 end

data/lib/lono/names.rb

@@ -21,30 +21,27 @@ module Lono
       vars.each do |var|
         string.gsub!(var, var_value(var))
       end
-      strip(string)
+      cleanse(string)
     end
 
     def var_value(unexpanded)
       name = unexpanded.sub(':','').downcase
       if respond_to?(name)
-        value = send(name).to_s
+        send(name).to_s # value
       else
         unexpanded
       end
     end
 
-    def strip(string)
+    def cleanse(string)
       string.sub(/^-+/,'').sub(/-+$/,'') # remove leading and trailing -
             .gsub(%r{-+},'-') # remove double dashes are more. IE: -- => -
             .gsub('_','-')    # underscores are not allowed in CloudFormation stack names
     end
 
-    def app
-      Lono.app
-    end
-
-    def env
-      Lono.env
+    delegate :app, :role, :env, :extra, to: :lono
+    def lono
+      Lono
     end
   end
 end

data/lib/lono/s3/bucket.rb

@@ -1,10 +1,11 @@
 module Lono::S3
   class Bucket
+    extend Lono::AwsServices
+    extend Memoist
     include Lono::AwsServices
     include Lono::Cfn::Concerns
     include Lono::Utils::Logging
-    extend Lono::AwsServices
-    extend Memoist
+    include Lono::Utils::Sure
 
     STACK_NAME = ENV['LONO_STACK_NAME'] || "lono"
     def initialize(options={})
@@ -115,20 +116,12 @@ module Lono::S3
 
 
     def are_you_sure?
-      return true if @options[:yes]
-
       if bucket_name.nil?
         logger.info "The lono stack and s3 bucket does not exist."
         exit
       end
 
-      logger.info "Are you yes you want the lono bucket #{bucket_name.color(:green)} to be emptied and deleted? (y/N)"
-      yes = $stdin.gets.strip
-      confirmed = yes =~ /^Y/i
-      unless confirmed
-        logger.info "Phew that was close."
-        exit
-      end
+      sure?("Will empty and delete bucket #{bucket_name.color(:green)}. Are you sure?")
     end
 
     def template_body
@@ -145,6 +138,10 @@ module Lono::S3
               Tags:
                 - Key: Name
                   Value: lono
+        Outputs:
+          Bucket:
+            Value:
+              Ref: Bucket
       YAML
     end
 
@@ -166,6 +163,15 @@ module Lono::S3
         bucket = stack_resources.find { |r| r.logical_resource_id == "Bucket" }
         @@name = bucket.physical_resource_id # actual bucket name
       end
+
+      @@ensure_exist = false
+      def ensure_exist
+        return if @@ensure_exist
+        bucket = Lono::S3::Bucket.new
+        return if bucket.exist?
+        bucket.deploy
+        @@ensure_exist = true
+      end
     end
   end
 end