lono 7.2.3 → 7.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93d21105409fe3d2c1173d636bc7b852c7a02d7846422e6688b064babd591b08
-  data.tar.gz: 9e38baab88bb7604187e80e5b6c2071ed52df1bc81762e40885190c19ac088f2
+  metadata.gz: 50634682d0b2bd1142fa36acb73b537e1fcceabde68d7ea88da24621c17b2267
+  data.tar.gz: 53a2e8a25610f8f039cdee8c6cb71ab15b24732642d0453a61ace628fb925ca4
 SHA512:
-  metadata.gz: 2a0c0b478ca67fc90289fa0e96bc212f75ffc6b22c6dbd92bc10a80eedf7b3198bd328b75e93ea877622c2d0cd205b3b19ab2eb23442c065ae537e64673ba577
-  data.tar.gz: d58eb137d8faf09aef321b413de9a2179858da95a34b3c38861c6ab8ef91c8517599efde1efff432af0c0f1e6a66b2005fc6bd5bfc2a7e745bc5d2c771760466
+  metadata.gz: a700f987afc86bdf10ffc561e0f9027d7ce7aa661b852395a6ff734d272f06a655f24ff4f44aff316b3cb0c225ce64affccab258bde7ac58c0393ce47eff09b5
+  data.tar.gz: 1ad6215fc323bd75bd88c085761fe96131d9e71e531cf40ee0d2940cf8f10483c97cb86984c382cf1c3b6ee1522edc60782275a514713b4fc05c5d6627bf184c

data/CHANGELOG.md

@@ -3,6 +3,9 @@
 All notable changes to this project will be documented in this file.
 This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [7.3.0]
+- #54 introduce lambda layer concept
+
 ## [7.2.3]
 - #53 improve configset lookup path lookup
 - cleanup starter blueprint

data/lib/lono/app_file/build.rb

@@ -15,8 +15,21 @@ module Lono::AppFile
     def build_all
       clean_output
       copy_to_output
+      build_layers
+      compress_output
+    end
+
+    def build_layers
+      layer_items = Registry.layers
+      layer_items.each do |item|
+        LambdaLayer.new(@blueprint, item).build
+      end
+    end
+
+    def compress_output
       Registry.items.each do |item|
-        if item.directory?
+        # type can be lambda_layer or file
+        if item.type == "lambda_layer" || item.directory?
           zip_directory(item)
         elsif item.file?
           zip_file(item)

data/lib/lono/app_file/build/lambda_layer.rb

@@ -0,0 +1,20 @@
+class Lono::AppFile::Build
+  class LambdaLayer
+    def initialize(blueprint, registry_item)
+      @blueprint, @registry_item = blueprint, registry_item
+    end
+
+    def build
+      lang = @registry_item.options[:lang]
+      unless lang =~ /ruby/
+        puts "WARN: Currently only support ruby lambda layers".color(:yellow)
+        return
+      end
+
+      klass_name = "Lono::AppFile::Build::LambdaLayer::#{lang.camelize}Packager"
+      klass = klass_name.constantize
+      packager = klass.new(@blueprint, @registry_item)
+      packager.build
+    end
+  end
+end

data/lib/lono/app_file/build/lambda_layer/ruby_packager.rb

@@ -0,0 +1,180 @@
+class Lono::AppFile::Build::LambdaLayer
+  # Based on jets
+  class RubyPackager
+    def initialize(blueprint, registry_item)
+      @blueprint, @registry_item = blueprint, registry_item
+
+      @registry_name = @registry_item.name.sub(/\/$/,'')
+      @app_root = "#{Lono.blueprint_root}/app/files/#{@registry_name}"
+    end
+
+    def build
+      return unless gemfile_exist?
+
+      bundle_install
+      package
+    end
+
+    # We consolidate all gems to opt
+    def package
+      setup_bundle_config(output_area)
+      # copy_cache_gems # TODO: might not need this cache
+      consolidate_gems_to_opt
+    end
+
+    # Also restructure the folder from:
+    #   vendor/gems/ruby/2.5.0
+    # To:
+    #   ruby/gems/2.5.0
+    #
+    # For Lambda Layer structure
+    def consolidate_gems_to_opt
+      src = "#{cache_area}/vendor/gems/ruby/#{ruby_folder}"
+      dest = "#{opt_area}/ruby/gems/#{ruby_folder}"
+      rsync_and_link(src, dest)
+    end
+
+    def rsync_and_link(src, dest)
+      FileUtils.mkdir_p(dest)
+      # Trailing slashes are required
+      sh "rsync -a --links #{src}/ #{dest}/"
+
+      # create symlink in output path not the cache path
+      symlink_dest = "#{output_area}/vendor/gems/ruby/#{ruby_folder}"
+      puts "symlink_dest #{symlink_dest}"
+      FileUtils.rm_rf(symlink_dest) # blow away original 2.5.0 folder
+
+      # Create symlink that will point to the gems in the Lambda Layer:
+      #   stage/opt/ruby/gems/2.5.0 -> /opt/ruby/gems/2.5.0
+      FileUtils.mkdir_p(File.dirname(symlink_dest))
+      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", symlink_dest)
+    end
+
+    def ruby_folder
+      major, minor, _ = RUBY_VERSION.split('.')
+      [major, minor, '0'].join('.') # 2.5.1 => 2.5.0
+    end
+
+    # Installs gems on the current target system: both compiled and non-compiled.
+    # If user is on a macosx machine, macosx gems will be installed.
+    # If user is on a linux machine, linux gems will be installed.
+    #
+    # Copies Gemfile* to /tmp/jets/demo/cache folder and installs
+    # gems with bundle install from there.
+    #
+    # We take the time to copy Gemfile and bundle into a separate directory
+    # because it gets left around to act as a 'cache'.  So, when the builds the
+    # project gets built again not all the gems from get installed from the
+    # beginning.
+    def bundle_install
+      puts "Bundling: running bundle install in cache area: #{cache_area}."
+
+      copy_gemfiles(@app_root)
+
+      # Uncomment out to always remove the cache/vendor/gems to debug
+      # FileUtils.rm_rf("#{cache_area}/vendor/gems")
+
+      # Remove .bundle folder so .bundle/config doesnt affect how Jets packages gems.
+      # Not using BUNDLE_IGNORE_CONFIG=1 to allow home ~/.bundle/config to affect bundling though.
+      # This is useful if you have private gems sources that require authentication. Example:
+      #
+      #    bundle config gems.myprivatesource.com user:pass
+      #
+
+      FileUtils.rm_rf("#{cache_area}/.bundle")
+      require "bundler" # dynamically require bundler so user can use any bundler
+      setup_bundle_config(cache_area)
+      Bundler.with_unbundled_env do
+        sh "cd #{cache_area} && env bundle install"
+      end
+
+      remove_bundled_with("#{cache_area}/Gemfile.lock")
+
+      # Copy the Gemfile.lock back to the project in case it was updated.
+      # For example we add the jets-rails to the Gemfile.
+      copy_back_gemfile_lock
+
+      puts 'Bundle install success.'
+    end
+
+    # Remove the BUNDLED WITH line since we don't control the bundler gem version on AWS Lambda
+    # And this can cause issues with require 'bundler/setup'
+    def remove_bundled_with(gemfile_lock)
+      lines = IO.readlines(gemfile_lock)
+
+      # amount is the number of lines to remove
+      new_lines, capture, count, amount = [], true, 0, 2
+      lines.each do |l|
+        capture = false if l.include?('BUNDLED WITH')
+        if capture
+          new_lines << l
+        end
+        if capture == false
+          count += 1
+          capture = count > amount # renable capture
+        end
+      end
+
+      content = new_lines.join('')
+      IO.write(gemfile_lock, content)
+    end
+
+    def copy_back_gemfile_lock
+      src = "#{cache_area}/Gemfile.lock"
+      dest = "#{@app_root}/Gemfile.lock"
+      FileUtils.cp(src, dest)
+    end
+
+    # Clean up extra unneeded files to reduce package size
+    def copy_gemfiles(app_root)
+      FileUtils.mkdir_p(cache_area)
+      FileUtils.cp("#{app_root}/Gemfile", "#{cache_area}/Gemfile")
+
+      gemfile_lock = "#{app_root}/Gemfile.lock"
+      dest = "#{cache_area}/Gemfile.lock"
+      return unless File.exist?(gemfile_lock)
+
+      FileUtils.cp(gemfile_lock, dest)
+    end
+
+    def setup_bundle_config(dir)
+      text =<<-EOL
+---
+BUNDLE_FROZEN: "true"
+BUNDLE_PATH: "vendor/gems"
+BUNDLE_WITHOUT: "development:test"
+EOL
+      bundle_config = "#{dir}/.bundle/config"
+      FileUtils.mkdir_p(File.dirname(bundle_config))
+      IO.write(bundle_config, text)
+    end
+
+    def output_area
+      "#{Lono.config.output_path}/#{@blueprint}/files/#{@registry_name}"
+    end
+
+    def build_area
+      "#{Lono.config.output_path}/#{@blueprint}/lambda_layers/#{@registry_name}"
+    end
+
+    def cache_area
+      "#{build_area}/cache"
+    end
+
+    def opt_area
+      "#{build_area}/opt"
+    end
+
+    def gemfile_exist?
+      gemfile_path = "#{@app_root}/Gemfile"
+      File.exist?(gemfile_path)
+    end
+
+    def sh(command)
+      puts "=> #{command}"
+      out = `#{command}`
+      puts out if ENV['LONO_DEBUG_LAMBDA_LAYER']
+      $?.success?
+    end
+  end
+end

data/lib/lono/app_file/registry.rb

@@ -1,13 +1,15 @@
 module Lono::AppFile
   class Registry
+    cattr_reader :items
     @@items = []
+
     class << self
       def register(name, blueprint, options={})
-        @@items << Item.new(name, blueprint, options) unless @@items.detect { |i| i.name == name }
+        @@items << Item.new(name, blueprint, options) unless @@items.detect { |i| i.name == name && i.type == options[:type] }
       end
 
-      def items
-        @@items
+      def layers
+        @@items.select { |i| i.type == "lambda_layer" }
       end
     end
   end

data/lib/lono/app_file/registry/item.rb

@@ -2,13 +2,18 @@ module Lono::AppFile
   class Registry
     # Holds metadata about the item in the regsitry.
     class Item
-      attr_reader :name, :options
+      attr_reader :name, :options, :type
       def initialize(name, blueprint, options={})
         @name, @blueprint, @options = name, blueprint, options
+        @type = options[:type] || "file"
       end
 
       def path
-        "#{Lono.root}/output/#{@blueprint}/files/#{@name}"
+        if @type == "file"
+          "#{Lono.root}/output/#{@blueprint}/files/#{@name}"
+        else
+          "#{Lono.root}/output/#{@blueprint}/lambda_layers/#{@name}/opt"
+        end
       end
 
       def directory?
@@ -20,7 +25,7 @@ module Lono::AppFile
       end
 
       def s3_path
-        file_path = zip_file_path.sub(%r{.*/output/[\w_-]+/files/}, '') # dont use basename. there might be subfolders
+        file_path = zip_file_path.sub(%r{.*/output/[\w_-]+/(files|lambda_layers)/}, '') # dont use basename. there might be subfolders
         "#{s3_prefix}/#{file_path}"
       end
 
@@ -30,7 +35,7 @@ module Lono::AppFile
       end
 
       def zip_file_name
-        "#{File.basename(path)}-#{Lono::Md5.sum(path)}.zip"
+        "#{File.basename(path)}-#{@type}-#{Lono::Md5.sum(path)}.zip"
       end
 
     private

data/lib/lono/template/post_processor.rb

@@ -14,15 +14,15 @@ class Lono::Template
     def replacements
       map = {}
       registry_items.each do |item|
-        if item.directory? || item.file?
-          replacement = item.s3_path
+        if item.type == "lambda_layer"
+          placeholder = "file://app/files/lambda_layer/#{item.name}"
+        elsif item.directory? || item.file?
+          placeholder = "file://app/files/file/#{item.name}"
         else
           puts "WARN: PostProcessor replacements Cannot find file: #{item.path}"
           next
         end
-
-        placeholder = "file://app/files/#{item.name}"
-        map[placeholder] = replacement
+        map[placeholder] = item.s3_path
       end
       map
     end

data/lib/lono/template/strategy/dsl/builder/helpers/s3_helper.rb

@@ -4,10 +4,12 @@ module Lono::Template::Strategy::Dsl::Builder::Helpers
       Lono::S3::Bucket.name
     end
 
-    def file_s3_key(name, options={})
+    def s3_key(name, options={})
+      default = {type: "file"}
+      options.reverse_merge!(default)
       Lono::AppFile::Registry.register(name, @blueprint, options)
-      "file://app/files/#{name}" # placeholder for post processing
+      "file://app/files/#{options[:type]}/#{name}" # placeholder for post processing
     end
-    alias_method :s3_key, :file_s3_key
+    alias_method :file_s3_key, :s3_key
   end
 end

data/lib/lono/version.rb

@@ -1,3 +1,3 @@
 module Lono
-  VERSION = "7.2.3"
+  VERSION = "7.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lono
 version: !ruby/object:Gem::Version
-  version: 7.2.3
+  version: 7.3.0
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-10 00:00:00.000000000 Z
+date: 2020-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -450,6 +450,8 @@ files:
 - lib/lono/api/verify.rb
 - lib/lono/app_file/base.rb
 - lib/lono/app_file/build.rb
+- lib/lono/app_file/build/lambda_layer.rb
+- lib/lono/app_file/build/lambda_layer/ruby_packager.rb
 - lib/lono/app_file/registry.rb
 - lib/lono/app_file/registry/item.rb
 - lib/lono/app_file/upload.rb