jets 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e0490cdce6f57474859bad4aa18687389e093c9c93bc3dd18d2277198455625
-  data.tar.gz: aba0604bffff78b7c9d77e919d214497ad50810c03634153b9dacbd42a86bd74
+  metadata.gz: bb0598c2fec261f9cf41a160261d7641963e09a62158e1590194e96eff34fbcb
+  data.tar.gz: f2e34c4c936ab77a6c75045d048a9c5ddebac2611b876f28a74ed11881e3d802
 SHA512:
-  metadata.gz: 766d84de9260cd6a068a2cead8fead75b8fd3a60b84815a153b787f2ce7c2bd420aad30b8398a7aace05006a9911c0aa0fffde7f91544b6327e81a2f5da6038f
-  data.tar.gz: d297334afecc43c4d8a13ddad7df20a3c94a18ac0658a080ccd9be35b6aa7d2d7d7ee73d6f1bce6b2529325ca26928442bbb86cd61237689997b0c31cf0f9cfb
+  metadata.gz: 603c3925ff7c16dac27e5c1695f948882b6880f91ee162346df13a74c8b3ee59a936555af15eca5b99c966b60da26adbd53c1a252a925aa64dceb3c82b62d9e1
+  data.tar.gz: c1336fde24ccee02f4ac5e2de60856960b9b33bb84ae5678365ee98f67982c297e8dae3272a126f0f0c4694467d8d5311a4461630bc4c3fea8c4075d83aaaa54

data/CHANGELOG.md

@@ -3,6 +3,14 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [1.3.0]
+- Official AWS Ruby Support
+- Ruby Version 2.5.3 upgrade
+- Gem Layer introduced
+- Removed node shim
+- Build purger: /tmp/jets/project is auto purge when major or minor version changes
+- Update default gems source to https://gems2.lambdagems.com
+
 ## [1.2.1]
 - remove comments about routes workaround, auto blue-green deployments resolves this
 

data/Gemfile.lock

@@ -11,7 +11,7 @@ GIT
 PATH
   remote: .
   specs:
-    jets (1.2.1)
+    jets (1.3.0)
       actionpack (>= 5.2.1)
       actionview (>= 5.2.1)
       activerecord (>= 5.2.1)
@@ -66,7 +66,7 @@ GEM
       tzinfo (~> 1.1)
     arel (9.0.0)
     aws-eventstream (1.0.1)
-    aws-partitions (1.122.0)
+    aws-partitions (1.123.0)
     aws-sdk-apigateway (1.23.0)
       aws-sdk-core (~> 3, >= 3.39.0)
       aws-sigv4 (~> 1.0)
@@ -76,7 +76,7 @@ GEM
     aws-sdk-cloudwatchlogs (1.12.0)
       aws-sdk-core (~> 3, >= 3.39.0)
       aws-sigv4 (~> 1.0)
-    aws-sdk-core (3.43.0)
+    aws-sdk-core (3.44.0)
       aws-eventstream (~> 1.0)
       aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.0)
@@ -109,7 +109,7 @@ GEM
     gems (1.1.1)
       json
     hashie (3.6.0)
-    i18n (1.1.1)
+    i18n (1.2.0)
       concurrent-ruby (~> 1.0)
     jmespath (1.4.0)
     json (2.1.0)

data/README.md

@@ -30,13 +30,13 @@ Jets supports writing AWS Lambda functions with Ruby. You define them in the `ap
 app/functions/simple.rb:
 
 ```ruby
-def handle(event, context)
+def handle(event:, context:)
   puts "hello world"
   {hello: "world"}
 end
 ```
 
-Though simple functions are supported by Jets, aside from the ability to use Ruby, they do not as much value as other ways to write code with Jets.
+Though simple functions are supported by Jets, they do not really add much value as other ways to write Ruby code with Jets. Classes like Controllers and Jobs add many conveniences and are more powerful to use. We’ll cover them next.
 
 ### Jets Controllers
 

data/lib/jets.rb

@@ -12,8 +12,7 @@ require "fileutils"
 require "memoist"
 
 module Jets
-  # When we update Jets::RUBY_VERSION, need to update jets-gems/base.rb: def jets_ruby_version also
-  RUBY_VERSION = "2.5.0"
+  RUBY_VERSION = "2.5.3"
 
   autoload :Application, "jets/application"
   autoload :AwsInfo, "jets/aws_info"
@@ -42,14 +41,14 @@ module Jets
   autoload :PolyFun, 'jets/poly_fun'
   autoload :Preheat, "jets/preheat"
   autoload :Processors, 'jets/processors'
+  autoload :RackServer, "jets/rack_server"
   autoload :Rdoc, "jets/rdoc"
   autoload :Resource, "jets/resource"
   autoload :Route, "jets/route"
   autoload :Router, "jets/router"
-  autoload :RubyServer, "jets/ruby_server"
   autoload :Rule, 'jets/rule'
-  autoload :Server, "jets/server"
   autoload :Stack, "jets/stack"
+  autoload :TmpLoader, "jets/tmp_loader"
   autoload :Turbine, 'jets/turbine'
   autoload :Util, "jets/util"
 

data/lib/jets/application.rb

@@ -52,7 +52,7 @@ class Jets::Application
 
     config.gems = ActiveSupport::OrderedOptions.new
     config.gems.sources = [
-      'https://gems.lambdagems.com'
+      Jets.default_gems_source
     ]
 
     config.inflections = ActiveSupport::OrderedOptions.new
@@ -65,7 +65,6 @@ class Jets::Application
     config.assets.cache_control = nil # IE: public, max-age=3600 , max_age is a shorter way to set cache_control.
 
     config.ruby = ActiveSupport::OrderedOptions.new
-    config.ruby.lazy_load = true # also set in config/environments files
 
     config.middleware = Jets::Middleware::Configurator.new
 
@@ -133,6 +132,18 @@ class Jets::Application
     load_app_config
     load_db_config
     load_environments_config
+    deprecated_configs_message
+  end
+
+  def deprecated_configs_message
+    unless config.ruby.lazy_load.nil?
+      puts "Detected config.ruby.lazy_load = #{config.ruby.lazy_load.inspect}".colorize(:yellow)
+      puts "Deprecated: config.ruby.lazy_load".colorize(:yellow)
+      puts "Gems are now bundled with with Lambda Layer and there's no need to lazy load them."
+      puts "Please remove the config in your config/application.rb or config/environments files."
+      puts "You can have Jets automatically do this by running:"
+      puts "  jets upgrade:v1"
+    end
   end
 
   def setup_auto_load_paths

data/lib/jets/builders.rb

@@ -1,9 +1,12 @@
 class Jets::Builders
   autoload :CodeBuilder, "jets/builders/code_builder"
+  autoload :CodeSize, "jets/builders/code_size"
   autoload :GemReplacer, "jets/builders/gem_replacer"
   autoload :HandlerGenerator, "jets/builders/handler_generator"
+  autoload :LambdaLayer, "jets/builders/lambda_layer"
   autoload :Md5, "jets/builders/md5"
   autoload :Md5Zip, "jets/builders/md5_zip"
+  autoload :Purger, "jets/builders/purger"
   autoload :RackPackager, "jets/builders/rack_packager"
   autoload :ReconfigureRails, "jets/builders/reconfigure_rails"
   autoload :RubyPackager, "jets/builders/ruby_packager"

data/lib/jets/builders/code_builder.rb

@@ -10,49 +10,10 @@ require "action_view"
 # /tmp/jets: build root where different jets projects get built.
 # /tmp/jets/project: each jets project gets built in a different subdirectory.
 #
-# The rest of the folders are subfolders under /tmp/jets/project:
+# The rest of the folders are subfolders under /tmp/jets/project.
 #
-# cache: Gemfile is here, this is where we run bundle install.
-# cache/bundled/gems: Vendored gems that get created as part of bundled install.
-#   Initially, macosx gems but then get replaced by linux gems where appropriate.
-# cache/downloads/rubies: ruby tarballs.
-# cache/downloads/gems: gem tarballs.
-# code: Where project gets copied into in order for us to configure it.
-# app_root/bundled/gems: Where vendored gems finally end up at.  The compiled
-#   gems at this point are only linux gems.
-# artifacts/code/code-md5sha.zip: code artifact that gets uploaded to lambda.
-#
-# Building Steps:
-#
-### Before copy
-# * compile assets: easier to do this before the copy
-#
-### copy project
-# * copy project: to app_root
-#
-### setup app_root project
-# * clean project: remove log and ignored files to reduce size
-# * reconfigure webpacker: config/webpacker.yml
-# * generate node shims: handlers
-#
-### build bundled in cache area
-# * bundle install: cache/bundled/gems
-#
-### setup bundled on app root from cache
-# * copy bundled to code: code/bundled
-# * extract linux ruby: cache/downloads/rubies:
-#                       cache/bundled/rbenv, cache/bundled/linuxbrew
-# * extract linux gems: cache/downloads/gems:
-#                       cache/bundled/gems, cache/bundled/linuxbrew
-# * setup bundled config: code/.bundle/config
-#
-### zip
-# * create zip file
 class Jets::Builders
   class CodeBuilder
-    # https://docs.aws.amazon.com/lambda/latest/dg/limits.html
-    AWS_CODE_SIZE_LIMIT = 250 * 1024 * 1024 # 250MB
-
     include Jets::AwsServices
     include Util
     extend Memoist
@@ -62,11 +23,13 @@ class Jets::Builders
       # Expanding to the full path and capture now.
       # Dir.chdir gets called later and we'll lose this info.
       @full_project_path = File.expand_path(Jets.root) + "/"
+      @version_purger = Purger.new
     end
 
     def build
-      cache_check_message
       check_ruby_version
+      @version_purger.purge
+      cache_check_message
 
       clean_start
       compile_assets # easier to do before we copy the project because node and yarn has been likely setup in the that dir
@@ -101,6 +64,7 @@ class Jets::Builders
 
     def create_zip_files
       folders = Md5.stage_folders
+      # Md5.stage_folders ["stage/bundled", "stage/code"]
       folders.each do |folder|
         zip = Md5Zip.new(folder)
         if exist_on_s3?(zip.md5_name)
@@ -122,56 +86,38 @@ class Jets::Builders
       end
     end
 
-    # Moves code/bundled and code/rack to build_root.
-    # These files will be packaged separated and lazy loaded as part of the
-    # node shim. This keeps the code zipfile smaller in size and helps
-    # with the 250MB extract limited. /tmp permits up to 512MB.
-    # AWS Lambda Limits: https://amzn.to/2A7y6v6
-    #
-    #   > Each Lambda function receives an additional 512MB of non-persistent disk space in its own /tmp directory. The /tmp directory can be used for loading additional resources like dependency libraries or data sets during function initialization.
-    #
-    def setup_tmp
-      tmp_symlink("bundled") if Jets.lazy_load?
-      tmp_symlink("rack")
-    end
-
     def stage_area
       "#{Jets.build_root}/stage"
     end
 
-    # Moves folder to a stage folder and create a symlink its place
-    # that links from /var/task to /tmp. Example:
-    #
-    #   /var/task/bundled => /tmp/bundled
-    #
-    def tmp_symlink(folder)
-      src = "#{full(tmp_code)}/#{folder}"
-      return unless File.exist?(src)
-
-      dest = "#{stage_area}/#{folder}"
-      dir = File.dirname(dest)
-      FileUtils.mkdir_p(dir) unless File.exist?(dir)
-      FileUtils.mv(src, dest)
-
-      # Create symlink
-      FileUtils.ln_sf("/tmp/#{folder}", "/#{full(tmp_code)}/#{folder}")
-    end
-
     def code_setup
       reconfigure_development_webpacker
     end
 
     def code_finish
-      update_lazy_load_config # at the top, must be called before Jets.lazy_load? is used
+      # Reconfigure code
       store_s3_base_url
       disable_webpacker_middleware
       copy_internal_jets_code
-      setup_tmp
+
+      # Code prep and zipping
+      build_lambda_layer
+      check_code_size!
       calculate_md5s # must be called before generate_node_shims and create_zip_files
       generate_node_shims
       create_zip_files
     end
 
+    def build_lambda_layer
+      return if Jets.poly_only?
+      lambda_layer = LambdaLayer.new
+      lambda_layer.build
+    end
+
+    def check_code_size!
+      CodeSize.check!
+    end
+
     # We copy the files into the project because we cannot require simple functions
     # directly since they are wrapped by an anonymous class.
     # TODO: Do this with the other files we required the same way.
@@ -185,17 +131,6 @@ class Jets::Builders
       end
     end
 
-    def update_lazy_load_config
-      size_limit = AWS_CODE_SIZE_LIMIT
-      code_size = dir_size(full(tmp_code))
-      if code_size > size_limit && !Jets.config.ruby.lazy_load
-        # override the setting because we dont have to a choice but to lazy load
-        mb_limit = AWS_CODE_SIZE_LIMIT / 1024 / 1024
-        puts "Code size close to AWS code size limit of #{mb_limit}MB. Lazy loading automatically enabled."
-        Jets.config.ruby.lazy_load = true
-      end
-    end
-
     # Thanks https://stackoverflow.com/questions/9354595/recursively-getting-the-size-of-a-directory
     # Seems to overestimate a little bit but close enough.
     def dir_size(folder)
@@ -210,8 +145,6 @@ class Jets::Builders
     # At this point the minimal stack exists, so we can grab it with the AWS API.
     # We do not want to grab this as part of the live request because it is slow.
     def store_s3_base_url
-      return if poly_only?
-
       write_s3_base_url("config/s3_base_url.txt")
       write_s3_base_url("rack/config/s3_base_url.txt") if Jets.rack?
     end
@@ -369,9 +302,9 @@ class Jets::Builders
       return if Jets.poly_only?
 
       ruby_packager.install
-      reconfigure_rails
+      reconfigure_rails # call here after full(tmp_code) is available
       rack_packager.install
-      ruby_packager.finish
+      ruby_packager.finish # by this time we have a /tmp/jets/demo/stage/code/bundled
       rack_packager.finish
     end
 

data/lib/jets/builders/code_size.rb

@@ -0,0 +1,55 @@
+class Jets::Builders
+  class CodeSize
+    LAMBDA_SIZE_LIMIT = 250 # Total lambda limit is 250MB
+    include Util
+
+    def self.check!
+      new.check
+    end
+
+    def check
+      return if within_lambda_limit?
+      say "Over the Lambda size limit of #{LAMBDA_SIZE_LIMIT}MB".colorize(:red)
+      say "Please reduce the size of your code."
+      display_sizes
+      exit 1
+    end
+
+    def within_lambda_limit?
+      total_size < LAMBDA_SIZE_LIMIT * 1024 # 120MB
+    end
+
+    def total_size
+      code_size = compute_size("#{stage_area}/code")
+      opt_size = compute_size("#{stage_area}/opt")
+      opt_size + code_size # total_size
+    end
+
+    def display_sizes
+      code_size = compute_size("#{stage_area}/code")
+      opt_size = compute_size("#{stage_area}/opt")
+      total_size = opt_size + code_size
+      overlimit = (LAMBDA_SIZE_LIMIT * 1024 - total_size) * -1
+      say "Sizes:"
+      say "Code: #{megabytes(code_size)} - #{stage_area}/code"
+      say "Gem Layer: #{megabytes(opt_size)} - #{stage_area}/opt"
+      say "Total Package: #{megabytes(total_size)}"
+      say "Over limit by: #{megabytes(overlimit)}"
+      # sh "du -csh #{stage_area}/*" unless ENV['TEST'] # uncomment to debug
+    end
+
+    def compute_size(path)
+      out = `du -s #{path}`
+      out.split(' ').first.to_i # bytes
+    end
+
+    def megabytes(bytes)
+      n = bytes / 1024.0
+      sprintf('%.1f', n) + 'MB'
+    end
+
+    def say(message)
+      puts message unless ENV['TEST']
+    end
+  end
+end

data/lib/jets/builders/gem_replacer.rb

@@ -17,15 +17,36 @@ class Jets::Builders
         gem_extractor.run
       end
 
+      # At this point the current compiled gems have been removed and compiled gems
+      # have been unpacked to code/opt. We can take the unpacked gems in opt and fully
+      # move them into vendor/bundle gems now.
+      move_opt_gems_to_vendor
+
       tidy
     end
 
-    def report_missing_gems
+    def move_opt_gems_to_vendor
+      code = "#{Jets.build_root}/stage/code"
+      opt_gems = "#{code}/opt/ruby/gems/#{Jets::Gems.ruby_folder}"
+      vendor_gems = "#{code}/vendor/bundle/ruby/#{Jets::Gems.ruby_folder}"
+      # https://stackoverflow.com/questions/23698183/how-to-force-cp-to-overwrite-directory-instead-of-creating-another-one-inside
+      # $ cp -TRv foo/ bar/
+      sh "cp -TR #{opt_gems} #{vendor_gems}"
+      # clean up opt compiled gems
+      FileUtils.rm_rf("#{code}/opt/ruby")
+    end
 
+    def sh(command)
+      puts "=> #{command}".colorize(:green)
+      success = system(command)
+      abort("Command Failed") unless success
+      success
     end
 
     # remove unnecessary files to reduce package size
     def tidy
+      # project_root: /tmp/jets/demo/stage/code/
+      # /tmp/jets/demo/stage/code/bundled
       tidy_gems("#{@options[:project_root]}/bundled/gems/ruby/*/gems/*")
       tidy_gems("#{@options[:project_root]}/bundled/gems/ruby/*/bundler/gems/*")
     end

data/lib/jets/builders/handler_generator.rb

@@ -14,21 +14,47 @@ class Jets::Builders
     end
 
     def build
-      common_base_shim
+      generate_data_yaml
       app_ruby_shims
       poly_shims
       shared_shims
       internal_shims
     end
 
+    # The handlers/data.yml is used by the shims
+    def generate_data_yaml
+      vars = Jets::Builders::ShimVars::Base.new
+      data = {
+        "s3_bucket" => vars.s3_bucket
+      }
+      data["rack_zip"] = vars.rack_zip if vars.rack_zip
+
+      content = YAML.dump(data)
+      path = "#{tmp_code}/handlers/data.yml"
+      FileUtils.mkdir_p(File.dirname(path))
+      IO.write(path, content)
+    end
+
     def app_ruby_shims
       app_files.each do |path|
-        # Generates one big node shim for a entire controller.
+        # Generates one shim for each app class: controller, job, etc
         vars = Jets::Builders::ShimVars::App.new(path)
-        generate_handler(vars)
+        if path.include?('app/functions')
+          copy_simple_function(path)
+        else
+          generate_handler(vars)
+        end
       end
     end
 
+    # source_path: app/functions/simple.rb
+    def copy_simple_function(source_path)
+      # Handler: handlers/controllers/posts_controller.handle
+      dest_path = source_path.sub('app/functions', 'handlers/functions')
+      FileUtils.mkdir_p(File.dirname(dest_path))
+      FileUtils.cp(source_path, dest_path)
+    end
+
     def app_files
       Jets::Commands::Build.app_files
     end
@@ -127,8 +153,8 @@ class Jets::Builders
     end
 
     def generate_handler(vars)
-      result = evaluate_template("handler.js", vars)
-      dest = "#{tmp_code}/#{vars.js_path}"
+      result = evaluate_template("handler.rb", vars)
+      dest = "#{tmp_code}/#{vars.dest_path}"
       FileUtils.mkdir_p(File.dirname(dest))
       IO.write(dest, result)
     end