jets 1.2.1 → 1.3.0

data/lib/jets/builders/lambda_layer.rb

@@ -0,0 +1,50 @@
+class Jets::Builders
+  class LambdaLayer
+    include Util
+
+    # At this point we gems have all been moved to stage/code/vendor/bundle, this includes
+    # binary gems, a good state. This method moves them:
+    #
+    #   from stage/code/vendor/bundle/ruby/2.5.0
+    #   to stage/opt/ruby/gems/2.5.0
+    #
+    # So we can move gems into the Lambda Layer. Important folders later:
+    #
+    #   stage/code/opt/lib
+    #   stage/code/opt/ruby
+    #
+    def build
+      move_opt_to_stage
+      move_vendor_to_opt
+      symlink_vendor_gems
+    end
+
+    def move_opt_to_stage
+      opt_original = "#{code_area}/opt"
+      opt = "#{stage_area}/opt"
+      FileUtils.mkdir_p(File.dirname(opt))
+      FileUtils.mv(opt_original, opt)
+    end
+
+    def move_vendor_to_opt
+      ruby_folder = Jets::Gems.ruby_folder
+      gems_original = "#{code_area}/vendor/bundle/ruby/#{ruby_folder}"
+      gems = "#{stage_area}/opt/ruby/gems/#{ruby_folder}"
+      FileUtils.mkdir_p(File.dirname(gems))
+      FileUtils.mv(gems_original, gems)
+      # Deleting in this way to make sure folders are empty before we delete them
+      FileUtils.rmdir("#{code_area}/vendor/bundle/ruby")
+      FileUtils.rmdir("#{code_area}/vendor/bundle")
+      FileUtils.rmdir("#{code_area}/vendor") if Dir.empty?("#{code_area}/vendor")
+    end
+
+    # Simple logic: vendor/bundle/ruby/2.5.0 -> /opt/ruby/gems/2.5.0
+    def symlink_vendor_gems
+      ruby_folder = Jets::Gems.ruby_folder
+      dest = "#{code_area}/vendor/bundle/ruby/#{ruby_folder}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      puts "ln -sf /opt/ruby/gems/#{ruby_folder} #{dest}" # uncommen to debug
+      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", dest)
+    end
+  end
+end

data/lib/jets/builders/md5.rb

@@ -4,7 +4,7 @@ require 'digest'
 # to reference files with the md5 checksum.  The files are the:
 #
 #   jets/code/rack-checksum.zip
-#   jets/code/bundled-checksum.zip
+#   jets/code/opt-checksum.zip
 #
 # We compute the checksums before we generate the node shim handlers.
 class Jets::Builders
@@ -17,14 +17,19 @@ class Jets::Builders
 
       def stage_folders
         paths = []
-        paths << "stage/bundled" if Jets.lazy_load?
-        paths << "stage/rack" if Jets.rack?
-        # Important to have stage/code at the end, since it will use the other
-        # 'symlinked' folders to adjust the md5 hash.
+        paths << "stage/opt" if folder_exist?("opt")
+        paths << "stage/rack" if folder_exist?("rack")
+        # Important to have stage/code at the end, since its md5 checksum depends
+        # on the previous folders.
         paths << "stage/code"
         paths
       end
 
+      def folder_exist?(folder)
+        path = "#{Jets.build_root}/stage/#{folder}"
+        File.directory?(path)
+      end
+
       def compute!
         stage_folders.each do |path|
           @@checksums[path] = dir(path)

data/lib/jets/builders/md5_zip.rb

@@ -20,6 +20,7 @@ class Jets::Builders
 
     def create
       headline "Creating zip file for #{@path}"
+      # => Creating zip file for /tmp/jets/demo/stage/bundled
 
       # https://serverfault.com/questions/265675/how-can-i-zip-compress-a-symlink
       command = "cd #{@path} && zip --symlinks -rq #{zip_file} ."

data/lib/jets/builders/purger.rb

@@ -0,0 +1,35 @@
+# When upgrading jets, automatically rm -rf /tmp/jets/project in case the structure has changed.
+class Jets::Builders
+  class Purger
+    def initialize
+      @project_name = Jets.config.project_name
+      @version_file = "/tmp/jets/#{@project_name}/jets_version.txt"
+    end
+
+    def purge
+      if version_changed?
+        last_version = @last_version || "unknown"
+        puts "The jets version has changed enough since the last build to merit freshing the build cache."
+        puts "Current jets version: #{Jets::VERSION} Last built jets version: #{last_version}"
+        puts "Removing /tmp/jets/#{@project_name} to start fresh."
+        FileUtils.rm_rf("/tmp/jets/#{@project_name}")
+      end
+      write_version
+    end
+
+    # When jets changes versions major or minor version consider it a big enough can to purge the cache
+    def version_changed?
+      return true unless File.exist?(@version_file)
+
+      @last_version = IO.read(@version_file).strip
+      last_major, last_minor, _ = @last_version.split('.')
+      current_major, current_minor, _ = Jets::VERSION.split('.')
+      last_major != current_major || last_minor != current_minor
+    end
+
+    def write_version
+      FileUtils.mkdir_p(File.dirname(@version_file))
+      IO.write(@version_file, Jets::VERSION)
+    end
+  end
+end

data/lib/jets/builders/rack_packager.rb

@@ -3,15 +3,17 @@ class Jets::Builders
     def finish
       return unless gemfile_exist?
 
-      symlink_rack_bundled
+      symlink_gems
+      rack_symlink
       copy_rackup_wrappers
     end
 
-    def symlink_rack_bundled
+    def symlink_gems
+      ruby_folder = Jets::Gems.ruby_folder
       # IE: @full_app_root: /tmp/jets/demo/stage/code/rack
-      rack_bundled = "#{@full_app_root}/bundled"
-      FileUtils.rm_f(rack_bundled) # looks like FileUtils.ln_sf doesnt remove existing symlinks
-      FileUtils.ln_sf("/var/task/bundled", rack_bundled)
+      dest = "#{@full_app_root}/vendor/bundle/ruby/#{ruby_folder}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", dest)
     end
 
     def copy_rackup_wrappers
@@ -25,5 +27,23 @@ class Jets::Builders
         FileUtils.chmod 0755, dest
       end
     end
+
+    # Moves folder to a stage folder and create a symlink its place
+    # that links from /var/task to /tmp. Example:
+    #
+    #   code_area/rack => /tmp/rack
+    #
+    def rack_symlink
+      src = "#{@full_app_root}/rack"
+      return unless File.exist?(src)
+
+      dest = "#{stage_area}/rack"
+      dir = File.dirname(dest)
+      FileUtils.mkdir_p(dir) unless File.exist?(dir)
+      FileUtils.mv(src, dest)
+
+      # Create symlink
+      FileUtils.ln_sf("/tmp/rack", "/#{@full_app_root}/rack")
+    end
   end
 end

data/lib/jets/builders/rackup_wrappers/rackup

@@ -16,8 +16,4 @@ unset BUNDLE_IGNORE_CONFIG
 # echo $$ > /tmp/jets-rackup.pid
 
 # Run the actual app using the bundled Ruby interpreter, with Bundler activated.
-if [ -n "$C9_USER" ]; then
-  exec ruby -rbundler/setup bin/rackup.rb "$@"
-else
-  exec "$PROJECTDIR/bundled/rbenv/shims/ruby" -rbundler/setup "$PROJECTDIR/bin/rackup.rb" "$@"
-fi
+exec ruby -rbundler/setup bin/rackup.rb "$@"

data/lib/jets/builders/ruby_packager.rb

@@ -18,12 +18,16 @@ class Jets::Builders
       setup_bundle_config
     end
 
+    #   build gems in vendor/bundle/ruby/2.5.0 (done in install phase)
+    # replace_compiled_gems:
+    #   remove binary gems in vendor/bundle/ruby/2.5.0
+    #   extract binary gems in opt/ruby/gems/2.5.0
+    #   move binary gems from opt/ruby/gems/2.5.0 to vendor/bundle/ruby/2.5.0
     def finish
       return unless gemfile_exist?
 
-      copy_bundled_cache
-      extract_ruby
-      extract_gems
+      copy_cache_gems
+      replace_compiled_gems
       tidy
     end
 
@@ -36,7 +40,7 @@ class Jets::Builders
     # If user is on a macosx machine, macosx gems will be installed.
     # If user is on a linux machine, linux gems will be installed.
     #
-    # Copies Gemfile* to /tmp/jetss/demo/bundled folder and installs
+    # Copies Gemfile* to /tmp/jets/demo/cache folder and installs
     # gems with bundle install from there.
     #
     # We take the time to copy Gemfile and bundle into a separate directory
@@ -44,8 +48,6 @@ class Jets::Builders
     # project gets built again not all the gems from get installed from the
     # beginning.
     def bundle_install
-      return if poly_only?
-
       full_project_path = @full_app_root
       headline "Bundling: running bundle install in cache area: #{cache_area}."
 
@@ -53,10 +55,9 @@ class Jets::Builders
 
       require "bundler" # dynamically require bundler so user can use any bundler
       Bundler.with_clean_env do
-        # cd /tmp/jets/demo
         sh(
           "cd #{cache_area} && " \
-          "env BUNDLE_IGNORE_CONFIG=1 bundle install --path #{cache_area}/bundled/gems --without development test"
+          "env BUNDLE_IGNORE_CONFIG=1 bundle install --path #{cache_area}/vendor/bundle --without development test"
         )
       end
 
@@ -95,7 +96,7 @@ class Jets::Builders
     end
 
     # When using submodules, bundler leaves old submodules behind. Over time this inflates
-    # the size of the the bundled gems.  So we'll clean it up.
+    # the size of the the cache gems.  So we'll clean it up.
     def clean_old_submodules
       # https://stackoverflow.com/questions/38800129/parsing-a-gemfile-lock-with-bundler
       lockfile = "#{cache_area}/Gemfile.lock"
@@ -117,8 +118,8 @@ class Jets::Builders
         md[1] # git_sha
       end
 
-      # IE: /tmp/jets/demo/cache/bundled/gems/ruby/2.5.0/bundler/gems/webpacker-a8c46614c675
-      Dir.glob("#{cache_area}/bundled/gems/ruby/2.5.0/bundler/gems/*").each do |path|
+      # IE: /tmp/jets/demo/cache/vendor/bundle/ruby/2.5.0/bundler/gems/webpacker-a8c46614c675
+      Dir.glob("#{cache_area}/vendor/bundle/ruby/2.5.0/bundler/gems/*").each do |path|
         sha = path.split('-').last[0..6] # only first 7 chars of the git sha
         unless git_shas.include?(sha)
           # puts "Removing old submoduled gem: #{path}" # uncomment to see and debug
@@ -139,7 +140,7 @@ class Jets::Builders
       # Override project's .bundle/config and ensure that .bundle/config matches
       # at these 2 spots:
       #   app_root/.bundle/config
-      #   bundled/gems/.bundle/config
+      #   vendor/bundle/.bundle/config
       cache_bundle_config = "#{cache_area}/.bundle/config"
       app_bundle_config = "#{@full_app_root}/.bundle/config"
       FileUtils.mkdir_p(File.dirname(app_bundle_config))
@@ -153,7 +154,8 @@ class Jets::Builders
     def ensure_build_cache_bundle_config_exists!
       text =<<-EOL
 ---
-BUNDLE_PATH: "bundled/gems"
+BUNDLE_FROZEN: "true"
+BUNDLE_PATH: "vendor/bundle"
 BUNDLE_WITHOUT: "development:test"
 EOL
       bundle_config = "#{cache_area}/.bundle/config"
@@ -169,25 +171,21 @@ EOL
       }
     end
 
-    def extract_ruby
-      headline "Setting up a vendored copy of ruby."
-      Jets::Gems::Extract::Ruby.new(Jets::RUBY_VERSION, gems_options).run
-    end
-
-    def extract_gems
+    def replace_compiled_gems
       headline "Replacing compiled gems with AWS Lambda Linux compiled versions."
       GemReplacer.new(Jets::RUBY_VERSION, gems_options).run
     end
 
-    def copy_bundled_cache
-      app_root_bundled = "#{@full_app_root}/bundled"
-      if File.exist?(app_root_bundled)
-        puts "Removing current bundled from project"
-        FileUtils.rm_rf(app_root_bundled)
+    def copy_cache_gems
+      vendor_bundle = "#{@full_app_root}/vendor/bundle"
+      if File.exist?(vendor_bundle)
+        puts "Removing current vendor_bundle from project"
+        FileUtils.rm_rf(vendor_bundle)
       end
-      # Leave #{Jets.build_root}/bundled behind to act as cache
-      if File.exist?("#{cache_area}/bundled")
-        FileUtils.cp_r("#{cache_area}/bundled", app_root_bundled)
+      # Leave #{Jets.build_root}/vendor_bundle behind to act as cache
+      if File.exist?("#{cache_area}/vendor/bundle")
+        FileUtils.mkdir_p(File.dirname(vendor_bundle))
+        FileUtils.cp_r("#{cache_area}/vendor/bundle", vendor_bundle)
       end
     end
 

data/lib/jets/builders/shim_vars/app.rb

@@ -8,7 +8,7 @@
 #
 #   functions: IE [:index, :show]
 #   handler_for(function_name): IE handlers/controllers/posts_controller.index
-#   js_path: IE: handlers/controllers/posts_controller.js
+#   dest_path: IE: handlers/controllers/posts_controller.js
 #
 module Jets::Builders::ShimVars
   class App < Base
@@ -69,10 +69,10 @@ module Jets::Builders::ShimVars
     end
 
     # Example return: "handlers/controllers/posts.js"
-    def js_path
+    # TODO: rename this to dest_path or something better now since using native ruby
+    def dest_path
       @relative_path
         .sub("app", "handlers")
-        .sub(/\.rb$/, ".js")
     end
   end
 end

data/lib/jets/builders/shim_vars/base.rb

@@ -8,13 +8,18 @@ module Jets::Builders::ShimVars
     end
 
     def rack_zip
-      checksum = Jets::Builders::Md5.checksums["stage/rack"]
-      "rack-#{checksum}.zip"
+      checksum_zip(:rack)
     end
 
     def bundled_zip
-      checksum = Jets::Builders::Md5.checksums["stage/bundled"]
-      "bundled-#{checksum}.zip"
+      checksum_zip(:bundled)
+    end
+
+  private
+    def checksum_zip(name)
+      checksum = Jets::Builders::Md5.checksums["stage/#{name}"]
+      return unless checksum
+      "#{name}-#{checksum}.zip"
     end
 
     def stage_area

data/lib/jets/builders/shim_vars/shared.rb

@@ -8,7 +8,7 @@
 #
 #   functions: IE [:index, :show]
 #   handler_for(function_name): IE handlers/controllers/posts_controller.index
-#   js_path: IE: handlers/controllers/posts_controller.js
+#   dest_path: IE: handlers/controllers/posts_controller.js
 #
 module Jets::Builders::ShimVars
   class Shared < Base
@@ -30,7 +30,7 @@ module Jets::Builders::ShimVars
     end
 
     # IE handlers/shared/functions/bob.js
-    def js_path
+    def dest_path
       @fun.handler_dest
     end
   end

data/lib/jets/builders/templates/handler.rb

@@ -0,0 +1,7 @@
+require "bundler/setup"
+require "jets"
+Jets.once  # runs once in lambda execution context
+
+<% @vars.functions.each do |function_name| -%>
+Jets.handler(self, "<%= @vars.handler_for(function_name) %>")
+<% end %>

data/lib/jets/builders/tidy.rb

@@ -48,9 +48,9 @@ class Jets::Builders
       (defaults + keep).uniq
     end
 
-    # folders to remove in the bundled folder regardless of the level of the folder
+    # folders to remove in the vendor/bundle folder regardless of the level of the folder
     def tidy_bundled
-      Dir.glob("#{@project_root}/bundled/**/*").each do |path|
+      Dir.glob("#{@project_root}/vendor/bundle/**/*").each do |path|
         next unless File.directory?(path)
         dir = File.basename(path)
         next unless always_removals.include?(dir)
@@ -65,7 +65,8 @@ class Jets::Builders
     # actual cache folders that they used.
     def remove_gem_cache
       ruby_minor_version = Jets::RUBY_VERSION.split('.')[0..1].join('.') + '.0'
-      FileUtils.rm_rf("#{@project_root}/bundled/gems/ruby/#{ruby_minor_version}/cache")
+      cache_path = "#{@project_root}/vendor/bundle/ruby/#{ruby_minor_version}/cache"
+      FileUtils.rm_rf(cache_path)
     end
 
     def rm_rf(path)

data/lib/jets/builders/util.rb

@@ -3,7 +3,11 @@ class Jets::Builders
     def sh(command)
       puts "=> #{command}".colorize(:green)
       success = system(command)
-      abort("#{command} failed to run") unless success
+      unless success
+        puts "#{command} failed to run.".colorize(:red)
+        puts caller[0]
+        exit 1
+      end
       success
     end
 
@@ -17,8 +21,12 @@ class Jets::Builders
       "#{Jets.build_root}/#{relative_path}"
     end
 
-    def poly_only?
-      Jets.poly_only?
+    def stage_area
+      "#{Jets.build_root}/stage"
+    end
+
+    def code_area
+      "#{stage_area}/code"
     end
   end
 end

data/lib/jets/cfn/builders/base_child_builder.rb

@@ -22,8 +22,11 @@ class Jets::Cfn::Builders
     end
 
     def add_common_parameters
-      add_parameter("IamRole", Description: "Iam Role that Lambda function uses.")
-      add_parameter("S3Bucket", Description: "S3 Bucket for source code.")
+      common_parameters = Jets::Resource::ChildStack::AppClass.common_parameters
+      common_parameters.each do |k,_|
+        add_parameter(k, Description: k)
+      end
+
       depends_on_params.each do |logical_id, desc|
         add_parameter(logical_id, Description: desc)
       end
@@ -58,7 +61,7 @@ class Jets::Cfn::Builders
     end
 
     def add_function(task)
-      resource = Jets::Resource::Function.new(task)
+      resource = Jets::Resource::Lambda::Function.new(task)
       add_resource(resource)
     end