jets-fs 1.6.10

data/lib/jets/builders/code_size.rb

@@ -0,0 +1,57 @@
+class Jets::Builders
+  class CodeSize
+    LAMBDA_SIZE_LIMIT = 250 # Total lambda limit is 250MB
+    include Util
+
+    def self.check!
+      new.check
+    end
+
+    def check
+      return if within_lambda_limit?
+      say "Over the Lambda size limit of #{LAMBDA_SIZE_LIMIT}MB".color(:red)
+      say "Please reduce the size of your code."
+      display_sizes
+      exit 1
+    end
+
+    def within_lambda_limit?
+      total_size < LAMBDA_SIZE_LIMIT * 1024 # 120MB
+    end
+
+    def total_size
+      code_size = compute_size("#{stage_area}/code")
+      opt_size = compute_size("#{stage_area}/opt")
+      opt_size + code_size # total_size
+    end
+
+    def display_sizes
+      code_size = compute_size("#{stage_area}/code")
+      opt_size = compute_size("#{stage_area}/opt")
+      total_size = opt_size + code_size
+      overlimit = (LAMBDA_SIZE_LIMIT * 1024 - total_size) * -1
+      say "Sizes:"
+      say "Code: #{megabytes(code_size)} - #{stage_area}/code"
+      say "Gem Layer: #{megabytes(opt_size)} - #{stage_area}/opt"
+      say "Total Package: #{megabytes(total_size)}"
+      say "Over limit by: #{megabytes(overlimit)}"
+      say "Sometimes blowing away the /tmp/jets cache will reduce the size: rm -rf /tmp/jets"
+      # sh "du -kcsh #{stage_area}/*" unless ENV['TEST'] # uncomment to debug
+    end
+
+    def compute_size(path)
+      # -k option is required for macosx but not for linux
+      out = `du -ks #{path}`
+      out.split(' ').first.to_i # bytes
+    end
+
+    def megabytes(bytes)
+      n = bytes / 1024.0
+      sprintf('%.1f', n) + 'MB'
+    end
+
+    def say(message)
+      puts message unless ENV['TEST']
+    end
+  end
+end

data/lib/jets/builders/gem_replacer.rb

@@ -0,0 +1,76 @@
+class Jets::Builders
+  class GemReplacer
+    extend Memoist
+    def initialize(options)
+      @options = options
+    end
+
+    def run
+      check = Jets::Gems::Check.new
+      if Jets.config.lambda.layers.empty?
+        found_gems = check.run! # exits early if missing gems found
+      else
+        # assumes missing gems are in the provided custom layer by the user
+        found_gems = check.run # does not exist early
+      end
+
+      # found gems will only have gems that were found
+      found_gems.each do |gem_name, source|
+        options = @options.merge(source_url: source)
+        gem_extractor = Jets::Gems::Extract::Gem.new(gem_name, options)
+        gem_extractor.run
+      end
+
+      tidy
+    end
+
+    def sh(command)
+      puts "=> #{command}".color(:green)
+      success = system(command)
+      abort("Command Failed") unless success
+      success
+    end
+
+    def ruby_folder
+      Jets::Gems.ruby_folder
+    end
+
+    # remove unnecessary files to reduce package size
+    def tidy
+      # project_root: /tmp/jets/demo/stage/code/
+      # /tmp/jets/demo/stage/code/bundled
+      tidy_gems("#{@options[:project_root]}/ruby/gems/#{ruby_folder}/*/gems/*")
+      tidy_gems("#{@options[:project_root]}/ruby/gems/#{ruby_folder}/*/bundler/gems/*")
+    end
+
+    def tidy_gems(gems_path)
+      Dir.glob(gems_path).each do |gem_path|
+        tidy_gem(gem_path)
+      end
+    end
+
+    # Clean up some unneeded files to try to keep the package size down
+    # In a generated jets app this made a decent 9% difference:
+    #  175M test2
+    #  191M test3
+    def tidy_gem(path)
+      # remove top level tests and cache folders
+      Dir.glob("#{path}/*").each do |path|
+        next unless File.directory?(path)
+        folder = File.basename(path)
+        if %w[test tests spec features benchmark cache doc].include?(folder)
+          FileUtils.rm_rf(path)
+        end
+      end
+
+      Dir.glob("#{path}/**/*").each do |path|
+        next unless File.file?(path)
+        ext = File.extname(path)
+        if %w[.rdoc .md .markdown].include?(ext) or
+           path =~ /LICENSE|CHANGELOG|README/
+          FileUtils.rm_f(path)
+        end
+      end
+    end
+  end
+end

data/lib/jets/builders/handler_generator.rb

@@ -0,0 +1,172 @@
+require "fileutils"
+require "erb"
+
+# Example:
+#
+# Jets::Builders::HandlerGenerator.new(
+#   "PostsController",
+#   :create, :update
+# )
+class Jets::Builders
+  class HandlerGenerator
+    def self.build!
+      new.build
+    end
+
+    def build
+      generate_data_yaml
+      app_ruby_shims
+      poly_shims
+      shared_shims
+      internal_shims
+    end
+
+    # The handlers/data.yml is used by the shims
+    def generate_data_yaml
+      vars = Jets::Builders::ShimVars::Base.new
+      data = {
+        "s3_bucket" => vars.s3_bucket
+      }
+      data["rack_zip"] = vars.rack_zip if vars.rack_zip
+
+      content = YAML.dump(data)
+      path = "#{tmp_code}/handlers/data.yml"
+      FileUtils.mkdir_p(File.dirname(path))
+      IO.write(path, content)
+    end
+
+    def app_ruby_shims
+      app_files.each do |path|
+        # Generates one shim for each app class: controller, job, etc
+        vars = Jets::Builders::ShimVars::App.new(path)
+        if path.include?('app/functions')
+          copy_simple_function(path)
+        else
+          generate_handler(vars)
+        end
+      end
+    end
+
+    # source_path: app/functions/simple.rb
+    def copy_simple_function(source_path)
+      # Handler: handlers/controllers/posts_controller.handle
+      dest_path = source_path.sub('app/functions', 'handlers/functions')
+      FileUtils.mkdir_p(File.dirname(dest_path))
+      FileUtils.cp(source_path, dest_path)
+    end
+
+    def app_files
+      Jets::Commands::Build.app_files
+    end
+
+    def poly_shims
+      missing = []
+
+      app_files.each do |path|
+        vars = Jets::Builders::ShimVars::App.new(path)
+        poly_tasks = vars.klass.tasks.select { |t| t.lang != :ruby }
+        poly_tasks.each do |task|
+          source_path = get_source_path(path, task)
+          if File.exist?(source_path)
+            native_function(path, task)
+          else
+            missing << source_path
+          end
+        end
+
+        unless missing.empty?
+          puts "ERROR: Missing source files. Please make sure these source files exist or remove their declarations".color(:red)
+          puts missing
+          exit 1
+        end
+      end
+    end
+
+    def shared_shims
+      Jets::Stack.subclasses.each do |subclass|
+        subclass.functions.each do |fun|
+          if fun.lang.to_s == "ruby"
+            shared_ruby_shim(fun)
+          else
+            copy_source_as_handler(fun)
+          end
+        end
+      end
+    end
+
+    def internal_shims
+      jets_base_path if Jets.custom_domain?
+    end
+
+    def jets_base_path
+      path = "jets/base_path.rb"
+      internal = File.expand_path("../internal", File.dirname(__FILE__))
+      src = "#{internal}/app/functions/#{path}"
+      result = Jets::Erb.result(src, stage_name: Jets::Resource::ApiGateway::Deployment.stage_name)
+      dest = "#{tmp_code}/handlers/functions/#{path}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
+    end
+
+    # app/shared/functions/kevin.py => /tmp/jets/demo/app_root/handlers/shared/functions/kevin.py
+    def copy_source_as_handler(fun)
+      source_path = fun.source_file
+      unless source_path
+        attributes = fun.template.values.first
+        function_name = attributes['Properties']['FunctionName']
+        puts "WARN: missing source file for: '#{function_name}' function".color(:yellow)
+        return
+      end
+
+      dest_path = "#{tmp_code}/#{fun.handler_dest}"
+      FileUtils.mkdir_p(File.dirname(dest_path))
+      FileUtils.cp(source_path, dest_path)
+    end
+
+    def get_source_path(original_path, task)
+      folder = original_path.sub(/\.rb$/,'')
+      lang_folder = "#{folder}/#{task.lang}"
+      root = Jets.root unless original_path.include?("lib/jets/internal")
+      "#{root}/#{lang_folder}/#{task.meth}#{task.lang_ext}"
+    end
+
+    # Builds and copies over the native source code: python or node
+    def native_function(original_path, task)
+      source_path = get_source_path(original_path, task)
+      # Handler: handlers/controllers/posts_controller.handle
+      dest_path = "#{tmp_code}/#{task.handler_path}"
+      FileUtils.mkdir_p(File.dirname(dest_path))
+      FileUtils.cp(source_path, dest_path)
+    end
+
+    def shared_ruby_shim(fun)
+      vars = Jets::Builders::ShimVars::Shared.new(fun)
+      generate_handler(vars)
+    end
+
+    def common_base_shim
+      vars = Jets::Builders::ShimVars::Base.new
+      result = evaluate_template("shim.js", vars)
+      dest = "#{tmp_code}/handlers/shim.js"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
+    end
+
+    def generate_handler(vars)
+      result = evaluate_template("handler.rb", vars)
+      dest = "#{tmp_code}/#{vars.dest_path}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
+    end
+
+    def evaluate_template(template_file, vars)
+      template_path = File.expand_path("../templates/#{template_file}", __FILE__)
+      Jets::Erb.result(template_path, vars: vars)
+    end
+
+    # TODO: move CodeBuilder.tmp_code to a common level for HandlerGenerator and CodeBuilder
+    def tmp_code
+      "#{Jets.build_root}/#{CodeBuilder.tmp_code}"
+    end
+  end
+end

data/lib/jets/builders/lambda_layer.rb

@@ -0,0 +1,69 @@
+class Jets::Builders
+  class LambdaLayer
+    include Util
+
+    # At this point gems are in the stage/code and stage/rack folders still.
+    # We consolidate all gems to stage/opt.
+    # Then replace the binary gems.
+    def build
+      consolidate_gems_to_opt
+      replace_compiled_gems
+    end
+
+    # Also restructure the folder from:
+    #   vendor/gems/ruby/2.5.0
+    # To:
+    #   ruby/gems/2.5.0
+    #
+    # For Lambda Layer structure
+    def consolidate_gems_to_opt
+      src = "#{stage_area}/code/vendor/gems/ruby/2.5.0"
+      dest = "#{stage_area}/opt/ruby/gems/2.5.0"
+      rsync_and_link(src, dest)
+
+      return unless Jets.rack?
+
+      src = "#{stage_area}/rack/vendor/gems/ruby/2.5.0"
+      rsync_and_link(src, dest)
+    end
+
+    def rsync_and_link(src, dest)
+      FileUtils.mkdir_p(dest)
+      # Trailing slashes are required
+      sh "rsync -a --links #{src}/ #{dest}/"
+
+      FileUtils.rm_rf(src) # blow away original 2.5.0 folder
+
+      # Create symlink that will point to the gems in the Lambda Layer:
+      #   stage/opt/ruby/gems/2.5.0 -> /opt/ruby/gems/2.5.0
+      FileUtils.ln_sf("/opt/ruby/gems/#{Jets::Gems.ruby_folder}", src)
+    end
+
+    # replace_compiled_gems:
+    #   remove binary gems in vendor/gems/ruby/2.5.0
+    #   extract binary gems in opt/ruby/gems/2.5.0
+    #   move binary gems from opt/ruby/gems/2.5.0 to vendor/gems/ruby/2.5.0
+    #
+    # After this point, gems have been replace in stage/code/vendor/gems with their
+    # binary extensions: a good state. This method moves these gems to the Lambda
+    # Layers structure and creates a symlinks to it.  First:
+    #
+    #   from stage/code/vendor/gems/ruby/2.5.0
+    #   to stage/opt/ruby/gems/2.5.0
+    #
+    # Then:
+    #
+    #   stage/code/vendor/gems/ruby/2.5.0 -> /opt/ruby/gems/2.5.0
+    #
+    def replace_compiled_gems
+      project_root = "#{stage_area}/opt"
+      headline "Replacing compiled gems with AWS Lambda Linux compiled versions: #{project_root}"
+      options = {
+        s3: "lambdagems2",
+        build_root: cache_area, # used in jets-gems
+        project_root: project_root, # used in gem_replacer and jets-gems
+      }
+      GemReplacer.new(options).run
+    end
+  end
+end

data/lib/jets/builders/md5.rb

@@ -0,0 +1,60 @@
+require 'digest'
+
+# Resolves the chicken-and-egg problem with md5 checksums. The handlers need
+# to reference files with the md5 checksum.  The files are the:
+#
+#   jets/code/rack-checksum.zip
+#   jets/code/opt-checksum.zip
+#
+# We compute the checksums before we generate the node shim handlers.
+class Jets::Builders
+  class Md5
+    class << self
+      @@checksums = {}
+      def checksums
+        @@checksums
+      end
+
+      def stage_folders
+        paths = []
+        paths << "stage/opt" if folder_exist?("opt")
+        paths << "stage/rack" if folder_exist?("rack")
+        # Important to have stage/code at the end, since its md5 checksum depends
+        # on the previous folders.
+        paths << "stage/code"
+        paths
+      end
+
+      def folder_exist?(folder)
+        path = "#{Jets.build_root}/stage/#{folder}"
+        File.directory?(path)
+      end
+
+      def compute!
+        stage_folders.each do |path|
+          @@checksums[path] = dir(path)
+        end
+        @@checksums
+      end
+
+      def dir(short_path)
+        path = "#{Jets.build_root}/#{short_path}"
+        files = Dir["#{path}/**/*"]
+        files.reject! { |f| File.directory?(f) }
+             .reject! { |f| File.symlink?(f) }
+        content = files.map do |f|
+          Digest::MD5.file(f).to_s[0..7]
+        end.join
+
+        # The stage/code md5 sha depends on the other 'symlinked' folders.
+        if short_path == "stage/code"
+          content += @@checksums.values.join
+        end
+
+        md5 = Digest::MD5.new
+        md5.update(content)
+        md5.hexdigest.to_s[0..7]
+      end
+    end
+  end
+end

data/lib/jets/builders/md5_zip.rb

@@ -0,0 +1,61 @@
+require 'action_view'
+
+# Examples:
+#
+#   zip = Jets::Builders::Md5Zip.new("/tmp/jets/demo/stage/code")
+#   zip.create
+#
+#   zip = Jets::Builders::Md5Zip.new("/tmp/jets/demo/stage/bundled")
+#   zip.create
+#
+class Jets::Builders
+  class Md5Zip
+    include ActionView::Helpers::NumberHelper # number_to_human_size
+    include Util
+
+    def initialize(folder)
+      @path = "#{Jets.build_root}/#{folder}"
+      @checksum = Md5.checksums[folder]
+    end
+
+    def create
+      headline "Creating zip file for #{@path}"
+      # => Creating zip file for /tmp/jets/demo/stage/bundled
+
+      # https://serverfault.com/questions/265675/how-can-i-zip-compress-a-symlink
+      command = "cd #{@path} && zip --symlinks -rq #{zip_file} ."
+      sh(command)
+      # move out of the lower folder to the stage folder
+      # mv /tmp/jets/demo/stage/code/code.zip /tmp/jets/demo/stage/code.zip
+      FileUtils.mkdir_p(File.dirname(zip_dest))
+      FileUtils.mv("#{@path}/#{zip_file}", zip_dest)
+
+      # mv /tmp/jets/demo/stage/zips/code.zip /tmp/jets/demo/stage/zips/code-a8a604aa.zip
+      FileUtils.mv(zip_dest, md5_dest)
+
+      file_size = number_to_human_size(File.size(md5_dest))
+      puts "Zip file created at: #{md5_dest.color(:green)} (#{file_size})"
+    end
+
+    # /tmp/jets/demo/stage/zips/code.zip
+    def zip_dest
+      stage_area, filename = File.dirname(@path), File.basename(@path)
+      zip_area = stage_area + '/zips' # /tmp/jets/demo/stage/zips
+      zip_file = filename + '.zip' # code.zip
+      "#{zip_area}/#{zip_file}" # /tmp/jets/demo/stage/zips/code.zip
+    end
+
+    def zip_file
+      File.basename(zip_dest)
+    end
+
+    # /tmp/jets/demo/stage/zips/code-SHA.zip
+    def md5_dest
+      zip_dest.sub(".zip", "-#{@checksum}.zip")
+    end
+
+    def md5_name
+      File.basename(md5_dest)
+    end
+  end
+end