jets.benforeva 3.0.17.pre.mount.pre.fix

data/lib/jets/builders/handler_generator.rb

@@ -0,0 +1,186 @@
+require "fileutils"
+require "erb"
+
+# Example:
+#
+# Jets::Builders::HandlerGenerator.new(
+#   "PostsController",
+#   :create, :update
+# )
+module Jets::Builders
+  class HandlerGenerator
+    def self.build!
+      new.build
+    end
+
+    def build
+      generate_data_yaml
+      app_ruby_shims
+      poly_shims
+      shared_shims
+      internal_shims
+    end
+
+    # The handlers/data.yml is used by the shims
+    def generate_data_yaml
+      vars = Jets::Builders::ShimVars::Base.new
+      data = {
+        "s3_bucket" => vars.s3_bucket
+      }
+      data["rack_zip"] = vars.rack_zip if vars.rack_zip
+
+      content = YAML.dump(data)
+      path = "#{tmp_code}/handlers/data.yml"
+      FileUtils.mkdir_p(File.dirname(path))
+      IO.write(path, content)
+    end
+
+    def app_ruby_shims
+      app_files.each do |path|
+        # Generates one shim for each app class: controller, job, etc
+        vars = Jets::Builders::ShimVars::App.new(path)
+        if path.include?('app/functions')
+          copy_simple_function(path)
+        else
+          generate_handler(vars)
+        end
+      end
+    end
+
+    # source_path: app/functions/simple.rb
+    def copy_simple_function(source_path)
+      # Handler: handlers/controllers/posts_controller.handle
+      dest_path = source_path.sub('app/functions', 'handlers/functions')
+      FileUtils.mkdir_p(File.dirname(dest_path))
+      FileUtils.cp(source_path, dest_path)
+    end
+
+    def app_files
+      Jets::Commands::Build.app_files
+    end
+
+    def poly_shims
+      missing = []
+
+      app_files.each do |path|
+        vars = Jets::Builders::ShimVars::App.new(path)
+        poly_tasks = vars.klass.tasks.select { |t| t.lang != :ruby }
+        poly_tasks.each do |task|
+          source_path = get_source_path(path, task)
+          if File.exist?(source_path)
+            native_function(path, task)
+          else
+            missing << source_path
+          end
+        end
+
+        unless missing.empty?
+          puts "ERROR: Missing source files. Please make sure these source files exist or remove their declarations".color(:red)
+          puts missing
+          exit 1
+        end
+      end
+    end
+
+    def shared_shims
+      Jets::Stack.subclasses.each do |subclass|
+        subclass.functions.each do |fun|
+          if fun.lang.to_s == "ruby"
+            shared_ruby_shim(fun)
+          else
+            copy_source_as_handler(fun)
+          end
+        end
+      end
+    end
+
+    def internal_shims
+      jets_base_path if Jets.custom_domain?
+      s3_bucket_config if Jets.s3_event?
+    end
+
+    def jets_base_path
+      copy_function_template("functions/jets/base_path.rb", stage_name: Jets::Resource::ApiGateway::Deployment.stage_name)
+    end
+
+    def s3_bucket_config
+      copy_function_template("shared/functions/jets/s3_bucket_config.rb")
+    end
+
+    # Copy code from internal folder to materialized app code
+    def copy_function_template(path, vars={})
+      internal = File.expand_path("../internal", File.dirname(__FILE__))
+      src = "#{internal}/app/#{path}"
+      result = Jets::Erb.result(src, vars)
+      dest = "#{tmp_code}/handlers/#{path}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
+    end
+
+    # app/shared/functions/kevin.py => /tmp/jets/demo/app_root/handlers/shared/functions/kevin.py
+    def copy_source_as_handler(fun)
+      return if fun.internal?
+
+      source_path = fun.source_file
+      unless source_path
+        attributes = fun.template.values.first
+        function_name = attributes['Properties']['FunctionName']
+        puts "WARN: missing source file for: '#{function_name}' function".color(:yellow)
+        return
+      end
+
+      dest_path = "#{tmp_code}/#{fun.handler_dest}"
+      FileUtils.mkdir_p(File.dirname(dest_path))
+      FileUtils.cp(source_path, dest_path)
+    end
+
+    def get_source_path(original_path, task)
+      folder = original_path.sub(/\.rb$/,'')
+      lang_folder = "#{folder}/#{task.lang}"
+      root = Jets.root unless original_path.include?("lib/jets/internal")
+      "#{root}/#{lang_folder}/#{task.meth}#{task.lang_ext}"
+    end
+
+    # Builds and copies over the native source code: python or node
+    def native_function(original_path, task)
+      source_path = get_source_path(original_path, task)
+      # Handler: handlers/controllers/posts_controller.handle
+      dest_path = "#{tmp_code}/#{task.handler_path}"
+      FileUtils.mkdir_p(File.dirname(dest_path))
+      FileUtils.cp(source_path, dest_path)
+    end
+
+    def shared_ruby_shim(fun)
+      # Cant use native_function because that requires task. Just re-implement
+      dest_path = fun.handler_dest
+      source_path = dest_path.sub(/^handlers/,'app')
+      FileUtils.mkdir_p(File.dirname(dest_path))
+      FileUtils.cp(source_path, dest_path)
+    end
+
+    def common_base_shim
+      vars = Jets::Builders::ShimVars::Base.new
+      result = evaluate_template("shim.js", vars)
+      dest = "#{tmp_code}/handlers/shim.js"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
+    end
+
+    def generate_handler(vars)
+      result = evaluate_template("handler.rb", vars)
+      dest = "#{tmp_code}/#{vars.dest_path}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
+    end
+
+    def evaluate_template(template_file, vars)
+      template_path = File.expand_path("../templates/#{template_file}", __FILE__)
+      Jets::Erb.result(template_path, vars: vars)
+    end
+
+    # TODO: move CodeBuilder.tmp_code to a common level for HandlerGenerator and CodeBuilder
+    def tmp_code
+      "#{Jets.build_root}/#{CodeBuilder.tmp_code}"
+    end
+  end
+end

data/lib/jets/builders/lambda_layer.rb

@@ -0,0 +1,68 @@
+module Jets::Builders
+  class LambdaLayer
+    include Util
+
+    # At this point gems are in the stage/code and stage/rack folders still.
+    # We consolidate all gems to stage/opt.
+    # Then replace the binary gems.
+    def build
+      consolidate_gems_to_opt
+      replace_compiled_gems unless Jets.config.gems.disable
+    end
+
+    # Also restructure the folder from:
+    #   vendor/gems/ruby/2.5.0
+    # To:
+    #   ruby/gems/2.5.0
+    #
+    # For Lambda Layer structure
+    def consolidate_gems_to_opt
+      src = "#{stage_area}/code/vendor/gems/ruby/#{Jets.ruby_folder}"
+      dest = "#{stage_area}/opt/ruby/gems/#{Jets.ruby_folder}"
+      rsync_and_link(src, dest)
+
+      return unless Jets.rack?
+
+      src = "#{stage_area}/rack/vendor/gems/ruby/#{Jets.ruby_folder}"
+      rsync_and_link(src, dest)
+    end
+
+    def rsync_and_link(src, dest)
+      FileUtils.mkdir_p(dest)
+      # Trailing slashes are required
+      sh "rsync -a --links #{src}/ #{dest}/"
+
+      FileUtils.rm_rf(src) # blow away original 2.5.0 folder
+
+      # Create symlink that will point to the gems in the Lambda Layer:
+      #   stage/opt/ruby/gems/2.5.0 -> /opt/ruby/gems/2.5.0
+      FileUtils.ln_sf("/opt/ruby/gems/#{Jets::Gems.ruby_folder}", src)
+    end
+
+    # replace_compiled_gems:
+    #   remove binary gems in vendor/gems/ruby/2.5.0
+    #   extract binary gems in opt/ruby/gems/2.5.0
+    #   move binary gems from opt/ruby/gems/2.5.0 to vendor/gems/ruby/2.5.0
+    #
+    # After this point, gems have been replace in stage/code/vendor/gems with their
+    # binary extensions: a good state. This method moves these gems to the Lambda
+    # Layers structure and creates a symlinks to it.  First:
+    #
+    #   from stage/code/vendor/gems/ruby/2.5.0
+    #   to stage/opt/ruby/gems/2.5.0
+    #
+    # Then:
+    #
+    #   stage/code/vendor/gems/ruby/2.5.0 -> /opt/ruby/gems/2.5.0
+    #
+    def replace_compiled_gems
+      project_root = "#{stage_area}/opt"
+      headline "Replacing compiled gems with AWS Lambda Linux compiled versions: #{project_root}"
+      options = {
+        build_root: cache_area, # used in jets-gems
+        project_root: project_root, # used in gem_replacer and jets-gems
+      }
+      GemReplacer.new(options).run
+    end
+  end
+end

data/lib/jets/builders/md5.rb

@@ -0,0 +1,60 @@
+require 'digest'
+
+# Resolves the chicken-and-egg problem with md5 checksums. The handlers need
+# to reference files with the md5 checksum.  The files are the:
+#
+#   jets/code/rack-checksum.zip
+#   jets/code/opt-checksum.zip
+#
+# We compute the checksums before we generate the node shim handlers.
+module Jets::Builders
+  class Md5
+    class << self
+      @@checksums = {}
+      def checksums
+        @@checksums
+      end
+
+      def stage_folders
+        paths = []
+        paths << "stage/opt" if folder_exist?("opt")
+        paths << "stage/rack" if folder_exist?("rack")
+        # Important to have stage/code at the end, since its md5 checksum depends
+        # on the previous folders.
+        paths << "stage/code"
+        paths
+      end
+
+      def folder_exist?(folder)
+        path = "#{Jets.build_root}/stage/#{folder}"
+        File.directory?(path)
+      end
+
+      def compute!
+        stage_folders.each do |path|
+          @@checksums[path] = dir(path)
+        end
+        @@checksums
+      end
+
+      def dir(short_path)
+        path = "#{Jets.build_root}/#{short_path}"
+        files = Dir["#{path}/**/*"]
+        files = files.reject { |f| File.directory?(f) }
+                     .reject { |f| File.symlink?(f) }
+        content = files.map do |f|
+          Digest::MD5.file(f).to_s[0..7]
+        end.join
+
+        # The stage/code md5 sha depends on the other 'symlinked' folders.
+        if short_path == "stage/code"
+          content += @@checksums.values.join
+        end
+
+        md5 = Digest::MD5.new
+        md5.update(content)
+        md5.hexdigest.to_s[0..7]
+      end
+    end
+  end
+end

data/lib/jets/builders/md5_zip.rb

@@ -0,0 +1,61 @@
+require "active_support/number_helper"
+
+# Examples:
+#
+#   zip = Jets::Builders::Md5Zip.new("/tmp/jets/demo/stage/code")
+#   zip.create
+#
+#   zip = Jets::Builders::Md5Zip.new("/tmp/jets/demo/stage/bundled")
+#   zip.create
+#
+module Jets::Builders
+  class Md5Zip
+    include ActiveSupport::NumberHelper # number_to_human_size
+    include Util
+
+    def initialize(folder)
+      @path = "#{Jets.build_root}/#{folder}"
+      @checksum = Md5.checksums[folder]
+    end
+
+    def create
+      headline "Creating zip file for #{@path}"
+      # => Creating zip file for /tmp/jets/demo/stage/bundled
+
+      # https://serverfault.com/questions/265675/how-can-i-zip-compress-a-symlink
+      command = "cd #{@path} && chmod -R 755 . && zip --symlinks -rq #{zip_file} ."
+      sh(command)
+      # move out of the lower folder to the stage folder
+      # mv /tmp/jets/demo/stage/code/code.zip /tmp/jets/demo/stage/code.zip
+      FileUtils.mkdir_p(File.dirname(zip_dest))
+      FileUtils.mv("#{@path}/#{zip_file}", zip_dest)
+
+      # mv /tmp/jets/demo/stage/zips/code.zip /tmp/jets/demo/stage/zips/code-a8a604aa.zip
+      FileUtils.mv(zip_dest, md5_dest)
+
+      file_size = number_to_human_size(File.size(md5_dest))
+      puts "Zip file created at: #{md5_dest.color(:green)} (#{file_size})"
+    end
+
+    # /tmp/jets/demo/stage/zips/code.zip
+    def zip_dest
+      stage_area, filename = File.dirname(@path), File.basename(@path)
+      zip_area = stage_area + '/zips' # /tmp/jets/demo/stage/zips
+      zip_file = filename + '.zip' # code.zip
+      "#{zip_area}/#{zip_file}" # /tmp/jets/demo/stage/zips/code.zip
+    end
+
+    def zip_file
+      File.basename(zip_dest)
+    end
+
+    # /tmp/jets/demo/stage/zips/code-SHA.zip
+    def md5_dest
+      zip_dest.sub(".zip", "-#{@checksum}.zip")
+    end
+
+    def md5_name
+      File.basename(md5_dest)
+    end
+  end
+end

data/lib/jets/builders/purger.rb

@@ -0,0 +1,35 @@
+# When upgrading jets, automatically rm -rf /tmp/jets/project in case the structure has changed.
+module Jets::Builders
+  class Purger
+    def initialize
+      @project_name = Jets.config.project_name
+      @version_file = "/tmp/jets/#{@project_name}/jets_version.txt"
+    end
+
+    def purge
+      if version_changed?
+        last_version = @last_version || "unknown"
+        puts "The jets version has changed enough since the last build to merit refreshing the build cache."
+        puts "Current jets version: #{Jets::VERSION} Last built jets version: #{last_version}"
+        puts "Removing /tmp/jets/#{@project_name} to start fresh."
+        FileUtils.rm_rf("/tmp/jets/#{@project_name}")
+      end
+      write_version
+    end
+
+    # When jets changes versions major or minor version consider it a big enough can to purge the cache
+    def version_changed?
+      return true unless File.exist?(@version_file)
+
+      @last_version = IO.read(@version_file).strip
+      last_major, last_minor, _ = @last_version.split('.')
+      current_major, current_minor, _ = Jets::VERSION.split('.')
+      last_major != current_major || last_minor != current_minor
+    end
+
+    def write_version
+      FileUtils.mkdir_p(File.dirname(@version_file))
+      IO.write(@version_file, Jets::VERSION)
+    end
+  end
+end

data/lib/jets/builders/rack_packager.rb

@@ -0,0 +1,51 @@
+module Jets::Builders
+  class RackPackager < RubyPackager
+    def finish
+      return unless gemfile_exist?
+
+      super
+
+      symlink_gems
+      copy_rackup_wrappers
+      rack_symlink
+    end
+
+    def symlink_gems
+      ruby_folder = Jets::Gems.ruby_folder
+      # IE: @full_app_root: /tmp/jets/demo/stage/code/rack
+      dest = "#{@full_app_root}/vendor/gems/ruby/#{ruby_folder}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", dest)
+    end
+
+    def copy_rackup_wrappers
+      # IE: @full_app_root: /tmp/jets/demo/stage/code/rack
+      rack_bin = "#{@full_app_root}/bin"
+      %w[rackup rackup.rb].each do |file|
+        src = File.expand_path("./rackup_wrappers/#{file}", File.dirname(__FILE__))
+        dest = "#{rack_bin}/#{file}"
+        FileUtils.mkdir_p(rack_bin) unless File.exist?(rack_bin)
+        FileUtils.cp(src, dest)
+        FileUtils.chmod 0755, dest
+      end
+    end
+
+    # Moves folder to a stage folder and create a symlink its place
+    # that links from /var/task to /tmp. Example:
+    #
+    #   stage/code/rack => /tmp/rack
+    #
+    def rack_symlink
+      src = @full_app_root
+      return unless File.exist?(src)
+
+      dest = "#{stage_area}/rack"
+      dir = File.dirname(dest)
+      FileUtils.mkdir_p(dir) unless File.exist?(dir)
+      FileUtils.mv(src, dest)
+
+      # Create symlink
+      FileUtils.ln_sf("/tmp/rack", src)
+    end
+  end
+end

data/lib/jets/builders/rackup_wrappers/rackup

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+export JETS_MEGA=1
+
+# Figure out where this script is located.
+PROJECTDIR="`dirname \"$0\"`"
+PROJECTDIR="`cd \"$PROJECTDIR/..\" && pwd`"
+
+unset BUNDLE_IGNORE_CONFIG
+# When jets packages the app code it creates .bundle/config in the project root.
+# We want that to be used.
+
+# Drop a pidfile
+# echo $$ > /tmp/jets-rackup.pid
+
+# Run the actual app using the bundled Ruby interpreter, with Bundler activated.
+exec ruby -rbundler/setup bin/rackup.rb "$@"

data/lib/jets/builders/rackup_wrappers/rackup.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Copied from: https://github.com/rack/rack/blob/master/bin/rackup
+
+require "rack"
+Rack::Server.start

data/lib/jets/builders/reconfigure_rails/config/initializers/jets.rb

@@ -0,0 +1,16 @@
+JetsRails.stage = ENV['JETS_STAGE'] || 'dev' # for jets-rails StageMiddleware
+
+# config/s3_base_url.txt is generated as part of jets build process
+s3_base_url = IO.read("#{Rails.root}/config/s3_base_url.txt").strip rescue "s3_base_url_placeholder"
+asset_host = "#{s3_base_url}/rack/public"
+Rails.application.config.action_controller.asset_host = asset_host
+Rails.application.config.public_file_server.enabled = true
+
+<% unless @api_mode -%>
+# Rails.application.config.assets.quiet = false
+Rails.application.config.assets.debug = false
+Rails.application.config.assets.compile = false
+<% end -%>
+
+# Looks better without colorizatiion in CloudWatch logs
+Rails.application.config.colorize_logging = false

data/lib/jets/builders/reconfigure_rails.rb

@@ -0,0 +1,114 @@
+# TODO: Move logic into plugin instead
+module Jets::Builders
+  class ReconfigureRails
+    def initialize(full_app_root)
+      # IE: @app_root: /tmp/jets/demo/stage/code/rack
+      @app_root = full_app_root
+    end
+
+    # Only support Rails right now. Maybe move into plugin when adding support
+    # to more frameworks. Or maybe better to just abstract but maintain in Jets.
+    def run
+      return unless rails?
+
+      puts "Mega Mode: Reconfiguring Rails app."
+      copy_initializer
+      update_gemfile
+      update_development_environment
+      set_favicon
+    end
+
+    def copy_initializer
+      templates = File.expand_path("./reconfigure_rails", File.dirname(__FILE__))
+      relative_path = "config/initializers/jets.rb"
+      src = "#{templates}/#{relative_path}"
+      result = Jets::Erb.result(src, api_mode: rails_api?)
+      dest = "#{@app_root}/#{relative_path}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
+    end
+
+    def api_mode?
+    end
+
+    def update_gemfile
+      gemfile = "#{@app_root}/Gemfile"
+      lines = IO.readlines(gemfile)
+      lines = add_jets_rails_gem(lines)
+      lines = comment_out_ruby_declaration(lines)
+      write_content(gemfile, lines)
+    end
+
+    def add_jets_rails_gem(lines)
+      return lines if lines.detect { |l| l =~ /jets-rails/ }
+      lines << "\n"
+      lines << %Q|gem "jets-rails"\n|
+      lines
+    end
+
+    # Jets packages up and uses only one version of ruby and different declarations
+    # of ruby can cause issues.
+    def comment_out_ruby_declaration(lines)
+      lines.map do |l|
+        l =~ /^(ruby[ (].*)/ ? "# #{$1} # commented out by jets" : l
+      end
+    end
+
+    def update_development_environment
+      env_file = "#{@app_root}/config/environments/development.rb"
+      lines = IO.readlines(env_file)
+      new_lines = lines.map do |line|
+        if line =~ /(config\.file_watcher.*)/
+          "  # #{$1} # commented out by jets\n"
+        else
+          line
+        end
+      end
+      write_content(env_file, new_lines)
+    end
+
+    # Evaluate the application layout and see if has a custom favicon defined yet.
+    # If not insert one and rewrite the application layout.
+    #
+    # This avoids serving binary assets from API gateway, instead serve it from s3 directly.
+    def set_favicon
+      app_layout = "#{@app_root}/app/views/layouts/application.html.erb"
+      return  unless File.exist?(app_layout)
+
+      favicon = '<link rel="shortcut icon" href="<%= asset_path("/favicon.ico") %>">'
+
+      lines = IO.readlines(app_layout)
+      has_favicon = !!lines.find { |l| l.include?(favicon) }
+      return if has_favicon
+
+      content = IO.read(app_layout)
+      content = content.sub('</head>', "\n    #{favicon}\n  </head>")
+
+      IO.write(app_layout, content)
+    end
+
+    # lines is an Array
+    def write_content(path, lines)
+      content = lines.join + "\n"
+      IO.write(path, content)
+    end
+
+    # Rudimentary rails detection
+    # Duplicated in builders/code_builders.rb
+    def rails?
+      config_ru = "#{@app_root}/config.ru"
+      return false unless File.exist?(config_ru)
+      !IO.readlines(config_ru).grep(/Rails.application/).empty?
+    end
+
+    # Rudimentary rails api detection
+    # Duplicated in builders/code_builders.rb
+    # Another way of checking is loading a rails console and checking Rails.application.config.api_only
+    # Using this way for simplicity.
+    def rails_api?
+      config_app = "#{@app_root}/config/application.rb"
+      return false unless File.exist?(config_app)
+      !IO.readlines(config_app).grep(/config.api_only.*=.*true/).empty?
+    end
+  end
+end