jets 0.10.4 → 1.0.0

data/lib/jets/builders/handler_generator.rb

@@ -9,16 +9,52 @@ require "erb"
 # )
 class Jets::Builders
   class HandlerGenerator
-    def initialize(path)
-      @path = path
+    def self.build!
+      new.build
     end
 
-    def generate
+    def build
+      common_base_shim
+      app_ruby_shims
       poly_shims
-      app_ruby_shim
       shared_shims
     end
 
+    def app_ruby_shims
+      app_files.each do |path|
+        # Generates one big node shim for a entire controller.
+        vars = Jets::Builders::ShimVars::App.new(path)
+        generate_handler(vars)
+      end
+    end
+
+    def app_files
+      Jets::Commands::Build.app_files
+    end
+
+    def poly_shims
+      missing = []
+
+      app_files.each do |path|
+        vars = Jets::Builders::ShimVars::App.new(path)
+        poly_tasks = vars.klass.tasks.select { |t| t.lang != :ruby }
+        poly_tasks.each do |task|
+          source_path = get_source_path(path, task)
+          if File.exist?(source_path)
+            native_function(path, task)
+          else
+            missing << source_path
+          end
+        end
+
+        unless missing.empty?
+          puts "ERROR: Missing source files. Please make sure these source files exist or remove their declarations".colorize(:red)
+          puts missing
+          exit 1
+        end
+      end
+    end
+
     def shared_shims
       Jets::Stack.subclasses.each do |subclass|
         subclass.functions.each do |fun|
@@ -41,32 +77,11 @@ class Jets::Builders
         return
       end
 
-      dest_path = "#{tmp_app_root}/#{fun.handler_dest}"
+      dest_path = "#{tmp_code}/#{fun.handler_dest}"
       FileUtils.mkdir_p(File.dirname(dest_path))
       FileUtils.cp(source_path, dest_path)
     end
 
-    def poly_shims
-      missing = []
-
-      deducer = Jets::Builders::Deducer.new(@path)
-      poly_tasks = deducer.klass.tasks.select { |t| t.lang != :ruby }
-      poly_tasks.each do |task|
-        source_path = get_source_path(@path, task)
-        if File.exist?(source_path)
-          native_function(@path, task)
-        else
-          missing << source_path
-        end
-      end
-
-      unless missing.empty?
-        puts "ERROR: Missing source files. Please make sure these source files exist or remove their declarations".colorize(:red)
-        puts missing
-        exit 1
-      end
-    end
-
     def get_source_path(original_path, task)
       folder = original_path.sub(/\.rb$/,'')
       lang_folder = "#{folder}/#{task.lang}"
@@ -78,35 +93,39 @@ class Jets::Builders
     def native_function(original_path, task)
       source_path = get_source_path(original_path, task)
       # Handler: handlers/controllers/posts_controller.handle
-      dest_path = "#{tmp_app_root}/#{task.handler_path}"
+      dest_path = "#{tmp_code}/#{task.handler_path}"
       FileUtils.mkdir_p(File.dirname(dest_path))
       FileUtils.cp(source_path, dest_path)
     end
 
     def shared_ruby_shim(fun)
-      deducer = Jets::Builders::SharedDeducer.new(fun)
-      generate_shim(deducer)
+      vars = Jets::Builders::ShimVars::Shared.new(fun)
+      generate_handler(vars)
     end
 
-    # Generates one big node shim for a entire controller.
-    def app_ruby_shim
-      deducer = Jets::Builders::Deducer.new(@path)
-      generate_shim(deducer)
+    def common_base_shim
+      vars = Jets::Builders::ShimVars::Base.new
+      result = evaluate_template("shim.js", vars)
+      dest = "#{tmp_code}/handlers/shim.js"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
     end
 
-    def generate_shim(deducer)
-      js_path = "#{tmp_app_root}/#{deducer.js_path}"
-      FileUtils.mkdir_p(File.dirname(js_path))
-
-      template_path = File.expand_path('../node-shim.js', __FILE__)
-      result = Jets::Erb.result(template_path, deducer: deducer)
+    def generate_handler(vars)
+      result = evaluate_template("handler.js", vars)
+      dest = "#{tmp_code}/#{vars.js_path}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
+    end
 
-      IO.write(js_path, result)
+    def evaluate_template(template_file, vars)
+      template_path = File.expand_path("../templates/#{template_file}", __FILE__)
+      Jets::Erb.result(template_path, vars: vars)
     end
 
-    # TODO: move CodeBuilder.tmp_app_root to a common level for HandlerGenerator and CodeBuilder
-    def tmp_app_root
-      "#{Jets.build_root}/#{CodeBuilder.tmp_app_root}"
+    # TODO: move CodeBuilder.tmp_code to a common level for HandlerGenerator and CodeBuilder
+    def tmp_code
+      "#{Jets.build_root}/#{CodeBuilder.tmp_code}"
     end
   end
 end

data/lib/jets/builders/md5.rb

@@ -0,0 +1,55 @@
+require 'digest'
+
+# Resolves the chicken-and-egg problem with md5 checksums. The handlers need
+# to reference files with the md5 checksum.  The files are the:
+#
+#   jets/code/rack-checksum.zip
+#   jets/code/bundled-checksum.zip
+#
+# We compute the checksums before we generate the node shim handlers.
+class Jets::Builders
+  class Md5
+    class << self
+      @@checksums = {}
+      def checksums
+        @@checksums
+      end
+
+      def stage_folders
+        paths = []
+        paths << "stage/bundled" if Jets.lazy_load?
+        paths << "stage/rack" if Jets.rack?
+        # Important to have stage/code at the end, since it will use the other
+        # 'symlinked' folders to adjust the md5 hash.
+        paths << "stage/code"
+        paths
+      end
+
+      def compute!
+        stage_folders.each do |path|
+          @@checksums[path] = dir(path)
+        end
+        @@checksums
+      end
+
+      def dir(short_path)
+        path = "#{Jets.build_root}/#{short_path}"
+        files = Dir["#{path}/**/*"]
+        files.reject! { |f| File.directory?(f) }
+             .reject! { |f| File.symlink?(f) }
+        content = files.map do |f|
+          Digest::MD5.file(f).to_s[0..7]
+        end.join
+
+        # The stage/code md5 sha depends on the other 'symlinked' folders.
+        if short_path == "stage/code"
+          content += @@checksums.values.join
+        end
+
+        md5 = Digest::MD5.new
+        md5.update(content)
+        md5.hexdigest.to_s[0..7]
+      end
+    end
+  end
+end

data/lib/jets/builders/md5_zip.rb

@@ -0,0 +1,60 @@
+require 'action_view'
+
+# Examples:
+#
+#   zip = Jets::Builders::Md5Zip.new("/tmp/jets/demo/stage/code")
+#   zip.create
+#
+#   zip = Jets::Builders::Md5Zip.new("/tmp/jets/demo/stage/bundled")
+#   zip.create
+#
+class Jets::Builders
+  class Md5Zip
+    include ActionView::Helpers::NumberHelper # number_to_human_size
+    include Util
+
+    def initialize(folder)
+      @path = "#{Jets.build_root}/#{folder}"
+      @checksum = Md5.checksums[folder]
+    end
+
+    def create
+      headline "Creating zip file for #{@path}"
+
+      # https://serverfault.com/questions/265675/how-can-i-zip-compress-a-symlink
+      command = "cd #{@path} && zip --symlinks -rq #{zip_file} ."
+      sh(command)
+      # move out of the lower folder to the stage folder
+      # mv /tmp/jets/demo/stage/code/code.zip /tmp/jets/demo/stage/code.zip
+      FileUtils.mkdir_p(File.dirname(zip_dest))
+      FileUtils.mv("#{@path}/#{zip_file}", zip_dest)
+
+      # mv /tmp/jets/demo/stage/zips/code.zip /tmp/jets/demo/stage/zips/code-a8a604aa.zip
+      FileUtils.mv(zip_dest, md5_dest)
+
+      file_size = number_to_human_size(File.size(md5_dest))
+      puts "Zip file created at: #{md5_dest.colorize(:green)} (#{file_size})"
+    end
+
+    # /tmp/jets/demo/stage/zips/code.zip
+    def zip_dest
+      stage_area, filename = File.dirname(@path), File.basename(@path)
+      zip_area = stage_area + '/zips' # /tmp/jets/demo/stage/zips
+      zip_file = filename + '.zip' # code.zip
+      "#{zip_area}/#{zip_file}" # /tmp/jets/demo/stage/zips/code.zip
+    end
+
+    def zip_file
+      File.basename(zip_dest)
+    end
+
+    # /tmp/jets/demo/stage/zips/code-SHA.zip
+    def md5_dest
+      zip_dest.sub(".zip", "-#{@checksum}.zip")
+    end
+
+    def md5_name
+      File.basename(md5_dest)
+    end
+  end
+end

data/lib/jets/builders/rack_packager.rb

@@ -0,0 +1,29 @@
+class Jets::Builders
+  class RackPackager < RubyPackager
+    def finish
+      return unless gemfile_exist?
+
+      symlink_rack_bundled
+      copy_rackup_wrappers
+    end
+
+    def symlink_rack_bundled
+      # IE: @full_app_root: /tmp/jets/demo/stage/code/rack
+      rack_bundled = "#{@full_app_root}/bundled"
+      FileUtils.rm_f(rack_bundled) # looks like FileUtils.ln_sf doesnt remove existing symlinks
+      FileUtils.ln_sf("/var/task/bundled", rack_bundled)
+    end
+
+    def copy_rackup_wrappers
+      # IE: @full_app_root: /tmp/jets/demo/stage/code/rack
+      rack_bin = "#{@full_app_root}/bin"
+      %w[rackup rackup.rb].each do |file|
+        src = File.expand_path("./rackup_wrappers/#{file}", File.dirname(__FILE__))
+        dest = "#{rack_bin}/#{file}"
+        FileUtils.mkdir_p(rack_bin) unless File.exist?(rack_bin)
+        FileUtils.cp(src, dest)
+        FileUtils.chmod 0755, dest
+      end
+    end
+  end
+end

data/lib/jets/builders/rackup_wrappers/rackup

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+export JETS_MEGA=1
+
+# Figure out where this script is located.
+PROJECTDIR="`dirname \"$0\"`"
+PROJECTDIR="`cd \"$PROJECTDIR/..\" && pwd`"
+
+unset BUNDLE_IGNORE_CONFIG
+# When jets packages the app code it creates .bundle/config in the project root.
+# We want that to be used.
+
+# Drop a pidfile
+# echo $$ > /tmp/jets-rackup.pid
+
+# Run the actual app using the bundled Ruby interpreter, with Bundler activated.
+if [ -n "$C9_USER" ]; then
+  exec ruby -rbundler/setup bin/rackup.rb "$@"
+else
+  exec "$PROJECTDIR/bundled/rbenv/shims/ruby" -rbundler/setup "$PROJECTDIR/bin/rackup.rb" "$@"
+fi

data/lib/jets/builders/rackup_wrappers/rackup.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Copied from: https://github.com/rack/rack/blob/master/bin/rackup
+
+require "rack"
+Rack::Server.start

data/lib/jets/builders/reconfigure_rails/config/initializers/jets.rb

@@ -0,0 +1,14 @@
+JetsRails.stage = ENV['JETS_STAGE'] || 'dev' # for jets-rails StageMiddleware
+
+# config/s3_base_url.txt is generated as part of jets build process
+s3_base_url = IO.read("#{Rails.root}/config/s3_base_url.txt").strip rescue "s3_base_url_placeholder"
+asset_host = "#{s3_base_url}/rack/public"
+Rails.application.config.action_controller.asset_host = asset_host
+
+# Rails.application.config.assets.quiet = false
+Rails.application.config.assets.debug = false
+Rails.application.config.assets.compile = false
+Rails.application.config.public_file_server.enabled = true
+
+# Looks better without colorizatiion in CloudWatch logs
+Rails.application.config.colorize_logging = false

data/lib/jets/builders/reconfigure_rails.rb

@@ -0,0 +1,99 @@
+# TODO: Move logic into plugin instead
+class Jets::Builders
+  class ReconfigureRails
+    def initialize(full_app_root)
+      # IE: @app_root: /tmp/jets/demo/stage/code/rack
+      @app_root = full_app_root
+    end
+
+    # Only support Rails right now. Maybe move into plugin when adding support
+    # to more frameworks. Or maybe better to just abstract but maintain in Jets.
+    def run
+      return unless rails?
+
+      puts "Mega Mode: Reconfiguring Rails app."
+      copy_initializer
+      update_gemfile
+      update_development_environment
+      set_favicon
+    end
+
+    def copy_initializer
+      templates = File.expand_path("./reconfigure_rails", File.dirname(__FILE__))
+      relative_path = "config/initializers/jets.rb"
+      src = "#{templates}/#{relative_path}"
+      dest = "#{@app_root}/#{relative_path}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      FileUtils.cp(src, dest)
+    end
+
+    def update_gemfile
+      gemfile = "#{@app_root}/Gemfile"
+      lines = IO.readlines(gemfile)
+      lines = add_jets_rails_gem(lines)
+      lines = comment_out_ruby_declaration(lines)
+      write_content(gemfile, lines)
+    end
+
+    def add_jets_rails_gem(lines)
+      return lines if lines.detect { |l| l =~ /jets-rails/ }
+      lines << "\n"
+      lines << %Q|gem "jets-rails"\n|
+      lines
+    end
+
+    # Jets packages up and uses only one version of ruby and different declarations
+    # of ruby can cause issues.
+    def comment_out_ruby_declaration(lines)
+      lines.map do |l|
+        l =~ /^(ruby[ (].*)/ ? "# #{$1} # commented out by jets" : l
+      end
+    end
+
+    def update_development_environment
+      env_file = "#{@app_root}/config/environments/development.rb"
+      lines = IO.readlines(env_file)
+      new_lines = lines.map do |line|
+        if line =~ /(config\.file_watcher.*)/
+          "  # #{$1} # commented out by jets\n"
+        else
+          line
+        end
+      end
+      write_content(env_file, new_lines)
+    end
+
+    # Evaluate the application layout and see if has a custom favicon defined yet.
+    # If not insert one and rewrite the application layout.
+    #
+    # This avoids serving binary assets from API gateway, instead serve it from s3 directly.
+    def set_favicon
+      app_layout = "#{@app_root}/app/views/layouts/application.html.erb"
+      return  unless File.exist?(app_layout)
+
+      favicon = '<link rel="shortcut icon" href="<%= asset_path("/favicon.ico") %>">'
+
+      lines = IO.readlines(app_layout)
+      has_favicon = !!lines.find { |l| l.include?(favicon) }
+      return if has_favicon
+
+      content = IO.read(app_layout)
+      content = content.sub('</head>', "\n    #{favicon}\n  </head>")
+
+      IO.write(app_layout, content)
+    end
+
+    # lines is an Array
+    def write_content(path, lines)
+      content = lines.join + "\n"
+      IO.write(path, content)
+    end
+
+    # Rudimentary rails detection
+    def rails?
+      config_ru = "#{@app_root}/config.ru"
+      return false unless File.exist?(config_ru)
+      !IO.readlines(config_ru).grep(/Rails.application/).empty?
+    end
+  end
+end

data/lib/jets/builders/ruby_packager.rb

@@ -0,0 +1,198 @@
+require "bundler" # for clean_old_submodules only
+
+class Jets::Builders
+  class RubyPackager
+    include Util
+
+    attr_reader :full_app_root
+    def initialize(relative_app_root)
+      @full_app_root = full(relative_app_root)
+    end
+
+    def install
+      return unless gemfile_exist?
+
+      reconfigure_ruby_version
+      clean_old_submodules
+      bundle_install
+      setup_bundle_config
+    end
+
+    def finish
+      return unless gemfile_exist?
+
+      copy_bundled_cache
+      extract_ruby
+      extract_gems
+      tidy
+    end
+
+    def gemfile_exist?
+      gemfile_path = "#{@full_app_root}/Gemfile"
+      File.exist?(gemfile_path)
+    end
+
+    # Installs gems on the current target system: both compiled and non-compiled.
+    # If user is on a macosx machine, macosx gems will be installed.
+    # If user is on a linux machine, linux gems will be installed.
+    #
+    # Copies Gemfile* to /tmp/jetss/demo/bundled folder and installs
+    # gems with bundle install from there.
+    #
+    # We take the time to copy Gemfile and bundle into a separate directory
+    # because it gets left around to act as a 'cache'.  So, when the builds the
+    # project gets built again not all the gems from get installed from the
+    # beginning.
+    def bundle_install
+      return if poly_only?
+
+      full_project_path = @full_app_root
+      headline "Bundling: running bundle install in cache area: #{cache_area}."
+
+      copy_gemfiles(full_project_path)
+
+      require "bundler" # dynamically require bundler so user can use any bundler
+      Bundler.with_clean_env do
+        # cd /tmp/jets/demo
+        sh(
+          "cd #{cache_area} && " \
+          "env BUNDLE_IGNORE_CONFIG=1 bundle install --path bundled/gems --without development test"
+        )
+      end
+
+      # Copy the Gemfile.lock back to the project in case it was updated.
+      # For example we add the jets-rails to the Gemfile.
+      copy_gemfile_lock
+
+      puts 'Bundle install success.'
+    end
+
+    def copy_gemfile_lock
+      src = "#{cache_area}/Gemfile.lock"
+      dest = "#{@full_app_root}/Gemfile.lock"
+      FileUtils.cp_r(src, dest)
+    end
+
+    # Clean up extra unneeded files to reduce package size
+    # Because we're removing files (something dangerous) use full paths.
+    def tidy
+      puts "Tidying project: removing ignored files to reduce package size."
+      tidy_project(@full_app_root)
+      # The rack sub project has it's own gitignore.
+      tidy_project(@full_app_root+"/rack")
+    end
+
+    def tidy_project(path)
+      Tidy.new(path).cleanup!
+    end
+
+    # This is in case the user has a 2.5.x variant.
+    # Force usage of ruby version that jets supports
+    # The lambda server only has ruby 2.5.0 installed.
+    def reconfigure_ruby_version
+      ruby_version = "#{@full_app_root}/.ruby-version"
+      IO.write(ruby_version, Jets::RUBY_VERSION)
+    end
+
+    # When using submodules, bundler leaves old submodules behind. Over time this inflates
+    # the size of the the bundled gems.  So we'll clean it up.
+    def clean_old_submodules
+      # https://stackoverflow.com/questions/38800129/parsing-a-gemfile-lock-with-bundler
+      lockfile = "#{cache_area}/Gemfile.lock"
+      return unless File.exist?(lockfile)
+
+      parser = Bundler::LockfileParser.new(Bundler.read_file(lockfile))
+      specs = parser.specs
+
+      # specs = Bundler.load.specs
+      # IE: spec.source.to_s: "https://github.com/tongueroo/webpacker.git (at jets@a8c4661)"
+      submoduled_specs = specs.select do |spec|
+        spec.source.to_s =~ /@\w+\)/
+      end
+
+      # find git shas to keep
+      # IE: ["a8c4661", "abc4661"]
+      git_shas = submoduled_specs.map do |spec|
+        md = spec.source.to_s.match(/@(\w+)\)/)
+        md[1] # git_sha
+      end
+
+      # IE: /tmp/jets/demo/cache/bundled/gems/ruby/2.5.0/bundler/gems/webpacker-a8c46614c675
+      Dir.glob("#{cache_area}/bundled/gems/ruby/2.5.0/bundler/gems/*").each do |path|
+        sha = path.split('-').last[0..6] # only first 7 chars of the git sha
+        unless git_shas.include?(sha)
+          # puts "Removing old submoduled gem: #{path}" # uncomment to see and debug
+          FileUtils.rm_rf(path) # REMOVE old submodule directory
+        end
+      end
+    end
+
+    def copy_gemfiles(full_project_path)
+      FileUtils.mkdir_p(cache_area)
+      FileUtils.cp("#{full_project_path}/Gemfile", "#{cache_area}/Gemfile")
+      FileUtils.cp("#{full_project_path}/Gemfile.lock", "#{cache_area}/Gemfile.lock")
+    end
+
+    def setup_bundle_config
+      ensure_build_cache_bundle_config_exists!
+
+      # Override project's .bundle/config and ensure that .bundle/config matches
+      # at these 2 spots:
+      #   app_root/.bundle/config
+      #   bundled/gems/.bundle/config
+      cache_bundle_config = "#{cache_area}/.bundle/config"
+      app_bundle_config = "#{@full_app_root}/.bundle/config"
+      FileUtils.mkdir_p(File.dirname(app_bundle_config))
+      FileUtils.cp(cache_bundle_config, app_bundle_config)
+    end
+
+    # On circleci the "#{Jets.build_root}/.bundle/config" doesnt exist
+    # this only happens with ssh debugging, not when the ci.sh script gets ran.
+    # But on macosx it exists.
+    # Dont know why this is the case.
+    def ensure_build_cache_bundle_config_exists!
+      text =<<-EOL
+---
+BUNDLE_PATH: "bundled/gems"
+BUNDLE_WITHOUT: "development:test"
+EOL
+      bundle_config = "#{cache_area}/.bundle/config"
+      FileUtils.mkdir_p(File.dirname(bundle_config))
+      IO.write(bundle_config, text)
+    end
+
+    def lambdagem_options
+      {
+        s3: "lambdagems",
+        build_root: cache_area, # used in lambdagem
+        project_root: @full_app_root, # used in gem_replacer and lambdagem
+      }
+    end
+
+    def extract_ruby
+      headline "Setting up a vendored copy of ruby."
+      Lambdagem.log_level = :info
+      Lambdagem::Extract::Ruby.new(Jets::RUBY_VERSION, lambdagem_options).run
+    end
+
+    def extract_gems
+      headline "Replacing compiled gems with AWS Lambda Linux compiled versions."
+      GemReplacer.new(Jets::RUBY_VERSION, lambdagem_options).run
+    end
+
+    def copy_bundled_cache
+      app_root_bundled = "#{@full_app_root}/bundled"
+      if File.exist?(app_root_bundled)
+        puts "Removing current bundled from project"
+        FileUtils.rm_rf(app_root_bundled)
+      end
+      # Leave #{Jets.build_root}/bundled behind to act as cache
+      FileUtils.cp_r("#{cache_area}/bundled", app_root_bundled)
+    end
+
+  private
+    def cache_area
+      "#{Jets.build_root}/cache" # cleaner to use full path for this setting
+    end
+  end
+end

data/lib/jets/builders/{deducer.rb → shim_vars/app.rb}

@@ -1,6 +1,17 @@
-# Build::Deducers figure out required values to create the node shim
-class Jets::Builders
-  class Deducer
+# Jets::Builders::ShimVars::Shared.new(path)
+#
+#   @vars.functions.each do |function_name|
+#     @vars.handler_for(function_name)
+#   end
+#
+# Implements:
+#
+#   functions
+#   handler_for(function_name)
+#   js_path
+#
+module Jets::Builders::ShimVars
+  class App < Base
     # Allow user to specify relative or full path. The right path gets used
     # internally. Example paths:
     #   app/controllers/posts_controller.rb
@@ -37,13 +48,6 @@ class Jets::Builders
     # Returns the public methods of the child_class.
     # Example: [:create, :update]
     def functions
-      # Example: require: /tmp/jets/demo/app_root/app/controllers/admin/posts_controller.rb
-      class_name = @full_path.sub(/.*\/app\/\w+\//,'').sub(/\.rb/,'') # admin/posts_controller
-      class_name = class_name.classify # Admin::PostsController
-      # Autoload instead of require to avoid accidentallly double requiring and
-      # it resulting WARNINGs when there are constants already defined
-      klass = class_name.constantize # autoload
-
       klass.lambda_functions
     end