jets-fs 1.6.10

data/lib/jets/builders/purger.rb

@@ -0,0 +1,35 @@
+# When upgrading jets, automatically rm -rf /tmp/jets/project in case the structure has changed.
+class Jets::Builders
+  class Purger
+    def initialize
+      @project_name = Jets.config.project_name
+      @version_file = "/tmp/jets/#{@project_name}/jets_version.txt"
+    end
+
+    def purge
+      if version_changed?
+        last_version = @last_version || "unknown"
+        puts "The jets version has changed enough since the last build to merit refreshing the build cache."
+        puts "Current jets version: #{Jets::VERSION} Last built jets version: #{last_version}"
+        puts "Removing /tmp/jets/#{@project_name} to start fresh."
+        FileUtils.rm_rf("/tmp/jets/#{@project_name}")
+      end
+      write_version
+    end
+
+    # When jets changes versions major or minor version consider it a big enough can to purge the cache
+    def version_changed?
+      return true unless File.exist?(@version_file)
+
+      @last_version = IO.read(@version_file).strip
+      last_major, last_minor, _ = @last_version.split('.')
+      current_major, current_minor, _ = Jets::VERSION.split('.')
+      last_major != current_major || last_minor != current_minor
+    end
+
+    def write_version
+      FileUtils.mkdir_p(File.dirname(@version_file))
+      IO.write(@version_file, Jets::VERSION)
+    end
+  end
+end

data/lib/jets/builders/rack_packager.rb

@@ -0,0 +1,51 @@
+class Jets::Builders
+  class RackPackager < RubyPackager
+    def finish
+      return unless gemfile_exist?
+
+      super
+
+      symlink_gems
+      copy_rackup_wrappers
+      rack_symlink
+    end
+
+    def symlink_gems
+      ruby_folder = Jets::Gems.ruby_folder
+      # IE: @full_app_root: /tmp/jets/demo/stage/code/rack
+      dest = "#{@full_app_root}/vendor/gems/ruby/#{ruby_folder}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", dest)
+    end
+
+    def copy_rackup_wrappers
+      # IE: @full_app_root: /tmp/jets/demo/stage/code/rack
+      rack_bin = "#{@full_app_root}/bin"
+      %w[rackup rackup.rb].each do |file|
+        src = File.expand_path("./rackup_wrappers/#{file}", File.dirname(__FILE__))
+        dest = "#{rack_bin}/#{file}"
+        FileUtils.mkdir_p(rack_bin) unless File.exist?(rack_bin)
+        FileUtils.cp(src, dest)
+        FileUtils.chmod 0755, dest
+      end
+    end
+
+    # Moves folder to a stage folder and create a symlink its place
+    # that links from /var/task to /tmp. Example:
+    #
+    #   stage/code/rack => /tmp/rack
+    #
+    def rack_symlink
+      src = @full_app_root
+      return unless File.exist?(src)
+
+      dest = "#{stage_area}/rack"
+      dir = File.dirname(dest)
+      FileUtils.mkdir_p(dir) unless File.exist?(dir)
+      FileUtils.mv(src, dest)
+
+      # Create symlink
+      FileUtils.ln_sf("/tmp/rack", src)
+    end
+  end
+end

data/lib/jets/builders/rackup_wrappers/rackup

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+export JETS_MEGA=1
+
+# Figure out where this script is located.
+PROJECTDIR="`dirname \"$0\"`"
+PROJECTDIR="`cd \"$PROJECTDIR/..\" && pwd`"
+
+unset BUNDLE_IGNORE_CONFIG
+# When jets packages the app code it creates .bundle/config in the project root.
+# We want that to be used.
+
+# Drop a pidfile
+# echo $$ > /tmp/jets-rackup.pid
+
+# Run the actual app using the bundled Ruby interpreter, with Bundler activated.
+exec ruby -rbundler/setup bin/rackup.rb "$@"

data/lib/jets/builders/rackup_wrappers/rackup.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Copied from: https://github.com/rack/rack/blob/master/bin/rackup
+
+require "rack"
+Rack::Server.start

data/lib/jets/builders/reconfigure_rails.rb

@@ -0,0 +1,114 @@
+# TODO: Move logic into plugin instead
+class Jets::Builders
+  class ReconfigureRails
+    def initialize(full_app_root)
+      # IE: @app_root: /tmp/jets/demo/stage/code/rack
+      @app_root = full_app_root
+    end
+
+    # Only support Rails right now. Maybe move into plugin when adding support
+    # to more frameworks. Or maybe better to just abstract but maintain in Jets.
+    def run
+      return unless rails?
+
+      puts "Mega Mode: Reconfiguring Rails app."
+      copy_initializer
+      update_gemfile
+      update_development_environment
+      set_favicon
+    end
+
+    def copy_initializer
+      templates = File.expand_path("./reconfigure_rails", File.dirname(__FILE__))
+      relative_path = "config/initializers/jets.rb"
+      src = "#{templates}/#{relative_path}"
+      result = Jets::Erb.result(src, api_mode: rails_api?)
+      dest = "#{@app_root}/#{relative_path}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, result)
+    end
+
+    def api_mode?
+    end
+
+    def update_gemfile
+      gemfile = "#{@app_root}/Gemfile"
+      lines = IO.readlines(gemfile)
+      lines = add_jets_rails_gem(lines)
+      lines = comment_out_ruby_declaration(lines)
+      write_content(gemfile, lines)
+    end
+
+    def add_jets_rails_gem(lines)
+      return lines if lines.detect { |l| l =~ /jets-rails/ }
+      lines << "\n"
+      lines << %Q|gem "jets-rails"\n|
+      lines
+    end
+
+    # Jets packages up and uses only one version of ruby and different declarations
+    # of ruby can cause issues.
+    def comment_out_ruby_declaration(lines)
+      lines.map do |l|
+        l =~ /^(ruby[ (].*)/ ? "# #{$1} # commented out by jets" : l
+      end
+    end
+
+    def update_development_environment
+      env_file = "#{@app_root}/config/environments/development.rb"
+      lines = IO.readlines(env_file)
+      new_lines = lines.map do |line|
+        if line =~ /(config\.file_watcher.*)/
+          "  # #{$1} # commented out by jets\n"
+        else
+          line
+        end
+      end
+      write_content(env_file, new_lines)
+    end
+
+    # Evaluate the application layout and see if has a custom favicon defined yet.
+    # If not insert one and rewrite the application layout.
+    #
+    # This avoids serving binary assets from API gateway, instead serve it from s3 directly.
+    def set_favicon
+      app_layout = "#{@app_root}/app/views/layouts/application.html.erb"
+      return  unless File.exist?(app_layout)
+
+      favicon = '<link rel="shortcut icon" href="<%= asset_path("/favicon.ico") %>">'
+
+      lines = IO.readlines(app_layout)
+      has_favicon = !!lines.find { |l| l.include?(favicon) }
+      return if has_favicon
+
+      content = IO.read(app_layout)
+      content = content.sub('</head>', "\n    #{favicon}\n  </head>")
+
+      IO.write(app_layout, content)
+    end
+
+    # lines is an Array
+    def write_content(path, lines)
+      content = lines.join + "\n"
+      IO.write(path, content)
+    end
+
+    # Rudimentary rails detection
+    # Duplicated in builders/code_builders.rb
+    def rails?
+      config_ru = "#{@app_root}/config.ru"
+      return false unless File.exist?(config_ru)
+      !IO.readlines(config_ru).grep(/Rails.application/).empty?
+    end
+
+    # Rudimentary rails api detection
+    # Duplicated in builders/code_builders.rb
+    # Another way of checking is loading a rails console and checking Rails.application.config.api_only
+    # Using this way for simplicity.
+    def rails_api?
+      config_app = "#{@app_root}/config/application.rb"
+      return false unless File.exist?(config_app)
+      !IO.readlines(config_app).grep(/config.api_only.*=.*true/).empty?
+    end
+  end
+end

data/lib/jets/builders/reconfigure_rails/config/initializers/jets.rb

@@ -0,0 +1,16 @@
+JetsRails.stage = ENV['JETS_STAGE'] || 'dev' # for jets-rails StageMiddleware
+
+# config/s3_base_url.txt is generated as part of jets build process
+s3_base_url = IO.read("#{Rails.root}/config/s3_base_url.txt").strip rescue "s3_base_url_placeholder"
+asset_host = "#{s3_base_url}/rack/public"
+Rails.application.config.action_controller.asset_host = asset_host
+Rails.application.config.public_file_server.enabled = true
+
+<% unless @api_mode -%>
+# Rails.application.config.assets.quiet = false
+Rails.application.config.assets.debug = false
+Rails.application.config.assets.compile = false
+<% end -%>
+
+# Looks better without colorizatiion in CloudWatch logs
+Rails.application.config.colorize_logging = false

data/lib/jets/builders/ruby_packager.rb

@@ -0,0 +1,207 @@
+require "bundler" # for clean_old_submodules only
+
+class Jets::Builders
+  class RubyPackager
+    include Util
+
+    attr_reader :full_app_root
+    def initialize(relative_app_root)
+      @full_app_root = "#{build_area}/#{relative_app_root}"
+    end
+
+    def install
+      return unless gemfile_exist?
+
+      reconfigure_ruby_version
+      clean_old_submodules
+      bundle_install
+      setup_bundle_config
+      copy_cache_gems
+    end
+
+    #   build gems in vendor/gems/ruby/2.5.0 (done in install phase)
+    def finish
+      return unless gemfile_exist?
+      tidy
+    end
+
+    def gemfile_exist?
+      gemfile_path = "#{@full_app_root}/Gemfile"
+      File.exist?(gemfile_path)
+    end
+
+    # Installs gems on the current target system: both compiled and non-compiled.
+    # If user is on a macosx machine, macosx gems will be installed.
+    # If user is on a linux machine, linux gems will be installed.
+    #
+    # Copies Gemfile* to /tmp/jets/demo/cache folder and installs
+    # gems with bundle install from there.
+    #
+    # We take the time to copy Gemfile and bundle into a separate directory
+    # because it gets left around to act as a 'cache'.  So, when the builds the
+    # project gets built again not all the gems from get installed from the
+    # beginning.
+    def bundle_install
+      full_project_path = @full_app_root
+      headline "Bundling: running bundle install in cache area: #{cache_area}."
+
+      copy_gemfiles(full_project_path)
+
+      # Uncomment out to always remove the cache/vendor/gems to debug
+      # FileUtils.rm_rf("#{cache_area}/vendor/gems")
+
+      require "bundler" # dynamically require bundler so user can use any bundler
+      Bundler.with_clean_env do
+        sh(
+          "cd #{cache_area} && " \
+          "env BUNDLE_IGNORE_CONFIG=1 bundle install --path #{cache_area}/vendor/gems --without development test"
+        )
+      end
+
+      remove_bundled_with("#{cache_area}/Gemfile.lock")
+
+      # Copy the Gemfile.lock back to the project in case it was updated.
+      # For example we add the jets-rails to the Gemfile.
+      copy_back_gemfile_lock
+
+      puts 'Bundle install success.'
+    end
+
+    def copy_back_gemfile_lock
+      src = "#{cache_area}/Gemfile.lock"
+      dest = "#{@full_app_root}/Gemfile.lock"
+      FileUtils.cp(src, dest)
+    end
+
+    # Clean up extra unneeded files to reduce package size
+    # Because we're removing files (something dangerous) use full paths.
+    def tidy
+      puts "Tidying project: removing ignored files to reduce package size."
+      tidy_project(@full_app_root)
+      # The rack sub project has it's own gitignore.
+      tidy_project(@full_app_root+"/rack")
+    end
+
+    def tidy_project(path)
+      Tidy.new(path).cleanup!
+    end
+
+    # This is in case the user has a 2.5.x variant.
+    # Force usage of ruby version that jets supports
+    # The lambda server only has ruby 2.5.0 installed.
+    def reconfigure_ruby_version
+      ruby_version = "#{@full_app_root}/.ruby-version"
+      IO.write(ruby_version, Jets::RUBY_VERSION)
+    end
+
+    # When using submodules, bundler leaves old submodules behind. Over time this inflates
+    # the size of the the cache gems.  So we'll clean it up.
+    def clean_old_submodules
+      # https://stackoverflow.com/questions/38800129/parsing-a-gemfile-lock-with-bundler
+      lockfile = "#{cache_area}/Gemfile.lock"
+      return unless File.exist?(lockfile)
+
+      return if Bundler.bundler_major_version <= 1 # LockfileParser only works for Bundler version 2+
+
+      parser = Bundler::LockfileParser.new(Bundler.read_file(lockfile))
+      specs = parser.specs
+
+      # specs = Bundler.load.specs
+      # IE: spec.source.to_s: "https://github.com/tongueroo/webpacker.git (at jets@a8c4661)"
+      submoduled_specs = specs.select do |spec|
+        spec.source.to_s =~ /@\w+\)/
+      end
+
+      # find git shas to keep
+      # IE: ["a8c4661", "abc4661"]
+      git_shas = submoduled_specs.map do |spec|
+        md = spec.source.to_s.match(/@(\w+)\)/)
+        md[1] # git_sha
+      end
+
+      # IE: /tmp/jets/demo/cache/vendor/gems/ruby/2.5.0/bundler/gems/webpacker-a8c46614c675
+      Dir.glob("#{cache_area}/vendor/gems/ruby/2.5.0/bundler/gems/*").each do |path|
+        sha = path.split('-').last[0..6] # only first 7 chars of the git sha
+        unless git_shas.include?(sha)
+          # puts "Removing old submoduled gem: #{path}" # uncomment to see and debug
+          FileUtils.rm_rf(path) # REMOVE old submodule directory
+        end
+      end
+    end
+
+    def copy_gemfiles(full_project_path)
+      FileUtils.mkdir_p(cache_area)
+      FileUtils.cp("#{full_project_path}/Gemfile", "#{cache_area}/Gemfile")
+
+      gemfile_lock = "#{full_project_path}/Gemfile.lock"
+      dest = "#{cache_area}/Gemfile.lock"
+      return unless File.exist?(gemfile_lock)
+
+      FileUtils.cp(gemfile_lock, dest)
+    end
+
+    # Remove the BUNDLED WITH line since we don't control the bundler gem version on AWS Lambda
+    # And this can cause issues with require 'bundler/setup'
+    def remove_bundled_with(gemfile_lock)
+      lines = IO.readlines(gemfile_lock)
+
+      # amount is the number of lines to remove
+      new_lines, capture, count, amount = [], true, 0, 2
+      lines.each do |l|
+        capture = false if l.include?('BUNDLED WITH')
+        if capture
+          new_lines << l
+        end
+        if capture == false
+          count += 1
+          capture = count > amount # renable capture
+        end
+      end
+
+      content = new_lines.join('')
+      IO.write(gemfile_lock, content)
+    end
+
+    def setup_bundle_config
+      ensure_build_cache_bundle_config_exists!
+
+      # Override project's .bundle/config and ensure that .bundle/config matches
+      # at these 2 spots:
+      #   app_root/.bundle/config
+      #   vendor/gems/.bundle/config
+      cache_bundle_config = "#{cache_area}/.bundle/config"
+      app_bundle_config = "#{@full_app_root}/.bundle/config"
+      FileUtils.mkdir_p(File.dirname(app_bundle_config))
+      FileUtils.cp(cache_bundle_config, app_bundle_config)
+    end
+
+    # On circleci the "#{Jets.build_root}/.bundle/config" doesnt exist
+    # this only happens with ssh debugging, not when the ci.sh script gets ran.
+    # But on macosx it exists.
+    # Dont know why this is the case.
+    def ensure_build_cache_bundle_config_exists!
+      text =<<-EOL
+---
+BUNDLE_FROZEN: "true"
+BUNDLE_PATH: "vendor/gems"
+BUNDLE_WITHOUT: "development:test"
+EOL
+      bundle_config = "#{cache_area}/.bundle/config"
+      FileUtils.mkdir_p(File.dirname(bundle_config))
+      IO.write(bundle_config, text)
+    end
+
+    def copy_cache_gems
+      vendor_gems = "#{@full_app_root}/vendor/gems"
+      if File.exist?(vendor_gems)
+        puts "Removing current vendor_gems from project"
+        FileUtils.rm_rf(vendor_gems)
+      end
+      # Leave #{Jets.build_root}/vendor_gems behind to act as cache
+      if File.exist?("#{cache_area}/vendor/gems")
+        FileUtils.mkdir_p(File.dirname(vendor_gems))
+        Jets::Util.cp_r("#{cache_area}/vendor/gems", vendor_gems)
+      end
+    end
+  end
+end