jets.benforeva 3.0.17.pre.mount.pre.fix

data/lib/jets/builders/ruby_packager.rb

@@ -0,0 +1,235 @@
+require "bundler" # for clean_old_submodules only
+
+module Jets::Builders
+  class RubyPackager
+    include Util
+
+    attr_reader :full_app_root
+    def initialize(relative_app_root)
+      @full_app_root = "#{build_area}/#{relative_app_root}"
+    end
+
+    def install
+      return unless gemfile_exist?
+
+      clean_old_submodules
+      bundle_install
+      copy_bundle_config
+      copy_cache_gems
+    end
+
+    #   build gems in vendor/gems/ruby/2.5.0 (done in install phase)
+    def finish
+      return unless gemfile_exist?
+      tidy
+    end
+
+    def gemfile_exist?
+      gemfile_path = "#{@full_app_root}/Gemfile"
+      File.exist?(gemfile_path)
+    end
+
+    # Installs gems on the current target system: both compiled and non-compiled.
+    # If user is on a macosx machine, macosx gems will be installed.
+    # If user is on a linux machine, linux gems will be installed.
+    #
+    # Copies Gemfile* to /tmp/jets/demo/cache folder and installs
+    # gems with bundle install from there.
+    #
+    # We take the time to copy Gemfile and bundle into a separate directory
+    # because it gets left around to act as a 'cache'.  So, when the builds the
+    # project gets built again not all the gems from get installed from the
+    # beginning.
+    def bundle_install
+      full_project_path = @full_app_root
+      headline "Bundling: running bundle install in cache area: #{cache_area}."
+
+      copy_gemfiles(full_project_path)
+      copy_bundled_gems(full_project_path)
+
+      # Uncomment out to always remove the cache/vendor/gems to debug
+      # FileUtils.rm_rf("#{cache_area}/vendor/gems")
+
+      # Remove .bundle folder so .bundle/config doesnt affect how Jets packages gems.
+      # Not using BUNDLE_IGNORE_CONFIG=1 to allow home ~/.bundle/config to affect bundling though.
+      # This is useful if you have private gems sources that require authentication. Example:
+      #
+      #    bundle config gems.myprivatesource.com user:pass
+      #
+
+      create_bundle_config
+      require "bundler" # dynamically require bundler so user can use any bundler
+      Bundler.with_unbundled_env do
+        sh(
+          "cd #{cache_area} && " \
+          "env bundle install"
+        )
+      end
+      create_bundle_config(frozen: true)
+
+      rewrite_gemfile_lock("#{cache_area}/Gemfile.lock")
+
+      # Copy the Gemfile.lock back to the project in case it was updated.
+      # For example we add the jets-rails to the Gemfile.
+      copy_back_gemfile_lock
+
+      puts 'Bundle install success.'
+    end
+
+    def copy_back_gemfile_lock
+      src = "#{cache_area}/Gemfile.lock"
+      dest = "#{@full_app_root}/Gemfile.lock"
+      FileUtils.cp(src, dest)
+    end
+
+    # Clean up extra unneeded files to reduce package size
+    # Because we're removing files (something dangerous) use full paths.
+    def tidy
+      puts "Tidying project: removing ignored files to reduce package size."
+      tidy_project(@full_app_root)
+      # The rack sub project has it's own gitignore.
+      tidy_project(@full_app_root+"/rack")
+    end
+
+    def tidy_project(path)
+      Tidy.new(path).cleanup!
+    end
+
+    # When using submodules, bundler leaves old submodules behind. Over time this inflates
+    # the size of the the cache gems.  So we'll clean it up.
+    def clean_old_submodules
+      # https://stackoverflow.com/questions/38800129/parsing-a-gemfile-lock-with-bundler
+      lockfile = "#{cache_area}/Gemfile.lock"
+      return unless File.exist?(lockfile)
+
+      return if Bundler.bundler_major_version <= 1 # LockfileParser only works for Bundler version 2+
+
+      parser = Bundler::LockfileParser.new(Bundler.read_file(lockfile))
+      specs = parser.specs
+
+      # specs = Bundler.load.specs
+      # IE: spec.source.to_s: "https://github.com/tongueroo/webpacker.git (at jets@a8c4661)"
+      submoduled_specs = specs.select do |spec|
+        spec.source.to_s =~ /@\w+\)/
+      end
+
+      # find git shas to keep
+      # IE: ["a8c4661", "abc4661"]
+      git_shas = submoduled_specs.map do |spec|
+        md = spec.source.to_s.match(/@(\w+)\)/)
+        md[1] # git_sha
+      end
+
+      # IE: /tmp/jets/demo/cache/vendor/gems/ruby/2.5.0/bundler/gems/webpacker-a8c46614c675
+      Dir.glob("#{cache_area}/vendor/gems/ruby/2.5.0/bundler/gems/*").each do |path|
+        sha = path.split('-').last[0..6] # only first 7 chars of the git sha
+        unless git_shas.include?(sha)
+          # puts "Removing old submoduled gem: #{path}" # uncomment to see and debug
+          FileUtils.rm_rf(path) # REMOVE old submodule directory
+        end
+      end
+    end
+
+    def copy_bundled_gems(full_project_path)
+      src = "#{full_project_path}/bundled_gems"
+      return unless File.exist?(src)
+      Jets::Util.cp_r(src, "#{cache_area}/bundled_gems")
+    end
+
+    def copy_gemfiles(full_project_path)
+      FileUtils.mkdir_p(cache_area)
+      FileUtils.cp("#{full_project_path}/Gemfile", "#{cache_area}/Gemfile")
+
+      gemfile_lock = "#{full_project_path}/Gemfile.lock"
+      dest = "#{cache_area}/Gemfile.lock"
+      return unless File.exist?(gemfile_lock)
+
+      FileUtils.cp(gemfile_lock, dest)
+    end
+
+    # Remove the BUNDLED WITH line since we don't control the bundler gem version on AWS Lambda
+    # And this can cause issues with require 'bundler/setup'
+    def rewrite_gemfile_lock(gemfile_lock)
+      lines = IO.readlines(gemfile_lock)
+
+      # Remove BUNDLED WITH
+      # amount is the number of lines to remove
+      new_lines, capture, count, amount = [], true, 0, 2
+      lines.each do |l|
+        capture = false if l.include?('BUNDLED WITH')
+        if capture
+          new_lines << l
+        end
+        if capture == false
+          count += 1
+          capture = count > amount # renable capture
+        end
+      end
+
+      # Replace things like nokogiri (1.11.1-x86_64-darwin) => nokogiri (1.11.1)
+      lines, new_lines = new_lines, []
+      lines.each do |l|
+        if l.include?("-x86_64-darwin")
+          l = l.sub('-x86_64-darwin','')
+        end
+        new_lines << l
+      end
+
+      # Make sure platform is ruby
+      lines, new_lines, marker = new_lines, [], false
+      lines.each do |l|
+        if marker # the next loop has the platform we want to replace
+          new_lines << "  ruby\n"
+          marker = false
+          next
+        end
+        marker = l.include?('PLATFORMS')
+        new_lines << l
+      end
+
+      content = new_lines.join('')
+      IO.write(gemfile_lock, content)
+    end
+
+    def copy_bundle_config
+      # Override project's .bundle/config and ensure that .bundle/config matches
+      # at these 2 spots:
+      #   app_root/.bundle/config
+      #   vendor/gems/.bundle/config
+      cache_bundle_config = "#{cache_area}/.bundle/config"
+      app_bundle_config = "#{@full_app_root}/.bundle/config"
+      FileUtils.mkdir_p(File.dirname(app_bundle_config))
+      FileUtils.cp(cache_bundle_config, app_bundle_config)
+    end
+
+    # On circleci the "#{Jets.build_root}/.bundle/config" doesnt exist
+    # this only happens with ssh debugging, not when the ci.sh script gets ran.
+    # But on macosx it exists.
+    # Dont know why this is the case.
+    def create_bundle_config(frozen: false)
+      FileUtils.rm_rf("#{cache_area}/.bundle")
+      frozen_line = %Q|BUNDLE_FROZEN: "true"\n| if frozen
+      text =<<-EOL
+---
+#{frozen_line}BUNDLE_PATH: "vendor/gems"
+BUNDLE_WITHOUT: "development:test"
+EOL
+      bundle_config = "#{cache_area}/.bundle/config"
+      FileUtils.mkdir_p(File.dirname(bundle_config))
+      IO.write(bundle_config, text)
+    end
+
+    def copy_cache_gems
+      vendor_gems = "#{@full_app_root}/vendor/gems"
+      if File.exist?(vendor_gems)
+        puts "Removing current vendor_gems from project"
+        FileUtils.rm_rf(vendor_gems)
+      end
+      # Leave #{Jets.build_root}/vendor_gems behind to act as cache
+      if File.exist?("#{cache_area}/vendor/gems")
+        FileUtils.mkdir_p(File.dirname(vendor_gems))
+        Jets::Util.cp_r("#{cache_area}/vendor/gems", vendor_gems)
+      end
+    end
+  end
+end

data/lib/jets/builders/shim_vars/app.rb

@@ -0,0 +1,78 @@
+# Jets::Builders::ShimVars::Shared.new(path)
+#
+#   @vars.functions.each do |function_name|
+#     @vars.handler_for(function_name)
+#   end
+#
+# Implements:
+#
+#   functions: IE [:index, :show]
+#   handler_for(function_name): IE handlers/controllers/posts_controller.index
+#   dest_path: IE: handlers/controllers/posts_controller.js
+#
+module Jets::Builders::ShimVars
+  class App < Base
+    # Allow user to specify relative or full path. The right path gets used
+    # internally. Example paths:
+    #   app/controllers/posts_controller.rb
+    #   app/jobs/hard_job.rb
+    #   /tmp/jets/build/app_root/app/jobs/hard_job.rb
+    #   /tmp/jets/build/app_root/app/functions/hello.rb
+    def initialize(path)
+      @full_path = full(path)
+      @relative_path = relative(path)
+    end
+
+    def full(path)
+      path = "#{Jets.root}/#{path}" unless path.starts_with?("/")
+      path
+    end
+
+    def relative(path)
+      full_path = full(path)
+      full_path.sub("#{Jets.root}/", "")
+               .sub(/.*internal\/app/, "app")
+
+    end
+
+    # process_type is key, it will either "controller" or "job".
+    # It is used to deduce klass, etc.
+    # We get the process_type from the path.
+    # Example paths:
+    #   app/controllers/posts_controller.rb
+    #   app/jobs/hard_job.rb
+    def process_type
+      @relative_path.split('/')[1].singularize # controller or job
+    end
+
+    # Returns the public methods of the child_class.
+    # Example: [:create, :update]
+    def functions
+      klass.lambda_functions
+    end
+
+    # Examples: PostsController, HardJob, Hello, HelloFunction
+    def klass
+      @klass ||= Jets::Klass.from_path(@relative_path)
+    end
+
+    def lang(meth)
+      klass.tasks.find
+    end
+
+    # This gets called in the node shim js template
+    # IE handlers/controllers/posts_controller.index
+    def handler_for(meth)
+      # possibly not include _function
+      underscored_name = @relative_path.sub(%r{app/(\w+)/},'').sub('.rb','')
+      "handlers/#{process_type.pluralize}/#{underscored_name}.#{meth}"
+    end
+
+    # Example return: "handlers/controllers/posts.js"
+    # TODO: rename this to dest_path or something better now since using native ruby
+    def dest_path
+      @relative_path
+        .sub("app", "handlers")
+    end
+  end
+end

data/lib/jets/builders/shim_vars/base.rb

@@ -0,0 +1,25 @@
+module Jets::Builders::ShimVars
+  class Base
+    include Jets::AwsServices
+    extend Memoist
+
+    def s3_bucket
+      Jets.aws.s3_bucket
+    end
+
+    def rack_zip
+      checksum_zip(:rack)
+    end
+
+    def bundled_zip
+      checksum_zip(:bundled)
+    end
+
+  private
+    def checksum_zip(name)
+      checksum = Jets::Builders::Md5.checksums["stage/#{name}"]
+      return unless checksum
+      "#{name}-#{checksum}.zip"
+    end
+  end
+end

data/lib/jets/builders/shim_vars/shared.rb

@@ -0,0 +1,37 @@
+# Jets::Builders::ShimVars::Shared.new(fun)
+#
+#   @deducer.functions.each do |function_name|
+#     @deducer.handler_for(function_name)
+#   end
+#
+# Implements:
+#
+#   functions: IE [:index, :show]
+#   handler_for(function_name): IE handlers/controllers/posts_controller.index
+#   dest_path: IE: handlers/controllers/posts_controller.js
+#
+module Jets::Builders::ShimVars
+  class Shared < Base
+    # fun is a Jets::Stack::Function
+    def initialize(fun)
+      @fun = fun
+    end
+
+    # Always only one element for shared functions
+    # functions: IE [:handle]
+    def functions
+      [@fun.meth] # function_names
+    end
+
+    # Dont need function_name arg but keeping the same interface as parent class
+    # IE handlers/shared/functions/bob.handle
+    def handler_for(function_name)
+      @fun.handler_dest
+    end
+
+    # IE handlers/shared/functions/bob.js
+    def dest_path
+      @fun.handler_dest
+    end
+  end
+end

data/lib/jets/builders/templates/handler.rb

@@ -0,0 +1,12 @@
+require "bundler/setup"
+require "jets"
+Jets.once  # runs once in lambda execution context
+
+<% @vars.functions.each do |function_name|
+  handler = @vars.handler_for(function_name)
+  meth = handler.split('.').last
+-%>
+def <%= meth -%>(event:, context:)
+  Jets.process(event, context, "<%= handler -%>")
+end
+<% end %>

data/lib/jets/builders/tidy.rb

@@ -0,0 +1,107 @@
+module Jets::Builders
+  class Tidy
+    include Util
+
+    def initialize(project_root, noop: false)
+      @project_root = project_root
+      @noop = noop
+    end
+
+    def cleanup!
+      removals.each do |removal|
+        removal = removal.sub(%r{^/},'') # remove leading slash
+        path = "#{@project_root}/#{removal}"
+        rm_rf(path)
+      end
+
+      clean_vendor_gems
+      clean_webpack_assets
+    end
+
+    # Clean out unnecessary src and compiled packs because Jets serves them out of s3.
+    # This keeps the code size down to help keep it in size limit so we can use the
+    # live Lambda console editor.
+    def clean_webpack_assets
+      FileUtils.rm_rf("#{@project_root}/app/javascript/src")
+
+      return unless File.exist?("#{@project_root}/public/packs") # this class works for rack subfolder too
+      FileUtils.mv("#{@project_root}/public/packs/manifest.json", "#{stage_area}/manifest.json")
+      FileUtils.rm_rf("#{@project_root}/public/packs")
+      FileUtils.mkdir_p("#{@project_root}/public/packs")
+      FileUtils.mv("#{stage_area}/manifest.json", "#{@project_root}/public/packs/manifest.json")
+    end
+
+    def removals
+      removals = always_removals
+      removals += get_removals("#{@project_root}/.gitignore")
+      removals += get_removals("#{@project_root}/.dockerignore")
+      removals += get_removals("#{@project_root}/.jetsignore")
+      removals = removals.reject do |p|
+        jetskeep.find do |keep|
+          p == keep
+        end
+      end
+      removals.uniq
+    end
+
+    def get_removals(file)
+      path = file
+      return [] unless File.exist?(path)
+
+      removal = File.read(path).split("\n")
+      removal.map {|i| i.strip}.reject {|i| i =~ /^#/ || i.empty?}
+      # IE: ["/handlers", "/bundled*", "/vendor/jets]
+    end
+
+    # We clean out ignored files pretty aggressively. So provide
+    # a way for users to keep files from being cleaned out.
+    def jetskeep
+      always = %w[.bundle /public/packs /public/packs-test vendor]
+      path = "#{@project_root}/.jetskeep"
+      return always unless File.exist?(path)
+
+      keep = IO.readlines(path)
+      keep = keep.map {|i| i.strip}.reject { |i| i =~ /^#/ || i.empty? }
+      (always + keep).uniq
+    end
+
+    # folders to remove in the vendor/gems folder regardless of the level of the folder
+    def clean_vendor_gems
+      # Thanks: https://stackoverflow.com/questions/11385795/ruby-list-directory-with-dir-including-dotfiles-but-not-and
+      Dir.glob("#{@project_root}/vendor/gems/**/*", File::FNM_DOTMATCH).each do |path|
+        next unless File.directory?(path)
+        dir = File.basename(path)
+        next unless always_removals.include?(dir)
+
+        rm_rf(path)
+      end
+
+      remove_gem_cache
+    end
+
+    # Reason do not remove the cache folder generally is because some gems have
+    # actual cache folders that they used.
+    def remove_gem_cache
+      cache_path = "#{@project_root}/vendor/gems/ruby/#{Jets.ruby_folder}/cache"
+      FileUtils.rm_rf(cache_path)
+    end
+
+    def rm_rf(path)
+      exists = File.exist?("#{path}/.gitkeep") || File.exist?("#{path}/.keep")
+      return if exists
+
+      # say "  rm -rf #{path}".color(:yellow) # uncomment to debug
+      system("rm -rf #{path}") unless @noop
+    end
+
+    # These directories will be removed regardless of dir level
+    def always_removals
+      %w[.git spec tmp]
+    end
+
+    def say(message)
+      message = "NOOP #{message}" if @noop
+      puts message
+    end
+  end
+end

data/lib/jets/builders/util.rb

@@ -0,0 +1,31 @@
+module Jets::Builders
+  module Util
+  private
+    def sh(command)
+      puts "=> #{command}".color(:green)
+      success = system(command)
+      unless success
+        puts "#{command} failed to run.".color(:red)
+        puts caller[0]
+        exit 1
+      end
+      success
+    end
+
+    def headline(message)
+      puts "=> #{message}".color(:cyan)
+    end
+
+    def build_area
+      Jets.build_root
+    end
+
+    def stage_area
+      "#{build_area}/stage"
+    end
+
+    def cache_area
+      "#{build_area}/cache" # cleaner to use full path for this setting
+    end
+  end
+end

data/lib/jets/bundle.rb

@@ -0,0 +1,88 @@
+require "jets/turbo"
+
+module Jets
+  # Named Bundle vs Bundler to avoid having to fully qualify ::Bundler
+  module Bundle
+    # Looks like for zeitwerk module autovivification to work `bundle exec` must be called.
+    # This allows zeitwork module autovivification to work even if the user has not called jets with `bundle exec jets`.
+    # Bundler.setup is essentially the same as `bundle exec`
+    # Reference: https://www.justinweiss.com/articles/what-are-the-differences-between-irb/
+    #
+    # The Bundler.setup is only necessary because we use Bundler.require after require "zeitwerk" is called.
+    #
+    # Note, this is called super early right before require "zeitwerk"
+    # The initially Bundler.setup does not include the Jets.env group.
+    # Later in Jets::Booter, Bundle.require is called and includes the Jets.env group.
+    #
+    def setup
+      return unless jets_project?
+      return unless bundler_enabled?
+      Kernel.require "bundler/setup"
+      Bundler.setup # Same as Bundler.setup(:default)
+    rescue LoadError => e
+      handle_error(e)
+    end
+
+    # Bundler.require called when environment boots up via Jets.boot.  This will eagerly require all gems in the
+    # Gemfile. This means the user will not have to explictly require dependencies.
+    #
+    # It also useful for when to loading Rake tasks in Jets::Commands::RakeTasks.load! For example, some gems like
+    # webpacker that load rake tasks are specified with a git based source:
+    #
+    #   gem "webpacker", git: "https://github.com/tongueroo/webpacker.git"
+    #
+    # This results in the user having to specific bundle exec in front of jets for those rake tasks to show up in
+    # jets help. Instead, when the user is within the project folder, jets automatically requires bundler for the
+    # user. So the rake tasks show up when calling jets help.
+    #
+    # When the user calls jets help from outside the project folder, bundler is not used and the load errors get
+    # rescued gracefully.  This is done in Jets::Commands::RakeTasks.load!  In the case when user is in another
+    # project with another Gemfile, the load errors will also be rescued.
+    def require
+      return unless jets_project?
+      return unless bundler_enabled?
+      Kernel.require "bundler/setup"
+      Bundler.require(*bundler_groups)
+    rescue LoadError => e
+      handle_error(e)
+    end
+
+    def handle_error(e)
+      puts e.message
+      puts <<~EOL.color(:yellow)
+        WARNING: Unable to require "bundler/setup"
+        There may be something funny with your ruby and bundler setup.
+        You can try upgrading bundler and rubygems:
+
+            gem update --system
+            gem install bundler
+
+        Here are some links that may be helpful:
+
+        * https://bundler.io/blog/2019/01/03/announcing-bundler-2.html
+        * https://community.rubyonjets.com/t/jets-1-9-8-install-issue-bundler-setup-missing/185/2
+
+        Also, running bundle exec in front of your command may remove this message.
+      EOL
+    end
+
+    # Also check for Afterburner mode since in that mode jets is a standalone tool.
+    def bundler_enabled?
+      !Jets::Turbo.afterburner? && gemfile?
+    end
+
+    def gemfile?
+      ENV['BUNDLE_GEMFILE'] || File.exist?("Gemfile")
+    end
+
+    def bundler_groups
+      [:default, Jets.env.to_sym]
+    end
+
+    def jets_project?
+      File.exist?("config/application.rb")
+    end
+
+    extend self
+  end
+end

data/lib/jets/camelizer.rb

@@ -0,0 +1,4 @@
+# Wrapper to CfnCamelizer gem
+module Jets
+  Camelizer = CfnCamelizer
+end

data/lib/jets/cfn/builders/api_deployment_builder.rb

@@ -0,0 +1,56 @@
+module Jets::Cfn::Builders
+  class ApiDeploymentBuilder
+    include Interface
+    include Jets::AwsServices
+
+    def initialize(options={})
+      @options = options
+      @template = ActiveSupport::HashWithIndifferentAccess.new(Resources: {})
+    end
+
+    # compose is an interface method
+    def compose
+      deployment = Jets::Resource::ApiGateway::Deployment.new
+      add_resource(deployment)
+      add_parameters(deployment.parameters)
+      add_outputs(deployment.outputs)
+
+      add_base_path_mapping
+    end
+
+    # Because Jets generates a new timestamped logical id for the API Deployment
+    # resource it also creates a new root base path mapping and fails.  Additionally,
+    # the base path mapping depends on the API Deploy for the stage name.
+    #
+    # We resolve this by using a custom resource that does an in-place update.
+    #
+    # Note, also tried to change the domain name of to something like demo-dev-[random].mydomain.com
+    # but that does not work because the domain name has to match the route53 record exactly.
+    #
+    def add_base_path_mapping
+      return unless Jets.custom_domain?
+
+      function = Jets::Resource::ApiGateway::BasePath::Function.new
+      add_resource(function)
+      add_outputs(function.outputs)
+
+      mapping = Jets::Resource::ApiGateway::BasePath::Mapping.new
+      add_resource(mapping)
+      add_outputs(mapping.outputs)
+
+      iam_role = Jets::Resource::ApiGateway::BasePath::Role.new
+      add_resource(iam_role)
+      add_outputs(iam_role.outputs)
+    end
+
+    # template_path is an interface method
+    def template_path
+      Jets::Naming.api_deployment_template_path
+    end
+
+    # do not bother writing a template if routes are empty
+    def write
+      super unless Jets::Router.routes.empty?
+    end
+  end
+end