jets 1.3.9 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5de0cb4913504da995988d705fc03c5248d398fbd2538d3ec356ee89124dd38f
-  data.tar.gz: abb22eb7ac6a16faa1914835e9c68c86263693cdd92aaebec44edda63a562e14
+  metadata.gz: ec914295c99192ef042110202c7ac3161003e644fb73a7b24cc196884d6295ae
+  data.tar.gz: 716b93e31d27fa740b11bfba46db5a5af8b344d0e8877363ad07a49d50728032
 SHA512:
-  metadata.gz: b7f26985d0b5dde339f8e887f508938e1565a306214520b563ebdd4c32fd87c0f05ff8f12366c35b9204e3fb35085fe959f852bb1255c99f673c896c94ccb227
-  data.tar.gz: ad759caa51dae3942176697965c0313a699702415c44c5f9d4eaf329028e72acc9702ce357c269fa3ebee95f5ee07e46d3af216fc518f5702cf91dd4ae78736c
+  metadata.gz: fc7e956e2c6a675fc14d325f3207d10cecb16d29a54e7ef81679de2161feff423055bbef8e0b25071dc059a9c57c4205a5d8d20094d0fe46315deed8b58f8062
+  data.tar.gz: 05067666cbda6be6986a12fc1acd6f757f8c33f8fa12a3debfe7182f72d1b08a17bd7321d0c23a2e2d0f827e19c7b16ea6386a2e4a177d71e66af16b41f42d42

data/CHANGELOG.md

@@ -3,6 +3,11 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [1.4.0]
+- Afterburner mode: Allows you to deploy from within a Rails app
+- #110 from tongueroo/jet-pack: Turbo charge mode: enabling afterburner
+- Afterburner is recommended over Mega Mode.
+
 ## [1.3.9]
 - #106 from eamsc/sqs-special-map
 - #107 from eamsc/sqs-resource

data/Gemfile.lock

@@ -11,7 +11,7 @@ GIT
 PATH
   remote: .
   specs:
-    jets (1.3.9)
+    jets (1.4.0)
       actionpack (>= 5.2.1)
       actionview (>= 5.2.1)
       activerecord (>= 5.2.1)
@@ -113,7 +113,7 @@ GEM
     gems (1.1.1)
       json
     hashie (3.6.0)
-    i18n (1.2.0)
+    i18n (1.3.0)
       concurrent-ruby (~> 1.0)
     jmespath (1.4.0)
     json (2.1.0)
@@ -123,7 +123,7 @@ GEM
       nokogiri (>= 1.5.9)
     memoist (0.16.0)
     method_source (0.9.2)
-    mimemagic (0.3.2)
+    mimemagic (0.3.3)
     mini_portile2 (2.4.0)
     minitest (5.11.3)
     mysql2 (0.5.2)

data/lib/jets.rb

@@ -50,6 +50,7 @@ module Jets
   autoload :Stack, "jets/stack"
   autoload :TmpLoader, "jets/tmp_loader"
   autoload :Turbine, 'jets/turbine'
+  autoload :Turbo, 'jets/turbo'
   autoload :Util, "jets/util"
 
   extend Core # root, logger, etc

data/lib/jets/booter.rb

@@ -4,8 +4,9 @@ class Jets::Booter
     def boot!(options={})
       return if @booted
 
+      turbo_charge
       confirm_jets_project!
-      require_bundle_gems
+      require_bundle_gems unless bypass_bundler_setup?
       Jets::Dotenv.load!
 
       Jets.application.setup!
@@ -20,6 +21,16 @@ class Jets::Booter
       @booted = true
     end
 
+    def bypass_bundler_setup?
+      command = ARGV.first
+      %w[build delete deploy].include?(command)
+    end
+
+    def turbo_charge
+      turbo = Jets::Turbo.new
+      turbo.charge
+    end
+
     # Builds and memoize stack so it only gets built on bootup
     def build_middleware_stack
       Jets.application.build_stack

data/lib/jets/builders/code_builder.rb

@@ -35,7 +35,7 @@ class Jets::Builders
       compile_assets # easier to do before we copy the project because node and yarn has been likely setup in the that dir
       compile_rails_assets
       copy_project
-      Dir.chdir(full(tmp_code)) do
+      Dir.chdir("#{stage_area}/code") do
         # These commands run from project root
         code_setup
         package_ruby
@@ -47,7 +47,7 @@ class Jets::Builders
     # to reference files with the md5 checksum.  The files are the:
     #
     #   jets/code/rack-checksum.zip
-    #   jets/code/bundled-checksum.zip
+    #   jets/code/opt-checksum.zip
     #
     # We compute the checksums before we generate the node shim handlers.
     def calculate_md5s
@@ -86,10 +86,6 @@ class Jets::Builders
       end
     end
 
-    def stage_area
-      "#{Jets.build_root}/stage"
-    end
-
     def code_setup
       reconfigure_development_webpacker
     end
@@ -101,19 +97,12 @@ class Jets::Builders
       copy_internal_jets_code
 
       # Code prep and zipping
-      build_lambda_layer
       check_code_size!
       calculate_md5s # must be called before generate_node_shims and create_zip_files
       generate_node_shims
       create_zip_files
     end
 
-    def build_lambda_layer
-      return if Jets.poly_only?
-      lambda_layer = LambdaLayer.new
-      lambda_layer.build
-    end
-
     def check_code_size!
       CodeSize.check!
     end
@@ -125,7 +114,7 @@ class Jets::Builders
       files = []
       files.each do |relative_path|
         src = File.expand_path("../internal/#{relative_path}", File.dirname(__FILE__))
-        dest = "#{full(tmp_code)}/#{relative_path}"
+        dest = "#{"#{stage_area}/code"}/#{relative_path}"
         FileUtils.mkdir_p(File.dirname(dest))
         FileUtils.cp(src, dest)
       end
@@ -145,12 +134,11 @@ class Jets::Builders
     # At this point the minimal stack exists, so we can grab it with the AWS API.
     # We do not want to grab this as part of the live request because it is slow.
     def store_s3_base_url
-      write_s3_base_url("config/s3_base_url.txt")
-      write_s3_base_url("rack/config/s3_base_url.txt") if Jets.rack?
+      write_s3_base_url("#{stage_area}/code/config/s3_base_url.txt")
+      write_s3_base_url("#{stage_area}/rack/config/s3_base_url.txt") if Jets.rack?
     end
 
-    def write_s3_base_url(relative_path)
-      full_path = "#{full(tmp_code)}/#{relative_path}"
+    def write_s3_base_url(full_path)
       FileUtils.mkdir_p(File.dirname(full_path))
       IO.write(full_path, s3_base_url)
     end
@@ -175,7 +163,7 @@ class Jets::Builders
     end
 
     def disable_webpacker_middleware
-      full_path = "#{full(tmp_code)}/config/disable-webpacker-middleware.txt"
+      full_path = "#{"#{stage_area}/code"}/config/disable-webpacker-middleware.txt"
       FileUtils.mkdir_p(File.dirname(full_path))
       FileUtils.touch(full_path)
     end
@@ -212,17 +200,25 @@ class Jets::Builders
 
       return unless Jets.rack?
 
+      # Need to capture JETS_ROOT since can be changed by Turbo mode
+      jets_root = Jets.root
       Bundler.with_clean_env do
-        rails_assets(:clobber)
-        rails_assets(:precompile)
+        gemfile = ENV['BUNDLE_GEMFILE']
+        ENV['BUNDLE_GEMFILE'] = "#{jets_root}/rack/Gemfile"
+
+        sh "cd #{jets_root} && bundle install"
+        rails_assets(:clobber, jets_root: jets_root)
+        rails_assets(:precompile, jets_root: jets_root)
+
+        ENV['BUNDLE_GEMFILE'] = gemfile
       end
     end
 
-    def rails_assets(cmd)
+    def rails_assets(cmd, jets_root:)
       # rake is available in both rails 4 and 5. rails command only in 5
       command = "bundle exec rake assets:#{cmd} --trace"
       command = "RAILS_ENV=#{Jets.env} #{command}" unless Jets.env.development?
-      sh("cd rack && #{command}")
+      sh("cd #{jets_root}rack && #{command}")
     end
 
     # Rudimentary rails detection
@@ -247,14 +243,14 @@ class Jets::Builders
     # directory untouched and we can also remove a bunch of unnecessary files like
     # logs before zipping it up.
     def copy_project
-      headline "Copying current project directory to temporary build area: #{full(tmp_code)}"
-      FileUtils.rm_rf(stage_area) # clear out from previous build
-      FileUtils.mkdir_p(stage_area)
-      FileUtils.rm_rf(full(tmp_code)) # remove current code folder
+      headline "Copying current project directory to temporary build area: #{"#{stage_area}/code"}"
+      FileUtils.rm_rf("#{build_area}/stage") # clear out from previous build's stage area
+      FileUtils.mkdir_p("#{build_area}/stage")
+      FileUtils.rm_rf("#{stage_area}/code") # remove current code folder
       move_node_modules(Jets.root, Jets.build_root)
       begin
-        # puts "cp -r #{@full_project_path} #{full(tmp_code)}".colorize(:yellow) # uncomment to debug
-        FileUtils.cp_r(@full_project_path, full(tmp_code))
+        # puts "cp -r #{@full_project_path} #{"#{stage_area}/code"}".colorize(:yellow) # uncomment to debug
+        FileUtils.cp_r(@full_project_path, "#{stage_area}/code")
       ensure
         move_node_modules(Jets.build_root, Jets.root) # move node_modules directory back
       end
@@ -279,7 +275,7 @@ class Jets::Builders
       return unless Jets.env.development?
       headline "Reconfiguring webpacker development settings for AWS Lambda."
 
-      webpacker_yml = "#{full(tmp_code)}/config/webpacker.yml"
+      webpacker_yml = "#{"#{stage_area}/code"}/config/webpacker.yml"
       return unless File.exist?(webpacker_yml)
 
       config = YAML.load_file(webpacker_yml)
@@ -302,15 +298,23 @@ class Jets::Builders
       return if Jets.poly_only?
 
       ruby_packager.install
-      reconfigure_rails # call here after full(tmp_code) is available
+      reconfigure_rails # call here after "#{stage_area}/code" is available
       rack_packager.install
-      ruby_packager.finish # by this time we have a /tmp/jets/demo/stage/code/bundled
+      ruby_packager.finish # by this time we have a /tmp/jets/demo/stage/code/vendor/gems
       rack_packager.finish
+
+      build_lambda_layer
+    end
+
+    def build_lambda_layer
+      return if Jets.poly_only?
+      lambda_layer = LambdaLayer.new
+      lambda_layer.build
     end
 
     # TODO: Move logic into plugin instead
     def reconfigure_rails
-      ReconfigureRails.new("#{full(tmp_code)}/rack").run
+      ReconfigureRails.new("#{"#{stage_area}/code"}/rack").run
     end
 
     def cache_check_message
@@ -321,9 +325,9 @@ class Jets::Builders
 
     def check_ruby_version
       unless ruby_version_supported?
-        puts "You are using ruby version #{RUBY_VERSION} which is not supported by Jets."
+        puts "You are using Ruby version #{RUBY_VERSION} which is not supported by Jets."
         ruby_variant = Jets::RUBY_VERSION.split('.')[0..1].join('.') + '.x'
-        abort("Jets uses ruby #{Jets::RUBY_VERSION}.  You should use a variant of ruby #{ruby_variant}".colorize(:red))
+        abort("Jets uses Ruby #{Jets::RUBY_VERSION}.  You should use a variant of Ruby #{ruby_variant}".colorize(:red))
       end
     end
 

data/lib/jets/builders/code_size.rb

@@ -35,11 +35,13 @@ class Jets::Builders
       say "Gem Layer: #{megabytes(opt_size)} - #{stage_area}/opt"
       say "Total Package: #{megabytes(total_size)}"
       say "Over limit by: #{megabytes(overlimit)}"
-      # sh "du -csh #{stage_area}/*" unless ENV['TEST'] # uncomment to debug
+      say "Sometimes blowing away the /tmp/jets cache will reduce the size: rm -rf /tmp/jets"
+      # sh "du -kcsh #{stage_area}/*" unless ENV['TEST'] # uncomment to debug
     end
 
     def compute_size(path)
-      out = `du -s #{path}`
+      # -k option is required for macosx but not for linux
+      out = `du -ks #{path}`
       out.split(' ').first.to_i # bytes
     end
 

data/lib/jets/builders/gem_replacer.rb

@@ -1,8 +1,7 @@
 class Jets::Builders
   class GemReplacer
     extend Memoist
-    def initialize(ruby_version, options)
-      @ruby_version = ruby_version
+    def initialize(options)
       @options = options
     end
 
@@ -17,25 +16,9 @@ class Jets::Builders
         gem_extractor.run
       end
 
-      # At this point the current compiled gems have been removed and compiled gems
-      # have been unpacked to code/opt. We can take the unpacked gems in opt and fully
-      # move them into vendor/gems gems now.
-      move_opt_gems_to_vendor
-
       tidy
     end
 
-    def move_opt_gems_to_vendor
-      code = "#{Jets.build_root}/stage/code"
-      opt_gems = "#{code}/opt/ruby/gems/#{Jets::Gems.ruby_folder}"
-      vendor_gems = "#{code}/vendor/gems/ruby/#{Jets::Gems.ruby_folder}"
-      # https://stackoverflow.com/questions/23698183/how-to-force-cp-to-overwrite-directory-instead-of-creating-another-one-inside
-      # Trailing slashes are required
-      sh "rsync -a --links #{opt_gems}/ #{vendor_gems}/"
-      # clean up opt compiled gems
-      FileUtils.rm_rf("#{code}/opt/ruby")
-    end
-
     def sh(command)
       puts "=> #{command}".colorize(:green)
       success = system(command)
@@ -43,12 +26,16 @@ class Jets::Builders
       success
     end
 
+    def ruby_folder
+      Jets::Gems.ruby_folder
+    end
+
     # remove unnecessary files to reduce package size
     def tidy
       # project_root: /tmp/jets/demo/stage/code/
       # /tmp/jets/demo/stage/code/bundled
-      tidy_gems("#{@options[:project_root]}/bundled/gems/ruby/*/gems/*")
-      tidy_gems("#{@options[:project_root]}/bundled/gems/ruby/*/bundler/gems/*")
+      tidy_gems("#{@options[:project_root]}/ruby/gems/#{ruby_folder}/*/gems/*")
+      tidy_gems("#{@options[:project_root]}/ruby/gems/#{ruby_folder}/*/bundler/gems/*")
     end
 
     def tidy_gems(gems_path)

data/lib/jets/builders/lambda_layer.rb

@@ -2,50 +2,68 @@ class Jets::Builders
   class LambdaLayer
     include Util
 
-    # At this point we gems have all been moved to stage/code/vendor/gems, this includes
-    # binary gems, a good state. This method moves them:
-    #
-    #   from stage/code/vendor/gems/ruby/2.5.0
-    #   to stage/opt/ruby/gems/2.5.0
-    #
-    # So we can move gems into the Lambda Layer. Important folders later:
-    #
-    #   stage/code/opt/lib
-    #   stage/code/opt/ruby
-    #
+    # At this point gems are in the stage/code and stage/rack folders still.
+    # We consolidate all gems to stage/opt.
+    # Then replace the binary gems.
     def build
-      move_opt_to_stage
-      move_vendor_to_opt
-      symlink_vendor_gems
+      consolidate_gems_to_opt
+      replace_compiled_gems
     end
 
-    def move_opt_to_stage
-      opt_original = "#{code_area}/opt"
-      opt = "#{stage_area}/opt"
-      FileUtils.mkdir_p(File.dirname(opt))
-      FileUtils.mv(opt_original, opt)
+    # Also restructure the folder from:
+    #   vendor/gems/ruby/2.5.0
+    # To:
+    #   ruby/gems/2.5.0
+    #
+    # For Lambda Layer structure
+    def consolidate_gems_to_opt
+      src = "#{stage_area}/code/vendor/gems/ruby/2.5.0"
+      dest = "#{stage_area}/opt/ruby/gems/2.5.0"
+      rsync_and_link(src, dest)
+
+      return unless Jets.rack?
+
+      src = "#{stage_area}/rack/vendor/gems/ruby/2.5.0"
+      rsync_and_link(src, dest)
     end
 
-    def move_vendor_to_opt
-      ruby_folder = Jets::Gems.ruby_folder
-      gems_original = "#{code_area}/vendor/gems/ruby/#{ruby_folder}"
-      gems = "#{stage_area}/opt/ruby/gems/#{ruby_folder}"
-
-      FileUtils.mkdir_p(File.dirname(gems))
-      FileUtils.mv(gems_original, gems)
-      # Deleting in this way to make sure folders are empty before we delete them
-      FileUtils.rmdir("#{code_area}/vendor/gems/ruby")
-      FileUtils.rmdir("#{code_area}/vendor/gems")
-      FileUtils.rmdir("#{code_area}/vendor") if Dir.empty?("#{code_area}/vendor")
+    def rsync_and_link(src, dest)
+      FileUtils.mkdir_p(dest)
+      # Trailing slashes are required
+      sh "rsync -a --links #{src}/ #{dest}/"
+
+      FileUtils.rm_rf(src) # blow away original 2.5.0 folder
+
+      # Create symlink that will point to the gems in the Lambda Layer:
+      #   stage/opt/ruby/gems/2.5.0 -> /opt/ruby/gems/2.5.0
+      FileUtils.ln_sf("/opt/ruby/gems/#{Jets::Gems.ruby_folder}", src)
     end
 
-    # Simple logic: vendor/gems/ruby/2.5.0 -> /opt/ruby/gems/2.5.0
-    def symlink_vendor_gems
-      ruby_folder = Jets::Gems.ruby_folder
-      dest = "#{code_area}/vendor/gems/ruby/#{ruby_folder}"
-      FileUtils.mkdir_p(File.dirname(dest))
-      # puts "ln -sf /opt/ruby/gems/#{ruby_folder} #{dest}" # uncomment to debug
-      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", dest)
+    # replace_compiled_gems:
+    #   remove binary gems in vendor/gems/ruby/2.5.0
+    #   extract binary gems in opt/ruby/gems/2.5.0
+    #   move binary gems from opt/ruby/gems/2.5.0 to vendor/gems/ruby/2.5.0
+    #
+    # After this point, gems have been replace in stage/code/vendor/gems with their
+    # binary extensions: a good state. This method moves these gems to the Lambda
+    # Layers structure and creates a symlinks to it.  First:
+    #
+    #   from stage/code/vendor/gems/ruby/2.5.0
+    #   to stage/opt/ruby/gems/2.5.0
+    #
+    # Then:
+    #
+    #   stage/code/vendor/gems/ruby/2.5.0 -> /opt/ruby/gems/2.5.0
+    #
+    def replace_compiled_gems
+      project_root = "#{stage_area}/opt"
+      headline "Replacing compiled gems with AWS Lambda Linux compiled versions: #{project_root}"
+      options = {
+        s3: "lambdagems2",
+        build_root: cache_area, # used in jets-gems
+        project_root: project_root, # used in gem_replacer and jets-gems
+      }
+      GemReplacer.new(options).run
     end
   end
 end

data/lib/jets/builders/rack_packager.rb

@@ -3,9 +3,11 @@ class Jets::Builders
     def finish
       return unless gemfile_exist?
 
+      super
+
       symlink_gems
-      rack_symlink
       copy_rackup_wrappers
+      rack_symlink
     end
 
     def symlink_gems
@@ -31,10 +33,10 @@ class Jets::Builders
     # Moves folder to a stage folder and create a symlink its place
     # that links from /var/task to /tmp. Example:
     #
-    #   code_area/rack => /tmp/rack
+    #   stage/code/rack => /tmp/rack
     #
     def rack_symlink
-      src = "#{@full_app_root}/rack"
+      src = @full_app_root
       return unless File.exist?(src)
 
       dest = "#{stage_area}/rack"
@@ -43,7 +45,7 @@ class Jets::Builders
       FileUtils.mv(src, dest)
 
       # Create symlink
-      FileUtils.ln_sf("/tmp/rack", "/#{@full_app_root}/rack")
+      FileUtils.ln_sf("/tmp/rack", src)
     end
   end
 end