jets 1.0.1 → 1.0.2

data/vendor/jets-gems/lib/jets-gems.rb

@@ -0,0 +1 @@
+require "jets/gems"

data/vendor/jets-gems/lib/jets/gems.rb

@@ -0,0 +1,16 @@
+require "jets/gems/version"
+
+module Jets
+  module Gems
+    autoload :Check, 'jets/gems/check'
+    autoload :Exist, 'jets/gems/exist'
+    autoload :Extract, 'jets/gems/extract'
+    autoload :Report, 'jets/gems/report'
+
+    def ruby_version_folder
+      major, minor, patch = RUBY_VERSION.split('.')
+      [major, minor, '0'].join('.') # 2.5.1 => 2.5.0
+    end
+    extend self
+  end
+end

data/vendor/jets-gems/lib/jets/gems/check.rb

@@ -0,0 +1,156 @@
+# Assumes gems were just built and checks the filesystem to find and detect for
+# compiled gems.  Unless the cli option is set to true, the it'll just check
+# based on the gemspecs.
+module Jets::Gems
+  class Check
+    extend Memoist
+
+    attr_reader :missing_gems
+    def initialize(options={})
+      @options = options
+      @missing_gems = [] # keeps track of gems that are not found in any of the lambdagems sources
+    end
+
+    # Checks whether the gem is found on at least one of the lambdagems sources.
+    # By the time the loop finishes, found_gems will hold a map of gem names to found
+    # url sources. Example:
+    #
+    #   found_gems = {
+    #     "nokogiri-1.8.4" => "https://lambdagems.com",
+    #     "pg-0.21.0" => "https://anothersource.com",
+    #   }
+    #
+    def run
+      puts "Checking projects gems for pre-built Lambda gems..."
+      found_gems = {}
+      compiled_gems.each do |gem_name|
+        puts "Checking #{gem_name}..." if @options[:cli]
+        gem_exists = false
+        Jets.config.lambdagems.sources.each do |source|
+          exist = Jets::Gems::Exist.new(source_url: source)
+          found = exist.check(gem_name)
+          # gem exists on at least of the lambdagem sources
+          if found
+            gem_exists = true
+            found_gems[gem_name] = source
+            break
+          end
+        end
+        unless gem_exists
+          @missing_gems << gem_name
+        end
+      end
+      found_gems
+    end
+
+    def missing?
+      !@missing_gems.empty?
+    end
+
+    def missing_message
+      template = <<-EOL
+Your project requires compiled gems were not available in any of your lambdagems sources.  Unavailable pre-compiled gems:
+<% missing_gems.each do |gem| %>
+* <%= gem -%>
+<% end %>
+
+Your current lambdagems sources:
+<% Jets.config.lambdagems.sources.map do |source| %>
+* <%= source -%>
+<% end %>
+
+Jets is unable to build a deployment package that will work on AWS Lambda without the required pre-compiled gems. To remedy this, you can:
+
+* Build the gem yourself and add it to your own custom lambdagems sources. Refer to the Lambda Gems Docs: http://rubyonjets.com/docs/lambdagems
+* Wait until it added to lambdagems.com. No need to report this to us, as we've already been notified.
+* Use another gem that does not require compilation.
+
+Compiled gems usually take some time to figure out how to build as they each depend on different libraries and packages. We make an effort add new gems as soon as we can.
+EOL
+      erb = ERB.new(template, nil, '-') # trim mode https://stackoverflow.com/questions/4632879/erb-template-removing-the-trailing-line
+      erb.result(binding)
+    end
+
+    # If there are subfolders compiled_gem_paths might have files deeper
+    # in the directory tree.  So lets grab the gem name and figure out the
+    # unique paths of the compiled gems from there.
+    def compiled_gems
+      # @use_gemspec option  finds compile gems with Gem::Specification
+      # The normal build process does not use this and checks the file system.
+      # So @use_gemspec is only used for this command:
+      #
+      #   jets gems:check
+      #
+      # This is because it seems like some gems like json are remove and screws things up.
+      # We'll filter out for the json gem as a hacky workaround, unsure if there are more
+      # gems though that exhibit this behavior.
+      if @options[:cli]
+        gemspec_compiled_gems
+      else
+        compiled_gems = compiled_gem_paths.map { |p| gem_name_from_path(p) }.uniq
+        # Double check that the gems are also in the gemspec list since that
+        # one is scoped to Bundler and will only included gems used in the project.
+        # This handles the possiblity of stale gems leftover from previous builds
+        # in the cache.
+        compiled_gems.select { |g| gemspec_compiled_gems.include?(g) }
+      end
+    end
+    memoize :compiled_gems
+
+    # Use pre-compiled gem because the gem could have development header shared
+    # object file dependencies.  The shared dependencies are packaged up as part
+    # of the pre-compiled gem so it is available in the Lambda execution environment.
+    #
+    # Example paths:
+    # Macosx:
+    #   bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/nokogiri-1.8.1
+    #   bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
+    # Official AWS Lambda Linux AMI:
+    #   bundled/gems/ruby/2.5.0/extensions/x86_64-linux/2.5.0-static/nokogiri-1.8.1
+    # Circleci Ubuntu based Linux:
+    #   bundled/gems/ruby/2.5.0/extensions/x86_64-linux/2.5.0/pg-0.21.0
+    def compiled_gem_paths
+      Dir.glob("#{Jets.build_root}/cache/bundled/gems/ruby/*/extensions/**/**/*.{so,bundle}")
+    end
+
+    # Input: bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
+    # Output: byebug-9.1.0
+    def gem_name_from_path(path)
+      regexp = /gems\/ruby\/\d+\.\d+\.\d+\/extensions\/.*?\/.*?\/(.*?)\//
+      gem_name = path.match(regexp)[1]
+    end
+
+    # So can also check for compiled gems with Gem::Specification
+    # But then also includes the json gem, which then bundler removes?
+    # We'll figure out the the json gems.
+    # https://gist.github.com/tongueroo/16f4aa5ac5393424103347b0e529495e
+    #
+    # This is a faster way to check but am unsure if there are more gems than just
+    # json that exhibit this behavior. So only using this technique for this commmand:
+    #
+    #   jets gems:check
+    #
+    # Thanks: https://gist.github.com/aelesbao/1414b169a79162b1d795 and
+    #   https://stackoverflow.com/questions/5165950/how-do-i-get-a-list-of-gems-that-are-installed-that-have-native-extensions
+    def specs_with_extensions
+      specs = Gem::Specification.each.select { |spec| spec.extensions.any?  }
+      specs.reject! { |spec| weird_gems.include?(spec.name) }
+      specs
+    end
+
+    def gemspec_compiled_gems
+      specs_with_extensions.map(&:full_name)
+    end
+
+    # Filter out the weird special case gems that bundler deletes?
+    # Probably to fix some bug.
+    #
+    #   $ bundle show json
+    #   The gem json has been deleted. It was installed at:
+    #   /home/ec2-user/.rbenv/versions/2.5.1/lib/ruby/gems/2.5.0/gems/json-2.1.0
+    #
+    def weird_gems
+      %w[json]
+    end
+  end
+end

data/vendor/jets-gems/lib/jets/gems/exist.rb

@@ -0,0 +1,54 @@
+module Jets::Gems
+  class Exist
+    def initialize(options)
+      @options = options
+    end
+
+    # We check all the availability before even downloading so we can provide a
+    # full list of gems they might want to research all at once instead of incrementally
+    #
+    # Examples:
+    #
+    #   check(single_gem)
+    #   check(list, of, gems)
+    def check(*gem_names)
+      gem_names = gem_names.flatten
+      exists = gem_names.inject({}) do |hash, gem_name|
+        exist = url_exists?(gem_url(gem_name))
+        hash[gem_name] = exist
+        hash.merge(hash)
+      end
+
+      exists.values.all? {|v| v } # all_exist
+    end
+
+    # Example url:
+    #   https://gems.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0-x86_64-linux.tgz
+    def url_exists?(url)
+      url = URI.parse(url)
+      req = Net::HTTP.new(url.host, url.port).tap do |http|
+        http.use_ssl = true
+      end
+      res = req.request_head(url.path)
+      res.code == "200"
+    rescue SocketError, OpenURI::HTTPError
+      false
+    end
+
+    def source_url
+      @options[:source_url] || s3_url
+    end
+
+    # gem_name: byebug-9.1.0
+    # Example url:
+    #   https://gems.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0-x86_64-linux.tgz
+    def gem_url(gem_name)
+      folder = gem_name.gsub(/-(\d+\.\d+\.\d+.*)/,'') # folder: byebug
+      "#{source_url}/gems/#{Jets::Gems.ruby_version_folder}/#{folder}/#{gem_name}-x86_64-linux.tgz"
+    end
+
+    def ruby_version_folder
+      Jets::Gems.ruby_version_folder
+    end
+  end
+end

data/vendor/jets-gems/lib/jets/gems/extract.rb

@@ -0,0 +1,7 @@
+module Jets::Gems
+  module Extract
+    autoload :Base, 'jets/gems/extract/base'
+    autoload :Gem, 'jets/gems/extract/gem'
+    autoload :Ruby, 'jets/gems/extract/ruby'
+  end
+end

data/vendor/{lambdagem/lib/lambdagem → jets-gems/lib/jets/gems}/extract/base.rb

@@ -1,13 +1,20 @@
 require "open-uri"
 
-module Lambdagem::Extract
-  autoload :ClassMethods, 'lambdagem/extract/class_methods'
+module Jets::Gems::Extract
+  class Base
+    class NotFound < RuntimeError; end
 
-  class Base < Lambdagem::Base
-    extend ClassMethods
+    attr_reader :source_url
+    def initialize(name, options={})
+      @name = name
+      @options = options
+
+      @downloads_root = options[:downloads_root] || "/tmp/jets/lambdagems"
+      @source_url = options[:source_url] || "https://gems.lambdagems.com"
+    end
 
     def clean_downloads(folder)
-      path = "#{@build_root}/downloads/#{folder}"
+      path = "#{@downloads_root}/downloads/#{folder}"
       say "Removing cache: #{path}"
       FileUtils.rm_rf(path)
     end
@@ -31,7 +38,8 @@ module Lambdagem::Extract
     end
 
     def url_exists?(url)
-      self.class.url_exists?(url)
+      exist = Jets::Gems::Exist.new(@options)
+      exist.url_exists?(url)
     end
 
     # Returns the dest path
@@ -58,5 +66,16 @@ module Lambdagem::Extract
     def project_root
       @options[:project_root] || "."
     end
+
+    @@log_level = :info # default level is :info
+    # @@log_level = :debug # uncomment to debug
+    def log_level=(val)
+      @@log_level = val
+    end
+
+    def say(message, level=:info)
+      enabled = @@log_level == :debug || level == :debug
+      puts(message) if enabled
+    end
   end
 end

data/vendor/{lambdagem/lib/lambdagem → jets-gems/lib/jets/gems}/extract/gem.rb

@@ -2,18 +2,17 @@ require "gems"
 
 # Usage:
 #
-#   Lambdagem::Extract::Gem.new("pg-0.21.0",
-#     s3: "lambdagems",
-#     build_root: cache_area, # defaults to /tmp/lambdagem
+#   Jets::Gems::Extract::Gem.new("pg-0.21.0",
+#     downloads_root: cache_area, # defaults to /tmp/lambdagem
 #     dest: cache_area, # defaults to . (project_root)
 #   ).run
 #
-module Lambdagem::Extract
+module Jets::Gems::Extract
   class Gem < Base
     VERSION_PATTERN = /-(\d+\.\d+\.\d+.*)/
 
     def run
-      say "Looking for #{full_gem_name} gem in: #{@options[:lambdagems_url]}"
+      say "Looking for #{full_gem_name} gem in: #{@options[:source_url]}"
       clean_downloads(:gems) if @options[:clean]
       tarball_path = download_gem
       remove_current_gem
@@ -25,7 +24,6 @@ module Lambdagem::Extract
     def full_gem_name
       return @full_gem_name if @full_gem_name
 
-      @name # TODO: make sure that we always have the full gem name with version number
       if @name.match(VERSION_PATTERN)
         @full_gem_name = @name
         return @full_gem_name
@@ -67,7 +65,7 @@ module Lambdagem::Extract
     end
 
     def download_path(filename)
-      "#{@build_root}/downloads/gems/#{filename}"
+      "#{@downloads_root}/downloads/gems/#{filename}"
     end
 
     # Finds any currently install gems that matched with the gem name and version
@@ -90,7 +88,11 @@ module Lambdagem::Extract
 
     # full_gem_name: byebug-9.1.0
     def gem_url
-      "#{lambdagems_url}/gems/#{jets_ruby_version}/#{gem_name}/#{full_gem_name}-x86_64-linux.tgz"
+      "#{source_url}/gems/#{ruby_version_folder}/#{gem_name}/#{full_gem_name}-x86_64-linux.tgz"
+    end
+
+    def ruby_version_folder
+      Jets::Gems.ruby_version_folder
     end
   end
 end

data/vendor/{lambdagem/lib/lambdagem → jets-gems/lib/jets/gems}/extract/ruby.rb

@@ -1,12 +1,11 @@
 # Usage:
 #
-#   Lambdagem::Extract::Ruby.new("2.5.0",
-#     s3: "lambdagems",
-#     build_root: cache_area, # defaults to /tmp/lambdagem
+#   Jets::Gems::Extract::Ruby.new("2.5.0",
+#     downloads_root: cache_area, # defaults to /tmp/lambdagem
 #     dest: cache_area, # defaults to . (project_root)
 #   ).run
 #
-module Lambdagem::Extract
+module Jets::Gems::Extract
   class Ruby < Base
     class NotFound < RuntimeError; end
 
@@ -25,7 +24,7 @@ module Lambdagem::Extract
         message = "Url: #{url} not found"
         if @options[:exit_on_error]
           say message
-          # exit # TODO: ADD BACK IN
+          exit
         else
           raise NotFound.new(message)
         end
@@ -35,7 +34,7 @@ module Lambdagem::Extract
     end
 
     def download_path(filename)
-      "#{@build_root}/downloads/rubies/#{filename}"
+      "#{@downloads_root}/downloads/rubies/#{filename}"
     end
 
     # If only the ruby version is given, then append ruby- in front. Otherwise
@@ -57,7 +56,7 @@ module Lambdagem::Extract
     end
 
     def ruby_url
-      "#{lambdagems_url}/rubies/#{full_ruby_name}"
+      "#{source_url}/rubies/#{full_ruby_name}"
     end
   end
 end

data/vendor/jets-gems/lib/jets/gems/report.rb

@@ -0,0 +1,53 @@
+require "net/http"
+
+module Jets::Gems
+  class Report
+    LAMBDAGEM_API_URL = ENV["LAMBDAGEM_API_URL"] || "https://api.lambdagems.com/api/v1"
+
+    def self.missing(gems)
+      new(gems).report
+    end
+
+    def initialize(gems)
+      @gems = gems
+    end
+
+    def report
+      version_pattern = /(.*)-(\d+\.\d+\.\d+.*)/
+      threads = []
+      @gems.each do |gem_name|
+        if md = gem_name.match(version_pattern)
+          name, version = md[1], md[2]
+          threads << Thread.new do
+            call_api("report/missing?name=#{name}&version=#{version}")
+          end
+        else
+          puts "WARN: Unable to extract the version from the gem name."
+        end
+      end
+      # Wait for request to finish because the command might finish before
+      # the Threads even send the request. So we join them just case
+      threads.each(&:join)
+    end
+
+    def api_url(path)
+      "#{LAMBDAGEM_API_URL}/#{path}"
+    end
+
+    def call_api(path)
+      # raise "HI"
+      uri = URI(api_url(path))
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      request = Net::HTTP::Get.new(uri)
+      response = http.request(request)
+      resp = {
+        status: response.code.to_i,
+        headers: response.each_header.to_h,
+        body: response.body,
+      }
+      # pp resp # Uncomment to debug
+      resp
+    end
+  end
+end

data/vendor/jets-gems/lib/jets/gems/version.rb

@@ -0,0 +1,5 @@
+module Jets
+  module Gems
+    VERSION = "0.1.0"
+  end
+end