jets 0.5.0 → 0.5.1

data/vendor/lambdagem/lib/lambdagem/command.rb

@@ -0,0 +1,38 @@
+require "thor"
+
+# Override thor's long_desc identation behavior
+# https://github.com/erikhuda/thor/issues/398
+class Thor
+  module Shell
+    class Basic
+      def print_wrapped(message, options = {})
+        message = "\n#{message}" unless message[0] == "\n"
+        stdout.puts message
+      end
+    end
+  end
+end
+
+module Lambdagem
+  class Command < Thor
+    class << self
+      def dispatch(m, args, options, config)
+        # Allow calling for help via:
+        #   lambdagem command help
+        #   lambdagem command -h
+        #   lambdagem command --help
+        #   lambdagem command -D
+        #
+        # as well thor's normal way:
+        #
+        #   lambdagem help command
+        help_flags = Thor::HELP_MAPPINGS + ["help"]
+        if args.length > 1 && !(args & help_flags).empty?
+          args -= help_flags
+          args.insert(-2, "help")
+        end
+        super
+      end
+    end
+  end
+end

data/vendor/lambdagem/lib/lambdagem/exist.rb

@@ -0,0 +1,50 @@
+class Lambdagem::Exist < Lambdagem::Base
+  def initialize(options)
+    @options = options
+  end
+
+  # We check all the availability before even downloading so we can provide a
+  # full list of gems they might want to research all at once instead of incrementally
+  #
+  # Examples:
+  #
+  #   check(single_gem)
+  #   check(list, of, gems)
+  def check(*gem_names)
+    gem_names = gem_names.flatten
+    exists = gem_names.inject({}) do |hash, gem_name|
+      exist = url_exists?(gem_url(gem_name))
+      hash[gem_name] = exist
+      hash.merge(hash)
+    end
+
+    exists.values.all? {|v| v } # all_exist
+  end
+
+  # Example url:
+  #   https://gems.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0-x86_64-linux.tgz
+  def url_exists?(url)
+    url = URI.parse(url)
+    req = Net::HTTP.new(url.host, url.port).tap do |http|
+      http.use_ssl = true
+    end
+    res = req.request_head(url.path)
+    res.code == "200"
+  rescue SocketError, OpenURI::HTTPError
+    false
+  end
+
+  def lambdagems_url
+    s3_bucket = @options[:s3] || "lambdagems"
+    s3_url = "https://s3.amazonaws.com/#{s3_bucket}"
+    @options[:lambdagems_url] || s3_url
+  end
+
+  # gem_name: byebug-9.1.0
+  # Example url:
+  #   https://gems.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0-x86_64-linux.tgz
+  def gem_url(gem_name)
+    folder = gem_name.gsub(/-(\d+\.\d+\.\d+.*)/,'') # folder: byebug
+    "#{lambdagems_url}/gems/#{jets_ruby_version}/#{folder}/#{gem_name}-x86_64-linux.tgz"
+  end
+end

data/vendor/lambdagem/lib/lambdagem/extract.rb

@@ -0,0 +1,8 @@
+module Lambdagem
+  module Extract
+    autoload :Base, "lambdagem/extract/base"
+    #autoload :ClassMethods, "lambdagem/extract/class_methods"
+    autoload :Gem, "lambdagem/extract/gem"
+    autoload :Ruby, "lambdagem/extract/ruby"
+  end
+end

data/vendor/lambdagem/lib/lambdagem/extract/base.rb

@@ -0,0 +1,62 @@
+require "open-uri"
+
+module Lambdagem::Extract
+  autoload :ClassMethods, 'lambdagem/extract/class_methods'
+
+  class Base < Lambdagem::Base
+    extend ClassMethods
+
+    def clean_downloads(folder)
+      path = "#{@build_root}/downloads/#{folder}"
+      puts "Removing cache: #{path}"
+      FileUtils.rm_rf(path)
+    end
+
+    def unpack_tarball(tarball_path)
+      dest = project_root
+      puts "Unpacking into #{dest}"
+      FileUtils.mkdir_p(dest)
+      untar(tarball_path, dest)
+    end
+
+    def untar(tarball_path, parent_folder_dest)
+      sh("tar -xzf #{tarball_path} -C #{parent_folder_dest}")
+    end
+
+    def sh(command)
+      puts "=> #{command}".colorize(:green)
+      success = system(command)
+      abort("Command Failed") unless success
+      success
+    end
+
+    def url_exists?(url)
+      self.class.url_exists?(url)
+    end
+
+    # Returns the dest path
+    def download_file(source_url, dest)
+      puts "Url #{source_url}"
+      return unless url_exists?(source_url)
+
+      if File.exist?(dest)
+        puts "File already downloaded #{dest}"
+        return dest
+      end
+
+      puts "Downloading..."
+      FileUtils.mkdir_p(File.dirname(dest)) # ensure parent folder exists
+
+      File.open(dest, 'wb') do |saved_file|
+        open(source_url, 'rb') do |read_file|
+          saved_file.write(read_file.read)
+        end
+      end
+      dest
+    end
+
+    def project_root
+      @options[:project_root] || "."
+    end
+  end
+end

data/vendor/lambdagem/lib/lambdagem/extract/class_methods.rb

@@ -0,0 +1,50 @@
+require "net/http"
+
+module Lambdagem::Extract::ClassMethods
+  # Exits early if not all the linux gems are available
+  # It better to error now then later on Lambda
+  # Provide users with instructions on how to compile gems
+  #
+  # We check all the availability before even downloading so we can provide a
+  # full list of gems they might want to research all at once instead of incrementally
+  def check_availability(gems)
+    availabilities = gems.inject({}) do |hash, gem_name|
+      exist = url_exists?(gem_url(gem_name))
+      hash[gem_name] = exist
+      hash.merge(hash)
+    end
+
+    all_available = availabilities.values.all? {|v| v }
+    unless all_available
+      puts "Your project requires some pre-compiled Linux gems that are not yet available as a pre-compiled lambda gem.  The build process will not continue because there's no point wasting your time deploying to Lambda and finding out later."
+      puts "The unavailable gems are:"
+      availabilities.each do |gem_name, available|
+        next if available
+        puts "  #{gem_name}"
+      end
+      puts <<-EOL
+How to fix this:
+
+  1. Build your jets project on an Amazon Lambda based EC2 Instance and compile your own gems with the proper shared libaries.
+  2. Configure jets to lookup your own pre-compiled gems url.
+  3. Add the the required gems to boltopslabs/lambdagems and submit a pull request.
+
+More info: http://lambdagems.com
+EOL
+      exit
+    end
+  end
+
+  # Example url:
+  #   https://gems.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0-x86_64-linux.tgz
+  def url_exists?(url)
+    url = URI.parse(url)
+    req = Net::HTTP.new(url.host, url.port).tap do |http|
+      http.use_ssl = true
+    end
+    res = req.request_head(url.path)
+    res.code == "200"
+  rescue SocketError, OpenURI::HTTPError
+    false
+  end
+end

data/vendor/lambdagem/lib/lambdagem/extract/gem.rb

@@ -0,0 +1,95 @@
+require "gems"
+
+# Usage:
+#
+#   Lambdagem::Extract::Gem.new("pg-0.21.0",
+#     s3: "lambdagems",
+#     build_root: cache_area, # defaults to /tmp/lambdagem
+#     dest: cache_area, # defaults to . (project_root)
+#   ).run
+#
+module Lambdagem::Extract
+  class Gem < Base
+    VERSION_PATTERN = /-(\d+\.\d+\.\d+.*)/
+
+    def run
+      puts "Looking for #{full_gem_name} gem in s3 bucket: #{@options[:s3]}"
+      clean_downloads(:gems) if @options[:clean]
+      tarball_path = download_gem
+      remove_current_gem
+      unpack_tarball(tarball_path)
+    end
+
+    # ensure that we always have the full gem name
+    def full_gem_name
+      return @full_gem_name if @full_gem_name
+
+      @name # TODO: make sure that we always have the full gem name with version number
+      if @name.match(VERSION_PATTERN)
+        @full_gem_name = @name
+        return @full_gem_name
+      end
+
+      # name doesnt have a version yet, so grab the latest version and add it
+      version = Gems.versions(@name).first
+      @full_gem_name = "#{@name}-#{version["number"]}"
+    end
+
+    def gem_name
+      full_gem_name.gsub(VERSION_PATTERN,'') # folder: byebug
+    end
+
+    # Downloads and extracts the linux gem into the proper directory.
+    # Extracts to: . (current directory)
+    #
+    # It produces a `bundled` folder.
+    # The folder contains the re-produced directory structure. Example with
+    # the gem: byebug-9.1.0
+    #
+    #   bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
+    #
+    def download_gem
+      # download - also move to /tmp/jets/demo/compiled_gems folder
+      url = gem_url
+      tarball_dest = download_file(url, download_path(File.basename(url)))
+      unless tarball_dest
+        message = "Url: #{url} not found"
+        if @options[:exit_on_error]
+          puts message
+          exit
+        else
+          raise NotFound.new(message)
+        end
+      end
+      puts "Tarball downloaded to: #{tarball_dest}"
+      tarball_dest
+    end
+
+    def download_path(filename)
+      "#{@build_root}/downloads/gems/#{filename}"
+    end
+
+    # Finds any currently install gems that matched with the gem name and version
+    # and remove them first.
+    # We clean up the current install gems first in case it was previously installed
+    # and has different *.so files that can be accidentally required.  This
+    # happened with the pg gem.
+    def remove_current_gem
+      puts "Removing current #{full_gem_name} gem installation:"
+      gem_dirs = Dir.glob("#{project_root}/**/*").select do |path|
+                  File.directory?(path) &&
+                  path =~ %r{bundled/gems} &&
+                  File.basename(path) == full_gem_name
+                end
+      gem_dirs.each do |dir|
+        puts "  rm -rf #{dir}"
+        FileUtils.rm_rf(dir)
+      end
+    end
+
+    # full_gem_name: byebug-9.1.0
+    def gem_url
+      "#{lambdagems_url}/gems/#{jets_ruby_version}/#{gem_name}/#{full_gem_name}-x86_64-linux.tgz"
+    end
+  end
+end

data/vendor/lambdagem/lib/lambdagem/extract/ruby.rb

@@ -0,0 +1,62 @@
+# Usage:
+#
+#   Lambdagem::Extract::Ruby.new("2.5.0",
+#     s3: "lambdagems",
+#     build_root: cache_area, # defaults to /tmp/lambdagem
+#     dest: cache_area, # defaults to . (project_root)
+#   ).run
+#
+module Lambdagem::Extract
+  class Ruby < Base
+    class NotFound < RuntimeError; end
+
+    def run
+      puts "Looking for #{full_ruby_name} in S3 bucket: #{@options[:s3]}"
+      clean_downloads(:rubies) if @options[:clean]
+      tarball_path = download_ruby
+      unpack_tarball(tarball_path)
+    end
+
+    def download_ruby
+      url = ruby_url
+      tarball_dest = download_file(url, download_path(File.basename(url)))
+      unless tarball_dest
+        message = "Url: #{url} not found"
+        if @options[:exit_on_error]
+          puts message
+          # exit # TODO: ADD BACK IN
+        else
+          raise NotFound.new(message)
+        end
+      end
+      puts "Tarball downloaded to: #{tarball_dest}"
+      tarball_dest
+    end
+
+    def download_path(filename)
+      "#{@build_root}/downloads/rubies/#{filename}"
+    end
+
+    # If only the ruby version is given, then append ruby- in front. Otherwise
+    # leave alone.
+    #
+    # Example:
+    #
+    #    2.5.0           -> ruby-2.5.0-linux-x86_64.tgz
+    #    ruby-2.5.0      -> ruby-2.5.0-linux-x86_64.tgz
+    #    test-ruby-2.5.0 -> test-ruby-2.5.0-linux-x86_64.tgz
+    def full_ruby_name
+      md = @name.match(/^(\d+\.\d+\.\d+)$/)
+      if md
+        ruby_version = md[1]
+        "ruby-#{ruby_version}-linux-x86_64.tgz"
+      else
+        "#{@name}-linux-x86_64.tgz"
+      end
+    end
+
+    def ruby_url
+      "#{lambdagems_url}/rubies/#{full_ruby_name}"
+    end
+  end
+end

data/vendor/lambdagem/lib/lambdagem/help.rb

@@ -0,0 +1,9 @@
+module Lambdagem::Help
+  class << self
+    def text(namespaced_command)
+      path = namespaced_command.to_s.gsub(':','/')
+      path = File.expand_path("../help/#{path}.md", __FILE__)
+      IO.read(path) if File.exist?(path)
+    end
+  end
+end

data/vendor/lambdagem/lib/lambdagem/help/all.md

@@ -0,0 +1,11 @@
+Builds, packages and uploads the gem to s3 all in one step. This is useful for simply compiled gems that do not required any shared .so files during runtime.
+
+Examples:
+
+  $ lambdagem all byebug --s3 lambdagems
+
+So the above command is the same as running:
+
+  $ lambdagem build byebug
+  $ lambdagem package byebug
+  $ lambdagem upload byebug --s3 lambdagems

data/vendor/lambdagem/lib/lambdagem/help/build.md

@@ -0,0 +1,14 @@
+Builds a gem. Examples:
+
+  lambdagem build byebug
+  lambdagem build byebug-0.9.1
+
+The lambdagem builds the gem in `/tmp/lambdagem`.  This is where you'll find the Gemfile that is produces and where you can test the gem.
+
+A slim down version of `/tmp/lambdagem/bundled/gems` the is initial created in `/tmp/lambdagem/artifacts/byebug-0.9.1/bundled/gems`.  This slim down version contains only the byebug gem files.
+
+When you are ready, the gem can be packaged to tarball with the `bundle package` command.  The tarball is stored at:
+
+```
+/tmp/lambdagem/artifacts/byebug-9.1.0-x86_64-linux.tar.gz
+```

data/vendor/lambdagem/lib/lambdagem/help/clean.md

@@ -0,0 +1,5 @@
+Cleans out tmp/downloads cache.
+
+cd your_app_root
+
+  $ lambdagem clean

data/vendor/lambdagem/lib/lambdagem/help/extract_gem.md

@@ -0,0 +1,5 @@
+Downloads pre-compiled gem from s3 and extracts it with the same structure as Jets. Examples:
+
+cd your_app_root
+
+  $ lambdagem extract_gem byebug-0.9.1

data/vendor/lambdagem/lib/lambdagem/help/extract_ruby.md

@@ -0,0 +1,5 @@
+Downloads pre-compiled ruby from s3 and extracts it with the same structure as Jets. Examples:
+
+cd your_app_root
+
+  $ lambdagem extract_ruby 2.5.0

data/vendor/lambdagem/lib/lambdagem/help/package.md

@@ -0,0 +1,20 @@
+Pakcages gem into a tarball. The tarball created is named after the gem name with version, but it contains the "bundled" folder with the underlying full structure.
+
+It is meant to be extracted via:
+
+  mkdir -p /var/task
+  tar zxf byebug-0.9.1.tgz -C /var/task
+  # creates /var/task/bundled
+
+Same as:
+
+  mkdir -p /var/task
+  cd /var/task && tar zxf byebug-0.9.1.tgz
+  # creates /var/task/bundled
+
+Examples:
+
+  $ lambdagem package byebug
+
+  $ lambdagem package byebug-0.9.1
+