jets 1.2.1 → 1.3.0

data/lib/jets/{server.rb → rack_server.rb}

@@ -1,10 +1,10 @@
 module Jets
-  class Server
+  class RackServer
     def self.start(options={})
       new(options).start
     end
 
-    def initialize(options)
+    def initialize(options={})
       @options = options
     end
 
@@ -44,6 +44,35 @@ module Jets
       end
     end
 
+    # blocks until rack server is up
+    def wait_for_socket
+      return unless Jets.rack?
+
+      retries = 0
+      max_retries = 30 # 15 seconds at a delay of 0.5s
+      delay = 0.5
+      if ENV['C9_USER'] # overrides for local testing
+        max_retries = 3
+        delay = 3
+      end
+      begin
+        server = TCPSocket.new('localhost', 9292)
+        server.close
+      rescue Errno::ECONNREFUSED
+        puts "Unable to connect to localhost:9292. Delay for #{delay} and will try to connect again."  if ENV['JETS_DEBUG']
+        sleep(delay)
+        retries += 1
+        if retries < max_retries
+          retry
+        else
+          puts "Giving up on trying to connect to localhost:9292"
+          return false
+        end
+      end
+      puts "Connected to localhost:9292 successfully"
+      true
+    end
+
     def rack_project
       "#{Jets.root}rack"
     end

data/lib/jets/resource.rb

@@ -7,8 +7,8 @@ class Jets::Resource
   autoload :ChildStack, 'jets/resource/child_stack'
   autoload :Config, 'jets/resource/config'
   autoload :Events, 'jets/resource/events'
-  autoload :Function, 'jets/resource/function'
   autoload :Iam, 'jets/resource/iam'
+  autoload :Lambda, 'jets/resource/lambda'
   autoload :Permission, 'jets/resource/permission'
   autoload :Replacer, 'jets/resource/replacer'
   autoload :Route53, 'jets/resource/route53'

data/lib/jets/resource/api_gateway/base_path/function.rb

@@ -1,6 +1,6 @@
 module Jets::Resource::ApiGateway::BasePath
   class Function < Jets::Resource::Base
-    include Jets::Resource::Function::Environment
+    include Jets::Resource::Lambda::Function::Environment
 
     def definition
       {

data/lib/jets/resource/child_stack/app_class.rb

@@ -50,15 +50,21 @@ module Jets::Resource::ChildStack
     end
 
     def parameters
-      common = {
-        IamRole: "!GetAtt IamRole.Arn",
-        S3Bucket: "!Ref S3Bucket",
-      }
+      common = self.class.common_parameters
       common.merge!(controller_params) if controller?
       common.merge!(depends_on_params) if depends_on
       common
     end
 
+    def self.common_parameters
+      parameters = {
+        IamRole: "!GetAtt IamRole.Arn",
+        S3Bucket: "!Ref S3Bucket",
+      }
+      parameters[:GemLayer] = "!Ref GemLayer" unless Jets.poly_only?
+      parameters
+    end
+
     def controller_params
       return {} if Jets::Router.routes.empty?
 

data/lib/jets/resource/lambda.rb

@@ -0,0 +1,5 @@
+module Jets::Resource::Lambda
+  autoload :Function, 'jets/resource/lambda/function'
+  autoload :GemLayer, 'jets/resource/lambda/gem_layer'
+  autoload :LayerVersion, 'jets/resource/lambda/layer_version'
+end

data/lib/jets/resource/{function.rb → lambda/function.rb}

@@ -1,6 +1,6 @@
-class Jets::Resource
+module Jets::Resource::Lambda
   class Function < Jets::Resource::Base
-    autoload :Environment, 'jets/resource/function/environment'
+    autoload :Environment, 'jets/resource/lambda/function/environment'
     include Environment
 
     def initialize(task)
@@ -120,15 +120,17 @@ class Jets::Resource
       properties
     end
 
-    # Properties managed by Jets with more finality.
+    # Properties managed by Jets with merged with finality.
     def finalize_properties!(props)
       handler = full_handler(props)
       runtime = get_runtime(props)
-      props.merge!(
+      managed = {
         function_name: function_name,
         handler: handler,
         runtime: runtime,
-      )
+      }
+      managed[:layers] = ["!Ref GemLayer"] if runtime =~ /^ruby/
+      props.merge!(managed)
     end
 
     def get_runtime(props)
@@ -139,7 +141,7 @@ class Jets::Resource
       map = {
         node: "nodejs8.10",
         python: "python3.6",
-        ruby: "nodejs8.10", # node shim for ruby support
+        ruby: "ruby2.5",
       }
       map[@task.lang]
     end

data/lib/jets/resource/{function → lambda/function}/environment.rb

@@ -1,4 +1,4 @@
-class Jets::Resource::Function
+class Jets::Resource::Lambda::Function
   module Environment
     def env_properties
       env_vars = Jets::Dotenv.load!(true)

data/lib/jets/resource/lambda/gem_layer.rb

@@ -0,0 +1,17 @@
+module Jets::Resource::Lambda
+  class GemLayer < LayerVersion
+    def description
+      "Jets Ruby Gems"
+    end
+
+    def layer_name
+      # Do not include the Jets.extra_env to group the layers in same app together
+      "#{Jets.config.short_env}-#{Jets.config.project_name}-gems"
+    end
+
+    def code_s3_key
+      checksum = Jets::Builders::Md5.checksums["stage/opt"]
+      "jets/code/opt-#{checksum}.zip" # s3_key
+    end
+  end
+end

data/lib/jets/resource/lambda/layer_version.rb

@@ -0,0 +1,46 @@
+# Type: "AWS::Lambda::LayerVersion"
+# Properties:
+#   CompatibleRuntimes:
+#     - String
+#     - ...
+#   Content:
+#     Content
+#   Description: String
+#   LayerName: String
+#   LicenseInfo: String
+module Jets::Resource::Lambda
+  class LayerVersion < Jets::Resource::Base
+    def definition
+      {
+        layer_version_logical_id => {
+          type: "AWS::Lambda::LayerVersion",
+          properties: {
+            # compatible_runtimes: ["ruby2.5"],
+            content: {
+              s3_bucket: s3_bucket,
+              s3_key: code_s3_key,
+              # s3_object_version: string,
+            },
+            description: description,
+            layer_name: layer_name,
+            license_info: "MIT",
+          }
+        }
+      }
+    end
+
+    def s3_bucket
+      "!Ref S3Bucket"
+    end
+
+    def layer_version_logical_id
+      self.class.name.split('::').last
+    end
+
+    def outputs
+      {
+        logical_id => "!Ref #{logical_id}",
+      }
+    end
+  end
+end

data/lib/jets/tmp_loader.rb

@@ -0,0 +1,52 @@
+module Jets
+  class TmpLoader
+    include AwsServices
+
+    def self.load!
+      new.load
+    end
+
+    def initialize(yaml_path=nil)
+      yaml_path ||= "#{Jets.root}handlers/data.yml"
+      @data = YAML.load_file(yaml_path)
+      @s3_bucket = @data['s3_bucket']
+      @rack_zip = @data['rack_zip']
+    end
+
+    def load
+      rack
+    end
+
+    def rack
+      return unless @rack_zip
+      download_and_extract(@rack_zip, '/tmp/rack')
+    end
+
+    def download_and_extract(zip_file, folder_dest)
+      s3_key = "jets/code/#{zip_file}" # jets/code/rack-checksum.zip
+      download_path = "/tmp/#{zip_file}" # /tmp/rack-checksum.zip
+
+      download(s3_key, download_path)
+      unzip(download_path, folder_dest)
+    end
+
+    def download(key, dest)
+      # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#get_object-instance_method
+      # stream object directly to disk
+      s3.get_object(response_target: dest,
+                    bucket: @s3_bucket,
+                    key: key)
+    end
+
+    def unzip(zipfile, folder_dest)
+      sh "unzip -qo #{zipfile} -d #{folder_dest}"
+    end
+
+    def sh(command)
+      puts "=> #{command}"
+      success = system(command)
+      raise "Command #{command} failed" unless success
+      success
+    end
+  end
+end

data/lib/jets/version.rb

@@ -1,3 +1,3 @@
 module Jets
-  VERSION = "1.2.1"
+  VERSION = "1.3.0"
 end

data/{README → readme}/testing.md

@@ -45,6 +45,11 @@ Run the remote integration script:
     eval "export $(cat demo/.env.development.remote)" # for DATABASE_URL for mega mode
     BASE_URL=xxx spec/integration/remote.sh
 
+Manually seed Rails data. Will add this to the remote.sh in the future.
+
+    cd demo/rack
+    rails runner 'Book.create(id: 1) unless Book.find_by(id: 1)'
+
 Example:
 
     eval "export $(cat demo/.env.development.remote)" # for DATABASE_URL for mega mode

data/vendor/jets-gems/lib/jets/gems.rb

@@ -8,7 +8,7 @@ module Jets
     autoload :Extract, 'jets/gems/extract'
     autoload :Report, 'jets/gems/report'
 
-    def ruby_version_folder
+    def ruby_folder
       major, minor, patch = RUBY_VERSION.split('.')
       [major, minor, '0'].join('.') # 2.5.1 => 2.5.0
     end

data/vendor/jets-gems/lib/jets/gems/check.rb

@@ -126,20 +126,20 @@ EOL
     #
     # Example paths:
     # Macosx:
-    #   bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/nokogiri-1.8.1
-    #   bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
+    #   vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/nokogiri-1.8.1
+    #   vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
     # Official AWS Lambda Linux AMI:
-    #   bundled/gems/ruby/2.5.0/extensions/x86_64-linux/2.5.0-static/nokogiri-1.8.1
+    #   vendor/bundle/ruby/2.5.0/extensions/x86_64-linux/2.5.0-static/nokogiri-1.8.1
     # Circleci Ubuntu based Linux:
-    #   bundled/gems/ruby/2.5.0/extensions/x86_64-linux/2.5.0/pg-0.21.0
+    #   vendor/bundle/ruby/2.5.0/extensions/x86_64-linux/2.5.0/pg-0.21.0
     def compiled_gem_paths
-      Dir.glob("#{Jets.build_root}/cache/bundled/gems/ruby/*/extensions/**/**/*.{so,bundle}")
+      Dir.glob("#{Jets.build_root}/cache/vendor/bundle/ruby/*/extensions/**/**/*.{so,bundle}")
     end
 
-    # Input: bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
+    # Input: vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
     # Output: byebug-9.1.0
     def gem_name_from_path(path)
-      regexp = /gems\/ruby\/\d+\.\d+\.\d+\/extensions\/.*?\/.*?\/(.*?)\//
+      regexp = /vendor\/bundle\/ruby\/\d+\.\d+\.\d+\/extensions\/.*?\/.*?\/(.*?)\//
       gem_name = path.match(regexp)[1]
     end
 

data/vendor/jets-gems/lib/jets/gems/exist.rb

@@ -25,7 +25,7 @@ module Jets::Gems
     end
 
     # Example url:
-    #   https://gems.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0-x86_64-linux.tgz
+    #   https://gems2.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0.zip
     def url_exists?(url)
       url = URI.parse(url)
       req = Net::HTTP.new(url.host, url.port).tap do |http|
@@ -43,14 +43,14 @@ module Jets::Gems
 
     # gem_name: byebug-9.1.0
     # Example url:
-    #   https://gems.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0-x86_64-linux.tgz
+    #   https://gems2.lambdagems.com/gems/2.5.0/byebug/byebug-9.1.0.zip
     def gem_url(gem_name)
       folder = gem_name.gsub(/-(\d+\.\d+\.\d+.*)/,'') # folder: byebug
-      "#{source_url}/gems/#{Jets::Gems.ruby_version_folder}/#{folder}/#{gem_name}-x86_64-linux.tgz"
+      "#{source_url}/gems/#{Jets::Gems.ruby_folder}/#{folder}/#{gem_name}.zip"
     end
 
-    def ruby_version_folder
-      Jets::Gems.ruby_version_folder
+    def ruby_folder
+      Jets::Gems.ruby_folder
     end
   end
 end

data/vendor/jets-gems/lib/jets/gems/extract/base.rb

@@ -9,8 +9,8 @@ module Jets::Gems::Extract
       @name = name
       @options = options
 
-      @downloads_root = options[:downloads_root] || "/tmp/jets/lambdagems"
-      @source_url = options[:source_url] || "https://gems.lambdagems.com"
+      @downloads_root = options[:downloads_root] || "/tmp/jets/#{Jets.config.project_name}/lambdagems"
+      @source_url = options[:source_url] || Jets.default_gems_source
     end
 
     def clean_downloads(folder)
@@ -19,15 +19,8 @@ module Jets::Gems::Extract
       FileUtils.rm_rf(path)
     end
 
-    def unpack_tarball(tarball_path)
-      dest = project_root
-      say "Unpacking into #{dest}"
-      FileUtils.mkdir_p(dest)
-      untar(tarball_path, dest)
-    end
-
-    def untar(tarball_path, parent_folder_dest)
-      sh("tar -xzf #{tarball_path} -C #{parent_folder_dest}")
+    def unzip(zipfile_path, parent_folder_dest)
+      sh("cd #{parent_folder_dest} && unzip -qo #{zipfile_path}")
     end
 
     def sh(command)

data/vendor/jets-gems/lib/jets/gems/extract/gem.rb

@@ -14,12 +14,19 @@ module Jets::Gems::Extract
     def run
       say "Looking for #{full_gem_name} gem in: #{@options[:source_url]}"
       clean_downloads(:gems) if @options[:clean]
-      tarball_path = download_gem
+      zipfile_path = download_gem
       remove_current_gem
-      unpack_tarball(tarball_path)
+      unzip_file(zipfile_path)
       say("Gem #{full_gem_name} unpacked at #{project_root}", :debug)
     end
 
+    def unzip_file(zipfile_path)
+      dest = "#{Jets.build_root}/stage/code/opt"
+      say "Unpacking into #{dest}"
+      FileUtils.mkdir_p(dest)
+      unzip(zipfile_path, dest)
+    end
+
     # ensure that we always have the full gem name
     def full_gem_name
       return @full_gem_name if @full_gem_name
@@ -45,7 +52,7 @@ module Jets::Gems::Extract
     # The folder contains the re-produced directory structure. Example with
     # the gem: byebug-9.1.0
     #
-    #   bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
+    #   vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
     #
     def download_gem
       # download - also move to /tmp/jets/demo/compiled_gems folder
@@ -60,7 +67,7 @@ module Jets::Gems::Extract
           raise NotFound.new(message)
         end
       end
-      say "Tarball downloaded to: #{tarball_dest}"
+      say "Downloaded to: #{tarball_dest}"
       tarball_dest
     end
 
@@ -77,7 +84,7 @@ module Jets::Gems::Extract
       say "Removing current #{full_gem_name} gem installation:"
       gem_dirs = Dir.glob("#{project_root}/**/*").select do |path|
                   File.directory?(path) &&
-                  path =~ %r{bundled/gems} &&
+                  path =~ %r{vendor/bundle} &&
                   File.basename(path) == full_gem_name
                 end
       gem_dirs.each do |dir|
@@ -88,11 +95,11 @@ module Jets::Gems::Extract
 
     # full_gem_name: byebug-9.1.0
     def gem_url
-      "#{source_url}/gems/#{ruby_version_folder}/#{gem_name}/#{full_gem_name}-x86_64-linux.tgz"
+      "#{source_url}/gems/#{ruby_folder}/#{gem_name}/#{full_gem_name}.zip"
     end
 
-    def ruby_version_folder
-      Jets::Gems.ruby_version_folder
+    def ruby_folder
+      Jets::Gems.ruby_folder
     end
   end
 end

data/vendor/jets-gems/lib/jets/gems/extract/ruby.rb

@@ -1,6 +1,6 @@
 # Usage:
 #
-#   Jets::Gems::Extract::Ruby.new("2.5.0",
+#   Jets::Gems::Extract::Ruby.new("2.5.3",
 #     downloads_root: cache_area, # defaults to /tmp/lambdagem
 #     dest: cache_area, # defaults to . (project_root)
 #   ).run
@@ -12,13 +12,22 @@ module Jets::Gems::Extract
     def run
       say "Looking for #{full_ruby_name}"
       clean_downloads(:rubies) if @options[:clean]
-      tarball_path = download_ruby
-      unpack_tarball(tarball_path)
-      say("Ruby #{full_ruby_name} unpacked at #{project_root}", :debug)
+      zip_path = download_ruby
+      unzip(zip_path)
+    end
+
+    def unzip(path)
+      dest = "#{Jets.build_root}/stage/code/opt"
+      say "Unpacking into #{dest}"
+      FileUtils.mkdir_p(dest)
+      # cd-ing dest unzips the files into that folder
+      sh("cd #{dest} && unzip -qo #{path}")
+      say("Ruby #{full_ruby_name} unziped at #{dest}", :debug)
     end
 
     def download_ruby
       url = ruby_url
+      puts "download ruby url #{url}"
       tarball_dest = download_file(url, download_path(File.basename(url)))
       unless tarball_dest
         message = "Url: #{url} not found"
@@ -29,7 +38,7 @@ module Jets::Gems::Extract
           raise NotFound.new(message)
         end
       end
-      say "Tarball downloaded to: #{tarball_dest}"
+      say "Downloaded to: #{tarball_dest}"
       tarball_dest
     end
 
@@ -42,16 +51,16 @@ module Jets::Gems::Extract
     #
     # Example:
     #
-    #    2.5.0           -> ruby-2.5.0-linux-x86_64.tgz
-    #    ruby-2.5.0      -> ruby-2.5.0-linux-x86_64.tgz
-    #    test-ruby-2.5.0 -> test-ruby-2.5.0-linux-x86_64.tgz
+    #    2.5.3           -> ruby-2.5.3.zip
+    #    ruby-2.5.3      -> ruby-2.5.3.zip
+    #    test-ruby-2.5.3 -> test-ruby-2.5.3.zip
     def full_ruby_name
       md = @name.match(/^(\d+\.\d+\.\d+)$/)
       if md
         ruby_version = md[1]
-        "ruby-#{ruby_version}-linux-x86_64.tgz"
+        "ruby-#{ruby_version}.zip"
       else
-        "#{@name}-linux-x86_64.tgz"
+        "#{@name}.zip"
       end
     end