jets 1.2.1 → 1.3.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jets
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-07 00:00:00.000000000 Z
+date: 2018-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionpack
@@ -437,8 +437,6 @@ files:
 - LICENSE.txt
 - Procfile
 - README.md
-- README/prerelease.md
-- README/testing.md
 - Rakefile
 - bin/release
 - buildspec.yml
@@ -452,10 +450,13 @@ files:
 - lib/jets/booter.rb
 - lib/jets/builders.rb
 - lib/jets/builders/code_builder.rb
+- lib/jets/builders/code_size.rb
 - lib/jets/builders/gem_replacer.rb
 - lib/jets/builders/handler_generator.rb
+- lib/jets/builders/lambda_layer.rb
 - lib/jets/builders/md5.rb
 - lib/jets/builders/md5_zip.rb
+- lib/jets/builders/purger.rb
 - lib/jets/builders/rack_packager.rb
 - lib/jets/builders/rackup_wrappers/rackup
 - lib/jets/builders/rackup_wrappers/rackup.rb
@@ -466,8 +467,7 @@ files:
 - lib/jets/builders/shim_vars/app.rb
 - lib/jets/builders/shim_vars/base.rb
 - lib/jets/builders/shim_vars/shared.rb
-- lib/jets/builders/templates/handler.js
-- lib/jets/builders/templates/shim.js
+- lib/jets/builders/templates/handler.rb
 - lib/jets/builders/tidy.rb
 - lib/jets/builders/util.rb
 - lib/jets/camelizer.rb
@@ -665,6 +665,13 @@ files:
 - lib/jets/middleware/layer.rb
 - lib/jets/middleware/stack.rb
 - lib/jets/naming.rb
+- lib/jets/overrides/lambda.rb
+- lib/jets/overrides/lambda/marshaller.rb
+- lib/jets/overrides/rails.rb
+- lib/jets/overrides/rails/asset_tag_helper.rb
+- lib/jets/overrides/rails/common_methods.rb
+- lib/jets/overrides/rails/rendering_helper.rb
+- lib/jets/overrides/rails/url_helper.rb
 - lib/jets/poly_fun.rb
 - lib/jets/poly_fun/base_executor.rb
 - lib/jets/poly_fun/lambda_executor.rb
@@ -676,11 +683,7 @@ files:
 - lib/jets/processors.rb
 - lib/jets/processors/deducer.rb
 - lib/jets/processors/main_processor.rb
-- lib/jets/rails_overrides.rb
-- lib/jets/rails_overrides/asset_tag_helper.rb
-- lib/jets/rails_overrides/common_methods.rb
-- lib/jets/rails_overrides/rendering_helper.rb
-- lib/jets/rails_overrides/url_helper.rb
+- lib/jets/rack_server.rb
 - lib/jets/rdoc.rb
 - lib/jets/resource.rb
 - lib/jets/resource/api_gateway.rb
@@ -710,8 +713,6 @@ files:
 - lib/jets/resource/config/managed_rule.rb
 - lib/jets/resource/events.rb
 - lib/jets/resource/events/rule.rb
-- lib/jets/resource/function.rb
-- lib/jets/resource/function/environment.rb
 - lib/jets/resource/iam.rb
 - lib/jets/resource/iam/application_role.rb
 - lib/jets/resource/iam/base_role_definition.rb
@@ -719,6 +720,11 @@ files:
 - lib/jets/resource/iam/function_role.rb
 - lib/jets/resource/iam/managed_policy.rb
 - lib/jets/resource/iam/policy_document.rb
+- lib/jets/resource/lambda.rb
+- lib/jets/resource/lambda/function.rb
+- lib/jets/resource/lambda/function/environment.rb
+- lib/jets/resource/lambda/gem_layer.rb
+- lib/jets/resource/lambda/layer_version.rb
 - lib/jets/resource/permission.rb
 - lib/jets/resource/replacer.rb
 - lib/jets/resource/route53.rb
@@ -728,11 +734,9 @@ files:
 - lib/jets/resource/standardizer.rb
 - lib/jets/route.rb
 - lib/jets/router.rb
-- lib/jets/ruby_server.rb
 - lib/jets/rule.rb
 - lib/jets/rule/base.rb
 - lib/jets/rule/dsl.rb
-- lib/jets/server.rb
 - lib/jets/stack.rb
 - lib/jets/stack/builder.rb
 - lib/jets/stack/definition.rb
@@ -751,9 +755,12 @@ files:
 - lib/jets/stack/parameter/dsl.rb
 - lib/jets/stack/resource.rb
 - lib/jets/stack/resource/dsl.rb
+- lib/jets/tmp_loader.rb
 - lib/jets/turbine.rb
 - lib/jets/util.rb
 - lib/jets/version.rb
+- readme/prerelease.md
+- readme/testing.md
 - vendor/dynomite/CHANGELOG.md
 - vendor/dynomite/Gemfile
 - vendor/dynomite/Gemfile.lock

data/lib/jets/builders/templates/handler.js

@@ -1,9 +0,0 @@
-'use strict';
-
-const shim = require("handlers/shim.js");
-
-shim.once(); // runs in lambda execution context
-
-<% @vars.functions.each do |function_name| -%>
-exports.<%= function_name %> = shim.handler("<%= @vars.handler_for(function_name) %>");
-<% end -%>

data/lib/jets/builders/templates/shim.js

@@ -1,275 +0,0 @@
-const TMP_LOG_PATH = '/tmp/shim-subprocess.log';
-const JETS_DEBUG = process.env.JETS_DEBUG; // set JETS_DEBUG=1 to see more debugging info
-const JETS_OUTPUT = '/tmp/jets-output.log';
-const S3_BUCKET = '<%= @vars.s3_bucket %>';
-<% if Jets.rack? -%>
-const RACK_ZIP = '<%= @vars.rack_zip %>';       // jets/code/rack-checksum.zip
-<% end -%>
-<% if Jets.lazy_load? -%>
-const BUNDLED_ZIP = '<%= @vars.bundled_zip %>'; // /tmp/bundled-checksum.zip
-<% end -%>
-
-
-const spawn = require('child_process').spawn;
-const fs = require('fs');
-const readline = require('readline');
-const subprocess_out = fs.openSync(TMP_LOG_PATH, 'a');
-const subprocess_err = fs.openSync(TMP_LOG_PATH, 'a');
-const { exec } = require('child_process');
-const AWS = require('aws-sdk/global');
-const S3 = require('aws-sdk/clients/s3'); // import individual service
-const net = require('net');
-
-////////////////////////////////////////////////////////////////////////////////
-// util methods
-const sh = async (command, args) => {
-  var promise = new Promise(function(resolve, reject) {
-    log(`=> ${command}`);
-    exec(command, (err, stdout, stderr) => {
-      log(`stdout: ${stdout}`);
-      log(`stderr: ${stderr}`);
-      if (err) {
-        resolve({success: false, command: command});
-      } else {
-        resolve({success: true, command: command});
-      }
-    });
-  });
-  return promise;
-};
-
-const download = async (key, dest) => {
-  var promise = new Promise(function(resolve, reject) {
-    var s3 = new AWS.S3();
-    var params = {Bucket: S3_BUCKET, Key: key};
-    var file = require('fs').createWriteStream(dest);
-    var stream = s3.getObject(params).createReadStream();
-    var pipe = stream.pipe(file);
-    file.on('finish', function () {
-      resolve({success: true, pipe: pipe});
-    });
-  });
-  return promise;
-};
-
-const downloadAndExtract = async (zip_file, folder_dest) => {
-  var s3_key = `jets/code/${zip_file}`;   // jets/code/bundled-checksum.zip
-  var download_path = `/tmp/${zip_file}`; // /tmp/bundled-checksum.zip
-
-  await download(s3_key, download_path);
-  await sh(`unzip -qo ${download_path} -d ${folder_dest}`);
-  // await sh(`ls ${folder_dest}*`);
-};
-
-// Clean out files to prevent confusing double output when debugging.
-// Also clean out the file whenever done printing out the output.
-function truncate(path) {
-  if (fs.existsSync(path)) {
-    fs.truncateSync(path);
-  }
-}
-
-// Produces an Error object that displays in the AWS Lambda test console nicely.
-// The backtrace are the ruby lines, not the nodejs shim error lines.
-// The json payload in the Lambda console looks something like this:
-//
-//   {
-//     "errorMessage": "RubyError: RuntimeError: error in submethod",
-//     "errorType": "RubyError",
-//     "stackTrace": [
-//       [
-//         "line1",
-//         "line2",
-//         "line3"
-//       ]
-//     ]
-//   }
-//
-function rubyError(resp) {
-  var name = resp["errorType"];
-  var message = resp["errorMessage"];
-  var stack = resp["stackTrace"];
-  stack.unshift(message); // JS error includes the error message at the top of the stacktrac also
-  stack = stack.join("\n");
-
-  var error = new Error(message);
-  error.name = name;
-  error.stack = stack;
-  return error;
-}
-
-// On AWS Lambda, we can log to either stdout or stderr and we're okay.
-// But locally when we're testing the shim, the log output can mess up piping
-// to jq. So not logging to stdout because when testing this shim locally the
-// stdout output messes up a pipe to jq.
-function log(text, level="info") {
-  if (level == "info" && JETS_DEBUG) {
-    // only log if JETS_DEBUG is set
-    console.error(text);
-  } else if (level == "debug") {
-    // always log if "debug" passed into method
-    console.error(text);
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-// Logic that runs once in the lambda execution context
-
-const downloadTmp = async () => {
-  var start = Date.now();
-
-<% if Jets.lazy_load? -%>
-  var bundled = downloadAndExtract(BUNDLED_ZIP, '/tmp/bundled');
-<% end -%>
-<% if Jets.rack? -%>
-  var rack = downloadAndExtract(RACK_ZIP, '/tmp/rack');
-<% end -%>
-<% if Jets.lazy_load? -%>
-  // Interestingly, doesnt seem to speed up download with parallization below.
-  // Think this is because node does not take advantage of multiple cores?
-  // Leaving this in place because speed seems the same as doing the await
-  // in serial and hope it is fast in the future.
-  await bundled;
-<% end -%>
-<% if Jets.rack? -%>
-  await rack;
-<% end -%>
-
-  var took = Date.now() - start;
-  log(`Download /tmp time: ${took}`);
-};
-
-const rubyServer = () => {
-  // start tcp server and detach
-  const subprocess = spawn('bin/ruby_server', {
-    detached: true,
-    stdio: [ 'ignore', subprocess_out, subprocess_err ]
-  });
-  subprocess.on('error', function(err) {
-    log('bin/ruby_server error', err);
-  });
-
-  subprocess.on('close', function(exit_code) {
-    log("Subprocess was shut down or deattached!");
-  });
-
-  // prevent the parent from waiting for a subprocess to exit
-  // https://nodejs.org/api/child_process.html
-  subprocess.unref();
-};
-
-// once runs in lambda execution context
-const once = async () => {
-  // Uncomment fake run once locally. No need to do this on real lambda environment.
-  // var filename = '/tmp/once.txt';
-  // if (fs.existsSync(filename)) {
-  //   log("fake run only once. /tmp/once.txt exists. remove to run once again.");
-  //   return;
-  // }
-  // fs.closeSync(fs.openSync(filename, 'w'));
-
-<% if Jets.rack? || Jets.lazy_load? -%>
-  await downloadTmp();
-<% end -%>
-  // The server related methods are fire and forget.
-  // Not using await here because had trouble resolving the promise on client.connect
-  rubyServer(); // no await
-};
-
-// currying function to make handler code prettier
-function handler(full_handler_name) {
-  return function(event, context, callback) {
-    request(event, context, full_handler_name, callback);
-  };
-}
-
-////////////////////////////////////////////////////////////////////////////////
-// main logic for handler
-function request(event, context, handler, callback) {
-  truncate(JETS_OUTPUT);
-  truncate(TMP_LOG_PATH);
-
-  log("event:");
-  log(event);
-  log("context:");
-  log(context);
-  var client = new net.Socket();
-  client.connect(8080, '127.0.0.01', function() {
-    log('Connected to socket');
-    client.write(JSON.stringify(event));
-    client.write("\r\n"); // important: \r\n is how server knows input is done
-    client.write(JSON.stringify(context));
-    client.write("\r\n"); // important: \r\n is how server knows input is done
-    client.write(handler);
-    client.write("\r\n"); // important: \r\n is how server knows input is done
-  });
-
-  // string concatation in javascript is faster than array concatation
-  // http://bit.ly/2gBMDs6
-  var response_buffer = ""; // response buffer
-  client.on('data', function(buffer) {
-  	log('Received data from socket: ' + buffer);
-    response_buffer += buffer;
-  });
-
-  client.on('close', function() {
-  	log('Socket connection closed');
-    // If server is not yet running, socket immediately closes and response_buffer
-    // is still empty. Return right away for this case, so request can retry.
-    if (response_buffer == "") {
-      return;
-  	}
-
-    if (fs.existsSync(JETS_OUTPUT)) {
-      // Thanks: https://stackoverflow.com/questions/6156501/read-a-file-one-line-at-a-time-in-node-js
-      var rd = readline.createInterface({
-        input: fs.createReadStream(JETS_OUTPUT),
-        // output: process.stdout,
-        console: false
-      });
-
-      rd.on('line', (line) => {
-        // Important to use console.error in case locally testing shim or we see
-        // stdout "twice", once from here and once from ruby-land and it's confusing.
-        // AWS lambda will write console.error and console.log to CloudWatch.
-        console.error(line); // output to AWS Lambda Logs
-      }).on('close', () => {
-        truncate(JETS_OUTPUT); // done reporting output, clear out file again
-      });
-    }
-
-    var resp = JSON.parse(response_buffer);
-    if (resp && resp["errorMessage"]) {
-      // Customize error object for lambda format
-      var error = rubyError(resp);
-      callback(error);
-    } else {
-      callback(null, resp);
-    }
-    client.destroy(); // kill client after server's response
-  });
-
-  client.on('error', function(error) {
-    log("Socket error:");
-    log(error);
-    log("Retrying request");
-    setTimeout(function() {
-      if (fs.existsSync(TMP_LOG_PATH)) {
-        var contents = fs.readFileSync(TMP_LOG_PATH, 'utf8');
-        if (contents != "") {
-          log("subprocess output:", "debug");
-          log(contents, "debug");
-        }
-      }
-      truncate(TMP_LOG_PATH);
-
-      log("Retrying request NOW");
-      request(event, context, handler, callback);
-    }, 500);
-  });
-}
-
-module.exports = {
-  handler: handler,
-  once: once
-};

data/lib/jets/rails_overrides.rb

@@ -1,4 +0,0 @@
-require "jets/rails_overrides/common_methods"
-require "jets/rails_overrides/url_helper"
-require "jets/rails_overrides/rendering_helper"
-require "jets/rails_overrides/asset_tag_helper"

data/lib/jets/ruby_server.rb

@@ -1,148 +0,0 @@
-require 'socket'
-require 'json'
-require 'stringio'
-
-# https://ruby-doc.org/stdlib-2.3.0/libdoc/socket/rdoc/TCPServer.html
-# https://stackoverflow.com/questions/806267/how-to-fire-and-forget-a-subprocess
-#
-# There's a generated bin/ruby_server which calls bin/ruby_server.rb as part of the
-# starter project template. Usage:
-#
-#    bin/ruby_server # background
-#    FOREGROUND=1 bin/ruby_server # foreground
-#
-module Jets
-  class RubyServer
-    PORT = 8080
-
-    def run
-      Jets.boot(stringio: true) # outside of child process for COW
-
-      # INT - ^C
-      trap('INT') do
-        puts "Shutting down ruby_server.rb..."
-        FileUtils.rm_f("/tmp/jets-rackup.pid") # remove the rack subprocess pid in case it exists
-        sleep 0.1
-        exit
-      end
-
-      if ENV['FOREGROUND'] # Usage above
-        # start_rack_server # commented out because we expect user to start the rack server in the foreground
-        serve # ruby_server
-        return
-      end
-
-      # Reaching here means we'll run the server in the "background"
-      pid = Process.fork
-
-      if pid.nil? # we're in child process
-        start_rack_server
-        serve # ruby_server
-      else # we're in parent process
-        # Detach main jets ruby server
-        Process.detach(pid) # dettached but still in the "foreground" since server loop runs in the foreground
-      end
-    end
-
-    # Megamode support
-    def start_rack_server
-      return unless Jets.rack?
-
-      # Fire and forget for concurrent, will wait with wait_for_rack_socket
-      Thread.new do
-        Jets::Server.start
-      end
-
-      wait_for_rack_socket # blocks until rack server is up
-    end
-
-    # blocks until rack server is up
-    def wait_for_rack_socket
-      return unless Jets.rack?
-
-      retries = 0
-      max_retries = 30 # 15 seconds at a delay of 0.5s
-      delay = 0.5
-      if ENV['C9_USER'] # overrides for local testing
-        max_retries = 3
-        delay = 3
-      end
-      begin
-        server = TCPSocket.new('localhost', 9292)
-        server.close
-      rescue Errno::ECONNREFUSED
-        puts "Unable to connect to localhost:9292. Delay for #{delay} and will try to connect again."  if ENV['JETS_DEBUG']
-        sleep(delay)
-        retries += 1
-        if retries < max_retries
-          retry
-        else
-          puts "Giving up on trying to connect to localhost:9292"
-          return false
-        end
-      end
-      puts "Connected to localhost:9292 successfully"
-      true
-    end
-
-    # runs in the child process
-    def serve
-      server = TCPServer.new(PORT) # Server bind to port 8080
-      puts "Ruby server started on port #{PORT}" if ENV['FOREGROUND'] || ENV['JETS_DEBUG'] || ENV['C9_USER']
-
-      loop do
-        client = server.accept    # Wait for a client to connect
-
-        input_completed, event, context, handler = nil, nil, nil
-        unless input_completed
-          event = client.gets&.strip # text or nil
-          context = client.gets&.strip # text or nil
-          handler = client.gets&.strip # text or nil
-          # The event is nil when a client connects and immediately disconnects without sending data
-          if event.nil?
-            # puts "event was nil" # uncomment to debug
-            next
-          end
-
-          input_completed = true
-        end
-
-        # puts "ruby_server.rb event: #{event.inspect}"
-        # puts "ruby_server.rb context: #{context.inspect}"
-
-        result = event['_prewarm'] ?
-          prewarm_request(event) :
-          standard_request(event, context, handler)
-
-        Jets::IO.flush # flush output and write to disk for node shim
-
-        client.puts(result)
-        client.close
-      end
-    end
-
-    def prewarm_request(event)
-      # JSON.dump("prewarmed_at" => Time.now.to_s)
-      Jets.increase_prewarm_count
-      Jets.logger.info("Prewarm request")
-      %Q|{"prewarmed_at":"#{Time.now.to_s}"}| # raw json for speed
-    end
-
-    def standard_request(event, context, handler)
-      Jets::Processors::MainProcessor.new(
-        event,
-        context,
-        handler).run
-    rescue Exception => e
-      JSON.dump(
-        "stackTrace" => e.backtrace,
-        "errorMessage" => e.message,
-        "errorType" => "RubyError",
-      )
-    end
-
-    def self.run
-      new.run
-    end
-  end
-end