jets 0.10.4 → 1.0.0

data/lib/jets/builders/shim_vars/base.rb

@@ -0,0 +1,24 @@
+module Jets::Builders::ShimVars
+  class Base
+    include Jets::AwsServices
+    extend Memoist
+
+    def s3_bucket
+      Jets.aws.s3_bucket
+    end
+
+    def rack_zip
+      checksum = Jets::Builders::Md5.checksums["stage/rack"]
+      "rack-#{checksum}.zip"
+    end
+
+    def bundled_zip
+      checksum = Jets::Builders::Md5.checksums["stage/bundled"]
+      "bundled-#{checksum}.zip"
+    end
+
+    def stage_area
+      "#{Jets.build_root}/stage"
+    end
+  end
+end

data/lib/jets/builders/{shared_deducer.rb → shim_vars/shared.rb}

@@ -1,4 +1,4 @@
-# Deducer.new(path)
+# Jets::Builders::ShimVars::Shared.new(fun)
 #
 #   @deducer.functions.each do |function_name|
 #     @deducer.handler_for(function_name)
@@ -10,8 +10,9 @@
 #   handler_for(function_name)
 #   js_path
 #
-class Jets::Builders
-  class SharedDeducer < Deducer
+module Jets::Builders::ShimVars
+  class Shared < Base
+    # fun is a Jets::Stack::Function
     def initialize(fun)
       @fun = fun
     end

data/lib/jets/builders/shim_vars.rb

@@ -0,0 +1,5 @@
+module Jets::Builders::ShimVars
+  autoload :App, "jets/builders/shim_vars/app"
+  autoload :Base, "jets/builders/shim_vars/base"
+  autoload :Shared, "jets/builders/shim_vars/shared"
+end

data/lib/jets/builders/templates/handler.js

@@ -0,0 +1,9 @@
+'use strict';
+
+const shim = require("handlers/shim.js");
+
+shim.once(); // runs in lambda execution context
+
+<% @vars.functions.each do |function_name| -%>
+exports.<%= function_name %> = shim.handler("<%= @vars.handler_for(function_name) %>");
+<% end -%>

data/lib/jets/builders/templates/shim.js

@@ -0,0 +1,271 @@
+const TMP_LOG_PATH = '/tmp/shim-subprocess.log';
+const JETS_DEBUG = process.env.JETS_DEBUG; // set JETS_DEBUG=1 to see more debugging info
+const JETS_OUTPUT = '/tmp/jets-output.log';
+const S3_BUCKET = '<%= @vars.s3_bucket %>';
+<% if Jets.rack? -%>
+const RACK_ZIP = '<%= @vars.rack_zip %>';       // jets/code/rack-checksum.zip
+<% end -%>
+<% if Jets.lazy_load? -%>
+const BUNDLED_ZIP = '<%= @vars.bundled_zip %>'; // /tmp/bundled-checksum.zip
+<% end -%>
+
+
+const spawn = require('child_process').spawn;
+const fs = require('fs');
+const readline = require('readline');
+const subprocess_out = fs.openSync(TMP_LOG_PATH, 'a');
+const subprocess_err = fs.openSync(TMP_LOG_PATH, 'a');
+const { exec } = require('child_process');
+const AWS = require('aws-sdk/global');
+const S3 = require('aws-sdk/clients/s3'); // import individual service
+const net = require('net');
+
+////////////////////////////////////////////////////////////////////////////////
+// util methods
+const sh = async (command, args) => {
+  var promise = new Promise(function(resolve, reject) {
+    log(`=> ${command}`);
+    exec(command, (err, stdout, stderr) => {
+      log(`stdout: ${stdout}`);
+      log(`stderr: ${stderr}`);
+      if (err) {
+        resolve({success: false, command: command});
+      } else {
+        resolve({success: true, command: command});
+      }
+    });
+  });
+  return promise;
+};
+
+const download = async (key, dest) => {
+  var promise = new Promise(function(resolve, reject) {
+    var s3 = new AWS.S3();
+    var params = {Bucket: S3_BUCKET, Key: key};
+    var file = require('fs').createWriteStream(dest);
+    var stream = s3.getObject(params).createReadStream();
+    var pipe = stream.pipe(file);
+    file.on('finish', function () {
+      resolve({success: true, pipe: pipe});
+    });
+  });
+  return promise;
+};
+
+const downloadAndExtract = async (zip_file, folder_dest) => {
+  var s3_key = `jets/code/${zip_file}`;   // jets/code/bundled-checksum.zip
+  var download_path = `/tmp/${zip_file}`; // /tmp/bundled-checksum.zip
+
+  await download(s3_key, download_path);
+  await sh(`unzip -qo ${download_path} -d ${folder_dest}`);
+  // await sh(`ls ${folder_dest}*`);
+};
+
+// Clean out files to prevent confusing double output when debugging.
+// Also clean out the file whenever done printing out the output.
+function truncate(path) {
+  if (fs.existsSync(path)) {
+    fs.truncateSync(path);
+  }
+}
+
+// Produces an Error object that displays in the AWS Lambda test console nicely.
+// The backtrace are the ruby lines, not the nodejs shim error lines.
+// The json payload in the Lambda console looks something like this:
+//
+//   {
+//     "errorMessage": "RubyError: RuntimeError: error in submethod",
+//     "errorType": "RubyError",
+//     "stackTrace": [
+//       [
+//         "line1",
+//         "line2",
+//         "line3"
+//       ]
+//     ]
+//   }
+//
+function rubyError(resp) {
+  var name = resp["errorType"];
+  var message = resp["errorMessage"];
+  var stack = resp["stackTrace"];
+  stack.unshift(message); // JS error includes the error message at the top of the stacktrac also
+  stack = stack.join("\n");
+
+  var error = new Error(message);
+  error.name = name;
+  error.stack = stack;
+  return error;
+}
+
+// On AWS Lambda, we can log to either stdout or stderr and we're okay.
+// But locally when we're testing the shim, the log output can mess up piping
+// to jq. So not logging to stdout because when testing this shim locally the
+// stdout output messes up a pipe to jq.
+function log(text, level="info") {
+  if (level == "info" && JETS_DEBUG) {
+    // only log if JETS_DEBUG is set
+    console.error(text);
+  } else if (level == "debug") {
+    // always log if "debug" passed into method
+    console.error(text);
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Logic that runs once in the lambda execution context
+
+const downloadTmp = async () => {
+  var start = Date.now();
+
+<% if Jets.lazy_load? -%>
+  var bundled = downloadAndExtract(BUNDLED_ZIP, '/tmp/bundled');
+<% end -%>
+<% if Jets.rack? -%>
+  var rack = downloadAndExtract(RACK_ZIP, '/tmp/rack');
+<% end -%>
+<% if Jets.lazy_load? -%>
+  // Interestingly, doesnt seem to speed up download with parallization below.
+  // Think this is because node does not take advantage of multiple cores?
+  // Leaving this in place because speed seems the same as doing the await
+  // in serial and hope it is fast in the future.
+  await bundled;
+<% end -%>
+<% if Jets.rack? -%>
+  await rack;
+<% end -%>
+
+  var took = Date.now() - start;
+  log(`Download /tmp time: ${took}`);
+};
+
+const rubyServer = () => {
+  // start tcp server and detach
+  const subprocess = spawn('bin/ruby_server', {
+    detached: true,
+    stdio: [ 'ignore', subprocess_out, subprocess_err ]
+  });
+  subprocess.on('error', function(err) {
+    log('bin/ruby_server error', err);
+  });
+
+  subprocess.on('close', function(exit_code) {
+    log("Subprocess was shut down or deattached!");
+  });
+
+  // prevent the parent from waiting for a subprocess to exit
+  // https://nodejs.org/api/child_process.html
+  subprocess.unref();
+};
+
+// once runs in lambda execution context
+const once = async () => {
+  // Uncomment fake run once locally. No need to do this on real lambda environment.
+  // var filename = '/tmp/once.txt';
+  // if (fs.existsSync(filename)) {
+  //   log("fake run only once. /tmp/once.txt exists. remove to run once again.");
+  //   return;
+  // }
+  // fs.closeSync(fs.openSync(filename, 'w'));
+
+<% if Jets.rack? || Jets.lazy_load? -%>
+  await downloadTmp();
+<% end -%>
+  // The server related methods are fire and forget.
+  // Not using await here because had trouble resolving the promise on client.connect
+  rubyServer(); // no await
+};
+
+// currying function to make handler code prettier
+function handler(full_handler_name) {
+  return function(event, context, callback) {
+    request(event, full_handler_name, callback);
+  };
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// main logic for handler
+function request(event, handler, callback) {
+  truncate(JETS_OUTPUT);
+  truncate(TMP_LOG_PATH);
+
+  log("event:");
+  log(event);
+  var client = new net.Socket();
+  client.connect(8080, '127.0.0.01', function() {
+    log('Connected to socket');
+    client.write(JSON.stringify(event));
+    client.write("\r\n"); // important: \r\n is how server knows input is done
+    client.write(handler);
+    client.write("\r\n"); // important: \r\n is how server knows input is done
+  });
+
+  // string concatation in javascript is faster than array concatation
+  // http://bit.ly/2gBMDs6
+  var response_buffer = ""; // response buffer
+  client.on('data', function(buffer) {
+  	log('Received data from socket: ' + buffer);
+    response_buffer += buffer;
+  });
+
+  client.on('close', function() {
+  	log('Socket connection closed');
+    // If server is not yet running, socket immediately closes and response_buffer
+    // is still empty. Return right away for this case, so request can retry.
+    if (response_buffer == "") {
+      return;
+  	}
+
+    if (fs.existsSync(JETS_OUTPUT)) {
+      // Thanks: https://stackoverflow.com/questions/6156501/read-a-file-one-line-at-a-time-in-node-js
+      var rd = readline.createInterface({
+        input: fs.createReadStream(JETS_OUTPUT),
+        // output: process.stdout,
+        console: false
+      });
+
+      rd.on('line', (line) => {
+        // Important to use console.error in case locally testing shim or we see
+        // stdout "twice", once from here and once from ruby-land and it's confusing.
+        // AWS lambda will write console.error and console.log to CloudWatch.
+        console.error(line); // output to AWS Lambda Logs
+      }).on('close', () => {
+        truncate(JETS_OUTPUT); // done reporting output, clear out file again
+      });
+    }
+
+    var resp = JSON.parse(response_buffer);
+    if (resp && resp["errorMessage"]) {
+      // Customize error object for lambda format
+      var error = rubyError(resp);
+      callback(error);
+    } else {
+      callback(null, resp);
+    }
+    client.destroy(); // kill client after server's response
+  });
+
+  client.on('error', function(error) {
+    log("Socket error:");
+    log(error);
+    log("Retrying request");
+    setTimeout(function() {
+      if (fs.existsSync(TMP_LOG_PATH)) {
+        var contents = fs.readFileSync(TMP_LOG_PATH, 'utf8');
+        if (contents != "") {
+          log("subprocess output:", "debug");
+          log(contents, "debug");
+        }
+      }
+      truncate(TMP_LOG_PATH);
+
+      log("Retrying request NOW");
+      request(event, handler, callback);
+    }, 500);
+  });
+}
+
+module.exports = {
+  handler: handler,
+  once: once
+};

data/lib/jets/builders/tidy.rb

@@ -0,0 +1,80 @@
+class Jets::Builders
+  class Tidy
+    def initialize(project_root, noop: false)
+      @project_root = project_root
+      @noop = noop
+    end
+
+    def cleanup!
+      removals.each do |removal|
+        removal = removal.sub(%r{^/},'') # remove leading slash
+        path = "#{@project_root}/#{removal}"
+        rm_rf(path)
+      end
+
+      tidy_bundled
+    end
+
+    def removals
+      removals = always_removals
+      removals += get_removals("#{@project_root}/.gitignore")
+      removals += get_removals("#{@project_root}/.dockerignore")
+      removals = removals.reject do |p|
+        jetskeep.find do |keep|
+          p.include?(keep)
+        end
+      end
+      removals.uniq
+    end
+
+    def get_removals(file)
+      path = file
+      return [] unless File.exist?(path)
+
+      removal = File.read(path).split("\n")
+      removal.map {|i| i.strip}.reject {|i| i =~ /^#/ || i.empty?}
+      # IE: ["/handlers", "/bundled*", "/vendor/jets]
+    end
+
+    # We clean out ignored files pretty aggressively. So provide
+    # a way for users to keep files from being cleaned out.
+    def jetskeep
+      defaults = %w[.bundle bundled pack handlers public/assets]
+      path = "#{@project_root}/.jetskeep"
+      return defaults unless File.exist?(path)
+
+      keep = IO.readlines(path)
+      keep = keep.map {|i| i.strip}.reject { |i| i =~ /^#/ || i.empty? }
+      (defaults + keep).uniq
+    end
+
+    # folders to remove in the bundled folder regardless of the level of the folder
+    def tidy_bundled
+      Dir.glob("#{@project_root}/bundled/**/*").each do |path|
+        next unless File.directory?(path)
+        dir = File.basename(path)
+        next unless always_removals.include?(dir)
+
+        rm_rf(path)
+      end
+    end
+
+    def rm_rf(path)
+      exists = File.exist?("#{path}/.gitkeep") || File.exist?("#{path}/.keep")
+      return if exists
+
+      # say "  rm -rf #{path}".colorize(:yellow) # uncomment to debug
+      system("rm -rf #{path}") unless @noop
+    end
+
+    # These directories will be removed regardless of dir level
+    def always_removals
+      %w[.git spec tmp]
+    end
+
+    def say(message)
+      message = "NOOP #{message}" if @noop
+      puts message
+    end
+  end
+end

data/lib/jets/builders/util.rb

@@ -0,0 +1,28 @@
+class Jets::Builders
+  module Util
+    def sh(command)
+      puts "=> #{command}".colorize(:green)
+      success = system(command)
+      abort("#{command} failed to run") unless success
+      success
+    end
+
+    def headline(message)
+      puts "=> #{message}".colorize(:cyan)
+    end
+
+    # Provide pretty clear way to desinate full path.
+    # full("bundled") => /tmp/jets/demo/bundled
+    def full(relative_path)
+      "#{Jets.build_root}/#{relative_path}"
+    end
+
+    # Finds out of the app has polymorphic functions only and zero ruby functions.
+    # In this case, we can skip a lot of the ruby related building and speed up the
+    # deploy process.
+    def poly_only?
+      return true if ENV['POLY_ONLY'] # bypass to allow rapid development of handlers
+      Jets::Commands::Build.poly_only?
+    end
+  end
+end

data/lib/jets/builders.rb

@@ -1,7 +1,13 @@
 class Jets::Builders
   autoload :CodeBuilder, "jets/builders/code_builder"
-  autoload :Deducer, "jets/builders/deducer"
   autoload :GemReplacer, "jets/builders/gem_replacer"
   autoload :HandlerGenerator, "jets/builders/handler_generator"
-  autoload :SharedDeducer, "jets/builders/shared_deducer"
+  autoload :Md5, "jets/builders/md5"
+  autoload :Md5Zip, "jets/builders/md5_zip"
+  autoload :RackPackager, "jets/builders/rack_packager"
+  autoload :ReconfigureRails, "jets/builders/reconfigure_rails"
+  autoload :RubyPackager, "jets/builders/ruby_packager"
+  autoload :ShimVars, "jets/builders/shim_vars"
+  autoload :Tidy, "jets/builders/tidy"
+  autoload :Util, "jets/builders/util"
 end

data/lib/jets/cfn/builders/function_builder.rb

@@ -10,15 +10,5 @@ class Jets::Cfn::Builders
       add_common_parameters
       add_functions
     end
-
-    # For function stacks, ensure there's a _function.yml at the end of the
-    # template_path name for easy identification.
-    def template_path
-      path = super
-      unless path.include?("function.yml")
-        path = path.sub(".yml", "_function.yml")
-      end
-      path
-    end
   end
 end

data/lib/jets/cfn/builders/parent_builder.rb

@@ -12,10 +12,8 @@ class Jets::Cfn::Builders
 
     # compose is an interface method
     def compose
-      puts "Building parent CloudFormation template."
-
       build_minimal_resources
-      build_child_resources if @options[:templates] || @options[:stack_type] == :full
+      build_child_resources if full?
     end
 
     # template_path is an interface method
@@ -29,6 +27,7 @@ class Jets::Cfn::Builders
       add_resource(resource)
       add_outputs(resource.outputs)
 
+      return unless full?
       # Add application-wide IAM policy from Jets.config.iam_role
       resource = Jets::Resource::Iam::ApplicationRole.new
       add_resource(resource)
@@ -36,8 +35,6 @@ class Jets::Cfn::Builders
     end
 
     def build_child_resources
-      puts "Building child CloudFormation templates."
-
       expression = "#{Jets::Naming.template_path_prefix}-app-*"
       # IE: path: #{Jets.build_root}/templates/demo-dev-2-comments_controller.yml
       Dir.glob(expression).each do |path|
@@ -53,12 +50,16 @@ class Jets::Cfn::Builders
         add_shared_resources(path)
       end
 
-      if (@options[:templates] || @options[:stack_type] == :full) and !Jets::Router.routes.empty?
+      if full? and !Jets::Router.routes.empty?
         add_api_gateway
         add_api_deployment
       end
     end
 
+    def full?
+      @options[:templates] || @options[:stack_type] == :full
+    end
+
     def add_app_class_stack(path)
       resource = Jets::Resource::ChildStack::AppClass.new(@options[:s3_bucket], path: path)
       add_child_resources(resource)

data/lib/jets/cfn/ship.rb

@@ -1,10 +1,7 @@
-require 'action_view'
-
 class Jets::Cfn
   class Ship
     include Jets::Timing
     include Jets::AwsServices
-    include ActionView::Helpers::NumberHelper # number_to_human_size
 
     def initialize(options)
       @options = options
@@ -13,8 +10,8 @@ class Jets::Cfn
     end
 
     def run
-      upload_to_s3 if @options[:stack_type] == :full # s3 bucket is available
-        # only when stack_type is full
+      # s3 bucket is available only when stack_type is full
+      upload_to_s3 if @options[:stack_type] == :full
 
       stack_in_progress?(@parent_stack_name)
 
@@ -81,11 +78,15 @@ class Jets::Cfn
     time :wait_for_stack
 
     def prewarm
+      if ENV['SKIP_PREWARMING']
+        puts "Skipping prewarming" # useful for testing
+        return
+      end
       return unless @options[:stack_type] == :full # s3 bucket is available
       return unless Jets.config.prewarm.enable
       return if Jets::Commands::Build.poly_only?
 
-      puts "Prewarming application..."
+      puts "Prewarming application."
       if Jets::PreheatJob::CONCURRENCY > 1
         Jets::PreheatJob.perform_now(:torch, {quiet: true})
       else
@@ -143,104 +144,9 @@ class Jets::Cfn
     def upload_to_s3
       raise "Did not specify @options[:s3_bucket] #{@options[:s3_bucket].inspect}" unless @options[:s3_bucket]
 
-      upload_cfn_templates
-      upload_code
-      upload_assets
+      uploader = Upload.new(@options[:s3_bucket])
+      uploader.upload
     end
     time :upload_to_s3
-
-    def bucket_name
-      @options[:s3_bucket]
-    end
-
-    def upload_cfn_templates
-      puts "Uploading child CloudFormation templates to S3"
-      expression = "#{Jets::Naming.template_path_prefix}-*"
-      Dir.glob(expression).each do |path|
-        next unless File.file?(path)
-
-        key = "jets/cfn-templates/#{File.basename(path)}"
-        obj = s3_resource.bucket(bucket_name).object(key)
-        obj.upload_file(path)
-      end
-    end
-
-    def upload_code
-      md5_code_zipfile = Jets::Naming.md5_code_zipfile
-      file_size = number_to_human_size(File.size(md5_code_zipfile))
-
-      puts "Uploading #{md5_code_zipfile} (#{file_size}) to S3"
-      start_time = Time.now
-      key = Jets::Naming.code_s3_key
-      obj = s3_resource.bucket(bucket_name).object(key)
-      obj.upload_file(md5_code_zipfile)
-      puts "Time to upload code to s3: #{pretty_time(Time.now-start_time).colorize(:green)}"
-    end
-
-    def upload_assets
-      puts "Uploading public assets"
-      start_time = Time.now
-      asset_folders = Jets.config.assets.folders
-      asset_folders.each do |folder|
-        upload_asset_folder(folder)
-      end
-      puts "Time to upload public assets to s3: #{pretty_time(Time.now-start_time).colorize(:green)}"
-    end
-
-    def upload_asset_folder(folder)
-      expression = "#{Jets.root}public/#{folder}/**/*"
-      group_size = 10
-      Dir.glob(expression).each_slice(group_size) do |paths|
-        threads = []
-        paths.each do |path|
-          next unless File.file?(path)
-
-          regexp = Regexp.new(".*/#{folder}/")
-          relative_path = path.sub(regexp,'')
-          file = "#{folder}/#{relative_path}"
-
-          threads << Thread.new do
-            upload_asset_file(file)
-          end
-        end
-        threads.each(&:join)
-      end
-    end
-
-    def upload_asset_file(file)
-      path = "#{Jets.root}public/#{file}"
-      key = "jets/public/#{file}"
-      puts "Uploading s3://#{bucket_name}/#{key}" # uncomment to see and debug
-      obj = s3_resource.bucket(bucket_name).object(key)
-      obj.upload_file(path, acl: "public-read", cache_control: cache_control)
-    end
-
-    # If cache_control is provided, then it will set the entire cache-control header.
-    # If only max_age is provided, then we'll generate a cache_control header.
-    # Using max_age is the shorter and simply way of setting the cache_control header.
-    def cache_control
-      cache_control = Jets.config.assets.cache_control
-      unless cache_control
-        max_age = Jets.config.assets.max_age # defaults to 3600 in jets/application.rb
-        cache_control = "public, max-age=#{max_age}"
-      end
-      cache_control
-    end
-
-    def s3_bucket
-      @options[:s3_bucket]
-    end
-
-    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
-    def pretty_time(total_seconds)
-      minutes = (total_seconds / 60) % 60
-      seconds = total_seconds % 60
-      if total_seconds < 60
-        "#{seconds.to_i}s"
-      else
-        "#{minutes.to_i}m #{seconds.to_i}s"
-      end
-    end
-
   end
 end