jets 0.2.0 → 0.5.0

data/lib/jets/builders/gem_replacer.rb

@@ -0,0 +1,154 @@
+# def extract_gems
+#   headline "Replacing compiled gems with Lambda Linux versions."
+#   Lambdagem::Extract::Gem.new(JETS_RUBY_VERSION,
+#     s3: "lambdagems",
+#     dest: full(cache_area),
+#   ).run
+# end
+class Jets::Builders
+  class GemReplacer
+    extend Memoist
+    attr_reader :missing_gems
+    def initialize(ruby_version, options)
+      @ruby_version = ruby_version
+      @options = options
+      @missing_gems = [] # keeps track of gems that are not found in any of the lambdagems sources
+    end
+
+    def run
+      # Checks whether the gem is found on at least one of the lambdagems sources.
+      # By the time the loop finishes, found_gems will hold a map of gem names to found
+      # url sources. Example:
+      #
+      #   found_gems = {
+      #     "nokogiri-1.8.4" => "https://lambdagems.com",
+      #     "pg-0.21.0" => "https://anothersource.com",
+      #   }
+      #
+      found_gems = {}
+      compiled_gems.each do |gem_name|
+        gem_exists = false
+        Jets.config.lambdagems.sources.each do |source|
+          exist = Lambdagem::Exist.new(lambdagems_url: source)
+          found = exist.check(gem_name)
+          # gem exists on at least of the lambdagem sources
+          if found
+            gem_exists = true
+            found_gems[gem_name] = source
+            break
+          end
+        end
+        unless gem_exists
+          @missing_gems << gem_name
+        end
+      end
+
+      # Exits early if not all the linux gems are available.
+      # It better to error now than deploy a broken package to AWS Lambda.
+      # Provide users with message about using their own lambdagems source.
+      unless @missing_gems.empty?
+        puts missing_gems_message
+        exit 1
+      end
+
+      # Reaching here means we can download and extract the gems
+      found_gems.each do |gem_name, source|
+        gem_extractor = Lambdagem::Extract::Gem.new(gem_name, @options.merge(lambdagems_url: source))
+        gem_extractor.run
+      end
+
+      tidy
+    end
+
+    def missing_gems_message
+      template = <<-EOL
+Your project requires compiled gems were not available in any of your lambdagems sources.  Unavailable pre-compiled gems:
+<% missing_gems.each do |gem| %>
+* <%= gem -%>
+<% end %>
+
+Your current lambdagems sources:
+<% Jets.config.lambdagems.sources.map do |source| %>
+* <%= source -%>
+<% end %>
+
+Jets is unable to build a deployment package that will work on AWS Lambda without the required pre-compiled gems. To remedy this, you can:
+
+* Build the gem yourself and add it to your own custom lambdagems sources. Refer to the Lambda Gems Docs: http://rubyonjets.com/docs/lambdagems
+* Wait until it added to lambdagems.com. No need to report this to us, as we've already been notified.
+* Use another gem that does not require compilation.
+
+Compiled gems usually take some time to figure out how to build as they each depend on different libraries and packages. We make an effort add new gems as soon as we can. You can support us by going to: http://rubyonjets.com/support-jets/
+EOL
+      erb = ERB.new(template, nil, '-') # trim mode https://stackoverflow.com/questions/4632879/erb-template-removing-the-trailing-line
+      erb.result(binding)
+    end
+
+    # remove unnecessary files to reduce package size
+    def tidy
+      tidy_gems("#{@options[:project_root]}/bundled/gems/ruby/*/gems/*")
+      tidy_gems("#{@options[:project_root]}/bundled/gems/ruby/*/bundler/gems/*")
+    end
+
+    def tidy_gems(gems_path)
+      Dir.glob(gems_path).each do |gem_path|
+        tidy_gem(gem_path)
+      end
+    end
+
+    # Clean up some unneeded files to try to keep the package size down
+    # In a generated jets app this made a decent 9% difference:
+    #  175M test2
+    #  191M test3
+    def tidy_gem(path)
+      # remove top level tests and cache folders
+      Dir.glob("#{path}/*").each do |path|
+        next unless File.directory?(path)
+        folder = File.basename(path)
+        if %w[test tests spec features benchmark cache doc].include?(folder)
+          FileUtils.rm_rf(path)
+        end
+      end
+
+      Dir.glob("#{path}/**/*").each do |path|
+        next unless File.file?(path)
+        ext = File.extname(path)
+        if %w[.rdoc .md .markdown].include?(ext) or
+           path =~ /LICENSE|CHANGELOG|README/
+          FileUtils.rm_f(path)
+        end
+      end
+    end
+
+    # If there are subfolders compiled_gem_paths might have files deeper
+    # in the directory tree.  So lets grab the gem name and figure out the
+    # unique paths of the compiled gems from there.
+    def compiled_gems
+      compiled_gem_paths.map { |p| gem_name_from_path(p) }.uniq# + ["whatever-0.0.1"]
+    end
+    memoize :compiled_gems
+
+    # Use pre-compiled gem because the gem could have development header shared
+    # object file dependencies.  The shared dependencies are packaged up as part
+    # of the pre-compiled gem so it is available in the Lambda execution environment.
+    #
+    # Example paths:
+    # Macosx:
+    #   bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/nokogiri-1.8.1
+    #   bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
+    # Official AWS Lambda Linux AMI:
+    #   bundled/gems/ruby/2.5.0/extensions/x86_64-linux/2.5.0-static/nokogiri-1.8.1
+    # Circleci Ubuntu based Linux:
+    #   bundled/gems/ruby/2.5.0/extensions/x86_64-linux/2.5.0/pg-0.21.0
+    def compiled_gem_paths
+      Dir.glob("#{Jets.build_root}/cache/bundled/gems/ruby/*/extensions/**/**/*.{so,bundle}")
+    end
+
+    # Input: bundled/gems/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0-static/byebug-9.1.0
+    # Output: byebug-9.1.0
+    def gem_name_from_path(path)
+      regexp = /gems\/ruby\/\d+\.\d+\.\d+\/extensions\/.*?\/.*?\/(.*?)\//
+      gem_name = path.match(regexp)[1]
+    end
+  end
+end

data/lib/jets/builders/handler_generator.rb

@@ -0,0 +1,76 @@
+require "fileutils"
+require "erb"
+
+# Example:
+#
+# Jets::Builders::HandlerGenerator.new(
+#   "PostsController",
+#   :create, :update
+# )
+class Jets::Builders
+  class HandlerGenerator
+    def initialize(path)
+      @path = path
+    end
+
+    def generate
+      poly_shims
+      ruby_node_shim
+    end
+
+    def poly_shims
+      missing = []
+
+      deducer = Jets::Builders::Deducer.new(@path)
+      poly_tasks = deducer.klass.tasks.select { |t| t.lang != :ruby }
+      poly_tasks.each do |task|
+        source_path = get_source_path(@path, task)
+        if File.exist?(source_path)
+          native_function(@path, task)
+        else
+          missing << source_path
+        end
+      end
+
+      unless missing.empty?
+        puts "ERROR: Missing source files. Please make sure these source files exist or remove their declarations".colorize(:red)
+        puts missing
+        exit 1
+      end
+    end
+
+    def get_source_path(original_path, task)
+      folder = original_path.sub(/\.rb$/,'')
+      lang_folder = "#{folder}/#{task.lang}"
+      root = Jets.root unless original_path.include?("lib/jets/internal")
+      "#{root}#{lang_folder}/#{task.meth}#{task.lang_ext}"
+    end
+
+    # Builds and copies over the native source code: python or node
+    def native_function(original_path, task)
+      source_path = get_source_path(original_path, task)
+      # Handler: handlers/controllers/posts_controller.handle
+      dest_path = "#{tmp_app_root}/#{task.handler_path}"
+      FileUtils.mkdir_p(File.dirname(dest_path))
+      FileUtils.cp(source_path, dest_path)
+    end
+
+    # Generates one big node shim for a entire controller.
+    def ruby_node_shim
+      deducer = Jets::Builders::Deducer.new(@path)
+
+      js_path = "#{tmp_app_root}/#{deducer.js_path}"
+      FileUtils.mkdir_p(File.dirname(js_path))
+
+      template_path = File.expand_path('../node-shim.js', __FILE__)
+      result = Jets::Erb.result(template_path, deducer: deducer)
+
+      IO.write(js_path, result)
+    end
+
+    # TODO: move CodeBuilder.tmp_app_root to a common level for HandlerGenerator and CodeBuilder
+    def tmp_app_root
+      "#{Jets.build_root}/#{CodeBuilder.tmp_app_root}"
+    end
+  end
+end

data/lib/jets/builders/node-hello.js

@@ -0,0 +1,73 @@
+'use strict';
+console.log('Loading hello world function');
+
+// https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-create-api-as-simple-proxy-for-lambda.html
+
+exports.handler = function(event, context, callback) {
+    let name = "you";
+    let city = 'World';
+    let time = 'day';
+    let day = '';
+    let responseCode = 200;
+    console.log("request: " + JSON.stringify(event));
+
+    // This is a simple illustration of app-specific logic to return the response.
+    // Although only 'event.queryStringParameters' are used here, other request data,
+    // such as 'event.headers', 'event.pathParameters', 'event.body', 'event.stageVariables',
+    // and 'event.requestContext' can be used to determine what response to return.
+    //
+    if (event.queryStringParameters !== null && event.queryStringParameters !== undefined) {
+        if (event.queryStringParameters.name !== undefined &&
+            event.queryStringParameters.name !== null &&
+            event.queryStringParameters.name !== "") {
+            console.log("Received name: " + event.queryStringParameters.name);
+            name = event.queryStringParameters.name;
+        }
+    }
+
+    if (event.pathParameters !== null && event.pathParameters !== undefined) {
+        if (event.pathParameters.proxy !== undefined &&
+            event.pathParameters.proxy !== null &&
+            event.pathParameters.proxy !== "") {
+            console.log("Received proxy: " + event.pathParameters.proxy);
+            city = event.pathParameters.proxy;
+        }
+    }
+
+    if (event.headers !== null && event.headers !== undefined) {
+        if (event.headers['day'] !== undefined && event.headers['day'] !== null && event.headers['day'] !== "") {
+            console.log("Received day: " + event.headers.day);
+            day = event.headers.day;
+        }
+    }
+
+    if (event.body !== null && event.body !== undefined) {
+        let body = JSON.parse(event.body)
+        if (body.time)
+            time = body.time;
+    }
+
+    let greeting = 'Good ' + time + ', ' + name + ' of ' + city + '. ';
+    if (day) greeting += 'Happy ' + day + '!';
+
+    var responseBody = {
+        message: greeting,
+        input: event
+    };
+
+    // The output from a Lambda proxy integration must be
+    // of the following JSON object. The 'headers' property
+    // is for custom response headers in addition to standard
+    // ones. The 'body' property  must be a JSON string. For
+    // base64-encoded payload, you must also set the 'isBase64Encoded'
+    // property to 'true'.
+    var response = {
+        statusCode: responseCode,
+        headers: {
+            "x-custom-header" : "my custom header value"
+        },
+        body: JSON.stringify(responseBody)
+    };
+    console.log("response: " + JSON.stringify(response))
+    callback(null, response);
+};

data/lib/jets/builders/node-shim.js

@@ -0,0 +1,151 @@
+'use strict';
+
+const spawn = require('child_process').spawn;
+const fs = require('fs');
+const TMP_LOG_PATH = '/tmp/shim-subprocess.log';
+const subprocess_out = fs.openSync(TMP_LOG_PATH, 'a');
+const subprocess_err = fs.openSync(TMP_LOG_PATH, 'a');
+
+function once() {
+  // Uncomment fake run once locally. No need to do this on real lambda environment.
+  // var filename = '/tmp/once.txt';
+  // if (fs.existsSync(filename)) {
+  //   return;
+  // }
+  // fs.closeSync(fs.openSync(filename, 'w'));
+
+  // start tcp server and detach
+  const subprocess = spawn('bin/ruby_server', {
+    detached: true,
+    stdio: [ 'ignore', subprocess_out, subprocess_err ]
+  });
+  subprocess.on('error', function(err) {
+    log('bin/ruby_server error', err);
+  });
+
+  subprocess.on('close', function(exit_code) {
+    log("Subprocess was shut down or deattached!");
+  });
+
+  // prevent the parent from waiting for a subprocess to exit
+  // https://nodejs.org/api/child_process.html
+  subprocess.unref();
+}
+
+once();
+
+// Produces an Error object that displays in the AWS Lambda test console nicely.
+// The backtrace are the ruby lines, not the nodejs shim error lines.
+// The json payload in the Lambda console looks something like this:
+//
+//   {
+//     "errorMessage": "RubyError: RuntimeError: error in submethod",
+//     "errorType": "RubyError",
+//     "stackTrace": [
+//       [
+//         "line1",
+//         "line2",
+//         "line3"
+//       ]
+//     ]
+//   }
+//
+function rubyError(resp) {
+  var name = resp["errorType"]
+  var message = resp["errorMessage"]
+  var lines = resp["stackTrace"]
+
+  var error = new Error(message)
+  error.name = name
+  error.stack = lines.join("\n")
+  return error
+}
+
+// On AWS Lambda, we can log to either stdout or stderr and we're okay.
+// But locally when we're testing the shim, the log output can mess up piping
+// to jq. So not logging to stdout because when testing this shim locally the
+// stdout output messes up a pipe to jq.
+function log(text) {
+  console.error(text);
+}
+
+////////////////////////////
+// main logic for handler
+const net = require('net');
+const PORT = 8080;
+const HOST = '127.0.0.1';
+
+function request(event, handler, callback) {
+  log("event:");
+  log(event);
+  var client = new net.Socket();
+  client.connect(PORT, HOST, function() {
+    log('Connected to socket');
+    client.write(JSON.stringify(event));
+    client.write("\r\n") // important: \r\n is how server knows input is done
+    client.write(handler);
+    client.write("\r\n") // important: \r\n is how server knows input is done
+  });
+
+  // string concatation in javascript is faster than array concatation
+  // http://bit.ly/2gBMDs6
+  var stdout_buffer = ""; // stdout buffer
+  client.on('data', function(buffer) {
+  	log('Received data from socket: ' + buffer);
+    stdout_buffer += buffer;
+  });
+
+  client.on('close', function() {
+  	log('Socket connection closed');
+  	// If server is not yet running, socket immediately closes and stdout_buffer
+  	// is still empty. Return right away for this case, so request can retry.
+  	if (stdout_buffer == "") {
+  	  return;
+  	}
+
+    var resp = JSON.parse(stdout_buffer);
+    if (resp["errorMessage"]) {
+      // Customize error object for lambda format
+      var error = rubyError(resp)
+      callback(error);
+    } else {
+      callback(null, resp);
+    }
+    client.destroy(); // kill client after server's response
+  });
+
+  client.on('error', function(error) {
+    log("Socket error %o", error);
+    log("Retrying request");
+    setTimeout(function() {
+      if (fs.existsSync(TMP_LOG_PATH)) {
+        var contents = fs.readFileSync(TMP_LOG_PATH, 'utf8');
+        log("subprocess output:");
+        log(contents);
+      }
+      request(event, handler, callback);
+    }, 500);
+  });
+};
+
+<% @deducer.functions.each do |function_name| %>
+exports.<%= function_name %> = (event, context, callback) => {
+  request(event, "<%= @deducer.handler_for(function_name) %>", callback);
+}
+<% end %>
+
+// for local testing
+if (require.main === module) {
+  // fake event and context
+  var event = {"hello": "world"}
+  // var event = {"body": {"hello": "world"}} // API Gateway wrapper structure
+  var context = {"fake": "context"}
+  exports.<%= @deducer.functions.first %>(event, context, (error, message) => {
+    log("\nLOCAL TESTING OUTPUT")
+    if (error) {
+      console.log("error message: %o", error)
+    } else {
+      console.log(JSON.stringify(message)) // stringify
+    }
+  })
+}