lam 0.0.1

data/lib/lam/build/templates/handler.js

@@ -0,0 +1,156 @@
+'use strict';
+
+// handler: <%= @handler %>
+const spawn = require('child_process').spawn;
+
+// Once hooked up to API Gateway can use the curl command to test:
+// curl -s -X POST -d @event.json https://endpoint | jq .
+
+// Filters out lines so only the error lines remain.
+// Uses the "RubyError: " marker to find the starting error lines.
+//
+// Input: String
+// random line
+// RubyError: RuntimeError: error in submethod
+// line1
+// line2
+// line3
+//
+// Output: String
+// RubyError: RuntimeError: error in submethod
+// line1
+// line2
+// line3
+function filterErrorLines(text) {
+  var lines = text.split("\n")
+  var markerIndex = lines.findIndex(line => line.startsWith("RubyError: ") )
+  lines = lines.filter((line, index) => index >= markerIndex )
+  return lines.join("\n")
+}
+
+// Produces an Error object that displays in the AWS Lambda test console nicely.
+// The backtrace are the ruby lines, not the nodejs shim error lines.
+// The json payload in the Lambda console looks something like this:
+//
+// {
+//   "errorMessage": "RubyError: RuntimeError: error in submethod",
+//   "errorType": "RubyError",
+//   "stackTrace": [
+//     [
+//       "line1",
+//       "line2",
+//       "line3"
+//     ]
+//   ]
+// }
+//
+// Input: String
+// RubyError: RuntimeError: error in submethod
+// line1
+// line2
+// line3
+//
+// Output: Error object
+// { RubyError: RuntimeError: error in submethod
+// line1
+// line2
+// line3 name: 'RubyError' }
+function customError(text) {
+  text = filterErrorLines(text) // filter for error lines only
+  var lines = text.split("\n")
+  var message = lines[0]
+  var error = new Error(message)
+  error.name = message.split(':')[0]
+  error.stack = lines.slice(0, lines.length-1) // drop final empty line
+                  .map(e => e.replace(/^\s+/g,'')) // trim leading whitespaces
+                  .join("\n")
+  return error
+}
+
+module.exports.<%= @js_method %> = (event, context, callback) => {
+  // To test on mac, set these environment variables:
+  // export RUBY_BIN=$HOME/.rbenv/shims/ruby
+  // export PROCESSOR_COMMAND="lam process controller"
+
+  // Command: lam process controller [event] [context] [handler]
+  const processor_command = process.env.PROCESSOR_COMMAND || "lam"
+  var args = [
+    "process",
+    "<%= @process_type %>",
+    JSON.stringify(event),
+    JSON.stringify(context),
+    "<%= @handler %>"
+  ]
+  // console.log("processor_command %o", processor_command)
+  // console.log("args %o", args)
+
+  var ruby = spawn("bin/lam", args);
+
+  // string concatation in javascript is faster than array concatation
+  // http://bit.ly/2gBMDs6
+  var stdout_buffer = ""; // stdout buffer
+  // In the processor_command we do NOT call puts directly and write to stdout
+  // because it will mess up the eventual response that we want API Gateway to
+  // process.
+  // The Lambda prints out function to whatever the return value the ruby method
+  ruby.stdout.on('data', function(data) {
+    // Not using console.log because it decorates output with a newline.
+    //
+    // Uncomment process.stdout.write to see stdout streamed for debugging.
+    // process.stdout.write(data)
+    stdout_buffer += data;
+  });
+
+  // react to potential errors
+  var stderr_buffer = "";
+  ruby.stderr.on('data', function(data) {
+    // not using console.error because it decorates output with a newline
+    stderr_buffer += data
+    process.stderr.write(data)
+  });
+
+  //finalize when ruby process is done.
+  ruby.on('close', function(exit_code) {
+    // http://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html#nodejs-prog-model-handler-callback
+
+    // succcess
+    if (exit_code == 0) {
+      var result
+      try {
+        result = JSON.parse(stdout_buffer)
+      } catch(e) {
+        // if json cannot be parse assume simple text output intended
+        process.stderr.write("WARN: error parsing json, assuming plain text is desired.")
+        result = stdout_buffer
+      }
+      callback(null, result);
+
+      // callback(null, stdout_buffer);
+    } else {
+
+      // TODO: if this works, allow a way to not decorate the error in case
+      // it actually errors in javascript land
+      // Customize error object with ruby error info
+      var error = customError(stderr_buffer)
+      callback(error);
+      // console.log("error!")
+    }
+  });
+}
+
+// for local testing
+if (process.platform == "darwin") {
+  // fake event and context
+  var event = {"hello": "world"}
+  // var event = {"body": {"hello": "world"}} // API Gateway wrapper structure
+  var context = {"fake": "context"}
+  module.exports.<%= @js_method %>(event, context, (error, message) => {
+    console.error("\nLOCAL TESTING OUTPUT")
+    if (error) {
+      console.error("error message: %o", error)
+    } else {
+      console.error("success message %o", message)
+      // console.log(JSON.stringify(message)) // stringify
+    }
+  })
+}

data/lib/lam/build/traveling_ruby.rb

@@ -0,0 +1,108 @@
+require "fileutils"
+require "open-uri"
+require "colorize"
+
+class Lam::Build
+  TRAVELING_RUBY_VERSION = 'http://d6r77u77i8pq3.cloudfront.net/releases/traveling-ruby-20150715-2.2.2-linux-x86_64.tar.gz'.freeze
+  TEMP_BUILD_DIR = '/tmp/lam_build'.freeze
+
+  class TravelingRuby
+    def build
+      if File.exist?("#{Lam.root}bundled")
+        puts "Ruby bundled already exists."
+        puts "To force rebundling: rm -rf bundled"
+        return
+      end
+
+      check_ruby_version
+
+      FileUtils.mkdir_p(TEMP_BUILD_DIR)
+      copy_gemfiles
+
+      Dir.chdir(TEMP_BUILD_DIR) do
+        download_traveling_ruby
+        unpack_traveling_ruby
+        bundle_install
+      end
+
+      move_bundled_to_project
+    end
+
+    def check_ruby_version
+      return if ENV['LAM_SKIP_RUBY_CHECK'] # only use if you absolutely need to
+      traveling_version = TRAVELING_RUBY_VERSION.match(/-((\d+)\.(\d+)\.(\d+))-/)[1]
+      if RUBY_VERSION != traveling_version
+        puts "You are using ruby version #{RUBY_VERSION}."
+        abort("You must use ruby #{traveling_version} to build the project because it's what Traveling Ruby uses.".colorize(:red))
+      end
+    end
+
+    def copy_gemfiles
+      FileUtils.cp("#{Lam.root}Gemfile", "#{TEMP_BUILD_DIR}/")
+      FileUtils.cp("#{Lam.root}Gemfile.lock", "#{TEMP_BUILD_DIR}/")
+    end
+
+    def download_traveling_ruby
+      puts "Downloading traveling ruby from #{traveling_ruby_url}."
+
+      FileUtils.rm_rf("#{TEMP_BUILD_DIR}/#{bundled_ruby_dest}")
+      File.open(traveling_ruby_tar_file, 'wb') do |saved_file|
+        # the following "open" is provided by open-uri
+        open(traveling_ruby_url, 'rb') do |read_file|
+          saved_file.write(read_file.read)
+        end
+      end
+
+      puts 'Download complete.'
+    end
+
+    def unpack_traveling_ruby
+      puts 'Unpacking traveling ruby.'
+
+      FileUtils.mkdir_p(bundled_ruby_dest)
+
+      success = system("tar -xzf #{traveling_ruby_tar_file} -C #{bundled_ruby_dest}")
+      abort('Unpacking traveling ruby failed') unless success
+      puts 'Unpacking traveling ruby successful.'
+
+      puts 'Removing tar.'
+      FileUtils.rm_rf(traveling_ruby_tar_file)
+    end
+
+    def bundle_install
+      puts 'Installing bundle.'
+      require "bundler" # dynamicaly require bundler so user can use any bundler
+      Bundler.with_clean_env do
+        success = system(
+          "cd #{TEMP_BUILD_DIR} && " \
+          'env BUNDLE_IGNORE_CONFIG=1 bundle install --path bundled/gems --without development'
+        )
+
+        abort('Bundle install failed, exiting.') unless success
+      end
+
+      puts 'Bundle install success.'
+    end
+
+    def move_bundled_to_project
+      if File.exist?("#{Lam.root}bundled")
+        puts "Removing current bundled folder"
+        FileUtils.rm_rf("#{Lam.root}bundled")
+      end
+      puts "Moving bundled ruby to your project."
+      FileUtils.mv("#{TEMP_BUILD_DIR}/bundled", Lam.root)
+    end
+
+    def bundled_ruby_dest
+      "bundled/ruby"
+    end
+
+    def traveling_ruby_url
+      TRAVELING_RUBY_VERSION
+    end
+
+    def traveling_ruby_tar_file
+      File.basename(traveling_ruby_url)
+    end
+  end
+end

data/lib/lam/cli.rb

@@ -0,0 +1,23 @@
+require "thor"
+require "lam/cli/help"
+
+module Lam
+
+  class CLI < Command
+    class_option :verbose, type: :boolean
+    class_option :noop, type: :boolean
+
+    desc "build", "Builds and prepares project for Lambda"
+    long_desc Help.build
+    option :force, type: :boolean, aliases: "-f", desc: "override existing starter files"
+    option :quiet, type: :boolean, aliases: "-q", desc: "silence the output"
+    option :format, type: :string, default: "yaml", desc: "starter project template format: json or yaml"
+    def build
+      Lam::Build.new(options).run
+    end
+
+    desc "process TYPE", "process subcommand tasks"
+    long_desc Help.process
+    subcommand "process", Lam::Process
+  end
+end

data/lib/lam/cli/help.rb

@@ -0,0 +1,19 @@
+module Lam
+  class CLI < Command
+    class Help
+      class << self
+        def build
+<<-EOL
+Builds and prepares project for AWS Lambda.  Generates a node shim and vendors Traveling Ruby.  Creates a zip file to be uploaded to Lambda for each handler.
+EOL
+        end
+
+        def process
+<<-EOL
+TODO: update process help menu
+EOL
+        end
+      end
+    end
+  end
+end

data/lib/lam/command.rb

@@ -0,0 +1,25 @@
+require "thor"
+
+module Lam
+  class Command < Thor
+    class << self
+      def dispatch(m, args, options, config)
+        # Allow calling for help via:
+        #   lam command help
+        #   lam command -h
+        #   lam command --help
+        #   lam command -D
+        #
+        # as well thor's normal way:
+        #
+        #   lam help command
+        help_flags = Thor::HELP_MAPPINGS + ["help"]
+        if args.length > 1 && !(args & help_flags).empty?
+          args -= help_flags
+          args.insert(-2, "help")
+        end
+        super
+      end
+    end
+  end
+end

data/lib/lam/process.rb

@@ -0,0 +1,18 @@
+class Lam::Process < Lam::Command
+  autoload :Help, 'lam/process/help'
+  autoload :ProcessorDeducer, 'lam/process/processor_deducer'
+  autoload :BaseProcessor, 'lam/process/base_processor'
+  autoload :ControllerProcessor, 'lam/process/controller_processor'
+
+  class_option :verbose, type: :boolean
+  class_option :noop, type: :boolean
+  class_option :project_root, desc: "Project folder.  Defaults to current directory", default: "."
+  class_option :region, desc: "AWS region"
+
+  desc "create STACK", "create a CloudFormation stack"
+  option :randomize_stack_name, type: :boolean, desc: "tack on random string at the end of the stack name", default: nil
+  long_desc Help.controller
+  def controller(event, context, handler)
+    ControllerProcessor.new(event, context, handler).run
+  end
+end

data/lib/lam/process/base_processor.rb

@@ -0,0 +1,23 @@
+require 'json'
+require_relative 'processor_deducer'
+
+# Global overrides for Lambda processing
+$stdout.sync = true
+# This might seem weird but we want puts to write to stderr which is set in
+# the node shim to write to stderr.  This directs the output to Lambda logs.
+# Printing to stdout can managle up the payload returned from Lambda function.
+# This is not desired if you want to return say a json payload to API Gateway
+# eventually.
+def puts(text)
+  $stderr.puts(text)
+end
+
+class Lam::Process::BaseProcessor
+  attr_reader :event, :context, :handler
+  def initialize(event, context, handler)
+    # assume valid json from Lambda
+    @event = JSON.parse(event)
+    @context = JSON.parse(context)
+    @handler = handler
+  end
+end

data/lib/lam/process/controller_processor.rb

@@ -0,0 +1,36 @@
+require_relative "base_processor"
+
+class Lam::Process
+  class ControllerProcessor < Lam::Process::BaseProcessor
+    def run
+      # Use the handler value (ie: posts.create) to deduce the user's business
+      # code to require and run.
+      deducer = ProcessorDeducer.new(handler)
+      path = deducer.controller[:path]
+      code = deducer.controller[:code]
+
+      begin
+        require path  # require "app/controllers/posts_controller.rb"
+        # Puts the return value of user's code to stdout because this is
+        # what eventually gets used by API Gateway.
+        # Explicitly using $stdout since puts redirected to $stderr.
+
+        # result = PostsController.new(event, context).create
+        result = instance_eval(code, path)
+
+        # JSON.dump is pretty robust.  If it cannot dump the structure into a
+        # json string, it just dumps it to a plain text string.
+        $stdout.puts JSON.dump(result) # only place where we write to stdout.
+      rescue Exception => e
+        # Customize error message slightly so nodejs shim can process the
+        # returned error message.
+        # The "RubyError: " is a marker that the javascript shim scans for.
+        $stderr.puts("RubyError: #{e.class}: #{e.message}") # js needs this as the first line
+        backtrace = e.backtrace.map {|l| "  #{l}" }
+        $stderr.puts(backtrace)
+        # $stderr.puts("END OF RUBY OUTPUT")
+        exit 1 # instead of re-raising to control the error backtrace output
+      end
+    end
+  end
+end

data/lib/lam/process/help.rb

@@ -0,0 +1,11 @@
+class Lam::Process::Help
+  class << self
+    def controller
+<<-EOL
+Examples:
+
+lam process controller '{ "we" : "love", "using" : "Lambda" }' '{"test": "1"}' "handlers/controllers/posts.create"
+EOL
+    end
+  end
+end

data/lib/lam/process/processor_deducer.rb

@@ -0,0 +1,52 @@
+class Lam::Process::ProcessorDeducer
+  def initialize(handler)
+    @handler = handler
+  end
+
+  # Deduces the path and method from the handler. Example:
+  #
+  #   ProcessorDeducer.new("handlers/functions/posts.create").function
+  #     => {path: "app/functions/posts.rb", code: "create(event, context)"}
+  #
+  # Summary:
+  #
+  # Input:
+  #   handler: handlers/functions/posts.create
+  # Output:
+  #   path: app/functions/posts.rb
+  #   code: create(event, context) # code to instance_eval
+  #
+  # Returns: {path: path, code: code}
+  def function
+    path, meth = @handler.split('.')
+    path = Lam.root + path.sub("handlers", "app") + ".rb"
+    code = "#{meth}(event, context)"
+    {path: path, code: code}
+  end
+
+  # Deduces the path and method from the handler. Example:
+  #
+  #   ProcessorDeducer.new("handlers/controllers/posts.create").controller
+  #     => {path: "controllers/posts_controller.rb", code: "create"}
+  #
+  # Summary:
+  #
+  # Input:
+  #   handler: handlers/controllers/posts.create
+  # Output:
+  #   path: app/controllers/posts_controller.rb
+  #   code: create # code to instance_eval
+  #
+  # Returns: {path: path, code: code}
+  def controller
+    handler_path, meth = @handler.split('.')
+
+    path = Lam.root + handler_path.sub("handlers", "app") + "_controller.rb"
+
+    controller_name = handler_path.sub(%r{.*handlers/controllers/}, "") + "_controller" # posts_controller
+    controller_class = controller_name.split('_').collect(&:capitalize).join # PostsController
+    code = "#{controller_class}.new(event, context).#{meth}" # PostsController.new(event, context).create
+
+    {path: path, code: code, class_name: controller_class}
+  end
+end