jets 0.2.0 → 0.5.0

data/lib/jets/lambda/functions.rb

@@ -0,0 +1,22 @@
+require 'json'
+
+# Jets::Lambda::Functions represents a collection of Lambda functions.
+#
+# Jets::Lambda::Functions is the superclass of:
+#   Jets::Controller::Base
+#   Jets::Job::Base
+module Jets::Lambda
+  class Functions
+    attr_reader :event, :context, :meth
+    def initialize(event, context, meth)
+      @event = event # Hash, JSON.parse(event) ran BaseProcessor
+      @context = context # Hash. JSON.parse(context) ran in BaseProcessor
+      @meth = meth
+      # store meth because it is useful to for identifying the which template
+      # to use later.
+    end
+
+    include Dsl # At the end so methods like event, context and method
+      # do not trigger method_added
+  end
+end

data/lib/jets/lambda/task.rb

@@ -0,0 +1,77 @@
+class Jets::Lambda::Task
+  attr_accessor :class_name, :type
+  attr_reader :meth, :properties, :lang
+  def initialize(class_name, meth, options={})
+    @class_name = class_name.to_s # use at EventsRuleMapper#full_task_name
+    @meth = meth
+    @options = options
+    @type = options[:type] || get_type  # controller, job, or function
+    @properties = options[:properties] || {}
+    @lang = options[:lang] || :ruby
+  end
+
+  def name
+    @meth
+  end
+
+  @@lang_exts = {
+    ruby: '.rb',
+    python: '.py',
+    node: '.js',
+  }
+  def lang_ext
+    @@lang_exts[@lang]
+  end
+
+  # The get_type method works for controller and job classes.
+  #
+  # Usually able to get the type from the class name. Examples:
+  #
+  #   PostsController => controller
+  #   HardJob => job
+  #
+  # However, for function types, we are not able to get the type for multiple of
+  # reasons.  First, function types are allowed to be named with or without
+  # _function.  Examples:
+  #
+  #   path => class => type
+  #   app/functions/hello.rb => Hello => function
+  #   app/functions/hello_function.rb => HelloFunction => function
+  #
+  # The second reason is that functions are not regular ruby classes. Instead they
+  # are anonymous classes created with Class.new.  When classes are created with
+  # Class.new the method_added hook has "" (blank string) as the self class name.
+  # We add the class_type to the task later on as we are constructing the class
+  # as part of the Class.new logic.
+  #
+  # For controller and job standard ruby classes though it can easily be
+  # determinated as part of initialization. So we get the type for convenience then.
+  #
+  # For anonymous function classes, we just set to nil and will later fix in
+  # FunctionConstructor.
+  #
+  # Returns: "controller", "job" or nil
+  def get_type
+    unless @class_name.empty? # when anonymous class is created with Class.new
+      @class_name.underscore.split('_').last # controller, job or rule
+    end
+  end
+
+  def full_handler(handler_function)
+    "#{handler_base}.#{handler_function}"
+  end
+
+  def handler_path
+    "#{handler_base}#{lang_ext}"
+  end
+
+  def handler_base
+    base = "handlers/#{@type.pluralize}/#{@class_name.underscore}"
+    base += "/#{@lang}" if @lang != :ruby
+    base += "/#{@meth}"
+  end
+
+  def poly_src_path
+    handler_path.sub("handlers/", "app/")
+  end
+end

data/lib/jets/naming.rb

@@ -0,0 +1,61 @@
+# This class groups the naming in one place.
+# Some naming is for CloudFormation
+# Some are for the Build process
+class Jets::Naming
+  # Mainly used by build.rb
+  class << self
+    def template_path(app_class)
+      underscored = app_class.to_s.underscore.gsub('/','-')
+      "#{template_path_prefix}-#{underscored}.yml"
+    end
+
+    def template_path_prefix
+      "#{Jets.build_root}/templates/#{Jets.config.project_namespace}"
+    end
+
+    # consider moving these methods into cfn/builder/helpers.rb or that area.
+    def parent_template_path
+      "#{template_path_prefix}.yml"
+    end
+
+    # consider moving these methods into cfn/builder/helpers.rb or that area.
+    def api_gateway_template_path
+      "#{template_path_prefix}-api-gateway.yml"
+    end
+
+    def api_gateway_deployment_template_path
+      "#{template_path_prefix}-api-gateway-deployment.yml"
+    end
+
+    def parent_stack_name
+      File.basename(parent_template_path, ".yml")
+    end
+
+    def gateway_api_name
+      "#{Jets.config.project_namespace}"
+    end
+
+    def code_s3_key
+      md5_zipfile = File.basename(md5_code_zipfile)
+      "jets/code/#{md5_zipfile}"
+    end
+
+    # build was already ran and that a file that contains the md5 path exists
+    # at Jets.build_root/code/current-md5-filename.txt
+    #
+    # md5_code_zipfile: /tmp/jets/demo/code/code-2e0e18f6.zip
+    def md5_code_zipfile
+      path = "#{Jets.build_root}/code/current-md5-filename.txt"
+      File.exist?(path) ? IO.read(path) : "current-md5-filename-doesnt-exist"
+    end
+    # The current-md5-filename.txt gets created as a part of CodeBuilder's build
+    # process.
+    # And is required to be used much later for cfn/ship and base_child_builder
+    # They need set an s3_key which requires the md5_zip_dest.
+    # It is a pain to pass this all the way up from the
+    # CodeBuilder class.
+    # We store the "/tmp/jets/demo/code/code-a8a604aa.zip" into a
+    # file that can be read from any places where this is needed.
+    # Can also just generate a "fake file" for specs
+  end
+end

data/lib/jets/pascalize.rb

@@ -0,0 +1,30 @@
+class Pascalize
+  class << self
+    # Specialized pascalize that will not pascalize keys under the
+    # Variables part of the hash structure.
+    # Based on: https://stackoverflow.com/questions/8706930/converting-nested-hash-keys-from-camelcase-to-snake-case-in-ruby
+    def pascalize(value, parent_key=nil)
+      case value
+        when Array
+          value.map { |v| pascalize(v) }
+        when Hash
+          initializer = value.map do |k, v|
+            new_key = pascal_key(k, parent_key)
+            [new_key, pascalize(v, new_key)]
+          end
+          Hash[initializer]
+        else
+          value
+       end
+    end
+
+    def pascal_key(k, parent_key=nil)
+      if parent_key == "Variables" # do not pascalize keys anything under Variables
+        k
+      else
+        k = k.to_s.camelize
+        k.slice(0,1).capitalize + k.slice(1..-1) # capitalize first letter only
+      end
+    end
+  end
+end

data/lib/jets/poly_fun.rb

@@ -0,0 +1,61 @@
+module Jets
+  class PolyFun
+    autoload :LambdaExecutor, 'jets/poly_fun/lambda_executor' # main class delegates to other classes
+
+    autoload :BaseExecutor, 'jets/poly_fun/base_executor'
+    autoload :PythonExecutor, 'jets/poly_fun/python_executor'
+    autoload :NodeExecutor, 'jets/poly_fun/node_executor'
+
+    autoload :PythonError, 'jets/poly_fun/python_error'
+    autoload :NodeError, 'jets/poly_fun/node_error'
+
+    extend Memoist
+
+    def initialize(app_class, app_meth)
+      @app_class = app_class # already a Constant, IE: PostController
+      @app_meth = app_meth.to_sym
+    end
+
+    def run(event, context={})
+      if task.lang == :ruby
+        # controller = PostsController.new(event, content)
+        # resp = controller.edit
+        @app_class.process(event, context, @app_meth)
+      else
+        executor = LambdaExecutor.new(task)
+        resp = executor.run(event, context)
+        if resp["errorMessage"]
+          raise_error(resp)
+        end
+        resp
+      end
+    end
+
+    def raise_error(resp)
+      backtrace = resp["stackTrace"] + caller
+      backtrace = backtrace.map { |l| l.sub(/^\s+/,'') }
+      # Adjust the paths from the tmp path to the app path to improve user debugging
+      # experience. Example:
+      # From:
+      #   File "/tmp/jets/lite/executor/20180917-16777-43a9e48/app/controllers/jets/public_controller/python/show.py", line 32
+      # To:
+      #   File "app/controllers/jets/public_controller/python/show.py", line 32      backtrace =
+      backtrace = backtrace.map do |l|
+        if l.include?(Jets.build_root) && !l.include?("lambda_executor.")
+          l.sub(/\/tmp\/jets.*executor\/\d{8}-+.*?\//, '')
+        else
+          l
+        end
+      end
+
+      # IE: Jets::PolyFun::PythonError
+      error_class = "Jets::PolyFun::#{task.lang.to_s.classify}Error".constantize
+      raise error_class.new(resp["errorMessage"], backtrace)
+    end
+
+    def task
+      @app_class.all_tasks[@app_meth]
+    end
+    memoize :task
+  end
+end

data/lib/jets/poly_fun/base_executor.rb

@@ -0,0 +1,129 @@
+require 'open3'
+require 'tmpdir'
+
+class Jets::PolyFun
+  class BaseExecutor
+    extend Memoist
+
+    def initialize(task)
+      @task = task
+    end
+
+    # Handler is in properties:
+    # 1. copy lambda function into tmp folder
+    # 2. generate Lang wrapper script
+    # 3. call wrapper script from ruby. Handle stdout and stderr and result. Pass info back to ruby
+    def run(event, context)
+      @temp_dir = create_tmpdir
+      copy_src_to_temp
+      write(code)
+      result = run_lambda_executor(event, context)
+      cleanup
+      result
+    end
+
+    def write(code)
+      puts "lambda_executor_script #{lambda_executor_script}" if ENV['KEEP_LAMBDA_WRAPPER']
+      IO.write(lambda_executor_script, code)
+    end
+
+    # Mimic Dir.mktmpdir randomness, not using Dir.mktmpdir because that generates
+    # the folder at the /tmp level only.
+    def create_tmpdir
+      random = "#{Time.now.strftime("%Y%d%H")}-#{Process.pid}-#{SecureRandom.hex[0..6]}"
+      tmpdir = "#{Jets.build_root}/executor/#{random}"
+      FileUtils.mkdir_p(tmpdir)
+      tmpdir
+    end
+
+    def copy_src_to_temp
+      app_class = @task.class_name.constantize
+      internal = app_class.respond_to?(:internal) && app_class.internal
+      src = internal ?
+        "#{File.expand_path("../internal", File.dirname(__FILE__))}/#{@task.poly_src_path}" :
+        "#{Jets.root}#{@task.poly_src_path}"
+      dest = "#{@temp_dir}/#{@task.poly_src_path}"
+
+      FileUtils.mkdir_p(File.dirname(dest))
+      FileUtils.cp(src, dest)
+    end
+
+    def lambda_executor_script
+      File.dirname("#{@temp_dir}/#{@task.poly_src_path}") + "/lambda_executor" + @task.lang_ext
+    end
+
+    # When polymorphic method errors, this method reproduces an error in the lambda format
+    # Here's some examples to help example:
+    #
+    # Example of what the raw python prints out to stderr:
+    #
+    #   Traceback (most recent call last):
+    #     File "/tmp/jets/lite/executor/20180804-10727-mcs6qk/lambda_executor.py", line 6, in <module>
+    #       resp = handle(event, context)
+    #     File "/tmp/jets/lite/executor/20180804-10727-mcs6qk/index.py", line 22, in handle
+    #       return response({'message': e.message}, 400)
+    #     File "/tmp/jets/lite/executor/20180804-10727-mcs6qk/index.py", line 5, in response
+    #       badcode
+    #   NameError: global name 'badcode' is not defined
+    #
+    # So last line has the error summary info.  Other lines have stack trace after the Traceback indicator.
+    #
+    # Example of the reproduced lambda error format:
+    #
+    #   {
+    #     "errorMessage": "'NameError' object has no attribute 'message'",
+    #     "errorType": "AttributeError",
+    #     "stackTrace": [
+    #       [
+    #         "/var/task/handlers/controllers/posts_controller/python/index.py",
+    #         22,
+    #         "handle",
+    #         "return response({'message': e.message}, 400)"
+    #       ]
+    #     ]
+    #   }
+    #
+    def run_lambda_executor(event, context)
+      interpreter = @task.lang
+      command = %Q|#{interpreter} #{lambda_executor_script} '#{JSON.dump(event)}' '#{JSON.dump(context)}'|
+      stdout, stderr, status = Open3.capture3(command)
+      # puts "=> #{command}".colorize(:green)
+      # puts "stdout #{stdout}"
+      # puts "stderr #{stderr}"
+      # puts "status #{status}"
+      if status.success?
+        stdout
+      else
+        # We'll mimic the way lambda displays an error.
+        # $stderr.puts(stderr) # uncomment to debug
+        error_lines = stderr.split("\n")
+        error_message = error_lines.pop
+        error_type = error_message.split(':').first
+        error_lines.shift # remove first line that has the Traceback
+        stack_trace = error_lines.reverse # python shows stack trace in opposite order from ruby
+        JSON.dump(
+          "errorMessage" => error_message,
+          "errorType" => error_type, # hardcode
+          "stackTrace" => stack_trace
+        )
+      end
+    end
+
+    def cleanup
+      FileUtils.rm_rf(@temp_dir) unless ENV['KEEP_LAMBDA_WRAPPER']
+    end
+
+    def handler
+      # Must use FunctionProperties to get the handler because that the class that combines
+      # the mutiple sources of how the handler can get set.
+      # puts "handler path #{@task.handler_path}"
+      #
+      # IE: Jets::Cfn::TemplateBuilders::FunctionProperties::PythonBuilder
+      builder_class = "Jets::Cfn::TemplateBuilders::FunctionProperties::#{@task.lang.to_s.classify}Builder".constantize
+      builder = builder_class.new(@task)
+      full_handler = builder.properties["Handler"] # full handler here
+      File.extname(full_handler).sub(/^./,'') # the extension of the full handler is the handler
+    end
+    memoize :handler
+  end
+end

data/lib/jets/poly_fun/lambda_executor.rb

@@ -0,0 +1,16 @@
+require 'json'
+
+class Jets::PolyFun
+  class LambdaExecutor
+    def initialize(task)
+      @task = task
+    end
+
+    def run(event, context)
+      executor_class = "Jets::PolyFun::#{@task.lang.capitalize}Executor".constantize
+      executor = executor_class.new(@task)
+      text = executor.run(event, context)
+      JSON.load(text)
+    end
+  end
+end

data/lib/jets/poly_fun/node_error.rb

@@ -0,0 +1,8 @@
+class Jets::PolyFun
+  class NodeError < StandardError
+    def initialize(message, backtrace)
+      super(message)
+      set_backtrace(backtrace)
+    end
+  end
+end

data/lib/jets/poly_fun/node_executor.rb

@@ -0,0 +1,54 @@
+class Jets::PolyFun
+  class NodeExecutor < BaseExecutor
+    # Code for wrapper script that mimics lambda execution. Wrapper script usage:
+    #
+    #   node WRAPPER_SCRIPT EVENT
+    #
+    # Example:
+    #
+    #   node /tmp/jets/demo/executor/20180804-12816-imqb9/lambda_executor.js '{}'
+    #
+    def code
+      if async_syntax?
+        async_code
+      else
+        callback_code
+      end
+    end
+
+    # https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
+    def async_syntax?
+      app_path = Jets.root + @task.handler_path.sub('handlers/', 'app/')
+      source_code = IO.read(app_path)
+      source_code.match(/=\s*async.*\(/)
+    end
+
+    def async_code
+      <<-EOL
+var event = process.argv[2]
+event = JSON.parse(event)
+var context = {}
+
+var app = require("./#{@task.meth}.js")
+app.#{handler}(event, context).then(resp => console.log(JSON.stringify(resp)))
+EOL
+    end
+
+
+    def callback_code
+      <<-EOL
+function callback(error, response) {
+  var text = JSON.stringify(response)
+  console.log(text)
+}
+
+var event = process.argv[2]
+event = JSON.parse(event)
+var context = {}
+
+var app = require("./#{@task.meth}.js")
+var resp = app.#{handler}(event, context, callback)
+EOL
+    end
+  end
+end