jets-fs 1.6.10

data/lib/jets/poly_fun/base_executor.rb

@@ -0,0 +1,125 @@
+require 'open3'
+require 'tmpdir'
+
+class Jets::PolyFun
+  class BaseExecutor
+    extend Memoist
+
+    def initialize(task)
+      @task = task
+    end
+
+    # Handler is in properties:
+    # 1. copy lambda function into tmp folder
+    # 2. generate Lang wrapper script
+    # 3. call wrapper script from ruby. Handle stdout and stderr and result. Pass info back to ruby
+    def run(event, context)
+      @temp_dir = create_tmpdir
+      copy_src_to_temp
+      write(code)
+      result = run_lambda_executor(event, context)
+      cleanup
+      result
+    end
+
+    def write(code)
+      puts "lambda_executor_script #{lambda_executor_script}" if ENV['KEEP_LAMBDA_WRAPPER']
+      IO.write(lambda_executor_script, code)
+    end
+
+    # Mimic Dir.mktmpdir randomness, not using Dir.mktmpdir because that generates
+    # the folder at the /tmp level only.
+    def create_tmpdir
+      random = "#{Time.now.strftime("%Y%d%H")}-#{Process.pid}-#{SecureRandom.hex[0..6]}"
+      tmpdir = "#{Jets.build_root}/executor/#{random}"
+      FileUtils.mkdir_p(tmpdir)
+      tmpdir
+    end
+
+    def copy_src_to_temp
+      app_class = @task.class_name.constantize
+      internal = app_class.respond_to?(:internal) && app_class.internal
+      src = internal ?
+        "#{File.expand_path("../internal", File.dirname(__FILE__))}/#{@task.poly_src_path}" :
+        "#{Jets.root}/#{@task.poly_src_path}"
+      dest = "#{@temp_dir}/#{@task.poly_src_path}"
+
+      FileUtils.mkdir_p(File.dirname(dest))
+      FileUtils.cp(src, dest)
+    end
+
+    def lambda_executor_script
+      File.dirname("#{@temp_dir}/#{@task.poly_src_path}") + "/lambda_executor" + @task.lang_ext
+    end
+
+    # When polymorphic method errors, this method reproduces an error in the lambda format
+    # Here's some examples to help example:
+    #
+    # Example of what the raw python prints out to stderr:
+    #
+    #   Traceback (most recent call last):
+    #     File "/tmp/jets/lite/executor/20180804-10727-mcs6qk/lambda_executor.py", line 6, in <module>
+    #       resp = handle(event, context)
+    #     File "/tmp/jets/lite/executor/20180804-10727-mcs6qk/index.py", line 22, in handle
+    #       return response({'message': e.message}, 400)
+    #     File "/tmp/jets/lite/executor/20180804-10727-mcs6qk/index.py", line 5, in response
+    #       badcode
+    #   NameError: global name 'badcode' is not defined
+    #
+    # So last line has the error summary info.  Other lines have stack trace after the Traceback indicator.
+    #
+    # Example of the reproduced lambda error format:
+    #
+    #   {
+    #     "errorMessage": "'NameError' object has no attribute 'message'",
+    #     "errorType": "AttributeError",
+    #     "stackTrace": [
+    #       [
+    #         "/var/task/handlers/controllers/posts_controller/python/index.py",
+    #         22,
+    #         "handle",
+    #         "return response({'message': e.message}, 400)"
+    #       ]
+    #     ]
+    #   }
+    #
+    def run_lambda_executor(event, context)
+      interpreter = @task.lang
+      command = %Q|#{interpreter} #{lambda_executor_script} '#{JSON.dump(event)}' '#{JSON.dump(context)}'|
+      stdout, stderr, status = Open3.capture3(command)
+      # puts "=> #{command}".color(:green)
+      # puts "stdout #{stdout}"
+      # puts "stderr #{stderr}"
+      # puts "status #{status}"
+      if status.success?
+        stdout
+      else
+        # We'll mimic the way lambda displays an error.
+        # $stderr.puts(stderr) # uncomment to debug
+        error_lines = stderr.split("\n")
+        error_message = error_lines.pop
+        error_type = error_message.split(':').first
+        error_lines.shift # remove first line that has the Traceback
+        stack_trace = error_lines.reverse # python shows stack trace in opposite order from ruby
+        JSON.dump(
+          "errorMessage" => error_message,
+          "errorType" => error_type, # hardcode
+          "stackTrace" => stack_trace
+        )
+      end
+    end
+
+    def cleanup
+      FileUtils.rm_rf(@temp_dir) unless ENV['KEEP_LAMBDA_WRAPPER']
+    end
+
+    def handler
+      # Must use the generated CloudFormation template to get the handler because
+      # the handler is derived from mutiple sources.
+      resource = Jets::Resource::Lambda::Function.new(@task)
+      full_handler = resource.properties["Handler"] # full handler here
+      File.extname(full_handler).sub(/^./,'') # the extension of the full handler is the handler
+    end
+    memoize :handler
+  end
+end

data/lib/jets/poly_fun/lambda_executor.rb

@@ -0,0 +1,16 @@
+require 'json'
+
+class Jets::PolyFun
+  class LambdaExecutor
+    def initialize(task)
+      @task = task
+    end
+
+    def run(event, context)
+      executor_class = "Jets::PolyFun::#{@task.lang.capitalize}Executor".constantize
+      executor = executor_class.new(@task)
+      text = executor.run(event, context)
+      JSON.load(text)
+    end
+  end
+end

data/lib/jets/poly_fun/node_error.rb

@@ -0,0 +1,8 @@
+class Jets::PolyFun
+  class NodeError < StandardError
+    def initialize(message, backtrace)
+      super(message)
+      set_backtrace(backtrace)
+    end
+  end
+end

data/lib/jets/poly_fun/node_executor.rb

@@ -0,0 +1,54 @@
+class Jets::PolyFun
+  class NodeExecutor < BaseExecutor
+    # Code for wrapper script that mimics lambda execution. Wrapper script usage:
+    #
+    #   node WRAPPER_SCRIPT EVENT
+    #
+    # Example:
+    #
+    #   node /tmp/jets/demo/executor/20180804-12816-imqb9/lambda_executor.js '{}'
+    #
+    def code
+      if async_syntax?
+        async_code
+      else
+        callback_code
+      end
+    end
+
+    # https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
+    def async_syntax?
+      app_path = "#{Jets.root}/" + @task.handler_path.sub('handlers/', 'app/')
+      source_code = IO.read(app_path)
+      source_code.match(/=\s*async.*\(/)
+    end
+
+    def async_code
+      <<-EOL
+var event = process.argv[2]
+event = JSON.parse(event)
+var context = {}
+
+var app = require("./#{@task.meth}.js")
+app.#{handler}(event, context).then(resp => console.log(JSON.stringify(resp)))
+EOL
+    end
+
+
+    def callback_code
+      <<-EOL
+function callback(error, response) {
+  var text = JSON.stringify(response)
+  console.log(text)
+}
+
+var event = process.argv[2]
+event = JSON.parse(event)
+var context = {}
+
+var app = require("./#{@task.meth}.js")
+var resp = app.#{handler}(event, context, callback)
+EOL
+    end
+  end
+end

data/lib/jets/poly_fun/python_error.rb

@@ -0,0 +1,8 @@
+class Jets::PolyFun
+  class PythonError < StandardError
+    def initialize(message, backtrace)
+      super(message)
+      set_backtrace(backtrace)
+    end
+  end
+end

data/lib/jets/poly_fun/python_executor.rb

@@ -0,0 +1,23 @@
+class Jets::PolyFun
+  class PythonExecutor < BaseExecutor
+    # Code for wrapper script that mimics lambda execution. Wrapper script usage:
+    #
+    #   python WRAPPER_SCRIPT EVENT
+    #
+    # Example:
+    #
+    #   python /tmp/jets/demo/executor/20180804-12816-imqb9/lambda_executor.py '{}'
+    def code
+      <<-EOL
+import sys
+import json
+from #{@task.meth} import #{handler}
+event = json.loads(sys.argv[1])
+context = {}
+resp = #{handler}(event, context)
+result = json.dumps(resp)
+print(result)
+EOL
+    end
+  end
+end

data/lib/jets/preheat.rb

@@ -0,0 +1,117 @@
+module Jets
+  class Preheat
+    extend Memoist
+
+    # Examples:
+    #
+    #   Jets::Preheat.warm("posts_controller-index")
+    #   Jets::Preheat.warm("jets-preheat_job-warm")
+    #
+    def self.warm(function_name, options={})
+      Preheat.new(options).warm(function_name)
+    end
+
+    def self.warm_all(options={})
+      Preheat.new(options).warm_all
+    end
+
+    def initialize(options)
+      @options = options # passed to Call.new options
+      @options[:mute_output] = true if @options[:mute_output].nil?
+      @options[:lambda_proxy] = false # do not transform controller event from {"event": "1"} to {"queryStringParameters":{"_prewarm":"1"}}
+    end
+
+    # Makes remote call to the Lambda function.
+    def warm(function_name)
+      Jets::Commands::Call.new(function_name, '{"_prewarm": "1"}', @options).run unless ENV['TEST']
+    end
+
+    # Loop through all methods for each class and makes special prewarm call to each method.
+    def warm_all
+      threads = []
+      all_functions.each do |function_name|
+        next if function_name.include?('jets-public_controller') # handled by warm_public_controller_more
+        next if function_name.include?('jets-rack_controller') # handled by warm_rack_controller_more
+        threads << Thread.new do
+          warm(function_name)
+        end
+      end
+      threads.each { |t| t.join }
+
+      # Warm the these controllers more since they can be hit more often
+      warm_public_controller_more
+      warm_rack_controller_more
+
+      # return the funciton names so we can see in the Lambda console
+      # the functions being prewarmed
+      all_functions
+    end
+
+    def warm_public_controller_more
+      function_name = 'jets-public_controller-show' # key function name
+      return unless all_functions.include?(function_name)
+
+      public_ratio = Jets.config.prewarm.public_ratio
+      return if public_ratio == 0
+
+      puts "Prewarming the public controller extra at a ratio of #{public_ratio}" unless @options[:mute]
+
+      threads = []
+      public_ratio.times do
+        threads << Thread.new do
+          warm(function_name)
+        end
+      end
+      threads.each { |t| t.join }
+    end
+
+    def warm_rack_controller_more
+      return unless Jets.rack?
+      function_name = 'jets-rack_controller-process' # key function name
+      return unless all_functions.include?(function_name)
+
+      rack_ratio = Jets.config.prewarm.rack_ratio
+      return if rack_ratio == 0
+
+      puts "Prewarming the rack controller extra at a ratio of #{rack_ratio}" unless @options[:mute]
+
+      threads = []
+      rack_ratio.times do
+        threads << Thread.new do
+          warm(function_name)
+        end
+      end
+      threads.each { |t| t.join }
+    end
+
+    # Returns:
+    #   [
+    #     "posts_controller-index",
+    #     "posts_controller-show",
+    #     ...
+    #   ]
+    def all_functions
+      classes.map do |klass|
+        tasks = klass.tasks.select { |t| t.lang == :ruby } # only prewarm ruby functions
+        tasks.map do |task|
+          meth = task.meth
+          underscored = klass.to_s.underscore.gsub('/','-')
+          "#{underscored}-#{meth}" # function_name
+        end
+      end.flatten.uniq.compact
+    end
+    memoize :all_functions
+
+    def classes
+      Jets::Commands::Build.app_files.map do |path|
+        next if path.include?("preheat_job.rb") # dont want to cause an infinite loop, just in case
+        next unless path =~ %r{app/controllers} # only prewarm controllers
+
+        class_path = path.sub(%r{.*app/\w+/},'').sub(/\.rb$/,'')
+        class_name = class_path.classify
+        # IE: PostsController
+        class_name.constantize # load app/**/* class definition
+      end.compact
+    end
+  end
+end

data/lib/jets/processors.rb

@@ -0,0 +1,4 @@
+module Jets::Processors
+  autoload :Deducer, 'jets/processors/deducer'
+  autoload :MainProcessor, 'jets/processors/main_processor'
+end

data/lib/jets/processors/deducer.rb

@@ -0,0 +1,65 @@
+# Jets::Processors::Deducer class figures out information that allows the
+# controller or job to be called. Sme key methods for deducer:
+#
+#   code - code to instance eval.  IE: PostsController.new(event, context).index
+#   path - full path to the app code. IE: #{Jets.root}app/controllers/posts_controller.rb
+#
+class Jets::Processors::Deducer
+  def initialize(handler)
+    @handler = handler # handlers/controllers/posts.show
+    # @handler_path: "handlers/controllers/posts"
+    # @handler_method: "show"
+    @handler_path, @handler_method = @handler.split('.')
+  end
+
+  def code
+    # code: "PostsController.process(event, context, meth: "show")"
+    # code: "HardJob.process(event, context, meth: "dig")"
+    %|#{class_name}.process(event, context, "#{@handler_method}")|
+  end
+
+  # Input: @handler_path: handlers/jobs/hard_job.rb
+  # Output: #{Jets.root/app/jobs/hard_job.rb
+  def path
+    @handler_path.sub("handlers", "app") + ".rb"
+  end
+
+  # process_type is key. It can be either "controller" or "job". It is used to
+  # deduce the rest of the methods: code, path.
+  def process_type
+    if shared?
+      "function" # all app/shared/functions are always function process_type
+    else
+      @handler.split('/')[1].singularize # controller, job, rule, etc
+    end
+  end
+
+  def shared?
+    @handler.include?("/shared/functions")
+  end
+
+  # Example underscored_class_name:
+  #   class_name = underscored_class_name
+  #   class_name = class_name # PostsController
+  def class_name
+    regexp = shared? ?
+      Regexp.new(".*handlers/shared/functions/") :
+      Regexp.new(".*handlers/#{process_type.pluralize}/")
+
+    # Example regexp:
+    #   /.*handlers\/controllers\//
+    #   /.*handlers\/jobs\//
+    class_name = @handler_path.sub(regexp, "")
+    # Example class names:
+    #   posts_controller
+    #   hard_job
+    #   hello
+    #   hello_function
+
+    class_name.classify
+  end
+
+  def load_class
+    Jets::Klass.from_path(path)
+  end
+end

data/lib/jets/processors/main_processor.rb

@@ -0,0 +1,59 @@
+require 'json'
+require 'stringio'
+
+# Node shim calls this class to process both controllers and jobs
+class Jets::Processors::MainProcessor
+  attr_reader :event, :context, :handler
+  def initialize(event, context, handler)
+    @event = event
+    @context = context
+    @handler = handler
+  end
+
+  def run
+    # Use the handler to deduce app code to run.
+    # Example handlers: handlers/controllers/posts.create, handlers/jobs/sleep.perform
+    #
+    #   deducer = Jets::Processors::Deducer.new("handlers/controllers/posts.create")
+    #
+    deducer = Jets::Processors::Deducer.new(handler)
+    begin
+      # Examples:
+      #   deducer.code => PostsController.process(event, context, "show")
+      #   deducer.path => app/controllers/posts_controller.rb
+      #
+      #   deducer.code => HardJob.process(event, context, "dig")
+      #   deducer.path => app/jobs/hard_job.rb
+      #
+      #   deducer.code => HelloFunction.process(event, context, "world")
+      #   deducer.path => app/functions/hello.rb
+      deducer.load_class
+      # result = PostsController.process(event, context, "create")
+      result = instance_eval(deducer.code, deducer.path)
+      result = HashWithIndifferentAccess.new(result) if result.is_a?(Hash)
+
+      Jets.increase_call_count
+
+      if result.is_a?(Hash) && result["headers"]
+        result["headers"]["x-jets-call-count"] = Jets.call_count
+        result["headers"]["x-jets-prewarm-count"] = Jets.prewarm_count
+      end
+
+      result
+    rescue Exception => e
+      unless ENV['TEST']
+        # Customize error message slightly so nodejs shim can process the
+        # returned error message.
+        # The "RubyError: " is a marker that the javascript shim scans for.
+        $stderr.puts("RubyError: #{e.class}: #{e.message}") # js needs this as the first line
+        backtrace = e.backtrace.map {|l| "  #{l}" }
+        $stderr.puts(backtrace)
+        # No need to having error in stderr above anymore because errors are handled in memory
+        # at ruby_server.rb but keeping around for posterity.
+      end
+
+      Jets.on_exception(e)
+      raise(e) # raise error to ruby_server.rb to rescue and handle
+    end
+  end
+end