jets 3.1.0 → 5.0.0

data/lib/jets/paths.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Jets
+  module Paths
+    # This object is an extended hash that behaves as root of the <tt>Jets::Paths</tt> system.
+    # It allows you to collect information about how you want to structure your application
+    # paths through a Hash-like API. It requires you to give a physical path on initialization.
+    #
+    #   root = Root.new "/jets"
+    #   root.add "app/controllers", eager_load: true
+    #
+    # The above command creates a new root object and adds "app/controllers" as a path.
+    # This means we can get a Jets::Paths::Path object back like below:
+    #
+    #   path = root["app/controllers"]
+    #   path.eager_load?               # => true
+    #   path.is_a?(Jets::Paths::Path) # => true
+    #
+    # The +Path+ object is simply an enumerable and allows you to easily add extra paths:
+    #
+    #   path.is_a?(Enumerable) # => true
+    #   path.to_ary.inspect    # => ["app/controllers"]
+    #
+    #   path << "lib/controllers"
+    #   path.to_ary.inspect    # => ["app/controllers", "lib/controllers"]
+    #
+    # Notice that when you add a path using +add+, the path object created already
+    # contains the path with the same path value given to +add+. In some situations,
+    # you may not want this behavior, so you can give <tt>:with</tt> as option.
+    #
+    #   root.add "config/routes", with: "config/routes.rb"
+    #   root["config/routes"].inspect # => ["config/routes.rb"]
+    #
+    # The +add+ method accepts the following options as arguments:
+    # eager_load, autoload, autoload_once, and glob.
+    #
+    # Finally, the +Path+ object also provides a few helpers:
+    #
+    #   root = Root.new "/jets"
+    #   root.add "app/controllers"
+    #
+    #   root["app/controllers"].expanded # => ["/jets/app/controllers"]
+    #   root["app/controllers"].existent # => ["/jets/app/controllers"]
+    #
+    # Check the <tt>Jets::Paths::Path</tt> documentation for more information.
+    class Root
+      attr_accessor :path
+
+      def initialize(path)
+        @path = path
+        @root = {}
+      end
+
+      def []=(path, value)
+        glob = self[path] ? self[path].glob : nil
+        add(path, with: value, glob: glob)
+      end
+
+      def add(path, options = {})
+        with = Array(options.fetch(:with, path))
+        @root[path] = Path.new(self, path, with, options)
+      end
+
+      def [](path)
+        @root[path]
+      end
+
+      def values
+        @root.values
+      end
+
+      def keys
+        @root.keys
+      end
+
+      def values_at(*list)
+        @root.values_at(*list)
+      end
+
+      def all_paths
+        values.tap(&:uniq!)
+      end
+
+      def autoload_once
+        filter_by(&:autoload_once?)
+      end
+
+      def eager_load
+        filter_by(&:eager_load?)
+      end
+
+      def autoload_paths
+        filter_by(&:autoload?)
+      end
+
+      def load_paths
+        filter_by(&:load_path?)
+      end
+
+    private
+      def filter_by(&block)
+        all_paths.find_all(&block).flat_map { |path|
+          paths = path.existent
+          paths - path.children.flat_map { |p| yield(p) ? [] : p.existent }
+        }.uniq
+      end
+    end
+
+    class Path
+      include Enumerable
+
+      attr_accessor :glob
+
+      def initialize(root, current, paths, options = {})
+        @paths   = paths
+        @current = current
+        @root    = root
+        @glob    = options[:glob]
+        @exclude = options[:exclude]
+
+        options[:autoload_once] ? autoload_once! : skip_autoload_once!
+        options[:eager_load]    ? eager_load!    : skip_eager_load!
+        options[:autoload]      ? autoload!      : skip_autoload!
+        options[:load_path]     ? load_path!     : skip_load_path!
+      end
+
+      def absolute_current # :nodoc:
+        File.expand_path(@current, @root.path)
+      end
+
+      def children
+        keys = @root.keys.find_all { |k|
+          k.start_with?(@current) && k != @current
+        }
+        @root.values_at(*keys.sort)
+      end
+
+      def first
+        expanded.first
+      end
+
+      def last
+        expanded.last
+      end
+
+      %w(autoload_once eager_load autoload load_path).each do |m|
+        class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          def #{m}!        # def eager_load!
+            @#{m} = true   #   @eager_load = true
+          end              # end
+                           #
+          def skip_#{m}!   # def skip_eager_load!
+            @#{m} = false  #   @eager_load = false
+          end              # end
+                           #
+          def #{m}?        # def eager_load?
+            @#{m}          #   @eager_load
+          end              # end
+        RUBY
+      end
+
+      def each(&block)
+        @paths.each(&block)
+      end
+
+      def <<(path)
+        @paths << path
+      end
+      alias :push :<<
+
+      def concat(paths)
+        @paths.concat paths
+      end
+
+      def unshift(*paths)
+        @paths.unshift(*paths)
+      end
+
+      def to_ary
+        @paths
+      end
+
+      def paths
+        raise "You need to set a path root" unless @root.path
+
+        map do |p|
+          Pathname.new(@root.path).join(p)
+        end
+      end
+
+      def extensions # :nodoc:
+        $1.split(",") if @glob =~ /\{([\S]+)\}/
+      end
+
+      # Expands all paths against the root and return all unique values.
+      def expanded
+        raise "You need to set a path root" unless @root.path
+        result = []
+
+        each do |path|
+          path = File.expand_path(path, @root.path)
+
+          if @glob && File.directory?(path)
+            result.concat files_in(path)
+          else
+            result << path
+          end
+        end
+
+        result.uniq!
+        result
+      end
+
+      # Returns all expanded paths but only if they exist in the filesystem.
+      def existent
+        expanded.select do |f|
+          does_exist = File.exist?(f)
+
+          if !does_exist && File.symlink?(f)
+            raise "File #{f.inspect} is a symlink that does not point to a valid file"
+          end
+          does_exist
+        end
+      end
+
+      def existent_directories
+        expanded.select { |d| File.directory?(d) }
+      end
+
+      alias to_a expanded
+
+      private
+        def files_in(path)
+          files = Dir.glob(@glob, base: path)
+          files -= @exclude if @exclude
+          files.map! { |file| File.join(path, file) }
+          files.sort
+        end
+    end
+  end
+end

data/lib/jets/{poly_fun → poly}/base_executor.rb

@@ -1,12 +1,12 @@
 require 'open3'
 require 'tmpdir'
 
-class Jets::PolyFun
+class Jets::Poly
   class BaseExecutor
     extend Memoist
 
-    def initialize(task)
-      @task = task
+    def initialize(definition)
+      @definition = definition
     end
 
     # Handler is in properties:
@@ -37,19 +37,19 @@ class Jets::PolyFun
     end
 
     def copy_src_to_temp
-      app_class = @task.class_name.constantize
+      app_class = @definition.class_name.constantize
       internal = app_class.respond_to?(:internal) && app_class.internal
       src = internal ?
-        "#{File.expand_path("../internal", File.dirname(__FILE__))}/#{@task.poly_src_path}" :
-        "#{Jets.root}/#{@task.poly_src_path}"
-      dest = "#{@temp_dir}/#{@task.poly_src_path}"
+        "#{File.expand_path("../internal", File.dirname(__FILE__))}/#{@definition.poly_src_path}" :
+        "#{Jets.root}/#{@definition.poly_src_path}"
+      dest = "#{@temp_dir}/#{@definition.poly_src_path}"
 
       FileUtils.mkdir_p(File.dirname(dest))
       FileUtils.cp(src, dest)
     end
 
     def lambda_executor_script
-      File.dirname("#{@temp_dir}/#{@task.poly_src_path}") + "/lambda_executor" + @task.lang_ext
+      File.dirname("#{@temp_dir}/#{@definition.poly_src_path}") + "/lambda_executor" + @definition.lang_ext
     end
 
     # When polymorphic method errors, this method reproduces an error in the lambda format
@@ -75,7 +75,7 @@ class Jets::PolyFun
     #     "errorType": "AttributeError",
     #     "stackTrace": [
     #       [
-    #         "/var/task/handlers/controllers/posts_controller/python/index.py",
+    #         "/var/definition/handlers/controllers/posts_controller/python/index.py",
     #         22,
     #         "handle",
     #         "return response({'message': e.message}, 400)"
@@ -84,7 +84,7 @@ class Jets::PolyFun
     #   }
     #
     def run_lambda_executor(event, context)
-      interpreter = @task.lang
+      interpreter = @definition.lang
       command = %Q|#{interpreter} #{lambda_executor_script} '#{JSON.dump(event)}' '#{JSON.dump(context)}'|
       stdout, stderr, status = Open3.capture3(command)
       # puts "=> #{command}".color(:green)
@@ -103,7 +103,7 @@ class Jets::PolyFun
         stack_trace = error_lines.reverse # python shows stack trace in opposite order from ruby
         JSON.dump(
           "errorMessage" => error_message,
-          "errorType" => error_type, # hardcode
+          "errorType" => error_type,
           "stackTrace" => stack_trace
         )
       end
@@ -116,8 +116,8 @@ class Jets::PolyFun
     def handler
       # Must use the generated CloudFormation template to get the handler because
       # the handler is derived from mutiple sources.
-      resource = Jets::Resource::Lambda::Function.new(@task)
-      full_handler = resource.properties["Handler"] # full handler here
+      resource = Jets::Cfn::Resource::Lambda::Function.new(@definition)
+      full_handler = resource.properties[:Handler] # full handler here
       File.extname(full_handler).sub(/^./,'') # the extension of the full handler is the handler
     end
     memoize :handler

data/lib/jets/{poly_fun → poly}/lambda_executor.rb

@@ -1,13 +1,13 @@
 require 'json'
 
-class Jets::PolyFun
+class Jets::Poly
   class LambdaExecutor
     def initialize(task)
       @task = task
     end
 
     def run(event, context)
-      executor_class = "Jets::PolyFun::#{@task.lang.capitalize}Executor".constantize
+      executor_class = "Jets::Poly::#{@task.lang.capitalize}Executor".constantize
       executor = executor_class.new(@task)
       text = executor.run(event, context)
       JSON.load(text)

data/lib/jets/{poly_fun → poly}/node_error.rb

@@ -1,4 +1,4 @@
-class Jets::PolyFun
+class Jets::Poly
   class NodeError < StandardError
     def initialize(message, backtrace)
       super(message)

data/lib/jets/{poly_fun → poly}/node_executor.rb

@@ -1,4 +1,4 @@
-class Jets::PolyFun
+class Jets::Poly
   class NodeExecutor < BaseExecutor
     # Code for wrapper script that mimics lambda execution. Wrapper script usage:
     #
@@ -18,7 +18,7 @@ class Jets::PolyFun
 
     # https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
     def async_syntax?
-      app_path = "#{Jets.root}/" + @task.handler_path.sub('handlers/', 'app/')
+      app_path = "#{Jets.root}/" + @definition.handler_path.sub('handlers/', 'app/')
       source_code = IO.read(app_path)
       source_code.match(/=\s*async.*\(/)
     end
@@ -29,7 +29,7 @@ var event = process.argv[2]
 event = JSON.parse(event)
 var context = {}
 
-var app = require("./#{@task.meth}.js")
+var app = require("./#{@definition.meth}.js")
 app.#{handler}(event, context).then(resp => console.log(JSON.stringify(resp)))
 EOL
     end
@@ -46,7 +46,7 @@ var event = process.argv[2]
 event = JSON.parse(event)
 var context = {}
 
-var app = require("./#{@task.meth}.js")
+var app = require("./#{@definition.meth}.js")
 var resp = app.#{handler}(event, context, callback)
 EOL
     end

data/lib/jets/{poly_fun → poly}/python_error.rb

@@ -1,4 +1,4 @@
-class Jets::PolyFun
+class Jets::Poly
   class PythonError < StandardError
     def initialize(message, backtrace)
       super(message)

data/lib/jets/{poly_fun → poly}/python_executor.rb

@@ -1,4 +1,4 @@
-class Jets::PolyFun
+class Jets::Poly
   class PythonExecutor < BaseExecutor
     # Code for wrapper script that mimics lambda execution. Wrapper script usage:
     #
@@ -11,7 +11,7 @@ class Jets::PolyFun
       <<-EOL
 import sys
 import json
-from #{@task.meth} import #{handler}
+from #{@definition.meth} import #{handler}
 event = json.loads(sys.argv[1])
 context = {}
 resp = #{handler}(event, context)

data/lib/jets/{poly_fun.rb → poly.rb}

@@ -1,5 +1,5 @@
 module Jets
-  class PolyFun
+  class Poly
     extend Memoist
 
     def initialize(app_class, app_meth)
@@ -10,12 +10,12 @@ module Jets
     def run(event, context={})
       check_private_method!
 
-      if task.lang == :ruby
+      if definition.lang == :ruby
         # controller = PostsController.new(event, content)
         # resp = controller.edit
         run_ruby_code(event, context)
       else
-        executor = LambdaExecutor.new(task)
+        executor = LambdaExecutor.new(definition)
         resp = executor.run(event, context)
         if resp["errorMessage"]
           raise_error(resp)
@@ -27,7 +27,7 @@ module Jets
     def run_ruby_code(event, context)
       @app_class.process(event, context, @app_meth)
     rescue Exception => e
-      Jets.on_exception(e)
+      Jets.report_exception(e)
       raise(e)
     end
 
@@ -48,23 +48,23 @@ module Jets
         end
       end
 
-      # IE: Jets::PolyFun::PythonError
-      error_class = "Jets::PolyFun::#{task.lang.to_s.camelize}Error".constantize
+      # IE: Jets::Poly::PythonError
+      error_class = "Jets::Poly::#{definition.lang.to_s.camelize}Error".constantize
       raise error_class.new(resp["errorMessage"], backtrace)
     end
 
-    def task
-      task = @app_class.all_public_tasks[@app_meth]
+    def definition
+      definition = @app_class.all_public_definitions[@app_meth]
       # Provider user a better error message to user than a nil failure.
-      unless task
+      unless definition
         raise "Unable to find #{@app_class}##{@app_meth}"
       end
-      task
+      definition
     end
-    memoize :task
+    memoize :definition
 
     def check_private_method!
-      private_detected = @app_class.all_private_tasks.keys.include?(@app_meth)
+      private_detected = @app_class.all_private_definitions.keys.include?(@app_meth)
       return unless private_detected # Ok to continue
 
       raise "The #{@app_class}##{@app_meth} is a private method.  Unable to call it unless it is public"

data/lib/jets/preheat.rb

@@ -1,6 +1,7 @@
 module Jets
   class Preheat
     extend Memoist
+    include Jets::AwsServices
 
     # Examples:
     #
@@ -23,7 +24,7 @@ module Jets
 
     # Makes remote call to the Lambda function.
     def warm(function_name)
-      Jets::Commands::Call.new(function_name, '{"_prewarm": "1"}', @options).run unless Jets.env.test?
+      Jets::Commands::Call::Caller.new(function_name, '{"_prewarm": "1"}', @options).run unless Jets.env.test?
     end
 
     # Loop through all methods for each class and makes special prewarm call to each method.
@@ -31,7 +32,6 @@ module Jets
       threads = []
       all_functions.each do |function_name|
         next if function_name.include?('jets-public_controller') # handled by warm_public_controller_more
-        next if function_name.include?('jets-rack_controller') # handled by warm_rack_controller_more
         threads << Thread.new do
           warm(function_name)
         end
@@ -40,7 +40,6 @@ module Jets
 
       # Warm the these controllers more since they can be hit more often
       warm_public_controller_more
-      warm_rack_controller_more
 
       # return the funciton names so we can see in the Lambda console
       # the functions being prewarmed
@@ -65,45 +64,39 @@ module Jets
       threads.each { |t| t.join }
     end
 
-    def warm_rack_controller_more
-      return unless Jets.rack?
-      function_name = 'jets-rack_controller-process' # key function name
-      return unless all_functions.include?(function_name)
-
-      rack_ratio = Jets.config.prewarm.rack_ratio
-      return if rack_ratio == 0
-
-      puts "Prewarming the rack controller extra at a ratio of #{rack_ratio}" unless @options[:mute]
-
-      threads = []
-      rack_ratio.times do
-        threads << Thread.new do
-          warm(function_name)
-        end
-      end
-      threads.each { |t| t.join }
-    end
-
     # Returns:
     #   [
-    #     "posts_controller-index",
-    #     "posts_controller-show",
+    #     "demo-posts_controller-index",
+    #     "demo-posts_controller-show",
+    #     ...
+    #   ]
+    #
+    #   or (for one lambda per controller)
+    #
+    #   [
+    #     "demo-posts_controller",
+    #     "demo-up_controller",
     #     ...
     #   ]
     def all_functions
-      classes.map do |klass|
-        tasks = klass.tasks.select { |t| t.lang == :ruby } # only prewarm ruby functions
-        tasks.map do |task|
-          meth = task.meth
-          underscored = klass.to_s.underscore.gsub('/','-')
-          "#{underscored}-#{meth}" # function_name
+      parent_stack = cfn.describe_stack_resources(stack_name: Jets::Names.parent_stack_name)
+      parent_resources = parent_stack.stack_resources.select do |resource|
+        resource.logical_resource_id =~ /Controller$/ # only controller functions
+      end
+      physical_resource_ids = parent_resources.map(&:physical_resource_id)
+      resources = physical_resource_ids.inject([]) do |acc, physical_resource_id|
+        stack_resources = cfn.describe_stack_resources(stack_name: physical_resource_id).stack_resources
+        stack_resources.each do |stack_resource|
+          acc << stack_resource if stack_resource.logical_resource_id.ends_with?('LambdaFunction') # only functions
         end
-      end.flatten.uniq.compact
+        acc
+      end
+      resources.map(&:physical_resource_id) # function names
     end
     memoize :all_functions
 
     def classes
-      Jets::Commands::Build.app_files.map do |path|
+      Jets::Cfn::Builder.app_files.map do |path|
         next if path.include?("preheat_job.rb") # dont want to cause an infinite loop, just in case
         next unless path =~ %r{app/controllers} # only prewarm controllers
 
@@ -114,4 +107,4 @@ module Jets
       end.compact
     end
   end
-end
+end

data/lib/jets/processors/deducer.rb

@@ -13,8 +13,8 @@ class Jets::Processors::Deducer
   end
 
   def code
-    # code: "PostsController.process(event, context, meth: "show")"
-    # code: "HardJob.process(event, context, meth: "dig")"
+    # code: "PostsController.process(event, context, "show")"
+    # code: "HardJob.process(event, context, "dig")"
     %|#{class_name}.process(event, context, "#{@handler_method}")|
   end
 

data/lib/jets/processors/main_processor.rb

@@ -42,20 +42,13 @@ class Jets::Processors::MainProcessor
       end
 
       result
-    rescue Exception => e
-      unless Jets.env.test?
-        # Customize error message slightly so nodejs shim can process the
-        # returned error message.
-        # The "RubyError: " is a marker that the javascript shim scans for.
-        $stderr.puts("RubyError: #{e.class}: #{e.message}") # js needs this as the first line
-        backtrace = e.backtrace.map {|l| "  #{l}" }
-        $stderr.puts(backtrace)
-        # No need to having error in stderr above anymore because errors are handled in memory
-        # at ruby_server.rb but keeping around for posterity.
-      end
 
-      Jets.on_exception(e)
-      raise(e) # raise error to ruby_server.rb to rescue and handle
+    # Additional rescue Exception as a paranoid measure.  We want to make sure
+    # that we always report the exception.  This is the last line of defense.
+    # Note: This only happens when the code is running in Lambda.
+    rescue => exception
+      Jets.report_exception(exception)
+      raise(exception)
     end
   end
 end

data/lib/jets/rack/logger.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/time/conversions"
+require "active_support/core_ext/object/blank"
+require "active_support/log_subscriber"
+require "rack/body_proxy"
+
+module Jets
+  module Rack
+    # Sets log tags, logs the request, calls the app, and flushes the logs.
+    #
+    # Log tags (+taggers+) can be an Array containing: methods that the +request+
+    # object responds to, objects that respond to +to_s+ or Proc objects that accept
+    # an instance of the +request+ object.
+    class Logger < ActiveSupport::LogSubscriber
+      def initialize(app, taggers = nil)
+        @app          = app
+        @taggers      = taggers || []
+      end
+
+      def call(env)
+        request = ActionDispatch::Request.new(env)
+
+        if logger.respond_to?(:tagged)
+          logger.tagged(compute_tags(request)) { call_app(request, env) }
+        else
+          call_app(request, env)
+        end
+      end
+
+      private
+        def call_app(request, env) # :doc:
+          instrumenter = ActiveSupport::Notifications.instrumenter
+          instrumenter_state = instrumenter.start "request.action_dispatch", request: request
+          instrumenter_finish = -> () {
+            instrumenter.finish_with_state(instrumenter_state, "request.action_dispatch", request: request)
+          }
+
+          logger.info { started_request_message(request) }
+          status, headers, body = @app.call(env)
+          body = ::Rack::BodyProxy.new(body, &instrumenter_finish)
+          [status, headers, body]
+        rescue Exception
+          instrumenter_finish.call
+          raise
+        ensure
+          ActiveSupport::LogSubscriber.flush_all!
+        end
+
+        # Started GET "/session/new" for 127.0.0.1 at 2012-09-26 14:51:42 -0700
+        def started_request_message(request) # :doc:
+          'Started %s "%s" for %s at %s' % [
+            request.raw_request_method,
+            request.filtered_path,
+            request.remote_ip,
+            Time.now.to_default_s ]
+        end
+
+        def compute_tags(request) # :doc:
+          @taggers.collect do |tag|
+            case tag
+            when Proc
+              tag.call(request)
+            when Symbol
+              request.send(tag)
+            else
+              tag
+            end
+          end
+        end
+
+        def logger
+          Jets.logger
+        end
+    end
+  end
+end