jets 5.0.14 → 6.0.0.beta1

data/lib/jets/lambda/function_constructor.rb

@@ -43,7 +43,7 @@ module Jets::Lambda
     # For anonymous classes method_added during definition registration contains ""
     # for the class name.  We adjust it here.
     def adjust_definitions(klass)
-      class_name = @code_path.to_s.sub(/.*\/functions\//,'').sub(/\.rb$/, '')
+      class_name = @code_path.to_s.sub(/.*\/functions\//, "").sub(/\.rb$/, "")
       class_name = class_name.camelize
       klass.definitions.each do |definition|
         definition.class_name = class_name
@@ -52,4 +52,3 @@ module Jets::Lambda
     end
   end
 end
-

data/lib/jets/lambda/functions.rb

@@ -1,29 +1,23 @@
-require 'json'
+require "json"
 
 # Jets::Lambda::Functions represents a collection of Lambda functions.
 #
 # Jets::Lambda::Functions is the superclass of:
-#   Jets::Controller::Base
-#   Jets::Job::Base
+#   Jets::Event::Base
 module Jets::Lambda
   class Functions
     include Jets::ExceptionReporting
+    include Jets::Util::Logging
 
     attr_reader :event, :context, :meth
     def initialize(event, context, meth)
       @event = HashWithIndifferentAccess.new(event) # Hash, JSON.parse(event) ran BaseProcessor
       @context = context # Hash. JSON.parse(context) ran in BaseProcessor
-      @meth = meth
-      # store meth because it is useful to for identifying the which template
-      # to use later.
-    end
-
-    def logger
-      Jets.logger
+      @meth = meth # useful to identify which template to use later.
     end
 
     include Dsl # At the end so methods like event, context and method
-      # do not trigger method_added
+    # do not trigger method_added
 
     # Pretty hacky since action_view/rendering.rb _normalize_options calls super
     def _normalize_options(options) # :doc:
@@ -31,19 +25,15 @@ module Jets::Lambda
     end
 
     class << self
+      include Jets::Util::Logging
+
       attr_reader :abstract
       alias_method :abstract?, :abstract
       @abstract = true
 
-      class << self
-        def inherited(klass) # :nodoc:
-          # Define the abstract ivar on subclasses so that we don't get
-          # uninitialized ivar warnings
-          unless klass.instance_variable_defined?(:@abstract)
-            klass.instance_variable_set(:@abstract, false)
-          end
-          super
-        end
+      def inherited(base)
+        super
+        subclasses << base if base.name
       end
 
       # Define a controller as abstract. See internal_methods for more details.
@@ -60,11 +50,6 @@ module Jets::Lambda
         @subclasses ||= []
       end
 
-      def inherited(base)
-        super
-        self.subclasses << base if base.name
-      end
-
       # Needed for depends_on. Got added due to stagger logic.
       def output_keys
         []

data/lib/jets/names.rb

@@ -11,7 +11,7 @@ class Jets::Names
     end
 
     def one_controller_template_path
-      "#{templates_folder}/jets-controller.yml"
+      "#{templates_folder}/controller.yml"
     end
 
     def app_template_path(app_class)
@@ -34,18 +34,6 @@ class Jets::Names
       "#{templates_folder}/api-gateway.yml"
     end
 
-    def api_resources_template_path(page_number)
-      "#{templates_folder}/api-resources-#{page_number}.yml"
-    end
-
-    def api_cors_template_path(page_number)
-      "#{templates_folder}/api-cors-#{page_number}.yml"
-    end
-
-    def api_methods_template_path(page_number)
-      "#{templates_folder}/api-methods-#{page_number}.yml"
-    end
-
     def api_deployment_template_path
       "#{templates_folder}/api-deployment.yml"
     end
@@ -59,21 +47,21 @@ class Jets::Names
     end
 
     def parent_stack_name
-      Jets.project_namespace
+      Jets.project.namespace
     end
 
     def gateway_api_name
-      Jets.project_namespace
+      Jets.project.namespace
     end
 
     def authorizer_template_path(path)
       underscored = underscore(path)
-      underscored.sub!(/^app-/, '')
+      underscored.sub!(/^app-/, "")
       "#{templates_folder}/#{underscored}.yml"
     end
 
     def underscore(s)
-      s.to_s.underscore.sub(/\.rb$/,'').gsub('/','-')
+      s.to_s.underscore.sub(/\.rb$/, "").tr("/", "-")
     end
   end
 end

data/lib/jets/prewarm.rb

@@ -0,0 +1,27 @@
+module Jets
+  class Prewarm
+    class << self
+      include Jets::Util::Logging
+
+      # Can use to prewarm post deploy
+      # Jets::Prewarm.handle
+      # Jets::Prewarm.handle(verbose: true, invocation_type: "RequestResponse")
+      # Note: verbose is only useful when invocation_type is "RequestResponse"
+      def handle(options = {})
+        defaults = {
+          function_name: "controller",
+          event: '{"_prewarm": 1}'
+        }
+        options = defaults.merge(options.symbolize_keys)
+        # Always calls Lambda, not local
+        # Use invoke so messages don't get printed
+        Jets::CLI::Call.new(options).invoke
+      rescue Jets::CLI::Call::Error => e
+        puts "ERROR: #{e.message}".color(:red)
+        puts "The stack may not be full deployed yet.  Please check the stack and try again."
+      end
+
+      delegate :stats, to: Jets::Shim::Adapter::Prewarm
+    end
+  end
+end

data/lib/jets/rdoc.rb

@@ -22,9 +22,9 @@ module Jets
         Rakefile
         bin
       ]
-      exclude = exclude.map { |word| ['-x', word] }.flatten
+      exclude = exclude.map { |word| ["-x", word] }.flatten
       ["-m", "README.md", "--markup", "tomdoc"] + exclude
     end
     extend self
   end
-end
+end

data/lib/jets/remote/base.rb

@@ -0,0 +1,11 @@
+module Jets::Remote
+  class Base
+    extend Memoist
+    include Jets::AwsServices
+    include Jets::Util::Logging
+
+    def initialize(options)
+      @options = options
+    end
+  end
+end

data/lib/jets/remote/download.rb

@@ -0,0 +1,65 @@
+module Jets::Remote
+  class Download < Base
+    delegate :s3_bucket, to: "Jets.project"
+
+    def initialize(command)
+      @command = command
+    end
+
+    def download_built
+      # build deploy ci:build ci:deploy waf:build waf:deploy
+      subcommand = @command.split(":").last
+      case subcommand
+      when "build", "deploy", "dockerfile"
+        download_templates
+        download_dockerfile
+      end
+    end
+
+    def download_dockerfile
+      s3_key = "jets/sigs/#{sig}/docker/Dockerfile"
+      object = s3_resource.bucket(s3_bucket).object(s3_key)
+      # Can not exists for: jets build --templates
+      if object.exists?
+        dest = "#{Jets.build_root}/docker/Dockerfile"
+        FileUtils.mkdir_p(File.dirname(dest))
+        object.download_file(dest)
+        if @command.split(":").last == "dockerfile"
+          log.info "Dockerfile at: #{dest}"
+        else
+          log.debug "Download: #{dest}"
+        end
+      end
+    end
+
+    def download_templates
+      clean_all_templates
+      download_all_templates
+    end
+
+    def clean_all_templates
+      FileUtils.rm_rf("#{Jets.build_root}/templates")
+    end
+
+    def download_all_templates
+      s3.list_objects(bucket: s3_bucket, prefix: "jets/sigs/#{sig}/templates/").each do |resp|
+        resp.contents.each do |object|
+          download_template(object.key)
+        end
+      end
+    end
+
+    def download_template(s3_key)
+      object = s3_resource.bucket(s3_bucket).object(s3_key)
+      path = object.key.sub("jets/sigs/#{sig}/templates/", "")
+      dest = "#{Jets.build_root}/templates/#{path}"
+      log.debug "Download: #{dest}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      object.download_file(dest)
+    end
+
+    def sig
+      Jets::Remote::Runner.sig
+    end
+  end
+end

data/lib/jets/remote/runner.rb

@@ -0,0 +1,101 @@
+module Jets::Remote
+  class Runner < Base
+    cattr_reader :sig
+    def run
+      code_zip_and_upload
+
+      command = @options[:command] || "deploy"
+      command_args = if @options[:version] # jets rollback 8
+        @options[:version]
+      elsif @options[:templates] # jets build --templates or jets deploy --templates
+        "--templates"
+      end
+
+      sig = Jets::Api::Sig.create(command: command, command_args: command_args, architecture: architecture)
+      @@sig = sig[:token]
+      @jets_go = sig[:jets_go]
+
+      params = {
+        buildspec_override: sig[:buildspec],
+        environment_variables_override: environment_variables_override,
+        project_name: project_name
+      }
+      fleet_override = Jets.bootstrap.config.codebuild.project.fleet_override
+      if fleet_override
+        params[:fleet_override] = {fleet_arn: fleet_override}
+      end
+
+      resp = codebuild.start_build(params)
+      Jets::Api::Sig.update(sig[:id], build_id: resp.build.id)
+
+      logger.info "Started remote run for #{command}"
+      Jets::CLI::Tip.show(:remote_run, disable_howto: false)
+      logger.info "Console Log Url:"
+      logger.info codebuild_log_url(resp.build.id)
+
+      tail_logs(resp.build.id)
+
+      Download.new(command).download_built
+
+      resp = codebuild.batch_get_builds(ids: [resp.build.id])
+      success = resp.builds.first.build_status == "SUCCEEDED"
+      exit 1 unless success
+      success
+    end
+
+    def code_zip_and_upload
+      code = Jets::Code.new(dummy: @options[:dummy])
+      @app_src = code.zip_and_upload # s3_location
+    end
+
+    # Get architecture of codebuild image being used
+    def architecture
+      project = codebuild.batch_get_projects(names: [project_name]).projects.first
+      image = project.environment.image # IE: aws/codebuild/amazonlinux-aarch64-lambda-standard:ruby3.2
+      if image.include?("aarch64") || image.include?("arm64")
+        "arm64"
+      else
+        "x86_64"
+      end
+    end
+
+    def project_name
+      stack_name = Jets::Names.parent_stack_name
+      stack = cfn.describe_stacks(stack_name: stack_name).stacks.first
+      logical_id = use_lambda_compute_type? ? "CodebuildLambda" : "Codebuild"
+      stack.outputs.find { |o| o.output_key == logical_id }.output_value
+    end
+    memoize :project_name
+
+    def use_lambda_compute_type?
+      return true if Jets.bootstrap.config.infra
+      return false unless Jets.bootstrap.config.codebuild.lambda.enable
+
+      command = ARGV.first
+      command == "ci" ||
+        command == "waf" ||
+        command == "dockerfile" ||
+        ARGV.include?("--templates")
+    end
+
+    def environment_variables_override
+      Jets::Cfn::Resource::Codebuild::Project::Env.new.pass_vars(
+        JETS_APP_SRC: @app_src,
+        JETS_GO: @jets_go,
+        JETS_SIG: @@sig
+      )
+    end
+
+    def tail_logs(build_id)
+      Tailer.new(@options, build_id).run
+    end
+
+    private
+
+    def codebuild_log_url(build_id)
+      build_id = build_id.split(":").last
+      region = Jets.aws.region
+      "https://#{region}.console.aws.amazon.com/codesuite/codebuild/projects/#{project_name}/build/#{project_name}%3A#{build_id}/log"
+    end
+  end
+end

data/lib/jets/remote/tailer.rb

@@ -0,0 +1,272 @@
+require "aws-logs"
+
+module Jets::Remote
+  class Tailer < Base
+    include Jets::Util::Pretty
+
+    def initialize(options, build_id)
+      @options, @build_id = options, build_id
+
+      @output = [] # for specs
+      @shown_phases = []
+      @thread = nil
+    end
+
+    @@delay = 2 # initial delay
+    def run
+      complete = false
+      until complete
+        build = find_build
+        unless build
+          puts "ERROR: Build id not found: #{@build_id}".color(:red)
+          return
+        end
+
+        # CodeBuild AWS Lambda Compute Type take slightly a few milliseconds to report phases.
+        # But return the build already. So we have to wait until the phases are available.
+        while build.phases.nil?
+          build = find_build # refresh
+          sleep 1
+        end
+
+        print_phases(build) unless @stop_printing_phases # once cloudwatch logs starts, stop printing phases
+        set_log_group_name(build)
+
+        complete = build.build_complete
+
+        next if ENV["JETS_TEST"]
+        if build_phase_started?(build)
+          start_cloudwatch_tail
+          @stop_printing_phases = true
+          @@delay = 5 # increase @@delay also
+        end
+
+        sleep @@delay
+      end
+
+      stop_cloudwatch_tail(build)
+      if Jets.bootstrap.config.codebuild.logging.final_phases
+        print_phases(build) # print final phases
+      end
+      display_failed_phases(build)
+    end
+
+    def build_phase_started?(build)
+      @build_phase_started ||= build.phases.any? do |phase|
+        phase.phase_type == "BUILD"
+      end
+    end
+
+    def display_failed_phases(build)
+      status = build.build_status.to_s # in case nil
+      status = (status != "SUCCEEDED") ? status.color(:red) : status.color(:green)
+      return if status == "SUCCEEDED"
+
+      failed_phases = build.phases.select do |phase|
+        phase.phase_status != "SUCCEEDED" && phase.phase_status.to_s != ""
+      end
+      return if failed_phases.empty?
+
+      puts "Failed Phases:"
+      failed_phases.each do |phase|
+        puts "#{phase.phase_type}: #{phase.phase_status.color(:red)}"
+        next unless phase.contexts # can be nil
+        context = phase.contexts.last
+        if context # show error details: Unable to pull customer's container image https://gist.github.com/tongueroo/22e4ca3d4cde002108ff506eba9062f6
+          message = context.message
+          puts message
+          if message.include?("CannotPullContainerError") && message.include?("access denied")
+            puts "See: https://docs.aws.amazon.com/codebuild/latest/userguide/sample-ecr.html"
+          end
+        end
+      end
+    end
+
+    def display_time_took(build)
+      puts "Remote runner took #{build_time(build)} to complete"
+    end
+
+    def find_build
+      resp = codebuild.batch_get_builds(ids: [@build_id])
+      resp.builds.first
+    rescue Aws::CodeBuild::Errors::ThrottlingException => e
+      log_arch "WARN: find_build codebuild.batch_get_builds ThrottlingException: #{e.message}"
+      @@delay = 10 # increase global @@delay also
+
+      # Also exponential backoff delay
+      # 2, 4, 8, 16, 32
+      @retries ||= 1
+      raise if @retries > 5 # give up after 5 retries
+      delay = 2**@retries
+      log_arch "Retrying in #{delay}s..."
+      sleep delay
+      @retries += 1
+      retry
+    end
+
+    def start_cloudwatch_tail
+      return if @cloudwatch_tail_started
+      return unless @log_group_name && @log_stream_name
+
+      @thread = Thread.new do
+        @cw_tail = cloudwatch_tail
+        @cw_tail.run
+      end
+      @cloudwatch_tail_started = true
+    end
+
+    def cloudwatch_tail
+      since = @options[:since] || "24h" # by default, search only 24h in the past
+      AwsLogs::Tail.new(
+        log_group_name: @log_group_name,
+        log_stream_names: [@log_stream_name],
+        since: since,
+        follow: true,
+        format: "plain",
+        show_if: show_if
+      )
+    end
+
+    def show_if
+      return true unless Jets.bootstrap.config.codebuild.logging.show == "filtered"
+
+      start_marker = "./jets"
+      end_marker = "Phase complete: BUILD"
+      proc do |event|
+        @display_showing ||= event.message.include?(start_marker)
+        if @display_showing && event.message.include?(end_marker)
+          @display_showing = false
+        end
+        @display_showing
+      end
+    end
+    memoize :show_if
+
+    def stop_cloudwatch_tail(build)
+      return if ENV["JETS_TEST"]
+      @cw_tail&.stop_follow!
+      @thread&.join
+    end
+
+    def logs_command?
+      ARGV.join(" ").include?("logs")
+    end
+
+    # build.build_status : The current status of the build. Valid values include:
+    #
+    #     FAILED : The build failed.
+    #     FAULT : The build faulted.
+    #     IN_PROGRESS : The build is still in progress.
+    #     STOPPED : The build stopped.
+    #     SUCCEEDED : The build succeeded.
+    #     TIMED_OUT : The build timed out.
+    #
+    def complete_failed?(build)
+      return if ENV["JETS_TEST"]
+      build.build_complete && build.build_status != "SUCCEEDED"
+    end
+
+    # Setting enables start_cloudwatch_tail
+    def set_log_group_name(build)
+      logs = build.logs
+      @log_group_name = logs.group_name if logs.group_name
+      @log_stream_name = logs.stream_name if logs.stream_name
+    end
+
+    def print_phases(build)
+      build.phases.each do |phase|
+        details = {
+          phase_type: phase.phase_type,
+          phase_status: phase.phase_status,
+          start_time: phase.start_time,
+          duration_in_seconds: phase.duration_in_seconds
+        }
+        display_phase(details)
+        @shown_phases << details
+      end
+    end
+
+    def display_phase(details)
+      already_shown = @shown_phases.detect do |p|
+        p[:phase_type] == details[:phase_type] &&
+          p[:phase_status] == details[:phase_status] &&
+          p[:start_time] == details[:start_time] &&
+          p[:duration_in_seconds] == details[:duration_in_seconds]
+      end
+      return if already_shown
+
+      return if filter_phase_type?(details)
+
+      phases = [
+        "Phase:", details[:phase_type]
+      ]
+
+      if details[:phase_type] == "COMPLETED"
+        # nothing to add
+      elsif details[:phase_status].nil?
+        phases << "Pending"
+      else
+        phases += [
+          phase_status(details[:phase_status]),
+          phase_duration(details[:duration_in_seconds])
+        ]
+      end
+
+      phases = phases.flatten.compact.join(" ")
+
+      say phases
+    end
+
+    def filter_phase_type?(details)
+      return false unless Jets.bootstrap.config.codebuild.logging.show == "filtered"
+
+      # Phase: SUBMITTED Status: SUCCEEDED Duration: 0s
+      # Phase: QUEUED Pending
+      # Phase: QUEUED Status: SUCCEEDED Duration: 0s
+      # Phase: PROVISIONING Pending
+      # Phase: PROVISIONING Status: SUCCEEDED Duration: 5s
+      # Phase: DOWNLOAD_SOURCE Status: SUCCEEDED Duration: 0s
+      # Phase: INSTALL Status: SUCCEEDED Duration: 0s
+      # Phase: PRE_BUILD Status: SUCCEEDED Duration: 0s
+      # Phase: BUILD Pending
+      # [Container] 2024/04/06 15:51:12.262049 Running command ...
+      #
+      # Becomes
+      #
+      # Phase: SUBMITTED Status: SUCCEEDED Duration: 0s
+      # Phase: QUEUED Pending
+      # Phase: QUEUED Status: SUCCEEDED Duration: 0s
+      # Phase: PROVISIONING Pending
+      # Phase: PROVISIONING Status: SUCCEEDED Duration: 5s
+      # [Container] 2024/04/06 15:51:12.262049 Running command ...
+      filtered_phase_types = %w[DOWNLOAD_SOURCE INSTALL PRE_BUILD BUILD]
+      filtered_phase_types.include?(details[:phase_type])
+    end
+
+    def phase_status(status)
+      return unless status # can be nil
+
+      text = (status == "SUCCEEDED") ? status : status.color(:red)
+      ["Status:", text]
+    end
+
+    def phase_duration(duration)
+      return unless duration # can be nil
+
+      ["Duration:", pretty_time(duration)]
+    end
+
+    def say(text)
+      ENV["JETS_TEST"] ? @output << text : puts(text)
+    end
+
+    def output
+      @output.join("\n") + "\n"
+    end
+
+    def build_time(build)
+      duration = build.phases.inject(0) { |sum, p| sum + p.duration_in_seconds.to_i }
+      pretty_time(duration)
+    end
+  end
+end

data/lib/jets/shim/adapter/alb.rb

@@ -0,0 +1,16 @@
+module Jets::Shim::Adapter
+  class Alb < Apigw
+    def env
+      super.merge(
+        "HTTP_PORT" => headers["x-forwarded-port"],
+        "SERVER_PORT" => headers["x-forwarded-port"],
+        "SERVER_PROTOCOL" => event.dig("requestContext", "protocol") || "HTTP/1.1"
+      )
+    end
+
+    def handle?
+      host =~ /elb\.amazonaws\.com/ ||
+        event.dig("requestContext", "elb")
+    end
+  end
+end

data/lib/jets/shim/adapter/apigw.rb

@@ -0,0 +1,37 @@
+module Jets::Shim::Adapter
+  class Apigw < Web
+    # See: https://github.com/rack/rack/blob/main/lib/rack/constants.rb
+    def env
+      {
+        # Request env keys
+        "HTTP_HOST" => host,
+        "HTTP_PORT" => headers["X-Forwarded-Port"],
+        "HTTPS" => https,
+        "PATH_INFO" => path_info,
+        "QUERY_STRING" => query_string,
+        "REQUEST_METHOD" => event["httpMethod"] || "GET", # useful to default to GET when testing with Lambda console
+        "REQUEST_PATH" => path_info,
+        "SCRIPT_NAME" => "",
+        "SERVER_NAME" => host,
+        "SERVER_PORT" => headers["X-Forwarded-Port"],
+        "SERVER_PROTOCOL" => event.dig("requestContext", "protocol") || "HTTP/1.1"
+      }
+    end
+
+    def path_info
+      event["path"] || "/" # always set by API Gateway, but setting to make shim testing easier
+    end
+
+    def handle?
+      host =~ /execute-api/ ||
+        event["resource"] && event.dig("requestContext", "stage")
+    end
+
+    private
+
+    def query_string
+      query = event["queryStringParameters"] || {} # always set with API Gateway but when testing shim might not be
+      Rack::Utils.build_nested_query(query)
+    end
+  end
+end