jets 5.0.17 → 6.0.0.beta1

data/lib/jets/code/stager.rb

@@ -0,0 +1,124 @@
+require "active_support"
+require "active_support/number_helper"
+
+class Jets::Code
+  class Stager
+    extend Memoist
+    include Jets::Util::Git
+    include Jets::Util::Logging
+    include Jets::Util::Sh
+
+    delegate :build_root, to: Jets
+    delegate :config, to: "Jets.bootstrap.config"
+
+    def build
+      clean
+      warn_large_codebase
+      stage_code # interface method
+      save_deploy_user
+      save_project_name
+      save_ruby_version
+      touch_deployed_at
+    end
+
+    # interface method: overridden by Dummy
+    def stage_code
+      log.debug "Copying project to #{build_root}/stage/code"
+      log.debug "Copy strategy: #{copy_strategy}"
+      copy_strategy.run
+    end
+
+    def copy_strategy
+      strategy = config.code.copy.strategy.to_s
+      if strategy == "auto"
+        auto_strategy
+      else # whatever user overrides it to
+        class_name = strategy.camelize
+        Copy.const_get(class_name) # Copy::GitInline
+      end
+    end
+    memoize :copy_strategy
+
+    def auto_strategy
+      if File.exist?("#{Jets.root}/.git") && rsync_installed?
+        Copy::Rsync
+      elsif git? # .git folder exists and git command available
+        Copy::GitCopy
+      else # full copy
+        Copy::Full # FileUtils.cp_r(Jets.root, "#{build_root}/stage/code")
+      end
+    end
+
+    @@rsync_installed = nil # only check once
+    def rsync_installed?
+      return @@rsync_installed unless @@rsync_installed.nil?
+      @@rsync_installed = system "type rsync > /dev/null 2>&1"
+    end
+
+    def save_deploy_user
+      User.new.save
+    end
+
+    # Save the project_name in .jets/info.yml if inferred
+    # Better to write the file here instead of within the Jets::Core::Config::Info class.
+    # Because Info assumes Jets.root is Dir.pwd
+    # We need to ensure it's written to stage/code not the current project root.
+    def save_project_name
+      return unless Jets.project.name_inferred?
+
+      jets_info = Jets::Core::Config::Info.instance
+      data = jets_info.data.merge(project_name: Jets.project.name)
+      dest = "#{build_root}/stage/code/#{jets_info.path}"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, data.to_h.to_yaml)
+    end
+
+    # Save by moving .ruby-version to .jets/ruby-version
+    # So it does not affect codebuild ruby version.
+    def save_ruby_version
+      Dir.chdir("#{build_root}/stage/code") do
+        move_ruby_version(".ruby-version")
+        move_ruby_version(".tool-versions") # future proofing for asdf
+      end
+    end
+
+    def move_ruby_version(filename = ".ruby-version")
+      if File.exist?(filename)
+        FileUtils.mkdir_p(".jets")
+        FileUtils.mv(filename, ".jets/#{filename}")
+      end
+    end
+
+    def touch_deployed_at
+      dest = "#{build_root}/stage/code/.jets/deployed_at"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, Time.now.utc.to_s)
+    end
+
+    def clean
+      FileUtils.rm_rf("#{build_root}/stage/code")
+      FileUtils.rm_rf("#{build_root}/stage/code-temp")
+      FileUtils.mkdir_p(File.dirname("#{build_root}/stage/code"))
+    end
+
+    def warn_large_codebase
+      return unless Jets.bootstrap.code.copy.warn_large
+      return unless system("type du > /dev/null 2>&1")
+      bytes = `du -bs #{Jets.root}`.split("\t").first.to_i
+      if bytes > 1024 * 1024 * 1024 # 1GB
+        size = ActiveSupport::NumberHelper.number_to_human_size(bytes)
+        log.info <<~EOL
+          WARNING: Large codebase detected: #{size}
+          Will take some time to zip and upload. Please be patient.
+          You can turn off this warning with
+
+          config/jets/bootstrap.rb
+
+              Jets.bootstrap.configure do
+                config.code.copy.warn_large = false
+              end
+        EOL
+      end
+    end
+  end
+end

data/lib/jets/code/user.rb

@@ -0,0 +1,24 @@
+require "aws-sdk-iam"
+
+class Jets::Code
+  class User
+    delegate :build_root, to: Jets
+
+    def save
+      user = iam_user || ENV["USER"] || ENV["JETS_DEPLOY_USER"]
+      FileUtils.mkdir_p(File.dirname(user_file))
+      IO.write(user_file, user)
+      user
+    end
+
+    def user_file
+      "#{build_root}/stage/code/.jets/deploy_user"
+    end
+
+    def iam_user
+      @iam ||= Aws::IAM::Client.new
+      @iam.get_user.user.user_name
+    rescue Aws::IAM::Errors::ValidationError
+    end
+  end
+end

data/lib/jets/code.rb

@@ -0,0 +1,80 @@
+module Jets
+  class Code
+    include Jets::AwsServices
+    include Jets::Util::Logging
+    include Jets::Util::Sh
+
+    delegate :s3_bucket, to: "Jets.project"
+    delegate :build_root, to: Jets
+
+    def initialize(options = {})
+      @options = options
+    end
+
+    def zip_and_upload
+      command = ARGV.reject { |s| s.match(/^-/) }.join(":") # remove options
+      log.info "Packaging code for #{command}: #{Jets.project.namespace}"
+      stage
+      zip
+      upload
+    end
+
+    def zip_path
+      "#{build_root}/code/code.zip"
+    end
+
+    @@s3_key = nil
+    def s3_key
+      return @@s3_key if @@s3_key
+
+      timestamp = Time.now.utc.strftime("%Y-%m-%dT%H-%M-%SZ")
+      git = Jets::Git::Info.new
+      branch, sha, dirty = git.params.values_at(:git_branch, :git_sha, :git_dirty)
+      sha = sha[0..6] if sha # short sha
+      dirty = dirty ? "dirty" : nil
+      key = ["jets/code/#{timestamp}", branch, sha, dirty].compact.join("-")
+      key = key.gsub(/[^a-zA-Z0-9\-_:\/]/, "-").squeeze("-") # sanitize
+      @@s3_key = key + ".zip"
+    end
+
+    def s3_location
+      "s3://#{s3_bucket}/#{s3_key}"
+    end
+
+    def upload
+      s3_resource.bucket(s3_bucket).object(s3_key).upload_file(zip_path)
+      logger.debug "Uploaded code to s3://#{s3_bucket}/#{s3_key}"
+      s3_location # Important to return the s3_location
+    end
+
+    def stage
+      if @options[:dummy]
+        Dummy.new.build
+      else
+        Stager.new.build
+      end
+    end
+
+    def zip
+      # Check if the folder exists
+      unless Dir.exist?("#{build_root}/stage/code")
+        logger.info "Error: Source folder not found: #{build_root}/stage/code"
+        return
+      end
+
+      # Setup
+      FileUtils.mkdir_p(File.dirname(zip_path))
+      FileUtils.rm_f(zip_path) # remove existing zip file. Else files are added to it
+
+      # Zip
+      quiet_sh "cd #{build_root}/stage/code && zip --symlinks -rq #{zip_path} ."
+
+      # Check if the zip operation was successful
+      unless $?.success?
+        logger.info "Error: Failed to zip the folder: #{build_root}/stage/code"
+      end
+
+      zip_path
+    end
+  end
+end

data/lib/jets/core/booter.rb

@@ -0,0 +1,58 @@
+require "securerandom"
+
+module Jets::Core
+  class Booter
+    class << self
+      extend Memoist
+
+      attr_reader :boot_at, :gid
+      def boot!
+        return false if @boot_at
+
+        Jets::Bundle.require # require all the gems in the Gemfile
+        require_config(:project) # for config.dotenv.overwrite
+
+        if require_bootstrap?
+          Jets::Dotenv.load!
+          require_config(:bootstrap)
+        end
+
+        initialize!
+
+        @gid = SecureRandom.uuid[0..7]
+        @boot_at = Time.now.utc
+      end
+
+      def initialize!
+        main = Jets::Autoloaders.main
+        main.configure(Jets.root)
+        main.setup
+      end
+
+      # Essentially deployment commands require the bootstrap to be run
+      def require_bootstrap?
+        args = ARGV.reject { |arg| arg.start_with?("-") }
+        %w[
+          bootstrap
+          build
+          delete
+          deploy
+          dockerfile
+          release:rollback
+        ].include?(args.last)
+      end
+
+      def require_config(name)
+        files = [
+          "config/jets/#{name}.rb",
+          "config/jets/#{name}/#{Jets.env}.rb"
+        ]
+        files.each do |file|
+          next unless File.exist?(file)
+          require "#{Jets.root}/#{file}"
+        end
+      end
+      memoize :require_config
+    end
+  end
+end

data/lib/jets/core/config/base.rb

@@ -0,0 +1,17 @@
+require "singleton"
+
+module Jets::Core::Config
+  class Base
+    extend Memoist
+    include Jets::Util::Camelize
+    include Singleton
+
+    def configure(&block)
+      instance_eval(&block) if block
+    end
+
+    def config
+      self
+    end
+  end
+end

data/lib/jets/core/config/bootstrap/cfn.rb

@@ -0,0 +1,12 @@
+class Jets::Core::Config::Bootstrap
+  module Cfn
+    attr_accessor :cfn
+
+    def initialize(*)
+      super
+
+      @cfn = ActiveSupport::OrderedOptions.new
+      @cfn.resource_tags = {} # tags to add to all resources
+    end
+  end
+end

data/lib/jets/core/config/bootstrap/code.rb

@@ -0,0 +1,16 @@
+class Jets::Core::Config::Bootstrap
+  module Code
+    attr_accessor :code
+
+    def initialize(*)
+      super
+
+      @code = ActiveSupport::OrderedOptions.new
+      @code.copy = ActiveSupport::OrderedOptions.new
+      @code.copy.always_keep = ["config/jets/env"]
+      @code.copy.always_remove = ["tmp"]
+      @code.copy.strategy = "auto"
+      @code.copy.warn_large = true
+    end
+  end
+end

data/lib/jets/core/config/bootstrap/codebuild.rb

@@ -0,0 +1,97 @@
+class Jets::Core::Config::Bootstrap
+  module Codebuild
+    attr_accessor :codebuild
+
+    def initialize(*)
+      super
+
+      @codebuild = ActiveSupport::OrderedOptions.new
+      @codebuild.project = ActiveSupport::OrderedOptions.new
+      @codebuild.project.compute_type = {
+        ComputeType: "BUILD_GENERAL1_SMALL",
+        Image: "aws/codebuild/amazonlinux2-aarch64-standard:3.0",
+        Type: "ARM_CONTAINER"
+      }
+      @codebuild.project.env = ActiveSupport::OrderedOptions.new
+      @codebuild.project.env.vars = {}
+      @codebuild.project.env.pass = []
+      # Be careful about default env.pass
+      # Jets::Dotenv will not load these variables as a part of the Lambda Function
+      # So we should be specific about what JETS_ vars we want to the codebuild remote runner
+      # Should not use regexp /JETS_/
+      @codebuild.project.env.default_pass = [
+        "JETS_API",
+        "JETS_DOCKER_IMAGE",
+        "JETS_ENV",
+        "JETS_EXTRA",
+        "JETS_GO_VERSION",
+        "JETS_PROJECT",
+        "JETS_REMOTE_VERSION",
+        "JETS_RESET"
+      ]
+      @codebuild.project.env.block = []
+      @codebuild.project.environment = {}
+      # fleet arn
+      @codebuild.project.fleet_override = ENV["JETS_CODEBUILD_FLEET_OVERRIDE"]
+      @codebuild.project.timeout_in_minutes = 60
+
+      # CodebuildLambda
+      # The only setting that is used for CodeBuildLambda compute_type
+      # Using a separate compute_type config because images are required to be different.
+      # A shared setting will not work.
+      #
+      # All other settings are shared between the CodeBuild and CodeBuildLambda projects
+      # IE: @codebuild.project.env.vars etc
+      # Think this keeps things simpler and easy to understand.
+      # IE: Less room for config errors.
+      @codebuild.lambda = ActiveSupport::OrderedOptions.new
+      @codebuild.lambda.enable = false
+      @codebuild.lambda.project = ActiveSupport::OrderedOptions.new
+      @codebuild.lambda.project.compute_type = {
+        ComputeType: "BUILD_LAMBDA_1GB",
+        Image: "aws/codebuild/amazonlinux-aarch64-lambda-standard:ruby3.2",
+        Type: "ARM_LAMBDA_CONTAINER"
+      }
+
+      @codebuild.fleet = ActiveSupport::OrderedOptions.new
+      @codebuild.fleet.base_capacity = 1
+      @codebuild.fleet.enable = !!ENV["JETS_CODEBUILD_FLEET_ENABLE"]
+
+      @codebuild.logging = ActiveSupport::OrderedOptions.new
+      @codebuild.logging.final_phases = false
+      @codebuild.logging.show = "filtered" # filtered or all
+      # Note: Do not use .display It's an ActiveSuppport method
+
+      @codebuild.iam = ActiveSupport::OrderedOptions.new
+      @codebuild.iam.policy = []
+      @codebuild.iam.managed_policy = []
+      @codebuild.iam.default_policy = %w[
+        apigateway
+        cloudformation
+        cloudfront
+        codebuild
+        dynamodb
+        ecr
+        ecr-public
+        events
+        iam
+        lambda
+        logs
+        route53
+        s3
+        sns
+        sqs
+        sts:GetServiceBearerToken
+        waf
+        wafv2
+      ]
+      @codebuild.iam.default_managed_policy = %w[
+        AmazonSSMReadOnlyAccess
+        AWSCertificateManagerReadOnly
+      ]
+      @codebuild.iam.default_vpc_policy = %w[
+        ec2
+      ]
+    end
+  end
+end

data/lib/jets/core/config/bootstrap/s3_bucket.rb

@@ -0,0 +1,19 @@
+class Jets::Core::Config::Bootstrap
+  module S3Bucket
+    attr_accessor :s3_bucket
+
+    def initialize(*)
+      super
+
+      @s3_bucket = ActiveSupport::OrderedOptions.new
+      @s3_bucket.cors_configuration = {
+        CorsRules: [{
+          AllowedHeaders: ["*"],
+          AllowedMethods: ["GET"],
+          AllowedOrigins: ["*"],
+          ExposedHeaders: []
+        }]
+      }
+    end
+  end
+end

data/lib/jets/core/config/bootstrap.rb

@@ -0,0 +1,24 @@
+module Jets::Core::Config
+  class Bootstrap < Base
+    include Helpers
+    # config settings
+    include Cfn
+    include Code
+    include Codebuild
+    include S3Bucket
+
+    attr_accessor :infra, :logger
+    def initialize(*)
+      super
+      @infra = false
+      @logger = default_logger
+    end
+
+    def default_logger
+      logger = ActiveSupport::Logger.new($stderr)
+      logger.formatter = ActiveSupport::Logger::SimpleFormatter.new # no timestamps
+      logger.level = ENV["JETS_LOG_LEVEL"] || :info
+      logger
+    end
+  end
+end

data/lib/jets/core/config/helpers/ssm.rb

@@ -0,0 +1,9 @@
+require "aws-sdk-ssm"
+
+module Jets::Core::Config::Helpers
+  module Ssm
+    def ssm_env
+      Jets::Dotenv::Convention.ssm_env
+    end
+  end
+end

data/lib/jets/core/config/helpers.rb

@@ -0,0 +1,5 @@
+module Jets::Core::Config
+  module Helpers
+    include Ssm
+  end
+end

data/lib/jets/core/config/info.rb

@@ -0,0 +1,27 @@
+require "singleton"
+
+module Jets::Core::Config
+  class Info
+    extend Memoist
+    include Singleton
+
+    def data
+      data = File.exist?(path) ? YAML.load_file(path) : {}
+      ActiveSupport::HashWithIndifferentAccess.new(data)
+    end
+    memoize :data
+
+    def method_missing(name, *args)
+      data.key?(name.to_sym) ? data[name.to_sym] : super
+    end
+
+    def respond_to_missing?(name, include_private = false)
+      data.key?(name.to_sym) || super
+    end
+
+    # Do not use absolute path.  This is because the path is written to the stage/code area
+    def path
+      "config/jets/info.yml"
+    end
+  end
+end

data/lib/jets/core/config/project.rb

@@ -0,0 +1,103 @@
+module Jets::Core::Config
+  # Intentionally keep this class simple.
+  # Frameworks should do the heavy lifting.
+  class Project < Base
+    attr_accessor(
+      :autoload_paths,
+      :ignore_paths,
+      :base64_encode,
+      :dotenv,
+      :tips
+    )
+    def initialize(*)
+      @autoload_paths = Set.new(%w[
+        app/events
+        app/extensions
+        shared/resources
+        shared/extensions
+      ])
+      @ignore_paths = Set.new(%w[
+        app/functions
+        shared/functions
+      ])
+      # Jets does not implement the concept of eager_load_paths like Rails for simplicity.
+      # Also Jets always does an eager load of app code that it manages above.
+      # This is because shared/extensions and shared/resources needed to be defined
+      # early to generate the CloudFormation templates.
+
+      @dotenv = ActiveSupport::OrderedOptions.new
+      @dotenv.ssm = ActiveSupport::OrderedOptions.new
+      @dotenv.ssm.autoload = ActiveSupport::OrderedOptions.new
+      @dotenv.ssm.autoload.default_skip = ["BASIC_AUTH_USERNAME", "BASIC_AUTH_PASSWORD", "BASIC_AUTH_CREDENTIALS"]
+      @dotenv.ssm.autoload.enable = true # autoloads parameters by path IE: /demo/dev/
+      @dotenv.ssm.autoload.skip = []
+      @dotenv.ssm.convention_resolver = nil # proc receives ssm_leaf_name
+      @dotenv.ssm.envs = ActiveSupport::OrderedOptions.new
+      @dotenv.ssm.envs.fallback = "dev"
+      @dotenv.ssm.envs.unique = ["dev", "prod"]
+      @dotenv.ssm.long_env_helper = false   # for completeness
+      @dotenv.ssm.long_env_name = false     # helps with Jets 5 legacy
+
+      @base64_encode = true
+
+      @tips = ActiveSupport::OrderedOptions.new
+      @tips.enable = true
+      @tips.concurrency_change = true
+      @tips.env_change = true
+      @tips.faster_deploy = true
+      @tips.remote_run = true
+      @tips.ssm_change = true
+    end
+
+    attr_writer :name
+    def name
+      if ENV["JETS_PROJECT"] && !ENV["JETS_PROJECT"].blank?
+        return ENV["JETS_PROJECT"]
+      end
+
+      # Too easy to call a method that requires the project name before Jets.boot.
+      # Ran into this a few times.
+      # IE: Ran into this with Jets API ping. That's no longer requires Jets.project.name
+      # But will leave this here in case we miss Jets.boot in the future.
+      Jets::Core::Booter.require_config(:project)
+      return @name if @name
+
+      if jets_info_project_name
+        jets_info_project_name
+      else
+        # when not set, conventionally infer app name from current directory
+        @name_inferred = true
+        Dir.pwd.split("/").last.gsub(/[^a-zA-Z0-9_]/, "-").squeeze("-")
+      end
+    end
+    memoize :name
+
+    def jets_info_project_name
+      info = Jets::Core::Config::Info.instance
+      info.project_name if info.respond_to?(:project_name)
+    end
+
+    def name_inferred?
+      name # trigger memoization
+      !!@name_inferred
+    end
+
+    def namespace
+      [name, Jets.env, Jets.extra].compact.join("-").tr("_", "-")
+    end
+
+    def s3_bucket
+      Jets::Cfn::Resource::S3::JetsBucket.name
+    end
+
+    def extension_paths
+      @autoload_paths.select { |path| path.ends_with?("/extensions") }
+    end
+
+    # Useful for jets rails engine
+    # Used to ignore all paths managed by Jets
+    def all_load_paths
+      @autoload_paths + @ignore_paths
+    end
+  end
+end