jets.benforeva 3.0.17.pre.mount.pre.fix

data/lib/jets/aws_info.rb

@@ -0,0 +1,111 @@
+
+module Jets
+  class AwsInfo
+    extend Memoist
+    include AwsServices
+
+    def region
+      return 'us-east-1' if Jets.env.test?
+
+      return ENV['JETS_AWS_REGION'] if ENV['JETS_AWS_REGION'] # highest precedence
+      return ENV['AWS_REGION'] if ENV['AWS_REGION']
+
+      region = nil
+
+      # First if aws binary is available
+      # try to get it from the ~/.aws/config
+      if which('aws')
+        region = `aws configure get region 2>&1`.strip rescue nil
+        exit_code = $?.exitstatus
+        if exit_code != 0
+          exception_message = region.split("\n").grep(/botocore\.exceptions/).first
+          if exception_message
+            puts "WARN: #{exception_message}".color(:yellow)
+          else
+            # show full message as warning
+            puts region.color(:yellow)
+          end
+          puts "You can also get rid of this message by setting AWS_REGION or configuring ~/.aws/config with the region"
+          region = nil
+        end
+        region = nil if region == ''
+        return region if region
+      end
+
+      # Second try the metadata endpoint, should be available on AWS Lambda environment
+      # https://stackoverflow.com/questions/4249488/find-region-from-within-an-ec2-instance
+      begin
+        az = `curl -s --max-time 3 --connect-timeout 5 http://169.254.169.254/latest/meta-data/placement/availability-zone`
+        region = az.strip.chop # remove last char
+        region = nil if region == ''
+      rescue
+      end
+      return region if region
+
+      'us-east-1' # default if all else fails
+    end
+    memoize :region
+
+    # aws sts get-caller-identity
+    def account
+      return '123456789' if Jets.env.test?
+      return ENV['JETS_AWS_ACCOUNT'] if ENV['JETS_AWS_ACCOUNT']
+
+      # ensure region set, required for sts.get_caller_identity.account to work
+      ENV['AWS_REGION'] ||= region
+      begin
+        sts.get_caller_identity.account
+      rescue Aws::Errors::MissingCredentialsError, Aws::Errors::NoSuchEndpointError, Aws::STS::Errors::InvalidClientTokenId
+        puts "INFO: You're missing AWS credentials. Only local services are currently available"
+      rescue Seahorse::Client::NetworkingError
+        puts "INFO: No internet connection available. Only local services are currently available"
+      end
+    end
+    memoize :account
+
+    # If bucket does not exist, do not use the cache value and check for the bucket again.
+    # This is because we can build the app before deploying it for the first time.
+    # The deploy sequence ensure an minimal stack exists and will return a s3 bucket
+    # value for real deployments though, just not for the `jets build` only command.
+    BUCKET_DOES_NOT_YET_EXIST = "bucket-does-not-yet-exist" # use const to save from misspellings
+    @@s3_bucket = BUCKET_DOES_NOT_YET_EXIST
+    def s3_bucket
+      return "fake-test-s3-bucket" if Jets.env.test?
+      return @@s3_bucket unless @@s3_bucket == BUCKET_DOES_NOT_YET_EXIST
+
+      resp = cfn.describe_stacks(stack_name: Jets::Naming.parent_stack_name)
+      stack = resp.stacks.first
+      output = stack["outputs"].find { |o| o["output_key"] == "S3Bucket" }
+      @@s3_bucket = output["output_value"] # s3_bucket
+    rescue Exception => e
+      # When user uses Jets::Application.default_iam_policy in their config/application.rb
+      # it looks up the s3 bucket for the iam policy, but the project name has
+      # not been loaded in the config yet.  We do some trickery with loading
+      # the config twice in Application#load_app_config
+      # The first load will result in a Aws::CloudFormation::Errors::ValidationError
+      # since the Jets::Naming.parent_stack_name has not been properly set yet.
+      #
+      # Rescuing all exceptions in case there are other exceptions dont know about yet
+      # Here are the known ones: Aws::CloudFormation::Errors::ValidationError, Aws::CloudFormation::Errors::InvalidClientTokenId
+      BUCKET_DOES_NOT_YET_EXIST
+    end
+
+    private
+
+    # Cross-platform way of finding an executable in the $PATH.
+    #
+    #   which('ruby') #=> /usr/bin/ruby
+    #
+    # source: https://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby
+    def which(cmd)
+      exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+      ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+        exts.each { |ext|
+          exe = File.join(path, "#{cmd}#{ext}")
+          return exe if File.executable?(exe) && !File.directory?(exe)
+        }
+      end
+      return nil
+    end
+  end
+end

data/lib/jets/aws_services/global_memoist.rb

@@ -0,0 +1,57 @@
+# We cache the clients globally to avoid re-instantiating them again after the initial Lambda cold start.
+#
+# Based on: https://hashrocket.com/blog/posts/implementing-a-macro-in-ruby-for-memoization
+# Except we use a global variable for the cache. So we'll use the same client across
+# all instances as well as across Lambda executions after the cold-start. Example:
+#
+#   class Foo
+#     def s3
+#       Aws::S3::Client.new
+#     end
+#     global_memoize :s3
+#   end
+#
+#   foo1 = Foo.new
+#   foo2 = Foo.new
+#   foo1.s3
+#   foo2.s3 # same client as foo1
+#
+# A prewarmed request after a cold-start will still use the same s3 client instance since it uses the
+# global variable $__memo_methods as the cache.
+#
+module Jets::AwsServices
+  module GlobalMemoist
+    def self.included(klass)
+      klass.extend(Macros)
+    end
+
+    module Macros
+      def global_memoize(*methods)
+        const_defined?(:"GlobalMemoist#{self.object_id}") ?
+          const_get(:"GlobalMemoist#{self.object_id}") : const_set(:"GlobalMemoist#{self.object_id}", Module.new)
+        mod = const_get(:"GlobalMemoist#{self.object_id}")
+
+        mod.class_eval do
+          methods.each do |method|
+            define_method(method) do |skip_cache = false|
+              $__memo_methods ||= {}
+              if $__memo_methods.include?(method) && !skip_cache
+                $__memo_methods[method]
+              else
+                $__memo_methods[method] = super()
+              end
+            end
+          end
+        end
+        prepend mod
+      end
+
+      def global_memoize_class_method(*methods)
+        singleton_class.class_eval do
+          include GlobalMemoist
+          global_memoize(*methods)
+        end
+      end
+    end
+  end
+end

data/lib/jets/aws_services/s3_bucket.rb

@@ -0,0 +1,34 @@
+module Jets::AwsServices
+  class S3Bucket
+    include Jets::AwsServices
+
+    def self.ensure_exists(bucket_name)
+      new(bucket_name).ensure_exists
+    end
+
+    def initialize(name)
+      @name = name
+    end
+
+    def ensure_exists
+      s3.create_bucket(bucket: @name) unless exists?
+    rescue Aws::S3::Errors::BucketAlreadyExists => e
+      puts "ERROR #{e.class}: #{e.message}".color(:red)
+      puts "Bucket name: #{@name}"
+      exit 1
+    end
+
+    def exists?
+      begin
+        s3.head_bucket(bucket: @name)
+        true
+      rescue Aws::S3::Errors::BucketAlreadyOwnedByYou, Aws::S3::Errors::Http301Error => e
+        # These exceptions indicate bucket already exists
+        # Aws::S3::Errors::Http301Error could be inaccurate but compromising for simplicity
+        true
+      rescue
+        false
+      end
+    end
+  end
+end

data/lib/jets/aws_services/stack_status.rb

@@ -0,0 +1,64 @@
+module Jets::AwsServices
+  module StackStatus
+    # Only cache if it is true because the initial deploy checks live status until a stack exists.
+    @@stack_exists_cache = [] # helps with CloudFormation rate limit
+    def stack_exists?(stack_name)
+      return false if Jets.env.test?
+      return true if ENV['JETS_BUILD_NO_INTERNET']
+      return true if @@stack_exists_cache.include?(stack_name)
+
+      exist = nil
+      begin
+        # When the stack does not exist an exception is raised. Example:
+        # Aws::CloudFormation::Errors::ValidationError: Stack with id blah does not exist
+        cfn.describe_stacks(stack_name: stack_name)
+        @@stack_exists_cache << stack_name
+        exist = true
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        if e.message =~ /does not exist/
+          exist = false
+        elsif e.message.include?("'stackName' failed to satisfy constraint")
+          # Example of e.message when describe_stack with invalid stack name
+          # "1 validation error detected: Value 'instance_and_route53' at 'stackName' failed to satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z][-a-zA-Z0-9]*|arn:[-a-zA-Z0-9:/._+]*"
+          puts "Invalid stack name: #{stack_name}"
+          puts "Full error message: #{e.message}"
+          exit 1
+        else
+          raise # re-raise exception  because unsure what other errors can happen
+        end
+      end
+      exist
+    end
+
+    # All CloudFormation states listed here: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-describing-stacks.html
+    #
+    # Returns resp so we can use it to grab data about the stack without calling api again.
+    def stack_in_progress?(stack_name)
+      return true if !stack_exists?(stack_name)
+
+      # Assumes stack exists
+      resp = cfn.describe_stacks(stack_name: stack_name)
+      status = resp.stacks[0].stack_status
+      if status =~ /_IN_PROGRESS$/
+        puts "The '#{stack_name}' stack status is #{status}. " \
+             "Please wait until the stack is ready and try again.".color(:red)
+        exit 0
+      elsif resp.stacks[0].outputs.empty? && status != 'ROLLBACK_COMPLETE'
+        # This Happens when the miminal stack fails at the very beginning.
+        # There is no s3 bucket at all.  User should delete the stack.
+        puts "The minimal stack failed to create. Please delete the stack first and try again. " \
+        "You can delete the CloudFormation stack or use the `jets delete` command"
+        exit 0
+      else
+        true
+      end
+    end
+
+    # Lookup output value.
+    # Used in Jets::Resource::ApiGateway::RestApi::* andJets::Commands::Url
+    def lookup(outputs, key)
+      out = outputs.find { |o| o.output_key == key }
+      out&.output_value
+    end
+  end
+end

data/lib/jets/aws_services.rb

@@ -0,0 +1,95 @@
+require "aws-sdk-apigateway"
+require "aws-sdk-cloudformation"
+require "aws-sdk-cloudwatchlogs"
+require "aws-sdk-dynamodb"
+require "aws-sdk-lambda"
+require "aws-sdk-s3"
+require "aws-sdk-sts"
+# Not used in Jets internally but convenient for shared resources
+require "aws-sdk-sns"
+require "aws-sdk-sqs"
+
+require "aws_mfa_secure/ext/aws" # add MFA support
+
+module Jets::AwsServices
+  include GlobalMemoist
+  include StackStatus
+
+  def apigateway
+    Aws::APIGateway::Client.new(aws_options)
+  end
+  global_memoize :apigateway
+
+  def cfn
+    Aws::CloudFormation::Client.new(aws_options)
+  end
+  global_memoize :cfn
+
+  def dynamodb
+    Aws::DynamoDB::Client.new(aws_options)
+  end
+  global_memoize :dynamodb
+
+  def aws_lambda
+    Aws::Lambda::Client.new(aws_options)
+  end
+  global_memoize :aws_lambda
+
+  def logs
+    Aws::CloudWatchLogs::Client.new(aws_options)
+  end
+  global_memoize :logs
+
+  def s3
+    Aws::S3::Client.new(aws_options)
+  end
+  global_memoize :s3
+
+  def s3_resource
+    Aws::S3::Resource.new(aws_options)
+  end
+  global_memoize :s3_resource
+
+  def sns
+    Aws::SNS::Client.new(aws_options)
+  end
+  global_memoize :sns
+
+  def sqs
+    Aws::SQS::Client.new(aws_options)
+  end
+  global_memoize :sqs
+
+  def sts
+    Aws::STS::Client.new(aws_options)
+  end
+  global_memoize :sts
+
+  # Override the AWS retry settings with Jets AWS clients.
+  #
+  # The aws-sdk-core has exponential backup with this formula:
+  #
+  #   2 ** c.retries * c.config.retry_base_delay
+  #
+  # So the max delay will be 2 ** 7 * 0.6 = 76.8s
+  #
+  # Only scoping this to deploy because dont want to affect people's application that use the aws sdk.
+  #
+  # There is also additional rate backoff logic elsewhere, since this is only scoped to deploys.
+  #
+  # Useful links:
+  #   https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/plugins/retry_errors.rb
+  #   https://docs.aws.amazon.com/apigateway/latest/developerguide/limits.html
+  #
+  def aws_options
+    options = {
+      retry_limit: 7, # default: 3
+      retry_base_delay: 0.6, # default: 0.3
+    }
+    options.merge!(
+      log_level: :debug,
+      logger: Logger.new($stdout),
+    ) if ENV['JETS_DEBUG_AWS_SDK']
+    options
+  end
+end

data/lib/jets/booter.rb

@@ -0,0 +1,167 @@
+require "jets/core_ext/kernel" # Hack prevents Rails const from being defined
+
+class Jets::Booter
+  class << self
+    @booted = false
+    def boot!
+      return if @booted
+
+      confirm_jets_project!
+      Jets::Bundle.require
+
+      Jets.application.setup!
+
+      # Turbines are loaded after setup_autoload_paths in Jets.application.setup!  Some Turbine options are defined
+      # in the project so setup must happen before internal Turbines are loaded.
+      load_internal_turbines
+
+      run_turbines(:initializers)
+      # Load configs after Turbine initializers so Turbines can defined some config options and they are available in
+      # user's project environment configs.
+      Jets.application.configs!
+      app_initializers
+      run_turbines(:after_initializers)
+      Jets.application.finish!
+
+      setup_db # establish db connections in Lambda Execution Context.
+      # The eager load calls connects_to in models and establish those connections in Lambda Execution Context also.
+      internal_finisher
+      eager_load
+
+      # TODO: Figure out how to build middleware during Jets.boot without breaking jets new and webpacker:install
+      # build_middleware_stack
+
+      @booted = true
+    end
+
+    # Runs right before eager_load
+    def internal_finisher
+      load_shared_extensions
+    end
+
+    # Shared extensions are added near the end because they require the Jets app load paths to first.
+    # We eager load the extensions and then use the loaded modules to extend Jets::Stack directly.
+    # Originally used an included hook but thats too early before app/shared/extensions is in the load_path.
+    def load_shared_extensions
+      Jets::Autoloaders.once.preload("#{Jets.root}/app/shared/extensions")
+      base_path = "#{Jets.root}/app/shared/extensions"
+      Dir.glob("#{base_path}/**/*.rb").each do |path|
+        next unless File.file?(path)
+
+        class_name = path.sub("#{base_path}/", '').sub(/\.rb/,'').camelize
+        mod = class_name.constantize # autoload
+        Jets::Stack.extend(mod)
+      end
+    end
+
+    def eager_load
+      preload_extensions
+      Jets::Autoloaders.main.eager_load # Eager load project code. Rather have user find out early than later on AWS Lambda.
+    end
+
+    def preload_extensions
+      base_path = "#{Jets.root}/app/extensions"
+      Dir.glob("#{base_path}/**/*.rb").each do |path|
+        next unless File.file?(path)
+
+        class_name = path.sub("#{base_path}/", '').sub(/\.rb/,'').camelize
+        klass = class_name.constantize # autoload
+        Jets::Lambda::Functions.extend(klass)
+      end
+    end
+
+    # Using ActiveRecord outside of Rails, so we need to set up the db connection ourself.
+    #
+    # Only connects to database for ActiveRecord and when config/database.yml exists.
+    # Dynomite handles connecting to the clients lazily.
+    def setup_db
+      return unless File.exist?("#{Jets.root}/config/database.yml")
+
+      db_configs = Jets.application.config.database
+      # DatabaseTasks.database_configuration for db:create db:migrate tasks
+      # Documented in DatabaseTasks that this is the right way to set it when
+      # using ActiveRecord rake tasks outside of Rails.
+      ActiveRecord::Tasks::DatabaseTasks.database_configuration = db_configs
+
+      if db_configs.configs_for(env_name: Jets.env).blank?
+        abort("ERROR: config/database.yml exists but no environment section configured for #{Jets.env}")
+      end
+      ActiveRecord::Base.configurations = db_configs
+      connect_db
+    end
+
+    # Eager connect to database, so connections are established in the Lambda Execution Context and get reused.
+    # Interestingly, the connections info is stored in the shared state but the connection doesnt show up on
+    # `show processlist` until after a query. Have confirmed that the connection is reused and the connection count stays
+    # the same.
+    def connect_db
+      primary_hash_config = ActiveRecord::Base.configurations.configs_for(env_name: Jets.env).find { |hash_config|
+        hash_config.name == "primary"
+      }
+      primary_config = primary_hash_config.configuration_hash # configuration_hash is a normal Ruby Hash
+      ActiveRecord::Base.establish_connection(primary_config)
+    end
+
+    def load_internal_turbines
+      Dir.glob("#{__dir__}/internal/turbines/**/*.rb").each do |path|
+        Jets::Autoloaders.once.preload(path)
+      end
+    end
+
+    # All Turbines
+    def turbine_initializers
+      Jets::Turbine.subclasses.each do |subclass|
+        initializers = subclass.initializers || []
+        initializers.each do |label, block|
+          block.call(Jets.application)
+        end
+      end
+    end
+
+    # All Turbines
+    def app_initializers
+      Dir.glob("#{Jets.root}/config/initializers/**/*").sort.each do |path|
+        load path
+      end
+    end
+
+    # run_turbines(:initializers)
+    # run_turbines(:after_initializers)
+    def run_turbines(name)
+      Jets::Turbine.subclasses.each do |subclass|
+        hooks = subclass.send(name) || []
+        hooks.each do |label, block|
+          block.call(Jets.application)
+        end
+      end
+    end
+
+    # Builds and memoize stack so it only gets built on bootup
+    def build_middleware_stack
+      Jets.application.build_stack
+    end
+
+    # Cannot call this for the jets new
+    def confirm_jets_project!
+      unless File.exist?("#{Jets.root}/config/application.rb")
+        puts "It does not look like you are running this command within a jets project.  Please confirm that you are in a jets project and try again.".color(:red)
+        exit 1
+      end
+    end
+
+    def message
+      "Jets booting up in #{Jets.env.color(:green)} mode!"
+    end
+
+    def check_config_ru!
+      config_ru = File.read("#{Jets.root}/config.ru")
+      unless config_ru.include?("Jets.boot")
+        puts 'The config.ru file is missing Jets.boot.  Please add Jets.boot after require "jets"'.color(:red)
+        puts "This was changed as made in Jets v1.1.0."
+        puts "To have Jets update the config.fu file for you, you can run:\n\n"
+        puts "  jets upgrade"
+        exit 1
+      end
+    end
+  end
+end