jets 3.1.4 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50e281f1856a1a82508629d805afe52156c44dc360f38d49528ca67e586ae8a4
-  data.tar.gz: ee5c33fe1a4f6929d1b285ba6a16acbfedcba1f229f34501f45c0983d4c71d6f
+  metadata.gz: e3693cb1f3b4143b90f9f41b3325d12c86a5095615c7fe843aa7a044561fa535
+  data.tar.gz: 0650ebb86666eac0779c167e4f926076c88b05481d8379f8ea7adaa2d6e35130
 SHA512:
-  metadata.gz: 531f6ccbc1f750cd08b4ac8575017c12de6f21104d014b983a14270e9b6a2760e097a03c5672de4ec989cbbefb52cf6f25d37a2a2459743c2c49e72c1b715985
-  data.tar.gz: 116483cc99bb26c464de5665a7abfaba09c95710a298026a25acfbcb4c42bfce0df48a6efba6f937031fb54db7b91485f83e18b0c048ec4c10837682c89d8076
+  metadata.gz: b8887e1a8af4cac8f4617aa422bc64daa52300d14abecfb130c074e27a45d35bafd4ba3090490d75de3cc091b4580450b03f39f27f815439d7cd31f34cb91ae7
+  data.tar.gz: 0c463af7c1675eb7658689eaec0d4026341db788e0c31c82f078b80963cce7a8939914bcf59f7c4434552bb7433536756375c1b8d582dc1166f29f3b337f9bca

data/CHANGELOG.md

@@ -3,6 +3,21 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/).
 
+## [3.2.0] - 2022-12-03
+- [#631](https://github.com/boltops-tools/jets/pull/631) Implement vpc_endpoint_ids configuration
+- [#634](https://github.com/boltops-tools/jets/pull/634) Custom domain messaging
+- [#635](https://github.com/boltops-tools/jets/pull/635) smarter apigw routes paging calculation
+- Introduce the storage of APIGW routes state in the jets managed s3 bucket under path jets/state/apigw/pages.json and jets/state/apigw/routes.json
+- This speeds up deployment for apps with a large number of routes. Depending on the internet connection this can be minutes faster.
+- Update internationalization.md
+- fix domain.name check when APIGW changes
+- improve change apigw endpoint change messaging
+
+## [3.1.5] - 2022-08-08
+- [#628](https://github.com/boltops-tools/jets/pull/628) Multiple Databases
+- [#629](https://github.com/boltops-tools/jets/pull/629) loosen zeitwerk dependency in generated project Gemfile
+- [#630](https://github.com/boltops-tools/jets/pull/630) compile assets with WEBPACKER_ASSET_HOST as s3 endpoint url
+
 ## [3.1.4] - 2022-07-02
 - [#627](https://github.com/boltops-tools/jets/pull/627) use >= for most gem dependencies
 

data/lib/jets/application/defaults.rb

@@ -77,10 +77,11 @@ class Jets::Application
       config.inflections.irregular = {}
 
       config.assets = ActiveSupport::OrderedOptions.new
-      config.assets.folders = %w[assets images packs]
       config.assets.base_url = nil # IE: https://cloudfront.com/my/base/path
-      config.assets.max_age = 3600
       config.assets.cache_control = nil # IE: public, max-age=3600 , max_age is a shorter way to set cache_control.
+      config.assets.folders = %w[assets images packs]
+      config.assets.max_age = 3600
+      config.assets.webpacker_asset_host = "s3_endpoint"  # true = use conventional by default
 
       config.ruby = ActiveSupport::OrderedOptions.new
 
@@ -98,6 +99,7 @@ class Jets::Application
       config.api.cors_authorization_type = nil # nil so ApiGateway::Cors#cors_authorization_type handles
       config.api.endpoint_policy = nil # required when endpoint_type is EDGE
       config.api.endpoint_type = 'EDGE' # PRIVATE, EDGE, REGIONAL
+      config.api.vpc_endpoint_ids = nil
 
       config.api.authorizers = ActiveSupport::OrderedOptions.new
       config.api.authorizers.default_token_source = "Auth" # method.request.header.Auth

data/lib/jets/aws_services.rb

@@ -86,10 +86,18 @@ module Jets::AwsServices
       retry_limit: 7, # default: 3
       retry_base_delay: 0.6, # default: 0.3
     }
+    # See debug logger. Noisy.
+    # Example:
+    #     D, [2022-12-02T13:18:55.298788 #26182] DEBUG -- : [Aws::APIGateway::Client 200 0.030837 0 retries] get_method(rest_api_id:"mke40eh6l0",resource_id:"zf8w2m",http_method:"GET")
     options.merge!(
       log_level: :debug,
       logger: Logger.new($stdout),
     ) if ENV['JETS_DEBUG_AWS_SDK']
+    # https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/debugging.html to enable http_wire_trace
+    # See the HTTP headers and JSON responses. Super noisy.
+    options.merge!(
+      http_wire_trace: true,
+    ) if ENV['JETS_DEBUG_AWS_SDK_HTTP_WIRE_TRACE']
     options
   end
 end

data/lib/jets/booter.rb

@@ -94,11 +94,23 @@ class Jets::Booter
     # `show processlist` until after a query. Have confirmed that the connection is reused and the connection count stays
     # the same.
     def connect_db
-      primary_hash_config = ActiveRecord::Base.configurations.configs_for(env_name: Jets.env).find { |hash_config|
-        hash_config.name == "primary"
-      }
-      primary_config = primary_hash_config.configuration_hash # configuration_hash is a normal Ruby Hash
-      ActiveRecord::Base.establish_connection(primary_config)
+      if ActiveRecord::Base.legacy_connection_handling
+        primary_hash_config = ActiveRecord::Base.configurations.configs_for(env_name: Jets.env).find { |hash_config|
+          hash_config.name == "primary"
+        }
+  
+        primary_config = primary_hash_config.configuration_hash # configuration_hash is a normal Ruby Hash
+
+        ActiveRecord::Base.establish_connection(primary_config)
+      else
+        configs = ActiveRecord::Base.configurations.configs_for(env_name: Jets.env, include_replicas: true)
+
+        databases = { }
+        databases[:writing] = :primary if configs.any? { |config| config.name == "primary" }
+        databases[:reading] = :primary_replica if configs.any? { |config| config.name == "primary_replica" }
+
+        ActiveRecord::Base.connects_to database: databases
+      end
     end
 
     def load_internal_turbines

data/lib/jets/builders/code_builder.rb

@@ -205,12 +205,45 @@ module Jets::Builders
       return unless webpacker_included?
 
       sh("yarn install")
+
+      ENV['WEBPACKER_ASSET_HOST'] = webpacker_asset_host if Jets.config.assets.webpacker_asset_host
       webpack_command = File.exist?("#{Jets.root}/bin/webpack") ?
           "bin/webpack" :
           `which webpack`.strip
       sh "JETS_ENV=#{Jets.env} #{webpack_command}"
     end
 
+    # Different url for these. Examples:
+    #
+    #   webpacker_asset_host https://demo-dev-s3bucket-lw5vq7ht8ip4.s3.us-west-2.amazonaws.com/jets/public/packs/media/images/boltops-0dd1c6bd.png
+    #   s3_base_url          https://s3-us-west-2.amazonaws.com/demo-dev-s3bucket-lw5vq7ht8ip4/jets/packs/media/images/boltops-0dd1c6bd.png
+    #
+    # Interesting: webpacker_asset_host works but s3_base_url does not for CORs. IE: reactjs or vuejs requests
+    # Thinking AWS configures the non-subdomain url endpoint to be more restrictive.
+    #
+    def webpacker_asset_host
+      # Allow user to set assets.webpacker_asset_host
+      #
+      #   Jets.application.configure do
+      #     config.assets.webpacker_asset_host = "https://cloudfront.com/my/base/path"
+      #   end
+      #
+      assets = Jets.config.assets
+      return assets.webpacker_asset_host if assets.webpacker_asset_host && assets.webpacker_asset_host != "s3_endpoint"
+      return assets.base_url if assets.base_url
+
+      # By default, will use the s3 url endpoint directly by convention
+      return unless assets.webpacker_asset_host == "s3_endpoint"
+
+      region = Jets.aws.region
+
+      asset_base_url = region == 'us-east-1' ?
+        "https://#{s3_bucket}.s3.amazonaws.com" :
+        "https://#{s3_bucket}.s3.#{region}.amazonaws.com"
+
+      "#{asset_base_url}/jets/public" # s3_base_url
+    end
+
     def webpacker_included?
       # Old code, leaving around for now:
       # Thanks: https://stackoverflow.com/questions/4195735/get-list-of-gems-being-used-by-a-bundler-project

data/lib/jets/cfn/builders/api_gateway_builder.rb

@@ -49,7 +49,7 @@ module Jets::Cfn::Builders
 
     def add_route53_dns
       dns = Jets::Resource::Route53::RecordSet.new
-      if !existing_domain_name?(dns.domain_name) or existing_dns_record_on_stack? 
+      if !existing_domain_name?(dns.domain_name) or existing_dns_record_on_stack?
         add_resource(dns)
         add_outputs(dns.outputs)
       end
@@ -57,11 +57,11 @@ module Jets::Cfn::Builders
 
     def create_domain_name()
       resource = Jets::Resource::ApiGateway::DomainName.new
-      
+
       return {
         "DomainName" => resource.domain_name
       } if (existing_domain_name?(resource) and !existing_domain_name_on_stack?)
-      
+
       add_resource(resource)
       return resource.outputs
     end
@@ -114,8 +114,9 @@ module Jets::Cfn::Builders
     def add_gateway_routes
       # Reject homepage. Otherwise we have 200 - 1 resources on the first page.
       # There's a next call in ApiResources.add_gateway_resources to skip the homepage.
-      all_paths = Jets::Router.all_paths.reject { |p| p == '' }
-      all_paths.each_slice(PAGE_LIMIT).each_with_index do |paths, i|
+      page_builder = Jets::Cfn::Builders::PageBuilder.new
+      pages = page_builder.build
+      pages.each_with_index do |paths, i|
         ApiResourcesBuilder.new(@options, paths, i+1).build
       end
     end

data/lib/jets/cfn/builders/page_builder.rb

@@ -0,0 +1,80 @@
+module Jets::Cfn::Builders
+  class PageBuilder
+    extend Memoist
+    cattr_reader :pages
+
+    # Build page slices
+    def build
+      map = build_map
+      pages = []
+      map.each do |path, existing_page|
+        if existing_page
+          pages[existing_page] ||= []
+          pages[existing_page] << path
+        end
+      end
+
+      # Remove existing paths from map. Leave behind new paths
+      pages.each do |page|
+        page.each do |i|
+          map.delete(i)
+        end
+      end
+
+      # Fill up available space in each page so all existing pages are full
+      keys = map.keys
+      pages.each do |page|
+        break if keys.empty?
+        while page.size < page_limit
+          path = keys.shift
+          break if path.nil?
+          page << path
+        end
+      end
+
+      # Add remaining slices to new additional pages
+      pages += keys.each_slice(page_limit).to_a
+
+      @@pages = pages
+
+      pages
+    end
+
+    # Build map that has paths as keys and page number as value
+    # Example: {"a1"=>0, "a2"=>0, "b1"=>1, "b2"=>1, "c1"=>2, "c2"=>2}
+    def build_map
+      map = {}
+      new_paths.each do |path|
+         map[path] = find_page_index(path)
+      end
+      map
+    end
+
+    def find_page_index(new_path)
+      pages = old_pages || []
+      pages.each_with_index do |slice, i|
+        slice.find do |old_path|
+          return i if old_path == new_path
+        end
+      end
+      nil
+    end
+
+    def old_pages
+      state = Jets::Router::State.new
+      state.load("pages")
+    end
+    memoize :old_pages
+
+    def new_paths
+      Jets::Router.all_paths.reject { |p| p == '' }
+    end
+
+    # Relevant is CloudFormation Outputs limit is 200
+    # JETS_APIGW_PAGE_LIMIT is based on that
+    # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
+    def page_limit
+      Integer(ENV['JETS_APIGW_PAGE_LIMIT'] || 200)
+    end
+  end
+end

data/lib/jets/cfn/ship.rb

@@ -42,6 +42,7 @@ module Jets::Cfn
         exit 1
       end
 
+      save_apigw_state
       prewarm
       clean_deploy_logs
       show_api_endpoint
@@ -88,6 +89,12 @@ module Jets::Cfn
       Jets::Cfn::Status.new(@options).wait
     end
 
+    def save_apigw_state
+      state = Jets::Router::State.new
+      state.save("pages", Jets::Cfn::Builders::PageBuilder.pages)
+      state.save("routes", Jets::Router.routes)
+    end
+
     def prewarm
       if ENV['SKIP_PREWARMING']
         puts "Skipping prewarming" # useful for testing

data/lib/jets/commands/templates/skeleton/Gemfile.tt

@@ -18,7 +18,7 @@ gem "mysql2", "~> 0.5.3"
 <% unless options[:mode] == 'job' -%>
 gem "dynomite"
 <% end -%>
-gem "zeitwerk", "~> 2.5.0"
+gem "zeitwerk", ">= 2.5.0"
 
 # development and test groups are not bundled as part of the deployment
 group :development, :test do

data/lib/jets/resource/api_gateway/rest_api/logical_id/message.rb

@@ -8,8 +8,7 @@ class Jets::Resource::ApiGateway::RestApi::LogicalId
     end
 
     def custom_domain
-      api = Jets::Resource::ApiGateway::DomainName.new
-      domain_name = api.domain_name
+      domain_name = Jets.config.domain.name
       if domain_name
         <<~EOL
         It looks like you have already set up a custom domain.
@@ -25,7 +24,18 @@ class Jets::Resource::ApiGateway::RestApi::LogicalId
         More info: custom domain docs: https://rubyonjets.com/docs/routing/custom-domain/
         EOL
       else
-        "Please set up a custom domain https://rubyonjets.com/docs/routing/custom-domain/"
+        <<~EOL
+        To avoid this prompt in the future, you can configure:
+
+        config/application.rb
+
+            config.api.auto_replace = true
+
+        However, you should also set up a custom domain for a "stable" endpoint.
+
+        https://rubyonjets.com/docs/routing/custom-domain/
+
+        EOL
       end
     end
 

data/lib/jets/resource/api_gateway/rest_api/routes/change/base.rb

@@ -7,51 +7,22 @@ class Jets::Resource::ApiGateway::RestApi::Routes::Change
       new.changed?
     end
 
-    # Build up deployed routes from the existing CloudFormation resources.
+    # Recreate routes from previously deployed stored state in s3
     def deployed_routes
-      routes = []
-
-      resources, position = [], true
-      while position
-        position = nil if position == true # start of loop
-        resp = apigateway.get_resources(
-          rest_api_id: rest_api_id,
-          position: position,
-          limit: 500, # default: 25 max: 500
+      state = Jets::Router::State.new
+      data = state.load("routes")
+      return [] if data.nil?
+
+      data.map do |item|
+        Jets::Router::Route.new(
+          path: item['path'],
+          method: item['options']['method'],
+          to: item['to'],
         )
-        resources += resp.items
-        position = resp.position
-      end
-
-      resources.each do |resource|
-        resource_methods = resource.resource_methods
-        next if resource_methods.nil?
-
-        resource_methods.each do |http_verb, resource_method|
-          # puts "#{http_verb} #{resource.path} | resource.id #{resource.id}"
-          # puts to(resource.id, http_verb)
-
-          # Test changing config.cors and CloudFormation does an in-place update
-          # on the resource. So no need to do bluegreen deployments for OPTIONS.
-          next if http_verb == "OPTIONS"
-
-          path = recreate_path(resource.path)
-          method = http_verb.downcase.to_sym
-          to = to(resource.id, http_verb)
-          route = Jets::Router::Route.new(path: path, method: method, to: to)
-          routes << route
-        end
       end
-      routes
     end
     memoize :deployed_routes
 
-    def recreate_path(path)
-      path = path.gsub(%r{^/},'')
-      path = path.gsub(/{([^}]*)\+}/, '*\1')
-      path.gsub(/{([^}]*)}/, ':\1')
-    end
-
     def to(resource_id, http_method)
       uri = method_uri(resource_id, http_method)
       recreate_to(uri) unless uri.nil?

data/lib/jets/resource/api_gateway/rest_api.rb

@@ -5,6 +5,7 @@ module Jets::Resource::ApiGateway
         name: Jets::Naming.gateway_api_name,
         endpoint_configuration: { types: endpoint_types }
       }
+      properties[:endpoint_configuration][:vpc_endpoint_ids] = vpce_ids if vpce_ids
       properties[:binary_media_types] = binary_media_types if binary_media_types
       properties[:policy] = endpoint_policy if endpoint_policy
 
@@ -60,5 +61,14 @@ module Jets::Resource::ApiGateway
 
       endpoint_policy
     end
+
+    private
+
+    def vpce_ids
+      ids = Jets.config.api.vpc_endpoint_ids
+      return nil if ids.nil? || ids.empty?
+
+      ids
+    end
   end
-end
+end

data/lib/jets/router/route.rb

@@ -191,6 +191,10 @@ class Jets::Router
       @options[:mount_class]
     end
 
+    def to_h
+      JSON.load(to_json)
+    end
+
   private
     def ensure_jets_format(path)
       path.split('/').map do |s|

data/lib/jets/router/state.rb

@@ -0,0 +1,47 @@
+class Jets::Router
+  class State
+    extend Memoist
+    include Jets::AwsServices
+
+    def load(filename)
+      resp = s3.get_object(
+        bucket: Jets.aws.s3_bucket,
+        key: s3_storage_path(filename),
+      )
+      text = resp.body.read
+      JSON.load(text)
+    rescue
+    end
+    memoize :load
+
+    # Save previously deployed APIGW routes state
+    def save(filename, data)
+      # body = Jets::Router.routes.to_json
+      # body = JSON.generate(Jets::Cfn::Builders::PageBuilder.pages)
+      body = data.respond_to?(:to_json) ? data.to_json : JSON.generate(data)
+      s3.put_object(
+        body: body,
+        bucket: Jets.aws.s3_bucket,
+        key: s3_storage_path(filename),
+      )
+    end
+
+    # Examples:
+    #
+    #   jets/state/apigw/pages.json
+    #   jets/state/apigw/routes.json
+    #
+    # Fetch or loaded in:
+    #
+    #   pages.json: Jets::Cfn::Builders::PageBuilder#old_pages
+    #   routes.json: Jets::Resource::ApiGateway::RestApi::Routes::Change::Base#deployed_routes
+    #
+    # Saved in:
+    #
+    #   Jets::Cfn::Ship#save_apigw_state
+    #
+    def s3_storage_path(filename)
+      "jets/state/apigw/#{filename}.json"
+    end
+  end
+end

data/lib/jets/router.rb

@@ -134,6 +134,18 @@ module Jets
         drawn_router.routes
       end
 
+      # So we can save state in s3 post deploy. Example of structure.
+      #
+      #   [
+      #     {"scope"=>{"options"=>{"as"=>"posts", "prefix"=>"posts", "param"=>nil, "from"=>"resources"}, "parent"=>{"options"=>{}, "parent"=>nil, "level"=>1}, "level"=>2}, "options"=>{"to"=>"posts#index", "from_scope"=>true, "path"=>"posts", "method"=>"get"}, "path"=>"posts", "to"=>"posts#index", "as"=>"posts"},
+      #     {"scope"=>{"options"=>{"as"=>"posts", "prefix"=>"posts", "param"=>nil, "from"=>"resources"}, "parent"=>{"options"=>{}, "parent"=>nil, "level"=>1}, "level"=>2}, "options"=>{"to"=>"posts#new", "from_scope"=>true, "path"=>"posts/new", "method"=>"get"}, "path"=>"posts/new", "to"=>"posts#new", "as"=>"new_post"},
+      #     ...
+      #   ]
+      #
+      def to_json
+        JSON.dump(routes.map(&:to_h))
+      end
+
       # Returns all paths including subpaths.
       # Example:
       # Input: ["posts/:id/edit"]

data/lib/jets/version.rb

@@ -1,3 +1,3 @@
 module Jets
-  VERSION = "3.1.4"
+  VERSION = "3.2.0"
 end

data/lib/jets.rb

@@ -17,6 +17,7 @@ require "active_support/ordered_options"
 require "cfn_camelizer"
 require "cfn_status"
 require "fileutils"
+require "json"
 require "memoist"
 require "rainbow/ext/string"
 require "serverlessgems"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jets
 version: !ruby/object:Gem::Version
-  version: 3.1.4
+  version: 3.2.0
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-07-02 00:00:00.000000000 Z
+date: 2022-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: actionmailer
@@ -661,6 +661,7 @@ files:
 - lib/jets/cfn/builders/function_builder.rb
 - lib/jets/cfn/builders/interface.rb
 - lib/jets/cfn/builders/job_builder.rb
+- lib/jets/cfn/builders/page_builder.rb
 - lib/jets/cfn/builders/parent_builder.rb
 - lib/jets/cfn/builders/parent_builder/stagger.rb
 - lib/jets/cfn/builders/rule_builder.rb
@@ -986,6 +987,7 @@ files:
 - lib/jets/router/route.rb
 - lib/jets/router/route/authorizer.rb
 - lib/jets/router/scope.rb
+- lib/jets/router/state.rb
 - lib/jets/router/util.rb
 - lib/jets/rule/base.rb
 - lib/jets/rule/dsl.rb