jets 2.1.1 → 2.1.2

data/lib/jets/commands/delete.rb

@@ -44,13 +44,26 @@ class Jets::Commands::Delete
   end
 
   def confirm_project_exists
+    retries = 0
     begin
-      resp = cfn.describe_stacks(stack_name: parent_stack_name)
+      cfn.describe_stacks(stack_name: parent_stack_name)
     rescue Aws::CloudFormation::Errors::ValidationError
       # Aws::CloudFormation::Errors::ValidationError is thrown when the stack
       # does not exist
       puts "The parent stack #{Jets.config.project_namespace.color(:green)} for the project #{Jets.config.project_name.color(:green)} does not exist. So it cannot be deleted."
       exit 0
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      retries += 1
+      seconds = 2 ** retries
+
+      puts "WARN: confirm_project_exists #{e.class} #{e.message}".color(:yellow)
+      puts "Backing off and will retry in #{seconds} seconds."
+      sleep(seconds)
+      if seconds > 90 # 2 ** 6 is 64 so will give up after 6 retries
+        puts "Giving up after #{retries} retries"
+      else
+        retry
+      end
     end
   end
 

data/lib/jets/commands/deploy.rb

@@ -1,3 +1,5 @@
+require "aws-sdk-core"
+
 module Jets::Commands
   class Deploy
     extend Memoist
@@ -7,6 +9,7 @@ module Jets::Commands
     end
 
     def run
+      aws_config_update!
       deployment_env = Jets.config.project_namespace.color(:green)
       puts "Deploying to Lambda #{deployment_env} environment..."
       return if @options[:noop]
@@ -34,6 +37,29 @@ module Jets::Commands
       ship(stack_type: :full, s3_bucket: s3_bucket)
     end
 
+    # Override the AWS retry settings during a deploy.
+    #
+    # The aws-sdk-core has expondential backup with this formula:
+    #
+    #   2 ** c.retries * c.config.retry_base_delay
+    #
+    # So the max delay will be 2 ** 7 * 0.6 = 76.8s
+    #
+    # Only scoping this to deploy because dont want to affect people's application that use the aws sdk.
+    #
+    # There is also additional rate backoff logic elsewhere, since this is only scoped to deploys.
+    #
+    # Useful links:
+    #   https://github.com/aws/aws-sdk-ruby/blob/master/gems/aws-sdk-core/lib/aws-sdk-core/plugins/retry_errors.rb
+    #   https://docs.aws.amazon.com/apigateway/latest/developerguide/limits.html
+    #
+    def aws_config_update!
+      Aws.config.update(
+        retry_limit: 7, # default: 3
+        retry_base_delay: 0.6, # default: 0.3
+      )
+    end
+
     def create_s3_event_buckets
       buckets = Jets::Job::Base.s3_events.keys
       buckets.each do |bucket|
@@ -60,19 +86,11 @@ module Jets::Commands
     end
 
     def validate_routes!
-      check_route_connected_functions
-    end
-
-    # Checks that all routes are validate and have corresponding lambda functions
-    def check_route_connected_functions
-      return if Jets::Router.all_routes_valid
-
-      puts "Deploy fail: The jets application contain invalid routes.".color(:red)
-      puts "Please double check the routes below map to valid controllers:"
-      Jets::Router.invalid_routes.each do |route|
-        puts "  /#{route.path} => #{route.controller_name}##{route.action_name}"
+      valid = Jets::Router.validate_routes!
+      unless valid
+        puts "Deploy fail: The jets application contain invalid routes.".color(:red)
+        exit 1
       end
-      exit 1
     end
 
     def ship(stack_options)
@@ -108,6 +126,7 @@ module Jets::Commands
     end
 
     def find_stack(stack_name)
+      retries = 0
       resp = cfn.describe_stacks(stack_name: stack_name)
       resp.stacks.first
     rescue Aws::CloudFormation::Errors::ValidationError => e
@@ -117,6 +136,18 @@ module Jets::Commands
       else
         raise
       end
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      retries += 1
+      seconds = 2 ** retries
+
+      puts "WARN: find_stack #{e.class} #{e.message}".color(:yellow)
+      puts "Backing off and will retry in #{seconds} seconds."
+      sleep(seconds)
+      if seconds > 90 # 2 ** 6 is 64 so will give up after 6 retries
+        puts "Giving up after #{retries} retries"
+      else
+        retry
+      end
     end
 
     # All CloudFormation states listed here: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-describing-stacks.html

data/lib/jets/commands/help/build.md

@@ -3,4 +3,4 @@ Builds a zip file package to be uploaded to AWS Lambda. This allows you to build
 * your application code
 * generated shims
 
-If the application has no Ruby code and only uses Polymorphic functions, then gems are not bundled up.
+If the application has no Ruby code and only uses Polymorphic functions, then gems are not bundled up.

data/lib/jets/commands/help/{destroy.md → degenerate.md}

@@ -2,7 +2,7 @@ This piggy backs off of the [rails scaffold destroy](https://guides.rubyonrails.
 
 ## Example
 
-    $ jets destroy scaffold Post title:string body:text published:boolean
+    $ jets degenerate scaffold post title:string body:text published:boolean
           invoke  active_record
           remove    db/migrate/20190225231821_create_posts.rb
           remove    app/models/post.rb

data/lib/jets/commands/help/upgrade.md

@@ -3,3 +3,5 @@ Upgrades the Jets project structure to the latest version. This command is desig
 ## Example
 
     $ jets upgrade
+
+Refer to https://rubyonjets.com/docs/extras/upgrading/

data/lib/jets/commands/main.rb

@@ -52,6 +52,7 @@ module Jets::Commands
     long_desc Help.text(:routes)
     def routes
       puts Jets::Router.help(Jets::Router.routes)
+      Jets::Router.validate_routes!
     end
 
     desc "console", "REPL console with Jets environment loaded"
@@ -103,7 +104,7 @@ module Jets::Commands
     end
 
     desc "degenerate [type] [args]", "Destroys things like scaffolds"
-    long_desc Help.text(:generate) # do use Jets::Generator.help as it'll load Rails const
+    long_desc Help.text(:degenerate) # do use Jets::Generator.help as it'll load Rails const
     def degenerate(generator, *args)
       Jets::Generator.revoke(generator, *args)
     end

data/lib/jets/commands/templates/skeleton/Gemfile.tt

@@ -19,6 +19,7 @@ gem "mysql2", "~> 0.5.2"
 gem "dynomite"
 <% end -%>
 
+# development and test groups are not bundled as part of the deployment
 group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console
   gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]

data/lib/jets/naming.rb

@@ -30,6 +30,10 @@ class Jets::Naming
       "#{template_path_prefix}-api-gateway.yml"
     end
 
+    def api_resources_template_path(page)
+      "#{template_path_prefix}-api-resources-#{page}.yml"
+    end
+
     def api_deployment_template_path
       "#{template_path_prefix}-api-deployment.yml"
     end

data/lib/jets/resource/api_gateway/resource.rb

@@ -37,16 +37,20 @@ module Jets::Resource::ApiGateway
       path.empty? ? 'Homepage route: /' : "Route for: /#{path}"
     end
 
-    def parent_id
+    def parent_path_parameter
       if @path.include?('/') # posts/:id or posts/:id/edit
         parent_path = @path.split('/')[0..-2].join('/')
         parent_logical_id = path_logical_id(parent_path)
-        "!Ref " + Jets::Resource.truncate_id("#{parent_logical_id}ApiResource")
+        Jets::Resource.truncate_id("#{parent_logical_id}ApiResource")
       else
-        "!GetAtt #{RestApi.logical_id(@internal)}.RootResourceId"
+        "RootResourceId"
       end
     end
 
+    def parent_id
+      "!Ref " + parent_path_parameter
+    end
+
     def path_part
       last_part = path.split('/').last
       last_part.split('/').map {|s| transform_capture(s) }.join('/') if last_part

data/lib/jets/resource/api_gateway/rest_api/change_detection.rb

@@ -4,39 +4,7 @@ class Jets::Resource::ApiGateway::RestApi
     include Jets::AwsServices
 
     def changed?
-      return false unless parent_stack_exists?
-      current_binary_media_types != new_binary_media_types ||
       Routes.changed?
     end
-
-    def new_binary_media_types
-      rest_api = Jets::Resource::ApiGateway::RestApi.new
-      rest_api.binary_media_types
-    end
-    memoize :new_binary_media_types
-
-    # Duplicated in rest_api/change_detection.rb, base_path/role.rb, rest_api/routes.rb
-    def current_binary_media_types
-      return nil unless parent_stack_exists?
-
-      stack = cfn.describe_stacks(stack_name: parent_stack_name).stacks.first
-
-      api_gateway_stack_arn = lookup(stack[:outputs], "ApiGateway")
-
-      stack = cfn.describe_stacks(stack_name: api_gateway_stack_arn).stacks.first
-      rest_api_id = lookup(stack[:outputs], "RestApi")
-
-      resp = apigateway.get_rest_api(rest_api_id: rest_api_id)
-      resp.binary_media_types
-    end
-    memoize :current_binary_media_types
-
-    def parent_stack_exists?
-      stack_exists?(parent_stack_name)
-    end
-
-    def parent_stack_name
-      Jets::Naming.parent_stack_name
-    end
   end
 end

data/lib/jets/resource/api_gateway/rest_api/routes/change.rb

@@ -1,8 +1,16 @@
 # Detects route changes
 class Jets::Resource::ApiGateway::RestApi::Routes
   class Change
+    include Jets::AwsServices
+
     def changed?
-      To.changed? || Variable.changed? || ENV['JETS_REPLACE_API']
+      return false unless parent_stack_exists?
+
+      MediaTypes.changed? || To.changed? || Variable.changed? || Page.changed? || ENV['JETS_REPLACE_API']
+    end
+
+    def parent_stack_exists?
+      stack_exists?(Jets::Naming.parent_stack_name)
     end
   end
 end

data/lib/jets/resource/api_gateway/rest_api/routes/change/base.rb

@@ -3,6 +3,10 @@ class Jets::Resource::ApiGateway::RestApi::Routes::Change
     extend Memoist
     include Jets::AwsServices
 
+    def self.changed?
+      new.changed?
+    end
+
     # Build up deployed routes from the existing CloudFormation resources.
     def deployed_routes
       routes = []
@@ -13,6 +17,7 @@ class Jets::Resource::ApiGateway::RestApi::Routes::Change
         resp = apigateway.get_resources(
           rest_api_id: rest_api_id,
           position: position,
+          limit: 500, # default: 25 max: 500
         )
         resources += resp.items
         position = resp.position
@@ -53,11 +58,27 @@ class Jets::Resource::ApiGateway::RestApi::Routes::Change
     end
 
     def method_uri(resource_id, http_method)
-      resp = apigateway.get_method(
-        rest_api_id: rest_api_id,
-        resource_id: resource_id,
-        http_method: http_method
-      )
+      # https://docs.aws.amazon.com/apigateway/latest/developerguide/limits.html
+      retries = 0
+      begin
+        resp = apigateway.get_method(
+          rest_api_id: rest_api_id,
+          resource_id: resource_id,
+          http_method: http_method
+        )
+      rescue Aws::APIGateway::Errors::TooManyRequestsException => e
+        retries += 1
+        seconds = 2 ** retries
+
+        puts "WARN: method_uri #{e.class} #{e.message}".color(:yellow)
+        puts "Backing off and will retry in #{seconds} seconds."
+        sleep(seconds)
+        if seconds > 90 # 2 ** 6 is 64 so will give up after 6 retries
+          puts "Giving up after #{retries} retries"
+        else
+          retry
+        end
+      end
       resp.method_integration.uri
     end
 

data/lib/jets/resource/api_gateway/rest_api/routes/change/media_types.rb

@@ -0,0 +1,36 @@
+class Jets::Resource::ApiGateway::RestApi::Routes::Change
+  class MediaTypes < Base
+    def changed?
+      current_binary_media_types != new_binary_media_types
+    end
+
+    def new_binary_media_types
+      rest_api = Jets::Resource::ApiGateway::RestApi.new
+      rest_api.binary_media_types
+    end
+    memoize :new_binary_media_types
+
+    def current_binary_media_types
+      return nil unless parent_stack_exists?
+
+      stack = cfn.describe_stacks(stack_name: parent_stack_name).stacks.first
+
+      api_gateway_stack_arn = lookup(stack[:outputs], "ApiGateway")
+
+      stack = cfn.describe_stacks(stack_name: api_gateway_stack_arn).stacks.first
+      rest_api_id = lookup(stack[:outputs], "RestApi")
+
+      resp = apigateway.get_rest_api(rest_api_id: rest_api_id)
+      resp.binary_media_types
+    end
+    memoize :current_binary_media_types
+
+    def parent_stack_exists?
+      stack_exists?(parent_stack_name)
+    end
+
+    def parent_stack_name
+      Jets::Naming.parent_stack_name
+    end
+  end
+end

data/lib/jets/resource/api_gateway/rest_api/routes/change/page.rb

@@ -0,0 +1,93 @@
+class Jets::Resource::ApiGateway::RestApi::Routes::Change
+  class Page < Base
+    def changed?
+      route_page_moved? || old_api_template?
+    end
+
+    def route_page_moved?
+      moved?(new_pages, deployed_pages)
+    end
+
+    # Routes page to logical ids
+    def moved?(new_pages, deployed_pages)
+      not_moved = true # page has not moved
+      new_pages.each do |logical_id, new_page_number|
+        if !deployed_pages[logical_id].nil? && deployed_pages[logical_id] != new_page_number
+          not_moved = false # page has moved
+          break
+        end
+      end
+      !not_moved # moved
+    end
+
+    def new_pages
+      local_logical_ids_map
+    end
+    memoize :new_pages
+
+    def deployed_pages
+      remote_logical_ids_map
+    end
+    memoize :deployed_pages
+
+    # logical id to page map
+    # Important: In Cfn::Builders::ApiGatewayBuilder, the add_gateway_routes and ApiResourcesBuilder needs to run
+    # before the parent add_gateway_rest_api method.
+    def local_logical_ids_map(path_expression="#{Jets::Naming.template_path_prefix}-api-resources-*.yml")
+      logical_ids = {} # logical id => page number
+
+      Dir.glob(path_expression).each do |path|
+        md = path.match(/-api-resources-(\d+).yml/)
+        page_number = md[1]
+
+        template = Jets::Cfn::BuiltTemplate.get(path)
+        template['Resources'].keys.each do |logical_id|
+          logical_ids[logical_id] = page_number
+        end
+      end
+
+      logical_ids
+    end
+
+    # aws cloudformation describe-stack-resources --stack-name demo-dev-ApiResources1-DYGLIEY3VAWT | jq -r '.StackResources[].LogicalResourceId'
+    def remote_logical_ids_map
+      logical_ids = {} # logical id => page number
+
+      parent_resources.each do |resource|
+        stack_name = resource.physical_resource_id # full physical id can be used as stack name also
+        regexp = Regexp.new("#{Jets.config.project_namespace}-ApiResources(\\d+)-") # tricky to escape \d pattern
+        md = stack_name.match(regexp)
+        if md
+          page_number = md[1]
+
+          resp = cfn.describe_stack_resources(stack_name: stack_name)
+          resp.stack_resources.map(&:logical_resource_id).each do |logical_id|
+            logical_ids[logical_id] = page_number
+          end
+        end
+      end
+
+      logical_ids
+    end
+
+    def old_api_template?
+      logical_resource_ids = parent_resources.map(&:logical_resource_id)
+
+      api_gateway_found = logical_resource_ids.detect do |logical_id|
+        logical_id == "ApiGateway"
+      end
+      return false unless api_gateway_found
+
+      api_resources_found = logical_resource_ids.detect do |logical_id|
+        logical_id.match(/^ApiResources\d+$/)
+      end
+      !api_resources_found # if api_resources_found then it's the new structure. so opposite is old structure
+    end
+
+    def parent_resources
+      resp = cfn.describe_stack_resources(stack_name: Jets::Naming.parent_stack_name)
+      resp.stack_resources
+    end
+    memoize :parent_resources
+  end
+end