jets 2.1.1 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82e56d77408579a757373a115134668315a2c47fdbcd297f8eae827951274f6a
-  data.tar.gz: 0152fbf8fe291033319761e987667026a77958b67398ba237d9152cc70a00615
+  metadata.gz: bdc43f68b08d9c92593eeb3ccfc2477a1080af2cd4430e32f4766c2756c70b23
+  data.tar.gz: f842a52a1c9a448545f3a7ade22e1c14c8ce2a4d556169c574b4f5a58675c450
 SHA512:
-  metadata.gz: f638230df0a341ee7b4e0a73355a89a650a78178d7ebb46326c4215d40ea4d33d6648a3d0909a98789e416b4c03c59bae8b5c8e5a94e254c45de78915e5214c4
-  data.tar.gz: 5a9e606725ae25d18c3ce4b1ea901780d7062245035c20a9d5139566b0e92ab998d9c3e6d2bb14081a5e77c04bfa5bb871ed95db51ac48467c42f186292b82ab
+  metadata.gz: e4efe902131423d3aa95b2e9e42e9b7cdefc78871d0c6eedbc1bbc98ee5cb2771ce7e9ba6c908c00f018960ddbba6aa1e3d5ea745b31c3cb1593770f20bb1ebc
+  data.tar.gz: 363c2292150bf101d9ee5196de2bf82465d59cbb255d3c541c4fc3e44e08421b83bafe52c797a6ef07a7bdae63a727cc476000eadc914397243d1a587ad9824d

data/.codebuild/README.md

@@ -1,68 +1,18 @@
-# Code Build Commands
+## Update Project
 
-## The code definitions are stored in the codebuild s3 bucket.
+To update the CodeBuild project:
 
-    aws s3 cp .codebuild/definitions/jets_base.json s3://$S3_BUCKET/codebuild/definitions/jets_base.json
-    aws s3 cp .codebuild/definitions/jets_main.json s3://$S3_BUCKET/codebuild/definitions/jets_main.json
+Main services:
 
-## JetsBase
+    export AWS_PROFILE=bolt-oss
+    cb deploy jets --type docs # mainly cli docs and subnav update
 
-The JetsBase codebuild project projects builds the Docker base image and pushes it to Docker hub.  Here are the commands to manage the codebuild project.
+## Start a Deploy
 
-    aws codebuild create-project --cli-input-json file://.codebuild/definitions/jets_base.json
-    aws codebuild update-project --cli-input-json file://.codebuild/definitions/jets_base.json
+To start a CodeBuild build which kicks off a deploy:
 
-    aws codebuild start-build --project-name JetsBase --source-version codebuild
+    cb start jets --type docs
 
-    aws codebuild batch-get-projects --names JetsBase
-    aws codebuild list-builds-for-project --project-name JetsBase
+To specify a branch:
 
-    BUILD_ID=$(aws codebuild list-builds-for-project --project-name JetsBase | jq -r '.ids[0]')
-    aws codebuild batch-get-builds --ids $BUILD_ID
-
-    STREAM=$(aws codebuild batch-get-builds --ids $BUILD_ID | jq -r '.builds[0].logs.streamName')
-    cw tail -f /aws/codebuild/JetsBase $STREAM
-
-If you want to manually build the Docker base image.  Run:
-
-    docker build -t tongueroo/jets:base -f Dockerfile.base .
-    docker push tongueroo/jets:base
-
-## JetsMain
-
-    aws codebuild create-project --cli-input-json file://.codebuild/definitions/jets_main.json
-    aws codebuild update-project --cli-input-json file://.codebuild/definitions/jets_main.json
-
-    aws codebuild start-build --project-name JetsMain --source-version codebuild
-
-    aws codebuild batch-get-projects --names JetsMain
-    aws codebuild list-builds-for-project --project-name JetsMain
-
-    BUILD_ID=$(aws codebuild list-builds-for-project --project-name JetsMain | jq -r '.ids[0]')
-    aws codebuild batch-get-builds --ids $BUILD_ID
-
-    STREAM=$(aws codebuild batch-get-builds --ids $BUILD_ID | jq -r '.builds[0].logs.streamName')
-    cw tail -f /aws/codebuild/JetsMain $STREAM
-
-## Run CodeBuild Locally
-
-    time docker run -it -v /var/run/docker.sock:/var/run/docker.sock \
-      -e "IMAGE_NAME=tongueroo/jets:base" \
-      -e "ARTIFACTS=/tmp/artifacts" \
-      -e "SOURCE=/home/ec2-user/environment/jets" \
-      -e "DB_USER=$DB_USER" \
-      -e "DB_PASS=$DB_PASS" \
-      -e "DB_HOST=$DB_HOST" \
-      amazon/aws-codebuild-local
-
-## Run ingreration.sh
-
-Can run the integration.sh test locally by running:
-
-    export DB_NAME=demo
-    export DB_USER=dbuser
-    export DB_PASS=dbpass
-    export DB_HOST=rdshost
-    .codebuild/integration.sh
-
-Note, you'll need to use a real RDS db instance.  Make sure DATABASE_URL is not set, this is working with the DB_* vars.
+    cb start jets --type docs --branch codebuild

data/.codebuild/docs/bin/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -eux
+
+.codebuild/docs/bin/git_setup.sh
+.codebuild/docs/bin/cli_docs.sh
+.codebuild/docs/bin/subnav.sh
+.codebuild/docs/bin/git_commit.sh

data/.codebuild/docs/bin/cli_docs.sh

@@ -0,0 +1,5 @@
+#!/bin/bash -eux
+
+# Generate docs
+bundle
+rake docs

data/.codebuild/docs/bin/git_commit.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -eux
+
+out=$(git status docs)
+if [[ "$out" = *"nothing to commit"* ]]; then
+  exit
+fi
+
+COMMIT_MESSAGE="docs updated by codebuild"
+
+# If the last commit already updated the docs, then exit.
+# Preventable measure to avoid infinite loop.
+if git log -1 --pretty=oneline | grep "$COMMIT_MESSAGE" ; then
+  exit
+fi
+
+# If reach here, we have some changes on docs that we should commit.
+git add docs
+git commit -m "$COMMIT_MESSAGE"
+
+# SSH_KEY_S3_PATH set as codebuild environment variable
+aws s3 cp $SSH_KEY_S3_PATH ~/.ssh/id_rsa
+chmod 400 ~/.ssh/id_rsa
+sed -i -- 's|https://github.com/|git@github.com:|g' .git/config
+
+# https://makandracards.com/makandra/12107-git-show-current-branch-name-only
+current_branch=$(git rev-parse --abbrev-ref HEAD)
+git push origin "$current_branch"

data/.codebuild/docs/bin/git_setup.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -eux
+
+# git push:
+#   CODEBUILD_WEBHOOK_HEAD_REF=refs/heads/codebuild
+#   CODEBUILD_WEBHOOK_TRIGGER=branch/codebuild
+#   CODEBUILD_SOURCE_VERSION=f0eb542 # resolved sha
+# cb start:
+#   CODEBUILD_SOURCE_VERSION=codebuild
+
+set +u # cb start will not have CODEBUILD_WEBHOOK_TRIGGER set
+if [ -n "$CODEBUILD_WEBHOOK_TRIGGER" ]; then # git push
+  BRANCH=$(echo $CODEBUILD_WEBHOOK_TRIGGER | sed "s/.*\///")
+elif [ -n "$CODEBUILD_SOURCE_VERSION" ]; then # cb start
+  BRANCH=$CODEBUILD_SOURCE_VERSION # contains the actual branch
+else
+  BRANCH=UNKNOWN-BRANCH
+fi
+git checkout $BRANCH
+set -u

data/.codebuild/docs/bin/subnav.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -eux
+
+cd docs
+# Unsure why but if cli_docs.sh runs bundle first, then bundle wont use the right bundler file
+export BUNDLE_GEMFILE=$(pwd)/Gemfile
+bundle
+bundle exec jekyll serve --detach
+
+# Error: front_matter_parser-0.2.1/lib/front_matter_parser/syntax_parser/multi_line_comment.rb:19:in `match': invalid byte sequence in US-ASCII (ArgumentError)
+# Workaround: https://stackoverflow.com/questions/17031651/invalid-byte-sequence-in-us-ascii-argument-error-when-i-run-rake-dbseed-in-ra
+export RUBYOPT="-KU -E utf-8:utf-8" # fixes front_matter_parser error within the `jekyll-sort reorder` call
+
+gem install jekyll-sort
+jekyll-sort reorder # Updates nav_order front matter of pages that are in the nav

data/.codebuild/docs/buildspec.yml

@@ -0,0 +1,8 @@
+version: 0.2
+
+phases:
+  build:
+    commands:
+      - git config --global user.email "tongueroo@gmail.com"
+      - git config --global user.name "Tung Nguyen"
+      - .codebuild/docs/bin/build.sh

data/.codebuild/docs/project.rb

@@ -0,0 +1,10 @@
+github_url("https://github.com/tongueroo/jets.git")
+linux_image("aws/codebuild/ruby:2.5.3-1.7.0")
+triggers(
+  webhook: true,
+  filter_groups: [[{type: "HEAD_REF", pattern: "master"}, {type: "EVENT", pattern: "PUSH"}]]
+)
+
+environment_variables(
+  SSH_KEY_S3_PATH: "ssm:/codebuild/jets/ssh_key_s3_path"
+)

data/.gitignore

@@ -1,12 +1,13 @@
 *.gem
 *.rbc
 .bundle
+.byebug_history
 .config
 .yardoc
-InstalledFiles
 _yardoc
 coverage
 doc/
+InstalledFiles
 lib/bundler/man
 pkg
 rdoc
@@ -15,10 +16,9 @@ test/tmp
 test/version_tmp
 tmp
 
-spec/fixtures/project/handlers
 .codebuild/definitions
-demo*
 /html
+/demo
+Gemfile.lock
 spec/fixtures/apps/franky/dynamodb/migrate
-Gemfile.lock 
-.byebug_history
+spec/fixtures/project/handlers

data/CHANGELOG.md

@@ -3,6 +3,14 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/).
 
+## [2.1.2]
+- improve start .env.development example for postgres
+- #351 Update Turbine documentation
+- #355 improve jets routes command to also validate routes
+- #356 improvements to named route helpers
+- #357 paginate api gateway resources to support more routes
+- #358 rate limit backoff
+
 ## [2.1.1]
 - #347 Add documentation about torch vs. warm
 - #348 Provide another minimal privileges example

data/jets.gemspec

@@ -7,11 +7,11 @@ require "jets/rdoc"
 Gem::Specification.new do |spec|
   spec.name          = "jets"
   spec.version       = Jets::VERSION
-  spec.authors       = ["Tung Nguyen"]
-  spec.email         = ["tongueroo@gmail.com"]
-  spec.summary       = "Ruby Serverless Framework on AWS Lambda"
-  spec.description   = "Jets is a framework that allows you to create serverless applications with a beautiful language: Ruby. It includes everything required to build an application and deploy it to AWS Lambda. Jets makes serverless accessible to everyone."
-  spec.homepage      = "http://rubyonjets.com"
+  spec.author        = "Tung Nguyen"
+  spec.email         = "tongueroo@gmail.com"
+  spec.summary       = "Ruby Serverless Framework"
+  spec.description   = "Jets is a framework that allows you to create serverless applications with a beautiful language: Ruby. It includes everything required to build and deploy an application.  Jets leverages the power of Ruby to make serverless joyful for everyone."
+  spec.homepage      = "https://rubyonjets.com"
   spec.license       = "MIT"
 
   spec.required_ruby_version = '~> 2.5'

data/lib/jets/aws_services/stack_status.rb

@@ -1,13 +1,17 @@
 module Jets::AwsServices
   module StackStatus
+    # Only cache if it is true because the initial deploy checks live status until a stack exists.
+    @@stack_exists_cache = [] # helps with CloudFormation rate limit
     def stack_exists?(stack_name)
       return false if ENV['TEST']
+      return true if @@stack_exists_cache.include?(stack_name)
 
       exist = nil
       begin
         # When the stack does not exist an exception is raised. Example:
         # Aws::CloudFormation::Errors::ValidationError: Stack with id blah does not exist
-        resp = cfn.describe_stacks(stack_name: stack_name)
+        cfn.describe_stacks(stack_name: stack_name)
+        @@stack_exists_cache << stack_name
         exist = true
       rescue Aws::CloudFormation::Errors::ValidationError => e
         if e.message =~ /does not exist/

data/lib/jets/cfn/builders/api_gateway_builder.rb

@@ -12,9 +12,9 @@ module Jets::Cfn::Builders
     def compose
       return unless @options[:templates] || @options[:stack_type] != :minimal
 
-      add_gateway_rest_api
-      add_custom_domain
-      add_gateway_routes
+      add_gateway_routes # "child template": build before add_gateway_rest_api. RestApi logical id and change detection is dependent on it.
+      add_gateway_rest_api # changes parent template
+      add_custom_domain    # changes parent template
     end
 
     # template_path is an interface method
@@ -57,23 +57,14 @@ module Jets::Cfn::Builders
     end
 
     # Adds route related Resources and Outputs
+    # Delegates to ApiResourcesBuilder
+    PAGE_LIMIT = Integer(ENV['JETS_AWS_OUTPUTS_LIMIT'] || 60) # Allow override for testing
     def add_gateway_routes
-      # The routes required a Gateway Resource to contain them.
-      # TODO: Support more routes. Right now outputing all routes in 1 template will hit the 60 routes limit.
-      # Will have to either output them as a joined string or break this up to multiple templates.
-      # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
-      # Outputs: Maximum number of outputs that you can declare in your AWS CloudFormation template. 60 outputs
-      # Output name: Maximum size of an output name. 255 characters.
-      #
-      # Note we must use .all_paths, not .routes here because we need to
-      # build the parent ApiGateway::Resource nodes also
-      Jets::Router.all_paths.each do |path|
-        homepage = path == ''
-        next if homepage # handled by RootResourceId output already
-
-        resource = Jets::Resource::ApiGateway::Resource.new(path, internal: true)
-        add_resource(resource)
-        add_outputs(resource.outputs)
+      # Reject homepage. Otherwise we have 60 - 1 resources on the first page.
+      # There's a next call in ApiResources.add_gateway_resources to skip the homepage.
+      all_paths = Jets::Router.all_paths.reject { |p| p == '' }
+      all_paths.each_slice(PAGE_LIMIT).each_with_index do |paths, i|
+        ApiResourcesBuilder.new(@options, paths, i+1).build
       end
     end
   end

data/lib/jets/cfn/builders/api_resources_builder.rb

@@ -0,0 +1,46 @@
+module Jets::Cfn::Builders
+  class ApiResourcesBuilder
+    include Interface
+    include Jets::AwsServices
+
+    def initialize(options={}, paths=[], page)
+      @options, @paths, @page = options, paths, page
+      @template = ActiveSupport::HashWithIndifferentAccess.new(Resources: {})
+    end
+
+    # compose is an interface method
+    def compose
+      return unless @options[:templates] || @options[:stack_type] != :minimal
+
+      add_rest_api_parameter
+      add_gateway_routes
+    end
+
+    # template_path is an interface method
+    def template_path
+      Jets::Naming.api_resources_template_path(@page)
+    end
+
+    def add_rest_api_parameter
+      add_parameter("RestApi", Description: "RestApi")
+    end
+
+    def add_gateway_routes
+      @paths.each do |path|
+        homepage = path == ''
+        next if homepage # handled by RootResourceId output already
+
+        resource = Jets::Resource::ApiGateway::Resource.new(path)
+        add_resource(resource)
+        add_outputs(resource.outputs)
+
+        parent_path = resource.parent_path_parameter
+        add_parameter(parent_path) unless part_of_template?(parent_path)
+      end
+    end
+
+    def part_of_template?(parent_path)
+      @template["Resources"].key?(parent_path)
+    end
+  end
+end

data/lib/jets/cfn/builders/parent_builder.rb

@@ -57,6 +57,7 @@ module Jets::Cfn::Builders
 
       if full? and !Jets::Router.routes.empty?
         add_api_gateway
+        add_api_resources
         add_api_deployment
       end
     end
@@ -80,6 +81,20 @@ module Jets::Cfn::Builders
       add_child_resources(resource)
     end
 
+    def add_api_resources
+      expression = "#{Jets::Naming.template_path_prefix}-api-resources-*"
+      # IE: path: #{Jets.build_root}/templates/demo-dev-2-api-resources-1.yml"
+      Dir.glob(expression).sort.each do |path|
+        next unless File.file?(path)
+
+        regexp = Regexp.new("#{Jets.config.project_namespace}-api-resources-(\\d+).yml") # tricky to escape \d pattern
+        md = path.match(regexp)
+        page = md[1]
+        resource = Jets::Resource::ChildStack::ApiResource.new(@options[:s3_bucket], page: page)
+        add_child_resources(resource)
+      end
+    end
+
     def add_api_deployment
       resource = Jets::Resource::ChildStack::ApiDeployment.new(@options[:s3_bucket])
       add_child_resources(resource)

data/lib/jets/cfn/built_template.rb

@@ -0,0 +1,15 @@
+module Jets::Cfn
+  # Caches the built template to reduce filesystem IO calls.
+  class BuiltTemplate
+    class << self
+      @@cache = {}
+      def get(path)
+        if @@cache[path]
+          @@cache[path] # using cache
+        else
+          @@cache[path] = YAML.load_file(path) # setting and using cache
+        end
+      end
+    end
+  end
+end

data/lib/jets/commands/clean/log.rb

@@ -19,12 +19,29 @@ class Jets::Commands::Clean
       say "Removing CloudWatch logs for #{prefix_guess}..."
       log_groups.each do |g|
         next if keep_log_group?(g.log_group_name)
-        logs.delete_log_group(log_group_name: g.log_group_name) unless @options[:noop]
+        delete_log_group(g.log_group_name) unless @options[:noop]
         say "Removed log group: #{g.log_group_name}"
       end
       say "Removed CloudWatch logs for #{prefix_guess}"
     end
 
+    def delete_log_group(log_group_name)
+      retries = 0
+      logs.delete_log_group(log_group_name: log_group_name)
+    rescue Aws::CloudWatchLogs::Errors::ThrottlingException => e
+      retries += 1
+      seconds = 2 ** retries
+
+      puts "WARN: delete_log_group #{e.class} #{e.message}".color(:yellow)
+      puts "Backing off and will retry in #{seconds} seconds."
+      sleep(seconds)
+      if seconds > 90 # 2 ** 6 is 64 so will give up after 6 retries
+        puts "Giving up after #{retries} retries"
+      else
+        retry
+      end
+    end
+
     def clean_deploys
       groups = deploy_log_groups.sort_by do |g|
         g.log_group_name