jets 0.6.7 → 0.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8dbba7d45f12322d2e782fa668a76f36a6818967e237681af10007bf95f2bdd
-  data.tar.gz: 4a9be05f3d832828ad0362b3f897ac57ad7c47a450b457f7b38fa4bdd3e6ab3e
+  metadata.gz: bb8be5eb8194cd0e2d6ac3f718e4ecdb2a4a14ef785b513e32bb097a9402e417
+  data.tar.gz: e70d4f88f3e0b8701a16c2115cdc5ad8a463447ba45ffc6adcf32e494a431690
 SHA512:
-  metadata.gz: 64b4ea7178ecfda89454067f08232b3f2cdb0f6df32ca6ff626026b7d5709dd1ac69fec5833da1a29762910f5fcb0963f5cc9f20895444c26666cea873601111
-  data.tar.gz: 6c008490354e438e024e839a48c26917a01e3d62e4727183314884ec19f7dde5404175ec971c30e7e7cb35e53c95a3f854830de53fe466a9ffb936de9b85432e
+  metadata.gz: eef24ef7e8533bab5e98b3ca06205f219ec46ad7e0a8ba84cdf9a687ee9122253b17942e661990b2fb82be4aa28501076eae9626b8aea4958b2347241ca7a75f
+  data.tar.gz: 98f1272457a69bc6ae50f093f32c2f130c3ca32dbf0235fbf82bd449612f2001edc3faab2f0957c7df7dd7c823548806d4329a62f1e11d9fa1346ca48d66c267

data/.codebuild/README.md

@@ -50,8 +50,19 @@ If you want to manually build the Docker base image.  Run:
       -e "IMAGE_NAME=tongueroo/jets:base" \
       -e "ARTIFACTS=/tmp/artifacts" \
       -e "SOURCE=/home/ec2-user/environment/jets" \
+      -e "DB_USER=$DB_USER" \
+      -e "DB_PASS=$DB_PASS" \
+      -e "DB_HOST=$DB_HOST" \
       amazon/aws-codebuild-local
 
-## Docker base image
+## Run ingreration.sh
 
+Can run the integration.sh test locally by running:
 
+    export DB_NAME=demo
+    export DB_USER=dbuser
+    export DB_PASS=dbpass
+    export DB_HOST=rdshost
+    .codebuild/integration.sh
+
+Note, you'll need to use a real RDS db instance.  Make sure DATABASE_URL is not set, this is working with the DB_* vars.

data/.codebuild/integration.sh

@@ -1,20 +1,37 @@
 #!/bin/bash -exu
 
-cp .codebuild/bin/jets /usr/local/bin/jets
-chmod a+x /usr/local/bin/jets
-export PATH=/usr/local/bin:$PATH
+# on codebuild make sure we use the bin/jets that was checked out
+set +u
+if [ -n "$CODEBUILD_SRC_DIR" ]; then
+  cp .codebuild/bin/jets /usr/local/bin/jets
+  chmod a+x /usr/local/bin/jets
+  export PATH=/usr/local/bin:$PATH
+  which jets
+fi
 
-which jets
+# Locally set the CODEBUILD_SRC_DIR to make script simpler
+# Do this before the cd-ing into the newly created directory.
+if [ -z $CODEBUILD_SRC_DIR ]; then
+  CODEBUILD_SRC_DIR=$(pwd)
+fi
+set -u
 
 APP_NAME=demo$(date +%s)
 jets new $APP_NAME # jets new runs bundle and webpacker:install
 cd $APP_NAME
 
-cat >>.env.development <<EOL
+cat >.env.development <<EOL
+DB_NAME=$DB_NAME
 DB_USER=$DB_USER
 DB_PASS=$DB_PASS
 DB_HOST=$DB_HOST
 EOL
+# IE: replace demo with demo1535227161 in case locally using demo and forget to set
+perl -i -pe "s/DB_NAME=demo/DB_NAME=${APP_NAME}_dev/" .env.development
+# Make sure database env vars matches whats in the file
+eval $(cat .env.development | sed 's/^/export /')
+# Make sure that nothing is .env.development.remote, dont want to use it for this case
+cp /dev/null .env.development.remote
 
 jets generate scaffold Post title:string
 # The DB_ environment variables are set up in the circleci environment variables
@@ -42,6 +59,7 @@ cat > jets.postman_environment.json <<EOL
   "_postman_exported_using": "Postman/6.2.3"
 }
 EOL
+
 cp $CODEBUILD_SRC_DIR/.codebuild/jets.postman_collection.json .
 
 npm install -g newman

data/.gitignore

@@ -17,3 +17,4 @@ tmp
 
 spec/fixtures/project/handlers
 .codebuild/definitions
+demo1*

data/CHANGELOG.md

@@ -1,7 +1,11 @@
 # Change Log
 
 All notable changes to this project will be documented in this file.
-This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
+This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
+
+## [0.6.8]
+- add jets clean:log and clean:build commands pull request #17
+- allow integration.sh test script to run locally
 
 ## [0.6.7]
 - eager load jets lib also, pull request #16

data/Gemfile.lock

@@ -11,12 +11,13 @@ GIT
 PATH
   remote: .
   specs:
-    jets (0.6.7)
+    jets (0.6.8)
       actionpack (>= 5.2.1)
       actionview (>= 5.2.1)
       activerecord (>= 5.2.1)
       activesupport (>= 5.2.1)
       aws-sdk-cloudformation
+      aws-sdk-cloudwatchlogs
       aws-sdk-dynamodb
       aws-sdk-lambda
       aws-sdk-s3
@@ -67,6 +68,9 @@ GEM
     aws-sdk-cloudformation (1.6.0)
       aws-sdk-core (~> 3)
       aws-sigv4 (~> 1.0)
+    aws-sdk-cloudwatchlogs (1.5.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
     aws-sdk-core (3.24.1)
       aws-eventstream (~> 1.0)
       aws-partitions (~> 1.0)

data/jets.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "recursive-open-struct"
   spec.add_dependency "aws-sdk-s3"
   spec.add_dependency "aws-sdk-cloudformation"
+  spec.add_dependency "aws-sdk-cloudwatchlogs"
   spec.add_dependency "aws-sdk-dynamodb"
   spec.add_dependency "aws-sdk-lambda"
   spec.add_dependency "kramdown"

data/lib/jets/aws_services.rb

@@ -1,5 +1,6 @@
 require "aws-sdk-s3"
 require "aws-sdk-cloudformation"
+require "aws-sdk-cloudwatchlogs"
 require "aws-sdk-lambda"
 require "aws-sdk-sts"
 
@@ -24,6 +25,10 @@ module Jets::AwsServices
     @sts ||= Aws::STS::Client.new
   end
 
+  def logs
+    @logs ||= Aws::CloudWatchLogs::Client.new
+  end
+
   def stack_exists?(stack_name)
     return false if ENV['TEST']
 

data/lib/jets/commands.rb

@@ -19,4 +19,5 @@ module Jets::Commands
   autoload :Dbconsole, "jets/commands/dbconsole"
   autoload :Url, "jets/commands/url"
   autoload :Markdown, "jets/commands/markdown"
+  autoload :Clean, "jets/commands/clean"
 end

data/lib/jets/commands/clean.rb

@@ -0,0 +1,23 @@
+module Jets::Commands
+  class Clean < Jets::Commands::Base
+    autoload :Base, 'jets/commands/clean/base'
+    autoload :Log, 'jets/commands/clean/log'
+    autoload :Build, 'jets/commands/clean/build'
+
+    class_option :noop, type: :boolean, desc: "noop or dry-run mode"
+    class_option :mute, type: :boolean, desc: "mute output"
+    class_option :sure, type: :boolean, desc: "bypass are you sure prompt"
+
+    desc "log", "Cleans CloudWatch log groups assocated with app"
+    long_desc Help.text('clean:log')
+    def log
+      Log.new(options).clean
+    end
+
+    desc "build", "Cleans jets build"
+    long_desc Help.text('clean:build')
+    def build
+      Build.new(options).clean
+    end
+  end
+end

data/lib/jets/commands/clean/base.rb

@@ -0,0 +1,24 @@
+class Jets::Commands::Clean
+  class Base
+    def initialize(options={})
+      @options = options
+    end
+
+  private
+    def say(message)
+      prefix = 'NOOP ' if @options[:noop]
+      puts "#{prefix}#{message}" unless @options[:mute]
+    end
+
+    def are_you_sure?(message)
+      return true if @options[:sure]
+
+      puts "Are you sure that you want to #{message}? (y/N)"
+      yes = $stdin.gets.strip
+      unless yes =~ /^y/
+        puts "Phew that was close!"
+        exit 0
+      end
+    end
+  end
+end

data/lib/jets/commands/clean/build.rb

@@ -0,0 +1,13 @@
+require 'fileutils'
+
+class Jets::Commands::Clean
+  class Build < Base
+    def clean
+      are_you_sure?("delete /tmp/jets")
+
+      say "Removing /tmp/jets..."
+      FileUtils.rm_rf("/tmp/jets") unless @options[:noop]
+      say "Removed /tmp/jets"
+    end
+  end
+end

data/lib/jets/commands/clean/log.rb

@@ -0,0 +1,89 @@
+# The thing that limits this implementation is that there needs to be at least
+# one lambda function created from an internal jets function. Example:
+#
+#   /aws/lambda/demo-dev-2-jets-preheat_job-warm
+#   /aws/lambda/demo-dev-2-jets-public_controller-show
+#
+# We're doing this because JETS_ENV_EXTRA environments can create additional matching
+# log groups and we don't want to overly-aggressively delete them.
+#
+# The `keep_prefixes(log_group_names)` method calcuates the log groups to keep.
+class Jets::Commands::Clean
+  class Log < Base
+    extend Memoist
+    include Jets::AwsServices
+
+    def clean
+      are_you_sure?("delete CloudWatch logs")
+
+      say "Removing CloudWatch logs for #{prefix_guess}..."
+      log_groups.each do |g|
+        next if keep_log_group?(g.log_group_name)
+        logs.delete_log_group(log_group_name: g.log_group_name) unless @options[:noop]
+        say "Removed log group: #{g.log_group_name}"
+      end
+      say "Removed CloudWatch logs for #{prefix_guess}"
+    end
+
+  private
+    def prefix_guess
+      Jets::Naming.parent_stack_name
+    end
+
+    def log_groups
+      groups, next_token = [], true
+      while next_token
+        next_token = nil if next_token == true # just at start the loop
+        resp = logs.describe_log_groups(
+          log_group_name_prefix: "/aws/lambda/#{prefix_guess}-",
+          next_token: next_token,
+        )
+        groups += resp.log_groups
+        next_token = resp.next_token
+      end
+      groups
+    end
+    memoize :log_groups
+
+    def log_group_names
+      log_groups.map(&:log_group_name)
+    end
+
+    def all_prefixes(log_group_names)
+      log_prefixes(log_group_names)
+    end
+    memoize :all_prefixes
+
+    # Check for the prefixes to keep. The slightly tricky thing to watch for is
+    # for the prefix matching addiitonal log groups that belong to other
+    # JETS_ENV_EXTRA=xxx created environments.
+    #
+    # We find and store the prefixes to keep so we don't over aggressively delete
+    # log groups.
+    def keep_prefixes(log_group_names)
+      names = log_group_names.reject do |name|
+        name =~ %r{/aws/lambda/#{prefix_guess}-jets}
+      end
+      log_prefixes(names)
+    end
+    memoize :keep_prefixes
+
+    # Strips -jets.* from the full log group name to leave only the prefix behind
+    def log_prefixes(names)
+      names = names.select do |name|
+        name.match(Regexp.new("#{prefix_guess}-.*jets"))
+      end
+      names.map do |name|
+        name.sub(/-jets.*/,'')
+      end.uniq.sort
+    end
+
+    # Check if it is safe to delete the log group
+    def keep_log_group?(log_group_name)
+      keep_prefixes = keep_prefixes(log_group_names)
+      !!keep_prefixes.detect do |keep_prefix|
+        log_group_name =~ Regexp.new(keep_prefix)
+      end
+    end
+  end
+end

data/lib/jets/commands/delete.rb

@@ -23,6 +23,9 @@ class Jets::Commands::Delete
     puts "Deleting #{Jets.config.project_namespace.colorize(:green)}..."
 
     wait_for_stack if @options[:wait]
+
+    delete_logs
+
     puts "Project #{Jets.config.project_namespace.colorize(:green)} deleted!"
   end
 
@@ -34,6 +37,12 @@ class Jets::Commands::Delete
     puts "Time took for deletion: #{status.pretty_time(took).green}."
   end
 
+  def delete_logs
+    puts "Deleting CloudWatch logs"
+    log = Jets::Commands::Clean::Log.new(mute: true, sure: true)
+    log.clean
+  end
+
   def confirm_project_exists
     begin
       resp = cfn.describe_stacks(stack_name: parent_stack_name)

data/lib/jets/commands/help/clean/build.md

@@ -0,0 +1,5 @@
+Removes the build files that jets creates. Essentially, deletes `/tmp/jets`. This will remove all build files for all jets projects. This is safe as jets uses `/tmp/jets` merely as a cache to speed up incrementally builds. Cleaning this out can clean up cruft in the `/tmp/jets` directory that builds over time.
+
+## Example
+
+    jets clean:build

data/lib/jets/commands/help/clean/log.md

@@ -0,0 +1,5 @@
+Essentially removes the CloudWatch groups assocated with the app. Lambda requests re-create the log groups so this is pretty safe to do.
+
+## Example
+
+    jets log:clean

data/lib/jets/version.rb

@@ -1,3 +1,3 @@
 module Jets
-  VERSION = "0.6.7"
+  VERSION = "0.6.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jets
 version: !ruby/object:Gem::Version
-  version: 0.6.7
+  version: 0.6.8
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-25 00:00:00.000000000 Z
+date: 2018-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -178,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudwatchlogs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-dynamodb
   requirement: !ruby/object:Gem::Requirement
@@ -461,6 +475,10 @@ files:
 - lib/jets/commands/call/autoload_guesser.rb
 - lib/jets/commands/call/base_guesser.rb
 - lib/jets/commands/call/guesser.rb
+- lib/jets/commands/clean.rb
+- lib/jets/commands/clean/base.rb
+- lib/jets/commands/clean/build.rb
+- lib/jets/commands/clean/log.rb
 - lib/jets/commands/console.rb
 - lib/jets/commands/db.rb
 - lib/jets/commands/db/environment-task.rake
@@ -474,6 +492,8 @@ files:
 - lib/jets/commands/help.rb
 - lib/jets/commands/help/build.md
 - lib/jets/commands/help/call.md
+- lib/jets/commands/help/clean/build.md
+- lib/jets/commands/help/clean/log.md
 - lib/jets/commands/help/console.md
 - lib/jets/commands/help/db/generate.md
 - lib/jets/commands/help/dbconsole.md