jets 0.10.4 → 1.0.0

data/lib/jets/cfn/upload.rb

@@ -0,0 +1,139 @@
+require 'action_view'
+require 'digest'
+
+class Jets::Cfn
+  class Upload
+    include Jets::AwsServices
+    include ActionView::Helpers::NumberHelper # number_to_human_size
+
+    attr_reader :bucket_name
+    def initialize(bucket_name)
+      @bucket_name = bucket_name
+    end
+
+    def upload
+      upload_cfn_templates
+      upload_zip_files
+      upload_assets
+    end
+
+    def upload_cfn_templates
+      puts "Uploading CloudFormation templates to S3."
+      expression = "#{Jets::Naming.template_path_prefix}-*"
+      Dir.glob(expression).each do |path|
+        next unless File.file?(path)
+
+        key = "jets/cfn-templates/#{File.basename(path)}"
+        obj = s3_resource.bucket(bucket_name).object(key)
+        obj.upload_file(path)
+      end
+    end
+
+    def upload_zip_files
+      puts "Uploading code zip files to S3."
+      zip_area = "#{Jets.build_root}/stage/zips"
+      Dir.glob("#{zip_area}/*").each do |file|
+        upload_zip_file(file)
+      end
+    end
+
+    def upload_zip_file(path)
+      file_size = number_to_human_size(File.size(path))
+
+      puts "Uploading #{path} (#{file_size}) to S3"
+      start_time = Time.now
+      s3_key = "jets/code/#{File.basename(path)}"
+      obj = s3_resource.bucket(bucket_name).object(s3_key)
+      obj.upload_file(path)
+      puts "Uploaded to s3://#{bucket_name}/#{s3_key}".colorize(:green)
+      puts "Time to upload code to s3: #{pretty_time(Time.now-start_time).colorize(:green)}"
+    end
+
+    def upload_assets
+      puts "Uploading modified public assets to S3."
+      start_time = Time.now
+      upload_public_assets
+      puts "Time to upload public assets to s3: #{pretty_time(Time.now-start_time).colorize(:green)}"
+    end
+
+    def upload_public_assets
+      asset_folders = Jets.config.assets.folders # IE: %w[public]
+      asset_folders = add_rack_assets(asset_folders)
+      asset_folders.each do |folder|
+        upload_asset_folder(folder)
+      end
+    end
+
+    def add_rack_assets(asset_folders)
+      return asset_folders unless Jets.rack?
+      asset_folders + %w[rack/public]
+    end
+
+    def upload_asset_folder(folder)
+      expression = "#{Jets.root}#{folder}/**/*"
+      group_size = 10
+      Dir.glob(expression).each_slice(group_size) do |paths|
+        threads = []
+        paths.each do |full_path|
+          next unless File.file?(full_path)
+
+          threads << Thread.new do
+            upload_to_s3(full_path)
+          end
+        end
+        threads.each(&:join)
+      end
+    end
+
+    def upload_to_s3(full_path)
+      return if identical_on_s3?(full_path)
+
+      key = s3_key(full_path)
+      obj = s3_resource.bucket(bucket_name).object(key)
+      puts "Uploading s3://#{bucket_name}/#{key}" # uncomment to see and debug
+      obj.upload_file(full_path, acl: "public-read", cache_control: cache_control)
+    end
+
+    def s3_key(full_path)
+      relative_path = full_path.sub(Jets.root.to_s, '')
+      "jets/#{relative_path}"
+    end
+
+    def identical_on_s3?(full_path)
+      local_md5 = ::Digest::MD5.file(full_path)
+      key = s3_key(full_path)
+      begin
+        resp = s3.head_object(bucket: bucket_name, key: key)
+      rescue Aws::S3::Errors::NotFound
+        return false
+      end
+
+      remote_md5 = resp.etag.delete_prefix('"').delete_suffix('"')
+      local_md5 == remote_md5
+    end
+
+    # If cache_control is provided, then it will set the entire cache-control header.
+    # If only max_age is provided, then we'll generate a cache_control header.
+    # Using max_age is the shorter and simply way of setting the cache_control header.
+    def cache_control
+      cache_control = Jets.config.assets.cache_control
+      unless cache_control
+        max_age = Jets.config.assets.max_age # defaults to 3600 in jets/application.rb
+        cache_control = "public, max-age=#{max_age}"
+      end
+      cache_control
+    end
+
+    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
+    def pretty_time(total_seconds)
+      minutes = (total_seconds / 60) % 60
+      seconds = total_seconds % 60
+      if total_seconds < 60
+        "#{seconds.to_i}s"
+      else
+        "#{minutes.to_i}m #{seconds.to_i}s"
+      end
+    end
+
+  end
+end

data/lib/jets/cfn.rb

@@ -2,4 +2,5 @@ class Jets::Cfn
   autoload :Builders, "jets/cfn/builders"
   autoload :Ship, "jets/cfn/ship"
   autoload :Status, "jets/cfn/status"
+  autoload :Upload, "jets/cfn/upload"
 end

data/lib/jets/commands/build.rb

@@ -31,16 +31,18 @@ module Jets::Commands
     time :build_code
 
     def build_templates
-      if @options[:templates] || @options[:stack_type] == :full
-        build_all_templates
-      else
-        build_minimal_template
-      end
+      puts "Building CloudFormation templates."
+      clean_templates
+      build_minimal_template
+      build_all_templates if full?
     end
     time :build_templates
 
+    def full?
+      @options[:templates] || @options[:stack_type] == :full
+    end
+
     def build_all_templates
-      clean_templates
       # CloudFormation templates
       # 1. Shared templates - child templates needs them
       build_api_gateway_templates
@@ -103,16 +105,15 @@ module Jets::Commands
       self.class.app_files
     end
 
-    # Crucial that the Dir.pwd is in the tmp_app_root because for
+    # Crucial that the Dir.pwd is in the tmp_code because for
     # because Jets.boot set ups autoload_paths and this is how project
     # classes are loaded.
-    # TODO: rework code so that Dir.pwd does not have to be in tmp_app_root for build to work.
+    # TODO: rework code so that Dir.pwd does not have to be in tmp_code for build to work.
     def self.app_files
       paths = []
       expression = "#{Jets.root}app/**/**/*.rb"
       Dir.glob(expression).each do |path|
         return false unless File.file?(path)
-        next if path.include?("app/functions") # cannot lazy load these because they are anonymous classes
         next unless app_file?(path)
 
         relative_path = path.sub(Jets.root.to_s, '')
@@ -144,15 +145,9 @@ module Jets::Commands
     def self.poly_only?
       # Scans all the app code and look for any methods that are ruby.
       # If any method is written in ruby then we know the app is not a
-      # soley polymorphic non-ruby app.
+      # solely polymorphic non-ruby app.
       has_ruby = app_files.detect do |path|
-        # 1. remove app/controllers or app/jobs, etc
-        # 2. remove .rb extension
-        app_file = path.sub(%r{app/\w+/},'').sub(/\.rb$/,'')
-
-        # Internal jets controllers like Welcome and Public need a different regexp
-        app_file = app_file.sub(%r{.*lib/jets/internal/},'')
-        app_class = app_file.classify.constantize # IE: PostsController, Jets::PublicController
+        app_class = Jets::Klass.from_path(path)  # IE: PostsController, Jets::PublicController
         langs = app_class.tasks.map(&:lang)
         langs.include?(:ruby)
       end
@@ -167,6 +162,9 @@ module Jets::Commands
       public_catchall = Jets::Router.has_controller?("Jets::PublicController")
       paths << "#{controllers}/public_controller.rb" if public_catchall
 
+      rack_catchall = Jets::Router.has_controller?("Jets::RackController")
+      paths << "#{controllers}/rack_controller.rb" if rack_catchall
+
       jobs = File.expand_path("../../internal/app/jobs/jets", __FILE__)
       paths << "#{jobs}/preheat_job.rb"
 
@@ -194,8 +192,8 @@ module Jets::Commands
       false
     end
 
-    def self.tmp_app_root(full_build_path=false)
-      full_build_path ? "#{Jets.build_root}/app_root" : "app_root"
+    def self.tmp_code(full_build_path=false)
+      full_build_path ? "#{Jets.build_root}/stage/code" : "stage/code"
     end
 
   end

data/lib/jets/commands/deploy.rb

@@ -21,6 +21,8 @@ module Jets::Commands
       exit_unless_updateable! # Stack could be in a weird rollback state or in progress state
       ship(stack_type: :minimal) if first_run?
 
+      Jets.application.reload_iam_policy!
+
       # Build code after the minimal stack because need s3 bucket for assets
       # on_aws? and s3_base_url logic
       build_code

data/lib/jets/commands/help/deploy.md

@@ -15,7 +15,7 @@ This builds the project and deploys it AWS Lambda. The deployment is mainly hand
     => Replacing compiled gems with AWS Lambda Linux compiled versions.
     Creating zip file.
     => cd /tmp/jets/demo/app_root && zip --symlinks -rq /tmp/jets/demo/code/code-temp.zip .
-    Building parent template.
+    Building CloudFormation templates.
     Deploying CloudFormation stack with jets app!
     Uploading /tmp/jets/demo/code/code-7169d0ac.zip (88.8 MB) to S3
     Time to upload code to s3: 1s
@@ -27,6 +27,6 @@ This builds the project and deploys it AWS Lambda. The deployment is mainly hand
     02:10:03AM UPDATE_COMPLETE AWS::CloudFormation::Stack demo-dev
     Stack success status: UPDATE_COMPLETE
     Time took for stack deployment: 1m 46s.
-    Prewarming application...
+    Prewarming application.
     API Gateway Endpoint: https://ewwnealfk0.execute-api.us-west-2.amazonaws.com/dev/
     $

data/lib/jets/commands/help/import/rack.md

@@ -0,0 +1,13 @@
+Imports a generic Rack application into a Jets project and configures it for [Mega Mode](http://rubyonjets.com/docs/megamode/).
+
+Note, generic rack projects will likely need some adjustments to take into account API Gateway stages and logging. For more info refer to [Mega Mode Considerations](http://rubyonjets.com//megamode-considerations/).
+
+## Example
+
+    jets import:rack http://github.com/tongueroo/jets-mega-rails.git
+
+## More Examples
+
+    jets import:rack tongueroo/jets-mega-rails # expands to github
+    jets import:rack git@github.com:tongueroo/jets-mega-rails.git
+    jets import:rack /path/to/folder/jets-mega-rails

data/lib/jets/commands/help/import/rails.md

@@ -0,0 +1,11 @@
+Imports a Rails application into a Jets project and configures it for [Mega Mode](http://rubyonjets.com/docs/rails-support/).
+
+## Example
+
+    jets import:rails http://github.com/tongueroo/jets-examples-rails.git
+
+## More Examples
+
+    jets import:rails tongueroo/jets-examples-rails # expands to github
+    jets import:rails git@github.com:tongueroo/jets-examples-rails.git
+    jets import:rails /path/to/folder/jets-examples-rails

data/lib/jets/commands/import/base.rb

@@ -0,0 +1,39 @@
+class Jets::Commands::Import
+  class Base < Sequence
+    # Since setup is public it will automatically run in the subclasses
+    def setup
+      start_message
+      create_rack_folder
+      configure_ruby
+      create_rackup_wrappers
+    end
+
+  private
+    def start_message
+      puts "Importing app into the rack folder..."
+    end
+
+    def create_rack_folder
+      repo? ? clone_project : copy_project
+    end
+
+    def configure_ruby
+      gsub_file 'rack/Gemfile', /^ruby(.*)/, '# ruby\1' # comment out ruby declaration
+      create_file "rack/.ruby-version", RUBY_VERSION, force: true
+    end
+
+    def create_rackup_wrappers
+      path = File.expand_path("../../builders/rackup_wrappers", File.dirname(__FILE__))
+      Dir.glob("#{path}/*").each do |p|
+        FileUtils.cp(p, "#{rack_folder}/bin/")
+      end
+      chmod "#{rack_folder}/bin/rackup", 0755
+    end
+
+    def bundle_install
+      Bundler.with_clean_env do
+        run "cd rack && bundle install"
+      end
+    end
+  end
+end

data/lib/jets/commands/import/rack.rb

@@ -0,0 +1,16 @@
+class Jets::Commands::Import
+  class Rack < Base
+    def install
+      bundle_install
+    end
+
+    def finish_message
+      puts <<~EOL
+        #{"="*30}
+        Congrats! The Rack project from #{@source} has been imported to the rack folder.
+
+        Note, generic rack projects will likely need some adjustments to take into account API Gateway stages and logging. For more info refer to [Mega Mode Considerations](http://rubyonjets.com//megamode-considerations/).
+      EOL
+    end
+  end
+end

data/lib/jets/commands/import/rail.rb

@@ -0,0 +1,68 @@
+class Jets::Commands::Import
+  class Rail < Base
+    # We add jets-rails gem even though it's only activated within the megamode
+    # rackup wrapper script so user can check run bundle install and determine early
+    # if dependencies are met. The jets-rails gem is also added as part of the deploy
+    # process but we add it here also so the user gets earlier possible errors and
+    # can fix before they hit the deploy phase.
+    def configure_gemfile
+      append_to_file 'rack/Gemfile' do
+        %Q|gem "jets-rails"\n|
+      end
+    end
+
+    def install
+      bundle_install
+    end
+
+    def configure_routes
+      # comment out current catchall route
+      comment_lines("config/routes.rb", 'any[ (][\'"]\*', verbose: true) # any "*catchall", to: ...
+
+      # Add catchall route for rack
+      insert_into_file "config/routes.rb", :before => /^end/ do
+        <<-CODE
+  # Enables Mega Mode Rails integration
+  any "*catchall", to: "jets/rack#process"
+CODE
+      end
+    end
+
+    def finish_message
+      puts <<~EOL
+        #{"="*30}
+        Congrats! The Rails project from #{@source} has been imported to the rack folder.  Here are some next steps:
+
+        # Local Testing
+
+        Check out the config/routes.rb file and noticed how a new catchall route has been added.  It looks something like this:
+
+            any "*catchall", to: "jets/rack#process"
+
+        The catchall route passes any route not handled by the Jets app as a request onto the Rails app.  You can modified the route to selectively route what you want.
+
+        Test the application locally. Test that the Rails app in the rack subfolder works independently.  You can start the application up with:
+
+            cd rack # cd into the imported Rails project
+            bundle exec rackup
+
+        The rack server starts up on http://localhost:9292  You might have to make sure that the database is configured.
+
+        Once tested, stop that server with CTRL-C.
+
+        Then you can start the jets server from the main jets project:
+
+            cd .. # back to the main jets projet
+            jets server # starts both jets and rack servers
+
+        The jets server starts up on http://localhost:8888  You can stop both servers with CTRL-C.
+
+        # Deploy
+
+        When you're ready deploy to AWS Lambda with:
+
+            jets deploy
+      EOL
+    end
+  end
+end

data/lib/jets/commands/import/sequence.rb

@@ -0,0 +1,68 @@
+require 'fileutils'
+require 'colorize'
+require 'active_support/core_ext/string'
+require 'thor'
+require 'bundler'
+
+class Jets::Commands::Import::Sequence < Thor::Group
+  include Thor::Actions
+  argument :source
+
+private
+  def clone_project
+    check_git_installed
+
+    command = "git clone #{repo_url} #{rack_folder}"
+    puts "=> #{command}"
+    if File.exist?(rack_folder)
+      abort "The folder #{rack_folder} already exists."
+    else
+      run command
+    end
+  end
+
+  def copy_project
+    puts "Creating rack folder"
+    template_path = File.expand_path(@source)
+    set_source_paths(template_path)
+    directory ".", rack_folder
+  end
+
+  def set_source_paths(*paths)
+    # Using string with instance_eval because block doesnt have access to
+    # path at runtime.
+    self.class.instance_eval %{
+      def self.source_paths
+        #{paths.flatten.inspect}
+      end
+    }
+  end
+
+  # normalize repo_url
+  def repo_url
+    if @source.include?('github.com')
+      @source # leave as is, user has provided full github url
+    else
+      "https://github.com/#{@source}"
+    end
+  end
+
+  def repo?
+    @source.include?('github.com')
+  end
+
+  def check_git_installed
+    return unless repo_url.include?('http') || repo_url.include?('git@')
+    unless git_installed?
+      abort "Unable to detect git installation on your system.  Git needs to be installed in order to clone the project."
+    end
+  end
+
+  def git_installed?
+    system("type git > /dev/null 2>&1")
+  end
+
+  def rack_folder
+    "#{Jets.root}rack"
+  end
+end

data/lib/jets/commands/import.rb

@@ -0,0 +1,14 @@
+module Jets::Commands
+  class Import < Jets::Commands::Base
+    autoload :Base, 'jets/commands/import/base'
+    autoload :Rack, 'jets/commands/import/rack'
+    autoload :Rail, 'jets/commands/import/rail'
+    autoload :Sequence, 'jets/commands/import/sequence'
+
+    long_desc Help.text('import:rails')
+    register(Jets::Commands::Import::Rail, "rails", "rails", "Imports rails project in the rack subfolder")
+
+    long_desc Help.text('import:rack')
+    register(Jets::Commands::Import::Rack, "rack", "rack", "Imports rack project in the rack subfolder")
+  end
+end

data/lib/jets/commands/main.rb

@@ -41,6 +41,7 @@ module Jets::Commands
       command = "bundle exec shotgun --port #{o[:port]} --host #{o[:host]}"
       puts "=> #{command}".colorize(:green)
       puts Jets::Booter.message
+      Jets::Rack::Server.start unless ENV['JETS_RACK'] == '0' # rack server runs in background by default
       system(command)
     end
 
@@ -108,7 +109,7 @@ module Jets::Commands
 
     long_desc Help.text(:new)
     Jets::Commands::New.cli_options.each do |args|
-      option *args
+      option(*args)
     end
     register(Jets::Commands::New, "new", "new", "Creates a starter skeleton jets project")
   end

data/lib/jets/commands/new.rb

@@ -9,7 +9,7 @@ module Jets::Commands
     def self.cli_options
       [
         [:bootstrap, type: :boolean, default: true, desc: "Install bootstrap css"], # same option in WebpackerTemplate
-        [:database, type: :boolean, default: true, desc: "Adds database"],
+        [:database, type: :string, default: 'mysql', desc: "Preconfigure database (options: mysql/postgresql)"],
         [:force, type: :boolean, desc: "Bypass overwrite are you sure prompt for existing files."],
         [:git, type: :boolean, default: true, desc: "Git initialize the project"],
         [:mode, default: 'html', desc: "mode: #{VALID_MODES.join(',')}"],

data/lib/jets/commands/sequence.rb

@@ -39,28 +39,32 @@ private
   end
 
   def copy_options
-    # list of words to include in the exclude pattern and will not be generated
-    words = %w[
-      Procfile
-      controllers
-      helpers
-      javascript
-      models/application_
-      views
-      config.ru
-      database.yml
-      dynamodb.yml
-      routes
-      db/
-      spec
-      yarn
-      public
-    ]
-
-    if @options[:mode] == 'job'
-      { exclude_pattern: Regexp.new(words.join('|')) }
-    else
-      {}
+    excludes = if @options[:mode] == 'job'
+      # For job mode: list of words to include in the exclude pattern and will not be generated.
+      %w[
+        Procfile
+        controllers
+        helpers
+        javascript
+        models/application_
+        views
+        config.ru
+        database.yml
+        dynamodb.yml
+        routes
+        db/
+        spec
+        yarn
+        public
+      ]
+    elsif !@options[:database]
+      # Do not even generated the config/database.yml because
+      # jets webpacker:install bombs and tries to load the db since it sees a
+      # config/database.yml but has there's no database pg gem configured.
+      %w[
+        database.yml
+        models/application_record
+      ]
     end
   end
 

data/lib/jets/commands/templates/skeleton/Gemfile.tt

@@ -6,9 +6,12 @@ gem "jets"
 # Include webpacker if you are you are building html pages
 gem "webpacker", git: "https://github.com/tongueroo/webpacker.git", branch: "jets"
 <% end -%>
-<% if @options[:database] %>
+<% if @options[:database] == 'postgresql' %>
 # Include pg gem if you are using ActiveRecord, remove if you are not
-gem "pg", "~> 0.21"
+gem "pg", "~> 1.1.3"
+<% elsif @options[:database] == 'mysql' %>
+# Include mysql2 gem if you are using ActiveRecord, remove if you are not
+gem "mysql2", "~> 0.5.2"
 <% end %>
 
 group :development, :test do

data/lib/jets/commands/templates/skeleton/README.md

@@ -1,4 +1,13 @@
-# Starter Jets Project
+# Jets Project
 
-This project demonstrates basic CRUD.
+This README would normally document whatever steps are necessary to get the application up and running.
 
+[![Gem Version](https://badge.fury.io/rb/GEMNAME.png)](http://badge.fury.io/rb/GEMNAME)
+
+Things you might want to cover:
+
+* Dependencies
+* Configuration
+* Database setup
+* How to run the test suite
+* Deployment instructions

data/lib/jets/commands/templates/skeleton/app/jobs/application_job.rb

@@ -1,4 +1,4 @@
 class ApplicationJob < Jets::Job::Base
   # Adjust to increase the default timeout for all Job classes
-  class_timeout 30
+  class_timeout 60
 end

data/lib/jets/commands/templates/skeleton/app/views/layouts/application.html.erb.tt

@@ -8,7 +8,7 @@
   <title>Jets App</title>
   <%%= javascript_pack_tag "application" %>
   <%%= stylesheet_pack_tag "theme" %>
-  <link rel="shortcut icon" href="<%%= asset_path("/images/favicon.ico") %>">
+  <link rel="shortcut icon" href="<%%= asset_path("/favicon.ico") %>">
 </head>
 <body>
 <section class="starter">

data/lib/jets/commands/templates/skeleton/config/application.rb.tt

@@ -2,6 +2,11 @@ Jets.application.configure do
   config.project_name = "<%= @project_name %>"
   config.mode = "<%= @options[:mode] %>"
 
+  # Lazy loads ruby and gems as part of first request. Prewarming will handles
+  # this. This also keeps the code package size small and allows us to use the
+  # Lambda console for inline code editing.
+  # config.ruby.lazy_load = true # default is true
+
   # config.prewarm.enable = true # default is true
   # config.prewarm.rate = '30 minutes' # default is '30 minutes'
   # config.prewarm.concurrency = 2 # default is 2
@@ -15,9 +20,9 @@ Jets.application.configure do
   config.cors = true # for '*''
   # config.cors = '*.mydomain.com' # for specific domain
 
-  config.function.timeout = 20
+  config.function.timeout = 30
   # config.function.role = "arn:aws:iam::#{Jets.aws.account}:role/service-role/pre-created"
-  # config.function.memory_size= 1536
+  # config.function.memory_size = 1536
 
   config.function.environment = {
     global_app_key1: "global_app_value1",
@@ -34,8 +39,10 @@ Jets.application.configure do
   # Underscored format can be used for keys to make it look more ruby-ish.
 
   # Assets settings
-  # config.assets.folders = %w[packs images assets] # default packs images assets
-  # config.assets.base_url = nil # IE: https://cloudfront.com/my/base/path , defaults to use s3
+  # config.assets.folders files are uploaded to s3 and are set to public-read to be served directly
+  # config.assets.folders = %w[public] # defaults to public
   # config.assets.max_age = 3600 # when to expire assets
-  # config.assets.cache_control = nil # IE: public, max-age=3600 , max_age is a shorter way to set cache_control.
+  # config.assets.cache_control = nil # IE: "public, max-age=3600" # override max_age for more fine-grain control.
+  # config.assets.base_url = nil # IE: https://cloudfront.com/my/base/path, defaults to the s3 bucket url
+  #                                IE: https://s3-us-west-2.amazonaws.com/demo-dev-s3bucket-1inlzkvujq8zb
 end