jets-fs 1.6.10

data/lib/jets/cfn/builders/rule_builder.rb

@@ -0,0 +1,24 @@
+# Implements:
+#
+#   compose
+#   template_path
+#
+class Jets::Cfn::Builders
+  class RuleBuilder < BaseChildBuilder
+    def compose
+      add_common_parameters
+      add_functions
+      add_resources
+      add_managed_rules
+    end
+
+    # Handle config_rules associated with aws managed rules.
+    # List of AWS Config Managed Rules: https://amzn.to/2BOt9KN
+    def add_managed_rules
+      @app_class.managed_rules.each do |rule|
+        resource = Jets::Resource.new(rule[:definition], rule[:replacements])
+        add_resource(resource)
+      end
+    end
+  end
+end

data/lib/jets/cfn/builders/shared_builder.rb

@@ -0,0 +1,14 @@
+class Jets::Cfn::Builders
+  class SharedBuilder < BaseChildBuilder
+    def compose
+      stack = @app_class.new # @app_class is subclass. IE: Alarm < Jets::Stack
+      builder = Jets::Stack::Builder.new(stack)
+      @template = builder.template # overwrite entire @template
+    end
+
+    # template_path is an interface method for Interface module
+    def template_path
+      Jets::Naming.shared_template_path(@app_class)
+    end
+  end
+end

data/lib/jets/cfn/ship.rb

@@ -0,0 +1,189 @@
+class Jets::Cfn
+  class Ship
+    extend Memoist
+    include Jets::AwsServices
+
+    def initialize(options)
+      @options = options
+      @parent_stack_name = Jets::Naming.parent_stack_name
+      @template_path = Jets::Naming.parent_template_path
+    end
+
+    def run
+      # s3 bucket is available only when stack_type is full
+      upload_to_s3 if @options[:stack_type] == :full
+
+      stack_in_progress?(@parent_stack_name)
+
+      puts "Deploying CloudFormation stack with jets app!"
+      begin
+        save_stack
+      rescue Aws::CloudFormation::Errors::InsufficientCapabilitiesException => e
+        capabilities = e.message.match(/\[(.*)\]/)[1]
+        confirm = prompt_for_iam(capabilities)
+        if confirm =~ /^y/
+          @options.merge!(capabilities: [capabilities])
+          puts "Re-running: #{command_with_iam(capabilities).color(:green)}"
+          retry
+        else
+          puts "Exited"
+          exit 1
+        end
+      end
+
+      success = wait_for_stack
+      unless success
+        puts <<~EOL
+          The Jets application failed to deploy. Jets creates a few CloudFormation stacks to deploy your application.
+          The logs above show the CloudFormation parent stack events and points to the stack with the error.
+          Please go to the CloudFormation console and look for the specific stack with the error.
+          The specific child stack usually shows more detailed information and can be used to resolve the issue.
+          Example of checking the CloudFormation console: http://rubyonjets.com/docs/debugging-cloudformation/
+        EOL
+        return
+      end
+
+      prewarm
+      clean_deploy_logs
+      show_api_endpoint
+      show_custom_domain
+    end
+
+    def save_stack
+      if stack_exists?(@parent_stack_name)
+        update_stack
+      else
+        create_stack
+      end
+    end
+
+    def create_stack
+      # parent stack template is on filesystem and child stacks templates is on s3
+      cfn.create_stack(stack_options)
+    end
+
+    def update_stack
+      begin
+        cfn.update_stack(stack_options)
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        puts "ERROR: #{e.message}".red
+        true # error
+      end
+    end
+
+    # options common to both create_stack and update_stack
+    def stack_options
+      {
+        stack_name: @parent_stack_name,
+        template_body: IO.read(@template_path),
+        capabilities: capabilities, # ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
+        # disable_rollback: !@options[:rollback],
+      }
+    end
+
+    # check for /(_COMPLETE|_FAILED)$/ status
+    def wait_for_stack
+      Jets::Cfn::Status.new(@options).wait
+    end
+
+    def prewarm
+      if ENV['SKIP_PREWARMING']
+        puts "Skipping prewarming" # useful for testing
+        return
+      end
+      return unless @options[:stack_type] == :full # s3 bucket is available
+      return unless Jets.config.prewarm.enable
+      return if Jets.poly_only?
+
+      puts "Prewarming application."
+      if Jets::PreheatJob::CONCURRENCY > 1
+        Jets::PreheatJob.perform_now(:torch, {quiet: true})
+      else
+        Jets::PreheatJob.perform_now(:warm, {quiet: true})
+      end
+    end
+
+    def clean_deploy_logs
+      Jets::Commands::Clean::Log.new.clean_deploys
+    end
+
+    def endpoint_unavailable?
+      return true unless @options[:stack_type] == :full # s3 bucket is available
+      return true if Jets::Router.routes.empty?
+      _, status = stack_status
+      return true if status.include?("ROLLBACK")
+      return true unless api_gateway
+    end
+
+    # Do not memoize this because on first stack run it will be nil
+    # It only gets called one more time so just let it get called.
+    def api_gateway
+      resp = cfn.describe_stack_resources(stack_name: @parent_stack_name)
+      resources = resp.stack_resources
+      resources.find { |resource| resource.logical_resource_id == "ApiGateway" }
+    end
+    memoize :api_gateway
+
+    def endpoint_available?
+      !endpoint_unavailable?
+    end
+
+    def show_api_endpoint
+      return unless endpoint_available?
+
+      stack_id = api_gateway["physical_resource_id"]
+
+      resp = cfn.describe_stacks(stack_name: stack_id)
+      stack = resp["stacks"].first
+      output = stack["outputs"].find { |o| o["output_key"] == "RestApiUrl" }
+      endpoint = output["output_value"]
+      puts "API Gateway Endpoint: #{endpoint}"
+    end
+
+    def show_custom_domain
+      return unless endpoint_available? && Jets.custom_domain? && Jets.config.domain.route53
+
+      domain_name = Jets::Resource::ApiGateway::DomainName.new
+      # Looks funny but its right.
+      # domain_name is a method on the Jets::Resource::ApiGateway::Domain instance
+      url = "https://#{domain_name.domain_name}"
+      puts "Custom Domain: #{url}"
+    end
+
+    # All CloudFormation states listed here:
+    # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-describing-stacks.html
+    def stack_status
+      resp = cfn.describe_stacks(stack_name: @parent_stack_name)
+      status = resp.stacks[0].stack_status
+      [resp, status]
+    end
+
+    def prompt_for_iam(capabilities)
+      puts "This stack will create IAM resources.  Please approve to run the command again with #{capabilities} capabilities."
+      puts "  #{command_with_iam(capabilities)}"
+
+      puts "Please confirm (y/n)"
+      $stdin.gets # confirm
+    end
+
+    def command_with_iam(capabilities)
+      "#{File.basename($0)} #{ARGV.join(' ')} --capabilities #{capabilities}"
+    end
+
+    def capabilities
+      ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"] # TODO: remove capabilities hardcode
+      # return @options[:capabilities] if @options[:capabilities]
+      # if @options[:iam]
+      #   ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
+      # end
+    end
+
+    # Upload both code and child templates to s3
+    def upload_to_s3
+      raise "Did not specify @options[:s3_bucket] #{@options[:s3_bucket].inspect}" unless @options[:s3_bucket]
+
+      uploader = Upload.new(@options[:s3_bucket])
+      uploader.upload
+    end
+  end
+end

data/lib/jets/cfn/status.rb

@@ -0,0 +1,212 @@
+class Jets::Cfn
+  class Status
+    include Jets::AwsServices
+
+    attr_reader :events
+    def initialize(options={})
+      @options = options
+      @stack_name = Jets::Naming.parent_stack_name
+      reset
+    end
+
+    # used for the jets status command
+    def run
+      unless stack_exists?(@stack_name)
+        puts "The stack #{@stack_name.color(:green)} does not exist."
+        return
+      end
+
+      resp = cfn.describe_stacks(stack_name: @stack_name)
+      stack = resp.stacks.first
+
+      puts "The current status for the stack #{@stack_name.color(:green)} is #{stack.stack_status.color(:green)}"
+      if stack.stack_status =~ /_IN_PROGRESS$/
+        puts "Stack events (tailing):"
+        # tail all events until done
+        @hide_time_took = true
+        wait
+      else
+        puts "Stack events:"
+        # show the last events that was user initiated
+        refresh_events
+        show_events(true)
+      end
+    end
+
+    def reset
+      @events = [] # constantly replaced with recent events
+      @last_shown_event_id = nil
+      @stack_deletion_completed = nil
+    end
+
+    # check for /(_COMPLETE|_FAILED)$/ status
+    def wait
+      start_time = Time.now
+
+      refresh_events
+      until completed || @stack_deletion_completed
+        show_events
+      end
+      show_events(true) # show the final event
+
+      if @stack_deletion_completed
+        puts "Stack #{@stack_name} deleted."
+        return
+      end
+
+      success = false
+      if last_event_status =~ /_FAILED/
+        puts "Stack failed: #{last_event_status}".color(:red)
+        puts "Stack reason #{@events[0]["resource_status_reason"]}".color(:red)
+      elsif last_event_status =~ /_ROLLBACK_/
+        puts "Stack rolled back: #{last_event_status}".color(:red)
+      else # success
+        puts "Stack success status: #{last_event_status}".color(:green)
+        success = true
+      end
+
+      # Never gets here when deleting a stack because the describe stack returns nothing
+      # once the stack is deleted. Gets here for stack create and update though.
+      return if @hide_time_took # set in run
+      took = Time.now - start_time
+      puts "Time took for stack deployment: #{pretty_time(took).color(:green)}."
+
+      success
+    end
+
+    def completed
+      last_event_status =~ /(_COMPLETE|_FAILED)$/ &&
+      @events[0]["logical_resource_id"] == @stack_name &&
+      @events[0]["resource_type"] == "AWS::CloudFormation::Stack"
+    end
+
+    def last_event_status
+      @events[0]["resource_status"]
+    end
+
+    # Only shows new events
+    def show_events(final=false)
+      if @last_shown_event_id.nil?
+        i = find_index(:start)
+        print_events(i)
+      else
+        i = find_index(:last_shown)
+        # puts "last_shown index #{i}"
+        print_events(i-1) unless i == 0
+      end
+
+      return if final
+      sleep 5 unless ENV['TEST']
+      refresh_events
+    end
+
+    def print_events(i)
+      @events[0..i].reverse.each do |e|
+        print_event(e)
+      end
+      @last_shown_event_id = @events[0]["event_id"]
+      # puts "@last_shown_event_id #{@last_shown_event_id.inspect}"
+    end
+
+    def print_event(e)
+      message = [
+        event_time(e["timestamp"]),
+        e["resource_status"],
+        e["resource_type"],
+        e["logical_resource_id"],
+        e["resource_status_reason"]
+      ].join(" ")
+      message = message.color(:red) if e["resource_status"] =~ /_FAILED/
+      puts message
+    end
+
+    # https://stackoverflow.com/questions/18000432/rails-12-hour-am-pm-range-for-a-day
+    def event_time(timestamp)
+      Time.parse(timestamp.to_s).localtime.strftime("%I:%M:%S%p")
+    end
+
+    # refreshes the loaded events in memory
+    def refresh_events
+      resp = cfn.describe_stack_events(stack_name: @stack_name)
+      @events = resp["stack_events"]
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      if e.message =~ /Stack .* does not exis/
+        @stack_deletion_completed = true
+      else
+        raise
+      end
+    end
+
+    def find_index(name)
+      send("#{name}_index")
+    end
+
+    def start_index
+      @events.find_index do |event|
+        event["resource_type"] == "AWS::CloudFormation::Stack" &&
+        event["resource_status_reason"] == "User Initiated"
+      end
+    end
+
+    def last_shown_index
+      @events.find_index do |event|
+        event["event_id"] == @last_shown_event_id
+      end
+    end
+
+    def success?
+      resource_status = @events[0]["resource_status"]
+      %w[CREATE_COMPLETE UPDATE_COMPLETE].include?(resource_status)
+    end
+
+    def update_rollback?
+      @events[0]["resource_status"] == "UPDATE_ROLLBACK_COMPLETE"
+    end
+
+    def find_update_failed_event
+      i = @events.find_index do |event|
+        event["resource_type"] == "AWS::CloudFormation::Stack" &&
+        event["resource_status_reason"] == "User Initiated"
+      end
+
+      @events[0..i].reverse.find do |e|
+        e["resource_status"] == "UPDATE_FAILED"
+      end
+    end
+
+    def rollback_error_message
+      return unless update_rollback?
+
+      event = find_update_failed_event
+      return unless event
+
+      reason = event["resource_status_reason"]
+      messages_map.each do |pattern, message|
+        if reason =~ pattern
+          return message
+        end
+      end
+
+      reason # default message is original reason if not found in messages map
+    end
+
+    def messages_map
+      {
+        /CloudFormation cannot update a stack when a custom-named resource requires replacing/ => "A workaround is to run ufo again with STATIC_NAME=0 and to switch to dynamic names for resources. Then run ufo again with STATIC_NAME=1 to get back to statically name resources. Note, there are caveats with the workaround.",
+        /cannot be associated with more than one load balancer/ => "There's was an issue updating the stack. Target groups can only be associated with one load balancer at a time. The workaround for this is to use UFO_FORCE_TARGET_GROUP=1 and run the command again. This will force the recreation of the target group resource.",
+        /SetSubnets is not supported for load balancers of type/ => "Changing subnets for Network Load Balancers is currently not supported. You can try workarouding this with UFO_FORCE_ELB=1 and run the command again. This will force the recreation of the elb resource."
+      }
+    end
+
+    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
+    def pretty_time(total_seconds)
+      minutes = (total_seconds / 60) % 60
+      seconds = total_seconds % 60
+      if total_seconds < 60
+        "#{seconds.to_i}s"
+      else
+        "#{minutes.to_i}m #{seconds.to_i}s"
+      end
+    end
+  end
+end

data/lib/jets/cfn/upload.rb

@@ -0,0 +1,139 @@
+require 'action_view'
+require 'digest'
+
+class Jets::Cfn
+  class Upload
+    include Jets::AwsServices
+    include ActionView::Helpers::NumberHelper # number_to_human_size
+
+    attr_reader :bucket_name
+    def initialize(bucket_name)
+      @bucket_name = bucket_name
+    end
+
+    def upload
+      upload_cfn_templates
+      upload_zip_files
+      upload_assets
+    end
+
+    def upload_cfn_templates
+      puts "Uploading CloudFormation templates to S3."
+      expression = "#{Jets::Naming.template_path_prefix}-*"
+      Dir.glob(expression).each do |path|
+        next unless File.file?(path)
+
+        key = "jets/cfn-templates/#{File.basename(path)}"
+        obj = s3_resource.bucket(bucket_name).object(key)
+        obj.upload_file(path)
+      end
+    end
+
+    def upload_zip_files
+      puts "Uploading code zip files to S3."
+      zip_area = "#{Jets.build_root}/stage/zips"
+      Dir.glob("#{zip_area}/*").each do |file|
+        upload_zip_file(file)
+      end
+    end
+
+    def upload_zip_file(path)
+      file_size = number_to_human_size(File.size(path))
+
+      puts "Uploading #{path} (#{file_size}) to S3"
+      start_time = Time.now
+      s3_key = "jets/code/#{File.basename(path)}"
+      obj = s3_resource.bucket(bucket_name).object(s3_key)
+      obj.upload_file(path)
+      puts "Uploaded to s3://#{bucket_name}/#{s3_key}".color(:green)
+      puts "Time to upload code to s3: #{pretty_time(Time.now-start_time).color(:green)}"
+    end
+
+    def upload_assets
+      puts "Checking for modified public assets and uploading to S3."
+      start_time = Time.now
+      upload_public_assets
+      puts "Time for public assets to s3: #{pretty_time(Time.now-start_time).color(:green)}"
+    end
+
+    def upload_public_assets
+      public_folders = %w[public]
+      public_folders = add_rack_assets(public_folders)
+      public_folders.each do |folder|
+        upload_asset_folder(folder)
+      end
+    end
+
+    def add_rack_assets(public_folders)
+      return public_folders unless Jets.rack?
+      public_folders + %w[rack/public]
+    end
+
+    def upload_asset_folder(folder)
+      expression = "#{Jets.root}/#{folder}/**/*"
+      group_size = 10
+      Dir.glob(expression).each_slice(group_size) do |paths|
+        threads = []
+        paths.each do |full_path|
+          next unless File.file?(full_path)
+
+          threads << Thread.new do
+            upload_to_s3(full_path)
+          end
+        end
+        threads.each(&:join)
+      end
+    end
+
+    def upload_to_s3(full_path)
+      return if identical_on_s3?(full_path)
+
+      key = s3_key(full_path)
+      obj = s3_resource.bucket(bucket_name).object(key)
+      puts "Uploading s3://#{bucket_name}/#{key}" # uncomment to see and debug
+      obj.upload_file(full_path, acl: "public-read", cache_control: cache_control)
+    end
+
+    def s3_key(full_path)
+      relative_path = full_path.sub("#{Jets.root}/", '')
+      "jets/#{relative_path}"
+    end
+
+    def identical_on_s3?(full_path)
+      local_md5 = ::Digest::MD5.file(full_path)
+      key = s3_key(full_path)
+      begin
+        resp = s3.head_object(bucket: bucket_name, key: key)
+      rescue Aws::S3::Errors::NotFound
+        return false
+      end
+
+      remote_md5 = resp.etag.delete_prefix('"').delete_suffix('"')
+      local_md5 == remote_md5
+    end
+
+    # If cache_control is provided, then it will set the entire cache-control header.
+    # If only max_age is provided, then we'll generate a cache_control header.
+    # Using max_age is the shorter and simply way of setting the cache_control header.
+    def cache_control
+      cache_control = Jets.config.assets.cache_control
+      unless cache_control
+        max_age = Jets.config.assets.max_age # defaults to 3600 in jets/application.rb
+        cache_control = "public, max-age=#{max_age}"
+      end
+      cache_control
+    end
+
+    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
+    def pretty_time(total_seconds)
+      minutes = (total_seconds / 60) % 60
+      seconds = total_seconds % 60
+      if total_seconds < 60
+        "#{seconds.to_i}s"
+      else
+        "#{minutes.to_i}m #{seconds.to_i}s"
+      end
+    end
+
+  end
+end