jets 0.2.0 → 0.5.0

data/lib/jets/cfn.rb

@@ -1,5 +1,6 @@
 class Jets::Cfn
-  autoload :Base, "jets/cfn/base"
-  autoload :Builder, "jets/cfn/builder"
-  autoload :Namer, "jets/cfn/namer"
-end
+  autoload :TemplateMappers, "jets/cfn/template_mappers"
+  autoload :TemplateBuilders, "jets/cfn/template_builders"
+  autoload :Ship, "jets/cfn/ship"
+  autoload :Status, "jets/cfn/status"
+end

data/lib/jets/cfn/ship.rb

@@ -0,0 +1,178 @@
+require 'action_view'
+
+class Jets::Cfn
+  class Ship
+    include Jets::Timing
+    include Jets::AwsServices
+    include ActionView::Helpers::NumberHelper # number_to_human_size
+
+    def initialize(options)
+      @options = options
+      @parent_stack_name = Jets::Naming.parent_stack_name
+      @template_path = Jets::Naming.parent_template_path
+    end
+
+    def run
+      upload_to_s3 if @options[:stack_type] == :full # s3 bucket is available
+        # only when stack_type is full
+
+      stack_in_progress?(@parent_stack_name)
+
+      puts "Deploying CloudFormation stack with jets app!"
+      begin
+        save_stack
+      rescue Aws::CloudFormation::Errors::InsufficientCapabilitiesException => e
+        capabilities = e.message.match(/\[(.*)\]/)[1]
+        confirm = prompt_for_iam(capabilities)
+        if confirm =~ /^y/
+          @options.merge!(capabilities: [capabilities])
+          puts "Re-running: #{command_with_iam(capabilities).colorize(:green)}"
+          retry
+        else
+          puts "Exited"
+          exit 1
+        end
+      end
+
+      wait_for_stack
+      prewarm
+      show_api_endpoint
+    end
+    time :run
+
+    def save_stack
+      if stack_exists?(@parent_stack_name)
+        update_stack
+      else
+        create_stack
+      end
+    end
+
+    def create_stack
+      # parent stack template is on filesystem and child stacks templates is on s3
+      template_body = IO.read(@template_path)
+      cfn.create_stack(stack_options)
+    end
+    time :create_stack
+
+    def update_stack
+      begin
+        cfn.update_stack(stack_options)
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        puts "ERROR: #{e.message}".red
+        error = true
+      end
+    end
+    time :update_stack
+
+    # options common to both create_stack and update_stack
+    def stack_options
+      {
+        stack_name: @parent_stack_name,
+        template_body: IO.read(@template_path),
+        capabilities: capabilities, # ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
+        # disable_rollback: !@options[:rollback],
+      }
+    end
+
+    # check for /(_COMPLETE|_FAILED)$/ status
+    def wait_for_stack
+      Jets::Cfn::Status.new(@options).wait
+    end
+    time :wait_for_stack
+
+    def prewarm
+      return unless @options[:stack_type] == :full # s3 bucket is available
+      puts "Prewarming application..."
+      if Jets::PreheatJob::CONCURRENCY > 1
+        Jets::PreheatJob.perform_now(:torch, {quiet: true})
+      else
+        Jets::PreheatJob.perform_now(:warm, {quiet: true})
+      end
+    end
+
+    def show_api_endpoint
+      return unless @options[:stack_type] == :full # s3 bucket is available
+      return if Jets::Router.routes.empty?
+
+      resp = cfn.describe_stack_resources(stack_name: @parent_stack_name)
+      resources = resp.stack_resources
+      api_gateway = resources.find { |resource| resource.logical_resource_id == "ApiGateway" }
+      stack_id = api_gateway["physical_resource_id"]
+
+      resp = cfn.describe_stacks(stack_name: stack_id)
+      stack = resp["stacks"].first
+      output = stack["outputs"].find { |o| o["output_key"] == "RestApiUrl" }
+      endpoint = output["output_value"]
+      puts "API Gateway Endpoint: #{endpoint}"
+    end
+
+    # All CloudFormation states listed here:
+    # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-describing-stacks.html
+    def stack_status
+      resp = cfn.describe_stacks(stack_name: @parent_stack_name)
+      status = resp.stacks[0].stack_status
+      [resp, status]
+    end
+
+    def prompt_for_iam(capabilities)
+      puts "This stack will create IAM resources.  Please approve to run the command again with #{capabilities} capabilities."
+      puts "  #{command_with_iam(capabilities)}"
+
+      puts "Please confirm (y/n)"
+      confirm = $stdin.gets
+    end
+
+    def command_with_iam(capabilities)
+      "#{File.basename($0)} #{ARGV.join(' ')} --capabilities #{capabilities}"
+    end
+
+    def capabilities
+      ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"] # TODO: remove capabilities hardcode
+      # return @options[:capabilities] if @options[:capabilities]
+      # if @options[:iam]
+      #   ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
+      # end
+    end
+
+    # Upload both code and child templates to s3
+    def upload_to_s3
+      raise "Did not specify @options[:s3_bucket] #{@options[:s3_bucket].inspect}" unless @options[:s3_bucket]
+
+      bucket_name = @options[:s3_bucket]
+
+      puts "Uploading child CloudFormation templates to S3"
+      expression = "#{Jets::Naming.template_path_prefix}-*"
+      Dir.glob(expression).each do |path|
+        next unless File.file?(path)
+
+        key = "jets/cfn-templates/#{File.basename(path)}"
+        obj = s3_resource.bucket(bucket_name).object(key)
+        obj.upload_file(path)
+      end
+
+      md5_code_zipfile = Jets::Naming.md5_code_zipfile
+      file_size = number_to_human_size(File.size(md5_code_zipfile))
+
+      puts "Uploading #{md5_code_zipfile} (#{file_size}) to S3"
+      start_time = Time.now
+      key = Jets::Naming.code_s3_key
+      obj = s3_resource.bucket(bucket_name).object(key)
+      obj.upload_file(md5_code_zipfile)
+      puts "Time to upload code to s3: #{pretty_time(Time.now-start_time).colorize(:green)}"
+    end
+    time :upload_to_s3
+
+    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
+    def pretty_time(total_seconds)
+      minutes = (total_seconds / 60) % 60
+      seconds = total_seconds % 60
+      if total_seconds < 60
+        "#{seconds.to_i}s"
+      else
+        "#{minutes.to_i}m #{seconds.to_i}s"
+      end
+    end
+
+  end
+end

data/lib/jets/cfn/status.rb

@@ -0,0 +1,208 @@
+class Jets::Cfn
+  class Status
+    include Jets::AwsServices
+
+    attr_reader :events
+    def initialize(options={})
+      @options = options
+      @stack_name = Jets::Naming.parent_stack_name
+      reset
+    end
+
+    # used for the jets status command
+    def run
+      unless stack_exists?(@stack_name)
+        puts "The stack #{@stack_name.colorize(:green)} does not exist."
+        return
+      end
+
+      resp = cfn.describe_stacks(stack_name: @stack_name)
+      stack = resp.stacks.first
+
+      puts "The current status for the stack #{@stack_name.colorize(:green)} is #{stack.stack_status.colorize(:green)}"
+      if stack.stack_status =~ /_IN_PROGRESS$/
+        puts "Stack events (tailing):"
+        # tail all events until done
+        @hide_time_took = true
+        wait
+      else
+        puts "Stack events:"
+        # show the last events that was user initiated
+        refresh_events
+        show_events(true)
+      end
+    end
+
+    def reset
+      @events = [] # constantly replaced with recent events
+      @last_shown_event_id = nil
+      @stack_deletion_completed = nil
+    end
+
+    # check for /(_COMPLETE|_FAILED)$/ status
+    def wait
+      start_time = Time.now
+
+      refresh_events
+      until completed || @stack_deletion_completed
+        show_events
+      end
+      show_events(true) # show the final event
+
+      if @stack_deletion_completed
+        puts "Stack #{@stack_name} deleted."
+        return
+      end
+
+      if last_event_status =~ /_FAILED/
+        puts "Stack failed: #{last_event_status}".colorize(:red)
+        puts "Stack reason #{@events[0]["resource_status_reason"]}".colorize(:red)
+      elsif last_event_status =~ /_ROLLBACK_/
+        puts "Stack rolled back: #{last_event_status}".colorize(:red)
+      else # success
+        puts "Stack success status: #{last_event_status}".colorize(:green)
+      end
+
+      # Never gets here when deleting a stack because the describe stack returns nothing
+      # once the stack is deleted. Gets here for stack create and update though.
+      return if @hide_time_took # set in run
+      took = Time.now - start_time
+      puts "Time took for stack deployment: #{pretty_time(took).green}."
+    end
+
+    def completed
+      last_event_status =~ /(_COMPLETE|_FAILED)$/ &&
+      @events[0]["logical_resource_id"] == @stack_name &&
+      @events[0]["resource_type"] == "AWS::CloudFormation::Stack"
+    end
+
+    def last_event_status
+      @events[0]["resource_status"]
+    end
+
+    # Only shows new events
+    def show_events(final=false)
+      if @last_shown_event_id.nil?
+        i = find_index(:start)
+        print_events(i)
+      else
+        i = find_index(:last_shown)
+        # puts "last_shown index #{i}"
+        print_events(i-1) unless i == 0
+      end
+
+      return if final
+      sleep 5 unless ENV['TEST']
+      refresh_events
+    end
+
+    def print_events(i)
+      @events[0..i].reverse.each do |e|
+        print_event(e)
+      end
+      @last_shown_event_id = @events[0]["event_id"]
+      # puts "@last_shown_event_id #{@last_shown_event_id.inspect}"
+    end
+
+    def print_event(e)
+      message = [
+        event_time(e["timestamp"]),
+        e["resource_status"],
+        e["resource_type"],
+        e["logical_resource_id"],
+        e["resource_status_reason"]
+      ].join(" ")
+      message = message.colorize(:red) if e["resource_status"] =~ /_FAILED/
+      puts message
+    end
+
+    # https://stackoverflow.com/questions/18000432/rails-12-hour-am-pm-range-for-a-day
+    def event_time(timestamp)
+      Time.parse(timestamp.to_s).localtime.strftime("%I:%M:%S%p")
+    end
+
+    # refreshes the loaded events in memory
+    def refresh_events
+      resp = cfn.describe_stack_events(stack_name: @stack_name)
+      @events = resp["stack_events"]
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      if e.message =~ /Stack .* does not exis/
+        @stack_deletion_completed = true
+      else
+        raise
+      end
+    end
+
+    def find_index(name)
+      send("#{name}_index")
+    end
+
+    def start_index
+      @events.find_index do |event|
+        event["resource_type"] == "AWS::CloudFormation::Stack" &&
+        event["resource_status_reason"] == "User Initiated"
+      end
+    end
+
+    def last_shown_index
+      @events.find_index do |event|
+        event["event_id"] == @last_shown_event_id
+      end
+    end
+
+    def success?
+      resource_status = @events[0]["resource_status"]
+      %w[CREATE_COMPLETE UPDATE_COMPLETE].include?(resource_status)
+    end
+
+    def update_rollback?
+      @events[0]["resource_status"] == "UPDATE_ROLLBACK_COMPLETE"
+    end
+
+    def find_update_failed_event
+      i = @events.find_index do |event|
+        event["resource_type"] == "AWS::CloudFormation::Stack" &&
+        event["resource_status_reason"] == "User Initiated"
+      end
+
+      @events[0..i].reverse.find do |e|
+        e["resource_status"] == "UPDATE_FAILED"
+      end
+    end
+
+    def rollback_error_message
+      return unless update_rollback?
+
+      event = find_update_failed_event
+      return unless event
+
+      reason = event["resource_status_reason"]
+      messages_map.each do |pattern, message|
+        if reason =~ pattern
+          return message
+        end
+      end
+
+      reason # default message is original reason if not found in messages map
+    end
+
+    def messages_map
+      {
+        /CloudFormation cannot update a stack when a custom-named resource requires replacing/ => "A workaround is to run ufo again with STATIC_NAME=0 and to switch to dynamic names for resources. Then run ufo again with STATIC_NAME=1 to get back to statically name resources. Note, there are caveats with the workaround.",
+        /cannot be associated with more than one load balancer/ => "There's was an issue updating the stack. Target groups can only be associated with one load balancer at a time. The workaround for this is to use UFO_FORCE_TARGET_GROUP=1 and run the command again. This will force the recreation of the target group resource.",
+        /SetSubnets is not supported for load balancers of type/ => "Changing subnets for Network Load Balancers is currently not supported. You can try workarouding this with UFO_FORCE_ELB=1 and run the command again. This will force the recreation of the elb resource."
+      }
+    end
+
+    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
+    def pretty_time(total_seconds)
+      minutes = (total_seconds / 60) % 60
+      seconds = total_seconds % 60
+      if total_seconds < 60
+        "#{seconds.to_i}s"
+      else
+        "#{minutes.to_i}m #{seconds.to_i}s"
+      end
+    end
+  end
+end

data/lib/jets/cfn/template_builders.rb

@@ -0,0 +1,21 @@
+require 'active_support/core_ext/hash'
+require 'yaml'
+
+class Jets::Cfn
+  class TemplateBuilders
+    autoload :Interface, "jets/cfn/template_builders/interface"
+    autoload :ParentBuilder, "jets/cfn/template_builders/parent_builder"
+
+    # These build the app/controllers, app/jobs, and app/functions
+    autoload :BaseChildBuilder, "jets/cfn/template_builders/base_child_builder"
+    autoload :ControllerBuilder, "jets/cfn/template_builders/controller_builder"
+    autoload :JobBuilder, "jets/cfn/template_builders/job_builder"
+    autoload :FunctionBuilder, "jets/cfn/template_builders/function_builder"
+    autoload :RuleBuilder, "jets/cfn/template_builders/rule_builder"
+
+    autoload :ApiGatewayBuilder, "jets/cfn/template_builders/api_gateway_builder"
+    autoload :ApiGatewayDeploymentBuilder, "jets/cfn/template_builders/api_gateway_deployment_builder"
+    # separate beast:
+    autoload :FunctionProperties, "jets/cfn/template_builders/function_properties"
+  end
+end

data/lib/jets/cfn/template_builders/api_gateway_builder.rb

@@ -0,0 +1,73 @@
+class Jets::Cfn::TemplateBuilders
+  class ApiGatewayBuilder
+    include Interface
+    include Jets::AwsServices
+
+    def initialize(options={})
+      @options = options
+      @template = ActiveSupport::HashWithIndifferentAccess.new(Resources: {})
+    end
+
+    # compose is an interface method
+    def compose
+      return if @options[:stack_type] == :minimal
+
+      puts "Building API Gateway template."
+      add_gateway_rest_api
+      add_gateway_routes
+    end
+
+    # template_path is an interface method
+    def template_path
+      Jets::Naming.api_gateway_template_path
+    end
+
+    # do not bother writing a template if routes are empty
+    def write
+      super unless Jets::Router.routes.empty?
+    end
+
+    # If the are routes in config/routes.rb add Gateway API in parent stack
+    def add_gateway_rest_api
+      add_resource("RestApi", "AWS::ApiGateway::RestApi",
+        Name: Jets::Naming.gateway_api_name
+      )
+
+      stage_name = Jets::Cfn::TemplateMappers::ApiGatewayDeploymentMapper.stage_name
+      add_output("RestApi", Value: "!Ref RestApi")
+      add_output("RestApiUrl", Value: "!Sub 'https://${RestApi}.execute-api.${AWS::Region}.amazonaws.com/#{stage_name}/'")
+      add_output("Region", Value: "!Ref AWS::Region")
+      add_output("RootResourceId", Value: "!GetAtt RestApi.RootResourceId")
+    end
+
+    # Adds route related Resources and Outputs
+    def add_gateway_routes
+      # The routes required a Gateway Resource to contain them.
+      # TODO: outputing all routes in 1 template will hit the 60 routes limit
+      # Will have to either output them as a joined string or
+      # break this up to multiple tempaltes.
+      # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
+      # Outputs: Maximum number of outputs that you can declare in your AWS CloudFormation template. 60 outputs
+      # Output name: Maximum size of an output name. 255 characters.
+      #
+      # Note we must use .all_paths, not .routes here because we need to
+      # build the parent ApiGateway::Resource nodes also
+      Jets::Router.all_paths.each do |path|
+        homepage = path == ''
+        next if homepage # handled by RootResourceId output already
+
+        map = Jets::Cfn::TemplateMappers::GatewayResourceMapper.new(path)
+
+        unless homepage # no AWS::ApiGateway::Resource for the top level route
+          add_resource(map.logical_id, "AWS::ApiGateway::Resource",
+            ParentId: map.parent_id,
+            PathPart: map.path_part,
+            RestApiId: "!Ref RestApi"
+          )
+        end
+
+        add_output(map.logical_id, Value: "!Ref #{map.logical_id}")
+      end
+    end
+  end
+end