ufo 5.0.7 → 6.0.0

data/lib/ufo/{stack → cfn/stack}/builder/resources/task_definition/reconstructor.rb

@@ -1,6 +1,6 @@
-class Ufo::Stack::Builder::Resources::TaskDefinition
+class Ufo::Cfn::Stack::Builder::Resources::TaskDefinition
   class Reconstructor
-    include Ufo::AwsService
+    include Ufo::AwsServices
 
     def initialize(task_definition, rollback=false)
       @task_definition, @rollback = task_definition, rollback
@@ -12,10 +12,10 @@ class Ufo::Stack::Builder::Resources::TaskDefinition
 
     def data
       if @rollback
-        resp = ecs.describe_task_definition(task_definition: @task_definition)
+        resp = ecs.describe_task_definition(task_definition: @task_definition.name)
         resp.task_definition.to_h
       else
-        path = "#{Ufo.root}/.ufo/output/#{@task_definition}.json"
+        path = "#{Ufo.root}/.ufo/output/task_definition.json"
         JSON.load(IO.read(path))
       end
     end

data/lib/ufo/{stack → cfn/stack}/builder/resources/task_definition.rb

@@ -1,7 +1,7 @@
-class Ufo::Stack::Builder::Resources
+class Ufo::Cfn::Stack::Builder::Resources
   class TaskDefinition < Base
     def build
-      return if @rollback_definition_arn
+      return if vars[:rollback_task_definition]
 
       {
         Type: "AWS::ECS::TaskDefinition",
@@ -15,8 +15,8 @@ class Ufo::Stack::Builder::Resources
       props = Reconstructor.new(@task_definition).reconstruct
 
       # Decorate with iam roles if needed
-      props[:TaskRoleArn] = {"Fn::GetAtt": "TaskRole.Arn"} if Roles::TaskRole.build?
-      props[:ExecutionRoleArn] = {"Fn::GetAtt": "ExecutionRole.Arn"} if Roles::ExecutionRole.build?
+      props[:TaskRoleArn] = {"Fn::GetAtt": "TaskRole.Arn"} if IamRoles::TaskRole.build?
+      props[:ExecutionRoleArn] = {"Fn::GetAtt": "ExecutionRole.Arn"} if IamRoles::ExecutionRole.build?
 
       props
     end

data/lib/ufo/cfn/stack/builder/resources.rb

@@ -0,0 +1,24 @@
+class Ufo::Cfn::Stack::Builder
+  class Resources < Base
+    def build
+      {
+        Dns: Dns.build(@options),
+        EcsService: EcsService.build(@options),
+        EcsSecurityGroup: SecurityGroup::Ecs.build(@options),
+        EcsSecurityGroupRule: SecurityGroup::EcsRule.build(@options),
+        Elb: Elb.build(@options),
+        ElbSecurityGroup: SecurityGroup::Elb.build(@options),
+        ExecutionRole: IamRoles::ExecutionRole.build(@options),
+        Listener: Listener.build(@options),
+        ListenerSsl: ListenerSsl.build(@options),
+        TargetGroup: TargetGroup.build(@options),
+        TaskDefinition: TaskDefinition.build(@options),
+        TaskRole: IamRoles::TaskRole.build(@options),
+        # ECS Service AutoScaling
+        ScalingRole: Scaling::Role.build(@options),
+        ScalingTarget: Scaling::Target.build(@options),
+        ScalingPolicy: Scaling::Policy.build(@options),
+      }
+    end
+  end
+end

data/lib/ufo/cfn/stack/builder.rb

@@ -0,0 +1,30 @@
+class Ufo::Cfn::Stack
+  class Builder < Ufo::Cfn::Base
+    include Ufo::Utils::Logging
+
+    def initialize(options={})
+      super
+      @vars = options[:vars]
+      @template = {Description: "UFO ECS stack #{@vars[:stack_name]}"}
+    end
+
+    def build
+      @template[:Parameters] = Parameters.build(@options)
+      @template[:Conditions] = Conditions.build(@options)
+      @template[:Resources] = Resources.build(@options)
+      @template[:Outputs] = Outputs.build(@options)
+      @template.deep_stringify_keys!
+      @template = Ufo::Utils::Squeezer.new(@template).squeeze
+      @template = CustomProperties.new(@template, @vars[:stack_name]).apply
+      write(@template)
+    end
+
+    def write(template)
+      text = YAML.dump(template)
+      path = ".ufo/output/template.yml"
+      IO.write("#{Ufo.root}/#{path}", text)
+      logger.info "Template built:        #{path}"
+      text
+    end
+  end
+end

data/lib/ufo/{stack → cfn/stack}/custom_properties.rb

@@ -1,7 +1,5 @@
-class Ufo::Stack
+class Ufo::Cfn::Stack
   class CustomProperties
-    include Ufo::Settings
-
     def initialize(template, stack_name)
       @template, @stack_name = template, stack_name
     end
@@ -19,9 +17,9 @@ class Ufo::Stack
     end
 
     # Keep backward compatiablity but encouraging CamelCase now because in the ufo init generator
-    # the .ufo/settings/cfn/default.yml is now CamelCase
+    # the .ufo/config/cfn/default.yml is now CamelCase
     def camelize(properties)
-      if ENV['UFO_CAMELIZE'] == '0' || settings[:auto_camelize] == false # provide a way to quickly test full camelize disable
+      if ENV['UFO_CAMELIZE'] == '0'
         return properties.deep_stringify_keys
       end
 
@@ -55,5 +53,23 @@ class Ufo::Stack
       end
       properties
     end
+
+    def cfn
+      folder = "#{Ufo.root}/.ufo/config/cfn/"
+      paths = ["base", Ufo.env].map do |file|
+        "#{folder}/#{file}.yml"
+      end
+      paths.inject({}) do |result, path|
+        data = yaml_load(path)
+        result.merge(data)
+      end
+    end
+
+    def yaml_load(path)
+      return {} unless File.exist?(path)
+      text = RenderMePretty.result(path)
+      result = YAML.load(text) # yaml file can contain nothing but comments
+      result.is_a?(Hash) ? result.deep_symbolize_keys : {}
+    end
   end
 end

data/lib/ufo/cfn/stack/params.rb

@@ -0,0 +1,51 @@
+class Ufo::Cfn::Stack
+  class Params < Ufo::Cfn::Stack::Builder::Base
+    def build
+      params = {
+        Vpc: vpc.id,
+        ElbSubnets: vpc.elb_subnets,
+        EcsSubnets: vpc.ecs_subnets,
+
+        CreateElb: vars[:create_elb] ? "true" : "false",
+        ElbTargetGroup: vars[:elb_target_group].to_s,
+
+        EcsSchedulingStrategy: Ufo.config.ecs.scheduling_strategy,
+      }
+      params[:EcsDesiredCount] = desired_count.to_s if desired_count # Note: cfn template is type String so it can be optional
+
+      params = Ufo::Utils::Squeezer.new(params).squeeze
+      parameters = params.map do |k,v|
+        if v == :use_previous_value
+          { parameter_key: k, use_previous_value: true }
+        else
+          { parameter_key: k, parameter_value: v }
+        end
+      end
+      save_params(parameters)
+      parameters
+    end
+
+  private
+    def desired_count
+      Ufo.config.ecs.desired_count
+    end
+
+    # No need to save template. That's already saved.
+    def save_params(parameters)
+      params = parameters.dup.map do |param|
+        param.transform_keys do |key|
+          key.to_s.camelize
+        end
+      end
+      path = "#{Ufo.root}/.ufo/output/params.json"
+      FileUtils.mkdir_p(File.dirname(path))
+      IO.write(path, JSON.pretty_generate(params))
+      logger.info "Parameters built:      #{pretty_path(path)}"
+    end
+
+    def vpc
+      Vpc.new(@options)
+    end
+    memoize :vpc
+  end
+end

data/lib/ufo/{stack → cfn/stack}/status.rb

@@ -23,9 +23,9 @@ require 'time'
 #   UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS
 #   UPDATE_ROLLBACK_IN_PROGRESS
 #
-class Ufo::Stack
+class Ufo::Cfn::Stack
   class Status
-    include Ufo::AwsService
+    include Ufo::AwsServices
     include Ufo::Util
 
     attr_reader :events

data/lib/ufo/cfn/stack/template.rb

@@ -0,0 +1,8 @@
+class Ufo::Cfn::Stack
+  class Template < Ufo::Cfn::Base
+    def body
+      builder = Builder.new(@options)
+      builder.build
+    end
+  end
+end

data/lib/ufo/cfn/stack/vars.rb

@@ -0,0 +1,110 @@
+class Ufo::Cfn::Stack
+  class Vars < Ufo::Cfn::Base
+    attr_reader :stack_name
+    def initialize(options={})
+      super
+      @cluster = options[:cluster].dup # Thor options are frozen, we thaw it because CustomProperties#substitute_variables does a sub!
+      @stack_name = options[:stack_name] || [Ufo.app, @task_definition.role, Ufo.env].join('-')
+      @stack = options[:stack]
+      @new_stack = !@stack
+    end
+
+    def values
+      {
+        cluster: @cluster,
+        stack_name: @stack_name, # used in custom_properties
+        container: container,
+        # to reconstruct TaskDefinition in the CloudFormation template
+        task_definition: @task_definition,
+        rollback_task_definition: @options[:rollback_task_definition],
+        # elb options remember that their 'state'
+        create_elb: create_elb?, # helps set Ecs DependsOn
+        elb_type: elb_type,
+        create_route53: create_elb? && dns_configured?,
+        default_target_group_protocol: default_target_group_protocol,
+        default_listener_protocol: default_listener_protocol,
+        default_listener_ssl_protocol: default_listener_ssl_protocol,
+        create_listener_ssl: create_listener_ssl?,
+      }
+    end
+
+    def dns_configured?
+      !!Ufo.config.dns.domain || !!Ufo.config.dns.name
+    end
+
+    def default_listener_protocol
+      port = Ufo.config.elb.port
+      if elb_type == 'network'
+        port == 443 ? 'TLS' : 'TCP'
+      else
+        port == 443 ? 'HTTPS' : 'HTTP'
+      end
+    end
+
+    def default_listener_ssl_protocol
+      elb_type == 'network' ? 'TLS' : 'HTTPS'
+    end
+
+    def default_target_group_protocol
+      elb_type == 'network' ? 'TCP' : 'HTTP'
+    end
+
+    # if the configuration is set to anything then enable it
+    def create_listener_ssl?
+      Ufo.config.elb.ssl.enabled
+    end
+
+    def create_elb?
+      elb = Ufo.config.elb
+      if elb.enabled.to_s == "auto"
+        container[:name] == "web" # convention
+      else
+        elb.enabled # true or false
+      end
+    end
+
+    def container
+      task_definition = Builder::Resources::TaskDefinition::Reconstructor.new(@task_definition, @options[:rollback]).reconstruct
+
+      container_def = task_definition["ContainerDefinitions"].first
+      requires_compatibilities = task_definition["RequiresCompatibilities"]
+      fargate = requires_compatibilities && requires_compatibilities == ["FARGATE"]
+      network_mode = task_definition["NetworkMode"]
+
+      mappings = container_def["PortMappings"] || []
+      unless mappings.empty?
+        port = mappings.first["ContainerPort"]
+      end
+
+      result =  {
+        name: container_def["Name"],
+        fargate: fargate,
+        network_mode: network_mode, # awsvpc, bridge, etc
+      }
+      result[:port] = port if port
+      result
+    end
+    memoize :container
+
+    def get_parameter_value(stack, key)
+      param = stack.parameters.find do |p|
+        p.parameter_key == key
+      end
+      param.parameter_value if param
+    end
+
+    def elb_type
+      elb_type = Ufo.config.elb.type
+      return elb_type if elb_type
+
+      # user is trying to create a network load balancer if subnet_mappings used
+      subnet_mappings = Ufo.config.elb.subnet_mappings
+      if subnet_mappings.empty?
+        "application" # default
+      else
+        "network"
+      end
+    end
+    memoize :elb_type
+  end
+end

data/lib/ufo/cfn/stack/vpc.rb

@@ -0,0 +1,31 @@
+class Ufo::Cfn::Stack
+  class Vpc < Ufo::Cfn::Base
+    extend Memoist
+    include Ufo::TaskDefinition::Helpers
+
+    def id
+      vpc.id ? vpc.id : default_vpc
+    end
+    alias_method :vpc_id, :id
+
+    def vpc
+      Ufo.config.vpc
+    end
+
+    def elb_subnets
+      subnets(vpc.subnets.elb)
+    end
+
+    def ecs_subnets
+      subnets(vpc.subnets.ecs)
+    end
+
+    def subnets(subnets)
+      if subnets
+        subnets.is_a?(String) ? subnets : subnets.join(',')
+      else
+        subnets_for(vpc_id).join(',') # default vpc subnets or all subnets for the configured vpc
+      end
+    end
+  end
+end

data/lib/ufo/cfn/stack.rb

@@ -0,0 +1,175 @@
+# CloudFormation status codes, full list:
+#   https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-describing-stacks.html
+#
+#   CREATE_COMPLETE
+#   ROLLBACK_COMPLETE
+#   DELETE_COMPLETE
+#   UPDATE_COMPLETE
+#   UPDATE_ROLLBACK_COMPLETE
+#
+#   CREATE_FAILED
+#   DELETE_FAILED
+#   ROLLBACK_FAILED
+#   UPDATE_ROLLBACK_FAILED
+#
+#   CREATE_IN_PROGRESS
+#   DELETE_IN_PROGRESS
+#   REVIEW_IN_PROGRESS
+#   ROLLBACK_IN_PROGRESS
+#   UPDATE_COMPLETE_CLEANUP_IN_PROGRESS
+#   UPDATE_IN_PROGRESS
+#   UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS
+#   UPDATE_ROLLBACK_IN_PROGRESS
+#
+module Ufo::Cfn
+  class Stack < Base
+    extend Memoist
+
+    def deploy
+      build
+      @stack = find_stack(@stack_name)
+      if @stack && rollback_complete?(@stack)
+        logger.info "Existing stack in ROLLBACK_COMPLETE state. Deleting stack before continuing."
+        cloudformation.delete_stack(stack_name: @stack_name)
+        status.wait
+        status.reset
+        @stack = nil # at this point stack has been deleted
+      end
+
+      exit_with_message(@stack) if @stack && !updatable?(@stack)
+
+      @stack ? perform(:update) : perform(:create)
+
+      stop_old_tasks if @options[:stop_old_task]
+
+      return unless @options[:wait]
+      status.wait
+
+      logger.info status.rollback_error_message if status.update_rollback?
+
+      status.success?
+    end
+
+    def perform(action)
+      logger.info "#{action[0..-2].capitalize}ing stack #{@stack_name.color(:green)}"
+      # Example: cloudformation.send("update_stack", stack_options)
+
+      # o = stack_options.dup
+      # o[:template_body] = '...' if o[:template_body]
+      # puts "stack_options:"
+      # pp o
+      # puts "parameters:"
+      # pp o[:parameters]
+
+      cloudformation.send("#{action}_stack", stack_options)
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      handle_stack_error(e)
+    end
+
+    def stack_options
+      options = {
+        capabilities: ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"],
+        parameters: @parameters,
+        stack_name: @stack_name,
+        template_body: @template_body,
+      }
+      cfn = Ufo.config.cfn
+      options[:notification_arns] = cfn.notification_arns if cfn.notification_arns
+      options[:disable_rollback] = cfn.disable_rollback unless cfn.disable_rollback.nil?
+      options[:tags] = tags(cfn.tags) if cfn.tags
+      options
+    end
+
+    def tags(hash)
+      hash.map do |k,v|
+        { key: v, value: v }
+      end
+    end
+
+    def build
+      options_with_vars = @options.dup.merge(vars: vars.values)
+      params = Params.new(options_with_vars)
+      @parameters = params.build
+      template = Template.new(options_with_vars)
+      @template_body = template.body
+    end
+
+    def vars
+      o = @options.merge(
+        cluster: @cluster,
+        stack_name: @stack_name,
+        stack: @stack,
+      )
+      o[:rollback_task_definition] = rollback_task_definition if rollback_task_definition
+      Vars.new(o)
+    end
+    memoize :vars
+
+    def scheduling_strategy
+      scheduling_strategy = Ufo.config.ecs.scheduling_strategy
+      scheduling_strategy.upcase if scheduling_strategy
+    end
+
+    def rollback_task_definition
+      return unless @options[:rollback]
+      @options[:rollback_task_definition]
+    end
+    memoize :rollback_task_definition
+
+    def exit_with_message(stack)
+      region = `aws configure get region`.strip rescue "us-east-1"
+      url = "https://console.aws.amazon.com/cloudformation/home?region=#{region}#/stacks"
+      logger.info "The stack is not in an updateable state: #{stack.stack_status.color(:yellow)}."
+      logger.info "Here's the CloudFormation url to check for more details #{url}"
+      exit 1
+    end
+
+    # Assume only first container_definition to get the container info.
+    # Stack:arn:aws:cloudformation:... is in ROLLBACK_COMPLETE state and can not be updated.
+    def handle_stack_error(e)
+      case e.message
+      when /state and can not be updated/
+        logger.info "The #{@stack_name} stack is in a state that cannot be updated. Deleted the stack and try again."
+        logger.info "ERROR: #{e.message}"
+        if message.include?('UPDATE_ROLLBACK_FAILED')
+          logger.info "You might be able to do a 'Continue Update Rollback' and skip some resources to get the stack back into a good state."
+        end
+        region = `aws configure get region`.strip rescue 'us-east-1'
+        url = "https://console.aws.amazon.com/cloudformation/home?region=#{region}"
+        logger.info "Here's the CloudFormation console url: #{url}"
+        exit 1
+      when /No updates are to be performed/
+        logger.info "There are no updates to be performed. Exiting.".color(:yellow)
+        exit 1
+      else
+        raise
+      end
+    end
+
+    def rollback_complete?(stack)
+      stack.stack_status == 'ROLLBACK_COMPLETE'
+    end
+
+    def updatable?(stack)
+      stack.stack_status =~ /_COMPLETE$/ || stack.stack_status == 'UPDATE_ROLLBACK_FAILED'
+    end
+
+    def cancel
+      stack = find_stack(@stack_name)
+      unless stack
+        logger.error "No #{@stack_name} stack to cancel".color(:red)
+        exit 1
+      end
+
+      if stack.stack_status == "CREATE_IN_PROGRESS"
+        cloudformation.delete_stack(stack_name: @stack_name)
+        logger.info "Canceling stack creation"
+      elsif stack.stack_status == "UPDATE_IN_PROGRESS"
+        cloudformation.cancel_update_stack(stack_name: @stack_name)
+        logger.info "Canceling stack update"
+      else
+        logger.info "The stack is not in a state to that is cancelable: #{stack.stack_status}"
+      end
+    end
+  end
+end

data/lib/ufo/cli/base.rb

@@ -0,0 +1,18 @@
+class Ufo::CLI
+  class Base
+    extend Memoist
+    include Ufo::AwsServices
+    include Ufo::Concerns
+    include Ufo::Utils::Logging
+    include Ufo::Utils::Pretty
+    include Ufo::Utils::Sure
+
+    attr_reader :task_definition
+    def initialize(options={})
+      @options = options
+      @task_definition = Ufo::TaskDefinition.new(options)
+      @stack_name = names.stack
+      @cluster = names.cluster
+    end
+  end
+end

data/lib/ufo/cli/build.rb

@@ -0,0 +1,30 @@
+class Ufo::CLI
+  class Build < Base
+    def build
+      docker
+      task_definition
+      Ufo::Cfn::Stack.new(@options).build
+    end
+    alias_method :all, :build
+
+    def for_deploy
+      docker
+      task_definition
+    end
+
+    def task_definition
+      Ufo::TaskDefinition::Builder.new(@options).build
+    end
+
+    def docker
+      return if @options[:docker] == false
+      # The config.docker.quiet only effects: ufo ship, not ufo docker build
+      quiet = Ufo.config.ship.docker.quiet
+      o = @options.dup.merge(quiet: quiet)
+      builder = Ufo::Docker::Builder.new(o)
+      builder.build
+      pusher = Ufo::Docker::Pusher.new(nil, o)
+      pusher.push
+    end
+  end
+end

data/lib/ufo/cli/cancel.rb

@@ -0,0 +1,18 @@
+class Ufo::CLI
+  class Cancel < Base
+    def run
+      are_you_sure?
+      stack = Ufo::Cfn::Stack.new(@options)
+      stack.cancel
+      stack.status.wait
+    end
+
+    def are_you_sure?
+      if @options[:yes]
+        logger.info "Canceling the #{@stack_name.color(:green)} stack"
+      else
+        sure?("You are about to cancel the #{@stack_name.color(:green)} stack")
+      end
+    end
+  end
+end

data/lib/ufo/cli/central/update.rb

@@ -0,0 +1,80 @@
+class Ufo::CLI::Central
+  class Update
+    include Ufo::Utils::Logging
+    include Ufo::Utils::Execute
+    include Ufo::Utils::Sure
+
+    def initialize(options={})
+      @options = options
+    end
+
+    def run
+      action = File.exist?(".ufo") ? "update" : "create"
+      sure?("Will #{action} the .ufo folder.") # IE: Will create the .ufo folder.
+      validate!
+      logger.info "Updating .ufo with #{repo}"
+      FileUtils.mkdir_p(tmp_area)
+      pull
+      sync
+      check_gitignore
+    end
+
+    def pull
+      logger.debug "Within #{tmp_area}"
+      Dir.chdir(tmp_area) do
+        if File.exist?(repo_name)
+          execute "cd #{repo_name} && git pull"
+        else
+          execute "git clone #{repo}"
+        end
+      end
+    end
+
+    def sync
+      FileUtils.mv(".ufo", ".ufo.bak") if File.exist?(".ufo")
+      src = "#{tmp_area}/#{repo_name}"
+      src += "/#{folder}" if folder
+      FileUtils.cp_r(src, ".ufo")
+      FileUtils.rm_rf(".ufo.bak")
+      logger.info "The .ufo folder has been updated"
+    end
+
+    def validate!
+      return if repo
+      logger.info "ERROR: Please set the env var: UFO_CENTRAL_REPO".color(:red)
+      exit 1
+    end
+
+    def repo_name
+      File.basename(repo)
+    end
+
+    def repo
+      ENV['UFO_CENTRAL_REPO']
+    end
+
+    def folder
+      ENV['UFO_CENTRAL_FOLDER']
+    end
+
+    def tmp_area
+      "/tmp/ufo/central"
+    end
+
+    def check_gitignore
+      ok = true
+      if File.exist?('.gitignore')
+        lines = IO.readlines('.gitignore')
+        ok = lines.detect do |line|
+          line =~ %r{/?.ufo/?$}
+        end
+      end
+      return if ok
+      logger.info "No .ufo found in your .gitignore file".color(:yellow)
+      logger.info <<~EOL
+        It's recommended to add .ufo to the .gitignore
+        When using ufo in a central fashion
+      EOL
+    end
+  end
+end