ufo 5.0.7 → 6.0.0

data/lib/ufo/task_definition/erb/json.rb

@@ -0,0 +1,28 @@
+class Ufo::TaskDefinition::Erb
+  class Json < Base
+    def data
+      text = IO.read(@path)
+      JSON.load(text)
+    rescue JSON::ParserError => e
+      # NOTE: JSON::ParserError e.message is not very useful
+      logger.error "ERROR: #{e.class}".color(:red)
+      logger.error <<~EOL
+        Rendered file contains invalid JSON. For debugging, files available at:
+
+        source:   #{@task_definition.path}
+        rendered: #{@path}
+
+      EOL
+      logger.error "Contents of the rendered file:\n\n"
+      print_code(text)
+      if jq_available?
+        system "cat #{@path} | jq"
+      end
+      exit 1
+    end
+
+    def jq_available?
+      system("type jq > /dev/null 2>&1")
+    end
+  end
+end

data/lib/ufo/task_definition/erb/yaml.rb

@@ -0,0 +1,18 @@
+class Ufo::TaskDefinition::Erb
+  class Yaml < Base
+    def data
+      text = IO.read(@path)
+      YAML.load(text)
+    rescue Psych::SyntaxError => e
+      logger.error "ERROR: #{e.class}: #{e.message}"
+      logger.error <<~EOL
+        Rendered file contains invalid YAML. For debugging, files available at:
+
+        source:   #{@task_definition.path}
+        rendered: #{@path}
+
+      EOL
+      print_code(text)
+    end
+  end
+end

data/lib/ufo/task_definition/erb.rb

@@ -0,0 +1,78 @@
+require 'ostruct'
+
+class Ufo::TaskDefinition
+  class Erb < Ufo::CLI::Base
+    extend Memoist
+    include Context
+
+    delegate :name, :role, to: :task_definition
+    alias_method :task_definition_name, :name
+    alias_method :family, :name
+
+    def run
+      logger.debug "Building Task Definition"
+      clean
+      load_context
+      data = evaluate_code
+      check_empty!(data)
+      data = squeeze(data)
+      write(data)
+      logger.info "Task Definition built: #{output_path}"
+    end
+
+    def check_empty!(data)
+      return unless data == true || data == false || data.empty?
+      logger.error "ERROR: Empty task definition results".color(:red)
+      logger.error <<~EOL
+        The resulting task definition is empty.
+        Please double check that the task definition code is not empty.
+
+            #{pretty_path(@task_definition.path)}
+
+      EOL
+      exit 1
+    end
+
+    def evaluate_code
+      path = @task_definition.path
+      text = RenderMePretty.result(path, context: self)
+      rendered_path = "/tmp/ufo/task_definition#{File.extname(path)}"
+      FileUtils.mkdir_p(File.dirname(rendered_path))
+      IO.write(rendered_path, text)
+
+      o = @options.merge(path: rendered_path, task_definition: @task_definition)
+      if path.ends_with?('.json')
+        Json.new(o).data
+      else
+        Yaml.new(o).data
+      end
+    end
+
+    def squeeze(data)
+      Ufo::Utils::Squeezer.new(data).squeeze
+    end
+
+    def write(data)
+      data = override_image(data)
+      json = JSON.pretty_generate(data)
+      FileUtils.mkdir_p(File.dirname(output_path))
+      IO.write(output_path, json)
+    end
+
+    def override_image(data)
+      return data unless @options[:image]
+      data["containerDefinitions"].each do |container_definition|
+        container_definition["image"] = @options[:image]
+      end
+      data
+    end
+
+    def output_path
+      "#{Ufo.root}/.ufo/output/task_definition.json".sub(/^\.\//,'') # remove leading ./
+    end
+
+    def clean
+      FileUtils.rm_rf("#{Ufo.root}/.ufo/output")
+    end
+  end
+end

data/lib/ufo/task_definition/helpers/aws_data_helper.rb

@@ -0,0 +1,18 @@
+module Ufo::TaskDefinition::Helpers
+  module AwsDataHelper
+    extend Memoist
+    extend ActiveSupport::Concern
+
+    included do
+      delegate :account, :region, to: :aws_data
+      alias_method :aws_region, :region
+      alias_method :current_region, :region
+    end
+
+    # Duplicated in vars.rb
+    def aws_data
+      AwsData.new
+    end
+    memoize :aws_data
+  end
+end

data/lib/ufo/task_definition/helpers/core.rb

@@ -0,0 +1,50 @@
+# Core helper methods.
+#
+# * docker_image - Docker image name to be used when a the docker image is build. This is defined in .ufo/config.rb
+# * dockerfile_port - Exposed port in the Dockerfile.  Only supports one exposed port, the first one that is encountered.
+#
+module Ufo::TaskDefinition::Helpers
+  module Core
+    extend Memoist
+
+    def dockerfile_port
+      dockerfile_path = "#{Ufo.root}/Dockerfile"
+      if File.exist?(dockerfile_path)
+        parse_for_dockerfile_port(dockerfile_path)
+      end
+    end
+
+    def docker_image
+      # Dont need to use @options here. Helps simplify the Helper initialization.
+      Ufo::Docker::Builder.new({}).docker_image
+    end
+
+    def env(text)
+      Vars.new(text: text).env
+    end
+    alias_method :env_vars, :env
+    alias_method :environment, :env
+
+    def env_file(path)
+      Vars.new(file: path).env
+    end
+
+    def secrets(text)
+      Vars.new(text: text).secrets
+    end
+
+    def secrets_file(path)
+      Vars.new(file: path).secrets
+    end
+
+    def parse_for_dockerfile_port(dockerfile_path)
+      lines = IO.read(dockerfile_path).split("\n")
+      expose_line = lines.find { |l| l =~ /^EXPOSE / }
+      if expose_line
+        md = expose_line.match(/EXPOSE (\d+)/)
+        port = md[1] if md
+      end
+      port.to_i if port
+    end
+  end
+end

data/lib/ufo/task_definition/helpers/ssm/fetcher.rb

@@ -0,0 +1,39 @@
+module Ufo::TaskDefinition::Helpers::Ssm
+  class Fetcher
+    include Ufo::AwsServices
+    include Ufo::Utils::Logging
+    include Ufo::Concerns::Names
+
+    def initialize(options={})
+      @options = options
+      @base64 = options[:base64]
+    end
+
+    def fetch(name)
+      name = names.expansion(name, dasherize: false)
+      parameter = fetch_parameter(name)
+      return unless parameter
+      value = parameter.value
+      value = Base64.strict_encode64(value).strip if base64?(parameter.type)
+      value
+    end
+
+    def base64?(type)
+      if @base64.nil?
+        type == "SecureString"
+      else
+        @base64
+      end
+    end
+
+    # Note: Cannot use logger if since if ssm helper is used in config it'll cause an infinite loop
+    def fetch_parameter(name)
+      resp = ssm_client.get_parameter(name: name, with_decryption: true)
+      resp.parameter
+    rescue Aws::SSM::Errors::ParameterNotFound => e
+      puts "WARN: name #{name} not found".color(:yellow)
+      puts e.message
+      nil
+    end
+  end
+end

data/lib/ufo/task_definition/helpers/ssm.rb

@@ -0,0 +1,8 @@
+module Ufo::TaskDefinition::Helpers
+  module Ssm
+    def ssm(name, options={})
+      fetcher = Ssm::Fetcher.new(options)
+      fetcher.fetch(name)
+    end
+  end
+end

data/lib/ufo/task_definition/helpers/stack_output.rb

@@ -0,0 +1,23 @@
+module Ufo::TaskDefinition::Helpers
+  module StackOutput
+    include Ufo::AwsServices
+    include Ufo::Concerns::Names
+
+    def stack_output(name)
+      stack_name, output_key = name.split(".")
+      stack_name = names.expansion(stack_name)
+      resp = cloudformation.describe_stacks(stack_name: stack_name)
+      stack = resp.stacks.first
+      if stack
+        o = stack.outputs.detect { |h| h.output_key == output_key }
+      end
+
+      if o
+        o.output_value
+      else
+        logger.info "WARN: NOT FOUND: output #{key} for stack #{stack_name}"
+        nil
+      end
+    end
+  end
+end

data/lib/ufo/{dsl/helper → task_definition/helpers}/vars.rb

@@ -1,8 +1,11 @@
 require "aws_data"
 
-class Ufo::DSL::Helper
+module Ufo::TaskDefinition::Helpers
   class Vars
     extend Memoist
+    include AwsDataHelper
+    include Ufo::Concerns::Names
+    include Ufo::Utils::Pretty
 
     def initialize(options={})
       # use either file or text. text takes higher precedence
@@ -17,7 +20,7 @@ class Ufo::DSL::Helper
     def read(path)
       full_path = "#{Ufo.root}/#{path}"
       unless File.exist?(full_path)
-        puts "The #{full_path} env file could not be found.  Are you sure it exists?"
+        puts "The #{pretty_path(full_path)} env file could not be found.  Are you sure it exists?"
         exit 1
       end
       IO.read(full_path)
@@ -41,24 +44,55 @@ class Ufo::DSL::Helper
       secrets = env
       secrets.map do |item|
         value = item.delete(:value)
-        item[:valueFrom] = substitute(expand_secret(value))
+        arn = normalize_to_arn(item[:name], value)
+        value = expansion(arn)
+        value = value.sub('parameter//','parameter/') # auto fix accidental leading slash for user
+        item[:valueFrom] = value
       end
       secrets
     end
 
-    def expand_secret(value)
+    def normalize_to_arn(name, value)
       case value
       when /^ssm:/i
         value.sub(/^ssm:/i, "arn:aws:ssm:#{region}:#{account}:parameter/")
       when /^secretsmanager:/i
         value.sub(/^secretsmanager:/i, "arn:aws:secretsmanager:#{region}:#{account}:secret:")
+      when '' # blank string will mean use convention
+        conventional_pattern(name, value)
       else
         value # assume full arn has been passed
       end
     end
 
-    def substitute(value)
-      value.gsub(":UFO_ENV", Ufo.env)
+    # arn:aws:ssm:us-west-2:536766270177:parameter/demo/dev/DB-NAME
+    # arn:aws:ssm:us-west-2:536766270177:parameter/demo/dev/DB-NAME
+    def expansion(arn)
+      # performance improvement only run names.expansion on the name portion
+      md = arn.match(/(.*:)(parameter\/|secret:)(.*)/)
+      prefix, type, name = md[1], md[2], md[3]
+      expanded_name = names.expansion(name, dasherize: false) # dasherize: false. dont turn SECRET_NAME => SECRET-NAME
+      "#{prefix}#{type}#{expanded_name}"
+    end
+
+    # Examples with config.secrets.provider = "ssm"
+    #
+    # .secrets
+    #
+    #      DB_NAME
+    #
+    # Results
+    #
+    #      DB_NAME=:APP/:ENV/:SECRET_NAME # expansion will use => demo/dev/DB_NAME
+    #
+    def conventional_pattern(name, value)
+      secrets = Ufo.config.secrets
+      provider = secrets.provider # ssm or secretsmanager
+      namespace = provider == "ssm" ? "parameter/" : "secret:"
+      pattern = secrets.pattern[provider] # IE: Ufo.config.secrets.pattern.ssm => :APP/:ENV/:SECRET_NAME
+      # replace :SECRET_NAME since names expand doesnt know how to nor do we want to add logic there
+      pattern = pattern.sub(':SECRET_NAME', name)
+      "arn:aws:#{provider}:#{region}:#{account}:#{namespace}#{pattern}"
     end
 
     def remove_surrounding_quotes(s)
@@ -80,18 +114,5 @@ class Ufo::DSL::Helper
       # filter out empty lines
       lines = lines.reject { |l| l.strip.empty? }
     end
-
-    def aws_data
-      AwsData.new
-    end
-    memoize :aws_data
-
-    def region
-      aws_data.region
-    end
-
-    def account
-      aws_data.account
-    end
   end
 end

data/lib/ufo/task_definition/helpers/vpc.rb

@@ -0,0 +1,48 @@
+module Ufo::TaskDefinition::Helpers
+  module Vpc
+    extend Memoist
+    include Ufo::AwsServices
+
+    def default_vpc
+      resp = ec2.describe_vpcs(filters: [name: "isDefault", values: ["true"]])
+      vpc = resp.vpcs.first
+      if vpc
+        vpc.vpc_id
+      else
+        logger.error "No default vpc found".color(:red)
+        logger.error <<~EOL
+          Please configure the `config.vpc` settings.
+
+          Docs: https://ufoships.com/config/vpc/
+
+        EOL
+        exit 1
+      end
+    end
+    memoize :default_vpc
+
+    def subnets_for(vpc_id)
+      resp = ec2.describe_subnets(filters: [name: "vpc-id", values: [vpc_id]])
+      subnets = resp.subnets
+      subnets.map(&:subnet_id)
+    end
+
+    def default_subnets
+      if default_vpc.nil?
+        logger.error "ERROR: no default subnets because no default vpc found".color(:red)
+        exit 1
+      end
+      resp = ec2.describe_subnets(filters: [name: "vpc-id", values: [default_vpc]])
+      subnets = resp.subnets
+      subnets.map(&:subnet_id)
+    end
+    memoize :default_subnets
+
+    def key_pairs(regexp=nil)
+      resp = ec2.describe_key_pairs
+      key_names = resp.key_pairs.map(&:key_name)
+      key_names.select! { |k| k =~ regexp } if regexp
+      key_names
+    end
+  end
+end

data/lib/ufo/task_definition/helpers.rb

@@ -0,0 +1,5 @@
+class Ufo::TaskDefinition
+  module Helpers
+    include_modules("helpers") # builtin helpers
+  end
+end

data/lib/ufo/task_definition.rb

@@ -0,0 +1,19 @@
+module Ufo
+  class TaskDefinition
+    extend Memoist
+    include Ufo::Concerns # for names
+
+    attr_reader :name, :role
+    def initialize(options={})
+      @options = options
+      @role = Ufo.role
+      @name = names.task_definition # IE: :APP-:ROLE-:ENV => demo-web-dev
+    end
+
+    def path
+      expr = "#{Ufo.root}/.ufo/resources/task_definitions/{#{@role},web,default}.{json,yml}"
+      Dir.glob(expr).first
+    end
+    memoize :path
+  end
+end

data/lib/ufo/upgrade/upgrade3.rb

@@ -38,7 +38,7 @@ class Ufo::Upgrade
       lines.map! do |line|
         line.sub(/[\/]?ufo/, '.ufo')
       end
-      lines << [".ufo/data\n"] # new ignore rule
+      lines << [".ufo/tmp/state\n"] # new ignore rule
       text = lines.join
       IO.write(".gitignore", text)
     end

data/lib/ufo/upgrade/upgrade4.rb

@@ -36,8 +36,8 @@ class Ufo::Upgrade
   private
     def update_task_definitions
       text = <<-EOL
-    # HINT: shows how Ufo.env_extra can to create different log groups
-    # awslogs_group: ["ecs/TASK_DEFINITION_NAME", Ufo.env_extra].compact.join('-'),
+    # HINT: shows how Ufo.extra can to create different log groups
+    # awslogs_group: ["ecs/TASK_DEFINITION_NAME", Ufo.extra].compact.join('-'),
 EOL
       insert_into_file ".ufo/task_definitions.rb", text, :before => /    awslogs_group:/
     end
@@ -112,7 +112,7 @@ EOL
     def upsert_gitignore
       text =<<-EOL
 .ufo/current
-.ufo/data
+.ufo/tmp/state
 .ufo/log
 .ufo/output
 EOL

data/lib/ufo/utils/execute.rb

@@ -0,0 +1,30 @@
+module Ufo::Utils
+  module Execute
+    def execute(command, options={})
+      log = options[:log]
+      if log
+        command += " >> #{log}"
+        logger.info "=> #{command}"
+        FileUtils.mkdir_p(File.dirname(log))
+        File.open(log, 'a') { |f| f.puts "=> #{command}" }
+        out = `#{command}`
+        success = $?.success?
+        unless success
+          logger.error out
+          exit 1
+        end
+        out
+      else
+        logger.info "=> #{command}"
+        system(command)
+      end
+    end
+
+    # TODO: remove these other metohds
+    # Custom user params from .ufo/params.yml
+    # Param keys are symbols for the aws-sdk calls.
+    def user_params
+      @user_params ||= Param.new.data
+    end
+  end
+end

data/lib/ufo/utils/logging.rb

@@ -0,0 +1,7 @@
+module Ufo::Utils
+  module Logging
+    def logger
+      Ufo.logger
+    end
+  end
+end

data/lib/ufo/utils/pretty.rb

@@ -0,0 +1,18 @@
+module Ufo::Utils
+  module Pretty
+    def pretty_path(path)
+      path.sub("#{Ufo.root}/",'').sub(/^\.\//,'')
+    end
+
+    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
+    def pretty_time(total_seconds)
+      minutes = (total_seconds / 60) % 60
+      seconds = total_seconds % 60
+      if total_seconds < 60
+        "#{seconds.to_i}s"
+      else
+        "#{minutes.to_i}m #{seconds.to_i}s"
+      end
+    end
+  end
+end

data/lib/ufo/utils/squeezer.rb

@@ -13,7 +13,7 @@ module Ufo::Utils
       when Hash
         data.each_with_object({}) do |(k,v), squeezed|
           # only remove nil and empty Array values within Hash structures
-          squeezed[k] = squeeze(v) unless v.nil? || v.is_a?(Array) && v.empty?
+          squeezed[k] = squeeze(v) unless v.nil? # || v.is_a?(Array) && v.empty?
           squeezed
         end
       else

data/lib/ufo/utils/sure.rb

@@ -0,0 +1,23 @@
+module Ufo::Utils
+  module Sure
+    def sure?(message=nil)
+      confirm = 'Are you sure?'
+      if @options[:yes]
+        yes = 'y'
+      else
+        out = if message
+                "#{message}\n#{confirm} (y/N) "
+              else
+                "#{confirm} (y/N) "
+              end
+        print out
+        yes = $stdin.gets
+      end
+
+      unless yes =~ /^y/
+        puts "Whew! Exiting."
+        exit 0
+      end
+    end
+  end
+end

data/lib/ufo/version.rb

@@ -1,3 +1,3 @@
 module Ufo
-  VERSION = "5.0.7"
+  VERSION = "6.0.0"
 end

data/lib/ufo/yaml/loader.rb

@@ -0,0 +1,48 @@
+require "yaml"
+
+class Ufo::Yaml
+  class Loader
+    def initialize(text)
+      @text = text
+    end
+
+    def load
+      add_domain_types!
+      YAML.load(@text)
+    end
+
+  private
+    def add_domain_types!
+      intrinsic_functions.each do |name|
+        YAML.add_domain_type('', name) do |type,val|
+          key = type.split('::').last
+          key = "Fn::" + key unless name == 'Ref'
+          { key => val }
+        end
+      end
+    end
+
+    def intrinsic_functions
+      %w[
+        And
+        Base64
+        Cidr
+        Equals
+        FindInMap
+        GetAtt
+        GetAZs
+        If
+        If
+        ImportValue
+        Join
+        Not
+        Or
+        Ref
+        Select
+        Split
+        Sub
+        Transform
+      ]
+    end
+  end
+end

data/lib/ufo/yaml/validator.rb

@@ -0,0 +1,51 @@
+class Ufo::Yaml
+  class Validator
+    def initialize(path)
+      @path = path
+    end
+
+    def validate!
+      validate_yaml(@path)
+    end
+
+    def validate_yaml(path)
+      text = IO.read(path)
+      begin
+        YAML.load(text)
+      rescue Psych::SyntaxError => e
+        handle_yaml_syntax_error(e, path)
+      end
+    end
+
+    def handle_yaml_syntax_error(e, path)
+      io = StringIO.new
+      io.puts "Invalid yaml. Output written for debugging: #{path}".color(:red)
+      io.puts "ERROR: #{e.message}".color(:red)
+
+      # Grab line info.  Example error:
+      #   ERROR: (<unknown>): could not find expected ':' while scanning a simple key at line 2 column 1
+      md = e.message.match(/at line (\d+) column (\d+)/)
+      line = md[1].to_i
+
+      lines = IO.read(path).split("\n")
+      context = 5 # lines of context
+      top, bottom = [line-context-1, 0].max, line+context-1
+      spacing = lines.size.to_s.size
+      lines[top..bottom].each_with_index do |line_content, index|
+        line_number = top+index+1
+        if line_number == line
+          io.printf("%#{spacing}d %s\n".color(:red), line_number, line_content)
+        else
+          io.printf("%#{spacing}d %s\n", line_number, line_content)
+        end
+      end
+
+      if ENV['LONO_TEST']
+        io.string
+      else
+        puts io.string
+        exit 1
+      end
+    end
+  end
+end