ufo 5.0.6 → 6.0.1

data/lib/ufo/{role → iam_role}/dsl.rb

@@ -1,11 +1,13 @@
-module Ufo::Role
+module Ufo::IamRole
   class DSL
+    include DslEvaluator
+
     def initialize(path)
       @path = path # IE: .ufo/iam_roles/task_role.rb
     end
 
     def evaluate
-      instance_eval(IO.read(@path), @path)
+      evaluate_file(@path)
     end
 
     def iam_policy(policy_name, statements)
@@ -17,5 +19,9 @@ module Ufo::Role
       role_type = File.basename(@path).sub('.rb','') # task_role or execution_role
       Registry.register_managed_policy(role_type, policies)
     end
+
+    def aws
+      AwsData.new
+    end
   end
 end

data/lib/ufo/{role → iam_role}/registry.rb

@@ -1,6 +1,6 @@
 require "set"
 
-module Ufo::Role
+module Ufo::IamRole
   class Registry
     class_attribute :policies
     self.policies = {}

data/lib/ufo/info.rb

@@ -1,5 +1,7 @@
 module Ufo
-  class Info < Base
+  class Info < Ufo::CLI::Base
+    include Ufo::AwsServices
+
     def run
       unless service
         puts no_service_message
@@ -7,12 +9,15 @@ module Ufo
       end
       puts "Resources:"
       stack_resources.each do |r|
-        # pp r
         puts "#{r.logical_resource_id} - #{r.resource_type}:".color(:green)
         puts "  #{r.physical_resource_id}"
       end
     end
 
+    def no_service_message
+      "No stack #{@stack_name.color(:green)} found"
+    end
+
     # Passing in service so method can be used else where.
     def load_balancer(service)
       load_balancer = service.load_balancers.first
@@ -33,24 +38,37 @@ module Ufo
       return unless stack
 
       service = stack_resources.find { |r| r.resource_type == "AWS::ECS::Service" }
+      return unless service # stack is still creating
       arn = service.physical_resource_id
       resp = ecs.describe_services(services: [arn], cluster: @cluster)
       resp.services.first
     end
     memoize :service
 
+    def service?
+      !!service
+    end
+
     def stack
       find_stack(@stack_name)
     end
     memoize :stack
 
-    def route53_dns
+    def url
       return unless stack
 
       output = stack.outputs.find do |o|
         o.output_key == "Route53Dns"
       end
-      output.output_value if output
+      dns_name = output.output_value if output
+      return unless dns_name
+
+      ssl = stack_resources.detect.find do |r|
+        r.logical_resource_id == "ListenerSsl"
+      end
+
+      protocol = ssl ? 'https' : 'http'
+      "#{protocol}://#{dns_name}"
     end
 
     def stack_resources

data/lib/ufo/layering/layer.rb

@@ -0,0 +1,95 @@
+require "aws_data"
+
+# Layers example
+#
+#     .ufo/vars.rb
+#     .ufo/vars/base.rb
+#     .ufo/vars/dev.rb
+#     .ufo/vars/prod.rb
+#     .ufo/vars/app1.rb
+#     .ufo/vars/app1/base.rb
+#     .ufo/vars/app1/dev.rb
+#     .ufo/vars/app1/prod.rb
+#
+module Ufo::Layering
+  class Layer
+    extend Memoist
+    include Ufo::Layering
+    include Ufo::Utils::Logging
+    include Ufo::Utils::Pretty
+
+    def initialize(task_definition)
+      @task_definition = task_definition
+    end
+
+    def paths
+      core = full_layers(".ufo/vars")
+      app = full_layers(".ufo/vars/#{Ufo.app}")
+      paths = core + app
+      add_ext!(paths)
+      paths.map! { |p| "#{Ufo.root}/#{p}" }
+      show_layers(paths)
+      paths
+    end
+
+    def full_layering
+      # layers defined in Lono::Layering module
+      all = layers.map { |layer| layer.sub(/\/$/,'') } # strip trailing slash
+      all.inject([]) do |sum, layer|
+        sum += layer_levels(layer) unless layer.nil?
+        sum
+      end
+    end
+
+    # interface method
+    def main_layers
+      ['']
+    end
+
+    # adds prefix and to each layer pair that has base and Ufo.env. IE:
+    #
+    #    "#{prefix}/base"
+    #    "#{prefix}/#{Ufo.env}"
+    #
+    def layer_levels(prefix=nil)
+      levels = ["base", Ufo.env]
+      levels.map! do |i|
+        # base layer has prefix of '', reject with blank so it doesnt produce '//'
+        [prefix, i].reject(&:blank?).join('/')
+      end
+      levels.unshift(prefix) # unless prefix.blank? # IE: params/us-west-2.txt
+      levels
+    end
+
+    def add_ext!(paths)
+      ext = "rb"
+      paths.map! do |path|
+        path = path.sub(/\/$/,'') if path.ends_with?('/')
+        "#{path}.rb"
+      end
+      paths
+    end
+
+    def full_layers(dir)
+      layers = full_layering.map do |layer|
+        "#{dir}/#{layer}"
+      end
+      role_layers = full_layering.map do |layer|
+        "#{dir}/#{@task_definition.role}/#{layer}" # Note: layer can be '' will clean up
+      end
+      layers += role_layers
+      layers.map { |l| l.gsub('//','/') } # cleanup // if layer is ''
+    end
+
+    @@shown_layers = false
+    def show_layers(paths)
+      return if @@shown_layers
+      logger.info "Vars Layers:" if ENV['UFO_SHOW_ALL_LAYERS']
+      paths.each do |path|
+        show_layer = File.exist?(path) && logger.level <= Logger::DEBUG
+        logger.info "    #{pretty_path(path)}" if show_layer || ENV['UFO_SHOW_ALL_LAYERS']
+      end
+      @@shown_layers = true
+    end
+  end
+end

data/lib/ufo/layering.rb

@@ -0,0 +1,23 @@
+require "active_support/lazy_load_hooks"
+
+module Ufo
+  module Layering
+    def layers
+      pre_layers + main_layers + post_layers
+    end
+
+    def main_layers
+      super
+    end
+
+    def pre_layers
+      []
+    end
+
+    def post_layers
+      []
+    end
+  end
+end
+
+ActiveSupport.run_load_hooks(:ufo_layering, Ufo::Layering)

data/lib/ufo/log_group.rb

@@ -1,24 +1,20 @@
 # Use to automatically create the CloudWatch group.
 # For some reason creating ECS does do this by default.
 module Ufo
-  class LogGroup
-    include AwsService
-
-    def initialize(task_definition, options)
-      @task_definition, @options = task_definition, options
-    end
+  class LogGroup < Ufo::CLI::Base
+    include Ufo::AwsServices
 
     def create
-      puts "Ensuring log group for #{@task_definition.color(:green)} task definition exists"
+      logger.debug "Ensuring log group for #{@task_definition.name.color(:green)} task definition exists"
       return if @options[:noop]
       return if @options[:rollback] # dont need to create log group because previously deployed
 
-      Ufo.check_task_definition!(@task_definition)
+      check_task_definition_exists!
       task_def = JSON.load(IO.read(task_def_path))
       task_def["containerDefinitions"].each do |container_def|
         begin
           log_group_name = container_def["logConfiguration"]["options"]["awslogs-group"]
-          puts "Log group name: #{log_group_name}"
+          logger.debug "Log group name: #{log_group_name}"
         rescue NoMethodError
           # silence when the logConfiguration is not specified
         end
@@ -34,7 +30,19 @@ module Ufo
     end
 
     def task_def_path
-      "#{Ufo.root}/.ufo/output/#{@task_definition}.json"
+      "#{Ufo.root}/.ufo/output/task_definition.json"
+    end
+
+    def check_task_definition_exists!
+      return if File.exist?(task_def_path)
+      logger.error "ERROR: Unable to find the task definition at #{task_def_path}.".color(:red)
+      logger.error <<~EOL
+          Please double check that it was built correctly with:
+
+              ufo build
+
+      EOL
+      exit 1
     end
   end
 end

data/lib/ufo/logger/formatter.rb

@@ -0,0 +1,13 @@
+class Ufo::Logger
+  class Formatter < ::Logger::Formatter
+    def call(severity, time, progname, msg)
+      # careful changing the format. All::Summary uses a regexp on this format to remove the timestamp
+      "[#{format_datetime(time)} ##{Process.pid} #{progname}]: #{msg}"
+    end
+
+  private
+    def format_datetime(time)
+      time.strftime("%Y-%m-%dT%H:%M:%S")
+    end
+  end
+end

data/lib/ufo/logger.rb

@@ -0,0 +1,32 @@
+require 'logger'
+
+module Ufo
+  class Logger < ::Logger
+    def initialize(*args)
+      super
+      self.formatter = Formatter.new
+      self.level = ENV['UFO_LOG_LEVEL'] || :info # note: only respected when config.logger not set in config/app.rb
+    end
+
+    def format_message(severity, datetime, progname, msg)
+      line = if @logdev.dev == $stdout || @logdev.dev == $stderr || @logdev.dev.is_a?(StringIO)
+        msg # super simple format if stdout
+      else
+        super # use the configured formatter
+      end
+      line =~ /\n$/ ? line : "#{line}\n"
+    end
+
+    # Used to allow output to always go to stdout
+    def stdout(msg, newline: true)
+      if newline
+        puts msg
+      else
+        print msg
+      end
+    end
+
+    public :print
+    public :printf
+  end
+end

data/lib/ufo/names.rb

@@ -0,0 +1,69 @@
+module Ufo
+  class Names
+    extend Memoist
+
+    attr_reader :role
+    def initialize
+      @role = Ufo.role
+    end
+
+    def cluster
+      expansion(Ufo.config.ecs.cluster) # IE: :ENV => dev
+    end
+    memoize :cluster
+
+    def stack
+      name = expansion(Ufo.config.names.stack) # IE: :APP-:ROLE-:ENV => demo-web-dev
+      [name, Ufo.extra].compact.join('-')
+    end
+    memoize :stack
+
+    def task_definition
+      expansion(Ufo.config.names.task_definition) # IE: :APP-:ROLE-:ENV => demo-web-dev
+    end
+    memoize :task_definition
+
+    def expansion(string, options={})
+      return string unless string.is_a?(String) # in case of nil
+
+      string = string.dup
+      vars = string.scan(/:\w+/) # => [":APP", ":ROLE", :ENV"]
+      vars.each do |var|
+        string.gsub!(var, var_value(var))
+      end
+      string = strip(string)
+      dashes = options[:dasherize].nil? ? true : options[:dasherize]
+      string = string.dasherize if dashes
+      string
+    end
+
+    def var_value(unexpanded)
+      name = unexpanded.sub(':','').downcase
+      if respond_to?(name)
+        send(name).to_s # pass value straight through
+      else
+        unexpanded
+      end
+    end
+
+    def strip(string)
+      string.sub(/^-+/,'').sub(/-+$/,'') # remove leading and trailing -
+            .gsub(%r{-+},'-') # remove double dashes are more. IE: -- => -
+    end
+
+    def app
+      Ufo.app
+    end
+
+    def env
+      Ufo.env
+    end
+    alias_method :ufo_env, :env
+
+    delegate :region, to: :aws
+    def aws
+      AwsData.new
+    end
+    memoize :aws
+  end
+end

data/lib/ufo/param.rb

@@ -18,7 +18,8 @@ module Ufo
     memoize :data
 
     def template_scope
-      @template_scope ||= Ufo::TemplateScope.new(Ufo::DSL::Helper.new, nil)
+      self # TODO: add access to helpers like network
+      # @template_scope ||= Ufo::TemplateScope.new(Ufo::DSL::Helper.new, nil)
     end
   end
 end

data/lib/ufo/task_definition/builder.rb

@@ -0,0 +1,7 @@
+class Ufo::TaskDefinition
+  class Builder < Ufo::CLI::Base
+    def build
+      Erb.new(@options).run
+    end
+  end
+end

data/lib/ufo/task_definition/context.rb

@@ -0,0 +1,45 @@
+class Ufo::TaskDefinition
+  module Context
+    include DslEvaluator
+    include Helpers
+
+    def load_context
+      load_variables
+      load_custom_helpers
+    end
+
+    def load_variables
+      layers = Ufo::Layering::Layer.new(@task_definition).paths
+      layers.each do |layer|
+        evaluate_file(layer)
+      end
+    end
+
+    def load_custom_helpers
+      load_helper_files("#{Ufo.root}/.ufo/helpers")
+    end
+
+    # Load custom project helper methods
+    def load_helper_files(dir)
+      paths = Dir.glob("#{dir}/**/*.rb")
+      paths.sort_by! { |p| p.size } # so namespaces are loaded first
+      paths.each do |path|
+        next unless File.file?(path)
+
+        filename = path.sub(%r{.*/helpers/},'').sub('.rb','')
+        module_name = filename.camelize
+
+        # Prepend a period so require works when UFO_ROOT is set to a relative path without a period.
+        #   Example: UFO_ROOT=tmp/ufo_project
+        first_char = path[0..0]
+        path = "./#{path}" unless %w[. /].include?(first_char)
+
+        # Examples:
+        #     path:        .ufo/helpers/custom_helper.rb
+        #     module_name: CustomHelper
+        require path
+        self.class.send :include, module_name.constantize
+      end
+    end
+  end
+end

data/lib/ufo/task_definition/erb/base.rb

@@ -0,0 +1,18 @@
+class Ufo::TaskDefinition::Erb
+  class Base < Ufo::CLI::Base
+    def initialize(options={})
+      super
+      @path = options[:path]
+      @task_definition = options[:task_definition]
+    end
+
+    def print_code(text)
+      lines = text.split("\n")
+      lpad = lines.size.to_s.size
+      lines.each_with_index do |line,n|
+        printf("%#{lpad}d %s\n", n+1, line)
+      end
+      nil
+    end
+  end
+end

data/lib/ufo/task_definition/erb/json.rb

@@ -0,0 +1,28 @@
+class Ufo::TaskDefinition::Erb
+  class Json < Base
+    def data
+      text = IO.read(@path)
+      JSON.load(text)
+    rescue JSON::ParserError => e
+      # NOTE: JSON::ParserError e.message is not very useful
+      logger.error "ERROR: #{e.class}".color(:red)
+      logger.error <<~EOL
+        Rendered file contains invalid JSON. For debugging, files available at:
+
+        source:   #{@task_definition.path}
+        rendered: #{@path}
+
+      EOL
+      logger.error "Contents of the rendered file:\n\n"
+      print_code(text)
+      if jq_available?
+        system "cat #{@path} | jq"
+      end
+      exit 1
+    end
+
+    def jq_available?
+      system("type jq > /dev/null 2>&1")
+    end
+  end
+end

data/lib/ufo/task_definition/erb/yaml.rb

@@ -0,0 +1,18 @@
+class Ufo::TaskDefinition::Erb
+  class Yaml < Base
+    def data
+      text = IO.read(@path)
+      YAML.load(text)
+    rescue Psych::SyntaxError => e
+      logger.error "ERROR: #{e.class}: #{e.message}"
+      logger.error <<~EOL
+        Rendered file contains invalid YAML. For debugging, files available at:
+
+        source:   #{@task_definition.path}
+        rendered: #{@path}
+
+      EOL
+      print_code(text)
+    end
+  end
+end

data/lib/ufo/task_definition/erb.rb

@@ -0,0 +1,78 @@
+require 'ostruct'
+
+class Ufo::TaskDefinition
+  class Erb < Ufo::CLI::Base
+    extend Memoist
+    include Context
+
+    delegate :name, :role, to: :task_definition
+    alias_method :task_definition_name, :name
+    alias_method :family, :name
+
+    def run
+      logger.debug "Building Task Definition"
+      clean
+      load_context
+      data = evaluate_code
+      check_empty!(data)
+      data = squeeze(data)
+      write(data)
+      logger.info "Task Definition built: #{output_path}"
+    end
+
+    def check_empty!(data)
+      return unless data == true || data == false || data.empty?
+      logger.error "ERROR: Empty task definition results".color(:red)
+      logger.error <<~EOL
+        The resulting task definition is empty.
+        Please double check that the task definition code is not empty.
+
+            #{pretty_path(@task_definition.path)}
+
+      EOL
+      exit 1
+    end
+
+    def evaluate_code
+      path = @task_definition.path
+      text = RenderMePretty.result(path, context: self)
+      rendered_path = "/tmp/ufo/task_definition#{File.extname(path)}"
+      FileUtils.mkdir_p(File.dirname(rendered_path))
+      IO.write(rendered_path, text)
+
+      o = @options.merge(path: rendered_path, task_definition: @task_definition)
+      if path.ends_with?('.json')
+        Json.new(o).data
+      else
+        Yaml.new(o).data
+      end
+    end
+
+    def squeeze(data)
+      Ufo::Utils::Squeezer.new(data).squeeze
+    end
+
+    def write(data)
+      data = override_image(data)
+      json = JSON.pretty_generate(data)
+      FileUtils.mkdir_p(File.dirname(output_path))
+      IO.write(output_path, json)
+    end
+
+    def override_image(data)
+      return data unless @options[:image]
+      data["containerDefinitions"].each do |container_definition|
+        container_definition["image"] = @options[:image]
+      end
+      data
+    end
+
+    def output_path
+      "#{Ufo.root}/.ufo/output/task_definition.json".sub(/^\.\//,'') # remove leading ./
+    end
+
+    def clean
+      FileUtils.rm_rf("#{Ufo.root}/.ufo/output")
+    end
+  end
+end

data/lib/ufo/task_definition/helpers/aws_data_helper.rb

@@ -0,0 +1,18 @@
+module Ufo::TaskDefinition::Helpers
+  module AwsDataHelper
+    extend Memoist
+    extend ActiveSupport::Concern
+
+    included do
+      delegate :account, :region, to: :aws_data
+      alias_method :aws_region, :region
+      alias_method :current_region, :region
+    end
+
+    # Duplicated in vars.rb
+    def aws_data
+      AwsData.new
+    end
+    memoize :aws_data
+  end
+end

data/lib/ufo/task_definition/helpers/core.rb

@@ -0,0 +1,50 @@
+# Core helper methods.
+#
+# * docker_image - Docker image name to be used when a the docker image is build. This is defined in .ufo/config.rb
+# * dockerfile_port - Exposed port in the Dockerfile.  Only supports one exposed port, the first one that is encountered.
+#
+module Ufo::TaskDefinition::Helpers
+  module Core
+    extend Memoist
+
+    def dockerfile_port
+      dockerfile_path = "#{Ufo.root}/Dockerfile"
+      if File.exist?(dockerfile_path)
+        parse_for_dockerfile_port(dockerfile_path)
+      end
+    end
+
+    def docker_image
+      # Dont need to use @options here. Helps simplify the Helper initialization.
+      Ufo::Docker::Builder.new({}).docker_image
+    end
+
+    def env(text)
+      Vars.new(text: text).env
+    end
+    alias_method :env_vars, :env
+    alias_method :environment, :env
+
+    def env_file(path)
+      Vars.new(file: path).env
+    end
+
+    def secrets(text)
+      Vars.new(text: text).secrets
+    end
+
+    def secrets_file(path)
+      Vars.new(file: path).secrets
+    end
+
+    def parse_for_dockerfile_port(dockerfile_path)
+      lines = IO.read(dockerfile_path).split("\n")
+      expose_line = lines.find { |l| l =~ /^EXPOSE / }
+      if expose_line
+        md = expose_line.match(/EXPOSE (\d+)/)
+        port = md[1] if md
+      end
+      port.to_i if port
+    end
+  end
+end