ufo 5.0.7 → 6.0.0

data/lib/ufo/config.rb

@@ -0,0 +1,187 @@
+require "singleton"
+
+module Ufo
+  class Config
+    extend Memoist
+    include DslEvaluator
+    include Singleton
+    include Ufo::Utils::Logging
+
+    include Ufo::TaskDefinition::Helpers
+
+    attr_reader :config
+    def initialize
+      @config = defaults
+    end
+
+    def defaults
+      config = ActiveSupport::OrderedOptions.new
+
+      config.app = nil # required unless UFO_APP is set
+
+      config.autoscaling = ActiveSupport::OrderedOptions.new
+      config.autoscaling.enabled = true
+      config.autoscaling.max_capacity = 5 # dont use max thats an OrderedOptions method
+      config.autoscaling.min_capacity = 1 # dont use min thats an OrderedOptions method
+      config.autoscaling.predefined_metric_type = "ECSServiceAverageCPUUtilization"
+      config.autoscaling.scale_in_cooldown = nil
+      config.autoscaling.scale_out_cooldown = nil
+      config.autoscaling.target_value = 75.0
+
+      config.cfn = ActiveSupport::OrderedOptions.new
+      config.cfn.disable_rollback = nil
+      config.cfn.notification_arns = nil
+      config.cfn.tags = nil # should be Ruby Hash
+
+      config.dns = ActiveSupport::OrderedOptions.new
+      config.dns.comment = "cname to load balancer created by ufo"
+      config.dns.domain = nil # only recommended option to set
+      config.dns.hosted_zone_id = nil
+      config.dns.hosted_zone_name = nil
+      config.dns.name = nil
+      config.dns.ttl = 60
+      config.dns.type = "CNAME"
+
+      config.docker = ActiveSupport::OrderedOptions.new
+      config.docker.clean_keep = nil
+      config.docker.ecr_keep = nil
+      config.docker.repo = nil # required IE: org/repo basename of the Docker image
+
+      config.ecs = ActiveSupport::OrderedOptions.new
+      config.ecs.cluster = ":ENV" # => dev
+      config.ecs.desired_count = nil # only respected when config.autoscaling.enabled = false
+      config.ecs.scheduling_strategy = "REPLICA"
+
+      config.elb = ActiveSupport::OrderedOptions.new
+      config.elb.subnet_mappings = nil # static IP addresses for network load balancer
+      config.elb.enabled = "auto" # "auto", true or false
+      config.elb.port = 80 # default listener port
+      config.elb.ssl = ActiveSupport::OrderedOptions.new
+      config.elb.ssl.certificates = nil
+      config.elb.ssl.enabled = false
+      config.elb.ssl.port = 443
+      config.elb.type = "application"
+      config.elb.default_actions = nil # full override
+      config.elb.redirect = ActiveSupport::OrderedOptions.new
+      config.elb.redirect.code = 302  # IE: 302 or 301
+      config.elb.redirect.enabled = false
+      config.elb.redirect.port = 443
+      config.elb.redirect.protocol = "HTTPS"
+
+      config.exec = ActiveSupport::OrderedOptions.new
+      config.exec.command = "/bin/bash" # aws ecs execute-command cli
+      config.exec.enabled = true        # EcsService EnableExecuteCommand
+
+      config.log = ActiveSupport::OrderedOptions.new
+      config.log.root = Ufo.log_root
+      config.logger = ufo_logger
+      config.logger.formatter = Logger::Formatter.new
+      config.logger.level = ENV['UFO_LOG_LEVEL'] || :info
+
+      config.logs = ActiveSupport::OrderedOptions.new
+      config.logs.filter_pattern = nil
+
+      config.names = ActiveSupport::OrderedOptions.new
+      config.names.stack = ":APP-:ROLE-:ENV" # => demo-web-dev
+      config.names.task_definition = ":APP-:ROLE-:ENV" # => demo-web-dev
+
+      config.ps = ActiveSupport::OrderedOptions.new
+      config.ps.format = "auto" # CliFormat.default_format
+      config.ps.hide_age = 5 # in minutes. IE: hide tasks that are older than 5 minutes
+      config.ps.summary = true
+
+      config.secrets = ActiveSupport::OrderedOptions.new
+      config.secrets.pattern = ActiveSupport::OrderedOptions.new
+      config.secrets.pattern.secretsmanager = ":APP-:ENV-:SECRET_NAME" # => demo-dev-DB_PASS
+      config.secrets.pattern.ssm = ":APP/:ENV/:SECRET_NAME" # => demo/dev/DB_PASS
+      config.secrets.provider = "ssm" # default provider for conventional expansion IE: ssm or secretsmanager
+
+      config.scale = ActiveSupport::OrderedOptions.new
+      config.scale.warning = true
+
+      config.ship = ActiveSupport::OrderedOptions.new
+      config.ship.docker = ActiveSupport::OrderedOptions.new
+      config.ship.docker.quiet = true # only affects ufo ship docker commands output
+
+      config.state = ActiveSupport::OrderedOptions.new
+      config.state.reminder = true
+
+      # When not set, the default vpc is used
+      config.vpc = ActiveSupport::OrderedOptions.new
+      config.vpc.id = nil
+      config.vpc.security_groups = ActiveSupport::OrderedOptions.new
+      config.vpc.security_groups.ecs = nil
+      config.vpc.security_groups.elb = nil
+      config.vpc.security_groups.managed = true
+      config.vpc.subnets = ActiveSupport::OrderedOptions.new
+      config.vpc.subnets.ecs = nil
+      config.vpc.subnets.elb = nil
+
+      config
+    end
+
+    def ufo_logger
+      Logger.new(ENV['UFO_LOG_PATH'] || $stderr)
+    end
+    memoize :ufo_logger
+
+    def configure
+      yield(@config)
+    end
+
+    # load_project_config gets called so early many things like logger is not available.
+    # Take care not to rely on things that rely on the the config or else will create
+    # and infinite loop.
+    def load_project_config
+      root = layer_levels(".ufo/config")
+      env  = layer_levels(".ufo/config/env")
+      role = layer_levels(".ufo/config/#{Ufo.role}")
+      layers = root + env + role
+      # Dont use Ufo.app or that'll cause infinite loop since it loads Ufo.config
+      if ENV['UFO_APP']
+        root = layer_levels(".ufo/config/#{Ufo.app}")
+        env  = layer_levels(".ufo/config/#{Ufo.app}/env")
+        role = layer_levels(".ufo/config/#{Ufo.app}/#{Ufo.role}")
+        layers += root + env + role
+      end
+      # load_project_config gets called so early that logger is not yet configured. use puts
+      puts "Config layers:" if ENV['UFO_SHOW_ALL_LAYERS']
+      layers.each do |layer|
+        path = "#{Ufo.root}/#{layer}"
+        puts "    #{layer}" if ENV['UFO_SHOW_ALL_LAYERS']
+        evaluate_file(path)
+      end
+    end
+
+    # Works similiar to Layering::Layer. Consider combining the logic and usin Layering::Layer
+    #
+    # Examples:
+    #
+    #     prefix: .ufo/config/#{Ufo.app}/env
+    #
+    # Returns
+    #
+    #     .ufo/config/#{Ufo.app}/env.rb
+    #     .ufo/config/#{Ufo.app}/env/base.rb
+    #     .ufo/config/#{Ufo.app}/env/#{Ufo.env}.rb
+    #
+    def layer_levels(prefix=nil)
+      levels = ["", "base", Ufo.env]
+      paths = levels.map do |i|
+        # base layer has prefix of '', reject with blank so it doesnt produce '//'
+        [prefix, i].join('/')
+      end
+      add_ext!(paths)
+    end
+
+      def add_ext!(paths)
+      ext = "rb"
+      paths.map! do |path|
+        path = path.sub(/\/$/,'') if path.ends_with?('/')
+        "#{path}.rb"
+      end
+      paths
+    end
+
+end
+end

data/lib/ufo/core.rb

@@ -4,65 +4,53 @@ require 'yaml'
 module Ufo
   module Core
     extend Memoist
-    include Ufo::Settings
 
-    def check_task_definition!(task_definition)
-      task_definition_path = "#{Ufo.root}/.ufo/output/#{task_definition}.json"
-      unless File.exist?(task_definition_path)
-        puts "ERROR: Unable to find the task definition at #{task_definition_path}.".color(:red)
-        puts "Are you sure you have defined it in .ufo/task_definitions.rb and it has been generated correctly in .ufo/output?".color(:red)
-        puts "If you are calling `ufo deploy` directly, you might want to generate the task definition first with `ufo tasks build`."
-        exit 1
-      end
+    def role
+      ENV['UFO_ROLE'] || 'web'
     end
 
-    def root
-      path = ENV['UFO_ROOT'] || '.'
-      Pathname.new(path)
+    def app
+      ENV['UFO_APP'] || config.app
     end
 
+    # v5: development is default
+    # v6: dev is default
     def env
-      # 2-way binding
-      ufo_env = env_from_profile || 'development'
-      ufo_env = ENV['UFO_ENV'] if ENV['UFO_ENV'] # highest precedence
-      ufo_env
+      ENV['UFO_ENV'] || 'dev'
     end
     memoize :env
 
-    def env_extra
-      env_extra = Current.env_extra
-      env_extra = ENV['UFO_ENV_EXTRA'] if ENV['UFO_ENV_EXTRA'] # highest precedence
-      return if env_extra&.empty?
-      env_extra
+    def extra
+      extra = ENV['UFO_EXTRA'] if ENV['UFO_EXTRA'] # highest precedence
+      return if extra&.empty?
+      extra
+    end
+    memoize :extra
+
+    def root
+      path = ENV['UFO_ROOT'] || '.'
+      Pathname.new(path)
+    end
+
+    def log_root
+      "#{root}/log"
     end
-    memoize :env_extra
 
-    # Overrides AWS_PROFILE based on the Ufo.env if set in .ufo/settings.yml
-    # 2-way binding.
-    def set_aws_profile!
-      return if ENV['TEST']
-      return unless File.exist?("#{Ufo.root}/.ufo/settings.yml") # for rake docs
-      return unless settings # Only load if within Ufo project and there's a settings.yml
-      data = settings || {}
-      if data[:aws_profile]
-        puts "Using AWS_PROFILE=#{data[:aws_profile]} from UFO_ENV=#{Ufo.env} in config/settings.yml"
-        ENV['AWS_PROFILE'] = data[:aws_profile]
-      end
+    def configure(&block)
+      Config.instance.configure(&block)
     end
 
-    def check_ufo_project!
-      check_path = "#{Ufo.root}/.ufo/settings.yml"
-      unless File.exist?(check_path)
-        puts "ERROR: No settings file at #{check_path}.  Are you sure you are in a project with ufo setup?".color(:red)
-        puts "Current directory: #{Dir.pwd}"
-        puts "If you want to set up ufo for this prjoect, please create a settings file via: ufo init"
-        exit 1 unless ENV['TEST']
-      end
+    # Generally, use the Lono.config instead of Config.instance.config since it guarantees the load_project_config call
+    def config
+      Config.instance.load_project_config
+      Config.instance.config
     end
+    memoize :config
 
-  private
-    def env_from_profile
-      Ufo::Setting.new.ufo_env
+    # allow different logger when running up all or rspec-lono
+    cattr_writer :logger
+    def logger
+      @@logger ||= config.logger
     end
   end
 end

data/lib/ufo/docker/builder.rb

@@ -1,12 +1,11 @@
-require 'active_support/core_ext/module/delegation'
-
-class Ufo::Docker
+module Ufo::Docker
   class Builder
-    include Ufo::Util
+    extend Memoist
+    include Concerns
 
     delegate :push, to: :pusher
-    def self.build(options)
-      builder = Builder.new(options) # outside if because it need builder.full_image_name
+    def self.build(options={})
+      builder = Builder.new(options) # outside if because it need builder.docker_image
       builder.build
       pusher = Pusher.new(nil, options)
       pusher.push
@@ -21,28 +20,27 @@ class Ufo::Docker
 
     def build
       start_time = Time.now
-      store_full_image_name
+      store_docker_image
 
-      command = "docker build #{build_options}-t #{full_image_name} -f #{@dockerfile} ."
-      say "Building docker image with:".color(:green)
-      say "  #{command}".color(:green)
+      logger.info "Building Docker Image"
       compile_dockerfile_erb
       check_dockerfile_exists
       update_auth_token
-      command = "cd #{Ufo.root} && #{command}"
-      success = execute(command, use_system: true)
+      command = "docker build #{build_options}-t #{docker_image} -f #{@dockerfile} ."
+      log = ".ufo/log/docker.log" if @options[:quiet]
+      success = execute(command, log: log)
       unless success
         docker_version_success = system("docker version > /dev/null 2>&1")
         unless docker_version_success
           docker_version_message = "  Are you sure the docker daemon is available?  Try running: docker version."
         end
-        puts "ERROR: The docker image fail to build.#{docker_version_message}".color(:red)
+        logger.info "ERROR: Fail to build Docker image.#{docker_version_message}".color(:red)
         exit 1
       end
 
       took = Time.now - start_time
-      say "Docker image #{full_image_name} built.  "
-      say "Docker build took #{pretty_time(took)}.".color(:green)
+      logger.info "Docker Image built: #{docker_image}"
+      logger.info "Took #{pretty_time(took)}"
     end
 
     def build_options
@@ -78,7 +76,7 @@ class Ufo::Docker
     end
 
     def pusher
-      @pusher ||= Pusher.new(full_image_name, @options)
+      @pusher ||= Pusher.new(docker_image, @options)
     end
 
     def compile_dockerfile_erb
@@ -91,32 +89,38 @@ class Ufo::Docker
       if File.exist?(erb_path)
         compile_dockerfile_erb
       else
-        puts "File #{erb_path.color(:green)} does not exist. Cannot compile it if it doesnt exist"
+        logger.info "File #{erb_path.color(:green)} does not exist. Cannot compile it if it doesnt exist"
       end
     end
 
     def check_dockerfile_exists
       unless File.exist?("#{Ufo.root}/#{@dockerfile}")
-        puts "#{@dockerfile} does not exist.  Are you sure it exists?"
+        logger.info "#{@dockerfile} does not exist.  Are you sure it exists?"
         exit 1
       end
     end
 
     # full_image - does not include the tag
     def image_name
-      settings[:image]
+      Ufo.config.docker.repo
     end
 
     # full_image - Includes the tag. Examples:
     #   123456789.dkr.ecr.us-west-2.amazonaws.com/myapp:ufo-2018-04-20T09-29-08-b7d51df
-    #   tongueroo/demo-ufo:ufo-2018-04-20T09-29-08-b7d51df
-    def full_image_name
+    #   org/repo:ufo-2018-04-20T09-29-08-b7d51df
+    def docker_image
       return generate_name if @options[:generate]
-      return "tongueroo/demo-ufo:ufo-12345678" if ENV['TEST']
 
       unless File.exist?(docker_name_path)
-        puts "Unable to find #{docker_name_path} which contains the last docker image name that was used as a part of `ufo docker build`.  Please run `ufo docker build` first."
-        exit 1
+        logger.info <<~EOL.color(:yellow)
+          WARN: Unable to find: #{pretty_path(docker_name_path)}
+          This contains the Docker image name that the build process uses.
+          Please first run:
+
+              ufo docker build
+
+        EOL
+        return "docker image not yet built"
       end
       IO.read(docker_name_path).strip
     end
@@ -125,44 +129,44 @@ class Ufo::Docker
     # and we want the image name to stay the same when the commands are run separate
     # in different processes.  So we store the state in a file.
     # Only when a new docker build command gets run will the image name state be updated.
-    def store_full_image_name
+    def store_docker_image
       dirname = File.dirname(docker_name_path)
       FileUtils.mkdir_p(dirname) unless File.exist?(dirname)
-      full_image_name = generate_name
-      IO.write(docker_name_path, full_image_name)
+      docker_image = generate_name
+      IO.write(docker_name_path, docker_image)
     end
 
     def generate_name
-      "#{image_name}:#{@image_namespace}-#{timestamp}-#{git_sha}"
+      ["#{image_name}:#{@image_namespace}-#{timestamp}", git_sha].compact.join('-') # compact in case git_sha is unavailable
     end
 
     def docker_name_path
       # output gets entirely wiped by tasks builder so dotn use that folder
-      "#{Ufo.root}/.ufo/data/docker_image_name_#{@image_namespace}.txt"
+      "#{Ufo.root}/.ufo/tmp/state/docker_image_name_#{@image_namespace}.txt"
     end
 
     def timestamp
-      @timestamp ||= Time.now.strftime('%Y-%m-%dT%H-%M-%S')
+      Time.now.strftime('%Y-%m-%dT%H-%M-%S')
     end
+    memoize :timestamp
 
     def git_sha
-      return @git_sha if @git_sha
-      # always call this and dont use the execute method because of the noop option
-      @git_sha = `cd #{Ufo.root} && git rev-parse --short HEAD`
-      @git_sha.strip!
+      sha = if File.exist?('.git')
+        `git rev-parse --short HEAD`
+      elsif ENV['CODEBUILD_RESOLVED_SOURCE_VERSION'] # AWS codebuild
+        ENV['CODEBUILD_RESOLVED_SOURCE_VERSION'][0..6] # first 7 chars
+      end
+      sha.strip if sha
     end
+    memoize :git_sha
 
     def update_dockerfile
       updater = if File.exist?("#{Ufo.root}/Dockerfile.erb") # dont use @dockerfile on purpose
-        Variables.new(full_image_name, @options)
+        State.new(docker_image, @options)
       else
-        Dockerfile.new(full_image_name, @options)
+        Dockerfile.new(docker_image, @options)
       end
       updater.update
     end
-
-    def say(msg)
-      puts msg unless @options[:mute]
-    end
   end
 end

data/lib/ufo/docker/cleaner.rb

@@ -1,26 +1,28 @@
-module Ufo
-  class Docker::Cleaner
-    include Util
+module Ufo::Docker
+  class Cleaner
+    include Ufo::Utils::Execute
+    include Ufo::Utils::Logging
 
     def initialize(docker_image_name, options)
       # docker_image_name does not containg the tag
       # Example: 123456789.dkr.ecr.us-east-1.amazonaws.com/image
       @docker_image_name = docker_image_name
       @options = options
-      @keep = options[:keep] || settings[:clean_keep] || 3
+      @keep = Ufo.config.docker.clean_keep
       @tag_prefix = options[:tag_prefix] || "ufo"
     end
 
     def cleanup
+      return if @keep.nil?
       return if delete_list.empty?
       command = "docker rmi #{delete_list}"
-      say "Cleaning up docker images...".color(:green)
-      say "=> #{"docker rmi #{delete_list}"}".color(:green)
+      logger.info "Cleaning docker images"
       return if @options[:noop]
-      execute(command, use_system: false) # to use_system: false silence output
+      execute(command, quiet: true)
     end
 
     def delete_list
+      return if @keep.nil?
       return [] if ENV['TEST'] || @options[:noop]
       return @delete_list if @delete_list
 
@@ -44,10 +46,5 @@ module Ufo
         @delete_list = delete_tags.map { |t| "#{@docker_image_name}:#{t}" }.join(' ')
       end
     end
-
-    def say(msg)
-      msg = "NOOP #{msg}" if @options[:noop]
-      puts msg unless @options[:mute]
-    end
   end
 end

data/lib/ufo/docker/compiler.rb

@@ -1,4 +1,4 @@
-class Ufo::Docker
+module Ufo::Docker
   class Compiler
     def initialize(dockerfile)
       @dockerfile = dockerfile
@@ -9,13 +9,19 @@ class Ufo::Docker
       return unless File.exist?(@erb_file)
 
       puts "Compiled #{File.basename(@erb_file).color(:green)} to #{File.basename(@dockerfile).color(:green)}"
-      path = "#{Ufo.root}/.ufo/settings/dockerfile_variables.yml"
+      path = "#{Ufo.root}/.ufo/state/data.yml"
       vars = YAML.load_file(path)[Ufo.env] if File.exist?(path)
       vars ||= {}
       result = RenderMePretty.result(@erb_file, vars)
       comment =<<~EOL.chop # remove the trailing newline
-        # Note this file was generated from #{File.basename(@erb_file)} as a part of running ufo ship"
-        # To update the FROM statement with the latest base docker image run: ufo docker base
+        # IMPORTANT: This file was generated from #{File.basename(@erb_file)} as a part of running:
+        #
+        #     ufo ship
+        #
+        # To update the FROM statement with the latest base docker image use:
+        #
+        #    ufo docker base
+        #
       EOL
       result = "#{comment}\n#{result}"
       IO.write(@dockerfile, result)

data/lib/ufo/docker/concerns.rb

@@ -0,0 +1,7 @@
+module Ufo::Docker
+  module Concerns
+    include Ufo::Utils::Execute
+    include Ufo::Utils::Logging
+    include Ufo::Utils::Pretty
+  end
+end

data/lib/ufo/docker/dockerfile.rb

@@ -1,7 +1,9 @@
-module Ufo
-  class Docker::Dockerfile
-    def initialize(full_image_name, options={})
-      @full_image_name, @options = full_image_name, options
+module Ufo::Docker
+  class Dockerfile
+    include Ufo::Utils::Logging
+
+    def initialize(docker_image, options={})
+      @docker_image, @options = docker_image, options
     end
 
     def update
@@ -21,7 +23,7 @@ module Ufo
       # replace FROM line
       new_lines = lines.map do |line|
                     if line =~ /^FROM /
-                      "FROM #{@full_image_name}"
+                      "FROM #{@docker_image}"
                     else
                       line
                     end
@@ -31,10 +33,12 @@ module Ufo
 
     def write_new_dockerfile
       IO.write(dockerfile_path, new_dockerfile)
-      unless @options[:mute]
-        puts "The Dockerfile FROM statement has been updated with the latest base image:".color(:green)
-        puts "  #{@full_image_name}".color(:green)
-      end
+      logger.debug <<~EOL
+        The Dockerfile FROM statement has been updated with the latest base image:
+
+            #{@docker_image}
+
+      EOL
     end
   end
 end

data/lib/ufo/docker/pusher.rb

@@ -1,36 +1,29 @@
-require 'active_support/core_ext/module/delegation'
-
-class Ufo::Docker
+module Ufo::Docker
   class Pusher
-    include Ufo::Util
+    include Concerns
 
-    delegate :full_image_name, to: :builder
+    delegate :docker_image, to: :builder
     attr_reader :last_image_name
     def initialize(image, options)
       @options = options
-      # full_image_name ultimately uses @options, so @last_image_name assignment
+      # docker_image ultimately uses @options, so @last_image_name assignment
       # line must be defined after setting @options.
-      @last_image_name = image || full_image_name
+      @last_image_name = image || docker_image
     end
 
     def push
       update_auth_token
       start_time = Time.now
-      message = "Pushed #{last_image_name} docker image."
-      if @options[:noop]
-        message = "NOOP #{message}"
-      else
-        command = "docker push #{last_image_name}"
-        puts "=> #{command}".color(:green)
-        success = execute(command, use_system: true)
-        unless success
-          puts "ERROR: The docker image fail to push.".color(:red)
-          exit 1
-        end
+      logger.info "Pushing Docker Image"
+      command = "docker push #{last_image_name}"
+      log = ".ufo/log/docker.log" if @options[:quiet]
+      success = execute(command, log: log)
+      unless success
+        logger.info "ERROR: The docker image fail to push.".color(:red)
+        exit 1
       end
       took = Time.now - start_time
-      message << "\nDocker push took #{pretty_time(took)}.".color(:green)
-      puts message unless @options[:mute]
+      logger.info "Took #{pretty_time(took)}"
     end
 
     def builder
@@ -42,10 +35,5 @@ class Ufo::Docker
       # wont update auth token unless the image being pushed in the ECR image format
       auth.update
     end
-
-    # full_image - does not include the tag
-    def image_name
-      settings[:image]
-    end
   end
 end