ufo 5.0.6 → 6.0.1

data/lib/ufo/core.rb

@@ -4,65 +4,53 @@ require 'yaml'
 module Ufo
   module Core
     extend Memoist
-    include Ufo::Settings
 
-    def check_task_definition!(task_definition)
-      task_definition_path = "#{Ufo.root}/.ufo/output/#{task_definition}.json"
-      unless File.exist?(task_definition_path)
-        puts "ERROR: Unable to find the task definition at #{task_definition_path}.".color(:red)
-        puts "Are you sure you have defined it in .ufo/task_definitions.rb and it has been generated correctly in .ufo/output?".color(:red)
-        puts "If you are calling `ufo deploy` directly, you might want to generate the task definition first with `ufo tasks build`."
-        exit 1
-      end
+    def role
+      ENV['UFO_ROLE'] || 'web'
     end
 
-    def root
-      path = ENV['UFO_ROOT'] || '.'
-      Pathname.new(path)
+    def app
+      ENV['UFO_APP'] || config.app
     end
 
+    # v5: development is default
+    # v6: dev is default
     def env
-      # 2-way binding
-      ufo_env = env_from_profile || 'development'
-      ufo_env = ENV['UFO_ENV'] if ENV['UFO_ENV'] # highest precedence
-      ufo_env
+      ENV['UFO_ENV'] || 'dev'
     end
     memoize :env
 
-    def env_extra
-      env_extra = Current.env_extra
-      env_extra = ENV['UFO_ENV_EXTRA'] if ENV['UFO_ENV_EXTRA'] # highest precedence
-      return if env_extra&.empty?
-      env_extra
+    def extra
+      extra = ENV['UFO_EXTRA'] if ENV['UFO_EXTRA'] # highest precedence
+      return if extra&.empty?
+      extra
+    end
+    memoize :extra
+
+    def root
+      path = ENV['UFO_ROOT'] || '.'
+      Pathname.new(path)
+    end
+
+    def log_root
+      "#{root}/log"
     end
-    memoize :env_extra
 
-    # Overrides AWS_PROFILE based on the Ufo.env if set in .ufo/settings.yml
-    # 2-way binding.
-    def set_aws_profile!
-      return if ENV['TEST']
-      return unless File.exist?("#{Ufo.root}/.ufo/settings.yml") # for rake docs
-      return unless settings # Only load if within Ufo project and there's a settings.yml
-      data = settings || {}
-      if data[:aws_profile]
-        puts "Using AWS_PROFILE=#{data[:aws_profile]} from UFO_ENV=#{Ufo.env} in config/settings.yml"
-        ENV['AWS_PROFILE'] = data[:aws_profile]
-      end
+    def configure(&block)
+      Config.instance.configure(&block)
     end
 
-    def check_ufo_project!
-      check_path = "#{Ufo.root}/.ufo/settings.yml"
-      unless File.exist?(check_path)
-        puts "ERROR: No settings file at #{check_path}.  Are you sure you are in a project with ufo setup?".color(:red)
-        puts "Current directory: #{Dir.pwd}"
-        puts "If you want to set up ufo for this prjoect, please create a settings file via: ufo init"
-        exit 1 unless ENV['TEST']
-      end
+    # Generally, use the Lono.config instead of Config.instance.config since it guarantees the load_project_config call
+    def config
+      Config.instance.load_project_config
+      Config.instance.config
     end
+    memoize :config
 
-  private
-    def env_from_profile
-      Ufo::Setting.new.ufo_env
+    # allow different logger when running up all or rspec-lono
+    cattr_writer :logger
+    def logger
+      @@logger ||= config.logger
     end
   end
 end

data/lib/ufo/docker/builder.rb

@@ -1,12 +1,11 @@
-require 'active_support/core_ext/module/delegation'
-
-class Ufo::Docker
+module Ufo::Docker
   class Builder
-    include Ufo::Util
+    extend Memoist
+    include Concerns
 
     delegate :push, to: :pusher
-    def self.build(options)
-      builder = Builder.new(options) # outside if because it need builder.full_image_name
+    def self.build(options={})
+      builder = Builder.new(options) # outside if because it need builder.docker_image
       builder.build
       pusher = Pusher.new(nil, options)
       pusher.push
@@ -21,28 +20,27 @@ class Ufo::Docker
 
     def build
       start_time = Time.now
-      store_full_image_name
+      store_docker_image
 
-      command = "docker build #{build_options}-t #{full_image_name} -f #{@dockerfile} ."
-      say "Building docker image with:".color(:green)
-      say "  #{command}".color(:green)
+      logger.info "Building Docker Image"
       compile_dockerfile_erb
       check_dockerfile_exists
       update_auth_token
-      command = "cd #{Ufo.root} && #{command}"
-      success = execute(command, use_system: true)
+      command = "docker build #{build_options}-t #{docker_image} -f #{@dockerfile} ."
+      log = ".ufo/log/docker.log" if @options[:quiet]
+      success = execute(command, log: log)
       unless success
         docker_version_success = system("docker version > /dev/null 2>&1")
         unless docker_version_success
           docker_version_message = "  Are you sure the docker daemon is available?  Try running: docker version."
         end
-        puts "ERROR: The docker image fail to build.#{docker_version_message}".color(:red)
+        logger.info "ERROR: Fail to build Docker image.#{docker_version_message}".color(:red)
         exit 1
       end
 
       took = Time.now - start_time
-      say "Docker image #{full_image_name} built.  "
-      say "Docker build took #{pretty_time(took)}.".color(:green)
+      logger.info "Docker Image built: #{docker_image}"
+      logger.info "Took #{pretty_time(took)}"
     end
 
     def build_options
@@ -78,7 +76,7 @@ class Ufo::Docker
     end
 
     def pusher
-      @pusher ||= Pusher.new(full_image_name, @options)
+      @pusher ||= Pusher.new(docker_image, @options)
     end
 
     def compile_dockerfile_erb
@@ -91,32 +89,38 @@ class Ufo::Docker
       if File.exist?(erb_path)
         compile_dockerfile_erb
       else
-        puts "File #{erb_path.color(:green)} does not exist. Cannot compile it if it doesnt exist"
+        logger.info "File #{erb_path.color(:green)} does not exist. Cannot compile it if it doesnt exist"
       end
     end
 
     def check_dockerfile_exists
       unless File.exist?("#{Ufo.root}/#{@dockerfile}")
-        puts "#{@dockerfile} does not exist.  Are you sure it exists?"
+        logger.info "#{@dockerfile} does not exist.  Are you sure it exists?"
         exit 1
       end
     end
 
     # full_image - does not include the tag
     def image_name
-      settings[:image]
+      Ufo.config.docker.repo
     end
 
     # full_image - Includes the tag. Examples:
     #   123456789.dkr.ecr.us-west-2.amazonaws.com/myapp:ufo-2018-04-20T09-29-08-b7d51df
-    #   tongueroo/demo-ufo:ufo-2018-04-20T09-29-08-b7d51df
-    def full_image_name
+    #   org/repo:ufo-2018-04-20T09-29-08-b7d51df
+    def docker_image
       return generate_name if @options[:generate]
-      return "tongueroo/demo-ufo:ufo-12345678" if ENV['TEST']
 
       unless File.exist?(docker_name_path)
-        puts "Unable to find #{docker_name_path} which contains the last docker image name that was used as a part of `ufo docker build`.  Please run `ufo docker build` first."
-        exit 1
+        logger.info <<~EOL.color(:yellow)
+          WARN: Unable to find: #{pretty_path(docker_name_path)}
+          This contains the Docker image name that the build process uses.
+          Please first run:
+
+              ufo docker build
+
+        EOL
+        return "docker image not yet built"
       end
       IO.read(docker_name_path).strip
     end
@@ -125,44 +129,44 @@ class Ufo::Docker
     # and we want the image name to stay the same when the commands are run separate
     # in different processes.  So we store the state in a file.
     # Only when a new docker build command gets run will the image name state be updated.
-    def store_full_image_name
+    def store_docker_image
       dirname = File.dirname(docker_name_path)
       FileUtils.mkdir_p(dirname) unless File.exist?(dirname)
-      full_image_name = generate_name
-      IO.write(docker_name_path, full_image_name)
+      docker_image = generate_name
+      IO.write(docker_name_path, docker_image)
     end
 
     def generate_name
-      "#{image_name}:#{@image_namespace}-#{timestamp}-#{git_sha}"
+      ["#{image_name}:#{@image_namespace}-#{timestamp}", git_sha].compact.join('-') # compact in case git_sha is unavailable
     end
 
     def docker_name_path
       # output gets entirely wiped by tasks builder so dotn use that folder
-      "#{Ufo.root}/.ufo/data/docker_image_name_#{@image_namespace}.txt"
+      "#{Ufo.root}/.ufo/tmp/state/docker_image_name_#{@image_namespace}.txt"
     end
 
     def timestamp
-      @timestamp ||= Time.now.strftime('%Y-%m-%dT%H-%M-%S')
+      Time.now.strftime('%Y-%m-%dT%H-%M-%S')
     end
+    memoize :timestamp
 
     def git_sha
-      return @git_sha if @git_sha
-      # always call this and dont use the execute method because of the noop option
-      @git_sha = `cd #{Ufo.root} && git rev-parse --short HEAD`
-      @git_sha.strip!
+      sha = if File.exist?('.git')
+        `git rev-parse --short HEAD`
+      elsif ENV['CODEBUILD_RESOLVED_SOURCE_VERSION'] # AWS codebuild
+        ENV['CODEBUILD_RESOLVED_SOURCE_VERSION'][0..6] # first 7 chars
+      end
+      sha.strip if sha
     end
+    memoize :git_sha
 
     def update_dockerfile
       updater = if File.exist?("#{Ufo.root}/Dockerfile.erb") # dont use @dockerfile on purpose
-        Variables.new(full_image_name, @options)
+        State.new(docker_image, @options)
       else
-        Dockerfile.new(full_image_name, @options)
+        Dockerfile.new(docker_image, @options)
       end
       updater.update
     end
-
-    def say(msg)
-      puts msg unless @options[:mute]
-    end
   end
 end

data/lib/ufo/docker/cleaner.rb

@@ -1,26 +1,28 @@
-module Ufo
-  class Docker::Cleaner
-    include Util
+module Ufo::Docker
+  class Cleaner
+    include Ufo::Utils::Execute
+    include Ufo::Utils::Logging
 
     def initialize(docker_image_name, options)
       # docker_image_name does not containg the tag
       # Example: 123456789.dkr.ecr.us-east-1.amazonaws.com/image
       @docker_image_name = docker_image_name
       @options = options
-      @keep = options[:keep] || settings[:clean_keep] || 3
+      @keep = Ufo.config.docker.clean_keep
       @tag_prefix = options[:tag_prefix] || "ufo"
     end
 
     def cleanup
+      return if @keep.nil?
       return if delete_list.empty?
       command = "docker rmi #{delete_list}"
-      say "Cleaning up docker images...".color(:green)
-      say "=> #{"docker rmi #{delete_list}"}".color(:green)
+      logger.info "Cleaning docker images"
       return if @options[:noop]
-      execute(command, use_system: false) # to use_system: false silence output
+      execute(command, quiet: true)
     end
 
     def delete_list
+      return if @keep.nil?
       return [] if ENV['TEST'] || @options[:noop]
       return @delete_list if @delete_list
 
@@ -44,10 +46,5 @@ module Ufo
         @delete_list = delete_tags.map { |t| "#{@docker_image_name}:#{t}" }.join(' ')
       end
     end
-
-    def say(msg)
-      msg = "NOOP #{msg}" if @options[:noop]
-      puts msg unless @options[:mute]
-    end
   end
 end

data/lib/ufo/docker/compiler.rb

@@ -1,4 +1,4 @@
-class Ufo::Docker
+module Ufo::Docker
   class Compiler
     def initialize(dockerfile)
       @dockerfile = dockerfile
@@ -9,13 +9,19 @@ class Ufo::Docker
       return unless File.exist?(@erb_file)
 
       puts "Compiled #{File.basename(@erb_file).color(:green)} to #{File.basename(@dockerfile).color(:green)}"
-      path = "#{Ufo.root}/.ufo/settings/dockerfile_variables.yml"
+      path = "#{Ufo.root}/.ufo/state/data.yml"
       vars = YAML.load_file(path)[Ufo.env] if File.exist?(path)
       vars ||= {}
       result = RenderMePretty.result(@erb_file, vars)
       comment =<<~EOL.chop # remove the trailing newline
-        # Note this file was generated from #{File.basename(@erb_file)} as a part of running ufo ship"
-        # To update the FROM statement with the latest base docker image run: ufo docker base
+        # IMPORTANT: This file was generated from #{File.basename(@erb_file)} as a part of running:
+        #
+        #     ufo ship
+        #
+        # To update the FROM statement with the latest base docker image use:
+        #
+        #    ufo docker base
+        #
       EOL
       result = "#{comment}\n#{result}"
       IO.write(@dockerfile, result)

data/lib/ufo/docker/concerns.rb

@@ -0,0 +1,7 @@
+module Ufo::Docker
+  module Concerns
+    include Ufo::Utils::Execute
+    include Ufo::Utils::Logging
+    include Ufo::Utils::Pretty
+  end
+end

data/lib/ufo/docker/dockerfile.rb

@@ -1,7 +1,9 @@
-module Ufo
-  class Docker::Dockerfile
-    def initialize(full_image_name, options={})
-      @full_image_name, @options = full_image_name, options
+module Ufo::Docker
+  class Dockerfile
+    include Ufo::Utils::Logging
+
+    def initialize(docker_image, options={})
+      @docker_image, @options = docker_image, options
     end
 
     def update
@@ -21,7 +23,7 @@ module Ufo
       # replace FROM line
       new_lines = lines.map do |line|
                     if line =~ /^FROM /
-                      "FROM #{@full_image_name}"
+                      "FROM #{@docker_image}"
                     else
                       line
                     end
@@ -31,10 +33,12 @@ module Ufo
 
     def write_new_dockerfile
       IO.write(dockerfile_path, new_dockerfile)
-      unless @options[:mute]
-        puts "The Dockerfile FROM statement has been updated with the latest base image:".color(:green)
-        puts "  #{@full_image_name}".color(:green)
-      end
+      logger.debug <<~EOL
+        The Dockerfile FROM statement has been updated with the latest base image:
+
+            #{@docker_image}
+
+      EOL
     end
   end
 end

data/lib/ufo/docker/pusher.rb

@@ -1,36 +1,29 @@
-require 'active_support/core_ext/module/delegation'
-
-class Ufo::Docker
+module Ufo::Docker
   class Pusher
-    include Ufo::Util
+    include Concerns
 
-    delegate :full_image_name, to: :builder
+    delegate :docker_image, to: :builder
     attr_reader :last_image_name
     def initialize(image, options)
       @options = options
-      # full_image_name ultimately uses @options, so @last_image_name assignment
+      # docker_image ultimately uses @options, so @last_image_name assignment
       # line must be defined after setting @options.
-      @last_image_name = image || full_image_name
+      @last_image_name = image || docker_image
     end
 
     def push
       update_auth_token
       start_time = Time.now
-      message = "Pushed #{last_image_name} docker image."
-      if @options[:noop]
-        message = "NOOP #{message}"
-      else
-        command = "docker push #{last_image_name}"
-        puts "=> #{command}".color(:green)
-        success = execute(command, use_system: true)
-        unless success
-          puts "ERROR: The docker image fail to push.".color(:red)
-          exit 1
-        end
+      logger.info "Pushing Docker Image"
+      command = "docker push #{last_image_name}"
+      log = ".ufo/log/docker.log" if @options[:quiet]
+      success = execute(command, log: log)
+      unless success
+        logger.info "ERROR: The docker image fail to push.".color(:red)
+        exit 1
       end
       took = Time.now - start_time
-      message << "\nDocker push took #{pretty_time(took)}.".color(:green)
-      puts message unless @options[:mute]
+      logger.info "Took #{pretty_time(took)}"
     end
 
     def builder
@@ -42,10 +35,5 @@ class Ufo::Docker
       # wont update auth token unless the image being pushed in the ECR image format
       auth.update
     end
-
-    # full_image - does not include the tag
-    def image_name
-      settings[:image]
-    end
   end
 end

data/lib/ufo/docker/state.rb

@@ -0,0 +1,63 @@
+module Ufo::Docker
+  class State
+    include Ufo::Utils::Logging
+    include Ufo::Utils::Pretty
+
+    def initialize(docker_image, options={})
+      @docker_image, @options = docker_image, options
+    end
+
+    def update
+      data = current_data
+      data[Ufo.env] ||= {}
+      data[Ufo.env]["base_image"] = @docker_image
+      pretty_path = state_path.sub("#{Ufo.root}/", "")
+      FileUtils.mkdir_p(File.dirname(state_path))
+      IO.write(state_path, YAML.dump(data))
+      logger.info "The #{pretty_path} base_image has been updated with the latest base image:".color(:green)
+      logger.info "  #{@docker_image}".color(:green)
+      reminder_message
+    end
+
+    def current_data
+      File.exist?(state_path) ? YAML.load_file(state_path) : {}
+    end
+
+    def state_path
+      path = "#{Ufo.root}/.ufo/state"
+      if ENV['UFO_APP'] # env var activates app path
+        path = "#{Ufo.root}/.ufo/state/#{Ufo.app}"
+      end
+      "#{path}/data.yml"
+    end
+
+    def reminder_message
+      return unless Ufo.config.state.reminder
+      repo = ENV['UFO_CENTRAL_REPO']
+      return unless repo
+      logger.info "It looks like you're using a central deployer pattern".color(:yellow)
+      logger.info <<~EOL
+        Remember to commit the state file:
+
+            state file: #{pretty_path(state_path)}
+            repo:       #{repo}
+
+      EOL
+
+      unless ENV['UFO_APP']
+        logger.info "WARN: It also doesnt look like UFO_ENV is set".color(:yellow)
+        logger.info "UFO_ENV should be set when you're using ufo in a central manner"
+      end
+
+      logger.info <<~EOL
+        You can disable these reminder messages with:
+
+        .ufo/config.rb
+
+            Ufo.configure do |config|
+              config.state.reminder = false
+            end
+      EOL
+    end
+  end
+end

data/lib/ufo/ecr/auth.rb

@@ -17,11 +17,12 @@ This class manipulates the ~/.docker/config.json file which is an internal docke
 =end
 module Ufo
   class Ecr::Auth
-    include AwsService
+    include Ufo::AwsServices
+    include Ufo::Utils::Logging
 
-    def initialize(full_image_name)
-      @full_image_name = full_image_name
-      @repo_domain = "#{full_image_name.split('/').first}"
+    def initialize(docker_image)
+      @docker_image = docker_image
+      @repo_domain = "#{docker_image.split('/').first}"
     end
 
     def update
@@ -32,16 +33,16 @@ module Ufo
       username, password = Base64.decode64(auth_token).split(':')
 
       command = "docker login -u #{username} --password-stdin #{@repo_domain}"
-      puts "=> #{command}".color(:green)
+      logger.debug "=> #{command}".color(:green)
       *, status = Open3.capture3(command, stdin_data: password)
       unless status.success?
-        puts "ERROR: The docker failed to login.".color(:red)
+        logger.error "ERROR: The docker failed to login.".color(:red)
         exit 1
       end
     end
 
     def ecr_image?
-      !!(@full_image_name =~ /\.amazonaws\.com/)
+      !!(@docker_image =~ /\.amazonaws\.com/)
     end
 
     def fetch_auth_token

data/lib/ufo/ecr/cleaner.rb

@@ -1,25 +1,23 @@
 require "json"
 
-# Can test:
-#
-#   ufo ship app-web --cluster prod --noop
 module Ufo
   class Ecr::Cleaner
-    include Util
-    include AwsService
+    include Ufo::AwsServices
+    include Ufo::Utils::Execute
+    include Ufo::Utils::Logging
 
     def initialize(docker_image_name, options={})
       # docker_image_name does not containg the tag
       # Example: 123456789.dkr.ecr.us-east-1.amazonaws.com/image
       @docker_image_name = docker_image_name
       @options = options
-      @keep = options[:ecr_keep] || settings[:ecr_keep]
+      @keep = options[:ecr_keep] || Ufo.config.docker.ecr_keep
       @tag_prefix = options[:tag_prefix] || "ufo"
     end
 
     def cleanup
-      return false unless ecr_image?
       return false unless @keep
+      return false unless ecr_image?
       update_auth_token
       image_tags = fetch_image_tags
       delete_tags = image_tags[@keep..-1] # ordered by most recent images first
@@ -36,16 +34,14 @@ module Ufo
 
     def delete_images(tags)
       return if tags.nil? || tags.empty?
-      unless @options[:mute]
-        puts "Keeping #{@keep} most recent ECR images."
-        puts "Deleting these ECR images:"
-        tag_list = tags.map { |t| "  #{repo_name}:#{t}" }
-        puts tag_list
-      end
+      logger.info "Keeping #{@keep} most recent ECR images."
+      logger.info "Deleting these ECR images:"
+      tag_list = tags.map { |t| "  #{repo_name}:#{t}" }
+      logger.info tag_list
       image_ids = tags.map { |tag| {image_tag: tag} }
       ecr.batch_delete_image(
         repository_name: repo_name,
-        image_ids: image_ids) unless @options[:noop]
+        image_ids: image_ids)
     end
 
     def update_auth_token

data/lib/ufo/ext/core/module.rb

@@ -0,0 +1,31 @@
+class Module
+  # Include all modules within the relative folder. IE: for dsl/syntax/mod/*
+  #
+  #    include Common
+  #    include Provider
+  #    # etc
+  #
+  # Caller lines are different for OSes:
+  #
+  #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/lono-1.1.1/lib/lono/builder.rb:34:in `build'"
+  #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/lono-1.1.1/lib/lono/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
+  #
+  def include_modules(dir)
+    caller_line = caller[0]
+    parts = caller_line.split(':')
+    calling_file = caller_line.match(/^[a-zA-Z]:/) ? parts[1] : parts[0]
+    parent_dir = File.dirname(calling_file)
+
+    full_dir = "#{parent_dir}/#{dir}"
+    paths = Dir.glob("#{full_dir}/**/*.rb")
+    if paths.empty?
+      raise "Empty include_modules full_dir: #{full_dir}"
+    end
+    paths.each do |path|
+      regexp = Regexp.new(".*/lib/")
+      mod = path.sub(regexp, '').sub('.rb','').camelize
+      c = mod.constantize
+      include c if c.class == Module
+    end
+  end
+end

data/lib/ufo/ext/core/nil_class.rb

@@ -0,0 +1,11 @@
+require "json"
+
+module Ufo
+  refine NilClass do
+    def to_json
+      JSON.generate(self)
+    end
+  end
+end
+
+using Ufo

data/lib/ufo/ext.rb

@@ -0,0 +1,2 @@
+require_relative "ext/core/module"
+require_relative "ext/core/nil_class"

data/lib/ufo/{role → iam_role}/builder.rb

@@ -1,4 +1,4 @@
-module Ufo::Role
+module Ufo::IamRole
   class Builder
     def initialize(role_type)
       @role_type = role_type