ufo 6.2.2 → 6.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e630cb6fc68958a6b7f7264e6b0ff03a2d41ae13d9a1648bd13fa601d9f9aae0
-  data.tar.gz: 2693b4ca4e3e5ffa31ff881f706de093792d93a4ff7263820308565d44685c38
+  metadata.gz: 69d0e411c4b48b8ec8a0a2148caf5501b3c038cb991585c4741027579a102e0c
+  data.tar.gz: f080bdf97e783b18bc54723ced3be52557b0749094451561de41766c052aba0e
 SHA512:
-  metadata.gz: 116244ca9466e2458845904b34d99bc1fd2e7d1d8379c32066d2f90d70cfe7eb84628552a8250f7d4cca13dcdcf1cdad0ba812e602ba1df25c4f0b383131c057
-  data.tar.gz: 7e7aa81c1595f231465f1694a80dc7e4f0a8f7f7c77399e28b12b71c109a2c5ea0b9674fd0e8a9b485792253e16c4fd4a27734a121affd01bb69e163fb6e9f51
+  metadata.gz: 860f1d782b915338e903d2c2c0cad43fbe4a915261b46a2b615a7d24870247170a3041de32b6e9de8b11161da1c8447dc3d12683e9179c8c6cd0af7ef3dc4d77
+  data.tar.gz: c02bc1f698f85055e2e1c99b7fbf6cf730044e23b7584509a5c2e66bd79259097b2c5ccf1706b186a64b329db31e5678fe6eacb3f120724dbfa896a0ab6fbb00

data/.cody/acceptance/bin/asg.sh

@@ -0,0 +1,15 @@
+function scale_asg_to() {
+  n=$1
+  ASG=$(asg)
+  echo "Scaling $ASG to $n"
+  aws autoscaling update-auto-scaling-group --auto-scaling-group-name $ASG \
+    --desired-capacity $n --min-size $n --max-size $n
+}
+
+# aws cloudformation describe-stacks --stack-name ecs-qa | jq -r '.Stacks[].Outputs[] | select(.OutputKey == "Asg") | .OutputValue'
+# aws cloudformation describe-stack-resources --stack-name ecs-qa | jq -r '.StackResources[] | select(.LogicalResourceId == "Asg") | .PhysicalResourceId'
+function asg() {
+  STACK_NAME=ecs-qa
+  ASG=$(aws cloudformation describe-stacks --stack-name $STACK_NAME | jq -r '.Stacks[].Outputs[] | select(.OutputKey == "Asg") | .OutputValue')
+  echo $ASG
+}

data/.cody/acceptance/bin/build.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$DIR/asg.sh"
+
 final_status=0
 function capture_status {
   if [ "$?" -ne "0" ] && [ $final_status -ne 1 ] ; then
@@ -40,6 +43,7 @@ cat .ufo/vars/base.rb
 cat .ufo/vars/dev.rb
 
 export UFO_ENV=qa
+scale_asg_to 1
 
 # Deploy
 ufo ship -y
@@ -95,3 +99,5 @@ ufo ps # see full output for debugging
 ufo ps 2>&1 | grep Stack | grep worker # should be success. IE: exit 0
 ufo destroy -y
 ufo ps 2>&1 | grep No | grep found # should be success. IE: exit 0
+
+scale_asg_to 0

data/.cody/acceptance/{role.rb → iam_role.rb}

@@ -1,5 +1,6 @@
 iam_policy(
   "application-autoscaling",
+  "autoscaling",
   "cloudformation",
   "ec2",
   "ecr",

data/.cody/acceptance/project.rb

@@ -1,8 +1,8 @@
-github_url("https://github.com/boltops-tools/ufo")
-linux_image("aws/codebuild/amazonlinux2-x86_64-standard:3.0")
-environment_variables(
-  DOCKER_USER: "ssm:/codebuild/ufo/DOCKER_USER",
-  DOCKER_PASS: "ssm:/codebuild/ufo/DOCKER_PASS",
+github("boltops-tools/ufo")
+image("aws/codebuild/amazonlinux2-x86_64-standard:3.0")
+env_vars(
+  DOCKER_USER: "ssm:/#{Cody.env}/DOCKER_USER",
+  DOCKER_PASS: "ssm:/#{Cody.env}/DOCKER_PASS",
 )
 
 # triggers(

data/.gitignore

@@ -19,3 +19,4 @@ tmp
 
 .ruby-version
 Gemfile.lock
+.cody/output

data/CHANGELOG.md

@@ -3,6 +3,18 @@
 All notable changes to this project will be documented in this file.
 This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [6.2.5] - 2022-03-24
+- [#159](https://github.com/tongueroo/ufo/pull/159) improve ufo call line
+- [#160](https://github.com/tongueroo/ufo/pull/160) conventionally lookup up secrets and env file
+- [#161](https://github.com/tongueroo/ufo/pull/161) layering support for env files
+- improve acceptance pipeline
+
+## [6.2.4] - 2022-03-20
+- [#158](https://github.com/tongueroo/ufo/pull/158) warn on missing env and secrets file instead of error
+
+## [6.2.3] - 2022-03-20
+- [#157](https://github.com/tongueroo/ufo/pull/157) layering.show_for_commands option
+
 ## [6.2.2] - 2022-03-20
 - [#155](https://github.com/tongueroo/ufo/pull/155) layering.show option ability to show layers
 - [#156](https://github.com/tongueroo/ufo/pull/156) extra layering support

data/lib/ufo/cli/scale.rb

@@ -54,6 +54,7 @@ class Ufo::CLI
 
     def register_scalable_target(scalable_target)
       # service/dev/app1-web-dev-EcsService-Q0XkN6VtxGWv|ecs:service:DesiredCount|ecs
+      return unless scalable_target && scalable_target.physical_resource_id # stack still creating
       resource_id, scalable_dimension, service_namespace = scalable_target.physical_resource_id.split('|')
       applicationautoscaling.register_scalable_target(
         max_capacity: @max,

data/lib/ufo/command.rb

@@ -78,7 +78,7 @@ module Ufo
       # See comment where configure_dsl_evaluator is used about Ufo.role
       def configure_dsl_evaluator
         DslEvaluator.configure do |config|
-          config.backtrace.select_pattern = Ufo.root.to_s
+          config.backtrace.select_pattern = "#{Ufo.root}/.ufo"
           config.logger = Ufo.logger
           config.on_exception = :exit
           config.root = Ufo.root

data/lib/ufo/config/parse.rb

@@ -0,0 +1,39 @@
+# Some limitations:
+#
+# * Only parsing one file: .ufo/config.rb
+# * If user is using Ruby code that cannot be parse will fallback to default
+#
+# Think it's worth it so user only has to configure
+#
+#     config.layering.show = true
+#
+class Ufo::Config
+  class Parse
+    def for(config, type: :boolean)
+      lines = IO.readlines("#{Ufo.root}/.ufo/config.rb")
+      config_line = lines.find do |l|
+        # IE: Regexp.new("config\.layering.show.*=")
+        regexp = Regexp.new("config\.#{config}.*=")
+        l =~ regexp && l !~ /^\s+#/
+      end
+      return false unless config_line # default is false
+      config_value = config_line.gsub(/.*=/,'').strip.gsub(/["']/,'')
+      case type
+      when :boolean
+        config_value != "false" && config_value != "nil"
+      when :array
+        eval(config_value) # IE: '["a"]' => ["a"]
+      else
+        raise "Type #{type.inspect} not supported"
+      end
+    rescue Exception => e
+      # if ENV['UFO_DEBUG']
+        puts "#{e.class} #{e.message}".color(:yellow)
+        puts "WARN: Unable to parse for config.layering.show".color(:yellow)
+        puts "Using default: config.layering.show = false"
+      # end
+      false
+    end
+  end
+end
+

data/lib/ufo/config.rb

@@ -86,7 +86,8 @@ module Ufo
       config.exec.enabled = true        # EcsService EnableExecuteCommand
 
       config.layering = ActiveSupport::OrderedOptions.new
-      config.layering.show = show_layers?
+      config.layering.show = parsed_layering_show
+      config.layering.show_for_commands = parsed_layering_show_for
 
       config.log = ActiveSupport::OrderedOptions.new
       config.log.root = Ufo.log_root
@@ -111,6 +112,7 @@ module Ufo
       config.secrets.pattern.secretsmanager = ":APP-:ENV-:SECRET_NAME" # => demo-dev-DB_PASS
       config.secrets.pattern.ssm = ":APP/:ENV/:SECRET_NAME" # => demo/dev/DB_PASS
       config.secrets.provider = "ssm" # default provider for conventional expansion IE: ssm or secretsmanager
+      config.secrets.warning = true
 
       config.ship = ActiveSupport::OrderedOptions.new
       config.ship.docker = ActiveSupport::OrderedOptions.new
@@ -165,7 +167,7 @@ module Ufo
       end
       # load_project_config gets called so early that logger is not yet configured.
       # Cannot use Ufo.config yet and cannot use logger which relies on Ufo.config
-      # Use puts and use show_layers? which parses for the config
+      # Use puts and use parsed_layering_show
       show = show_layers?
       puts "Config Layers" if show
       layers.each do |layer|
@@ -180,34 +182,25 @@ module Ufo
     end
 
     def show_layers?
-      ENV['UFO_LAYERS'] || parse_for_layering_show
+      show_for = parsed_layering_show_for
+      command = ARGV[0]
+      parsed_layering_show && show_for.include?(command)
     end
-    private :show_layers?
 
-    # Some limitations:
-    #
-    # * Only parsing one file: .ufo/config.rb
-    # * If user is using Ruby code that cannot be parse will fallback to default
-    #
-    # Think it's worth it so user only has to configure
-    #
-    #     config.layering.show = true
-    #
-    def parse_for_layering_show
-      lines = IO.readlines("#{Ufo.root}/.ufo/config.rb")
-      config_line = lines.find { |l| l =~ /config\.layering.show.*=/ && l !~ /^\s+#/ }
-      return false unless config_line # default is false
-      config_value = config_line.gsub(/.*=/,'').strip.gsub(/["']/,'')
-      config_value != "false" && config_value != "nil"
-    rescue Exception => e
-      if ENV['UFO_DEBUG']
-        puts "#{e.class} #{e.message}".color(:yellow)
-        puts "WARN: Unable to parse for config.layering.show".color(:yellow)
-        puts "Using default: config.layering.show = false"
-      end
-      false
+    def parsed_layering_show_for
+      parse.for("layering.show_for_commands", type: :array) || %w[build ship] # IE: ps exec logs are not shown
+    end
+    memoize :parsed_layering_show_for
+
+    def parsed_layering_show
+      ENV['UFO_LAYERS'] || parse.for("layering.show", type: :boolean)
+    end
+    private :parsed_layering_show
+
+    def parse
+      Parse.new
     end
-    memoize :parse_for_layering_show
+    memoize :parse
 
     # Works similiar to Layering::Layer. Consider combining the logic and usin Layering::Layer
     #

data/lib/ufo/info.rb

@@ -28,6 +28,7 @@ module Ufo
       )
       target_group = resp.target_groups.first
       load_balancer_arn = target_group.load_balancer_arns.first # assume first only
+      return unless load_balancer_arn # can occur while stack is being deleted
 
       resp = elb.describe_load_balancers(load_balancer_arns: [load_balancer_arn])
       resp.load_balancers.first

data/lib/ufo/layering/layer.rb

@@ -68,9 +68,7 @@ module Ufo::Layering
       paths
     end
 
-    @@shown = false
     def show_layers(paths)
-      return if @@shown
       logger.debug "Layers:"
       paths.each do |path|
         if ENV['UFO_LAYERS_ALL']
@@ -79,7 +77,6 @@ module Ufo::Layering
           logger.info "    #{pretty_path(path)}" if File.exist?(path)
         end
       end
-      @@shown = true
     end
   end
 end

data/lib/ufo/task_definition/helpers/acm.rb

@@ -14,7 +14,7 @@ module Ufo::TaskDefinition::Helpers
       else
         # Logger causes infinite loop when waf helper used in .ufo/
         logger.warn "WARN: ACM cert not found: #{domain}".color(:yellow)
-        call_line = ufo_config_call_line
+        call_line = ufo_call_line
         DslEvaluator.print_code(call_line)
         nil
       end

data/lib/ufo/task_definition/helpers/ecr.rb

@@ -12,7 +12,7 @@ module Ufo::TaskDefinition::Helpers
       resp.repositories.first
     rescue Aws::ECR::Errors::RepositoryNotFoundException => e
       logger.warn "WARN: #{e.class} #{e.message}".color(:yellow)
-      call_line = ufo_config_call_line
+      call_line = ufo_call_line
       DslEvaluator.print_code(call_line)
       nil
     end

data/lib/ufo/task_definition/helpers/stack_output.rb

@@ -9,7 +9,7 @@ module Ufo::TaskDefinition::Helpers
       stack = find_stack(stack_name)
       unless stack
         logger.error "ERROR: Stack not found: #{stack_name}".color(:red)
-        call_line = ufo_config_call_line
+        call_line = ufo_call_line
         DslEvaluator.print_code(call_line)
         return
       end

data/lib/ufo/task_definition/helpers/vars/builder.rb

@@ -5,8 +5,10 @@ module Ufo::TaskDefinition::Helpers::Vars
     extend Memoist
     include AwsHelper
     include Ufo::Concerns::Names
-    include Ufo::Utils::Pretty
     include Ufo::Config::CallableOption::Concern
+    include Ufo::Utils::CallLine
+    include Ufo::Utils::Logging
+    include Ufo::Utils::Pretty
 
     def initialize(options={})
       # use either file or text. text takes higher precedence
@@ -15,25 +17,61 @@ module Ufo::TaskDefinition::Helpers::Vars
     end
 
     def content
-      @text || read(@file)
+      @text if @text
+      read(*find_files)
     end
 
-    def read(path)
-      full_path = "#{Ufo.root}/#{path}"
-      unless File.exist?(full_path)
-        puts "The #{pretty_path(full_path)} env file could not be found.  Are you sure it exists?"
-        exit 1
+    # Not considering .env files in project root since this is more for deployment
+    # Also ufo supports a smarter format than the normal .env files
+    def find_files
+      return @file if @file
+      layers = [
+        "base",
+        "#{Ufo.env}",
+        "#{Ufo.app}",
+        "#{Ufo.app}/base",
+        "#{Ufo.app}/#{Ufo.env}",
+        "#{Ufo.app}/#{Ufo.role}",
+        "#{Ufo.app}/#{Ufo.role}/base",
+        "#{Ufo.app}/#{Ufo.role}/#{Ufo.env}",
+      ]
+      layers.map! { |l| ".ufo/env_files/#{l}#{@ext}" }
+      show_layers(layers)
+      layers.select! { |l| File.exist?(l) }
+      layers
+    end
+
+    def show_layers(paths)
+      label = @ext.sub('.','').capitalize
+      paths.each do |path|
+        if ENV['UFO_LAYERS_ALL']
+          logger.info "    #{path}"
+        elsif Ufo.config.layering.show
+          logger.info "    #{path} "if File.exist?(path)
+        end
       end
-      IO.read(full_path)
     end
 
-    def env
+    def read(*paths)
+      text= ""
+      paths.compact.each do |path|
+        text << IO.read("#{Ufo.root}/#{path}")
+        text << "\n"
+      end
+      text
+    end
+
+    def env(ext='.env')
+      @ext = ext # assign instance variable so dont have to pass around
       lines = filtered_lines(content)
       lines.map do |line|
+        line = line.sub('export ', '') # allow user to use export. ufo ignores it
         key,*value = line.strip.split("=").map do |x|
           remove_surrounding_quotes(x.strip)
         end
         value = value.join('=')
+        # Note: env vars do NOT support valueFrom
+        # Docs: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_environment
         {
           name: key,
           value: value,
@@ -42,7 +80,7 @@ module Ufo::TaskDefinition::Helpers::Vars
     end
 
     def secrets
-      secrets = env
+      secrets = env('.secrets')
       secrets.map do |item|
         value = item.delete(:value)
         arn = normalize_to_arn(item[:name], value)
@@ -69,11 +107,15 @@ module Ufo::TaskDefinition::Helpers::Vars
     # arn:aws:ssm:us-west-2:111111111111:parameter/demo/dev/DB-NAME
     # arn:aws:ssm:us-west-2:111111111111:parameter/demo/dev/DB-NAME
     def expansion(arn)
-      # performance improvement only run names.expansion on the name portion
       md = arn.match(/(.*:)(parameter\/|secret:)(.*)/)
-      prefix, type, name = md[1], md[2], md[3]
-      expanded_name = names.expansion(name, dasherize: false) # dasherize: false. dont turn SECRET_NAME => SECRET-NAME
-      "#{prefix}#{type}#{expanded_name}"
+      if md
+        prefix, type, name = md[1], md[2], md[3]
+        # performance improvement only run names.expansion on the name portion
+        expanded_name = names.expansion(name, dasherize: false) # dasherize: false. dont turn SECRET_NAME => SECRET-NAME
+        "#{prefix}#{type}#{expanded_name}"
+      else # not arn full value. In case user accidentally puts value in .secrets file KEY=value
+        names.expansion(arn, dasherize: false) # dasherize: false. dont turn SECRET_NAME => SECRET-NAME
+      end
     end
 
     # Examples with config.secrets.provider = "ssm"

data/lib/ufo/task_definition/helpers/vars.rb

@@ -6,7 +6,7 @@ module Ufo::TaskDefinition::Helpers
     alias_method :env_vars, :env
     alias_method :environment, :env
 
-    def env_file(path)
+    def env_file(path=nil)
       Builder.new(file: path).env
     end
 
@@ -14,7 +14,7 @@ module Ufo::TaskDefinition::Helpers
       Builder.new(text: text).secrets
     end
 
-    def secrets_file(path)
+    def secrets_file(path=nil)
       Builder.new(file: path).secrets
     end
   end

data/lib/ufo/task_definition/helpers/waf.rb

@@ -20,7 +20,7 @@ module Ufo::TaskDefinition::Helpers
       else
         # Logger causes infinite loop when waf helper used in .ufo/
         logger.warn "WARN: Web ACL not found: #{name}".color(:yellow)
-        call_line = ufo_config_call_line
+        call_line = ufo_call_line
         DslEvaluator.print_code(call_line)
       end
     end

data/lib/ufo/utils/call_line.rb

@@ -2,7 +2,7 @@ module Ufo::Utils
   module CallLine
     include Pretty
 
-    def ufo_config_call_line
+    def ufo_call_line
       caller.find { |l| l.include?('.ufo/') }
     end
   end

data/lib/ufo/version.rb

@@ -1,3 +1,3 @@
 module Ufo
-  VERSION = "6.2.2"
+  VERSION = "6.2.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ufo
 version: !ruby/object:Gem::Version
-  version: 6.2.2
+  version: 6.2.5
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-03-20 00:00:00.000000000 Z
+date: 2022-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-logs
@@ -453,10 +453,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".cody/README.md"
+- ".cody/acceptance/bin/asg.sh"
 - ".cody/acceptance/bin/build.sh"
 - ".cody/acceptance/buildspec.yml"
+- ".cody/acceptance/iam_role.rb"
 - ".cody/acceptance/project.rb"
-- ".cody/acceptance/role.rb"
 - ".cody/shared/script/install.sh"
 - ".cody/shared/script/install/ufo.sh"
 - ".github/ISSUE_TEMPLATE.md"
@@ -594,6 +595,7 @@ files:
 - lib/ufo/config/callable_option.rb
 - lib/ufo/config/callable_option/concern.rb
 - lib/ufo/config/inits.rb
+- lib/ufo/config/parse.rb
 - lib/ufo/core.rb
 - lib/ufo/docker/builder.rb
 - lib/ufo/docker/cleaner.rb