ufo 5.0.6 → 6.0.1

data/lib/ufo/task_definition/helpers/ssm/fetcher.rb

@@ -0,0 +1,39 @@
+module Ufo::TaskDefinition::Helpers::Ssm
+  class Fetcher
+    include Ufo::AwsServices
+    include Ufo::Utils::Logging
+    include Ufo::Concerns::Names
+
+    def initialize(options={})
+      @options = options
+      @base64 = options[:base64]
+    end
+
+    def fetch(name)
+      name = names.expansion(name, dasherize: false)
+      parameter = fetch_parameter(name)
+      return unless parameter
+      value = parameter.value
+      value = Base64.strict_encode64(value).strip if base64?(parameter.type)
+      value
+    end
+
+    def base64?(type)
+      if @base64.nil?
+        type == "SecureString"
+      else
+        @base64
+      end
+    end
+
+    # Note: Cannot use logger if since if ssm helper is used in config it'll cause an infinite loop
+    def fetch_parameter(name)
+      resp = ssm_client.get_parameter(name: name, with_decryption: true)
+      resp.parameter
+    rescue Aws::SSM::Errors::ParameterNotFound => e
+      puts "WARN: name #{name} not found".color(:yellow)
+      puts e.message
+      nil
+    end
+  end
+end

data/lib/ufo/task_definition/helpers/ssm.rb

@@ -0,0 +1,8 @@
+module Ufo::TaskDefinition::Helpers
+  module Ssm
+    def ssm(name, options={})
+      fetcher = Ssm::Fetcher.new(options)
+      fetcher.fetch(name)
+    end
+  end
+end

data/lib/ufo/task_definition/helpers/stack_output.rb

@@ -0,0 +1,23 @@
+module Ufo::TaskDefinition::Helpers
+  module StackOutput
+    include Ufo::AwsServices
+    include Ufo::Concerns::Names
+
+    def stack_output(name)
+      stack_name, output_key = name.split(".")
+      stack_name = names.expansion(stack_name)
+      resp = cloudformation.describe_stacks(stack_name: stack_name)
+      stack = resp.stacks.first
+      if stack
+        o = stack.outputs.detect { |h| h.output_key == output_key }
+      end
+
+      if o
+        o.output_value
+      else
+        logger.info "WARN: NOT FOUND: output #{key} for stack #{stack_name}"
+        nil
+      end
+    end
+  end
+end

data/lib/ufo/{dsl/helper → task_definition/helpers}/vars.rb

@@ -1,8 +1,11 @@
 require "aws_data"
 
-class Ufo::DSL::Helper
+module Ufo::TaskDefinition::Helpers
   class Vars
     extend Memoist
+    include AwsDataHelper
+    include Ufo::Concerns::Names
+    include Ufo::Utils::Pretty
 
     def initialize(options={})
       # use either file or text. text takes higher precedence
@@ -17,7 +20,7 @@ class Ufo::DSL::Helper
     def read(path)
       full_path = "#{Ufo.root}/#{path}"
       unless File.exist?(full_path)
-        puts "The #{full_path} env file could not be found.  Are you sure it exists?"
+        puts "The #{pretty_path(full_path)} env file could not be found.  Are you sure it exists?"
         exit 1
       end
       IO.read(full_path)
@@ -41,24 +44,55 @@ class Ufo::DSL::Helper
       secrets = env
       secrets.map do |item|
         value = item.delete(:value)
-        item[:valueFrom] = substitute(expand_secret(value))
+        arn = normalize_to_arn(item[:name], value)
+        value = expansion(arn)
+        value = value.sub('parameter//','parameter/') # auto fix accidental leading slash for user
+        item[:valueFrom] = value
       end
       secrets
     end
 
-    def expand_secret(value)
+    def normalize_to_arn(name, value)
       case value
       when /^ssm:/i
         value.sub(/^ssm:/i, "arn:aws:ssm:#{region}:#{account}:parameter/")
       when /^secretsmanager:/i
         value.sub(/^secretsmanager:/i, "arn:aws:secretsmanager:#{region}:#{account}:secret:")
+      when '' # blank string will mean use convention
+        conventional_pattern(name, value)
       else
         value # assume full arn has been passed
       end
     end
 
-    def substitute(value)
-      value.gsub(":UFO_ENV", Ufo.env)
+    # arn:aws:ssm:us-west-2:111111111111:parameter/demo/dev/DB-NAME
+    # arn:aws:ssm:us-west-2:111111111111:parameter/demo/dev/DB-NAME
+    def expansion(arn)
+      # performance improvement only run names.expansion on the name portion
+      md = arn.match(/(.*:)(parameter\/|secret:)(.*)/)
+      prefix, type, name = md[1], md[2], md[3]
+      expanded_name = names.expansion(name, dasherize: false) # dasherize: false. dont turn SECRET_NAME => SECRET-NAME
+      "#{prefix}#{type}#{expanded_name}"
+    end
+
+    # Examples with config.secrets.provider = "ssm"
+    #
+    # .secrets
+    #
+    #      DB_NAME
+    #
+    # Results
+    #
+    #      DB_NAME=:APP/:ENV/:SECRET_NAME # expansion will use => demo/dev/DB_NAME
+    #
+    def conventional_pattern(name, value)
+      secrets = Ufo.config.secrets
+      provider = secrets.provider # ssm or secretsmanager
+      namespace = provider == "ssm" ? "parameter/" : "secret:"
+      pattern = secrets.pattern[provider] # IE: Ufo.config.secrets.pattern.ssm => :APP/:ENV/:SECRET_NAME
+      # replace :SECRET_NAME since names expand doesnt know how to nor do we want to add logic there
+      pattern = pattern.sub(':SECRET_NAME', name)
+      "arn:aws:#{provider}:#{region}:#{account}:#{namespace}#{pattern}"
     end
 
     def remove_surrounding_quotes(s)
@@ -80,18 +114,5 @@ class Ufo::DSL::Helper
       # filter out empty lines
       lines = lines.reject { |l| l.strip.empty? }
     end
-
-    def aws_data
-      AwsData.new
-    end
-    memoize :aws_data
-
-    def region
-      aws_data.region
-    end
-
-    def account
-      aws_data.account
-    end
   end
 end

data/lib/ufo/task_definition/helpers/vpc.rb

@@ -0,0 +1,48 @@
+module Ufo::TaskDefinition::Helpers
+  module Vpc
+    extend Memoist
+    include Ufo::AwsServices
+
+    def default_vpc
+      resp = ec2.describe_vpcs(filters: [name: "isDefault", values: ["true"]])
+      vpc = resp.vpcs.first
+      if vpc
+        vpc.vpc_id
+      else
+        logger.error "No default vpc found".color(:red)
+        logger.error <<~EOL
+          Please configure the `config.vpc` settings.
+
+          Docs: https://ufoships.com/config/vpc/
+
+        EOL
+        exit 1
+      end
+    end
+    memoize :default_vpc
+
+    def subnets_for(vpc_id)
+      resp = ec2.describe_subnets(filters: [name: "vpc-id", values: [vpc_id]])
+      subnets = resp.subnets
+      subnets.map(&:subnet_id)
+    end
+
+    def default_subnets
+      if default_vpc.nil?
+        logger.error "ERROR: no default subnets because no default vpc found".color(:red)
+        exit 1
+      end
+      resp = ec2.describe_subnets(filters: [name: "vpc-id", values: [default_vpc]])
+      subnets = resp.subnets
+      subnets.map(&:subnet_id)
+    end
+    memoize :default_subnets
+
+    def key_pairs(regexp=nil)
+      resp = ec2.describe_key_pairs
+      key_names = resp.key_pairs.map(&:key_name)
+      key_names.select! { |k| k =~ regexp } if regexp
+      key_names
+    end
+  end
+end

data/lib/ufo/task_definition/helpers.rb

@@ -0,0 +1,5 @@
+class Ufo::TaskDefinition
+  module Helpers
+    include_modules("helpers") # builtin helpers
+  end
+end

data/lib/ufo/task_definition.rb

@@ -0,0 +1,19 @@
+module Ufo
+  class TaskDefinition
+    extend Memoist
+    include Ufo::Concerns # for names
+
+    attr_reader :name, :role
+    def initialize(options={})
+      @options = options
+      @role = Ufo.role
+      @name = names.task_definition # IE: :APP-:ROLE-:ENV => demo-web-dev
+    end
+
+    def path
+      expr = "#{Ufo.root}/.ufo/resources/task_definitions/{#{@role},web,default}.{json,yml}"
+      Dir.glob(expr).first
+    end
+    memoize :path
+  end
+end

data/lib/ufo/upgrade/upgrade3.rb

@@ -38,7 +38,7 @@ class Ufo::Upgrade
       lines.map! do |line|
         line.sub(/[\/]?ufo/, '.ufo')
       end
-      lines << [".ufo/data\n"] # new ignore rule
+      lines << [".ufo/tmp/state\n"] # new ignore rule
       text = lines.join
       IO.write(".gitignore", text)
     end

data/lib/ufo/upgrade/upgrade4.rb

@@ -36,8 +36,8 @@ class Ufo::Upgrade
   private
     def update_task_definitions
       text = <<-EOL
-    # HINT: shows how Ufo.env_extra can to create different log groups
-    # awslogs_group: ["ecs/TASK_DEFINITION_NAME", Ufo.env_extra].compact.join('-'),
+    # HINT: shows how Ufo.extra can to create different log groups
+    # awslogs_group: ["ecs/TASK_DEFINITION_NAME", Ufo.extra].compact.join('-'),
 EOL
       insert_into_file ".ufo/task_definitions.rb", text, :before => /    awslogs_group:/
     end
@@ -112,7 +112,7 @@ EOL
     def upsert_gitignore
       text =<<-EOL
 .ufo/current
-.ufo/data
+.ufo/tmp/state
 .ufo/log
 .ufo/output
 EOL

data/lib/ufo/utils/execute.rb

@@ -0,0 +1,30 @@
+module Ufo::Utils
+  module Execute
+    def execute(command, options={})
+      log = options[:log]
+      if log
+        command += " >> #{log}"
+        logger.info "=> #{command}"
+        FileUtils.mkdir_p(File.dirname(log))
+        File.open(log, 'a') { |f| f.puts "=> #{command}" }
+        out = `#{command}`
+        success = $?.success?
+        unless success
+          logger.error out
+          exit 1
+        end
+        out
+      else
+        logger.info "=> #{command}"
+        system(command)
+      end
+    end
+
+    # TODO: remove these other metohds
+    # Custom user params from .ufo/params.yml
+    # Param keys are symbols for the aws-sdk calls.
+    def user_params
+      @user_params ||= Param.new.data
+    end
+  end
+end

data/lib/ufo/utils/logging.rb

@@ -0,0 +1,7 @@
+module Ufo::Utils
+  module Logging
+    def logger
+      Ufo.logger
+    end
+  end
+end

data/lib/ufo/utils/pretty.rb

@@ -0,0 +1,18 @@
+module Ufo::Utils
+  module Pretty
+    def pretty_path(path)
+      path.sub("#{Ufo.root}/",'').sub(/^\.\//,'')
+    end
+
+    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
+    def pretty_time(total_seconds)
+      minutes = (total_seconds / 60) % 60
+      seconds = total_seconds % 60
+      if total_seconds < 60
+        "#{seconds.to_i}s"
+      else
+        "#{minutes.to_i}m #{seconds.to_i}s"
+      end
+    end
+  end
+end

data/lib/ufo/utils/squeezer.rb

@@ -13,7 +13,7 @@ module Ufo::Utils
       when Hash
         data.each_with_object({}) do |(k,v), squeezed|
           # only remove nil and empty Array values within Hash structures
-          squeezed[k] = squeeze(v) unless v.nil? || v.is_a?(Array) && v.empty?
+          squeezed[k] = squeeze(v) unless v.nil? # || v.is_a?(Array) && v.empty?
           squeezed
         end
       else

data/lib/ufo/utils/sure.rb

@@ -0,0 +1,23 @@
+module Ufo::Utils
+  module Sure
+    def sure?(message=nil)
+      confirm = 'Are you sure?'
+      if @options[:yes]
+        yes = 'y'
+      else
+        out = if message
+                "#{message}\n#{confirm} (y/N) "
+              else
+                "#{confirm} (y/N) "
+              end
+        print out
+        yes = $stdin.gets
+      end
+
+      unless yes =~ /^y/
+        puts "Whew! Exiting."
+        exit 0
+      end
+    end
+  end
+end

data/lib/ufo/version.rb

@@ -1,3 +1,3 @@
 module Ufo
-  VERSION = "5.0.6"
+  VERSION = "6.0.1"
 end

data/lib/ufo/yaml/loader.rb

@@ -0,0 +1,48 @@
+require "yaml"
+
+class Ufo::Yaml
+  class Loader
+    def initialize(text)
+      @text = text
+    end
+
+    def load
+      add_domain_types!
+      YAML.load(@text)
+    end
+
+  private
+    def add_domain_types!
+      intrinsic_functions.each do |name|
+        YAML.add_domain_type('', name) do |type,val|
+          key = type.split('::').last
+          key = "Fn::" + key unless name == 'Ref'
+          { key => val }
+        end
+      end
+    end
+
+    def intrinsic_functions
+      %w[
+        And
+        Base64
+        Cidr
+        Equals
+        FindInMap
+        GetAtt
+        GetAZs
+        If
+        If
+        ImportValue
+        Join
+        Not
+        Or
+        Ref
+        Select
+        Split
+        Sub
+        Transform
+      ]
+    end
+  end
+end

data/lib/ufo/yaml/validator.rb

@@ -0,0 +1,51 @@
+class Ufo::Yaml
+  class Validator
+    def initialize(path)
+      @path = path
+    end
+
+    def validate!
+      validate_yaml(@path)
+    end
+
+    def validate_yaml(path)
+      text = IO.read(path)
+      begin
+        YAML.load(text)
+      rescue Psych::SyntaxError => e
+        handle_yaml_syntax_error(e, path)
+      end
+    end
+
+    def handle_yaml_syntax_error(e, path)
+      io = StringIO.new
+      io.puts "Invalid yaml. Output written for debugging: #{path}".color(:red)
+      io.puts "ERROR: #{e.message}".color(:red)
+
+      # Grab line info.  Example error:
+      #   ERROR: (<unknown>): could not find expected ':' while scanning a simple key at line 2 column 1
+      md = e.message.match(/at line (\d+) column (\d+)/)
+      line = md[1].to_i
+
+      lines = IO.read(path).split("\n")
+      context = 5 # lines of context
+      top, bottom = [line-context-1, 0].max, line+context-1
+      spacing = lines.size.to_s.size
+      lines[top..bottom].each_with_index do |line_content, index|
+        line_number = top+index+1
+        if line_number == line
+          io.printf("%#{spacing}d %s\n".color(:red), line_number, line_content)
+        else
+          io.printf("%#{spacing}d %s\n", line_number, line_content)
+        end
+      end
+
+      if ENV['LONO_TEST']
+        io.string
+      else
+        puts io.string
+        exit 1
+      end
+    end
+  end
+end

data/lib/ufo/yaml.rb

@@ -0,0 +1,13 @@
+module Ufo
+  class Yaml
+    class << self
+      def load(text)
+        path = "/tmp/ufo/temp.yml"
+        FileUtils.mkdir_p(File.dirname(path))
+        IO.write(path, text)
+        Validator.new(path).validate!
+        Loader.new(text).load
+      end
+    end
+  end
+end

data/lib/ufo.rb

@@ -1,19 +1,30 @@
 $stdout.sync = true unless ENV["UFO_STDOUT_SYNC"] == "0"
 
-$:.unshift(File.expand_path('../', __FILE__))
-require 'active_support/all'
+require 'active_support'
+require 'active_support/core_ext/class'
+require 'active_support/core_ext/hash'
+require 'active_support/core_ext/module/delegation'
+require 'cli-format'
 require 'deep_merge/rails_compat'
+require 'dsl_evaluator'
 require 'fileutils'
 require 'memoist'
 require 'rainbow/ext/string'
 require 'render_me_pretty'
+require 'ufo/ext'
 require 'ufo/version'
+require 'yaml'
 
 require "ufo/autoloader"
 Ufo::Autoloader.setup
 
 module Ufo
+  class UfoError < RuntimeError; end
+  class ShipmentOverridden < UfoError; end
+
   extend Core
 end
 
-Ufo.set_aws_profile!
+CliFormat.default_format = "table"
+
+Ufo::Booter.boot

data/spec/spec_helper.rb

@@ -1,42 +1,20 @@
-ENV["TEST"] = "1"
+ENV["UFO_TEST"] = "1"
 # Ensures aws api never called. Fixture home does not contain ~/.aws/credentials
 ENV['HOME'] = "spec/fixtures/home"
 
-# CodeClimate test coverage: https://docs.codeclimate.com/docs/configuring-test-coverage
-# require 'simplecov'
-# SimpleCov.start
-
 require "pp"
 require "byebug"
 root = File.expand_path("../", File.dirname(__FILE__))
 require "#{root}/lib/ufo"
 
-ENV['UFO_ROOT'] = "tmp/project"
-
 module Helpers
-  def create_ufo_project
-    FileUtils.rm_rf(Ufo.root)
-    execute("exe/ufo init --app demo --image tongueroo/demo-ufo")
-    create_test_settings
-  end
-
-  # modify the generated settings so we can spec the settings themselves
-  def create_test_settings
-    FileUtils.cp("spec/fixtures/settings.yml", "#{Ufo.root}/.ufo/settings.yml")
-  end
-
   def execute(cmd)
-    puts "Running: #{cmd}" if show_command?
+    show_command = ENV['DEBUG'] || ENV['SHOW_COMMAND']
+    puts "Running: #{cmd}" if show_command
     out = `#{cmd}`
-    puts out if show_command?
+    puts out if show_command
     out
   end
-
-  # Added SHOW_COMMAND because DEBUG is also used by other libraries like
-  # bundler and it shows its internal debugging logging also.
-  def show_command?
-    ENV['DEBUG'] || ENV['SHOW_COMMAND']
-  end
 end
 
 RSpec.configure do |c|

data/spec/{lib → ufo}/ecr_cleaner_spec.rb

@@ -3,9 +3,7 @@ describe Ufo::Ecr::Cleaner do
   let(:repo_domain) { "123456789.dkr.ecr.us-east-1.amazonaws.com" }
   let(:cleaner) do
     Ufo::Ecr::Cleaner.new(docker_image_name,
-      project_root: "spec/fixtures/hi",
       ecr_keep: 3, # using 3 to test, default is 30
-      mute: true
     )
   end
   before(:each) do

data/spec/{lib/role → ufo/iam_role}/builder_spec.rb

@@ -1,25 +1,25 @@
-describe Ufo::Role::Builder do
+describe Ufo::IamRole::Builder do
   let(:builder) { described_class.new(role_type) }
   let(:role_type) { "task_role" }
 
   before(:each) do
-    Ufo::Role::Registry.register_policy("task_role",
+    Ufo::IamRole::Registry.register_policy("task_role",
       "AmazonS3ReadOnlyAccess",
       {:Action=>["s3:Get*", "s3:List*"], :Effect=>"Allow", :Resource=>"*"}
     )
-    Ufo::Role::Registry.register_policy("task_role",
+    Ufo::IamRole::Registry.register_policy("task_role",
       "CloudwatchWrite",
       {:Action=>["cloudwatch:PutMetricData"], :Effect=>"Allow", :Resource=>"*"}
     )
     # Called twice on purpose to show that duplicated items in the set wont create doubles.
     # This allows the DSL evaluate to be ran multiple times.
-    Ufo::Role::Registry.register_policy("task_role",
+    Ufo::IamRole::Registry.register_policy("task_role",
       "CloudwatchWrite",
       {:Action=>["cloudwatch:PutMetricData"], :Effect=>"Allow", :Resource=>"*"}
     )
 
 
-    Ufo::Role::Registry.register_managed_policy("task_role",
+    Ufo::IamRole::Registry.register_managed_policy("task_role",
       "AmazonS3ReadOnlyAccess", "AmazonEC2ReadOnlyAccess"
     )
   end

data/spec/{lib/role → ufo/iam_role}/dsl_spec.rb

@@ -1,12 +1,12 @@
-describe Ufo::Role::DSL do
+describe Ufo::IamRole::DSL do
   let(:dsl) { described_class.new(path) }
   let(:path) { "spec/fixtures/iam_roles/task_role.rb" }
 
   context "evaluate" do
     it "registers policies from role DSL" do
       dsl.evaluate
-      expect(Ufo::Role::Registry.policies).not_to be_empty
-      expect(Ufo::Role::Registry.managed_policies).not_to be_empty
+      expect(Ufo::IamRole::Registry.policies).not_to be_empty
+      expect(Ufo::IamRole::Registry.managed_policies).not_to be_empty
     end
   end
 end

data/spec/{lib → ufo}/logs_spec.rb

@@ -1,6 +1,6 @@
-describe Ufo::Logs do
+describe Ufo::CLI::Logs do
   let(:logs) do
-    logs = Ufo::Logs.new("test-web", {})
+    logs = Ufo::CLI::Logs.new
     allow(logs).to receive(:exit).and_return(null)
     allow(logs).to receive(:info).and_return(null)
     allow(logs).to receive(:ecs).and_return(ecs)