ufo 6.1.5 → 6.2.2

data/lib/ufo/config.rb

@@ -85,6 +85,9 @@ module Ufo
       config.exec.command = "/bin/bash" # aws ecs execute-command cli
       config.exec.enabled = true        # EcsService EnableExecuteCommand
 
+      config.layering = ActiveSupport::OrderedOptions.new
+      config.layering.show = show_layers?
+
       config.log = ActiveSupport::OrderedOptions.new
       config.log.root = Ufo.log_root
       config.logger = ufo_logger
@@ -114,7 +117,10 @@ module Ufo
       config.ship.docker.quiet = false # only affects ufo ship docker commands output
 
       config.state = ActiveSupport::OrderedOptions.new
+      config.state.bucket = nil # Set to use existing bucket. When not set ufo creates a managed s3 bucket
+      config.state.managed = true # false will disable creation of managed bucket entirely
       config.state.reminder = true
+      config.state.storage = "s3" # s3 or file
 
       config.waf = ActiveSupport::OrderedOptions.new
       config.waf.web_acl_arn = nil
@@ -157,15 +163,52 @@ module Ufo
         role = layer_levels(".ufo/config/#{Ufo.app}/#{Ufo.role}")
         layers += root + env + role
       end
-      # load_project_config gets called so early that logger is not yet configured. use puts
-      puts "Config layers:" if ENV['UFO_SHOW_ALL_LAYERS']
+      # load_project_config gets called so early that logger is not yet configured.
+      # Cannot use Ufo.config yet and cannot use logger which relies on Ufo.config
+      # Use puts and use show_layers? which parses for the config
+      show = show_layers?
+      puts "Config Layers" if show
       layers.each do |layer|
         path = "#{Ufo.root}/#{layer}"
-        puts "    #{layer}" if ENV['UFO_SHOW_ALL_LAYERS']
+        if ENV['UFO_LAYERS_ALL']
+          puts "    #{pretty_path(path)}"
+        elsif show
+          puts "    #{pretty_path(path)}" if File.exist?(path)
+        end
         evaluate_file(path)
       end
     end
 
+    def show_layers?
+      ENV['UFO_LAYERS'] || parse_for_layering_show
+    end
+    private :show_layers?
+
+    # Some limitations:
+    #
+    # * Only parsing one file: .ufo/config.rb
+    # * If user is using Ruby code that cannot be parse will fallback to default
+    #
+    # Think it's worth it so user only has to configure
+    #
+    #     config.layering.show = true
+    #
+    def parse_for_layering_show
+      lines = IO.readlines("#{Ufo.root}/.ufo/config.rb")
+      config_line = lines.find { |l| l =~ /config\.layering.show.*=/ && l !~ /^\s+#/ }
+      return false unless config_line # default is false
+      config_value = config_line.gsub(/.*=/,'').strip.gsub(/["']/,'')
+      config_value != "false" && config_value != "nil"
+    rescue Exception => e
+      if ENV['UFO_DEBUG']
+        puts "#{e.class} #{e.message}".color(:yellow)
+        puts "WARN: Unable to parse for config.layering.show".color(:yellow)
+        puts "Using default: config.layering.show = false"
+      end
+      false
+    end
+    memoize :parse_for_layering_show
+
     # Works similiar to Layering::Layer. Consider combining the logic and usin Layering::Layer
     #
     # Examples:
@@ -180,6 +223,7 @@ module Ufo
     #
     def layer_levels(prefix=nil)
       levels = ["", "base", Ufo.env]
+      levels << "#{Ufo.env}-#{Ufo.extra}" if Ufo.extra
       paths = levels.map do |i|
         # base layer has prefix of '', reject with blank so it doesnt produce '//'
         [prefix, i].join('/')

data/lib/ufo/docker/builder.rb

@@ -162,7 +162,7 @@ module Ufo::Docker
 
     def update_dockerfile
       updater = if File.exist?("#{Ufo.root}/Dockerfile.erb") # dont use @dockerfile on purpose
-        State.new(docker_image, @options)
+        State.new(@options.merge(base_image: docker_image))
       else
         Dockerfile.new(docker_image, @options)
       end

data/lib/ufo/docker/compiler.rb

@@ -9,9 +9,9 @@ module Ufo::Docker
       return unless File.exist?(@erb_file)
 
       puts "Compiled #{File.basename(@erb_file).color(:green)} to #{File.basename(@dockerfile).color(:green)}"
-      path = "#{Ufo.root}/.ufo/state/data.yml"
-      vars = YAML.load_file(path)[Ufo.env] if File.exist?(path)
-      vars ||= {}
+
+      state = State.new
+      vars = state.read
       result = RenderMePretty.result(@erb_file, vars)
       comment =<<~EOL.chop # remove the trailing newline
         # IMPORTANT: This file was generated from #{File.basename(@erb_file)} as a part of running:

data/lib/ufo/docker/state/base.rb

@@ -0,0 +1,14 @@
+class Ufo::Docker::State
+  class Base
+    include Ufo::Utils::Logging
+    include Ufo::Utils::Pretty
+
+    def initialize(options={})
+      @options = options
+      # base_image only passed in with: ufo docker base
+      # State#update uses it.
+      # State#read wont have access to it and gets it from stored state
+      @base_image = options[:base_image]
+    end
+  end
+end

data/lib/ufo/docker/state/bucket.rb

@@ -0,0 +1,2 @@
+class Ufo::Docker::State
+  class S3 < Base

data/lib/ufo/docker/state/file.rb

@@ -0,0 +1,52 @@
+class Ufo::Docker::State
+  class File < Base
+    def read
+      current_data
+    end
+
+    def update
+      data = current_data
+      data["base_image"] = @base_image
+
+      pretty_path = state_path.sub("#{Ufo.root}/", "")
+      FileUtils.mkdir_p(::File.dirname(state_path))
+      IO.write(state_path, YAML.dump(data))
+
+      logger.info "The #{pretty_path} base_image has been updated with the latest base image:".color(:green)
+      logger.info "    #{@base_image}".color(:green)
+      reminder_message
+    end
+
+    def current_data
+      ::File.exist?(state_path) ? YAML.load_file(state_path) : {}
+    end
+
+    def state_path
+      "#{Ufo.root}/.ufo/state/#{Ufo.app}/#{Ufo.env}/data.yml"
+    end
+
+    def reminder_message
+      return unless Ufo.config.state.reminder
+      repo = ENV['UFO_CENTRAL_REPO']
+      return unless repo
+      logger.info "It looks like you're using a central deployer pattern".color(:yellow)
+      logger.info <<~EOL
+        Remember to commit the state file:
+
+            state file: #{pretty_path(state_path)}
+            repo:       #{repo}
+
+      EOL
+
+      logger.info <<~EOL
+        You can disable these reminder messages with:
+
+        .ufo/config.rb
+
+            Ufo.configure do |config|
+              config.state.reminder = false
+            end
+      EOL
+    end
+  end
+end

data/lib/ufo/docker/state/s3.rb

@@ -0,0 +1,80 @@
+class Ufo::Docker::State
+  class S3 < Base
+    extend Memoist
+    include Ufo::AwsServices
+
+    def read
+      current_data
+    end
+
+    def update
+      data = current_data
+      data["base_image"] = @base_image
+
+      # write data to s3
+      body = YAML.dump(data)
+      s3.put_object(
+        body: body,
+        bucket: s3_bucket,
+        key: s3_key,
+      )
+      logger.info "Updated base image in s3://#{s3_bucket}/#{s3_key}"
+      logger.info "    #{@base_image}".color(:green)
+    end
+
+    # TODO: edge cases: no bucket, no permission
+    def current_data
+      resp = s3.get_object(bucket: s3_bucket, key: s3_key)
+      YAML.load(resp.body)
+    rescue Aws::S3::Errors::NoSuchKey
+      logger.debug "WARN: s3 key does not exist: #{s3_key}"
+      {}
+    rescue Aws::S3::Errors::NoSuchBucket
+      logger.error "ERROR: S3 bucket does not exist to store state: #{s3_bucket}".color(:red)
+      logger.error <<~EOL
+          Please double check the config.
+
+          See: http://ufoships.com/docs/config/state/
+
+      EOL
+      exit 1
+    end
+
+    def s3_key
+      "ufo/state/#{app}/#{Ufo.env}/data.yml"
+    end
+
+    # ufo docker base is called before Ufo.config is loaded. This ensures it is loaded
+    def app
+      Ufo.config
+      Ufo.app
+    end
+
+    def s3_bucket
+      state = Ufo.config.state
+      if state.bucket
+        state.bucket
+      elsif state.managed
+        ensure_s3_bucket_exist
+        Ufo::S3::Bucket.name
+      else
+        logger.error "ERROR: No s3 bucket to store state".color(:red)
+        logger.error <<~EOL
+          UFO needs a bucket to store the built docker base image.
+
+          Configure an existing bucket or enable UFO to create a bucket.
+
+          See: http://ufoships.com/docs/config/state/
+        EOL
+        exit 1
+      end
+    end
+
+    def ensure_s3_bucket_exist
+      bucket = Ufo::S3::Bucket.new
+      return if bucket.exist?
+      bucket.deploy
+    end
+    memoize :ensure_s3_bucket_exist
+  end
+end

data/lib/ufo/docker/state.rb

@@ -1,63 +1,29 @@
 module Ufo::Docker
   class State
-    include Ufo::Utils::Logging
-    include Ufo::Utils::Pretty
+    extend Memoist
 
-    def initialize(docker_image, options={})
-      @docker_image, @options = docker_image, options
+    def initialize(options={})
+      @options = options
     end
 
     def update
-      data = current_data
-      data[Ufo.env] ||= {}
-      data[Ufo.env]["base_image"] = @docker_image
-      pretty_path = state_path.sub("#{Ufo.root}/", "")
-      FileUtils.mkdir_p(File.dirname(state_path))
-      IO.write(state_path, YAML.dump(data))
-      logger.info "The #{pretty_path} base_image has been updated with the latest base image:".color(:green)
-      logger.info "  #{@docker_image}".color(:green)
-      reminder_message
+      storage.update
     end
 
-    def current_data
-      File.exist?(state_path) ? YAML.load_file(state_path) : {}
+    def read
+      storage.read
     end
 
-    def state_path
-      path = "#{Ufo.root}/.ufo/state"
-      if ENV['UFO_APP'] # env var activates app path
-        path = "#{Ufo.root}/.ufo/state/#{Ufo.app}"
-      end
-      "#{path}/data.yml"
-    end
-
-    def reminder_message
-      return unless Ufo.config.state.reminder
-      repo = ENV['UFO_CENTRAL_REPO']
-      return unless repo
-      logger.info "It looks like you're using a central deployer pattern".color(:yellow)
-      logger.info <<~EOL
-        Remember to commit the state file:
-
-            state file: #{pretty_path(state_path)}
-            repo:       #{repo}
-
-      EOL
-
-      unless ENV['UFO_APP']
-        logger.info "WARN: It also doesnt look like UFO_ENV is set".color(:yellow)
-        logger.info "UFO_ENV should be set when you're using ufo in a central manner"
-      end
-
-      logger.info <<~EOL
-        You can disable these reminder messages with:
-
-        .ufo/config.rb
-
-            Ufo.configure do |config|
-              config.state.reminder = false
-            end
-      EOL
+  private
+    # Examples:
+    #   File.new(@docker_image, @options)
+    #   S3.new(@docker_image, @options)
+    def storage
+      storage = Ufo.config.state.storage
+      class_name = "Ufo::Docker::State::#{storage.camelize}"
+      klass = class_name.constantize
+      klass.new(@options)
     end
+    memoize :storage
   end
 end

data/lib/ufo/info.rb

@@ -73,7 +73,7 @@ module Ufo
     end
 
     def stack_resources
-      resp = cloudformation.describe_stack_resources(stack_name: @stack_name)
+      resp = cfn.describe_stack_resources(stack_name: @stack_name)
       resp.stack_resources
     end
     memoize :stack_resources

data/lib/ufo/layering/layer.rb

@@ -23,46 +23,44 @@ module Ufo::Layering
     end
 
     def paths
-      core = full_layers(".ufo/vars")
-      app = full_layers(".ufo/vars/#{Ufo.app}")
-      paths = core + app
+      # core = full_layers(".ufo/vars")
+      # app = full_layers(".ufo/vars/#{Ufo.app}")
+
+      core =     layer_levels(".ufo/vars")
+      role =     layer_levels(".ufo/vars/#{@task_definition.role}")
+      app =      layer_levels(".ufo/vars/#{Ufo.app}")
+      app_role = layer_levels(".ufo/vars/#{Ufo.app}/#{@task_definition.role}")
+
+      paths = core + role + app + app_role
       add_ext!(paths)
       paths.map! { |p| "#{Ufo.root}/#{p}" }
       show_layers(paths)
       paths
     end
 
-    def full_layering
-      # layers defined in Lono::Layering module
-      all = layers.map { |layer| layer.sub(/\/$/,'') } # strip trailing slash
-      all.inject([]) do |sum, layer|
-        sum += layer_levels(layer) unless layer.nil?
-        sum
-      end
-    end
-
-    # interface method
-    def main_layers
-      ['']
-    end
-
     # adds prefix and to each layer pair that has base and Ufo.env. IE:
     #
     #    "#{prefix}/base"
     #    "#{prefix}/#{Ufo.env}"
     #
     def layer_levels(prefix=nil)
-      levels = ["base", Ufo.env]
+      levels = ["", "base", Ufo.env]
+      levels << "#{Ufo.env}-#{Ufo.extra}" if Ufo.extra
       levels.map! do |i|
         # base layer has prefix of '', reject with blank so it doesnt produce '//'
         [prefix, i].reject(&:blank?).join('/')
       end
-      levels.unshift(prefix) # unless prefix.blank? # IE: params/us-west-2.txt
+      levels.map! { |level| level.sub(/\/$/,'') } # strip trailing slash
+      # levels.unshift(prefix) # unless prefix.blank? # IE: params/us-west-2.txt
       levels
     end
 
+    # interface method
+    def main_layers
+      ['']
+    end
+
     def add_ext!(paths)
-      ext = "rb"
       paths.map! do |path|
         path = path.sub(/\/$/,'') if path.ends_with?('/')
         "#{path}.rb"
@@ -70,26 +68,18 @@ module Ufo::Layering
       paths
     end
 
-    def full_layers(dir)
-      layers = full_layering.map do |layer|
-        "#{dir}/#{layer}"
-      end
-      role_layers = full_layering.map do |layer|
-        "#{dir}/#{@task_definition.role}/#{layer}" # Note: layer can be '' will clean up
-      end
-      layers += role_layers
-      layers.map { |l| l.gsub('//','/') } # cleanup // if layer is ''
-    end
-
-    @@shown_layers = false
+    @@shown = false
     def show_layers(paths)
-      return if @@shown_layers
-      logger.info "Vars Layers:" if ENV['UFO_SHOW_ALL_LAYERS']
+      return if @@shown
+      logger.debug "Layers:"
       paths.each do |path|
-        show_layer = File.exist?(path) && logger.level <= Logger::DEBUG
-        logger.info "    #{pretty_path(path)}" if show_layer || ENV['UFO_SHOW_ALL_LAYERS']
+        if ENV['UFO_LAYERS_ALL']
+          logger.info "    #{pretty_path(path)}"
+        elsif Ufo.config.layering.show
+          logger.info "    #{pretty_path(path)}" if File.exist?(path)
+        end
       end
-      @@shown_layers = true
+      @@shown = true
     end
   end
 end

data/lib/ufo/s3/aws_setup.rb

@@ -0,0 +1,17 @@
+module Ufo::S3
+  class AwsSetup
+    include Ufo::AwsServices
+    include Ufo::Utils::Logging
+
+    def check!
+      s3.config.region
+    rescue Aws::Errors::MissingRegionError => e
+      logger.info "ERROR: #{e.class}: #{e.message}".color(:red)
+      logger.info <<~EOL
+        Unable to detect the AWS_REGION to make AWS API calls. This is might be because the AWS access
+        has not been set up yet. Please either your ~/.aws files.
+      EOL
+      exit 1
+    end
+  end
+end

data/lib/ufo/s3/bucket.rb

@@ -0,0 +1,174 @@
+module Ufo::S3
+  class Bucket
+    extend Memoist
+    extend Ufo::AwsServices
+    include Ufo::AwsServices
+    include Ufo::Utils::Logging
+
+    STACK_NAME = ENV['UFO_STACK_NAME'] || "ufo"
+    def initialize(options={})
+      @options = options
+    end
+
+    def deploy
+      stack = find_stack
+      if rollback.complete?
+        logger.info "Existing '#{STACK_NAME}' stack in ROLLBACK_COMPLETE state. Deleting stack before continuing."
+        disable_termination_protection
+        cfn.delete_stack(stack_name: STACK_NAME)
+        status.wait
+        stack = nil
+      end
+
+      if stack
+        update
+      else
+        create
+      end
+    end
+
+    def exist?
+      !!bucket_name
+    end
+
+    def bucket_name
+      self.class.name
+    end
+
+    def show
+      if bucket_name
+        logger.info "UFO bucket name: #{bucket_name}"
+      else
+        logger.info "UFO bucket does not exist yet."
+      end
+    end
+
+    # Launches a cloudformation to create an s3 bucket
+    def create
+      logger.info "Creating #{STACK_NAME} stack for s3 bucket to store state"
+      cfn.create_stack(
+        stack_name: STACK_NAME,
+        template_body: template_body,
+        enable_termination_protection: true,
+      )
+      success = status.wait
+      unless success
+        logger.info "ERROR: Unable to create UFO stack with managed s3 bucket".color(:red)
+        exit 1
+      end
+    end
+
+    def update
+      logger.info "Updating #{STACK_NAME} stack with the s3 bucket"
+      cfn.update_stack(stack_name: STACK_NAME, template_body: template_body)
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      raise unless e.message.include?("No updates are to be performed")
+    end
+
+    def delete
+      are_you_sure?
+
+      logger.info "Deleting #{STACK_NAME} stack with the s3 bucket"
+      disable_termination_protection
+      empty_bucket!
+      cfn.delete_stack(stack_name: STACK_NAME)
+    end
+
+    def disable_termination_protection
+      cfn.update_termination_protection(
+        stack_name: STACK_NAME,
+        enable_termination_protection: false,
+      )
+    end
+
+    def find_stack
+      resp = cfn.describe_stacks(stack_name: STACK_NAME)
+      resp.stacks.first
+    rescue Aws::CloudFormation::Errors::ValidationError
+      nil
+    end
+
+    def status
+      CfnStatus.new(STACK_NAME)
+    end
+
+  private
+
+    def empty_bucket!
+      return unless bucket_name # in case of UFO stack ROLLBACK_COMPLETE from failed bucket creation
+
+      resp = s3.list_objects(bucket: bucket_name)
+      if resp.contents.size > 0
+        # IE: objects = [{key: "objectkey1"}, {key: "objectkey2"}]
+        objects = resp.contents.map { |item| {key: item.key} }
+        s3.delete_objects(
+          bucket: bucket_name,
+          delete: {
+            objects: objects,
+            quiet: false,
+          }
+        )
+        empty_bucket! # keep deleting objects until bucket is empty
+      end
+    end
+
+
+    def are_you_sure?
+      return true if @options[:yes]
+
+      if bucket_name.nil?
+        logger.info "The UFO stack and s3 bucket does not exist."
+        exit
+      end
+
+      logger.info "Are you yes you want the UFO bucket #{bucket_name.color(:green)} to be emptied and deleted? (y/N)"
+      yes = $stdin.gets.strip
+      confirmed = yes =~ /^Y/i
+      unless confirmed
+        logger.info "Phew that was close."
+        exit
+      end
+    end
+
+    def template_body
+      <<~YAML
+        Description: UFO managed s3 bucket
+        Resources:
+          Bucket:
+            Type: AWS::S3::Bucket
+            Properties:
+              BucketEncryption:
+                 ServerSideEncryptionConfiguration:
+                 - ServerSideEncryptionByDefault:
+                    SSEAlgorithm: AES256
+              Tags:
+                - Key: Name
+                  Value: UFO
+        Outputs:
+          Bucket:
+            Value:
+              Ref: Bucket
+      YAML
+    end
+
+    def rollback
+      Rollback.new(STACK_NAME)
+    end
+
+    class << self
+      @@name = nil
+      def name
+        return @@name if @@name # only memoize once bucket has been created
+
+        AwsSetup.new.check!
+
+        stack = new.find_stack
+        return unless stack
+
+        stack_resources = find_stack_resources(STACK_NAME)
+        bucket = stack_resources.find { |r| r.logical_resource_id == "Bucket" }
+        @@name = bucket.physical_resource_id # actual bucket name
+      end
+    end
+  end
+end

data/lib/ufo/s3/rollback.rb

@@ -0,0 +1,52 @@
+module Ufo::S3
+  class Rollback
+    extend Memoist
+    include Ufo::AwsServices
+    include Ufo::Utils::Logging
+
+    def initialize(stack)
+      @stack = stack
+    end
+
+    def delete_stack
+      return unless complete?
+      logger.info "Existing stack in ROLLBACK_COMPLETE state. Deleting stack before continuing."
+      cfn.delete_stack(stack_name: @stack)
+      status.wait
+      status.reset
+      true
+    end
+
+    def continue_update
+      continue_update?
+      begin
+        cfn.continue_update_rollback(stack_name: @stack)
+      rescue Aws::CloudFormation::Errors::ValidationError => e
+        logger.info "ERROR: Continue update: #{e.message}".color(:red)
+        quit 1
+      end
+    end
+
+    def continue_update?
+      logger.info <<~EOL
+        The stack is in the UPDATE_ROLLBACK_FAILED state. More info here: https://amzn.to/2IiEjc5
+        Would you like to try to continue the update rollback? (y/N)
+      EOL
+
+      yes = @options[:yes] ? "y" : $stdin.gets
+      unless yes =~ /^y/
+        logger.info "Exiting without continuing the update rollback."
+        quit 0
+      end
+    end
+
+    def complete?
+      stack&.stack_status == 'ROLLBACK_COMPLETE'
+    end
+
+    def stack
+      find_stack(@stack)
+    end
+    memoize :stack
+  end
+end