ufo 5.0.7 → 6.0.0

data/lib/ufo/task.rb

@@ -1,225 +0,0 @@
-module Ufo
-  class Task < Base
-    extend Memoist
-
-    include AwsService
-    include Ufo::Settings
-    include Util
-
-    def initialize(task_definition, options)
-      @task_definition = task_definition
-      @options = options
-      # Assume task_definition is the same name as the ecs service name
-      @cluster = @options[:cluster] || default_cluster(task_definition)
-    end
-
-    def run
-      puts "Running task_definition: #{@task_definition}".color(:green) unless @options[:mute]
-      return if @options[:noop]
-
-      task_options = {
-        cluster: @cluster,
-        task_definition: @task_definition
-      }
-      task_options = adjust_fargate_options(task_options)
-      task_options = task_options.merge(user_params[:run_task] || {})
-      task_options = adjust_security_groups(task_options)
-
-      if @options[:command]
-        task_options.merge!(overrides: overrides)
-        puts "Running task with container overrides."
-        puts "Command: #{command_in_human_readable_form}"
-      end
-
-      unless @options[:mute]
-        puts "Running task with params:"
-        display_params(task_options)
-      end
-
-      ensure_log_group_exist
-
-      resp = run_task(task_options)
-      exit_if_failures!(resp)
-      unless @options[:mute]
-        task_arn = resp.tasks[0].task_arn
-        puts "Task ARN: #{task_arn}"
-        puts "  aws ecs describe-tasks --tasks #{task_arn} --cluster #{@cluster}"
-        cloudwatch_info(task_arn)
-      end
-    end
-
-    def ensure_log_group_exist
-      LogGroup.new(@task_definition, @options).create
-    end
-
-    # Pretty hard to produce this edge case.  Happens when:
-    #   launch_type: EC2
-    #   network_mode: awsvpc
-    #   assign_public_ip: DISABLED
-    def exit_if_failures!(resp)
-      return if resp[:failures].nil? || resp[:failures].empty?
-
-      puts "There was a failure running the ECS task.".color(:red)
-      puts "This might be happen if you have a network_mode of awsvpc and have assigned_public_ip to DISABLED."
-      puts "This cryptic error also shows up if the network settings have security groups and subnets that are not in the same vpc as the ECS cluster container instances.  Please double check that."
-      puts "You can use this command to quickly reconfigure the network settings:"
-      puts "  ufo network init --vpc-id XXX."
-      puts "More details on the can be found under the 'Task Networking Considerations' section at: "
-      puts "https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-networking.html"
-      puts "Original response with failures:"
-      pp resp
-      exit 1
-    end
-
-    def run_task(options)
-      puts "Equivalent aws cli command:"
-      puts "  aws ecs run-task --cluster #{@cluster} --task-definition #{options[:task_definition]}".color(:green)
-      run_task_result = ecs.run_task(options)
-      if @options[:wait]
-        task_arn = run_task_result.tasks[0].task_arn
-        raise SystemExit, exit_status_of_task(task_arn)
-      else
-        run_task_result
-      end
-    rescue Aws::ECS::Errors::ClientException => e
-      if e.message =~ /ECS was unable to assume the role/
-        puts "ERROR: #{e.class} #{e.message}".color(:red)
-        puts "Please double check the executionRoleArn in your task definition."
-        exit 1
-      else
-        raise
-      end
-    rescue Aws::ECS::Errors::InvalidParameterException => e
-      if e.message =~ /Network Configuration must be provided when networkMode 'awsvpc' is specified/
-        puts "ERROR: #{e.class} #{e.message}".color(:red)
-        puts "Please double check .ufo/params.yml and make sure that network_configuration is set."
-        puts "Or run change the task definition template in .ufo/templates so it does not use vpcmode."
-        exit 1
-      else
-        raise
-      end
-    end
-
-  private
-    # adjust network_configuration based on fargate and network mode of awsvpc
-    def adjust_fargate_options(options)
-      task_def = recent_task_definition
-      return options unless task_def[:network_mode] == "awsvpc"
-
-      awsvpc_conf = { subnets: network[:ecs_subnets] }
-      if task_def[:requires_compatibilities] == ["FARGATE"]
-        awsvpc_conf[:assign_public_ip] = "ENABLED"
-        options[:launch_type] = "FARGATE"
-      end
-
-      options[:network_configuration] = { awsvpc_configuration: awsvpc_conf }
-      options
-    end
-
-    # Ensures at least 1 security group is assigned if awsvpc_configuration
-    # is provided.
-    def adjust_security_groups(options)
-      return options unless options[:network_configuration] &&
-                     options[:network_configuration][:awsvpc_configuration]
-
-      awsvpc_conf = options[:network_configuration][:awsvpc_configuration]
-
-      security_groups = awsvpc_conf[:security_groups]
-      if [nil, '', 'nil'].include?(security_groups)
-        security_groups = []
-      end
-      if security_groups.empty?
-        fetch = Network::Fetch.new(network[:vpc])
-        sg = fetch.security_group_id
-        security_groups << sg
-        security_groups.uniq!
-      end
-
-      # override security groups
-      options[:network_configuration][:awsvpc_configuration][:security_groups] = security_groups
-      options
-    end
-
-    def cloudwatch_info(task_arn)
-      config = container_definition[:log_configuration]
-      container_name = container_definition[:name]
-
-      return unless config && config[:log_driver] == "awslogs"
-
-      log_group = config[:options]["awslogs-group"]
-      log_stream_prefix = config[:options]["awslogs-stream-prefix"]
-      task_id = task_arn.split('/').last
-      log_stream = "#{log_stream_prefix}/#{container_name}/#{task_id}"
-      # website/web/d473440a-9a0e-4262-a53d-f9e345cf2b7e
-      region = `aws configure get region`.strip rescue 'us-east-1'
-      url = "https://#{region}.console.aws.amazon.com/cloudwatch/home?region=#{region}#logEventViewer:group=#{log_group};stream=#{log_stream}"
-
-      puts "To see the task output visit CloudWatch:\n  #{url}"
-      puts "NOTE: It will take some time for the log to show up because it takes time for the task to start. Run the `aws ecs describe-tasks` above for the task status."
-    end
-
-    # only using the overrides to override the container command
-    def overrides
-      command = @options[:command] # Thor parser ensure this is always an array
-      {
-        container_overrides: [
-          {
-            name: container_definition[:name],
-            command: command,
-            environment: container_definition[:environment],
-          },
-        ]
-      }
-    end
-
-    # Usually most recent task definition.
-    # If user has specified task_definition with specific version like
-    #   demo-web:8
-    # Then it'll be that exact task definnition.
-    def recent_task_definition
-      arns = task_definition_arns(@task_definition)
-      # "arn:aws:ecs:us-east-1:<aws_account_id>:task-definition/wordpress:6",
-      last_definition_arn = arns.first
-      # puts "last_definition_arn #{last_definition_arn}"
-      task_name = last_definition_arn.split("/").last
-      resp = ecs.describe_task_definition(task_definition: task_name)
-
-      resp.task_definition
-    end
-    memoize :recent_task_definition
-
-    # container definition from the most recent task definition
-    def container_definition
-      recent_task_definition.container_definitions[0].to_h
-    end
-
-    def exit_status_of_task(task_arn)
-      waiter_max_attempts = @options[:timeout] / 5
-      waiter_delay = 5 # seconds long polling.
-
-      print "Waiting for task to complete."
-      ecs.wait_until(:tasks_stopped, { cluster: @cluster, tasks: [task_arn] }) do |waiter|
-        waiter.before_wait { print '.' }
-        waiter.delay = waiter_delay
-        waiter.max_attempts = waiter_max_attempts
-      end
-      puts
-      container = ecs.describe_tasks(cluster: @cluster, tasks: [task_arn]).tasks[0].containers[0]
-
-      if container.exit_code.nil?
-        puts "Command failed!".color(:red)
-        puts "Reason: #{container.reason}"
-        1
-      else
-        container.exit_code
-      end
-    rescue Aws::Waiters::Errors::WaiterFailed
-      puts "It took longer than #{options[:timeout]} seconds to run #{command_in_human_readable_form} (#{task_arn})"
-      exit 1
-    end
-
-    def command_in_human_readable_form
-      @options[:command].join(' ')
-    end
-  end
-end

data/lib/ufo/tasks/builder.rb

@@ -1,40 +0,0 @@
-module Ufo
-  class Tasks::Builder
-    # ship: build and registers task definitions together
-    def self.ship(task_definition, options)
-      # When handling task definitions in with the ship command and class, we always want to
-      # build and register task definitions. There is little point of running them independently
-      # This method helps us do that.
-      build(options)
-    end
-
-    # ship: build and registers task definitions together
-    def self.build(options)
-      Tasks::Builder.new(options).build
-    end
-
-    def initialize(options={})
-      @options = options
-    end
-
-    def build
-      puts "Building Task Definitions...".color(:green) unless @options[:mute]
-      check_templates_definitions_path
-      dsl = DSL.new(template_definitions_path, @options.merge(quiet: false, mute: true))
-      dsl.run
-      puts "Task Definitions built in .ufo/output" unless @options[:mute]
-    end
-
-    def check_templates_definitions_path
-      unless File.exist?(template_definitions_path)
-        pretty_path = template_definitions_path.sub("#{Ufo.root}/", '')
-        puts "ERROR: #{pretty_path} does not exist.  Run: `ufo init` to create a starter file" unless @options[:mute]
-        exit 1
-      end
-    end
-
-    def template_definitions_path
-      "#{Ufo.root}/.ufo/task_definitions.rb"
-    end
-  end
-end

data/lib/ufo/tasks/register.rb

@@ -1,90 +0,0 @@
-require 'plissken' # Hash#to_snake_keys
-require 'json'
-
-module Ufo
-  class Tasks::Register
-    include Util
-    include AwsService
-
-    def self.register(task_name, options={})
-      Dir.glob("#{Ufo.root}/.ufo/output/*").each do |path|
-        if task_name == :all or path.include?(task_name)
-          task_register = Tasks::Register.new(path, options)
-          task_register.register
-        end
-      end
-    end
-
-    def initialize(template_definition_path, options={})
-      @template_definition_path = template_definition_path
-      @options = options
-    end
-
-    # aws ecs register-task-definition --cli-input-json file://.ufo/output/demo-web-prod.json
-    def register
-      data = JSON.parse(IO.read(@template_definition_path))
-      data = rubyize_format(data)
-
-      message = "#{data[:family]} task definition registered."
-      if @options[:noop]
-        message = "NOOP: #{message}"
-      else
-        register_task_definition(data)
-      end
-
-      unless @options[:mute]
-        puts "Equivalent aws cli command:"
-        file_path = "file://#{@template_definition_path.sub(/^\.\//,'')}"
-        puts "  aws ecs register-task-definition --cli-input-json #{file_path}".color(:green)
-        puts message
-      end
-    end
-
-    def register_task_definition(data)
-      if ENV["UFO_SHOW_REGISTER_TASK_DEFINITION"]
-        puts "Registering task definition with:"
-        display_params(data)
-      end
-
-      ecs.register_task_definition(data)
-    rescue Aws::ECS::Errors::ClientException => e
-      if e.message =~ /No Fargate configuration exists for given values/
-        puts "ERROR: #{e.message}".color(:red)
-        puts "Configured values are: cpu #{data[:cpu]} memory #{data[:memory]}"
-        puts "Check that the cpu and memory values are a supported combination by Fargate."
-        puts "More info: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html"
-        exit 1
-      else
-        raise
-      end
-    end
-
-    # The ruby aws-sdk expects symbols for keys and AWS docs for the task
-    # definition uses json camelCase for the keys.  This method transforms
-    # the keys to the expected ruby aws-sdk format.
-    #
-    # One quirk is that the logConfiguration options casing should not be
-    # transformed.
-    def rubyize_format(original_data)
-      data = original_data.to_snake_keys.deep_symbolize_keys
-
-      definitions = data[:container_definitions]
-      definitions.each_with_index do |definition, i|
-        next unless definition[:log_configuration] || definition[:firelens_configuration]
-        { log_configuration: 'logConfiguration',
-          firelens_configuration: 'firelensConfiguration' }.each_pair do |key, value|
-
-          next unless definition.dig(key, :options)
-
-          # LogConfiguration and firelensConfiguration options do not get transformed and
-          # keep their original structure:
-          #   https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/ECS/Types/ContainerDefinition.html
-          original_definition = original_data["containerDefinitions"][i]
-          definition[key][:options] = original_definition[value]["options"]
-        end
-      end
-
-      data
-    end
-  end
-end

data/lib/ufo/template_scope.rb

@@ -1,57 +0,0 @@
-module Ufo
-  class TemplateScope
-    extend Memoist
-    include Ufo::Settings
-
-    attr_reader :helper
-    attr_reader :task_definition_name
-    def initialize(helper=nil, task_definition_name=nil)
-      @helper = helper
-      @task_definition_name = task_definition_name # only available from task_definition
-        # not available from params
-      load_variables_file("base")
-      load_variables_file(Ufo.env)
-    end
-
-    # Load the variables defined in ufo/variables/* to make available in the
-    # template blocks in ufo/templates/*.
-    #
-    # Example:
-    #
-    #   `ufo/variables/base.rb`:
-    #     @name = "docker-process-name"
-    #     @image = "docker-image-name"
-    #
-    #   `ufo/templates/main.json.erb`:
-    #   {
-    #     "containerDefinitions": [
-    #       {
-    #          "name": "<%= @name %>",
-    #          "image": "<%= @image %>",
-    #      ....
-    #   }
-    #
-    # NOTE: Only able to make instance variables avaialble with instance_eval
-    #   Wasnt able to make local variables available.
-    def load_variables_file(filename)
-      path = "#{Ufo.root}/.ufo/variables/#{filename}.rb"
-      instance_eval(IO.read(path), path) if File.exist?(path)
-    end
-
-    # Add additional instance variables to template_scope
-    def assign_instance_variables(vars)
-      vars.each do |k,v|
-        instance_variable_set("@#{k}".to_sym, v)
-      end
-    end
-
-    def pretty_name?
-      # env variable takes highest precedence
-      if ENV["STATIC_NAME"]
-        ENV["STATIC_NAME"] != "0"
-      else
-        settings[:pretty_name]
-      end
-    end
-  end
-end

data/lib/ufo/util.rb

@@ -1,69 +0,0 @@
-require 'active_support/core_ext/hash'
-
-module Ufo
-  module Util
-    # The default cluster normally defaults to the Ufo.env value.
-    # But it can be overriden by ufo/settings.yml cluster
-    #
-    # More info: http://ufoships.com/docs/settings/
-    def default_cluster(service)
-      # to_s.to_sym in case service is nil
-      settings.dig(:service_cluster, service.to_s.to_sym) ||
-      settings[:cluster] ||
-      Ufo.env
-    end
-
-    # Keys are strings for simplicity.
-    def settings
-      @settings ||= Ufo.settings
-    end
-
-    # Custom user params from .ufo/params.yml
-    # Param keys are symbols for the aws-sdk calls.
-    def user_params
-      @user_params ||= Param.new.data
-    end
-
-    def execute(command, local_options={})
-      if @options[:noop] && !local_options[:live]
-        say "NOOP: #{command}"
-        result = true # always success with no noop for specs
-      else
-        if local_options[:use_system]
-          result = system(command)
-        else
-          result = `#{command}`
-        end
-      end
-      result
-    end
-
-    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
-    def pretty_time(total_seconds)
-      minutes = (total_seconds / 60) % 60
-      seconds = total_seconds % 60
-      if total_seconds < 60
-        "#{seconds.to_i}s"
-      else
-        "#{minutes.to_i}m #{seconds.to_i}s"
-      end
-    end
-
-    def display_params(options)
-      puts YAML.dump(options.deep_stringify_keys)
-    end
-
-    def task_definition_arns(service, max_items=10)
-      resp = ecs.list_task_definitions(
-        family_prefix: service,
-        sort: "DESC",
-      )
-      arns = resp.task_definition_arns
-      arns = arns.select do |arn|
-        task_definition = arn.split('/').last.split(':').first
-        task_definition == service
-      end
-      arns[0..max_items]
-    end
-  end
-end

data/spec/lib/apps_spec.rb

@@ -1,20 +0,0 @@
-describe Ufo::Apps do
-  before(:each) do
-    create_ufo_project
-  end
-  let(:apps) do
-    Ufo::Apps.new({})
-  end
-
-  context "running tasks" do
-    let(:describe_tasks_response) do
-      JSON.load(IO.read("spec/fixtures/apps/describe_services.json"))
-    end
-    it "displays info" do
-      allow(apps).to receive(:service_info).and_return([
-        "demo-web", "task-def", "1", "EC2", "yes"
-      ])
-      apps.display_info(describe_tasks_response)
-    end
-  end
-end

data/spec/lib/builder_spec.rb

@@ -1,23 +0,0 @@
-describe Ufo::Docker::Builder do
-  before(:all) do
-    create_ufo_project
-  end
-
-  let(:builder) { Ufo::Docker::Builder.new }
-
-  context "dockerfile uses ecr image for FROM instruction" do
-    it "updates the auth token before building the image" do
-      # builder.from_ecr_image?("spec/fixtures/dockerfiles/dockerhub/Dockerfile")
-      names = builder.ecr_image_names("spec/fixtures/dockerfiles/ecr/Dockerfile")
-      expect(names).not_to be_empty
-    end
-  end
-
-  context "dockerfile uses dockerhub image for FROM instruction" do
-    it "does not update the auth token before building the image" do
-      # builder.from_ecr_image?("spec/fixtures/dockerfiles/dockerhub/Dockerfile")
-      names = builder.ecr_image_names("spec/fixtures/dockerfiles/dockerhub/Dockerfile")
-      expect(names).to be_empty
-    end
-  end
-end

data/spec/lib/cli_spec.rb

@@ -1,73 +0,0 @@
-describe Ufo::CLI do
-  before(:all) do
-    create_ufo_project
-    @args = "--noop"
-  end
-
-  describe "ufo" do
-    context "docker" do
-      it "build builds image" do
-        out = execute("exe/ufo docker build #{@args}")
-        expect(out).to include("docker build -t tongueroo/demo-ufo")
-      end
-
-      it "tag shows the tag" do
-        out = execute("exe/ufo docker name #{@args}")
-        expect(out).to match(%r{tongueroo/demo-ufo:ufo-.{7}})
-      end
-
-      it "compiles Dockerfile from Dockerfile.erb" do
-        FileUtils.cp("spec/fixtures/dockerfiles/Dockerfile.erb", "#{Ufo.root}/Dockerfile.erb")
-        out = execute("exe/ufo docker compile #{@args}")
-        expect(out).to include("Compiled Dockerfile.erb to Dockerfile")
-      end
-    end
-
-    context "tasks" do
-      before(:each) do
-        FileUtils.mkdir_p("#{Ufo.root}/.ufo/data/")
-        FileUtils.touch("#{Ufo.root}/.ufo/data/docker_image_name_ufo.txt")
-      end
-      after(:each) do
-        FileUtils.rm_f("#{Ufo.root}/.ufo/data/docker_image_name_ufo.txt")
-      end
-
-      it "build builds task definition" do
-        out = execute("exe/ufo tasks build #{@args}")
-        expect(out).to include("Task Definitions built")
-      end
-
-      it "register it registers all the output task definitions" do
-        out = execute("exe/ufo tasks register #{@args}")
-        expect(out).to include("register")
-      end
-    end
-
-    context "ship" do
-      it "deploys software" do
-        out = execute("exe/ufo ship demo-web-prod #{@args} --no-wait")
-        # cannot look for Software shipped! because
-        #   ship.deploy unless ENV['TEST'] # to allow me to quickly test CLI portion only
-        # just testing the CLI portion.  The ship class itself is tested via ship_spec.rb
-        expect(out).to include("Task Definitions built")
-      end
-    end
-
-    context "ships" do
-      it "deploys software to multiple services" do
-        out = execute("exe/ufo ships demo-web-prod demo-worker-prod #{@args} --no-wait")
-        # cannot look for Software shipped! because
-        #   ship.deploy unless ENV['TEST'] # to allow me to quickly test CLI portion only
-        # just testing the CLI portion.  The ship class itself is tested via ship_spec.rb
-        expect(out).to include("Task Definitions built")
-      end
-    end
-
-    context "task" do
-      it "runs one time task" do
-        out = execute("exe/ufo completion ship name")
-        expect(out).to include("--help")
-      end
-    end
-  end
-end

data/spec/lib/completion_spec.rb

@@ -1,18 +0,0 @@
-describe Ufo::CLI do
-  describe "ufo completion" do
-    commands = {
-      "ship" => "service",
-      "ship service" => "--task",
-      "docker" =>  "build",
-      "docker build" => "--push",
-      "docker clean" => "image_name",
-      "init" => "--image",
-    }
-    commands.each do |command, expected_word|
-      it "#{command}" do
-        out = execute("exe/ufo completion #{command}")
-        expect(out).to include(expected_word) # only checking for one word for simplicity
-      end
-    end
-  end
-end

data/spec/lib/ps_spec.rb

@@ -1,14 +0,0 @@
-describe Ufo::Ps do
-  let(:ps) do
-    Ufo::Ps.new("test-web", {})
-  end
-
-  context "running tasks" do
-    let(:task_arns) do
-      JSON.load(IO.read("spec/fixtures/ps/describe_tasks.json"))["tasks"]
-    end
-    it "display_tasks" do
-      ps.display_tasks(task_arns)
-    end
-  end
-end

data/spec/lib/register_spec.rb

@@ -1,49 +0,0 @@
-describe Ufo::Tasks::Register do
-  before(:all) do
-    create_ufo_project
-  end
-
-  let(:register) { Ufo::Tasks::Register.new("fake_task_definition_path") }
-
-  context "syslog" do
-    let(:data) do
-      {
-        "containerDefinitions" => [{
-          "logConfiguration" => {
-            "logDriver" => "syslog"
-          }
-        }]
-      }
-    end
-
-    it "#rubyize_format" do
-      result = register.rubyize_format(data)
-      driver = result[:container_definitions][0][:log_configuration][:log_driver]
-      expect(driver).to eq "syslog"
-    end
-  end
-
-  context "awslogs" do
-    let(:data) do
-      {
-        "containerDefinitions" => [{
-          "logConfiguration" => {
-            "logDriver" => "awslogs",
-            "options" => {
-              "awslogs-group" => "mygroup",
-              "awslogs-region" => "us-east-1",
-              "awslogs-stream-prefix" => "mystream"
-            }
-          }
-        }]
-      }
-    end
-
-    it "rubyize_format" do
-      result = register.rubyize_format(data)
-      log_configuration = result[:container_definitions][0][:log_configuration]
-      expect(log_configuration[:log_driver]).to eq "awslogs"
-      expect(log_configuration[:options].keys).to include("awslogs-group")
-    end
-  end
-end