ufo 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.rspec +3 -0
- data/CHANGELOG.md +12 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +128 -0
- data/Guardfile +12 -0
- data/LICENSE.txt +22 -0
- data/README.md +227 -0
- data/Rakefile +6 -0
- data/bin/ufo +10 -0
- data/lib/starter_project/.env.prod +3 -0
- data/lib/starter_project/Dockerfile +20 -0
- data/lib/starter_project/bin/deploy +12 -0
- data/lib/starter_project/ufo/settings.yml +7 -0
- data/lib/starter_project/ufo/task_definitions.rb +44 -0
- data/lib/starter_project/ufo/templates/main.json.erb +39 -0
- data/lib/ufo.rb +26 -0
- data/lib/ufo/aws_services.rb +21 -0
- data/lib/ufo/cli.rb +136 -0
- data/lib/ufo/cli/help.rb +164 -0
- data/lib/ufo/defaults.rb +46 -0
- data/lib/ufo/destroy.rb +60 -0
- data/lib/ufo/docker_builder.rb +120 -0
- data/lib/ufo/docker_cleaner.rb +52 -0
- data/lib/ufo/dockerfile_updater.rb +42 -0
- data/lib/ufo/dsl.rb +93 -0
- data/lib/ufo/dsl/helper.rb +47 -0
- data/lib/ufo/dsl/outputter.rb +40 -0
- data/lib/ufo/dsl/task_definition.rb +65 -0
- data/lib/ufo/ecr_auth.rb +35 -0
- data/lib/ufo/ecr_cleaner.rb +66 -0
- data/lib/ufo/execute.rb +19 -0
- data/lib/ufo/init.rb +83 -0
- data/lib/ufo/pretty_time.rb +14 -0
- data/lib/ufo/scale.rb +34 -0
- data/lib/ufo/settings.rb +33 -0
- data/lib/ufo/ship.rb +436 -0
- data/lib/ufo/tasks_builder.rb +30 -0
- data/lib/ufo/tasks_register.rb +51 -0
- data/lib/ufo/templates/default.json.erb +39 -0
- data/lib/ufo/version.rb +3 -0
- data/spec/fixtures/home_existing/.docker/config.json +10 -0
- data/spec/lib/cli_spec.rb +50 -0
- data/spec/lib/ecr_auth_spec.rb +39 -0
- data/spec/lib/ecr_cleaner_spec.rb +32 -0
- data/spec/lib/ship_spec.rb +77 -0
- data/spec/spec_helper.rb +28 -0
- data/ufo.gemspec +34 -0
- metadata +267 -0
@@ -0,0 +1,40 @@
|
|
1
|
+
module Ufo
|
2
|
+
class DSL
|
3
|
+
class Outputter
|
4
|
+
def initialize(name, erb_result, options={})
|
5
|
+
@name = name
|
6
|
+
@erb_result = erb_result
|
7
|
+
@options = options
|
8
|
+
@pretty = options[:pretty].nil? ? true : options[:pretty]
|
9
|
+
end
|
10
|
+
|
11
|
+
def write
|
12
|
+
output_path = "#{@options[:project_root]}/ufo/output"
|
13
|
+
FileUtils.rm_rf(output_path) if @options[:clean]
|
14
|
+
FileUtils.mkdir(output_path) unless File.exist?(output_path)
|
15
|
+
|
16
|
+
path = "#{output_path}/#{@name}.json"
|
17
|
+
puts "Generated task definition at: #{path}" unless @options[:quiet]
|
18
|
+
validate(@erb_result, path)
|
19
|
+
json = @pretty ?
|
20
|
+
JSON.pretty_generate(JSON.parse(@erb_result)) :
|
21
|
+
@erb_result
|
22
|
+
File.open(path, 'w') {|f| f.write(output_json(json)) }
|
23
|
+
end
|
24
|
+
|
25
|
+
def validate(json, path)
|
26
|
+
begin
|
27
|
+
JSON.parse(json)
|
28
|
+
rescue JSON::ParserError => e
|
29
|
+
puts "Invalid json. Output written to #{path} for debugging".colorize(:red)
|
30
|
+
File.open(path, 'w') {|f| f.write(json) }
|
31
|
+
exit 1
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def output_json(json)
|
36
|
+
@options[:pretty] ? JSON.pretty_generate(JSON.parse(json)) : json
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require "erb"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
module Ufo
|
5
|
+
class DSL
|
6
|
+
class TaskDefinition
|
7
|
+
attr_reader :task_definition_name
|
8
|
+
def initialize(dsl, task_definition_name, options={}, &block)
|
9
|
+
@dsl = dsl
|
10
|
+
@task_definition_name = task_definition_name
|
11
|
+
@block = block
|
12
|
+
@options = options
|
13
|
+
@project_root = @options[:project_root] || '.'
|
14
|
+
end
|
15
|
+
|
16
|
+
# delegate helper method back up to dsl
|
17
|
+
def helper
|
18
|
+
@dsl.helper
|
19
|
+
end
|
20
|
+
|
21
|
+
def build
|
22
|
+
instance_eval(&@block)
|
23
|
+
erb_template = IO.read(source_path)
|
24
|
+
ERB.new(erb_template).result(binding)
|
25
|
+
end
|
26
|
+
|
27
|
+
# at this point instance_eval has been called and source has possibly been called
|
28
|
+
def source(name)
|
29
|
+
@source = name
|
30
|
+
end
|
31
|
+
|
32
|
+
def variables(vars={})
|
33
|
+
vars.each do |var,value|
|
34
|
+
if instance_variable_defined?("@#{var}")
|
35
|
+
puts "WARNING: The instance variable @#{var} is already used internally with ufo. Please name you variable another name!"
|
36
|
+
end
|
37
|
+
instance_variable_set("@#{var}", value)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def source_path
|
42
|
+
if @source # this means that source has been called
|
43
|
+
path = "#{@project_root}/ufo/templates/#{@source}.json.erb"
|
44
|
+
check_source_path(path)
|
45
|
+
else
|
46
|
+
# default source path
|
47
|
+
path = File.expand_path("../../templates/default.json.erb", __FILE__)
|
48
|
+
puts "#{task_definition_name} template definition using default template: #{path}" unless @options[:mute]
|
49
|
+
end
|
50
|
+
path
|
51
|
+
end
|
52
|
+
|
53
|
+
def check_source_path(path)
|
54
|
+
unless File.exist?(path)
|
55
|
+
friendly_path = path.sub("#{@project_root}/", '')
|
56
|
+
puts "ERROR: Could not find the #{friendly_path} template. Are sure it exists? Check where you called source in ufo/task_definitions.rb"
|
57
|
+
exit 1
|
58
|
+
else
|
59
|
+
puts "#{task_definition_name} template definition using project template: #{path}" unless @options[:mute]
|
60
|
+
end
|
61
|
+
path
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/lib/ufo/ecr_auth.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
module Ufo
|
2
|
+
class EcrAuth
|
3
|
+
include AwsServices
|
4
|
+
|
5
|
+
def initialize(repo_domain)
|
6
|
+
@repo_domain = repo_domain
|
7
|
+
end
|
8
|
+
|
9
|
+
def update
|
10
|
+
auth_token = fetch_auth_token
|
11
|
+
if File.exist?(docker_config)
|
12
|
+
data = JSON.load(IO.read(docker_config))
|
13
|
+
data["auths"][@repo_domain] = {auth: auth_token}
|
14
|
+
else
|
15
|
+
data = {auths: {@repo_domain => {auth: auth_token}}}
|
16
|
+
end
|
17
|
+
ensure_dotdocker_exists
|
18
|
+
IO.write(docker_config, JSON.pretty_generate(data))
|
19
|
+
end
|
20
|
+
|
21
|
+
def fetch_auth_token
|
22
|
+
ecr.get_authorization_token.authorization_data.first.authorization_token
|
23
|
+
end
|
24
|
+
|
25
|
+
def docker_config
|
26
|
+
"#{ENV['HOME']}/.docker/config.json"
|
27
|
+
end
|
28
|
+
|
29
|
+
def ensure_dotdocker_exists
|
30
|
+
dirname = File.dirname(docker_config)
|
31
|
+
FileUtils.mkdir_p(dirname) unless File.exist?(dirname)
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require "json"
|
2
|
+
|
3
|
+
# Can test:
|
4
|
+
#
|
5
|
+
# ufo ship app-web --cluster prod --noop
|
6
|
+
module Ufo
|
7
|
+
class EcrCleaner
|
8
|
+
include AwsServices
|
9
|
+
include Defaults
|
10
|
+
|
11
|
+
def initialize(docker_image_name, options={})
|
12
|
+
# docker_image_name does not containg the tag
|
13
|
+
# Example: 123456789.dkr.ecr.us-east-1.amazonaws.com/image
|
14
|
+
@docker_image_name = docker_image_name
|
15
|
+
@options = options
|
16
|
+
@keep = options[:ecr_keep] || settings.data["ecr_keep"]
|
17
|
+
@tag_prefix = options[:tag_prefix] || "ufo"
|
18
|
+
end
|
19
|
+
|
20
|
+
def cleanup
|
21
|
+
return false unless ecr_image?
|
22
|
+
return false unless @keep
|
23
|
+
update_auth_token
|
24
|
+
image_tags = fetch_image_tags
|
25
|
+
delete_tags = image_tags[@keep..-1] # ordered by most recent images first
|
26
|
+
delete_images(delete_tags)
|
27
|
+
end
|
28
|
+
|
29
|
+
def fetch_image_tags
|
30
|
+
ecr.list_images(repository_name: repo_name).
|
31
|
+
image_ids.
|
32
|
+
map { |image_id| image_id.image_tag }.
|
33
|
+
select { |image_tag| image_tag =~ Regexp.new("^#{@tag_prefix}-") }.
|
34
|
+
sort.reverse
|
35
|
+
end
|
36
|
+
|
37
|
+
def delete_images(tags)
|
38
|
+
return if tags.nil? || tags.empty?
|
39
|
+
unless @options[:mute]
|
40
|
+
puts "Keeping #{@keep} most recent ECR images."
|
41
|
+
puts "Deleting these ECR images:"
|
42
|
+
tag_list = tags.map { |t| " #{repo_name}:#{t}" }
|
43
|
+
puts tag_list
|
44
|
+
end
|
45
|
+
image_ids = tags.map { |tag| {image_tag: tag} }
|
46
|
+
ecr.batch_delete_image(
|
47
|
+
repository_name: repo_name,
|
48
|
+
image_ids: image_ids) unless @options[:noop]
|
49
|
+
end
|
50
|
+
|
51
|
+
def update_auth_token
|
52
|
+
repo_domain = "https://#{@docker_image_name.split('/').first}"
|
53
|
+
auth = EcrAuth.new(repo_domain)
|
54
|
+
auth.update
|
55
|
+
end
|
56
|
+
|
57
|
+
def repo_name
|
58
|
+
# @docker_image_name example: 123456789.dkr.ecr.us-east-1.amazonaws.com/image
|
59
|
+
@docker_image_name.split('/').last
|
60
|
+
end
|
61
|
+
|
62
|
+
def ecr_image?
|
63
|
+
@docker_image_name =~ /\.amazonaws\.com/
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
data/lib/ufo/execute.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
module Ufo
|
2
|
+
module Execute
|
3
|
+
def execute(command, local_options={})
|
4
|
+
command = "cd #{@project_root} && #{command}"
|
5
|
+
# local_options[:live] overrides the global @options[:noop]
|
6
|
+
if @options[:noop] && !local_options[:live]
|
7
|
+
say "NOOP: #{command}"
|
8
|
+
result = true # always success with no noop for specs
|
9
|
+
else
|
10
|
+
if local_options[:use_system]
|
11
|
+
result = system(command)
|
12
|
+
else
|
13
|
+
result = `#{command}`
|
14
|
+
end
|
15
|
+
end
|
16
|
+
result
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/ufo/init.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'colorize'
|
2
|
+
require 'erb'
|
3
|
+
|
4
|
+
# quick test:
|
5
|
+
# rm -rf ufo ; /Users/tung/src/tongueroo/ufo/bin/ufo init --cluster default2 --image tongueroo/ruby ; cat ufo/settings.yml
|
6
|
+
module Ufo
|
7
|
+
class Init
|
8
|
+
def initialize(options = {})
|
9
|
+
@options = options
|
10
|
+
@project_root = options[:project_root] || '.'
|
11
|
+
end
|
12
|
+
|
13
|
+
def setup
|
14
|
+
puts "Setting up ufo project...".blue unless @options[:quiet]
|
15
|
+
source_root = File.expand_path("../../starter_project", __FILE__)
|
16
|
+
# https://ruby-doc.org/core-2.2.0/Dir.html
|
17
|
+
# use the File::FNM_DOTMATCH flag or something like "{*,.*}".
|
18
|
+
paths = Dir.glob("#{source_root}/**/{*,.*}").
|
19
|
+
select {|p| File.file?(p) }
|
20
|
+
paths.each do |src|
|
21
|
+
dest = src.gsub(%r{.*starter_project/},'')
|
22
|
+
dest = "#{@project_root}/#{dest}"
|
23
|
+
|
24
|
+
if File.exist?(dest) and !@options[:force]
|
25
|
+
puts "exists: #{dest}".yellow unless @options[:quiet]
|
26
|
+
else
|
27
|
+
dirname = File.dirname(dest)
|
28
|
+
FileUtils.mkdir_p(dirname) unless File.exist?(dirname)
|
29
|
+
if dest =~ /\.erb$/
|
30
|
+
FileUtils.cp(src, dest)
|
31
|
+
else
|
32
|
+
write_erb_result(src, dest)
|
33
|
+
end
|
34
|
+
puts "created: #{dest}".green unless @options[:quiet]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
puts "Starter ufo files created.".blue
|
38
|
+
File.chmod(0755, "#{@project_root}/bin/deploy")
|
39
|
+
add_gitignore
|
40
|
+
end
|
41
|
+
|
42
|
+
def write_erb_result(src, dest)
|
43
|
+
source = IO.read(src)
|
44
|
+
b = ERBContext.new(@options).get_binding
|
45
|
+
output = ERB.new(source).result(b)
|
46
|
+
IO.write(dest, output)
|
47
|
+
end
|
48
|
+
|
49
|
+
def add_gitignore
|
50
|
+
gitignore_path = "#{@project_root}/.gitignore"
|
51
|
+
if File.exist?(gitignore_path)
|
52
|
+
ignores = IO.read(gitignore_path)
|
53
|
+
has_ignore = ignores.include?("ufo/output")
|
54
|
+
ignores << ufo_ignores unless has_ignore
|
55
|
+
else
|
56
|
+
ignores = ufo_ignores
|
57
|
+
end
|
58
|
+
IO.write(gitignore_path, ignores)
|
59
|
+
end
|
60
|
+
|
61
|
+
def ufo_ignores
|
62
|
+
ignores =<<-EOL
|
63
|
+
/ufo/output
|
64
|
+
/ufo/docker_image_name*.txt
|
65
|
+
/ufo/version
|
66
|
+
EOL
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# http://stackoverflow.com/questions/1338960/ruby-templates-how-to-pass-variables-into-inlined-erb
|
73
|
+
class ERBContext
|
74
|
+
def initialize(hash)
|
75
|
+
hash.each_pair do |key, value|
|
76
|
+
instance_variable_set('@' + key.to_s, value)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def get_binding
|
81
|
+
binding
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Ufo
|
2
|
+
module PrettyTime
|
3
|
+
# http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
|
4
|
+
def pretty_time(total_seconds)
|
5
|
+
minutes = (total_seconds / 60) % 60
|
6
|
+
seconds = total_seconds % 60
|
7
|
+
if total_seconds < 60
|
8
|
+
"#{seconds.to_i}s"
|
9
|
+
else
|
10
|
+
"#{minutes.to_i}m #{seconds.to_i}s"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/ufo/scale.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
module Ufo
|
2
|
+
class Scale
|
3
|
+
include Defaults
|
4
|
+
include AwsServices
|
5
|
+
|
6
|
+
def initialize(service, count, options={})
|
7
|
+
@service = service
|
8
|
+
@count = count
|
9
|
+
@options = options
|
10
|
+
@cluster = @options[:cluster] || default_cluster
|
11
|
+
end
|
12
|
+
|
13
|
+
def update
|
14
|
+
unless service_exists?
|
15
|
+
puts "Unable to find the #{@service} service on #{@cluster} cluster."
|
16
|
+
puts "Are you sure you are trying to scale the right service on the right cluster?"
|
17
|
+
exit
|
18
|
+
end
|
19
|
+
ecs.update_service(
|
20
|
+
service: @service,
|
21
|
+
cluster: @cluster,
|
22
|
+
desired_count: @count
|
23
|
+
)
|
24
|
+
puts "Scale #{@service} service in #{@cluster} cluster to #{@count}" unless @options[:mute]
|
25
|
+
end
|
26
|
+
|
27
|
+
def service_exists?
|
28
|
+
cluster = ecs.describe_clusters(clusters: [@cluster]).clusters.first
|
29
|
+
return false unless cluster
|
30
|
+
service = ecs.describe_services(services: [@service], cluster: @cluster).services.first
|
31
|
+
!!service
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
data/lib/ufo/settings.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Ufo
|
4
|
+
class Settings
|
5
|
+
def initialize(project_root='.')
|
6
|
+
@project_root = project_root
|
7
|
+
end
|
8
|
+
|
9
|
+
# data contains the settings.yml config. The order or precedence for settings
|
10
|
+
# is the project ufo/settings.yml and then the ~/.ufo/settings.yml.
|
11
|
+
def data
|
12
|
+
return @data if @data
|
13
|
+
|
14
|
+
if File.exist?(settings_path)
|
15
|
+
@data = YAML.load_file(settings_path)
|
16
|
+
@data = user_settings.merge(@data)
|
17
|
+
else
|
18
|
+
puts "ERROR: No settings file file at #{settings_path}"
|
19
|
+
puts "Please create a settings file via: ufo init"
|
20
|
+
exit 1
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def user_settings
|
25
|
+
path = "#{ENV['HOME']}/.ufo/settings.yml"
|
26
|
+
File.exist?(path) ? YAML.load_file(path) : {}
|
27
|
+
end
|
28
|
+
|
29
|
+
def settings_path
|
30
|
+
"#{@project_root}/ufo/settings.yml"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/ufo/ship.rb
ADDED
@@ -0,0 +1,436 @@
|
|
1
|
+
require 'colorize'
|
2
|
+
|
3
|
+
module Ufo
|
4
|
+
# Creating this class pass so we can have a reference we do not have describe
|
5
|
+
# all services and look up service_name creating more API calls.
|
6
|
+
#
|
7
|
+
# Also this class allows us to pass one object around instead of both
|
8
|
+
# cluster_name and service_name.
|
9
|
+
module ECS
|
10
|
+
Service = Struct.new(:cluster_arn, :service_arn) do
|
11
|
+
def cluster_name
|
12
|
+
cluster_arn.split('/').last
|
13
|
+
end
|
14
|
+
|
15
|
+
def service_name
|
16
|
+
service_arn.split('/').last
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class UfoError < RuntimeError; end
|
22
|
+
class ShipmentOverridden < UfoError; end
|
23
|
+
|
24
|
+
class Ship
|
25
|
+
include Defaults
|
26
|
+
include AwsServices
|
27
|
+
include PrettyTime
|
28
|
+
|
29
|
+
# service can be a pattern
|
30
|
+
def initialize(service, task_definition, options={})
|
31
|
+
@service = service
|
32
|
+
@task_definition = task_definition
|
33
|
+
@options = options
|
34
|
+
@project_root = options[:project_root] || '.'
|
35
|
+
@elb_prompt = @options[:elb_prompt].nil? ? true : @options[:elb_prompt]
|
36
|
+
@cluster = @options[:cluster] || default_cluster
|
37
|
+
@wait_for_deployment = @options[:wait].nil? ? true : @options[:wait]
|
38
|
+
@stop_old_tasks = @options[:stop_old_tasks].nil? ? false : @options[:stop_old_tasks]
|
39
|
+
end
|
40
|
+
|
41
|
+
# Find all the matching services on a cluster and update them.
|
42
|
+
# If no service is found at all, then create a service on the very first specified cluster
|
43
|
+
# only.
|
44
|
+
#
|
45
|
+
# If it looks like one service is passed in then it'll automatically create the service
|
46
|
+
# on the first cluster.
|
47
|
+
|
48
|
+
# If it looks like a regexp is passed in then it'll only update the services
|
49
|
+
# This is because regpex cannot be used to determined a list of service_names.
|
50
|
+
#
|
51
|
+
# Example:
|
52
|
+
# No way to map: hi-.*-prod -> hi-web-prod hi-worker-prod hi-clock-prod
|
53
|
+
def deploy
|
54
|
+
puts "Shipping #{@service}...".green unless @options[:mute]
|
55
|
+
|
56
|
+
ensure_cluster_exist
|
57
|
+
process_single_service
|
58
|
+
|
59
|
+
puts "Software shipped!" unless @options[:mute]
|
60
|
+
end
|
61
|
+
|
62
|
+
# A single service name shouold had been passed and the service automatically
|
63
|
+
# gets created if it does not exist.
|
64
|
+
def process_single_service
|
65
|
+
ecs_service = find_ecs_service
|
66
|
+
deployed_service = if ecs_service
|
67
|
+
# update all existing service
|
68
|
+
update_service(ecs_service)
|
69
|
+
else
|
70
|
+
# create service on the first cluster
|
71
|
+
create_service
|
72
|
+
end
|
73
|
+
|
74
|
+
wait_for_deployment(deployed_service) if @wait_for_deployment && !@options[:noop]
|
75
|
+
stop_old_task(deployed_service) if @stop_old_tasks
|
76
|
+
end
|
77
|
+
|
78
|
+
def process_multiple_services
|
79
|
+
puts "Multi services mode" unless @options[:mute]
|
80
|
+
services_to_deploy = []
|
81
|
+
find_all_ecs_services do |ecs_service|
|
82
|
+
if service_pattern_match?(ecs_service.service_name)
|
83
|
+
services_to_deploy << ecs_service
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
deployed_services = services_to_deploy.map do |ecs_service|
|
88
|
+
update_service(ecs_service)
|
89
|
+
end
|
90
|
+
|
91
|
+
wait_for_all_deployments(deployed_services) if @wait_for_deployment && !@options[:noop]
|
92
|
+
stop_old_tasks(deployed_services) if @stop_old_tasks
|
93
|
+
end
|
94
|
+
|
95
|
+
def service_tasks(cluster, service)
|
96
|
+
all_task_arns = ecs.list_tasks(cluster: cluster, service_name: service).task_arns
|
97
|
+
return [] if all_task_arns.empty?
|
98
|
+
ecs.describe_tasks(cluster: cluster, tasks: all_task_arns).tasks
|
99
|
+
end
|
100
|
+
|
101
|
+
def old_task?(deployed_task_definition_arn, task_definition_arn)
|
102
|
+
puts "deployed_task_definition_arn: #{deployed_task_definition_arn.inspect}"
|
103
|
+
puts "task_definition_arn: #{task_definition_arn.inspect}"
|
104
|
+
deployed_version = deployed_task_definition_arn.split(':').last.to_i
|
105
|
+
version = task_definition_arn.split(':').last.to_i
|
106
|
+
puts "deployed_version #{deployed_version.inspect}"
|
107
|
+
puts "version #{version.inspect}"
|
108
|
+
is_old = version < deployed_version
|
109
|
+
puts "is_old #{is_old.inspect}"
|
110
|
+
is_old
|
111
|
+
end
|
112
|
+
|
113
|
+
def stop_old_tasks(services)
|
114
|
+
services.each do |service|
|
115
|
+
stop_old_task(service)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
# aws ecs list-tasks --cluster prod-hi --service-name gr-web-prod
|
120
|
+
# aws ecs describe-tasks --tasks arn:aws:ecs:us-east-1:467446852200:task/09038fd2-f989-4903-a8c6-1bc41761f93f --cluster prod-hi
|
121
|
+
def stop_old_task(deployed_service)
|
122
|
+
deployed_task_definition_arn = deployed_service.task_definition
|
123
|
+
puts "deployed_task_definition_arn #{deployed_task_definition_arn.inspect}"
|
124
|
+
|
125
|
+
# cannot use @serivce because of multiple mode
|
126
|
+
all_tasks = service_tasks(@cluster, deployed_service.service_name)
|
127
|
+
old_tasks = all_tasks.select do |task|
|
128
|
+
old_task?(deployed_task_definition_arn, task.task_definition_arn)
|
129
|
+
end
|
130
|
+
|
131
|
+
reason = "Ufo #{Ufo::VERSION} has deployed new code and waited until the newer code is running."
|
132
|
+
puts reason
|
133
|
+
# Stopping old tasks after we have confirmed that the new task definition has the same
|
134
|
+
# number of desired_count and running_count speeds up clean up and ensure that we
|
135
|
+
# dont have any stale code being served. It seems to take a long time for the
|
136
|
+
# ELB to drain the register container otherwise. This might cut off some requests but
|
137
|
+
# providing this as an option that can be turned of beause I've seen deploys go way too
|
138
|
+
# slow.
|
139
|
+
puts "@options[:stop_old_tasks] #{@options[:stop_old_tasks].inspect}"
|
140
|
+
puts "old_tasks.size #{old_tasks.size}"
|
141
|
+
old_tasks.each do |task|
|
142
|
+
puts "stopping task.task_definition_arn #{task.task_definition_arn.inspect}"
|
143
|
+
ecs.stop_task(cluster: @cluster, task: task.task_arn, reason: reason)
|
144
|
+
end if @options[:stop_old_tasks]
|
145
|
+
end
|
146
|
+
|
147
|
+
# service is the returned object from aws-sdk not the @service which is just a String.
|
148
|
+
# Returns [service_name, time_took]
|
149
|
+
def wait_for_deployment(deployed_service, quiet=false)
|
150
|
+
start_time = Time.now
|
151
|
+
deployed_task_name = task_name(deployed_service.task_definition)
|
152
|
+
puts "Waiting for deployment of task definition #{deployed_task_name.green} to complete" unless quiet
|
153
|
+
begin
|
154
|
+
until deployment_complete(deployed_service)
|
155
|
+
print '.'
|
156
|
+
sleep 5
|
157
|
+
end
|
158
|
+
rescue ShipmentOverridden => e
|
159
|
+
puts "This deployed was overridden by another deploy"
|
160
|
+
puts e.message
|
161
|
+
end
|
162
|
+
puts '' unless quiet
|
163
|
+
took = Time.now - start_time
|
164
|
+
puts "Time waiting for ECS deployment: #{pretty_time(took).green}." unless quiet
|
165
|
+
[deployed_service.service_name, took]
|
166
|
+
end
|
167
|
+
|
168
|
+
def wait_for_all_deployments(deployed_services)
|
169
|
+
start_time = Time.now
|
170
|
+
threads = deployed_services.map do |deployed_service|
|
171
|
+
Thread.new do
|
172
|
+
# http://stackoverflow.com/questions/1383390/how-can-i-return-a-value-from-a-thread-in-ruby
|
173
|
+
Thread.current[:output] = wait_for_deployment(deployed_service, quiet=true)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
threads.each { |t| t.join }
|
177
|
+
total_took = Time.now - start_time
|
178
|
+
puts ""
|
179
|
+
puts "Shipments for all #{deployed_service.size} services took a total of #{pretty_time(total_took).green}."
|
180
|
+
puts "Each deployment took:"
|
181
|
+
threads.each do |t|
|
182
|
+
service_name, took = t[:output]
|
183
|
+
puts " #{service_name}: #{pretty_time(took)}"
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
# used for polling
|
188
|
+
# must pass in a service and cannot use @service for the case of multi_services mode
|
189
|
+
def find_updated_service(service)
|
190
|
+
ecs.describe_services(services: [service.service_name], cluster: @cluster).services.first
|
191
|
+
end
|
192
|
+
|
193
|
+
# aws ecs describe-services --services hi-web-prod --cluster prod-hi
|
194
|
+
# Passing in the service because we need to capture the deployed task_definition
|
195
|
+
# that was actually deployed. We use it to pull the describe_services
|
196
|
+
# until all the paramters we expect upon a completed deployment are updated.
|
197
|
+
#
|
198
|
+
def deployment_complete(deployed_service)
|
199
|
+
deployed_task_definition = deployed_service.task_definition # want the stale task_definition out of the wa
|
200
|
+
service = find_updated_service(deployed_service) # polling
|
201
|
+
deployment = service.deployments.first
|
202
|
+
# Edge case when another deploy superseds this deploy in this case break out of this loop
|
203
|
+
deployed_task_version = task_version(deployed_task_definition)
|
204
|
+
current_task_version = task_version(service.task_definition)
|
205
|
+
if current_task_version > deployed_task_version
|
206
|
+
raise ShipmentOverridden.new("deployed_task_version was #{deployed_task_version} but task_version is now #{current_task_version}")
|
207
|
+
end
|
208
|
+
|
209
|
+
(deployment.task_definition == deployed_task_definition &&
|
210
|
+
deployment.desired_count == deployment.running_count)
|
211
|
+
end
|
212
|
+
|
213
|
+
# $ aws ecs create-service --generate-cli-skeleton
|
214
|
+
# {
|
215
|
+
# "cluster": "",
|
216
|
+
# "serviceName": "",
|
217
|
+
# "taskDefinition": "",
|
218
|
+
# "desiredCount": 0,
|
219
|
+
# "loadBalancers": [
|
220
|
+
# {
|
221
|
+
# "targetGroupArn": "",
|
222
|
+
# "containerName": "",
|
223
|
+
# "containerPort": 0
|
224
|
+
# }
|
225
|
+
# ],
|
226
|
+
# "role": "",
|
227
|
+
# "clientToken": "",
|
228
|
+
# "deploymentConfiguration": {
|
229
|
+
# "maximumPercent": 0,
|
230
|
+
# "minimumHealthyPercent": 0
|
231
|
+
# }
|
232
|
+
# }
|
233
|
+
#
|
234
|
+
# If the service needs to be created it will get created with some default settings.
|
235
|
+
# When does a normal deploy where an update happens only the only thing that ufo
|
236
|
+
# will update is the task_definition. The other settings should normally be updated with
|
237
|
+
# the ECS console. `ufo scale` will allow you to updated the desired_count from the
|
238
|
+
# CLI though.
|
239
|
+
def create_service
|
240
|
+
container = container_info(@task_definition)
|
241
|
+
target_group = create_service_prompt(container)
|
242
|
+
|
243
|
+
message = "#{@service} service created on #{@cluster} cluster"
|
244
|
+
if @options[:noop]
|
245
|
+
message = "NOOP #{message}"
|
246
|
+
else
|
247
|
+
options = {
|
248
|
+
cluster: @cluster,
|
249
|
+
service_name: @service,
|
250
|
+
desired_count: default_desired_count,
|
251
|
+
deployment_configuration: {
|
252
|
+
maximum_percent: default_maximum_percent,
|
253
|
+
minimum_healthy_percent: default_minimum_healthy_percent
|
254
|
+
},
|
255
|
+
task_definition: @task_definition
|
256
|
+
}
|
257
|
+
unless target_group.nil? || target_group.empty?
|
258
|
+
add_load_balancer!(container, options)
|
259
|
+
end
|
260
|
+
response = ecs.create_service(options)
|
261
|
+
service = response.service # must set service here since this might never be called if @wait_for_deployment is false
|
262
|
+
end
|
263
|
+
puts message unless @options[:mute]
|
264
|
+
service
|
265
|
+
end
|
266
|
+
|
267
|
+
# $ aws ecs update-service --generate-cli-skeleton
|
268
|
+
# {
|
269
|
+
# "cluster": "",
|
270
|
+
# "service": "",
|
271
|
+
# "taskDefinition": "",
|
272
|
+
# "desiredCount": 0,
|
273
|
+
# "deploymentConfiguration": {
|
274
|
+
# "maximumPercent": 0,
|
275
|
+
# "minimumHealthyPercent": 0
|
276
|
+
# }
|
277
|
+
# }
|
278
|
+
# Only thing we want to change is the task-definition
|
279
|
+
def update_service(ecs_service)
|
280
|
+
message = "#{ecs_service.service_name} service updated on #{ecs_service.cluster_name} cluster with task #{@task_definition}"
|
281
|
+
if @options[:noop]
|
282
|
+
message = "NOOP #{message}"
|
283
|
+
else
|
284
|
+
response = ecs.update_service(
|
285
|
+
cluster: ecs_service.cluster_arn, # can use the cluster name also since it is unique
|
286
|
+
service: ecs_service.service_arn, # can use the service name also since it is unique
|
287
|
+
task_definition: @task_definition
|
288
|
+
)
|
289
|
+
service = response.service # must set service here since this might never be called if @wait_for_deployment is false
|
290
|
+
end
|
291
|
+
puts message unless @options[:mute]
|
292
|
+
service
|
293
|
+
end
|
294
|
+
|
295
|
+
# Only support Application Load Balancer
|
296
|
+
# Think there is an AWS bug that complains about not having the LB
|
297
|
+
# name but you cannot pass both a LB Name and a Target Group.
|
298
|
+
def add_load_balancer!(container, options)
|
299
|
+
options.merge!(
|
300
|
+
role: "ecsServiceRole", # assumption that we're using the ecsServiceRole
|
301
|
+
load_balancers: [
|
302
|
+
{
|
303
|
+
container_name: container[:name],
|
304
|
+
container_port: container[:port],
|
305
|
+
target_group_arn: @options[:target_group],
|
306
|
+
}
|
307
|
+
]
|
308
|
+
)
|
309
|
+
end
|
310
|
+
|
311
|
+
# Returns the target_group.
|
312
|
+
# Will only allow an target_group and the service to use a load balancer
|
313
|
+
# if the container name is "web".
|
314
|
+
def create_service_prompt(container)
|
315
|
+
return if @options[:noop]
|
316
|
+
return unless @elb_prompt
|
317
|
+
if container[:name] != "web" and @options[:target_group]
|
318
|
+
puts "WARNING: A --target-group #{@options[:target_group]} was provided but it will not be used because this not a web container. Container name: #{container[:name].inspect}."
|
319
|
+
end
|
320
|
+
return unless container[:name] == 'web'
|
321
|
+
return @options[:target_group] if @options[:target_group]
|
322
|
+
|
323
|
+
puts "This service #{@service} does not yet exist in the #{@cluster} cluster. This deploy will create it."
|
324
|
+
puts "Would you like this service to be associated with an Application Load Balancer?"
|
325
|
+
puts "If yes, please provide the Application Load Balancer Target Group ARN."
|
326
|
+
puts "If no, simply press enter."
|
327
|
+
print "Target Group ARN: "
|
328
|
+
|
329
|
+
arn = $stdin.gets.strip
|
330
|
+
until arn == '' or validate_target_group(arn)
|
331
|
+
puts "You have provided an invalid Application Load Balancer Target Group ARN: #{arn}."
|
332
|
+
puts "It should be in the form: arn:aws:elasticloadbalancing:us-east-1:123456789:targetgroup/target-name/2378947392743"
|
333
|
+
puts "Please try again or skip adding a Target Group by just pressing enter."
|
334
|
+
print "Target Group ARN: "
|
335
|
+
arn = $stdin.gets.strip
|
336
|
+
end
|
337
|
+
arn
|
338
|
+
end
|
339
|
+
|
340
|
+
def validate_target_group(arn)
|
341
|
+
elb.describe_target_groups(target_group_arns: [arn])
|
342
|
+
true
|
343
|
+
rescue Aws::ElasticLoadBalancingV2::Errors::ValidationError
|
344
|
+
false
|
345
|
+
end
|
346
|
+
|
347
|
+
# assume only 1 container_definition
|
348
|
+
# assume only 1 port mapping in that container_defintion
|
349
|
+
def container_info(task_definition)
|
350
|
+
task_definition_path = "ufo/output/#{task_definition}.json"
|
351
|
+
task_definition_full_path = "#{@project_root}/#{task_definition_path}"
|
352
|
+
unless File.exist?(task_definition_full_path)
|
353
|
+
puts "ERROR: Unable to find the task definition at #{task_definition_path}."
|
354
|
+
puts "Are you sure you have defined it in ufo/template_definitions.rb?"
|
355
|
+
exit
|
356
|
+
end
|
357
|
+
task_definition = JSON.load(IO.read(task_definition_full_path))
|
358
|
+
container_def = task_definition["containerDefinitions"].first
|
359
|
+
mappings = container_def["portMappings"]
|
360
|
+
if mappings
|
361
|
+
map = mappings.first
|
362
|
+
port = map["containerPort"]
|
363
|
+
end
|
364
|
+
{
|
365
|
+
name: container_def["name"],
|
366
|
+
port: port
|
367
|
+
}
|
368
|
+
end
|
369
|
+
|
370
|
+
def service_exact_match?(service_name)
|
371
|
+
service_name == @service
|
372
|
+
end
|
373
|
+
|
374
|
+
# @service is changed to a pattern to be search with.
|
375
|
+
#
|
376
|
+
# Examples:
|
377
|
+
# @service == "hi-.*-prod"
|
378
|
+
def service_pattern_match?(service_name)
|
379
|
+
service_patttern = Regexp.new(@service)
|
380
|
+
service_name =~ service_patttern
|
381
|
+
end
|
382
|
+
|
383
|
+
def find_ecs_service
|
384
|
+
find_all_ecs_services.find { |ecs_service| ecs_service.service_name == @service }
|
385
|
+
end
|
386
|
+
|
387
|
+
# find all services on a cluster
|
388
|
+
# yields ECS::Service object
|
389
|
+
def find_all_ecs_services
|
390
|
+
ecs_services = []
|
391
|
+
service_arns.each do |service_arn|
|
392
|
+
ecs_service = ECS::Service.new(cluster_arn, service_arn)
|
393
|
+
yield(ecs_service) if block_given?
|
394
|
+
ecs_services << ecs_service
|
395
|
+
end
|
396
|
+
ecs_services
|
397
|
+
end
|
398
|
+
|
399
|
+
def service_arns
|
400
|
+
ecs.list_services(cluster: @cluster).service_arns
|
401
|
+
end
|
402
|
+
|
403
|
+
def cluster_arn
|
404
|
+
@cluster_arn ||= ecs_clusters.first.cluster_arn
|
405
|
+
end
|
406
|
+
|
407
|
+
def ensure_cluster_exist
|
408
|
+
cluster_exist = ecs_clusters.first
|
409
|
+
unless cluster_exist
|
410
|
+
message = "#{@cluster} cluster created."
|
411
|
+
if @options[:noop]
|
412
|
+
message = "NOOP #{message}"
|
413
|
+
else
|
414
|
+
ecs.create_cluster(cluster_name: @cluster)
|
415
|
+
end
|
416
|
+
puts message unless @options[:mute]
|
417
|
+
end
|
418
|
+
end
|
419
|
+
|
420
|
+
def ecs_clusters
|
421
|
+
ecs.describe_clusters(clusters: [@cluster]).clusters
|
422
|
+
end
|
423
|
+
|
424
|
+
def task_name(task_definition)
|
425
|
+
# "arn:aws:ecs:us-east-1:123456789:task-definition/hi-web-prod:72"
|
426
|
+
# ->
|
427
|
+
# "task-definition/hi-web-prod:72"
|
428
|
+
task_definition.split('/').last
|
429
|
+
end
|
430
|
+
|
431
|
+
def task_version(task_definition)
|
432
|
+
# "task-definition/hi-web-prod:72" -> 72
|
433
|
+
task_name(task_definition).split(':').last.to_i
|
434
|
+
end
|
435
|
+
end
|
436
|
+
end
|