ufo 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.rspec +3 -0
- data/CHANGELOG.md +12 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +128 -0
- data/Guardfile +12 -0
- data/LICENSE.txt +22 -0
- data/README.md +227 -0
- data/Rakefile +6 -0
- data/bin/ufo +10 -0
- data/lib/starter_project/.env.prod +3 -0
- data/lib/starter_project/Dockerfile +20 -0
- data/lib/starter_project/bin/deploy +12 -0
- data/lib/starter_project/ufo/settings.yml +7 -0
- data/lib/starter_project/ufo/task_definitions.rb +44 -0
- data/lib/starter_project/ufo/templates/main.json.erb +39 -0
- data/lib/ufo.rb +26 -0
- data/lib/ufo/aws_services.rb +21 -0
- data/lib/ufo/cli.rb +136 -0
- data/lib/ufo/cli/help.rb +164 -0
- data/lib/ufo/defaults.rb +46 -0
- data/lib/ufo/destroy.rb +60 -0
- data/lib/ufo/docker_builder.rb +120 -0
- data/lib/ufo/docker_cleaner.rb +52 -0
- data/lib/ufo/dockerfile_updater.rb +42 -0
- data/lib/ufo/dsl.rb +93 -0
- data/lib/ufo/dsl/helper.rb +47 -0
- data/lib/ufo/dsl/outputter.rb +40 -0
- data/lib/ufo/dsl/task_definition.rb +65 -0
- data/lib/ufo/ecr_auth.rb +35 -0
- data/lib/ufo/ecr_cleaner.rb +66 -0
- data/lib/ufo/execute.rb +19 -0
- data/lib/ufo/init.rb +83 -0
- data/lib/ufo/pretty_time.rb +14 -0
- data/lib/ufo/scale.rb +34 -0
- data/lib/ufo/settings.rb +33 -0
- data/lib/ufo/ship.rb +436 -0
- data/lib/ufo/tasks_builder.rb +30 -0
- data/lib/ufo/tasks_register.rb +51 -0
- data/lib/ufo/templates/default.json.erb +39 -0
- data/lib/ufo/version.rb +3 -0
- data/spec/fixtures/home_existing/.docker/config.json +10 -0
- data/spec/lib/cli_spec.rb +50 -0
- data/spec/lib/ecr_auth_spec.rb +39 -0
- data/spec/lib/ecr_cleaner_spec.rb +32 -0
- data/spec/lib/ship_spec.rb +77 -0
- data/spec/spec_helper.rb +28 -0
- data/ufo.gemspec +34 -0
- metadata +267 -0
@@ -0,0 +1,40 @@
|
|
1
|
+
module Ufo
|
2
|
+
class DSL
|
3
|
+
class Outputter
|
4
|
+
def initialize(name, erb_result, options={})
|
5
|
+
@name = name
|
6
|
+
@erb_result = erb_result
|
7
|
+
@options = options
|
8
|
+
@pretty = options[:pretty].nil? ? true : options[:pretty]
|
9
|
+
end
|
10
|
+
|
11
|
+
def write
|
12
|
+
output_path = "#{@options[:project_root]}/ufo/output"
|
13
|
+
FileUtils.rm_rf(output_path) if @options[:clean]
|
14
|
+
FileUtils.mkdir(output_path) unless File.exist?(output_path)
|
15
|
+
|
16
|
+
path = "#{output_path}/#{@name}.json"
|
17
|
+
puts "Generated task definition at: #{path}" unless @options[:quiet]
|
18
|
+
validate(@erb_result, path)
|
19
|
+
json = @pretty ?
|
20
|
+
JSON.pretty_generate(JSON.parse(@erb_result)) :
|
21
|
+
@erb_result
|
22
|
+
File.open(path, 'w') {|f| f.write(output_json(json)) }
|
23
|
+
end
|
24
|
+
|
25
|
+
def validate(json, path)
|
26
|
+
begin
|
27
|
+
JSON.parse(json)
|
28
|
+
rescue JSON::ParserError => e
|
29
|
+
puts "Invalid json. Output written to #{path} for debugging".colorize(:red)
|
30
|
+
File.open(path, 'w') {|f| f.write(json) }
|
31
|
+
exit 1
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def output_json(json)
|
36
|
+
@options[:pretty] ? JSON.pretty_generate(JSON.parse(json)) : json
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require "erb"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
module Ufo
|
5
|
+
class DSL
|
6
|
+
class TaskDefinition
|
7
|
+
attr_reader :task_definition_name
|
8
|
+
def initialize(dsl, task_definition_name, options={}, &block)
|
9
|
+
@dsl = dsl
|
10
|
+
@task_definition_name = task_definition_name
|
11
|
+
@block = block
|
12
|
+
@options = options
|
13
|
+
@project_root = @options[:project_root] || '.'
|
14
|
+
end
|
15
|
+
|
16
|
+
# delegate helper method back up to dsl
|
17
|
+
def helper
|
18
|
+
@dsl.helper
|
19
|
+
end
|
20
|
+
|
21
|
+
def build
|
22
|
+
instance_eval(&@block)
|
23
|
+
erb_template = IO.read(source_path)
|
24
|
+
ERB.new(erb_template).result(binding)
|
25
|
+
end
|
26
|
+
|
27
|
+
# at this point instance_eval has been called and source has possibly been called
|
28
|
+
def source(name)
|
29
|
+
@source = name
|
30
|
+
end
|
31
|
+
|
32
|
+
def variables(vars={})
|
33
|
+
vars.each do |var,value|
|
34
|
+
if instance_variable_defined?("@#{var}")
|
35
|
+
puts "WARNING: The instance variable @#{var} is already used internally with ufo. Please name you variable another name!"
|
36
|
+
end
|
37
|
+
instance_variable_set("@#{var}", value)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def source_path
|
42
|
+
if @source # this means that source has been called
|
43
|
+
path = "#{@project_root}/ufo/templates/#{@source}.json.erb"
|
44
|
+
check_source_path(path)
|
45
|
+
else
|
46
|
+
# default source path
|
47
|
+
path = File.expand_path("../../templates/default.json.erb", __FILE__)
|
48
|
+
puts "#{task_definition_name} template definition using default template: #{path}" unless @options[:mute]
|
49
|
+
end
|
50
|
+
path
|
51
|
+
end
|
52
|
+
|
53
|
+
def check_source_path(path)
|
54
|
+
unless File.exist?(path)
|
55
|
+
friendly_path = path.sub("#{@project_root}/", '')
|
56
|
+
puts "ERROR: Could not find the #{friendly_path} template. Are sure it exists? Check where you called source in ufo/task_definitions.rb"
|
57
|
+
exit 1
|
58
|
+
else
|
59
|
+
puts "#{task_definition_name} template definition using project template: #{path}" unless @options[:mute]
|
60
|
+
end
|
61
|
+
path
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/lib/ufo/ecr_auth.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
module Ufo
|
2
|
+
class EcrAuth
|
3
|
+
include AwsServices
|
4
|
+
|
5
|
+
def initialize(repo_domain)
|
6
|
+
@repo_domain = repo_domain
|
7
|
+
end
|
8
|
+
|
9
|
+
def update
|
10
|
+
auth_token = fetch_auth_token
|
11
|
+
if File.exist?(docker_config)
|
12
|
+
data = JSON.load(IO.read(docker_config))
|
13
|
+
data["auths"][@repo_domain] = {auth: auth_token}
|
14
|
+
else
|
15
|
+
data = {auths: {@repo_domain => {auth: auth_token}}}
|
16
|
+
end
|
17
|
+
ensure_dotdocker_exists
|
18
|
+
IO.write(docker_config, JSON.pretty_generate(data))
|
19
|
+
end
|
20
|
+
|
21
|
+
def fetch_auth_token
|
22
|
+
ecr.get_authorization_token.authorization_data.first.authorization_token
|
23
|
+
end
|
24
|
+
|
25
|
+
def docker_config
|
26
|
+
"#{ENV['HOME']}/.docker/config.json"
|
27
|
+
end
|
28
|
+
|
29
|
+
def ensure_dotdocker_exists
|
30
|
+
dirname = File.dirname(docker_config)
|
31
|
+
FileUtils.mkdir_p(dirname) unless File.exist?(dirname)
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require "json"
|
2
|
+
|
3
|
+
# Can test:
|
4
|
+
#
|
5
|
+
# ufo ship app-web --cluster prod --noop
|
6
|
+
module Ufo
|
7
|
+
class EcrCleaner
|
8
|
+
include AwsServices
|
9
|
+
include Defaults
|
10
|
+
|
11
|
+
def initialize(docker_image_name, options={})
|
12
|
+
# docker_image_name does not containg the tag
|
13
|
+
# Example: 123456789.dkr.ecr.us-east-1.amazonaws.com/image
|
14
|
+
@docker_image_name = docker_image_name
|
15
|
+
@options = options
|
16
|
+
@keep = options[:ecr_keep] || settings.data["ecr_keep"]
|
17
|
+
@tag_prefix = options[:tag_prefix] || "ufo"
|
18
|
+
end
|
19
|
+
|
20
|
+
def cleanup
|
21
|
+
return false unless ecr_image?
|
22
|
+
return false unless @keep
|
23
|
+
update_auth_token
|
24
|
+
image_tags = fetch_image_tags
|
25
|
+
delete_tags = image_tags[@keep..-1] # ordered by most recent images first
|
26
|
+
delete_images(delete_tags)
|
27
|
+
end
|
28
|
+
|
29
|
+
def fetch_image_tags
|
30
|
+
ecr.list_images(repository_name: repo_name).
|
31
|
+
image_ids.
|
32
|
+
map { |image_id| image_id.image_tag }.
|
33
|
+
select { |image_tag| image_tag =~ Regexp.new("^#{@tag_prefix}-") }.
|
34
|
+
sort.reverse
|
35
|
+
end
|
36
|
+
|
37
|
+
def delete_images(tags)
|
38
|
+
return if tags.nil? || tags.empty?
|
39
|
+
unless @options[:mute]
|
40
|
+
puts "Keeping #{@keep} most recent ECR images."
|
41
|
+
puts "Deleting these ECR images:"
|
42
|
+
tag_list = tags.map { |t| " #{repo_name}:#{t}" }
|
43
|
+
puts tag_list
|
44
|
+
end
|
45
|
+
image_ids = tags.map { |tag| {image_tag: tag} }
|
46
|
+
ecr.batch_delete_image(
|
47
|
+
repository_name: repo_name,
|
48
|
+
image_ids: image_ids) unless @options[:noop]
|
49
|
+
end
|
50
|
+
|
51
|
+
def update_auth_token
|
52
|
+
repo_domain = "https://#{@docker_image_name.split('/').first}"
|
53
|
+
auth = EcrAuth.new(repo_domain)
|
54
|
+
auth.update
|
55
|
+
end
|
56
|
+
|
57
|
+
def repo_name
|
58
|
+
# @docker_image_name example: 123456789.dkr.ecr.us-east-1.amazonaws.com/image
|
59
|
+
@docker_image_name.split('/').last
|
60
|
+
end
|
61
|
+
|
62
|
+
def ecr_image?
|
63
|
+
@docker_image_name =~ /\.amazonaws\.com/
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
data/lib/ufo/execute.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
module Ufo
|
2
|
+
module Execute
|
3
|
+
def execute(command, local_options={})
|
4
|
+
command = "cd #{@project_root} && #{command}"
|
5
|
+
# local_options[:live] overrides the global @options[:noop]
|
6
|
+
if @options[:noop] && !local_options[:live]
|
7
|
+
say "NOOP: #{command}"
|
8
|
+
result = true # always success with no noop for specs
|
9
|
+
else
|
10
|
+
if local_options[:use_system]
|
11
|
+
result = system(command)
|
12
|
+
else
|
13
|
+
result = `#{command}`
|
14
|
+
end
|
15
|
+
end
|
16
|
+
result
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/ufo/init.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'colorize'
|
2
|
+
require 'erb'
|
3
|
+
|
4
|
+
# quick test:
|
5
|
+
# rm -rf ufo ; /Users/tung/src/tongueroo/ufo/bin/ufo init --cluster default2 --image tongueroo/ruby ; cat ufo/settings.yml
|
6
|
+
module Ufo
|
7
|
+
class Init
|
8
|
+
def initialize(options = {})
|
9
|
+
@options = options
|
10
|
+
@project_root = options[:project_root] || '.'
|
11
|
+
end
|
12
|
+
|
13
|
+
def setup
|
14
|
+
puts "Setting up ufo project...".blue unless @options[:quiet]
|
15
|
+
source_root = File.expand_path("../../starter_project", __FILE__)
|
16
|
+
# https://ruby-doc.org/core-2.2.0/Dir.html
|
17
|
+
# use the File::FNM_DOTMATCH flag or something like "{*,.*}".
|
18
|
+
paths = Dir.glob("#{source_root}/**/{*,.*}").
|
19
|
+
select {|p| File.file?(p) }
|
20
|
+
paths.each do |src|
|
21
|
+
dest = src.gsub(%r{.*starter_project/},'')
|
22
|
+
dest = "#{@project_root}/#{dest}"
|
23
|
+
|
24
|
+
if File.exist?(dest) and !@options[:force]
|
25
|
+
puts "exists: #{dest}".yellow unless @options[:quiet]
|
26
|
+
else
|
27
|
+
dirname = File.dirname(dest)
|
28
|
+
FileUtils.mkdir_p(dirname) unless File.exist?(dirname)
|
29
|
+
if dest =~ /\.erb$/
|
30
|
+
FileUtils.cp(src, dest)
|
31
|
+
else
|
32
|
+
write_erb_result(src, dest)
|
33
|
+
end
|
34
|
+
puts "created: #{dest}".green unless @options[:quiet]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
puts "Starter ufo files created.".blue
|
38
|
+
File.chmod(0755, "#{@project_root}/bin/deploy")
|
39
|
+
add_gitignore
|
40
|
+
end
|
41
|
+
|
42
|
+
def write_erb_result(src, dest)
|
43
|
+
source = IO.read(src)
|
44
|
+
b = ERBContext.new(@options).get_binding
|
45
|
+
output = ERB.new(source).result(b)
|
46
|
+
IO.write(dest, output)
|
47
|
+
end
|
48
|
+
|
49
|
+
def add_gitignore
|
50
|
+
gitignore_path = "#{@project_root}/.gitignore"
|
51
|
+
if File.exist?(gitignore_path)
|
52
|
+
ignores = IO.read(gitignore_path)
|
53
|
+
has_ignore = ignores.include?("ufo/output")
|
54
|
+
ignores << ufo_ignores unless has_ignore
|
55
|
+
else
|
56
|
+
ignores = ufo_ignores
|
57
|
+
end
|
58
|
+
IO.write(gitignore_path, ignores)
|
59
|
+
end
|
60
|
+
|
61
|
+
def ufo_ignores
|
62
|
+
ignores =<<-EOL
|
63
|
+
/ufo/output
|
64
|
+
/ufo/docker_image_name*.txt
|
65
|
+
/ufo/version
|
66
|
+
EOL
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# http://stackoverflow.com/questions/1338960/ruby-templates-how-to-pass-variables-into-inlined-erb
|
73
|
+
class ERBContext
|
74
|
+
def initialize(hash)
|
75
|
+
hash.each_pair do |key, value|
|
76
|
+
instance_variable_set('@' + key.to_s, value)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def get_binding
|
81
|
+
binding
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Ufo
|
2
|
+
module PrettyTime
|
3
|
+
# http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
|
4
|
+
def pretty_time(total_seconds)
|
5
|
+
minutes = (total_seconds / 60) % 60
|
6
|
+
seconds = total_seconds % 60
|
7
|
+
if total_seconds < 60
|
8
|
+
"#{seconds.to_i}s"
|
9
|
+
else
|
10
|
+
"#{minutes.to_i}m #{seconds.to_i}s"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
data/lib/ufo/scale.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
module Ufo
|
2
|
+
class Scale
|
3
|
+
include Defaults
|
4
|
+
include AwsServices
|
5
|
+
|
6
|
+
def initialize(service, count, options={})
|
7
|
+
@service = service
|
8
|
+
@count = count
|
9
|
+
@options = options
|
10
|
+
@cluster = @options[:cluster] || default_cluster
|
11
|
+
end
|
12
|
+
|
13
|
+
def update
|
14
|
+
unless service_exists?
|
15
|
+
puts "Unable to find the #{@service} service on #{@cluster} cluster."
|
16
|
+
puts "Are you sure you are trying to scale the right service on the right cluster?"
|
17
|
+
exit
|
18
|
+
end
|
19
|
+
ecs.update_service(
|
20
|
+
service: @service,
|
21
|
+
cluster: @cluster,
|
22
|
+
desired_count: @count
|
23
|
+
)
|
24
|
+
puts "Scale #{@service} service in #{@cluster} cluster to #{@count}" unless @options[:mute]
|
25
|
+
end
|
26
|
+
|
27
|
+
def service_exists?
|
28
|
+
cluster = ecs.describe_clusters(clusters: [@cluster]).clusters.first
|
29
|
+
return false unless cluster
|
30
|
+
service = ecs.describe_services(services: [@service], cluster: @cluster).services.first
|
31
|
+
!!service
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
data/lib/ufo/settings.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Ufo
|
4
|
+
class Settings
|
5
|
+
def initialize(project_root='.')
|
6
|
+
@project_root = project_root
|
7
|
+
end
|
8
|
+
|
9
|
+
# data contains the settings.yml config. The order or precedence for settings
|
10
|
+
# is the project ufo/settings.yml and then the ~/.ufo/settings.yml.
|
11
|
+
def data
|
12
|
+
return @data if @data
|
13
|
+
|
14
|
+
if File.exist?(settings_path)
|
15
|
+
@data = YAML.load_file(settings_path)
|
16
|
+
@data = user_settings.merge(@data)
|
17
|
+
else
|
18
|
+
puts "ERROR: No settings file file at #{settings_path}"
|
19
|
+
puts "Please create a settings file via: ufo init"
|
20
|
+
exit 1
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def user_settings
|
25
|
+
path = "#{ENV['HOME']}/.ufo/settings.yml"
|
26
|
+
File.exist?(path) ? YAML.load_file(path) : {}
|
27
|
+
end
|
28
|
+
|
29
|
+
def settings_path
|
30
|
+
"#{@project_root}/ufo/settings.yml"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/ufo/ship.rb
ADDED
@@ -0,0 +1,436 @@
|
|
1
|
+
require 'colorize'
|
2
|
+
|
3
|
+
module Ufo
|
4
|
+
# Creating this class pass so we can have a reference we do not have describe
|
5
|
+
# all services and look up service_name creating more API calls.
|
6
|
+
#
|
7
|
+
# Also this class allows us to pass one object around instead of both
|
8
|
+
# cluster_name and service_name.
|
9
|
+
module ECS
|
10
|
+
Service = Struct.new(:cluster_arn, :service_arn) do
|
11
|
+
def cluster_name
|
12
|
+
cluster_arn.split('/').last
|
13
|
+
end
|
14
|
+
|
15
|
+
def service_name
|
16
|
+
service_arn.split('/').last
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class UfoError < RuntimeError; end
|
22
|
+
class ShipmentOverridden < UfoError; end
|
23
|
+
|
24
|
+
class Ship
|
25
|
+
include Defaults
|
26
|
+
include AwsServices
|
27
|
+
include PrettyTime
|
28
|
+
|
29
|
+
# service can be a pattern
|
30
|
+
def initialize(service, task_definition, options={})
|
31
|
+
@service = service
|
32
|
+
@task_definition = task_definition
|
33
|
+
@options = options
|
34
|
+
@project_root = options[:project_root] || '.'
|
35
|
+
@elb_prompt = @options[:elb_prompt].nil? ? true : @options[:elb_prompt]
|
36
|
+
@cluster = @options[:cluster] || default_cluster
|
37
|
+
@wait_for_deployment = @options[:wait].nil? ? true : @options[:wait]
|
38
|
+
@stop_old_tasks = @options[:stop_old_tasks].nil? ? false : @options[:stop_old_tasks]
|
39
|
+
end
|
40
|
+
|
41
|
+
# Find all the matching services on a cluster and update them.
|
42
|
+
# If no service is found at all, then create a service on the very first specified cluster
|
43
|
+
# only.
|
44
|
+
#
|
45
|
+
# If it looks like one service is passed in then it'll automatically create the service
|
46
|
+
# on the first cluster.
|
47
|
+
|
48
|
+
# If it looks like a regexp is passed in then it'll only update the services
|
49
|
+
# This is because regpex cannot be used to determined a list of service_names.
|
50
|
+
#
|
51
|
+
# Example:
|
52
|
+
# No way to map: hi-.*-prod -> hi-web-prod hi-worker-prod hi-clock-prod
|
53
|
+
def deploy
|
54
|
+
puts "Shipping #{@service}...".green unless @options[:mute]
|
55
|
+
|
56
|
+
ensure_cluster_exist
|
57
|
+
process_single_service
|
58
|
+
|
59
|
+
puts "Software shipped!" unless @options[:mute]
|
60
|
+
end
|
61
|
+
|
62
|
+
# A single service name shouold had been passed and the service automatically
|
63
|
+
# gets created if it does not exist.
|
64
|
+
def process_single_service
|
65
|
+
ecs_service = find_ecs_service
|
66
|
+
deployed_service = if ecs_service
|
67
|
+
# update all existing service
|
68
|
+
update_service(ecs_service)
|
69
|
+
else
|
70
|
+
# create service on the first cluster
|
71
|
+
create_service
|
72
|
+
end
|
73
|
+
|
74
|
+
wait_for_deployment(deployed_service) if @wait_for_deployment && !@options[:noop]
|
75
|
+
stop_old_task(deployed_service) if @stop_old_tasks
|
76
|
+
end
|
77
|
+
|
78
|
+
def process_multiple_services
|
79
|
+
puts "Multi services mode" unless @options[:mute]
|
80
|
+
services_to_deploy = []
|
81
|
+
find_all_ecs_services do |ecs_service|
|
82
|
+
if service_pattern_match?(ecs_service.service_name)
|
83
|
+
services_to_deploy << ecs_service
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
deployed_services = services_to_deploy.map do |ecs_service|
|
88
|
+
update_service(ecs_service)
|
89
|
+
end
|
90
|
+
|
91
|
+
wait_for_all_deployments(deployed_services) if @wait_for_deployment && !@options[:noop]
|
92
|
+
stop_old_tasks(deployed_services) if @stop_old_tasks
|
93
|
+
end
|
94
|
+
|
95
|
+
def service_tasks(cluster, service)
|
96
|
+
all_task_arns = ecs.list_tasks(cluster: cluster, service_name: service).task_arns
|
97
|
+
return [] if all_task_arns.empty?
|
98
|
+
ecs.describe_tasks(cluster: cluster, tasks: all_task_arns).tasks
|
99
|
+
end
|
100
|
+
|
101
|
+
def old_task?(deployed_task_definition_arn, task_definition_arn)
|
102
|
+
puts "deployed_task_definition_arn: #{deployed_task_definition_arn.inspect}"
|
103
|
+
puts "task_definition_arn: #{task_definition_arn.inspect}"
|
104
|
+
deployed_version = deployed_task_definition_arn.split(':').last.to_i
|
105
|
+
version = task_definition_arn.split(':').last.to_i
|
106
|
+
puts "deployed_version #{deployed_version.inspect}"
|
107
|
+
puts "version #{version.inspect}"
|
108
|
+
is_old = version < deployed_version
|
109
|
+
puts "is_old #{is_old.inspect}"
|
110
|
+
is_old
|
111
|
+
end
|
112
|
+
|
113
|
+
def stop_old_tasks(services)
|
114
|
+
services.each do |service|
|
115
|
+
stop_old_task(service)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
# aws ecs list-tasks --cluster prod-hi --service-name gr-web-prod
|
120
|
+
# aws ecs describe-tasks --tasks arn:aws:ecs:us-east-1:467446852200:task/09038fd2-f989-4903-a8c6-1bc41761f93f --cluster prod-hi
|
121
|
+
def stop_old_task(deployed_service)
|
122
|
+
deployed_task_definition_arn = deployed_service.task_definition
|
123
|
+
puts "deployed_task_definition_arn #{deployed_task_definition_arn.inspect}"
|
124
|
+
|
125
|
+
# cannot use @serivce because of multiple mode
|
126
|
+
all_tasks = service_tasks(@cluster, deployed_service.service_name)
|
127
|
+
old_tasks = all_tasks.select do |task|
|
128
|
+
old_task?(deployed_task_definition_arn, task.task_definition_arn)
|
129
|
+
end
|
130
|
+
|
131
|
+
reason = "Ufo #{Ufo::VERSION} has deployed new code and waited until the newer code is running."
|
132
|
+
puts reason
|
133
|
+
# Stopping old tasks after we have confirmed that the new task definition has the same
|
134
|
+
# number of desired_count and running_count speeds up clean up and ensure that we
|
135
|
+
# dont have any stale code being served. It seems to take a long time for the
|
136
|
+
# ELB to drain the register container otherwise. This might cut off some requests but
|
137
|
+
# providing this as an option that can be turned of beause I've seen deploys go way too
|
138
|
+
# slow.
|
139
|
+
puts "@options[:stop_old_tasks] #{@options[:stop_old_tasks].inspect}"
|
140
|
+
puts "old_tasks.size #{old_tasks.size}"
|
141
|
+
old_tasks.each do |task|
|
142
|
+
puts "stopping task.task_definition_arn #{task.task_definition_arn.inspect}"
|
143
|
+
ecs.stop_task(cluster: @cluster, task: task.task_arn, reason: reason)
|
144
|
+
end if @options[:stop_old_tasks]
|
145
|
+
end
|
146
|
+
|
147
|
+
# service is the returned object from aws-sdk not the @service which is just a String.
|
148
|
+
# Returns [service_name, time_took]
|
149
|
+
def wait_for_deployment(deployed_service, quiet=false)
|
150
|
+
start_time = Time.now
|
151
|
+
deployed_task_name = task_name(deployed_service.task_definition)
|
152
|
+
puts "Waiting for deployment of task definition #{deployed_task_name.green} to complete" unless quiet
|
153
|
+
begin
|
154
|
+
until deployment_complete(deployed_service)
|
155
|
+
print '.'
|
156
|
+
sleep 5
|
157
|
+
end
|
158
|
+
rescue ShipmentOverridden => e
|
159
|
+
puts "This deployed was overridden by another deploy"
|
160
|
+
puts e.message
|
161
|
+
end
|
162
|
+
puts '' unless quiet
|
163
|
+
took = Time.now - start_time
|
164
|
+
puts "Time waiting for ECS deployment: #{pretty_time(took).green}." unless quiet
|
165
|
+
[deployed_service.service_name, took]
|
166
|
+
end
|
167
|
+
|
168
|
+
def wait_for_all_deployments(deployed_services)
|
169
|
+
start_time = Time.now
|
170
|
+
threads = deployed_services.map do |deployed_service|
|
171
|
+
Thread.new do
|
172
|
+
# http://stackoverflow.com/questions/1383390/how-can-i-return-a-value-from-a-thread-in-ruby
|
173
|
+
Thread.current[:output] = wait_for_deployment(deployed_service, quiet=true)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
threads.each { |t| t.join }
|
177
|
+
total_took = Time.now - start_time
|
178
|
+
puts ""
|
179
|
+
puts "Shipments for all #{deployed_service.size} services took a total of #{pretty_time(total_took).green}."
|
180
|
+
puts "Each deployment took:"
|
181
|
+
threads.each do |t|
|
182
|
+
service_name, took = t[:output]
|
183
|
+
puts " #{service_name}: #{pretty_time(took)}"
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
# used for polling
|
188
|
+
# must pass in a service and cannot use @service for the case of multi_services mode
|
189
|
+
def find_updated_service(service)
|
190
|
+
ecs.describe_services(services: [service.service_name], cluster: @cluster).services.first
|
191
|
+
end
|
192
|
+
|
193
|
+
# aws ecs describe-services --services hi-web-prod --cluster prod-hi
|
194
|
+
# Passing in the service because we need to capture the deployed task_definition
|
195
|
+
# that was actually deployed. We use it to pull the describe_services
|
196
|
+
# until all the paramters we expect upon a completed deployment are updated.
|
197
|
+
#
|
198
|
+
def deployment_complete(deployed_service)
|
199
|
+
deployed_task_definition = deployed_service.task_definition # want the stale task_definition out of the wa
|
200
|
+
service = find_updated_service(deployed_service) # polling
|
201
|
+
deployment = service.deployments.first
|
202
|
+
# Edge case when another deploy superseds this deploy in this case break out of this loop
|
203
|
+
deployed_task_version = task_version(deployed_task_definition)
|
204
|
+
current_task_version = task_version(service.task_definition)
|
205
|
+
if current_task_version > deployed_task_version
|
206
|
+
raise ShipmentOverridden.new("deployed_task_version was #{deployed_task_version} but task_version is now #{current_task_version}")
|
207
|
+
end
|
208
|
+
|
209
|
+
(deployment.task_definition == deployed_task_definition &&
|
210
|
+
deployment.desired_count == deployment.running_count)
|
211
|
+
end
|
212
|
+
|
213
|
+
# $ aws ecs create-service --generate-cli-skeleton
|
214
|
+
# {
|
215
|
+
# "cluster": "",
|
216
|
+
# "serviceName": "",
|
217
|
+
# "taskDefinition": "",
|
218
|
+
# "desiredCount": 0,
|
219
|
+
# "loadBalancers": [
|
220
|
+
# {
|
221
|
+
# "targetGroupArn": "",
|
222
|
+
# "containerName": "",
|
223
|
+
# "containerPort": 0
|
224
|
+
# }
|
225
|
+
# ],
|
226
|
+
# "role": "",
|
227
|
+
# "clientToken": "",
|
228
|
+
# "deploymentConfiguration": {
|
229
|
+
# "maximumPercent": 0,
|
230
|
+
# "minimumHealthyPercent": 0
|
231
|
+
# }
|
232
|
+
# }
|
233
|
+
#
|
234
|
+
# If the service needs to be created it will get created with some default settings.
|
235
|
+
# When does a normal deploy where an update happens only the only thing that ufo
|
236
|
+
# will update is the task_definition. The other settings should normally be updated with
|
237
|
+
# the ECS console. `ufo scale` will allow you to updated the desired_count from the
|
238
|
+
# CLI though.
|
239
|
+
def create_service
|
240
|
+
container = container_info(@task_definition)
|
241
|
+
target_group = create_service_prompt(container)
|
242
|
+
|
243
|
+
message = "#{@service} service created on #{@cluster} cluster"
|
244
|
+
if @options[:noop]
|
245
|
+
message = "NOOP #{message}"
|
246
|
+
else
|
247
|
+
options = {
|
248
|
+
cluster: @cluster,
|
249
|
+
service_name: @service,
|
250
|
+
desired_count: default_desired_count,
|
251
|
+
deployment_configuration: {
|
252
|
+
maximum_percent: default_maximum_percent,
|
253
|
+
minimum_healthy_percent: default_minimum_healthy_percent
|
254
|
+
},
|
255
|
+
task_definition: @task_definition
|
256
|
+
}
|
257
|
+
unless target_group.nil? || target_group.empty?
|
258
|
+
add_load_balancer!(container, options)
|
259
|
+
end
|
260
|
+
response = ecs.create_service(options)
|
261
|
+
service = response.service # must set service here since this might never be called if @wait_for_deployment is false
|
262
|
+
end
|
263
|
+
puts message unless @options[:mute]
|
264
|
+
service
|
265
|
+
end
|
266
|
+
|
267
|
+
# $ aws ecs update-service --generate-cli-skeleton
|
268
|
+
# {
|
269
|
+
# "cluster": "",
|
270
|
+
# "service": "",
|
271
|
+
# "taskDefinition": "",
|
272
|
+
# "desiredCount": 0,
|
273
|
+
# "deploymentConfiguration": {
|
274
|
+
# "maximumPercent": 0,
|
275
|
+
# "minimumHealthyPercent": 0
|
276
|
+
# }
|
277
|
+
# }
|
278
|
+
# Only thing we want to change is the task-definition
|
279
|
+
def update_service(ecs_service)
|
280
|
+
message = "#{ecs_service.service_name} service updated on #{ecs_service.cluster_name} cluster with task #{@task_definition}"
|
281
|
+
if @options[:noop]
|
282
|
+
message = "NOOP #{message}"
|
283
|
+
else
|
284
|
+
response = ecs.update_service(
|
285
|
+
cluster: ecs_service.cluster_arn, # can use the cluster name also since it is unique
|
286
|
+
service: ecs_service.service_arn, # can use the service name also since it is unique
|
287
|
+
task_definition: @task_definition
|
288
|
+
)
|
289
|
+
service = response.service # must set service here since this might never be called if @wait_for_deployment is false
|
290
|
+
end
|
291
|
+
puts message unless @options[:mute]
|
292
|
+
service
|
293
|
+
end
|
294
|
+
|
295
|
+
# Only support Application Load Balancer
|
296
|
+
# Think there is an AWS bug that complains about not having the LB
|
297
|
+
# name but you cannot pass both a LB Name and a Target Group.
|
298
|
+
def add_load_balancer!(container, options)
|
299
|
+
options.merge!(
|
300
|
+
role: "ecsServiceRole", # assumption that we're using the ecsServiceRole
|
301
|
+
load_balancers: [
|
302
|
+
{
|
303
|
+
container_name: container[:name],
|
304
|
+
container_port: container[:port],
|
305
|
+
target_group_arn: @options[:target_group],
|
306
|
+
}
|
307
|
+
]
|
308
|
+
)
|
309
|
+
end
|
310
|
+
|
311
|
+
# Returns the target_group.
|
312
|
+
# Will only allow an target_group and the service to use a load balancer
|
313
|
+
# if the container name is "web".
|
314
|
+
def create_service_prompt(container)
|
315
|
+
return if @options[:noop]
|
316
|
+
return unless @elb_prompt
|
317
|
+
if container[:name] != "web" and @options[:target_group]
|
318
|
+
puts "WARNING: A --target-group #{@options[:target_group]} was provided but it will not be used because this not a web container. Container name: #{container[:name].inspect}."
|
319
|
+
end
|
320
|
+
return unless container[:name] == 'web'
|
321
|
+
return @options[:target_group] if @options[:target_group]
|
322
|
+
|
323
|
+
puts "This service #{@service} does not yet exist in the #{@cluster} cluster. This deploy will create it."
|
324
|
+
puts "Would you like this service to be associated with an Application Load Balancer?"
|
325
|
+
puts "If yes, please provide the Application Load Balancer Target Group ARN."
|
326
|
+
puts "If no, simply press enter."
|
327
|
+
print "Target Group ARN: "
|
328
|
+
|
329
|
+
arn = $stdin.gets.strip
|
330
|
+
until arn == '' or validate_target_group(arn)
|
331
|
+
puts "You have provided an invalid Application Load Balancer Target Group ARN: #{arn}."
|
332
|
+
puts "It should be in the form: arn:aws:elasticloadbalancing:us-east-1:123456789:targetgroup/target-name/2378947392743"
|
333
|
+
puts "Please try again or skip adding a Target Group by just pressing enter."
|
334
|
+
print "Target Group ARN: "
|
335
|
+
arn = $stdin.gets.strip
|
336
|
+
end
|
337
|
+
arn
|
338
|
+
end
|
339
|
+
|
340
|
+
def validate_target_group(arn)
|
341
|
+
elb.describe_target_groups(target_group_arns: [arn])
|
342
|
+
true
|
343
|
+
rescue Aws::ElasticLoadBalancingV2::Errors::ValidationError
|
344
|
+
false
|
345
|
+
end
|
346
|
+
|
347
|
+
# assume only 1 container_definition
|
348
|
+
# assume only 1 port mapping in that container_defintion
|
349
|
+
def container_info(task_definition)
|
350
|
+
task_definition_path = "ufo/output/#{task_definition}.json"
|
351
|
+
task_definition_full_path = "#{@project_root}/#{task_definition_path}"
|
352
|
+
unless File.exist?(task_definition_full_path)
|
353
|
+
puts "ERROR: Unable to find the task definition at #{task_definition_path}."
|
354
|
+
puts "Are you sure you have defined it in ufo/template_definitions.rb?"
|
355
|
+
exit
|
356
|
+
end
|
357
|
+
task_definition = JSON.load(IO.read(task_definition_full_path))
|
358
|
+
container_def = task_definition["containerDefinitions"].first
|
359
|
+
mappings = container_def["portMappings"]
|
360
|
+
if mappings
|
361
|
+
map = mappings.first
|
362
|
+
port = map["containerPort"]
|
363
|
+
end
|
364
|
+
{
|
365
|
+
name: container_def["name"],
|
366
|
+
port: port
|
367
|
+
}
|
368
|
+
end
|
369
|
+
|
370
|
+
def service_exact_match?(service_name)
|
371
|
+
service_name == @service
|
372
|
+
end
|
373
|
+
|
374
|
+
# @service is changed to a pattern to be search with.
|
375
|
+
#
|
376
|
+
# Examples:
|
377
|
+
# @service == "hi-.*-prod"
|
378
|
+
def service_pattern_match?(service_name)
|
379
|
+
service_patttern = Regexp.new(@service)
|
380
|
+
service_name =~ service_patttern
|
381
|
+
end
|
382
|
+
|
383
|
+
def find_ecs_service
|
384
|
+
find_all_ecs_services.find { |ecs_service| ecs_service.service_name == @service }
|
385
|
+
end
|
386
|
+
|
387
|
+
# find all services on a cluster
|
388
|
+
# yields ECS::Service object
|
389
|
+
def find_all_ecs_services
|
390
|
+
ecs_services = []
|
391
|
+
service_arns.each do |service_arn|
|
392
|
+
ecs_service = ECS::Service.new(cluster_arn, service_arn)
|
393
|
+
yield(ecs_service) if block_given?
|
394
|
+
ecs_services << ecs_service
|
395
|
+
end
|
396
|
+
ecs_services
|
397
|
+
end
|
398
|
+
|
399
|
+
def service_arns
|
400
|
+
ecs.list_services(cluster: @cluster).service_arns
|
401
|
+
end
|
402
|
+
|
403
|
+
def cluster_arn
|
404
|
+
@cluster_arn ||= ecs_clusters.first.cluster_arn
|
405
|
+
end
|
406
|
+
|
407
|
+
def ensure_cluster_exist
|
408
|
+
cluster_exist = ecs_clusters.first
|
409
|
+
unless cluster_exist
|
410
|
+
message = "#{@cluster} cluster created."
|
411
|
+
if @options[:noop]
|
412
|
+
message = "NOOP #{message}"
|
413
|
+
else
|
414
|
+
ecs.create_cluster(cluster_name: @cluster)
|
415
|
+
end
|
416
|
+
puts message unless @options[:mute]
|
417
|
+
end
|
418
|
+
end
|
419
|
+
|
420
|
+
def ecs_clusters
|
421
|
+
ecs.describe_clusters(clusters: [@cluster]).clusters
|
422
|
+
end
|
423
|
+
|
424
|
+
def task_name(task_definition)
|
425
|
+
# "arn:aws:ecs:us-east-1:123456789:task-definition/hi-web-prod:72"
|
426
|
+
# ->
|
427
|
+
# "task-definition/hi-web-prod:72"
|
428
|
+
task_definition.split('/').last
|
429
|
+
end
|
430
|
+
|
431
|
+
def task_version(task_definition)
|
432
|
+
# "task-definition/hi-web-prod:72" -> 72
|
433
|
+
task_name(task_definition).split(':').last.to_i
|
434
|
+
end
|
435
|
+
end
|
436
|
+
end
|