ufo 2.3.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +57 -0
  3. data/.gitmodules +3 -0
  4. data/.rspec +1 -0
  5. data/CHANGELOG.md +16 -1
  6. data/Gemfile.lock +16 -3
  7. data/README.md +5 -1
  8. data/docs/_docs/auto-completion.md +27 -0
  9. data/docs/_docs/automated-cleanup.md +1 -1
  10. data/docs/_docs/conventions.md +2 -2
  11. data/docs/_docs/helpers.md +6 -6
  12. data/docs/_docs/run-in-pieces.md +15 -8
  13. data/docs/_docs/settings.md +61 -49
  14. data/docs/_docs/structure.md +2 -2
  15. data/docs/_docs/tutorial-ufo-docker-build.md +10 -3
  16. data/docs/_docs/tutorial-ufo-init.md +48 -16
  17. data/docs/_docs/tutorial-ufo-ship.md +14 -7
  18. data/docs/_docs/tutorial-ufo-ships.md +1 -1
  19. data/docs/_docs/tutorial-ufo-tasks-build.md +23 -14
  20. data/docs/_docs/ufo-deploy.md +30 -0
  21. data/docs/_docs/ufo-docker-base.md +3 -3
  22. data/docs/_docs/ufo-docker-build.md +3 -3
  23. data/docs/_docs/ufo-docker-push.md +43 -0
  24. data/docs/_docs/ufo-env.md +17 -15
  25. data/docs/_docs/ufo-init.md +14 -1
  26. data/docs/_docs/ufo-scale.md +2 -4
  27. data/docs/_docs/ufo-ships.md +2 -2
  28. data/docs/_docs/variables.md +6 -6
  29. data/docs/_includes/commands.html +4 -4
  30. data/docs/_includes/subnav.html +3 -0
  31. data/docs/_includes/summary.html +2 -2
  32. data/docs/_includes/ufo-ship-options.md +0 -2
  33. data/docs/docs.md +5 -1
  34. data/docs/quick-start.md +19 -10
  35. data/lib/{starter_project → template}/.env +0 -0
  36. data/lib/template/.ufo/settings.yml.tt +27 -0
  37. data/lib/{starter_project/ufo/task_definitions.rb → template/.ufo/task_definitions.rb.tt} +0 -0
  38. data/lib/{starter_project/ufo → template/.ufo}/templates/main.json.erb +0 -0
  39. data/lib/{starter_project/ufo → template/.ufo}/variables/base.rb +0 -0
  40. data/lib/{starter_project/ufo → template/.ufo}/variables/development.rb +0 -0
  41. data/lib/{starter_project/ufo → template/.ufo}/variables/production.rb +0 -0
  42. data/lib/{starter_project → template}/Dockerfile +0 -0
  43. data/lib/{starter_project/bin/deploy → template/bin/deploy.tt} +0 -0
  44. data/lib/ufo.rb +9 -2
  45. data/lib/ufo/cli.rb +34 -29
  46. data/lib/ufo/completer.rb +86 -64
  47. data/lib/ufo/core.rb +42 -0
  48. data/lib/ufo/default.rb +4 -6
  49. data/lib/ufo/default/settings.yml +24 -22
  50. data/lib/ufo/deploy.rb +0 -0
  51. data/lib/ufo/docker.rb +12 -2
  52. data/lib/ufo/docker/builder.rb +19 -49
  53. data/lib/ufo/docker/cleaner.rb +4 -2
  54. data/lib/ufo/docker/dockerfile.rb +1 -2
  55. data/lib/ufo/docker/pusher.rb +53 -0
  56. data/lib/ufo/dsl.rb +1 -2
  57. data/lib/ufo/dsl/helper.rb +3 -4
  58. data/lib/ufo/dsl/outputter.rb +1 -1
  59. data/lib/ufo/dsl/task_definition.rb +17 -37
  60. data/lib/ufo/ecr/auth.rb +22 -2
  61. data/lib/ufo/ecs.rb +5 -0
  62. data/lib/ufo/ecs/service.rb +21 -0
  63. data/lib/ufo/help/completion.md +1 -1
  64. data/lib/ufo/help/completion_script.md +1 -1
  65. data/lib/ufo/help/deploy.md +14 -0
  66. data/lib/ufo/help/docker/name.md +13 -2
  67. data/lib/ufo/help/docker/push.md +11 -0
  68. data/lib/ufo/init.rb +48 -65
  69. data/lib/ufo/log_group.rb +5 -2
  70. data/lib/ufo/sequence.rb +27 -0
  71. data/lib/ufo/setting.rb +18 -8
  72. data/lib/ufo/ship.rb +23 -46
  73. data/lib/ufo/tasks/builder.rb +8 -11
  74. data/lib/ufo/tasks/register.rb +2 -3
  75. data/lib/ufo/upgrade3.rb +64 -0
  76. data/lib/ufo/util.rb +0 -2
  77. data/lib/ufo/version.rb +1 -1
  78. data/spec/fixtures/home_existing/.docker/config.json +1 -1
  79. data/spec/fixtures/settings.yml +23 -0
  80. data/spec/lib/cli_spec.rb +1 -9
  81. data/spec/lib/completion_spec.rb +18 -0
  82. data/spec/lib/core_spec.rb +16 -0
  83. data/spec/lib/ecr_auth_spec.rb +1 -3
  84. data/spec/lib/ecr_cleaner_spec.rb +1 -3
  85. data/spec/lib/setting_spec.rb +12 -0
  86. data/spec/lib/ship_spec.rb +2 -4
  87. data/spec/lib/task_spec.rb +0 -2
  88. data/spec/spec_helper.rb +12 -2
  89. data/ufo.gemspec +2 -0
  90. metadata +47 -13
  91. data/lib/starter_project/ufo/settings.yml +0 -18
  92. data/lib/ufo/env.rb +0 -18
  93. data/lib/ufo/help/sub/goodbye.md +0 -5
@@ -1,4 +1,5 @@
1
- # Use to automatically create the CloudWatch group
1
+ # Use to automatically create the CloudWatch group.
2
+ # For some reason creating ECS does do this by default.
2
3
  module Ufo
3
4
  class LogGroup
4
5
  include AwsService
@@ -10,6 +11,8 @@ module Ufo
10
11
  def create
11
12
  puts "Ensuring log group for #{@task_definition} exists"
12
13
  return if @options[:noop]
14
+
15
+ Ufo.check_task_definition!(@task_definition)
13
16
  task_def = JSON.load(IO.read(task_def_path))
14
17
  task_def["containerDefinitions"].each do |container_def|
15
18
  begin
@@ -29,7 +32,7 @@ module Ufo
29
32
  end
30
33
 
31
34
  def task_def_path
32
- "./ufo/output/#{@task_definition}.json"
35
+ "#{Ufo.root}/.ufo/output/#{@task_definition}.json"
33
36
  end
34
37
  end
35
38
  end
@@ -0,0 +1,27 @@
1
+ require 'fileutils'
2
+ require 'colorize'
3
+ require 'thor'
4
+
5
+ module Ufo
6
+ class Sequence < Thor::Group
7
+ include Thor::Actions
8
+
9
+ def self.source_root
10
+ File.expand_path("../../template", __FILE__)
11
+ end
12
+
13
+ private
14
+ def confirm_cli_project
15
+ cli_project = File.exist?("#{project_name}/config/application.rb")
16
+ unless cli_project
17
+ puts "It does not look like the repo #{options[:repo]} is a cli project. Maybe double check that it is? Exited.".colorize(:red)
18
+ exit 1
19
+ end
20
+ end
21
+
22
+ def copy_project
23
+ puts "Creating new project called #{project_name}."
24
+ directory ".", project_name
25
+ end
26
+ end
27
+ end
@@ -2,8 +2,7 @@ require 'yaml'
2
2
 
3
3
  module Ufo
4
4
  class Setting
5
- def initialize(project_root='.', check_ufo_project=true)
6
- @project_root = project_root
5
+ def initialize(check_ufo_project=true)
7
6
  @check_ufo_project = check_ufo_project
8
7
  end
9
8
 
@@ -14,13 +13,12 @@ module Ufo
14
13
 
15
14
  if @check_ufo_project && !File.exist?(project_settings_path)
16
15
  puts "ERROR: No settings file at #{project_settings_path}. Are you sure you are in a project with ufo setup?"
17
- puts "Please create a settings file via: ufo init"
16
+ puts "If you want to set up ufo for this prjoect, please create a settings file via: ufo init"
18
17
  exit 1
19
18
  end
20
19
 
21
- project = File.exist?(project_settings_path) ?
22
- YAML.load_file(project_settings_path) :
23
- {}
20
+ # project based settings files
21
+ project = load_file(project_settings_path)
24
22
 
25
23
  user_file = "#{ENV['HOME']}/.ufo/settings.yml"
26
24
  user = File.exist?(user_file) ? YAML.load_file(user_file) : {}
@@ -28,12 +26,24 @@ module Ufo
28
26
  default_file = File.expand_path("../default/settings.yml", __FILE__)
29
27
  default = YAML.load_file(default_file)
30
28
 
31
- @settings_yaml = default.merge(user.merge(project))
29
+ @settings_yaml = default.deep_merge(user.deep_merge(project))[Ufo.env]
32
30
  end
33
31
 
34
32
  private
33
+ def load_file(path)
34
+ content = RenderMePretty.result(path)
35
+ data = File.exist?(path) ? YAML.load(content) : {}
36
+ # automatically add base settings to the rest of the environments
37
+ data.each do |ufo_env, _setting|
38
+ base = data["base"] || {}
39
+ env = data[ufo_env] || {}
40
+ data[ufo_env] = base.merge(env) unless ufo_env == "base"
41
+ end
42
+ data
43
+ end
44
+
35
45
  def project_settings_path
36
- "#{@project_root}/ufo/settings.yml"
46
+ "#{Ufo.root}/.ufo/settings.yml"
37
47
  end
38
48
  end
39
49
  end
@@ -1,23 +1,6 @@
1
1
  require 'colorize'
2
2
 
3
3
  module Ufo
4
- # Creating this class pass so we can have a reference we do not have describe
5
- # all services and look up service_name creating more API calls.
6
- #
7
- # Also this class allows us to pass one object around instead of both
8
- # cluster_name and service_name.
9
- module ECS
10
- Service = Struct.new(:cluster_arn, :service_arn) do
11
- def cluster_name
12
- cluster_arn.split('/').last
13
- end
14
-
15
- def service_name
16
- service_arn.split('/').last
17
- end
18
- end
19
- end
20
-
21
4
  class UfoError < RuntimeError; end
22
5
  class ShipmentOverridden < UfoError; end
23
6
 
@@ -30,25 +13,12 @@ module Ufo
30
13
  @service = service
31
14
  @task_definition = task_definition
32
15
  @options = options
33
- @project_root = options[:project_root] || '.'
34
16
  @target_group_prompt = @options[:target_group_prompt].nil? ? true : @options[:target_group_prompt]
35
17
  @cluster = @options[:cluster] || default_cluster
36
18
  @wait_for_deployment = @options[:wait].nil? ? true : @options[:wait]
37
19
  @stop_old_tasks = @options[:stop_old_tasks].nil? ? false : @options[:stop_old_tasks]
38
20
  end
39
21
 
40
- # Find all the matching services on a cluster and update them.
41
- # If no service is found at all, then create a service on the very first specified cluster
42
- # only.
43
- #
44
- # If it looks like one service is passed in then it'll automatically create the service
45
- # on the first cluster.
46
-
47
- # If it looks like a regexp is passed in then it'll only update the services
48
- # This is because regpex cannot be used to determined a list of service_names.
49
- #
50
- # Example:
51
- # No way to map: hi-.*-prod -> hi-web-prod hi-worker-prod hi-clock-prod
52
22
  def deploy
53
23
  message = "Shipping #{@service}..."
54
24
  unless @options[:mute]
@@ -60,15 +30,18 @@ module Ufo
60
30
  end
61
31
  end
62
32
 
33
+ ensure_log_group_exist
63
34
  ensure_cluster_exist
64
- process_single_service
35
+ process_deployment
65
36
 
66
37
  puts "Software shipped!" unless @options[:mute]
67
38
  end
68
39
 
69
- # A single service name shouold had been passed and the service automatically
70
- # gets created if it does not exist.
71
- def process_single_service
40
+ def ensure_log_group_exist
41
+ LogGroup.new(@task_definition, @options).create
42
+ end
43
+
44
+ def process_deployment
72
45
  ecs_service = find_ecs_service
73
46
  deployed_service = if ecs_service
74
47
  # update all existing service
@@ -272,11 +245,12 @@ module Ufo
272
245
  if @options[:noop]
273
246
  message = "NOOP #{message}"
274
247
  else
275
- response = ecs.update_service(
248
+ params = {
276
249
  cluster: ecs_service.cluster_arn, # can use the cluster name also since it is unique
277
250
  service: ecs_service.service_arn, # can use the service name also since it is unique
278
251
  task_definition: @task_definition
279
- )
252
+ }
253
+ response = ecs.update_service(params)
280
254
  service = response.service # must set service here since this might never be called if @wait_for_deployment is false
281
255
  end
282
256
  puts message unless @options[:mute]
@@ -338,14 +312,9 @@ module Ufo
338
312
  # assume only 1 container_definition
339
313
  # assume only 1 port mapping in that container_defintion
340
314
  def container_info(task_definition)
315
+ Ufo.check_task_definition!(task_definition)
341
316
  task_definition_path = "ufo/output/#{task_definition}.json"
342
- task_definition_full_path = "#{@project_root}/#{task_definition_path}"
343
- unless File.exist?(task_definition_full_path)
344
- puts "ERROR: Unable to find the task definition at #{task_definition_path}.".colorize(:red)
345
- puts "Are you sure you have defined it in ufo/template_definitions.rb?".colorize(:red)
346
- exit
347
- end
348
- task_definition = JSON.load(IO.read(task_definition_full_path))
317
+ task_definition = JSON.load(IO.read(task_definition_path))
349
318
  container_def = task_definition["containerDefinitions"].first
350
319
  mappings = container_def["portMappings"]
351
320
  if mappings
@@ -363,11 +332,11 @@ module Ufo
363
332
  end
364
333
 
365
334
  # find all services on a cluster
366
- # yields ECS::Service object
335
+ # yields Ufo::ECS::Service object
367
336
  def find_all_ecs_services
368
337
  ecs_services = []
369
338
  service_arns.each do |service_arn|
370
- ecs_service = ECS::Service.new(cluster_arn, service_arn)
339
+ ecs_service = Ufo::ECS::Service.new(cluster_arn, service_arn)
371
340
  yield(ecs_service) if block_given?
372
341
  ecs_services << ecs_service
373
342
  end
@@ -375,7 +344,13 @@ module Ufo
375
344
  end
376
345
 
377
346
  def service_arns
378
- ecs.list_services(cluster: @cluster).service_arns
347
+ services = ecs.list_services(cluster: @cluster)
348
+ list_service_arns = services.service_arns
349
+ while services.next_token != nil
350
+ services = ecs.list_services(cluster: @cluster, next_token: services.next_token)
351
+ list_service_arns += services.service_arns
352
+ end
353
+ list_service_arns
379
354
  end
380
355
 
381
356
  def cluster_arn
@@ -390,6 +365,8 @@ module Ufo
390
365
  message = "NOOP #{message}"
391
366
  else
392
367
  ecs.create_cluster(cluster_name: @cluster)
368
+ # TODO: Aad Waiter logic, sometimes the cluster does not exist by the time
369
+ # we create the service
393
370
  end
394
371
  puts message unless @options[:mute]
395
372
  end
@@ -1,19 +1,16 @@
1
1
  module Ufo
2
2
  class Tasks::Builder
3
- # build and registers together
4
- def self.register(task_definition, options)
5
- # task definition and deploy logic are coupled in the Ship class.
6
- # Example: We need to know if the task defintion is a web service to see if we need to
7
- # add the elb target group. The web service information is in the Tasks::Builder
8
- # and the elb target group gets set in the Ship class.
9
- # So we always call these together.
3
+ # ship: build and registers task definitions together
4
+ def self.ship(task_definition, options)
5
+ # When handling task definitions in with the ship command and class, we always want to
6
+ # build and register task definitions. There is little point of running them independently
7
+ # This method helps us do that.
10
8
  Tasks::Builder.new(options).build
11
9
  Tasks::Register.register(task_definition, options)
12
10
  end
13
11
 
14
12
  def initialize(options={})
15
13
  @options = options
16
- @project_root = options[:project_root] || '.'
17
14
  end
18
15
 
19
16
  def build
@@ -21,19 +18,19 @@ module Ufo
21
18
  check_templates_definitions_path
22
19
  dsl = DSL.new(template_definitions_path, @options.merge(quiet: false, mute: true))
23
20
  dsl.run
24
- puts "Task Definitions built in ufo/output." unless @options[:mute]
21
+ puts "Task Definitions built in .ufo/output" unless @options[:mute]
25
22
  end
26
23
 
27
24
  def check_templates_definitions_path
28
25
  unless File.exist?(template_definitions_path)
29
- pretty_path = template_definitions_path.sub("#{@project_root}/", '')
26
+ pretty_path = template_definitions_path.sub("#{Ufo.root}/", '')
30
27
  puts "ERROR: #{pretty_path} does not exist. Run: `ufo init` to create a starter file" unless @options[:mute]
31
28
  exit 1
32
29
  end
33
30
  end
34
31
 
35
32
  def template_definitions_path
36
- "#{@project_root}/ufo/task_definitions.rb"
33
+ "#{Ufo.root}/.ufo/task_definitions.rb"
37
34
  end
38
35
  end
39
36
  end
@@ -6,8 +6,7 @@ module Ufo
6
6
  include AwsService
7
7
 
8
8
  def self.register(task_name, options={})
9
- project_root = options[:project_root] || '.'
10
- Dir.glob("#{project_root}/ufo/output/*").each do |path|
9
+ Dir.glob("#{Ufo.root}/.ufo/output/*").each do |path|
11
10
  if task_name == :all or path.include?(task_name)
12
11
  task_register = Tasks::Register.new(path, options)
13
12
  task_register.register
@@ -20,7 +19,7 @@ module Ufo
20
19
  @options = options
21
20
  end
22
21
 
23
- # aws ecs register-task-definition --cli-input-json file://ufo/output/hi-web-prod.json
22
+ # aws ecs register-task-definition --cli-input-json file://.ufo/output/hi-web-prod.json
24
23
  def register
25
24
  data = JSON.parse(IO.read(@template_definition_path), symbolize_names: true)
26
25
  data = data.to_snake_keys
@@ -0,0 +1,64 @@
1
+ require 'fileutils'
2
+ require 'yaml'
3
+
4
+ module Ufo
5
+ class Upgrade3
6
+ def initialize(options)
7
+ @options = options
8
+ end
9
+
10
+ def run
11
+ if File.exist?("#{Ufo.root}/.ufo")
12
+ puts "It looks like you already have a .ufo folder in your project. This is the new project structure so exiting without updating anything."
13
+ return
14
+ end
15
+
16
+ if !File.exist?("#{Ufo.root}/ufo")
17
+ puts "Could not find a ufo folder in your project. Maybe you want to run ufo init to initialize a new ufo project instead?"
18
+ return
19
+ end
20
+
21
+ puts "Upgrading structure of your current project to the new ufo version 3 project structure"
22
+ upgrade_settings("ufo/settings.yml")
23
+ user_settings_path = "#{ENV['HOME']}/.ufo/settings.yml"
24
+ if File.exist?(user_settings_path)
25
+ upgrade_settings(user_settings_path)
26
+ end
27
+ mv("ufo", ".ufo")
28
+ puts "Upgrade complete."
29
+ end
30
+
31
+ def upgrade_settings(path)
32
+ data = YAML.load_file(path)
33
+ return if data.key?("base") # already in new format
34
+
35
+ new_structure = {}
36
+
37
+ (data["aws_profile_ufo_env_map"] || {}).each do |aws_profile, ufo_env|
38
+ new_structure[ufo_env] ||= {}
39
+ new_structure[ufo_env]["aws_profiles"] ||= []
40
+ new_structure[ufo_env]["aws_profiles"] << aws_profile
41
+ end
42
+ data.delete("aws_profile_ufo_env_map")
43
+
44
+ (data["ufo_env_cluster_map"] || {}).each do |ufo_env, cluster|
45
+ new_structure[ufo_env] ||= {}
46
+ new_structure[ufo_env]["cluster"] = cluster
47
+ end
48
+ data.delete("ufo_env_cluster_map")
49
+
50
+ new_structure["base"] = data
51
+ text = YAML.dump(new_structure)
52
+ IO.write(path, text)
53
+ puts "Upgraded settings: #{path}"
54
+ if path.include?(ENV['HOME'])
55
+ puts "NOTE: Your ~/.ufo/settings.yml file was also upgraded to the new format. If you are using ufo in other projects those will have to be upgraded also."
56
+ end
57
+ end
58
+
59
+ def mv(src, dest)
60
+ puts "mv #{src} #{dest}"
61
+ FileUtils.mv(src, dest)
62
+ end
63
+ end
64
+ end
@@ -1,8 +1,6 @@
1
1
  module Ufo
2
2
  module Util
3
3
  def execute(command, local_options={})
4
- command = "cd #{@project_root} && #{command}"
5
- # local_options[:live] overrides the global @options[:noop]
6
4
  if @options[:noop] && !local_options[:live]
7
5
  say "NOOP: #{command}"
8
6
  result = true # always success with no noop for specs
@@ -1,3 +1,3 @@
1
1
  module Ufo
2
- VERSION = "2.3.0"
2
+ VERSION = "3.0.0"
3
3
  end
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "auths": {
3
- "https://123456789.dkr.ecr.us-east-1.amazonaws.com": {
3
+ "123456789.dkr.ecr.us-east-1.amazonaws.com": {
4
4
  "auth": "crazylongawsecrtoken"
5
5
  },
6
6
  "https://index.docker.io/v1/": {
@@ -0,0 +1,23 @@
1
+ # More info: http://ufoships.com/docs/ufo-settings/
2
+ base: &base
3
+ image: <%= @image %>
4
+ # clean_keep: 30 # cleans up docker images on your docker server.
5
+ # ecr_keep: 30 # cleans up images on ECR and keeps this remaining amount. Defaults to keep all.
6
+ # defaults when an new ECS service is created by ufo ship
7
+ new_service:
8
+ maximum_percent: 200
9
+ minimum_healthy_percent: 100
10
+ desired_count: 1
11
+
12
+ development:
13
+ <<: *base
14
+ cluster: dev
15
+ aws_profiles:
16
+ - dev_profile1
17
+ - dev_profile2
18
+
19
+ production:
20
+ <<: *base
21
+ cluster: prod
22
+ aws_profiles:
23
+ - prod_profile
@@ -1,15 +1,7 @@
1
- require 'spec_helper'
2
-
3
- # to run specs with what's remembered from vcr
4
- # $ rake
5
- #
6
- # to run specs with new fresh data from aws api calls
7
- # $ rake clean:vcr ; time rake
8
1
  describe Ufo::CLI do
9
2
  before(:all) do
10
3
  create_starter_project_fixture
11
- project_root = File.expand_path("../../fixtures/hi", __FILE__)
12
- @args = "--noop --project-root=#{project_root}"
4
+ @args = "--noop"
13
5
  end
14
6
 
15
7
  describe "ufo" do
@@ -0,0 +1,18 @@
1
+ describe Ufo::CLI do
2
+ describe "ufo completion" do
3
+ commands = {
4
+ "ship" => "service",
5
+ "ship service" => "--task",
6
+ "docker" => "build",
7
+ "docker build" => "--push",
8
+ "docker clean" => "image_name",
9
+ "init" => "--image",
10
+ }
11
+ commands.each do |command, expected_word|
12
+ it "#{command}" do
13
+ out = execute("exe/ufo completion #{command}")
14
+ expect(out).to include(expected_word) # only checking for one word for simplicity
15
+ end
16
+ end
17
+ end
18
+ end