ufo 4.6.3 → 5.0.4

data/lib/ufo/stack/custom_properties.rb

@@ -0,0 +1,59 @@
+class Ufo::Stack
+  class CustomProperties
+    include Ufo::Settings
+
+    def initialize(template, stack_name)
+      @template, @stack_name = template, stack_name
+    end
+
+    def apply
+      customizations = camelize(cfn)
+      @template["Resources"].each do |logical_id, attrs|
+        custom_props = customizations[logical_id]
+        next unless custom_props
+        attrs["Properties"].deeper_merge!(custom_props, {overwrite_arrays: true})
+      end
+
+      substitute_variables!(@template["Resources"])
+      @template
+    end
+
+    # Keep backward compatiablity but encouraging CamelCase now because in the ufo init generator
+    # the .ufo/settings/cfn/default.yml is now CamelCase
+    def camelize(properties)
+      if ENV['UFO_CAMELIZE'] == '0' || settings[:auto_camelize] == false # provide a way to quickly test full camelize disable
+        return properties.deep_stringify_keys
+      end
+
+      # transform keys: camelize
+      properties.deep_stringify_keys.deep_transform_keys do |key|
+        if key == key.upcase # trying to generalize special rule for dns.TTL
+          key # leave key alone if key is already in all upcase
+        else
+          key.camelize
+        end
+      end
+    end
+
+    # Substitute special variables that cannot be baked into the template
+    # because they are dynamically assigned. Only one special variable:
+    #
+    #   {stack_name}
+    def substitute_variables!(properties)
+      # transform values and substitute for special values
+      # https://stackoverflow.com/questions/34595142/process-nested-hash-to-convert-all-values-to-strings
+      #
+      # Examples:
+      #   "{stack_name}.stag.boltops.com." => development-demo-web.stag.boltops.com.
+      #   "{stack_name}.stag.boltops.com." => dev-demo-web-2.stag.boltops.com.
+      properties.deep_merge(properties) do |_,_,v|
+        if v.is_a?(String)
+          v.sub!('{stack_name}', @stack_name) # need shebang, updating in-place
+        else
+          v
+        end
+      end
+      properties
+    end
+  end
+end

data/lib/ufo/stack/helper.rb

@@ -2,6 +2,7 @@ class Ufo::Stack
   module Helper
     include Ufo::AwsService
     include Ufo::Util
+    include Ufo::Settings
     extend Memoist
 
     def find_stack(stack_name)
@@ -34,15 +35,11 @@ class Ufo::Stack
       when "append_nothing", "prepend_nothing"
         [service, Ufo.env_extra]
       else # new default. ufo v4.5 and above
-        [service, Ufo.env, Ufo.env_extra]
+        [service, Ufo.env.to_s, Ufo.env_extra]
       end
       parts.reject {|x| x==''}.compact.join('-') # stack_name
     end
 
-    def cfn
-      Ufo::Setting::Profile.new(:cfn, settings[:cfn_profile]).data
-    end
-
     def status
       Status.new(@stack_name)
     end

data/lib/ufo/stack/template_body.rb

@@ -0,0 +1,13 @@
+class Ufo::Stack
+  class TemplateBody
+    def initialize(context)
+      @context = context
+    end
+
+    def build
+      builder = Builder.new(@context)
+      builder.build
+    end
+  end
+end
+

data/lib/ufo/task.rb

@@ -2,8 +2,9 @@ module Ufo
   class Task < Base
     extend Memoist
 
-    include Util
     include AwsService
+    include Ufo::Settings
+    include Util
 
     def initialize(task_definition, options)
       @task_definition = task_definition
@@ -139,12 +140,6 @@ module Ufo
       options
     end
 
-    def network
-      settings = Ufo.settings
-      Setting::Profile.new(:network, settings[:network_profile]).data
-    end
-    memoize :network
-
     def cloudwatch_info(task_arn)
       config = container_definition[:log_configuration]
       container_name = container_definition[:name]

data/lib/ufo/tasks.rb

@@ -2,7 +2,7 @@ module Ufo
   class Tasks < Command
     desc "build", "Build task definitions."
     long_desc Help.text("tasks:build")
-    option :pretty, type: :boolean, default: true, desc: "Pretty format the json for the task definitions"
+    option :image_override, desc: "Override image in task definition for quick testing"
     def build
       Tasks::Builder.new(options).build
     end

data/lib/ufo/tasks/builder.rb

@@ -6,7 +6,6 @@ module Ufo
       # build and register task definitions. There is little point of running them independently
       # This method helps us do that.
       build(options)
-      Tasks::Register.register(task_definition, options)
     end
 
     # ship: build and registers task definitions together

data/lib/ufo/template_scope.rb

@@ -1,6 +1,7 @@
 module Ufo
   class TemplateScope
     extend Memoist
+    include Ufo::Settings
 
     attr_reader :helper
     attr_reader :task_definition_name
@@ -44,72 +45,6 @@ module Ufo
       end
     end
 
-    def network
-      Ufo::Setting::Profile.new(:network, settings[:network_profile]).data
-    end
-    memoize :network
-
-    def cfn
-      Ufo::Setting::Profile.new(:cfn, settings[:cfn_profile]).data
-    end
-    memoize :cfn
-
-    def settings
-      Ufo.settings
-    end
-
-    def custom_properties(resource)
-      resource = resource.to_s.underscore
-      properties = cfn[resource.to_sym]
-      return unless properties
-
-      # transform keys: camelize
-      properties = properties.deep_stringify_keys.deep_transform_keys do |key|
-        if key == key.upcase # trying to generalize special rule for dns.TTL
-          key # leave key alone if key is already in all upcase
-        else
-          key.camelize
-        end
-      end
-
-      substitute_variables!(properties)
-
-      yaml = YAML.dump(properties)
-      # add spaces in front on each line
-      yaml.split("\n")[1..-1].map do |line|
-        "      #{line}"
-      end.join("\n") + "\n"
-    end
-
-    # Substitute special variables that cannot be baked into the template
-    # because they are dynamically assigned. Only one special variable:
-    #
-    #   {stack_name}
-    def substitute_variables!(properties)
-      # transform values and substitute for special values
-      # https://stackoverflow.com/questions/34595142/process-nested-hash-to-convert-all-values-to-strings
-      #
-      # Examples:
-      #   "{stack_name}.stag.boltops.com." => development-demo-web.stag.boltops.com.
-      #   "{stack_name}.stag.boltops.com." => dev-demo-web-2.stag.boltops.com.
-      properties.deep_merge(properties) do |_,_,v|
-        if v.is_a?(String)
-          v.sub!('{stack_name}', @stack_name) # unsure why need shebang, but it works
-        else
-          v
-        end
-      end
-      properties
-    end
-
-    def default_target_group_protocol
-      default_elb_protocol
-    end
-
-    def default_elb_protocol
-      @elb_type == "application" ? "HTTP" : "TCP"
-    end
-
     def pretty_name?
       # env variable takes highest precedence
       if ENV["STATIC_NAME"]

data/lib/ufo/utils/squeezer.rb

@@ -0,0 +1,24 @@
+module Ufo::Utils
+  class Squeezer
+    def initialize(data)
+      @data = data
+    end
+
+    def squeeze(new_data=nil)
+      data = new_data.nil? ? @data : new_data
+
+      case data
+      when Array
+        data.map! { |v| squeeze(v) }
+      when Hash
+        data.each_with_object({}) do |(k,v), squeezed|
+          # only remove nil and empty Array values within Hash structures
+          squeezed[k] = squeeze(v) unless v.nil? || v.is_a?(Array) && v.empty?
+          squeezed
+        end
+      else
+        data # do not transform
+      end
+    end
+  end
+end

data/lib/ufo/version.rb

@@ -1,3 +1,3 @@
 module Ufo
-  VERSION = "4.6.3"
+  VERSION = "5.0.4"
 end

data/spec/fixtures/iam_roles/task_role.rb

@@ -0,0 +1,17 @@
+iam_policy("AmazonS3ReadOnlyAccess",
+  Action: [
+    "s3:Get*",
+    "s3:List*"
+  ],
+  Effect: "Allow",
+  Resource: "*"
+)
+iam_policy("CloudwatchWrite",
+  Action: [
+    "cloudwatch:PutMetricData",
+  ],
+  Effect: "Allow",
+  Resource: "*"
+)
+
+managed_iam_policy("AmazonS3ReadOnlyAccess", "AmazonEC2ReadOnlyAccess")

data/spec/lib/ecr_auth_spec.rb

@@ -1,36 +1,48 @@
 describe Ufo::Ecr::Auth do
   let(:repo_domain) { "123456789.dkr.ecr.us-east-1.amazonaws.com" }
+  let(:username) { "user" }
+  let(:password) { "opensesame" }
   let(:auth) { Ufo::Ecr::Auth.new(repo_domain) }
   before(:each) do
-    allow(auth).to receive(:fetch_auth_token).and_return("opensesame")
+    allow(auth).to receive(:fetch_auth_token).and_return(Base64.encode64("#{username}:#{password}"))
   end
 
   context("update") do
-    before(:each) do
-      clean_home
-    end
+    context("with ecr repo") do
+      context("when login successful") do
+        it "should create the auth token" do
+          command = "docker login -u #{username} --password-stdin #{repo_domain}"
+          command_result = double(success?: true)
+          expect(Open3).to receive(:capture3)
+            .with(command, stdin_data: password)
+            .and_return(['', '', command_result])
 
-    context("missing ~/.docker/config.json") do
-      it "should create the auth token" do
-        auth.update
-        data = JSON.load(IO.read("spec/fixtures/home/.docker/config.json"))
-        auth_token = data["auths"][repo_domain]["auth"]
-        expect(auth_token).to eq("opensesame")
+          auth.update
+        end
+      end
+
+      context("when login failed") do
+        it "should exit with code 1" do
+          command = "docker login -u #{username} --password-stdin #{repo_domain}"
+          command_result = double(success?: false)
+          expect(Open3).to receive(:capture3)
+            .with(command, stdin_data: password)
+            .and_return(['', '', command_result])
+          expect(auth).to receive(:exit).with(1)
+
+          auth.update
+        end
       end
     end
 
-    context("existing ~/.docker/config.json") do
-      it "should update the auth token" do
+    context("with not ecr repo") do
+      let(:repo_domain) { "example/test" }
+
+      it "should not update credentials" do
+        expect(Open3).not_to receive(:capture3)
+
         auth.update
-        data = JSON.load(IO.read("spec/fixtures/home/.docker/config.json"))
-        auth_token = data["auths"][repo_domain]["auth"]
-        expect(auth_token).to eq("opensesame")
       end
     end
   end
-
-  def clean_home
-    FileUtils.rm_rf("spec/fixtures/home")
-    FileUtils.cp_r("spec/fixtures/home_existing", "spec/fixtures/home")
-  end
 end

data/spec/lib/role/builder_spec.rb

@@ -0,0 +1,67 @@
+describe Ufo::Role::Builder do
+  let(:builder) { described_class.new(role_type) }
+  let(:role_type) { "task_role" }
+
+  before(:each) do
+    Ufo::Role::Registry.register_policy("task_role",
+      "AmazonS3ReadOnlyAccess",
+      {:Action=>["s3:Get*", "s3:List*"], :Effect=>"Allow", :Resource=>"*"}
+    )
+    Ufo::Role::Registry.register_policy("task_role",
+      "CloudwatchWrite",
+      {:Action=>["cloudwatch:PutMetricData"], :Effect=>"Allow", :Resource=>"*"}
+    )
+    # Called twice on purpose to show that duplicated items in the set wont create doubles.
+    # This allows the DSL evaluate to be ran multiple times.
+    Ufo::Role::Registry.register_policy("task_role",
+      "CloudwatchWrite",
+      {:Action=>["cloudwatch:PutMetricData"], :Effect=>"Allow", :Resource=>"*"}
+    )
+
+
+    Ufo::Role::Registry.register_managed_policy("task_role",
+      "AmazonS3ReadOnlyAccess", "AmazonEC2ReadOnlyAccess"
+    )
+  end
+
+  context "build" do
+    it "builds role" do
+      resource = builder.build
+      expected = <<YAML
+---
+Type: AWS::IAM::Role
+Properties:
+  AssumeRolePolicyDocument:
+    Version: '2012-10-17'
+    Statement:
+    - Effect: Allow
+      Principal:
+        Service: ecs-tasks.amazonaws.com
+      Action: sts:AssumeRole
+  Policies:
+  - PolicyName: AmazonS3ReadOnlyAccess
+    PolicyDocument:
+      Version: '2012-10-17'
+      Statement:
+      - Action:
+        - s3:Get*
+        - s3:List*
+        Effect: Allow
+        Resource: "*"
+  - PolicyName: CloudwatchWrite
+    PolicyDocument:
+      Version: '2012-10-17'
+      Statement:
+      - Action:
+        - cloudwatch:PutMetricData
+        Effect: Allow
+        Resource: "*"
+  ManagedPolicyArns:
+  - arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess
+  - arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess
+YAML
+      yaml = YAML.dump(resource)
+      expect(yaml).to eq(expected)
+    end
+  end
+end

data/spec/lib/role/dsl_spec.rb

@@ -0,0 +1,12 @@
+describe Ufo::Role::DSL do
+  let(:dsl) { described_class.new(path) }
+  let(:path) { "spec/fixtures/iam_roles/task_role.rb" }
+
+  context "evaluate" do
+    it "registers policies from role DSL" do
+      dsl.evaluate
+      expect(Ufo::Role::Registry.policies).not_to be_empty
+      expect(Ufo::Role::Registry.managed_policies).not_to be_empty
+    end
+  end
+end

data/ufo.gemspec

@@ -19,13 +19,14 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "aws-logs"
-  spec.add_dependency "aws-mfa-secure"
+  spec.add_dependency "aws-mfa-secure", "~> 0.4.3"
   spec.add_dependency "aws-sdk-cloudformation"
   spec.add_dependency "aws-sdk-cloudwatchlogs"
   spec.add_dependency "aws-sdk-ec2"
   spec.add_dependency "aws-sdk-ecr"
   spec.add_dependency "aws-sdk-ecs"
   spec.add_dependency "aws-sdk-elasticloadbalancingv2"
+  spec.add_dependency "aws_data"
   spec.add_dependency "rainbow"
   spec.add_dependency "deep_merge"
   spec.add_dependency "memoist"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ufo
 version: !ruby/object:Gem::Version
-  version: 4.6.3
+  version: 5.0.4
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-27 00:00:00.000000000 Z
+date: 2021-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-logs
@@ -28,16 +28,16 @@ dependencies:
   name: aws-mfa-secure
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.4.3
 - !ruby/object:Gem::Dependency
   name: aws-sdk-cloudformation
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws_data
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rainbow
   requirement: !ruby/object:Gem::Requirement
@@ -337,6 +351,7 @@ files:
 - docs/_docs/extras/ecs-network-mode.md
 - docs/_docs/extras/load-balancer.md
 - docs/_docs/extras/minimal-deploy-iam.md
+- docs/_docs/extras/notification-arns.md
 - docs/_docs/extras/redirection-support.md
 - docs/_docs/extras/route53-support.md
 - docs/_docs/extras/security-groups.md
@@ -344,6 +359,7 @@ files:
 - docs/_docs/faq.md
 - docs/_docs/fargate.md
 - docs/_docs/helpers.md
+- docs/_docs/iam-roles.md
 - docs/_docs/install.md
 - docs/_docs/more/auto-completion.md
 - docs/_docs/more/automated-cleanup.md
@@ -355,10 +371,12 @@ files:
 - docs/_docs/more/why-cloudformation.md
 - docs/_docs/next-steps.md
 - docs/_docs/quick-start-ec2.md
+- docs/_docs/secrets.md
 - docs/_docs/settings.md
 - docs/_docs/settings/aws_profile.md
 - docs/_docs/settings/cfn.md
 - docs/_docs/settings/cluster.md
+- docs/_docs/settings/manage-security-groups.md
 - docs/_docs/settings/network.md
 - docs/_docs/ssl_errors.md
 - docs/_docs/structure.md
@@ -377,6 +395,7 @@ files:
 - docs/_docs/upgrading.md
 - docs/_docs/upgrading/upgrade4.5.md
 - docs/_docs/upgrading/upgrade4.md
+- docs/_docs/upgrading/upgrade5.md
 - docs/_docs/variables.md
 - docs/_includes/about.html
 - docs/_includes/cfn-customize.md
@@ -493,8 +512,10 @@ files:
 - docs/utils/test-aws-api-access.rb
 - docs/utils/update-cert-chains.sh
 - exe/ufo
-- lib/cfn/stack.yml
 - lib/template/.env
+- lib/template/.secrets
+- lib/template/.ufo/iam_roles/execution_role.rb
+- lib/template/.ufo/iam_roles/task_role.rb
 - lib/template/.ufo/params.yml.tt
 - lib/template/.ufo/settings.yml.tt
 - lib/template/.ufo/settings/cfn/default.yml.tt
@@ -535,6 +556,7 @@ files:
 - lib/ufo/docker/variables.rb
 - lib/ufo/dsl.rb
 - lib/ufo/dsl/helper.rb
+- lib/ufo/dsl/helper/vars.rb
 - lib/ufo/dsl/outputter.rb
 - lib/ufo/dsl/task_definition.rb
 - lib/ufo/ecr/auth.rb
@@ -586,16 +608,45 @@ files:
 - lib/ufo/ps.rb
 - lib/ufo/ps/task.rb
 - lib/ufo/releases.rb
+- lib/ufo/role/builder.rb
+- lib/ufo/role/dsl.rb
+- lib/ufo/role/registry.rb
 - lib/ufo/rollback.rb
 - lib/ufo/scale.rb
 - lib/ufo/sequence.rb
 - lib/ufo/setting.rb
 - lib/ufo/setting/profile.rb
+- lib/ufo/setting/security_groups.rb
+- lib/ufo/settings.rb
 - lib/ufo/ship.rb
 - lib/ufo/stack.rb
+- lib/ufo/stack/builder.rb
+- lib/ufo/stack/builder/base.rb
+- lib/ufo/stack/builder/conditions.rb
+- lib/ufo/stack/builder/outputs.rb
+- lib/ufo/stack/builder/parameters.rb
+- lib/ufo/stack/builder/resources.rb
+- lib/ufo/stack/builder/resources/base.rb
+- lib/ufo/stack/builder/resources/dns.rb
+- lib/ufo/stack/builder/resources/ecs.rb
+- lib/ufo/stack/builder/resources/elb.rb
+- lib/ufo/stack/builder/resources/listener.rb
+- lib/ufo/stack/builder/resources/listener_ssl.rb
+- lib/ufo/stack/builder/resources/roles/base.rb
+- lib/ufo/stack/builder/resources/roles/execution_role.rb
+- lib/ufo/stack/builder/resources/roles/task_role.rb
+- lib/ufo/stack/builder/resources/security_group/base.rb
+- lib/ufo/stack/builder/resources/security_group/ecs.rb
+- lib/ufo/stack/builder/resources/security_group/ecs_rule.rb
+- lib/ufo/stack/builder/resources/security_group/elb.rb
+- lib/ufo/stack/builder/resources/target_group.rb
+- lib/ufo/stack/builder/resources/task_definition.rb
+- lib/ufo/stack/builder/resources/task_definition/reconstructor.rb
 - lib/ufo/stack/context.rb
+- lib/ufo/stack/custom_properties.rb
 - lib/ufo/stack/helper.rb
 - lib/ufo/stack/status.rb
+- lib/ufo/stack/template_body.rb
 - lib/ufo/status.rb
 - lib/ufo/stop.rb
 - lib/ufo/task.rb
@@ -610,6 +661,7 @@ files:
 - lib/ufo/upgrade/upgrade4.rb
 - lib/ufo/upgrade/upgrade43to45.rb
 - lib/ufo/util.rb
+- lib/ufo/utils/squeezer.rb
 - lib/ufo/version.rb
 - spec/fixtures/apps/describe_services.json
 - spec/fixtures/cfn/stack-events-complete.json
@@ -621,6 +673,7 @@ files:
 - spec/fixtures/dockerfiles/ecr/Dockerfile
 - spec/fixtures/home_existing/.aws/config
 - spec/fixtures/home_existing/.docker/config.json
+- spec/fixtures/iam_roles/task_role.rb
 - spec/fixtures/mocks/logs/awslogs.json
 - spec/fixtures/mocks/logs/no-awslogs.json
 - spec/fixtures/ps/describe_tasks.json
@@ -634,6 +687,8 @@ files:
 - spec/lib/logs_spec.rb
 - spec/lib/ps_spec.rb
 - spec/lib/register_spec.rb
+- spec/lib/role/builder_spec.rb
+- spec/lib/role/dsl_spec.rb
 - spec/lib/setting_spec.rb
 - spec/lib/ship_spec.rb
 - spec/lib/stack/status_spec.rb
@@ -660,7 +715,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.5
 signing_key: 
 specification_version: 4
 summary: AWS ECS Deploy Tool
@@ -675,6 +730,7 @@ test_files:
 - spec/fixtures/dockerfiles/ecr/Dockerfile
 - spec/fixtures/home_existing/.aws/config
 - spec/fixtures/home_existing/.docker/config.json
+- spec/fixtures/iam_roles/task_role.rb
 - spec/fixtures/mocks/logs/awslogs.json
 - spec/fixtures/mocks/logs/no-awslogs.json
 - spec/fixtures/ps/describe_tasks.json
@@ -688,6 +744,8 @@ test_files:
 - spec/lib/logs_spec.rb
 - spec/lib/ps_spec.rb
 - spec/lib/register_spec.rb
+- spec/lib/role/builder_spec.rb
+- spec/lib/role/dsl_spec.rb
 - spec/lib/setting_spec.rb
 - spec/lib/ship_spec.rb
 - spec/lib/stack/status_spec.rb