ecs_deploy_cli 0.2.1 → 0.5.0

data/lib/ecs_deploy_cli/dsl/cluster.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module EcsDeployCli
+  module DSL
+    class Cluster
+      include AutoOptions
+
+      allowed_options :instances_count, :instance_type, :ebs_volume_size, :keypair_name
+
+      def initialize(name, config)
+        @config = config
+        _options[:name] = name.to_s
+      end
+
+      def vpc(id = nil, &block)
+        @vpc = VPC.new(id)
+        @vpc.instance_exec(&block)
+      end
+
+      def as_definition
+        {
+          instances_count: 1,
+
+          device_name: '/dev/xvda',
+          ebs_volume_size: 22,
+          ebs_volume_type: 'gp2',
+
+          root_device_name: '/dev/xvdcz',
+          root_ebs_volume_size: 30,
+
+          vpc: @vpc&.as_definition
+        }.merge(_options)
+      end
+
+      class VPC
+        include AutoOptions
+        allowed_options :cidr, :subnet1, :subnet2, :subnet3
+
+        def initialize(id)
+          _options[:id] = id
+        end
+
+        def availability_zones(*values)
+          _options[:availability_zones] = values.join(',')
+        end
+
+        def subnet_ids(*values)
+          _options[:subnet_ids] = values.join(',')
+        end
+
+        def as_definition
+          validate! if _options[:id]
+
+          {
+            cidr: '10.0.0.0/16',
+            subnet1: '10.0.0.0/24',
+            subnet2: '10.0.1.0/24',
+            subnet3: '10.0.2.0/24'
+          }.merge(_options)
+        end
+
+        def validate!
+          [
+            :subnet1, :subnet_ids, :availability_zones
+          ].each { |key| raise "Missing required parameter #{key}" unless _options[key] }
+        end
+      end
+    end
+  end
+end

data/lib/ecs_deploy_cli/dsl/container.rb

@@ -56,6 +56,10 @@ module EcsDeployCli
 
       def as_definition
         {
+          cpu: 0,
+          mount_points: [],
+          port_mappings: [],
+          volumes_from: [],
           memory_reservation: nil,
           essential: true
         }.merge(_options)

data/lib/ecs_deploy_cli/dsl/parser.rb

@@ -40,8 +40,11 @@ module EcsDeployCli
         @crons[name].instance_exec(&block)
       end
 
-      def cluster(name)
+      def cluster(name, &block)
         config[:cluster] = name
+        @cluster ||= {}.with_indifferent_access
+        @cluster = Cluster.new(name, config)
+        @cluster.instance_exec(&block) if block
       end
 
       def config
@@ -58,7 +61,8 @@ module EcsDeployCli
         resolved_containers = (@containers || {}).transform_values(&:as_definition)
         resolved_tasks = (@tasks || {}).transform_values { |t| t.as_definition(resolved_containers) }
         resolved_crons = (@crons || {}).transform_values { |t| t.as_definition(resolved_tasks) }
-        [@services, resolved_tasks, resolved_crons]
+        resolved_cluster = @cluster.as_definition
+        [@services, resolved_tasks, resolved_crons, resolved_cluster]
       end
 
       def self.load(file)

data/lib/ecs_deploy_cli/dsl/service.rb

@@ -18,13 +18,42 @@ module EcsDeployCli
         _options
       end
 
+      def load_balancer(name, &block)
+        @load_balancers ||= []
+
+        load_balancer = LoadBalancer.new(name, @config)
+        load_balancer.instance_exec(&block)
+
+        @load_balancers << load_balancer
+      end
+
       def as_definition(task)
         {
           cluster: @config[:cluster],
-          service: service_name,
-          task_definition: task
+          service: _options[:service],
+          task_definition: task,
+          load_balancers: @load_balancers&.map(&:as_definition) || []
         }
       end
+
+      class LoadBalancer
+        include AutoOptions
+        allowed_options :container_name, :container_port
+
+        def initialize(name, config)
+          _options[:load_balancer_name] = name
+          @config = config
+        end
+
+        def target_group_arn(value)
+          _options[:target_group_arn] = "arn:aws:elasticloadbalancing:#{@config[:aws_region]}:#{@config[:aws_profile_id]}:targetgroup/#{value}"
+          _options.delete(:load_balancer_name)
+        end
+
+        def as_definition
+          _options
+        end
+      end
     end
   end
 end

data/lib/ecs_deploy_cli/runner.rb

@@ -7,6 +7,7 @@ require 'ecs_deploy_cli/runners/diff'
 require 'ecs_deploy_cli/runners/update_crons'
 require 'ecs_deploy_cli/runners/update_services'
 require 'ecs_deploy_cli/runners/run_task'
+require 'ecs_deploy_cli/runners/setup'
 
 module EcsDeployCli
   class Runner
@@ -14,6 +15,10 @@ module EcsDeployCli
       @parser = parser
     end
 
+    def setup!
+      EcsDeployCli::Runners::Setup.new(@parser).run!
+    end
+
     def validate!
       EcsDeployCli::Runners::Validate.new(@parser).run!
     end
@@ -26,8 +31,8 @@ module EcsDeployCli
       EcsDeployCli::Runners::RunTask.new(@parser).run!(task_name, launch_type: launch_type, security_groups: security_groups, subnets: subnets)
     end
 
-    def ssh
-      EcsDeployCli::Runners::SSH.new(@parser).run!
+    def ssh(**options)
+      EcsDeployCli::Runners::SSH.new(@parser).run!(options)
     end
 
     def diff

data/lib/ecs_deploy_cli/runners/base.rb

@@ -11,14 +11,35 @@ module EcsDeployCli
         raise NotImplementedError, 'abstract method'
       end
 
+      def update_task(definition)
+        _update_task(definition)
+      end
+
       protected
 
       def _update_task(definition)
+        definition[:container_definitions].each do |container|
+          next unless container.dig(:log_configuration, :log_driver) == 'awslogs'
+
+          _create_cloudwatch_logs_if_needed(container.dig(:log_configuration, :options, 'awslogs-group'))
+        end
+
         ecs_client.register_task_definition(
           definition
         ).to_h[:task_definition]
       end
 
+      def _create_cloudwatch_logs_if_needed(prefix)
+        log_group = cwl_client.describe_log_groups(log_group_name_prefix: prefix, limit: 1).to_h[:log_groups]
+        return if log_group.any?
+
+        cwl_client.create_log_group(log_group_name: prefix)
+        cwl_client.put_retention_policy(
+          log_group_name: prefix,
+          retention_in_days: 14
+        )
+      end
+
       def ec2_client
         @ec2_client ||= begin
           require 'aws-sdk-ec2'
@@ -46,6 +67,46 @@ module EcsDeployCli
         end
       end
 
+      def ssm_client
+        @cwl_client ||= begin
+          require 'aws-sdk-ssm'
+          Aws::SSM::Client.new(
+            profile: ENV.fetch('AWS_PROFILE', 'default'),
+            region: config[:aws_region]
+          )
+        end
+      end
+
+      def cwl_client
+        @cwl_client ||= begin
+          require 'aws-sdk-cloudwatchlogs'
+          Aws::CloudWatchLogs::Client.new(
+            profile: ENV.fetch('AWS_PROFILE', 'default'),
+            region: config[:aws_region]
+          )
+        end
+      end
+
+      def cf_client
+        @cl_client ||= begin
+          require 'aws-sdk-cloudformation'
+          Aws::CloudFormation::Client.new(
+            profile: ENV.fetch('AWS_PROFILE', 'default'),
+            region: config[:aws_region]
+          )
+        end
+      end
+
+      def iam_client
+        @iam_client ||= begin
+          require 'aws-sdk-iam'
+          Aws::IAM::Client.new(
+            profile: ENV.fetch('AWS_PROFILE', 'default'),
+            region: config[:aws_region]
+          )
+        end
+      end
+
       def config
         @parser.config
       end

data/lib/ecs_deploy_cli/runners/diff.rb

@@ -15,27 +15,57 @@ module EcsDeployCli
 
           result = ecs_client.describe_task_definition(task_definition: task_name).to_h
 
-          current = result[:task_definition].except(:revision, :status, :registered_at, :registered_by, :requires_attributes, :task_definition_arn)
+          current = cleanup_source_task(result[:task_definition])
+          definition = cleanup_source_task(definition)
 
           print_diff Hashdiff.diff(current.except(:container_definitions), definition.except(:container_definitions))
 
-          current[:container_definitions].zip(definition[:container_definitions]).each do |a, b|
-            EcsDeployCli.logger.info "Container #{a&.dig(:name) || 'NONE'} <=> #{b&.dig(:name) || 'NONE'}"
+          diff_container_definitions(
+            current[:container_definitions],
+            definition[:container_definitions]
+          )
 
-            print_diff Hashdiff.diff(a, b) if a && b
-          end
           EcsDeployCli.logger.info '---'
         end
       end
 
+      private
+
+      def diff_container_definitions(first, second)
+        first.zip(second).each do |a, b|
+          EcsDeployCli.logger.info "Container #{a&.dig(:name) || 'NONE'} <=> #{b&.dig(:name) || 'NONE'}"
+
+          next if !a || !b
+
+          sort_envs! a
+          sort_envs! b
+
+          print_diff Hashdiff.diff(a.delete_if { |_, v| v.nil? }, b.delete_if { |_, v| v.nil? })
+        end
+      end
+
+      def sort_envs!(definition)
+        return unless definition[:environment]
+
+        definition[:environment].sort_by! { |e| e[:name] }
+      end
+
+      def cleanup_source_task(task)
+        task.except(
+          :revision, :compatibilities, :status, :registered_at, :registered_by,
+          :requires_attributes, :task_definition_arn
+        ).delete_if { |_, v| v.nil? }
+      end
+
       def print_diff(diff)
         diff.each do |(op, path, *values)|
-          if op == '-'
+          case op
+          when '-'
             EcsDeployCli.logger.info "#{op} #{path} => #{values.join(' ')}".colorize(:red)
-          elsif op == '+'
+          when '+'
             EcsDeployCli.logger.info "#{op} #{path} => #{values.join(' ')}".colorize(:green)
           else
-            EcsDeployCli.logger.info "#{op} #{path} => #{values.join(' ')}".colorize(:light_blue)
+            EcsDeployCli.logger.info "#{op} #{path} => #{values.join(' ')}".colorize(:yellow)
           end
         end
       end

data/lib/ecs_deploy_cli/runners/logs.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module EcsDeployCli
+  module Runners
+    class Logs < Base
+      def run!
+        # _, tasks, = @parser.resolve
+
+        # tasks.
+        # cwl_client.
+      end
+    end
+  end
+end

data/lib/ecs_deploy_cli/runners/setup.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module EcsDeployCli
+  module Runners
+    class Setup < Base
+      REQUIRED_ECS_ROLES = {
+        'ecsInstanceRole' => 'https://docs.aws.amazon.com/batch/latest/userguide/instance_IAM_role.html',
+        'ecsTaskExecutionRole' => 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_execution_IAM_role.html'
+      }.freeze
+      class SetupError < StandardError; end
+
+      def run!
+        services, resolved_tasks, _, cluster_options = @parser.resolve
+
+        ensure_ecs_roles_exists!
+
+        setup_cluster! cluster_options
+        setup_services! services, resolved_tasks: resolved_tasks
+      rescue SetupError => e
+        EcsDeployCli.logger.info e.message
+      end
+
+      private
+
+      def setup_cluster!(cluster_options)
+        if cluster_exists?
+          EcsDeployCli.logger.info 'Cluster already created, skipping.'
+          return
+        end
+
+        EcsDeployCli.logger.info "Creating cluster #{config[:cluster]}..."
+
+        create_keypair_if_required! cluster_options
+        params = create_params(cluster_options)
+
+        ecs_client.create_cluster(
+          cluster_name: config[:cluster]
+        )
+        EcsDeployCli.logger.info 'Cluster created, now running cloudformation...'
+
+        stack_name = "EC2ContainerService-#{config[:cluster]}"
+
+        cf_client.create_stack(
+          stack_name: stack_name,
+          template_body: File.read(File.join(__dir__, '..', 'cloudformation', 'default.yml')),
+          on_failure: 'ROLLBACK',
+          parameters: format_cloudformation_params(params)
+        )
+
+        cf_client.wait_until(:stack_create_complete, { stack_name: stack_name }, delay: 30, max_attempts: 120)
+        EcsDeployCli.logger.info "Cluster #{config[:cluster]} created! 🎉"
+      end
+
+      def setup_services!(services, resolved_tasks:)
+        services.each do |service_name, service_definition|
+          existing_services = ecs_client.describe_services(cluster: config[:cluster], services: [service_name]).to_h[:services].filter { |s| s[:status] != 'INACTIVE' }
+          if existing_services.any?
+            EcsDeployCli.logger.info "Service #{service_name} already created, skipping."
+            next
+          end
+
+          EcsDeployCli.logger.info "Creating service #{service_name}..."
+          task_definition = _update_task resolved_tasks[service_definition.options[:task]]
+          task_name = "#{task_definition[:family]}:#{task_definition[:revision]}"
+
+          ecs_client.create_service(
+            cluster: config[:cluster],
+            desired_count: 1, # FIXME: this should be a parameter
+            load_balancers: service_definition.as_definition(task_definition)[:load_balancers],
+            service_name: service_name,
+            task_definition: task_name
+          )
+          EcsDeployCli.logger.info "Service #{service_name} created!"
+        end
+      end
+
+      def create_params(cluster_options)
+        raise ArgumentError, 'Missing vpc configuration' unless cluster_options[:vpc]
+
+        {
+          'AsgMaxSize' => cluster_options[:instances_count],
+          'AutoAssignPublicIp' => 'INHERIT',
+          'ConfigureDataVolume' => false,
+          'ConfigureRootVolume' => true,
+          'DeviceName' => cluster_options[:device_name],
+          'EbsVolumeSize' => cluster_options[:ebs_volume_size],
+          'EbsVolumeType' => cluster_options[:ebs_volume_type],
+          'EcsAmiId' => load_ami_id,
+          'EcsClusterName' => config[:cluster],
+          'EcsEndpoint' => nil,
+          'EcsInstanceType' => cluster_options[:instance_type],
+          'IamRoleInstanceProfile' => "arn:aws:iam::#{config[:aws_profile_id]}:instance-profile/ecsInstanceRole",
+          'IamSpotFleetRoleArn' => nil,
+          'IsWindows' => false,
+          'KeyName' => cluster_options[:keypair_name],
+          'RootDeviceName' => cluster_options[:root_device_name],
+          'RootEbsVolumeSize' => cluster_options[:root_ebs_volume_size],
+
+          ##### TODO: Implement this feature
+          'SecurityGroupId' => nil,
+          'SecurityIngressCidrIp' => '0.0.0.0/0',
+          'SecurityIngressFromPort' => 80,
+          'SecurityIngressToPort' =>	80,
+          #####
+
+          ##### TODO: Implement this feature
+          'SpotAllocationStrategy' =>	'diversified',
+          'SpotPrice' => nil,
+          'UseSpot' =>	false,
+          #####
+
+          'UserData' => "#!/bin/bash\necho ECS_CLUSTER=#{config[:cluster]} >> /etc/ecs/ecs.config;echo ECS_BACKEND_HOST= >> /etc/ecs/ecs.config;",
+          'VpcAvailabilityZones' => cluster_options.dig(:vpc, :availability_zones),
+          'VpcCidr' => cluster_options.dig(:vpc, :cidr),
+          'SubnetCidr1' => cluster_options.dig(:vpc, :subnet1),
+          'SubnetCidr2' => cluster_options.dig(:vpc, :subnet2),
+          'SubnetCidr3' => cluster_options.dig(:vpc, :subnet3),
+
+          'VpcId' => cluster_options.dig(:vpc, :id),
+          'SubnetIds' => cluster_options.dig(:vpc, :subnet_ids)
+        }
+      end
+
+      def cluster_exists?
+        clusters = ecs_client.describe_clusters(clusters: [config[:cluster]]).to_h[:clusters]
+
+        clusters.filter { |c| c[:status] != 'INACTIVE' }.length == 1
+      end
+
+      def ensure_ecs_roles_exists!
+        REQUIRED_ECS_ROLES.each do |role_name, link|
+          iam_client.get_role(role_name: role_name).to_h
+        rescue Aws::IAM::Errors::NoSuchEntity
+          raise SetupError, "IAM Role #{role_name} does not exist. Please create it: #{link}."
+        end
+      end
+
+      def create_keypair_if_required!(cluster_options)
+        ec2_client.describe_key_pairs(key_names: [cluster_options[:keypair_name]]).to_h[:key_pairs]
+      rescue Aws::EC2::Errors::InvalidKeyPairNotFound
+        EcsDeployCli.logger.info "Keypair \"#{cluster_options[:keypair_name]}\" not found, creating it..."
+        key_material = ec2_client.create_key_pair(key_name: cluster_options[:keypair_name]).to_h[:key_material]
+        File.write("#{cluster_options[:keypair_name]}.pem", key_material)
+        EcsDeployCli.logger.info "Created PEM file at #{Dir.pwd}/#{cluster_options[:keypair_name]}.pem"
+      end
+
+      def format_cloudformation_params(params)
+        params.map { |k, v| { parameter_key: k, parameter_value: v.to_s } }
+      end
+
+      def load_ami_id
+        ami_data = ssm_client.get_parameter(
+          name: '/aws/service/ecs/optimized-ami/amazon-linux-2/recommended'
+        ).to_h[:parameter]
+
+        ami_details = JSON.parse(ami_data[:value]).with_indifferent_access
+
+        ami_details[:image_id]
+      end
+    end
+  end
+end