ecs_deploy_cli 0.2.2 → 0.5.1

data/lib/ecs_deploy_cli/runners/update_crons.rb

@@ -8,16 +8,13 @@ module EcsDeployCli
 
         crons.each do |cron_name, cron_definition|
           task_definition = tasks[cron_definition[:task_name]]
-          raise "Undefined task #{cron_definition[:task_name].inspect} in (#{tasks.keys.inspect})" unless task_definition
+          unless task_definition
+            raise "Undefined task #{cron_definition[:task_name].inspect} in (#{tasks.keys.inspect})"
+          end
 
           updated_task = _update_task(task_definition)
 
-          current_target = cwe_client.list_targets_by_rule(
-            {
-              rule: cron_name,
-              limit: 1
-            }
-          ).to_h[:targets].first
+          current_target = load_or_init_target(cron_name)
 
           cwe_client.put_rule(
             cron_definition[:rule]
@@ -36,6 +33,18 @@ module EcsDeployCli
           EcsDeployCli.logger.info "Deployed scheduled task \"#{cron_name}\"!"
         end
       end
+
+      private
+
+      def load_or_init_target(cron_name)
+        cwe_client.list_targets_by_rule({ rule: cron_name, limit: 1 }).to_h[:targets].first
+      rescue Aws::CloudWatchEvents::Errors::ResourceNotFoundException
+        {
+          id: cron_name,
+          arn: "arn:aws:ecs:#{config[:aws_region]}:#{config[:aws_profile_id]}:cluster/#{config[:cluster]}",
+          role_arn: "arn:aws:iam::#{config[:aws_profile_id]}:role/ecsEventsRole"
+        }
+      end
     end
   end
 end

data/lib/ecs_deploy_cli/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EcsDeployCli
-  VERSION = '0.2.2'
+  VERSION = '0.5.1'
 end

data/spec/ecs_deploy_cli/cli_spec.rb

@@ -48,6 +48,14 @@ describe EcsDeployCli::CLI do
       described_class.start(['run-task', 'yourproject', '--subnets', 'subnet-123123', '--file', 'spec/support/ECSFile'])
     end
 
+    it 'runs setup' do
+      expect(runner).to receive(:setup!)
+      described_class.no_commands do
+        expect_any_instance_of(described_class).to receive(:runner).at_least(:once).and_return(runner)
+      end
+      expect { described_class.start(['setup', '--file', 'spec/support/ECSFile']) }.to output(/[WARNING]/).to_stdout
+    end
+
     it 'runs deploy' do
       expect(runner).to receive(:update_crons!)
       expect(runner).to receive(:update_services!).with(timeout: 500)

data/spec/ecs_deploy_cli/dsl/cluster_spec.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe EcsDeployCli::DSL::Cluster do
+  context 'defines cluster data' do
+    subject { described_class.new('mydata-cluster', { aws_profile_id: '123123', aws_region: 'eu-central-1' }) }
+
+    it '#vpc' do
+      subject.instances_count 1
+      subject.instance_type 't2.small'
+      subject.keypair_name 'test'
+
+      subject.vpc do
+        cidr '11.0.0.0/16'
+        subnet1 '11.0.0.0/24'
+        subnet2 '11.0.1.0/24'
+        subnet3 '11.0.2.0/24'
+        subnet_ids 'subnet-123', 'subnet-321', 'subnet-333'
+
+        availability_zones 'eu-central-1a', 'eu-central-1b', 'eu-central-1c'
+      end
+
+      expect(subject.as_definition).to eq(
+        {
+          device_name: '/dev/xvda',
+          ebs_volume_size: 22,
+          ebs_volume_type: 'gp2',
+          instances_count: 1,
+          instance_type: 't2.small',
+          keypair_name: 'test',
+          name: 'mydata-cluster',
+          root_device_name: '/dev/xvdcz',
+          root_ebs_volume_size: 30,
+          vpc: {
+            availability_zones: 'eu-central-1a,eu-central-1b,eu-central-1c',
+            cidr: '11.0.0.0/16',
+            id: nil,
+            subnet1: '11.0.0.0/24',
+            subnet2: '11.0.1.0/24',
+            subnet3: '11.0.2.0/24',
+            subnet_ids: 'subnet-123,subnet-321,subnet-333'
+          }
+        }
+      )
+    end
+  end
+end

data/spec/ecs_deploy_cli/dsl/service_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe EcsDeployCli::DSL::Service do
+  context 'defines service data' do
+    subject { described_class.new('test', { aws_profile_id: '123123', aws_region: 'eu-central-1' }) }
+
+    it 'has the correct name' do
+      expect(subject.as_definition({})[:service]).to eq('test')
+    end
+
+    it '#load_balancer' do
+      subject.load_balancer :'yourproject-load-balancer' do
+        target_group_arn 'loader-target-group/123abc'
+        container_name :web
+        container_port 80
+      end
+
+      expect(subject.as_definition({})[:load_balancers]).to eq(
+        [
+          {
+            container_name: :web,
+            container_port: 80,
+            target_group_arn: 'arn:aws:elasticloadbalancing:eu-central-1:123123:targetgroup/loader-target-group/123abc'
+          }
+        ]
+      )
+    end
+  end
+end

data/spec/ecs_deploy_cli/runner_spec.rb

@@ -2,14 +2,22 @@
 
 require 'spec_helper'
 require 'aws-sdk-cloudwatchevents'
+require 'aws-sdk-cloudwatchlogs'
 require 'aws-sdk-ec2'
+require 'aws-sdk-ssm'
+require 'aws-sdk-cloudformation'
+require 'aws-sdk-iam'
 
 describe EcsDeployCli::Runner do
   context 'defines task data' do
     let(:parser) { EcsDeployCli::DSL::Parser.load('spec/support/ECSFile') }
     subject { described_class.new(parser) }
+    let(:mock_iam_client) { Aws::IAM::Client.new(stub_responses: true) }
+    let(:mock_cf_client) { Aws::CloudFormation::Client.new(stub_responses: true) }
+    let(:mock_ssm_client) { Aws::SSM::Client.new(stub_responses: true) }
     let(:mock_ecs_client) { Aws::ECS::Client.new(stub_responses: true) }
     let(:mock_ec2_client) { Aws::EC2::Client.new(stub_responses: true) }
+    let(:mock_cwl_client) { Aws::CloudWatchLogs::Client.new(stub_responses: true) }
     let(:mock_cwe_client) do
       Aws::CloudWatchEvents::Client.new(stub_responses: true)
     end
@@ -84,28 +92,142 @@ describe EcsDeployCli::Runner do
         ENV['AWS_REGION'] = nil
       end
 
-      it '#ssh' do
-        expect(mock_ecs_client).to receive(:list_container_instances).and_return({ container_instance_arns: ['arn:123123'] })
-        expect(mock_ecs_client).to receive(:describe_container_instances).and_return(double(container_instances: [double(ec2_instance_id: 'i-123123')]))
-
-        expect(mock_ec2_client).to receive(:describe_instances)
-          .with(instance_ids: ['i-123123'])
-          .and_return(
-            double(reservations: [
-                     double(instances: [double(public_dns_name: 'test.com')])
-                   ])
+      context '#setup!' do
+        before do
+          mock_ssm_client.stub_responses(
+            :get_parameter, {
+              parameter: {
+                name: '/aws/service/ecs/optimized-ami/amazon-linux-2/recommended',
+                type: 'String',
+                value: '{"schema_version":1,"image_name":"amzn2-ami-ecs-hvm-2.0.20210331-x86_64-ebs","image_id":"ami-03bbf53329af34379","os":"Amazon Linux 2","ecs_runtime_version":"Docker version 19.03.13-ce","ecs_agent_version":"1.51.0"}'
+              }
+            }
           )
+        end
+
+        it 'setups the cluster correctly' do
+          expect(mock_ec2_client).to receive(:describe_key_pairs).and_return(key_pairs: [{ key_id: 'some' }])
+
+          expect(mock_iam_client).to receive(:get_role).at_least(:once).and_return({ role: { arn: 'some' } })
+          expect(mock_cf_client).to receive(:wait_until)
+          expect(mock_ecs_client).to receive(:create_service)
 
-        expect(Process).to receive(:fork) do |&block|
-          block.call
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ec2_client).at_least(:once).and_return(mock_ec2_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:iam_client).at_least(:once).and_return(mock_iam_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwl_client).at_least(:once).and_return(mock_cwl_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ssm_client).at_least(:once).and_return(mock_ssm_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cf_client).at_least(:once).and_return(mock_cf_client)
+
+          subject.setup!
         end
-        expect(Process).to receive(:wait)
 
-        expect_any_instance_of(EcsDeployCli::Runners::SSH).to receive(:exec).with('ssh ec2-user@test.com')
-        expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
-        expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ec2_client).at_least(:once).and_return(mock_ec2_client)
+        it 'fails if the IAM role is not setup' do
+          expect(EcsDeployCli.logger).to receive(:info).at_least(:once) do |message|
+            puts message
+          end
+
+          expect(mock_iam_client).to receive(:get_role).at_least(:once) do
+            raise Aws::IAM::Errors::NoSuchEntity.new(nil, 'some')
+          end
+
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:iam_client).at_least(:once).and_return(mock_iam_client)
+
+          expect { subject.setup! }.to output(/IAM Role ecsInstanceRole does not exist./).to_stdout
+        end
+
+        it 'fails if the cluster is already there' do
+          expect(mock_ecs_client).to receive(:describe_clusters).and_return(clusters: [{}])
+
+          expect(EcsDeployCli.logger).to receive(:info).at_least(:once) do |message|
+            puts message
+          end
+
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwl_client).at_least(:once).and_return(mock_cwl_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:iam_client).at_least(:once).and_return(mock_iam_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
+
+          expect { subject.setup! }.to output(/Cluster already created, skipping./).to_stdout
+        end
+
+        it 'creates the keypair if not there' do
+          expect(mock_ec2_client).to receive(:describe_key_pairs) do
+            raise Aws::EC2::Errors::InvalidKeyPairNotFound.new(nil, 'some')
+          end
+
+          expect(mock_ec2_client).to receive(:create_key_pair) { raise 'created keypair' }
+
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ec2_client).at_least(:once).and_return(mock_ec2_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:iam_client).at_least(:once).and_return(mock_iam_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
+
+          expect { subject.setup! }.to raise_error('created keypair')
+        end
+      end
+
+      context '#ssh' do
+        it 'runs ssh on a single container instance' do
+          expect(mock_ecs_client).to receive(:list_tasks).and_return({ task_arns: ['arn:123123'] })
+          expect(mock_ecs_client).to receive(:describe_tasks).and_return({ tasks: [{ container_instance_arn: 'arn:instance:123123' }] })
+          expect(mock_ecs_client).to receive(:describe_container_instances).and_return(double(container_instances: [double(ec2_instance_id: 'i-123123')]))
+
+          expect(mock_ec2_client).to receive(:describe_instances)
+            .with(instance_ids: ['i-123123'])
+            .and_return(
+              double(
+                reservations: [
+                  double(instances: [double(public_dns_name: 'test.com')])
+                ]
+              )
+            )
+
+          expect(Process).to receive(:fork) do |&block|
+            block.call
+          end
+          expect(Process).to receive(:wait)
+
+          expect_any_instance_of(EcsDeployCli::Runners::SSH).to receive(:exec).with('ssh ec2-user@test.com')
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ec2_client).at_least(:once).and_return(mock_ec2_client)
+
+          subject.ssh
+        end
+
+        it 'prompts which instance if there are multiple ones' do
+          expect(mock_ecs_client).to receive(:list_tasks).and_return({ task_arns: ['arn:123123', 'arn:321321'] })
+          expect(mock_ecs_client).to receive(:describe_tasks).and_return(
+            {
+              tasks: [
+                { container_instance_arn: 'arn:instance:123123' },
+                { container_instance_arn: 'arn:instance:321321' }
+              ]
+            }
+          )
+          expect(mock_ecs_client).to receive(:describe_container_instances).and_return(
+            double(container_instances: [double(ec2_instance_id: 'i-123123'), double(ec2_instance_id: 'i-321321')])
+          )
+
+          expect(STDIN).to receive(:gets).and_return('2')
 
-        subject.ssh
+          expect(mock_ec2_client).to receive(:describe_instances)
+            .with(instance_ids: ['i-321321'])
+            .and_return(
+              double(reservations: [
+                       double(instances: [double(public_dns_name: 'test.com')])
+                     ])
+            )
+
+          expect(Process).to receive(:fork) do |&block|
+            block.call
+          end
+          expect(Process).to receive(:wait)
+
+          expect_any_instance_of(EcsDeployCli::Runners::SSH).to receive(:exec).with('ssh ec2-user@test.com')
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ec2_client).at_least(:once).and_return(mock_ec2_client)
+
+          subject.ssh
+        end
       end
 
       it '#diff' do
@@ -124,20 +246,38 @@ describe EcsDeployCli::Runner do
 
         mock_cwe_client.stub_responses(:run_task)
 
+        expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwl_client).at_least(:once).and_return(mock_cwl_client)
         expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
 
         subject.run_task!('yourproject-cron', launch_type: 'FARGATE', security_groups: [], subnets: [])
       end
 
-      it '#update_crons!' do
-        mock_ecs_client.stub_responses(:register_task_definition, { task_definition: { family: 'some', revision: 1, task_definition_arn: 'arn:task:eu-central-1:xxxx' } })
+      context '#update_crons!' do
+        it 'creates missing crons' do
+          mock_ecs_client.stub_responses(:register_task_definition, { task_definition: { family: 'some', revision: 1, task_definition_arn: 'arn:task:eu-central-1:xxxx' } })
 
-        mock_cwe_client.stub_responses(:list_targets_by_rule, { targets: [{ id: '123', arn: 'arn:123' }] })
+          expect(mock_cwe_client).to receive(:list_targets_by_rule) do
+            raise Aws::CloudWatchEvents::Errors::ResourceNotFoundException.new(nil, 'some')
+          end
 
-        expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
-        expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwe_client).at_least(:once).and_return(mock_cwe_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwl_client).at_least(:once).and_return(mock_cwl_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwe_client).at_least(:once).and_return(mock_cwe_client)
+
+          subject.update_crons!
+        end
+
+        it 'updates existing crons' do
+          mock_ecs_client.stub_responses(:register_task_definition, { task_definition: { family: 'some', revision: 1, task_definition_arn: 'arn:task:eu-central-1:xxxx' } })
 
-        subject.update_crons!
+          mock_cwe_client.stub_responses(:list_targets_by_rule, { targets: [{ id: '123', arn: 'arn:123' }] })
+
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwl_client).at_least(:once).and_return(mock_cwl_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
+          expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwe_client).at_least(:once).and_return(mock_cwe_client)
+
+          subject.update_crons!
+        end
       end
 
       it '#update_services!' do
@@ -149,6 +289,7 @@ describe EcsDeployCli::Runner do
         )
         expect(mock_ecs_client).to receive(:wait_until)
 
+        expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwl_client).at_least(:once).and_return(mock_cwl_client)
         expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
 
         subject.update_services!

data/spec/ecs_deploy_cli/runners/base_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'aws-sdk-cloudwatchevents'
+require 'aws-sdk-cloudwatchlogs'
+require 'aws-sdk-ec2'
+require 'aws-sdk-ssm'
+require 'aws-sdk-cloudformation'
+
+describe EcsDeployCli::Runners::Base do
+  let(:parser) { EcsDeployCli::DSL::Parser.load('spec/support/ECSFile') }
+  subject { described_class.new(parser) }
+  let(:mock_cf_client) { Aws::CloudFormation::Client.new(stub_responses: true) }
+  let(:mock_ssm_client) { Aws::SSM::Client.new(stub_responses: true) }
+  let(:mock_ecs_client) { Aws::ECS::Client.new(stub_responses: true) }
+  let(:mock_ec2_client) { Aws::EC2::Client.new(stub_responses: true) }
+  let(:mock_cwl_client) { Aws::CloudWatchLogs::Client.new(stub_responses: true) }
+  let(:mock_cwe_client) do
+    Aws::CloudWatchEvents::Client.new(stub_responses: true)
+  end
+
+  around(:each) do |example|
+    ENV['AWS_PROFILE_ID'] = '123123123'
+    ENV['AWS_REGION'] = 'us-east-1'
+    example.run
+    ENV['AWS_PROFILE_ID'] = nil
+    ENV['AWS_REGION'] = nil
+  end
+
+  context '#update_task' do
+    it 'creates cloud watch logs if missing' do
+      _, tasks, = parser.resolve
+
+      expect(mock_cwl_client).to receive(:describe_log_groups).at_least(:once).and_return({ log_groups: [] })
+      expect(mock_cwl_client).to receive(:create_log_group).at_least(:once)
+      expect(mock_ecs_client).to receive(:register_task_definition).at_least(:once).and_return({ task_definition: { family: 'some', revision: '1' } })
+
+      expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwl_client).at_least(:once).and_return(mock_cwl_client)
+      expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
+
+      subject.update_task(tasks.values.first)
+    end
+
+    it 'creates no cloudwatch log group if is already there' do
+      _, tasks, = parser.resolve
+
+      expect(mock_cwl_client).to receive(:describe_log_groups).at_least(:once).and_return({ log_groups: [{}] })
+      expect(mock_cwl_client).not_to receive(:create_log_group)
+      expect(mock_ecs_client).to receive(:register_task_definition).at_least(:once).and_return({ task_definition: { family: 'some', revision: '1' } })
+
+      expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:cwl_client).at_least(:once).and_return(mock_cwl_client)
+      expect_any_instance_of(EcsDeployCli::Runners::Base).to receive(:ecs_client).at_least(:once).and_return(mock_ecs_client)
+
+      subject.update_task(tasks.values.first)
+    end
+  end
+end

data/spec/support/ECSFile

@@ -4,7 +4,14 @@ aws_region ENV.fetch('AWS_REGION', 'eu-central-1')
 aws_profile_id ENV['AWS_PROFILE_ID']
 
 # Defining the cluster name
-cluster 'yourproject-cluster'
+cluster 'yourproject-cluster' do
+  instance_type 't3.small'
+  keypair_name 'test'
+
+  vpc do
+    availability_zones 'eu-central-1a', 'eu-central-1b', 'eu-central-1c'
+  end
+end
 
 # This is used as a template for the next two containers, it will not be used inside a task
 container :base_container do
@@ -49,6 +56,11 @@ end
 # The main service
 service :'yourproject-service' do
   task :yourproject
+  load_balancer :'yourproject-load-balancer' do
+    target_group_arn 'loader-target-group/123abc'
+    container_name :web
+    container_port 80
+  end
 end
 
 # A task for cron jobs