ecs_deploy 0.3.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b00504a25168d2bc3b6c700c8ec8ff5ccded3ff3
-  data.tar.gz: 97c42178d1adec1876d319ddb8776cf28f687231
+SHA256:
+  metadata.gz: f4d2cb1f5439702efac5ec26e9cd0c47c6225e0106c05b6f73b7860098098488
+  data.tar.gz: 29007dc6fdeb7b64b82a0113ac78ad00025ec9693bca1b238ccca3c507065c01
 SHA512:
-  metadata.gz: fa259524dc7c1c0d0b2a61228d61014106df9ecc9c5b0ae21eda2300917935103f4122d8d2682e51f4232da6d9037b79a7f4646ff189915a64c12330bb39a33c
-  data.tar.gz: b2d4350db7dd4083fceef62a38c9dc229e20bdfec903a09ee5cfbf5ba8a4a9089c1177fb23a7dfb2ba1659515f2b94e12c9a3df15b4ebaa9c03f3c6f6942c6f3
+  metadata.gz: 42481f7631cebe62057fbd9e2c0fc6ba61ff0ee198e5d537958934df8a001c76ef5ddb6f8965c2f7db57d0098aac3b79cb43a677cdbaa95609107d29826892b8
+  data.tar.gz: 28a2d8072e7f94a84301e0f0607fc02c469561d1f39da439cf87d3f7b64c12e301bb77a13bb96e0823870b43af36f5f35ee3e4984567ac57ebe5ae759342d61c

data/.gitignore

@@ -8,4 +8,5 @@
 /spec/reports/
 /tmp/
 
+.rspec_status
 .envrc

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - ruby-2.4.9
+  - ruby-2.5.7
+  - ruby-2.6.5

data/CHANGELOG.md

@@ -0,0 +1,150 @@
+# v1.0
+
+## Release v1.0.3 - 2021/11/17
+
+### Bug fixes
+* Fix bug that `InstanceFluctuationManager#decrement` tries to detach instances whose status is 'DEREGISTERING'
+  https://github.com/reproio/ecs_deploy/pull/72
+
+### Enhancement
+* Add a cluster name to deployment logs
+  https://github.com/reproio/ecs_deploy/pull/71
+
+
+## Release v1.0.2 - 2021/05/26
+
+### Enhancement
+
+* add option enable_execute_command to support ECS Exec
+  https://github.com/reproio/ecs_deploy/pull/69
+
+## Release v1.0.1 - 2021/05/19
+
+### Enhancement
+
+* retry register_task_definition by AWS SDK feature
+  https://github.com/reproio/ecs_deploy/pull/67
+* Support Ruby 3.0
+  https://github.com/reproio/ecs_deploy/pull/66
+* Wait until stop old tasks
+  https://github.com/reproio/ecs_deploy/pull/65
+* Add prioritized_over_upscale_triggers option to triggers
+  https://github.com/reproio/ecs_deploy/pull/62
+* Display only unstable services in EcsDeploy::Service#wait_all_running
+  https://github.com/reproio/ecs_deploy/pull/61
+
+## Release v1.0.0 - 2019/12/24
+
+### New feature
+
+* Add tasks to deploy the application faster
+  https://github.com/reproio/ecs_deploy/pull/57
+
+### Enhancement
+
+* Add parameters `ecs_wait_until_services_stable_max_attempts` and `ecs_wait_until_services_stable_delay`
+  https://github.com/reproio/ecs_deploy/pull/30
+* Detect region automatically according to AWS SDK
+  https://github.com/reproio/ecs_deploy/pull/31
+* Support new features of ECS to support Fargate
+  https://github.com/reproio/ecs_deploy/pull/32
+* Ignore running tasks which don't belong to the ECS services on deregistering container instances
+  https://github.com/reproio/ecs_deploy/pull/33
+* Drop AWS SDK 2 support
+  https://github.com/reproio/ecs_deploy/pull/34
+* Support scheduling_strategy option
+  https://github.com/reproio/ecs_deploy/pull/35
+* Support execution_role_arn on task_definition
+  https://github.com/reproio/ecs_deploy/pull/36
+* Support spot fleet requests and container instance draining
+  https://github.com/reproio/ecs_deploy/pull/40
+* Add network_configuration paramters to ScheduledTask
+  https://github.com/reproio/ecs_deploy/pull/46
+* Support tagging ECS resources
+  https://github.com/reproio/ecs_deploy/pull/48
+  https://github.com/reproio/ecs_deploy/pull/49
+* Wait for stopping tasks until tasks stop
+  https://github.com/reproio/ecs_deploy/pull/50
+* Improve performance when start tasks
+  https://github.com/reproio/ecs_deploy/pull/53
+* Improve stability of auto scaling groups managed by ecs_auto_scaler
+  https://github.com/reproio/ecs_deploy/pull/55
+
+### Bug fixes
+
+* Fix infinite loop that occurs when there are more than 100 container instances
+  https://github.com/reproio/ecs_deploy/pull/38
+* Fix errors that occur on decreasing more than 20 container instances
+  https://github.com/reproio/ecs_deploy/pull/39
+
+# Ancient releases
+
+## Release v0.3.2 - 2017/23/10
+
+### Enhancement
+
+* Remove execution feature
+  https://github.com/reproio/ecs_deploy/pull/24
+* Support container overrides in scheduled tasks
+  https://github.com/reproio/ecs_deploy/pull/26
+
+### Bug fixes
+
+* Fix deployment errors that occur when `ecs_scheduled_tasks` is not set
+  https://github.com/reproio/ecs_deploy/pull/27
+
+## Release v0.3.1 - 2017/04/08
+
+### Bug fixes
+
+* Fix block parameter name
+
+## Release v0.3.0 - 2017/03/08
+
+### New feature
+
+* Support ScheduledTask deployment
+  https://github.com/reproio/ecs_deploy/pull/22
+
+### Enhancement
+
+* Support network_mode and placement_constraints
+* Introduce `ecs_registered_tasks` capistrano variable
+  https://github.com/reproio/ecs_deploy/pull/23
+
+### Bug fixes
+
+* Filter inactive services
+  https://github.com/reproio/ecs_deploy/pull/19
+* Wait 10 services at once
+  https://github.com/reproio/ecs_deploy/pull/20
+  https://github.com/reproio/ecs_deploy/pull/21
+
+## Release v0.2.0 - 2016/31/10
+
+### Enhancement
+
+* Support task role arn
+  https://github.com/reproio/ecs_deploy/pull/13
+* Make the scale-in process safe
+  https://github.com/reproio/ecs_deploy/pull/14
+* Support ALB
+  https://github.com/reproio/ecs_deploy/pull/15
+
+## Release v0.1.2 - 2016/28/07
+
+### Bug fixes
+
+* Fix rollback bug
+  https://github.com/reproio/ecs_deploy/pull/11
+
+## Release v0.1.1 - 2016/03/07
+
+### Bug fixes
+
+* Add missing desired_count for backend services
+  https://github.com/reproio/ecs_deploy/pull/9
+
+## Release v0.1.0 - 2016/27/06
+
+Initial release.

data/README.md

@@ -31,6 +31,8 @@ set :ecs_default_cluster, "ecs-cluster-name"
 set :ecs_region, %w(ap-northeast-1) # optional, if nil, use environment variable
 set :ecs_service_role, "customEcsServiceRole" # default: ecsServiceRole
 set :ecs_deploy_wait_timeout, 600 # default: 300
+set :ecs_wait_until_services_stable_max_attempts, 40 # optional
+set :ecs_wait_until_services_stable_delay, 15 # optional
 
 set :ecs_tasks, [
   {
@@ -92,7 +94,7 @@ set :ecs_scheduled_tasks, [
   {
     cluster: "default", # Unless this key, use fetch(:ecs_default_cluster)
     rule_name: "schedule_name",
-    schedule_expression: "cron(0 12 * * ? *"),
+    schedule_expression: "cron(0 12 * * ? *)",
     description: "schedule_description", # Optional
     target_id: "task_name", # Unless this key, use task_definition_name
     task_definition_name: "myapp-#{fetch(:rails_env)}",
@@ -187,7 +189,16 @@ And rollback
 
 ## Autoscaler
 
-Write config file (YAML format).
+The autoscaler of `ecs_deploy` supports auto scaling of ECS services and clusters.
+
+### Prerequisits
+
+* You use a ECS cluster whose instances belong to either an auto scaling group or a spot fleet request
+* You have CloudWatch alarms and you want to scale services when their state changes
+
+### How to use autoscaler
+
+First, write a configuration file (YAML format) like below:
 
 ```yaml
 # ポーリング時にupscale_triggersに指定した状態のalarmがあればstep分serviceとinstanceを増やす (max_task_countまで)
@@ -200,37 +211,275 @@ polling_interval: 60
 auto_scaling_groups:
   - name: ecs-cluster-nodes
     region: ap-northeast-1
-    buffer: 1 # タスク数に対する余剰のインスタンス数
-
-services:
-  - name: repro-api-production
     cluster: ecs-cluster
+    # autoscaler will set the capacity to (buffer + desired_tasks * required_capacity).
+    # Adjust this value if it takes much time to prepare ECS instances and launch new tasks.
+    buffer: 1
+    services:
+      - name: repro-api-production
+        step: 1
+        idle_time: 240
+        max_task_count: [10, 25]
+        scheduled_min_task_count:
+          - {from: "1:45", to: "4:30", count: 8}
+        cooldown_time_for_reach_max: 600
+        min_task_count: 0
+        # Required capacity per task (default: 1)
+        # You should specify "binpack" as task placement strategy if the value is less than 1 and you use an auto scaling group.
+        required_capacity: 0.5
+        upscale_triggers:
+          - alarm_name: "ECS [repro-api-production] CPUUtilization"
+            state: ALARM
+          - alarm_name: "ELB repro-api-a HTTPCode_Backend_5XX"
+            state: ALARM
+            step: 2
+        downscale_triggers:
+          - alarm_name: "ECS [repro-api-production] CPUUtilization (low)"
+            state: OK
+
+spot_fleet_requests:
+  - id: sfr-354de735-2c17-4565-88c9-10ada5b957e5
     region: ap-northeast-1
-    auto_scaling_group_name: ecs-cluster-nodes
-    step: 1
-    idle_time: 240
-    max_task_count: [10, 25]
-    scheduled_min_task_count:
-      - {from: "1:45", to: "4:30", count: 8}
-    cooldown_time_for_reach_max: 600
-    min_task_count: 0
-    upscale_triggers:
-      - alarm_name: "ECS [repro-api-production] CPUUtilization"
-        state: ALARM
-      - alarm_name: "ELB repro-api-a HTTPCode_Backend_5XX"
-        state: ALARM
-        step: 2
-    downscale_triggers:
-      - alarm_name: "ECS [repro-api-production] CPUUtilization (low)"
-        state: OK
+    cluster: ecs-cluster-for-worker
+    buffer: 1
+    services:
+      - name: repro-worker-production
+        step: 1
+        idle_time: 240
+        cooldown_time_for_reach_max: 600
+        min_task_count: 0
+        # Required capacity per task (default: 1)
+        # The capacity assumes that WeightedCapacity is equal to the number of vCPUs.
+        required_capacity: 2
+        upscale_triggers:
+          - alarm_name: "ECS [repro-worker-production] CPUUtilization"
+            state: ALARM
+        downscale_triggers:
+          - alarm_name: "ECS [repro-worker-production] CPUUtilization (low)"
+            state: OK
+          - alarm_name: "Aurora DMLLatency is high"
+            state: ALARM
+            prioritized_over_upscale_triggers: true
+
+# If you specify `spot_instance_intrp_warns_queue_urls` as SQS queue for spot instance interruption warnings,
+# autoscaler will polls them and set the state of instances to be intrrupted to "DRAINING".
+# autoscaler will also waits for the capacity of active instances in the cluster being decreased
+# when the capacity of spot fleet request is decreased,
+# so you should specify URLs or set the state of the instances to "DRAINING" manually.
+spot_instance_intrp_warns_queue_urls:
+  - https://sqs.ap-northeast-1.amazonaws.com/<account-id>/spot-instance-intrp-warns
 ```
 
+Then, execute the following command:
+
 ```sh
 ecs_auto_scaler <config yaml>
 ```
 
 I recommends deploy `ecs_auto_scaler` on ECS too.
 
+### Signals
+
+ Signal    | Description
+-----------|------------------------------------------------------------
+ TERM, INT | Shutdown gracefully
+ CONT      | Resume auto scaling
+ TSTP      | Pause auto scaling (Run only container instance draining)
+
+### IAM policy for autoscaler
+
+The following permissions are required for the preceding configuration of "repro-api-production" service:
+
+```
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "autoscaling:DescribeAutoScalingGroups",
+        "cloudwatch:DescribeAlarms",
+        "ec2:DescribeInstances",
+        "ec2:TerminateInstances",
+        "ecs:ListTasks"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeServices",
+        "ecs:UpdateService"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:service/ecs-cluster/repro-api-production"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeTasks"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:task/ecs-cluster/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "autoscaling:DetachInstances",
+        "autoscaling:UpdateAutoScalingGroup"
+      ],
+      "Resource": [
+        "arn:aws:autoscaling:ap-northeast-1:<account-id>:autoScalingGroup:<group-id>:autoScalingGroupName/ecs-cluster-nodes"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeContainerInstances"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:container-instance/ecs-cluster/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DeregisterContainerInstance",
+        "ecs:ListContainerInstances"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:cluster/ecs-cluster"
+      ]
+    }
+  ]
+}
+```
+
+If you use spot instances, additional permissions are required like below:
+
+```
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "ecs:UpdateContainerInstancesState",
+      "Resource": "arn:aws:ecs:ap-northeast-1:<account-id>:container-instance/ecs-cluster/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sqs:DeleteMessage",
+        "sqs:DeleteMessageBatch",
+        "sqs:ReceiveMessage"
+      ],
+      "Resource": "arn:aws:sqs:ap-northeast-1:<account-id>:spot-instance-intrp-warns"
+    }
+  ]
+}
+```
+
+The following permissions are required for the preceding configuration of "repro-worker-production" service:
+
+```
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sqs:DeleteMessage",
+        "sqs:DeleteMessageBatch",
+        "sqs:ReceiveMessage"
+      ],
+      "Resource": "arn:aws:sqs:ap-northeast-1:<account-id>:spot-instance-intrp-warns"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "cloudwatch:DescribeAlarms",
+        "ec2:DescribeInstances",
+        "ec2:DescribeSpotFleetInstances",
+        "ec2:DescribeSpotFleetRequests",
+        "ec2:ModifySpotFleetRequest",
+        "ec2:TerminateInstances",
+        "ecs:ListTasks"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeServices",
+        "ecs:UpdateService"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:service/ecs-cluster-for-worker/repro-worker-production"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeTasks"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:task/ecs-cluster-for-worker/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeContainerInstances",
+        "ecs:UpdateContainerInstancesState"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:container-instance/ecs-cluster-for-worker/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:ListContainerInstances"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:cluster/ecs-cluster-for-worker"
+      ]
+    }
+  ]
+}
+```
+
+### How to deploy faster with Auto Scaling Group
+
+Add following configuration to your deploy.rb and hooks if you need.
+
+```ruby
+# deploy.rb
+set :ecs_instance_fluctuation_manager_configs, [
+  {
+    region: "ap-northeast-1",
+    cluster: "CLUSTER_NAME",
+    auto_scaling_group_name: "AUTO_SCALING_GROUP_NAME",
+    desired_capacity: 20, # original capacity of auto scaling group
+  }
+]
+```
+
+This configuration enables tasks `ecs:increase_instances_to_max_size` and `ecs:terminate_redundant_instances`.
+If this configuration is not set, the above tasks do nothing.
+The task `ecs:increase_instances_to_max_size` will increase ECS instances.
+The task `ecs:terminate_redundant_instances` will decrease ECS instances considering AZ balance.
+
+Hook configuration example:
+
+```ruby
+after "deploy:updating", "ecs:increase_instances_to_max_size"
+after "deploy:finished", "ecs:terminate_redundant_instances"
+after "deploy:failed", "ecs:terminate_redundant_instances"
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -1,2 +1,6 @@
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
 task :default => :spec

data/ecs_deploy.gemspec

@@ -18,10 +18,16 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "aws-sdk", "~> 2.9"
+  spec.add_runtime_dependency "aws-sdk-autoscaling", "~> 1"
+  spec.add_runtime_dependency "aws-sdk-cloudwatch", "~> 1"
+  spec.add_runtime_dependency "aws-sdk-cloudwatchevents", "~> 1"
+  spec.add_runtime_dependency "aws-sdk-ec2", "~> 1"
+  spec.add_runtime_dependency "aws-sdk-ecs", "~> 1"
+  spec.add_runtime_dependency "aws-sdk-sqs", "~> 1"
   spec.add_runtime_dependency "terminal-table"
   spec.add_runtime_dependency "paint"
 
-  spec.add_development_dependency "bundler", "~> 1.11"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", ">= 1.11", "< 3"
+  spec.add_development_dependency "rake", ">= 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
 end