ecs_deploy 0.3.0 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a565132ff142d819e1bdc64368264ebc89de1030
-  data.tar.gz: d0b5d389d5a9311853d8524e45c56bd54943450a
+SHA256:
+  metadata.gz: 3988729fd28cd6cfb8fb6f36869855b1bbc468899d453a38e7415552a81c13aa
+  data.tar.gz: 73543aa973eadf609e049355064cd98e6ce18c448b69be6c1920e6cfee9fffa7
 SHA512:
-  metadata.gz: 50c6f3c9aecdacc26de0d61cb487f4196bf31031b8c43b798b4bebae1e2246e35ebf1bb1c6079a62b7d5f54c3f504fefb69ca08f08689d9717c601746db63bdb
-  data.tar.gz: f4fa32bf5a67e71a6265842eaffbff31647d17337fdb23e01007a6082f13b894a45ec127ca098a7c9125935937f9ac04c5d88025223915938da591a3259119e2
+  metadata.gz: 95495a3d6fdfaaaf4c7e5197f2f2107905310a0df36dc26197a43e8defe4e1721b9de45eed68a89e8aa50d3187df54590814544806b0e1b719e387bc774005af
+  data.tar.gz: 5e09d4a796fa051db972f5fd70c52ceb96058690123a3cc583aaf4642ebabf9e80cd225afcf64e67af9078944cb4ba013142f9923d9b6d8549d654401fa5edb6

data/.gitignore

@@ -8,4 +8,5 @@
 /spec/reports/
 /tmp/
 
+.rspec_status
 .envrc

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - ruby-2.4.9
+  - ruby-2.5.7
+  - ruby-2.6.5

data/CHANGELOG.md

@@ -0,0 +1,139 @@
+# v1.0
+
+## Release v1.0.2 - 2021/05/26
+
+### Enhancement
+
+* add option enable_execute_command to support ECS Exec
+  https://github.com/reproio/ecs_deploy/pull/69
+
+## Release v1.0.1 - 2021/05/19
+
+### Enhancement
+
+* retry register_task_definition by AWS SDK feature
+  https://github.com/reproio/ecs_deploy/pull/67
+* Support Ruby 3.0
+  https://github.com/reproio/ecs_deploy/pull/66
+* Wait until stop old tasks
+  https://github.com/reproio/ecs_deploy/pull/65
+* Add prioritized_over_upscale_triggers option to triggers
+  https://github.com/reproio/ecs_deploy/pull/62
+* Display only unstable services in EcsDeploy::Service#wait_all_running
+  https://github.com/reproio/ecs_deploy/pull/61
+
+## Release v1.0.0 - 2019/12/24
+
+### New feature
+
+* Add tasks to deploy the application faster
+  https://github.com/reproio/ecs_deploy/pull/57
+
+### Enhancement
+
+* Add parameters `ecs_wait_until_services_stable_max_attempts` and `ecs_wait_until_services_stable_delay`
+  https://github.com/reproio/ecs_deploy/pull/30
+* Detect region automatically according to AWS SDK
+  https://github.com/reproio/ecs_deploy/pull/31
+* Support new features of ECS to support Fargate
+  https://github.com/reproio/ecs_deploy/pull/32
+* Ignore running tasks which don't belong to the ECS services on deregistering container instances
+  https://github.com/reproio/ecs_deploy/pull/33
+* Drop AWS SDK 2 support
+  https://github.com/reproio/ecs_deploy/pull/34
+* Support scheduling_strategy option
+  https://github.com/reproio/ecs_deploy/pull/35
+* Support execution_role_arn on task_definition
+  https://github.com/reproio/ecs_deploy/pull/36
+* Support spot fleet requests and container instance draining
+  https://github.com/reproio/ecs_deploy/pull/40
+* Add network_configuration paramters to ScheduledTask
+  https://github.com/reproio/ecs_deploy/pull/46
+* Support tagging ECS resources
+  https://github.com/reproio/ecs_deploy/pull/48
+  https://github.com/reproio/ecs_deploy/pull/49
+* Wait for stopping tasks until tasks stop
+  https://github.com/reproio/ecs_deploy/pull/50
+* Improve performance when start tasks
+  https://github.com/reproio/ecs_deploy/pull/53
+* Improve stability of auto scaling groups managed by ecs_auto_scaler
+  https://github.com/reproio/ecs_deploy/pull/55
+
+### Bug fixes
+
+* Fix infinite loop that occurs when there are more than 100 container instances
+  https://github.com/reproio/ecs_deploy/pull/38
+* Fix errors that occur on decreasing more than 20 container instances
+  https://github.com/reproio/ecs_deploy/pull/39
+
+# Ancient releases
+
+## Release v0.3.2 - 2017/23/10
+
+### Enhancement
+
+* Remove execution feature
+  https://github.com/reproio/ecs_deploy/pull/24
+* Support container overrides in scheduled tasks
+  https://github.com/reproio/ecs_deploy/pull/26
+
+### Bug fixes
+
+* Fix deployment errors that occur when `ecs_scheduled_tasks` is not set
+  https://github.com/reproio/ecs_deploy/pull/27
+
+## Release v0.3.1 - 2017/04/08
+
+### Bug fixes
+
+* Fix block parameter name
+
+## Release v0.3.0 - 2017/03/08
+
+### New feature
+
+* Support ScheduledTask deployment
+  https://github.com/reproio/ecs_deploy/pull/22
+
+### Enhancement
+
+* Support network_mode and placement_constraints
+* Introduce `ecs_registered_tasks` capistrano variable
+  https://github.com/reproio/ecs_deploy/pull/23
+
+### Bug fixes
+
+* Filter inactive services
+  https://github.com/reproio/ecs_deploy/pull/19
+* Wait 10 services at once
+  https://github.com/reproio/ecs_deploy/pull/20
+  https://github.com/reproio/ecs_deploy/pull/21
+
+## Release v0.2.0 - 2016/31/10
+
+### Enhancement
+
+* Support task role arn
+  https://github.com/reproio/ecs_deploy/pull/13
+* Make the scale-in process safe
+  https://github.com/reproio/ecs_deploy/pull/14
+* Support ALB
+  https://github.com/reproio/ecs_deploy/pull/15
+
+## Release v0.1.2 - 2016/28/07
+
+### Bug fixes
+
+* Fix rollback bug
+  https://github.com/reproio/ecs_deploy/pull/11
+
+## Release v0.1.1 - 2016/03/07
+
+### Bug fixes
+
+* Add missing desired_count for backend services
+  https://github.com/reproio/ecs_deploy/pull/9
+
+## Release v0.1.0 - 2016/27/06
+
+Initial release.

data/README.md

@@ -28,11 +28,11 @@ require 'ecs_deploy/capistrano'
 
 # deploy.rb
 set :ecs_default_cluster, "ecs-cluster-name"
-set :ecs_access_key_id, "dummy" # optional, if nil, use environment variable
-set :ecs_secret_access_key, "dummy" # optional, if nil, use environment variable
 set :ecs_region, %w(ap-northeast-1) # optional, if nil, use environment variable
 set :ecs_service_role, "customEcsServiceRole" # default: ecsServiceRole
 set :ecs_deploy_wait_timeout, 600 # default: 300
+set :ecs_wait_until_services_stable_max_attempts, 40 # optional
+set :ecs_wait_until_services_stable_delay, 15 # optional
 
 set :ecs_tasks, [
   {
@@ -87,12 +87,27 @@ set :ecs_tasks, [
       }
     ],
     volumes: [{name: "sockets_path", host: {}}],
-    executions: [ # execution task on deploy timing
-      {container_overrides: [{name: "myapp", command: ["db_migrate"]}]},
-    ]
   },
 ]
 
+set :ecs_scheduled_tasks, [
+  {
+    cluster: "default", # Unless this key, use fetch(:ecs_default_cluster)
+    rule_name: "schedule_name",
+    schedule_expression: "cron(0 12 * * ? *)",
+    description: "schedule_description", # Optional
+    target_id: "task_name", # Unless this key, use task_definition_name
+    task_definition_name: "myapp-#{fetch(:rails_env)}",
+    task_count: 2, # Default 1
+    revision: 12, # Optional
+    role_arn: "TaskRoleArn", # Optional
+    container_overrides: [ # Optional
+      name: "myapp-main",
+      command: ["ls"],
+    ]
+  }
+]
+
 set :ecs_services, [
   {
     name: "myapp-#{fetch(:rails_env)}",
@@ -116,6 +131,7 @@ set :ecs_services, [
 
 ```sh
 cap <stage> ecs:register_task_definition # register ecs_tasks as TaskDefinition
+cap <stage> ecs:deploy_scheduled_task # register ecs_scheduled_tasks to CloudWatchEvent
 cap <stage> ecs:deploy # create or update Service by ecs_services info
 
 cap <stage> ecs:rollback # deregister current task definition and update Service by previous revision of current task definition
@@ -173,7 +189,16 @@ And rollback
 
 ## Autoscaler
 
-Write config file (YAML format).
+The autoscaler of `ecs_deploy` supports auto scaling of ECS services and clusters.
+
+### Prerequisits
+
+* You use a ECS cluster whose instances belong to either an auto scaling group or a spot fleet request
+* You have CloudWatch alarms and you want to scale services when their state changes
+
+### How to use autoscaler
+
+First, write a configuration file (YAML format) like below:
 
 ```yaml
 # ポーリング時にupscale_triggersに指定した状態のalarmがあればstep分serviceとinstanceを増やす (max_task_countまで)
@@ -186,37 +211,275 @@ polling_interval: 60
 auto_scaling_groups:
   - name: ecs-cluster-nodes
     region: ap-northeast-1
-    buffer: 1 # タスク数に対する余剰のインスタンス数
-
-services:
-  - name: repro-api-production
     cluster: ecs-cluster
+    # autoscaler will set the capacity to (buffer + desired_tasks * required_capacity).
+    # Adjust this value if it takes much time to prepare ECS instances and launch new tasks.
+    buffer: 1
+    services:
+      - name: repro-api-production
+        step: 1
+        idle_time: 240
+        max_task_count: [10, 25]
+        scheduled_min_task_count:
+          - {from: "1:45", to: "4:30", count: 8}
+        cooldown_time_for_reach_max: 600
+        min_task_count: 0
+        # Required capacity per task (default: 1)
+        # You should specify "binpack" as task placement strategy if the value is less than 1 and you use an auto scaling group.
+        required_capacity: 0.5
+        upscale_triggers:
+          - alarm_name: "ECS [repro-api-production] CPUUtilization"
+            state: ALARM
+          - alarm_name: "ELB repro-api-a HTTPCode_Backend_5XX"
+            state: ALARM
+            step: 2
+        downscale_triggers:
+          - alarm_name: "ECS [repro-api-production] CPUUtilization (low)"
+            state: OK
+
+spot_fleet_requests:
+  - id: sfr-354de735-2c17-4565-88c9-10ada5b957e5
     region: ap-northeast-1
-    auto_scaling_group_name: ecs-cluster-nodes
-    step: 1
-    idle_time: 240
-    max_task_count: [10, 25]
-    scheduled_min_task_count:
-      - {from: "1:45", to: "4:30", count: 8}
-    cooldown_time_for_reach_max: 600
-    min_task_count: 0
-    upscale_triggers:
-      - alarm_name: "ECS [repro-api-production] CPUUtilization"
-        state: ALARM
-      - alarm_name: "ELB repro-api-a HTTPCode_Backend_5XX"
-        state: ALARM
-        step: 2
-    downscale_triggers:
-      - alarm_name: "ECS [repro-api-production] CPUUtilization (low)"
-        state: OK
+    cluster: ecs-cluster-for-worker
+    buffer: 1
+    services:
+      - name: repro-worker-production
+        step: 1
+        idle_time: 240
+        cooldown_time_for_reach_max: 600
+        min_task_count: 0
+        # Required capacity per task (default: 1)
+        # The capacity assumes that WeightedCapacity is equal to the number of vCPUs.
+        required_capacity: 2
+        upscale_triggers:
+          - alarm_name: "ECS [repro-worker-production] CPUUtilization"
+            state: ALARM
+        downscale_triggers:
+          - alarm_name: "ECS [repro-worker-production] CPUUtilization (low)"
+            state: OK
+          - alarm_name: "Aurora DMLLatency is high"
+            state: ALARM
+            prioritized_over_upscale_triggers: true
+
+# If you specify `spot_instance_intrp_warns_queue_urls` as SQS queue for spot instance interruption warnings,
+# autoscaler will polls them and set the state of instances to be intrrupted to "DRAINING".
+# autoscaler will also waits for the capacity of active instances in the cluster being decreased
+# when the capacity of spot fleet request is decreased,
+# so you should specify URLs or set the state of the instances to "DRAINING" manually.
+spot_instance_intrp_warns_queue_urls:
+  - https://sqs.ap-northeast-1.amazonaws.com/<account-id>/spot-instance-intrp-warns
 ```
 
+Then, execute the following command:
+
 ```sh
 ecs_auto_scaler <config yaml>
 ```
 
 I recommends deploy `ecs_auto_scaler` on ECS too.
 
+### Signals
+
+ Signal    | Description
+-----------|------------------------------------------------------------
+ TERM, INT | Shutdown gracefully
+ CONT      | Resume auto scaling
+ TSTP      | Pause auto scaling (Run only container instance draining)
+
+### IAM policy for autoscaler
+
+The following permissions are required for the preceding configuration of "repro-api-production" service:
+
+```
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "autoscaling:DescribeAutoScalingGroups",
+        "cloudwatch:DescribeAlarms",
+        "ec2:DescribeInstances",
+        "ec2:TerminateInstances",
+        "ecs:ListTasks"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeServices",
+        "ecs:UpdateService"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:service/ecs-cluster/repro-api-production"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeTasks"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:task/ecs-cluster/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "autoscaling:DetachInstances",
+        "autoscaling:UpdateAutoScalingGroup"
+      ],
+      "Resource": [
+        "arn:aws:autoscaling:ap-northeast-1:<account-id>:autoScalingGroup:<group-id>:autoScalingGroupName/ecs-cluster-nodes"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeContainerInstances"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:container-instance/ecs-cluster/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DeregisterContainerInstance",
+        "ecs:ListContainerInstances"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:cluster/ecs-cluster"
+      ]
+    }
+  ]
+}
+```
+
+If you use spot instances, additional permissions are required like below:
+
+```
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "ecs:UpdateContainerInstancesState",
+      "Resource": "arn:aws:ecs:ap-northeast-1:<account-id>:container-instance/ecs-cluster/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sqs:DeleteMessage",
+        "sqs:DeleteMessageBatch",
+        "sqs:ReceiveMessage"
+      ],
+      "Resource": "arn:aws:sqs:ap-northeast-1:<account-id>:spot-instance-intrp-warns"
+    }
+  ]
+}
+```
+
+The following permissions are required for the preceding configuration of "repro-worker-production" service:
+
+```
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sqs:DeleteMessage",
+        "sqs:DeleteMessageBatch",
+        "sqs:ReceiveMessage"
+      ],
+      "Resource": "arn:aws:sqs:ap-northeast-1:<account-id>:spot-instance-intrp-warns"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "cloudwatch:DescribeAlarms",
+        "ec2:DescribeInstances",
+        "ec2:DescribeSpotFleetInstances",
+        "ec2:DescribeSpotFleetRequests",
+        "ec2:ModifySpotFleetRequest",
+        "ec2:TerminateInstances",
+        "ecs:ListTasks"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeServices",
+        "ecs:UpdateService"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:service/ecs-cluster-for-worker/repro-worker-production"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeTasks"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:task/ecs-cluster-for-worker/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DescribeContainerInstances",
+        "ecs:UpdateContainerInstancesState"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:container-instance/ecs-cluster-for-worker/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecs:ListContainerInstances"
+      ],
+      "Resource": [
+        "arn:aws:ecs:ap-northeast-1:<account-id>:cluster/ecs-cluster-for-worker"
+      ]
+    }
+  ]
+}
+```
+
+### How to deploy faster with Auto Scaling Group
+
+Add following configuration to your deploy.rb and hooks if you need.
+
+```ruby
+# deploy.rb
+set :ecs_instance_fluctuation_manager_configs, [
+  {
+    region: "ap-northeast-1",
+    cluster: "CLUSTER_NAME",
+    auto_scaling_group_name: "AUTO_SCALING_GROUP_NAME",
+    desired_capacity: 20, # original capacity of auto scaling group
+  }
+]
+```
+
+This configuration enables tasks `ecs:increase_instances_to_max_size` and `ecs:terminate_redundant_instances`.
+If this configuration is not set, the above tasks do nothing.
+The task `ecs:increase_instances_to_max_size` will increase ECS instances.
+The task `ecs:terminate_redundant_instances` will decrease ECS instances considering AZ balance.
+
+Hook configuration example:
+
+```ruby
+after "deploy:updating", "ecs:increase_instances_to_max_size"
+after "deploy:finished", "ecs:terminate_redundant_instances"
+after "deploy:failed", "ecs:terminate_redundant_instances"
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.