hako 1.6.2 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: de23a44f8ddc813bbeab7fd82eb3db64f439e82e
-  data.tar.gz: c086e946fea0dc907a98733b44085219be86727d
+  metadata.gz: f81c77dc43aeaaf63eaf41066e2988b3402a7ab4
+  data.tar.gz: e7d8de4361f7b24ca5aa1b8a3dfa8a9de9cdfaed
 SHA512:
-  metadata.gz: c068d2f8857df91fd803c6fa4a7eb043dbc9b6a01814599ca544a82fcc3036603c58734be4c1ec65985062b146703ab28c50e01ba97a032077e123cc50272f2c
-  data.tar.gz: b08335811bff88a721352f3bb20b5597a889ca81ce8bd0a9bfdb9831a6950c7b2635ad3f51c5cf667aab439bc7e2a18453f346e5747185b2567adf34887b7bc6
+  metadata.gz: ae3dd38326c2d3533f18b774b70282f93d1338d8b2a17bfd7383d2bcf62d2b2cc48fa2918c9051ba0a3ca59ad62de84a167d65f35be9c18f074ef953a64128f5
+  data.tar.gz: f559ea0ab2183dfd04f324f9507475e4887fe1a8f130c214a3b1270c52adbebdcfd7310f6b5f19bc12cfd46c1abf286fe239642f8f3f9641b66a8a49fdb069d5

data/CHANGELOG.md

@@ -1,4 +1,10 @@
-# 1.6.1 (2017-07-11)
+# 1.7.0 (2017-08-29)
+## New features
+- Add experimental `autoscaling_topic_for_oneshot` option to ECS scheduler
+    - It publishes scale-out request to SNS topic.
+    - Administrators is expected to receive SNS event and initiate scale-out.
+
+# 1.6.2 (2017-07-11)
 ## Bug fixes
 - Exclude unusable instances when checking remaining capacity
 

data/README.md

@@ -50,6 +50,21 @@ I, [2016-05-02T13:07:12.959116 #10961]  INFO -- : Updated service: arn:aws:ecs:a
 I, [2016-05-02T13:08:27.280686 #10961]  INFO -- : Deployment completed
 ```
 
+Run oneshot command.
+
+```
+% hako oneshot examples/hello.yml date
+I, [2017-07-18T18:14:06.099763 #6627]  INFO -- : Task definition isn't changed: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task-definition/hello-oneshot:32
+I, [2017-07-18T18:14:06.147062 #6627]  INFO -- : Started task: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task/01234567-89ab-cdef-0123-456789abcdef
+I, [2017-07-18T18:14:06.193860 #6627]  INFO -- : Container instance is arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:container-instance/01234567-89ab-cdef-0123-456789abcdef (i-0123456789abcdef0)
+I, [2017-07-18T18:14:37.826389 #6627]  INFO -- : Started at 2017-07-18 18:14:37 +0900
+I, [2017-07-18T18:14:37.826482 #6627]  INFO -- : Stopped at 2017-07-18 18:14:37 +0900 (reason: Essential container in task exited)
+I, [2017-07-18T18:14:37.826520 #6627]  INFO -- : Oneshot task finished
+I, [2017-07-18T18:14:37.826548 #6627]  INFO -- : app has stopped with exit_code=0
+```
+
+See also [docs/ecs-task-notification.md](docs/ecs-task-notification.md).
+
 ## Front image
 The front container receives these environment variables.
 

data/docs/ecs-task-notification.md

@@ -0,0 +1,24 @@
+# ECS task notification
+In ECS scheduler, `hako oneshot` supports multiple methods of detecting task finish.
+
+## ecs:DescribeTasks (default)
+Use [DescribeTasks](http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_DescribeTasks.html) API to get the task status.
+This method can be used without any preparation or configuration, but the DescribeTasks API can return "Rate exceeded" error when there's several running `hako oneshot` processes.
+
+## s3:GetObject
+Amazon ECS has integration with Amazon CloudWatch Events. The integration notifies ECS task state changes to AWS Lambda, Amazon SNS, Amazon SQS, and so on.
+http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_cwe_events.html#ecs_task_events
+
+Amazon S3 is a good storage for polling, so connecting CloudWatch Events to AWS Lambda and put the payload to S3 is more scalable than ecs:DescribeTasks.
+
+The example implementation of AWS Lambda can be found in [../examples/put-ecs-container-status-to-s3](../examples/put-ecs-container-status-to-s3) directory.
+
+To enable task notification with S3, you have to configure scheduler in YAML.
+
+```yaml
+scheduler:
+  type: ecs
+  oneshot_notification_prefix: 's3://ecs-task-notifications/task_statuses?region=ap-northeast-1'
+```
+
+It uses ecs-task-notifications bucket in ap-northeast-1 region.

data/examples/put-ecs-container-status-to-s3/index.js

@@ -0,0 +1,40 @@
+const AWS = require('aws-sdk');
+
+const BUCKET = 'ecs-task-notifications';
+
+exports.handler = (event, context, callback) => {
+  const s3 = new AWS.S3();
+  // console.log(JSON.stringify(event, null, 2));
+  const taskArn = event.detail.taskArn;
+  if (event.detail.lastStatus === 'RUNNING' && event.detail.containers.every((container) => container.lastStatus === 'RUNNING')) {
+    console.log(`${taskArn} started`);
+    s3.putObject({
+      Bucket: BUCKET,
+      Key: `task_statuses/${taskArn}/started.json`,
+      Body: JSON.stringify(event),
+    }, (err, data) => {
+      if (err) {
+        console.log(err);
+        callback(err);
+      } else {
+        callback(null, 'Started');
+      }
+    });
+  } else if (event.detail.lastStatus === 'STOPPED') {
+    console.log(`${taskArn} stopped`);
+    s3.putObject({
+      Bucket: BUCKET,
+      Key: `task_statuses/${taskArn}/stopped.json`,
+      Body: JSON.stringify(event),
+    }, (err, data) => {
+      if (err) {
+        console.log(err);
+        callback(err);
+      } else {
+        callback(null, 'Stopped');
+      }
+    });
+  } else {
+    callback(null, 'Skipped');
+  }
+};

data/examples/put-ecs-container-status-to-s3/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "put-ecs-container-status-to-s3",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "private": true
+}

data/lib/hako/schedulers/ecs.rb

@@ -38,6 +38,10 @@ module Hako
           @autoscaling = EcsAutoscaling.new(options.fetch('autoscaling'), dry_run: @dry_run)
         end
         @autoscaling_group_for_oneshot = options.fetch('autoscaling_group_for_oneshot', nil)
+        @autoscaling_topic_for_oneshot = options.fetch('autoscaling_topic_for_oneshot', nil)
+        if @autoscaling_topic_for_oneshot && !@autoscaling_group_for_oneshot
+          validation_error!('autoscaling_group_for_oneshot must be set when autoscaling_topic_for_oneshot is set')
+        end
         @oneshot_notification_prefix = options.fetch('oneshot_notification_prefix', nil)
         @deployment_configuration = {}
         %i[maximum_percent minimum_healthy_percent].each do |key|
@@ -638,7 +642,7 @@ module Hako
 
       # @param [Aws::ECS::Types::Task] task
       # @return [Hash<String, Aws::ECS::Types::Container>]
-      # Experimental: Get stopped container status from S3.
+      # Get stopped container status from S3.
       # The advantage is scalability; ecs:DescribeTasks is heavily
       # rate-limited, but s3:GetObject is much more scalable.
       # The JSON is supposed to be stored from Amazon ECS Event Stream.
@@ -854,8 +858,6 @@ module Hako
         raise "Unable to find rollback target. #{task_definition.task_definition_arn} is INACTIVE?"
       end
 
-      MIN_ASG_INTERVAL = 1
-      MAX_ASG_INTERVAL = 120
       # @param [Aws::ECS::Types::TaskDefinition] task_definition
       # @return [Boolean] true if the capacity is reserved
       def on_no_tasks_started(task_definition)
@@ -863,6 +865,35 @@ module Hako
           return false
         end
 
+        if @autoscaling_topic_for_oneshot
+          try_scale_out_with_sns(task_definition)
+        else
+          try_scale_out_with_as(task_definition)
+        end
+      end
+
+      RUN_TASK_INTERVAL = 10
+      def try_scale_out_with_sns(task_definition)
+        required_cpu, required_memory = task_definition.container_definitions.inject([0, 0]) { |(cpu, memory), d| [cpu + d.cpu, memory + d.memory] }
+        @hako_task_id ||= SecureRandom.uuid
+        message = JSON.dump(
+          group_name: @autoscaling_group_for_oneshot,
+          cluster: @cluster,
+          cpu: required_cpu,
+          memory: required_memory,
+          hako_task_id: @hako_task_id,
+        )
+        Hako.logger.info("Unable to start tasks. Publish message to #{@autoscaling_topic_for_oneshot}: #{message}")
+        sns_client = Aws::SNS::Client.new
+        resp = sns_client.publish(topic_arn: @autoscaling_topic_for_oneshot, message: message)
+        Hako.logger.info("Sent message_id=#{resp.message_id}")
+        sleep(RUN_TASK_INTERVAL)
+        true
+      end
+
+      MIN_ASG_INTERVAL = 1
+      MAX_ASG_INTERVAL = 120
+      def try_scale_out_with_as(task_definition)
         autoscaling = Aws::AutoScaling::Client.new
         interval = MIN_ASG_INTERVAL
         Hako.logger.info("Unable to start tasks. Start trying scaling out '#{@autoscaling_group_for_oneshot}'")

data/lib/hako/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Hako
-  VERSION = '1.6.2'
+  VERSION = '1.7.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hako
 version: !ruby/object:Gem::Version
-  version: 1.6.2
+  version: 1.7.0
 platform: ruby
 authors:
 - Kohei Suzuki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-11 00:00:00.000000000 Z
+date: 2017-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -143,6 +143,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- docs/ecs-task-notification.md
 - examples/create_aws_cloud_watch_logs_log_group.yml
 - examples/front.yml
 - examples/hello-autoscaling-group.yml
@@ -153,6 +154,8 @@ files:
 - examples/hello-privileged-app.yml
 - examples/hello.env
 - examples/hello.yml
+- examples/put-ecs-container-status-to-s3/index.js
+- examples/put-ecs-container-status-to-s3/package.json
 - exe/hako
 - hako.gemspec
 - lib/hako.rb