hako 1.6.2 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: de23a44f8ddc813bbeab7fd82eb3db64f439e82e
4
- data.tar.gz: c086e946fea0dc907a98733b44085219be86727d
3
+ metadata.gz: f81c77dc43aeaaf63eaf41066e2988b3402a7ab4
4
+ data.tar.gz: e7d8de4361f7b24ca5aa1b8a3dfa8a9de9cdfaed
5
5
  SHA512:
6
- metadata.gz: c068d2f8857df91fd803c6fa4a7eb043dbc9b6a01814599ca544a82fcc3036603c58734be4c1ec65985062b146703ab28c50e01ba97a032077e123cc50272f2c
7
- data.tar.gz: b08335811bff88a721352f3bb20b5597a889ca81ce8bd0a9bfdb9831a6950c7b2635ad3f51c5cf667aab439bc7e2a18453f346e5747185b2567adf34887b7bc6
6
+ metadata.gz: ae3dd38326c2d3533f18b774b70282f93d1338d8b2a17bfd7383d2bcf62d2b2cc48fa2918c9051ba0a3ca59ad62de84a167d65f35be9c18f074ef953a64128f5
7
+ data.tar.gz: f559ea0ab2183dfd04f324f9507475e4887fe1a8f130c214a3b1270c52adbebdcfd7310f6b5f19bc12cfd46c1abf286fe239642f8f3f9641b66a8a49fdb069d5
@@ -1,4 +1,10 @@
1
- # 1.6.1 (2017-07-11)
1
+ # 1.7.0 (2017-08-29)
2
+ ## New features
3
+ - Add experimental `autoscaling_topic_for_oneshot` option to ECS scheduler
4
+ - It publishes scale-out request to SNS topic.
5
+ - Administrators is expected to receive SNS event and initiate scale-out.
6
+
7
+ # 1.6.2 (2017-07-11)
2
8
  ## Bug fixes
3
9
  - Exclude unusable instances when checking remaining capacity
4
10
 
data/README.md CHANGED
@@ -50,6 +50,21 @@ I, [2016-05-02T13:07:12.959116 #10961] INFO -- : Updated service: arn:aws:ecs:a
50
50
  I, [2016-05-02T13:08:27.280686 #10961] INFO -- : Deployment completed
51
51
  ```
52
52
 
53
+ Run oneshot command.
54
+
55
+ ```
56
+ % hako oneshot examples/hello.yml date
57
+ I, [2017-07-18T18:14:06.099763 #6627] INFO -- : Task definition isn't changed: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task-definition/hello-oneshot:32
58
+ I, [2017-07-18T18:14:06.147062 #6627] INFO -- : Started task: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task/01234567-89ab-cdef-0123-456789abcdef
59
+ I, [2017-07-18T18:14:06.193860 #6627] INFO -- : Container instance is arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:container-instance/01234567-89ab-cdef-0123-456789abcdef (i-0123456789abcdef0)
60
+ I, [2017-07-18T18:14:37.826389 #6627] INFO -- : Started at 2017-07-18 18:14:37 +0900
61
+ I, [2017-07-18T18:14:37.826482 #6627] INFO -- : Stopped at 2017-07-18 18:14:37 +0900 (reason: Essential container in task exited)
62
+ I, [2017-07-18T18:14:37.826520 #6627] INFO -- : Oneshot task finished
63
+ I, [2017-07-18T18:14:37.826548 #6627] INFO -- : app has stopped with exit_code=0
64
+ ```
65
+
66
+ See also [docs/ecs-task-notification.md](docs/ecs-task-notification.md).
67
+
53
68
  ## Front image
54
69
  The front container receives these environment variables.
55
70
 
@@ -0,0 +1,24 @@
1
+ # ECS task notification
2
+ In ECS scheduler, `hako oneshot` supports multiple methods of detecting task finish.
3
+
4
+ ## ecs:DescribeTasks (default)
5
+ Use [DescribeTasks](http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_DescribeTasks.html) API to get the task status.
6
+ This method can be used without any preparation or configuration, but the DescribeTasks API can return "Rate exceeded" error when there's several running `hako oneshot` processes.
7
+
8
+ ## s3:GetObject
9
+ Amazon ECS has integration with Amazon CloudWatch Events. The integration notifies ECS task state changes to AWS Lambda, Amazon SNS, Amazon SQS, and so on.
10
+ http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_cwe_events.html#ecs_task_events
11
+
12
+ Amazon S3 is a good storage for polling, so connecting CloudWatch Events to AWS Lambda and put the payload to S3 is more scalable than ecs:DescribeTasks.
13
+
14
+ The example implementation of AWS Lambda can be found in [../examples/put-ecs-container-status-to-s3](../examples/put-ecs-container-status-to-s3) directory.
15
+
16
+ To enable task notification with S3, you have to configure scheduler in YAML.
17
+
18
+ ```yaml
19
+ scheduler:
20
+ type: ecs
21
+ oneshot_notification_prefix: 's3://ecs-task-notifications/task_statuses?region=ap-northeast-1'
22
+ ```
23
+
24
+ It uses ecs-task-notifications bucket in ap-northeast-1 region.
@@ -0,0 +1,40 @@
1
+ const AWS = require('aws-sdk');
2
+
3
+ const BUCKET = 'ecs-task-notifications';
4
+
5
+ exports.handler = (event, context, callback) => {
6
+ const s3 = new AWS.S3();
7
+ // console.log(JSON.stringify(event, null, 2));
8
+ const taskArn = event.detail.taskArn;
9
+ if (event.detail.lastStatus === 'RUNNING' && event.detail.containers.every((container) => container.lastStatus === 'RUNNING')) {
10
+ console.log(`${taskArn} started`);
11
+ s3.putObject({
12
+ Bucket: BUCKET,
13
+ Key: `task_statuses/${taskArn}/started.json`,
14
+ Body: JSON.stringify(event),
15
+ }, (err, data) => {
16
+ if (err) {
17
+ console.log(err);
18
+ callback(err);
19
+ } else {
20
+ callback(null, 'Started');
21
+ }
22
+ });
23
+ } else if (event.detail.lastStatus === 'STOPPED') {
24
+ console.log(`${taskArn} stopped`);
25
+ s3.putObject({
26
+ Bucket: BUCKET,
27
+ Key: `task_statuses/${taskArn}/stopped.json`,
28
+ Body: JSON.stringify(event),
29
+ }, (err, data) => {
30
+ if (err) {
31
+ console.log(err);
32
+ callback(err);
33
+ } else {
34
+ callback(null, 'Stopped');
35
+ }
36
+ });
37
+ } else {
38
+ callback(null, 'Skipped');
39
+ }
40
+ };
@@ -0,0 +1,7 @@
1
+ {
2
+ "name": "put-ecs-container-status-to-s3",
3
+ "version": "1.0.0",
4
+ "main": "index.js",
5
+ "license": "MIT",
6
+ "private": true
7
+ }
@@ -38,6 +38,10 @@ module Hako
38
38
  @autoscaling = EcsAutoscaling.new(options.fetch('autoscaling'), dry_run: @dry_run)
39
39
  end
40
40
  @autoscaling_group_for_oneshot = options.fetch('autoscaling_group_for_oneshot', nil)
41
+ @autoscaling_topic_for_oneshot = options.fetch('autoscaling_topic_for_oneshot', nil)
42
+ if @autoscaling_topic_for_oneshot && !@autoscaling_group_for_oneshot
43
+ validation_error!('autoscaling_group_for_oneshot must be set when autoscaling_topic_for_oneshot is set')
44
+ end
41
45
  @oneshot_notification_prefix = options.fetch('oneshot_notification_prefix', nil)
42
46
  @deployment_configuration = {}
43
47
  %i[maximum_percent minimum_healthy_percent].each do |key|
@@ -638,7 +642,7 @@ module Hako
638
642
 
639
643
  # @param [Aws::ECS::Types::Task] task
640
644
  # @return [Hash<String, Aws::ECS::Types::Container>]
641
- # Experimental: Get stopped container status from S3.
645
+ # Get stopped container status from S3.
642
646
  # The advantage is scalability; ecs:DescribeTasks is heavily
643
647
  # rate-limited, but s3:GetObject is much more scalable.
644
648
  # The JSON is supposed to be stored from Amazon ECS Event Stream.
@@ -854,8 +858,6 @@ module Hako
854
858
  raise "Unable to find rollback target. #{task_definition.task_definition_arn} is INACTIVE?"
855
859
  end
856
860
 
857
- MIN_ASG_INTERVAL = 1
858
- MAX_ASG_INTERVAL = 120
859
861
  # @param [Aws::ECS::Types::TaskDefinition] task_definition
860
862
  # @return [Boolean] true if the capacity is reserved
861
863
  def on_no_tasks_started(task_definition)
@@ -863,6 +865,35 @@ module Hako
863
865
  return false
864
866
  end
865
867
 
868
+ if @autoscaling_topic_for_oneshot
869
+ try_scale_out_with_sns(task_definition)
870
+ else
871
+ try_scale_out_with_as(task_definition)
872
+ end
873
+ end
874
+
875
+ RUN_TASK_INTERVAL = 10
876
+ def try_scale_out_with_sns(task_definition)
877
+ required_cpu, required_memory = task_definition.container_definitions.inject([0, 0]) { |(cpu, memory), d| [cpu + d.cpu, memory + d.memory] }
878
+ @hako_task_id ||= SecureRandom.uuid
879
+ message = JSON.dump(
880
+ group_name: @autoscaling_group_for_oneshot,
881
+ cluster: @cluster,
882
+ cpu: required_cpu,
883
+ memory: required_memory,
884
+ hako_task_id: @hako_task_id,
885
+ )
886
+ Hako.logger.info("Unable to start tasks. Publish message to #{@autoscaling_topic_for_oneshot}: #{message}")
887
+ sns_client = Aws::SNS::Client.new
888
+ resp = sns_client.publish(topic_arn: @autoscaling_topic_for_oneshot, message: message)
889
+ Hako.logger.info("Sent message_id=#{resp.message_id}")
890
+ sleep(RUN_TASK_INTERVAL)
891
+ true
892
+ end
893
+
894
+ MIN_ASG_INTERVAL = 1
895
+ MAX_ASG_INTERVAL = 120
896
+ def try_scale_out_with_as(task_definition)
866
897
  autoscaling = Aws::AutoScaling::Client.new
867
898
  interval = MIN_ASG_INTERVAL
868
899
  Hako.logger.info("Unable to start tasks. Start trying scaling out '#{@autoscaling_group_for_oneshot}'")
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Hako
4
- VERSION = '1.6.2'
4
+ VERSION = '1.7.0'
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hako
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.6.2
4
+ version: 1.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kohei Suzuki
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-07-11 00:00:00.000000000 Z
11
+ date: 2017-08-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk
@@ -143,6 +143,7 @@ files:
143
143
  - Rakefile
144
144
  - bin/console
145
145
  - bin/setup
146
+ - docs/ecs-task-notification.md
146
147
  - examples/create_aws_cloud_watch_logs_log_group.yml
147
148
  - examples/front.yml
148
149
  - examples/hello-autoscaling-group.yml
@@ -153,6 +154,8 @@ files:
153
154
  - examples/hello-privileged-app.yml
154
155
  - examples/hello.env
155
156
  - examples/hello.yml
157
+ - examples/put-ecs-container-status-to-s3/index.js
158
+ - examples/put-ecs-container-status-to-s3/package.json
156
159
  - exe/hako
157
160
  - hako.gemspec
158
161
  - lib/hako.rb