hako 1.6.2 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -1
- data/README.md +15 -0
- data/docs/ecs-task-notification.md +24 -0
- data/examples/put-ecs-container-status-to-s3/index.js +40 -0
- data/examples/put-ecs-container-status-to-s3/package.json +7 -0
- data/lib/hako/schedulers/ecs.rb +34 -3
- data/lib/hako/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f81c77dc43aeaaf63eaf41066e2988b3402a7ab4
|
4
|
+
data.tar.gz: e7d8de4361f7b24ca5aa1b8a3dfa8a9de9cdfaed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ae3dd38326c2d3533f18b774b70282f93d1338d8b2a17bfd7383d2bcf62d2b2cc48fa2918c9051ba0a3ca59ad62de84a167d65f35be9c18f074ef953a64128f5
|
7
|
+
data.tar.gz: f559ea0ab2183dfd04f324f9507475e4887fe1a8f130c214a3b1270c52adbebdcfd7310f6b5f19bc12cfd46c1abf286fe239642f8f3f9641b66a8a49fdb069d5
|
data/CHANGELOG.md
CHANGED
@@ -1,4 +1,10 @@
|
|
1
|
-
# 1.
|
1
|
+
# 1.7.0 (2017-08-29)
|
2
|
+
## New features
|
3
|
+
- Add experimental `autoscaling_topic_for_oneshot` option to ECS scheduler
|
4
|
+
- It publishes scale-out request to SNS topic.
|
5
|
+
- Administrators is expected to receive SNS event and initiate scale-out.
|
6
|
+
|
7
|
+
# 1.6.2 (2017-07-11)
|
2
8
|
## Bug fixes
|
3
9
|
- Exclude unusable instances when checking remaining capacity
|
4
10
|
|
data/README.md
CHANGED
@@ -50,6 +50,21 @@ I, [2016-05-02T13:07:12.959116 #10961] INFO -- : Updated service: arn:aws:ecs:a
|
|
50
50
|
I, [2016-05-02T13:08:27.280686 #10961] INFO -- : Deployment completed
|
51
51
|
```
|
52
52
|
|
53
|
+
Run oneshot command.
|
54
|
+
|
55
|
+
```
|
56
|
+
% hako oneshot examples/hello.yml date
|
57
|
+
I, [2017-07-18T18:14:06.099763 #6627] INFO -- : Task definition isn't changed: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task-definition/hello-oneshot:32
|
58
|
+
I, [2017-07-18T18:14:06.147062 #6627] INFO -- : Started task: arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:task/01234567-89ab-cdef-0123-456789abcdef
|
59
|
+
I, [2017-07-18T18:14:06.193860 #6627] INFO -- : Container instance is arn:aws:ecs:ap-northeast-1:XXXXXXXXXXXX:container-instance/01234567-89ab-cdef-0123-456789abcdef (i-0123456789abcdef0)
|
60
|
+
I, [2017-07-18T18:14:37.826389 #6627] INFO -- : Started at 2017-07-18 18:14:37 +0900
|
61
|
+
I, [2017-07-18T18:14:37.826482 #6627] INFO -- : Stopped at 2017-07-18 18:14:37 +0900 (reason: Essential container in task exited)
|
62
|
+
I, [2017-07-18T18:14:37.826520 #6627] INFO -- : Oneshot task finished
|
63
|
+
I, [2017-07-18T18:14:37.826548 #6627] INFO -- : app has stopped with exit_code=0
|
64
|
+
```
|
65
|
+
|
66
|
+
See also [docs/ecs-task-notification.md](docs/ecs-task-notification.md).
|
67
|
+
|
53
68
|
## Front image
|
54
69
|
The front container receives these environment variables.
|
55
70
|
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# ECS task notification
|
2
|
+
In ECS scheduler, `hako oneshot` supports multiple methods of detecting task finish.
|
3
|
+
|
4
|
+
## ecs:DescribeTasks (default)
|
5
|
+
Use [DescribeTasks](http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_DescribeTasks.html) API to get the task status.
|
6
|
+
This method can be used without any preparation or configuration, but the DescribeTasks API can return "Rate exceeded" error when there's several running `hako oneshot` processes.
|
7
|
+
|
8
|
+
## s3:GetObject
|
9
|
+
Amazon ECS has integration with Amazon CloudWatch Events. The integration notifies ECS task state changes to AWS Lambda, Amazon SNS, Amazon SQS, and so on.
|
10
|
+
http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_cwe_events.html#ecs_task_events
|
11
|
+
|
12
|
+
Amazon S3 is a good storage for polling, so connecting CloudWatch Events to AWS Lambda and put the payload to S3 is more scalable than ecs:DescribeTasks.
|
13
|
+
|
14
|
+
The example implementation of AWS Lambda can be found in [../examples/put-ecs-container-status-to-s3](../examples/put-ecs-container-status-to-s3) directory.
|
15
|
+
|
16
|
+
To enable task notification with S3, you have to configure scheduler in YAML.
|
17
|
+
|
18
|
+
```yaml
|
19
|
+
scheduler:
|
20
|
+
type: ecs
|
21
|
+
oneshot_notification_prefix: 's3://ecs-task-notifications/task_statuses?region=ap-northeast-1'
|
22
|
+
```
|
23
|
+
|
24
|
+
It uses ecs-task-notifications bucket in ap-northeast-1 region.
|
@@ -0,0 +1,40 @@
|
|
1
|
+
const AWS = require('aws-sdk');
|
2
|
+
|
3
|
+
const BUCKET = 'ecs-task-notifications';
|
4
|
+
|
5
|
+
exports.handler = (event, context, callback) => {
|
6
|
+
const s3 = new AWS.S3();
|
7
|
+
// console.log(JSON.stringify(event, null, 2));
|
8
|
+
const taskArn = event.detail.taskArn;
|
9
|
+
if (event.detail.lastStatus === 'RUNNING' && event.detail.containers.every((container) => container.lastStatus === 'RUNNING')) {
|
10
|
+
console.log(`${taskArn} started`);
|
11
|
+
s3.putObject({
|
12
|
+
Bucket: BUCKET,
|
13
|
+
Key: `task_statuses/${taskArn}/started.json`,
|
14
|
+
Body: JSON.stringify(event),
|
15
|
+
}, (err, data) => {
|
16
|
+
if (err) {
|
17
|
+
console.log(err);
|
18
|
+
callback(err);
|
19
|
+
} else {
|
20
|
+
callback(null, 'Started');
|
21
|
+
}
|
22
|
+
});
|
23
|
+
} else if (event.detail.lastStatus === 'STOPPED') {
|
24
|
+
console.log(`${taskArn} stopped`);
|
25
|
+
s3.putObject({
|
26
|
+
Bucket: BUCKET,
|
27
|
+
Key: `task_statuses/${taskArn}/stopped.json`,
|
28
|
+
Body: JSON.stringify(event),
|
29
|
+
}, (err, data) => {
|
30
|
+
if (err) {
|
31
|
+
console.log(err);
|
32
|
+
callback(err);
|
33
|
+
} else {
|
34
|
+
callback(null, 'Stopped');
|
35
|
+
}
|
36
|
+
});
|
37
|
+
} else {
|
38
|
+
callback(null, 'Skipped');
|
39
|
+
}
|
40
|
+
};
|
data/lib/hako/schedulers/ecs.rb
CHANGED
@@ -38,6 +38,10 @@ module Hako
|
|
38
38
|
@autoscaling = EcsAutoscaling.new(options.fetch('autoscaling'), dry_run: @dry_run)
|
39
39
|
end
|
40
40
|
@autoscaling_group_for_oneshot = options.fetch('autoscaling_group_for_oneshot', nil)
|
41
|
+
@autoscaling_topic_for_oneshot = options.fetch('autoscaling_topic_for_oneshot', nil)
|
42
|
+
if @autoscaling_topic_for_oneshot && !@autoscaling_group_for_oneshot
|
43
|
+
validation_error!('autoscaling_group_for_oneshot must be set when autoscaling_topic_for_oneshot is set')
|
44
|
+
end
|
41
45
|
@oneshot_notification_prefix = options.fetch('oneshot_notification_prefix', nil)
|
42
46
|
@deployment_configuration = {}
|
43
47
|
%i[maximum_percent minimum_healthy_percent].each do |key|
|
@@ -638,7 +642,7 @@ module Hako
|
|
638
642
|
|
639
643
|
# @param [Aws::ECS::Types::Task] task
|
640
644
|
# @return [Hash<String, Aws::ECS::Types::Container>]
|
641
|
-
#
|
645
|
+
# Get stopped container status from S3.
|
642
646
|
# The advantage is scalability; ecs:DescribeTasks is heavily
|
643
647
|
# rate-limited, but s3:GetObject is much more scalable.
|
644
648
|
# The JSON is supposed to be stored from Amazon ECS Event Stream.
|
@@ -854,8 +858,6 @@ module Hako
|
|
854
858
|
raise "Unable to find rollback target. #{task_definition.task_definition_arn} is INACTIVE?"
|
855
859
|
end
|
856
860
|
|
857
|
-
MIN_ASG_INTERVAL = 1
|
858
|
-
MAX_ASG_INTERVAL = 120
|
859
861
|
# @param [Aws::ECS::Types::TaskDefinition] task_definition
|
860
862
|
# @return [Boolean] true if the capacity is reserved
|
861
863
|
def on_no_tasks_started(task_definition)
|
@@ -863,6 +865,35 @@ module Hako
|
|
863
865
|
return false
|
864
866
|
end
|
865
867
|
|
868
|
+
if @autoscaling_topic_for_oneshot
|
869
|
+
try_scale_out_with_sns(task_definition)
|
870
|
+
else
|
871
|
+
try_scale_out_with_as(task_definition)
|
872
|
+
end
|
873
|
+
end
|
874
|
+
|
875
|
+
RUN_TASK_INTERVAL = 10
|
876
|
+
def try_scale_out_with_sns(task_definition)
|
877
|
+
required_cpu, required_memory = task_definition.container_definitions.inject([0, 0]) { |(cpu, memory), d| [cpu + d.cpu, memory + d.memory] }
|
878
|
+
@hako_task_id ||= SecureRandom.uuid
|
879
|
+
message = JSON.dump(
|
880
|
+
group_name: @autoscaling_group_for_oneshot,
|
881
|
+
cluster: @cluster,
|
882
|
+
cpu: required_cpu,
|
883
|
+
memory: required_memory,
|
884
|
+
hako_task_id: @hako_task_id,
|
885
|
+
)
|
886
|
+
Hako.logger.info("Unable to start tasks. Publish message to #{@autoscaling_topic_for_oneshot}: #{message}")
|
887
|
+
sns_client = Aws::SNS::Client.new
|
888
|
+
resp = sns_client.publish(topic_arn: @autoscaling_topic_for_oneshot, message: message)
|
889
|
+
Hako.logger.info("Sent message_id=#{resp.message_id}")
|
890
|
+
sleep(RUN_TASK_INTERVAL)
|
891
|
+
true
|
892
|
+
end
|
893
|
+
|
894
|
+
MIN_ASG_INTERVAL = 1
|
895
|
+
MAX_ASG_INTERVAL = 120
|
896
|
+
def try_scale_out_with_as(task_definition)
|
866
897
|
autoscaling = Aws::AutoScaling::Client.new
|
867
898
|
interval = MIN_ASG_INTERVAL
|
868
899
|
Hako.logger.info("Unable to start tasks. Start trying scaling out '#{@autoscaling_group_for_oneshot}'")
|
data/lib/hako/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hako
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kohei Suzuki
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-08-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk
|
@@ -143,6 +143,7 @@ files:
|
|
143
143
|
- Rakefile
|
144
144
|
- bin/console
|
145
145
|
- bin/setup
|
146
|
+
- docs/ecs-task-notification.md
|
146
147
|
- examples/create_aws_cloud_watch_logs_log_group.yml
|
147
148
|
- examples/front.yml
|
148
149
|
- examples/hello-autoscaling-group.yml
|
@@ -153,6 +154,8 @@ files:
|
|
153
154
|
- examples/hello-privileged-app.yml
|
154
155
|
- examples/hello.env
|
155
156
|
- examples/hello.yml
|
157
|
+
- examples/put-ecs-container-status-to-s3/index.js
|
158
|
+
- examples/put-ecs-container-status-to-s3/package.json
|
156
159
|
- exe/hako
|
157
160
|
- hako.gemspec
|
158
161
|
- lib/hako.rb
|